From webhook-mailer at python.org  Sun Sep  1 04:18:44 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sun, 01 Sep 2019 08:18:44 -0000
Subject: [Python-checkins] bpo-36543: Remove old-deprecated ElementTree
 features. (GH-12707)
Message-ID: <mailman.203.1567325926.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f02ea6225bc3b71bd5fe66224d199a6e3e23b14d
commit: f02ea6225bc3b71bd5fe66224d199a6e3e23b14d
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-01T11:18:35+03:00
summary:

bpo-36543: Remove old-deprecated ElementTree features. (GH-12707)

Remove methods Element.getchildren(), Element.getiterator() and
ElementTree.getiterator() and the xml.etree.cElementTree module.

files:
A Misc/NEWS.d/next/Library/2019-04-06-20-08-12.bpo-36543.RPjmUz.rst
M Doc/library/xml.etree.elementtree.rst
M Doc/whatsnew/3.9.rst
M Lib/test/test_xml_etree.py
M Lib/xml/etree/ElementTree.py
M Modules/_elementtree.c
M Modules/clinic/_elementtree.c.h

diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst
index 6047e6e29b92..1b4e2c9b142e 100644
--- a/Doc/library/xml.etree.elementtree.rst
+++ b/Doc/library/xml.etree.elementtree.rst
@@ -873,18 +873,6 @@ Element Objects
       in the expression into the given namespace.
 
 
-   .. method:: getchildren()
-
-      .. deprecated-removed:: 3.2 3.9
-         Use ``list(elem)`` or iteration.
-
-
-   .. method:: getiterator(tag=None)
-
-      .. deprecated-removed:: 3.2 3.9
-         Use method :meth:`Element.iter` instead.
-
-
    .. method:: insert(index, subelement)
 
       Inserts *subelement* at the given position in this element.  Raises
@@ -1019,12 +1007,6 @@ ElementTree Objects
       Same as :meth:`Element.findtext`, starting at the root of the tree.
 
 
-   .. method:: getiterator(tag=None)
-
-      .. deprecated-removed:: 3.2 3.9
-         Use method :meth:`ElementTree.iter` instead.
-
-
    .. method:: getroot()
 
       Returns the root element for this tree.
diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
index e20ae47462eb..0a0b91670ca5 100644
--- a/Doc/whatsnew/3.9.rst
+++ b/Doc/whatsnew/3.9.rst
@@ -207,6 +207,12 @@ Removed
   Use :meth:`~threading.Thread.is_alive()` instead.
   (Contributed by Dong-hee Na in :issue:`37804`.)
 
+* Methods ``getchildren()`` and ``getiterator()`` in the
+  :mod:`~xml.etree.ElementTree` module have been removed.  They were
+  deprecated in Python 3.2.  Use functions :func:`list` and :func:`iter`
+  instead.  The ``xml.etree.cElementTree`` module has been removed.
+  (Contributed by Serhiy Storchaka in :issue:`36543`.)
+
 
 Porting to Python 3.9
 =====================
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
index db06aceb1461..87a6fa146462 100644
--- a/Lib/test/test_xml_etree.py
+++ b/Lib/test/test_xml_etree.py
@@ -242,7 +242,6 @@ def check_method(method):
         check_method(element.extend)
         check_method(element.insert)
         check_method(element.remove)
-        check_method(element.getchildren)
         check_method(element.find)
         check_method(element.iterfind)
         check_method(element.findall)
@@ -254,7 +253,6 @@ def check_method(method):
         check_method(element.items)
         check_method(element.iter)
         check_method(element.itertext)
-        check_method(element.getiterator)
 
         # These methods return an iterable. See bug 6472.
 
@@ -741,24 +739,20 @@ def end_ns(self, prefix):
                 ('end-ns', ''),
             ])
 
-    # Element.getchildren() and ElementTree.getiterator() are deprecated.
-    @checkwarnings(("This method will be removed in future versions.  "
-                    "Use .+ instead.",
-                    DeprecationWarning))
-    def test_getchildren(self):
-        # Test Element.getchildren()
+    def test_children(self):
+        # Test Element children iteration
 
         with open(SIMPLE_XMLFILE, "rb") as f:
             tree = ET.parse(f)
-        self.assertEqual([summarize_list(elem.getchildren())
+        self.assertEqual([summarize_list(elem)
                           for elem in tree.getroot().iter()], [
                 ['element', 'element', 'empty-element'],
                 [],
                 [],
                 [],
             ])
-        self.assertEqual([summarize_list(elem.getchildren())
-                          for elem in tree.getiterator()], [
+        self.assertEqual([summarize_list(elem)
+                          for elem in tree.iter()], [
                 ['element', 'element', 'empty-element'],
                 [],
                 [],
@@ -766,13 +760,13 @@ def test_getchildren(self):
             ])
 
         elem = ET.XML(SAMPLE_XML)
-        self.assertEqual(len(elem.getchildren()), 3)
-        self.assertEqual(len(elem[2].getchildren()), 1)
-        self.assertEqual(elem[:], elem.getchildren())
+        self.assertEqual(len(list(elem)), 3)
+        self.assertEqual(len(list(elem[2])), 1)
+        self.assertEqual(elem[:], list(elem))
         child1 = elem[0]
         child2 = elem[2]
         del elem[1:2]
-        self.assertEqual(len(elem.getchildren()), 2)
+        self.assertEqual(len(list(elem)), 2)
         self.assertEqual(child1, elem[0])
         self.assertEqual(child2, elem[1])
         elem[0:2] = [child2, child1]
@@ -780,7 +774,7 @@ def test_getchildren(self):
         self.assertEqual(child1, elem[1])
         self.assertNotEqual(child1, elem[0])
         elem.clear()
-        self.assertEqual(elem.getchildren(), [])
+        self.assertEqual(list(elem), [])
 
     def test_writestring(self):
         elem = ET.XML("<html><body>text</body></html>")
@@ -2955,40 +2949,6 @@ def test_iter_by_tag(self):
         self.assertEqual(self._ilist(doc), all_tags)
         self.assertEqual(self._ilist(doc, '*'), all_tags)
 
-    # Element.getiterator() is deprecated.
-    @checkwarnings(("This method will be removed in future versions.  "
-                    "Use .+ instead.", DeprecationWarning))
-    def test_getiterator(self):
-        doc = ET.XML('''
-            <document>
-                <house>
-                    <room>bedroom1</room>
-                    <room>bedroom2</room>
-                </house>
-                <shed>nothing here
-                </shed>
-                <house>
-                    <room>bedroom8</room>
-                </house>
-            </document>''')
-
-        self.assertEqual(summarize_list(doc.getiterator('room')),
-                         ['room'] * 3)
-        self.assertEqual(summarize_list(doc.getiterator('house')),
-                         ['house'] * 2)
-
-        # test that getiterator also accepts 'tag' as a keyword arg
-        self.assertEqual(
-            summarize_list(doc.getiterator(tag='room')),
-            ['room'] * 3)
-
-        # make sure both tag=None and tag='*' return all tags
-        all_tags = ['document', 'house', 'room', 'room',
-                    'shed', 'house', 'room']
-        self.assertEqual(summarize_list(doc.getiterator()), all_tags)
-        self.assertEqual(summarize_list(doc.getiterator(None)), all_tags)
-        self.assertEqual(summarize_list(doc.getiterator('*')), all_tags)
-
     def test_copy(self):
         a = ET.Element('a')
         it = a.iter()
diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py
index 431ecd0dddf1..beb2d68803a5 100644
--- a/Lib/xml/etree/ElementTree.py
+++ b/Lib/xml/etree/ElementTree.py
@@ -273,19 +273,6 @@ def remove(self, subelement):
         # assert iselement(element)
         self._children.remove(subelement)
 
-    def getchildren(self):
-        """(Deprecated) Return all subelements.
-
-        Elements are returned in document order.
-
-        """
-        warnings.warn(
-            "This method will be removed in future versions.  "
-            "Use 'list(elem)' or iteration over elem instead.",
-            DeprecationWarning, stacklevel=2
-            )
-        return self._children
-
     def find(self, path, namespaces=None):
         """Find first matching element by tag name or path.
 
@@ -409,15 +396,6 @@ def iter(self, tag=None):
         for e in self._children:
             yield from e.iter(tag)
 
-    # compatibility
-    def getiterator(self, tag=None):
-        warnings.warn(
-            "This method will be removed in future versions.  "
-            "Use 'elem.iter()' or 'list(elem.iter())' instead.",
-            DeprecationWarning, stacklevel=2
-        )
-        return list(self.iter(tag))
-
     def itertext(self):
         """Create text iterator.
 
@@ -617,15 +595,6 @@ def iter(self, tag=None):
         # assert self._root is not None
         return self._root.iter(tag)
 
-    # compatibility
-    def getiterator(self, tag=None):
-        warnings.warn(
-            "This method will be removed in future versions.  "
-            "Use 'tree.iter()' or 'list(tree.iter())' instead.",
-            DeprecationWarning, stacklevel=2
-        )
-        return list(self.iter(tag))
-
     def find(self, path, namespaces=None):
         """Find first matching element by tag name or path.
 
diff --git a/Misc/NEWS.d/next/Library/2019-04-06-20-08-12.bpo-36543.RPjmUz.rst b/Misc/NEWS.d/next/Library/2019-04-06-20-08-12.bpo-36543.RPjmUz.rst
new file mode 100644
index 000000000000..1153de872bc7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-04-06-20-08-12.bpo-36543.RPjmUz.rst
@@ -0,0 +1,2 @@
+Removed methods Element.getchildren(), Element.getiterator() and
+ElementTree.getiterator() and the xml.etree.cElementTree module.
diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
index 830ce8635eae..b88e8a1d6e9e 100644
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -1426,42 +1426,6 @@ _elementtree_Element_get_impl(ElementObject *self, PyObject *key,
     return value;
 }
 
-/*[clinic input]
-_elementtree.Element.getchildren
-
-[clinic start generated code]*/
-
-static PyObject *
-_elementtree_Element_getchildren_impl(ElementObject *self)
-/*[clinic end generated code: output=e50ffe118637b14f input=0f754dfded150d5f]*/
-{
-    Py_ssize_t i;
-    PyObject* list;
-
-    if (PyErr_WarnEx(PyExc_DeprecationWarning,
-                     "This method will be removed in future versions.  "
-                     "Use 'list(elem)' or iteration over elem instead.",
-                     1) < 0) {
-        return NULL;
-    }
-
-    if (!self->extra)
-        return PyList_New(0);
-
-    list = PyList_New(self->extra->length);
-    if (!list)
-        return NULL;
-
-    for (i = 0; i < self->extra->length; i++) {
-        PyObject* item = self->extra->children[i];
-        Py_INCREF(item);
-        PyList_SET_ITEM(list, i, item);
-    }
-
-    return list;
-}
-
-
 static PyObject *
 create_elementiter(ElementObject *self, PyObject *tag, int gettext);
 
@@ -1492,27 +1456,6 @@ _elementtree_Element_iter_impl(ElementObject *self, PyObject *tag)
 }
 
 
-/*[clinic input]
-_elementtree.Element.getiterator
-
-    tag: object = None
-
-[clinic start generated code]*/
-
-static PyObject *
-_elementtree_Element_getiterator_impl(ElementObject *self, PyObject *tag)
-/*[clinic end generated code: output=cb69ff4a3742dfa1 input=500da1a03f7b9e28]*/
-{
-    if (PyErr_WarnEx(PyExc_DeprecationWarning,
-                     "This method will be removed in future versions.  "
-                     "Use 'tree.iter()' or 'list(tree.iter())' instead.",
-                     1) < 0) {
-        return NULL;
-    }
-    return _elementtree_Element_iter_impl(self, tag);
-}
-
-
 /*[clinic input]
 _elementtree.Element.itertext
 
@@ -4220,9 +4163,6 @@ static PyMethodDef element_methods[] = {
     _ELEMENTTREE_ELEMENT_ITERTEXT_METHODDEF
     _ELEMENTTREE_ELEMENT_ITERFIND_METHODDEF
 
-    _ELEMENTTREE_ELEMENT_GETITERATOR_METHODDEF
-    _ELEMENTTREE_ELEMENT_GETCHILDREN_METHODDEF
-
     _ELEMENTTREE_ELEMENT_ITEMS_METHODDEF
     _ELEMENTTREE_ELEMENT_KEYS_METHODDEF
 
diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h
index 324e549d486a..a58ad737f854 100644
--- a/Modules/clinic/_elementtree.c.h
+++ b/Modules/clinic/_elementtree.c.h
@@ -355,23 +355,6 @@ _elementtree_Element_get(ElementObject *self, PyObject *const *args, Py_ssize_t
     return return_value;
 }
 
-PyDoc_STRVAR(_elementtree_Element_getchildren__doc__,
-"getchildren($self, /)\n"
-"--\n"
-"\n");
-
-#define _ELEMENTTREE_ELEMENT_GETCHILDREN_METHODDEF    \
-    {"getchildren", (PyCFunction)_elementtree_Element_getchildren, METH_NOARGS, _elementtree_Element_getchildren__doc__},
-
-static PyObject *
-_elementtree_Element_getchildren_impl(ElementObject *self);
-
-static PyObject *
-_elementtree_Element_getchildren(ElementObject *self, PyObject *Py_UNUSED(ignored))
-{
-    return _elementtree_Element_getchildren_impl(self);
-}
-
 PyDoc_STRVAR(_elementtree_Element_iter__doc__,
 "iter($self, /, tag=None)\n"
 "--\n"
@@ -408,42 +391,6 @@ _elementtree_Element_iter(ElementObject *self, PyObject *const *args, Py_ssize_t
     return return_value;
 }
 
-PyDoc_STRVAR(_elementtree_Element_getiterator__doc__,
-"getiterator($self, /, tag=None)\n"
-"--\n"
-"\n");
-
-#define _ELEMENTTREE_ELEMENT_GETITERATOR_METHODDEF    \
-    {"getiterator", (PyCFunction)(void(*)(void))_elementtree_Element_getiterator, METH_FASTCALL|METH_KEYWORDS, _elementtree_Element_getiterator__doc__},
-
-static PyObject *
-_elementtree_Element_getiterator_impl(ElementObject *self, PyObject *tag);
-
-static PyObject *
-_elementtree_Element_getiterator(ElementObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
-{
-    PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"tag", NULL};
-    static _PyArg_Parser _parser = {NULL, _keywords, "getiterator", 0};
-    PyObject *argsbuf[1];
-    Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
-    PyObject *tag = Py_None;
-
-    args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
-    if (!args) {
-        goto exit;
-    }
-    if (!noptargs) {
-        goto skip_optional_pos;
-    }
-    tag = args[0];
-skip_optional_pos:
-    return_value = _elementtree_Element_getiterator_impl(self, tag);
-
-exit:
-    return return_value;
-}
-
 PyDoc_STRVAR(_elementtree_Element_itertext__doc__,
 "itertext($self, /)\n"
 "--\n"
@@ -969,4 +916,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args,
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=50e0b1954c5f9e0f input=a9049054013a1b77]*/
+/*[clinic end generated code: output=f5dbf9b4a095d310 input=a9049054013a1b77]*/


From webhook-mailer at python.org  Sun Sep  1 05:03:43 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sun, 01 Sep 2019 09:03:43 -0000
Subject: [Python-checkins] bpo-37994: Fix silencing all errors if an
 attribute lookup fails. (GH-15630)
Message-ID: <mailman.204.1567328624.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/41c57b335330ff48af098d47e379e0f9ba09d233
commit: 41c57b335330ff48af098d47e379e0f9ba09d233
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-01T12:03:39+03:00
summary:

bpo-37994: Fix silencing all errors if an attribute lookup fails. (GH-15630)

Only AttributeError should be silenced.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-08-31-11-13-25.bpo-37994.Rj6S4j.rst
M Modules/_csv.c
M Modules/_datetimemodule.c
M Modules/_pickle.c
M Modules/_threadmodule.c
M Modules/pyexpat.c
M Objects/bytearrayobject.c
M Objects/descrobject.c
M Objects/fileobject.c
M Objects/setobject.c
M Objects/typeobject.c
M Objects/weakrefobject.c
M Python/bltinmodule.c
M Python/sysmodule.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-31-11-13-25.bpo-37994.Rj6S4j.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-31-11-13-25.bpo-37994.Rj6S4j.rst
new file mode 100644
index 000000000000..103ac5a30905
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-31-11-13-25.bpo-37994.Rj6S4j.rst	
@@ -0,0 +1,2 @@
+Fixed silencing arbitrary errors if an attribute lookup fails in several
+sites. Only AttributeError should be silenced.
diff --git a/Modules/_csv.c b/Modules/_csv.c
index 9653ff92ee9a..aaf377650c0b 100644
--- a/Modules/_csv.c
+++ b/Modules/_csv.c
@@ -1382,7 +1382,10 @@ csv_writer(PyObject *module, PyObject *args, PyObject *keyword_args)
         Py_DECREF(self);
         return NULL;
     }
-    self->write = _PyObject_GetAttrId(output_file, &PyId_write);
+    if (_PyObject_LookupAttrId(output_file, &PyId_write, &self->write) < 0) {
+        Py_DECREF(self);
+        return NULL;
+    }
     if (self->write == NULL || !PyCallable_Check(self->write)) {
         PyErr_SetString(PyExc_TypeError,
                         "argument 1 must have a \"write\" method");
diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c
index 8f4fa21da141..c1b24073436e 100644
--- a/Modules/_datetimemodule.c
+++ b/Modules/_datetimemodule.c
@@ -3607,24 +3607,24 @@ tzinfo_reduce(PyObject *self, PyObject *Py_UNUSED(ignored))
     _Py_IDENTIFIER(__getinitargs__);
     _Py_IDENTIFIER(__getstate__);
 
-    getinitargs = _PyObject_GetAttrId(self, &PyId___getinitargs__);
+    if (_PyObject_LookupAttrId(self, &PyId___getinitargs__, &getinitargs) < 0) {
+        return NULL;
+    }
     if (getinitargs != NULL) {
         args = PyObject_CallNoArgs(getinitargs);
         Py_DECREF(getinitargs);
-        if (args == NULL) {
-            return NULL;
-        }
     }
     else {
-        PyErr_Clear();
-
         args = PyTuple_New(0);
-        if (args == NULL) {
-            return NULL;
-        }
+    }
+    if (args == NULL) {
+        return NULL;
     }
 
-    getstate = _PyObject_GetAttrId(self, &PyId___getstate__);
+    if (_PyObject_LookupAttrId(self, &PyId___getstate__, &getstate) < 0) {
+        Py_DECREF(args);
+        return NULL;
+    }
     if (getstate != NULL) {
         state = PyObject_CallNoArgs(getstate);
         Py_DECREF(getstate);
@@ -3635,7 +3635,6 @@ tzinfo_reduce(PyObject *self, PyObject *Py_UNUSED(ignored))
     }
     else {
         PyObject **dictptr;
-        PyErr_Clear();
         state = Py_None;
         dictptr = _PyObject_GetDictPtr(self);
         if (dictptr && *dictptr && PyDict_GET_SIZE(*dictptr)) {
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index 0c53f2e18685..9e82d145cc66 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -4383,7 +4383,6 @@ save(PicklerObject *self, PyObject *obj, int pers_save)
         _Py_IDENTIFIER(__reduce__);
         _Py_IDENTIFIER(__reduce_ex__);
 
-
         /* XXX: If the __reduce__ method is defined, __reduce_ex__ is
            automatically defined as __reduce__. While this is convenient, this
            make it impossible to know which method was actually called. Of
@@ -4404,14 +4403,15 @@ save(PicklerObject *self, PyObject *obj, int pers_save)
             }
         }
         else {
-            PickleState *st = _Pickle_GetGlobalState();
-
             /* Check for a __reduce__ method. */
-            reduce_func = _PyObject_GetAttrId(obj, &PyId___reduce__);
+            if (_PyObject_LookupAttrId(obj, &PyId___reduce__, &reduce_func) < 0) {
+                goto error;
+            }
             if (reduce_func != NULL) {
                 reduce_value = PyObject_CallNoArgs(reduce_func);
             }
             else {
+                PickleState *st = _Pickle_GetGlobalState();
                 PyErr_Format(st->PicklingError,
                              "can't pickle '%.200s' object: %R",
                              type->tp_name, obj);
@@ -6446,7 +6446,9 @@ do_append(UnpicklerObject *self, Py_ssize_t x)
         PyObject *extend_func;
         _Py_IDENTIFIER(extend);
 
-        extend_func = _PyObject_GetAttrId(list, &PyId_extend);
+        if (_PyObject_LookupAttrId(list, &PyId_extend, &extend_func) < 0) {
+            return -1;
+        }
         if (extend_func != NULL) {
             slice = Pdata_poplist(self->stack, x);
             if (!slice) {
@@ -6466,7 +6468,6 @@ do_append(UnpicklerObject *self, Py_ssize_t x)
             /* Even if the PEP 307 requires extend() and append() methods,
                fall back on append() if the object has no extend() method
                for backward compatibility. */
-            PyErr_Clear();
             append_func = _PyObject_GetAttrId(list, &PyId_append);
             if (append_func == NULL)
                 return -1;
diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c
index 6665827fe2ba..a3ecd2e74e8c 100644
--- a/Modules/_threadmodule.c
+++ b/Modules/_threadmodule.c
@@ -1313,6 +1313,7 @@ static int
 thread_excepthook_file(PyObject *file, PyObject *exc_type, PyObject *exc_value,
                        PyObject *exc_traceback, PyObject *thread)
 {
+    _Py_IDENTIFIER(name);
     /* print(f"Exception in thread {thread.name}:", file=file) */
     if (PyFile_WriteString("Exception in thread ", file) < 0) {
         return -1;
@@ -1320,7 +1321,9 @@ thread_excepthook_file(PyObject *file, PyObject *exc_type, PyObject *exc_value,
 
     PyObject *name = NULL;
     if (thread != Py_None) {
-        name = PyObject_GetAttrString(thread, "name");
+        if (_PyObject_LookupAttrId(thread, &PyId_name, &name) < 0) {
+            return -1;
+        }
     }
     if (name != NULL) {
         if (PyFile_WriteObject(name, file, Py_PRINT_RAW) < 0) {
@@ -1330,8 +1333,6 @@ thread_excepthook_file(PyObject *file, PyObject *exc_type, PyObject *exc_value,
         Py_DECREF(name);
     }
     else {
-        PyErr_Clear();
-
         unsigned long ident = PyThread_get_thread_ident();
         PyObject *str = PyUnicode_FromFormat("%lu", ident);
         if (str != NULL) {
diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c
index b0096e664780..5d284cddcfc7 100644
--- a/Modules/pyexpat.c
+++ b/Modules/pyexpat.c
@@ -810,7 +810,9 @@ pyexpat_xmlparser_ParseFile(xmlparseobject *self, PyObject *file)
     PyObject *readmethod = NULL;
     _Py_IDENTIFIER(read);
 
-    readmethod = _PyObject_GetAttrId(file, &PyId_read);
+    if (_PyObject_LookupAttrId(file, &PyId_read, &readmethod) < 0) {
+        return NULL;
+    }
     if (readmethod == NULL) {
         PyErr_SetString(PyExc_TypeError,
                         "argument must have 'read' attribute");
diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c
index 9dd67127b614..8c16cc630d71 100644
--- a/Objects/bytearrayobject.c
+++ b/Objects/bytearrayobject.c
@@ -2061,9 +2061,10 @@ _common_reduce(PyByteArrayObject *self, int proto)
     _Py_IDENTIFIER(__dict__);
     char *buf;
 
-    dict = _PyObject_GetAttrId((PyObject *)self, &PyId___dict__);
+    if (_PyObject_LookupAttrId((PyObject *)self, &PyId___dict__, &dict) < 0) {
+        return NULL;
+    }
     if (dict == NULL) {
-        PyErr_Clear();
         dict = Py_None;
         Py_INCREF(dict);
     }
diff --git a/Objects/descrobject.c b/Objects/descrobject.c
index 9e1b281c4603..c50fe00ce80b 100644
--- a/Objects/descrobject.c
+++ b/Objects/descrobject.c
@@ -1616,29 +1616,25 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset,
     /* if no docstring given and the getter has one, use that one */
     if ((doc == NULL || doc == Py_None) && fget != NULL) {
         _Py_IDENTIFIER(__doc__);
-        PyObject *get_doc = _PyObject_GetAttrId(fget, &PyId___doc__);
-        if (get_doc) {
-            if (Py_TYPE(self) == &PyProperty_Type) {
-                Py_XSETREF(self->prop_doc, get_doc);
-            }
-            else {
-                /* If this is a property subclass, put __doc__
-                in dict of the subclass instance instead,
-                otherwise it gets shadowed by __doc__ in the
-                class's dict. */
-                int err = _PyObject_SetAttrId((PyObject *)self, &PyId___doc__, get_doc);
-                Py_DECREF(get_doc);
-                if (err < 0)
-                    return -1;
-            }
-            self->getter_doc = 1;
+        PyObject *get_doc;
+        int rc = _PyObject_LookupAttrId(fget, &PyId___doc__, &get_doc);
+        if (rc <= 0) {
+            return rc;
         }
-        else if (PyErr_ExceptionMatches(PyExc_Exception)) {
-            PyErr_Clear();
+        if (Py_TYPE(self) == &PyProperty_Type) {
+            Py_XSETREF(self->prop_doc, get_doc);
         }
         else {
-            return -1;
+            /* If this is a property subclass, put __doc__
+               in dict of the subclass instance instead,
+               otherwise it gets shadowed by __doc__ in the
+               class's dict. */
+            int err = _PyObject_SetAttrId((PyObject *)self, &PyId___doc__, get_doc);
+            Py_DECREF(get_doc);
+            if (err < 0)
+                return -1;
         }
+        self->getter_doc = 1;
     }
 
     return 0;
diff --git a/Objects/fileobject.c b/Objects/fileobject.c
index 0faf7e70b5d8..61c94280937b 100644
--- a/Objects/fileobject.c
+++ b/Objects/fileobject.c
@@ -185,8 +185,10 @@ PyObject_AsFileDescriptor(PyObject *o)
     if (PyLong_Check(o)) {
         fd = _PyLong_AsInt(o);
     }
-    else if ((meth = _PyObject_GetAttrId(o, &PyId_fileno)) != NULL)
-    {
+    else if (_PyObject_LookupAttrId(o, &PyId_fileno, &meth) < 0) {
+        return -1;
+    }
+    else if (meth != NULL) {
         PyObject *fno = _PyObject_CallNoArg(meth);
         Py_DECREF(meth);
         if (fno == NULL)
diff --git a/Objects/setobject.c b/Objects/setobject.c
index fafc2fa9e46d..924885d7505a 100644
--- a/Objects/setobject.c
+++ b/Objects/setobject.c
@@ -1970,9 +1970,10 @@ set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored))
     args = PyTuple_Pack(1, keys);
     if (args == NULL)
         goto done;
-    dict = _PyObject_GetAttrId((PyObject *)so, &PyId___dict__);
+    if (_PyObject_LookupAttrId((PyObject *)so, &PyId___dict__, &dict) < 0) {
+        goto done;
+    }
     if (dict == NULL) {
-        PyErr_Clear();
         dict = Py_None;
         Py_INCREF(dict);
     }
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index c816c3b6df23..1d8216d29887 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1551,17 +1551,10 @@ tail_contains(PyObject *tuple, int whence, PyObject *o)
 static PyObject *
 class_name(PyObject *cls)
 {
-    PyObject *name = _PyObject_GetAttrId(cls, &PyId___name__);
-    if (name == NULL) {
-        PyErr_Clear();
+    PyObject *name;
+    if (_PyObject_LookupAttrId(cls, &PyId___name__, &name) == 0) {
         name = PyObject_Repr(cls);
     }
-    if (name == NULL)
-        return NULL;
-    if (!PyUnicode_Check(name)) {
-        Py_DECREF(name);
-        return NULL;
-    }
     return name;
 }
 
@@ -1579,13 +1572,15 @@ check_duplicates(PyObject *tuple)
             if (PyTuple_GET_ITEM(tuple, j) == o) {
                 o = class_name(o);
                 if (o != NULL) {
-                    PyErr_Format(PyExc_TypeError,
-                                 "duplicate base class %U",
-                                 o);
+                    if (PyUnicode_Check(o)) {
+                        PyErr_Format(PyExc_TypeError,
+                                     "duplicate base class %U", o);
+                    }
+                    else {
+                        PyErr_SetString(PyExc_TypeError,
+                                        "duplicate base class");
+                    }
                     Py_DECREF(o);
-                } else {
-                    PyErr_SetString(PyExc_TypeError,
-                                 "duplicate base class");
                 }
                 return -1;
             }
@@ -1629,13 +1624,20 @@ consistent method resolution\norder (MRO) for bases");
     i = 0;
     while (PyDict_Next(set, &i, &k, &v) && (size_t)off < sizeof(buf)) {
         PyObject *name = class_name(k);
-        const char *name_str;
+        const char *name_str = NULL;
         if (name != NULL) {
-            name_str = PyUnicode_AsUTF8(name);
-            if (name_str == NULL)
+            if (PyUnicode_Check(name)) {
+                name_str = PyUnicode_AsUTF8(name);
+            }
+            else {
                 name_str = "?";
-        } else
-            name_str = "?";
+            }
+        }
+        if (name_str == NULL) {
+            Py_XDECREF(name);
+            Py_DECREF(set);
+            return;
+        }
         off += PyOS_snprintf(buf + off, sizeof(buf) - off, " %s", name_str);
         Py_XDECREF(name);
         if (--n && (size_t)(off+1) < sizeof(buf)) {
@@ -3422,10 +3424,10 @@ merge_class_dict(PyObject *dict, PyObject *aclass)
     assert(aclass);
 
     /* Merge in the type's dict (if any). */
-    classdict = _PyObject_GetAttrId(aclass, &PyId___dict__);
-    if (classdict == NULL)
-        PyErr_Clear();
-    else {
+    if (_PyObject_LookupAttrId(aclass, &PyId___dict__, &classdict) < 0) {
+        return -1;
+    }
+    if (classdict != NULL) {
         int status = PyDict_Update(dict, classdict);
         Py_DECREF(classdict);
         if (status < 0)
@@ -3433,15 +3435,17 @@ merge_class_dict(PyObject *dict, PyObject *aclass)
     }
 
     /* Recursively merge in the base types' (if any) dicts. */
-    bases = _PyObject_GetAttrId(aclass, &PyId___bases__);
-    if (bases == NULL)
-        PyErr_Clear();
-    else {
+    if (_PyObject_LookupAttrId(aclass, &PyId___bases__, &bases) < 0) {
+        return -1;
+    }
+    if (bases != NULL) {
         /* We have no guarantee that bases is a real tuple */
         Py_ssize_t i, n;
         n = PySequence_Size(bases); /* This better be right */
-        if (n < 0)
-            PyErr_Clear();
+        if (n < 0) {
+            Py_DECREF(bases);
+            return -1;
+        }
         else {
             for (i = 0; i < n; i++) {
                 int status;
@@ -4730,9 +4734,10 @@ object___dir___impl(PyObject *self)
     PyObject *itsclass = NULL;
 
     /* Get __dict__ (which may or may not be a real dict...) */
-    dict = _PyObject_GetAttrId(self, &PyId___dict__);
+    if (_PyObject_LookupAttrId(self, &PyId___dict__, &dict) < 0) {
+        return NULL;
+    }
     if (dict == NULL) {
-        PyErr_Clear();
         dict = PyDict_New();
     }
     else if (!PyDict_Check(dict)) {
@@ -4750,12 +4755,12 @@ object___dir___impl(PyObject *self)
         goto error;
 
     /* Merge in attrs reachable from its class. */
-    itsclass = _PyObject_GetAttrId(self, &PyId___class__);
-    if (itsclass == NULL)
-        /* XXX(tomer): Perhaps fall back to obj->ob_type if no
-                       __class__ exists? */
-        PyErr_Clear();
-    else if (merge_class_dict(dict, itsclass) != 0)
+    if (_PyObject_LookupAttrId(self, &PyId___class__, &itsclass) < 0) {
+        goto error;
+    }
+    /* XXX(tomer): Perhaps fall back to obj->ob_type if no
+                   __class__ exists? */
+    if (itsclass != NULL && merge_class_dict(dict, itsclass) < 0)
         goto error;
 
     result = PyDict_Keys(dict);
@@ -6111,16 +6116,19 @@ method_is_overloaded(PyObject *left, PyObject *right, struct _Py_Identifier *nam
     PyObject *a, *b;
     int ok;
 
-    b = _PyObject_GetAttrId((PyObject *)(Py_TYPE(right)), name);
+    if (_PyObject_LookupAttrId((PyObject *)(Py_TYPE(right)), name, &b) < 0) {
+        return -1;
+    }
     if (b == NULL) {
-        PyErr_Clear();
         /* If right doesn't have it, it's not overloaded */
         return 0;
     }
 
-    a = _PyObject_GetAttrId((PyObject *)(Py_TYPE(left)), name);
+    if (_PyObject_LookupAttrId((PyObject *)(Py_TYPE(left)), name, &a) < 0) {
+        Py_DECREF(b);
+        return -1;
+    }
     if (a == NULL) {
-        PyErr_Clear();
         Py_DECREF(b);
         /* If right has it but left doesn't, it's overloaded */
         return 1;
@@ -6129,11 +6137,6 @@ method_is_overloaded(PyObject *left, PyObject *right, struct _Py_Identifier *nam
     ok = PyObject_RichCompareBool(a, b, Py_NE);
     Py_DECREF(a);
     Py_DECREF(b);
-    if (ok < 0) {
-        PyErr_Clear();
-        return 0;
-    }
-
     return ok;
 }
 
@@ -6151,16 +6154,20 @@ FUNCNAME(PyObject *self, PyObject *other) \
     if (Py_TYPE(self)->tp_as_number != NULL && \
         Py_TYPE(self)->tp_as_number->SLOTNAME == TESTFUNC) { \
         PyObject *r; \
-        if (do_other && \
-            PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self)) && \
-            method_is_overloaded(self, other, &rop_id)) { \
-            stack[0] = other; \
-            stack[1] = self; \
-            r = vectorcall_maybe(&rop_id, stack, 2); \
-            if (r != Py_NotImplemented) \
-                return r; \
-            Py_DECREF(r); \
-            do_other = 0; \
+        if (do_other && PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) { \
+            int ok = method_is_overloaded(self, other, &rop_id); \
+            if (ok < 0) { \
+                return NULL; \
+            } \
+            if (ok) { \
+                stack[0] = other; \
+                stack[1] = self; \
+                r = vectorcall_maybe(&rop_id, stack, 2); \
+                if (r != Py_NotImplemented) \
+                    return r; \
+                Py_DECREF(r); \
+                do_other = 0; \
+            } \
         } \
         stack[0] = self; \
         stack[1] = other; \
@@ -7753,7 +7760,9 @@ supercheck(PyTypeObject *type, PyObject *obj)
         /* Try the slow way */
         PyObject *class_attr;
 
-        class_attr = _PyObject_GetAttrId(obj, &PyId___class__);
+        if (_PyObject_LookupAttrId(obj, &PyId___class__, &class_attr) < 0) {
+            return NULL;
+        }
         if (class_attr != NULL &&
             PyType_Check(class_attr) &&
             (PyTypeObject *)class_attr != Py_TYPE(obj))
@@ -7763,11 +7772,7 @@ supercheck(PyTypeObject *type, PyObject *obj)
             if (ok)
                 return (PyTypeObject *)class_attr;
         }
-
-        if (class_attr == NULL)
-            PyErr_Clear();
-        else
-            Py_DECREF(class_attr);
+        Py_XDECREF(class_attr);
     }
 
     PyErr_SetString(PyExc_TypeError,
diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c
index e8a429ab5b54..daee476444a4 100644
--- a/Objects/weakrefobject.c
+++ b/Objects/weakrefobject.c
@@ -163,10 +163,10 @@ weakref_repr(PyWeakReference *self)
     if (PyWeakref_GET_OBJECT(self) == Py_None)
         return PyUnicode_FromFormat("<weakref at %p; dead>", self);
 
-    name = _PyObject_GetAttrId(PyWeakref_GET_OBJECT(self), &PyId___name__);
+    if (_PyObject_LookupAttrId(PyWeakref_GET_OBJECT(self), &PyId___name__, &name) < 0) {
+        return NULL;
+    }
     if (name == NULL || !PyUnicode_Check(name)) {
-        if (name == NULL)
-            PyErr_Clear();
         repr = PyUnicode_FromFormat(
             "<weakref at %p; to '%s' at %p>",
             self,
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 63e58128651a..5053f7a1748f 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -2255,16 +2255,12 @@ builtin_vars(PyObject *self, PyObject *args)
         return NULL;
     if (v == NULL) {
         d = PyEval_GetLocals();
-        if (d == NULL)
-            return NULL;
-        Py_INCREF(d);
+        Py_XINCREF(d);
     }
     else {
-        d = _PyObject_GetAttrId(v, &PyId___dict__);
-        if (d == NULL) {
+        if (_PyObject_LookupAttrId(v, &PyId___dict__, &d) == 0) {
             PyErr_SetString(PyExc_TypeError,
                 "vars() argument must have __dict__ attribute");
-            return NULL;
         }
     }
     return d;
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 0635e9d834cd..8509aaf86693 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -224,16 +224,12 @@ PySys_Audit(const char *event, const char *argFormat, ...)
         ts->tracing++;
         ts->use_tracing = 0;
         while ((hook = PyIter_Next(hooks)) != NULL) {
+            _Py_IDENTIFIER(__cantrace__);
             PyObject *o;
-            int canTrace = -1;
-            o = PyObject_GetAttrString(hook, "__cantrace__");
+            int canTrace = _PyObject_LookupAttrId(hook, &PyId___cantrace__, &o);
             if (o) {
                 canTrace = PyObject_IsTrue(o);
                 Py_DECREF(o);
-            } else if (_PyErr_Occurred(ts) &&
-                       _PyErr_ExceptionMatches(ts, PyExc_AttributeError)) {
-                _PyErr_Clear(ts);
-                canTrace = 0;
             }
             if (canTrace < 0) {
                 break;
@@ -579,7 +575,10 @@ sys_displayhook_unencodable(PyThreadState *tstate, PyObject *outf, PyObject *o)
     if (encoded == NULL)
         goto error;
 
-    buffer = _PyObject_GetAttrId(outf, &PyId_buffer);
+    if (_PyObject_LookupAttrId(outf, &PyId_buffer, &buffer) < 0) {
+        Py_DECREF(encoded);
+        goto error;
+    }
     if (buffer) {
         result = _PyObject_CallMethodIdOneArg(buffer, &PyId_write, encoded);
         Py_DECREF(buffer);
@@ -589,7 +588,6 @@ sys_displayhook_unencodable(PyThreadState *tstate, PyObject *outf, PyObject *o)
         Py_DECREF(result);
     }
     else {
-        _PyErr_Clear(tstate);
         escaped_str = PyUnicode_FromEncodedObject(encoded,
                                                   stdout_encoding_str,
                                                   "strict");


From webhook-mailer at python.org  Sun Sep  1 05:11:47 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sun, 01 Sep 2019 09:11:47 -0000
Subject: [Python-checkins] bpo-15999: Always pass bool instead of int to the
 expat parser. (GH-15622)
Message-ID: <mailman.205.1567329108.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/eb8974616bc58f44b2a3c3e4ca2326894ae42c8f
commit: eb8974616bc58f44b2a3c3e4ca2326894ae42c8f
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-01T12:11:43+03:00
summary:

bpo-15999: Always pass bool instead of int to the expat parser. (GH-15622)

files:
M Lib/test/test_pyexpat.py
M Lib/xml/dom/expatbuilder.py
M Lib/xml/etree/ElementTree.py
M Lib/xml/sax/expatreader.py
M Lib/xmlrpc/client.py

diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py
index a0ba59008ece..b2b4dea06053 100644
--- a/Lib/test/test_pyexpat.py
+++ b/Lib/test/test_pyexpat.py
@@ -231,7 +231,7 @@ def test_parse_bytes(self):
         parser = expat.ParserCreate(namespace_separator='!')
         self._hookup_callbacks(parser, out)
 
-        parser.Parse(data, 1)
+        parser.Parse(data, True)
 
         operations = out.out
         self._verify_parse_output(operations)
@@ -243,7 +243,7 @@ def test_parse_str(self):
         parser = expat.ParserCreate(namespace_separator='!')
         self._hookup_callbacks(parser, out)
 
-        parser.Parse(data.decode('iso-8859-1'), 1)
+        parser.Parse(data.decode('iso-8859-1'), True)
 
         operations = out.out
         self._verify_parse_output(operations)
@@ -316,7 +316,7 @@ def collector(name, *args):
             L.append(name)
         p.StartElementHandler = collector
         p.EndElementHandler = collector
-        p.Parse(b"<e> <e/> <e></e> </e>", 1)
+        p.Parse(b"<e> <e/> <e></e> </e>", True)
         tag = L[0]
         self.assertEqual(len(L), 6)
         for entry in L:
@@ -332,14 +332,14 @@ def __init__(self, parser):
 
             def ExternalEntityRefHandler(self, context, base, sysId, pubId):
                 external_parser = self.parser.ExternalEntityParserCreate("")
-                self.parser_result = external_parser.Parse(b"", 1)
+                self.parser_result = external_parser.Parse(b"", True)
                 return 1
 
         parser = expat.ParserCreate(namespace_separator='!')
         parser.buffer_text = 1
         out = ExternalOutputter(parser)
         parser.ExternalEntityRefHandler = out.ExternalEntityRefHandler
-        parser.Parse(data, 1)
+        parser.Parse(data, True)
         self.assertEqual(out.parser_result, 1)
 
 
@@ -383,7 +383,7 @@ def test_default_to_disabled(self):
     def test_buffering_enabled(self):
         # Make sure buffering is turned on
         self.assertTrue(self.parser.buffer_text)
-        self.parser.Parse(b"<a>1<b/>2<c/>3</a>", 1)
+        self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True)
         self.assertEqual(self.stuff, ['123'],
                          "buffered text not properly collapsed")
 
@@ -391,39 +391,39 @@ def test1(self):
         # XXX This test exposes more detail of Expat's text chunking than we
         # XXX like, but it tests what we need to concisely.
         self.setHandlers(["StartElementHandler"])
-        self.parser.Parse(b"<a>1<b buffer-text='no'/>2\n3<c buffer-text='yes'/>4\n5</a>", 1)
+        self.parser.Parse(b"<a>1<b buffer-text='no'/>2\n3<c buffer-text='yes'/>4\n5</a>", True)
         self.assertEqual(self.stuff,
                          ["<a>", "1", "<b>", "2", "\n", "3", "<c>", "4\n5"],
                          "buffering control not reacting as expected")
 
     def test2(self):
-        self.parser.Parse(b"<a>1<b/>&lt;2&gt;<c/>&#32;\n&#x20;3</a>", 1)
+        self.parser.Parse(b"<a>1<b/>&lt;2&gt;<c/>&#32;\n&#x20;3</a>", True)
         self.assertEqual(self.stuff, ["1<2> \n 3"],
                          "buffered text not properly collapsed")
 
     def test3(self):
         self.setHandlers(["StartElementHandler"])
-        self.parser.Parse(b"<a>1<b/>2<c/>3</a>", 1)
+        self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True)
         self.assertEqual(self.stuff, ["<a>", "1", "<b>", "2", "<c>", "3"],
                          "buffered text not properly split")
 
     def test4(self):
         self.setHandlers(["StartElementHandler", "EndElementHandler"])
         self.parser.CharacterDataHandler = None
-        self.parser.Parse(b"<a>1<b/>2<c/>3</a>", 1)
+        self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True)
         self.assertEqual(self.stuff,
                          ["<a>", "<b>", "</b>", "<c>", "</c>", "</a>"])
 
     def test5(self):
         self.setHandlers(["StartElementHandler", "EndElementHandler"])
-        self.parser.Parse(b"<a>1<b></b>2<c/>3</a>", 1)
+        self.parser.Parse(b"<a>1<b></b>2<c/>3</a>", True)
         self.assertEqual(self.stuff,
             ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3", "</a>"])
 
     def test6(self):
         self.setHandlers(["CommentHandler", "EndElementHandler",
                     "StartElementHandler"])
-        self.parser.Parse(b"<a>1<b/>2<c></c>345</a> ", 1)
+        self.parser.Parse(b"<a>1<b/>2<c></c>345</a> ", True)
         self.assertEqual(self.stuff,
             ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "345", "</a>"],
             "buffered text not properly split")
@@ -431,7 +431,7 @@ def test6(self):
     def test7(self):
         self.setHandlers(["CommentHandler", "EndElementHandler",
                     "StartElementHandler"])
-        self.parser.Parse(b"<a>1<b/>2<c></c>3<!--abc-->4<!--def-->5</a> ", 1)
+        self.parser.Parse(b"<a>1<b/>2<c></c>3<!--abc-->4<!--def-->5</a> ", True)
         self.assertEqual(self.stuff,
                          ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3",
                           "<!--abc-->", "4", "<!--def-->", "5", "</a>"],
@@ -451,7 +451,7 @@ def test_exception(self):
         parser = expat.ParserCreate()
         parser.StartElementHandler = self.StartElementHandler
         try:
-            parser.Parse(b"<a><b><c/></b></a>", 1)
+            parser.Parse(b"<a><b><c/></b></a>", True)
             self.fail()
         except RuntimeError as e:
             self.assertEqual(e.args[0], 'a',
@@ -499,7 +499,7 @@ def test(self):
                               ('e', 15, 3, 6), ('e', 17, 4, 1), ('e', 22, 5, 0)]
 
         xml = b'<a>\n <b>\n  <c/>\n </b>\n</a>'
-        self.parser.Parse(xml, 1)
+        self.parser.Parse(xml, True)
 
 
 class sf1296433Test(unittest.TestCase):
@@ -579,7 +579,7 @@ def test_disabling_buffer(self):
 
         # Parse one chunk of XML
         self.n = 0
-        parser.Parse(xml1, 0)
+        parser.Parse(xml1, False)
         self.assertEqual(parser.buffer_size, 1024)
         self.assertEqual(self.n, 1)
 
@@ -588,13 +588,13 @@ def test_disabling_buffer(self):
         self.assertFalse(parser.buffer_text)
         self.assertEqual(parser.buffer_size, 1024)
         for i in range(10):
-            parser.Parse(xml2, 0)
+            parser.Parse(xml2, False)
         self.assertEqual(self.n, 11)
 
         parser.buffer_text = 1
         self.assertTrue(parser.buffer_text)
         self.assertEqual(parser.buffer_size, 1024)
-        parser.Parse(xml3, 1)
+        parser.Parse(xml3, True)
         self.assertEqual(self.n, 12)
 
     def counting_handler(self, text):
@@ -621,10 +621,10 @@ def test_change_size_1(self):
         self.assertEqual(parser.buffer_size, 1024)
 
         self.n = 0
-        parser.Parse(xml1, 0)
+        parser.Parse(xml1, False)
         parser.buffer_size *= 2
         self.assertEqual(parser.buffer_size, 2048)
-        parser.Parse(xml2, 1)
+        parser.Parse(xml2, True)
         self.assertEqual(self.n, 2)
 
     def test_change_size_2(self):
@@ -637,10 +637,10 @@ def test_change_size_2(self):
         self.assertEqual(parser.buffer_size, 2048)
 
         self.n=0
-        parser.Parse(xml1, 0)
+        parser.Parse(xml1, False)
         parser.buffer_size = parser.buffer_size // 2
         self.assertEqual(parser.buffer_size, 1024)
-        parser.Parse(xml2, 1)
+        parser.Parse(xml2, True)
         self.assertEqual(self.n, 4)
 
 class MalformedInputTest(unittest.TestCase):
diff --git a/Lib/xml/dom/expatbuilder.py b/Lib/xml/dom/expatbuilder.py
index 2bd835b03594..199c22d0af34 100644
--- a/Lib/xml/dom/expatbuilder.py
+++ b/Lib/xml/dom/expatbuilder.py
@@ -204,11 +204,11 @@ def parseFile(self, file):
                 buffer = file.read(16*1024)
                 if not buffer:
                     break
-                parser.Parse(buffer, 0)
+                parser.Parse(buffer, False)
                 if first_buffer and self.document.documentElement:
                     self._setup_subset(buffer)
                 first_buffer = False
-            parser.Parse("", True)
+            parser.Parse(b"", True)
         except ParseEscape:
             pass
         doc = self.document
@@ -637,7 +637,7 @@ def parseString(self, string):
         nsattrs = self._getNSattrs() # get ns decls from node's ancestors
         document = _FRAGMENT_BUILDER_TEMPLATE % (ident, subset, nsattrs)
         try:
-            parser.Parse(document, 1)
+            parser.Parse(document, True)
         except:
             self.reset()
             raise
@@ -697,7 +697,7 @@ def external_entity_ref_handler(self, context, base, systemId, publicId):
             self.fragment = self.document.createDocumentFragment()
             self.curNode = self.fragment
             try:
-                parser.Parse(self._source, 1)
+                parser.Parse(self._source, True)
             finally:
                 self.curNode = old_cur_node
                 self.document = old_document
diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py
index beb2d68803a5..e75200a1b8a3 100644
--- a/Lib/xml/etree/ElementTree.py
+++ b/Lib/xml/etree/ElementTree.py
@@ -1710,14 +1710,14 @@ def _default(self, text):
     def feed(self, data):
         """Feed encoded data to parser."""
         try:
-            self.parser.Parse(data, 0)
+            self.parser.Parse(data, False)
         except self._error as v:
             self._raiseerror(v)
 
     def close(self):
         """Finish feeding data to parser and return element structure."""
         try:
-            self.parser.Parse("", 1) # end of data
+            self.parser.Parse(b"", True) # end of data
         except self._error as v:
             self._raiseerror(v)
         try:
diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py
index 5066ffc2fa51..e334ac9fea0d 100644
--- a/Lib/xml/sax/expatreader.py
+++ b/Lib/xml/sax/expatreader.py
@@ -93,7 +93,7 @@ def __init__(self, namespaceHandling=0, bufsize=2**16-20):
         self._parser = None
         self._namespaces = namespaceHandling
         self._lex_handler_prop = None
-        self._parsing = 0
+        self._parsing = False
         self._entity_stack = []
         self._external_ges = 0
         self._interning = None
@@ -203,10 +203,10 @@ def setProperty(self, name, value):
 
     # IncrementalParser methods
 
-    def feed(self, data, isFinal = 0):
+    def feed(self, data, isFinal=False):
         if not self._parsing:
             self.reset()
-            self._parsing = 1
+            self._parsing = True
             self._cont_handler.startDocument()
 
         try:
@@ -237,13 +237,13 @@ def close(self):
             # If we are completing an external entity, do nothing here
             return
         try:
-            self.feed("", isFinal = 1)
+            self.feed(b"", isFinal=True)
             self._cont_handler.endDocument()
-            self._parsing = 0
+            self._parsing = False
             # break cycle created by expat handlers pointing to our methods
             self._parser = None
         finally:
-            self._parsing = 0
+            self._parsing = False
             if self._parser is not None:
                 # Keep ErrorColumnNumber and ErrorLineNumber after closing.
                 parser = _ClosedParser()
@@ -307,7 +307,7 @@ def reset(self):
         self._parser.SetParamEntityParsing(
             expat.XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE)
 
-        self._parsing = 0
+        self._parsing = False
         self._entity_stack = []
 
     # Locator methods
diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py
index 246ef27ffc24..b4b2941ea5b4 100644
--- a/Lib/xmlrpc/client.py
+++ b/Lib/xmlrpc/client.py
@@ -442,7 +442,7 @@ def __init__(self, target):
         target.xml(encoding, None)
 
     def feed(self, data):
-        self._parser.Parse(data, 0)
+        self._parser.Parse(data, False)
 
     def close(self):
         try:


From webhook-mailer at python.org  Sun Sep  1 05:12:55 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sun, 01 Sep 2019 09:12:55 -0000
Subject: [Python-checkins] bpo-15999: Always pass bool instead of int to
 socket.setblocking(). (GH-15621)
Message-ID: <mailman.206.1567329176.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5eca7f3f3836cc734dfe8dc5ec669f3b4e9333fe
commit: 5eca7f3f3836cc734dfe8dc5ec669f3b4e9333fe
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-01T12:12:52+03:00
summary:

bpo-15999: Always pass bool instead of int to socket.setblocking(). (GH-15621)

files:
M Doc/howto/sockets.rst
M Lib/asyncore.py
M Lib/test/test_asyncio/functional.py
M Lib/test/test_socket.py
M Lib/test/test_ssl.py
M Lib/test/test_timeout.py
M Modules/socketmodule.c

diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst
index bc71d85a83e9..4655f2806089 100644
--- a/Doc/howto/sockets.rst
+++ b/Doc/howto/sockets.rst
@@ -317,7 +317,7 @@ know about the mechanics of using sockets. You'll still use the same calls, in
 much the same ways. It's just that, if you do it right, your app will be almost
 inside-out.
 
-In Python, you use ``socket.setblocking(0)`` to make it non-blocking. In C, it's
+In Python, you use ``socket.setblocking(False)`` to make it non-blocking. In C, it's
 more complex, (for one thing, you'll need to choose between the BSD flavor
 ``O_NONBLOCK`` and the almost indistinguishable Posix flavor ``O_NDELAY``, which
 is completely different from ``TCP_NODELAY``), but it's the exact same idea. You
diff --git a/Lib/asyncore.py b/Lib/asyncore.py
index 0e92be3ad191..ce16f11a2f5c 100644
--- a/Lib/asyncore.py
+++ b/Lib/asyncore.py
@@ -228,7 +228,7 @@ def __init__(self, sock=None, map=None):
         if sock:
             # Set to nonblocking just to make sure for cases where we
             # get a socket from a blocking source.
-            sock.setblocking(0)
+            sock.setblocking(False)
             self.set_socket(sock, map)
             self.connected = True
             # The constructor no longer requires that the socket
@@ -280,7 +280,7 @@ def del_channel(self, map=None):
     def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM):
         self.family_and_type = family, type
         sock = socket.socket(family, type)
-        sock.setblocking(0)
+        sock.setblocking(False)
         self.set_socket(sock)
 
     def set_socket(self, sock, map=None):
diff --git a/Lib/test/test_asyncio/functional.py b/Lib/test/test_asyncio/functional.py
index 70cd140f4796..4620d3e7a331 100644
--- a/Lib/test/test_asyncio/functional.py
+++ b/Lib/test/test_asyncio/functional.py
@@ -225,7 +225,7 @@ def stop(self):
     def run(self):
         try:
             with self._sock:
-                self._sock.setblocking(0)
+                self._sock.setblocking(False)
                 self._run()
         finally:
             self._s1.close()
diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
index ce816cd603ec..14ff561da94f 100644
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -4597,7 +4597,7 @@ def _testInheritFlagsTimeout(self):
 
     def testAccept(self):
         # Testing non-blocking accept
-        self.serv.setblocking(0)
+        self.serv.setblocking(False)
 
         # connect() didn't start: non-blocking accept() fails
         start_time = time.monotonic()
@@ -4628,7 +4628,7 @@ def testRecv(self):
         # Testing non-blocking recv
         conn, addr = self.serv.accept()
         self.addCleanup(conn.close)
-        conn.setblocking(0)
+        conn.setblocking(False)
 
         # the server didn't send data yet: non-blocking recv() fails
         with self.assertRaises(BlockingIOError):
@@ -5698,15 +5698,15 @@ def test_SOCK_NONBLOCK(self):
         with socket.socket(socket.AF_INET,
                            socket.SOCK_STREAM | socket.SOCK_NONBLOCK) as s:
             self.checkNonblock(s)
-            s.setblocking(1)
+            s.setblocking(True)
             self.checkNonblock(s, nonblock=False)
-            s.setblocking(0)
+            s.setblocking(False)
             self.checkNonblock(s)
             s.settimeout(None)
             self.checkNonblock(s, nonblock=False)
             s.settimeout(2.0)
             self.checkNonblock(s, timeout=2.0)
-            s.setblocking(1)
+            s.setblocking(True)
             self.checkNonblock(s, nonblock=False)
         # defaulttimeout
         t = socket.getdefaulttimeout()
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index afc5be9a7e99..6ad0b61d8a57 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -2209,7 +2209,7 @@ def __init__(self, server, connsock, addr):
             self.running = False
             self.sock = connsock
             self.addr = addr
-            self.sock.setblocking(1)
+            self.sock.setblocking(True)
             self.sslconn = None
             threading.Thread.__init__(self)
             self.daemon = True
@@ -3255,7 +3255,7 @@ def test_starttls(self):
         wrapped = False
         with server:
             s = socket.socket()
-            s.setblocking(1)
+            s.setblocking(True)
             s.connect((HOST, server.port))
             if support.verbose:
                 sys.stdout.write("\n")
diff --git a/Lib/test/test_timeout.py b/Lib/test/test_timeout.py
index b07c07cbfc4d..0fe4c7ab6e75 100644
--- a/Lib/test/test_timeout.py
+++ b/Lib/test/test_timeout.py
@@ -79,24 +79,24 @@ def testRangeCheck(self):
     def testTimeoutThenBlocking(self):
         # Test settimeout() followed by setblocking()
         self.sock.settimeout(10)
-        self.sock.setblocking(1)
+        self.sock.setblocking(True)
         self.assertEqual(self.sock.gettimeout(), None)
-        self.sock.setblocking(0)
+        self.sock.setblocking(False)
         self.assertEqual(self.sock.gettimeout(), 0.0)
 
         self.sock.settimeout(10)
-        self.sock.setblocking(0)
+        self.sock.setblocking(False)
         self.assertEqual(self.sock.gettimeout(), 0.0)
-        self.sock.setblocking(1)
+        self.sock.setblocking(True)
         self.assertEqual(self.sock.gettimeout(), None)
 
     def testBlockingThenTimeout(self):
         # Test setblocking() followed by settimeout()
-        self.sock.setblocking(0)
+        self.sock.setblocking(False)
         self.sock.settimeout(1)
         self.assertEqual(self.sock.gettimeout(), 1)
 
-        self.sock.setblocking(1)
+        self.sock.setblocking(True)
         self.sock.settimeout(1)
         self.assertEqual(self.sock.gettimeout(), 1)
 
diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c
index d4f2098e1e6f..3548b0c87875 100644
--- a/Modules/socketmodule.c
+++ b/Modules/socketmodule.c
@@ -146,7 +146,7 @@ recvfrom_into(buffer[, nbytes, [, flags])\n\
 sendall(data[, flags]) -- send all data\n\
 send(data[, flags]) -- send data, may not send all of it\n\
 sendto(data[, flags], addr) -- send data to a given address\n\
-setblocking(0 | 1) -- set or clear the blocking I/O flag\n\
+setblocking(bool) -- set or clear the blocking I/O flag\n\
 getblocking() -- return True if socket is blocking, False if non-blocking\n\
 setsockopt(level, optname, value[, optlen]) -- set socket options\n\
 settimeout(None | float) -- set or clear the timeout\n\


From webhook-mailer at python.org  Sun Sep  1 05:16:54 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sun, 01 Sep 2019 09:16:54 -0000
Subject: [Python-checkins] bpo-15999: Clean up of handling boolean
 arguments. (GH-15610)
Message-ID: <mailman.207.1567329416.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1f21eaa15e8a0d2b0f78d0e3f2b9e5b458eb0a70
commit: 1f21eaa15e8a0d2b0f78d0e3f2b9e5b458eb0a70
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-01T12:16:51+03:00
summary:

bpo-15999: Clean up of handling boolean arguments. (GH-15610)

* Use the 'p' format unit instead of manually called PyObject_IsTrue().
* Pass boolean value instead 0/1 integers to functions that needs boolean.
* Convert some arguments to boolean only once.

files:
M Lib/_pyio.py
M Lib/codeop.py
M Lib/doctest.py
M Lib/quopri.py
M Lib/test/datetimetester.py
M Lib/test/lock_tests.py
M Lib/test/test_builtin.py
M Lib/test/test_ioctl.py
M Lib/test/test_ordered_dict.py
M Lib/test/test_unicode.py
M Lib/threading.py
M Lib/tkinter/__init__.py
M Modules/_io/_iomodule.c
M Modules/_io/stringio.c
M Modules/_io/textio.c
M Modules/itertoolsmodule.c
M Modules/main.c
M Modules/parsermodule.c
M Objects/fileobject.c
M Python/bltinmodule.c
M Python/pylifecycle.c

diff --git a/Lib/_pyio.py b/Lib/_pyio.py
index 0d3f974c0724..c1bdac791319 100644
--- a/Lib/_pyio.py
+++ b/Lib/_pyio.py
@@ -2295,7 +2295,7 @@ def _read_chunk(self):
         return not eof
 
     def _pack_cookie(self, position, dec_flags=0,
-                           bytes_to_feed=0, need_eof=0, chars_to_skip=0):
+                           bytes_to_feed=0, need_eof=False, chars_to_skip=0):
         # The meaning of a tell() cookie is: seek to position, set the
         # decoder flags to dec_flags, read bytes_to_feed bytes, feed them
         # into the decoder with need_eof as the EOF flag, then skip
@@ -2309,7 +2309,7 @@ def _unpack_cookie(self, bigint):
         rest, dec_flags = divmod(rest, 1<<64)
         rest, bytes_to_feed = divmod(rest, 1<<64)
         need_eof, chars_to_skip = divmod(rest, 1<<64)
-        return position, dec_flags, bytes_to_feed, need_eof, chars_to_skip
+        return position, dec_flags, bytes_to_feed, bool(need_eof), chars_to_skip
 
     def tell(self):
         if not self._seekable:
@@ -2383,7 +2383,7 @@ def tell(self):
             # (a point where the decoder has nothing buffered, so seek()
             # can safely start from there and advance to this location).
             bytes_fed = 0
-            need_eof = 0
+            need_eof = False
             # Chars decoded since `start_pos`
             chars_decoded = 0
             for i in range(skip_bytes, len(next_input)):
@@ -2400,7 +2400,7 @@ def tell(self):
             else:
                 # We didn't get enough decoded data; signal EOF to get more.
                 chars_decoded += len(decoder.decode(b'', final=True))
-                need_eof = 1
+                need_eof = True
                 if chars_decoded < chars_to_skip:
                     raise OSError("can't reconstruct logical file position")
 
diff --git a/Lib/codeop.py b/Lib/codeop.py
index fb759da42ad1..e5c7adea54fb 100644
--- a/Lib/codeop.py
+++ b/Lib/codeop.py
@@ -130,7 +130,7 @@ def __init__(self):
         self.flags = PyCF_DONT_IMPLY_DEDENT
 
     def __call__(self, source, filename, symbol):
-        codeob = compile(source, filename, symbol, self.flags, 1)
+        codeob = compile(source, filename, symbol, self.flags, True)
         for feature in _features:
             if codeob.co_flags & feature.compiler_flag:
                 self.flags |= feature.compiler_flag
diff --git a/Lib/doctest.py b/Lib/doctest.py
index bf4889f59e0d..8fca6280b8aa 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -1326,7 +1326,7 @@ def __run(self, test, compileflags, out):
             try:
                 # Don't blink!  This is where the user's code gets run.
                 exec(compile(example.source, filename, "single",
-                             compileflags, 1), test.globs)
+                             compileflags, True), test.globs)
                 self.debugger.set_continue() # ==== Example Finished ====
                 exception = None
             except KeyboardInterrupt:
diff --git a/Lib/quopri.py b/Lib/quopri.py
index cbd979abdffc..08899c5cb73a 100755
--- a/Lib/quopri.py
+++ b/Lib/quopri.py
@@ -204,11 +204,11 @@ def main():
         print("-t: quote tabs")
         print("-d: decode; default encode")
         sys.exit(2)
-    deco = 0
-    tabs = 0
+    deco = False
+    tabs = False
     for o, a in opts:
-        if o == '-t': tabs = 1
-        if o == '-d': deco = 1
+        if o == '-t': tabs = True
+        if o == '-d': deco = True
     if tabs and deco:
         sys.stdout = sys.stderr
         print("-t and -d are mutually exclusive")
diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index d09255d79bfb..d5de5e91a982 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -6160,7 +6160,7 @@ class TZInfoSubclass(tzinfo):
     def test_date_from_date(self):
         exp_date = date(1993, 8, 26)
 
-        for macro in [0, 1]:
+        for macro in False, True:
             with self.subTest(macro=macro):
                 c_api_date = _testcapi.get_date_fromdate(
                     macro,
@@ -6173,7 +6173,7 @@ def test_date_from_date(self):
     def test_datetime_from_dateandtime(self):
         exp_date = datetime(1993, 8, 26, 22, 12, 55, 99999)
 
-        for macro in [0, 1]:
+        for macro in False, True:
             with self.subTest(macro=macro):
                 c_api_date = _testcapi.get_datetime_fromdateandtime(
                     macro,
@@ -6191,7 +6191,7 @@ def test_datetime_from_dateandtimeandfold(self):
         exp_date = datetime(1993, 8, 26, 22, 12, 55, 99999)
 
         for fold in [0, 1]:
-            for macro in [0, 1]:
+            for macro in False, True:
                 with self.subTest(macro=macro, fold=fold):
                     c_api_date = _testcapi.get_datetime_fromdateandtimeandfold(
                         macro,
@@ -6210,7 +6210,7 @@ def test_datetime_from_dateandtimeandfold(self):
     def test_time_from_time(self):
         exp_time = time(22, 12, 55, 99999)
 
-        for macro in [0, 1]:
+        for macro in False, True:
             with self.subTest(macro=macro):
                 c_api_time = _testcapi.get_time_fromtime(
                     macro,
@@ -6225,7 +6225,7 @@ def test_time_from_timeandfold(self):
         exp_time = time(22, 12, 55, 99999)
 
         for fold in [0, 1]:
-            for macro in [0, 1]:
+            for macro in False, True:
                 with self.subTest(macro=macro, fold=fold):
                     c_api_time = _testcapi.get_time_fromtimeandfold(
                         macro,
@@ -6241,7 +6241,7 @@ def test_time_from_timeandfold(self):
     def test_delta_from_dsu(self):
         exp_delta = timedelta(26, 55, 99999)
 
-        for macro in [0, 1]:
+        for macro in False, True:
             with self.subTest(macro=macro):
                 c_api_delta = _testcapi.get_delta_fromdsu(
                     macro,
@@ -6254,7 +6254,7 @@ def test_delta_from_dsu(self):
     def test_date_from_timestamp(self):
         ts = datetime(1995, 4, 12).timestamp()
 
-        for macro in [0, 1]:
+        for macro in False, True:
             with self.subTest(macro=macro):
                 d = _testcapi.get_date_fromtimestamp(int(ts), macro)
 
@@ -6272,7 +6272,7 @@ def test_datetime_from_timestamp(self):
 
         from_timestamp = _testcapi.get_datetime_fromtimestamp
         for case in cases:
-            for macro in [0, 1]:
+            for macro in False, True:
                 with self.subTest(case=case, macro=macro):
                     dtup, tzinfo, usetz = case
                     dt_orig = datetime(*dtup, tzinfo=tzinfo)
diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py
index d12a86e7e934..cd1155d34e99 100644
--- a/Lib/test/lock_tests.py
+++ b/Lib/test/lock_tests.py
@@ -182,7 +182,7 @@ def f():
     def test_timeout(self):
         lock = self.locktype()
         # Can't set timeout if not blocking
-        self.assertRaises(ValueError, lock.acquire, 0, 1)
+        self.assertRaises(ValueError, lock.acquire, False, 1)
         # Invalid timeout values
         self.assertRaises(ValueError, lock.acquire, timeout=-100)
         self.assertRaises(OverflowError, lock.acquire, timeout=1e100)
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 1100c49e9b88..db2f6cd49b2c 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -320,8 +320,8 @@ def test_compile(self):
         bom = b'\xef\xbb\xbf'
         compile(bom + b'print(1)\n', '', 'exec')
         compile(source='pass', filename='?', mode='exec')
-        compile(dont_inherit=0, filename='tmp', source='0', mode='eval')
-        compile('pass', '?', dont_inherit=1, mode='exec')
+        compile(dont_inherit=False, filename='tmp', source='0', mode='eval')
+        compile('pass', '?', dont_inherit=True, mode='exec')
         compile(memoryview(b"text"), "name", "exec")
         self.assertRaises(TypeError, compile)
         self.assertRaises(ValueError, compile, 'print(42)\n', '<string>', 'badmode')
@@ -1853,7 +1853,7 @@ def test_basic(self):
         self.assertEqual(data, sorted(copy, key=lambda x: -x))
         self.assertNotEqual(data, copy)
         random.shuffle(copy)
-        self.assertEqual(data, sorted(copy, reverse=1))
+        self.assertEqual(data, sorted(copy, reverse=True))
         self.assertNotEqual(data, copy)
 
     def test_bad_arguments(self):
diff --git a/Lib/test/test_ioctl.py b/Lib/test/test_ioctl.py
index d1a5db90810d..a2873582cf98 100644
--- a/Lib/test/test_ioctl.py
+++ b/Lib/test/test_ioctl.py
@@ -48,7 +48,7 @@ def _check_ioctl_mutate_len(self, nbytes=None):
         else:
             buf.append(fill)
         with open("/dev/tty", "rb") as tty:
-            r = fcntl.ioctl(tty, termios.TIOCGPGRP, buf, 1)
+            r = fcntl.ioctl(tty, termios.TIOCGPGRP, buf, True)
         rpgrp = buf[0]
         self.assertEqual(r, 0)
         self.assertIn(rpgrp, ids)
diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py
index 148a9bdc35ee..f337be8eb855 100644
--- a/Lib/test/test_ordered_dict.py
+++ b/Lib/test/test_ordered_dict.py
@@ -407,9 +407,9 @@ def test_move_to_end(self):
         self.assertEqual(list(od), list('abcde'))
         od.move_to_end('c')
         self.assertEqual(list(od), list('abdec'))
-        od.move_to_end('c', 0)
+        od.move_to_end('c', False)
         self.assertEqual(list(od), list('cabde'))
-        od.move_to_end('c', 0)
+        od.move_to_end('c', False)
         self.assertEqual(list(od), list('cabde'))
         od.move_to_end('e')
         self.assertEqual(list(od), list('cabde'))
@@ -418,7 +418,7 @@ def test_move_to_end(self):
         with self.assertRaises(KeyError):
             od.move_to_end('x')
         with self.assertRaises(KeyError):
-            od.move_to_end('x', 0)
+            od.move_to_end('x', False)
 
     def test_move_to_end_issue25406(self):
         OrderedDict = self.OrderedDict
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index 7bd7f51b592b..8636f2b0bd50 100644
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -2820,15 +2820,15 @@ def test_asucs4(self):
         for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600',
                   'a\ud800b\udfffc', '\ud834\udd1e']:
             l = len(s)
-            self.assertEqual(unicode_asucs4(s, l, 1), s+'\0')
-            self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff')
-            self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff')
-            self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff')
-            self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1)
-            self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0)
+            self.assertEqual(unicode_asucs4(s, l, True), s+'\0')
+            self.assertEqual(unicode_asucs4(s, l, False), s+'\uffff')
+            self.assertEqual(unicode_asucs4(s, l+1, True), s+'\0\uffff')
+            self.assertEqual(unicode_asucs4(s, l+1, False), s+'\0\uffff')
+            self.assertRaises(SystemError, unicode_asucs4, s, l-1, True)
+            self.assertRaises(SystemError, unicode_asucs4, s, l-2, False)
             s = '\0'.join([s, s])
-            self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0')
-            self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff')
+            self.assertEqual(unicode_asucs4(s, len(s), True), s+'\0')
+            self.assertEqual(unicode_asucs4(s, len(s), False), s+'\uffff')
 
     # Test PyUnicode_AsUTF8()
     @support.cpython_only
diff --git a/Lib/threading.py b/Lib/threading.py
index c3bbb19acbe2..59323679c80d 100644
--- a/Lib/threading.py
+++ b/Lib/threading.py
@@ -262,7 +262,7 @@ def _acquire_restore(self, x):
     def _is_owned(self):
         # Return True if lock is owned by current_thread.
         # This method is called only if _lock doesn't have _is_owned().
-        if self._lock.acquire(0):
+        if self._lock.acquire(False):
             self._lock.release()
             return False
         else:
diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py
index 9258484af5fd..5d5cf90e0bdd 100644
--- a/Lib/tkinter/__init__.py
+++ b/Lib/tkinter/__init__.py
@@ -2241,7 +2241,7 @@ class Tk(Misc, Wm):
     _w = '.'
 
     def __init__(self, screenName=None, baseName=None, className='Tk',
-                 useTk=1, sync=0, use=None):
+                 useTk=True, sync=False, use=None):
         """Return a new Toplevel widget on screen SCREENNAME. A new Tcl interpreter will
         be created. BASENAME will be used for the identification of the profile file (see
         readprofile).
@@ -2259,7 +2259,7 @@ def __init__(self, screenName=None, baseName=None, className='Tk',
             baseName, ext = os.path.splitext(baseName)
             if ext not in ('.py', '.pyc'):
                 baseName = baseName + ext
-        interactive = 0
+        interactive = False
         self.tk = _tkinter.create(screenName, baseName, className, interactive, wantobjects, useTk, sync, use)
         if useTk:
             self._loadtk()
@@ -2361,7 +2361,7 @@ def __getattr__(self, attr):
 # copied into the Pack, Place or Grid class.
 
 
-def Tcl(screenName=None, baseName=None, className='Tk', useTk=0):
+def Tcl(screenName=None, baseName=None, className='Tk', useTk=False):
     return Tk(screenName, baseName, className, useTk)
 
 
diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c
index 96426e0276ab..162b288852b3 100644
--- a/Modules/_io/_iomodule.c
+++ b/Modules/_io/_iomodule.c
@@ -383,8 +383,10 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode,
             encoding = "utf-8";
         }
 #endif
-        raw = PyObject_CallFunction(RawIO_class,
-                                    "OsiO", path_or_fd, rawmode, closefd, opener);
+        raw = PyObject_CallFunction(RawIO_class, "OsOO",
+                                    path_or_fd, rawmode,
+                                    closefd ? Py_True : Py_False,
+                                    opener);
     }
 
     if (raw == NULL)
@@ -476,10 +478,10 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode,
 
     /* wraps into a TextIOWrapper */
     wrapper = PyObject_CallFunction((PyObject *)&PyTextIOWrapper_Type,
-                                    "Osssi",
+                                    "OsssO",
                                     buffer,
                                     encoding, errors, newline,
-                                    line_buffering);
+                                    line_buffering ? Py_True : Py_False);
     if (wrapper == NULL)
         goto error;
     result = wrapper;
diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c
index 3f7afad56347..89b29bb8fbb7 100644
--- a/Modules/_io/stringio.c
+++ b/Modules/_io/stringio.c
@@ -714,9 +714,9 @@ _io_StringIO___init___impl(stringio *self, PyObject *value,
     }
 
     if (self->readuniversal) {
-        self->decoder = PyObject_CallFunction(
+        self->decoder = PyObject_CallFunctionObjArgs(
             (PyObject *)&PyIncrementalNewlineDecoder_Type,
-            "Oi", Py_None, (int) self->readtranslate);
+            Py_None, self->readtranslate ? Py_True : Py_False, NULL);
         if (self->decoder == NULL)
             return -1;
     }
diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c
index 7ddac8062a96..1db8d94ed12a 100644
--- a/Modules/_io/textio.c
+++ b/Modules/_io/textio.c
@@ -880,9 +880,9 @@ _textiowrapper_set_decoder(textio *self, PyObject *codec_info,
         return -1;
 
     if (self->readuniversal) {
-        PyObject *incrementalDecoder = PyObject_CallFunction(
+        PyObject *incrementalDecoder = PyObject_CallFunctionObjArgs(
             (PyObject *)&PyIncrementalNewlineDecoder_Type,
-            "Oi", self->decoder, (int)self->readtranslate);
+            self->decoder, self->readtranslate ? Py_True : Py_False, NULL);
         if (incrementalDecoder == NULL)
             return -1;
         Py_CLEAR(self->decoder);
@@ -2591,8 +2591,8 @@ _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence)
         }
         Py_XSETREF(self->snapshot, snapshot);
 
-        decoded = _PyObject_CallMethodId(self->decoder, &PyId_decode,
-            "Oi", input_chunk, (int)cookie.need_eof);
+        decoded = _PyObject_CallMethodIdObjArgs(self->decoder, &PyId_decode,
+            input_chunk, cookie.need_eof ? Py_True : Py_False, NULL);
 
         if (check_decoded(decoded) < 0)
             goto fail;
@@ -2819,7 +2819,7 @@ _io_TextIOWrapper_tell_impl(textio *self)
     if (input == input_end) {
         /* We didn't get enough decoded data; signal EOF to get more. */
         PyObject *decoded = _PyObject_CallMethodId(
-            self->decoder, &PyId_decode, "yi", "", /* final = */ 1);
+            self->decoder, &PyId_decode, "yO", "", /* final = */ Py_True);
         if (check_decoded(decoded) < 0)
             goto fail;
         chars_decoded += PyUnicode_GET_LENGTH(decoded);
diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c
index ab473e29fbd5..2e398425a146 100644
--- a/Modules/itertoolsmodule.c
+++ b/Modules/itertoolsmodule.c
@@ -1059,10 +1059,10 @@ cycle_reduce(cycleobject *lz, PyObject *Py_UNUSED(ignored))
             }
             Py_DECREF(res);
         }
-        return Py_BuildValue("O(N)(Oi)", Py_TYPE(lz), it, lz->saved, 1);
+        return Py_BuildValue("O(N)(OO)", Py_TYPE(lz), it, lz->saved, Py_True);
     }
-    return Py_BuildValue("O(O)(Oi)", Py_TYPE(lz), lz->it, lz->saved,
-                         lz->firstpass);
+    return Py_BuildValue("O(O)(OO)", Py_TYPE(lz), lz->it, lz->saved,
+                         lz->firstpass ? Py_True : Py_False);
 }
 
 static PyObject *
diff --git a/Modules/main.c b/Modules/main.c
index b8a1c9b79ce3..fdc7e0dabf17 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -292,7 +292,7 @@ pymain_run_module(const wchar_t *modname, int set_argv0)
         Py_DECREF(runmodule);
         return pymain_exit_err_print();
     }
-    runargs = Py_BuildValue("(Oi)", module, set_argv0);
+    runargs = PyTuple_Pack(2, module, set_argv0 ? Py_True : Py_False);
     if (runargs == NULL) {
         fprintf(stderr,
             "Could not create arguments for runpy._run_module_as_main\n");
diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c
index b2495fc8b322..ef63ca936e91 100644
--- a/Modules/parsermodule.c
+++ b/Modules/parsermodule.c
@@ -1080,25 +1080,20 @@ parser__pickler(PyObject *self, PyObject *args)
     NOTE(ARGUNUSED(self))
     PyObject *result = NULL;
     PyObject *st = NULL;
-    PyObject *empty_dict = NULL;
 
     if (PyArg_ParseTuple(args, "O!:_pickler", &PyST_Type, &st)) {
         PyObject *newargs;
         PyObject *tuple;
 
-        if ((empty_dict = PyDict_New()) == NULL)
-            goto finally;
-        if ((newargs = Py_BuildValue("Oi", st, 1)) == NULL)
-            goto finally;
-        tuple = parser_st2tuple((PyST_Object*)NULL, newargs, empty_dict);
+        if ((newargs = PyTuple_Pack(2, st, Py_True)) == NULL)
+            return NULL;
+        tuple = parser_st2tuple((PyST_Object*)NULL, newargs, NULL);
         if (tuple != NULL) {
             result = Py_BuildValue("O(O)", pickle_constructor, tuple);
             Py_DECREF(tuple);
         }
         Py_DECREF(newargs);
     }
-  finally:
-    Py_XDECREF(empty_dict);
 
     return (result);
 }
diff --git a/Objects/fileobject.c b/Objects/fileobject.c
index 61c94280937b..3ec5a00f30f6 100644
--- a/Objects/fileobject.c
+++ b/Objects/fileobject.c
@@ -38,9 +38,9 @@ PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const c
     io = PyImport_ImportModule("_io");
     if (io == NULL)
         return NULL;
-    stream = _PyObject_CallMethodId(io, &PyId_open, "isisssi", fd, mode,
+    stream = _PyObject_CallMethodId(io, &PyId_open, "isisssO", fd, mode,
                                  buffering, encoding, errors,
-                                 newline, closefd);
+                                 newline, closefd ? Py_True : Py_False);
     Py_DECREF(io);
     if (stream == NULL)
         return NULL;
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 5053f7a1748f..3b0d64ff14ab 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -1820,8 +1820,9 @@ static PyObject *
 builtin_print(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     static const char * const _keywords[] = {"sep", "end", "file", "flush", 0};
-    static struct _PyArg_Parser _parser = {"|OOOO:print", _keywords, 0};
-    PyObject *sep = NULL, *end = NULL, *file = NULL, *flush = NULL;
+    static struct _PyArg_Parser _parser = {"|OOOp:print", _keywords, 0};
+    PyObject *sep = NULL, *end = NULL, *file = NULL;
+    int flush = 0;
     int i, err;
 
     if (kwnames != NULL &&
@@ -1883,18 +1884,11 @@ builtin_print(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject
     if (err)
         return NULL;
 
-    if (flush != NULL) {
-        PyObject *tmp;
-        int do_flush = PyObject_IsTrue(flush);
-        if (do_flush == -1)
+    if (flush) {
+        PyObject *tmp = _PyObject_CallMethodIdNoArgs(file, &PyId_flush);
+        if (tmp == NULL)
             return NULL;
-        else if (do_flush) {
-            tmp = _PyObject_CallMethodIdNoArgs(file, &PyId_flush);
-            if (tmp == NULL)
-                return NULL;
-            else
-                Py_DECREF(tmp);
-        }
+        Py_DECREF(tmp);
     }
 
     Py_RETURN_NONE;
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 7cda44dc29d3..ea89b3a626bd 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1735,10 +1735,10 @@ create_stdio(const PyConfig *config, PyObject* io,
         mode = "wb";
     else
         mode = "rb";
-    buf = _PyObject_CallMethodId(io, &PyId_open, "isiOOOi",
+    buf = _PyObject_CallMethodId(io, &PyId_open, "isiOOOO",
                                  fd, mode, buffering,
                                  Py_None, Py_None, /* encoding, errors */
-                                 Py_None, 0); /* newline, closefd */
+                                 Py_None, Py_False); /* newline, closefd */
     if (buf == NULL)
         goto error;
 


From webhook-mailer at python.org  Sun Sep  1 07:01:09 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sun, 01 Sep 2019 11:01:09 -0000
Subject: [Python-checkins] [3.8] bpo-37994: Fix silencing all errors if an
 attribute lookup fails. (GH-15630) (GH-15635)
Message-ID: <mailman.208.1567335671.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/353053d9ad08fea0e205e6c008b8a4350c0188e6
commit: 353053d9ad08fea0e205e6c008b8a4350c0188e6
branch: 3.8
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-01T14:01:05+03:00
summary:

[3.8] bpo-37994: Fix silencing all errors if an attribute lookup fails. (GH-15630) (GH-15635)

Only AttributeError should be silenced.
(cherry picked from commit 41c57b335330ff48af098d47e379e0f9ba09d233)

files:
A Misc/NEWS.d/next/Core and Builtins/2019-08-31-11-13-25.bpo-37994.Rj6S4j.rst
M Modules/_csv.c
M Modules/_datetimemodule.c
M Modules/_pickle.c
M Modules/_threadmodule.c
M Modules/pyexpat.c
M Objects/bytearrayobject.c
M Objects/descrobject.c
M Objects/fileobject.c
M Objects/setobject.c
M Objects/typeobject.c
M Objects/weakrefobject.c
M Python/bltinmodule.c
M Python/sysmodule.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-31-11-13-25.bpo-37994.Rj6S4j.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-31-11-13-25.bpo-37994.Rj6S4j.rst
new file mode 100644
index 000000000000..103ac5a30905
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-31-11-13-25.bpo-37994.Rj6S4j.rst	
@@ -0,0 +1,2 @@
+Fixed silencing arbitrary errors if an attribute lookup fails in several
+sites. Only AttributeError should be silenced.
diff --git a/Modules/_csv.c b/Modules/_csv.c
index 014cbb4e0231..46d414383cbd 100644
--- a/Modules/_csv.c
+++ b/Modules/_csv.c
@@ -1382,7 +1382,10 @@ csv_writer(PyObject *module, PyObject *args, PyObject *keyword_args)
         Py_DECREF(self);
         return NULL;
     }
-    self->write = _PyObject_GetAttrId(output_file, &PyId_write);
+    if (_PyObject_LookupAttrId(output_file, &PyId_write, &self->write) < 0) {
+        Py_DECREF(self);
+        return NULL;
+    }
     if (self->write == NULL || !PyCallable_Check(self->write)) {
         PyErr_SetString(PyExc_TypeError,
                         "argument 1 must have a \"write\" method");
diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c
index d3c3f0905386..41c3f34269dd 100644
--- a/Modules/_datetimemodule.c
+++ b/Modules/_datetimemodule.c
@@ -3610,24 +3610,24 @@ tzinfo_reduce(PyObject *self, PyObject *Py_UNUSED(ignored))
     _Py_IDENTIFIER(__getinitargs__);
     _Py_IDENTIFIER(__getstate__);
 
-    getinitargs = _PyObject_GetAttrId(self, &PyId___getinitargs__);
+    if (_PyObject_LookupAttrId(self, &PyId___getinitargs__, &getinitargs) < 0) {
+        return NULL;
+    }
     if (getinitargs != NULL) {
         args = _PyObject_CallNoArg(getinitargs);
         Py_DECREF(getinitargs);
-        if (args == NULL) {
-            return NULL;
-        }
     }
     else {
-        PyErr_Clear();
-
         args = PyTuple_New(0);
-        if (args == NULL) {
-            return NULL;
-        }
+    }
+    if (args == NULL) {
+        return NULL;
     }
 
-    getstate = _PyObject_GetAttrId(self, &PyId___getstate__);
+    if (_PyObject_LookupAttrId(self, &PyId___getstate__, &getstate) < 0) {
+        Py_DECREF(args);
+        return NULL;
+    }
     if (getstate != NULL) {
         state = _PyObject_CallNoArg(getstate);
         Py_DECREF(getstate);
@@ -3638,7 +3638,6 @@ tzinfo_reduce(PyObject *self, PyObject *Py_UNUSED(ignored))
     }
     else {
         PyObject **dictptr;
-        PyErr_Clear();
         state = Py_None;
         dictptr = _PyObject_GetDictPtr(self);
         if (dictptr && *dictptr && PyDict_GET_SIZE(*dictptr)) {
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index bbe26034881a..fa2ef7d9dcd0 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -4385,7 +4385,6 @@ save(PicklerObject *self, PyObject *obj, int pers_save)
         _Py_IDENTIFIER(__reduce__);
         _Py_IDENTIFIER(__reduce_ex__);
 
-
         /* XXX: If the __reduce__ method is defined, __reduce_ex__ is
            automatically defined as __reduce__. While this is convenient, this
            make it impossible to know which method was actually called. Of
@@ -4406,14 +4405,15 @@ save(PicklerObject *self, PyObject *obj, int pers_save)
             }
         }
         else {
-            PickleState *st = _Pickle_GetGlobalState();
-
             /* Check for a __reduce__ method. */
-            reduce_func = _PyObject_GetAttrId(obj, &PyId___reduce__);
+            if (_PyObject_LookupAttrId(obj, &PyId___reduce__, &reduce_func) < 0) {
+                goto error;
+            }
             if (reduce_func != NULL) {
                 reduce_value = _PyObject_CallNoArg(reduce_func);
             }
             else {
+                PickleState *st = _Pickle_GetGlobalState();
                 PyErr_Format(st->PicklingError,
                              "can't pickle '%.200s' object: %R",
                              type->tp_name, obj);
@@ -6448,7 +6448,9 @@ do_append(UnpicklerObject *self, Py_ssize_t x)
         PyObject *extend_func;
         _Py_IDENTIFIER(extend);
 
-        extend_func = _PyObject_GetAttrId(list, &PyId_extend);
+        if (_PyObject_LookupAttrId(list, &PyId_extend, &extend_func) < 0) {
+            return -1;
+        }
         if (extend_func != NULL) {
             slice = Pdata_poplist(self->stack, x);
             if (!slice) {
@@ -6468,7 +6470,6 @@ do_append(UnpicklerObject *self, Py_ssize_t x)
             /* Even if the PEP 307 requires extend() and append() methods,
                fall back on append() if the object has no extend() method
                for backward compatibility. */
-            PyErr_Clear();
             append_func = _PyObject_GetAttrId(list, &PyId_append);
             if (append_func == NULL)
                 return -1;
diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c
index d5e40ef999e3..fadf57aaa762 100644
--- a/Modules/_threadmodule.c
+++ b/Modules/_threadmodule.c
@@ -1305,6 +1305,7 @@ static int
 thread_excepthook_file(PyObject *file, PyObject *exc_type, PyObject *exc_value,
                        PyObject *exc_traceback, PyObject *thread)
 {
+    _Py_IDENTIFIER(name);
     /* print(f"Exception in thread {thread.name}:", file=file) */
     if (PyFile_WriteString("Exception in thread ", file) < 0) {
         return -1;
@@ -1312,7 +1313,9 @@ thread_excepthook_file(PyObject *file, PyObject *exc_type, PyObject *exc_value,
 
     PyObject *name = NULL;
     if (thread != Py_None) {
-        name = PyObject_GetAttrString(thread, "name");
+        if (_PyObject_LookupAttrId(thread, &PyId_name, &name) < 0) {
+            return -1;
+        }
     }
     if (name != NULL) {
         if (PyFile_WriteObject(name, file, Py_PRINT_RAW) < 0) {
@@ -1322,8 +1325,6 @@ thread_excepthook_file(PyObject *file, PyObject *exc_type, PyObject *exc_value,
         Py_DECREF(name);
     }
     else {
-        PyErr_Clear();
-
         unsigned long ident = PyThread_get_thread_ident();
         PyObject *str = PyUnicode_FromFormat("%lu", ident);
         if (str != NULL) {
diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c
index 45a1e684d19f..bb5e25cc585d 100644
--- a/Modules/pyexpat.c
+++ b/Modules/pyexpat.c
@@ -810,7 +810,9 @@ pyexpat_xmlparser_ParseFile(xmlparseobject *self, PyObject *file)
     PyObject *readmethod = NULL;
     _Py_IDENTIFIER(read);
 
-    readmethod = _PyObject_GetAttrId(file, &PyId_read);
+    if (_PyObject_LookupAttrId(file, &PyId_read, &readmethod) < 0) {
+        return NULL;
+    }
     if (readmethod == NULL) {
         PyErr_SetString(PyExc_TypeError,
                         "argument must have 'read' attribute");
diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c
index 1bb19a9271b6..590b80605618 100644
--- a/Objects/bytearrayobject.c
+++ b/Objects/bytearrayobject.c
@@ -2063,9 +2063,10 @@ _common_reduce(PyByteArrayObject *self, int proto)
     _Py_IDENTIFIER(__dict__);
     char *buf;
 
-    dict = _PyObject_GetAttrId((PyObject *)self, &PyId___dict__);
+    if (_PyObject_LookupAttrId((PyObject *)self, &PyId___dict__, &dict) < 0) {
+        return NULL;
+    }
     if (dict == NULL) {
-        PyErr_Clear();
         dict = Py_None;
         Py_INCREF(dict);
     }
diff --git a/Objects/descrobject.c b/Objects/descrobject.c
index dbb44de846fd..119be35db08d 100644
--- a/Objects/descrobject.c
+++ b/Objects/descrobject.c
@@ -1622,29 +1622,25 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset,
     /* if no docstring given and the getter has one, use that one */
     if ((doc == NULL || doc == Py_None) && fget != NULL) {
         _Py_IDENTIFIER(__doc__);
-        PyObject *get_doc = _PyObject_GetAttrId(fget, &PyId___doc__);
-        if (get_doc) {
-            if (Py_TYPE(self) == &PyProperty_Type) {
-                Py_XSETREF(self->prop_doc, get_doc);
-            }
-            else {
-                /* If this is a property subclass, put __doc__
-                in dict of the subclass instance instead,
-                otherwise it gets shadowed by __doc__ in the
-                class's dict. */
-                int err = _PyObject_SetAttrId((PyObject *)self, &PyId___doc__, get_doc);
-                Py_DECREF(get_doc);
-                if (err < 0)
-                    return -1;
-            }
-            self->getter_doc = 1;
+        PyObject *get_doc;
+        int rc = _PyObject_LookupAttrId(fget, &PyId___doc__, &get_doc);
+        if (rc <= 0) {
+            return rc;
         }
-        else if (PyErr_ExceptionMatches(PyExc_Exception)) {
-            PyErr_Clear();
+        if (Py_TYPE(self) == &PyProperty_Type) {
+            Py_XSETREF(self->prop_doc, get_doc);
         }
         else {
-            return -1;
+            /* If this is a property subclass, put __doc__
+               in dict of the subclass instance instead,
+               otherwise it gets shadowed by __doc__ in the
+               class's dict. */
+            int err = _PyObject_SetAttrId((PyObject *)self, &PyId___doc__, get_doc);
+            Py_DECREF(get_doc);
+            if (err < 0)
+                return -1;
         }
+        self->getter_doc = 1;
     }
 
     return 0;
diff --git a/Objects/fileobject.c b/Objects/fileobject.c
index 3791241e5c7c..dd42d516898a 100644
--- a/Objects/fileobject.c
+++ b/Objects/fileobject.c
@@ -185,8 +185,10 @@ PyObject_AsFileDescriptor(PyObject *o)
     if (PyLong_Check(o)) {
         fd = _PyLong_AsInt(o);
     }
-    else if ((meth = _PyObject_GetAttrId(o, &PyId_fileno)) != NULL)
-    {
+    else if (_PyObject_LookupAttrId(o, &PyId_fileno, &meth) < 0) {
+        return -1;
+    }
+    else if (meth != NULL) {
         PyObject *fno = _PyObject_CallNoArg(meth);
         Py_DECREF(meth);
         if (fno == NULL)
diff --git a/Objects/setobject.c b/Objects/setobject.c
index 8cd95ba890dd..f8ae0c05e273 100644
--- a/Objects/setobject.c
+++ b/Objects/setobject.c
@@ -1954,9 +1954,10 @@ set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored))
     args = PyTuple_Pack(1, keys);
     if (args == NULL)
         goto done;
-    dict = _PyObject_GetAttrId((PyObject *)so, &PyId___dict__);
+    if (_PyObject_LookupAttrId((PyObject *)so, &PyId___dict__, &dict) < 0) {
+        goto done;
+    }
     if (dict == NULL) {
-        PyErr_Clear();
         dict = Py_None;
         Py_INCREF(dict);
     }
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index bf0c3e072919..8ece21827519 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1547,17 +1547,10 @@ tail_contains(PyObject *tuple, int whence, PyObject *o)
 static PyObject *
 class_name(PyObject *cls)
 {
-    PyObject *name = _PyObject_GetAttrId(cls, &PyId___name__);
-    if (name == NULL) {
-        PyErr_Clear();
+    PyObject *name;
+    if (_PyObject_LookupAttrId(cls, &PyId___name__, &name) == 0) {
         name = PyObject_Repr(cls);
     }
-    if (name == NULL)
-        return NULL;
-    if (!PyUnicode_Check(name)) {
-        Py_DECREF(name);
-        return NULL;
-    }
     return name;
 }
 
@@ -1575,13 +1568,15 @@ check_duplicates(PyObject *tuple)
             if (PyTuple_GET_ITEM(tuple, j) == o) {
                 o = class_name(o);
                 if (o != NULL) {
-                    PyErr_Format(PyExc_TypeError,
-                                 "duplicate base class %U",
-                                 o);
+                    if (PyUnicode_Check(o)) {
+                        PyErr_Format(PyExc_TypeError,
+                                     "duplicate base class %U", o);
+                    }
+                    else {
+                        PyErr_SetString(PyExc_TypeError,
+                                        "duplicate base class");
+                    }
                     Py_DECREF(o);
-                } else {
-                    PyErr_SetString(PyExc_TypeError,
-                                 "duplicate base class");
                 }
                 return -1;
             }
@@ -1625,13 +1620,20 @@ consistent method resolution\norder (MRO) for bases");
     i = 0;
     while (PyDict_Next(set, &i, &k, &v) && (size_t)off < sizeof(buf)) {
         PyObject *name = class_name(k);
-        const char *name_str;
+        const char *name_str = NULL;
         if (name != NULL) {
-            name_str = PyUnicode_AsUTF8(name);
-            if (name_str == NULL)
+            if (PyUnicode_Check(name)) {
+                name_str = PyUnicode_AsUTF8(name);
+            }
+            else {
                 name_str = "?";
-        } else
-            name_str = "?";
+            }
+        }
+        if (name_str == NULL) {
+            Py_XDECREF(name);
+            Py_DECREF(set);
+            return;
+        }
         off += PyOS_snprintf(buf + off, sizeof(buf) - off, " %s", name_str);
         Py_XDECREF(name);
         if (--n && (size_t)(off+1) < sizeof(buf)) {
@@ -3418,10 +3420,10 @@ merge_class_dict(PyObject *dict, PyObject *aclass)
     assert(aclass);
 
     /* Merge in the type's dict (if any). */
-    classdict = _PyObject_GetAttrId(aclass, &PyId___dict__);
-    if (classdict == NULL)
-        PyErr_Clear();
-    else {
+    if (_PyObject_LookupAttrId(aclass, &PyId___dict__, &classdict) < 0) {
+        return -1;
+    }
+    if (classdict != NULL) {
         int status = PyDict_Update(dict, classdict);
         Py_DECREF(classdict);
         if (status < 0)
@@ -3429,15 +3431,17 @@ merge_class_dict(PyObject *dict, PyObject *aclass)
     }
 
     /* Recursively merge in the base types' (if any) dicts. */
-    bases = _PyObject_GetAttrId(aclass, &PyId___bases__);
-    if (bases == NULL)
-        PyErr_Clear();
-    else {
+    if (_PyObject_LookupAttrId(aclass, &PyId___bases__, &bases) < 0) {
+        return -1;
+    }
+    if (bases != NULL) {
         /* We have no guarantee that bases is a real tuple */
         Py_ssize_t i, n;
         n = PySequence_Size(bases); /* This better be right */
-        if (n < 0)
-            PyErr_Clear();
+        if (n < 0) {
+            Py_DECREF(bases);
+            return -1;
+        }
         else {
             for (i = 0; i < n; i++) {
                 int status;
@@ -4725,9 +4729,10 @@ object___dir___impl(PyObject *self)
     PyObject *itsclass = NULL;
 
     /* Get __dict__ (which may or may not be a real dict...) */
-    dict = _PyObject_GetAttrId(self, &PyId___dict__);
+    if (_PyObject_LookupAttrId(self, &PyId___dict__, &dict) < 0) {
+        return NULL;
+    }
     if (dict == NULL) {
-        PyErr_Clear();
         dict = PyDict_New();
     }
     else if (!PyDict_Check(dict)) {
@@ -4745,12 +4750,12 @@ object___dir___impl(PyObject *self)
         goto error;
 
     /* Merge in attrs reachable from its class. */
-    itsclass = _PyObject_GetAttrId(self, &PyId___class__);
-    if (itsclass == NULL)
-        /* XXX(tomer): Perhaps fall back to obj->ob_type if no
-                       __class__ exists? */
-        PyErr_Clear();
-    else if (merge_class_dict(dict, itsclass) != 0)
+    if (_PyObject_LookupAttrId(self, &PyId___class__, &itsclass) < 0) {
+        goto error;
+    }
+    /* XXX(tomer): Perhaps fall back to obj->ob_type if no
+                   __class__ exists? */
+    if (itsclass != NULL && merge_class_dict(dict, itsclass) < 0)
         goto error;
 
     result = PyDict_Keys(dict);
@@ -6105,16 +6110,19 @@ method_is_overloaded(PyObject *left, PyObject *right, struct _Py_Identifier *nam
     PyObject *a, *b;
     int ok;
 
-    b = _PyObject_GetAttrId((PyObject *)(Py_TYPE(right)), name);
+    if (_PyObject_LookupAttrId((PyObject *)(Py_TYPE(right)), name, &b) < 0) {
+        return -1;
+    }
     if (b == NULL) {
-        PyErr_Clear();
         /* If right doesn't have it, it's not overloaded */
         return 0;
     }
 
-    a = _PyObject_GetAttrId((PyObject *)(Py_TYPE(left)), name);
+    if (_PyObject_LookupAttrId((PyObject *)(Py_TYPE(left)), name, &a) < 0) {
+        Py_DECREF(b);
+        return -1;
+    }
     if (a == NULL) {
-        PyErr_Clear();
         Py_DECREF(b);
         /* If right has it but left doesn't, it's overloaded */
         return 1;
@@ -6123,11 +6131,6 @@ method_is_overloaded(PyObject *left, PyObject *right, struct _Py_Identifier *nam
     ok = PyObject_RichCompareBool(a, b, Py_NE);
     Py_DECREF(a);
     Py_DECREF(b);
-    if (ok < 0) {
-        PyErr_Clear();
-        return 0;
-    }
-
     return ok;
 }
 
@@ -6145,15 +6148,19 @@ FUNCNAME(PyObject *self, PyObject *other) \
     if (Py_TYPE(self)->tp_as_number != NULL && \
         Py_TYPE(self)->tp_as_number->SLOTNAME == TESTFUNC) { \
         PyObject *r; \
-        if (do_other && \
-            PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self)) && \
-            method_is_overloaded(self, other, &rop_id)) { \
-            stack[0] = self; \
-            r = call_maybe(other, &rop_id, stack, 1); \
-            if (r != Py_NotImplemented) \
-                return r; \
-            Py_DECREF(r); \
-            do_other = 0; \
+        if (do_other && PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) { \
+            int ok = method_is_overloaded(self, other, &rop_id); \
+            if (ok < 0) { \
+                return NULL; \
+            } \
+            if (ok) { \
+                stack[0] = self; \
+                r = call_maybe(other, &rop_id, stack, 1); \
+                if (r != Py_NotImplemented) \
+                    return r; \
+                Py_DECREF(r); \
+                do_other = 0; \
+            } \
         } \
         stack[0] = other; \
         r = call_maybe(self, &op_id, stack, 1); \
@@ -7739,7 +7746,9 @@ supercheck(PyTypeObject *type, PyObject *obj)
         /* Try the slow way */
         PyObject *class_attr;
 
-        class_attr = _PyObject_GetAttrId(obj, &PyId___class__);
+        if (_PyObject_LookupAttrId(obj, &PyId___class__, &class_attr) < 0) {
+            return NULL;
+        }
         if (class_attr != NULL &&
             PyType_Check(class_attr) &&
             (PyTypeObject *)class_attr != Py_TYPE(obj))
@@ -7749,11 +7758,7 @@ supercheck(PyTypeObject *type, PyObject *obj)
             if (ok)
                 return (PyTypeObject *)class_attr;
         }
-
-        if (class_attr == NULL)
-            PyErr_Clear();
-        else
-            Py_DECREF(class_attr);
+        Py_XDECREF(class_attr);
     }
 
     PyErr_SetString(PyExc_TypeError,
diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c
index 8b8e71031afa..4b81450de83f 100644
--- a/Objects/weakrefobject.c
+++ b/Objects/weakrefobject.c
@@ -163,10 +163,10 @@ weakref_repr(PyWeakReference *self)
     if (PyWeakref_GET_OBJECT(self) == Py_None)
         return PyUnicode_FromFormat("<weakref at %p; dead>", self);
 
-    name = _PyObject_GetAttrId(PyWeakref_GET_OBJECT(self), &PyId___name__);
+    if (_PyObject_LookupAttrId(PyWeakref_GET_OBJECT(self), &PyId___name__, &name) < 0) {
+        return NULL;
+    }
     if (name == NULL || !PyUnicode_Check(name)) {
-        if (name == NULL)
-            PyErr_Clear();
         repr = PyUnicode_FromFormat(
             "<weakref at %p; to '%s' at %p>",
             self,
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index b85bfb2b91e3..8f9e81329c50 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -2256,16 +2256,12 @@ builtin_vars(PyObject *self, PyObject *args)
         return NULL;
     if (v == NULL) {
         d = PyEval_GetLocals();
-        if (d == NULL)
-            return NULL;
-        Py_INCREF(d);
+        Py_XINCREF(d);
     }
     else {
-        d = _PyObject_GetAttrId(v, &PyId___dict__);
-        if (d == NULL) {
+        if (_PyObject_LookupAttrId(v, &PyId___dict__, &d) == 0) {
             PyErr_SetString(PyExc_TypeError,
                 "vars() argument must have __dict__ attribute");
-            return NULL;
         }
     }
     return d;
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 93ffce2a4783..11c20cf3c713 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -201,16 +201,12 @@ PySys_Audit(const char *event, const char *argFormat, ...)
         ts->tracing++;
         ts->use_tracing = 0;
         while ((hook = PyIter_Next(hooks)) != NULL) {
+            _Py_IDENTIFIER(__cantrace__);
             PyObject *o;
-            int canTrace = -1;
-            o = PyObject_GetAttrString(hook, "__cantrace__");
+            int canTrace = _PyObject_LookupAttrId(hook, &PyId___cantrace__, &o);
             if (o) {
                 canTrace = PyObject_IsTrue(o);
                 Py_DECREF(o);
-            } else if (PyErr_Occurred() &&
-                       PyErr_ExceptionMatches(PyExc_AttributeError)) {
-                PyErr_Clear();
-                canTrace = 0;
             }
             if (canTrace < 0) {
                 break;
@@ -538,7 +534,10 @@ sys_displayhook_unencodable(PyObject *outf, PyObject *o)
     if (encoded == NULL)
         goto error;
 
-    buffer = _PyObject_GetAttrId(outf, &PyId_buffer);
+    if (_PyObject_LookupAttrId(outf, &PyId_buffer, &buffer) < 0) {
+        Py_DECREF(encoded);
+        goto error;
+    }
     if (buffer) {
         result = _PyObject_CallMethodIdObjArgs(buffer, &PyId_write, encoded, NULL);
         Py_DECREF(buffer);
@@ -548,7 +547,6 @@ sys_displayhook_unencodable(PyObject *outf, PyObject *o)
         Py_DECREF(result);
     }
     else {
-        PyErr_Clear();
         escaped_str = PyUnicode_FromEncodedObject(encoded,
                                                   stdout_encoding_str,
                                                   "strict");


From webhook-mailer at python.org  Mon Sep  2 11:08:07 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Mon, 02 Sep 2019 15:08:07 -0000
Subject: [Python-checkins] bpo-38010 Sync importlib.metadata with
 importlib_metadata 0.20. (GH-15646)
Message-ID: <mailman.209.1567436889.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/102e9b40ff6ee45086a5f0d34d9c60c581a1e5e5
commit: 102e9b40ff6ee45086a5f0d34d9c60c581a1e5e5
branch: master
author: Jason R. Coombs <jaraco at jaraco.com>
committer: GitHub <noreply at github.com>
date: 2019-09-02T11:08:03-04:00
summary:

bpo-38010 Sync importlib.metadata with importlib_metadata 0.20. (GH-15646)

Sync importlib.metadata with importlib_metadata 0.20.

files:
A Misc/NEWS.d/next/Library/2019-09-02-14-30-39.bpo-38010.JOnz9Z.rst
M Doc/library/importlib.metadata.rst
M Lib/importlib/metadata.py
M Lib/test/test_importlib/test_metadata_api.py

diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst
index 2126498e728c..23c8cdeb21c2 100644
--- a/Doc/library/importlib.metadata.rst
+++ b/Doc/library/importlib.metadata.rst
@@ -158,6 +158,13 @@ Once you have the file, you can also read its contents::
             return s.encode('utf-8')
         return s
 
+In the case where the metadata file listing files
+(RECORD or SOURCES.txt) is missing, ``files()`` will
+return ``None``. The caller may wish to wrap calls to
+``files()`` in `always_iterable
+<https://more-itertools.readthedocs.io/en/stable/api.html#more_itertools.always_iterable>`_
+or otherwise guard against this condition if the target
+distribution is not known to have the metadata present.
 
 .. _requirements:
 
diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py
index e80f4fa6409b..3b46142231ec 100644
--- a/Lib/importlib/metadata.py
+++ b/Lib/importlib/metadata.py
@@ -213,6 +213,15 @@ def entry_points(self):
 
     @property
     def files(self):
+        """Files in this distribution.
+
+        :return: Iterable of PackagePath for this distribution or None
+
+        Result is `None` if the metadata file that enumerates files
+        (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is
+        missing.
+        Result may be empty if the metadata exists but is empty.
+        """
         file_lines = self._read_files_distinfo() or self._read_files_egginfo()
 
         def make_file(name, hash=None, size_str=None):
@@ -245,8 +254,7 @@ def requires(self):
         return self._read_dist_info_reqs() or self._read_egg_info_reqs()
 
     def _read_dist_info_reqs(self):
-        spec = self.metadata['Requires-Dist']
-        return spec and filter(None, spec.splitlines())
+        return self.metadata.get_all('Requires-Dist')
 
     def _read_egg_info_reqs(self):
         source = self.read_text('requires.txt')
@@ -318,7 +326,11 @@ def find_distributions(self, name=None, path=None):
 
 class PathDistribution(Distribution):
     def __init__(self, path):
-        """Construct a distribution from a path to the metadata directory."""
+        """Construct a distribution from a path to the metadata directory.
+
+        :param path: A pathlib.Path or similar object supporting
+                     .joinpath(), __div__, .parent, and .read_text().
+        """
         self._path = path
 
     def read_text(self, filename):
diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py
index 899777f4b1ad..af3bab3558ef 100644
--- a/Lib/test/test_importlib/test_metadata_api.py
+++ b/Lib/test/test_importlib/test_metadata_api.py
@@ -109,6 +109,8 @@ def test_requires(self):
     def test_requires_dist_info(self):
         deps = list(requires('distinfo-pkg'))
         assert deps and all(deps)
+        assert 'wheel >= 1.0' in deps
+        assert "pytest; extra == 'test'" in deps
 
     def test_more_complex_deps_requires_text(self):
         requires = textwrap.dedent("""
diff --git a/Misc/NEWS.d/next/Library/2019-09-02-14-30-39.bpo-38010.JOnz9Z.rst b/Misc/NEWS.d/next/Library/2019-09-02-14-30-39.bpo-38010.JOnz9Z.rst
new file mode 100644
index 000000000000..a97f9b7d9a40
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-02-14-30-39.bpo-38010.JOnz9Z.rst
@@ -0,0 +1 @@
+In ``importlib.metadata`` sync with ``importlib_metadata`` 0.20, clarifying behavior of ``files()`` and fixing issue where only one requirement was returned for ``requires()`` on ``dist-info`` packages.


From webhook-mailer at python.org  Mon Sep  2 12:01:28 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Mon, 02 Sep 2019 16:01:28 -0000
Subject: [Python-checkins] bpo-36853: Fix suspicious.py to actually print
 the unused rules (#13579)
Message-ID: <mailman.210.1567440089.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e1786b54162e2bfb01ca5aafa19d596c4af5a803
commit: e1786b54162e2bfb01ca5aafa19d596c4af5a803
branch: master
author: Anthony Sottile <asottile at umich.edu>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-02T12:01:23-04:00
summary:

bpo-36853: Fix suspicious.py to actually print the unused rules (#13579)

* Fix suspicious.py to actually print the unused rules

* Fix the other `self.warn` calls

files:
M Doc/tools/extensions/suspicious.py

diff --git a/Doc/tools/extensions/suspicious.py b/Doc/tools/extensions/suspicious.py
index 34a0112f5a00..9e814fb94d2b 100644
--- a/Doc/tools/extensions/suspicious.py
+++ b/Doc/tools/extensions/suspicious.py
@@ -115,10 +115,12 @@ def write_doc(self, docname, doctree):
     def finish(self):
         unused_rules = [rule for rule in self.rules if not rule.used]
         if unused_rules:
-            self.logger.warn('Found %s/%s unused rules:' %
-                             (len(unused_rules), len(self.rules)))
-            for rule in unused_rules:
-                self.logger.info(repr(rule))
+            self.logger.warning(
+                'Found %s/%s unused rules: %s' % (
+                    len(unused_rules), len(self.rules),
+                    ''.join(repr(rule) for rule in unused_rules),
+                )
+            )
         return
 
     def check_issue(self, line, lineno, issue):
@@ -151,14 +153,15 @@ def report_issue(self, text, lineno, issue):
         self.any_issue = True
         self.write_log_entry(lineno, issue, text)
         if py3:
-            self.logger.warn('[%s:%d] "%s" found in "%-.120s"' %
-                             (self.docname, lineno, issue, text))
+            self.logger.warning('[%s:%d] "%s" found in "%-.120s"' %
+                                (self.docname, lineno, issue, text))
         else:
-            self.logger.warn('[%s:%d] "%s" found in "%-.120s"' % (
-                self.docname.encode(sys.getdefaultencoding(),'replace'),
-                lineno,
-                issue.encode(sys.getdefaultencoding(),'replace'),
-                text.strip().encode(sys.getdefaultencoding(),'replace')))
+            self.logger.warning(
+                '[%s:%d] "%s" found in "%-.120s"' % (
+                    self.docname.encode(sys.getdefaultencoding(),'replace'),
+                    lineno,
+                    issue.encode(sys.getdefaultencoding(),'replace'),
+                    text.strip().encode(sys.getdefaultencoding(),'replace')))
         self.app.statuscode = 1
 
     def write_log_entry(self, lineno, issue, text):


From webhook-mailer at python.org  Mon Sep  2 12:11:07 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Mon, 02 Sep 2019 16:11:07 -0000
Subject: [Python-checkins] bpo-38010 Sync importlib.metadata with
 importlib_metadata 0.20. (GH-15646) (GH-15648)
Message-ID: <mailman.211.1567440669.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bf69e160c431dc40d6530d427cec71c6a97522f5
commit: bf69e160c431dc40d6530d427cec71c6a97522f5
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-02T12:11:01-04:00
summary:

bpo-38010 Sync importlib.metadata with importlib_metadata 0.20. (GH-15646) (GH-15648)

Sync importlib.metadata with importlib_metadata 0.20.
(cherry picked from commit 102e9b40ff6ee45086a5f0d34d9c60c581a1e5e5)

Co-authored-by: Jason R. Coombs <jaraco at jaraco.com>

files:
A Misc/NEWS.d/next/Library/2019-09-02-14-30-39.bpo-38010.JOnz9Z.rst
M Doc/library/importlib.metadata.rst
M Lib/importlib/metadata.py
M Lib/test/test_importlib/test_metadata_api.py

diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst
index 2126498e728c..23c8cdeb21c2 100644
--- a/Doc/library/importlib.metadata.rst
+++ b/Doc/library/importlib.metadata.rst
@@ -158,6 +158,13 @@ Once you have the file, you can also read its contents::
             return s.encode('utf-8')
         return s
 
+In the case where the metadata file listing files
+(RECORD or SOURCES.txt) is missing, ``files()`` will
+return ``None``. The caller may wish to wrap calls to
+``files()`` in `always_iterable
+<https://more-itertools.readthedocs.io/en/stable/api.html#more_itertools.always_iterable>`_
+or otherwise guard against this condition if the target
+distribution is not known to have the metadata present.
 
 .. _requirements:
 
diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py
index e80f4fa6409b..3b46142231ec 100644
--- a/Lib/importlib/metadata.py
+++ b/Lib/importlib/metadata.py
@@ -213,6 +213,15 @@ def entry_points(self):
 
     @property
     def files(self):
+        """Files in this distribution.
+
+        :return: Iterable of PackagePath for this distribution or None
+
+        Result is `None` if the metadata file that enumerates files
+        (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is
+        missing.
+        Result may be empty if the metadata exists but is empty.
+        """
         file_lines = self._read_files_distinfo() or self._read_files_egginfo()
 
         def make_file(name, hash=None, size_str=None):
@@ -245,8 +254,7 @@ def requires(self):
         return self._read_dist_info_reqs() or self._read_egg_info_reqs()
 
     def _read_dist_info_reqs(self):
-        spec = self.metadata['Requires-Dist']
-        return spec and filter(None, spec.splitlines())
+        return self.metadata.get_all('Requires-Dist')
 
     def _read_egg_info_reqs(self):
         source = self.read_text('requires.txt')
@@ -318,7 +326,11 @@ def find_distributions(self, name=None, path=None):
 
 class PathDistribution(Distribution):
     def __init__(self, path):
-        """Construct a distribution from a path to the metadata directory."""
+        """Construct a distribution from a path to the metadata directory.
+
+        :param path: A pathlib.Path or similar object supporting
+                     .joinpath(), __div__, .parent, and .read_text().
+        """
         self._path = path
 
     def read_text(self, filename):
diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py
index 899777f4b1ad..af3bab3558ef 100644
--- a/Lib/test/test_importlib/test_metadata_api.py
+++ b/Lib/test/test_importlib/test_metadata_api.py
@@ -109,6 +109,8 @@ def test_requires(self):
     def test_requires_dist_info(self):
         deps = list(requires('distinfo-pkg'))
         assert deps and all(deps)
+        assert 'wheel >= 1.0' in deps
+        assert "pytest; extra == 'test'" in deps
 
     def test_more_complex_deps_requires_text(self):
         requires = textwrap.dedent("""
diff --git a/Misc/NEWS.d/next/Library/2019-09-02-14-30-39.bpo-38010.JOnz9Z.rst b/Misc/NEWS.d/next/Library/2019-09-02-14-30-39.bpo-38010.JOnz9Z.rst
new file mode 100644
index 000000000000..a97f9b7d9a40
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-02-14-30-39.bpo-38010.JOnz9Z.rst
@@ -0,0 +1 @@
+In ``importlib.metadata`` sync with ``importlib_metadata`` 0.20, clarifying behavior of ``files()`` and fixing issue where only one requirement was returned for ``requires()`` on ``dist-info`` packages.


From webhook-mailer at python.org  Mon Sep  2 12:12:23 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Mon, 02 Sep 2019 16:12:23 -0000
Subject: [Python-checkins] bpo-36853: Fix suspicious.py to actually print
 the unused rules (GH-13579) (GH-15649)
Message-ID: <mailman.212.1567440744.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b365cfae4675ae90df329cb1179a5664e8283c13
commit: b365cfae4675ae90df329cb1179a5664e8283c13
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-02T12:12:19-04:00
summary:

bpo-36853: Fix suspicious.py to actually print the unused rules (GH-13579) (GH-15649)

* Fix suspicious.py to actually print the unused rules

* Fix the other `self.warn` calls
(cherry picked from commit e1786b54162e2bfb01ca5aafa19d596c4af5a803)

Co-authored-by: Anthony Sottile <asottile at umich.edu>

files:
M Doc/tools/extensions/suspicious.py

diff --git a/Doc/tools/extensions/suspicious.py b/Doc/tools/extensions/suspicious.py
index 34a0112f5a00..9e814fb94d2b 100644
--- a/Doc/tools/extensions/suspicious.py
+++ b/Doc/tools/extensions/suspicious.py
@@ -115,10 +115,12 @@ def write_doc(self, docname, doctree):
     def finish(self):
         unused_rules = [rule for rule in self.rules if not rule.used]
         if unused_rules:
-            self.logger.warn('Found %s/%s unused rules:' %
-                             (len(unused_rules), len(self.rules)))
-            for rule in unused_rules:
-                self.logger.info(repr(rule))
+            self.logger.warning(
+                'Found %s/%s unused rules: %s' % (
+                    len(unused_rules), len(self.rules),
+                    ''.join(repr(rule) for rule in unused_rules),
+                )
+            )
         return
 
     def check_issue(self, line, lineno, issue):
@@ -151,14 +153,15 @@ def report_issue(self, text, lineno, issue):
         self.any_issue = True
         self.write_log_entry(lineno, issue, text)
         if py3:
-            self.logger.warn('[%s:%d] "%s" found in "%-.120s"' %
-                             (self.docname, lineno, issue, text))
+            self.logger.warning('[%s:%d] "%s" found in "%-.120s"' %
+                                (self.docname, lineno, issue, text))
         else:
-            self.logger.warn('[%s:%d] "%s" found in "%-.120s"' % (
-                self.docname.encode(sys.getdefaultencoding(),'replace'),
-                lineno,
-                issue.encode(sys.getdefaultencoding(),'replace'),
-                text.strip().encode(sys.getdefaultencoding(),'replace')))
+            self.logger.warning(
+                '[%s:%d] "%s" found in "%-.120s"' % (
+                    self.docname.encode(sys.getdefaultencoding(),'replace'),
+                    lineno,
+                    issue.encode(sys.getdefaultencoding(),'replace'),
+                    text.strip().encode(sys.getdefaultencoding(),'replace')))
         self.app.statuscode = 1
 
     def write_log_entry(self, lineno, issue, text):


From webhook-mailer at python.org  Mon Sep  2 15:17:29 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Mon, 02 Sep 2019 19:17:29 -0000
Subject: [Python-checkins] bpo-36853: Fix suspicious.py to actually print
 the unused rules (#13579) (#15653)
Message-ID: <mailman.213.1567451849.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ebe709dc1d7c1f9f07dc7d77e53674d2500b223e
commit: ebe709dc1d7c1f9f07dc7d77e53674d2500b223e
branch: 3.7
author: Anthony Sottile <asottile at umich.edu>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-02T15:17:18-04:00
summary:

bpo-36853: Fix suspicious.py to actually print the unused rules (#13579) (#15653)

* Fix suspicious.py to actually print the unused rules

* Fix the other `self.warn` calls

(cherry picked from commit e1786b54162e2bfb01ca5aafa19d596c4af5a803)

files:
M Doc/tools/extensions/suspicious.py

diff --git a/Doc/tools/extensions/suspicious.py b/Doc/tools/extensions/suspicious.py
index 8d80f6759bff..fd50f318170b 100644
--- a/Doc/tools/extensions/suspicious.py
+++ b/Doc/tools/extensions/suspicious.py
@@ -115,10 +115,12 @@ def write_doc(self, docname, doctree):
     def finish(self):
         unused_rules = [rule for rule in self.rules if not rule.used]
         if unused_rules:
-            self.warn('Found %s/%s unused rules:' %
-                      (len(unused_rules), len(self.rules)))
-            for rule in unused_rules:
-                self.logger.info(repr(rule))
+            self.logger.warning(
+                'Found %s/%s unused rules: %s' % (
+                    len(unused_rules), len(self.rules),
+                    ''.join(repr(rule) for rule in unused_rules),
+                )
+            )
         return
 
     def check_issue(self, line, lineno, issue):
@@ -152,14 +154,15 @@ def report_issue(self, text, lineno, issue):
         self.any_issue = True
         self.write_log_entry(lineno, issue, text)
         if py3:
-            self.warn('[%s:%d] "%s" found in "%-.120s"' %
-                      (self.docname, lineno, issue, text))
+            self.logger.warning('[%s:%d] "%s" found in "%-.120s"' %
+                                (self.docname, lineno, issue, text))
         else:
-            self.warn('[%s:%d] "%s" found in "%-.120s"' % (
-                self.docname.encode(sys.getdefaultencoding(),'replace'),
-                lineno,
-                issue.encode(sys.getdefaultencoding(),'replace'),
-                text.strip().encode(sys.getdefaultencoding(),'replace')))
+            self.logger.warning(
+                '[%s:%d] "%s" found in "%-.120s"' % (
+                    self.docname.encode(sys.getdefaultencoding(),'replace'),
+                    lineno,
+                    issue.encode(sys.getdefaultencoding(),'replace'),
+                    text.strip().encode(sys.getdefaultencoding(),'replace')))
         self.app.statuscode = 1
 
     def write_log_entry(self, lineno, issue, text):


From webhook-mailer at python.org  Tue Sep  3 00:21:38 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 03 Sep 2019 04:21:38 -0000
Subject: [Python-checkins] Enforce PEP 257 conventions in ftplib.py
 (GH-15604)
Message-ID: <mailman.214.1567484499.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/efa3b51fd060352cc6220b27a1026e4d4d5401bd
commit: efa3b51fd060352cc6220b27a1026e4d4d5401bd
branch: master
author: Alan Yee <alanyee at users.noreply.github.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-02T21:21:33-07:00
summary:

Enforce PEP 257 conventions in ftplib.py  (GH-15604)



-`"""` over `'''`
-no blank line either before or after the docstring.
-place the closing quotes on a line by themselves

files:
M Lib/ftplib.py

diff --git a/Lib/ftplib.py b/Lib/ftplib.py
index 58a46bca4a36..c339eb25bc2d 100644
--- a/Lib/ftplib.py
+++ b/Lib/ftplib.py
@@ -72,7 +72,6 @@ class error_proto(Error): pass          # response does not begin with [1-5]
 
 # The class itself
 class FTP:
-
     '''An FTP client class.
 
     To create a connection, call the class using these arguments:
@@ -105,12 +104,13 @@ class FTP:
     passiveserver = 1
     encoding = "latin-1"
 
-    # Initialization method (called by class instantiation).
-    # Initialize host to localhost, port to standard ftp port
-    # Optional arguments are host (for connect()),
-    # and user, passwd, acct (for login())
     def __init__(self, host='', user='', passwd='', acct='',
                  timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        """Initialization method (called by class instantiation).
+        Initialize host to localhost, port to standard ftp port.
+        Optional arguments are host (for connect()),
+        and user, passwd, acct (for login()).
+        """
         self.source_address = source_address
         self.timeout = timeout
         if host:
@@ -823,7 +823,6 @@ def parse227(resp):
     '''Parse the '227' response for a PASV request.
     Raises error_proto if it does not contain '(h1,h2,h3,h4,p1,p2)'
     Return ('host.addr.as.numbers', port#) tuple.'''
-
     if resp[:3] != '227':
         raise error_reply(resp)
     global _227_re
@@ -843,7 +842,6 @@ def parse229(resp, peer):
     '''Parse the '229' response for an EPSV request.
     Raises error_proto if it does not contain '(|||port|)'
     Return ('host.addr.as.numbers', port#) tuple.'''
-
     if resp[:3] != '229':
         raise error_reply(resp)
     left = resp.find('(')
@@ -865,7 +863,6 @@ def parse257(resp):
     '''Parse the '257' response for a MKD or PWD request.
     This is a response to a MKD or PWD request: a directory name.
     Returns the directoryname in the 257 reply.'''
-
     if resp[:3] != '257':
         raise error_reply(resp)
     if resp[3:5] != ' "':


From webhook-mailer at python.org  Tue Sep  3 01:17:04 2019
From: webhook-mailer at python.org (Terry Jan Reedy)
Date: Tue, 03 Sep 2019 05:17:04 -0000
Subject: [Python-checkins] bpo-35771: IDLE: Fix flaky tool-tip hover delay
 tests (GH-15634)
Message-ID: <mailman.215.1567487825.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/132acaba5a7f01373ca624b1a5975b190fe866f5
commit: 132acaba5a7f01373ca624b1a5975b190fe866f5
branch: master
author: Tal Einat <taleinat+github at gmail.com>
committer: Terry Jan Reedy <tjreedy at udel.edu>
date: 2019-09-03T01:17:00-04:00
summary:

bpo-35771: IDLE: Fix flaky tool-tip hover delay tests (GH-15634)

Extending the hover delay in test_tooltip should avoid spurious test_idle failures.
One longer delay instead of two shorter delays results in a net speedup.

files:
A Misc/NEWS.d/next/IDLE/2019-09-01-10-22-55.bpo-35771.tdbmbP.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/idle_test/test_tooltip.py
M Lib/idlelib/tooltip.py

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 949b30bd99ca..47c2291d2377 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,9 @@ Released on 2019-10-20?
 ======================================
 
 
+bpo-35771: To avoid occasional spurious test_idle failures on slower
+machines, increase the ``hover_delay`` in test_tooltip.
+
 bpo-37824: Properly handle user input warnings in IDLE shell.
 Cease turning SyntaxWarnings into SyntaxErrors.
 
diff --git a/Lib/idlelib/idle_test/test_tooltip.py b/Lib/idlelib/idle_test/test_tooltip.py
index 44ea1110e155..c616d4fde3b6 100644
--- a/Lib/idlelib/idle_test/test_tooltip.py
+++ b/Lib/idlelib/idle_test/test_tooltip.py
@@ -1,3 +1,10 @@
+"""Test tooltip, coverage 100%.
+
+Coverage is 100% after excluding 6 lines with "# pragma: no cover".
+They involve TclErrors that either should or should not happen in a
+particular situation, and which are 'pass'ed if they do.
+"""
+
 from idlelib.tooltip import TooltipBase, Hovertip
 from test.support import requires
 requires('gui')
@@ -12,16 +19,13 @@ def setUpModule():
     global root
     root = Tk()
 
-def root_update():
-    global root
-    root.update()
-
 def tearDownModule():
     global root
     root.update_idletasks()
     root.destroy()
     del root
 
+
 def add_call_counting(func):
     @wraps(func)
     def wrapped_func(*args, **kwargs):
@@ -65,22 +69,25 @@ class HovertipTest(unittest.TestCase):
     def setUp(self):
         self.top, self.button = _make_top_and_button(self)
 
+    def is_tipwindow_shown(self, tooltip):
+        return tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()
+
     def test_showtip(self):
         tooltip = Hovertip(self.button, 'ToolTip text')
         self.addCleanup(tooltip.hidetip)
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
         tooltip.showtip()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertTrue(self.is_tipwindow_shown(tooltip))
 
     def test_showtip_twice(self):
         tooltip = Hovertip(self.button, 'ToolTip text')
         self.addCleanup(tooltip.hidetip)
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
         tooltip.showtip()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertTrue(self.is_tipwindow_shown(tooltip))
         orig_tipwindow = tooltip.tipwindow
         tooltip.showtip()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertTrue(self.is_tipwindow_shown(tooltip))
         self.assertIs(tooltip.tipwindow, orig_tipwindow)
 
     def test_hidetip(self):
@@ -88,59 +95,67 @@ def test_hidetip(self):
         self.addCleanup(tooltip.hidetip)
         tooltip.showtip()
         tooltip.hidetip()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
 
     def test_showtip_on_mouse_enter_no_delay(self):
         tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=None)
         self.addCleanup(tooltip.hidetip)
         tooltip.showtip = add_call_counting(tooltip.showtip)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        root.update()
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
         self.button.event_generate('<Enter>', x=0, y=0)
-        root_update()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        root.update()
+        self.assertTrue(self.is_tipwindow_shown(tooltip))
         self.assertGreater(len(tooltip.showtip.call_args_list), 0)
 
-    def test_showtip_on_mouse_enter_hover_delay(self):
-        tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=50)
-        self.addCleanup(tooltip.hidetip)
-        tooltip.showtip = add_call_counting(tooltip.showtip)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+    def test_hover_with_delay(self):
+        # Run multiple tests requiring an actual delay simultaneously.
+
+        # Test #1: A hover tip with a non-zero delay appears after the delay.
+        tooltip1 = Hovertip(self.button, 'ToolTip text', hover_delay=100)
+        self.addCleanup(tooltip1.hidetip)
+        tooltip1.showtip = add_call_counting(tooltip1.showtip)
+        root.update()
+        self.assertFalse(self.is_tipwindow_shown(tooltip1))
         self.button.event_generate('<Enter>', x=0, y=0)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
-        time.sleep(0.1)
-        root_update()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
-        self.assertGreater(len(tooltip.showtip.call_args_list), 0)
+        root.update()
+        self.assertFalse(self.is_tipwindow_shown(tooltip1))
+
+        # Test #2: A hover tip with a non-zero delay doesn't appear when
+        # the mouse stops hovering over the base widget before the delay
+        # expires.
+        tooltip2 = Hovertip(self.button, 'ToolTip text', hover_delay=100)
+        self.addCleanup(tooltip2.hidetip)
+        tooltip2.showtip = add_call_counting(tooltip2.showtip)
+        root.update()
+        self.button.event_generate('<Enter>', x=0, y=0)
+        root.update()
+        self.button.event_generate('<Leave>', x=0, y=0)
+        root.update()
+
+        time.sleep(0.15)
+        root.update()
+
+        # Test #1 assertions.
+        self.assertTrue(self.is_tipwindow_shown(tooltip1))
+        self.assertGreater(len(tooltip1.showtip.call_args_list), 0)
+
+        # Test #2 assertions.
+        self.assertFalse(self.is_tipwindow_shown(tooltip2))
+        self.assertEqual(tooltip2.showtip.call_args_list, [])
 
     def test_hidetip_on_mouse_leave(self):
         tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=None)
         self.addCleanup(tooltip.hidetip)
         tooltip.showtip = add_call_counting(tooltip.showtip)
-        root_update()
+        root.update()
         self.button.event_generate('<Enter>', x=0, y=0)
-        root_update()
+        root.update()
         self.button.event_generate('<Leave>', x=0, y=0)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        root.update()
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
         self.assertGreater(len(tooltip.showtip.call_args_list), 0)
 
-    def test_dont_show_on_mouse_leave_before_delay(self):
-        tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=50)
-        self.addCleanup(tooltip.hidetip)
-        tooltip.showtip = add_call_counting(tooltip.showtip)
-        root_update()
-        self.button.event_generate('<Enter>', x=0, y=0)
-        root_update()
-        self.button.event_generate('<Leave>', x=0, y=0)
-        root_update()
-        time.sleep(0.1)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
-        self.assertEqual(tooltip.showtip.call_args_list, [])
-
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/tooltip.py b/Lib/idlelib/tooltip.py
index f54ea36f059d..69658264dbd4 100644
--- a/Lib/idlelib/tooltip.py
+++ b/Lib/idlelib/tooltip.py
@@ -75,7 +75,7 @@ def hidetip(self):
         if tw:
             try:
                 tw.destroy()
-            except TclError:
+            except TclError:  # pragma: no cover
                 pass
 
 
@@ -103,8 +103,8 @@ def __init__(self, anchor_widget, hover_delay=1000):
     def __del__(self):
         try:
             self.anchor_widget.unbind("<Enter>", self._id1)
-            self.anchor_widget.unbind("<Leave>", self._id2)
-            self.anchor_widget.unbind("<Button>", self._id3)
+            self.anchor_widget.unbind("<Leave>", self._id2)  # pragma: no cover
+            self.anchor_widget.unbind("<Button>", self._id3) # pragma: no cover
         except TclError:
             pass
         super(OnHoverTooltipBase, self).__del__()
@@ -137,7 +137,7 @@ def hidetip(self):
         """hide the tooltip"""
         try:
             self.unschedule()
-        except TclError:
+        except TclError:  # pragma: no cover
             pass
         super(OnHoverTooltipBase, self).hidetip()
 
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-01-10-22-55.bpo-35771.tdbmbP.rst b/Misc/NEWS.d/next/IDLE/2019-09-01-10-22-55.bpo-35771.tdbmbP.rst
new file mode 100644
index 000000000000..b96e53f5a5df
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-01-10-22-55.bpo-35771.tdbmbP.rst
@@ -0,0 +1,2 @@
+To avoid occasional spurious test_idle failures on slower machines, 
+increase the ``hover_delay`` in test_tooltip.


From webhook-mailer at python.org  Tue Sep  3 01:35:23 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 03 Sep 2019 05:35:23 -0000
Subject: [Python-checkins] bpo-35771: IDLE: Fix flaky tool-tip hover delay
 tests (GH-15634)
Message-ID: <mailman.216.1567488924.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/48058050cee5f6600150392100c162f223b4317f
commit: 48058050cee5f6600150392100c162f223b4317f
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-02T22:35:19-07:00
summary:

bpo-35771: IDLE: Fix flaky tool-tip hover delay tests (GH-15634)


Extending the hover delay in test_tooltip should avoid spurious test_idle failures.
One longer delay instead of two shorter delays results in a net speedup.
(cherry picked from commit 132acaba5a7f01373ca624b1a5975b190fe866f5)

Co-authored-by: Tal Einat <taleinat+github at gmail.com>

files:
A Misc/NEWS.d/next/IDLE/2019-09-01-10-22-55.bpo-35771.tdbmbP.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/idle_test/test_tooltip.py
M Lib/idlelib/tooltip.py

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index e414ead0762f..b368ea9a3375 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,9 @@ Released on 2019-09-30?
 ======================================
 
 
+bpo-35771: To avoid occasional spurious test_idle failures on slower
+machines, increase the ``hover_delay`` in test_tooltip.
+
 bpo-37824: Properly handle user input warnings in IDLE shell.
 Cease turning SyntaxWarnings into SyntaxErrors.
 
diff --git a/Lib/idlelib/idle_test/test_tooltip.py b/Lib/idlelib/idle_test/test_tooltip.py
index 44ea1110e155..c616d4fde3b6 100644
--- a/Lib/idlelib/idle_test/test_tooltip.py
+++ b/Lib/idlelib/idle_test/test_tooltip.py
@@ -1,3 +1,10 @@
+"""Test tooltip, coverage 100%.
+
+Coverage is 100% after excluding 6 lines with "# pragma: no cover".
+They involve TclErrors that either should or should not happen in a
+particular situation, and which are 'pass'ed if they do.
+"""
+
 from idlelib.tooltip import TooltipBase, Hovertip
 from test.support import requires
 requires('gui')
@@ -12,16 +19,13 @@ def setUpModule():
     global root
     root = Tk()
 
-def root_update():
-    global root
-    root.update()
-
 def tearDownModule():
     global root
     root.update_idletasks()
     root.destroy()
     del root
 
+
 def add_call_counting(func):
     @wraps(func)
     def wrapped_func(*args, **kwargs):
@@ -65,22 +69,25 @@ class HovertipTest(unittest.TestCase):
     def setUp(self):
         self.top, self.button = _make_top_and_button(self)
 
+    def is_tipwindow_shown(self, tooltip):
+        return tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()
+
     def test_showtip(self):
         tooltip = Hovertip(self.button, 'ToolTip text')
         self.addCleanup(tooltip.hidetip)
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
         tooltip.showtip()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertTrue(self.is_tipwindow_shown(tooltip))
 
     def test_showtip_twice(self):
         tooltip = Hovertip(self.button, 'ToolTip text')
         self.addCleanup(tooltip.hidetip)
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
         tooltip.showtip()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertTrue(self.is_tipwindow_shown(tooltip))
         orig_tipwindow = tooltip.tipwindow
         tooltip.showtip()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertTrue(self.is_tipwindow_shown(tooltip))
         self.assertIs(tooltip.tipwindow, orig_tipwindow)
 
     def test_hidetip(self):
@@ -88,59 +95,67 @@ def test_hidetip(self):
         self.addCleanup(tooltip.hidetip)
         tooltip.showtip()
         tooltip.hidetip()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
 
     def test_showtip_on_mouse_enter_no_delay(self):
         tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=None)
         self.addCleanup(tooltip.hidetip)
         tooltip.showtip = add_call_counting(tooltip.showtip)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        root.update()
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
         self.button.event_generate('<Enter>', x=0, y=0)
-        root_update()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        root.update()
+        self.assertTrue(self.is_tipwindow_shown(tooltip))
         self.assertGreater(len(tooltip.showtip.call_args_list), 0)
 
-    def test_showtip_on_mouse_enter_hover_delay(self):
-        tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=50)
-        self.addCleanup(tooltip.hidetip)
-        tooltip.showtip = add_call_counting(tooltip.showtip)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+    def test_hover_with_delay(self):
+        # Run multiple tests requiring an actual delay simultaneously.
+
+        # Test #1: A hover tip with a non-zero delay appears after the delay.
+        tooltip1 = Hovertip(self.button, 'ToolTip text', hover_delay=100)
+        self.addCleanup(tooltip1.hidetip)
+        tooltip1.showtip = add_call_counting(tooltip1.showtip)
+        root.update()
+        self.assertFalse(self.is_tipwindow_shown(tooltip1))
         self.button.event_generate('<Enter>', x=0, y=0)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
-        time.sleep(0.1)
-        root_update()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
-        self.assertGreater(len(tooltip.showtip.call_args_list), 0)
+        root.update()
+        self.assertFalse(self.is_tipwindow_shown(tooltip1))
+
+        # Test #2: A hover tip with a non-zero delay doesn't appear when
+        # the mouse stops hovering over the base widget before the delay
+        # expires.
+        tooltip2 = Hovertip(self.button, 'ToolTip text', hover_delay=100)
+        self.addCleanup(tooltip2.hidetip)
+        tooltip2.showtip = add_call_counting(tooltip2.showtip)
+        root.update()
+        self.button.event_generate('<Enter>', x=0, y=0)
+        root.update()
+        self.button.event_generate('<Leave>', x=0, y=0)
+        root.update()
+
+        time.sleep(0.15)
+        root.update()
+
+        # Test #1 assertions.
+        self.assertTrue(self.is_tipwindow_shown(tooltip1))
+        self.assertGreater(len(tooltip1.showtip.call_args_list), 0)
+
+        # Test #2 assertions.
+        self.assertFalse(self.is_tipwindow_shown(tooltip2))
+        self.assertEqual(tooltip2.showtip.call_args_list, [])
 
     def test_hidetip_on_mouse_leave(self):
         tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=None)
         self.addCleanup(tooltip.hidetip)
         tooltip.showtip = add_call_counting(tooltip.showtip)
-        root_update()
+        root.update()
         self.button.event_generate('<Enter>', x=0, y=0)
-        root_update()
+        root.update()
         self.button.event_generate('<Leave>', x=0, y=0)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        root.update()
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
         self.assertGreater(len(tooltip.showtip.call_args_list), 0)
 
-    def test_dont_show_on_mouse_leave_before_delay(self):
-        tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=50)
-        self.addCleanup(tooltip.hidetip)
-        tooltip.showtip = add_call_counting(tooltip.showtip)
-        root_update()
-        self.button.event_generate('<Enter>', x=0, y=0)
-        root_update()
-        self.button.event_generate('<Leave>', x=0, y=0)
-        root_update()
-        time.sleep(0.1)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
-        self.assertEqual(tooltip.showtip.call_args_list, [])
-
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/tooltip.py b/Lib/idlelib/tooltip.py
index f54ea36f059d..69658264dbd4 100644
--- a/Lib/idlelib/tooltip.py
+++ b/Lib/idlelib/tooltip.py
@@ -75,7 +75,7 @@ def hidetip(self):
         if tw:
             try:
                 tw.destroy()
-            except TclError:
+            except TclError:  # pragma: no cover
                 pass
 
 
@@ -103,8 +103,8 @@ def __init__(self, anchor_widget, hover_delay=1000):
     def __del__(self):
         try:
             self.anchor_widget.unbind("<Enter>", self._id1)
-            self.anchor_widget.unbind("<Leave>", self._id2)
-            self.anchor_widget.unbind("<Button>", self._id3)
+            self.anchor_widget.unbind("<Leave>", self._id2)  # pragma: no cover
+            self.anchor_widget.unbind("<Button>", self._id3) # pragma: no cover
         except TclError:
             pass
         super(OnHoverTooltipBase, self).__del__()
@@ -137,7 +137,7 @@ def hidetip(self):
         """hide the tooltip"""
         try:
             self.unschedule()
-        except TclError:
+        except TclError:  # pragma: no cover
             pass
         super(OnHoverTooltipBase, self).hidetip()
 
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-01-10-22-55.bpo-35771.tdbmbP.rst b/Misc/NEWS.d/next/IDLE/2019-09-01-10-22-55.bpo-35771.tdbmbP.rst
new file mode 100644
index 000000000000..b96e53f5a5df
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-01-10-22-55.bpo-35771.tdbmbP.rst
@@ -0,0 +1,2 @@
+To avoid occasional spurious test_idle failures on slower machines, 
+increase the ``hover_delay`` in test_tooltip.


From webhook-mailer at python.org  Tue Sep  3 01:39:11 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 03 Sep 2019 05:39:11 -0000
Subject: [Python-checkins] bpo-35771: IDLE: Fix flaky tool-tip hover delay
 tests (GH-15634)
Message-ID: <mailman.217.1567489153.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1c18aec3bb7d59061742854d3204439a5274e201
commit: 1c18aec3bb7d59061742854d3204439a5274e201
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-02T22:39:08-07:00
summary:

bpo-35771: IDLE: Fix flaky tool-tip hover delay tests (GH-15634)


Extending the hover delay in test_tooltip should avoid spurious test_idle failures.
One longer delay instead of two shorter delays results in a net speedup.
(cherry picked from commit 132acaba5a7f01373ca624b1a5975b190fe866f5)

Co-authored-by: Tal Einat <taleinat+github at gmail.com>

files:
A Misc/NEWS.d/next/IDLE/2019-09-01-10-22-55.bpo-35771.tdbmbP.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/idle_test/test_tooltip.py
M Lib/idlelib/tooltip.py

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 949b30bd99ca..47c2291d2377 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,9 @@ Released on 2019-10-20?
 ======================================
 
 
+bpo-35771: To avoid occasional spurious test_idle failures on slower
+machines, increase the ``hover_delay`` in test_tooltip.
+
 bpo-37824: Properly handle user input warnings in IDLE shell.
 Cease turning SyntaxWarnings into SyntaxErrors.
 
diff --git a/Lib/idlelib/idle_test/test_tooltip.py b/Lib/idlelib/idle_test/test_tooltip.py
index 44ea1110e155..c616d4fde3b6 100644
--- a/Lib/idlelib/idle_test/test_tooltip.py
+++ b/Lib/idlelib/idle_test/test_tooltip.py
@@ -1,3 +1,10 @@
+"""Test tooltip, coverage 100%.
+
+Coverage is 100% after excluding 6 lines with "# pragma: no cover".
+They involve TclErrors that either should or should not happen in a
+particular situation, and which are 'pass'ed if they do.
+"""
+
 from idlelib.tooltip import TooltipBase, Hovertip
 from test.support import requires
 requires('gui')
@@ -12,16 +19,13 @@ def setUpModule():
     global root
     root = Tk()
 
-def root_update():
-    global root
-    root.update()
-
 def tearDownModule():
     global root
     root.update_idletasks()
     root.destroy()
     del root
 
+
 def add_call_counting(func):
     @wraps(func)
     def wrapped_func(*args, **kwargs):
@@ -65,22 +69,25 @@ class HovertipTest(unittest.TestCase):
     def setUp(self):
         self.top, self.button = _make_top_and_button(self)
 
+    def is_tipwindow_shown(self, tooltip):
+        return tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()
+
     def test_showtip(self):
         tooltip = Hovertip(self.button, 'ToolTip text')
         self.addCleanup(tooltip.hidetip)
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
         tooltip.showtip()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertTrue(self.is_tipwindow_shown(tooltip))
 
     def test_showtip_twice(self):
         tooltip = Hovertip(self.button, 'ToolTip text')
         self.addCleanup(tooltip.hidetip)
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
         tooltip.showtip()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertTrue(self.is_tipwindow_shown(tooltip))
         orig_tipwindow = tooltip.tipwindow
         tooltip.showtip()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertTrue(self.is_tipwindow_shown(tooltip))
         self.assertIs(tooltip.tipwindow, orig_tipwindow)
 
     def test_hidetip(self):
@@ -88,59 +95,67 @@ def test_hidetip(self):
         self.addCleanup(tooltip.hidetip)
         tooltip.showtip()
         tooltip.hidetip()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
 
     def test_showtip_on_mouse_enter_no_delay(self):
         tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=None)
         self.addCleanup(tooltip.hidetip)
         tooltip.showtip = add_call_counting(tooltip.showtip)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        root.update()
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
         self.button.event_generate('<Enter>', x=0, y=0)
-        root_update()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        root.update()
+        self.assertTrue(self.is_tipwindow_shown(tooltip))
         self.assertGreater(len(tooltip.showtip.call_args_list), 0)
 
-    def test_showtip_on_mouse_enter_hover_delay(self):
-        tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=50)
-        self.addCleanup(tooltip.hidetip)
-        tooltip.showtip = add_call_counting(tooltip.showtip)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+    def test_hover_with_delay(self):
+        # Run multiple tests requiring an actual delay simultaneously.
+
+        # Test #1: A hover tip with a non-zero delay appears after the delay.
+        tooltip1 = Hovertip(self.button, 'ToolTip text', hover_delay=100)
+        self.addCleanup(tooltip1.hidetip)
+        tooltip1.showtip = add_call_counting(tooltip1.showtip)
+        root.update()
+        self.assertFalse(self.is_tipwindow_shown(tooltip1))
         self.button.event_generate('<Enter>', x=0, y=0)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
-        time.sleep(0.1)
-        root_update()
-        self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
-        self.assertGreater(len(tooltip.showtip.call_args_list), 0)
+        root.update()
+        self.assertFalse(self.is_tipwindow_shown(tooltip1))
+
+        # Test #2: A hover tip with a non-zero delay doesn't appear when
+        # the mouse stops hovering over the base widget before the delay
+        # expires.
+        tooltip2 = Hovertip(self.button, 'ToolTip text', hover_delay=100)
+        self.addCleanup(tooltip2.hidetip)
+        tooltip2.showtip = add_call_counting(tooltip2.showtip)
+        root.update()
+        self.button.event_generate('<Enter>', x=0, y=0)
+        root.update()
+        self.button.event_generate('<Leave>', x=0, y=0)
+        root.update()
+
+        time.sleep(0.15)
+        root.update()
+
+        # Test #1 assertions.
+        self.assertTrue(self.is_tipwindow_shown(tooltip1))
+        self.assertGreater(len(tooltip1.showtip.call_args_list), 0)
+
+        # Test #2 assertions.
+        self.assertFalse(self.is_tipwindow_shown(tooltip2))
+        self.assertEqual(tooltip2.showtip.call_args_list, [])
 
     def test_hidetip_on_mouse_leave(self):
         tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=None)
         self.addCleanup(tooltip.hidetip)
         tooltip.showtip = add_call_counting(tooltip.showtip)
-        root_update()
+        root.update()
         self.button.event_generate('<Enter>', x=0, y=0)
-        root_update()
+        root.update()
         self.button.event_generate('<Leave>', x=0, y=0)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
+        root.update()
+        self.assertFalse(self.is_tipwindow_shown(tooltip))
         self.assertGreater(len(tooltip.showtip.call_args_list), 0)
 
-    def test_dont_show_on_mouse_leave_before_delay(self):
-        tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=50)
-        self.addCleanup(tooltip.hidetip)
-        tooltip.showtip = add_call_counting(tooltip.showtip)
-        root_update()
-        self.button.event_generate('<Enter>', x=0, y=0)
-        root_update()
-        self.button.event_generate('<Leave>', x=0, y=0)
-        root_update()
-        time.sleep(0.1)
-        root_update()
-        self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable())
-        self.assertEqual(tooltip.showtip.call_args_list, [])
-
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/tooltip.py b/Lib/idlelib/tooltip.py
index f54ea36f059d..69658264dbd4 100644
--- a/Lib/idlelib/tooltip.py
+++ b/Lib/idlelib/tooltip.py
@@ -75,7 +75,7 @@ def hidetip(self):
         if tw:
             try:
                 tw.destroy()
-            except TclError:
+            except TclError:  # pragma: no cover
                 pass
 
 
@@ -103,8 +103,8 @@ def __init__(self, anchor_widget, hover_delay=1000):
     def __del__(self):
         try:
             self.anchor_widget.unbind("<Enter>", self._id1)
-            self.anchor_widget.unbind("<Leave>", self._id2)
-            self.anchor_widget.unbind("<Button>", self._id3)
+            self.anchor_widget.unbind("<Leave>", self._id2)  # pragma: no cover
+            self.anchor_widget.unbind("<Button>", self._id3) # pragma: no cover
         except TclError:
             pass
         super(OnHoverTooltipBase, self).__del__()
@@ -137,7 +137,7 @@ def hidetip(self):
         """hide the tooltip"""
         try:
             self.unschedule()
-        except TclError:
+        except TclError:  # pragma: no cover
             pass
         super(OnHoverTooltipBase, self).hidetip()
 
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-01-10-22-55.bpo-35771.tdbmbP.rst b/Misc/NEWS.d/next/IDLE/2019-09-01-10-22-55.bpo-35771.tdbmbP.rst
new file mode 100644
index 000000000000..b96e53f5a5df
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-01-10-22-55.bpo-35771.tdbmbP.rst
@@ -0,0 +1,2 @@
+To avoid occasional spurious test_idle failures on slower machines, 
+increase the ``hover_delay`` in test_tooltip.


From webhook-mailer at python.org  Tue Sep  3 05:21:52 2019
From: webhook-mailer at python.org (Tal Einat)
Date: Tue, 03 Sep 2019 09:21:52 -0000
Subject: [Python-checkins] bpo-37798: Fix _statistics module doc (GH-15546)
Message-ID: <mailman.218.1567502514.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0cf832a9ef84be6e18aad02464814530d80b82b2
commit: 0cf832a9ef84be6e18aad02464814530d80b82b2
branch: master
author: Dong-hee Na <donghee.na92 at gmail.com>
committer: Tal Einat <taleinat+github at gmail.com>
date: 2019-09-03T12:21:45+03:00
summary:

bpo-37798: Fix _statistics module doc (GH-15546)

files:
M Modules/_statisticsmodule.c

diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c
index 676d3fb33803..16a992485237 100644
--- a/Modules/_statisticsmodule.c
+++ b/Modules/_statisticsmodule.c
@@ -110,10 +110,13 @@ static PyMethodDef statistics_methods[] = {
     {NULL, NULL, 0, NULL}
 };
 
+PyDoc_STRVAR(statistics_doc,
+"Accelerators for the statistics module.\n");
+
 static struct PyModuleDef statisticsmodule = {
         PyModuleDef_HEAD_INIT,
         "_statistics",
-        _statistics__normal_dist_inv_cdf__doc__,
+        statistics_doc,
         -1,
         statistics_methods,
         NULL,


From webhook-mailer at python.org  Tue Sep  3 06:22:06 2019
From: webhook-mailer at python.org (Tal Einat)
Date: Tue, 03 Sep 2019 10:22:06 -0000
Subject: [Python-checkins] bpo-37798: Fix _statistics module doc (GH-15546)
Message-ID: <mailman.219.1567506128.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/58067d2cf6e81187f9782aff03cc8bec3d878778
commit: 58067d2cf6e81187f9782aff03cc8bec3d878778
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Tal Einat <taleinat+github at gmail.com>
date: 2019-09-03T13:21:59+03:00
summary:

bpo-37798: Fix _statistics module doc (GH-15546)

(cherry picked from commit 0cf832a9ef84be6e18aad02464814530d80b82b2)

Co-authored-by: Dong-hee Na <donghee.na92 at gmail.com>

files:
M Modules/_statisticsmodule.c

diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c
index 676d3fb33803..16a992485237 100644
--- a/Modules/_statisticsmodule.c
+++ b/Modules/_statisticsmodule.c
@@ -110,10 +110,13 @@ static PyMethodDef statistics_methods[] = {
     {NULL, NULL, 0, NULL}
 };
 
+PyDoc_STRVAR(statistics_doc,
+"Accelerators for the statistics module.\n");
+
 static struct PyModuleDef statisticsmodule = {
         PyModuleDef_HEAD_INIT,
         "_statistics",
-        _statistics__normal_dist_inv_cdf__doc__,
+        statistics_doc,
         -1,
         statistics_methods,
         NULL,


From webhook-mailer at python.org  Tue Sep  3 12:42:59 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 03 Sep 2019 16:42:59 -0000
Subject: [Python-checkins] [3.7] bpo-37764: Fix infinite loop when parsing
 unstructured email headers. (GH-15239) (GH-15654)
Message-ID: <mailman.220.1567528981.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ea21389dda401457198fb214aa2c981a45ed9528
commit: ea21389dda401457198fb214aa2c981a45ed9528
branch: 3.7
author: Ashwin Ramaswami <aramaswamis at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-03T09:42:53-07:00
summary:

[3.7] bpo-37764: Fix infinite loop when parsing unstructured email headers. (GH-15239) (GH-15654)



?aders. (GH-15239)

Fixes a case in which email._header_value_parser.get_unstructured hangs the system for some invalid headers. This covers the cases in which the header contains either:
- a case without trailing whitespace
- an invalid encoded word

https://bugs.python.org/issue37764

This fix should also be backported to 3.7 and 3.8

https://bugs.python.org/issue37764
(cherry picked from commit c5b242f87f31286ad38991bc3868cf4cfbf2b681)

Co-authored-by: Ashwin Ramaswami <aramaswamis at gmail.com>





https://bugs.python.org/issue37764

files:
A Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst
M Lib/email/_header_value_parser.py
M Lib/test/test_email/test__header_value_parser.py
M Lib/test/test_email/test_email.py
M Misc/ACKS

diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index 2306879fc9d7..7cc9a4672601 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -933,6 +933,10 @@ def __str__(self):
         return ''
 
 
+class _InvalidEwError(errors.HeaderParseError):
+    """Invalid encoded word found while parsing headers."""
+
+
 # XXX these need to become classes and used as instances so
 # that a program can't change them in a parse tree and screw
 # up other parse trees.  Maybe should have  tests for that, too.
@@ -1037,7 +1041,10 @@ def get_encoded_word(value):
         raise errors.HeaderParseError(
             "expected encoded word but found {}".format(value))
     remstr = ''.join(remainder)
-    if len(remstr) > 1 and remstr[0] in hexdigits and remstr[1] in hexdigits:
+    if (len(remstr) > 1 and
+        remstr[0] in hexdigits and
+        remstr[1] in hexdigits and
+        tok.count('?') < 2):
         # The ? after the CTE was followed by an encoded word escape (=XX).
         rest, *remainder = remstr.split('?=', 1)
         tok = tok + '?=' + rest
@@ -1049,7 +1056,7 @@ def get_encoded_word(value):
     try:
         text, charset, lang, defects = _ew.decode('=?' + tok + '?=')
     except ValueError:
-        raise errors.HeaderParseError(
+        raise _InvalidEwError(
             "encoded word format invalid: '{}'".format(ew.cte))
     ew.charset = charset
     ew.lang = lang
@@ -1099,9 +1106,12 @@ def get_unstructured(value):
             token, value = get_fws(value)
             unstructured.append(token)
             continue
+        valid_ew = True
         if value.startswith('=?'):
             try:
                 token, value = get_encoded_word(value)
+            except _InvalidEwError:
+                valid_ew = False
             except errors.HeaderParseError:
                 # XXX: Need to figure out how to register defects when
                 # appropriate here.
@@ -1123,7 +1133,10 @@ def get_unstructured(value):
         # Split in the middle of an atom if there is a rfc2047 encoded word
         # which does not have WSP on both sides. The defect will be registered
         # the next time through the loop.
-        if rfc2047_matcher.search(tok):
+        # This needs to only be performed when the encoded word is valid;
+        # otherwise, performing it on an invalid encoded word can cause
+        # the parser to go in an infinite loop.
+        if valid_ew and rfc2047_matcher.search(tok):
             tok, *remainder = value.partition('=?')
         vtext = ValueTerminal(tok, 'vtext')
         _validate_xtext(vtext)
diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py
index dd6975784568..8bb0365739f7 100644
--- a/Lib/test/test_email/test__header_value_parser.py
+++ b/Lib/test/test_email/test__header_value_parser.py
@@ -383,6 +383,22 @@ def test_get_unstructured_ew_without_trailing_whitespace(self):
             [errors.InvalidHeaderDefect],
             '')
 
+    def test_get_unstructured_without_trailing_whitespace_hang_case(self):
+        self._test_get_x(self._get_unst,
+            '=?utf-8?q?somevalue?=aa',
+            'somevalueaa',
+            'somevalueaa',
+            [errors.InvalidHeaderDefect],
+            '')
+
+    def test_get_unstructured_invalid_ew(self):
+        self._test_get_x(self._get_unst,
+            '=?utf-8?q?=somevalue?=',
+            '=?utf-8?q?=somevalue?=',
+            '=?utf-8?q?=somevalue?=',
+            [],
+            '')
+
     # get_qp_ctext
 
     def test_get_qp_ctext_only(self):
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index aa775881c552..5414cf070cc1 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -5381,6 +5381,27 @@ def test_rfc2231_unencoded_then_encoded_segments(self):
         eq(language, 'en-us')
         eq(s, 'My Document For You')
 
+    def test_should_not_hang_on_invalid_ew_messages(self):
+        messages = ["""From: user at host.com
+To: user at host.com
+Bad-Header:
+ =?us-ascii?Q?LCSwrV11+IB0rSbSker+M9vWR7wEDSuGqmHD89Gt=ea0nJFSaiz4vX3XMJPT4vrE?=
+ =?us-ascii?Q?xGUZeOnp0o22pLBB7CYLH74Js=wOlK6Tfru2U47qR?=
+ =?us-ascii?Q?72OfyEY2p2=2FrA9xNFyvH+fBTCmazxwzF8nGkK6D?=
+
+Hello!
+""", """From: ??????????????? ???????????????????????? <xxx at xxx>
+To: "xxx" <xxx at xxx>
+Subject:   ????????? ?????????????????????????????? ??????????????? ??????????????? ??? ??????????????????????????? ?????? ????????????
+MIME-Version: 1.0
+Content-Type: text/plain; charset="windows-1251";
+Content-Transfer-Encoding: 8bit
+
+?????? ??????????????? ??? ???????????? ?????????????????? ????????? ????????????????????????
+"""]
+        for m in messages:
+            with self.subTest(m=m):
+                msg = email.message_from_string(m)
 
 
 # Tests to ensure that signed parts of an email are completely preserved, as
diff --git a/Misc/ACKS b/Misc/ACKS
index ade26f5e86d2..ec14bdc7bb59 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1307,6 +1307,7 @@ Burton Radons
 Abhilash Raj
 Shorya Raj
 Dhushyanth Ramasamy
+Ashwin Ramaswami
 Jeff Ramnani
 Bayard Randel
 Varpu Rantala
diff --git a/Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst b/Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst
new file mode 100644
index 000000000000..27fa8e192f0c
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst
@@ -0,0 +1 @@
+Fixes email._header_value_parser.get_unstructured going into an infinite loop for a specific case in which the email header does not have trailing whitespace, and the case in which it contains an invalid encoded word. Patch by Ashwin Ramaswami.
\ No newline at end of file


From webhook-mailer at python.org  Tue Sep  3 13:08:45 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 03 Sep 2019 17:08:45 -0000
Subject: [Python-checkins] [3.8] bpo-21315: Fix parsing of encoded words
 with missing leading ws (GH-13425) (GH-15655)
Message-ID: <mailman.221.1567530526.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/59e8fba7189d0e86d428a1125744afb8b0f40b5d
commit: 59e8fba7189d0e86d428a1125744afb8b0f40b5d
branch: 3.8
author: Ashwin Ramaswami <aramaswamis at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-03T10:08:39-07:00
summary:

[3.8] bpo-21315: Fix parsing of encoded words with missing leading ws (GH-13425) (GH-15655)



* [bpo-21315](https://bugs.python.org/issue21315): Fix parsing of encoded words with missing leading ws.

Because of missing leading whitespace, encoded word would get parsed as
unstructured token. This patch fixes that by looking for encoded words when
splitting tokens with whitespace.

Missing trailing whitespace around encoded word now register a defect
instead.

Original patch suggestion by David R. Murray on [bpo-21315](https://bugs.python.org/issue21315).
(cherry picked from commit 66c4f3f38b867d8329b28c032bb907fd1a2f22d2)

Co-authored-by: Abhilash Raj <maxking at users.noreply.github.com>
(cherry picked from commit dc20fc4311dece19488299a7cd11317ffbe4d3c3)

Co-authored-by: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>





https://bugs.python.org/issue21315

files:
A Misc/NEWS.d/next/Library/2019-05-19-10-48-46.bpo-21315.PgXVqF.rst
M Lib/email/_header_value_parser.py
M Lib/test/test_email/test__header_value_parser.py
M Lib/test/test_email/test_headerregistry.py

diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index ea7083fc88b8..b5003943ab0d 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -96,6 +96,18 @@
 def quote_string(value):
     return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"'
 
+# Match a RFC 2047 word, looks like =?utf-8?q?someword?=
+rfc2047_matcher = re.compile(r'''
+   =\?            # literal =?
+   [^?]*          # charset
+   \?             # literal ?
+   [qQbB]         # literal 'q' or 'b', case insensitive
+   \?             # literal ?
+  .*?             # encoded word
+  \?=             # literal ?=
+''', re.VERBOSE | re.MULTILINE)
+
+
 #
 # TokenList and its subclasses
 #
@@ -1054,6 +1066,10 @@ def get_encoded_word(value):
         _validate_xtext(vtext)
         ew.append(vtext)
         text = ''.join(remainder)
+    # Encoded words should be followed by a WS
+    if value and value[0] not in WSP:
+        ew.defects.append(errors.InvalidHeaderDefect(
+            "missing trailing whitespace after encoded-word"))
     return ew, value
 
 def get_unstructured(value):
@@ -1106,6 +1122,11 @@ def get_unstructured(value):
                 unstructured.append(token)
                 continue
         tok, *remainder = _wsp_splitter(value, 1)
+        # Split in the middle of an atom if there is a rfc2047 encoded word
+        # which does not have WSP on both sides. The defect will be registered
+        # the next time through the loop.
+        if rfc2047_matcher.search(tok):
+            tok, *remainder = value.partition('=?')
         vtext = ValueTerminal(tok, 'vtext')
         _validate_xtext(vtext)
         unstructured.append(vtext)
diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py
index 198cf17bbf3e..bad4333dbc43 100644
--- a/Lib/test/test_email/test__header_value_parser.py
+++ b/Lib/test/test_email/test__header_value_parser.py
@@ -118,7 +118,7 @@ def test_get_encoded_word_gets_first_even_if_no_space(self):
                          '=?us-ascii?q?first?==?utf-8?q?second?=',
                          'first',
                          'first',
-                         [],
+                         [errors.InvalidHeaderDefect],
                          '=?utf-8?q?second?=')
 
     def test_get_encoded_word_sets_extra_attributes(self):
@@ -361,6 +361,25 @@ def test_get_unstructured_no_whitespace_between_ews(self):
             '=?utf-8?q?foo?==?utf-8?q?bar?=',
             'foobar',
             'foobar',
+            [errors.InvalidHeaderDefect,
+            errors.InvalidHeaderDefect],
+            '')
+
+    def test_get_unstructured_ew_without_leading_whitespace(self):
+        self._test_get_x(
+            self._get_unst,
+            'nowhitespace=?utf-8?q?somevalue?=',
+            'nowhitespacesomevalue',
+            'nowhitespacesomevalue',
+            [errors.InvalidHeaderDefect],
+            '')
+
+    def test_get_unstructured_ew_without_trailing_whitespace(self):
+        self._test_get_x(
+            self._get_unst,
+            '=?utf-8?q?somevalue?=nowhitespace',
+            'somevaluenowhitespace',
+            'somevaluenowhitespace',
             [errors.InvalidHeaderDefect],
             '')
 
@@ -550,7 +569,8 @@ def test_encoded_word_inside_quotes(self):
             '"=?utf-8?Q?not_really_valid?="',
             '"not really valid"',
             'not really valid',
-            [errors.InvalidHeaderDefect],
+            [errors.InvalidHeaderDefect,
+             errors.InvalidHeaderDefect],
             '')
 
     # get_comment
diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py
index a6b48385aeac..8d89c5dd5832 100644
--- a/Lib/test/test_email/test_headerregistry.py
+++ b/Lib/test/test_email/test_headerregistry.py
@@ -1180,7 +1180,8 @@ class TestAddressHeader(TestHeaderBase):
 
         'rfc2047_atom_in_quoted_string_is_decoded':
             ('"=?utf-8?q?=C3=89ric?=" <foo at example.com>',
-            [errors.InvalidHeaderDefect],
+            [errors.InvalidHeaderDefect,
+            errors.InvalidHeaderDefect],
             '?ric <foo at example.com>',
             '?ric',
             'foo at example.com',
diff --git a/Misc/NEWS.d/next/Library/2019-05-19-10-48-46.bpo-21315.PgXVqF.rst b/Misc/NEWS.d/next/Library/2019-05-19-10-48-46.bpo-21315.PgXVqF.rst
new file mode 100644
index 000000000000..dd0dd7f72c0a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-19-10-48-46.bpo-21315.PgXVqF.rst
@@ -0,0 +1,4 @@
+Email headers containing RFC2047 encoded words are parsed despite the missing
+whitespace, and a defect registered. Also missing trailing whitespace after
+encoded words is now registered as a defect.
+


From webhook-mailer at python.org  Tue Sep  3 15:50:57 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 03 Sep 2019 19:50:57 -0000
Subject: [Python-checkins] bpo-38020: Fixes crash in os.readlink() on
 Windows (GH-15663)
Message-ID: <mailman.222.1567540257.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/993ac92418839427d4068d6ae8e618b06b5d9294
commit: 993ac92418839427d4068d6ae8e618b06b5d9294
branch: master
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-03T12:50:51-07:00
summary:

bpo-38020: Fixes crash in os.readlink() on Windows (GH-15663)

files:
A Misc/NEWS.d/next/Windows/2019-09-03-11-47-37.bpo-38020.xFZ2j0.rst
M Modules/posixmodule.c

diff --git a/Misc/NEWS.d/next/Windows/2019-09-03-11-47-37.bpo-38020.xFZ2j0.rst b/Misc/NEWS.d/next/Windows/2019-09-03-11-47-37.bpo-38020.xFZ2j0.rst
new file mode 100644
index 000000000000..e6ddaba2f67b
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-03-11-47-37.bpo-38020.xFZ2j0.rst
@@ -0,0 +1,2 @@
+Fixes potential crash when calling :func:`os.readlink` (or indirectly
+through :func:`~os.path.realpath`) on a file that is not a supported link.
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 2302678ccc14..c412d07a0e11 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -7818,7 +7818,7 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd)
     HANDLE reparse_point_handle;
     char target_buffer[_Py_MAXIMUM_REPARSE_DATA_BUFFER_SIZE];
     _Py_REPARSE_DATA_BUFFER *rdb = (_Py_REPARSE_DATA_BUFFER *)target_buffer;
-    PyObject *result;
+    PyObject *result = NULL;
 
     /* First get a handle to the reparse point */
     Py_BEGIN_ALLOW_THREADS
@@ -7872,7 +7872,7 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd)
             name[1] = L'\\';
         }
         result = PyUnicode_FromWideChar(name, nameLen);
-        if (path->narrow) {
+        if (result && path->narrow) {
             Py_SETREF(result, PyUnicode_EncodeFSDefault(result));
         }
     }


From webhook-mailer at python.org  Tue Sep  3 16:13:47 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 03 Sep 2019 20:13:47 -0000
Subject: [Python-checkins] bpo-38020: Fixes crash in os.readlink() on
 Windows (GH-15663)
Message-ID: <mailman.223.1567541628.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/54dac6c0f4fa24d03025c45899ee7c5f2a4c367e
commit: 54dac6c0f4fa24d03025c45899ee7c5f2a4c367e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-03T13:13:41-07:00
summary:

bpo-38020: Fixes crash in os.readlink() on Windows (GH-15663)

(cherry picked from commit 993ac92418839427d4068d6ae8e618b06b5d9294)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-03-11-47-37.bpo-38020.xFZ2j0.rst
M Modules/posixmodule.c

diff --git a/Misc/NEWS.d/next/Windows/2019-09-03-11-47-37.bpo-38020.xFZ2j0.rst b/Misc/NEWS.d/next/Windows/2019-09-03-11-47-37.bpo-38020.xFZ2j0.rst
new file mode 100644
index 000000000000..e6ddaba2f67b
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-03-11-47-37.bpo-38020.xFZ2j0.rst
@@ -0,0 +1,2 @@
+Fixes potential crash when calling :func:`os.readlink` (or indirectly
+through :func:`~os.path.realpath`) on a file that is not a supported link.
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index c178facc4bee..b9e8c0d94c20 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -7821,7 +7821,7 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd)
     HANDLE reparse_point_handle;
     char target_buffer[_Py_MAXIMUM_REPARSE_DATA_BUFFER_SIZE];
     _Py_REPARSE_DATA_BUFFER *rdb = (_Py_REPARSE_DATA_BUFFER *)target_buffer;
-    PyObject *result;
+    PyObject *result = NULL;
 
     /* First get a handle to the reparse point */
     Py_BEGIN_ALLOW_THREADS
@@ -7875,7 +7875,7 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd)
             name[1] = L'\\';
         }
         result = PyUnicode_FromWideChar(name, nameLen);
-        if (path->narrow) {
+        if (result && path->narrow) {
             Py_SETREF(result, PyUnicode_EncodeFSDefault(result));
         }
     }


From webhook-mailer at python.org  Tue Sep  3 16:53:04 2019
From: webhook-mailer at python.org (Terry Jan Reedy)
Date: Tue, 03 Sep 2019 20:53:04 -0000
Subject: [Python-checkins] bpo-38022: IDLE: upgrade help.html to sphinx 2.x
 HTML5 output (GH-15664)
Message-ID: <mailman.224.1567543985.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/580bdb0ece681537eadb360f0c796123ead7a559
commit: 580bdb0ece681537eadb360f0c796123ead7a559
branch: master
author: Tal Einat <taleinat+github at gmail.com>
committer: Terry Jan Reedy <tjreedy at udel.edu>
date: 2019-09-03T16:52:58-04:00
summary:

bpo-38022: IDLE: upgrade help.html to sphinx 2.x HTML5 output (GH-15664)

The HTML5 output from Sphinx 2.x adds  '<p>' tags within list elements.  Using a new prevtag attribute, ignore these instead of emitting unwanted '\n\n'.

Also stop looking for 'first' classes on tags (no longer present) and fix the bug of double-spacing instead of single spacing after <pre> blocks.

files:
M Lib/idlelib/help.html
M Lib/idlelib/help.py

diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html
index 91e66a4b9117..0754f2453baf 100644
--- a/Lib/idlelib/help.html
+++ b/Lib/idlelib/help.html
@@ -1,11 +1,9 @@
 
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!DOCTYPE html>
 
 <html xmlns="http://www.w3.org/1999/xhtml">
   <head>
-    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
-    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta charset="utf-8" />
     <title>IDLE &#8212; Python 3.9.0a0 documentation</title>
     <link rel="stylesheet" href="../_static/pydoctheme.css" type="text/css" />
     <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
@@ -108,17 +106,17 @@ <h3>Navigation</h3>
 <p>IDLE is Python?s Integrated Development and Learning Environment.</p>
 <p>IDLE has the following features:</p>
 <ul class="simple">
-<li>coded in 100% pure Python, using the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><code class="xref py py-mod docutils literal notranslate"><span class="pre">tkinter</span></code></a> GUI toolkit</li>
-<li>cross-platform: works mostly the same on Windows, Unix, and macOS</li>
-<li>Python shell window (interactive interpreter) with colorizing
-of code input, output, and error messages</li>
-<li>multi-window text editor with multiple undo, Python colorizing,
-smart indent, call tips, auto completion, and other features</li>
-<li>search within any window, replace within editor windows, and search
-through multiple files (grep)</li>
-<li>debugger with persistent breakpoints, stepping, and viewing
-of global and local namespaces</li>
-<li>configuration, browsers, and other dialogs</li>
+<li><p>coded in 100% pure Python, using the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><code class="xref py py-mod docutils literal notranslate"><span class="pre">tkinter</span></code></a> GUI toolkit</p></li>
+<li><p>cross-platform: works mostly the same on Windows, Unix, and macOS</p></li>
+<li><p>Python shell window (interactive interpreter) with colorizing
+of code input, output, and error messages</p></li>
+<li><p>multi-window text editor with multiple undo, Python colorizing,
+smart indent, call tips, auto completion, and other features</p></li>
+<li><p>search within any window, replace within editor windows, and search
+through multiple files (grep)</p></li>
+<li><p>debugger with persistent breakpoints, stepping, and viewing
+of global and local namespaces</p></li>
+<li><p>configuration, browsers, and other dialogs</p></li>
 </ul>
 <div class="section" id="menus">
 <h2>Menus<a class="headerlink" href="#menus" title="Permalink to this headline">?</a></h2>
@@ -134,161 +132,216 @@ <h2>Menus<a class="headerlink" href="#menus" title="Permalink to this headline">
 described below are moved around to conform to Apple guidelines.</p>
 <div class="section" id="file-menu-shell-and-editor">
 <h3>File menu (Shell and Editor)<a class="headerlink" href="#file-menu-shell-and-editor" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>New File</dt><dd>Create a new file editing window.</dd>
-<dt>Open?</dt><dd>Open an existing file with an Open dialog.</dd>
-<dt>Recent Files</dt><dd>Open a list of recent files.  Click one to open it.</dd>
-<dt>Open Module?</dt><dd>Open an existing module (searches sys.path).</dd>
+<dl class="simple">
+<dt>New File</dt><dd><p>Create a new file editing window.</p>
+</dd>
+<dt>Open?</dt><dd><p>Open an existing file with an Open dialog.</p>
+</dd>
+<dt>Recent Files</dt><dd><p>Open a list of recent files.  Click one to open it.</p>
+</dd>
+<dt>Open Module?</dt><dd><p>Open an existing module (searches sys.path).</p>
+</dd>
 </dl>
-<dl class="docutils" id="index-1">
-<dt>Class Browser</dt><dd>Show functions, classes, and methods in the current Editor file in a
-tree structure.  In the shell, open a module first.</dd>
-<dt>Path Browser</dt><dd>Show sys.path directories, modules, functions, classes and methods in a
-tree structure.</dd>
-<dt>Save</dt><dd>Save the current window to the associated file, if there is one.  Windows
+<dl class="simple" id="index-1">
+<dt>Class Browser</dt><dd><p>Show functions, classes, and methods in the current Editor file in a
+tree structure.  In the shell, open a module first.</p>
+</dd>
+<dt>Path Browser</dt><dd><p>Show sys.path directories, modules, functions, classes and methods in a
+tree structure.</p>
+</dd>
+<dt>Save</dt><dd><p>Save the current window to the associated file, if there is one.  Windows
 that have been changed since being opened or last saved have a * before
 and after the window title.  If there is no associated file,
-do Save As instead.</dd>
-<dt>Save As?</dt><dd>Save the current window with a Save As dialog.  The file saved becomes the
-new associated file for the window.</dd>
-<dt>Save Copy As?</dt><dd>Save the current window to different file without changing the associated
-file.</dd>
-<dt>Print Window</dt><dd>Print the current window to the default printer.</dd>
-<dt>Close</dt><dd>Close the current window (ask to save if unsaved).</dd>
-<dt>Exit</dt><dd>Close all windows and quit IDLE (ask to save unsaved windows).</dd>
+do Save As instead.</p>
+</dd>
+<dt>Save As?</dt><dd><p>Save the current window with a Save As dialog.  The file saved becomes the
+new associated file for the window.</p>
+</dd>
+<dt>Save Copy As?</dt><dd><p>Save the current window to different file without changing the associated
+file.</p>
+</dd>
+<dt>Print Window</dt><dd><p>Print the current window to the default printer.</p>
+</dd>
+<dt>Close</dt><dd><p>Close the current window (ask to save if unsaved).</p>
+</dd>
+<dt>Exit</dt><dd><p>Close all windows and quit IDLE (ask to save unsaved windows).</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="edit-menu-shell-and-editor">
 <h3>Edit menu (Shell and Editor)<a class="headerlink" href="#edit-menu-shell-and-editor" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>Undo</dt><dd>Undo the last change to the current window.  A maximum of 1000 changes may
-be undone.</dd>
-<dt>Redo</dt><dd>Redo the last undone change to the current window.</dd>
-<dt>Cut</dt><dd>Copy selection into the system-wide clipboard; then delete the selection.</dd>
-<dt>Copy</dt><dd>Copy selection into the system-wide clipboard.</dd>
-<dt>Paste</dt><dd>Insert contents of the system-wide clipboard into the current window.</dd>
+<dl class="simple">
+<dt>Undo</dt><dd><p>Undo the last change to the current window.  A maximum of 1000 changes may
+be undone.</p>
+</dd>
+<dt>Redo</dt><dd><p>Redo the last undone change to the current window.</p>
+</dd>
+<dt>Cut</dt><dd><p>Copy selection into the system-wide clipboard; then delete the selection.</p>
+</dd>
+<dt>Copy</dt><dd><p>Copy selection into the system-wide clipboard.</p>
+</dd>
+<dt>Paste</dt><dd><p>Insert contents of the system-wide clipboard into the current window.</p>
+</dd>
 </dl>
 <p>The clipboard functions are also available in context menus.</p>
-<dl class="docutils">
-<dt>Select All</dt><dd>Select the entire contents of the current window.</dd>
-<dt>Find?</dt><dd>Open a search dialog with many options</dd>
-<dt>Find Again</dt><dd>Repeat the last search, if there is one.</dd>
-<dt>Find Selection</dt><dd>Search for the currently selected string, if there is one.</dd>
-<dt>Find in Files?</dt><dd>Open a file search dialog.  Put results in a new output window.</dd>
-<dt>Replace?</dt><dd>Open a search-and-replace dialog.</dd>
-<dt>Go to Line</dt><dd>Move cursor to the line number requested and make that line visible.</dd>
-<dt>Show Completions</dt><dd>Open a scrollable list allowing selection of keywords and attributes. See
-<a class="reference internal" href="#completions"><span class="std std-ref">Completions</span></a> in the Editing and navigation section below.</dd>
-<dt>Expand Word</dt><dd>Expand a prefix you have typed to match a full word in the same window;
-repeat to get a different expansion.</dd>
-<dt>Show call tip</dt><dd>After an unclosed parenthesis for a function, open a small window with
+<dl class="simple">
+<dt>Select All</dt><dd><p>Select the entire contents of the current window.</p>
+</dd>
+<dt>Find?</dt><dd><p>Open a search dialog with many options</p>
+</dd>
+<dt>Find Again</dt><dd><p>Repeat the last search, if there is one.</p>
+</dd>
+<dt>Find Selection</dt><dd><p>Search for the currently selected string, if there is one.</p>
+</dd>
+<dt>Find in Files?</dt><dd><p>Open a file search dialog.  Put results in a new output window.</p>
+</dd>
+<dt>Replace?</dt><dd><p>Open a search-and-replace dialog.</p>
+</dd>
+<dt>Go to Line</dt><dd><p>Move cursor to the line number requested and make that line visible.</p>
+</dd>
+<dt>Show Completions</dt><dd><p>Open a scrollable list allowing selection of keywords and attributes. See
+<a class="reference internal" href="#completions"><span class="std std-ref">Completions</span></a> in the Editing and navigation section below.</p>
+</dd>
+<dt>Expand Word</dt><dd><p>Expand a prefix you have typed to match a full word in the same window;
+repeat to get a different expansion.</p>
+</dd>
+<dt>Show call tip</dt><dd><p>After an unclosed parenthesis for a function, open a small window with
 function parameter hints.  See <a class="reference internal" href="#calltips"><span class="std std-ref">Calltips</span></a> in the
-Editing and navigation section below.</dd>
-<dt>Show surrounding parens</dt><dd>Highlight the surrounding parenthesis.</dd>
+Editing and navigation section below.</p>
+</dd>
+<dt>Show surrounding parens</dt><dd><p>Highlight the surrounding parenthesis.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="format-menu-editor-window-only">
 <span id="format-menu"></span><h3>Format menu (Editor window only)<a class="headerlink" href="#format-menu-editor-window-only" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>Indent Region</dt><dd>Shift selected lines right by the indent width (default 4 spaces).</dd>
-<dt>Dedent Region</dt><dd>Shift selected lines left by the indent width (default 4 spaces).</dd>
-<dt>Comment Out Region</dt><dd>Insert ## in front of selected lines.</dd>
-<dt>Uncomment Region</dt><dd>Remove leading # or ## from selected lines.</dd>
-<dt>Tabify Region</dt><dd>Turn <em>leading</em> stretches of spaces into tabs. (Note: We recommend using
-4 space blocks to indent Python code.)</dd>
-<dt>Untabify Region</dt><dd>Turn <em>all</em> tabs into the correct number of spaces.</dd>
-<dt>Toggle Tabs</dt><dd>Open a dialog to switch between indenting with spaces and tabs.</dd>
-<dt>New Indent Width</dt><dd>Open a dialog to change indent width. The accepted default by the Python
-community is 4 spaces.</dd>
-<dt>Format Paragraph</dt><dd>Reformat the current blank-line-delimited paragraph in comment block or
+<dl class="simple">
+<dt>Indent Region</dt><dd><p>Shift selected lines right by the indent width (default 4 spaces).</p>
+</dd>
+<dt>Dedent Region</dt><dd><p>Shift selected lines left by the indent width (default 4 spaces).</p>
+</dd>
+<dt>Comment Out Region</dt><dd><p>Insert ## in front of selected lines.</p>
+</dd>
+<dt>Uncomment Region</dt><dd><p>Remove leading # or ## from selected lines.</p>
+</dd>
+<dt>Tabify Region</dt><dd><p>Turn <em>leading</em> stretches of spaces into tabs. (Note: We recommend using
+4 space blocks to indent Python code.)</p>
+</dd>
+<dt>Untabify Region</dt><dd><p>Turn <em>all</em> tabs into the correct number of spaces.</p>
+</dd>
+<dt>Toggle Tabs</dt><dd><p>Open a dialog to switch between indenting with spaces and tabs.</p>
+</dd>
+<dt>New Indent Width</dt><dd><p>Open a dialog to change indent width. The accepted default by the Python
+community is 4 spaces.</p>
+</dd>
+<dt>Format Paragraph</dt><dd><p>Reformat the current blank-line-delimited paragraph in comment block or
 multiline string or selected line in a string.  All lines in the
-paragraph will be formatted to less than N columns, where N defaults to 72.</dd>
-<dt>Strip trailing whitespace</dt><dd>Remove trailing space and other whitespace characters after the last
+paragraph will be formatted to less than N columns, where N defaults to 72.</p>
+</dd>
+<dt>Strip trailing whitespace</dt><dd><p>Remove trailing space and other whitespace characters after the last
 non-whitespace character of a line by applying str.rstrip to each line,
-including lines within multiline strings.</dd>
+including lines within multiline strings.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="run-menu-editor-window-only">
 <span id="index-2"></span><h3>Run menu (Editor window only)<a class="headerlink" href="#run-menu-editor-window-only" title="Permalink to this headline">?</a></h3>
-<dl class="docutils" id="run-module">
-<dt>Run Module</dt><dd>Do <a class="reference internal" href="#check-module"><span class="std std-ref">Check Module</span></a>.  If no error, restart the shell to clean the
+<dl class="simple" id="run-module">
+<dt>Run Module</dt><dd><p>Do <a class="reference internal" href="#check-module"><span class="std std-ref">Check Module</span></a>.  If no error, restart the shell to clean the
 environment, then execute the module.  Output is displayed in the Shell
 window.  Note that output requires use of <code class="docutils literal notranslate"><span class="pre">print</span></code> or <code class="docutils literal notranslate"><span class="pre">write</span></code>.
 When execution is complete, the Shell retains focus and displays a prompt.
 At this point, one may interactively explore the result of execution.
 This is similar to executing a file with <code class="docutils literal notranslate"><span class="pre">python</span> <span class="pre">-i</span> <span class="pre">file</span></code> at a command
-line.</dd>
+line.</p>
+</dd>
 </dl>
-<dl class="docutils" id="run-custom">
-<dt>Run? Customized</dt><dd>Same as <a class="reference internal" href="#run-module"><span class="std std-ref">Run Module</span></a>, but run the module with customized
+<dl class="simple" id="run-custom">
+<dt>Run? Customized</dt><dd><p>Same as <a class="reference internal" href="#run-module"><span class="std std-ref">Run Module</span></a>, but run the module with customized
 settings.  <em>Command Line Arguments</em> extend <a class="reference internal" href="sys.html#sys.argv" title="sys.argv"><code class="xref py py-data docutils literal notranslate"><span class="pre">sys.argv</span></code></a> as if passed
-on a command line. The module can be run in the Shell without restarting.</dd>
+on a command line. The module can be run in the Shell without restarting.</p>
+</dd>
 </dl>
-<dl class="docutils" id="check-module">
-<dt>Check Module</dt><dd>Check the syntax of the module currently open in the Editor window. If the
+<dl class="simple" id="check-module">
+<dt>Check Module</dt><dd><p>Check the syntax of the module currently open in the Editor window. If the
 module has not been saved IDLE will either prompt the user to save or
 autosave, as selected in the General tab of the Idle Settings dialog.  If
 there is a syntax error, the approximate location is indicated in the
-Editor window.</dd>
+Editor window.</p>
+</dd>
 </dl>
-<dl class="docutils" id="python-shell">
-<dt>Python Shell</dt><dd>Open or wake up the Python Shell window.</dd>
+<dl class="simple" id="python-shell">
+<dt>Python Shell</dt><dd><p>Open or wake up the Python Shell window.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="shell-menu-shell-window-only">
 <h3>Shell menu (Shell window only)<a class="headerlink" href="#shell-menu-shell-window-only" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>View Last Restart</dt><dd>Scroll the shell window to the last Shell restart.</dd>
-<dt>Restart Shell</dt><dd>Restart the shell to clean the environment.</dd>
-<dt>Previous History</dt><dd>Cycle through earlier commands in history which match the current entry.</dd>
-<dt>Next History</dt><dd>Cycle through later commands in history which match the current entry.</dd>
-<dt>Interrupt Execution</dt><dd>Stop a running program.</dd>
+<dl class="simple">
+<dt>View Last Restart</dt><dd><p>Scroll the shell window to the last Shell restart.</p>
+</dd>
+<dt>Restart Shell</dt><dd><p>Restart the shell to clean the environment.</p>
+</dd>
+<dt>Previous History</dt><dd><p>Cycle through earlier commands in history which match the current entry.</p>
+</dd>
+<dt>Next History</dt><dd><p>Cycle through later commands in history which match the current entry.</p>
+</dd>
+<dt>Interrupt Execution</dt><dd><p>Stop a running program.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="debug-menu-shell-window-only">
 <h3>Debug menu (Shell window only)<a class="headerlink" href="#debug-menu-shell-window-only" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>Go to File/Line</dt><dd>Look on the current line. with the cursor, and the line above for a filename
+<dl class="simple">
+<dt>Go to File/Line</dt><dd><p>Look on the current line. with the cursor, and the line above for a filename
 and line number.  If found, open the file if not already open, and show the
 line.  Use this to view source lines referenced in an exception traceback
 and lines found by Find in Files. Also available in the context menu of
-the Shell window and Output windows.</dd>
+the Shell window and Output windows.</p>
+</dd>
 </dl>
-<dl class="docutils" id="index-3">
-<dt>Debugger (toggle)</dt><dd>When activated, code entered in the Shell or run from an Editor will run
+<dl class="simple" id="index-3">
+<dt>Debugger (toggle)</dt><dd><p>When activated, code entered in the Shell or run from an Editor will run
 under the debugger.  In the Editor, breakpoints can be set with the context
-menu.  This feature is still incomplete and somewhat experimental.</dd>
-<dt>Stack Viewer</dt><dd>Show the stack traceback of the last exception in a tree widget, with
-access to locals and globals.</dd>
-<dt>Auto-open Stack Viewer</dt><dd>Toggle automatically opening the stack viewer on an unhandled exception.</dd>
+menu.  This feature is still incomplete and somewhat experimental.</p>
+</dd>
+<dt>Stack Viewer</dt><dd><p>Show the stack traceback of the last exception in a tree widget, with
+access to locals and globals.</p>
+</dd>
+<dt>Auto-open Stack Viewer</dt><dd><p>Toggle automatically opening the stack viewer on an unhandled exception.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="options-menu-shell-and-editor">
 <h3>Options menu (Shell and Editor)<a class="headerlink" href="#options-menu-shell-and-editor" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>Configure IDLE</dt><dd>Open a configuration dialog and change preferences for the following:
+<dl class="simple">
+<dt>Configure IDLE</dt><dd><p>Open a configuration dialog and change preferences for the following:
 fonts, indentation, keybindings, text color themes, startup windows and
 size, additional help sources, and extensions.  On macOS, open the
 configuration dialog by selecting Preferences in the application
 menu. For more details, see
-<a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a> under Help and preferences.</dd>
+<a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a> under Help and preferences.</p>
+</dd>
 </dl>
 <p>Most configuration options apply to all windows or all future windows.
 The option items below only apply to the active window.</p>
-<dl class="docutils">
-<dt>Show/Hide Code Context (Editor Window only)</dt><dd>Open a pane at the top of the edit window which shows the block context
+<dl class="simple">
+<dt>Show/Hide Code Context (Editor Window only)</dt><dd><p>Open a pane at the top of the edit window which shows the block context
 of the code which has scrolled above the top of the window.  See
 <a class="reference internal" href="#code-context"><span class="std std-ref">Code Context</span></a> in the Editing and Navigation section
-below.</dd>
-<dt>Show/Hide Line Numbers (Editor Window only)</dt><dd>Open a column to the left of the edit window which shows the number
+below.</p>
+</dd>
+<dt>Show/Hide Line Numbers (Editor Window only)</dt><dd><p>Open a column to the left of the edit window which shows the number
 of each line of text.  The default is off, which may be changed in the
-preferences (see <a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a>).</dd>
-<dt>Zoom/Restore Height</dt><dd>Toggles the window between normal size and maximum height. The initial size
+preferences (see <a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a>).</p>
+</dd>
+<dt>Zoom/Restore Height</dt><dd><p>Toggles the window between normal size and maximum height. The initial size
 defaults to 40 lines by 80 chars unless changed on the General tab of the
 Configure IDLE dialog.  The maximum height for a screen is determined by
 momentarily maximizing a window the first time one is zoomed on the screen.
 Changing screen settings may invalidate the saved height.  This toggle has
-no effect when a window is maximized.</dd>
+no effect when a window is maximized.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="window-menu-shell-and-editor">
@@ -298,13 +351,17 @@ <h3>Window menu (Shell and Editor)<a class="headerlink" href="#window-menu-shell
 </div>
 <div class="section" id="help-menu-shell-and-editor">
 <h3>Help menu (Shell and Editor)<a class="headerlink" href="#help-menu-shell-and-editor" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>About IDLE</dt><dd>Display version, copyright, license, credits, and more.</dd>
-<dt>IDLE Help</dt><dd>Display this IDLE document, detailing the menu options, basic editing and
-navigation, and other tips.</dd>
-<dt>Python Docs</dt><dd>Access local Python documentation, if installed, or start a web browser
-and open docs.python.org showing the latest Python documentation.</dd>
-<dt>Turtle Demo</dt><dd>Run the turtledemo module with example Python code and turtle drawings.</dd>
+<dl class="simple">
+<dt>About IDLE</dt><dd><p>Display version, copyright, license, credits, and more.</p>
+</dd>
+<dt>IDLE Help</dt><dd><p>Display this IDLE document, detailing the menu options, basic editing and
+navigation, and other tips.</p>
+</dd>
+<dt>Python Docs</dt><dd><p>Access local Python documentation, if installed, or start a web browser
+and open docs.python.org showing the latest Python documentation.</p>
+</dd>
+<dt>Turtle Demo</dt><dd><p>Run the turtledemo module with example Python code and turtle drawings.</p>
+</dd>
 </dl>
 <p>Additional help sources may be added here with the Configure IDLE dialog under
 the General tab. See the <a class="reference internal" href="#help-sources"><span class="std std-ref">Help sources</span></a> subsection below
@@ -314,27 +371,34 @@ <h3>Help menu (Shell and Editor)<a class="headerlink" href="#help-menu-shell-and
 <span id="index-4"></span><h3>Context Menus<a class="headerlink" href="#context-menus" title="Permalink to this headline">?</a></h3>
 <p>Open a context menu by right-clicking in a window (Control-click on macOS).
 Context menus have the standard clipboard functions also on the Edit menu.</p>
-<dl class="docutils">
-<dt>Cut</dt><dd>Copy selection into the system-wide clipboard; then delete the selection.</dd>
-<dt>Copy</dt><dd>Copy selection into the system-wide clipboard.</dd>
-<dt>Paste</dt><dd>Insert contents of the system-wide clipboard into the current window.</dd>
+<dl class="simple">
+<dt>Cut</dt><dd><p>Copy selection into the system-wide clipboard; then delete the selection.</p>
+</dd>
+<dt>Copy</dt><dd><p>Copy selection into the system-wide clipboard.</p>
+</dd>
+<dt>Paste</dt><dd><p>Insert contents of the system-wide clipboard into the current window.</p>
+</dd>
 </dl>
 <p>Editor windows also have breakpoint functions.  Lines with a breakpoint set are
 specially marked.  Breakpoints only have an effect when running under the
 debugger.  Breakpoints for a file are saved in the user?s .idlerc directory.</p>
-<dl class="docutils">
-<dt>Set Breakpoint</dt><dd>Set a breakpoint on the current line.</dd>
-<dt>Clear Breakpoint</dt><dd>Clear the breakpoint on that line.</dd>
+<dl class="simple">
+<dt>Set Breakpoint</dt><dd><p>Set a breakpoint on the current line.</p>
+</dd>
+<dt>Clear Breakpoint</dt><dd><p>Clear the breakpoint on that line.</p>
+</dd>
 </dl>
 <p>Shell and Output windows also have the following.</p>
-<dl class="docutils">
-<dt>Go to file/line</dt><dd>Same as in Debug menu.</dd>
+<dl class="simple">
+<dt>Go to file/line</dt><dd><p>Same as in Debug menu.</p>
+</dd>
 </dl>
 <p>The Shell window also has an output squeezing facility explained in the <em>Python
 Shell window</em> subsection below.</p>
-<dl class="docutils">
-<dt>Squeeze</dt><dd>If the cursor is over an output line, squeeze all the output between
-the code above and the prompt below down to a ?Squeezed text? label.</dd>
+<dl class="simple">
+<dt>Squeeze</dt><dd><p>If the cursor is over an output line, squeeze all the output between
+the code above and the prompt below down to a ?Squeezed text? label.</p>
+</dd>
 </dl>
 </div>
 </div>
@@ -357,32 +421,26 @@ <h3>Key bindings<a class="headerlink" href="#key-bindings" title="Permalink to t
 <p>In this section, ?C? refers to the <kbd class="kbd docutils literal notranslate">Control</kbd> key on Windows and Unix and
 the <kbd class="kbd docutils literal notranslate">Command</kbd> key on macOS.</p>
 <ul>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">Backspace</kbd> deletes to the left; <kbd class="kbd docutils literal notranslate">Del</kbd> deletes to the right</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-Backspace</kbd> delete word left; <kbd class="kbd docutils literal notranslate">C-Del</kbd> delete word to the right</p>
-</li>
-<li><p class="first">Arrow keys and <kbd class="kbd docutils literal notranslate">Page Up</kbd>/<kbd class="kbd docutils literal notranslate">Page Down</kbd> to move around</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-LeftArrow</kbd> and <kbd class="kbd docutils literal notranslate">C-RightArrow</kbd> moves by words</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">Home</kbd>/<kbd class="kbd docutils literal notranslate">End</kbd> go to begin/end of line</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-Home</kbd>/<kbd class="kbd docutils literal notranslate">C-End</kbd> go to begin/end of file</p>
-</li>
-<li><p class="first">Some useful Emacs bindings are inherited from Tcl/Tk:</p>
+<li><p><kbd class="kbd docutils literal notranslate">Backspace</kbd> deletes to the left; <kbd class="kbd docutils literal notranslate">Del</kbd> deletes to the right</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-Backspace</kbd> delete word left; <kbd class="kbd docutils literal notranslate">C-Del</kbd> delete word to the right</p></li>
+<li><p>Arrow keys and <kbd class="kbd docutils literal notranslate">Page Up</kbd>/<kbd class="kbd docutils literal notranslate">Page Down</kbd> to move around</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-LeftArrow</kbd> and <kbd class="kbd docutils literal notranslate">C-RightArrow</kbd> moves by words</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">Home</kbd>/<kbd class="kbd docutils literal notranslate">End</kbd> go to begin/end of line</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-Home</kbd>/<kbd class="kbd docutils literal notranslate">C-End</kbd> go to begin/end of file</p></li>
+<li><p>Some useful Emacs bindings are inherited from Tcl/Tk:</p>
 <blockquote>
 <div><ul class="simple">
-<li><kbd class="kbd docutils literal notranslate">C-a</kbd> beginning of line</li>
-<li><kbd class="kbd docutils literal notranslate">C-e</kbd> end of line</li>
-<li><kbd class="kbd docutils literal notranslate">C-k</kbd> kill line (but doesn?t put it in clipboard)</li>
-<li><kbd class="kbd docutils literal notranslate">C-l</kbd> center window around the insertion point</li>
-<li><kbd class="kbd docutils literal notranslate">C-b</kbd> go backward one character without deleting (usually you can
-also use the cursor key for this)</li>
-<li><kbd class="kbd docutils literal notranslate">C-f</kbd> go forward one character without deleting (usually you can
-also use the cursor key for this)</li>
-<li><kbd class="kbd docutils literal notranslate">C-p</kbd> go up one line (usually you can also use the cursor key for
-this)</li>
-<li><kbd class="kbd docutils literal notranslate">C-d</kbd> delete next character</li>
+<li><p><kbd class="kbd docutils literal notranslate">C-a</kbd> beginning of line</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-e</kbd> end of line</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-k</kbd> kill line (but doesn?t put it in clipboard)</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-l</kbd> center window around the insertion point</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-b</kbd> go backward one character without deleting (usually you can
+also use the cursor key for this)</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-f</kbd> go forward one character without deleting (usually you can
+also use the cursor key for this)</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-p</kbd> go up one line (usually you can also use the cursor key for
+this)</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-d</kbd> delete next character</p></li>
 </ul>
 </div></blockquote>
 </li>
@@ -485,17 +543,15 @@ <h3>Python Shell window<a class="headerlink" href="#python-shell-window" title="
 <p>The editing features described in previous subsections work when entering
 code interactively.  IDLE?s Shell window also responds to the following keys.</p>
 <ul>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-c</kbd> interrupts executing command</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-d</kbd> sends end-of-file; closes window if typed at a <code class="docutils literal notranslate"><span class="pre">&gt;&gt;&gt;</span></code> prompt</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">Alt-/</kbd> (Expand word) is also useful to reduce typing</p>
+<li><p><kbd class="kbd docutils literal notranslate">C-c</kbd> interrupts executing command</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-d</kbd> sends end-of-file; closes window if typed at a <code class="docutils literal notranslate"><span class="pre">&gt;&gt;&gt;</span></code> prompt</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">Alt-/</kbd> (Expand word) is also useful to reduce typing</p>
 <p>Command history</p>
 <ul class="simple">
-<li><kbd class="kbd docutils literal notranslate">Alt-p</kbd> retrieves previous command matching what you have typed. On
-macOS use <kbd class="kbd docutils literal notranslate">C-p</kbd>.</li>
-<li><kbd class="kbd docutils literal notranslate">Alt-n</kbd> retrieves next. On macOS use <kbd class="kbd docutils literal notranslate">C-n</kbd>.</li>
-<li><kbd class="kbd docutils literal notranslate">Return</kbd> while on any previous command retrieves that command</li>
+<li><p><kbd class="kbd docutils literal notranslate">Alt-p</kbd> retrieves previous command matching what you have typed. On
+macOS use <kbd class="kbd docutils literal notranslate">C-p</kbd>.</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">Alt-n</kbd> retrieves next. On macOS use <kbd class="kbd docutils literal notranslate">C-n</kbd>.</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">Return</kbd> while on any previous command retrieves that command</p></li>
 </ul>
 </li>
 </ul>
@@ -545,12 +601,12 @@ <h3>Command line usage<a class="headerlink" href="#command-line-usage" title="Pe
 </div>
 <p>If there are arguments:</p>
 <ul class="simple">
-<li>If <code class="docutils literal notranslate"><span class="pre">-</span></code>, <code class="docutils literal notranslate"><span class="pre">-c</span></code>, or <code class="docutils literal notranslate"><span class="pre">r</span></code> is used, all arguments are placed in
+<li><p>If <code class="docutils literal notranslate"><span class="pre">-</span></code>, <code class="docutils literal notranslate"><span class="pre">-c</span></code>, or <code class="docutils literal notranslate"><span class="pre">r</span></code> is used, all arguments are placed in
 <code class="docutils literal notranslate"><span class="pre">sys.argv[1:...]</span></code> and <code class="docutils literal notranslate"><span class="pre">sys.argv[0]</span></code> is set to <code class="docutils literal notranslate"><span class="pre">''</span></code>, <code class="docutils literal notranslate"><span class="pre">'-c'</span></code>,
 or <code class="docutils literal notranslate"><span class="pre">'-r'</span></code>.  No editor window is opened, even if that is the default
-set in the Options dialog.</li>
-<li>Otherwise, arguments are files opened for editing and
-<code class="docutils literal notranslate"><span class="pre">sys.argv</span></code> reflects the arguments passed to IDLE itself.</li>
+set in the Options dialog.</p></li>
+<li><p>Otherwise, arguments are files opened for editing and
+<code class="docutils literal notranslate"><span class="pre">sys.argv</span></code> reflects the arguments passed to IDLE itself.</p></li>
 </ul>
 </div>
 <div class="section" id="startup-failure">
@@ -903,11 +959,11 @@ <h3>Navigation</h3>
 <br />
     <br />
 
-    Last updated on Aug 04, 2019.
+    Last updated on Sep 01, 2019.
     <a href="https://docs.python.org/3/bugs.html">Found a bug</a>?
     <br />
 
-    Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 2.1.1.
+    Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 2.1.2.
     </div>
 
   </body>
diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py
index e19d3615016f..ba2c6ec55481 100644
--- a/Lib/idlelib/help.py
+++ b/Lib/idlelib/help.py
@@ -62,6 +62,7 @@ def __init__(self, text):
         self.simplelist = False  # simple list (no double spacing)
         self.toc = []            # pair headers with text indexes for toc
         self.header = ''         # text within header tags for toc
+        self.prevtag = None      # info about previous tag (was opener, tag)
 
     def indent(self, amt=1):
         self.level += amt
@@ -78,8 +79,11 @@ def handle_starttag(self, tag, attrs):
             self.show = True    # start of main content
         elif tag == 'div' and class_ == 'sphinxsidebar':
             self.show = False   # end of main content
-        elif tag == 'p' and class_ != 'first':
-            s = '\n\n'
+        elif tag == 'p' and self.prevtag and not self.prevtag[0]:
+            # begin a new block for <p> tags after a closed tag
+            # avoid extra lines, e.g. after <pre> tags
+            lastline = self.text.get('end-1c linestart', 'end-1c')
+            s = '\n\n' if lastline and not lastline.isspace() else '\n'
         elif tag == 'span' and class_ == 'pre':
             self.chartags = 'pre'
         elif tag == 'span' and class_ == 'versionmodified':
@@ -120,6 +124,7 @@ def handle_starttag(self, tag, attrs):
             self.tags = tag
         if self.show:
             self.text.insert('end', s, (self.tags, self.chartags))
+        self.prevtag = (True, tag)
 
     def handle_endtag(self, tag):
         "Handle endtags in help.html."
@@ -139,6 +144,7 @@ def handle_endtag(self, tag):
             self.tags = ''
         elif tag in ['ul', 'dd', 'ol']:
             self.indent(amt=-1)
+        self.prevtag = (False, tag)
 
     def handle_data(self, data):
         "Handle date segments in help.html."


From webhook-mailer at python.org  Tue Sep  3 17:10:01 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 03 Sep 2019 21:10:01 -0000
Subject: [Python-checkins] bpo-38022: IDLE: upgrade help.html to sphinx 2.x
 HTML5 output (GH-15664)
Message-ID: <mailman.225.1567545003.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/52fd6616235412d4a49ef28c1046bd133b30bc90
commit: 52fd6616235412d4a49ef28c1046bd133b30bc90
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-03T14:09:57-07:00
summary:

bpo-38022: IDLE: upgrade help.html to sphinx 2.x HTML5 output (GH-15664)


The HTML5 output from Sphinx 2.x adds  '<p>' tags within list elements.  Using a new prevtag attribute, ignore these instead of emitting unwanted '\n\n'.

Also stop looking for 'first' classes on tags (no longer present) and fix the bug of double-spacing instead of single spacing after <pre> blocks.
(cherry picked from commit 580bdb0ece681537eadb360f0c796123ead7a559)

Co-authored-by: Tal Einat <taleinat+github at gmail.com>

files:
M Lib/idlelib/help.html
M Lib/idlelib/help.py

diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html
index 91e66a4b9117..0754f2453baf 100644
--- a/Lib/idlelib/help.html
+++ b/Lib/idlelib/help.html
@@ -1,11 +1,9 @@
 
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!DOCTYPE html>
 
 <html xmlns="http://www.w3.org/1999/xhtml">
   <head>
-    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
-    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta charset="utf-8" />
     <title>IDLE &#8212; Python 3.9.0a0 documentation</title>
     <link rel="stylesheet" href="../_static/pydoctheme.css" type="text/css" />
     <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
@@ -108,17 +106,17 @@ <h3>Navigation</h3>
 <p>IDLE is Python?s Integrated Development and Learning Environment.</p>
 <p>IDLE has the following features:</p>
 <ul class="simple">
-<li>coded in 100% pure Python, using the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><code class="xref py py-mod docutils literal notranslate"><span class="pre">tkinter</span></code></a> GUI toolkit</li>
-<li>cross-platform: works mostly the same on Windows, Unix, and macOS</li>
-<li>Python shell window (interactive interpreter) with colorizing
-of code input, output, and error messages</li>
-<li>multi-window text editor with multiple undo, Python colorizing,
-smart indent, call tips, auto completion, and other features</li>
-<li>search within any window, replace within editor windows, and search
-through multiple files (grep)</li>
-<li>debugger with persistent breakpoints, stepping, and viewing
-of global and local namespaces</li>
-<li>configuration, browsers, and other dialogs</li>
+<li><p>coded in 100% pure Python, using the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><code class="xref py py-mod docutils literal notranslate"><span class="pre">tkinter</span></code></a> GUI toolkit</p></li>
+<li><p>cross-platform: works mostly the same on Windows, Unix, and macOS</p></li>
+<li><p>Python shell window (interactive interpreter) with colorizing
+of code input, output, and error messages</p></li>
+<li><p>multi-window text editor with multiple undo, Python colorizing,
+smart indent, call tips, auto completion, and other features</p></li>
+<li><p>search within any window, replace within editor windows, and search
+through multiple files (grep)</p></li>
+<li><p>debugger with persistent breakpoints, stepping, and viewing
+of global and local namespaces</p></li>
+<li><p>configuration, browsers, and other dialogs</p></li>
 </ul>
 <div class="section" id="menus">
 <h2>Menus<a class="headerlink" href="#menus" title="Permalink to this headline">?</a></h2>
@@ -134,161 +132,216 @@ <h2>Menus<a class="headerlink" href="#menus" title="Permalink to this headline">
 described below are moved around to conform to Apple guidelines.</p>
 <div class="section" id="file-menu-shell-and-editor">
 <h3>File menu (Shell and Editor)<a class="headerlink" href="#file-menu-shell-and-editor" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>New File</dt><dd>Create a new file editing window.</dd>
-<dt>Open?</dt><dd>Open an existing file with an Open dialog.</dd>
-<dt>Recent Files</dt><dd>Open a list of recent files.  Click one to open it.</dd>
-<dt>Open Module?</dt><dd>Open an existing module (searches sys.path).</dd>
+<dl class="simple">
+<dt>New File</dt><dd><p>Create a new file editing window.</p>
+</dd>
+<dt>Open?</dt><dd><p>Open an existing file with an Open dialog.</p>
+</dd>
+<dt>Recent Files</dt><dd><p>Open a list of recent files.  Click one to open it.</p>
+</dd>
+<dt>Open Module?</dt><dd><p>Open an existing module (searches sys.path).</p>
+</dd>
 </dl>
-<dl class="docutils" id="index-1">
-<dt>Class Browser</dt><dd>Show functions, classes, and methods in the current Editor file in a
-tree structure.  In the shell, open a module first.</dd>
-<dt>Path Browser</dt><dd>Show sys.path directories, modules, functions, classes and methods in a
-tree structure.</dd>
-<dt>Save</dt><dd>Save the current window to the associated file, if there is one.  Windows
+<dl class="simple" id="index-1">
+<dt>Class Browser</dt><dd><p>Show functions, classes, and methods in the current Editor file in a
+tree structure.  In the shell, open a module first.</p>
+</dd>
+<dt>Path Browser</dt><dd><p>Show sys.path directories, modules, functions, classes and methods in a
+tree structure.</p>
+</dd>
+<dt>Save</dt><dd><p>Save the current window to the associated file, if there is one.  Windows
 that have been changed since being opened or last saved have a * before
 and after the window title.  If there is no associated file,
-do Save As instead.</dd>
-<dt>Save As?</dt><dd>Save the current window with a Save As dialog.  The file saved becomes the
-new associated file for the window.</dd>
-<dt>Save Copy As?</dt><dd>Save the current window to different file without changing the associated
-file.</dd>
-<dt>Print Window</dt><dd>Print the current window to the default printer.</dd>
-<dt>Close</dt><dd>Close the current window (ask to save if unsaved).</dd>
-<dt>Exit</dt><dd>Close all windows and quit IDLE (ask to save unsaved windows).</dd>
+do Save As instead.</p>
+</dd>
+<dt>Save As?</dt><dd><p>Save the current window with a Save As dialog.  The file saved becomes the
+new associated file for the window.</p>
+</dd>
+<dt>Save Copy As?</dt><dd><p>Save the current window to different file without changing the associated
+file.</p>
+</dd>
+<dt>Print Window</dt><dd><p>Print the current window to the default printer.</p>
+</dd>
+<dt>Close</dt><dd><p>Close the current window (ask to save if unsaved).</p>
+</dd>
+<dt>Exit</dt><dd><p>Close all windows and quit IDLE (ask to save unsaved windows).</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="edit-menu-shell-and-editor">
 <h3>Edit menu (Shell and Editor)<a class="headerlink" href="#edit-menu-shell-and-editor" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>Undo</dt><dd>Undo the last change to the current window.  A maximum of 1000 changes may
-be undone.</dd>
-<dt>Redo</dt><dd>Redo the last undone change to the current window.</dd>
-<dt>Cut</dt><dd>Copy selection into the system-wide clipboard; then delete the selection.</dd>
-<dt>Copy</dt><dd>Copy selection into the system-wide clipboard.</dd>
-<dt>Paste</dt><dd>Insert contents of the system-wide clipboard into the current window.</dd>
+<dl class="simple">
+<dt>Undo</dt><dd><p>Undo the last change to the current window.  A maximum of 1000 changes may
+be undone.</p>
+</dd>
+<dt>Redo</dt><dd><p>Redo the last undone change to the current window.</p>
+</dd>
+<dt>Cut</dt><dd><p>Copy selection into the system-wide clipboard; then delete the selection.</p>
+</dd>
+<dt>Copy</dt><dd><p>Copy selection into the system-wide clipboard.</p>
+</dd>
+<dt>Paste</dt><dd><p>Insert contents of the system-wide clipboard into the current window.</p>
+</dd>
 </dl>
 <p>The clipboard functions are also available in context menus.</p>
-<dl class="docutils">
-<dt>Select All</dt><dd>Select the entire contents of the current window.</dd>
-<dt>Find?</dt><dd>Open a search dialog with many options</dd>
-<dt>Find Again</dt><dd>Repeat the last search, if there is one.</dd>
-<dt>Find Selection</dt><dd>Search for the currently selected string, if there is one.</dd>
-<dt>Find in Files?</dt><dd>Open a file search dialog.  Put results in a new output window.</dd>
-<dt>Replace?</dt><dd>Open a search-and-replace dialog.</dd>
-<dt>Go to Line</dt><dd>Move cursor to the line number requested and make that line visible.</dd>
-<dt>Show Completions</dt><dd>Open a scrollable list allowing selection of keywords and attributes. See
-<a class="reference internal" href="#completions"><span class="std std-ref">Completions</span></a> in the Editing and navigation section below.</dd>
-<dt>Expand Word</dt><dd>Expand a prefix you have typed to match a full word in the same window;
-repeat to get a different expansion.</dd>
-<dt>Show call tip</dt><dd>After an unclosed parenthesis for a function, open a small window with
+<dl class="simple">
+<dt>Select All</dt><dd><p>Select the entire contents of the current window.</p>
+</dd>
+<dt>Find?</dt><dd><p>Open a search dialog with many options</p>
+</dd>
+<dt>Find Again</dt><dd><p>Repeat the last search, if there is one.</p>
+</dd>
+<dt>Find Selection</dt><dd><p>Search for the currently selected string, if there is one.</p>
+</dd>
+<dt>Find in Files?</dt><dd><p>Open a file search dialog.  Put results in a new output window.</p>
+</dd>
+<dt>Replace?</dt><dd><p>Open a search-and-replace dialog.</p>
+</dd>
+<dt>Go to Line</dt><dd><p>Move cursor to the line number requested and make that line visible.</p>
+</dd>
+<dt>Show Completions</dt><dd><p>Open a scrollable list allowing selection of keywords and attributes. See
+<a class="reference internal" href="#completions"><span class="std std-ref">Completions</span></a> in the Editing and navigation section below.</p>
+</dd>
+<dt>Expand Word</dt><dd><p>Expand a prefix you have typed to match a full word in the same window;
+repeat to get a different expansion.</p>
+</dd>
+<dt>Show call tip</dt><dd><p>After an unclosed parenthesis for a function, open a small window with
 function parameter hints.  See <a class="reference internal" href="#calltips"><span class="std std-ref">Calltips</span></a> in the
-Editing and navigation section below.</dd>
-<dt>Show surrounding parens</dt><dd>Highlight the surrounding parenthesis.</dd>
+Editing and navigation section below.</p>
+</dd>
+<dt>Show surrounding parens</dt><dd><p>Highlight the surrounding parenthesis.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="format-menu-editor-window-only">
 <span id="format-menu"></span><h3>Format menu (Editor window only)<a class="headerlink" href="#format-menu-editor-window-only" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>Indent Region</dt><dd>Shift selected lines right by the indent width (default 4 spaces).</dd>
-<dt>Dedent Region</dt><dd>Shift selected lines left by the indent width (default 4 spaces).</dd>
-<dt>Comment Out Region</dt><dd>Insert ## in front of selected lines.</dd>
-<dt>Uncomment Region</dt><dd>Remove leading # or ## from selected lines.</dd>
-<dt>Tabify Region</dt><dd>Turn <em>leading</em> stretches of spaces into tabs. (Note: We recommend using
-4 space blocks to indent Python code.)</dd>
-<dt>Untabify Region</dt><dd>Turn <em>all</em> tabs into the correct number of spaces.</dd>
-<dt>Toggle Tabs</dt><dd>Open a dialog to switch between indenting with spaces and tabs.</dd>
-<dt>New Indent Width</dt><dd>Open a dialog to change indent width. The accepted default by the Python
-community is 4 spaces.</dd>
-<dt>Format Paragraph</dt><dd>Reformat the current blank-line-delimited paragraph in comment block or
+<dl class="simple">
+<dt>Indent Region</dt><dd><p>Shift selected lines right by the indent width (default 4 spaces).</p>
+</dd>
+<dt>Dedent Region</dt><dd><p>Shift selected lines left by the indent width (default 4 spaces).</p>
+</dd>
+<dt>Comment Out Region</dt><dd><p>Insert ## in front of selected lines.</p>
+</dd>
+<dt>Uncomment Region</dt><dd><p>Remove leading # or ## from selected lines.</p>
+</dd>
+<dt>Tabify Region</dt><dd><p>Turn <em>leading</em> stretches of spaces into tabs. (Note: We recommend using
+4 space blocks to indent Python code.)</p>
+</dd>
+<dt>Untabify Region</dt><dd><p>Turn <em>all</em> tabs into the correct number of spaces.</p>
+</dd>
+<dt>Toggle Tabs</dt><dd><p>Open a dialog to switch between indenting with spaces and tabs.</p>
+</dd>
+<dt>New Indent Width</dt><dd><p>Open a dialog to change indent width. The accepted default by the Python
+community is 4 spaces.</p>
+</dd>
+<dt>Format Paragraph</dt><dd><p>Reformat the current blank-line-delimited paragraph in comment block or
 multiline string or selected line in a string.  All lines in the
-paragraph will be formatted to less than N columns, where N defaults to 72.</dd>
-<dt>Strip trailing whitespace</dt><dd>Remove trailing space and other whitespace characters after the last
+paragraph will be formatted to less than N columns, where N defaults to 72.</p>
+</dd>
+<dt>Strip trailing whitespace</dt><dd><p>Remove trailing space and other whitespace characters after the last
 non-whitespace character of a line by applying str.rstrip to each line,
-including lines within multiline strings.</dd>
+including lines within multiline strings.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="run-menu-editor-window-only">
 <span id="index-2"></span><h3>Run menu (Editor window only)<a class="headerlink" href="#run-menu-editor-window-only" title="Permalink to this headline">?</a></h3>
-<dl class="docutils" id="run-module">
-<dt>Run Module</dt><dd>Do <a class="reference internal" href="#check-module"><span class="std std-ref">Check Module</span></a>.  If no error, restart the shell to clean the
+<dl class="simple" id="run-module">
+<dt>Run Module</dt><dd><p>Do <a class="reference internal" href="#check-module"><span class="std std-ref">Check Module</span></a>.  If no error, restart the shell to clean the
 environment, then execute the module.  Output is displayed in the Shell
 window.  Note that output requires use of <code class="docutils literal notranslate"><span class="pre">print</span></code> or <code class="docutils literal notranslate"><span class="pre">write</span></code>.
 When execution is complete, the Shell retains focus and displays a prompt.
 At this point, one may interactively explore the result of execution.
 This is similar to executing a file with <code class="docutils literal notranslate"><span class="pre">python</span> <span class="pre">-i</span> <span class="pre">file</span></code> at a command
-line.</dd>
+line.</p>
+</dd>
 </dl>
-<dl class="docutils" id="run-custom">
-<dt>Run? Customized</dt><dd>Same as <a class="reference internal" href="#run-module"><span class="std std-ref">Run Module</span></a>, but run the module with customized
+<dl class="simple" id="run-custom">
+<dt>Run? Customized</dt><dd><p>Same as <a class="reference internal" href="#run-module"><span class="std std-ref">Run Module</span></a>, but run the module with customized
 settings.  <em>Command Line Arguments</em> extend <a class="reference internal" href="sys.html#sys.argv" title="sys.argv"><code class="xref py py-data docutils literal notranslate"><span class="pre">sys.argv</span></code></a> as if passed
-on a command line. The module can be run in the Shell without restarting.</dd>
+on a command line. The module can be run in the Shell without restarting.</p>
+</dd>
 </dl>
-<dl class="docutils" id="check-module">
-<dt>Check Module</dt><dd>Check the syntax of the module currently open in the Editor window. If the
+<dl class="simple" id="check-module">
+<dt>Check Module</dt><dd><p>Check the syntax of the module currently open in the Editor window. If the
 module has not been saved IDLE will either prompt the user to save or
 autosave, as selected in the General tab of the Idle Settings dialog.  If
 there is a syntax error, the approximate location is indicated in the
-Editor window.</dd>
+Editor window.</p>
+</dd>
 </dl>
-<dl class="docutils" id="python-shell">
-<dt>Python Shell</dt><dd>Open or wake up the Python Shell window.</dd>
+<dl class="simple" id="python-shell">
+<dt>Python Shell</dt><dd><p>Open or wake up the Python Shell window.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="shell-menu-shell-window-only">
 <h3>Shell menu (Shell window only)<a class="headerlink" href="#shell-menu-shell-window-only" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>View Last Restart</dt><dd>Scroll the shell window to the last Shell restart.</dd>
-<dt>Restart Shell</dt><dd>Restart the shell to clean the environment.</dd>
-<dt>Previous History</dt><dd>Cycle through earlier commands in history which match the current entry.</dd>
-<dt>Next History</dt><dd>Cycle through later commands in history which match the current entry.</dd>
-<dt>Interrupt Execution</dt><dd>Stop a running program.</dd>
+<dl class="simple">
+<dt>View Last Restart</dt><dd><p>Scroll the shell window to the last Shell restart.</p>
+</dd>
+<dt>Restart Shell</dt><dd><p>Restart the shell to clean the environment.</p>
+</dd>
+<dt>Previous History</dt><dd><p>Cycle through earlier commands in history which match the current entry.</p>
+</dd>
+<dt>Next History</dt><dd><p>Cycle through later commands in history which match the current entry.</p>
+</dd>
+<dt>Interrupt Execution</dt><dd><p>Stop a running program.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="debug-menu-shell-window-only">
 <h3>Debug menu (Shell window only)<a class="headerlink" href="#debug-menu-shell-window-only" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>Go to File/Line</dt><dd>Look on the current line. with the cursor, and the line above for a filename
+<dl class="simple">
+<dt>Go to File/Line</dt><dd><p>Look on the current line. with the cursor, and the line above for a filename
 and line number.  If found, open the file if not already open, and show the
 line.  Use this to view source lines referenced in an exception traceback
 and lines found by Find in Files. Also available in the context menu of
-the Shell window and Output windows.</dd>
+the Shell window and Output windows.</p>
+</dd>
 </dl>
-<dl class="docutils" id="index-3">
-<dt>Debugger (toggle)</dt><dd>When activated, code entered in the Shell or run from an Editor will run
+<dl class="simple" id="index-3">
+<dt>Debugger (toggle)</dt><dd><p>When activated, code entered in the Shell or run from an Editor will run
 under the debugger.  In the Editor, breakpoints can be set with the context
-menu.  This feature is still incomplete and somewhat experimental.</dd>
-<dt>Stack Viewer</dt><dd>Show the stack traceback of the last exception in a tree widget, with
-access to locals and globals.</dd>
-<dt>Auto-open Stack Viewer</dt><dd>Toggle automatically opening the stack viewer on an unhandled exception.</dd>
+menu.  This feature is still incomplete and somewhat experimental.</p>
+</dd>
+<dt>Stack Viewer</dt><dd><p>Show the stack traceback of the last exception in a tree widget, with
+access to locals and globals.</p>
+</dd>
+<dt>Auto-open Stack Viewer</dt><dd><p>Toggle automatically opening the stack viewer on an unhandled exception.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="options-menu-shell-and-editor">
 <h3>Options menu (Shell and Editor)<a class="headerlink" href="#options-menu-shell-and-editor" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>Configure IDLE</dt><dd>Open a configuration dialog and change preferences for the following:
+<dl class="simple">
+<dt>Configure IDLE</dt><dd><p>Open a configuration dialog and change preferences for the following:
 fonts, indentation, keybindings, text color themes, startup windows and
 size, additional help sources, and extensions.  On macOS, open the
 configuration dialog by selecting Preferences in the application
 menu. For more details, see
-<a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a> under Help and preferences.</dd>
+<a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a> under Help and preferences.</p>
+</dd>
 </dl>
 <p>Most configuration options apply to all windows or all future windows.
 The option items below only apply to the active window.</p>
-<dl class="docutils">
-<dt>Show/Hide Code Context (Editor Window only)</dt><dd>Open a pane at the top of the edit window which shows the block context
+<dl class="simple">
+<dt>Show/Hide Code Context (Editor Window only)</dt><dd><p>Open a pane at the top of the edit window which shows the block context
 of the code which has scrolled above the top of the window.  See
 <a class="reference internal" href="#code-context"><span class="std std-ref">Code Context</span></a> in the Editing and Navigation section
-below.</dd>
-<dt>Show/Hide Line Numbers (Editor Window only)</dt><dd>Open a column to the left of the edit window which shows the number
+below.</p>
+</dd>
+<dt>Show/Hide Line Numbers (Editor Window only)</dt><dd><p>Open a column to the left of the edit window which shows the number
 of each line of text.  The default is off, which may be changed in the
-preferences (see <a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a>).</dd>
-<dt>Zoom/Restore Height</dt><dd>Toggles the window between normal size and maximum height. The initial size
+preferences (see <a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a>).</p>
+</dd>
+<dt>Zoom/Restore Height</dt><dd><p>Toggles the window between normal size and maximum height. The initial size
 defaults to 40 lines by 80 chars unless changed on the General tab of the
 Configure IDLE dialog.  The maximum height for a screen is determined by
 momentarily maximizing a window the first time one is zoomed on the screen.
 Changing screen settings may invalidate the saved height.  This toggle has
-no effect when a window is maximized.</dd>
+no effect when a window is maximized.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="window-menu-shell-and-editor">
@@ -298,13 +351,17 @@ <h3>Window menu (Shell and Editor)<a class="headerlink" href="#window-menu-shell
 </div>
 <div class="section" id="help-menu-shell-and-editor">
 <h3>Help menu (Shell and Editor)<a class="headerlink" href="#help-menu-shell-and-editor" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>About IDLE</dt><dd>Display version, copyright, license, credits, and more.</dd>
-<dt>IDLE Help</dt><dd>Display this IDLE document, detailing the menu options, basic editing and
-navigation, and other tips.</dd>
-<dt>Python Docs</dt><dd>Access local Python documentation, if installed, or start a web browser
-and open docs.python.org showing the latest Python documentation.</dd>
-<dt>Turtle Demo</dt><dd>Run the turtledemo module with example Python code and turtle drawings.</dd>
+<dl class="simple">
+<dt>About IDLE</dt><dd><p>Display version, copyright, license, credits, and more.</p>
+</dd>
+<dt>IDLE Help</dt><dd><p>Display this IDLE document, detailing the menu options, basic editing and
+navigation, and other tips.</p>
+</dd>
+<dt>Python Docs</dt><dd><p>Access local Python documentation, if installed, or start a web browser
+and open docs.python.org showing the latest Python documentation.</p>
+</dd>
+<dt>Turtle Demo</dt><dd><p>Run the turtledemo module with example Python code and turtle drawings.</p>
+</dd>
 </dl>
 <p>Additional help sources may be added here with the Configure IDLE dialog under
 the General tab. See the <a class="reference internal" href="#help-sources"><span class="std std-ref">Help sources</span></a> subsection below
@@ -314,27 +371,34 @@ <h3>Help menu (Shell and Editor)<a class="headerlink" href="#help-menu-shell-and
 <span id="index-4"></span><h3>Context Menus<a class="headerlink" href="#context-menus" title="Permalink to this headline">?</a></h3>
 <p>Open a context menu by right-clicking in a window (Control-click on macOS).
 Context menus have the standard clipboard functions also on the Edit menu.</p>
-<dl class="docutils">
-<dt>Cut</dt><dd>Copy selection into the system-wide clipboard; then delete the selection.</dd>
-<dt>Copy</dt><dd>Copy selection into the system-wide clipboard.</dd>
-<dt>Paste</dt><dd>Insert contents of the system-wide clipboard into the current window.</dd>
+<dl class="simple">
+<dt>Cut</dt><dd><p>Copy selection into the system-wide clipboard; then delete the selection.</p>
+</dd>
+<dt>Copy</dt><dd><p>Copy selection into the system-wide clipboard.</p>
+</dd>
+<dt>Paste</dt><dd><p>Insert contents of the system-wide clipboard into the current window.</p>
+</dd>
 </dl>
 <p>Editor windows also have breakpoint functions.  Lines with a breakpoint set are
 specially marked.  Breakpoints only have an effect when running under the
 debugger.  Breakpoints for a file are saved in the user?s .idlerc directory.</p>
-<dl class="docutils">
-<dt>Set Breakpoint</dt><dd>Set a breakpoint on the current line.</dd>
-<dt>Clear Breakpoint</dt><dd>Clear the breakpoint on that line.</dd>
+<dl class="simple">
+<dt>Set Breakpoint</dt><dd><p>Set a breakpoint on the current line.</p>
+</dd>
+<dt>Clear Breakpoint</dt><dd><p>Clear the breakpoint on that line.</p>
+</dd>
 </dl>
 <p>Shell and Output windows also have the following.</p>
-<dl class="docutils">
-<dt>Go to file/line</dt><dd>Same as in Debug menu.</dd>
+<dl class="simple">
+<dt>Go to file/line</dt><dd><p>Same as in Debug menu.</p>
+</dd>
 </dl>
 <p>The Shell window also has an output squeezing facility explained in the <em>Python
 Shell window</em> subsection below.</p>
-<dl class="docutils">
-<dt>Squeeze</dt><dd>If the cursor is over an output line, squeeze all the output between
-the code above and the prompt below down to a ?Squeezed text? label.</dd>
+<dl class="simple">
+<dt>Squeeze</dt><dd><p>If the cursor is over an output line, squeeze all the output between
+the code above and the prompt below down to a ?Squeezed text? label.</p>
+</dd>
 </dl>
 </div>
 </div>
@@ -357,32 +421,26 @@ <h3>Key bindings<a class="headerlink" href="#key-bindings" title="Permalink to t
 <p>In this section, ?C? refers to the <kbd class="kbd docutils literal notranslate">Control</kbd> key on Windows and Unix and
 the <kbd class="kbd docutils literal notranslate">Command</kbd> key on macOS.</p>
 <ul>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">Backspace</kbd> deletes to the left; <kbd class="kbd docutils literal notranslate">Del</kbd> deletes to the right</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-Backspace</kbd> delete word left; <kbd class="kbd docutils literal notranslate">C-Del</kbd> delete word to the right</p>
-</li>
-<li><p class="first">Arrow keys and <kbd class="kbd docutils literal notranslate">Page Up</kbd>/<kbd class="kbd docutils literal notranslate">Page Down</kbd> to move around</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-LeftArrow</kbd> and <kbd class="kbd docutils literal notranslate">C-RightArrow</kbd> moves by words</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">Home</kbd>/<kbd class="kbd docutils literal notranslate">End</kbd> go to begin/end of line</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-Home</kbd>/<kbd class="kbd docutils literal notranslate">C-End</kbd> go to begin/end of file</p>
-</li>
-<li><p class="first">Some useful Emacs bindings are inherited from Tcl/Tk:</p>
+<li><p><kbd class="kbd docutils literal notranslate">Backspace</kbd> deletes to the left; <kbd class="kbd docutils literal notranslate">Del</kbd> deletes to the right</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-Backspace</kbd> delete word left; <kbd class="kbd docutils literal notranslate">C-Del</kbd> delete word to the right</p></li>
+<li><p>Arrow keys and <kbd class="kbd docutils literal notranslate">Page Up</kbd>/<kbd class="kbd docutils literal notranslate">Page Down</kbd> to move around</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-LeftArrow</kbd> and <kbd class="kbd docutils literal notranslate">C-RightArrow</kbd> moves by words</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">Home</kbd>/<kbd class="kbd docutils literal notranslate">End</kbd> go to begin/end of line</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-Home</kbd>/<kbd class="kbd docutils literal notranslate">C-End</kbd> go to begin/end of file</p></li>
+<li><p>Some useful Emacs bindings are inherited from Tcl/Tk:</p>
 <blockquote>
 <div><ul class="simple">
-<li><kbd class="kbd docutils literal notranslate">C-a</kbd> beginning of line</li>
-<li><kbd class="kbd docutils literal notranslate">C-e</kbd> end of line</li>
-<li><kbd class="kbd docutils literal notranslate">C-k</kbd> kill line (but doesn?t put it in clipboard)</li>
-<li><kbd class="kbd docutils literal notranslate">C-l</kbd> center window around the insertion point</li>
-<li><kbd class="kbd docutils literal notranslate">C-b</kbd> go backward one character without deleting (usually you can
-also use the cursor key for this)</li>
-<li><kbd class="kbd docutils literal notranslate">C-f</kbd> go forward one character without deleting (usually you can
-also use the cursor key for this)</li>
-<li><kbd class="kbd docutils literal notranslate">C-p</kbd> go up one line (usually you can also use the cursor key for
-this)</li>
-<li><kbd class="kbd docutils literal notranslate">C-d</kbd> delete next character</li>
+<li><p><kbd class="kbd docutils literal notranslate">C-a</kbd> beginning of line</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-e</kbd> end of line</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-k</kbd> kill line (but doesn?t put it in clipboard)</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-l</kbd> center window around the insertion point</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-b</kbd> go backward one character without deleting (usually you can
+also use the cursor key for this)</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-f</kbd> go forward one character without deleting (usually you can
+also use the cursor key for this)</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-p</kbd> go up one line (usually you can also use the cursor key for
+this)</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-d</kbd> delete next character</p></li>
 </ul>
 </div></blockquote>
 </li>
@@ -485,17 +543,15 @@ <h3>Python Shell window<a class="headerlink" href="#python-shell-window" title="
 <p>The editing features described in previous subsections work when entering
 code interactively.  IDLE?s Shell window also responds to the following keys.</p>
 <ul>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-c</kbd> interrupts executing command</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-d</kbd> sends end-of-file; closes window if typed at a <code class="docutils literal notranslate"><span class="pre">&gt;&gt;&gt;</span></code> prompt</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">Alt-/</kbd> (Expand word) is also useful to reduce typing</p>
+<li><p><kbd class="kbd docutils literal notranslate">C-c</kbd> interrupts executing command</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-d</kbd> sends end-of-file; closes window if typed at a <code class="docutils literal notranslate"><span class="pre">&gt;&gt;&gt;</span></code> prompt</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">Alt-/</kbd> (Expand word) is also useful to reduce typing</p>
 <p>Command history</p>
 <ul class="simple">
-<li><kbd class="kbd docutils literal notranslate">Alt-p</kbd> retrieves previous command matching what you have typed. On
-macOS use <kbd class="kbd docutils literal notranslate">C-p</kbd>.</li>
-<li><kbd class="kbd docutils literal notranslate">Alt-n</kbd> retrieves next. On macOS use <kbd class="kbd docutils literal notranslate">C-n</kbd>.</li>
-<li><kbd class="kbd docutils literal notranslate">Return</kbd> while on any previous command retrieves that command</li>
+<li><p><kbd class="kbd docutils literal notranslate">Alt-p</kbd> retrieves previous command matching what you have typed. On
+macOS use <kbd class="kbd docutils literal notranslate">C-p</kbd>.</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">Alt-n</kbd> retrieves next. On macOS use <kbd class="kbd docutils literal notranslate">C-n</kbd>.</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">Return</kbd> while on any previous command retrieves that command</p></li>
 </ul>
 </li>
 </ul>
@@ -545,12 +601,12 @@ <h3>Command line usage<a class="headerlink" href="#command-line-usage" title="Pe
 </div>
 <p>If there are arguments:</p>
 <ul class="simple">
-<li>If <code class="docutils literal notranslate"><span class="pre">-</span></code>, <code class="docutils literal notranslate"><span class="pre">-c</span></code>, or <code class="docutils literal notranslate"><span class="pre">r</span></code> is used, all arguments are placed in
+<li><p>If <code class="docutils literal notranslate"><span class="pre">-</span></code>, <code class="docutils literal notranslate"><span class="pre">-c</span></code>, or <code class="docutils literal notranslate"><span class="pre">r</span></code> is used, all arguments are placed in
 <code class="docutils literal notranslate"><span class="pre">sys.argv[1:...]</span></code> and <code class="docutils literal notranslate"><span class="pre">sys.argv[0]</span></code> is set to <code class="docutils literal notranslate"><span class="pre">''</span></code>, <code class="docutils literal notranslate"><span class="pre">'-c'</span></code>,
 or <code class="docutils literal notranslate"><span class="pre">'-r'</span></code>.  No editor window is opened, even if that is the default
-set in the Options dialog.</li>
-<li>Otherwise, arguments are files opened for editing and
-<code class="docutils literal notranslate"><span class="pre">sys.argv</span></code> reflects the arguments passed to IDLE itself.</li>
+set in the Options dialog.</p></li>
+<li><p>Otherwise, arguments are files opened for editing and
+<code class="docutils literal notranslate"><span class="pre">sys.argv</span></code> reflects the arguments passed to IDLE itself.</p></li>
 </ul>
 </div>
 <div class="section" id="startup-failure">
@@ -903,11 +959,11 @@ <h3>Navigation</h3>
 <br />
     <br />
 
-    Last updated on Aug 04, 2019.
+    Last updated on Sep 01, 2019.
     <a href="https://docs.python.org/3/bugs.html">Found a bug</a>?
     <br />
 
-    Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 2.1.1.
+    Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 2.1.2.
     </div>
 
   </body>
diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py
index e19d3615016f..ba2c6ec55481 100644
--- a/Lib/idlelib/help.py
+++ b/Lib/idlelib/help.py
@@ -62,6 +62,7 @@ def __init__(self, text):
         self.simplelist = False  # simple list (no double spacing)
         self.toc = []            # pair headers with text indexes for toc
         self.header = ''         # text within header tags for toc
+        self.prevtag = None      # info about previous tag (was opener, tag)
 
     def indent(self, amt=1):
         self.level += amt
@@ -78,8 +79,11 @@ def handle_starttag(self, tag, attrs):
             self.show = True    # start of main content
         elif tag == 'div' and class_ == 'sphinxsidebar':
             self.show = False   # end of main content
-        elif tag == 'p' and class_ != 'first':
-            s = '\n\n'
+        elif tag == 'p' and self.prevtag and not self.prevtag[0]:
+            # begin a new block for <p> tags after a closed tag
+            # avoid extra lines, e.g. after <pre> tags
+            lastline = self.text.get('end-1c linestart', 'end-1c')
+            s = '\n\n' if lastline and not lastline.isspace() else '\n'
         elif tag == 'span' and class_ == 'pre':
             self.chartags = 'pre'
         elif tag == 'span' and class_ == 'versionmodified':
@@ -120,6 +124,7 @@ def handle_starttag(self, tag, attrs):
             self.tags = tag
         if self.show:
             self.text.insert('end', s, (self.tags, self.chartags))
+        self.prevtag = (True, tag)
 
     def handle_endtag(self, tag):
         "Handle endtags in help.html."
@@ -139,6 +144,7 @@ def handle_endtag(self, tag):
             self.tags = ''
         elif tag in ['ul', 'dd', 'ol']:
             self.indent(amt=-1)
+        self.prevtag = (False, tag)
 
     def handle_data(self, data):
         "Handle date segments in help.html."


From webhook-mailer at python.org  Tue Sep  3 17:18:52 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 03 Sep 2019 21:18:52 -0000
Subject: [Python-checkins] bpo-38022: IDLE: upgrade help.html to sphinx 2.x
 HTML5 output (GH-15664)
Message-ID: <mailman.226.1567545533.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/952ea67289ffbd2f4785a9e537884a63d1208101
commit: 952ea67289ffbd2f4785a9e537884a63d1208101
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-03T14:18:45-07:00
summary:

bpo-38022: IDLE: upgrade help.html to sphinx 2.x HTML5 output (GH-15664)


The HTML5 output from Sphinx 2.x adds  '<p>' tags within list elements.  Using a new prevtag attribute, ignore these instead of emitting unwanted '\n\n'.

Also stop looking for 'first' classes on tags (no longer present) and fix the bug of double-spacing instead of single spacing after <pre> blocks.
(cherry picked from commit 580bdb0ece681537eadb360f0c796123ead7a559)

Co-authored-by: Tal Einat <taleinat+github at gmail.com>

files:
M Lib/idlelib/help.html
M Lib/idlelib/help.py

diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html
index 91e66a4b9117..0754f2453baf 100644
--- a/Lib/idlelib/help.html
+++ b/Lib/idlelib/help.html
@@ -1,11 +1,9 @@
 
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!DOCTYPE html>
 
 <html xmlns="http://www.w3.org/1999/xhtml">
   <head>
-    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
-    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta charset="utf-8" />
     <title>IDLE &#8212; Python 3.9.0a0 documentation</title>
     <link rel="stylesheet" href="../_static/pydoctheme.css" type="text/css" />
     <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
@@ -108,17 +106,17 @@ <h3>Navigation</h3>
 <p>IDLE is Python?s Integrated Development and Learning Environment.</p>
 <p>IDLE has the following features:</p>
 <ul class="simple">
-<li>coded in 100% pure Python, using the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><code class="xref py py-mod docutils literal notranslate"><span class="pre">tkinter</span></code></a> GUI toolkit</li>
-<li>cross-platform: works mostly the same on Windows, Unix, and macOS</li>
-<li>Python shell window (interactive interpreter) with colorizing
-of code input, output, and error messages</li>
-<li>multi-window text editor with multiple undo, Python colorizing,
-smart indent, call tips, auto completion, and other features</li>
-<li>search within any window, replace within editor windows, and search
-through multiple files (grep)</li>
-<li>debugger with persistent breakpoints, stepping, and viewing
-of global and local namespaces</li>
-<li>configuration, browsers, and other dialogs</li>
+<li><p>coded in 100% pure Python, using the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><code class="xref py py-mod docutils literal notranslate"><span class="pre">tkinter</span></code></a> GUI toolkit</p></li>
+<li><p>cross-platform: works mostly the same on Windows, Unix, and macOS</p></li>
+<li><p>Python shell window (interactive interpreter) with colorizing
+of code input, output, and error messages</p></li>
+<li><p>multi-window text editor with multiple undo, Python colorizing,
+smart indent, call tips, auto completion, and other features</p></li>
+<li><p>search within any window, replace within editor windows, and search
+through multiple files (grep)</p></li>
+<li><p>debugger with persistent breakpoints, stepping, and viewing
+of global and local namespaces</p></li>
+<li><p>configuration, browsers, and other dialogs</p></li>
 </ul>
 <div class="section" id="menus">
 <h2>Menus<a class="headerlink" href="#menus" title="Permalink to this headline">?</a></h2>
@@ -134,161 +132,216 @@ <h2>Menus<a class="headerlink" href="#menus" title="Permalink to this headline">
 described below are moved around to conform to Apple guidelines.</p>
 <div class="section" id="file-menu-shell-and-editor">
 <h3>File menu (Shell and Editor)<a class="headerlink" href="#file-menu-shell-and-editor" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>New File</dt><dd>Create a new file editing window.</dd>
-<dt>Open?</dt><dd>Open an existing file with an Open dialog.</dd>
-<dt>Recent Files</dt><dd>Open a list of recent files.  Click one to open it.</dd>
-<dt>Open Module?</dt><dd>Open an existing module (searches sys.path).</dd>
+<dl class="simple">
+<dt>New File</dt><dd><p>Create a new file editing window.</p>
+</dd>
+<dt>Open?</dt><dd><p>Open an existing file with an Open dialog.</p>
+</dd>
+<dt>Recent Files</dt><dd><p>Open a list of recent files.  Click one to open it.</p>
+</dd>
+<dt>Open Module?</dt><dd><p>Open an existing module (searches sys.path).</p>
+</dd>
 </dl>
-<dl class="docutils" id="index-1">
-<dt>Class Browser</dt><dd>Show functions, classes, and methods in the current Editor file in a
-tree structure.  In the shell, open a module first.</dd>
-<dt>Path Browser</dt><dd>Show sys.path directories, modules, functions, classes and methods in a
-tree structure.</dd>
-<dt>Save</dt><dd>Save the current window to the associated file, if there is one.  Windows
+<dl class="simple" id="index-1">
+<dt>Class Browser</dt><dd><p>Show functions, classes, and methods in the current Editor file in a
+tree structure.  In the shell, open a module first.</p>
+</dd>
+<dt>Path Browser</dt><dd><p>Show sys.path directories, modules, functions, classes and methods in a
+tree structure.</p>
+</dd>
+<dt>Save</dt><dd><p>Save the current window to the associated file, if there is one.  Windows
 that have been changed since being opened or last saved have a * before
 and after the window title.  If there is no associated file,
-do Save As instead.</dd>
-<dt>Save As?</dt><dd>Save the current window with a Save As dialog.  The file saved becomes the
-new associated file for the window.</dd>
-<dt>Save Copy As?</dt><dd>Save the current window to different file without changing the associated
-file.</dd>
-<dt>Print Window</dt><dd>Print the current window to the default printer.</dd>
-<dt>Close</dt><dd>Close the current window (ask to save if unsaved).</dd>
-<dt>Exit</dt><dd>Close all windows and quit IDLE (ask to save unsaved windows).</dd>
+do Save As instead.</p>
+</dd>
+<dt>Save As?</dt><dd><p>Save the current window with a Save As dialog.  The file saved becomes the
+new associated file for the window.</p>
+</dd>
+<dt>Save Copy As?</dt><dd><p>Save the current window to different file without changing the associated
+file.</p>
+</dd>
+<dt>Print Window</dt><dd><p>Print the current window to the default printer.</p>
+</dd>
+<dt>Close</dt><dd><p>Close the current window (ask to save if unsaved).</p>
+</dd>
+<dt>Exit</dt><dd><p>Close all windows and quit IDLE (ask to save unsaved windows).</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="edit-menu-shell-and-editor">
 <h3>Edit menu (Shell and Editor)<a class="headerlink" href="#edit-menu-shell-and-editor" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>Undo</dt><dd>Undo the last change to the current window.  A maximum of 1000 changes may
-be undone.</dd>
-<dt>Redo</dt><dd>Redo the last undone change to the current window.</dd>
-<dt>Cut</dt><dd>Copy selection into the system-wide clipboard; then delete the selection.</dd>
-<dt>Copy</dt><dd>Copy selection into the system-wide clipboard.</dd>
-<dt>Paste</dt><dd>Insert contents of the system-wide clipboard into the current window.</dd>
+<dl class="simple">
+<dt>Undo</dt><dd><p>Undo the last change to the current window.  A maximum of 1000 changes may
+be undone.</p>
+</dd>
+<dt>Redo</dt><dd><p>Redo the last undone change to the current window.</p>
+</dd>
+<dt>Cut</dt><dd><p>Copy selection into the system-wide clipboard; then delete the selection.</p>
+</dd>
+<dt>Copy</dt><dd><p>Copy selection into the system-wide clipboard.</p>
+</dd>
+<dt>Paste</dt><dd><p>Insert contents of the system-wide clipboard into the current window.</p>
+</dd>
 </dl>
 <p>The clipboard functions are also available in context menus.</p>
-<dl class="docutils">
-<dt>Select All</dt><dd>Select the entire contents of the current window.</dd>
-<dt>Find?</dt><dd>Open a search dialog with many options</dd>
-<dt>Find Again</dt><dd>Repeat the last search, if there is one.</dd>
-<dt>Find Selection</dt><dd>Search for the currently selected string, if there is one.</dd>
-<dt>Find in Files?</dt><dd>Open a file search dialog.  Put results in a new output window.</dd>
-<dt>Replace?</dt><dd>Open a search-and-replace dialog.</dd>
-<dt>Go to Line</dt><dd>Move cursor to the line number requested and make that line visible.</dd>
-<dt>Show Completions</dt><dd>Open a scrollable list allowing selection of keywords and attributes. See
-<a class="reference internal" href="#completions"><span class="std std-ref">Completions</span></a> in the Editing and navigation section below.</dd>
-<dt>Expand Word</dt><dd>Expand a prefix you have typed to match a full word in the same window;
-repeat to get a different expansion.</dd>
-<dt>Show call tip</dt><dd>After an unclosed parenthesis for a function, open a small window with
+<dl class="simple">
+<dt>Select All</dt><dd><p>Select the entire contents of the current window.</p>
+</dd>
+<dt>Find?</dt><dd><p>Open a search dialog with many options</p>
+</dd>
+<dt>Find Again</dt><dd><p>Repeat the last search, if there is one.</p>
+</dd>
+<dt>Find Selection</dt><dd><p>Search for the currently selected string, if there is one.</p>
+</dd>
+<dt>Find in Files?</dt><dd><p>Open a file search dialog.  Put results in a new output window.</p>
+</dd>
+<dt>Replace?</dt><dd><p>Open a search-and-replace dialog.</p>
+</dd>
+<dt>Go to Line</dt><dd><p>Move cursor to the line number requested and make that line visible.</p>
+</dd>
+<dt>Show Completions</dt><dd><p>Open a scrollable list allowing selection of keywords and attributes. See
+<a class="reference internal" href="#completions"><span class="std std-ref">Completions</span></a> in the Editing and navigation section below.</p>
+</dd>
+<dt>Expand Word</dt><dd><p>Expand a prefix you have typed to match a full word in the same window;
+repeat to get a different expansion.</p>
+</dd>
+<dt>Show call tip</dt><dd><p>After an unclosed parenthesis for a function, open a small window with
 function parameter hints.  See <a class="reference internal" href="#calltips"><span class="std std-ref">Calltips</span></a> in the
-Editing and navigation section below.</dd>
-<dt>Show surrounding parens</dt><dd>Highlight the surrounding parenthesis.</dd>
+Editing and navigation section below.</p>
+</dd>
+<dt>Show surrounding parens</dt><dd><p>Highlight the surrounding parenthesis.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="format-menu-editor-window-only">
 <span id="format-menu"></span><h3>Format menu (Editor window only)<a class="headerlink" href="#format-menu-editor-window-only" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>Indent Region</dt><dd>Shift selected lines right by the indent width (default 4 spaces).</dd>
-<dt>Dedent Region</dt><dd>Shift selected lines left by the indent width (default 4 spaces).</dd>
-<dt>Comment Out Region</dt><dd>Insert ## in front of selected lines.</dd>
-<dt>Uncomment Region</dt><dd>Remove leading # or ## from selected lines.</dd>
-<dt>Tabify Region</dt><dd>Turn <em>leading</em> stretches of spaces into tabs. (Note: We recommend using
-4 space blocks to indent Python code.)</dd>
-<dt>Untabify Region</dt><dd>Turn <em>all</em> tabs into the correct number of spaces.</dd>
-<dt>Toggle Tabs</dt><dd>Open a dialog to switch between indenting with spaces and tabs.</dd>
-<dt>New Indent Width</dt><dd>Open a dialog to change indent width. The accepted default by the Python
-community is 4 spaces.</dd>
-<dt>Format Paragraph</dt><dd>Reformat the current blank-line-delimited paragraph in comment block or
+<dl class="simple">
+<dt>Indent Region</dt><dd><p>Shift selected lines right by the indent width (default 4 spaces).</p>
+</dd>
+<dt>Dedent Region</dt><dd><p>Shift selected lines left by the indent width (default 4 spaces).</p>
+</dd>
+<dt>Comment Out Region</dt><dd><p>Insert ## in front of selected lines.</p>
+</dd>
+<dt>Uncomment Region</dt><dd><p>Remove leading # or ## from selected lines.</p>
+</dd>
+<dt>Tabify Region</dt><dd><p>Turn <em>leading</em> stretches of spaces into tabs. (Note: We recommend using
+4 space blocks to indent Python code.)</p>
+</dd>
+<dt>Untabify Region</dt><dd><p>Turn <em>all</em> tabs into the correct number of spaces.</p>
+</dd>
+<dt>Toggle Tabs</dt><dd><p>Open a dialog to switch between indenting with spaces and tabs.</p>
+</dd>
+<dt>New Indent Width</dt><dd><p>Open a dialog to change indent width. The accepted default by the Python
+community is 4 spaces.</p>
+</dd>
+<dt>Format Paragraph</dt><dd><p>Reformat the current blank-line-delimited paragraph in comment block or
 multiline string or selected line in a string.  All lines in the
-paragraph will be formatted to less than N columns, where N defaults to 72.</dd>
-<dt>Strip trailing whitespace</dt><dd>Remove trailing space and other whitespace characters after the last
+paragraph will be formatted to less than N columns, where N defaults to 72.</p>
+</dd>
+<dt>Strip trailing whitespace</dt><dd><p>Remove trailing space and other whitespace characters after the last
 non-whitespace character of a line by applying str.rstrip to each line,
-including lines within multiline strings.</dd>
+including lines within multiline strings.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="run-menu-editor-window-only">
 <span id="index-2"></span><h3>Run menu (Editor window only)<a class="headerlink" href="#run-menu-editor-window-only" title="Permalink to this headline">?</a></h3>
-<dl class="docutils" id="run-module">
-<dt>Run Module</dt><dd>Do <a class="reference internal" href="#check-module"><span class="std std-ref">Check Module</span></a>.  If no error, restart the shell to clean the
+<dl class="simple" id="run-module">
+<dt>Run Module</dt><dd><p>Do <a class="reference internal" href="#check-module"><span class="std std-ref">Check Module</span></a>.  If no error, restart the shell to clean the
 environment, then execute the module.  Output is displayed in the Shell
 window.  Note that output requires use of <code class="docutils literal notranslate"><span class="pre">print</span></code> or <code class="docutils literal notranslate"><span class="pre">write</span></code>.
 When execution is complete, the Shell retains focus and displays a prompt.
 At this point, one may interactively explore the result of execution.
 This is similar to executing a file with <code class="docutils literal notranslate"><span class="pre">python</span> <span class="pre">-i</span> <span class="pre">file</span></code> at a command
-line.</dd>
+line.</p>
+</dd>
 </dl>
-<dl class="docutils" id="run-custom">
-<dt>Run? Customized</dt><dd>Same as <a class="reference internal" href="#run-module"><span class="std std-ref">Run Module</span></a>, but run the module with customized
+<dl class="simple" id="run-custom">
+<dt>Run? Customized</dt><dd><p>Same as <a class="reference internal" href="#run-module"><span class="std std-ref">Run Module</span></a>, but run the module with customized
 settings.  <em>Command Line Arguments</em> extend <a class="reference internal" href="sys.html#sys.argv" title="sys.argv"><code class="xref py py-data docutils literal notranslate"><span class="pre">sys.argv</span></code></a> as if passed
-on a command line. The module can be run in the Shell without restarting.</dd>
+on a command line. The module can be run in the Shell without restarting.</p>
+</dd>
 </dl>
-<dl class="docutils" id="check-module">
-<dt>Check Module</dt><dd>Check the syntax of the module currently open in the Editor window. If the
+<dl class="simple" id="check-module">
+<dt>Check Module</dt><dd><p>Check the syntax of the module currently open in the Editor window. If the
 module has not been saved IDLE will either prompt the user to save or
 autosave, as selected in the General tab of the Idle Settings dialog.  If
 there is a syntax error, the approximate location is indicated in the
-Editor window.</dd>
+Editor window.</p>
+</dd>
 </dl>
-<dl class="docutils" id="python-shell">
-<dt>Python Shell</dt><dd>Open or wake up the Python Shell window.</dd>
+<dl class="simple" id="python-shell">
+<dt>Python Shell</dt><dd><p>Open or wake up the Python Shell window.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="shell-menu-shell-window-only">
 <h3>Shell menu (Shell window only)<a class="headerlink" href="#shell-menu-shell-window-only" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>View Last Restart</dt><dd>Scroll the shell window to the last Shell restart.</dd>
-<dt>Restart Shell</dt><dd>Restart the shell to clean the environment.</dd>
-<dt>Previous History</dt><dd>Cycle through earlier commands in history which match the current entry.</dd>
-<dt>Next History</dt><dd>Cycle through later commands in history which match the current entry.</dd>
-<dt>Interrupt Execution</dt><dd>Stop a running program.</dd>
+<dl class="simple">
+<dt>View Last Restart</dt><dd><p>Scroll the shell window to the last Shell restart.</p>
+</dd>
+<dt>Restart Shell</dt><dd><p>Restart the shell to clean the environment.</p>
+</dd>
+<dt>Previous History</dt><dd><p>Cycle through earlier commands in history which match the current entry.</p>
+</dd>
+<dt>Next History</dt><dd><p>Cycle through later commands in history which match the current entry.</p>
+</dd>
+<dt>Interrupt Execution</dt><dd><p>Stop a running program.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="debug-menu-shell-window-only">
 <h3>Debug menu (Shell window only)<a class="headerlink" href="#debug-menu-shell-window-only" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>Go to File/Line</dt><dd>Look on the current line. with the cursor, and the line above for a filename
+<dl class="simple">
+<dt>Go to File/Line</dt><dd><p>Look on the current line. with the cursor, and the line above for a filename
 and line number.  If found, open the file if not already open, and show the
 line.  Use this to view source lines referenced in an exception traceback
 and lines found by Find in Files. Also available in the context menu of
-the Shell window and Output windows.</dd>
+the Shell window and Output windows.</p>
+</dd>
 </dl>
-<dl class="docutils" id="index-3">
-<dt>Debugger (toggle)</dt><dd>When activated, code entered in the Shell or run from an Editor will run
+<dl class="simple" id="index-3">
+<dt>Debugger (toggle)</dt><dd><p>When activated, code entered in the Shell or run from an Editor will run
 under the debugger.  In the Editor, breakpoints can be set with the context
-menu.  This feature is still incomplete and somewhat experimental.</dd>
-<dt>Stack Viewer</dt><dd>Show the stack traceback of the last exception in a tree widget, with
-access to locals and globals.</dd>
-<dt>Auto-open Stack Viewer</dt><dd>Toggle automatically opening the stack viewer on an unhandled exception.</dd>
+menu.  This feature is still incomplete and somewhat experimental.</p>
+</dd>
+<dt>Stack Viewer</dt><dd><p>Show the stack traceback of the last exception in a tree widget, with
+access to locals and globals.</p>
+</dd>
+<dt>Auto-open Stack Viewer</dt><dd><p>Toggle automatically opening the stack viewer on an unhandled exception.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="options-menu-shell-and-editor">
 <h3>Options menu (Shell and Editor)<a class="headerlink" href="#options-menu-shell-and-editor" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>Configure IDLE</dt><dd>Open a configuration dialog and change preferences for the following:
+<dl class="simple">
+<dt>Configure IDLE</dt><dd><p>Open a configuration dialog and change preferences for the following:
 fonts, indentation, keybindings, text color themes, startup windows and
 size, additional help sources, and extensions.  On macOS, open the
 configuration dialog by selecting Preferences in the application
 menu. For more details, see
-<a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a> under Help and preferences.</dd>
+<a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a> under Help and preferences.</p>
+</dd>
 </dl>
 <p>Most configuration options apply to all windows or all future windows.
 The option items below only apply to the active window.</p>
-<dl class="docutils">
-<dt>Show/Hide Code Context (Editor Window only)</dt><dd>Open a pane at the top of the edit window which shows the block context
+<dl class="simple">
+<dt>Show/Hide Code Context (Editor Window only)</dt><dd><p>Open a pane at the top of the edit window which shows the block context
 of the code which has scrolled above the top of the window.  See
 <a class="reference internal" href="#code-context"><span class="std std-ref">Code Context</span></a> in the Editing and Navigation section
-below.</dd>
-<dt>Show/Hide Line Numbers (Editor Window only)</dt><dd>Open a column to the left of the edit window which shows the number
+below.</p>
+</dd>
+<dt>Show/Hide Line Numbers (Editor Window only)</dt><dd><p>Open a column to the left of the edit window which shows the number
 of each line of text.  The default is off, which may be changed in the
-preferences (see <a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a>).</dd>
-<dt>Zoom/Restore Height</dt><dd>Toggles the window between normal size and maximum height. The initial size
+preferences (see <a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a>).</p>
+</dd>
+<dt>Zoom/Restore Height</dt><dd><p>Toggles the window between normal size and maximum height. The initial size
 defaults to 40 lines by 80 chars unless changed on the General tab of the
 Configure IDLE dialog.  The maximum height for a screen is determined by
 momentarily maximizing a window the first time one is zoomed on the screen.
 Changing screen settings may invalidate the saved height.  This toggle has
-no effect when a window is maximized.</dd>
+no effect when a window is maximized.</p>
+</dd>
 </dl>
 </div>
 <div class="section" id="window-menu-shell-and-editor">
@@ -298,13 +351,17 @@ <h3>Window menu (Shell and Editor)<a class="headerlink" href="#window-menu-shell
 </div>
 <div class="section" id="help-menu-shell-and-editor">
 <h3>Help menu (Shell and Editor)<a class="headerlink" href="#help-menu-shell-and-editor" title="Permalink to this headline">?</a></h3>
-<dl class="docutils">
-<dt>About IDLE</dt><dd>Display version, copyright, license, credits, and more.</dd>
-<dt>IDLE Help</dt><dd>Display this IDLE document, detailing the menu options, basic editing and
-navigation, and other tips.</dd>
-<dt>Python Docs</dt><dd>Access local Python documentation, if installed, or start a web browser
-and open docs.python.org showing the latest Python documentation.</dd>
-<dt>Turtle Demo</dt><dd>Run the turtledemo module with example Python code and turtle drawings.</dd>
+<dl class="simple">
+<dt>About IDLE</dt><dd><p>Display version, copyright, license, credits, and more.</p>
+</dd>
+<dt>IDLE Help</dt><dd><p>Display this IDLE document, detailing the menu options, basic editing and
+navigation, and other tips.</p>
+</dd>
+<dt>Python Docs</dt><dd><p>Access local Python documentation, if installed, or start a web browser
+and open docs.python.org showing the latest Python documentation.</p>
+</dd>
+<dt>Turtle Demo</dt><dd><p>Run the turtledemo module with example Python code and turtle drawings.</p>
+</dd>
 </dl>
 <p>Additional help sources may be added here with the Configure IDLE dialog under
 the General tab. See the <a class="reference internal" href="#help-sources"><span class="std std-ref">Help sources</span></a> subsection below
@@ -314,27 +371,34 @@ <h3>Help menu (Shell and Editor)<a class="headerlink" href="#help-menu-shell-and
 <span id="index-4"></span><h3>Context Menus<a class="headerlink" href="#context-menus" title="Permalink to this headline">?</a></h3>
 <p>Open a context menu by right-clicking in a window (Control-click on macOS).
 Context menus have the standard clipboard functions also on the Edit menu.</p>
-<dl class="docutils">
-<dt>Cut</dt><dd>Copy selection into the system-wide clipboard; then delete the selection.</dd>
-<dt>Copy</dt><dd>Copy selection into the system-wide clipboard.</dd>
-<dt>Paste</dt><dd>Insert contents of the system-wide clipboard into the current window.</dd>
+<dl class="simple">
+<dt>Cut</dt><dd><p>Copy selection into the system-wide clipboard; then delete the selection.</p>
+</dd>
+<dt>Copy</dt><dd><p>Copy selection into the system-wide clipboard.</p>
+</dd>
+<dt>Paste</dt><dd><p>Insert contents of the system-wide clipboard into the current window.</p>
+</dd>
 </dl>
 <p>Editor windows also have breakpoint functions.  Lines with a breakpoint set are
 specially marked.  Breakpoints only have an effect when running under the
 debugger.  Breakpoints for a file are saved in the user?s .idlerc directory.</p>
-<dl class="docutils">
-<dt>Set Breakpoint</dt><dd>Set a breakpoint on the current line.</dd>
-<dt>Clear Breakpoint</dt><dd>Clear the breakpoint on that line.</dd>
+<dl class="simple">
+<dt>Set Breakpoint</dt><dd><p>Set a breakpoint on the current line.</p>
+</dd>
+<dt>Clear Breakpoint</dt><dd><p>Clear the breakpoint on that line.</p>
+</dd>
 </dl>
 <p>Shell and Output windows also have the following.</p>
-<dl class="docutils">
-<dt>Go to file/line</dt><dd>Same as in Debug menu.</dd>
+<dl class="simple">
+<dt>Go to file/line</dt><dd><p>Same as in Debug menu.</p>
+</dd>
 </dl>
 <p>The Shell window also has an output squeezing facility explained in the <em>Python
 Shell window</em> subsection below.</p>
-<dl class="docutils">
-<dt>Squeeze</dt><dd>If the cursor is over an output line, squeeze all the output between
-the code above and the prompt below down to a ?Squeezed text? label.</dd>
+<dl class="simple">
+<dt>Squeeze</dt><dd><p>If the cursor is over an output line, squeeze all the output between
+the code above and the prompt below down to a ?Squeezed text? label.</p>
+</dd>
 </dl>
 </div>
 </div>
@@ -357,32 +421,26 @@ <h3>Key bindings<a class="headerlink" href="#key-bindings" title="Permalink to t
 <p>In this section, ?C? refers to the <kbd class="kbd docutils literal notranslate">Control</kbd> key on Windows and Unix and
 the <kbd class="kbd docutils literal notranslate">Command</kbd> key on macOS.</p>
 <ul>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">Backspace</kbd> deletes to the left; <kbd class="kbd docutils literal notranslate">Del</kbd> deletes to the right</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-Backspace</kbd> delete word left; <kbd class="kbd docutils literal notranslate">C-Del</kbd> delete word to the right</p>
-</li>
-<li><p class="first">Arrow keys and <kbd class="kbd docutils literal notranslate">Page Up</kbd>/<kbd class="kbd docutils literal notranslate">Page Down</kbd> to move around</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-LeftArrow</kbd> and <kbd class="kbd docutils literal notranslate">C-RightArrow</kbd> moves by words</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">Home</kbd>/<kbd class="kbd docutils literal notranslate">End</kbd> go to begin/end of line</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-Home</kbd>/<kbd class="kbd docutils literal notranslate">C-End</kbd> go to begin/end of file</p>
-</li>
-<li><p class="first">Some useful Emacs bindings are inherited from Tcl/Tk:</p>
+<li><p><kbd class="kbd docutils literal notranslate">Backspace</kbd> deletes to the left; <kbd class="kbd docutils literal notranslate">Del</kbd> deletes to the right</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-Backspace</kbd> delete word left; <kbd class="kbd docutils literal notranslate">C-Del</kbd> delete word to the right</p></li>
+<li><p>Arrow keys and <kbd class="kbd docutils literal notranslate">Page Up</kbd>/<kbd class="kbd docutils literal notranslate">Page Down</kbd> to move around</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-LeftArrow</kbd> and <kbd class="kbd docutils literal notranslate">C-RightArrow</kbd> moves by words</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">Home</kbd>/<kbd class="kbd docutils literal notranslate">End</kbd> go to begin/end of line</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-Home</kbd>/<kbd class="kbd docutils literal notranslate">C-End</kbd> go to begin/end of file</p></li>
+<li><p>Some useful Emacs bindings are inherited from Tcl/Tk:</p>
 <blockquote>
 <div><ul class="simple">
-<li><kbd class="kbd docutils literal notranslate">C-a</kbd> beginning of line</li>
-<li><kbd class="kbd docutils literal notranslate">C-e</kbd> end of line</li>
-<li><kbd class="kbd docutils literal notranslate">C-k</kbd> kill line (but doesn?t put it in clipboard)</li>
-<li><kbd class="kbd docutils literal notranslate">C-l</kbd> center window around the insertion point</li>
-<li><kbd class="kbd docutils literal notranslate">C-b</kbd> go backward one character without deleting (usually you can
-also use the cursor key for this)</li>
-<li><kbd class="kbd docutils literal notranslate">C-f</kbd> go forward one character without deleting (usually you can
-also use the cursor key for this)</li>
-<li><kbd class="kbd docutils literal notranslate">C-p</kbd> go up one line (usually you can also use the cursor key for
-this)</li>
-<li><kbd class="kbd docutils literal notranslate">C-d</kbd> delete next character</li>
+<li><p><kbd class="kbd docutils literal notranslate">C-a</kbd> beginning of line</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-e</kbd> end of line</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-k</kbd> kill line (but doesn?t put it in clipboard)</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-l</kbd> center window around the insertion point</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-b</kbd> go backward one character without deleting (usually you can
+also use the cursor key for this)</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-f</kbd> go forward one character without deleting (usually you can
+also use the cursor key for this)</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-p</kbd> go up one line (usually you can also use the cursor key for
+this)</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-d</kbd> delete next character</p></li>
 </ul>
 </div></blockquote>
 </li>
@@ -485,17 +543,15 @@ <h3>Python Shell window<a class="headerlink" href="#python-shell-window" title="
 <p>The editing features described in previous subsections work when entering
 code interactively.  IDLE?s Shell window also responds to the following keys.</p>
 <ul>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-c</kbd> interrupts executing command</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">C-d</kbd> sends end-of-file; closes window if typed at a <code class="docutils literal notranslate"><span class="pre">&gt;&gt;&gt;</span></code> prompt</p>
-</li>
-<li><p class="first"><kbd class="kbd docutils literal notranslate">Alt-/</kbd> (Expand word) is also useful to reduce typing</p>
+<li><p><kbd class="kbd docutils literal notranslate">C-c</kbd> interrupts executing command</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">C-d</kbd> sends end-of-file; closes window if typed at a <code class="docutils literal notranslate"><span class="pre">&gt;&gt;&gt;</span></code> prompt</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">Alt-/</kbd> (Expand word) is also useful to reduce typing</p>
 <p>Command history</p>
 <ul class="simple">
-<li><kbd class="kbd docutils literal notranslate">Alt-p</kbd> retrieves previous command matching what you have typed. On
-macOS use <kbd class="kbd docutils literal notranslate">C-p</kbd>.</li>
-<li><kbd class="kbd docutils literal notranslate">Alt-n</kbd> retrieves next. On macOS use <kbd class="kbd docutils literal notranslate">C-n</kbd>.</li>
-<li><kbd class="kbd docutils literal notranslate">Return</kbd> while on any previous command retrieves that command</li>
+<li><p><kbd class="kbd docutils literal notranslate">Alt-p</kbd> retrieves previous command matching what you have typed. On
+macOS use <kbd class="kbd docutils literal notranslate">C-p</kbd>.</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">Alt-n</kbd> retrieves next. On macOS use <kbd class="kbd docutils literal notranslate">C-n</kbd>.</p></li>
+<li><p><kbd class="kbd docutils literal notranslate">Return</kbd> while on any previous command retrieves that command</p></li>
 </ul>
 </li>
 </ul>
@@ -545,12 +601,12 @@ <h3>Command line usage<a class="headerlink" href="#command-line-usage" title="Pe
 </div>
 <p>If there are arguments:</p>
 <ul class="simple">
-<li>If <code class="docutils literal notranslate"><span class="pre">-</span></code>, <code class="docutils literal notranslate"><span class="pre">-c</span></code>, or <code class="docutils literal notranslate"><span class="pre">r</span></code> is used, all arguments are placed in
+<li><p>If <code class="docutils literal notranslate"><span class="pre">-</span></code>, <code class="docutils literal notranslate"><span class="pre">-c</span></code>, or <code class="docutils literal notranslate"><span class="pre">r</span></code> is used, all arguments are placed in
 <code class="docutils literal notranslate"><span class="pre">sys.argv[1:...]</span></code> and <code class="docutils literal notranslate"><span class="pre">sys.argv[0]</span></code> is set to <code class="docutils literal notranslate"><span class="pre">''</span></code>, <code class="docutils literal notranslate"><span class="pre">'-c'</span></code>,
 or <code class="docutils literal notranslate"><span class="pre">'-r'</span></code>.  No editor window is opened, even if that is the default
-set in the Options dialog.</li>
-<li>Otherwise, arguments are files opened for editing and
-<code class="docutils literal notranslate"><span class="pre">sys.argv</span></code> reflects the arguments passed to IDLE itself.</li>
+set in the Options dialog.</p></li>
+<li><p>Otherwise, arguments are files opened for editing and
+<code class="docutils literal notranslate"><span class="pre">sys.argv</span></code> reflects the arguments passed to IDLE itself.</p></li>
 </ul>
 </div>
 <div class="section" id="startup-failure">
@@ -903,11 +959,11 @@ <h3>Navigation</h3>
 <br />
     <br />
 
-    Last updated on Aug 04, 2019.
+    Last updated on Sep 01, 2019.
     <a href="https://docs.python.org/3/bugs.html">Found a bug</a>?
     <br />
 
-    Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 2.1.1.
+    Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 2.1.2.
     </div>
 
   </body>
diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py
index e19d3615016f..ba2c6ec55481 100644
--- a/Lib/idlelib/help.py
+++ b/Lib/idlelib/help.py
@@ -62,6 +62,7 @@ def __init__(self, text):
         self.simplelist = False  # simple list (no double spacing)
         self.toc = []            # pair headers with text indexes for toc
         self.header = ''         # text within header tags for toc
+        self.prevtag = None      # info about previous tag (was opener, tag)
 
     def indent(self, amt=1):
         self.level += amt
@@ -78,8 +79,11 @@ def handle_starttag(self, tag, attrs):
             self.show = True    # start of main content
         elif tag == 'div' and class_ == 'sphinxsidebar':
             self.show = False   # end of main content
-        elif tag == 'p' and class_ != 'first':
-            s = '\n\n'
+        elif tag == 'p' and self.prevtag and not self.prevtag[0]:
+            # begin a new block for <p> tags after a closed tag
+            # avoid extra lines, e.g. after <pre> tags
+            lastline = self.text.get('end-1c linestart', 'end-1c')
+            s = '\n\n' if lastline and not lastline.isspace() else '\n'
         elif tag == 'span' and class_ == 'pre':
             self.chartags = 'pre'
         elif tag == 'span' and class_ == 'versionmodified':
@@ -120,6 +124,7 @@ def handle_starttag(self, tag, attrs):
             self.tags = tag
         if self.show:
             self.text.insert('end', s, (self.tags, self.chartags))
+        self.prevtag = (True, tag)
 
     def handle_endtag(self, tag):
         "Handle endtags in help.html."
@@ -139,6 +144,7 @@ def handle_endtag(self, tag):
             self.tags = ''
         elif tag in ['ul', 'dd', 'ol']:
             self.indent(amt=-1)
+        self.prevtag = (False, tag)
 
     def handle_data(self, data):
         "Handle date segments in help.html."


From webhook-mailer at python.org  Tue Sep  3 22:46:07 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Wed, 04 Sep 2019 02:46:07 -0000
Subject: [Python-checkins] closes bpo-37966: Fully implement the UAX #15
 quick-check algorithm. (GH-15558)
Message-ID: <mailman.227.1567565169.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2f09413947d1ce0043de62ed2346f9a2b4e5880b
commit: 2f09413947d1ce0043de62ed2346f9a2b4e5880b
branch: master
author: Greg Price <gnprice at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-03T19:45:44-07:00
summary:

closes bpo-37966: Fully implement the UAX #15 quick-check algorithm. (GH-15558)

The purpose of the `unicodedata.is_normalized` function is to answer
the question `str == unicodedata.normalized(form, str)` more
efficiently than writing just that, by using the "quick check"
optimization described in the Unicode standard in UAX #15.

However, it turns out the code doesn't implement the full algorithm
from the standard, and as a result we often miss the optimization and
end up having to compute the whole normalized string after all.

Implement the standard's algorithm.  This greatly speeds up
`unicodedata.is_normalized` in many cases where our partial variant
of quick-check had been returning MAYBE and the standard algorithm
returns NO.

At a quick test on my desktop, the existing code takes about 4.4 ms/MB
(so 4.4 ns per byte) when the partial quick-check returns MAYBE and it
has to do the slow normalize-and-compare:

  $ build.base/python -m timeit -s 'import unicodedata; s = "\uf900"*500000' \
      -- 'unicodedata.is_normalized("NFD", s)'
  50 loops, best of 5: 4.39 msec per loop

With this patch, it gets the answer instantly (58 ns) on the same 1 MB
string:

  $ build.dev/python -m timeit -s 'import unicodedata; s = "\uf900"*500000' \
      -- 'unicodedata.is_normalized("NFD", s)'
  5000000 loops, best of 5: 58.2 nsec per loop

This restores a small optimization that the original version of this
code had for the `unicodedata.normalize` use case.

With this, that case is actually faster than in master!

$ build.base/python -m timeit -s 'import unicodedata; s = "\u0338"*500000' \
    -- 'unicodedata.normalize("NFD", s)'
500 loops, best of 5: 561 usec per loop

$ build.dev/python -m timeit -s 'import unicodedata; s = "\u0338"*500000' \
    -- 'unicodedata.normalize("NFD", s)'
500 loops, best of 5: 512 usec per loop

files:
A Misc/NEWS.d/next/Core and Builtins/2019-08-27-21-21-36.bpo-37966.5OBLez.rst
M Doc/whatsnew/3.8.rst
M Lib/test/test_unicodedata.py
M Modules/unicodedata.c

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index bcdb60d86d85..4a1362d943c8 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -1090,8 +1090,9 @@ unicodedata
   <http://blog.unicode.org/2019/05/unicode-12-1-en.html>`_ release.
 
 * New function :func:`~unicodedata.is_normalized` can be used to verify a string
-  is in a specific normal form. (Contributed by Max Belanger and David Euresti in
-  :issue:`32285`).
+  is in a specific normal form, often much faster than by actually normalizing
+  the string.  (Contributed by Max Belanger, David Euresti, and Greg Price in
+  :issue:`32285` and :issue:`37966`).
 
 
 unittest
diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py
index 7bc196be362b..9ec2f1149726 100644
--- a/Lib/test/test_unicodedata.py
+++ b/Lib/test/test_unicodedata.py
@@ -208,6 +208,8 @@ def test_issue29456(self):
         self.assertEqual(self.db.normalize('NFC', u11a7_str_a), u11a7_str_b)
         self.assertEqual(self.db.normalize('NFC', u11c3_str_a), u11c3_str_b)
 
+    # For tests of unicodedata.is_normalized / self.db.is_normalized ,
+    # see test_normalization.py .
 
     def test_east_asian_width(self):
         eaw = self.db.east_asian_width
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-27-21-21-36.bpo-37966.5OBLez.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-27-21-21-36.bpo-37966.5OBLez.rst
new file mode 100644
index 000000000000..6b9d69c5b3a9
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-27-21-21-36.bpo-37966.5OBLez.rst	
@@ -0,0 +1,3 @@
+The implementation of :func:`~unicodedata.is_normalized` has been greatly
+sped up on strings that aren't normalized, by implementing the full
+normalization-quick-check algorithm from the Unicode standard.
diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c
index ae0d4e46f9a4..5e8ba602d668 100644
--- a/Modules/unicodedata.c
+++ b/Modules/unicodedata.c
@@ -19,6 +19,8 @@
 #include "ucnhash.h"
 #include "structmember.h"
 
+#include <stdbool.h>
+
 _Py_IDENTIFIER(NFC);
 _Py_IDENTIFIER(NFD);
 _Py_IDENTIFIER(NFKC);
@@ -775,25 +777,40 @@ nfc_nfkc(PyObject *self, PyObject *input, int k)
     return result;
 }
 
-typedef enum {YES, NO, MAYBE} NormalMode;
-
-/* Return YES if the input is certainly normalized, NO or MAYBE if it might not be. */
-static NormalMode
-is_normalized(PyObject *self, PyObject *input, int nfc, int k)
+// This needs to match the logic in makeunicodedata.py
+// which constructs the quickcheck data.
+typedef enum {YES = 0, MAYBE = 1, NO = 2} QuickcheckResult;
+
+/* Run the Unicode normalization "quickcheck" algorithm.
+ *
+ * Return YES or NO if quickcheck determines the input is certainly
+ * normalized or certainly not, and MAYBE if quickcheck is unable to
+ * tell.
+ *
+ * If `yes_only` is true, then return MAYBE as soon as we determine
+ * the answer is not YES.
+ *
+ * For background and details on the algorithm, see UAX #15:
+ *   https://www.unicode.org/reports/tr15/#Detecting_Normalization_Forms
+ */
+static QuickcheckResult
+is_normalized_quickcheck(PyObject *self, PyObject *input,
+                         int nfc, int k, bool yes_only)
 {
-    Py_ssize_t i, len;
-    int kind;
-    void *data;
-    unsigned char prev_combining = 0, quickcheck_mask;
-
     /* An older version of the database is requested, quickchecks must be
        disabled. */
     if (self && UCD_Check(self))
         return NO;
 
-    /* The two quickcheck bits at this shift mean 0=Yes, 1=Maybe, 2=No,
-       as described in http://unicode.org/reports/tr15/#Annex8. */
-    quickcheck_mask = 3 << ((nfc ? 4 : 0) + (k ? 2 : 0));
+    Py_ssize_t i, len;
+    int kind;
+    void *data;
+    unsigned char prev_combining = 0;
+
+    /* The two quickcheck bits at this shift have type QuickcheckResult. */
+    int quickcheck_shift = (nfc ? 4 : 0) + (k ? 2 : 0);
+
+    QuickcheckResult result = YES; /* certainly normalized, unless we find something */
 
     i = 0;
     kind = PyUnicode_KIND(input);
@@ -802,16 +819,26 @@ is_normalized(PyObject *self, PyObject *input, int nfc, int k)
     while (i < len) {
         Py_UCS4 ch = PyUnicode_READ(kind, data, i++);
         const _PyUnicode_DatabaseRecord *record = _getrecord_ex(ch);
-        unsigned char combining = record->combining;
-        unsigned char quickcheck = record->normalization_quick_check;
 
-        if (quickcheck & quickcheck_mask)
-            return MAYBE; /* this string might need normalization */
+        unsigned char combining = record->combining;
         if (combining && prev_combining > combining)
             return NO; /* non-canonical sort order, not normalized */
         prev_combining = combining;
+
+        unsigned char quickcheck_whole = record->normalization_quick_check;
+        if (yes_only) {
+            if (quickcheck_whole & (3 << quickcheck_shift))
+                return MAYBE;
+        } else {
+            switch ((quickcheck_whole >> quickcheck_shift) & 3) {
+            case NO:
+              return NO;
+            case MAYBE:
+              result = MAYBE; /* this string might need normalization */
+            }
+        }
     }
-    return YES; /* certainly normalized */
+    return result;
 }
 
 /*[clinic input]
@@ -844,7 +871,7 @@ unicodedata_UCD_is_normalized_impl(PyObject *self, PyObject *form,
     PyObject *result;
     int nfc = 0;
     int k = 0;
-    NormalMode m;
+    QuickcheckResult m;
 
     PyObject *cmp;
     int match = 0;
@@ -867,7 +894,7 @@ unicodedata_UCD_is_normalized_impl(PyObject *self, PyObject *form,
         return NULL;
     }
 
-    m = is_normalized(self, input, nfc, k);
+    m = is_normalized_quickcheck(self, input, nfc, k, false);
 
     if (m == MAYBE) {
         cmp = (nfc ? nfc_nfkc : nfd_nfkd)(self, input, k);
@@ -913,28 +940,28 @@ unicodedata_UCD_normalize_impl(PyObject *self, PyObject *form,
     }
 
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFC)) {
-        if (is_normalized(self, input, 1, 0) == YES) {
+        if (is_normalized_quickcheck(self, input, 1, 0, true) == YES) {
             Py_INCREF(input);
             return input;
         }
         return nfc_nfkc(self, input, 0);
     }
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFKC)) {
-        if (is_normalized(self, input, 1, 1) == YES) {
+        if (is_normalized_quickcheck(self, input, 1, 1, true) == YES) {
             Py_INCREF(input);
             return input;
         }
         return nfc_nfkc(self, input, 1);
     }
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFD)) {
-        if (is_normalized(self, input, 0, 0) == YES) {
+        if (is_normalized_quickcheck(self, input, 0, 0, true) == YES) {
             Py_INCREF(input);
             return input;
         }
         return nfd_nfkd(self, input, 0);
     }
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFKD)) {
-        if (is_normalized(self, input, 0, 1) == YES) {
+        if (is_normalized_quickcheck(self, input, 0, 1, true) == YES) {
             Py_INCREF(input);
             return input;
         }


From webhook-mailer at python.org  Tue Sep  3 23:03:44 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 04 Sep 2019 03:03:44 -0000
Subject: [Python-checkins] closes bpo-37966: Fully implement the UAX GH-15
 quick-check algorithm. (GH-15558)
Message-ID: <mailman.228.1567566224.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4dd1c9d9c2bca4744c70c9556b7051f4465ede3e
commit: 4dd1c9d9c2bca4744c70c9556b7051f4465ede3e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-03T20:03:37-07:00
summary:

closes bpo-37966: Fully implement the UAX GH-15 quick-check algorithm. (GH-15558)


The purpose of the `unicodedata.is_normalized` function is to answer
the question `str == unicodedata.normalized(form, str)` more
efficiently than writing just that, by using the "quick check"
optimization described in the Unicode standard in UAX GH-15.

However, it turns out the code doesn't implement the full algorithm
from the standard, and as a result we often miss the optimization and
end up having to compute the whole normalized string after all.

Implement the standard's algorithm.  This greatly speeds up
`unicodedata.is_normalized` in many cases where our partial variant
of quick-check had been returning MAYBE and the standard algorithm
returns NO.

At a quick test on my desktop, the existing code takes about 4.4 ms/MB
(so 4.4 ns per byte) when the partial quick-check returns MAYBE and it
has to do the slow normalize-and-compare:

  $ build.base/python -m timeit -s 'import unicodedata; s = "\uf900"*500000' \
      -- 'unicodedata.is_normalized("NFD", s)'
  50 loops, best of 5: 4.39 msec per loop

With this patch, it gets the answer instantly (58 ns) on the same 1 MB
string:

  $ build.dev/python -m timeit -s 'import unicodedata; s = "\uf900"*500000' \
      -- 'unicodedata.is_normalized("NFD", s)'
  5000000 loops, best of 5: 58.2 nsec per loop

This restores a small optimization that the original version of this
code had for the `unicodedata.normalize` use case.

With this, that case is actually faster than in master!

$ build.base/python -m timeit -s 'import unicodedata; s = "\u0338"*500000' \
    -- 'unicodedata.normalize("NFD", s)'
500 loops, best of 5: 561 usec per loop

$ build.dev/python -m timeit -s 'import unicodedata; s = "\u0338"*500000' \
    -- 'unicodedata.normalize("NFD", s)'
500 loops, best of 5: 512 usec per loop
(cherry picked from commit 2f09413947d1ce0043de62ed2346f9a2b4e5880b)

Co-authored-by: Greg Price <gnprice at gmail.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2019-08-27-21-21-36.bpo-37966.5OBLez.rst
M Doc/whatsnew/3.8.rst
M Lib/test/test_unicodedata.py
M Modules/unicodedata.c

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index bcdb60d86d85..4a1362d943c8 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -1090,8 +1090,9 @@ unicodedata
   <http://blog.unicode.org/2019/05/unicode-12-1-en.html>`_ release.
 
 * New function :func:`~unicodedata.is_normalized` can be used to verify a string
-  is in a specific normal form. (Contributed by Max Belanger and David Euresti in
-  :issue:`32285`).
+  is in a specific normal form, often much faster than by actually normalizing
+  the string.  (Contributed by Max Belanger, David Euresti, and Greg Price in
+  :issue:`32285` and :issue:`37966`).
 
 
 unittest
diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py
index a52b6de547fb..07d717688b0c 100644
--- a/Lib/test/test_unicodedata.py
+++ b/Lib/test/test_unicodedata.py
@@ -220,6 +220,8 @@ def test_issue29456(self):
         self.assertEqual(self.db.normalize('NFC', u11a7_str_a), u11a7_str_b)
         self.assertEqual(self.db.normalize('NFC', u11c3_str_a), u11c3_str_b)
 
+    # For tests of unicodedata.is_normalized / self.db.is_normalized ,
+    # see test_normalization.py .
 
     def test_east_asian_width(self):
         eaw = self.db.east_asian_width
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-27-21-21-36.bpo-37966.5OBLez.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-27-21-21-36.bpo-37966.5OBLez.rst
new file mode 100644
index 000000000000..6b9d69c5b3a9
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-27-21-21-36.bpo-37966.5OBLez.rst	
@@ -0,0 +1,3 @@
+The implementation of :func:`~unicodedata.is_normalized` has been greatly
+sped up on strings that aren't normalized, by implementing the full
+normalization-quick-check algorithm from the Unicode standard.
diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c
index ae0d4e46f9a4..5e8ba602d668 100644
--- a/Modules/unicodedata.c
+++ b/Modules/unicodedata.c
@@ -19,6 +19,8 @@
 #include "ucnhash.h"
 #include "structmember.h"
 
+#include <stdbool.h>
+
 _Py_IDENTIFIER(NFC);
 _Py_IDENTIFIER(NFD);
 _Py_IDENTIFIER(NFKC);
@@ -775,25 +777,40 @@ nfc_nfkc(PyObject *self, PyObject *input, int k)
     return result;
 }
 
-typedef enum {YES, NO, MAYBE} NormalMode;
-
-/* Return YES if the input is certainly normalized, NO or MAYBE if it might not be. */
-static NormalMode
-is_normalized(PyObject *self, PyObject *input, int nfc, int k)
+// This needs to match the logic in makeunicodedata.py
+// which constructs the quickcheck data.
+typedef enum {YES = 0, MAYBE = 1, NO = 2} QuickcheckResult;
+
+/* Run the Unicode normalization "quickcheck" algorithm.
+ *
+ * Return YES or NO if quickcheck determines the input is certainly
+ * normalized or certainly not, and MAYBE if quickcheck is unable to
+ * tell.
+ *
+ * If `yes_only` is true, then return MAYBE as soon as we determine
+ * the answer is not YES.
+ *
+ * For background and details on the algorithm, see UAX #15:
+ *   https://www.unicode.org/reports/tr15/#Detecting_Normalization_Forms
+ */
+static QuickcheckResult
+is_normalized_quickcheck(PyObject *self, PyObject *input,
+                         int nfc, int k, bool yes_only)
 {
-    Py_ssize_t i, len;
-    int kind;
-    void *data;
-    unsigned char prev_combining = 0, quickcheck_mask;
-
     /* An older version of the database is requested, quickchecks must be
        disabled. */
     if (self && UCD_Check(self))
         return NO;
 
-    /* The two quickcheck bits at this shift mean 0=Yes, 1=Maybe, 2=No,
-       as described in http://unicode.org/reports/tr15/#Annex8. */
-    quickcheck_mask = 3 << ((nfc ? 4 : 0) + (k ? 2 : 0));
+    Py_ssize_t i, len;
+    int kind;
+    void *data;
+    unsigned char prev_combining = 0;
+
+    /* The two quickcheck bits at this shift have type QuickcheckResult. */
+    int quickcheck_shift = (nfc ? 4 : 0) + (k ? 2 : 0);
+
+    QuickcheckResult result = YES; /* certainly normalized, unless we find something */
 
     i = 0;
     kind = PyUnicode_KIND(input);
@@ -802,16 +819,26 @@ is_normalized(PyObject *self, PyObject *input, int nfc, int k)
     while (i < len) {
         Py_UCS4 ch = PyUnicode_READ(kind, data, i++);
         const _PyUnicode_DatabaseRecord *record = _getrecord_ex(ch);
-        unsigned char combining = record->combining;
-        unsigned char quickcheck = record->normalization_quick_check;
 
-        if (quickcheck & quickcheck_mask)
-            return MAYBE; /* this string might need normalization */
+        unsigned char combining = record->combining;
         if (combining && prev_combining > combining)
             return NO; /* non-canonical sort order, not normalized */
         prev_combining = combining;
+
+        unsigned char quickcheck_whole = record->normalization_quick_check;
+        if (yes_only) {
+            if (quickcheck_whole & (3 << quickcheck_shift))
+                return MAYBE;
+        } else {
+            switch ((quickcheck_whole >> quickcheck_shift) & 3) {
+            case NO:
+              return NO;
+            case MAYBE:
+              result = MAYBE; /* this string might need normalization */
+            }
+        }
     }
-    return YES; /* certainly normalized */
+    return result;
 }
 
 /*[clinic input]
@@ -844,7 +871,7 @@ unicodedata_UCD_is_normalized_impl(PyObject *self, PyObject *form,
     PyObject *result;
     int nfc = 0;
     int k = 0;
-    NormalMode m;
+    QuickcheckResult m;
 
     PyObject *cmp;
     int match = 0;
@@ -867,7 +894,7 @@ unicodedata_UCD_is_normalized_impl(PyObject *self, PyObject *form,
         return NULL;
     }
 
-    m = is_normalized(self, input, nfc, k);
+    m = is_normalized_quickcheck(self, input, nfc, k, false);
 
     if (m == MAYBE) {
         cmp = (nfc ? nfc_nfkc : nfd_nfkd)(self, input, k);
@@ -913,28 +940,28 @@ unicodedata_UCD_normalize_impl(PyObject *self, PyObject *form,
     }
 
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFC)) {
-        if (is_normalized(self, input, 1, 0) == YES) {
+        if (is_normalized_quickcheck(self, input, 1, 0, true) == YES) {
             Py_INCREF(input);
             return input;
         }
         return nfc_nfkc(self, input, 0);
     }
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFKC)) {
-        if (is_normalized(self, input, 1, 1) == YES) {
+        if (is_normalized_quickcheck(self, input, 1, 1, true) == YES) {
             Py_INCREF(input);
             return input;
         }
         return nfc_nfkc(self, input, 1);
     }
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFD)) {
-        if (is_normalized(self, input, 0, 0) == YES) {
+        if (is_normalized_quickcheck(self, input, 0, 0, true) == YES) {
             Py_INCREF(input);
             return input;
         }
         return nfd_nfkd(self, input, 0);
     }
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFKD)) {
-        if (is_normalized(self, input, 0, 1) == YES) {
+        if (is_normalized_quickcheck(self, input, 0, 1, true) == YES) {
             Py_INCREF(input);
             return input;
         }


From webhook-mailer at python.org  Wed Sep  4 02:04:28 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 04 Sep 2019 06:04:28 -0000
Subject: [Python-checkins] Fix grammar in asyncio-dev.rst (GH-15672)
Message-ID: <mailman.229.1567577069.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/675d17cec49ed7f7eb01d1bc3ae6999c728e070d
commit: 675d17cec49ed7f7eb01d1bc3ae6999c728e070d
branch: master
author: Roger Iyengar <ri at rogeriyengar.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-03T23:04:09-07:00
summary:

Fix grammar in asyncio-dev.rst (GH-15672)



Automerge-Triggered-By: @ned-deily

files:
M Doc/library/asyncio-dev.rst

diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst
index b72880361929..101e7817a95e 100644
--- a/Doc/library/asyncio-dev.rst
+++ b/Doc/library/asyncio-dev.rst
@@ -119,7 +119,7 @@ all concurrent asyncio Tasks and IO operations would be delayed
 by 1 second.
 
 An executor can be used to run a task in a different thread or even in
-a different process to avoid blocking block the OS thread with the
+a different process to avoid blocking the OS thread with the
 event loop.  See the :meth:`loop.run_in_executor` method for more
 details.
 


From webhook-mailer at python.org  Wed Sep  4 02:10:51 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 04 Sep 2019 06:10:51 -0000
Subject: [Python-checkins] Fix grammar in asyncio-dev.rst (GH-15672)
Message-ID: <mailman.230.1567577452.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5e194f57c08898cb8da457c6584402a2be2c828d
commit: 5e194f57c08898cb8da457c6584402a2be2c828d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-03T23:10:45-07:00
summary:

Fix grammar in asyncio-dev.rst (GH-15672)


Automerge-Triggered-By: @ned-deily
(cherry picked from commit 675d17cec49ed7f7eb01d1bc3ae6999c728e070d)

Co-authored-by: Roger Iyengar <ri at rogeriyengar.com>

files:
M Doc/library/asyncio-dev.rst

diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst
index b72880361929..101e7817a95e 100644
--- a/Doc/library/asyncio-dev.rst
+++ b/Doc/library/asyncio-dev.rst
@@ -119,7 +119,7 @@ all concurrent asyncio Tasks and IO operations would be delayed
 by 1 second.
 
 An executor can be used to run a task in a different thread or even in
-a different process to avoid blocking block the OS thread with the
+a different process to avoid blocking the OS thread with the
 event loop.  See the :meth:`loop.run_in_executor` method for more
 details.
 


From webhook-mailer at python.org  Wed Sep  4 02:13:28 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 04 Sep 2019 06:13:28 -0000
Subject: [Python-checkins] Fix grammar in asyncio-dev.rst (GH-15672)
Message-ID: <mailman.231.1567577609.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6b8121a261917256198ac5bc3a02306a05cea169
commit: 6b8121a261917256198ac5bc3a02306a05cea169
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-03T23:13:24-07:00
summary:

Fix grammar in asyncio-dev.rst (GH-15672)


Automerge-Triggered-By: @ned-deily
(cherry picked from commit 675d17cec49ed7f7eb01d1bc3ae6999c728e070d)

Co-authored-by: Roger Iyengar <ri at rogeriyengar.com>

files:
M Doc/library/asyncio-dev.rst

diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst
index b72880361929..101e7817a95e 100644
--- a/Doc/library/asyncio-dev.rst
+++ b/Doc/library/asyncio-dev.rst
@@ -119,7 +119,7 @@ all concurrent asyncio Tasks and IO operations would be delayed
 by 1 second.
 
 An executor can be used to run a task in a different thread or even in
-a different process to avoid blocking block the OS thread with the
+a different process to avoid blocking the OS thread with the
 event loop.  See the :meth:`loop.run_in_executor` method for more
 details.
 


From webhook-mailer at python.org  Wed Sep  4 09:58:09 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 04 Sep 2019 13:58:09 -0000
Subject: [Python-checkins] bpo-36030: Fix a possible segfault in
 PyTuple_New() (GH-15670)
Message-ID: <mailman.232.1567605491.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/60bd1f88f21073965a444c8b39c4202d015da5d6
commit: 60bd1f88f21073965a444c8b39c4202d015da5d6
branch: master
author: Zackery Spytz <zspytz at gmail.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-04T15:58:04+02:00
summary:

bpo-36030: Fix a possible segfault in PyTuple_New() (GH-15670)

files:
M Objects/tupleobject.c

diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c
index 3419baa529a6..a72257f95b08 100644
--- a/Objects/tupleobject.c
+++ b/Objects/tupleobject.c
@@ -146,6 +146,9 @@ PyTuple_New(Py_ssize_t size)
     }
 #endif
     op = tuple_alloc(size);
+    if (op == NULL) {
+        return NULL;
+    }
     for (Py_ssize_t i = 0; i < size; i++) {
         op->ob_item[i] = NULL;
     }


From webhook-mailer at python.org  Wed Sep  4 17:43:04 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Wed, 04 Sep 2019 21:43:04 -0000
Subject: [Python-checkins] bpo-38030: Fix os.stat failures on block devices
 on Windows (GH-15681)
Message-ID: <mailman.233.1567633385.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/772ec0fad57412daa53d16d7019b6b2fe6e94942
commit: 772ec0fad57412daa53d16d7019b6b2fe6e94942
branch: master
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-04T14:42:54-07:00
summary:

bpo-38030: Fix os.stat failures on block devices on Windows (GH-15681)

files:
A Misc/NEWS.d/next/Windows/2019-09-04-14-01-08.bpo-38030._USdtk.rst
M Lib/test/test_os.py
M Modules/posixmodule.c

diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 440cd6c1cf73..8ff0296fad0b 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -591,6 +591,14 @@ def test_access_denied(self):
         result = os.stat(fname)
         self.assertNotEqual(result.st_size, 0)
 
+    @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
+    def test_stat_block_device(self):
+        # bpo-38030: os.stat fails for block devices
+        # Test a filename like "//./C:"
+        fname = "//./" + os.path.splitdrive(os.getcwd())[0]
+        result = os.stat(fname)
+        self.assertEqual(result.st_mode, stat.S_IFBLK)
+
 
 class UtimeTests(unittest.TestCase):
     def setUp(self):
diff --git a/Misc/NEWS.d/next/Windows/2019-09-04-14-01-08.bpo-38030._USdtk.rst b/Misc/NEWS.d/next/Windows/2019-09-04-14-01-08.bpo-38030._USdtk.rst
new file mode 100644
index 000000000000..f1be8a1e1c86
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-04-14-01-08.bpo-38030._USdtk.rst
@@ -0,0 +1 @@
+Fixes :func:`os.stat` failing for block devices on Windows
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index c412d07a0e11..81704eec321c 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -1794,13 +1794,13 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result,
             case ERROR_INVALID_PARAMETER:
             case ERROR_INVALID_FUNCTION:
             case ERROR_NOT_SUPPORTED:
-                retval = -1;
+                /* Volumes and physical disks are block devices, e.g.
+                   \\.\C: and \\.\PhysicalDrive0. */
+                memset(result, 0, sizeof(*result));
+                result->st_mode = 0x6000; /* S_IFBLK */
                 goto cleanup;
             }
-            /* Volumes and physical disks are block devices, e.g.
-               \\.\C: and \\.\PhysicalDrive0. */
-            memset(result, 0, sizeof(*result));
-            result->st_mode = 0x6000; /* S_IFBLK */
+            retval = -1;
             goto cleanup;
         }
     }
@@ -1827,7 +1827,14 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result,
 
 cleanup:
     if (hFile != INVALID_HANDLE_VALUE) {
-        CloseHandle(hFile);
+        /* Preserve last error if we are failing */
+        error = retval ? GetLastError() : 0;
+        if (!CloseHandle(hFile)) {
+            retval = -1;
+        } else if (retval) {
+            /* Restore last error */
+            SetLastError(error);
+        }
     }
 
     return retval;


From webhook-mailer at python.org  Wed Sep  4 18:18:08 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 04 Sep 2019 22:18:08 -0000
Subject: [Python-checkins] bpo-38030: Fix os.stat failures on block devices
 on Windows (GH-15681)
Message-ID: <mailman.234.1567635489.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/cad7abf8abe657b696b9c8deb4b727e0cefaf36d
commit: cad7abf8abe657b696b9c8deb4b727e0cefaf36d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-04T15:18:05-07:00
summary:

bpo-38030: Fix os.stat failures on block devices on Windows (GH-15681)

(cherry picked from commit 772ec0fad57412daa53d16d7019b6b2fe6e94942)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-04-14-01-08.bpo-38030._USdtk.rst
M Lib/test/test_os.py
M Modules/posixmodule.c

diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 440cd6c1cf73..8ff0296fad0b 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -591,6 +591,14 @@ def test_access_denied(self):
         result = os.stat(fname)
         self.assertNotEqual(result.st_size, 0)
 
+    @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
+    def test_stat_block_device(self):
+        # bpo-38030: os.stat fails for block devices
+        # Test a filename like "//./C:"
+        fname = "//./" + os.path.splitdrive(os.getcwd())[0]
+        result = os.stat(fname)
+        self.assertEqual(result.st_mode, stat.S_IFBLK)
+
 
 class UtimeTests(unittest.TestCase):
     def setUp(self):
diff --git a/Misc/NEWS.d/next/Windows/2019-09-04-14-01-08.bpo-38030._USdtk.rst b/Misc/NEWS.d/next/Windows/2019-09-04-14-01-08.bpo-38030._USdtk.rst
new file mode 100644
index 000000000000..f1be8a1e1c86
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-04-14-01-08.bpo-38030._USdtk.rst
@@ -0,0 +1 @@
+Fixes :func:`os.stat` failing for block devices on Windows
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index b9e8c0d94c20..771c5616152c 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -1793,13 +1793,13 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result,
             case ERROR_INVALID_PARAMETER:
             case ERROR_INVALID_FUNCTION:
             case ERROR_NOT_SUPPORTED:
-                retval = -1;
+                /* Volumes and physical disks are block devices, e.g.
+                   \\.\C: and \\.\PhysicalDrive0. */
+                memset(result, 0, sizeof(*result));
+                result->st_mode = 0x6000; /* S_IFBLK */
                 goto cleanup;
             }
-            /* Volumes and physical disks are block devices, e.g.
-               \\.\C: and \\.\PhysicalDrive0. */
-            memset(result, 0, sizeof(*result));
-            result->st_mode = 0x6000; /* S_IFBLK */
+            retval = -1;
             goto cleanup;
         }
     }
@@ -1826,7 +1826,14 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result,
 
 cleanup:
     if (hFile != INVALID_HANDLE_VALUE) {
-        CloseHandle(hFile);
+        /* Preserve last error if we are failing */
+        error = retval ? GetLastError() : 0;
+        if (!CloseHandle(hFile)) {
+            retval = -1;
+        } else if (retval) {
+            /* Restore last error */
+            SetLastError(error);
+        }
     }
 
     return retval;


From webhook-mailer at python.org  Wed Sep  4 20:20:14 2019
From: webhook-mailer at python.org (Terry Jan Reedy)
Date: Thu, 05 Sep 2019 00:20:14 -0000
Subject: [Python-checkins] Fix idlelib.help comments (GH-15669)
Message-ID: <mailman.235.1567642815.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6cd9666ce93658ae91f07b396aa6932b362a61d3
commit: 6cd9666ce93658ae91f07b396aa6932b362a61d3
branch: master
author: Terry Jan Reedy <tjreedy at udel.edu>
committer: GitHub <noreply at github.com>
date: 2019-09-04T20:20:08-04:00
summary:

Fix idlelib.help comments (GH-15669)

files:
M Lib/idlelib/help.py

diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py
index ba2c6ec55481..9f63ea0d3990 100644
--- a/Lib/idlelib/help.py
+++ b/Lib/idlelib/help.py
@@ -50,21 +50,22 @@ class HelpParser(HTMLParser):
     """
     def __init__(self, text):
         HTMLParser.__init__(self, convert_charrefs=True)
-        self.text = text         # text widget we're rendering into
-        self.tags = ''           # current block level text tags to apply
-        self.chartags = ''       # current character level text tags
-        self.show = False        # used so we exclude page navigation
-        self.hdrlink = False     # used so we don't show header links
-        self.level = 0           # indentation level
-        self.pre = False         # displaying preformatted text
-        self.hprefix = ''        # prefix such as '25.5' to strip from headings
-        self.nested_dl = False   # if we're in a nested <dl>
-        self.simplelist = False  # simple list (no double spacing)
-        self.toc = []            # pair headers with text indexes for toc
-        self.header = ''         # text within header tags for toc
-        self.prevtag = None      # info about previous tag (was opener, tag)
+        self.text = text         # Text widget we're rendering into.
+        self.tags = ''           # Current block level text tags to apply.
+        self.chartags = ''       # Current character level text tags.
+        self.show = False        # Exclude html page navigation.
+        self.hdrlink = False     # Exclude html header links.
+        self.level = 0           # Track indentation level.
+        self.pre = False         # Displaying preformatted text?
+        self.hprefix = ''        # Heading prefix (like '25.5'?) to remove.
+        self.nested_dl = False   # In a nested <dl>?
+        self.simplelist = False  # In a simple list (no double spacing)?
+        self.toc = []            # Pair headers with text indexes for toc.
+        self.header = ''         # Text within header tags for toc.
+        self.prevtag = None      # Previous tag info (opener?, tag).
 
     def indent(self, amt=1):
+        "Change indent (+1, 0, -1) and tags."
         self.level += amt
         self.tags = '' if self.level == 0 else 'l'+str(self.level)
 
@@ -76,12 +77,12 @@ def handle_starttag(self, tag, attrs):
                 class_ = v
         s = ''
         if tag == 'div' and class_ == 'section':
-            self.show = True    # start of main content
+            self.show = True    # Start main content.
         elif tag == 'div' and class_ == 'sphinxsidebar':
-            self.show = False   # end of main content
+            self.show = False   # End main content.
         elif tag == 'p' and self.prevtag and not self.prevtag[0]:
-            # begin a new block for <p> tags after a closed tag
-            # avoid extra lines, e.g. after <pre> tags
+            # Begin a new block for <p> tags after a closed tag.
+            # Avoid extra lines, e.g. after <pre> tags.
             lastline = self.text.get('end-1c linestart', 'end-1c')
             s = '\n\n' if lastline and not lastline.isspace() else '\n'
         elif tag == 'span' and class_ == 'pre':
@@ -103,7 +104,7 @@ def handle_starttag(self, tag, attrs):
         elif tag == 'li':
             s = '\n* ' if self.simplelist else '\n\n* '
         elif tag == 'dt':
-            s = '\n\n' if not self.nested_dl else '\n'  # avoid extra line
+            s = '\n\n' if not self.nested_dl else '\n'  # Avoid extra line.
             self.nested_dl = False
         elif tag == 'dd':
             self.indent()
@@ -129,12 +130,13 @@ def handle_starttag(self, tag, attrs):
     def handle_endtag(self, tag):
         "Handle endtags in help.html."
         if tag in ['h1', 'h2', 'h3']:
-            self.indent(0)  # clear tag, reset indent
+            assert self.level == 0
             if self.show:
                 indent = ('        ' if tag == 'h3' else
                           '    ' if tag == 'h2' else
                           '')
                 self.toc.append((indent+self.header, self.text.index('insert')))
+            self.tags = ''
         elif tag in ['span', 'em']:
             self.chartags = ''
         elif tag == 'a':
@@ -143,7 +145,7 @@ def handle_endtag(self, tag):
             self.pre = False
             self.tags = ''
         elif tag in ['ul', 'dd', 'ol']:
-            self.indent(amt=-1)
+            self.indent(-1)
         self.prevtag = (False, tag)
 
     def handle_data(self, data):
@@ -169,7 +171,7 @@ def __init__(self, parent, filename):
         "Configure tags and feed file to parser."
         uwide = idleConf.GetOption('main', 'EditorWindow', 'width', type='int')
         uhigh = idleConf.GetOption('main', 'EditorWindow', 'height', type='int')
-        uhigh = 3 * uhigh // 4  # lines average 4/3 of editor line height
+        uhigh = 3 * uhigh // 4  # Lines average 4/3 of editor line height.
         Text.__init__(self, parent, wrap='word', highlightthickness=0,
                       padx=5, borderwidth=0, width=uwide, height=uhigh)
 
@@ -209,7 +211,6 @@ class HelpFrame(Frame):
     "Display html text, scrollbar, and toc."
     def __init__(self, parent, filename):
         Frame.__init__(self, parent)
-        # keep references to widgets for test access.
         self.text = text = HelpText(self, filename)
         self['background'] = text['background']
         self.toc = toc = self.toc_menu(text)
@@ -217,7 +218,7 @@ def __init__(self, parent, filename):
         text['yscrollcommand'] = scroll.set
 
         self.rowconfigure(0, weight=1)
-        self.columnconfigure(1, weight=1)  # text
+        self.columnconfigure(1, weight=1)  # Only expand the text widget.
         toc.grid(row=0, column=0, sticky='nw')
         text.grid(row=0, column=1, sticky='nsew')
         scroll.grid(row=0, column=2, sticky='ns')
@@ -279,7 +280,7 @@ def show_idlehelp(parent):
     "Create HelpWindow; called from Idle Help event handler."
     filename = join(abspath(dirname(__file__)), 'help.html')
     if not isfile(filename):
-        # try copy_strip, present message
+        # Try copy_strip, present message.
         return
     HelpWindow(parent, filename, 'IDLE Help (%s)' % python_version())
 


From webhook-mailer at python.org  Wed Sep  4 20:34:39 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 05 Sep 2019 00:34:39 -0000
Subject: [Python-checkins] bpo-22347: Update mimetypes.guess_type to allow
 proper parsing of URLs (GH-15522)
Message-ID: <mailman.236.1567643679.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/87bd2071c756188b6cd577889fb1682831142ceb
commit: 87bd2071c756188b6cd577889fb1682831142ceb
branch: master
author: Dong-hee Na <donghee.na92 at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-04T17:34:35-07:00
summary:

bpo-22347: Update mimetypes.guess_type to allow proper parsing of URLs (GH-15522)



https://bugs.python.org/issue22347

files:
A Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst
M Lib/mimetypes.py
M Lib/test/test_mimetypes.py
M Lib/test/test_urllib2.py

diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py
index 01a16fdf9aa1..f38005c9d295 100644
--- a/Lib/mimetypes.py
+++ b/Lib/mimetypes.py
@@ -114,7 +114,8 @@ def guess_type(self, url, strict=True):
         but non-standard types.
         """
         url = os.fspath(url)
-        scheme, url = urllib.parse._splittype(url)
+        p = urllib.parse.urlparse(url)
+        scheme, url = p.scheme, p.path
         if scheme == 'data':
             # syntax of data URLs:
             # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py
index bfd5eeedaa77..7761c3fe867a 100644
--- a/Lib/test/test_mimetypes.py
+++ b/Lib/test/test_mimetypes.py
@@ -51,6 +51,14 @@ def test_non_standard_types(self):
         eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None))
         eq(self.db.guess_extension('image/jpg', strict=False), '.jpg')
 
+    def test_url(self):
+        result = self.db.guess_type('http://host.html')
+        msg = 'URL only has a host name, not a file'
+        self.assertSequenceEqual(result, (None, None), msg)
+        result = self.db.guess_type('http://example.com/host.html')
+        msg = 'Should be text/html'
+        self.assertSequenceEqual(result, ('text/html', None), msg)
+
     def test_guess_all_types(self):
         eq = self.assertEqual
         unless = self.assertTrue
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
index 7b576db4e3aa..c228fa7d1dc1 100644
--- a/Lib/test/test_urllib2.py
+++ b/Lib/test/test_urllib2.py
@@ -742,7 +742,7 @@ def connect_ftp(self, user, passwd, host, port, dirs,
              ["foo", "bar"], "", None),
             ("ftp://localhost/baz.gif;type=a",
              "localhost", ftplib.FTP_PORT, "", "", "A",
-             [], "baz.gif", None),  # XXX really this should guess image/gif
+             [], "baz.gif", "image/gif"),
             ]:
             req = Request(url)
             req.timeout = None
diff --git a/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst b/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst
new file mode 100644
index 000000000000..1a3c19938217
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst
@@ -0,0 +1,2 @@
+Update mimetypes.guess_type to allow proper parsing of URLs with only a host name.
+Patch by Dong-hee Na.


From webhook-mailer at python.org  Wed Sep  4 20:39:38 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 05 Sep 2019 00:39:38 -0000
Subject: [Python-checkins] Fix idlelib.help comments (GH-15669)
Message-ID: <mailman.237.1567643979.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/29825a33926db63bb73601ba9e76ecd3c6cfa0f6
commit: 29825a33926db63bb73601ba9e76ecd3c6cfa0f6
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-04T17:39:34-07:00
summary:

Fix idlelib.help comments (GH-15669)

(cherry picked from commit 6cd9666ce93658ae91f07b396aa6932b362a61d3)

Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu>

files:
M Lib/idlelib/help.py

diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py
index ba2c6ec55481..9f63ea0d3990 100644
--- a/Lib/idlelib/help.py
+++ b/Lib/idlelib/help.py
@@ -50,21 +50,22 @@ class HelpParser(HTMLParser):
     """
     def __init__(self, text):
         HTMLParser.__init__(self, convert_charrefs=True)
-        self.text = text         # text widget we're rendering into
-        self.tags = ''           # current block level text tags to apply
-        self.chartags = ''       # current character level text tags
-        self.show = False        # used so we exclude page navigation
-        self.hdrlink = False     # used so we don't show header links
-        self.level = 0           # indentation level
-        self.pre = False         # displaying preformatted text
-        self.hprefix = ''        # prefix such as '25.5' to strip from headings
-        self.nested_dl = False   # if we're in a nested <dl>
-        self.simplelist = False  # simple list (no double spacing)
-        self.toc = []            # pair headers with text indexes for toc
-        self.header = ''         # text within header tags for toc
-        self.prevtag = None      # info about previous tag (was opener, tag)
+        self.text = text         # Text widget we're rendering into.
+        self.tags = ''           # Current block level text tags to apply.
+        self.chartags = ''       # Current character level text tags.
+        self.show = False        # Exclude html page navigation.
+        self.hdrlink = False     # Exclude html header links.
+        self.level = 0           # Track indentation level.
+        self.pre = False         # Displaying preformatted text?
+        self.hprefix = ''        # Heading prefix (like '25.5'?) to remove.
+        self.nested_dl = False   # In a nested <dl>?
+        self.simplelist = False  # In a simple list (no double spacing)?
+        self.toc = []            # Pair headers with text indexes for toc.
+        self.header = ''         # Text within header tags for toc.
+        self.prevtag = None      # Previous tag info (opener?, tag).
 
     def indent(self, amt=1):
+        "Change indent (+1, 0, -1) and tags."
         self.level += amt
         self.tags = '' if self.level == 0 else 'l'+str(self.level)
 
@@ -76,12 +77,12 @@ def handle_starttag(self, tag, attrs):
                 class_ = v
         s = ''
         if tag == 'div' and class_ == 'section':
-            self.show = True    # start of main content
+            self.show = True    # Start main content.
         elif tag == 'div' and class_ == 'sphinxsidebar':
-            self.show = False   # end of main content
+            self.show = False   # End main content.
         elif tag == 'p' and self.prevtag and not self.prevtag[0]:
-            # begin a new block for <p> tags after a closed tag
-            # avoid extra lines, e.g. after <pre> tags
+            # Begin a new block for <p> tags after a closed tag.
+            # Avoid extra lines, e.g. after <pre> tags.
             lastline = self.text.get('end-1c linestart', 'end-1c')
             s = '\n\n' if lastline and not lastline.isspace() else '\n'
         elif tag == 'span' and class_ == 'pre':
@@ -103,7 +104,7 @@ def handle_starttag(self, tag, attrs):
         elif tag == 'li':
             s = '\n* ' if self.simplelist else '\n\n* '
         elif tag == 'dt':
-            s = '\n\n' if not self.nested_dl else '\n'  # avoid extra line
+            s = '\n\n' if not self.nested_dl else '\n'  # Avoid extra line.
             self.nested_dl = False
         elif tag == 'dd':
             self.indent()
@@ -129,12 +130,13 @@ def handle_starttag(self, tag, attrs):
     def handle_endtag(self, tag):
         "Handle endtags in help.html."
         if tag in ['h1', 'h2', 'h3']:
-            self.indent(0)  # clear tag, reset indent
+            assert self.level == 0
             if self.show:
                 indent = ('        ' if tag == 'h3' else
                           '    ' if tag == 'h2' else
                           '')
                 self.toc.append((indent+self.header, self.text.index('insert')))
+            self.tags = ''
         elif tag in ['span', 'em']:
             self.chartags = ''
         elif tag == 'a':
@@ -143,7 +145,7 @@ def handle_endtag(self, tag):
             self.pre = False
             self.tags = ''
         elif tag in ['ul', 'dd', 'ol']:
-            self.indent(amt=-1)
+            self.indent(-1)
         self.prevtag = (False, tag)
 
     def handle_data(self, data):
@@ -169,7 +171,7 @@ def __init__(self, parent, filename):
         "Configure tags and feed file to parser."
         uwide = idleConf.GetOption('main', 'EditorWindow', 'width', type='int')
         uhigh = idleConf.GetOption('main', 'EditorWindow', 'height', type='int')
-        uhigh = 3 * uhigh // 4  # lines average 4/3 of editor line height
+        uhigh = 3 * uhigh // 4  # Lines average 4/3 of editor line height.
         Text.__init__(self, parent, wrap='word', highlightthickness=0,
                       padx=5, borderwidth=0, width=uwide, height=uhigh)
 
@@ -209,7 +211,6 @@ class HelpFrame(Frame):
     "Display html text, scrollbar, and toc."
     def __init__(self, parent, filename):
         Frame.__init__(self, parent)
-        # keep references to widgets for test access.
         self.text = text = HelpText(self, filename)
         self['background'] = text['background']
         self.toc = toc = self.toc_menu(text)
@@ -217,7 +218,7 @@ def __init__(self, parent, filename):
         text['yscrollcommand'] = scroll.set
 
         self.rowconfigure(0, weight=1)
-        self.columnconfigure(1, weight=1)  # text
+        self.columnconfigure(1, weight=1)  # Only expand the text widget.
         toc.grid(row=0, column=0, sticky='nw')
         text.grid(row=0, column=1, sticky='nsew')
         scroll.grid(row=0, column=2, sticky='ns')
@@ -279,7 +280,7 @@ def show_idlehelp(parent):
     "Create HelpWindow; called from Idle Help event handler."
     filename = join(abspath(dirname(__file__)), 'help.html')
     if not isfile(filename):
-        # try copy_strip, present message
+        # Try copy_strip, present message.
         return
     HelpWindow(parent, filename, 'IDLE Help (%s)' % python_version())
 


From webhook-mailer at python.org  Wed Sep  4 20:41:48 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 05 Sep 2019 00:41:48 -0000
Subject: [Python-checkins] Fix idlelib.help comments (GH-15669)
Message-ID: <mailman.238.1567644109.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b4f6e444b1d8a0c81e331701f2084cd5487502dc
commit: b4f6e444b1d8a0c81e331701f2084cd5487502dc
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-04T17:41:45-07:00
summary:

Fix idlelib.help comments (GH-15669)

(cherry picked from commit 6cd9666ce93658ae91f07b396aa6932b362a61d3)

Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu>

files:
M Lib/idlelib/help.py

diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py
index ba2c6ec55481..9f63ea0d3990 100644
--- a/Lib/idlelib/help.py
+++ b/Lib/idlelib/help.py
@@ -50,21 +50,22 @@ class HelpParser(HTMLParser):
     """
     def __init__(self, text):
         HTMLParser.__init__(self, convert_charrefs=True)
-        self.text = text         # text widget we're rendering into
-        self.tags = ''           # current block level text tags to apply
-        self.chartags = ''       # current character level text tags
-        self.show = False        # used so we exclude page navigation
-        self.hdrlink = False     # used so we don't show header links
-        self.level = 0           # indentation level
-        self.pre = False         # displaying preformatted text
-        self.hprefix = ''        # prefix such as '25.5' to strip from headings
-        self.nested_dl = False   # if we're in a nested <dl>
-        self.simplelist = False  # simple list (no double spacing)
-        self.toc = []            # pair headers with text indexes for toc
-        self.header = ''         # text within header tags for toc
-        self.prevtag = None      # info about previous tag (was opener, tag)
+        self.text = text         # Text widget we're rendering into.
+        self.tags = ''           # Current block level text tags to apply.
+        self.chartags = ''       # Current character level text tags.
+        self.show = False        # Exclude html page navigation.
+        self.hdrlink = False     # Exclude html header links.
+        self.level = 0           # Track indentation level.
+        self.pre = False         # Displaying preformatted text?
+        self.hprefix = ''        # Heading prefix (like '25.5'?) to remove.
+        self.nested_dl = False   # In a nested <dl>?
+        self.simplelist = False  # In a simple list (no double spacing)?
+        self.toc = []            # Pair headers with text indexes for toc.
+        self.header = ''         # Text within header tags for toc.
+        self.prevtag = None      # Previous tag info (opener?, tag).
 
     def indent(self, amt=1):
+        "Change indent (+1, 0, -1) and tags."
         self.level += amt
         self.tags = '' if self.level == 0 else 'l'+str(self.level)
 
@@ -76,12 +77,12 @@ def handle_starttag(self, tag, attrs):
                 class_ = v
         s = ''
         if tag == 'div' and class_ == 'section':
-            self.show = True    # start of main content
+            self.show = True    # Start main content.
         elif tag == 'div' and class_ == 'sphinxsidebar':
-            self.show = False   # end of main content
+            self.show = False   # End main content.
         elif tag == 'p' and self.prevtag and not self.prevtag[0]:
-            # begin a new block for <p> tags after a closed tag
-            # avoid extra lines, e.g. after <pre> tags
+            # Begin a new block for <p> tags after a closed tag.
+            # Avoid extra lines, e.g. after <pre> tags.
             lastline = self.text.get('end-1c linestart', 'end-1c')
             s = '\n\n' if lastline and not lastline.isspace() else '\n'
         elif tag == 'span' and class_ == 'pre':
@@ -103,7 +104,7 @@ def handle_starttag(self, tag, attrs):
         elif tag == 'li':
             s = '\n* ' if self.simplelist else '\n\n* '
         elif tag == 'dt':
-            s = '\n\n' if not self.nested_dl else '\n'  # avoid extra line
+            s = '\n\n' if not self.nested_dl else '\n'  # Avoid extra line.
             self.nested_dl = False
         elif tag == 'dd':
             self.indent()
@@ -129,12 +130,13 @@ def handle_starttag(self, tag, attrs):
     def handle_endtag(self, tag):
         "Handle endtags in help.html."
         if tag in ['h1', 'h2', 'h3']:
-            self.indent(0)  # clear tag, reset indent
+            assert self.level == 0
             if self.show:
                 indent = ('        ' if tag == 'h3' else
                           '    ' if tag == 'h2' else
                           '')
                 self.toc.append((indent+self.header, self.text.index('insert')))
+            self.tags = ''
         elif tag in ['span', 'em']:
             self.chartags = ''
         elif tag == 'a':
@@ -143,7 +145,7 @@ def handle_endtag(self, tag):
             self.pre = False
             self.tags = ''
         elif tag in ['ul', 'dd', 'ol']:
-            self.indent(amt=-1)
+            self.indent(-1)
         self.prevtag = (False, tag)
 
     def handle_data(self, data):
@@ -169,7 +171,7 @@ def __init__(self, parent, filename):
         "Configure tags and feed file to parser."
         uwide = idleConf.GetOption('main', 'EditorWindow', 'width', type='int')
         uhigh = idleConf.GetOption('main', 'EditorWindow', 'height', type='int')
-        uhigh = 3 * uhigh // 4  # lines average 4/3 of editor line height
+        uhigh = 3 * uhigh // 4  # Lines average 4/3 of editor line height.
         Text.__init__(self, parent, wrap='word', highlightthickness=0,
                       padx=5, borderwidth=0, width=uwide, height=uhigh)
 
@@ -209,7 +211,6 @@ class HelpFrame(Frame):
     "Display html text, scrollbar, and toc."
     def __init__(self, parent, filename):
         Frame.__init__(self, parent)
-        # keep references to widgets for test access.
         self.text = text = HelpText(self, filename)
         self['background'] = text['background']
         self.toc = toc = self.toc_menu(text)
@@ -217,7 +218,7 @@ def __init__(self, parent, filename):
         text['yscrollcommand'] = scroll.set
 
         self.rowconfigure(0, weight=1)
-        self.columnconfigure(1, weight=1)  # text
+        self.columnconfigure(1, weight=1)  # Only expand the text widget.
         toc.grid(row=0, column=0, sticky='nw')
         text.grid(row=0, column=1, sticky='nsew')
         scroll.grid(row=0, column=2, sticky='ns')
@@ -279,7 +280,7 @@ def show_idlehelp(parent):
     "Create HelpWindow; called from Idle Help event handler."
     filename = join(abspath(dirname(__file__)), 'help.html')
     if not isfile(filename):
-        # try copy_strip, present message
+        # Try copy_strip, present message.
         return
     HelpWindow(parent, filename, 'IDLE Help (%s)' % python_version())
 


From webhook-mailer at python.org  Wed Sep  4 20:55:02 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 05 Sep 2019 00:55:02 -0000
Subject: [Python-checkins] bpo-22347: Update mimetypes.guess_type to allow
 proper parsing of URLs (GH-15522)
Message-ID: <mailman.239.1567644904.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6d7a786d2e4b48a6b50614e042ace9ff996f0238
commit: 6d7a786d2e4b48a6b50614e042ace9ff996f0238
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-04T17:54:59-07:00
summary:

bpo-22347: Update mimetypes.guess_type to allow proper parsing of URLs (GH-15522)


https://bugs.python.org/issue22347
(cherry picked from commit 87bd2071c756188b6cd577889fb1682831142ceb)

Co-authored-by: Dong-hee Na <donghee.na92 at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst
M Lib/mimetypes.py
M Lib/test/test_mimetypes.py
M Lib/test/test_urllib2.py

diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py
index 01a16fdf9aa1..f38005c9d295 100644
--- a/Lib/mimetypes.py
+++ b/Lib/mimetypes.py
@@ -114,7 +114,8 @@ def guess_type(self, url, strict=True):
         but non-standard types.
         """
         url = os.fspath(url)
-        scheme, url = urllib.parse._splittype(url)
+        p = urllib.parse.urlparse(url)
+        scheme, url = p.scheme, p.path
         if scheme == 'data':
             # syntax of data URLs:
             # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py
index bfd5eeedaa77..7761c3fe867a 100644
--- a/Lib/test/test_mimetypes.py
+++ b/Lib/test/test_mimetypes.py
@@ -51,6 +51,14 @@ def test_non_standard_types(self):
         eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None))
         eq(self.db.guess_extension('image/jpg', strict=False), '.jpg')
 
+    def test_url(self):
+        result = self.db.guess_type('http://host.html')
+        msg = 'URL only has a host name, not a file'
+        self.assertSequenceEqual(result, (None, None), msg)
+        result = self.db.guess_type('http://example.com/host.html')
+        msg = 'Should be text/html'
+        self.assertSequenceEqual(result, ('text/html', None), msg)
+
     def test_guess_all_types(self):
         eq = self.assertEqual
         unless = self.assertTrue
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
index c6d275e16b09..810ee5e6ea28 100644
--- a/Lib/test/test_urllib2.py
+++ b/Lib/test/test_urllib2.py
@@ -744,7 +744,7 @@ def connect_ftp(self, user, passwd, host, port, dirs,
              ["foo", "bar"], "", None),
             ("ftp://localhost/baz.gif;type=a",
              "localhost", ftplib.FTP_PORT, "", "", "A",
-             [], "baz.gif", None),  # XXX really this should guess image/gif
+             [], "baz.gif", "image/gif"),
             ]:
             req = Request(url)
             req.timeout = None
diff --git a/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst b/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst
new file mode 100644
index 000000000000..1a3c19938217
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst
@@ -0,0 +1,2 @@
+Update mimetypes.guess_type to allow proper parsing of URLs with only a host name.
+Patch by Dong-hee Na.


From webhook-mailer at python.org  Wed Sep  4 21:16:42 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 05 Sep 2019 01:16:42 -0000
Subject: [Python-checkins] [3.7] bpo-22347: Update mimetypes.guess_type to
 allow proper parsing of URLs (GH-15522) (GH-15687)
Message-ID: <mailman.240.1567646202.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8873bff2871078e9f23e6c7d942d3a8edbd0921f
commit: 8873bff2871078e9f23e6c7d942d3a8edbd0921f
branch: 3.7
author: Dong-hee Na <donghee.na92 at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-04T18:16:39-07:00
summary:

[3.7] bpo-22347: Update mimetypes.guess_type to allow proper parsing of URLs (GH-15522) (GH-15687)



https://bugs.python.org/issue22347
(cherry picked from commit 87bd2071c756188b6cd577889fb1682831142ceb)


https://bugs.python.org/issue22347



Automerge-Triggered-By: @maxking

files:
A Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst
M Lib/mimetypes.py
M Lib/test/test_mimetypes.py
M Lib/test/test_urllib2.py

diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py
index bcf522835f4c..b5e1ce1ca104 100644
--- a/Lib/mimetypes.py
+++ b/Lib/mimetypes.py
@@ -114,7 +114,8 @@ def guess_type(self, url, strict=True):
         but non-standard types.
         """
         url = os.fspath(url)
-        scheme, url = urllib.parse.splittype(url)
+        p = urllib.parse.urlparse(url)
+        scheme, url = p.scheme, p.path
         if scheme == 'data':
             # syntax of data URLs:
             # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py
index adbec8dab426..2f26edeb9903 100644
--- a/Lib/test/test_mimetypes.py
+++ b/Lib/test/test_mimetypes.py
@@ -50,6 +50,14 @@ def test_non_standard_types(self):
         eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None))
         eq(self.db.guess_extension('image/jpg', strict=False), '.jpg')
 
+    def test_url(self):
+        result = self.db.guess_type('http://host.html')
+        msg = 'URL only has a host name, not a file'
+        self.assertSequenceEqual(result, (None, None), msg)
+        result = self.db.guess_type('http://example.com/host.html')
+        msg = 'Should be text/html'
+        self.assertSequenceEqual(result, ('text/html', None), msg)
+
     def test_guess_all_types(self):
         eq = self.assertEqual
         unless = self.assertTrue
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
index 876fcd4199fb..0677390c2b0b 100644
--- a/Lib/test/test_urllib2.py
+++ b/Lib/test/test_urllib2.py
@@ -746,7 +746,7 @@ def connect_ftp(self, user, passwd, host, port, dirs,
              ["foo", "bar"], "", None),
             ("ftp://localhost/baz.gif;type=a",
              "localhost", ftplib.FTP_PORT, "", "", "A",
-             [], "baz.gif", None),  # XXX really this should guess image/gif
+             [], "baz.gif", "image/gif"),
             ]:
             req = Request(url)
             req.timeout = None
diff --git a/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst b/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst
new file mode 100644
index 000000000000..1a3c19938217
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst
@@ -0,0 +1,2 @@
+Update mimetypes.guess_type to allow proper parsing of URLs with only a host name.
+Patch by Dong-hee Na.


From webhook-mailer at python.org  Wed Sep  4 21:20:45 2019
From: webhook-mailer at python.org (Abhilash Raj)
Date: Thu, 05 Sep 2019 01:20:45 -0000
Subject: [Python-checkins] [3.8] bpo-37764: Fix infinite loop when parsing
 unstructured email headers. (GH-15239) (GH-15686)
Message-ID: <mailman.241.1567646446.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6ad0a2c45f78020f7994e47620c1cf7b225f8197
commit: 6ad0a2c45f78020f7994e47620c1cf7b225f8197
branch: 3.8
author: Abhilash Raj <maxking at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-04T18:20:40-07:00
summary:

[3.8] bpo-37764: Fix infinite loop when parsing unstructured email headers. (GH-15239) (GH-15686)

Fixes a case in which email._header_value_parser.get_unstructured hangs the system for some invalid headers. This covers the cases in which the header contains either:
- a case without trailing whitespace
- an invalid encoded word

https://bugs.python.org/issue37764

This fix should also be backported to 3.7 and 3.8

https://bugs.python.org/issue37764
(cherry picked from commit c5b242f87f31286ad38991bc3868cf4cfbf2b681)

Co-authored-by: Ashwin Ramaswami <aramaswamis at gmail.com>

files:
A Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst
M Lib/email/_header_value_parser.py
M Lib/test/test_email/test__header_value_parser.py
M Lib/test/test_email/test_email.py
M Misc/ACKS

diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index b5003943ab0d..16c19907d68d 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -935,6 +935,10 @@ def __str__(self):
         return ''
 
 
+class _InvalidEwError(errors.HeaderParseError):
+    """Invalid encoded word found while parsing headers."""
+
+
 # XXX these need to become classes and used as instances so
 # that a program can't change them in a parse tree and screw
 # up other parse trees.  Maybe should have  tests for that, too.
@@ -1039,7 +1043,10 @@ def get_encoded_word(value):
         raise errors.HeaderParseError(
             "expected encoded word but found {}".format(value))
     remstr = ''.join(remainder)
-    if len(remstr) > 1 and remstr[0] in hexdigits and remstr[1] in hexdigits:
+    if (len(remstr) > 1 and
+        remstr[0] in hexdigits and
+        remstr[1] in hexdigits and
+        tok.count('?') < 2):
         # The ? after the CTE was followed by an encoded word escape (=XX).
         rest, *remainder = remstr.split('?=', 1)
         tok = tok + '?=' + rest
@@ -1051,7 +1058,7 @@ def get_encoded_word(value):
     try:
         text, charset, lang, defects = _ew.decode('=?' + tok + '?=')
     except ValueError:
-        raise errors.HeaderParseError(
+        raise _InvalidEwError(
             "encoded word format invalid: '{}'".format(ew.cte))
     ew.charset = charset
     ew.lang = lang
@@ -1101,9 +1108,12 @@ def get_unstructured(value):
             token, value = get_fws(value)
             unstructured.append(token)
             continue
+        valid_ew = True
         if value.startswith('=?'):
             try:
                 token, value = get_encoded_word(value)
+            except _InvalidEwError:
+                valid_ew = False
             except errors.HeaderParseError:
                 # XXX: Need to figure out how to register defects when
                 # appropriate here.
@@ -1125,7 +1135,10 @@ def get_unstructured(value):
         # Split in the middle of an atom if there is a rfc2047 encoded word
         # which does not have WSP on both sides. The defect will be registered
         # the next time through the loop.
-        if rfc2047_matcher.search(tok):
+        # This needs to only be performed when the encoded word is valid;
+        # otherwise, performing it on an invalid encoded word can cause
+        # the parser to go in an infinite loop.
+        if valid_ew and rfc2047_matcher.search(tok):
             tok, *remainder = value.partition('=?')
         vtext = ValueTerminal(tok, 'vtext')
         _validate_xtext(vtext)
diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py
index bad4333dbc43..dd33b065c804 100644
--- a/Lib/test/test_email/test__header_value_parser.py
+++ b/Lib/test/test_email/test__header_value_parser.py
@@ -383,6 +383,22 @@ def test_get_unstructured_ew_without_trailing_whitespace(self):
             [errors.InvalidHeaderDefect],
             '')
 
+    def test_get_unstructured_without_trailing_whitespace_hang_case(self):
+        self._test_get_x(self._get_unst,
+            '=?utf-8?q?somevalue?=aa',
+            'somevalueaa',
+            'somevalueaa',
+            [errors.InvalidHeaderDefect],
+            '')
+
+    def test_get_unstructured_invalid_ew(self):
+        self._test_get_x(self._get_unst,
+            '=?utf-8?q?=somevalue?=',
+            '=?utf-8?q?=somevalue?=',
+            '=?utf-8?q?=somevalue?=',
+            [],
+            '')
+
     # get_qp_ctext
 
     def test_get_qp_ctext_only(self):
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index aa775881c552..5414cf070cc1 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -5381,6 +5381,27 @@ def test_rfc2231_unencoded_then_encoded_segments(self):
         eq(language, 'en-us')
         eq(s, 'My Document For You')
 
+    def test_should_not_hang_on_invalid_ew_messages(self):
+        messages = ["""From: user at host.com
+To: user at host.com
+Bad-Header:
+ =?us-ascii?Q?LCSwrV11+IB0rSbSker+M9vWR7wEDSuGqmHD89Gt=ea0nJFSaiz4vX3XMJPT4vrE?=
+ =?us-ascii?Q?xGUZeOnp0o22pLBB7CYLH74Js=wOlK6Tfru2U47qR?=
+ =?us-ascii?Q?72OfyEY2p2=2FrA9xNFyvH+fBTCmazxwzF8nGkK6D?=
+
+Hello!
+""", """From: ??????????????? ???????????????????????? <xxx at xxx>
+To: "xxx" <xxx at xxx>
+Subject:   ????????? ?????????????????????????????? ??????????????? ??????????????? ??? ??????????????????????????? ?????? ????????????
+MIME-Version: 1.0
+Content-Type: text/plain; charset="windows-1251";
+Content-Transfer-Encoding: 8bit
+
+?????? ??????????????? ??? ???????????? ?????????????????? ????????? ????????????????????????
+"""]
+        for m in messages:
+            with self.subTest(m=m):
+                msg = email.message_from_string(m)
 
 
 # Tests to ensure that signed parts of an email are completely preserved, as
diff --git a/Misc/ACKS b/Misc/ACKS
index 24e327a5f86e..def874b0071e 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1330,6 +1330,7 @@ Burton Radons
 Abhilash Raj
 Shorya Raj
 Dhushyanth Ramasamy
+Ashwin Ramaswami
 Jeff Ramnani
 Bayard Randel
 Varpu Rantala
diff --git a/Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst b/Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst
new file mode 100644
index 000000000000..27fa8e192f0c
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst
@@ -0,0 +1 @@
+Fixes email._header_value_parser.get_unstructured going into an infinite loop for a specific case in which the email header does not have trailing whitespace, and the case in which it contains an invalid encoded word. Patch by Ashwin Ramaswami.
\ No newline at end of file


From webhook-mailer at python.org  Wed Sep  4 21:33:37 2019
From: webhook-mailer at python.org (Terry Jan Reedy)
Date: Thu, 05 Sep 2019 01:33:37 -0000
Subject: [Python-checkins] bpo-37902: IDLE: Add scrolling for IDLE browsers.
 (#15368)
Message-ID: <mailman.242.1567647219.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2cd902585815582eb059e3b40e014ebe4e7fdee7
commit: 2cd902585815582eb059e3b40e014ebe4e7fdee7
branch: master
author: GeeTransit <geetransit at gmail.com>
committer: Terry Jan Reedy <tjreedy at udel.edu>
date: 2019-09-04T21:33:33-04:00
summary:

bpo-37902: IDLE: Add scrolling for IDLE browsers. (#15368)

Modify the wheel event handler so it can also be used for module, path, and stack browsers.
Patch by George Zhang.

files:
A Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/editor.py
M Lib/idlelib/idle_test/test_multicall.py
M Lib/idlelib/idle_test/test_tree.py
M Lib/idlelib/tree.py
M Misc/ACKS

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 47c2291d2377..c9e846a6fba6 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,9 @@ Released on 2019-10-20?
 ======================================
 
 
+bpo-37092: Add mousewheel scrolling for IDLE module, path, and stack
+browsers.  Patch by George Zhang.
+
 bpo-35771: To avoid occasional spurious test_idle failures on slower
 machines, increase the ``hover_delay`` in test_tooltip.
 
diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py
index 793ed3afaed0..5cbf704ab27b 100644
--- a/Lib/idlelib/editor.py
+++ b/Lib/idlelib/editor.py
@@ -26,6 +26,7 @@
 from idlelib import query
 from idlelib import replace
 from idlelib import search
+from idlelib.tree import wheel_event
 from idlelib import window
 
 # The default tab setting for a Text widget, in average-width characters.
@@ -151,9 +152,10 @@ def __init__(self, flist=None, filename=None, key=None, root=None):
         else:
             # Elsewhere, use right-click for popup menus.
             text.bind("<3>",self.right_menu_event)
-        text.bind('<MouseWheel>', self.mousescroll)
-        text.bind('<Button-4>', self.mousescroll)
-        text.bind('<Button-5>', self.mousescroll)
+
+        text.bind('<MouseWheel>', wheel_event)
+        text.bind('<Button-4>', wheel_event)
+        text.bind('<Button-5>', wheel_event)
         text.bind('<Configure>', self.handle_winconfig)
         text.bind("<<cut>>", self.cut)
         text.bind("<<copy>>", self.copy)
@@ -502,23 +504,6 @@ def handle_yview(self, event, *args):
         self.text.yview(event, *args)
         return 'break'
 
-    def mousescroll(self, event):
-        """Handle scrollwheel event.
-
-        For wheel up, event.delta = 120*n on Windows, -1*n on darwin,
-        where n can be > 1 if one scrolls fast.  Flicking the wheel
-        generates up to maybe 20 events with n up to 10 or more 1.
-        Macs use wheel down (delta = 1*n) to scroll up, so positive
-        delta means to scroll up on both systems.
-
-        X-11 sends Control-Button-4 event instead.
-        """
-        up = {EventType.MouseWheel: event.delta > 0,
-              EventType.Button: event.num == 4}
-        lines = -5 if up[event.type] else 5
-        self.text.yview_scroll(lines, 'units')
-        return 'break'
-
     rmenu = None
 
     def right_menu_event(self, event):
diff --git a/Lib/idlelib/idle_test/test_multicall.py b/Lib/idlelib/idle_test/test_multicall.py
index 68156a743d7b..ba582bb3ca51 100644
--- a/Lib/idlelib/idle_test/test_multicall.py
+++ b/Lib/idlelib/idle_test/test_multicall.py
@@ -35,6 +35,14 @@ def test_init(self):
         mctext = self.mc(self.root)
         self.assertIsInstance(mctext._MultiCall__binders, list)
 
+    def test_yview(self):
+        # Added for tree.wheel_event
+        # (it depends on yview to not be overriden)
+        mc = self.mc
+        self.assertIs(mc.yview, Text.yview)
+        mctext = self.mc(self.root)
+        self.assertIs(mctext.yview.__func__, Text.yview)
+
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/idle_test/test_tree.py b/Lib/idlelib/idle_test/test_tree.py
index 9be9abee361f..b3e4c10cf9e3 100644
--- a/Lib/idlelib/idle_test/test_tree.py
+++ b/Lib/idlelib/idle_test/test_tree.py
@@ -4,7 +4,7 @@
 import unittest
 from test.support import requires
 requires('gui')
-from tkinter import Tk
+from tkinter import Tk, EventType, SCROLL
 
 
 class TreeTest(unittest.TestCase):
@@ -29,5 +29,32 @@ def test_init(self):
         node.expand()
 
 
+class TestScrollEvent(unittest.TestCase):
+
+    def test_wheel_event(self):
+        # Fake widget class containing `yview` only.
+        class _Widget:
+            def __init__(widget, *expected):
+                widget.expected = expected
+            def yview(widget, *args):
+                self.assertTupleEqual(widget.expected, args)
+        # Fake event class
+        class _Event:
+            pass
+        #        (type, delta, num, amount)
+        tests = ((EventType.MouseWheel, 120, -1, -5),
+                 (EventType.MouseWheel, -120, -1, 5),
+                 (EventType.ButtonPress, -1, 4, -5),
+                 (EventType.ButtonPress, -1, 5, 5))
+
+        event = _Event()
+        for ty, delta, num, amount in tests:
+            event.type = ty
+            event.delta = delta
+            event.num = num
+            res = tree.wheel_event(event, _Widget(SCROLL, amount, "units"))
+            self.assertEqual(res, "break")
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py
index 21426cbb33e0..6229be4e5a8a 100644
--- a/Lib/idlelib/tree.py
+++ b/Lib/idlelib/tree.py
@@ -56,6 +56,30 @@ def listicons(icondir=ICONDIR):
             column = 0
     root.images = images
 
+def wheel_event(event, widget=None):
+    """Handle scrollwheel event.
+
+    For wheel up, event.delta = 120*n on Windows, -1*n on darwin,
+    where n can be > 1 if one scrolls fast.  Flicking the wheel
+    generates up to maybe 20 events with n up to 10 or more 1.
+    Macs use wheel down (delta = 1*n) to scroll up, so positive
+    delta means to scroll up on both systems.
+
+    X-11 sends Control-Button-4,5 events instead.
+
+    The widget parameter is needed so browser label bindings can pass
+    the underlying canvas.
+
+    This function depends on widget.yview to not be overridden by
+    a subclass.
+    """
+    up = {EventType.MouseWheel: event.delta > 0,
+          EventType.ButtonPress: event.num == 4}
+    lines = -5 if up[event.type] else 5
+    widget = event.widget if widget is None else widget
+    widget.yview(SCROLL, lines, 'units')
+    return 'break'
+
 
 class TreeNode:
 
@@ -260,6 +284,9 @@ def drawtext(self):
                                        anchor="nw", window=self.label)
         self.label.bind("<1>", self.select_or_edit)
         self.label.bind("<Double-1>", self.flip)
+        self.label.bind("<MouseWheel>", lambda e: wheel_event(e, self.canvas))
+        self.label.bind("<Button-4>", lambda e: wheel_event(e, self.canvas))
+        self.label.bind("<Button-5>", lambda e: wheel_event(e, self.canvas))
         self.text_id = id
 
     def select_or_edit(self, event=None):
@@ -410,6 +437,7 @@ def GetSubList(self):
 # A canvas widget with scroll bars and some useful bindings
 
 class ScrolledCanvas:
+
     def __init__(self, master, **opts):
         if 'yscrollincrement' not in opts:
             opts['yscrollincrement'] = 17
@@ -431,6 +459,9 @@ def __init__(self, master, **opts):
         self.canvas.bind("<Key-Next>", self.page_down)
         self.canvas.bind("<Key-Up>", self.unit_up)
         self.canvas.bind("<Key-Down>", self.unit_down)
+        self.canvas.bind("<MouseWheel>", wheel_event)
+        self.canvas.bind("<Button-4>", wheel_event)
+        self.canvas.bind("<Button-5>", wheel_event)
         #if isinstance(master, Toplevel) or isinstance(master, Tk):
         self.canvas.bind("<Alt-Key-2>", self.zoom_height)
         self.canvas.focus_set()
diff --git a/Misc/ACKS b/Misc/ACKS
index ce8b144900eb..ce427b8222ed 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1871,6 +1871,7 @@ Nickolai Zeldovich
 Yuxiao Zeng
 Uwe Zessin
 Cheng Zhang
+George Zhang
 Kai Zhu
 Tarek Ziad?
 Jelle Zijlstra
diff --git a/Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst b/Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst
new file mode 100644
index 000000000000..24b414248469
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst
@@ -0,0 +1,2 @@
+Add mousewheel scrolling for IDLE module, path, and stack browsers.
+Patch by George Zhang.


From webhook-mailer at python.org  Wed Sep  4 21:53:51 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 05 Sep 2019 01:53:51 -0000
Subject: [Python-checkins] bpo-37902: IDLE: Add scrolling for IDLE browsers.
 (GH-15368)
Message-ID: <mailman.243.1567648432.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9c2654d1aa85968fede1b888fba86aebc06c5be6
commit: 9c2654d1aa85968fede1b888fba86aebc06c5be6
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-04T18:53:47-07:00
summary:

bpo-37902: IDLE: Add scrolling for IDLE browsers. (GH-15368)


Modify the wheel event handler so it can also be used for module, path, and stack browsers.
Patch by George Zhang.
(cherry picked from commit 2cd902585815582eb059e3b40e014ebe4e7fdee7)

Co-authored-by: GeeTransit <geetransit at gmail.com>

files:
A Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/editor.py
M Lib/idlelib/idle_test/test_multicall.py
M Lib/idlelib/idle_test/test_tree.py
M Lib/idlelib/tree.py
M Misc/ACKS

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 47c2291d2377..c9e846a6fba6 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,9 @@ Released on 2019-10-20?
 ======================================
 
 
+bpo-37092: Add mousewheel scrolling for IDLE module, path, and stack
+browsers.  Patch by George Zhang.
+
 bpo-35771: To avoid occasional spurious test_idle failures on slower
 machines, increase the ``hover_delay`` in test_tooltip.
 
diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py
index 793ed3afaed0..5cbf704ab27b 100644
--- a/Lib/idlelib/editor.py
+++ b/Lib/idlelib/editor.py
@@ -26,6 +26,7 @@
 from idlelib import query
 from idlelib import replace
 from idlelib import search
+from idlelib.tree import wheel_event
 from idlelib import window
 
 # The default tab setting for a Text widget, in average-width characters.
@@ -151,9 +152,10 @@ def __init__(self, flist=None, filename=None, key=None, root=None):
         else:
             # Elsewhere, use right-click for popup menus.
             text.bind("<3>",self.right_menu_event)
-        text.bind('<MouseWheel>', self.mousescroll)
-        text.bind('<Button-4>', self.mousescroll)
-        text.bind('<Button-5>', self.mousescroll)
+
+        text.bind('<MouseWheel>', wheel_event)
+        text.bind('<Button-4>', wheel_event)
+        text.bind('<Button-5>', wheel_event)
         text.bind('<Configure>', self.handle_winconfig)
         text.bind("<<cut>>", self.cut)
         text.bind("<<copy>>", self.copy)
@@ -502,23 +504,6 @@ def handle_yview(self, event, *args):
         self.text.yview(event, *args)
         return 'break'
 
-    def mousescroll(self, event):
-        """Handle scrollwheel event.
-
-        For wheel up, event.delta = 120*n on Windows, -1*n on darwin,
-        where n can be > 1 if one scrolls fast.  Flicking the wheel
-        generates up to maybe 20 events with n up to 10 or more 1.
-        Macs use wheel down (delta = 1*n) to scroll up, so positive
-        delta means to scroll up on both systems.
-
-        X-11 sends Control-Button-4 event instead.
-        """
-        up = {EventType.MouseWheel: event.delta > 0,
-              EventType.Button: event.num == 4}
-        lines = -5 if up[event.type] else 5
-        self.text.yview_scroll(lines, 'units')
-        return 'break'
-
     rmenu = None
 
     def right_menu_event(self, event):
diff --git a/Lib/idlelib/idle_test/test_multicall.py b/Lib/idlelib/idle_test/test_multicall.py
index 68156a743d7b..ba582bb3ca51 100644
--- a/Lib/idlelib/idle_test/test_multicall.py
+++ b/Lib/idlelib/idle_test/test_multicall.py
@@ -35,6 +35,14 @@ def test_init(self):
         mctext = self.mc(self.root)
         self.assertIsInstance(mctext._MultiCall__binders, list)
 
+    def test_yview(self):
+        # Added for tree.wheel_event
+        # (it depends on yview to not be overriden)
+        mc = self.mc
+        self.assertIs(mc.yview, Text.yview)
+        mctext = self.mc(self.root)
+        self.assertIs(mctext.yview.__func__, Text.yview)
+
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/idle_test/test_tree.py b/Lib/idlelib/idle_test/test_tree.py
index 9be9abee361f..b3e4c10cf9e3 100644
--- a/Lib/idlelib/idle_test/test_tree.py
+++ b/Lib/idlelib/idle_test/test_tree.py
@@ -4,7 +4,7 @@
 import unittest
 from test.support import requires
 requires('gui')
-from tkinter import Tk
+from tkinter import Tk, EventType, SCROLL
 
 
 class TreeTest(unittest.TestCase):
@@ -29,5 +29,32 @@ def test_init(self):
         node.expand()
 
 
+class TestScrollEvent(unittest.TestCase):
+
+    def test_wheel_event(self):
+        # Fake widget class containing `yview` only.
+        class _Widget:
+            def __init__(widget, *expected):
+                widget.expected = expected
+            def yview(widget, *args):
+                self.assertTupleEqual(widget.expected, args)
+        # Fake event class
+        class _Event:
+            pass
+        #        (type, delta, num, amount)
+        tests = ((EventType.MouseWheel, 120, -1, -5),
+                 (EventType.MouseWheel, -120, -1, 5),
+                 (EventType.ButtonPress, -1, 4, -5),
+                 (EventType.ButtonPress, -1, 5, 5))
+
+        event = _Event()
+        for ty, delta, num, amount in tests:
+            event.type = ty
+            event.delta = delta
+            event.num = num
+            res = tree.wheel_event(event, _Widget(SCROLL, amount, "units"))
+            self.assertEqual(res, "break")
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py
index 21426cbb33e0..6229be4e5a8a 100644
--- a/Lib/idlelib/tree.py
+++ b/Lib/idlelib/tree.py
@@ -56,6 +56,30 @@ def listicons(icondir=ICONDIR):
             column = 0
     root.images = images
 
+def wheel_event(event, widget=None):
+    """Handle scrollwheel event.
+
+    For wheel up, event.delta = 120*n on Windows, -1*n on darwin,
+    where n can be > 1 if one scrolls fast.  Flicking the wheel
+    generates up to maybe 20 events with n up to 10 or more 1.
+    Macs use wheel down (delta = 1*n) to scroll up, so positive
+    delta means to scroll up on both systems.
+
+    X-11 sends Control-Button-4,5 events instead.
+
+    The widget parameter is needed so browser label bindings can pass
+    the underlying canvas.
+
+    This function depends on widget.yview to not be overridden by
+    a subclass.
+    """
+    up = {EventType.MouseWheel: event.delta > 0,
+          EventType.ButtonPress: event.num == 4}
+    lines = -5 if up[event.type] else 5
+    widget = event.widget if widget is None else widget
+    widget.yview(SCROLL, lines, 'units')
+    return 'break'
+
 
 class TreeNode:
 
@@ -260,6 +284,9 @@ def drawtext(self):
                                        anchor="nw", window=self.label)
         self.label.bind("<1>", self.select_or_edit)
         self.label.bind("<Double-1>", self.flip)
+        self.label.bind("<MouseWheel>", lambda e: wheel_event(e, self.canvas))
+        self.label.bind("<Button-4>", lambda e: wheel_event(e, self.canvas))
+        self.label.bind("<Button-5>", lambda e: wheel_event(e, self.canvas))
         self.text_id = id
 
     def select_or_edit(self, event=None):
@@ -410,6 +437,7 @@ def GetSubList(self):
 # A canvas widget with scroll bars and some useful bindings
 
 class ScrolledCanvas:
+
     def __init__(self, master, **opts):
         if 'yscrollincrement' not in opts:
             opts['yscrollincrement'] = 17
@@ -431,6 +459,9 @@ def __init__(self, master, **opts):
         self.canvas.bind("<Key-Next>", self.page_down)
         self.canvas.bind("<Key-Up>", self.unit_up)
         self.canvas.bind("<Key-Down>", self.unit_down)
+        self.canvas.bind("<MouseWheel>", wheel_event)
+        self.canvas.bind("<Button-4>", wheel_event)
+        self.canvas.bind("<Button-5>", wheel_event)
         #if isinstance(master, Toplevel) or isinstance(master, Tk):
         self.canvas.bind("<Alt-Key-2>", self.zoom_height)
         self.canvas.focus_set()
diff --git a/Misc/ACKS b/Misc/ACKS
index def874b0071e..290f8ea33269 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1863,6 +1863,7 @@ Nickolai Zeldovich
 Yuxiao Zeng
 Uwe Zessin
 Cheng Zhang
+George Zhang
 Kai Zhu
 Tarek Ziad?
 Jelle Zijlstra
diff --git a/Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst b/Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst
new file mode 100644
index 000000000000..24b414248469
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst
@@ -0,0 +1,2 @@
+Add mousewheel scrolling for IDLE module, path, and stack browsers.
+Patch by George Zhang.


From webhook-mailer at python.org  Wed Sep  4 21:58:26 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 05 Sep 2019 01:58:26 -0000
Subject: [Python-checkins] bpo-37902: IDLE: Add scrolling for IDLE browsers.
 (GH-15368)
Message-ID: <mailman.244.1567648708.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/16af39aa84cc3553c51d57461964ab4e28029184
commit: 16af39aa84cc3553c51d57461964ab4e28029184
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-04T18:58:22-07:00
summary:

bpo-37902: IDLE: Add scrolling for IDLE browsers. (GH-15368)


Modify the wheel event handler so it can also be used for module, path, and stack browsers.
Patch by George Zhang.
(cherry picked from commit 2cd902585815582eb059e3b40e014ebe4e7fdee7)

Co-authored-by: GeeTransit <geetransit at gmail.com>

files:
A Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/editor.py
M Lib/idlelib/idle_test/test_multicall.py
M Lib/idlelib/idle_test/test_tree.py
M Lib/idlelib/tree.py
M Misc/ACKS

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index b368ea9a3375..372c7ef5a9bc 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,9 @@ Released on 2019-09-30?
 ======================================
 
 
+bpo-37092: Add mousewheel scrolling for IDLE module, path, and stack
+browsers.  Patch by George Zhang.
+
 bpo-35771: To avoid occasional spurious test_idle failures on slower
 machines, increase the ``hover_delay`` in test_tooltip.
 
diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py
index 793ed3afaed0..5cbf704ab27b 100644
--- a/Lib/idlelib/editor.py
+++ b/Lib/idlelib/editor.py
@@ -26,6 +26,7 @@
 from idlelib import query
 from idlelib import replace
 from idlelib import search
+from idlelib.tree import wheel_event
 from idlelib import window
 
 # The default tab setting for a Text widget, in average-width characters.
@@ -151,9 +152,10 @@ def __init__(self, flist=None, filename=None, key=None, root=None):
         else:
             # Elsewhere, use right-click for popup menus.
             text.bind("<3>",self.right_menu_event)
-        text.bind('<MouseWheel>', self.mousescroll)
-        text.bind('<Button-4>', self.mousescroll)
-        text.bind('<Button-5>', self.mousescroll)
+
+        text.bind('<MouseWheel>', wheel_event)
+        text.bind('<Button-4>', wheel_event)
+        text.bind('<Button-5>', wheel_event)
         text.bind('<Configure>', self.handle_winconfig)
         text.bind("<<cut>>", self.cut)
         text.bind("<<copy>>", self.copy)
@@ -502,23 +504,6 @@ def handle_yview(self, event, *args):
         self.text.yview(event, *args)
         return 'break'
 
-    def mousescroll(self, event):
-        """Handle scrollwheel event.
-
-        For wheel up, event.delta = 120*n on Windows, -1*n on darwin,
-        where n can be > 1 if one scrolls fast.  Flicking the wheel
-        generates up to maybe 20 events with n up to 10 or more 1.
-        Macs use wheel down (delta = 1*n) to scroll up, so positive
-        delta means to scroll up on both systems.
-
-        X-11 sends Control-Button-4 event instead.
-        """
-        up = {EventType.MouseWheel: event.delta > 0,
-              EventType.Button: event.num == 4}
-        lines = -5 if up[event.type] else 5
-        self.text.yview_scroll(lines, 'units')
-        return 'break'
-
     rmenu = None
 
     def right_menu_event(self, event):
diff --git a/Lib/idlelib/idle_test/test_multicall.py b/Lib/idlelib/idle_test/test_multicall.py
index 68156a743d7b..ba582bb3ca51 100644
--- a/Lib/idlelib/idle_test/test_multicall.py
+++ b/Lib/idlelib/idle_test/test_multicall.py
@@ -35,6 +35,14 @@ def test_init(self):
         mctext = self.mc(self.root)
         self.assertIsInstance(mctext._MultiCall__binders, list)
 
+    def test_yview(self):
+        # Added for tree.wheel_event
+        # (it depends on yview to not be overriden)
+        mc = self.mc
+        self.assertIs(mc.yview, Text.yview)
+        mctext = self.mc(self.root)
+        self.assertIs(mctext.yview.__func__, Text.yview)
+
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/idle_test/test_tree.py b/Lib/idlelib/idle_test/test_tree.py
index 9be9abee361f..b3e4c10cf9e3 100644
--- a/Lib/idlelib/idle_test/test_tree.py
+++ b/Lib/idlelib/idle_test/test_tree.py
@@ -4,7 +4,7 @@
 import unittest
 from test.support import requires
 requires('gui')
-from tkinter import Tk
+from tkinter import Tk, EventType, SCROLL
 
 
 class TreeTest(unittest.TestCase):
@@ -29,5 +29,32 @@ def test_init(self):
         node.expand()
 
 
+class TestScrollEvent(unittest.TestCase):
+
+    def test_wheel_event(self):
+        # Fake widget class containing `yview` only.
+        class _Widget:
+            def __init__(widget, *expected):
+                widget.expected = expected
+            def yview(widget, *args):
+                self.assertTupleEqual(widget.expected, args)
+        # Fake event class
+        class _Event:
+            pass
+        #        (type, delta, num, amount)
+        tests = ((EventType.MouseWheel, 120, -1, -5),
+                 (EventType.MouseWheel, -120, -1, 5),
+                 (EventType.ButtonPress, -1, 4, -5),
+                 (EventType.ButtonPress, -1, 5, 5))
+
+        event = _Event()
+        for ty, delta, num, amount in tests:
+            event.type = ty
+            event.delta = delta
+            event.num = num
+            res = tree.wheel_event(event, _Widget(SCROLL, amount, "units"))
+            self.assertEqual(res, "break")
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py
index 21426cbb33e0..6229be4e5a8a 100644
--- a/Lib/idlelib/tree.py
+++ b/Lib/idlelib/tree.py
@@ -56,6 +56,30 @@ def listicons(icondir=ICONDIR):
             column = 0
     root.images = images
 
+def wheel_event(event, widget=None):
+    """Handle scrollwheel event.
+
+    For wheel up, event.delta = 120*n on Windows, -1*n on darwin,
+    where n can be > 1 if one scrolls fast.  Flicking the wheel
+    generates up to maybe 20 events with n up to 10 or more 1.
+    Macs use wheel down (delta = 1*n) to scroll up, so positive
+    delta means to scroll up on both systems.
+
+    X-11 sends Control-Button-4,5 events instead.
+
+    The widget parameter is needed so browser label bindings can pass
+    the underlying canvas.
+
+    This function depends on widget.yview to not be overridden by
+    a subclass.
+    """
+    up = {EventType.MouseWheel: event.delta > 0,
+          EventType.ButtonPress: event.num == 4}
+    lines = -5 if up[event.type] else 5
+    widget = event.widget if widget is None else widget
+    widget.yview(SCROLL, lines, 'units')
+    return 'break'
+
 
 class TreeNode:
 
@@ -260,6 +284,9 @@ def drawtext(self):
                                        anchor="nw", window=self.label)
         self.label.bind("<1>", self.select_or_edit)
         self.label.bind("<Double-1>", self.flip)
+        self.label.bind("<MouseWheel>", lambda e: wheel_event(e, self.canvas))
+        self.label.bind("<Button-4>", lambda e: wheel_event(e, self.canvas))
+        self.label.bind("<Button-5>", lambda e: wheel_event(e, self.canvas))
         self.text_id = id
 
     def select_or_edit(self, event=None):
@@ -410,6 +437,7 @@ def GetSubList(self):
 # A canvas widget with scroll bars and some useful bindings
 
 class ScrolledCanvas:
+
     def __init__(self, master, **opts):
         if 'yscrollincrement' not in opts:
             opts['yscrollincrement'] = 17
@@ -431,6 +459,9 @@ def __init__(self, master, **opts):
         self.canvas.bind("<Key-Next>", self.page_down)
         self.canvas.bind("<Key-Up>", self.unit_up)
         self.canvas.bind("<Key-Down>", self.unit_down)
+        self.canvas.bind("<MouseWheel>", wheel_event)
+        self.canvas.bind("<Button-4>", wheel_event)
+        self.canvas.bind("<Button-5>", wheel_event)
         #if isinstance(master, Toplevel) or isinstance(master, Tk):
         self.canvas.bind("<Alt-Key-2>", self.zoom_height)
         self.canvas.focus_set()
diff --git a/Misc/ACKS b/Misc/ACKS
index ec14bdc7bb59..c7d6f6c35b61 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1830,6 +1830,7 @@ Nickolai Zeldovich
 Yuxiao Zeng
 Uwe Zessin
 Cheng Zhang
+George Zhang
 Kai Zhu
 Tarek Ziad?
 Jelle Zijlstra
diff --git a/Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst b/Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst
new file mode 100644
index 000000000000..24b414248469
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst
@@ -0,0 +1,2 @@
+Add mousewheel scrolling for IDLE module, path, and stack browsers.
+Patch by George Zhang.


From webhook-mailer at python.org  Thu Sep  5 00:07:13 2019
From: webhook-mailer at python.org (Inada Naoki)
Date: Thu, 05 Sep 2019 04:07:13 -0000
Subject: [Python-checkins] bpo-38026: fix inspect.getattr_static (GH-15676)
Message-ID: <mailman.245.1567656433.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8f9cc8771ffb8d0e21be287eaed42ae06087acca
commit: 8f9cc8771ffb8d0e21be287eaed42ae06087acca
branch: master
author: Inada Naoki <songofacandy at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-05T13:07:08+09:00
summary:

bpo-38026: fix inspect.getattr_static (GH-15676)

It should avoid dynamic lookup including `isinstance`.

This is a regression caused by GH-5351.

files:
A Misc/NEWS.d/next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst
M Lib/inspect.py

diff --git a/Lib/inspect.py b/Lib/inspect.py
index 99a580bd2f23..a616f2d49b7d 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -1558,7 +1558,7 @@ def _shadowed_dict(klass):
         except KeyError:
             pass
         else:
-            if not (isinstance(class_dict, types.GetSetDescriptorType) and
+            if not (type(class_dict) is types.GetSetDescriptorType and
                     class_dict.__name__ == "__dict__" and
                     class_dict.__objclass__ is entry):
                 return class_dict
@@ -1580,7 +1580,7 @@ def getattr_static(obj, attr, default=_sentinel):
         klass = type(obj)
         dict_attr = _shadowed_dict(klass)
         if (dict_attr is _sentinel or
-            isinstance(dict_attr, types.MemberDescriptorType)):
+            type(dict_attr) is types.MemberDescriptorType):
             instance_result = _check_instance(obj, attr)
     else:
         klass = obj
diff --git a/Misc/NEWS.d/next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst b/Misc/NEWS.d/next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst
new file mode 100644
index 000000000000..166cd867514c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst
@@ -0,0 +1,2 @@
+Fixed :func:`inspect.getattr_static` used ``isinstance`` while it should
+avoid dynamic lookup.


From webhook-mailer at python.org  Thu Sep  5 00:26:03 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 05 Sep 2019 04:26:03 -0000
Subject: [Python-checkins] bpo-38026: fix inspect.getattr_static (GH-15676)
Message-ID: <mailman.246.1567657564.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bdcbb83c6640c2b30d0e1a2a497e9ba34cc53b26
commit: bdcbb83c6640c2b30d0e1a2a497e9ba34cc53b26
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-04T21:25:59-07:00
summary:

bpo-38026: fix inspect.getattr_static (GH-15676)


It should avoid dynamic lookup including `isinstance`.

This is a regression caused by GH-5351.
(cherry picked from commit 8f9cc8771ffb8d0e21be287eaed42ae06087acca)

Co-authored-by: Inada Naoki <songofacandy at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst
M Lib/inspect.py

diff --git a/Lib/inspect.py b/Lib/inspect.py
index 99a580bd2f23..a616f2d49b7d 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -1558,7 +1558,7 @@ def _shadowed_dict(klass):
         except KeyError:
             pass
         else:
-            if not (isinstance(class_dict, types.GetSetDescriptorType) and
+            if not (type(class_dict) is types.GetSetDescriptorType and
                     class_dict.__name__ == "__dict__" and
                     class_dict.__objclass__ is entry):
                 return class_dict
@@ -1580,7 +1580,7 @@ def getattr_static(obj, attr, default=_sentinel):
         klass = type(obj)
         dict_attr = _shadowed_dict(klass)
         if (dict_attr is _sentinel or
-            isinstance(dict_attr, types.MemberDescriptorType)):
+            type(dict_attr) is types.MemberDescriptorType):
             instance_result = _check_instance(obj, attr)
     else:
         klass = obj
diff --git a/Misc/NEWS.d/next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst b/Misc/NEWS.d/next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst
new file mode 100644
index 000000000000..166cd867514c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst
@@ -0,0 +1,2 @@
+Fixed :func:`inspect.getattr_static` used ``isinstance`` while it should
+avoid dynamic lookup.


From webhook-mailer at python.org  Thu Sep  5 03:18:51 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Thu, 05 Sep 2019 07:18:51 -0000
Subject: [Python-checkins] bpo-36324: Apply review comments from Allen
 Downey (GH-15693)
Message-ID: <mailman.247.1567667932.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e4810b2a6c1d0db1a27ad046831b8fa3b57967a4
commit: e4810b2a6c1d0db1a27ad046831b8fa3b57967a4
branch: master
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-05T00:18:47-07:00
summary:

bpo-36324:  Apply review comments from Allen Downey (GH-15693)

files:
M Doc/library/statistics.rst
M Lib/statistics.py
M Misc/ACKS

diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index 04b731dcc444..62d856bf248c 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -26,10 +26,10 @@ numeric (:class:`Real`-valued) data.
    Unless explicitly noted otherwise, these functions support :class:`int`,
    :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`.
    Behaviour with other types (whether in the numeric tower or not) is
-   currently unsupported.  Mixed types are also undefined and
-   implementation-dependent.  If your input data consists of mixed types,
-   you may be able to use :func:`map` to ensure a consistent result, e.g.
-   ``map(float, input_data)``.
+   currently unsupported.  Collections with a mix of types are also undefined
+   and implementation-dependent.  If your input data consists of mixed types,
+   you may be able to use :func:`map` to ensure a consistent result, for
+   example: ``map(float, input_data)``.
 
 Averages and measures of central location
 -----------------------------------------
@@ -102,11 +102,9 @@ However, for reading convenience, most of the examples show sorted sequences.
    .. note::
 
       The mean is strongly affected by outliers and is not a robust estimator
-      for central location: the mean is not necessarily a typical example of the
-      data points.  For more robust, although less efficient, measures of
-      central location, see :func:`median` and :func:`mode`.  (In this case,
-      "efficient" refers to statistical efficiency rather than computational
-      efficiency.)
+      for central location: the mean is not necessarily a typical example of
+      the data points.  For more robust measures of central location, see
+      :func:`median` and :func:`mode`.
 
       The sample mean gives an unbiased estimate of the true population mean,
       which means that, taken on average over all the possible samples,
@@ -120,9 +118,8 @@ However, for reading convenience, most of the examples show sorted sequences.
    Convert *data* to floats and compute the arithmetic mean.
 
    This runs faster than the :func:`mean` function and it always returns a
-   :class:`float`.  The result is highly accurate but not as perfect as
-   :func:`mean`.  If the input dataset is empty, raises a
-   :exc:`StatisticsError`.
+   :class:`float`.  The *data* may be a sequence or iterator.  If the input
+   dataset is empty, raises a :exc:`StatisticsError`.
 
    .. doctest::
 
@@ -136,15 +133,20 @@ However, for reading convenience, most of the examples show sorted sequences.
 
    Convert *data* to floats and compute the geometric mean.
 
+   The geometric mean indicates the central tendency or typical value of the
+   *data* using the product of the values (as opposed to the arithmetic mean
+   which uses their sum).
+
    Raises a :exc:`StatisticsError` if the input dataset is empty,
    if it contains a zero, or if it contains a negative value.
+   The *data* may be a sequence or iterator.
 
    No special efforts are made to achieve exact results.
    (However, this may change in the future.)
 
    .. doctest::
 
-      >>> round(geometric_mean([54, 24, 36]), 9)
+      >>> round(geometric_mean([54, 24, 36]), 1)
       36.0
 
    .. versionadded:: 3.8
@@ -174,7 +176,7 @@ However, for reading convenience, most of the examples show sorted sequences.
       3.6
 
    Using the arithmetic mean would give an average of about 5.167, which
-   is too high.
+   is well over the aggregate P/E ratio.
 
    :exc:`StatisticsError` is raised if *data* is empty, or any element
    is less than zero.
@@ -312,10 +314,10 @@ However, for reading convenience, most of the examples show sorted sequences.
    The mode (when it exists) is the most typical value and serves as a
    measure of central location.
 
-   If there are multiple modes, returns the first one encountered in the *data*.
-   If the smallest or largest of multiple modes is desired instead, use
-   ``min(multimode(data))`` or ``max(multimode(data))``.  If the input *data* is
-   empty, :exc:`StatisticsError` is raised.
+   If there are multiple modes with the same frequency, returns the first one
+   encountered in the *data*.  If the smallest or largest of those is
+   desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``.
+   If the input *data* is empty, :exc:`StatisticsError` is raised.
 
    ``mode`` assumes discrete data, and returns a single value. This is the
    standard treatment of the mode as commonly taught in schools:
@@ -325,8 +327,8 @@ However, for reading convenience, most of the examples show sorted sequences.
       >>> mode([1, 1, 2, 3, 3, 3, 3, 4])
       3
 
-   The mode is unique in that it is the only statistic which also applies
-   to nominal (non-numeric) data:
+   The mode is unique in that it is the only statistic in this package that
+   also applies to nominal (non-numeric) data:
 
    .. doctest::
 
@@ -368,15 +370,16 @@ However, for reading convenience, most of the examples show sorted sequences.
 
 .. function:: pvariance(data, mu=None)
 
-   Return the population variance of *data*, a non-empty iterable of real-valued
-   numbers.  Variance, or second moment about the mean, is a measure of the
-   variability (spread or dispersion) of data.  A large variance indicates that
-   the data is spread out; a small variance indicates it is clustered closely
-   around the mean.
+   Return the population variance of *data*, a non-empty sequence or iterator
+   of real-valued numbers.  Variance, or second moment about the mean, is a
+   measure of the variability (spread or dispersion) of data.  A large
+   variance indicates that the data is spread out; a small variance indicates
+   it is clustered closely around the mean.
 
-   If the optional second argument *mu* is given, it should be the mean of
-   *data*.  If it is missing or ``None`` (the default), the mean is
-   automatically calculated.
+   If the optional second argument *mu* is given, it is typically the mean of
+   the *data*.  It can also be used to compute the second moment around a
+   point that is not the mean.  If it is missing or ``None`` (the default),
+   the arithmetic mean is automatically calculated.
 
    Use this function to calculate the variance from the entire population.  To
    estimate the variance from a sample, the :func:`variance` function is usually
@@ -401,10 +404,6 @@ However, for reading convenience, most of the examples show sorted sequences.
       >>> pvariance(data, mu)
       1.25
 
-   This function does not attempt to verify that you have passed the actual mean
-   as *mu*.  Using arbitrary values for *mu* may lead to invalid or impossible
-   results.
-
    Decimals and Fractions are supported:
 
    .. doctest::
@@ -423,11 +422,11 @@ However, for reading convenience, most of the examples show sorted sequences.
       ??.  When called on a sample instead, this is the biased sample variance
       s?, also known as variance with N degrees of freedom.
 
-      If you somehow know the true population mean ?, you may use this function
-      to calculate the variance of a sample, giving the known population mean as
-      the second argument.  Provided the data points are representative
-      (e.g. independent and identically distributed), the result will be an
-      unbiased estimate of the population variance.
+      If you somehow know the true population mean ?, you may use this
+      function to calculate the variance of a sample, giving the known
+      population mean as the second argument.  Provided the data points are a
+      random sample of the population, the result will be an unbiased estimate
+      of the population variance.
 
 
 .. function:: stdev(data, xbar=None)
@@ -502,19 +501,19 @@ However, for reading convenience, most of the examples show sorted sequences.
       :func:`pvariance` function as the *mu* parameter to get the variance of a
       sample.
 
-.. function:: quantiles(dist, *, n=4, method='exclusive')
+.. function:: quantiles(data, *, n=4, method='exclusive')
 
-   Divide *dist* into *n* continuous intervals with equal probability.
+   Divide *data* into *n* continuous intervals with equal probability.
    Returns a list of ``n - 1`` cut points separating the intervals.
 
    Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.  Set
    *n* to 100 for percentiles which gives the 99 cuts points that separate
-   *dist* in to 100 equal sized groups.  Raises :exc:`StatisticsError` if *n*
+   *data* in to 100 equal sized groups.  Raises :exc:`StatisticsError` if *n*
    is not least 1.
 
-   The *dist* can be any iterable containing sample data or it can be an
+   The *data* can be any iterable containing sample data or it can be an
    instance of a class that defines an :meth:`~inv_cdf` method.  For meaningful
-   results, the number of data points in *dist* should be larger than *n*.
+   results, the number of data points in *data* should be larger than *n*.
    Raises :exc:`StatisticsError` if there are not at least two data points.
 
    For sample data, the cut points are linearly interpolated from the
@@ -523,7 +522,7 @@ However, for reading convenience, most of the examples show sorted sequences.
    cut-point will evaluate to ``104``.
 
    The *method* for computing quantiles can be varied depending on
-   whether the data in *dist* includes or excludes the lowest and
+   whether the data in *data* includes or excludes the lowest and
    highest possible values from the population.
 
    The default *method* is "exclusive" and is used for data sampled from
@@ -535,14 +534,14 @@ However, for reading convenience, most of the examples show sorted sequences.
 
    Setting the *method* to "inclusive" is used for describing population
    data or for samples that are known to include the most extreme values
-   from the population.  The minimum value in *dist* is treated as the 0th
+   from the population.  The minimum value in *data* is treated as the 0th
    percentile and the maximum value is treated as the 100th percentile.
    The portion of the population falling below the *i-th* of *m* sorted
    data points is computed as ``(i - 1) / (m - 1)``.  Given 11 sample
    values, the method sorts them and assigns the following percentiles:
    0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%.
 
-   If *dist* is an instance of a class that defines an
+   If *data* is an instance of a class that defines an
    :meth:`~inv_cdf` method, setting *method* has no effect.
 
    .. doctest::
@@ -580,7 +579,7 @@ A single exception is defined:
 :class:`NormalDist` is a tool for creating and manipulating normal
 distributions of a `random variable
 <http://www.stat.yale.edu/Courses/1997-98/101/ranvar.htm>`_.  It is a
-composite class that treats the mean and standard deviation of data
+class that treats the mean and standard deviation of data
 measurements as a single entity.
 
 Normal distributions arise from the `Central Limit Theorem
@@ -616,13 +615,14 @@ of applications in statistics.
 
     .. classmethod:: NormalDist.from_samples(data)
 
-       Makes a normal distribution instance computed from sample data.  The
-       *data* can be any :term:`iterable` and should consist of values that
-       can be converted to type :class:`float`.
+       Makes a normal distribution instance with *mu* and *sigma* parameters
+       estimated from the *data* using :func:`fmean` and :func:`stdev`.
 
-       If *data* does not contain at least two elements, raises
-       :exc:`StatisticsError` because it takes at least one point to estimate
-       a central value and at least two points to estimate dispersion.
+       The *data* can be any :term:`iterable` and should consist of values
+       that can be converted to type :class:`float`.  If *data* does not
+       contain at least two elements, raises :exc:`StatisticsError` because it
+       takes at least one point to estimate a central value and at least two
+       points to estimate dispersion.
 
     .. method:: NormalDist.samples(n, *, seed=None)
 
@@ -636,10 +636,10 @@ of applications in statistics.
     .. method:: NormalDist.pdf(x)
 
        Using a `probability density function (pdf)
-       <https://en.wikipedia.org/wiki/Probability_density_function>`_,
-       compute the relative likelihood that a random variable *X* will be near
-       the given value *x*.  Mathematically, it is the ratio ``P(x <= X <
-       x+dx) / dx``.
+       <https://en.wikipedia.org/wiki/Probability_density_function>`_, compute
+       the relative likelihood that a random variable *X* will be near the
+       given value *x*.  Mathematically, it is the limit of the ratio ``P(x <=
+       X < x+dx) / dx`` as *dx* approaches zero.
 
        The relative likelihood is computed as the probability of a sample
        occurring in a narrow range divided by the width of the range (hence
@@ -667,8 +667,10 @@ of applications in statistics.
 
     .. method:: NormalDist.overlap(other)
 
-       Returns a value between 0.0 and 1.0 giving the overlapping area for
-       the two probability density functions.
+       Measures the agreement between two normal probability distributions.
+       Returns a value between 0.0 and 1.0 giving `the overlapping area for
+       the two probability density functions
+       <https://www.rasch.org/rmt/rmt101r.htm>`_.
 
     Instances of :class:`NormalDist` support addition, subtraction,
     multiplication and division by a constant.  These operations
@@ -740,12 +742,11 @@ Carlo simulation <https://en.wikipedia.org/wiki/Monte_Carlo_method>`_:
     ...     return (3*x + 7*x*y - 5*y) / (11 * z)
     ...
     >>> n = 100_000
-    >>> seed = 86753099035768
-    >>> X = NormalDist(10, 2.5).samples(n, seed=seed)
-    >>> Y = NormalDist(15, 1.75).samples(n, seed=seed)
-    >>> Z = NormalDist(50, 1.25).samples(n, seed=seed)
-    >>> NormalDist.from_samples(map(model, X, Y, Z))     # doctest: +SKIP
-    NormalDist(mu=1.8661894803304777, sigma=0.65238717376862)
+    >>> X = NormalDist(10, 2.5).samples(n, seed=3652260728)
+    >>> Y = NormalDist(15, 1.75).samples(n, seed=4582495471)
+    >>> Z = NormalDist(50, 1.25).samples(n, seed=6582483453)
+    >>> quantiles(map(model, X, Y, Z))       # doctest: +SKIP
+    [1.4591308524824727, 1.8035946855390597, 2.175091447274739]
 
 Normal distributions commonly arise in machine learning problems.
 
diff --git a/Lib/statistics.py b/Lib/statistics.py
index c7d6568145e0..4b172662770f 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -322,7 +322,6 @@ def fmean(data):
     """Convert data to floats and compute the arithmetic mean.
 
     This runs faster than the mean() function and it always returns a float.
-    The result is highly accurate but not as perfect as mean().
     If the input dataset is empty, it raises a StatisticsError.
 
     >>> fmean([3.5, 4.0, 5.25])
@@ -538,15 +537,16 @@ def mode(data):
     ``mode`` assumes discrete data, and returns a single value. This is the
     standard treatment of the mode as commonly taught in schools:
 
-    >>> mode([1, 1, 2, 3, 3, 3, 3, 4])
-    3
+        >>> mode([1, 1, 2, 3, 3, 3, 3, 4])
+        3
 
     This also works with nominal (non-numeric) data:
 
-    >>> mode(["red", "blue", "blue", "red", "green", "red", "red"])
-    'red'
+        >>> mode(["red", "blue", "blue", "red", "green", "red", "red"])
+        'red'
 
-    If there are multiple modes, return the first one encountered.
+    If there are multiple modes with same frequency, return the first one
+    encountered:
 
         >>> mode(['red', 'red', 'green', 'blue', 'blue'])
         'red'
@@ -615,28 +615,28 @@ def multimode(data):
 # position is that fewer options make for easier choices and that
 # external packages can be used for anything more advanced.
 
-def quantiles(dist, /, *, n=4, method='exclusive'):
-    """Divide *dist* into *n* continuous intervals with equal probability.
+def quantiles(data, /, *, n=4, method='exclusive'):
+    """Divide *data* into *n* continuous intervals with equal probability.
 
     Returns a list of (n - 1) cut points separating the intervals.
 
     Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.
     Set *n* to 100 for percentiles which gives the 99 cuts points that
-    separate *dist* in to 100 equal sized groups.
+    separate *data* in to 100 equal sized groups.
 
-    The *dist* can be any iterable containing sample data or it can be
+    The *data* can be any iterable containing sample data or it can be
     an instance of a class that defines an inv_cdf() method.  For sample
     data, the cut points are linearly interpolated between data points.
 
-    If *method* is set to *inclusive*, *dist* is treated as population
+    If *method* is set to *inclusive*, *data* is treated as population
     data.  The minimum value is treated as the 0th percentile and the
     maximum value is treated as the 100th percentile.
     """
     if n < 1:
         raise StatisticsError('n must be at least 1')
-    if hasattr(dist, 'inv_cdf'):
-        return [dist.inv_cdf(i / n) for i in range(1, n)]
-    data = sorted(dist)
+    if hasattr(data, 'inv_cdf'):
+        return [data.inv_cdf(i / n) for i in range(1, n)]
+    data = sorted(data)
     ld = len(data)
     if ld < 2:
         raise StatisticsError('must have at least two data points')
@@ -745,7 +745,7 @@ def variance(data, xbar=None):
 def pvariance(data, mu=None):
     """Return the population variance of ``data``.
 
-    data should be an iterable of Real-valued numbers, with at least one
+    data should be a sequence or iterator of Real-valued numbers, with at least one
     value. The optional argument mu, if given, should be the mean of
     the data. If it is missing or None, the mean is automatically calculated.
 
@@ -766,10 +766,6 @@ def pvariance(data, mu=None):
     >>> pvariance(data, mu)
     1.25
 
-    This function does not check that ``mu`` is actually the mean of ``data``.
-    Giving arbitrary values for ``mu`` may lead to invalid or impossible
-    results.
-
     Decimals and Fractions are supported:
 
     >>> from decimal import Decimal as D
@@ -913,8 +909,8 @@ def __init__(self, mu=0.0, sigma=1.0):
         "NormalDist where mu is the mean and sigma is the standard deviation."
         if sigma < 0.0:
             raise StatisticsError('sigma must be non-negative')
-        self._mu = mu
-        self._sigma = sigma
+        self._mu = float(mu)
+        self._sigma = float(sigma)
 
     @classmethod
     def from_samples(cls, data):
diff --git a/Misc/ACKS b/Misc/ACKS
index ce427b8222ed..63189984e64a 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -416,6 +416,7 @@ Dima Dorfman
 Yves Dorfsman
 Michael Dorman
 Steve Dower
+Allen Downey
 Cesar Douady
 Dean Draayer
 Fred L. Drake, Jr.


From webhook-mailer at python.org  Thu Sep  5 03:42:32 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Thu, 05 Sep 2019 07:42:32 -0000
Subject: [Python-checkins] bpo-36324: Apply review comments from Allen
 Downey (GH-15693) (GH-15694)
Message-ID: <mailman.248.1567669353.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/dafbe3265657dc5a5c46e762080023f0aa25ec58
commit: dafbe3265657dc5a5c46e762080023f0aa25ec58
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-05T00:42:22-07:00
summary:

bpo-36324:  Apply review comments from Allen Downey (GH-15693) (GH-15694)

(cherry picked from commit e4810b2a6c1d0db1a27ad046831b8fa3b57967a4)

Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com>

files:
M Doc/library/statistics.rst
M Lib/statistics.py
M Misc/ACKS

diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index cbe2b8223faf..9f9fc86b3a13 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -26,10 +26,10 @@ numeric (:class:`Real`-valued) data.
    Unless explicitly noted otherwise, these functions support :class:`int`,
    :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`.
    Behaviour with other types (whether in the numeric tower or not) is
-   currently unsupported.  Mixed types are also undefined and
-   implementation-dependent.  If your input data consists of mixed types,
-   you may be able to use :func:`map` to ensure a consistent result, e.g.
-   ``map(float, input_data)``.
+   currently unsupported.  Collections with a mix of types are also undefined
+   and implementation-dependent.  If your input data consists of mixed types,
+   you may be able to use :func:`map` to ensure a consistent result, for
+   example: ``map(float, input_data)``.
 
 Averages and measures of central location
 -----------------------------------------
@@ -102,11 +102,9 @@ However, for reading convenience, most of the examples show sorted sequences.
    .. note::
 
       The mean is strongly affected by outliers and is not a robust estimator
-      for central location: the mean is not necessarily a typical example of the
-      data points.  For more robust, although less efficient, measures of
-      central location, see :func:`median` and :func:`mode`.  (In this case,
-      "efficient" refers to statistical efficiency rather than computational
-      efficiency.)
+      for central location: the mean is not necessarily a typical example of
+      the data points.  For more robust measures of central location, see
+      :func:`median` and :func:`mode`.
 
       The sample mean gives an unbiased estimate of the true population mean,
       which means that, taken on average over all the possible samples,
@@ -120,9 +118,8 @@ However, for reading convenience, most of the examples show sorted sequences.
    Convert *data* to floats and compute the arithmetic mean.
 
    This runs faster than the :func:`mean` function and it always returns a
-   :class:`float`.  The result is highly accurate but not as perfect as
-   :func:`mean`.  If the input dataset is empty, raises a
-   :exc:`StatisticsError`.
+   :class:`float`.  The *data* may be a sequence or iterator.  If the input
+   dataset is empty, raises a :exc:`StatisticsError`.
 
    .. doctest::
 
@@ -136,15 +133,20 @@ However, for reading convenience, most of the examples show sorted sequences.
 
    Convert *data* to floats and compute the geometric mean.
 
+   The geometric mean indicates the central tendency or typical value of the
+   *data* using the product of the values (as opposed to the arithmetic mean
+   which uses their sum).
+
    Raises a :exc:`StatisticsError` if the input dataset is empty,
    if it contains a zero, or if it contains a negative value.
+   The *data* may be a sequence or iterator.
 
    No special efforts are made to achieve exact results.
    (However, this may change in the future.)
 
    .. doctest::
 
-      >>> round(geometric_mean([54, 24, 36]), 9)
+      >>> round(geometric_mean([54, 24, 36]), 1)
       36.0
 
    .. versionadded:: 3.8
@@ -174,7 +176,7 @@ However, for reading convenience, most of the examples show sorted sequences.
       3.6
 
    Using the arithmetic mean would give an average of about 5.167, which
-   is too high.
+   is well over the aggregate P/E ratio.
 
    :exc:`StatisticsError` is raised if *data* is empty, or any element
    is less than zero.
@@ -312,10 +314,10 @@ However, for reading convenience, most of the examples show sorted sequences.
    The mode (when it exists) is the most typical value and serves as a
    measure of central location.
 
-   If there are multiple modes, returns the first one encountered in the *data*.
-   If the smallest or largest of multiple modes is desired instead, use
-   ``min(multimode(data))`` or ``max(multimode(data))``.  If the input *data* is
-   empty, :exc:`StatisticsError` is raised.
+   If there are multiple modes with the same frequency, returns the first one
+   encountered in the *data*.  If the smallest or largest of those is
+   desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``.
+   If the input *data* is empty, :exc:`StatisticsError` is raised.
 
    ``mode`` assumes discrete data, and returns a single value. This is the
    standard treatment of the mode as commonly taught in schools:
@@ -325,8 +327,8 @@ However, for reading convenience, most of the examples show sorted sequences.
       >>> mode([1, 1, 2, 3, 3, 3, 3, 4])
       3
 
-   The mode is unique in that it is the only statistic which also applies
-   to nominal (non-numeric) data:
+   The mode is unique in that it is the only statistic in this package that
+   also applies to nominal (non-numeric) data:
 
    .. doctest::
 
@@ -368,15 +370,16 @@ However, for reading convenience, most of the examples show sorted sequences.
 
 .. function:: pvariance(data, mu=None)
 
-   Return the population variance of *data*, a non-empty iterable of real-valued
-   numbers.  Variance, or second moment about the mean, is a measure of the
-   variability (spread or dispersion) of data.  A large variance indicates that
-   the data is spread out; a small variance indicates it is clustered closely
-   around the mean.
+   Return the population variance of *data*, a non-empty sequence or iterator
+   of real-valued numbers.  Variance, or second moment about the mean, is a
+   measure of the variability (spread or dispersion) of data.  A large
+   variance indicates that the data is spread out; a small variance indicates
+   it is clustered closely around the mean.
 
-   If the optional second argument *mu* is given, it should be the mean of
-   *data*.  If it is missing or ``None`` (the default), the mean is
-   automatically calculated.
+   If the optional second argument *mu* is given, it is typically the mean of
+   the *data*.  It can also be used to compute the second moment around a
+   point that is not the mean.  If it is missing or ``None`` (the default),
+   the arithmetic mean is automatically calculated.
 
    Use this function to calculate the variance from the entire population.  To
    estimate the variance from a sample, the :func:`variance` function is usually
@@ -401,10 +404,6 @@ However, for reading convenience, most of the examples show sorted sequences.
       >>> pvariance(data, mu)
       1.25
 
-   This function does not attempt to verify that you have passed the actual mean
-   as *mu*.  Using arbitrary values for *mu* may lead to invalid or impossible
-   results.
-
    Decimals and Fractions are supported:
 
    .. doctest::
@@ -423,11 +422,11 @@ However, for reading convenience, most of the examples show sorted sequences.
       ??.  When called on a sample instead, this is the biased sample variance
       s?, also known as variance with N degrees of freedom.
 
-      If you somehow know the true population mean ?, you may use this function
-      to calculate the variance of a sample, giving the known population mean as
-      the second argument.  Provided the data points are representative
-      (e.g. independent and identically distributed), the result will be an
-      unbiased estimate of the population variance.
+      If you somehow know the true population mean ?, you may use this
+      function to calculate the variance of a sample, giving the known
+      population mean as the second argument.  Provided the data points are a
+      random sample of the population, the result will be an unbiased estimate
+      of the population variance.
 
 
 .. function:: stdev(data, xbar=None)
@@ -502,19 +501,19 @@ However, for reading convenience, most of the examples show sorted sequences.
       :func:`pvariance` function as the *mu* parameter to get the variance of a
       sample.
 
-.. function:: quantiles(dist, *, n=4, method='exclusive')
+.. function:: quantiles(data, *, n=4, method='exclusive')
 
-   Divide *dist* into *n* continuous intervals with equal probability.
+   Divide *data* into *n* continuous intervals with equal probability.
    Returns a list of ``n - 1`` cut points separating the intervals.
 
    Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.  Set
    *n* to 100 for percentiles which gives the 99 cuts points that separate
-   *dist* in to 100 equal sized groups.  Raises :exc:`StatisticsError` if *n*
+   *data* in to 100 equal sized groups.  Raises :exc:`StatisticsError` if *n*
    is not least 1.
 
-   The *dist* can be any iterable containing sample data or it can be an
+   The *data* can be any iterable containing sample data or it can be an
    instance of a class that defines an :meth:`~inv_cdf` method.  For meaningful
-   results, the number of data points in *dist* should be larger than *n*.
+   results, the number of data points in *data* should be larger than *n*.
    Raises :exc:`StatisticsError` if there are not at least two data points.
 
    For sample data, the cut points are linearly interpolated from the
@@ -523,7 +522,7 @@ However, for reading convenience, most of the examples show sorted sequences.
    cut-point will evaluate to ``104``.
 
    The *method* for computing quantiles can be varied depending on
-   whether the data in *dist* includes or excludes the lowest and
+   whether the data in *data* includes or excludes the lowest and
    highest possible values from the population.
 
    The default *method* is "exclusive" and is used for data sampled from
@@ -535,14 +534,14 @@ However, for reading convenience, most of the examples show sorted sequences.
 
    Setting the *method* to "inclusive" is used for describing population
    data or for samples that are known to include the most extreme values
-   from the population.  The minimum value in *dist* is treated as the 0th
+   from the population.  The minimum value in *data* is treated as the 0th
    percentile and the maximum value is treated as the 100th percentile.
    The portion of the population falling below the *i-th* of *m* sorted
    data points is computed as ``(i - 1) / (m - 1)``.  Given 11 sample
    values, the method sorts them and assigns the following percentiles:
    0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%.
 
-   If *dist* is an instance of a class that defines an
+   If *data* is an instance of a class that defines an
    :meth:`~inv_cdf` method, setting *method* has no effect.
 
    .. doctest::
@@ -580,7 +579,7 @@ A single exception is defined:
 :class:`NormalDist` is a tool for creating and manipulating normal
 distributions of a `random variable
 <http://www.stat.yale.edu/Courses/1997-98/101/ranvar.htm>`_.  It is a
-composite class that treats the mean and standard deviation of data
+class that treats the mean and standard deviation of data
 measurements as a single entity.
 
 Normal distributions arise from the `Central Limit Theorem
@@ -616,13 +615,14 @@ of applications in statistics.
 
     .. classmethod:: NormalDist.from_samples(data)
 
-       Makes a normal distribution instance computed from sample data.  The
-       *data* can be any :term:`iterable` and should consist of values that
-       can be converted to type :class:`float`.
+       Makes a normal distribution instance with *mu* and *sigma* parameters
+       estimated from the *data* using :func:`fmean` and :func:`stdev`.
 
-       If *data* does not contain at least two elements, raises
-       :exc:`StatisticsError` because it takes at least one point to estimate
-       a central value and at least two points to estimate dispersion.
+       The *data* can be any :term:`iterable` and should consist of values
+       that can be converted to type :class:`float`.  If *data* does not
+       contain at least two elements, raises :exc:`StatisticsError` because it
+       takes at least one point to estimate a central value and at least two
+       points to estimate dispersion.
 
     .. method:: NormalDist.samples(n, *, seed=None)
 
@@ -636,10 +636,10 @@ of applications in statistics.
     .. method:: NormalDist.pdf(x)
 
        Using a `probability density function (pdf)
-       <https://en.wikipedia.org/wiki/Probability_density_function>`_,
-       compute the relative likelihood that a random variable *X* will be near
-       the given value *x*.  Mathematically, it is the ratio ``P(x <= X <
-       x+dx) / dx``.
+       <https://en.wikipedia.org/wiki/Probability_density_function>`_, compute
+       the relative likelihood that a random variable *X* will be near the
+       given value *x*.  Mathematically, it is the limit of the ratio ``P(x <=
+       X < x+dx) / dx`` as *dx* approaches zero.
 
        The relative likelihood is computed as the probability of a sample
        occurring in a narrow range divided by the width of the range (hence
@@ -667,8 +667,10 @@ of applications in statistics.
 
     .. method:: NormalDist.overlap(other)
 
-       Returns a value between 0.0 and 1.0 giving the overlapping area for
-       the two probability density functions.
+       Measures the agreement between two normal probability distributions.
+       Returns a value between 0.0 and 1.0 giving `the overlapping area for
+       the two probability density functions
+       <https://www.rasch.org/rmt/rmt101r.htm>`_.
 
     Instances of :class:`NormalDist` support addition, subtraction,
     multiplication and division by a constant.  These operations
@@ -740,12 +742,11 @@ Carlo simulation <https://en.wikipedia.org/wiki/Monte_Carlo_method>`_:
     ...     return (3*x + 7*x*y - 5*y) / (11 * z)
     ...
     >>> n = 100_000
-    >>> seed = 86753099035768
-    >>> X = NormalDist(10, 2.5).samples(n, seed=seed)
-    >>> Y = NormalDist(15, 1.75).samples(n, seed=seed)
-    >>> Z = NormalDist(50, 1.25).samples(n, seed=seed)
-    >>> NormalDist.from_samples(map(model, X, Y, Z))     # doctest: +SKIP
-    NormalDist(mu=1.8661894803304777, sigma=0.65238717376862)
+    >>> X = NormalDist(10, 2.5).samples(n, seed=3652260728)
+    >>> Y = NormalDist(15, 1.75).samples(n, seed=4582495471)
+    >>> Z = NormalDist(50, 1.25).samples(n, seed=6582483453)
+    >>> quantiles(map(model, X, Y, Z))       # doctest: +SKIP
+    [1.4591308524824727, 1.8035946855390597, 2.175091447274739]
 
 Normal distributions commonly arise in machine learning problems.
 
diff --git a/Lib/statistics.py b/Lib/statistics.py
index c7d6568145e0..4b172662770f 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -322,7 +322,6 @@ def fmean(data):
     """Convert data to floats and compute the arithmetic mean.
 
     This runs faster than the mean() function and it always returns a float.
-    The result is highly accurate but not as perfect as mean().
     If the input dataset is empty, it raises a StatisticsError.
 
     >>> fmean([3.5, 4.0, 5.25])
@@ -538,15 +537,16 @@ def mode(data):
     ``mode`` assumes discrete data, and returns a single value. This is the
     standard treatment of the mode as commonly taught in schools:
 
-    >>> mode([1, 1, 2, 3, 3, 3, 3, 4])
-    3
+        >>> mode([1, 1, 2, 3, 3, 3, 3, 4])
+        3
 
     This also works with nominal (non-numeric) data:
 
-    >>> mode(["red", "blue", "blue", "red", "green", "red", "red"])
-    'red'
+        >>> mode(["red", "blue", "blue", "red", "green", "red", "red"])
+        'red'
 
-    If there are multiple modes, return the first one encountered.
+    If there are multiple modes with same frequency, return the first one
+    encountered:
 
         >>> mode(['red', 'red', 'green', 'blue', 'blue'])
         'red'
@@ -615,28 +615,28 @@ def multimode(data):
 # position is that fewer options make for easier choices and that
 # external packages can be used for anything more advanced.
 
-def quantiles(dist, /, *, n=4, method='exclusive'):
-    """Divide *dist* into *n* continuous intervals with equal probability.
+def quantiles(data, /, *, n=4, method='exclusive'):
+    """Divide *data* into *n* continuous intervals with equal probability.
 
     Returns a list of (n - 1) cut points separating the intervals.
 
     Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.
     Set *n* to 100 for percentiles which gives the 99 cuts points that
-    separate *dist* in to 100 equal sized groups.
+    separate *data* in to 100 equal sized groups.
 
-    The *dist* can be any iterable containing sample data or it can be
+    The *data* can be any iterable containing sample data or it can be
     an instance of a class that defines an inv_cdf() method.  For sample
     data, the cut points are linearly interpolated between data points.
 
-    If *method* is set to *inclusive*, *dist* is treated as population
+    If *method* is set to *inclusive*, *data* is treated as population
     data.  The minimum value is treated as the 0th percentile and the
     maximum value is treated as the 100th percentile.
     """
     if n < 1:
         raise StatisticsError('n must be at least 1')
-    if hasattr(dist, 'inv_cdf'):
-        return [dist.inv_cdf(i / n) for i in range(1, n)]
-    data = sorted(dist)
+    if hasattr(data, 'inv_cdf'):
+        return [data.inv_cdf(i / n) for i in range(1, n)]
+    data = sorted(data)
     ld = len(data)
     if ld < 2:
         raise StatisticsError('must have at least two data points')
@@ -745,7 +745,7 @@ def variance(data, xbar=None):
 def pvariance(data, mu=None):
     """Return the population variance of ``data``.
 
-    data should be an iterable of Real-valued numbers, with at least one
+    data should be a sequence or iterator of Real-valued numbers, with at least one
     value. The optional argument mu, if given, should be the mean of
     the data. If it is missing or None, the mean is automatically calculated.
 
@@ -766,10 +766,6 @@ def pvariance(data, mu=None):
     >>> pvariance(data, mu)
     1.25
 
-    This function does not check that ``mu`` is actually the mean of ``data``.
-    Giving arbitrary values for ``mu`` may lead to invalid or impossible
-    results.
-
     Decimals and Fractions are supported:
 
     >>> from decimal import Decimal as D
@@ -913,8 +909,8 @@ def __init__(self, mu=0.0, sigma=1.0):
         "NormalDist where mu is the mean and sigma is the standard deviation."
         if sigma < 0.0:
             raise StatisticsError('sigma must be non-negative')
-        self._mu = mu
-        self._sigma = sigma
+        self._mu = float(mu)
+        self._sigma = float(sigma)
 
     @classmethod
     def from_samples(cls, data):
diff --git a/Misc/ACKS b/Misc/ACKS
index 290f8ea33269..55aab6b6b2b3 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -414,6 +414,7 @@ Dima Dorfman
 Yves Dorfsman
 Michael Dorman
 Steve Dower
+Allen Downey
 Cesar Douady
 Dean Draayer
 Fred L. Drake, Jr.


From webhook-mailer at python.org  Thu Sep  5 04:03:21 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Thu, 05 Sep 2019 08:03:21 -0000
Subject: [Python-checkins] bpo-36324: Apply review comment from Jake
 Vanderplas (GH-15695)
Message-ID: <mailman.249.1567670602.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9b51570ffd0494c07dafe10c7d2afe865754694c
commit: 9b51570ffd0494c07dafe10c7d2afe865754694c
branch: master
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-05T01:03:14-07:00
summary:

bpo-36324:  Apply review comment from Jake Vanderplas (GH-15695)

files:
M Doc/library/statistics.rst

diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index 62d856bf248c..27d0518b4a79 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -727,9 +727,9 @@ Find the `quartiles <https://en.wikipedia.org/wiki/Quartile>`_ and `deciles
 
 .. doctest::
 
-    >>> [round(sat.inv_cdf(p)) for p in (0.25, 0.50, 0.75)]
+    >>> list(map(round, quantiles(sat)))
     [928, 1060, 1192]
-    >>> [round(sat.inv_cdf(p / 10)) for p in range(1, 10)]
+    >>> list(map(round, quantiles(sat, n=10)))
     [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310]
 
 To estimate the distribution for a model than isn't easy to solve


From webhook-mailer at python.org  Thu Sep  5 04:10:45 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Thu, 05 Sep 2019 08:10:45 -0000
Subject: [Python-checkins] bpo-36324: Apply review comment from Jake
 Vanderplas (GH-15695) (GH-15696)
Message-ID: <mailman.250.1567671045.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f5649bfe7622447b302ef55e4db3a96b5840f8e8
commit: f5649bfe7622447b302ef55e4db3a96b5840f8e8
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-05T01:10:40-07:00
summary:

bpo-36324:  Apply review comment from Jake Vanderplas (GH-15695) (GH-15696)

(cherry picked from commit 9b51570ffd0494c07dafe10c7d2afe865754694c)

Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com>

files:
M Doc/library/statistics.rst

diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index 9f9fc86b3a13..2ddd393a4593 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -727,9 +727,9 @@ Find the `quartiles <https://en.wikipedia.org/wiki/Quartile>`_ and `deciles
 
 .. doctest::
 
-    >>> [round(sat.inv_cdf(p)) for p in (0.25, 0.50, 0.75)]
+    >>> list(map(round, quantiles(sat)))
     [928, 1060, 1192]
-    >>> [round(sat.inv_cdf(p / 10)) for p in range(1, 10)]
+    >>> list(map(round, quantiles(sat, n=10)))
     [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310]
 
 To estimate the distribution for a model than isn't easy to solve


From webhook-mailer at python.org  Thu Sep  5 04:11:39 2019
From: webhook-mailer at python.org (Ronald Oussoren)
Date: Thu, 05 Sep 2019 08:11:39 -0000
Subject: [Python-checkins] bpo-36409: Remove old plistlib API deprecated in
 3.4 (GH-15615)
Message-ID: <mailman.251.1567671100.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ce81a925ef2d466b0d801deaba9af44a0dfc2aa1
commit: ce81a925ef2d466b0d801deaba9af44a0dfc2aa1
branch: master
author: Jon Janzen <jjjonjanzen at gmail.com>
committer: Ronald Oussoren <ronaldoussoren at mac.com>
date: 2019-09-05T10:11:35+02:00
summary:

bpo-36409: Remove old plistlib API deprecated in 3.4 (GH-15615)

* Remove implementation for old plistlib API deprecated in 3.4

files:
A Misc/NEWS.d/next/Library/2019-03-23-16-42-46.bpo-36409.ZYVKao.rst
M Doc/library/plistlib.rst
M Doc/whatsnew/3.9.rst
M Lib/plistlib.py
M Lib/test/test_plistlib.py

diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst
index d84fcac0ef23..f71aaa2354e9 100644
--- a/Doc/library/plistlib.rst
+++ b/Doc/library/plistlib.rst
@@ -30,7 +30,7 @@ To work with plist data in bytes objects, use :func:`dumps`
 and :func:`loads`.
 
 Values can be strings, integers, floats, booleans, tuples, lists, dictionaries
-(but only with string keys), :class:`Data`, :class:`bytes`, :class:`bytesarray`
+(but only with string keys), :class:`bytes`, :class:`bytearray`
 or :class:`datetime.datetime` objects.
 
 .. versionchanged:: 3.4
@@ -40,6 +40,9 @@ or :class:`datetime.datetime` objects.
    Support added for reading and writing :class:`UID` tokens in binary plists as used
    by NSKeyedArchiver and NSKeyedUnarchiver.
 
+.. versionchanged:: 3.9
+   Old API removed.
+
 .. seealso::
 
    `PList manual page <https://developer.apple.com/library/content/documentation/Cocoa/Conceptual/PropertyLists/>`_
@@ -48,7 +51,7 @@ or :class:`datetime.datetime` objects.
 
 This module defines the following functions:
 
-.. function:: load(fp, \*, fmt=None, use_builtin_types=True, dict_type=dict)
+.. function:: load(fp, \*, fmt=None, dict_type=dict)
 
    Read a plist file. *fp* should be a readable and binary file object.
    Return the unpacked root object (which usually is a
@@ -62,10 +65,6 @@ This module defines the following functions:
 
    * :data:`FMT_BINARY`: Binary plist format
 
-   If *use_builtin_types* is true (the default) binary data will be returned
-   as instances of :class:`bytes`, otherwise it is returned as instances of
-   :class:`Data`.
-
    The *dict_type* is the type used for dictionaries that are read from the
    plist file.
 
@@ -80,7 +79,7 @@ This module defines the following functions:
    .. versionadded:: 3.4
 
 
-.. function:: loads(data, \*, fmt=None, use_builtin_types=True, dict_type=dict)
+.. function:: loads(data, \*, fmt=None, dict_type=dict)
 
    Load a plist from a bytes object. See :func:`load` for an explanation of
    the keyword arguments.
@@ -124,75 +123,6 @@ This module defines the following functions:
 
    .. versionadded:: 3.4
 
-The following functions are deprecated:
-
-.. function:: readPlist(pathOrFile)
-
-   Read a plist file. *pathOrFile* may be either a file name or a (readable
-   and binary) file object. Returns the unpacked root object (which usually
-   is a dictionary).
-
-   This function calls :func:`load` to do the actual work, see the documentation
-   of :func:`that function <load>` for an explanation of the keyword arguments.
-
-   .. deprecated:: 3.4 Use :func:`load` instead.
-
-   .. versionchanged:: 3.7
-      Dict values in the result are now normal dicts.  You no longer can use
-      attribute access to access items of these dictionaries.
-
-
-.. function:: writePlist(rootObject, pathOrFile)
-
-   Write *rootObject* to an XML plist file. *pathOrFile* may be either a file name
-   or a (writable and binary) file object
-
-   .. deprecated:: 3.4 Use :func:`dump` instead.
-
-
-.. function:: readPlistFromBytes(data)
-
-   Read a plist data from a bytes object.  Return the root object.
-
-   See :func:`load` for a description of the keyword arguments.
-
-   .. deprecated:: 3.4 Use :func:`loads` instead.
-
-   .. versionchanged:: 3.7
-      Dict values in the result are now normal dicts.  You no longer can use
-      attribute access to access items of these dictionaries.
-
-
-.. function:: writePlistToBytes(rootObject)
-
-   Return *rootObject* as an XML plist-formatted bytes object.
-
-   .. deprecated:: 3.4 Use :func:`dumps` instead.
-
-
-The following classes are available:
-
-.. class:: Data(data)
-
-   Return a "data" wrapper object around the bytes object *data*.  This is used
-   in functions converting from/to plists to represent the ``<data>`` type
-   available in plists.
-
-   It has one attribute, :attr:`data`, that can be used to retrieve the Python
-   bytes object stored in it.
-
-   .. deprecated:: 3.4 Use a :class:`bytes` object instead.
-
-.. class:: UID(data)
-
-   Wraps an :class:`int`.  This is used when reading or writing NSKeyedArchiver
-   encoded data, which contains UID (see PList manual).
-
-   It has one attribute, :attr:`data` which can be used to retrieve the int value
-   of the UID.  :attr:`data` must be in the range `0 <= data <= 2**64`.
-
-   .. versionadded:: 3.8
-
 
 The following constants are available:
 
diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
index 0a0b91670ca5..009a66adff7d 100644
--- a/Doc/whatsnew/3.9.rst
+++ b/Doc/whatsnew/3.9.rst
@@ -213,6 +213,12 @@ Removed
   instead.  The ``xml.etree.cElementTree`` module has been removed.
   (Contributed by Serhiy Storchaka in :issue:`36543`.)
 
+* The old :mod:`plistlib` API has been removed, it was deprecated since Python
+  3.4. Use the :func:`load`, :func:`loads`, :func:`dump`, and :func:`dumps`
+  functions. Additionally, the ``use_builtin_types`` parameter was removed,
+  standard :class:`bytes` objects are always used.
+  (Contributed by Jon Janzen in :issue:`36409`.)
+
 
 Porting to Python 3.9
 =====================
diff --git a/Lib/plistlib.py b/Lib/plistlib.py
index 04f8a87634d3..a84bb57371ef 100644
--- a/Lib/plistlib.py
+++ b/Lib/plistlib.py
@@ -46,9 +46,7 @@
     print(pl["aKey"])
 """
 __all__ = [
-    "readPlist", "writePlist", "readPlistFromBytes", "writePlistToBytes",
-    "Data", "InvalidFileException", "FMT_XML", "FMT_BINARY",
-    "load", "dump", "loads", "dumps", "UID"
+    "InvalidFileException", "FMT_XML", "FMT_BINARY", "load", "dump", "loads", "dumps", "UID"
 ]
 
 import binascii
@@ -69,112 +67,6 @@
 globals().update(PlistFormat.__members__)
 
 
-#
-#
-# Deprecated functionality
-#
-#
-
-
- at contextlib.contextmanager
-def _maybe_open(pathOrFile, mode):
-    if isinstance(pathOrFile, str):
-        with open(pathOrFile, mode) as fp:
-            yield fp
-
-    else:
-        yield pathOrFile
-
-
-def readPlist(pathOrFile):
-    """
-    Read a .plist from a path or file. pathOrFile should either
-    be a file name, or a readable binary file object.
-
-    This function is deprecated, use load instead.
-    """
-    warn("The readPlist function is deprecated, use load() instead",
-        DeprecationWarning, 2)
-
-    with _maybe_open(pathOrFile, 'rb') as fp:
-        return load(fp, fmt=None, use_builtin_types=False)
-
-def writePlist(value, pathOrFile):
-    """
-    Write 'value' to a .plist file. 'pathOrFile' may either be a
-    file name or a (writable) file object.
-
-    This function is deprecated, use dump instead.
-    """
-    warn("The writePlist function is deprecated, use dump() instead",
-        DeprecationWarning, 2)
-    with _maybe_open(pathOrFile, 'wb') as fp:
-        dump(value, fp, fmt=FMT_XML, sort_keys=True, skipkeys=False)
-
-
-def readPlistFromBytes(data):
-    """
-    Read a plist data from a bytes object. Return the root object.
-
-    This function is deprecated, use loads instead.
-    """
-    warn("The readPlistFromBytes function is deprecated, use loads() instead",
-        DeprecationWarning, 2)
-    return load(BytesIO(data), fmt=None, use_builtin_types=False)
-
-
-def writePlistToBytes(value):
-    """
-    Return 'value' as a plist-formatted bytes object.
-
-    This function is deprecated, use dumps instead.
-    """
-    warn("The writePlistToBytes function is deprecated, use dumps() instead",
-        DeprecationWarning, 2)
-    f = BytesIO()
-    dump(value, f, fmt=FMT_XML, sort_keys=True, skipkeys=False)
-    return f.getvalue()
-
-
-class Data:
-    """
-    Wrapper for binary data.
-
-    This class is deprecated, use a bytes object instead.
-    """
-
-    def __init__(self, data):
-        if not isinstance(data, bytes):
-            raise TypeError("data must be as bytes")
-        self.data = data
-
-    @classmethod
-    def fromBase64(cls, data):
-        # base64.decodebytes just calls binascii.a2b_base64;
-        # it seems overkill to use both base64 and binascii.
-        return cls(_decode_base64(data))
-
-    def asBase64(self, maxlinelength=76):
-        return _encode_base64(self.data, maxlinelength)
-
-    def __eq__(self, other):
-        if isinstance(other, self.__class__):
-            return self.data == other.data
-        elif isinstance(other, bytes):
-            return self.data == other
-        else:
-            return NotImplemented
-
-    def __repr__(self):
-        return "%s(%s)" % (self.__class__.__name__, repr(self.data))
-
-#
-#
-# End of deprecated functionality
-#
-#
-
-
 class UID:
     def __init__(self, data):
         if not isinstance(data, int):
@@ -202,7 +94,6 @@ def __eq__(self, other):
     def __hash__(self):
         return hash(self.data)
 
-
 #
 # XML support
 #
@@ -273,11 +164,10 @@ def _escape(text):
     return text
 
 class _PlistParser:
-    def __init__(self, use_builtin_types, dict_type):
+    def __init__(self, dict_type):
         self.stack = []
         self.current_key = None
         self.root = None
-        self._use_builtin_types = use_builtin_types
         self._dict_type = dict_type
 
     def parse(self, fileobj):
@@ -366,11 +256,7 @@ def end_string(self):
         self.add_object(self.get_data())
 
     def end_data(self):
-        if self._use_builtin_types:
-            self.add_object(_decode_base64(self.get_data()))
-
-        else:
-            self.add_object(Data.fromBase64(self.get_data()))
+        self.add_object(_decode_base64(self.get_data()))
 
     def end_date(self):
         self.add_object(_date_from_string(self.get_data()))
@@ -452,9 +338,6 @@ def write_value(self, value):
         elif isinstance(value, dict):
             self.write_dict(value)
 
-        elif isinstance(value, Data):
-            self.write_data(value)
-
         elif isinstance(value, (bytes, bytearray)):
             self.write_bytes(value)
 
@@ -467,9 +350,6 @@ def write_value(self, value):
         else:
             raise TypeError("unsupported type: %s" % type(value))
 
-    def write_data(self, data):
-        self.write_bytes(data.data)
-
     def write_bytes(self, data):
         self.begin_element("data")
         self._indent_level -= 1
@@ -563,8 +443,7 @@ class _BinaryPlistParser:
 
     see also: http://opensource.apple.com/source/CF/CF-744.18/CFBinaryPList.c
     """
-    def __init__(self, use_builtin_types, dict_type):
-        self._use_builtin_types = use_builtin_types
+    def __init__(self, dict_type):
         self._dict_type = dict_type
 
     def parse(self, fp):
@@ -664,10 +543,7 @@ def _read_object(self, ref):
 
         elif tokenH == 0x40:  # data
             s = self._get_size(tokenL)
-            if self._use_builtin_types:
-                result = self._fp.read(s)
-            else:
-                result = Data(self._fp.read(s))
+            result = self._fp.read(s)
 
         elif tokenH == 0x50:  # ascii string
             s = self._get_size(tokenL)
@@ -783,10 +659,6 @@ def _flatten(self, value):
             if (type(value), value) in self._objtable:
                 return
 
-        elif isinstance(value, Data):
-            if (type(value.data), value.data) in self._objtable:
-                return
-
         elif id(value) in self._objidtable:
             return
 
@@ -795,8 +667,6 @@ def _flatten(self, value):
         self._objlist.append(value)
         if isinstance(value, _scalars):
             self._objtable[(type(value), value)] = refnum
-        elif isinstance(value, Data):
-            self._objtable[(type(value.data), value.data)] = refnum
         else:
             self._objidtable[id(value)] = refnum
 
@@ -826,8 +696,6 @@ def _flatten(self, value):
     def _getrefnum(self, value):
         if isinstance(value, _scalars):
             return self._objtable[(type(value), value)]
-        elif isinstance(value, Data):
-            return self._objtable[(type(value.data), value.data)]
         else:
             return self._objidtable[id(value)]
 
@@ -885,10 +753,6 @@ def _write_object(self, value):
             f = (value - datetime.datetime(2001, 1, 1)).total_seconds()
             self._fp.write(struct.pack('>Bd', 0x33, f))
 
-        elif isinstance(value, Data):
-            self._write_size(0x40, len(value.data))
-            self._fp.write(value.data)
-
         elif isinstance(value, (bytes, bytearray)):
             self._write_size(0x40, len(value))
             self._fp.write(value)
@@ -970,7 +834,7 @@ def _is_fmt_binary(header):
 }
 
 
-def load(fp, *, fmt=None, use_builtin_types=True, dict_type=dict):
+def load(fp, *, fmt=None, dict_type=dict):
     """Read a .plist file. 'fp' should be a readable and binary file object.
     Return the unpacked root object (which usually is a dictionary).
     """
@@ -988,17 +852,16 @@ def load(fp, *, fmt=None, use_builtin_types=True, dict_type=dict):
     else:
         P = _FORMATS[fmt]['parser']
 
-    p = P(use_builtin_types=use_builtin_types, dict_type=dict_type)
+    p = P(dict_type=dict_type)
     return p.parse(fp)
 
 
-def loads(value, *, fmt=None, use_builtin_types=True, dict_type=dict):
+def loads(value, *, fmt=None, dict_type=dict):
     """Read a .plist file from a bytes object.
     Return the unpacked root object (which usually is a dictionary).
     """
     fp = BytesIO(value)
-    return load(
-        fp, fmt=fmt, use_builtin_types=use_builtin_types, dict_type=dict_type)
+    return load(fp, fmt=fmt, dict_type=dict_type)
 
 
 def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False):
diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py
index 5c2d0265079a..d3ef6f8ffe3a 100644
--- a/Lib/test/test_plistlib.py
+++ b/Lib/test/test_plistlib.py
@@ -263,14 +263,12 @@ def test_uid_copy(self):
         self.assertEqual(copy.deepcopy(UID(1)), UID(1))
 
     def test_appleformatting(self):
-        for use_builtin_types in (True, False):
-            for fmt in ALL_FORMATS:
-                with self.subTest(fmt=fmt, use_builtin_types=use_builtin_types):
-                    pl = plistlib.loads(TESTDATA[fmt],
-                        use_builtin_types=use_builtin_types)
-                    data = plistlib.dumps(pl, fmt=fmt)
-                    self.assertEqual(data, TESTDATA[fmt],
-                        "generated data was not identical to Apple's output")
+        for fmt in ALL_FORMATS:
+            with self.subTest(fmt=fmt):
+                pl = plistlib.loads(TESTDATA[fmt])
+                data = plistlib.dumps(pl, fmt=fmt)
+                self.assertEqual(data, TESTDATA[fmt],
+                    "generated data was not identical to Apple's output")
 
 
     def test_appleformattingfromliteral(self):
@@ -524,8 +522,7 @@ def test_dump_duplicates(self):
         # Test effectiveness of saving duplicated objects
         for x in (None, False, True, 12345, 123.45, 'abcde', b'abcde',
                   datetime.datetime(2004, 10, 26, 10, 33, 33),
-                  plistlib.Data(b'abcde'), bytearray(b'abcde'),
-                  [12, 345], (12, 345), {'12': 345}):
+                  bytearray(b'abcde'), [12, 345], (12, 345), {'12': 345}):
             with self.subTest(x=x):
                 data = plistlib.dumps([x]*1000, fmt=plistlib.FMT_BINARY)
                 self.assertLess(len(data), 1100, repr(data))
@@ -533,8 +530,7 @@ def test_dump_duplicates(self):
     def test_identity(self):
         for x in (None, False, True, 12345, 123.45, 'abcde', b'abcde',
                   datetime.datetime(2004, 10, 26, 10, 33, 33),
-                  plistlib.Data(b'abcde'), bytearray(b'abcde'),
-                  [12, 345], (12, 345), {'12': 345}):
+                  bytearray(b'abcde'), [12, 345], (12, 345), {'12': 345}):
             with self.subTest(x=x):
                 data = plistlib.dumps([x]*2, fmt=plistlib.FMT_BINARY)
                 a, b = plistlib.loads(data)
@@ -621,95 +617,6 @@ def test_invalid_binary(self):
                 plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY)
 
 
-class TestPlistlibDeprecated(unittest.TestCase):
-    def test_io_deprecated(self):
-        pl_in = {
-            'key': 42,
-            'sub': {
-                'key': 9,
-                'alt': 'value',
-                'data': b'buffer',
-            }
-        }
-        pl_out = {
-            'key': 42,
-            'sub': {
-                'key': 9,
-                'alt': 'value',
-                'data': plistlib.Data(b'buffer'),
-            }
-        }
-
-        self.addCleanup(support.unlink, support.TESTFN)
-        with self.assertWarns(DeprecationWarning):
-            plistlib.writePlist(pl_in, support.TESTFN)
-
-        with self.assertWarns(DeprecationWarning):
-            pl2 = plistlib.readPlist(support.TESTFN)
-
-        self.assertEqual(pl_out, pl2)
-
-        os.unlink(support.TESTFN)
-
-        with open(support.TESTFN, 'wb') as fp:
-            with self.assertWarns(DeprecationWarning):
-                plistlib.writePlist(pl_in, fp)
-
-        with open(support.TESTFN, 'rb') as fp:
-            with self.assertWarns(DeprecationWarning):
-                pl2 = plistlib.readPlist(fp)
-
-        self.assertEqual(pl_out, pl2)
-
-    def test_bytes_deprecated(self):
-        pl = {
-            'key': 42,
-            'sub': {
-                'key': 9,
-                'alt': 'value',
-                'data': b'buffer',
-            }
-        }
-        with self.assertWarns(DeprecationWarning):
-            data = plistlib.writePlistToBytes(pl)
-
-        with self.assertWarns(DeprecationWarning):
-            pl2 = plistlib.readPlistFromBytes(data)
-
-        self.assertIsInstance(pl2, dict)
-        self.assertEqual(pl2, dict(
-            key=42,
-            sub=dict(
-                key=9,
-                alt='value',
-                data=plistlib.Data(b'buffer'),
-            )
-        ))
-
-        with self.assertWarns(DeprecationWarning):
-            data2 = plistlib.writePlistToBytes(pl2)
-        self.assertEqual(data, data2)
-
-    def test_dataobject_deprecated(self):
-        in_data = { 'key': plistlib.Data(b'hello') }
-        out_data = { 'key': b'hello' }
-
-        buf = plistlib.dumps(in_data)
-
-        cur = plistlib.loads(buf)
-        self.assertEqual(cur, out_data)
-        self.assertEqual(cur, in_data)
-
-        cur = plistlib.loads(buf, use_builtin_types=False)
-        self.assertEqual(cur, out_data)
-        self.assertEqual(cur, in_data)
-
-        with self.assertWarns(DeprecationWarning):
-            cur = plistlib.readPlistFromBytes(buf)
-        self.assertEqual(cur, out_data)
-        self.assertEqual(cur, in_data)
-
-
 class TestKeyedArchive(unittest.TestCase):
     def test_keyed_archive_data(self):
         # This is the structure of a NSKeyedArchive packed plist
@@ -749,7 +656,7 @@ def test__all__(self):
 
 
 def test_main():
-    support.run_unittest(TestPlistlib, TestPlistlibDeprecated, TestKeyedArchive, MiscTestCase)
+    support.run_unittest(TestPlistlib, TestKeyedArchive, MiscTestCase)
 
 
 if __name__ == '__main__':
diff --git a/Misc/NEWS.d/next/Library/2019-03-23-16-42-46.bpo-36409.ZYVKao.rst b/Misc/NEWS.d/next/Library/2019-03-23-16-42-46.bpo-36409.ZYVKao.rst
new file mode 100644
index 000000000000..e516d3190f81
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-03-23-16-42-46.bpo-36409.ZYVKao.rst
@@ -0,0 +1 @@
+Remove the old plistlib API deprecated in Python 3.4
\ No newline at end of file


From webhook-mailer at python.org  Thu Sep  5 07:10:53 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 05 Sep 2019 11:10:53 -0000
Subject: [Python-checkins] Correct minor gramatical mistake in sys.settrace
 doc (GH-15637)
Message-ID: <mailman.252.1567681854.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3038e87ba848023470f571242a8bb5a206c24430
commit: 3038e87ba848023470f571242a8bb5a206c24430
branch: master
author: Andre Delfino <adelfino at gmail.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-05T13:10:37+02:00
summary:

Correct minor gramatical mistake in sys.settrace doc (GH-15637)

files:
M Doc/library/sys.rst

diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 6119bee10ab3..01df026a6e8a 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -1253,7 +1253,8 @@ always available.
 
    The trace function is invoked (with *event* set to ``'call'``) whenever a new
    local scope is entered; it should return a reference to a local trace
-   function to be used that scope, or ``None`` if the scope shouldn't be traced.
+   function to be used for the new scope, or ``None`` if the scope shouldn't be
+   traced.
 
    The local trace function should return a reference to itself (or to another
    function for further tracing in that scope), or ``None`` to turn off tracing


From webhook-mailer at python.org  Thu Sep  5 07:17:46 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 05 Sep 2019 11:17:46 -0000
Subject: [Python-checkins] Correct minor gramatical mistake in sys.settrace
 doc (GH-15637)
Message-ID: <mailman.253.1567682267.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7eaeddad75cc736d327b8ece9380ef6ee23a0d9a
commit: 7eaeddad75cc736d327b8ece9380ef6ee23a0d9a
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-05T04:17:41-07:00
summary:

Correct minor gramatical mistake in sys.settrace doc (GH-15637)

(cherry picked from commit 3038e87ba848023470f571242a8bb5a206c24430)

Co-authored-by: Andre Delfino <adelfino at gmail.com>

files:
M Doc/library/sys.rst

diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 09a987ca32ac..be1af371d428 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -1276,7 +1276,8 @@ always available.
 
    The trace function is invoked (with *event* set to ``'call'``) whenever a new
    local scope is entered; it should return a reference to a local trace
-   function to be used that scope, or ``None`` if the scope shouldn't be traced.
+   function to be used for the new scope, or ``None`` if the scope shouldn't be
+   traced.
 
    The local trace function should return a reference to itself (or to another
    function for further tracing in that scope), or ``None`` to turn off tracing


From webhook-mailer at python.org  Thu Sep  5 07:31:53 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 05 Sep 2019 11:31:53 -0000
Subject: [Python-checkins] Correct minor gramatical mistake in sys.settrace
 doc (GH-15637)
Message-ID: <mailman.254.1567683114.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/256f03befb1f144340e11bc6eee92d3afd84ee23
commit: 256f03befb1f144340e11bc6eee92d3afd84ee23
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-05T04:31:50-07:00
summary:

Correct minor gramatical mistake in sys.settrace doc (GH-15637)

(cherry picked from commit 3038e87ba848023470f571242a8bb5a206c24430)

Co-authored-by: Andre Delfino <adelfino at gmail.com>

files:
M Doc/library/sys.rst

diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index e54f2526a8df..121464f9470b 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -1185,7 +1185,8 @@ always available.
 
    The trace function is invoked (with *event* set to ``'call'``) whenever a new
    local scope is entered; it should return a reference to a local trace
-   function to be used that scope, or ``None`` if the scope shouldn't be traced.
+   function to be used for the new scope, or ``None`` if the scope shouldn't be
+   traced.
 
    The local trace function should return a reference to itself (or to another
    function for further tracing in that scope), or ``None`` to turn off tracing


From webhook-mailer at python.org  Thu Sep  5 10:55:02 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 05 Sep 2019 14:55:02 -0000
Subject: [Python-checkins] bpo-37064: Add option -k to
 Tools/scripts/pathfix.py (GH-15548)
Message-ID: <mailman.255.1567695303.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/50254ac4c179cb412e90682098c97db786143929
commit: 50254ac4c179cb412e90682098c97db786143929
branch: master
author: PatrikKopkan <kopkanpatrik at gmail.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-05T16:54:54+02:00
summary:

bpo-37064: Add option -k to Tools/scripts/pathfix.py (GH-15548)

Add flag -k to pathscript.py script: preserve shebang flags.

files:
A Lib/test/test_tools/test_pathfix.py
A Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
M Tools/scripts/pathfix.py

diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py
new file mode 100644
index 000000000000..5c0dd1d5a60c
--- /dev/null
+++ b/Lib/test/test_tools/test_pathfix.py
@@ -0,0 +1,64 @@
+import os
+import subprocess
+import sys
+import unittest
+from test import support
+from test.test_tools import import_tool, scriptsdir
+
+
+class TestPathfixFunctional(unittest.TestCase):
+    script = os.path.join(scriptsdir, 'pathfix.py')
+
+    def setUp(self):
+        self.temp_file = support.TESTFN
+        self.addCleanup(support.unlink, support.TESTFN)
+
+    def pathfix(self, shebang, pathfix_flags):
+        with open(self.temp_file, 'w', encoding='utf8') as f:
+            f.write(f'{shebang}\n' + 'print("Hello world")\n')
+
+        proc = subprocess.run(
+            [sys.executable, self.script,
+             *pathfix_flags, '-n', self.temp_file],
+            capture_output=True)
+        self.assertEqual(proc.returncode, 0, proc)
+
+        with open(self.temp_file, 'r', encoding='utf8') as f:
+            output = f.read()
+
+        lines = output.split('\n')
+        self.assertEqual(lines[1:], ['print("Hello world")', ''])
+        shebang = lines[0]
+        return shebang
+
+    def test_pathfix(self):
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python',
+                ['-i', '/usr/bin/python3',]),
+            '#! /usr/bin/python3',
+        )
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python -R',
+                ['-i', '/usr/bin/python3', ]),
+            '#! /usr/bin/python3',
+        )
+
+    def test_pathfix_keeping_flags(self):
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python -R',
+                ['-i', '/usr/bin/python3', '-k',]),
+            '#! /usr/bin/python3 -R',
+        )
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python',
+                ['-i', '/usr/bin/python3', '-k',]),
+            '#! /usr/bin/python3',
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst b/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
new file mode 100644
index 000000000000..02599077c9d5
--- /dev/null
+++ b/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
@@ -0,0 +1 @@
+Add flag -k to pathscript.py script: preserve shebang flags.
\ No newline at end of file
diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py
index 1a0cf1c9e69c..eee684e3502d 100755
--- a/Tools/scripts/pathfix.py
+++ b/Tools/scripts/pathfix.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Change the #! line occurring in Python scripts.  The new interpreter
+# Change the #! line (shebang) occurring in Python scripts.  The new interpreter
 # pathname must be given with a -i option.
 #
 # Command line arguments are files or directories to be processed.
@@ -10,7 +10,12 @@
 # arguments).
 # The original file is kept as a back-up (with a "~" attached to its name),
 # -n flag can be used to disable this.
-#
+
+# Sometimes you may find shebangs with flags such as `#! /usr/bin/env python -si`.
+# Normally, pathfix overwrites the entire line, including the flags.
+# To change interpreter and keep flags from the original shebang line, use -k.
+
+
 # Undoubtedly you can do this using find and sed or perl, but this is
 # a nice example of Python code that recurses down a directory tree
 # and uses regular expressions.  Also note several subtleties like
@@ -33,16 +38,19 @@
 new_interpreter = None
 preserve_timestamps = False
 create_backup = True
+keep_flags = False
 
 
 def main():
     global new_interpreter
     global preserve_timestamps
     global create_backup
-    usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' %
+    global keep_flags
+
+    usage = ('usage: %s -i /interpreter -p -n -k file-or-directory ...\n' %
              sys.argv[0])
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'i:pn')
+        opts, args = getopt.getopt(sys.argv[1:], 'i:kpn')
     except getopt.error as msg:
         err(str(msg) + '\n')
         err(usage)
@@ -54,6 +62,8 @@ def main():
             preserve_timestamps = True
         if o == '-n':
             create_backup = False
+        if o == '-k':
+            keep_flags = True
     if not new_interpreter or not new_interpreter.startswith(b'/') or \
            not args:
         err('-i option or file-or-directory missing\n')
@@ -70,10 +80,14 @@ def main():
             if fix(arg): bad = 1
     sys.exit(bad)
 
+
 ispythonprog = re.compile(r'^[a-zA-Z0-9_]+\.py$')
+
+
 def ispython(name):
     return bool(ispythonprog.match(name))
 
+
 def recursedown(dirname):
     dbg('recursedown(%r)\n' % (dirname,))
     bad = 0
@@ -96,6 +110,7 @@ def recursedown(dirname):
         if recursedown(fullname): bad = 1
     return bad
 
+
 def fix(filename):
 ##  dbg('fix(%r)\n' % (filename,))
     try:
@@ -164,12 +179,26 @@ def fix(filename):
     # Return success
     return 0
 
+
+def parse_shebang(shebangline):
+    shebangline = shebangline.rstrip(b'\n')
+    start = shebangline.find(b' -')
+    if start == -1:
+        return b''
+    return shebangline[start:]
+
+
 def fixline(line):
     if not line.startswith(b'#!'):
         return line
+
     if b"python" not in line:
         return line
-    return b'#! ' + new_interpreter + b'\n'
+    flags = b''
+    if keep_flags:
+        flags = parse_shebang(line)
+    return b'#! ' + new_interpreter + flags + b'\n'
+
 
 if __name__ == '__main__':
     main()


From webhook-mailer at python.org  Thu Sep  5 11:06:52 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 05 Sep 2019 15:06:52 -0000
Subject: [Python-checkins] bpo-36797: Fix a dead link in
 Doc/distutils/apiref (GH-15700)
Message-ID: <mailman.256.1567696013.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2c2b561967c1916855399f809e30ae0ba7e09ae2
commit: 2c2b561967c1916855399f809e30ae0ba7e09ae2
branch: master
author: Miro Hron?ok <miro at hroncok.cz>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-05T08:06:45-07:00
summary:

bpo-36797: Fix a dead link in  Doc/distutils/apiref (GH-15700)



https://bugs.python.org/issue36797

files:
A Misc/NEWS.d/next/Documentation/2019-09-05-14-47-51.bpo-36797.KN9Ga5.rst
M Doc/distutils/apiref.rst

diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst
index 937f19f57be9..a42d2d3559f2 100644
--- a/Doc/distutils/apiref.rst
+++ b/Doc/distutils/apiref.rst
@@ -6,12 +6,12 @@ API Reference
 
 .. seealso::
 
-   `New and changed setup.py arguments in setuptools <setuptools-setup-py>`_
+   `New and changed setup.py arguments in setuptools`_
       The ``setuptools`` project adds new capabilities to the ``setup`` function
       and other APIs, makes the API consistent across different Python versions,
       and is hence recommended over using ``distutils`` directly.
 
-.. _setuptools-setup-py: https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords
+.. _New and changed setup.py arguments in setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords
 
 .. include:: ./_setuptools_disclaimer.rst
 
diff --git a/Misc/NEWS.d/next/Documentation/2019-09-05-14-47-51.bpo-36797.KN9Ga5.rst b/Misc/NEWS.d/next/Documentation/2019-09-05-14-47-51.bpo-36797.KN9Ga5.rst
new file mode 100644
index 000000000000..62a3c176b1c7
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-09-05-14-47-51.bpo-36797.KN9Ga5.rst
@@ -0,0 +1 @@
+Fix a dead link in the distutils API Reference.


From webhook-mailer at python.org  Thu Sep  5 12:06:55 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 05 Sep 2019 16:06:55 -0000
Subject: [Python-checkins] bpo-37878: Remove PyThreadState_DeleteCurrent()
 function (GH-15315)
Message-ID: <mailman.257.1567699616.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2bc43cdc015eda4f1a651bb2b308a17a83c38e14
commit: 2bc43cdc015eda4f1a651bb2b308a17a83c38e14
branch: master
author: Joannah Nanjekye <33177550+nanjekyejoannah at users.noreply.github.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-05T18:06:49+02:00
summary:

bpo-37878: Remove PyThreadState_DeleteCurrent() function (GH-15315)

* Rename PyThreadState_DeleteCurrent()
  to _PyThreadState_DeleteCurrent()
* Move it to the internal C API

Co-Authored-By: Carol Willing <carolcode at willingconsulting.com>

files:
A Misc/NEWS.d/next/Documentation/2019-08-16-20-01-10.bpo-37878.MvA6rZ.rst
M Doc/whatsnew/3.9.rst
M Include/internal/pycore_pylifecycle.h
M Include/pystate.h
M Modules/_threadmodule.c
M Modules/_tracemalloc.c
M Python/pystate.c

diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
index 009a66adff7d..756690d93793 100644
--- a/Doc/whatsnew/3.9.rst
+++ b/Doc/whatsnew/3.9.rst
@@ -219,6 +219,9 @@ Removed
   standard :class:`bytes` objects are always used.
   (Contributed by Jon Janzen in :issue:`36409`.)
 
+* ``PyThreadState_DeleteCurrent()`` has been removed. It was not documented.
+  (Contributed by Joannah Nanjekye in :issue:`37878`.)
+
 
 Porting to Python 3.9
 =====================
diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index 7ea4488a9364..cf844cc063f8 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -110,6 +110,8 @@ PyAPI_FUNC(void) _PyErr_Print(PyThreadState *tstate);
 PyAPI_FUNC(void) _PyErr_Display(PyObject *file, PyObject *exception,
                                 PyObject *value, PyObject *tb);
 
+PyAPI_FUNC(void) _PyThreadState_DeleteCurrent(_PyRuntimeState *runtime);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/pystate.h b/Include/pystate.h
index 4c25e3f7ef84..1cb2305696f2 100644
--- a/Include/pystate.h
+++ b/Include/pystate.h
@@ -50,7 +50,6 @@ PyAPI_FUNC(PyObject*) PyState_FindModule(struct PyModuleDef*);
 PyAPI_FUNC(PyThreadState *) PyThreadState_New(PyInterpreterState *);
 PyAPI_FUNC(void) PyThreadState_Clear(PyThreadState *);
 PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *);
-PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void);
 
 /* Get the current thread state.
 
diff --git a/Misc/NEWS.d/next/Documentation/2019-08-16-20-01-10.bpo-37878.MvA6rZ.rst b/Misc/NEWS.d/next/Documentation/2019-08-16-20-01-10.bpo-37878.MvA6rZ.rst
new file mode 100644
index 000000000000..8da4eb9717d8
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-08-16-20-01-10.bpo-37878.MvA6rZ.rst
@@ -0,0 +1 @@
+Make :c:func:`PyThreadState_DeleteCurrent` Internal.
diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c
index a3ecd2e74e8c..1c7df3f87414 100644
--- a/Modules/_threadmodule.c
+++ b/Modules/_threadmodule.c
@@ -993,6 +993,7 @@ struct bootstate {
     PyObject *args;
     PyObject *keyw;
     PyThreadState *tstate;
+    _PyRuntimeState *runtime;
 };
 
 static void
@@ -1000,11 +1001,13 @@ t_bootstrap(void *boot_raw)
 {
     struct bootstate *boot = (struct bootstate *) boot_raw;
     PyThreadState *tstate;
+    _PyRuntimeState *runtime;
     PyObject *res;
 
+    runtime = boot->runtime;
     tstate = boot->tstate;
     tstate->thread_id = PyThread_get_thread_ident();
-    _PyThreadState_Init(&_PyRuntime, tstate);
+    _PyThreadState_Init(runtime, tstate);
     PyEval_AcquireThread(tstate);
     tstate->interp->num_threads++;
     res = PyObject_Call(boot->func, boot->args, boot->keyw);
@@ -1025,13 +1028,14 @@ t_bootstrap(void *boot_raw)
     PyMem_DEL(boot_raw);
     tstate->interp->num_threads--;
     PyThreadState_Clear(tstate);
-    PyThreadState_DeleteCurrent();
+    _PyThreadState_DeleteCurrent(runtime);
     PyThread_exit_thread();
 }
 
 static PyObject *
 thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs)
 {
+    _PyRuntimeState *runtime = &_PyRuntime;
     PyObject *func, *args, *keyw = NULL;
     struct bootstate *boot;
     unsigned long ident;
@@ -1062,6 +1066,7 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs)
     boot->args = args;
     boot->keyw = keyw;
     boot->tstate = _PyThreadState_Prealloc(boot->interp);
+    boot->runtime = runtime;
     if (boot->tstate == NULL) {
         PyMem_DEL(boot);
         return PyErr_NoMemory();
diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c
index ee32ac29b7ee..26d76004aaa5 100644
--- a/Modules/_tracemalloc.c
+++ b/Modules/_tracemalloc.c
@@ -34,7 +34,7 @@ static struct {
 #if defined(TRACE_RAW_MALLOC)
 /* This lock is needed because tracemalloc_free() is called without
    the GIL held from PyMem_RawFree(). It cannot acquire the lock because it
-   would introduce a deadlock in PyThreadState_DeleteCurrent(). */
+   would introduce a deadlock in _PyThreadState_DeleteCurrent(). */
 static PyThread_type_lock tables_lock;
 #  define TABLES_LOCK() PyThread_acquire_lock(tables_lock, 1)
 #  define TABLES_UNLOCK() PyThread_release_lock(tables_lock)
@@ -728,7 +728,7 @@ tracemalloc_free(void *ctx, void *ptr)
         return;
 
      /* GIL cannot be locked in PyMem_RawFree() because it would introduce
-        a deadlock in PyThreadState_DeleteCurrent(). */
+        a deadlock in _PyThreadState_DeleteCurrent(). */
 
     alloc->free(alloc->ctx, ptr);
 
diff --git a/Python/pystate.c b/Python/pystate.c
index dc5240048ba0..02bc90394361 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -809,7 +809,7 @@ PyThreadState_Clear(PyThreadState *tstate)
 }
 
 
-/* Common code for PyThreadState_Delete() and PyThreadState_DeleteCurrent() */
+/* Common code for PyThreadState_Delete() and _PyThreadState_DeleteCurrent() */
 static void
 tstate_delete_common(_PyRuntimeState *runtime, PyThreadState *tstate)
 {
@@ -858,14 +858,14 @@ PyThreadState_Delete(PyThreadState *tstate)
 }
 
 
-static void
+void
 _PyThreadState_DeleteCurrent(_PyRuntimeState *runtime)
 {
     struct _gilstate_runtime_state *gilstate = &runtime->gilstate;
     PyThreadState *tstate = _PyRuntimeGILState_GetThreadState(gilstate);
     if (tstate == NULL)
         Py_FatalError(
-            "PyThreadState_DeleteCurrent: no current tstate");
+            "_PyThreadState_DeleteCurrent: no current tstate");
     tstate_delete_common(runtime, tstate);
     if (gilstate->autoInterpreterState &&
         PyThread_tss_get(&gilstate->autoTSSkey) == tstate)
@@ -876,12 +876,6 @@ _PyThreadState_DeleteCurrent(_PyRuntimeState *runtime)
     PyEval_ReleaseLock();
 }
 
-void
-PyThreadState_DeleteCurrent()
-{
-    _PyThreadState_DeleteCurrent(&_PyRuntime);
-}
-
 
 /*
  * Delete all thread states except the one passed as argument.


From webhook-mailer at python.org  Thu Sep  5 12:09:50 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 05 Sep 2019 16:09:50 -0000
Subject: [Python-checkins] bpo-37064: Skip test_tools.test_pathfix if
 installed (GH-15705)
Message-ID: <mailman.258.1567699790.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3f43ceff186da09978d0aff257bb18b8ac7611f7
commit: 3f43ceff186da09978d0aff257bb18b8ac7611f7
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-05T18:09:46+02:00
summary:

bpo-37064: Skip test_tools.test_pathfix if installed (GH-15705)

If Python is installed, skip test_tools.test_pathfix test because
Tools/scripts/pathfix.py script is not installed.

files:
M Lib/test/test_tools/test_pathfix.py

diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py
index 5c0dd1d5a60c..0c1aea721170 100644
--- a/Lib/test/test_tools/test_pathfix.py
+++ b/Lib/test/test_tools/test_pathfix.py
@@ -3,7 +3,11 @@
 import sys
 import unittest
 from test import support
-from test.test_tools import import_tool, scriptsdir
+from test.test_tools import import_tool, scriptsdir, skip_if_missing
+
+
+# need Tools/script/ directory: skip if run on Python installed on the system
+skip_if_missing()
 
 
 class TestPathfixFunctional(unittest.TestCase):


From webhook-mailer at python.org  Fri Sep  6 02:01:00 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Fri, 06 Sep 2019 06:01:00 -0000
Subject: [Python-checkins] replace inline function `is_small_int` with a
 macro version (GH-15710)
Message-ID: <mailman.259.1567749660.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6b519985d23bd0f0bd072b5d5d5f2c60a81a19f2
commit: 6b519985d23bd0f0bd072b5d5d5f2c60a81a19f2
branch: master
author: animalize <animalize at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-05T23:00:56-07:00
summary:

replace inline function `is_small_int` with a macro version (GH-15710)

files:
M Misc/NEWS.d/next/Core and Builtins/2019-08-23-22-46-25.bpo-37812.vsWZwS.rst
M Objects/longobject.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-23-22-46-25.bpo-37812.vsWZwS.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-23-22-46-25.bpo-37812.vsWZwS.rst
index 90c3ff3f7ef9..f1085a1791c8 100644
--- a/Misc/NEWS.d/next/Core and Builtins/2019-08-23-22-46-25.bpo-37812.vsWZwS.rst	
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-23-22-46-25.bpo-37812.vsWZwS.rst	
@@ -1,3 +1,2 @@
 The ``CHECK_SMALL_INT`` macro used inside :file:`Object/longobject.c` has
-been replaced with an explicit ``return`` at each call site, conditioned on
-a ``static inline`` function ``is_small_int``.
+been replaced with an explicit ``return`` at each call site.
diff --git a/Objects/longobject.c b/Objects/longobject.c
index 475b9bda59f9..4cf2b0726e81 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -42,11 +42,7 @@ PyObject *_PyLong_One = NULL;
 */
 static PyLongObject small_ints[NSMALLNEGINTS + NSMALLPOSINTS];
 
-static inline int
-is_small_int(long long ival)
-{
-    return -NSMALLNEGINTS <= ival && ival < NSMALLPOSINTS;
-}
+#define IS_SMALL_INT(ival) (-NSMALLNEGINTS <= (ival) && (ival) < NSMALLPOSINTS)
 
 #ifdef COUNT_ALLOCS
 Py_ssize_t _Py_quick_int_allocs, _Py_quick_neg_int_allocs;
@@ -56,7 +52,7 @@ static PyObject *
 get_small_int(sdigit ival)
 {
     PyObject *v;
-    assert(is_small_int(ival));
+    assert(IS_SMALL_INT(ival));
     v = (PyObject *)&small_ints[ival + NSMALLNEGINTS];
     Py_INCREF(v);
 #ifdef COUNT_ALLOCS
@@ -73,7 +69,7 @@ maybe_small_long(PyLongObject *v)
 {
     if (v && Py_ABS(Py_SIZE(v)) <= 1) {
         sdigit ival = MEDIUM_VALUE(v);
-        if (is_small_int(ival)) {
+        if (IS_SMALL_INT(ival)) {
             Py_DECREF(v);
             return (PyLongObject *)get_small_int(ival);
         }
@@ -81,8 +77,8 @@ maybe_small_long(PyLongObject *v)
     return v;
 }
 #else
-#define is_small_int(ival) 0
-#define get_small_int(ival) (assert(0), NULL)
+#define IS_SMALL_INT(ival) 0
+#define get_small_int(ival) (Py_UNREACHABLE(), NULL)
 #define maybe_small_long(val) (val)
 #endif
 
@@ -297,7 +293,7 @@ _PyLong_Copy(PyLongObject *src)
         i = -(i);
     if (i < 2) {
         sdigit ival = MEDIUM_VALUE(src);
-        if (is_small_int(ival)) {
+        if (IS_SMALL_INT(ival)) {
             return get_small_int(ival);
         }
     }
@@ -321,7 +317,7 @@ PyLong_FromLong(long ival)
     int ndigits = 0;
     int sign;
 
-    if (is_small_int(ival)) {
+    if (IS_SMALL_INT(ival)) {
         return get_small_int((sdigit)ival);
     }
 
@@ -1154,7 +1150,7 @@ PyLong_FromLongLong(long long ival)
     int ndigits = 0;
     int negative = 0;
 
-    if (is_small_int(ival)) {
+    if (IS_SMALL_INT(ival)) {
         return get_small_int((sdigit)ival);
     }
 
@@ -1229,7 +1225,7 @@ PyLong_FromSsize_t(Py_ssize_t ival)
     int ndigits = 0;
     int negative = 0;
 
-    if (is_small_int(ival)) {
+    if (IS_SMALL_INT(ival)) {
         return get_small_int((sdigit)ival);
     }
 


From webhook-mailer at python.org  Fri Sep  6 02:02:31 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Fri, 06 Sep 2019 06:02:31 -0000
Subject: [Python-checkins] More refinements to the statistics docs (GH-15713)
Message-ID: <mailman.260.1567749751.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d8c93aa5d29d3cab537357018d5806a57452a8fe
commit: d8c93aa5d29d3cab537357018d5806a57452a8fe
branch: master
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-05T23:02:27-07:00
summary:

More refinements to the statistics docs (GH-15713)

files:
M Doc/library/statistics.rst

diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index 27d0518b4a79..0798ae29118e 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -19,17 +19,21 @@
 --------------
 
 This module provides functions for calculating mathematical statistics of
-numeric (:class:`Real`-valued) data.
-
-.. note::
-
-   Unless explicitly noted otherwise, these functions support :class:`int`,
-   :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`.
-   Behaviour with other types (whether in the numeric tower or not) is
-   currently unsupported.  Collections with a mix of types are also undefined
-   and implementation-dependent.  If your input data consists of mixed types,
-   you may be able to use :func:`map` to ensure a consistent result, for
-   example: ``map(float, input_data)``.
+numeric (:class:`~numbers.Real`-valued) data.
+
+The module is not intended to be a competitor to third-party libraries such
+as `NumPy <https://numpy.org>`_, `SciPy <https://www.scipy.org/>`_, or
+proprietary full-featured statistics packages aimed at professional
+statisticians such as Minitab, SAS and Matlab. It is aimed at the level of
+graphing and scientific calculators.
+
+Unless explicitly noted, these functions support :class:`int`,
+:class:`float`, :class:`~decimal.Decimal` and :class:`~fractions.Fraction`.
+Behaviour with other types (whether in the numeric tower or not) is
+currently unsupported.  Collections with a mix of types are also undefined
+and implementation-dependent.  If your input data consists of mixed types,
+you may be able to use :func:`map` to ensure a consistent result, for
+example: ``map(float, input_data)``.
 
 Averages and measures of central location
 -----------------------------------------
@@ -107,7 +111,7 @@ However, for reading convenience, most of the examples show sorted sequences.
       :func:`median` and :func:`mode`.
 
       The sample mean gives an unbiased estimate of the true population mean,
-      which means that, taken on average over all the possible samples,
+      so that when taken on average over all the possible samples,
       ``mean(sample)`` converges on the true mean of the entire population.  If
       *data* represents the entire population rather than a sample, then
       ``mean(data)`` is equivalent to calculating the true population mean ?.
@@ -163,8 +167,16 @@ However, for reading convenience, most of the examples show sorted sequences.
    will be equivalent to ``3/(1/a + 1/b + 1/c)``.
 
    The harmonic mean is a type of average, a measure of the central
-   location of the data.  It is often appropriate when averaging quantities
-   which are rates or ratios, for example speeds. For example:
+   location of the data.  It is often appropriate when averaging
+   rates or ratios, for example speeds.
+
+   Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr.
+   What is the average speed?
+
+   .. doctest::
+
+      >>> harmonic_mean([40, 60])
+      48.0
 
    Suppose an investor purchases an equal value of shares in each of
    three companies, with P/E (price/earning) ratios of 2.5, 3 and 10.
@@ -175,9 +187,6 @@ However, for reading convenience, most of the examples show sorted sequences.
       >>> harmonic_mean([2.5, 3, 10])  # For an equal investment portfolio.
       3.6
 
-   Using the arithmetic mean would give an average of about 5.167, which
-   is well over the aggregate P/E ratio.
-
    :exc:`StatisticsError` is raised if *data* is empty, or any element
    is less than zero.
 
@@ -190,9 +199,9 @@ However, for reading convenience, most of the examples show sorted sequences.
    middle two" method.  If *data* is empty, :exc:`StatisticsError` is raised.
    *data* can be a sequence or iterator.
 
-   The median is a robust measure of central location, and is less affected by
-   the presence of outliers in your data.  When the number of data points is
-   odd, the middle data point is returned:
+   The median is a robust measure of central location and is less affected by
+   the presence of outliers.  When the number of data points is odd, the
+   middle data point is returned:
 
    .. doctest::
 
@@ -210,13 +219,10 @@ However, for reading convenience, most of the examples show sorted sequences.
    This is suited for when your data is discrete, and you don't mind that the
    median may not be an actual data point.
 
-   If your data is ordinal (supports order operations) but not numeric (doesn't
-   support addition), you should use :func:`median_low` or :func:`median_high`
+   If the data is ordinal (supports order operations) but not numeric (doesn't
+   support addition), consider using :func:`median_low` or :func:`median_high`
    instead.
 
-   .. seealso:: :func:`median_low`, :func:`median_high`, :func:`median_grouped`
-
-
 .. function:: median_low(data)
 
    Return the low median of numeric data.  If *data* is empty,
@@ -319,7 +325,7 @@ However, for reading convenience, most of the examples show sorted sequences.
    desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``.
    If the input *data* is empty, :exc:`StatisticsError` is raised.
 
-   ``mode`` assumes discrete data, and returns a single value. This is the
+   ``mode`` assumes discrete data and returns a single value. This is the
    standard treatment of the mode as commonly taught in schools:
 
    .. doctest::
@@ -522,7 +528,7 @@ However, for reading convenience, most of the examples show sorted sequences.
    cut-point will evaluate to ``104``.
 
    The *method* for computing quantiles can be varied depending on
-   whether the data in *data* includes or excludes the lowest and
+   whether the *data* includes or excludes the lowest and
    highest possible values from the population.
 
    The default *method* is "exclusive" and is used for data sampled from


From webhook-mailer at python.org  Fri Sep  6 02:19:28 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Fri, 06 Sep 2019 06:19:28 -0000
Subject: [Python-checkins] More refinements to the statistics docs
 (GH-15713) (GH-15715)
Message-ID: <mailman.261.1567750770.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b8c66779c7003071f1a330428d58bbbb34c7ae12
commit: b8c66779c7003071f1a330428d58bbbb34c7ae12
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-05T23:19:13-07:00
summary:

More refinements to the statistics docs (GH-15713) (GH-15715)

(cherry picked from commit d8c93aa5d29d3cab537357018d5806a57452a8fe)

Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com>

files:
M Doc/library/statistics.rst

diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index 2ddd393a4593..9e39828a5700 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -19,17 +19,21 @@
 --------------
 
 This module provides functions for calculating mathematical statistics of
-numeric (:class:`Real`-valued) data.
-
-.. note::
-
-   Unless explicitly noted otherwise, these functions support :class:`int`,
-   :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`.
-   Behaviour with other types (whether in the numeric tower or not) is
-   currently unsupported.  Collections with a mix of types are also undefined
-   and implementation-dependent.  If your input data consists of mixed types,
-   you may be able to use :func:`map` to ensure a consistent result, for
-   example: ``map(float, input_data)``.
+numeric (:class:`~numbers.Real`-valued) data.
+
+The module is not intended to be a competitor to third-party libraries such
+as `NumPy <https://numpy.org>`_, `SciPy <https://www.scipy.org/>`_, or
+proprietary full-featured statistics packages aimed at professional
+statisticians such as Minitab, SAS and Matlab. It is aimed at the level of
+graphing and scientific calculators.
+
+Unless explicitly noted, these functions support :class:`int`,
+:class:`float`, :class:`~decimal.Decimal` and :class:`~fractions.Fraction`.
+Behaviour with other types (whether in the numeric tower or not) is
+currently unsupported.  Collections with a mix of types are also undefined
+and implementation-dependent.  If your input data consists of mixed types,
+you may be able to use :func:`map` to ensure a consistent result, for
+example: ``map(float, input_data)``.
 
 Averages and measures of central location
 -----------------------------------------
@@ -107,7 +111,7 @@ However, for reading convenience, most of the examples show sorted sequences.
       :func:`median` and :func:`mode`.
 
       The sample mean gives an unbiased estimate of the true population mean,
-      which means that, taken on average over all the possible samples,
+      so that when taken on average over all the possible samples,
       ``mean(sample)`` converges on the true mean of the entire population.  If
       *data* represents the entire population rather than a sample, then
       ``mean(data)`` is equivalent to calculating the true population mean ?.
@@ -163,8 +167,16 @@ However, for reading convenience, most of the examples show sorted sequences.
    will be equivalent to ``3/(1/a + 1/b + 1/c)``.
 
    The harmonic mean is a type of average, a measure of the central
-   location of the data.  It is often appropriate when averaging quantities
-   which are rates or ratios, for example speeds. For example:
+   location of the data.  It is often appropriate when averaging
+   rates or ratios, for example speeds.
+
+   Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr.
+   What is the average speed?
+
+   .. doctest::
+
+      >>> harmonic_mean([40, 60])
+      48.0
 
    Suppose an investor purchases an equal value of shares in each of
    three companies, with P/E (price/earning) ratios of 2.5, 3 and 10.
@@ -175,9 +187,6 @@ However, for reading convenience, most of the examples show sorted sequences.
       >>> harmonic_mean([2.5, 3, 10])  # For an equal investment portfolio.
       3.6
 
-   Using the arithmetic mean would give an average of about 5.167, which
-   is well over the aggregate P/E ratio.
-
    :exc:`StatisticsError` is raised if *data* is empty, or any element
    is less than zero.
 
@@ -190,9 +199,9 @@ However, for reading convenience, most of the examples show sorted sequences.
    middle two" method.  If *data* is empty, :exc:`StatisticsError` is raised.
    *data* can be a sequence or iterator.
 
-   The median is a robust measure of central location, and is less affected by
-   the presence of outliers in your data.  When the number of data points is
-   odd, the middle data point is returned:
+   The median is a robust measure of central location and is less affected by
+   the presence of outliers.  When the number of data points is odd, the
+   middle data point is returned:
 
    .. doctest::
 
@@ -210,13 +219,10 @@ However, for reading convenience, most of the examples show sorted sequences.
    This is suited for when your data is discrete, and you don't mind that the
    median may not be an actual data point.
 
-   If your data is ordinal (supports order operations) but not numeric (doesn't
-   support addition), you should use :func:`median_low` or :func:`median_high`
+   If the data is ordinal (supports order operations) but not numeric (doesn't
+   support addition), consider using :func:`median_low` or :func:`median_high`
    instead.
 
-   .. seealso:: :func:`median_low`, :func:`median_high`, :func:`median_grouped`
-
-
 .. function:: median_low(data)
 
    Return the low median of numeric data.  If *data* is empty,
@@ -319,7 +325,7 @@ However, for reading convenience, most of the examples show sorted sequences.
    desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``.
    If the input *data* is empty, :exc:`StatisticsError` is raised.
 
-   ``mode`` assumes discrete data, and returns a single value. This is the
+   ``mode`` assumes discrete data and returns a single value. This is the
    standard treatment of the mode as commonly taught in schools:
 
    .. doctest::
@@ -522,7 +528,7 @@ However, for reading convenience, most of the examples show sorted sequences.
    cut-point will evaluate to ``104``.
 
    The *method* for computing quantiles can be varied depending on
-   whether the data in *data* includes or excludes the lowest and
+   whether the *data* includes or excludes the lowest and
    highest possible values from the population.
 
    The default *method* is "exclusive" and is used for data sampled from


From webhook-mailer at python.org  Fri Sep  6 05:14:42 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Fri, 06 Sep 2019 09:14:42 -0000
Subject: [Python-checkins] bpo-37380: subprocess: don't use _active on win
 (GH-14360) (GH-15706)
Message-ID: <mailman.262.1567761283.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1e2707d7e82aedf73c59772bc7aa228286323c3c
commit: 1e2707d7e82aedf73c59772bc7aa228286323c3c
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-06T11:14:24+02:00
summary:

bpo-37380: subprocess: don't use _active on win (GH-14360) (GH-15706)

As noted by @eryksun in [1] and [2], using _cleanup and _active(in
__del__) is not necessary on Windows, since:

> Unlike Unix, a process in Windows doesn't have to be waited on by
> its parent to avoid a zombie. Keeping the handle open will actually
> create a zombie until the next _cleanup() call, which may be never
> if Popen() isn't called again.

This patch simply defines `subprocess._active` as `None`, for which we already
have the proper logic in place in `subprocess.Popen.__del__`, that prevents it
from trying to append the process to the `_active`. This patch also defines
`subprocess._cleanup` as a noop for Windows.

[1] https://bugs.python.org/issue37380GH-msg346333
[2] https://bugs.python.org/issue36067GH-msg336262

Signed-off-by: Ruslan Kuprieiev <ruslan at iterative.ai>
(cherry picked from commit 042821ae3cf537e01963c9ec85d1a454d921e826)

Co-authored-by: Ruslan Kuprieiev <kupruser at gmail.com>

files:
A Misc/NEWS.d/next/Windows/2019-06-25-04-15-22.bpo-37380.tPxjuz.rst
M Lib/subprocess.py
M Lib/test/test_subprocess.py

diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 53a5e7209974..2c30e344e61f 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -217,22 +217,38 @@ def __repr__(self):
         __str__ = __repr__
 
 
-# This lists holds Popen instances for which the underlying process had not
-# exited at the time its __del__ method got called: those processes are wait()ed
-# for synchronously from _cleanup() when a new Popen object is created, to avoid
-# zombie processes.
-_active = []
-
-def _cleanup():
-    for inst in _active[:]:
-        res = inst._internal_poll(_deadstate=sys.maxsize)
-        if res is not None:
-            try:
-                _active.remove(inst)
-            except ValueError:
-                # This can happen if two threads create a new Popen instance.
-                # It's harmless that it was already removed, so ignore.
-                pass
+if _mswindows:
+    # On Windows we just need to close `Popen._handle` when we no longer need
+    # it, so that the kernel can free it. `Popen._handle` gets closed
+    # implicitly when the `Popen` instance is finalized (see `Handle.__del__`,
+    # which is calling `CloseHandle` as requested in [1]), so there is nothing
+    # for `_cleanup` to do.
+    #
+    # [1] https://docs.microsoft.com/en-us/windows/desktop/ProcThread/
+    # creating-processes
+    _active = None
+
+    def _cleanup():
+        pass
+else:
+    # This lists holds Popen instances for which the underlying process had not
+    # exited at the time its __del__ method got called: those processes are
+    # wait()ed for synchronously from _cleanup() when a new Popen object is
+    # created, to avoid zombie processes.
+    _active = []
+
+    def _cleanup():
+        if _active is None:
+            return
+        for inst in _active[:]:
+            res = inst._internal_poll(_deadstate=sys.maxsize)
+            if res is not None:
+                try:
+                    _active.remove(inst)
+                except ValueError:
+                    # This can happen if two threads create a new Popen instance.
+                    # It's harmless that it was already removed, so ignore.
+                    pass
 
 PIPE = -1
 STDOUT = -2
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index 8419061b2a90..41e6a0901585 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -59,10 +59,14 @@ def setUp(self):
         support.reap_children()
 
     def tearDown(self):
-        for inst in subprocess._active:
-            inst.wait()
-        subprocess._cleanup()
-        self.assertFalse(subprocess._active, "subprocess._active not empty")
+        if not mswindows:
+            # subprocess._active is not used on Windows and is set to None.
+            for inst in subprocess._active:
+                inst.wait()
+            subprocess._cleanup()
+            self.assertFalse(
+                subprocess._active, "subprocess._active not empty"
+            )
         self.doCleanups()
         support.reap_children()
 
@@ -2622,8 +2626,12 @@ def test_zombie_fast_process_del(self):
         with support.check_warnings(('', ResourceWarning)):
             p = None
 
-        # check that p is in the active processes list
-        self.assertIn(ident, [id(o) for o in subprocess._active])
+        if mswindows:
+            # subprocess._active is not used on Windows and is set to None.
+            self.assertIsNone(subprocess._active)
+        else:
+            # check that p is in the active processes list
+            self.assertIn(ident, [id(o) for o in subprocess._active])
 
     def test_leak_fast_process_del_killed(self):
         # Issue #12650: on Unix, if Popen.__del__() was called before the
@@ -2644,8 +2652,12 @@ def test_leak_fast_process_del_killed(self):
             p = None
 
         os.kill(pid, signal.SIGKILL)
-        # check that p is in the active processes list
-        self.assertIn(ident, [id(o) for o in subprocess._active])
+        if mswindows:
+            # subprocess._active is not used on Windows and is set to None.
+            self.assertIsNone(subprocess._active)
+        else:
+            # check that p is in the active processes list
+            self.assertIn(ident, [id(o) for o in subprocess._active])
 
         # let some time for the process to exit, and create a new Popen: this
         # should trigger the wait() of p
@@ -2657,7 +2669,11 @@ def test_leak_fast_process_del_killed(self):
                 pass
         # p should have been wait()ed on, and removed from the _active list
         self.assertRaises(OSError, os.waitpid, pid, 0)
-        self.assertNotIn(ident, [id(o) for o in subprocess._active])
+        if mswindows:
+            # subprocess._active is not used on Windows and is set to None.
+            self.assertIsNone(subprocess._active)
+        else:
+            self.assertNotIn(ident, [id(o) for o in subprocess._active])
 
     def test_close_fds_after_preexec(self):
         fd_status = support.findfile("fd_status.py", subdir="subprocessdata")
diff --git a/Misc/NEWS.d/next/Windows/2019-06-25-04-15-22.bpo-37380.tPxjuz.rst b/Misc/NEWS.d/next/Windows/2019-06-25-04-15-22.bpo-37380.tPxjuz.rst
new file mode 100644
index 000000000000..facce27954a6
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-06-25-04-15-22.bpo-37380.tPxjuz.rst
@@ -0,0 +1,2 @@
+Don't collect unfinished processes with ``subprocess._active`` on Windows to
+cleanup later. Patch by Ruslan Kuprieiev.


From webhook-mailer at python.org  Fri Sep  6 05:14:42 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Fri, 06 Sep 2019 09:14:42 -0000
Subject: [Python-checkins] bpo-37380: subprocess: don't use _active on win
 (GH-14360) (GH-15707)
Message-ID: <mailman.263.1567761285.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4d1abedce9422473af2ac78047e55cde73208208
commit: 4d1abedce9422473af2ac78047e55cde73208208
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-06T11:14:31+02:00
summary:

bpo-37380: subprocess: don't use _active on win (GH-14360) (GH-15707)

As noted by @eryksun in [1] and [2], using _cleanup and _active(in
__del__) is not necessary on Windows, since:

> Unlike Unix, a process in Windows doesn't have to be waited on by
> its parent to avoid a zombie. Keeping the handle open will actually
> create a zombie until the next _cleanup() call, which may be never
> if Popen() isn't called again.

This patch simply defines `subprocess._active` as `None`, for which we already
have the proper logic in place in `subprocess.Popen.__del__`, that prevents it
from trying to append the process to the `_active`. This patch also defines
`subprocess._cleanup` as a noop for Windows.

[1] https://bugs.python.org/issue37380GH-msg346333
[2] https://bugs.python.org/issue36067GH-msg336262

Signed-off-by: Ruslan Kuprieiev <ruslan at iterative.ai>
(cherry picked from commit 042821ae3cf537e01963c9ec85d1a454d921e826)

Co-authored-by: Ruslan Kuprieiev <kupruser at gmail.com>

files:
A Misc/NEWS.d/next/Windows/2019-06-25-04-15-22.bpo-37380.tPxjuz.rst
M Lib/subprocess.py
M Lib/test/test_subprocess.py

diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index c0bda96cbc03..5bbeba47a374 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -218,22 +218,38 @@ def __repr__(self):
         _PopenSelector = selectors.SelectSelector
 
 
-# This lists holds Popen instances for which the underlying process had not
-# exited at the time its __del__ method got called: those processes are wait()ed
-# for synchronously from _cleanup() when a new Popen object is created, to avoid
-# zombie processes.
-_active = []
-
-def _cleanup():
-    for inst in _active[:]:
-        res = inst._internal_poll(_deadstate=sys.maxsize)
-        if res is not None:
-            try:
-                _active.remove(inst)
-            except ValueError:
-                # This can happen if two threads create a new Popen instance.
-                # It's harmless that it was already removed, so ignore.
-                pass
+if _mswindows:
+    # On Windows we just need to close `Popen._handle` when we no longer need
+    # it, so that the kernel can free it. `Popen._handle` gets closed
+    # implicitly when the `Popen` instance is finalized (see `Handle.__del__`,
+    # which is calling `CloseHandle` as requested in [1]), so there is nothing
+    # for `_cleanup` to do.
+    #
+    # [1] https://docs.microsoft.com/en-us/windows/desktop/ProcThread/
+    # creating-processes
+    _active = None
+
+    def _cleanup():
+        pass
+else:
+    # This lists holds Popen instances for which the underlying process had not
+    # exited at the time its __del__ method got called: those processes are
+    # wait()ed for synchronously from _cleanup() when a new Popen object is
+    # created, to avoid zombie processes.
+    _active = []
+
+    def _cleanup():
+        if _active is None:
+            return
+        for inst in _active[:]:
+            res = inst._internal_poll(_deadstate=sys.maxsize)
+            if res is not None:
+                try:
+                    _active.remove(inst)
+                except ValueError:
+                    # This can happen if two threads create a new Popen instance.
+                    # It's harmless that it was already removed, so ignore.
+                    pass
 
 PIPE = -1
 STDOUT = -2
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index 9bfc21123cc0..e58d0925df3b 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -52,10 +52,14 @@ def setUp(self):
         support.reap_children()
 
     def tearDown(self):
-        for inst in subprocess._active:
-            inst.wait()
-        subprocess._cleanup()
-        self.assertFalse(subprocess._active, "subprocess._active not empty")
+        if not mswindows:
+            # subprocess._active is not used on Windows and is set to None.
+            for inst in subprocess._active:
+                inst.wait()
+            subprocess._cleanup()
+            self.assertFalse(
+                subprocess._active, "subprocess._active not empty"
+            )
         self.doCleanups()
         support.reap_children()
 
@@ -2672,8 +2676,12 @@ def test_zombie_fast_process_del(self):
         with support.check_warnings(('', ResourceWarning)):
             p = None
 
-        # check that p is in the active processes list
-        self.assertIn(ident, [id(o) for o in subprocess._active])
+        if mswindows:
+            # subprocess._active is not used on Windows and is set to None.
+            self.assertIsNone(subprocess._active)
+        else:
+            # check that p is in the active processes list
+            self.assertIn(ident, [id(o) for o in subprocess._active])
 
     def test_leak_fast_process_del_killed(self):
         # Issue #12650: on Unix, if Popen.__del__() was called before the
@@ -2694,8 +2702,12 @@ def test_leak_fast_process_del_killed(self):
             p = None
 
         os.kill(pid, signal.SIGKILL)
-        # check that p is in the active processes list
-        self.assertIn(ident, [id(o) for o in subprocess._active])
+        if mswindows:
+            # subprocess._active is not used on Windows and is set to None.
+            self.assertIsNone(subprocess._active)
+        else:
+            # check that p is in the active processes list
+            self.assertIn(ident, [id(o) for o in subprocess._active])
 
         # let some time for the process to exit, and create a new Popen: this
         # should trigger the wait() of p
@@ -2707,7 +2719,11 @@ def test_leak_fast_process_del_killed(self):
                 pass
         # p should have been wait()ed on, and removed from the _active list
         self.assertRaises(OSError, os.waitpid, pid, 0)
-        self.assertNotIn(ident, [id(o) for o in subprocess._active])
+        if mswindows:
+            # subprocess._active is not used on Windows and is set to None.
+            self.assertIsNone(subprocess._active)
+        else:
+            self.assertNotIn(ident, [id(o) for o in subprocess._active])
 
     def test_close_fds_after_preexec(self):
         fd_status = support.findfile("fd_status.py", subdir="subprocessdata")
diff --git a/Misc/NEWS.d/next/Windows/2019-06-25-04-15-22.bpo-37380.tPxjuz.rst b/Misc/NEWS.d/next/Windows/2019-06-25-04-15-22.bpo-37380.tPxjuz.rst
new file mode 100644
index 000000000000..facce27954a6
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-06-25-04-15-22.bpo-37380.tPxjuz.rst
@@ -0,0 +1,2 @@
+Don't collect unfinished processes with ``subprocess._active`` on Windows to
+cleanup later. Patch by Ruslan Kuprieiev.


From webhook-mailer at python.org  Fri Sep  6 11:42:02 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Fri, 06 Sep 2019 15:42:02 -0000
Subject: [Python-checkins] bpo-15088 : Remove PyGen_NeedsFinalizing()
 (GH-15702)
Message-ID: <mailman.264.1567784522.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/74b662cf202753d224d82d5503974cce881f7436
commit: 74b662cf202753d224d82d5503974cce881f7436
branch: master
author: Joannah Nanjekye <33177550+nanjekyejoannah at users.noreply.github.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-06T17:41:38+02:00
summary:

bpo-15088 : Remove PyGen_NeedsFinalizing() (GH-15702)

Remove PyGen_NeedsFinalizing(): it was not
documented, tested or used anywhere within CPython after
the implementation of PEP 442.

files:
A Misc/NEWS.d/next/C API/2019-09-05-14-17-21.bpo-15088.plt8Em.rst
M Doc/whatsnew/3.9.rst
M Include/genobject.h
M Objects/genobject.c
M Python/ceval.c

diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
index 756690d93793..5670cb59f481 100644
--- a/Doc/whatsnew/3.9.rst
+++ b/Doc/whatsnew/3.9.rst
@@ -222,6 +222,11 @@ Removed
 * ``PyThreadState_DeleteCurrent()`` has been removed. It was not documented.
   (Contributed by Joannah Nanjekye in :issue:`37878`.)
 
+* The C function ``PyGen_NeedsFinalizing`` has been removed. It was not
+  documented, tested or used anywhere within CPython after the implementation
+  of :pep:`442`. Patch by Joannah Nanjekye.
+  (Contributed by Joannah Nanjekye in :issue:`15088`)
+
 
 Porting to Python 3.9
 =====================
diff --git a/Include/genobject.h b/Include/genobject.h
index 6755963f332f..3b3b68b56b90 100644
--- a/Include/genobject.h
+++ b/Include/genobject.h
@@ -43,7 +43,6 @@ PyAPI_DATA(PyTypeObject) PyGen_Type;
 PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
 PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(struct _frame *,
     PyObject *name, PyObject *qualname);
-PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *);
 PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *);
 PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
 PyAPI_FUNC(PyObject *) _PyGen_Send(PyGenObject *, PyObject *);
diff --git a/Misc/NEWS.d/next/C API/2019-09-05-14-17-21.bpo-15088.plt8Em.rst b/Misc/NEWS.d/next/C API/2019-09-05-14-17-21.bpo-15088.plt8Em.rst
new file mode 100644
index 000000000000..8a27afc9678a
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-09-05-14-17-21.bpo-15088.plt8Em.rst	
@@ -0,0 +1,4 @@
+The C function ``PyGen_NeedsFinalizing`` has been removed. It was not
+documented, tested or used anywhere within CPython after the implementation
+of :pep:`442`. Patch by Joannah Nanjekye.
+(Patch by Joannah Nanjekye)
\ No newline at end of file
diff --git a/Objects/genobject.c b/Objects/genobject.c
index 5e0bfa4f43de..9f490b4e2e48 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -819,22 +819,6 @@ PyGen_New(PyFrameObject *f)
     return gen_new_with_qualname(&PyGen_Type, f, NULL, NULL);
 }
 
-int
-PyGen_NeedsFinalizing(PyGenObject *gen)
-{
-    PyFrameObject *f = gen->gi_frame;
-
-    if (f == NULL || f->f_stacktop == NULL)
-        return 0; /* no frame or empty blockstack == no finalization */
-
-    /* Any (exception-handling) block type requires cleanup. */
-    if (f->f_iblock > 0)
-        return 1;
-
-    /* No blocks, it's safe to skip finalization. */
-    return 0;
-}
-
 /* Coroutine Object */
 
 typedef struct {
diff --git a/Python/ceval.c b/Python/ceval.c
index 7a3f42c3437f..f9e03b3097f0 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -3227,11 +3227,6 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
         }
 
         case TARGET(SETUP_FINALLY): {
-            /* NOTE: If you add any new block-setup opcodes that
-               are not try/except/finally handlers, you may need
-               to update the PyGen_NeedsFinalizing() function.
-               */
-
             PyFrame_BlockSetup(f, SETUP_FINALLY, INSTR_OFFSET() + oparg,
                                STACK_LEVEL());
             DISPATCH();


From webhook-mailer at python.org  Fri Sep  6 13:54:51 2019
From: webhook-mailer at python.org (Terry Jan Reedy)
Date: Fri, 06 Sep 2019 17:54:51 -0000
Subject: [Python-checkins] bpo-38041: Refine IDLE Shell restart lines.
 (GH-15709)
Message-ID: <mailman.265.1567792493.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/38da805d563422cf1bb9cd9be24c73806840fe30
commit: 38da805d563422cf1bb9cd9be24c73806840fe30
branch: master
author: Terry Jan Reedy <tjreedy at udel.edu>
committer: GitHub <noreply at github.com>
date: 2019-09-06T13:54:45-04:00
summary:

 bpo-38041: Refine IDLE Shell restart lines.  (GH-15709)

 Restart lines now always start with '=' and never end with ' ' and fill the width of the window unless that would require ending with ' ', which could be wrapped by itself and possible confusing the user.

files:
A Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/idle_test/test_pyshell.py
M Lib/idlelib/pyshell.py

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index c9e846a6fba6..559ffd0cf4f3 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,10 @@ Released on 2019-10-20?
 ======================================
 
 
+bpo-38401: Shell restart lines now fill the window width, always start
+with '=', and avoid wrapping unnecessarily. The line will still wrap
+if the included file name is long relative to the width.
+
 bpo-37092: Add mousewheel scrolling for IDLE module, path, and stack
 browsers.  Patch by George Zhang.
 
diff --git a/Lib/idlelib/idle_test/test_pyshell.py b/Lib/idlelib/idle_test/test_pyshell.py
index 581444ca5ef2..4a096676f257 100644
--- a/Lib/idlelib/idle_test/test_pyshell.py
+++ b/Lib/idlelib/idle_test/test_pyshell.py
@@ -7,6 +7,28 @@
 from tkinter import Tk
 
 
+class FunctionTest(unittest.TestCase):
+    # Test stand-alone module level non-gui functions.
+
+    def test_restart_line_wide(self):
+        eq = self.assertEqual
+        for file, mul, extra in (('', 22, ''), ('finame', 21, '=')):
+            width = 60
+            bar = mul * '='
+            with self.subTest(file=file, bar=bar):
+                file = file or 'Shell'
+                line = pyshell.restart_line(width, file)
+                eq(len(line), width)
+                eq(line, f"{bar+extra} RESTART: {file} {bar}")
+
+    def test_restart_line_narrow(self):
+        expect, taglen = "= RESTART: Shell", 16
+        for width in (taglen-1, taglen, taglen+1):
+            with self.subTest(width=width):
+                self.assertEqual(pyshell.restart_line(width, ''), expect)
+        self.assertEqual(pyshell.restart_line(taglen+2, ''), expect+' =')
+
+
 class PyShellFileListTest(unittest.TestCase):
 
     @classmethod
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index ea9465568bd9..08716a9733b7 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -387,6 +387,19 @@ def handle_EOF(self):
         "Override the base class - just re-raise EOFError"
         raise EOFError
 
+def restart_line(width, filename):  # See bpo-38141.
+    """Return width long restart line formatted with filename.
+
+    Fill line with balanced '='s, with any extras and at least one at
+    the beginning.  Do not end with a trailing space.
+    """
+    tag = f"= RESTART: {filename or 'Shell'} ="
+    if width >= len(tag):
+        div, mod = divmod((width -len(tag)), 2)
+        return f"{(div+mod)*'='}{tag}{div*'='}"
+    else:
+        return tag[:-2]  # Remove ' ='.
+
 
 class ModifiedInterpreter(InteractiveInterpreter):
 
@@ -491,9 +504,8 @@ def restart_subprocess(self, with_cwd=False, filename=''):
         console.stop_readline()
         # annotate restart in shell window and mark it
         console.text.delete("iomark", "end-1c")
-        tag = 'RESTART: ' + (filename if filename else 'Shell')
-        halfbar = ((int(console.width) -len(tag) - 4) // 2) * '='
-        console.write("\n{0} {1} {0}".format(halfbar, tag))
+        console.write('\n')
+        console.write(restart_line(console.width, filename))
         console.text.mark_set("restart", "end-1c")
         console.text.mark_gravity("restart", "left")
         if not filename:
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst b/Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst
new file mode 100644
index 000000000000..0aa254e8ca70
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst
@@ -0,0 +1,3 @@
+Shell restart lines now fill the window width, always start with '=',
+and avoid wrapping unnecessarily. The line will still wrap if the
+included file name is long relative to the width.


From webhook-mailer at python.org  Fri Sep  6 14:14:59 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 06 Sep 2019 18:14:59 -0000
Subject: [Python-checkins] bpo-38041: Refine IDLE Shell restart lines.
 (GH-15709)
Message-ID: <mailman.266.1567793700.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4009a8522d774c36918005527dfb0975f389c8c2
commit: 4009a8522d774c36918005527dfb0975f389c8c2
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-06T11:14:49-07:00
summary:

bpo-38041: Refine IDLE Shell restart lines.  (GH-15709)


 Restart lines now always start with '=' and never end with ' ' and fill the width of the window unless that would require ending with ' ', which could be wrapped by itself and possible confusing the user.
(cherry picked from commit 38da805d563422cf1bb9cd9be24c73806840fe30)

Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu>

files:
A Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/idle_test/test_pyshell.py
M Lib/idlelib/pyshell.py

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index c9e846a6fba6..559ffd0cf4f3 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,10 @@ Released on 2019-10-20?
 ======================================
 
 
+bpo-38401: Shell restart lines now fill the window width, always start
+with '=', and avoid wrapping unnecessarily. The line will still wrap
+if the included file name is long relative to the width.
+
 bpo-37092: Add mousewheel scrolling for IDLE module, path, and stack
 browsers.  Patch by George Zhang.
 
diff --git a/Lib/idlelib/idle_test/test_pyshell.py b/Lib/idlelib/idle_test/test_pyshell.py
index 581444ca5ef2..4a096676f257 100644
--- a/Lib/idlelib/idle_test/test_pyshell.py
+++ b/Lib/idlelib/idle_test/test_pyshell.py
@@ -7,6 +7,28 @@
 from tkinter import Tk
 
 
+class FunctionTest(unittest.TestCase):
+    # Test stand-alone module level non-gui functions.
+
+    def test_restart_line_wide(self):
+        eq = self.assertEqual
+        for file, mul, extra in (('', 22, ''), ('finame', 21, '=')):
+            width = 60
+            bar = mul * '='
+            with self.subTest(file=file, bar=bar):
+                file = file or 'Shell'
+                line = pyshell.restart_line(width, file)
+                eq(len(line), width)
+                eq(line, f"{bar+extra} RESTART: {file} {bar}")
+
+    def test_restart_line_narrow(self):
+        expect, taglen = "= RESTART: Shell", 16
+        for width in (taglen-1, taglen, taglen+1):
+            with self.subTest(width=width):
+                self.assertEqual(pyshell.restart_line(width, ''), expect)
+        self.assertEqual(pyshell.restart_line(taglen+2, ''), expect+' =')
+
+
 class PyShellFileListTest(unittest.TestCase):
 
     @classmethod
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index ea9465568bd9..08716a9733b7 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -387,6 +387,19 @@ def handle_EOF(self):
         "Override the base class - just re-raise EOFError"
         raise EOFError
 
+def restart_line(width, filename):  # See bpo-38141.
+    """Return width long restart line formatted with filename.
+
+    Fill line with balanced '='s, with any extras and at least one at
+    the beginning.  Do not end with a trailing space.
+    """
+    tag = f"= RESTART: {filename or 'Shell'} ="
+    if width >= len(tag):
+        div, mod = divmod((width -len(tag)), 2)
+        return f"{(div+mod)*'='}{tag}{div*'='}"
+    else:
+        return tag[:-2]  # Remove ' ='.
+
 
 class ModifiedInterpreter(InteractiveInterpreter):
 
@@ -491,9 +504,8 @@ def restart_subprocess(self, with_cwd=False, filename=''):
         console.stop_readline()
         # annotate restart in shell window and mark it
         console.text.delete("iomark", "end-1c")
-        tag = 'RESTART: ' + (filename if filename else 'Shell')
-        halfbar = ((int(console.width) -len(tag) - 4) // 2) * '='
-        console.write("\n{0} {1} {0}".format(halfbar, tag))
+        console.write('\n')
+        console.write(restart_line(console.width, filename))
         console.text.mark_set("restart", "end-1c")
         console.text.mark_gravity("restart", "left")
         if not filename:
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst b/Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst
new file mode 100644
index 000000000000..0aa254e8ca70
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst
@@ -0,0 +1,3 @@
+Shell restart lines now fill the window width, always start with '=',
+and avoid wrapping unnecessarily. The line will still wrap if the
+included file name is long relative to the width.


From webhook-mailer at python.org  Fri Sep  6 14:19:34 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 06 Sep 2019 18:19:34 -0000
Subject: [Python-checkins] bpo-38041: Refine IDLE Shell restart lines.
 (GH-15709)
Message-ID: <mailman.267.1567793974.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/084ba337a93facac4694459b460cf329d58d2b62
commit: 084ba337a93facac4694459b460cf329d58d2b62
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-06T11:19:30-07:00
summary:

bpo-38041: Refine IDLE Shell restart lines.  (GH-15709)


 Restart lines now always start with '=' and never end with ' ' and fill the width of the window unless that would require ending with ' ', which could be wrapped by itself and possible confusing the user.
(cherry picked from commit 38da805d563422cf1bb9cd9be24c73806840fe30)

Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu>

files:
A Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/idle_test/test_pyshell.py
M Lib/idlelib/pyshell.py

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 372c7ef5a9bc..34cdfbd099cb 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,10 @@ Released on 2019-09-30?
 ======================================
 
 
+bpo-38401: Shell restart lines now fill the window width, always start
+with '=', and avoid wrapping unnecessarily. The line will still wrap
+if the included file name is long relative to the width.
+
 bpo-37092: Add mousewheel scrolling for IDLE module, path, and stack
 browsers.  Patch by George Zhang.
 
diff --git a/Lib/idlelib/idle_test/test_pyshell.py b/Lib/idlelib/idle_test/test_pyshell.py
index 581444ca5ef2..4a096676f257 100644
--- a/Lib/idlelib/idle_test/test_pyshell.py
+++ b/Lib/idlelib/idle_test/test_pyshell.py
@@ -7,6 +7,28 @@
 from tkinter import Tk
 
 
+class FunctionTest(unittest.TestCase):
+    # Test stand-alone module level non-gui functions.
+
+    def test_restart_line_wide(self):
+        eq = self.assertEqual
+        for file, mul, extra in (('', 22, ''), ('finame', 21, '=')):
+            width = 60
+            bar = mul * '='
+            with self.subTest(file=file, bar=bar):
+                file = file or 'Shell'
+                line = pyshell.restart_line(width, file)
+                eq(len(line), width)
+                eq(line, f"{bar+extra} RESTART: {file} {bar}")
+
+    def test_restart_line_narrow(self):
+        expect, taglen = "= RESTART: Shell", 16
+        for width in (taglen-1, taglen, taglen+1):
+            with self.subTest(width=width):
+                self.assertEqual(pyshell.restart_line(width, ''), expect)
+        self.assertEqual(pyshell.restart_line(taglen+2, ''), expect+' =')
+
+
 class PyShellFileListTest(unittest.TestCase):
 
     @classmethod
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index ea9465568bd9..08716a9733b7 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -387,6 +387,19 @@ def handle_EOF(self):
         "Override the base class - just re-raise EOFError"
         raise EOFError
 
+def restart_line(width, filename):  # See bpo-38141.
+    """Return width long restart line formatted with filename.
+
+    Fill line with balanced '='s, with any extras and at least one at
+    the beginning.  Do not end with a trailing space.
+    """
+    tag = f"= RESTART: {filename or 'Shell'} ="
+    if width >= len(tag):
+        div, mod = divmod((width -len(tag)), 2)
+        return f"{(div+mod)*'='}{tag}{div*'='}"
+    else:
+        return tag[:-2]  # Remove ' ='.
+
 
 class ModifiedInterpreter(InteractiveInterpreter):
 
@@ -491,9 +504,8 @@ def restart_subprocess(self, with_cwd=False, filename=''):
         console.stop_readline()
         # annotate restart in shell window and mark it
         console.text.delete("iomark", "end-1c")
-        tag = 'RESTART: ' + (filename if filename else 'Shell')
-        halfbar = ((int(console.width) -len(tag) - 4) // 2) * '='
-        console.write("\n{0} {1} {0}".format(halfbar, tag))
+        console.write('\n')
+        console.write(restart_line(console.width, filename))
         console.text.mark_set("restart", "end-1c")
         console.text.mark_gravity("restart", "left")
         if not filename:
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst b/Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst
new file mode 100644
index 000000000000..0aa254e8ca70
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst
@@ -0,0 +1,3 @@
+Shell restart lines now fill the window width, always start with '=',
+and avoid wrapping unnecessarily. The line will still wrap if the
+included file name is long relative to the width.


From webhook-mailer at python.org  Sat Sep  7 01:24:09 2019
From: webhook-mailer at python.org (larryhastings)
Date: Sat, 07 Sep 2019 05:24:09 -0000
Subject: [Python-checkins] [3.5] bpo-34155: Dont parse domains containing @
 (GH-13079) (#15317)
Message-ID: <mailman.268.1567833849.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/063eba280a11d3c9a5dd9ee5abe4de640907951b
commit: 063eba280a11d3c9a5dd9ee5abe4de640907951b
branch: 3.5
author: Abhilash Raj <maxking at users.noreply.github.com>
committer: larryhastings <larry at hastings.org>
date: 2019-09-07T06:24:05+01:00
summary:

[3.5] bpo-34155: Dont parse domains containing @ (GH-13079) (#15317)

https://bugs.python.org/issue34155
(cherry picked from commit 8cb65d1381b027f0b09ee36bfed7f35bb4dec9a9)

Co-authored-by: jpic <jpic at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Security/2019-05-04-13-33-37.bpo-34155.MJll68.rst
M Lib/email/_header_value_parser.py
M Lib/email/_parseaddr.py
M Lib/test/test_email/test__header_value_parser.py
M Lib/test/test_email/test_email.py

diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index 37a9fbcbb67d..facc208fde12 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -1964,6 +1964,8 @@ def get_domain(value):
         token, value = get_dot_atom(value)
     except errors.HeaderParseError:
         token, value = get_atom(value)
+    if value and value[0] == '@':
+        raise errors.HeaderParseError('Invalid Domain')
     if leader is not None:
         token[:0] = [leader]
     domain.append(token)
diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py
index cdfa3729adc7..41ff6f8c000d 100644
--- a/Lib/email/_parseaddr.py
+++ b/Lib/email/_parseaddr.py
@@ -379,7 +379,12 @@ def getaddrspec(self):
         aslist.append('@')
         self.pos += 1
         self.gotonext()
-        return EMPTYSTRING.join(aslist) + self.getdomain()
+        domain = self.getdomain()
+        if not domain:
+            # Invalid domain, return an empty address instead of returning a
+            # local part to denote failed parsing.
+            return EMPTYSTRING
+        return EMPTYSTRING.join(aslist) + domain
 
     def getdomain(self):
         """Get the complete domain name from an address."""
@@ -394,6 +399,10 @@ def getdomain(self):
             elif self.field[self.pos] == '.':
                 self.pos += 1
                 sdlist.append('.')
+            elif self.field[self.pos] == '@':
+                # bpo-34155: Don't parse domains with two `@` like
+                # `a at malicious.org@important.com`.
+                return EMPTYSTRING
             elif self.field[self.pos] in self.atomends:
                 break
             else:
diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py
index b1e7dff2405d..8ae617a6bd58 100644
--- a/Lib/test/test_email/test__header_value_parser.py
+++ b/Lib/test/test_email/test__header_value_parser.py
@@ -1418,6 +1418,16 @@ def test_get_addr_spec_dot_atom(self):
         self.assertEqual(addr_spec.domain, 'example.com')
         self.assertEqual(addr_spec.addr_spec, 'star.a.star at example.com')
 
+    def test_get_addr_spec_multiple_domains(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_addr_spec('star at a.star@example.com')
+
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_addr_spec('star at a@example.com')
+
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_addr_spec('star at 172.17.0.1@example.com')
+
     # get_obs_route
 
     def test_get_obs_route_simple(self):
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index 9b86a2aee9ac..d4c3447e63d0 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -3015,6 +3015,20 @@ def test_parseaddr_empty(self):
         self.assertEqual(utils.parseaddr('<>'), ('', ''))
         self.assertEqual(utils.formataddr(utils.parseaddr('<>')), '')
 
+    def test_parseaddr_multiple_domains(self):
+        self.assertEqual(
+            utils.parseaddr('a at b@c'),
+            ('', '')
+        )
+        self.assertEqual(
+            utils.parseaddr('a at b.c@c'),
+            ('', '')
+        )
+        self.assertEqual(
+            utils.parseaddr('a at 172.17.0.1@c'),
+            ('', '')
+        )
+
     def test_noquote_dump(self):
         self.assertEqual(
             utils.formataddr(('A Silly Person', 'person at dom.ain')),
diff --git a/Misc/NEWS.d/next/Security/2019-05-04-13-33-37.bpo-34155.MJll68.rst b/Misc/NEWS.d/next/Security/2019-05-04-13-33-37.bpo-34155.MJll68.rst
new file mode 100644
index 000000000000..50292e29ed1d
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-05-04-13-33-37.bpo-34155.MJll68.rst
@@ -0,0 +1 @@
+Fix parsing of invalid email addresses with more than one ``@`` (e.g. a at b@c.com.) to not return the part before 2nd ``@`` as valid email address. Patch by maxking & jpic.


From webhook-mailer at python.org  Sat Sep  7 02:06:33 2019
From: webhook-mailer at python.org (larryhastings)
Date: Sat, 07 Sep 2019 06:06:33 -0000
Subject: [Python-checkins] bpo-36576: Skip test_ssl and test_asyncio tests
 failing with OpenSSL 1.1.1 (#12694)
Message-ID: <mailman.269.1567836394.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4d1c2541c125fe9d211016193ebfd5899a8511aa
commit: 4d1c2541c125fe9d211016193ebfd5899a8511aa
branch: 3.5
author: Victor Stinner <vstinner at redhat.com>
committer: larryhastings <larry at hastings.org>
date: 2019-09-07T07:06:27+01:00
summary:

bpo-36576: Skip test_ssl and test_asyncio tests failing with OpenSSL 1.1.1 (#12694)

Some test_ssl and test_asyncio tests were written for OpenSSL 1.0 and TLS 1.0, but fail with OpenSSL 1.1.1 and TLS 1.3.  Fixing these requires backporting new ssl flags like ssl.OP_NO_TLSv1_3 or ssl.OP_NO_COMPRESSION, which is inappropriate at this stage in Python 3.5's lifetime. Moreover, it's not really worth it: the code works fine, the problem is just in the tests.  This patch disables those problematic tests when Python 3.5 is built using newer versions of OpenSSL.

files:
A Misc/NEWS.d/next/Tests/2019-04-05-10-34-29.bpo-36576.7Cp2kK.rst
M Lib/test/test_asyncio/test_events.py
M Lib/test/test_ssl.py

diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py
index 492a84a2313b..b23889b20dea 100644
--- a/Lib/test/test_asyncio/test_events.py
+++ b/Lib/test/test_asyncio/test_events.py
@@ -38,6 +38,12 @@
     from asyncio import test_support as support
 
 
+if ssl is not None:
+    IS_OPENSSL_1_1_1 = ssl.OPENSSL_VERSION_INFO >= (1, 1, 1)
+else:
+    IS_OPENSSL_1_1_1 = False
+
+
 def data_file(filename):
     if hasattr(support, 'TEST_HOME_DIR'):
         fullname = os.path.join(support.TEST_HOME_DIR, filename)
@@ -1145,6 +1151,7 @@ def test_legacy_create_unix_server_ssl_verify_failed(self):
             self.test_create_unix_server_ssl_verify_failed()
 
     @unittest.skipIf(ssl is None, 'No ssl module')
+    @unittest.skipIf(IS_OPENSSL_1_1_1, "bpo-36576: fail on OpenSSL 1.1.1")
     def test_create_server_ssl_match_failed(self):
         proto = MyProto(loop=self.loop)
         server, host, port = self._make_ssl_server(
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 6c78601b4589..30a64ee0a4c7 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -25,6 +25,7 @@
 HOST = support.HOST
 IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL')
 IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0)
+IS_OPENSSL_1_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 1)
 
 
 def data_file(*name):
@@ -857,6 +858,7 @@ def test_ciphers(self):
             ctx.set_ciphers("^$:,;?*'dorothyx")
 
     @skip_if_broken_ubuntu_ssl
+    @unittest.skipIf(IS_OPENSSL_1_1_1, "bpo-36576: fail on OpenSSL 1.1.1")
     def test_options(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value
@@ -3047,6 +3049,7 @@ def test_version_basic(self):
                 self.assertIs(s.version(), None)
 
         @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL")
+        @unittest.skipIf(IS_OPENSSL_1_1_1, "bpo-36576: fail on OpenSSL 1.1.1")
         def test_default_ecdh_curve(self):
             # Issue #21015: elliptic curve-based Diffie Hellman key exchange
             # should be enabled by default on SSL contexts.
@@ -3176,6 +3179,7 @@ def test_selected_alpn_protocol_if_server_uses_alpn(self):
             self.assertIs(stats['client_alpn_protocol'], None)
 
         @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support needed for this test")
+        @unittest.skipIf(IS_OPENSSL_1_1_1, "bpo-36576: fail on OpenSSL 1.1.1")
         def test_alpn_protocols(self):
             server_protocols = ['foo', 'bar', 'milkshake']
             protocol_tests = [
@@ -3356,6 +3360,7 @@ def cb_wrong_return_type(ssl_sock, server_name, initial_context):
             self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR')
             self.assertIn("TypeError", stderr.getvalue())
 
+        @unittest.skipIf(IS_OPENSSL_1_1_1, "bpo-36576: fail on OpenSSL 1.1.1")
         def test_shared_ciphers(self):
             server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             server_context.load_cert_chain(SIGNED_CERTFILE)
diff --git a/Misc/NEWS.d/next/Tests/2019-04-05-10-34-29.bpo-36576.7Cp2kK.rst b/Misc/NEWS.d/next/Tests/2019-04-05-10-34-29.bpo-36576.7Cp2kK.rst
new file mode 100644
index 000000000000..4d15bdf42796
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-04-05-10-34-29.bpo-36576.7Cp2kK.rst
@@ -0,0 +1 @@
+Skip test_ssl and test_asyncio tests failing with OpenSSL 1.1.1.


From webhook-mailer at python.org  Sat Sep  7 02:33:28 2019
From: webhook-mailer at python.org (larryhastings)
Date: Sat, 07 Sep 2019 06:33:28 -0000
Subject: [Python-checkins] bpo-36742: Corrects fix to handle decomposition
 in usernames (GH-13812) (GH-13814) (#14772)
Message-ID: <mailman.270.1567838009.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/095373c32d16df575ba5fcb5f44bf44119b26193
commit: 095373c32d16df575ba5fcb5f44bf44119b26193
branch: 3.5
author: Victor Stinner <vstinner at redhat.com>
committer: larryhastings <larry at hastings.org>
date: 2019-09-07T07:33:24+01:00
summary:

bpo-36742: Corrects fix to handle decomposition in usernames (GH-13812) (GH-13814) (#14772)

(cherry picked from commit 8d0ef0b5edeae52960c7ed05ae8a12388324f87e)

Co-authored-by: Steve Dower <steve.dower at python.org>
(cherry picked from commit fd1771dbdd28709716bd531580c40ae5ed814468)

files:
M Lib/test/test_urlparse.py
M Lib/urllib/parse.py

diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
index 1e90e18609b2..ebb30ea01c76 100644
--- a/Lib/test/test_urlparse.py
+++ b/Lib/test/test_urlparse.py
@@ -994,11 +994,12 @@ def test_urlsplit_normalization(self):
             urllib.parse.urlsplit('http://\u30d5\u309a\ufe1380')
 
         for scheme in ["http", "https", "ftp"]:
-            for c in denorm_chars:
-                url = "{}://netloc{}false.netloc/path".format(scheme, c)
-                with self.subTest(url=url, char='{:04X}'.format(ord(c))):
-                    with self.assertRaises(ValueError):
-                        urllib.parse.urlsplit(url)
+            for netloc in ["netloc{}false.netloc", "n{}user at netloc"]:
+                for c in denorm_chars:
+                    url = "{}://{}/path".format(scheme, netloc.format(c))
+                    with self.subTest(url=url, char='{:04X}'.format(ord(c))):
+                        with self.assertRaises(ValueError):
+                            urllib.parse.urlsplit(url)
 
 class Utility_Tests(unittest.TestCase):
     """Testcase to test the various utility functions in the urllib."""
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
index 7405d660fc4e..60fd5e4e0f3f 100644
--- a/Lib/urllib/parse.py
+++ b/Lib/urllib/parse.py
@@ -333,9 +333,9 @@ def _checknetloc(netloc):
     # looking for characters like \u2100 that expand to 'a/c'
     # IDNA uses NFKC equivalence, so normalize for this check
     import unicodedata
-    n = netloc.rpartition('@')[2] # ignore anything to the left of '@'
-    n = n.replace(':', '')        # ignore characters already included
-    n = n.replace('#', '')        # but not the surrounding text
+    n = netloc.replace('@', '')   # ignore characters already included
+    n = n.replace(':', '')        # but not the surrounding text
+    n = n.replace('#', '')
     n = n.replace('?', '')
     netloc2 = unicodedata.normalize('NFKC', n)
     if n == netloc2:


From webhook-mailer at python.org  Sat Sep  7 03:05:33 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sat, 07 Sep 2019 07:05:33 -0000
Subject: [Python-checkins] bpo-20806: Reference both times(2) and times(3)
 and link to MSDN. (GH-15479)
Message-ID: <mailman.271.1567839934.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3ccdbc33385a849c60a268def578cb06b8d41be6
commit: 3ccdbc33385a849c60a268def578cb06b8d41be6
branch: master
author: Joannah Nanjekye <33177550+nanjekyejoannah at users.noreply.github.com>
committer: Serhiy Storchaka <storchaka at gmail.com>
date: 2019-09-07T10:05:29+03:00
summary:

bpo-20806: Reference both times(2) and times(3) and link to MSDN. (GH-15479)

files:
M Doc/library/os.rst

diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index 6cbfb7432841..67fe36b54e2f 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -3879,7 +3879,9 @@ written in Python, such as a mail server's external command delivery program.
    :attr:`children_system`, and :attr:`elapsed` in that order.
 
    See the Unix manual page
-   :manpage:`times(2)` or the corresponding Windows Platform API documentation.
+   :manpage:`times(2)` and :manpage:`times(3)` manual page on Unix or `the GetProcessTimes MSDN
+   <https://docs.microsoft.com/windows/win32/api/processthreadsapi/nf-processthreadsapi-getprocesstimes>`
+   _ on Windows.
    On Windows, only :attr:`user` and :attr:`system` are known; the other
    attributes are zero.
 


From webhook-mailer at python.org  Sat Sep  7 03:08:56 2019
From: webhook-mailer at python.org (larryhastings)
Date: Sat, 07 Sep 2019 07:08:56 -0000
Subject: [Python-checkins] [3.5] bpo-37461: Fix infinite loop in parsing of
 specially crafted email headers (GH-14794) (#15446)
Message-ID: <mailman.272.1567840137.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c28e4a5160d3283b12514c7c28ed6e0a2a52271a
commit: c28e4a5160d3283b12514c7c28ed6e0a2a52271a
branch: 3.5
author: Abhilash Raj <maxking at users.noreply.github.com>
committer: larryhastings <larry at hastings.org>
date: 2019-09-07T08:08:53+01:00
summary:

[3.5] bpo-37461: Fix infinite loop in parsing of specially crafted email headers (GH-14794) (#15446)

* [3.5] bpo-37461: Fix infinite loop in parsing of specially crafted email headers (GH-14794)

Some crafted email header would cause the get_parameter method to run in an
infinite loop causing a DoS attack surface when parsing those headers. This
patch fixes that by making sure the DQUOTE character is handled to prevent
going into an infinite loop.
(cherry picked from commit a4a994bd3e619cbaff97610a1cee8ffa87c672f5)

Co-authored-by: Abhilash Raj <maxking at users.noreply.github.com>
Co-Authored-By: Ashwin Ramaswami <aramaswamis at gmail.com>

files:
A Misc/NEWS.d/next/Security/2019-07-16-08-11-00.bpo-37461.1Ahz7O.rst
M Lib/email/_header_value_parser.py
M Lib/test/test_email/test__header_value_parser.py

diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index facc208fde12..5c4eb75f4981 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -2771,6 +2771,9 @@ def get_parameter(value):
         while value:
             if value[0] in WSP:
                 token, value = get_fws(value)
+            elif value[0] == '"':
+                token = ValueTerminal('"', 'DQUOTE')
+                value = value[1:]
             else:
                 token, value = get_qcontent(value)
             v.append(token)
diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py
index 8ae617a6bd58..0cc5898b6275 100644
--- a/Lib/test/test_email/test__header_value_parser.py
+++ b/Lib/test/test_email/test__header_value_parser.py
@@ -2573,6 +2573,13 @@ def mime_parameters_as_value(self,
             # Defects are apparent missing *0*, and two 'out of sequence'.
             [errors.InvalidHeaderDefect]*3),
 
+        # bpo-37461: Check that we don't go into an infinite loop.
+        'extra_dquote': (
+            'r*="\'a\'\\"',
+            ' r="\\""',
+            'r*=\'a\'"',
+            [('r', '"')],
+            [errors.InvalidHeaderDefect]*2),
     }
 
 @parameterize
diff --git a/Misc/NEWS.d/next/Security/2019-07-16-08-11-00.bpo-37461.1Ahz7O.rst b/Misc/NEWS.d/next/Security/2019-07-16-08-11-00.bpo-37461.1Ahz7O.rst
new file mode 100644
index 000000000000..9d47578c6277
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-07-16-08-11-00.bpo-37461.1Ahz7O.rst
@@ -0,0 +1,2 @@
+Fix an infinite loop when parsing specially crafted email headers. Patch by
+Abhilash Raj.


From webhook-mailer at python.org  Sat Sep  7 03:12:38 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 07 Sep 2019 07:12:38 -0000
Subject: [Python-checkins] bpo-20806: Reference both times(2) and times(3)
 and link to MSDN. (GH-15479)
Message-ID: <mailman.273.1567840360.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/cc51a6d7c7b6b06fb537860428347d88776d802b
commit: cc51a6d7c7b6b06fb537860428347d88776d802b
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-07T00:12:34-07:00
summary:

bpo-20806: Reference both times(2) and times(3) and link to MSDN. (GH-15479)

(cherry picked from commit 3ccdbc33385a849c60a268def578cb06b8d41be6)

Co-authored-by: Joannah Nanjekye <33177550+nanjekyejoannah at users.noreply.github.com>

files:
M Doc/library/os.rst

diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index 6cbfb7432841..67fe36b54e2f 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -3879,7 +3879,9 @@ written in Python, such as a mail server's external command delivery program.
    :attr:`children_system`, and :attr:`elapsed` in that order.
 
    See the Unix manual page
-   :manpage:`times(2)` or the corresponding Windows Platform API documentation.
+   :manpage:`times(2)` and :manpage:`times(3)` manual page on Unix or `the GetProcessTimes MSDN
+   <https://docs.microsoft.com/windows/win32/api/processthreadsapi/nf-processthreadsapi-getprocesstimes>`
+   _ on Windows.
    On Windows, only :attr:`user` and :attr:`system` are known; the other
    attributes are zero.
 


From webhook-mailer at python.org  Sat Sep  7 03:16:31 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 07 Sep 2019 07:16:31 -0000
Subject: [Python-checkins] bpo-20806: Reference both times(2) and times(3)
 and link to MSDN. (GH-15479)
Message-ID: <mailman.274.1567840591.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a6eac83c1804fd14ed076b1776ffeea8dcb9478a
commit: a6eac83c1804fd14ed076b1776ffeea8dcb9478a
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-07T00:16:27-07:00
summary:

bpo-20806: Reference both times(2) and times(3) and link to MSDN. (GH-15479)

(cherry picked from commit 3ccdbc33385a849c60a268def578cb06b8d41be6)

Co-authored-by: Joannah Nanjekye <33177550+nanjekyejoannah at users.noreply.github.com>

files:
M Doc/library/os.rst

diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index 5572b62420fe..1556422a1704 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -3602,7 +3602,9 @@ written in Python, such as a mail server's external command delivery program.
    :attr:`children_system`, and :attr:`elapsed` in that order.
 
    See the Unix manual page
-   :manpage:`times(2)` or the corresponding Windows Platform API documentation.
+   :manpage:`times(2)` and :manpage:`times(3)` manual page on Unix or `the GetProcessTimes MSDN
+   <https://docs.microsoft.com/windows/win32/api/processthreadsapi/nf-processthreadsapi-getprocesstimes>`
+   _ on Windows.
    On Windows, only :attr:`user` and :attr:`system` are known; the other
    attributes are zero.
 


From webhook-mailer at python.org  Sat Sep  7 03:44:41 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sat, 07 Sep 2019 07:44:41 -0000
Subject: [Python-checkins] Remove some unused defines in multiprocessing.h.
 (GH-15661)
Message-ID: <mailman.275.1567842282.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/32f825393e5836ab71de843596379fa3f9e23c7a
commit: 32f825393e5836ab71de843596379fa3f9e23c7a
branch: master
author: Sergey Fedoseev <fedoseev.sergey at gmail.com>
committer: Serhiy Storchaka <storchaka at gmail.com>
date: 2019-09-07T10:44:36+03:00
summary:

Remove some unused defines in multiprocessing.h. (GH-15661)

It looks like they are unused since
87cf220972c9cb400ddcd577962883dcc5dca51a.

files:
M Modules/_multiprocessing/multiprocessing.h

diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h
index 512bc17f23d4..fe78135d4669 100644
--- a/Modules/_multiprocessing/multiprocessing.h
+++ b/Modules/_multiprocessing/multiprocessing.h
@@ -27,14 +27,6 @@
 #    include <semaphore.h>
      typedef sem_t *SEM_HANDLE;
 #  endif
-#  define HANDLE int
-#  define SOCKET int
-#  define BOOL int
-#  define UINT32 uint32_t
-#  define INT32 int32_t
-#  define TRUE 1
-#  define FALSE 0
-#  define INVALID_HANDLE_VALUE (-1)
 #endif
 
 /*
@@ -72,8 +64,6 @@
 #  define T_HANDLE T_POINTER
 #  define F_SEM_HANDLE F_HANDLE
 #  define T_SEM_HANDLE T_HANDLE
-#  define F_DWORD "k"
-#  define T_DWORD T_ULONG
 #else
 #  define F_HANDLE "i"
 #  define T_HANDLE T_INT


From webhook-mailer at python.org  Sun Sep  8 04:54:19 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sun, 08 Sep 2019 08:54:19 -0000
Subject: [Python-checkins] bpo-38038: Remove urllib.parse._splittype from
 xmlrpc.client. (GH-15703)
Message-ID: <mailman.276.1567932859.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9c4c459ac66c86a4511c8fec1997e8760e15ec17
commit: 9c4c459ac66c86a4511c8fec1997e8760e15ec17
branch: master
author: Dong-hee Na <donghee.na92 at gmail.com>
committer: Serhiy Storchaka <storchaka at gmail.com>
date: 2019-09-08T11:54:02+03:00
summary:

bpo-38038: Remove urllib.parse._splittype from xmlrpc.client. (GH-15703)

files:
M Lib/xmlrpc/client.py

diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py
index b4b2941ea5b4..d15d60d2937a 100644
--- a/Lib/xmlrpc/client.py
+++ b/Lib/xmlrpc/client.py
@@ -1421,15 +1421,14 @@ def __init__(self, uri, transport=None, encoding=None, verbose=False,
         # establish a "logical" server connection
 
         # get the url
-        type, uri = urllib.parse._splittype(uri)
-        if type not in ("http", "https"):
+        p = urllib.parse.urlparse(uri)
+        if p.scheme not in ("http", "https"):
             raise OSError("unsupported XML-RPC protocol")
-        self.__host, self.__handler = urllib.parse._splithost(uri)
-        if not self.__handler:
-            self.__handler = "/RPC2"
+        self.__host = p.netloc
+        self.__handler = p.path or "/RPC2"
 
         if transport is None:
-            if type == "https":
+            if p.scheme == "https":
                 handler = SafeTransport
                 extra_kwargs = {"context": context}
             else:


From webhook-mailer at python.org  Sun Sep  8 05:42:21 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sun, 08 Sep 2019 09:42:21 -0000
Subject: [Python-checkins] Correct Roman-numeral example in Unicode HOWTO.
 (GH-15541)
Message-ID: <mailman.277.1567935742.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/32a960f8e1015b64b4b955b3d62920c5903d4c6f
commit: 32a960f8e1015b64b4b955b3d62920c5903d4c6f
branch: master
author: Greg Price <gnprice at gmail.com>
committer: Serhiy Storchaka <storchaka at gmail.com>
date: 2019-09-08T12:42:13+03:00
summary:

Correct Roman-numeral example in Unicode HOWTO. (GH-15541)

files:
M Doc/howto/unicode.rst

diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst
index d5632bc799d2..51bd64bfc232 100644
--- a/Doc/howto/unicode.rst
+++ b/Doc/howto/unicode.rst
@@ -57,14 +57,14 @@ their corresponding code points:
    ...
    007B    '{'; LEFT CURLY BRACKET
    ...
-   2167    '?': ROMAN NUMERAL SEVEN
-   2168    '?': ROMAN NUMERAL NINE
+   2167    '?'; ROMAN NUMERAL EIGHT
+   2168    '?'; ROMAN NUMERAL NINE
    ...
-   265E    '?': BLACK CHESS KNIGHT
-   265F    '?': BLACK CHESS PAWN
+   265E    '?'; BLACK CHESS KNIGHT
+   265F    '?'; BLACK CHESS PAWN
    ...
-   1F600   '?': GRINNING FACE
-   1F609   '?': WINKING FACE
+   1F600   '?'; GRINNING FACE
+   1F609   '?'; WINKING FACE
    ...
 
 Strictly, these definitions imply that it's meaningless to say 'this is


From webhook-mailer at python.org  Sun Sep  8 06:16:01 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sun, 08 Sep 2019 10:16:01 -0000
Subject: [Python-checkins] bpo-36946:Fix possible signed integer overflow
 when handling slices. (GH-15639)
Message-ID: <mailman.278.1567937762.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3c87a667bb367ace1de6bd1577fdb4f66947da52
commit: 3c87a667bb367ace1de6bd1577fdb4f66947da52
branch: master
author: HongWeipeng <hongweichen8888 at sina.com>
committer: Serhiy Storchaka <storchaka at gmail.com>
date: 2019-09-08T13:15:56+03:00
summary:

bpo-36946:Fix possible signed integer overflow when handling slices. (GH-15639)

This is a complement to PR 13375.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-02-16-17-42.bpo-36946._lAuSR.rst
M Lib/test/test_list.py
M Modules/_ctypes/_ctypes.c
M Objects/listobject.c

diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py
index c5002b12732c..fe4b2cd365f0 100644
--- a/Lib/test/test_list.py
+++ b/Lib/test/test_list.py
@@ -150,6 +150,11 @@ def test_reversed_pickle(self):
             a[:] = data
             self.assertEqual(list(it), [])
 
+    def test_step_overflow(self):
+        a = [0, 1, 2, 3, 4]
+        a[1::sys.maxsize] = [0]
+        self.assertEqual(a[3::sys.maxsize], [3])
+
     def test_no_comdat_folding(self):
         # Issue 8847: In the PGO build, the MSVC linker's COMDAT folding
         # optimization causes failures in code that relies on distinct
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-02-16-17-42.bpo-36946._lAuSR.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-02-16-17-42.bpo-36946._lAuSR.rst
new file mode 100644
index 000000000000..e0c5d712f001
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-02-16-17-42.bpo-36946._lAuSR.rst	
@@ -0,0 +1 @@
+Fix possible signed integer overflow when handling slices. Patch by hongweipeng.
diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index 95bfe9d6348e..d2f6391fa6db 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -5195,7 +5195,8 @@ Pointer_subscript(PyObject *myself, PyObject *item)
         PyObject *np;
         StgDictObject *stgdict, *itemdict;
         PyObject *proto;
-        Py_ssize_t i, len, cur;
+        Py_ssize_t i, len;
+        size_t cur;
 
         /* Since pointers have no length, and we want to apply
            different semantics to negative indices than normal
diff --git a/Objects/listobject.c b/Objects/listobject.c
index 5fca08ea363a..645742b801fa 100644
--- a/Objects/listobject.c
+++ b/Objects/listobject.c
@@ -2789,7 +2789,8 @@ list_subscript(PyListObject* self, PyObject* item)
         return list_item(self, i);
     }
     else if (PySlice_Check(item)) {
-        Py_ssize_t start, stop, step, slicelength, cur, i;
+        Py_ssize_t start, stop, step, slicelength, i;
+        size_t cur;
         PyObject* result;
         PyObject* it;
         PyObject **src, **dest;
@@ -2925,7 +2926,8 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value)
             /* assign slice */
             PyObject *ins, *seq;
             PyObject **garbage, **seqitems, **selfitems;
-            Py_ssize_t cur, i;
+            Py_ssize_t i;
+            size_t cur;
 
             /* protect against a[::-1] = a */
             if (self == (PyListObject*)value) {


From webhook-mailer at python.org  Sun Sep  8 06:17:28 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sun, 08 Sep 2019 10:17:28 -0000
Subject: [Python-checkins] [3.8] Correct Roman-numeral example in Unicode
 HOWTO. (GH-15541). (GH-15728)
Message-ID: <mailman.279.1567937850.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3be4b107490be27470cb9a101a8dfecf446fd992
commit: 3be4b107490be27470cb9a101a8dfecf446fd992
branch: 3.8
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-08T13:17:24+03:00
summary:

[3.8] Correct Roman-numeral example in Unicode HOWTO. (GH-15541). (GH-15728)

(cherry picked from commit 32a960f8e1015b64b4b955b3d62920c5903d4c6f)

Co-authored-by: Greg Price <gnprice at gmail.com>

files:
M Doc/howto/unicode.rst

diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst
index 24c3235e4add..51bd64bfc232 100644
--- a/Doc/howto/unicode.rst
+++ b/Doc/howto/unicode.rst
@@ -57,14 +57,14 @@ their corresponding code points:
    ...
    007B    '{'; LEFT CURLY BRACKET
    ...
-   2167    '?': ROMAN NUMERAL EIGHT
-   2168    '?': ROMAN NUMERAL NINE
+   2167    '?'; ROMAN NUMERAL EIGHT
+   2168    '?'; ROMAN NUMERAL NINE
    ...
-   265E    '?': BLACK CHESS KNIGHT
-   265F    '?': BLACK CHESS PAWN
+   265E    '?'; BLACK CHESS KNIGHT
+   265F    '?'; BLACK CHESS PAWN
    ...
-   1F600   '?': GRINNING FACE
-   1F609   '?': WINKING FACE
+   1F600   '?'; GRINNING FACE
+   1F609   '?'; WINKING FACE
    ...
 
 Strictly, these definitions imply that it's meaningless to say 'this is


From webhook-mailer at python.org  Sun Sep  8 06:28:44 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sun, 08 Sep 2019 10:28:44 -0000
Subject: [Python-checkins] [3.8] Correct Roman-numeral example in Unicode
 HOWTO. (GH-15541). (GH-15728)
Message-ID: <mailman.280.1567938525.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/30933d57fc4fd3e52123554b67723f13756a70ca
commit: 30933d57fc4fd3e52123554b67723f13756a70ca
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-08T03:28:40-07:00
summary:

[3.8] Correct Roman-numeral example in Unicode HOWTO. (GH-15541). (GH-15728)


(cherry picked from commit 32a960f8e1015b64b4b955b3d62920c5903d4c6f)

Co-authored-by: Greg Price <gnprice at gmail.com>
(cherry picked from commit 3be4b107490be27470cb9a101a8dfecf446fd992)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
M Doc/howto/unicode.rst

diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst
index d4b1b2a70726..1ac4102034d3 100644
--- a/Doc/howto/unicode.rst
+++ b/Doc/howto/unicode.rst
@@ -57,14 +57,14 @@ their corresponding code points:
    ...
    007B    '{'; LEFT CURLY BRACKET
    ...
-   2167    '?': ROMAN NUMERAL EIGHT
-   2168    '?': ROMAN NUMERAL NINE
+   2167    '?'; ROMAN NUMERAL EIGHT
+   2168    '?'; ROMAN NUMERAL NINE
    ...
-   265E    '?': BLACK CHESS KNIGHT
-   265F    '?': BLACK CHESS PAWN
+   265E    '?'; BLACK CHESS KNIGHT
+   265F    '?'; BLACK CHESS PAWN
    ...
-   1F600   '?': GRINNING FACE
-   1F609   '?': WINKING FACE
+   1F600   '?'; GRINNING FACE
+   1F609   '?'; WINKING FACE
    ...
 
 Strictly, these definitions imply that it's meaningless to say 'this is


From webhook-mailer at python.org  Sun Sep  8 06:36:42 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sun, 08 Sep 2019 10:36:42 -0000
Subject: [Python-checkins] bpo-36946:Fix possible signed integer overflow
 when handling slices. (GH-15639)
Message-ID: <mailman.281.1567939003.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/021e5db20bc19d678a5b94247a5cdcf689eff006
commit: 021e5db20bc19d678a5b94247a5cdcf689eff006
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-08T03:36:38-07:00
summary:

bpo-36946:Fix possible signed integer overflow when handling slices. (GH-15639)


This is a complement to PR 13375.
(cherry picked from commit 3c87a667bb367ace1de6bd1577fdb4f66947da52)

Co-authored-by: HongWeipeng <hongweichen8888 at sina.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-02-16-17-42.bpo-36946._lAuSR.rst
M Lib/test/test_list.py
M Modules/_ctypes/_ctypes.c
M Objects/listobject.c

diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py
index def4badbf557..5078d4bc1ced 100644
--- a/Lib/test/test_list.py
+++ b/Lib/test/test_list.py
@@ -149,6 +149,11 @@ def test_reversed_pickle(self):
             a[:] = data
             self.assertEqual(list(it), [])
 
+    def test_step_overflow(self):
+        a = [0, 1, 2, 3, 4]
+        a[1::sys.maxsize] = [0]
+        self.assertEqual(a[3::sys.maxsize], [3])
+
     def test_no_comdat_folding(self):
         # Issue 8847: In the PGO build, the MSVC linker's COMDAT folding
         # optimization causes failures in code that relies on distinct
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-02-16-17-42.bpo-36946._lAuSR.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-02-16-17-42.bpo-36946._lAuSR.rst
new file mode 100644
index 000000000000..e0c5d712f001
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-02-16-17-42.bpo-36946._lAuSR.rst	
@@ -0,0 +1 @@
+Fix possible signed integer overflow when handling slices. Patch by hongweipeng.
diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index 8fa662755261..8008a481e275 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -5019,7 +5019,8 @@ Pointer_subscript(PyObject *myself, PyObject *item)
         PyObject *np;
         StgDictObject *stgdict, *itemdict;
         PyObject *proto;
-        Py_ssize_t i, len, cur;
+        Py_ssize_t i, len;
+        size_t cur;
 
         /* Since pointers have no length, and we want to apply
            different semantics to negative indices than normal
diff --git a/Objects/listobject.c b/Objects/listobject.c
index 78aa8dea08ac..c5e7553efcf9 100644
--- a/Objects/listobject.c
+++ b/Objects/listobject.c
@@ -2729,7 +2729,8 @@ list_subscript(PyListObject* self, PyObject* item)
         return list_item(self, i);
     }
     else if (PySlice_Check(item)) {
-        Py_ssize_t start, stop, step, slicelength, cur, i;
+        Py_ssize_t start, stop, step, slicelength, i;
+        size_t cur;
         PyObject* result;
         PyObject* it;
         PyObject **src, **dest;
@@ -2865,7 +2866,8 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value)
             /* assign slice */
             PyObject *ins, *seq;
             PyObject **garbage, **seqitems, **selfitems;
-            Py_ssize_t cur, i;
+            Py_ssize_t i;
+            size_t cur;
 
             /* protect against a[::-1] = a */
             if (self == (PyListObject*)value) {


From webhook-mailer at python.org  Sun Sep  8 19:58:07 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Sun, 08 Sep 2019 23:58:07 -0000
Subject: [Python-checkins] bpo-36018: Address more reviewer feedback
 (GH-15733)
Message-ID: <mailman.282.1567987088.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4db25d5c39e369f4b55eab52dc8f87f390233892
commit: 4db25d5c39e369f4b55eab52dc8f87f390233892
branch: master
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-08T16:57:58-07:00
summary:

bpo-36018: Address more reviewer feedback (GH-15733)

files:
M Doc/library/statistics.rst
M Lib/statistics.py
M Lib/test/test_statistics.py

diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index 0798ae29118e..bdd706d0a93e 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -514,15 +514,14 @@ However, for reading convenience, most of the examples show sorted sequences.
 
    Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.  Set
    *n* to 100 for percentiles which gives the 99 cuts points that separate
-   *data* in to 100 equal sized groups.  Raises :exc:`StatisticsError` if *n*
+   *data* into 100 equal sized groups.  Raises :exc:`StatisticsError` if *n*
    is not least 1.
 
-   The *data* can be any iterable containing sample data or it can be an
-   instance of a class that defines an :meth:`~inv_cdf` method.  For meaningful
+   The *data* can be any iterable containing sample data.  For meaningful
    results, the number of data points in *data* should be larger than *n*.
    Raises :exc:`StatisticsError` if there are not at least two data points.
 
-   For sample data, the cut points are linearly interpolated from the
+   The cut points are linearly interpolated from the
    two nearest data points.  For example, if a cut point falls one-third
    of the distance between two sample values, ``100`` and ``112``, the
    cut-point will evaluate to ``104``.
@@ -547,9 +546,6 @@ However, for reading convenience, most of the examples show sorted sequences.
    values, the method sorts them and assigns the following percentiles:
    0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%.
 
-   If *data* is an instance of a class that defines an
-   :meth:`~inv_cdf` method, setting *method* has no effect.
-
    .. doctest::
 
         # Decile cut points for empirically sampled data
@@ -561,11 +557,6 @@ However, for reading convenience, most of the examples show sorted sequences.
         >>> [round(q, 1) for q in quantiles(data, n=10)]
         [81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0]
 
-        >>> # Quartile cut points for the standard normal distribution
-        >>> Z = NormalDist()
-        >>> [round(q, 4) for q in quantiles(Z, n=4)]
-        [-0.6745, 0.0, 0.6745]
-
    .. versionadded:: 3.8
 
 
@@ -607,6 +598,18 @@ of applications in statistics.
        <https://en.wikipedia.org/wiki/Arithmetic_mean>`_ of a normal
        distribution.
 
+    .. attribute:: median
+
+       A read-only property for the `median
+       <https://en.wikipedia.org/wiki/Median>`_ of a normal
+       distribution.
+
+    .. attribute:: mode
+
+       A read-only property for the `mode
+       <https://en.wikipedia.org/wiki/Mode_(statistics)>`_ of a normal
+       distribution.
+
     .. attribute:: stdev
 
        A read-only property for the `standard deviation
@@ -678,6 +681,16 @@ of applications in statistics.
        the two probability density functions
        <https://www.rasch.org/rmt/rmt101r.htm>`_.
 
+    .. method:: NormalDist.quantiles()
+
+        Divide the normal distribution into *n* continuous intervals with
+        equal probability.  Returns a list of (n - 1) cut points separating
+        the intervals.
+
+        Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.
+        Set *n* to 100 for percentiles which gives the 99 cuts points that
+        separate the normal distribution into 100 equal sized groups.
+
     Instances of :class:`NormalDist` support addition, subtraction,
     multiplication and division by a constant.  These operations
     are used for translation and scaling.  For example:
@@ -733,9 +746,9 @@ Find the `quartiles <https://en.wikipedia.org/wiki/Quartile>`_ and `deciles
 
 .. doctest::
 
-    >>> list(map(round, quantiles(sat)))
+    >>> list(map(round, sat.quantiles()))
     [928, 1060, 1192]
-    >>> list(map(round, quantiles(sat, n=10)))
+    >>> list(map(round, sat.quantiles(n=10)))
     [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310]
 
 To estimate the distribution for a model than isn't easy to solve
diff --git a/Lib/statistics.py b/Lib/statistics.py
index 4b172662770f..70c48d605d19 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -624,9 +624,8 @@ def quantiles(data, /, *, n=4, method='exclusive'):
     Set *n* to 100 for percentiles which gives the 99 cuts points that
     separate *data* in to 100 equal sized groups.
 
-    The *data* can be any iterable containing sample data or it can be
-    an instance of a class that defines an inv_cdf() method.  For sample
-    data, the cut points are linearly interpolated between data points.
+    The *data* can be any iterable containing sample.
+    The cut points are linearly interpolated between data points.
 
     If *method* is set to *inclusive*, *data* is treated as population
     data.  The minimum value is treated as the 0th percentile and the
@@ -634,8 +633,6 @@ def quantiles(data, /, *, n=4, method='exclusive'):
     """
     if n < 1:
         raise StatisticsError('n must be at least 1')
-    if hasattr(data, 'inv_cdf'):
-        return [data.inv_cdf(i / n) for i in range(1, n)]
     data = sorted(data)
     ld = len(data)
     if ld < 2:
@@ -955,6 +952,17 @@ def inv_cdf(self, p):
             raise StatisticsError('cdf() not defined when sigma at or below zero')
         return _normal_dist_inv_cdf(p, self._mu, self._sigma)
 
+    def quantiles(self, n=4):
+        """Divide into *n* continuous intervals with equal probability.
+
+        Returns a list of (n - 1) cut points separating the intervals.
+
+        Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.
+        Set *n* to 100 for percentiles which gives the 99 cuts points that
+        separate the normal distribution in to 100 equal sized groups.
+        """
+        return [self.inv_cdf(i / n) for i in range(1, n)]
+
     def overlap(self, other):
         """Compute the overlapping coefficient (OVL) between two normal distributions.
 
@@ -994,6 +1002,20 @@ def mean(self):
         "Arithmetic mean of the normal distribution."
         return self._mu
 
+    @property
+    def median(self):
+        "Return the median of the normal distribution"
+        return self._mu
+
+    @property
+    def mode(self):
+        """Return the mode of the normal distribution
+
+        The mode is the value x where which the probability density
+        function (pdf) takes its maximum value.
+        """
+        return self._mu
+
     @property
     def stdev(self):
         "Standard deviation of the normal distribution."
diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py
index 01b317c3281e..af26473e8fdf 100644
--- a/Lib/test/test_statistics.py
+++ b/Lib/test/test_statistics.py
@@ -2198,16 +2198,6 @@ def f(x):
             exp = list(map(f, expected))
             act = quantiles(map(f, data), n=n)
             self.assertTrue(all(math.isclose(e, a) for e, a in zip(exp, act)))
-        # Quartiles of a standard normal distribution
-        for n, expected in [
-            (1, []),
-            (2, [0.0]),
-            (3, [-0.4307, 0.4307]),
-            (4 ,[-0.6745, 0.0, 0.6745]),
-                ]:
-            actual = quantiles(statistics.NormalDist(), n=n)
-            self.assertTrue(all(math.isclose(e, a, abs_tol=0.0001)
-                            for e, a in zip(expected, actual)))
         # Q2 agrees with median()
         for k in range(2, 60):
             data = random.choices(range(100), k=k)
@@ -2248,16 +2238,6 @@ def f(x):
             exp = list(map(f, expected))
             act = quantiles(map(f, data), n=n, method="inclusive")
             self.assertTrue(all(math.isclose(e, a) for e, a in zip(exp, act)))
-        # Quartiles of a standard normal distribution
-        for n, expected in [
-            (1, []),
-            (2, [0.0]),
-            (3, [-0.4307, 0.4307]),
-            (4 ,[-0.6745, 0.0, 0.6745]),
-                ]:
-            actual = quantiles(statistics.NormalDist(), n=n, method="inclusive")
-            self.assertTrue(all(math.isclose(e, a, abs_tol=0.0001)
-                            for e, a in zip(expected, actual)))
         # Natural deciles
         self.assertEqual(quantiles([0, 100], n=10, method='inclusive'),
                          [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0])
@@ -2546,6 +2526,19 @@ def test_inv_cdf(self):
         # Special values
         self.assertTrue(math.isnan(Z.inv_cdf(float('NaN'))))
 
+    def test_quantiles(self):
+        # Quartiles of a standard normal distribution
+        Z = self.module.NormalDist()
+        for n, expected in [
+            (1, []),
+            (2, [0.0]),
+            (3, [-0.4307, 0.4307]),
+            (4 ,[-0.6745, 0.0, 0.6745]),
+                ]:
+            actual = Z.quantiles(n=n)
+            self.assertTrue(all(math.isclose(e, a, abs_tol=0.0001)
+                            for e, a in zip(expected, actual)))
+
     def test_overlap(self):
         NormalDist = self.module.NormalDist
 
@@ -2612,6 +2605,8 @@ def overlap_numeric(X, Y, *, steps=8_192, z=5):
     def test_properties(self):
         X = self.module.NormalDist(100, 15)
         self.assertEqual(X.mean, 100)
+        self.assertEqual(X.median, 100)
+        self.assertEqual(X.mode, 100)
         self.assertEqual(X.stdev, 15)
         self.assertEqual(X.variance, 225)
 


From webhook-mailer at python.org  Sun Sep  8 21:40:10 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Mon, 09 Sep 2019 01:40:10 -0000
Subject: [Python-checkins] [3.8] bpo-36018: Address more reviewer feedback
 (GH-15733) (GH-15734)
Message-ID: <mailman.289.1567993211.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/cc1bdf91d53b1a4751be84ef607e24e69a327a9b
commit: cc1bdf91d53b1a4751be84ef607e24e69a327a9b
branch: 3.8
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-08T18:40:06-07:00
summary:

[3.8] bpo-36018: Address more reviewer feedback (GH-15733) (GH-15734)

files:
M Doc/library/statistics.rst
M Lib/statistics.py
M Lib/test/test_statistics.py

diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index 9e39828a5700..bdd706d0a93e 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -514,15 +514,14 @@ However, for reading convenience, most of the examples show sorted sequences.
 
    Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.  Set
    *n* to 100 for percentiles which gives the 99 cuts points that separate
-   *data* in to 100 equal sized groups.  Raises :exc:`StatisticsError` if *n*
+   *data* into 100 equal sized groups.  Raises :exc:`StatisticsError` if *n*
    is not least 1.
 
-   The *data* can be any iterable containing sample data or it can be an
-   instance of a class that defines an :meth:`~inv_cdf` method.  For meaningful
+   The *data* can be any iterable containing sample data.  For meaningful
    results, the number of data points in *data* should be larger than *n*.
    Raises :exc:`StatisticsError` if there are not at least two data points.
 
-   For sample data, the cut points are linearly interpolated from the
+   The cut points are linearly interpolated from the
    two nearest data points.  For example, if a cut point falls one-third
    of the distance between two sample values, ``100`` and ``112``, the
    cut-point will evaluate to ``104``.
@@ -547,9 +546,6 @@ However, for reading convenience, most of the examples show sorted sequences.
    values, the method sorts them and assigns the following percentiles:
    0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%.
 
-   If *data* is an instance of a class that defines an
-   :meth:`~inv_cdf` method, setting *method* has no effect.
-
    .. doctest::
 
         # Decile cut points for empirically sampled data
@@ -561,11 +557,6 @@ However, for reading convenience, most of the examples show sorted sequences.
         >>> [round(q, 1) for q in quantiles(data, n=10)]
         [81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0]
 
-        >>> # Quartile cut points for the standard normal distibution
-        >>> Z = NormalDist()
-        >>> [round(q, 4) for q in quantiles(Z, n=4)]
-        [-0.6745, 0.0, 0.6745]
-
    .. versionadded:: 3.8
 
 
@@ -607,6 +598,18 @@ of applications in statistics.
        <https://en.wikipedia.org/wiki/Arithmetic_mean>`_ of a normal
        distribution.
 
+    .. attribute:: median
+
+       A read-only property for the `median
+       <https://en.wikipedia.org/wiki/Median>`_ of a normal
+       distribution.
+
+    .. attribute:: mode
+
+       A read-only property for the `mode
+       <https://en.wikipedia.org/wiki/Mode_(statistics)>`_ of a normal
+       distribution.
+
     .. attribute:: stdev
 
        A read-only property for the `standard deviation
@@ -678,6 +681,16 @@ of applications in statistics.
        the two probability density functions
        <https://www.rasch.org/rmt/rmt101r.htm>`_.
 
+    .. method:: NormalDist.quantiles()
+
+        Divide the normal distribution into *n* continuous intervals with
+        equal probability.  Returns a list of (n - 1) cut points separating
+        the intervals.
+
+        Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.
+        Set *n* to 100 for percentiles which gives the 99 cuts points that
+        separate the normal distribution into 100 equal sized groups.
+
     Instances of :class:`NormalDist` support addition, subtraction,
     multiplication and division by a constant.  These operations
     are used for translation and scaling.  For example:
@@ -733,9 +746,9 @@ Find the `quartiles <https://en.wikipedia.org/wiki/Quartile>`_ and `deciles
 
 .. doctest::
 
-    >>> list(map(round, quantiles(sat)))
+    >>> list(map(round, sat.quantiles()))
     [928, 1060, 1192]
-    >>> list(map(round, quantiles(sat, n=10)))
+    >>> list(map(round, sat.quantiles(n=10)))
     [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310]
 
 To estimate the distribution for a model than isn't easy to solve
diff --git a/Lib/statistics.py b/Lib/statistics.py
index 4b172662770f..70c48d605d19 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -624,9 +624,8 @@ def quantiles(data, /, *, n=4, method='exclusive'):
     Set *n* to 100 for percentiles which gives the 99 cuts points that
     separate *data* in to 100 equal sized groups.
 
-    The *data* can be any iterable containing sample data or it can be
-    an instance of a class that defines an inv_cdf() method.  For sample
-    data, the cut points are linearly interpolated between data points.
+    The *data* can be any iterable containing sample.
+    The cut points are linearly interpolated between data points.
 
     If *method* is set to *inclusive*, *data* is treated as population
     data.  The minimum value is treated as the 0th percentile and the
@@ -634,8 +633,6 @@ def quantiles(data, /, *, n=4, method='exclusive'):
     """
     if n < 1:
         raise StatisticsError('n must be at least 1')
-    if hasattr(data, 'inv_cdf'):
-        return [data.inv_cdf(i / n) for i in range(1, n)]
     data = sorted(data)
     ld = len(data)
     if ld < 2:
@@ -955,6 +952,17 @@ def inv_cdf(self, p):
             raise StatisticsError('cdf() not defined when sigma at or below zero')
         return _normal_dist_inv_cdf(p, self._mu, self._sigma)
 
+    def quantiles(self, n=4):
+        """Divide into *n* continuous intervals with equal probability.
+
+        Returns a list of (n - 1) cut points separating the intervals.
+
+        Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.
+        Set *n* to 100 for percentiles which gives the 99 cuts points that
+        separate the normal distribution in to 100 equal sized groups.
+        """
+        return [self.inv_cdf(i / n) for i in range(1, n)]
+
     def overlap(self, other):
         """Compute the overlapping coefficient (OVL) between two normal distributions.
 
@@ -994,6 +1002,20 @@ def mean(self):
         "Arithmetic mean of the normal distribution."
         return self._mu
 
+    @property
+    def median(self):
+        "Return the median of the normal distribution"
+        return self._mu
+
+    @property
+    def mode(self):
+        """Return the mode of the normal distribution
+
+        The mode is the value x where which the probability density
+        function (pdf) takes its maximum value.
+        """
+        return self._mu
+
     @property
     def stdev(self):
         "Standard deviation of the normal distribution."
diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py
index 01b317c3281e..af26473e8fdf 100644
--- a/Lib/test/test_statistics.py
+++ b/Lib/test/test_statistics.py
@@ -2198,16 +2198,6 @@ def f(x):
             exp = list(map(f, expected))
             act = quantiles(map(f, data), n=n)
             self.assertTrue(all(math.isclose(e, a) for e, a in zip(exp, act)))
-        # Quartiles of a standard normal distribution
-        for n, expected in [
-            (1, []),
-            (2, [0.0]),
-            (3, [-0.4307, 0.4307]),
-            (4 ,[-0.6745, 0.0, 0.6745]),
-                ]:
-            actual = quantiles(statistics.NormalDist(), n=n)
-            self.assertTrue(all(math.isclose(e, a, abs_tol=0.0001)
-                            for e, a in zip(expected, actual)))
         # Q2 agrees with median()
         for k in range(2, 60):
             data = random.choices(range(100), k=k)
@@ -2248,16 +2238,6 @@ def f(x):
             exp = list(map(f, expected))
             act = quantiles(map(f, data), n=n, method="inclusive")
             self.assertTrue(all(math.isclose(e, a) for e, a in zip(exp, act)))
-        # Quartiles of a standard normal distribution
-        for n, expected in [
-            (1, []),
-            (2, [0.0]),
-            (3, [-0.4307, 0.4307]),
-            (4 ,[-0.6745, 0.0, 0.6745]),
-                ]:
-            actual = quantiles(statistics.NormalDist(), n=n, method="inclusive")
-            self.assertTrue(all(math.isclose(e, a, abs_tol=0.0001)
-                            for e, a in zip(expected, actual)))
         # Natural deciles
         self.assertEqual(quantiles([0, 100], n=10, method='inclusive'),
                          [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0])
@@ -2546,6 +2526,19 @@ def test_inv_cdf(self):
         # Special values
         self.assertTrue(math.isnan(Z.inv_cdf(float('NaN'))))
 
+    def test_quantiles(self):
+        # Quartiles of a standard normal distribution
+        Z = self.module.NormalDist()
+        for n, expected in [
+            (1, []),
+            (2, [0.0]),
+            (3, [-0.4307, 0.4307]),
+            (4 ,[-0.6745, 0.0, 0.6745]),
+                ]:
+            actual = Z.quantiles(n=n)
+            self.assertTrue(all(math.isclose(e, a, abs_tol=0.0001)
+                            for e, a in zip(expected, actual)))
+
     def test_overlap(self):
         NormalDist = self.module.NormalDist
 
@@ -2612,6 +2605,8 @@ def overlap_numeric(X, Y, *, steps=8_192, z=5):
     def test_properties(self):
         X = self.module.NormalDist(100, 15)
         self.assertEqual(X.mean, 100)
+        self.assertEqual(X.median, 100)
+        self.assertEqual(X.mode, 100)
         self.assertEqual(X.stdev, 15)
         self.assertEqual(X.variance, 225)
 


From webhook-mailer at python.org  Mon Sep  9 04:18:34 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Mon, 09 Sep 2019 08:18:34 -0000
Subject: [Python-checkins] Revert "Raise a RuntimeError when tee iterator is
 consumed from different threads (GH-15567)" (GH-15736)
Message-ID: <mailman.290.1568017115.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/918b468b7d5ebf1ec5e604cb0d99605cee38d983
commit: 918b468b7d5ebf1ec5e604cb0d99605cee38d983
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T11:18:16+03:00
summary:

Revert "Raise a RuntimeError when tee iterator is consumed from different threads (GH-15567)" (GH-15736)

This reverts commit fa220ec7633e9674baccc28dde987f29d7f65141.

files:
D Misc/NEWS.d/next/Library/2019-08-29-10-26-57.bpo-34410.ttCIpm.rst
M Doc/library/itertools.rst
M Modules/itertoolsmodule.c

diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst
index 8e7899e6b613..a3f403a5b40b 100644
--- a/Doc/library/itertools.rst
+++ b/Doc/library/itertools.rst
@@ -643,8 +643,7 @@ loops that truncate the stream.
 
    Once :func:`tee` has made a split, the original *iterable* should not be
    used anywhere else; otherwise, the *iterable* could get advanced without
-   the tee objects being informed. the :func:`tee` iterator can not be consumed
-   from different threads, even if an underlying iterator is thread-safe.
+   the tee objects being informed.
 
    This itertool may require significant auxiliary storage (depending on how
    much temporary data needs to be stored). In general, if one iterator uses
diff --git a/Misc/NEWS.d/next/Library/2019-08-29-10-26-57.bpo-34410.ttCIpm.rst b/Misc/NEWS.d/next/Library/2019-08-29-10-26-57.bpo-34410.ttCIpm.rst
deleted file mode 100644
index caddada835fc..000000000000
--- a/Misc/NEWS.d/next/Library/2019-08-29-10-26-57.bpo-34410.ttCIpm.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Raise a RuntimeError when itertools.tee() iterator is consumed from different
-threads. Patch by hongweipeng.
diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c
index 2e398425a146..cf419ad85325 100644
--- a/Modules/itertoolsmodule.c
+++ b/Modules/itertoolsmodule.c
@@ -452,7 +452,6 @@ typedef struct {
     teedataobject *dataobj;
     int index;                  /* 0 <= index <= LINKCELLS */
     PyObject *weakreflist;
-    unsigned long thread_id;
 } teeobject;
 
 static PyTypeObject teedataobject_type;
@@ -681,11 +680,6 @@ tee_next(teeobject *to)
 {
     PyObject *value, *link;
 
-    if (to->thread_id != PyThread_get_thread_ident()) {
-        PyErr_SetString(PyExc_RuntimeError,
-            "tee() iterator can not be consumed from different threads.");
-        return NULL;
-    }
     if (to->index >= LINKCELLS) {
         link = teedataobject_jumplink(to->dataobj);
         if (link == NULL)
@@ -719,7 +713,6 @@ tee_copy(teeobject *to, PyObject *Py_UNUSED(ignored))
     newto->dataobj = to->dataobj;
     newto->index = to->index;
     newto->weakreflist = NULL;
-    newto->thread_id = to->thread_id;
     PyObject_GC_Track(newto);
     return (PyObject *)newto;
 }
@@ -752,7 +745,6 @@ tee_fromiterable(PyObject *iterable)
 
     to->index = 0;
     to->weakreflist = NULL;
-    to->thread_id = PyThread_get_thread_ident();
     PyObject_GC_Track(to);
 done:
     Py_XDECREF(it);


From webhook-mailer at python.org  Mon Sep  9 04:47:18 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Mon, 09 Sep 2019 08:47:18 -0000
Subject: [Python-checkins] bpo-34410: Fix a crash in the tee iterator when
 re-enter it. (GH-15625)
Message-ID: <mailman.291.1568018839.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/526a01467b3277f9fcf7f91e66c23321caa1245d
commit: 526a01467b3277f9fcf7f91e66c23321caa1245d
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T11:47:14+03:00
summary:

bpo-34410: Fix a crash in the tee iterator when re-enter it. (GH-15625)

RuntimeError is now raised in this case.

files:
A Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
M Doc/library/itertools.rst
M Lib/test/test_itertools.py
M Modules/itertoolsmodule.c

diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst
index a3f403a5b40b..8d134d438065 100644
--- a/Doc/library/itertools.rst
+++ b/Doc/library/itertools.rst
@@ -645,6 +645,10 @@ loops that truncate the stream.
    used anywhere else; otherwise, the *iterable* could get advanced without
    the tee objects being informed.
 
+   ``tee`` iterators are not threadsafe. A :exc:`RuntimeError` may be
+   raised when using simultaneously iterators returned by the same :func:`tee`
+   call, even if the original *iterable* is threadsafe.
+
    This itertool may require significant auxiliary storage (depending on how
    much temporary data needs to be stored). In general, if one iterator uses
    most or all of the data before another iterator starts, it is faster to use
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
index 98b8c8373189..eaa6197bec39 100644
--- a/Lib/test/test_itertools.py
+++ b/Lib/test/test_itertools.py
@@ -11,6 +11,7 @@
 from functools import reduce
 import sys
 import struct
+import threading
 maxsize = support.MAX_Py_ssize_t
 minsize = -maxsize-1
 
@@ -1494,6 +1495,42 @@ def test_tee_del_backward(self):
             del forward, backward
             raise
 
+    def test_tee_reenter(self):
+        class I:
+            first = True
+            def __iter__(self):
+                return self
+            def __next__(self):
+                first = self.first
+                self.first = False
+                if first:
+                    return next(b)
+
+        a, b = tee(I())
+        with self.assertRaisesRegex(RuntimeError, "tee"):
+            next(a)
+
+    def test_tee_concurrent(self):
+        start = threading.Event()
+        finish = threading.Event()
+        class I:
+            def __iter__(self):
+                return self
+            def __next__(self):
+                start.set()
+                finish.wait()
+
+        a, b = tee(I())
+        thread = threading.Thread(target=next, args=[a])
+        thread.start()
+        try:
+            start.wait()
+            with self.assertRaisesRegex(RuntimeError, "tee"):
+                next(b)
+        finally:
+            finish.set()
+            thread.join()
+
     def test_StopIteration(self):
         self.assertRaises(StopIteration, next, zip())
 
diff --git a/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst b/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
new file mode 100644
index 000000000000..64e778ee0913
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
@@ -0,0 +1,2 @@
+Fixed a crash in the :func:`tee` iterator when re-enter it. RuntimeError is
+now raised in this case.
diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c
index cf419ad85325..e60ad5bec43c 100644
--- a/Modules/itertoolsmodule.c
+++ b/Modules/itertoolsmodule.c
@@ -443,6 +443,7 @@ typedef struct {
     PyObject_HEAD
     PyObject *it;
     int numread;                /* 0 <= numread <= LINKCELLS */
+    int running;
     PyObject *nextlink;
     PyObject *(values[LINKCELLS]);
 } teedataobject;
@@ -465,6 +466,7 @@ teedataobject_newinternal(PyObject *it)
     if (tdo == NULL)
         return NULL;
 
+    tdo->running = 0;
     tdo->numread = 0;
     tdo->nextlink = NULL;
     Py_INCREF(it);
@@ -493,7 +495,14 @@ teedataobject_getitem(teedataobject *tdo, int i)
     else {
         /* this is the lead iterator, so fetch more data */
         assert(i == tdo->numread);
+        if (tdo->running) {
+            PyErr_SetString(PyExc_RuntimeError,
+                            "cannot re-enter the tee iterator");
+            return NULL;
+        }
+        tdo->running = 1;
         value = PyIter_Next(tdo->it);
+        tdo->running = 0;
         if (value == NULL)
             return NULL;
         tdo->numread++;


From webhook-mailer at python.org  Mon Sep  9 05:05:01 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Mon, 09 Sep 2019 09:05:01 -0000
Subject: [Python-checkins] Fix assertions regarding magic methods function
 body that was not executed (GH-14154)
Message-ID: <mailman.292.1568019902.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/aa515082749687c1e3bc9ec5e2296368191b9f84
commit: aa515082749687c1e3bc9ec5e2296368191b9f84
branch: master
author: Xtreak <tir.karthi at gmail.com>
committer: Lisa Roach <lisaroach14 at gmail.com>
date: 2019-09-09T10:04:57+01:00
summary:

Fix assertions regarding magic methods function body that was not executed (GH-14154)

files:
M Lib/unittest/test/testmock/testasync.py

diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index fa906e4f7152..865cfdc00264 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -361,17 +361,14 @@ def test_add_side_effect_iterable(self):
 
 
 class AsyncContextManagerTest(unittest.TestCase):
+
     class WithAsyncContextManager:
-        def __init__(self):
-            self.entered = False
-            self.exited = False
 
         async def __aenter__(self, *args, **kwargs):
-            self.entered = True
             return self
 
         async def __aexit__(self, *args, **kwargs):
-            self.exited = True
+            pass
 
     def test_magic_methods_are_async_mocks(self):
         mock = MagicMock(self.WithAsyncContextManager())
@@ -390,11 +387,7 @@ def test_mock_supports_async_context_manager(self):
             return result
 
         result = asyncio.run(use_context_manager())
-        self.assertFalse(instance.entered)
-        self.assertFalse(instance.exited)
         self.assertTrue(called)
-        self.assertTrue(mock_instance.entered)
-        self.assertTrue(mock_instance.exited)
         self.assertTrue(mock_instance.__aenter__.called)
         self.assertTrue(mock_instance.__aexit__.called)
         self.assertIsNot(mock_instance, result)


From webhook-mailer at python.org  Mon Sep  9 05:06:41 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Mon, 09 Sep 2019 09:06:41 -0000
Subject: [Python-checkins] bpo-15817: gdbinit: Document commands after
 defining them (GH-15021)
Message-ID: <mailman.293.1568020002.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1f86fdcfc57270ee569cc58269a4e08afe7608ec
commit: 1f86fdcfc57270ee569cc58269a4e08afe7608ec
branch: master
author: Florian Bruhin <me at the-compiler.org>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-09T04:06:37-05:00
summary:

bpo-15817: gdbinit: Document commands after defining them (GH-15021)

The gdb manual[1] says the following for "document":

  The command commandname must already be defined.

[1] https://sourceware.org/gdb/current/onlinedocs/gdb/Define.html

And indeed when trying to use the gdbinit file with gdb 8.3, I get:

  .../cpython/Misc/gdbinit:17: Error in sourced command file:
  Undefined command: "pyo".  Try "help".

Fix this by moving all documentation blocks after the define blocks.

This was introduced in GH-6384.

files:
M Misc/gdbinit

diff --git a/Misc/gdbinit b/Misc/gdbinit
index afefe0818e4c..45e79fcf6f46 100644
--- a/Misc/gdbinit
+++ b/Misc/gdbinit
@@ -14,30 +14,27 @@
 # with embedded macros that you may find superior to what is in here.
 # See Tools/gdb/libpython.py and http://bugs.python.org/issue8032.
 
-document pyo
-  Prints a representation of the object to stderr, along with the
-  number of reference counts it currently has and the hex address the
-  object is allocated at.  The argument must be a PyObject*
-end
 define pyo
     # side effect of calling _PyObject_Dump is to dump the object's
     # info - assigning just prevents gdb from printing the
     # NULL return value
     set $_unused_void = _PyObject_Dump($arg0)
 end
-
-document pyg
+document pyo
   Prints a representation of the object to stderr, along with the
   number of reference counts it currently has and the hex address the
-  object is allocated at.  The argument must be a PyGC_Head*
+  object is allocated at.  The argument must be a PyObject*
 end
+
 define pyg
     print _PyGC_Dump($arg0)
 end
-
-document pylocals
-  Print the local variables of the current frame.
+document pyg
+  Prints a representation of the object to stderr, along with the
+  number of reference counts it currently has and the hex address the
+  object is allocated at.  The argument must be a PyGC_Head*
 end
+
 define pylocals
     set $_i = 0
     while $_i < f->f_code->co_nlocals
@@ -50,6 +47,9 @@ define pylocals
         set $_i = $_i + 1
     end
 end
+document pylocals
+  Print the local variables of the current frame.
+end
 
 # A rewrite of the Python interpreter's line number calculator in GDB's
 # command language
@@ -75,13 +75,13 @@ define lineno
     printf "%d", $__li
 end
 
-document pyframev
-  Print the current frame - verbose
-end
 define pyframev
     pyframe
     pylocals
 end
+document pyframev
+  Print the current frame - verbose
+end
 
 define pyframe
     set $__fn = PyUnicode_AsUTF8(f->f_code->co_filename)
@@ -134,9 +134,6 @@ end
 # the interpreter you may will have to change the functions you compare with
 # $pc.
 
-document pystack
-  Print the entire Python call stack
-end
 define pystack
     while $pc < Py_Main || $pc > Py_GetArgcArgv
         if $pc > PyEval_EvalFrameEx && $pc < _PyEval_EvalFrameDefault
@@ -146,10 +143,10 @@ define pystack
     end
     select-frame 0
 end
-
-document pystackv
-  Print the entire Python call stack - verbose mode
+document pystack
+  Print the entire Python call stack
 end
+
 define pystackv
     while $pc < Py_Main || $pc > Py_GetArgcArgv
         if $pc > PyEval_EvalFrameEx && $pc < _PyEval_EvalFrameDefault
@@ -159,10 +156,10 @@ define pystackv
     end
     select-frame 0
 end
-
-document pu
-  Generally useful macro to print a Unicode string
+document pystackv
+  Print the entire Python call stack - verbose mode
 end
+
 def pu
   set $uni = $arg0
   set $i = 0
@@ -174,3 +171,6 @@ def pu
     end
   end
 end
+document pu
+  Generally useful macro to print a Unicode string
+end


From webhook-mailer at python.org  Mon Sep  9 05:07:55 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 09:07:55 -0000
Subject: [Python-checkins] bpo-34410: Fix a crash in the tee iterator when
 re-enter it. (GH-15625)
Message-ID: <mailman.294.1568020076.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6e3809c7ce9fbee11c3a3f89dd7e89829b7581ac
commit: 6e3809c7ce9fbee11c3a3f89dd7e89829b7581ac
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T02:07:51-07:00
summary:

bpo-34410: Fix a crash in the tee iterator when re-enter it. (GH-15625)


RuntimeError is now raised in this case.
(cherry picked from commit 526a01467b3277f9fcf7f91e66c23321caa1245d)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
M Doc/library/itertools.rst
M Lib/test/test_itertools.py
M Modules/itertoolsmodule.c

diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst
index a3f403a5b40b..8d134d438065 100644
--- a/Doc/library/itertools.rst
+++ b/Doc/library/itertools.rst
@@ -645,6 +645,10 @@ loops that truncate the stream.
    used anywhere else; otherwise, the *iterable* could get advanced without
    the tee objects being informed.
 
+   ``tee`` iterators are not threadsafe. A :exc:`RuntimeError` may be
+   raised when using simultaneously iterators returned by the same :func:`tee`
+   call, even if the original *iterable* is threadsafe.
+
    This itertool may require significant auxiliary storage (depending on how
    much temporary data needs to be stored). In general, if one iterator uses
    most or all of the data before another iterator starts, it is faster to use
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
index 98b8c8373189..eaa6197bec39 100644
--- a/Lib/test/test_itertools.py
+++ b/Lib/test/test_itertools.py
@@ -11,6 +11,7 @@
 from functools import reduce
 import sys
 import struct
+import threading
 maxsize = support.MAX_Py_ssize_t
 minsize = -maxsize-1
 
@@ -1494,6 +1495,42 @@ def test_tee_del_backward(self):
             del forward, backward
             raise
 
+    def test_tee_reenter(self):
+        class I:
+            first = True
+            def __iter__(self):
+                return self
+            def __next__(self):
+                first = self.first
+                self.first = False
+                if first:
+                    return next(b)
+
+        a, b = tee(I())
+        with self.assertRaisesRegex(RuntimeError, "tee"):
+            next(a)
+
+    def test_tee_concurrent(self):
+        start = threading.Event()
+        finish = threading.Event()
+        class I:
+            def __iter__(self):
+                return self
+            def __next__(self):
+                start.set()
+                finish.wait()
+
+        a, b = tee(I())
+        thread = threading.Thread(target=next, args=[a])
+        thread.start()
+        try:
+            start.wait()
+            with self.assertRaisesRegex(RuntimeError, "tee"):
+                next(b)
+        finally:
+            finish.set()
+            thread.join()
+
     def test_StopIteration(self):
         self.assertRaises(StopIteration, next, zip())
 
diff --git a/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst b/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
new file mode 100644
index 000000000000..64e778ee0913
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
@@ -0,0 +1,2 @@
+Fixed a crash in the :func:`tee` iterator when re-enter it. RuntimeError is
+now raised in this case.
diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c
index 22c04f29353e..eba59ba1b880 100644
--- a/Modules/itertoolsmodule.c
+++ b/Modules/itertoolsmodule.c
@@ -443,6 +443,7 @@ typedef struct {
     PyObject_HEAD
     PyObject *it;
     int numread;                /* 0 <= numread <= LINKCELLS */
+    int running;
     PyObject *nextlink;
     PyObject *(values[LINKCELLS]);
 } teedataobject;
@@ -465,6 +466,7 @@ teedataobject_newinternal(PyObject *it)
     if (tdo == NULL)
         return NULL;
 
+    tdo->running = 0;
     tdo->numread = 0;
     tdo->nextlink = NULL;
     Py_INCREF(it);
@@ -493,7 +495,14 @@ teedataobject_getitem(teedataobject *tdo, int i)
     else {
         /* this is the lead iterator, so fetch more data */
         assert(i == tdo->numread);
+        if (tdo->running) {
+            PyErr_SetString(PyExc_RuntimeError,
+                            "cannot re-enter the tee iterator");
+            return NULL;
+        }
+        tdo->running = 1;
         value = PyIter_Next(tdo->it);
+        tdo->running = 0;
         if (value == NULL)
             return NULL;
         tdo->numread++;


From webhook-mailer at python.org  Mon Sep  9 05:11:26 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 09:11:26 -0000
Subject: [Python-checkins] bpo-34410: Fix a crash in the tee iterator when
 re-enter it. (GH-15625)
Message-ID: <mailman.295.1568020287.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5190b7193c184268d5c8a9440b3a5a8bcd84a23e
commit: 5190b7193c184268d5c8a9440b3a5a8bcd84a23e
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T02:11:22-07:00
summary:

bpo-34410: Fix a crash in the tee iterator when re-enter it. (GH-15625)


RuntimeError is now raised in this case.
(cherry picked from commit 526a01467b3277f9fcf7f91e66c23321caa1245d)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
M Doc/library/itertools.rst
M Lib/test/test_itertools.py
M Modules/itertoolsmodule.c

diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst
index 959424ff9143..76899ce75368 100644
--- a/Doc/library/itertools.rst
+++ b/Doc/library/itertools.rst
@@ -628,6 +628,10 @@ loops that truncate the stream.
    used anywhere else; otherwise, the *iterable* could get advanced without
    the tee objects being informed.
 
+   ``tee`` iterators are not threadsafe. A :exc:`RuntimeError` may be
+   raised when using simultaneously iterators returned by the same :func:`tee`
+   call, even if the original *iterable* is threadsafe.
+
    This itertool may require significant auxiliary storage (depending on how
    much temporary data needs to be stored). In general, if one iterator uses
    most or all of the data before another iterator starts, it is faster to use
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
index cbbb4c4f71d3..721a17556f7e 100644
--- a/Lib/test/test_itertools.py
+++ b/Lib/test/test_itertools.py
@@ -11,6 +11,7 @@
 from functools import reduce
 import sys
 import struct
+import threading
 maxsize = support.MAX_Py_ssize_t
 minsize = -maxsize-1
 
@@ -1476,6 +1477,42 @@ def test_tee_del_backward(self):
             del forward, backward
             raise
 
+    def test_tee_reenter(self):
+        class I:
+            first = True
+            def __iter__(self):
+                return self
+            def __next__(self):
+                first = self.first
+                self.first = False
+                if first:
+                    return next(b)
+
+        a, b = tee(I())
+        with self.assertRaisesRegex(RuntimeError, "tee"):
+            next(a)
+
+    def test_tee_concurrent(self):
+        start = threading.Event()
+        finish = threading.Event()
+        class I:
+            def __iter__(self):
+                return self
+            def __next__(self):
+                start.set()
+                finish.wait()
+
+        a, b = tee(I())
+        thread = threading.Thread(target=next, args=[a])
+        thread.start()
+        try:
+            start.wait()
+            with self.assertRaisesRegex(RuntimeError, "tee"):
+                next(b)
+        finally:
+            finish.set()
+            thread.join()
+
     def test_StopIteration(self):
         self.assertRaises(StopIteration, next, zip())
 
diff --git a/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst b/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
new file mode 100644
index 000000000000..64e778ee0913
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
@@ -0,0 +1,2 @@
+Fixed a crash in the :func:`tee` iterator when re-enter it. RuntimeError is
+now raised in this case.
diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c
index bff9df608304..0ddd98dea124 100644
--- a/Modules/itertoolsmodule.c
+++ b/Modules/itertoolsmodule.c
@@ -395,6 +395,7 @@ typedef struct {
     PyObject_HEAD
     PyObject *it;
     int numread;                /* 0 <= numread <= LINKCELLS */
+    int running;
     PyObject *nextlink;
     PyObject *(values[LINKCELLS]);
 } teedataobject;
@@ -417,6 +418,7 @@ teedataobject_newinternal(PyObject *it)
     if (tdo == NULL)
         return NULL;
 
+    tdo->running = 0;
     tdo->numread = 0;
     tdo->nextlink = NULL;
     Py_INCREF(it);
@@ -445,7 +447,14 @@ teedataobject_getitem(teedataobject *tdo, int i)
     else {
         /* this is the lead iterator, so fetch more data */
         assert(i == tdo->numread);
+        if (tdo->running) {
+            PyErr_SetString(PyExc_RuntimeError,
+                            "cannot re-enter the tee iterator");
+            return NULL;
+        }
+        tdo->running = 1;
         value = PyIter_Next(tdo->it);
+        tdo->running = 0;
         if (value == NULL)
             return NULL;
         tdo->numread++;


From webhook-mailer at python.org  Mon Sep  9 05:15:17 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 09:15:17 -0000
Subject: [Python-checkins] bpo-11953: Extend table of Windows WSA* error
 codes (GH-15004)
Message-ID: <mailman.296.1568020519.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/71ea688d662a74ddf39a3874e06c73e58df55c02
commit: 71ea688d662a74ddf39a3874e06c73e58df55c02
branch: master
author: Ngalim Siregar <ngalim.siregar at gmail.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-09T02:15:14-07:00
summary:

bpo-11953: Extend table of Windows WSA* error codes (GH-15004)

files:
A Misc/NEWS.d/next/Library/2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst
M Lib/socket.py

diff --git a/Lib/socket.py b/Lib/socket.py
index 3016b03104d4..87b097f2c071 100644
--- a/Lib/socket.py
+++ b/Lib/socket.py
@@ -104,24 +104,105 @@ def _intenum_converter(value, enum_klass):
     except ValueError:
         return value
 
+
 # WSA error codes
 if sys.platform.lower().startswith("win"):
     errorTab = {}
+    errorTab[6] = "Specified event object handle is invalid."
+    errorTab[8] = "Insufficient memory available."
+    errorTab[87] = "One or more parameters are invalid."
+    errorTab[995] = "Overlapped operation aborted."
+    errorTab[996] = "Overlapped I/O event object not in signaled state."
+    errorTab[997] = "Overlapped operation will complete later."
     errorTab[10004] = "The operation was interrupted."
     errorTab[10009] = "A bad file handle was passed."
     errorTab[10013] = "Permission denied."
-    errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT
+    errorTab[10014] = "A fault occurred on the network??"  # WSAEFAULT
     errorTab[10022] = "An invalid operation was attempted."
+    errorTab[10024] = "Too many open files."
     errorTab[10035] = "The socket operation would block"
     errorTab[10036] = "A blocking operation is already in progress."
+    errorTab[10037] = "Operation already in progress."
+    errorTab[10038] = "Socket operation on nonsocket."
+    errorTab[10039] = "Destination address required."
+    errorTab[10040] = "Message too long."
+    errorTab[10041] = "Protocol wrong type for socket."
+    errorTab[10042] = "Bad protocol option."
+    errorTab[10043] = "Protocol not supported."
+    errorTab[10044] = "Socket type not supported."
+    errorTab[10045] = "Operation not supported."
+    errorTab[10046] = "Protocol family not supported."
+    errorTab[10047] = "Address family not supported by protocol family."
     errorTab[10048] = "The network address is in use."
+    errorTab[10049] = "Cannot assign requested address."
+    errorTab[10050] = "Network is down."
+    errorTab[10051] = "Network is unreachable."
+    errorTab[10052] = "Network dropped connection on reset."
+    errorTab[10053] = "Software caused connection abort."
     errorTab[10054] = "The connection has been reset."
+    errorTab[10055] = "No buffer space available."
+    errorTab[10056] = "Socket is already connected."
+    errorTab[10057] = "Socket is not connected."
     errorTab[10058] = "The network has been shut down."
+    errorTab[10059] = "Too many references."
     errorTab[10060] = "The operation timed out."
     errorTab[10061] = "Connection refused."
+    errorTab[10062] = "Cannot translate name."
     errorTab[10063] = "The name is too long."
     errorTab[10064] = "The host is down."
     errorTab[10065] = "The host is unreachable."
+    errorTab[10066] = "Directory not empty."
+    errorTab[10067] = "Too many processes."
+    errorTab[10068] = "User quota exceeded."
+    errorTab[10069] = "Disk quota exceeded."
+    errorTab[10070] = "Stale file handle reference."
+    errorTab[10071] = "Item is remote."
+    errorTab[10091] = "Network subsystem is unavailable."
+    errorTab[10092] = "Winsock.dll version out of range."
+    errorTab[10093] = "Successful WSAStartup not yet performed."
+    errorTab[10101] = "Graceful shutdown in progress."
+    errorTab[10102] = "No more results from WSALookupServiceNext."
+    errorTab[10103] = "Call has been canceled."
+    errorTab[10104] = "Procedure call table is invalid."
+    errorTab[10105] = "Service provider is invalid."
+    errorTab[10106] = "Service provider failed to initialize."
+    errorTab[10107] = "System call failure."
+    errorTab[10108] = "Service not found."
+    errorTab[10109] = "Class type not found."
+    errorTab[10110] = "No more results from WSALookupServiceNext."
+    errorTab[10111] = "Call was canceled."
+    errorTab[10112] = "Database query was refused."
+    errorTab[11001] = "Host not found."
+    errorTab[11002] = "Nonauthoritative host not found."
+    errorTab[11003] = "This is a nonrecoverable error."
+    errorTab[11004] = "Valid name, no data record requested type."
+    errorTab[11005] = "QoS receivers."
+    errorTab[11006] = "QoS senders."
+    errorTab[11007] = "No QoS senders."
+    errorTab[11008] = "QoS no receivers."
+    errorTab[11009] = "QoS request confirmed."
+    errorTab[11010] = "QoS admission error."
+    errorTab[11011] = "QoS policy failure."
+    errorTab[11012] = "QoS bad style."
+    errorTab[11013] = "QoS bad object."
+    errorTab[11014] = "QoS traffic control error."
+    errorTab[11015] = "QoS generic error."
+    errorTab[11016] = "QoS service type error."
+    errorTab[11017] = "QoS flowspec error."
+    errorTab[11018] = "Invalid QoS provider buffer."
+    errorTab[11019] = "Invalid QoS filter style."
+    errorTab[11020] = "Invalid QoS filter style."
+    errorTab[11021] = "Incorrect QoS filter count."
+    errorTab[11022] = "Invalid QoS object length."
+    errorTab[11023] = "Incorrect QoS flow count."
+    errorTab[11024] = "Unrecognized QoS object."
+    errorTab[11025] = "Invalid QoS policy object."
+    errorTab[11026] = "Invalid QoS flow descriptor."
+    errorTab[11027] = "Invalid QoS provider-specific flowspec."
+    errorTab[11028] = "Invalid QoS provider-specific filterspec."
+    errorTab[11029] = "Invalid QoS shape discard mode object."
+    errorTab[11030] = "Invalid QoS shaping rate object."
+    errorTab[11031] = "Reserved policy QoS element type."
     __all__.append("errorTab")
 
 
diff --git a/Misc/NEWS.d/next/Library/2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst b/Misc/NEWS.d/next/Library/2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst
new file mode 100644
index 000000000000..62f3c44c284e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst
@@ -0,0 +1 @@
+Completing WSA* error codes in :mod:`socket`.


From webhook-mailer at python.org  Mon Sep  9 05:16:35 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Mon, 09 Sep 2019 09:16:35 -0000
Subject: [Python-checkins] bpo-38043: Use `bool` for boolean flags on
 is_normalized_quickcheck. (GH-15711)
Message-ID: <mailman.297.1568020596.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7669cb8b21c7c9cef758609c44017c09d1ce4658
commit: 7669cb8b21c7c9cef758609c44017c09d1ce4658
branch: master
author: Greg Price <gnprice at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-09T02:16:31-07:00
summary:

bpo-38043: Use `bool` for boolean flags on is_normalized_quickcheck. (GH-15711)

files:
M Modules/unicodedata.c

diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c
index 5e8ba602d668..bb09b7d21f36 100644
--- a/Modules/unicodedata.c
+++ b/Modules/unicodedata.c
@@ -795,7 +795,7 @@ typedef enum {YES = 0, MAYBE = 1, NO = 2} QuickcheckResult;
  */
 static QuickcheckResult
 is_normalized_quickcheck(PyObject *self, PyObject *input,
-                         int nfc, int k, bool yes_only)
+                         bool nfc, bool k, bool yes_only)
 {
     /* An older version of the database is requested, quickchecks must be
        disabled. */
@@ -869,25 +869,25 @@ unicodedata_UCD_is_normalized_impl(PyObject *self, PyObject *form,
     }
 
     PyObject *result;
-    int nfc = 0;
-    int k = 0;
+    bool nfc = false;
+    bool k = false;
     QuickcheckResult m;
 
     PyObject *cmp;
     int match = 0;
 
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFC)) {
-        nfc = 1;
+        nfc = true;
     }
     else if (_PyUnicode_EqualToASCIIId(form, &PyId_NFKC)) {
-        nfc = 1;
-        k = 1;
+        nfc = true;
+        k = true;
     }
     else if (_PyUnicode_EqualToASCIIId(form, &PyId_NFD)) {
         /* matches default values for `nfc` and `k` */
     }
     else if (_PyUnicode_EqualToASCIIId(form, &PyId_NFKD)) {
-        k = 1;
+        k = true;
     }
     else {
         PyErr_SetString(PyExc_ValueError, "invalid normalization form");
@@ -940,28 +940,28 @@ unicodedata_UCD_normalize_impl(PyObject *self, PyObject *form,
     }
 
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFC)) {
-        if (is_normalized_quickcheck(self, input, 1, 0, true) == YES) {
+        if (is_normalized_quickcheck(self, input, true,  false, true) == YES) {
             Py_INCREF(input);
             return input;
         }
         return nfc_nfkc(self, input, 0);
     }
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFKC)) {
-        if (is_normalized_quickcheck(self, input, 1, 1, true) == YES) {
+        if (is_normalized_quickcheck(self, input, true,  true,  true) == YES) {
             Py_INCREF(input);
             return input;
         }
         return nfc_nfkc(self, input, 1);
     }
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFD)) {
-        if (is_normalized_quickcheck(self, input, 0, 0, true) == YES) {
+        if (is_normalized_quickcheck(self, input, false, false, true) == YES) {
             Py_INCREF(input);
             return input;
         }
         return nfd_nfkd(self, input, 0);
     }
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFKD)) {
-        if (is_normalized_quickcheck(self, input, 0, 1, true) == YES) {
+        if (is_normalized_quickcheck(self, input, false, true,  true) == YES) {
             Py_INCREF(input);
             return input;
         }


From webhook-mailer at python.org  Mon Sep  9 05:20:43 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 09:20:43 -0000
Subject: [Python-checkins] bpo-37445: Include FORMAT_MESSAGE_IGNORE_INSERTS
 in FormatMessageW() calls (GH-14462)
Message-ID: <mailman.298.1568020844.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a6563650c835d50f7302971a5b145e94f9d0dc68
commit: a6563650c835d50f7302971a5b145e94f9d0dc68
branch: master
author: Zackery Spytz <zspytz at gmail.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-09T02:20:38-07:00
summary:

bpo-37445: Include FORMAT_MESSAGE_IGNORE_INSERTS in FormatMessageW() calls (GH-14462)

If FormatMessageW() is passed the FORMAT_MESSAGE_FROM_SYSTEM flag without FORMAT_MESSAGE_IGNORE_INSERTS, it will fail if there are insert sequences in the message definition.

files:
A Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
M Modules/_ctypes/callproc.c
M Modules/overlapped.c
M PC/bdist_wininst/install.c

diff --git a/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst b/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
new file mode 100644
index 000000000000..e4805b4e02fb
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
@@ -0,0 +1,2 @@
+Include the ``FORMAT_MESSAGE_IGNORE_INSERTS`` flag in ``FormatMessageW()``
+calls.
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
index 85fa79983ec1..a13c89fa8262 100644
--- a/Modules/_ctypes/callproc.c
+++ b/Modules/_ctypes/callproc.c
@@ -242,7 +242,9 @@ static WCHAR *FormatError(DWORD code)
 {
     WCHAR *lpMsgBuf;
     DWORD n;
-    n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+    n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
+                       FORMAT_MESSAGE_FROM_SYSTEM |
+                       FORMAT_MESSAGE_IGNORE_INSERTS,
                        NULL,
                        code,
                        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), /* Default language */
diff --git a/Modules/overlapped.c b/Modules/overlapped.c
index 44a0a5a83463..52ed0bc284bc 100644
--- a/Modules/overlapped.c
+++ b/Modules/overlapped.c
@@ -500,7 +500,8 @@ overlapped_FormatMessage(PyObject *ignore, PyObject *args)
         return NULL;
 
     n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
-                       FORMAT_MESSAGE_FROM_SYSTEM,
+                       FORMAT_MESSAGE_FROM_SYSTEM |
+                       FORMAT_MESSAGE_IGNORE_INSERTS,
                        NULL,
                        code,
                        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
diff --git a/PC/bdist_wininst/install.c b/PC/bdist_wininst/install.c
index 0219a195bc98..5d77542582a0 100644
--- a/PC/bdist_wininst/install.c
+++ b/PC/bdist_wininst/install.c
@@ -938,7 +938,8 @@ static BOOL SystemError(int error, char *msg)
         LPVOID lpMsgBuf;
         FormatMessage(
             FORMAT_MESSAGE_ALLOCATE_BUFFER |
-            FORMAT_MESSAGE_FROM_SYSTEM,
+            FORMAT_MESSAGE_FROM_SYSTEM |
+            FORMAT_MESSAGE_IGNORE_INSERTS,
             NULL,
             error,
             MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),


From webhook-mailer at python.org  Mon Sep  9 05:26:19 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 09:26:19 -0000
Subject: [Python-checkins] bpo-32587: Make winreg.REG_MULTI_SZ support
 zero-length strings (#13239)
Message-ID: <mailman.299.1568021180.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e223ba13d8d871ee58570dfca4e82a591189cc2f
commit: e223ba13d8d871ee58570dfca4e82a591189cc2f
branch: master
author: Zackery Spytz <zspytz at gmail.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-09T02:26:15-07:00
summary:

bpo-32587: Make winreg.REG_MULTI_SZ support zero-length strings (#13239)

* bpo-32587: Make winreg.REG_MULTI_SZ support PendingFileRenameOperations

* Address review comments.

files:
A Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst
M Lib/test/test_winreg.py
M PC/winreg.c

diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py
index 91a2bbc066b1..5c25ec8f7ec6 100644
--- a/Lib/test/test_winreg.py
+++ b/Lib/test/test_winreg.py
@@ -41,6 +41,7 @@
     ("String Val",    "A string value",                        REG_SZ),
     ("StringExpand",  "The path is %path%",                    REG_EXPAND_SZ),
     ("Multi-string",  ["Lots", "of", "string", "values"],      REG_MULTI_SZ),
+    ("Multi-nul",     ["", "", "", ""],                        REG_MULTI_SZ),
     ("Raw Data",      b"binary\x00data",                       REG_BINARY),
     ("Big String",    "x"*(2**14-1),                           REG_SZ),
     ("Big Binary",    b"x"*(2**14),                            REG_BINARY),
diff --git a/Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst b/Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst
new file mode 100644
index 000000000000..41483aa8b74a
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst
@@ -0,0 +1 @@
+Make :data:`winreg.REG_MULTI_SZ` support zero-length strings.
diff --git a/PC/winreg.c b/PC/winreg.c
index d0df7ef0ad47..37bc2c72dcde 100644
--- a/PC/winreg.c
+++ b/PC/winreg.c
@@ -518,11 +518,18 @@ fixupMultiSZ(wchar_t **str, wchar_t *data, int len)
     int i;
     wchar_t *Q;
 
-    Q = data + len;
-    for (P = data, i = 0; P < Q && *P != '\0'; P++, i++) {
+    if (len > 0 && data[len - 1] == '\0') {
+        Q = data + len - 1;
+    }
+    else {
+        Q = data + len;
+    }
+
+    for (P = data, i = 0; P < Q; P++, i++) {
         str[i] = P;
-        for (; P < Q && *P != '\0'; P++)
+        for (; P < Q && *P != '\0'; P++) {
             ;
+        }
     }
 }
 
@@ -530,12 +537,20 @@ static int
 countStrings(wchar_t *data, int len)
 {
     int strings;
-    wchar_t *P;
-    wchar_t *Q = data + len;
+    wchar_t *P, *Q;
+
+    if (len > 0 && data[len - 1] == '\0') {
+        Q = data + len - 1;
+    }
+    else {
+        Q = data + len;
+    }
 
-    for (P = data, strings = 0; P < Q && *P != '\0'; P++, strings++)
-        for (; P < Q && *P != '\0'; P++)
+    for (P = data, strings = 0; P < Q; P++, strings++) {
+        for (; P < Q && *P != '\0'; P++) {
             ;
+        }
+    }
     return strings;
 }
 
@@ -749,21 +764,15 @@ Reg2Py(BYTE *retDataBuf, DWORD retDataSize, DWORD typ)
                 }
                 for (index = 0; index < s; index++)
                 {
-                    size_t len = wcslen(str[index]);
-                    if (len > INT_MAX) {
-                        PyErr_SetString(PyExc_OverflowError,
-                            "registry string is too long for a Python string");
-                        Py_DECREF(obData);
-                        PyMem_Free(str);
-                        return NULL;
-                    }
-                    PyObject *uni = PyUnicode_FromWideChar(str[index], len);
+                    size_t slen = wcsnlen(str[index], len);
+                    PyObject *uni = PyUnicode_FromWideChar(str[index], slen);
                     if (uni == NULL) {
                         Py_DECREF(obData);
                         PyMem_Free(str);
                         return NULL;
                     }
                     PyList_SET_ITEM(obData, index, uni);
+                    len -= slen + 1;
                 }
                 PyMem_Free(str);
 


From webhook-mailer at python.org  Mon Sep  9 05:34:54 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 09:34:54 -0000
Subject: [Python-checkins] bpo-37936: Avoid ignoring files that we actually
 do track. (GH-15451)
Message-ID: <mailman.300.1568021695.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5e5e9515029f70836003a8cfb30433166fcc8db7
commit: 5e5e9515029f70836003a8cfb30433166fcc8db7
branch: master
author: Greg Price <gnprice at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-09T02:34:50-07:00
summary:

bpo-37936: Avoid ignoring files that we actually do track. (GH-15451)



There were about 14 files that are actually in the repo but that are
covered by the rules in .gitignore.

Git itself takes no notice of what .gitignore says about files that
it's already tracking... but the discrepancy can be confusing to a
human that adds a new file unexpectedly covered by these rules, as
well as to non-Git software that looks at .gitignore but doesn't
implement this wrinkle in its semantics.  (E.g., `rg`.)

Several of these are from rules that apply more broadly than
intended: for example, `Makefile` applies to `Doc/Makefile` and
`Tools/freeze/test/Makefile`, whereas `/Makefile` means only the
`Makefile` at the repo's root.

And the `Modules/Setup` rule simply wasn't updated after 961d54c5c.


https://bugs.python.org/issue37936

files:
A Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst
M .gitignore

diff --git a/.gitignore b/.gitignore
index 9f3b1ac83b51..54c598e109e0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,12 +27,12 @@ Include/pydtrace_probes.h
 Lib/distutils/command/*.pdb
 Lib/lib2to3/*.pickle
 Lib/test/data/*
-Makefile
+!Lib/test/data/README
+/Makefile
 Makefile.pre
 Misc/python.pc
 Misc/python-embed.pc
 Misc/python-config.sh
-Modules/Setup
 Modules/Setup.config
 Modules/Setup.local
 Modules/config.c
@@ -79,6 +79,7 @@ config.log
 config.status
 config.status.lineno
 core
+!Tools/msi/core/
 db_home
 .hg/
 .idea/
@@ -89,7 +90,7 @@ libpython*.dylib
 libpython*.dll
 platform
 pybuilddir.txt
-pyconfig.h
+/pyconfig.h
 python-config
 python-config.py
 python.bat
diff --git a/Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst b/Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst
new file mode 100644
index 000000000000..4c6486103881
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst
@@ -0,0 +1,2 @@
+The :file:`.gitignore` file no longer applies to any files that are in fact
+tracked in the Git repository.  Patch by Greg Price.


From webhook-mailer at python.org  Mon Sep  9 05:35:12 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 09:35:12 -0000
Subject: [Python-checkins] bpo-37705: Improve the implementation of
 winerror_to_errno() (GH-15623)
Message-ID: <mailman.301.1568021713.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/19052a11314e7be7ba003fd6cdbb5400a5d77d96
commit: 19052a11314e7be7ba003fd6cdbb5400a5d77d96
branch: master
author: Zackery Spytz <zspytz at gmail.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-09T02:35:08-07:00
summary:

bpo-37705: Improve the implementation of winerror_to_errno() (GH-15623)

winerror_to_errno() is no longer automatically generated.
Do not rely on the old _dosmapperr() function.
Add ERROR_NO_UNICODE_TRANSLATION (1113) -> EILSEQ.

files:
A Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst
D PC/generrmap.c
M PC/errmap.h

diff --git a/Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst b/Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst
new file mode 100644
index 000000000000..a374c3a2965a
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst
@@ -0,0 +1 @@
+Improve the implementation of ``winerror_to_errno()``.
diff --git a/PC/errmap.h b/PC/errmap.h
index 985f673a4649..a7489ab75c65 100644
--- a/PC/errmap.h
+++ b/PC/errmap.h
@@ -1,80 +1,140 @@
-/* Generated file. Do not edit. */
-int winerror_to_errno(int winerror)
+int
+winerror_to_errno(int winerror)
 {
-    switch(winerror) {
-        case 2: return 2;
-        case 3: return 2;
-        case 4: return 24;
-        case 5: return 13;
-        case 6: return 9;
-        case 7: return 12;
-        case 8: return 12;
-        case 9: return 12;
-        case 10: return 7;
-        case 11: return 8;
-        case 15: return 2;
-        case 16: return 13;
-        case 17: return 18;
-        case 18: return 2;
-        case 19: return 13;
-        case 20: return 13;
-        case 21: return 13;
-        case 22: return 13;
-        case 23: return 13;
-        case 24: return 13;
-        case 25: return 13;
-        case 26: return 13;
-        case 27: return 13;
-        case 28: return 13;
-        case 29: return 13;
-        case 30: return 13;
-        case 31: return 13;
-        case 32: return 13;
-        case 33: return 13;
-        case 34: return 13;
-        case 35: return 13;
-        case 36: return 13;
-        case 53: return 2;
-        case 65: return 13;
-        case 67: return 2;
-        case 80: return 17;
-        case 82: return 13;
-        case 83: return 13;
-        case 89: return 11;
-        case 108: return 13;
-        case 109: return 32;
-        case 112: return 28;
-        case 114: return 9;
-        case 128: return 10;
-        case 129: return 10;
-        case 130: return 9;
-        case 132: return 13;
-        case 145: return 41;
-        case 158: return 13;
-        case 161: return 2;
-        case 164: return 11;
-        case 167: return 13;
-        case 183: return 17;
-        case 188: return 8;
-        case 189: return 8;
-        case 190: return 8;
-        case 191: return 8;
-        case 192: return 8;
-        case 193: return 8;
-        case 194: return 8;
-        case 195: return 8;
-        case 196: return 8;
-        case 197: return 8;
-        case 198: return 8;
-        case 199: return 8;
-        case 200: return 8;
-        case 201: return 8;
-        case 202: return 8;
-        case 206: return 2;
-        case 215: return 11;
-        case 232: return 32;
-        case 267: return 20;
-        case 1816: return 12;
-        default: return EINVAL;
+    // Unwrap FACILITY_WIN32 HRESULT errors.
+    if ((winerror & 0xFFFF0000) == 0x80070000) {
+        winerror &= 0x0000FFFF;
+    }
+
+    // Winsock error codes (10000-11999) are errno values.
+    if (winerror >= 10000 && winerror < 12000) {
+        switch (winerror) {
+        case WSAEINTR:
+        case WSAEBADF:
+        case WSAEACCES:
+        case WSAEFAULT:
+        case WSAEINVAL:
+        case WSAEMFILE:
+            // Winsock definitions of errno values. See WinSock2.h
+            return winerror - 10000;
+        default:
+            return winerror;
+        }
+    }
+
+    switch (winerror) {
+    case ERROR_FILE_NOT_FOUND:            //    2
+    case ERROR_PATH_NOT_FOUND:            //    3
+    case ERROR_INVALID_DRIVE:             //   15
+    case ERROR_NO_MORE_FILES:             //   18
+    case ERROR_BAD_NETPATH:               //   53
+    case ERROR_BAD_NET_NAME:              //   67
+    case ERROR_BAD_PATHNAME:              //  161
+    case ERROR_FILENAME_EXCED_RANGE:      //  206
+        return ENOENT;
+
+    case ERROR_BAD_ENVIRONMENT:           //   10
+        return E2BIG;
+
+    case ERROR_BAD_FORMAT:                //   11
+    case ERROR_INVALID_STARTING_CODESEG:  //  188
+    case ERROR_INVALID_STACKSEG:          //  189
+    case ERROR_INVALID_MODULETYPE:        //  190
+    case ERROR_INVALID_EXE_SIGNATURE:     //  191
+    case ERROR_EXE_MARKED_INVALID:        //  192
+    case ERROR_BAD_EXE_FORMAT:            //  193
+    case ERROR_ITERATED_DATA_EXCEEDS_64k: //  194
+    case ERROR_INVALID_MINALLOCSIZE:      //  195
+    case ERROR_DYNLINK_FROM_INVALID_RING: //  196
+    case ERROR_IOPL_NOT_ENABLED:          //  197
+    case ERROR_INVALID_SEGDPL:            //  198
+    case ERROR_AUTODATASEG_EXCEEDS_64k:   //  199
+    case ERROR_RING2SEG_MUST_BE_MOVABLE:  //  200
+    case ERROR_RELOC_CHAIN_XEEDS_SEGLIM:  //  201
+    case ERROR_INFLOOP_IN_RELOC_CHAIN:    //  202
+        return ENOEXEC;
+
+    case ERROR_INVALID_HANDLE:            //    6
+    case ERROR_INVALID_TARGET_HANDLE:     //  114
+    case ERROR_DIRECT_ACCESS_HANDLE:      //  130
+        return EBADF;
+
+    case ERROR_WAIT_NO_CHILDREN:          //  128
+    case ERROR_CHILD_NOT_COMPLETE:        //  129
+        return ECHILD;
+
+    case ERROR_NO_PROC_SLOTS:             //   89
+    case ERROR_MAX_THRDS_REACHED:         //  164
+    case ERROR_NESTING_NOT_ALLOWED:       //  215
+        return EAGAIN;
+
+    case ERROR_ARENA_TRASHED:             //    7
+    case ERROR_NOT_ENOUGH_MEMORY:         //    8
+    case ERROR_INVALID_BLOCK:             //    9
+    case ERROR_NOT_ENOUGH_QUOTA:          // 1816
+        return ENOMEM;
+
+    case ERROR_ACCESS_DENIED:             //    5
+    case ERROR_CURRENT_DIRECTORY:         //   16
+    case ERROR_WRITE_PROTECT:             //   19
+    case ERROR_BAD_UNIT:                  //   20
+    case ERROR_NOT_READY:                 //   21
+    case ERROR_BAD_COMMAND:               //   22
+    case ERROR_CRC:                       //   23
+    case ERROR_BAD_LENGTH:                //   24
+    case ERROR_SEEK:                      //   25
+    case ERROR_NOT_DOS_DISK:              //   26
+    case ERROR_SECTOR_NOT_FOUND:          //   27
+    case ERROR_OUT_OF_PAPER:              //   28
+    case ERROR_WRITE_FAULT:               //   29
+    case ERROR_READ_FAULT:                //   30
+    case ERROR_GEN_FAILURE:               //   31
+    case ERROR_SHARING_VIOLATION:         //   32
+    case ERROR_LOCK_VIOLATION:            //   33
+    case ERROR_WRONG_DISK:                //   34
+    case ERROR_SHARING_BUFFER_EXCEEDED:   //   36
+    case ERROR_NETWORK_ACCESS_DENIED:     //   65
+    case ERROR_CANNOT_MAKE:               //   82
+    case ERROR_FAIL_I24:                  //   83
+    case ERROR_DRIVE_LOCKED:              //  108
+    case ERROR_SEEK_ON_DEVICE:            //  132
+    case ERROR_NOT_LOCKED:                //  158
+    case ERROR_LOCK_FAILED:               //  167
+    case 35:                              //   35 (undefined)
+        return EACCES;
+
+    case ERROR_FILE_EXISTS:               //   80
+    case ERROR_ALREADY_EXISTS:            //  183
+        return EEXIST;
+
+    case ERROR_NOT_SAME_DEVICE:           //   17
+        return EXDEV;
+
+    case ERROR_DIRECTORY:                 //  267 (bpo-12802)
+        return ENOTDIR;
+
+    case ERROR_TOO_MANY_OPEN_FILES:       //    4
+        return EMFILE;
+
+    case ERROR_DISK_FULL:                 //  112
+        return ENOSPC;
+
+    case ERROR_BROKEN_PIPE:               //  109
+    case ERROR_NO_DATA:                   //  232 (bpo-13063)
+        return EPIPE;
+
+    case ERROR_DIR_NOT_EMPTY:             //  145
+        return ENOTEMPTY;
+
+    case ERROR_NO_UNICODE_TRANSLATION:    // 1113
+        return EILSEQ;
+
+    case ERROR_INVALID_FUNCTION:          //    1
+    case ERROR_INVALID_ACCESS:            //   12
+    case ERROR_INVALID_DATA:              //   13
+    case ERROR_INVALID_PARAMETER:         //   87
+    case ERROR_NEGATIVE_SEEK:             //  131
+    default:
+        return EINVAL;
     }
 }
diff --git a/PC/generrmap.c b/PC/generrmap.c
deleted file mode 100644
index 953344c0d795..000000000000
--- a/PC/generrmap.c
+++ /dev/null
@@ -1,32 +0,0 @@
-#include <windows.h>
-#include <fcntl.h>
-#include <io.h>
-#include <stdio.h>
-#include <errno.h>
-
-/* Extract the mapping of Win32 error codes to errno */
-
-int main()
-{
-    int i;
-    _setmode(fileno(stdout), O_BINARY);
-    printf("/* Generated file. Do not edit. */\n");
-    printf("int winerror_to_errno(int winerror)\n");
-    printf("{\n    switch(winerror) {\n");
-    for(i=1; i < 65000; i++) {
-        _dosmaperr(i);
-        if (errno == EINVAL) {
-            /* Issue #12802 */
-            if (i == ERROR_DIRECTORY)
-                errno = ENOTDIR;
-            /* Issue #13063 */
-            else if (i == ERROR_NO_DATA)
-                errno = EPIPE;
-            else
-                continue;
-        }
-        printf("        case %d: return %d;\n", i, errno);
-    }
-    printf("        default: return EINVAL;\n");
-    printf("    }\n}\n");
-}


From webhook-mailer at python.org  Mon Sep  9 05:38:09 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Mon, 09 Sep 2019 09:38:09 -0000
Subject: [Python-checkins] [2.7] bpo-34410: Fix a crash in the tee iterator
 when re-enter it. (GH-15625) (GH-15740)
Message-ID: <mailman.302.1568021889.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2fb6921ab296f933caf361a662e6471e143abefc
commit: 2fb6921ab296f933caf361a662e6471e143abefc
branch: 2.7
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T12:38:05+03:00
summary:

[2.7] bpo-34410: Fix a crash in the tee iterator when re-enter it. (GH-15625) (GH-15740)

RuntimeError is now raised in this case.
(cherry picked from commit 526a01467b3277f9fcf7f91e66c23321caa1245d)

files:
A Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
M Doc/library/itertools.rst
M Lib/test/test_itertools.py
M Modules/itertoolsmodule.c

diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst
index 17303dd01cef..18477f129903 100644
--- a/Doc/library/itertools.rst
+++ b/Doc/library/itertools.rst
@@ -659,6 +659,10 @@ loops that truncate the stream.
    used anywhere else; otherwise, the *iterable* could get advanced without
    the tee objects being informed.
 
+   ``tee`` iterators are not threadsafe. A :exc:`RuntimeError` may be
+   raised when using simultaneously iterators returned by the same :func:`tee`
+   call, even if the original *iterable* is threadsafe.
+
    This itertool may require significant auxiliary storage (depending on how
    much temporary data needs to be stored). In general, if one iterator uses
    most or all of the data before another iterator starts, it is faster to use
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
index 04279792065e..2cdcbb21efcd 100644
--- a/Lib/test/test_itertools.py
+++ b/Lib/test/test_itertools.py
@@ -10,6 +10,10 @@
 import copy
 import pickle
 from functools import reduce
+try:
+    import threading
+except ImportError:
+    threading = None
 maxsize = test_support.MAX_Py_ssize_t
 minsize = -maxsize-1
 
@@ -984,6 +988,43 @@ def test_tee_del_backward(self):
             del forward, backward
             raise
 
+    def test_tee_reenter(self):
+        class I:
+            first = True
+            def __iter__(self):
+                return self
+            def next(self):
+                first = self.first
+                self.first = False
+                if first:
+                    return next(b)
+
+        a, b = tee(I())
+        with self.assertRaisesRegexp(RuntimeError, "tee"):
+            next(a)
+
+    @unittest.skipUnless(threading, 'Threading required for this test.')
+    def test_tee_concurrent(self):
+        start = threading.Event()
+        finish = threading.Event()
+        class I:
+            def __iter__(self):
+                return self
+            def next(self):
+                start.set()
+                finish.wait()
+
+        a, b = tee(I())
+        thread = threading.Thread(target=next, args=[a])
+        thread.start()
+        try:
+            start.wait()
+            with self.assertRaisesRegexp(RuntimeError, "tee"):
+                next(b)
+        finally:
+            finish.set()
+            thread.join()
+
     def test_StopIteration(self):
         self.assertRaises(StopIteration, izip().next)
 
diff --git a/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst b/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
new file mode 100644
index 000000000000..64e778ee0913
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
@@ -0,0 +1,2 @@
+Fixed a crash in the :func:`tee` iterator when re-enter it. RuntimeError is
+now raised in this case.
diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c
index 04076fd08709..edd21be337cf 100644
--- a/Modules/itertoolsmodule.c
+++ b/Modules/itertoolsmodule.c
@@ -314,6 +314,7 @@ typedef struct {
     PyObject_HEAD
     PyObject *it;
     int numread;
+    int running;
     PyObject *nextlink;
     PyObject *(values[LINKCELLS]);
 } teedataobject;
@@ -336,6 +337,7 @@ teedataobject_new(PyObject *it)
     if (tdo == NULL)
         return NULL;
 
+    tdo->running = 0;
     tdo->numread = 0;
     tdo->nextlink = NULL;
     Py_INCREF(it);
@@ -364,7 +366,14 @@ teedataobject_getitem(teedataobject *tdo, int i)
     else {
         /* this is the lead iterator, so fetch more data */
         assert(i == tdo->numread);
+        if (tdo->running) {
+            PyErr_SetString(PyExc_RuntimeError,
+                            "cannot re-enter the tee iterator");
+            return NULL;
+        }
+        tdo->running = 1;
         value = PyIter_Next(tdo->it);
+        tdo->running = 0;
         if (value == NULL)
             return NULL;
         tdo->numread++;


From webhook-mailer at python.org  Mon Sep  9 05:50:35 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 09:50:35 -0000
Subject: [Python-checkins] bpo-37445: Include FORMAT_MESSAGE_IGNORE_INSERTS
 in FormatMessageW() calls (GH-14462)
Message-ID: <mailman.303.1568022637.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e103732f5df13a97f610a8b80883895f7a273573
commit: e103732f5df13a97f610a8b80883895f7a273573
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T02:50:30-07:00
summary:

bpo-37445: Include FORMAT_MESSAGE_IGNORE_INSERTS in FormatMessageW() calls (GH-14462)


If FormatMessageW() is passed the FORMAT_MESSAGE_FROM_SYSTEM flag without FORMAT_MESSAGE_IGNORE_INSERTS, it will fail if there are insert sequences in the message definition.
(cherry picked from commit a6563650c835d50f7302971a5b145e94f9d0dc68)

Co-authored-by: Zackery Spytz <zspytz at gmail.com>

files:
A Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
M Modules/_ctypes/callproc.c
M Modules/overlapped.c
M PC/bdist_wininst/install.c

diff --git a/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst b/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
new file mode 100644
index 000000000000..e4805b4e02fb
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
@@ -0,0 +1,2 @@
+Include the ``FORMAT_MESSAGE_IGNORE_INSERTS`` flag in ``FormatMessageW()``
+calls.
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
index 1de86e48d719..d9bdd9881333 100644
--- a/Modules/_ctypes/callproc.c
+++ b/Modules/_ctypes/callproc.c
@@ -242,7 +242,9 @@ static WCHAR *FormatError(DWORD code)
 {
     WCHAR *lpMsgBuf;
     DWORD n;
-    n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+    n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
+                       FORMAT_MESSAGE_FROM_SYSTEM |
+                       FORMAT_MESSAGE_IGNORE_INSERTS,
                        NULL,
                        code,
                        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), /* Default language */
diff --git a/Modules/overlapped.c b/Modules/overlapped.c
index 44a0a5a83463..52ed0bc284bc 100644
--- a/Modules/overlapped.c
+++ b/Modules/overlapped.c
@@ -500,7 +500,8 @@ overlapped_FormatMessage(PyObject *ignore, PyObject *args)
         return NULL;
 
     n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
-                       FORMAT_MESSAGE_FROM_SYSTEM,
+                       FORMAT_MESSAGE_FROM_SYSTEM |
+                       FORMAT_MESSAGE_IGNORE_INSERTS,
                        NULL,
                        code,
                        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
diff --git a/PC/bdist_wininst/install.c b/PC/bdist_wininst/install.c
index 6d01ad5c2d50..72d9837fe4f7 100644
--- a/PC/bdist_wininst/install.c
+++ b/PC/bdist_wininst/install.c
@@ -938,7 +938,8 @@ static BOOL SystemError(int error, char *msg)
         LPVOID lpMsgBuf;
         FormatMessage(
             FORMAT_MESSAGE_ALLOCATE_BUFFER |
-            FORMAT_MESSAGE_FROM_SYSTEM,
+            FORMAT_MESSAGE_FROM_SYSTEM |
+            FORMAT_MESSAGE_IGNORE_INSERTS,
             NULL,
             error,
             MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),


From webhook-mailer at python.org  Mon Sep  9 05:56:07 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 09:56:07 -0000
Subject: [Python-checkins] bpo-37445: Include FORMAT_MESSAGE_IGNORE_INSERTS
 in FormatMessageW() calls (GH-14462)
Message-ID: <mailman.304.1568022968.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1af2c0ec2f63cc6516eb814c3e29d94451a52194
commit: 1af2c0ec2f63cc6516eb814c3e29d94451a52194
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T02:56:03-07:00
summary:

bpo-37445: Include FORMAT_MESSAGE_IGNORE_INSERTS in FormatMessageW() calls (GH-14462)


If FormatMessageW() is passed the FORMAT_MESSAGE_FROM_SYSTEM flag without FORMAT_MESSAGE_IGNORE_INSERTS, it will fail if there are insert sequences in the message definition.
(cherry picked from commit a6563650c835d50f7302971a5b145e94f9d0dc68)

Co-authored-by: Zackery Spytz <zspytz at gmail.com>

files:
A Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
M Modules/_ctypes/callproc.c
M Modules/overlapped.c
M PC/bdist_wininst/install.c

diff --git a/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst b/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
new file mode 100644
index 000000000000..e4805b4e02fb
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
@@ -0,0 +1,2 @@
+Include the ``FORMAT_MESSAGE_IGNORE_INSERTS`` flag in ``FormatMessageW()``
+calls.
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
index ec854c864cc2..5861042398d4 100644
--- a/Modules/_ctypes/callproc.c
+++ b/Modules/_ctypes/callproc.c
@@ -239,7 +239,9 @@ static WCHAR *FormatError(DWORD code)
 {
     WCHAR *lpMsgBuf;
     DWORD n;
-    n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+    n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
+                       FORMAT_MESSAGE_FROM_SYSTEM |
+                       FORMAT_MESSAGE_IGNORE_INSERTS,
                        NULL,
                        code,
                        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), /* Default language */
diff --git a/Modules/overlapped.c b/Modules/overlapped.c
index 7798856bee73..596ee33027ed 100644
--- a/Modules/overlapped.c
+++ b/Modules/overlapped.c
@@ -486,7 +486,8 @@ overlapped_FormatMessage(PyObject *ignore, PyObject *args)
         return NULL;
 
     n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
-                       FORMAT_MESSAGE_FROM_SYSTEM,
+                       FORMAT_MESSAGE_FROM_SYSTEM |
+                       FORMAT_MESSAGE_IGNORE_INSERTS,
                        NULL,
                        code,
                        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
diff --git a/PC/bdist_wininst/install.c b/PC/bdist_wininst/install.c
index 04323ebd1706..7a0e2e8c5dd7 100644
--- a/PC/bdist_wininst/install.c
+++ b/PC/bdist_wininst/install.c
@@ -939,7 +939,8 @@ static BOOL SystemError(int error, char *msg)
         LPVOID lpMsgBuf;
         FormatMessage(
             FORMAT_MESSAGE_ALLOCATE_BUFFER |
-            FORMAT_MESSAGE_FROM_SYSTEM,
+            FORMAT_MESSAGE_FROM_SYSTEM |
+            FORMAT_MESSAGE_IGNORE_INSERTS,
             NULL,
             error,
             MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),


From webhook-mailer at python.org  Mon Sep  9 05:56:41 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Mon, 09 Sep 2019 09:56:41 -0000
Subject: [Python-checkins] [3.8] bpo-15817: gdbinit: Document commands after
 defining them (GH-15021) (#15744)
Message-ID: <mailman.305.1568023003.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/14f7de72b62ec8e73a8a57b25ad8fef230dc6a3c
commit: 14f7de72b62ec8e73a8a57b25ad8fef230dc6a3c
branch: 3.8
author: Zachary Ware <zachary.ware at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T04:56:38-05:00
summary:

[3.8] bpo-15817: gdbinit: Document commands after defining them (GH-15021) (#15744)

The gdb manual[1] says the following for "document":

  The command commandname must already be defined.

[1] https://sourceware.org/gdb/current/onlinedocs/gdb/Define.html

And indeed when trying to use the gdbinit file with gdb 8.3, I get:

  .../cpython/Misc/gdbinit:17: Error in sourced command file:
  Undefined command: "pyo".  Try "help".

Fix this by moving all documentation blocks after the define blocks.

This was introduced in GH-6384.
(cherry picked from commit 1f86fdcfc57270ee569cc58269a4e08afe7608ec)

Authored-by: Florian Bruhin <me at the-compiler.org>

files:
M Misc/gdbinit

diff --git a/Misc/gdbinit b/Misc/gdbinit
index afefe0818e4c..45e79fcf6f46 100644
--- a/Misc/gdbinit
+++ b/Misc/gdbinit
@@ -14,30 +14,27 @@
 # with embedded macros that you may find superior to what is in here.
 # See Tools/gdb/libpython.py and http://bugs.python.org/issue8032.
 
-document pyo
-  Prints a representation of the object to stderr, along with the
-  number of reference counts it currently has and the hex address the
-  object is allocated at.  The argument must be a PyObject*
-end
 define pyo
     # side effect of calling _PyObject_Dump is to dump the object's
     # info - assigning just prevents gdb from printing the
     # NULL return value
     set $_unused_void = _PyObject_Dump($arg0)
 end
-
-document pyg
+document pyo
   Prints a representation of the object to stderr, along with the
   number of reference counts it currently has and the hex address the
-  object is allocated at.  The argument must be a PyGC_Head*
+  object is allocated at.  The argument must be a PyObject*
 end
+
 define pyg
     print _PyGC_Dump($arg0)
 end
-
-document pylocals
-  Print the local variables of the current frame.
+document pyg
+  Prints a representation of the object to stderr, along with the
+  number of reference counts it currently has and the hex address the
+  object is allocated at.  The argument must be a PyGC_Head*
 end
+
 define pylocals
     set $_i = 0
     while $_i < f->f_code->co_nlocals
@@ -50,6 +47,9 @@ define pylocals
         set $_i = $_i + 1
     end
 end
+document pylocals
+  Print the local variables of the current frame.
+end
 
 # A rewrite of the Python interpreter's line number calculator in GDB's
 # command language
@@ -75,13 +75,13 @@ define lineno
     printf "%d", $__li
 end
 
-document pyframev
-  Print the current frame - verbose
-end
 define pyframev
     pyframe
     pylocals
 end
+document pyframev
+  Print the current frame - verbose
+end
 
 define pyframe
     set $__fn = PyUnicode_AsUTF8(f->f_code->co_filename)
@@ -134,9 +134,6 @@ end
 # the interpreter you may will have to change the functions you compare with
 # $pc.
 
-document pystack
-  Print the entire Python call stack
-end
 define pystack
     while $pc < Py_Main || $pc > Py_GetArgcArgv
         if $pc > PyEval_EvalFrameEx && $pc < _PyEval_EvalFrameDefault
@@ -146,10 +143,10 @@ define pystack
     end
     select-frame 0
 end
-
-document pystackv
-  Print the entire Python call stack - verbose mode
+document pystack
+  Print the entire Python call stack
 end
+
 define pystackv
     while $pc < Py_Main || $pc > Py_GetArgcArgv
         if $pc > PyEval_EvalFrameEx && $pc < _PyEval_EvalFrameDefault
@@ -159,10 +156,10 @@ define pystackv
     end
     select-frame 0
 end
-
-document pu
-  Generally useful macro to print a Unicode string
+document pystackv
+  Print the entire Python call stack - verbose mode
 end
+
 def pu
   set $uni = $arg0
   set $i = 0
@@ -174,3 +171,6 @@ def pu
     end
   end
 end
+document pu
+  Generally useful macro to print a Unicode string
+end


From webhook-mailer at python.org  Mon Sep  9 06:04:58 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 10:04:58 -0000
Subject: [Python-checkins] bpo-32587: Make winreg.REG_MULTI_SZ support
 zero-length strings (GH-13239)
Message-ID: <mailman.306.1568023500.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bbf02da42e2368ae6b40015d6e92eaac4120f2dc
commit: bbf02da42e2368ae6b40015d6e92eaac4120f2dc
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T03:04:54-07:00
summary:

bpo-32587: Make winreg.REG_MULTI_SZ support zero-length strings (GH-13239)


* bpo-32587: Make winreg.REG_MULTI_SZ support PendingFileRenameOperations

* Address review comments.
(cherry picked from commit e223ba13d8d871ee58570dfca4e82a591189cc2f)

Co-authored-by: Zackery Spytz <zspytz at gmail.com>

files:
A Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst
M Lib/test/test_winreg.py
M PC/winreg.c

diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py
index 11d054e16cdb..be9313db7855 100644
--- a/Lib/test/test_winreg.py
+++ b/Lib/test/test_winreg.py
@@ -41,6 +41,7 @@
     ("String Val",    "A string value",                        REG_SZ),
     ("StringExpand",  "The path is %path%",                    REG_EXPAND_SZ),
     ("Multi-string",  ["Lots", "of", "string", "values"],      REG_MULTI_SZ),
+    ("Multi-nul",     ["", "", "", ""],                        REG_MULTI_SZ),
     ("Raw Data",      b"binary\x00data",                       REG_BINARY),
     ("Big String",    "x"*(2**14-1),                           REG_SZ),
     ("Big Binary",    b"x"*(2**14),                            REG_BINARY),
diff --git a/Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst b/Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst
new file mode 100644
index 000000000000..41483aa8b74a
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst
@@ -0,0 +1 @@
+Make :data:`winreg.REG_MULTI_SZ` support zero-length strings.
diff --git a/PC/winreg.c b/PC/winreg.c
index 5f18876895c3..0016b320cac0 100644
--- a/PC/winreg.c
+++ b/PC/winreg.c
@@ -517,11 +517,18 @@ fixupMultiSZ(wchar_t **str, wchar_t *data, int len)
     int i;
     wchar_t *Q;
 
-    Q = data + len;
-    for (P = data, i = 0; P < Q && *P != '\0'; P++, i++) {
+    if (len > 0 && data[len - 1] == '\0') {
+        Q = data + len - 1;
+    }
+    else {
+        Q = data + len;
+    }
+
+    for (P = data, i = 0; P < Q; P++, i++) {
         str[i] = P;
-        for (; P < Q && *P != '\0'; P++)
+        for (; P < Q && *P != '\0'; P++) {
             ;
+        }
     }
 }
 
@@ -529,12 +536,20 @@ static int
 countStrings(wchar_t *data, int len)
 {
     int strings;
-    wchar_t *P;
-    wchar_t *Q = data + len;
+    wchar_t *P, *Q;
+
+    if (len > 0 && data[len - 1] == '\0') {
+        Q = data + len - 1;
+    }
+    else {
+        Q = data + len;
+    }
 
-    for (P = data, strings = 0; P < Q && *P != '\0'; P++, strings++)
-        for (; P < Q && *P != '\0'; P++)
+    for (P = data, strings = 0; P < Q; P++, strings++) {
+        for (; P < Q && *P != '\0'; P++) {
             ;
+        }
+    }
     return strings;
 }
 
@@ -749,21 +764,15 @@ Reg2Py(BYTE *retDataBuf, DWORD retDataSize, DWORD typ)
                 }
                 for (index = 0; index < s; index++)
                 {
-                    size_t len = wcslen(str[index]);
-                    if (len > INT_MAX) {
-                        PyErr_SetString(PyExc_OverflowError,
-                            "registry string is too long for a Python string");
-                        Py_DECREF(obData);
-                        PyMem_Free(str);
-                        return NULL;
-                    }
-                    PyObject *uni = PyUnicode_FromWideChar(str[index], len);
+                    size_t slen = wcsnlen(str[index], len);
+                    PyObject *uni = PyUnicode_FromWideChar(str[index], slen);
                     if (uni == NULL) {
                         Py_DECREF(obData);
                         PyMem_Free(str);
                         return NULL;
                     }
                     PyList_SET_ITEM(obData, index, uni);
+                    len -= slen + 1;
                 }
                 PyMem_Free(str);
 


From webhook-mailer at python.org  Mon Sep  9 06:11:05 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 10:11:05 -0000
Subject: [Python-checkins] bpo-32587: Make winreg.REG_MULTI_SZ support
 zero-length strings (GH-13239)
Message-ID: <mailman.307.1568023866.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ebca7eb093f31052ff9f245b306d38941c28a1ad
commit: ebca7eb093f31052ff9f245b306d38941c28a1ad
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T03:11:00-07:00
summary:

bpo-32587: Make winreg.REG_MULTI_SZ support zero-length strings (GH-13239)


* bpo-32587: Make winreg.REG_MULTI_SZ support PendingFileRenameOperations

* Address review comments.
(cherry picked from commit e223ba13d8d871ee58570dfca4e82a591189cc2f)

Co-authored-by: Zackery Spytz <zspytz at gmail.com>

files:
A Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst
M Lib/test/test_winreg.py
M PC/winreg.c

diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py
index 91a2bbc066b1..5c25ec8f7ec6 100644
--- a/Lib/test/test_winreg.py
+++ b/Lib/test/test_winreg.py
@@ -41,6 +41,7 @@
     ("String Val",    "A string value",                        REG_SZ),
     ("StringExpand",  "The path is %path%",                    REG_EXPAND_SZ),
     ("Multi-string",  ["Lots", "of", "string", "values"],      REG_MULTI_SZ),
+    ("Multi-nul",     ["", "", "", ""],                        REG_MULTI_SZ),
     ("Raw Data",      b"binary\x00data",                       REG_BINARY),
     ("Big String",    "x"*(2**14-1),                           REG_SZ),
     ("Big Binary",    b"x"*(2**14),                            REG_BINARY),
diff --git a/Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst b/Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst
new file mode 100644
index 000000000000..41483aa8b74a
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst
@@ -0,0 +1 @@
+Make :data:`winreg.REG_MULTI_SZ` support zero-length strings.
diff --git a/PC/winreg.c b/PC/winreg.c
index d0df7ef0ad47..37bc2c72dcde 100644
--- a/PC/winreg.c
+++ b/PC/winreg.c
@@ -518,11 +518,18 @@ fixupMultiSZ(wchar_t **str, wchar_t *data, int len)
     int i;
     wchar_t *Q;
 
-    Q = data + len;
-    for (P = data, i = 0; P < Q && *P != '\0'; P++, i++) {
+    if (len > 0 && data[len - 1] == '\0') {
+        Q = data + len - 1;
+    }
+    else {
+        Q = data + len;
+    }
+
+    for (P = data, i = 0; P < Q; P++, i++) {
         str[i] = P;
-        for (; P < Q && *P != '\0'; P++)
+        for (; P < Q && *P != '\0'; P++) {
             ;
+        }
     }
 }
 
@@ -530,12 +537,20 @@ static int
 countStrings(wchar_t *data, int len)
 {
     int strings;
-    wchar_t *P;
-    wchar_t *Q = data + len;
+    wchar_t *P, *Q;
+
+    if (len > 0 && data[len - 1] == '\0') {
+        Q = data + len - 1;
+    }
+    else {
+        Q = data + len;
+    }
 
-    for (P = data, strings = 0; P < Q && *P != '\0'; P++, strings++)
-        for (; P < Q && *P != '\0'; P++)
+    for (P = data, strings = 0; P < Q; P++, strings++) {
+        for (; P < Q && *P != '\0'; P++) {
             ;
+        }
+    }
     return strings;
 }
 
@@ -749,21 +764,15 @@ Reg2Py(BYTE *retDataBuf, DWORD retDataSize, DWORD typ)
                 }
                 for (index = 0; index < s; index++)
                 {
-                    size_t len = wcslen(str[index]);
-                    if (len > INT_MAX) {
-                        PyErr_SetString(PyExc_OverflowError,
-                            "registry string is too long for a Python string");
-                        Py_DECREF(obData);
-                        PyMem_Free(str);
-                        return NULL;
-                    }
-                    PyObject *uni = PyUnicode_FromWideChar(str[index], len);
+                    size_t slen = wcsnlen(str[index], len);
+                    PyObject *uni = PyUnicode_FromWideChar(str[index], slen);
                     if (uni == NULL) {
                         Py_DECREF(obData);
                         PyMem_Free(str);
                         return NULL;
                     }
                     PyList_SET_ITEM(obData, index, uni);
+                    len -= slen + 1;
                 }
                 PyMem_Free(str);
 


From webhook-mailer at python.org  Mon Sep  9 06:25:08 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 10:25:08 -0000
Subject: [Python-checkins] bpo-37936: Avoid ignoring files that we actually
 do track. (GH-15451)
Message-ID: <mailman.308.1568024709.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c837ad408e85eed9d20ba8331751df15e14f6aef
commit: c837ad408e85eed9d20ba8331751df15e14f6aef
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T03:25:04-07:00
summary:

bpo-37936: Avoid ignoring files that we actually do track. (GH-15451)


There were about 14 files that are actually in the repo but that are
covered by the rules in .gitignore.

Git itself takes no notice of what .gitignore says about files that
it's already tracking... but the discrepancy can be confusing to a
human that adds a new file unexpectedly covered by these rules, as
well as to non-Git software that looks at .gitignore but doesn't
implement this wrinkle in its semantics.  (E.g., `rg`.)

Several of these are from rules that apply more broadly than
intended: for example, `Makefile` applies to `Doc/Makefile` and
`Tools/freeze/test/Makefile`, whereas `/Makefile` means only the
`Makefile` at the repo's root.

And the `Modules/Setup` rule simply wasn't updated after 961d54c5c.

https://bugs.python.org/issue37936
(cherry picked from commit 5e5e9515029f70836003a8cfb30433166fcc8db7)

Co-authored-by: Greg Price <gnprice at gmail.com>

files:
A Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst
M .gitignore

diff --git a/.gitignore b/.gitignore
index 9c0c2ef07a1d..648f07e765b1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,12 +27,12 @@ Include/pydtrace_probes.h
 Lib/distutils/command/*.pdb
 Lib/lib2to3/*.pickle
 Lib/test/data/*
-Makefile
+!Lib/test/data/README
+/Makefile
 Makefile.pre
 Misc/python.pc
 Misc/python-embed.pc
 Misc/python-config.sh
-Modules/Setup
 Modules/Setup.config
 Modules/Setup.local
 Modules/config.c
@@ -78,6 +78,7 @@ config.log
 config.status
 config.status.lineno
 core
+!Tools/msi/core/
 db_home
 .hg/
 .idea/
@@ -88,7 +89,7 @@ libpython*.dylib
 libpython*.dll
 platform
 pybuilddir.txt
-pyconfig.h
+/pyconfig.h
 python-config
 python-config.py
 python.bat
diff --git a/Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst b/Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst
new file mode 100644
index 000000000000..4c6486103881
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst
@@ -0,0 +1,2 @@
+The :file:`.gitignore` file no longer applies to any files that are in fact
+tracked in the Git repository.  Patch by Greg Price.


From webhook-mailer at python.org  Mon Sep  9 06:36:08 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 10:36:08 -0000
Subject: [Python-checkins] bpo-11953: Extend table of Windows WSA* error
 codes (GH-15004)
Message-ID: <mailman.309.1568025369.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/eb02196bd95ea12fcccff3960f36601596811570
commit: eb02196bd95ea12fcccff3960f36601596811570
branch: 3.8
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-09T03:36:04-07:00
summary:

bpo-11953: Extend table of Windows WSA* error codes (GH-15004)

files:
A Misc/NEWS.d/next/Library/2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst
M Lib/socket.py

diff --git a/Lib/socket.py b/Lib/socket.py
index 0dd8ec70e168..af2ed0e76a49 100644
--- a/Lib/socket.py
+++ b/Lib/socket.py
@@ -109,21 +109,101 @@ def _intenum_converter(value, enum_klass):
 # WSA error codes
 if sys.platform.lower().startswith("win"):
     errorTab = {}
+    errorTab[6] = "Specified event object handle is invalid."
+    errorTab[8] = "Insufficient memory available."
+    errorTab[87] = "One or more parameters are invalid."
+    errorTab[995] = "Overlapped operation aborted."
+    errorTab[996] = "Overlapped I/O event object not in signaled state."
+    errorTab[997] = "Overlapped operation will complete later."
     errorTab[10004] = "The operation was interrupted."
     errorTab[10009] = "A bad file handle was passed."
     errorTab[10013] = "Permission denied."
-    errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT
+    errorTab[10014] = "A fault occurred on the network??"  # WSAEFAULT
     errorTab[10022] = "An invalid operation was attempted."
+    errorTab[10024] = "Too many open files."
     errorTab[10035] = "The socket operation would block"
     errorTab[10036] = "A blocking operation is already in progress."
+    errorTab[10037] = "Operation already in progress."
+    errorTab[10038] = "Socket operation on nonsocket."
+    errorTab[10039] = "Destination address required."
+    errorTab[10040] = "Message too long."
+    errorTab[10041] = "Protocol wrong type for socket."
+    errorTab[10042] = "Bad protocol option."
+    errorTab[10043] = "Protocol not supported."
+    errorTab[10044] = "Socket type not supported."
+    errorTab[10045] = "Operation not supported."
+    errorTab[10046] = "Protocol family not supported."
+    errorTab[10047] = "Address family not supported by protocol family."
     errorTab[10048] = "The network address is in use."
+    errorTab[10049] = "Cannot assign requested address."
+    errorTab[10050] = "Network is down."
+    errorTab[10051] = "Network is unreachable."
+    errorTab[10052] = "Network dropped connection on reset."
+    errorTab[10053] = "Software caused connection abort."
     errorTab[10054] = "The connection has been reset."
+    errorTab[10055] = "No buffer space available."
+    errorTab[10056] = "Socket is already connected."
+    errorTab[10057] = "Socket is not connected."
     errorTab[10058] = "The network has been shut down."
+    errorTab[10059] = "Too many references."
     errorTab[10060] = "The operation timed out."
     errorTab[10061] = "Connection refused."
+    errorTab[10062] = "Cannot translate name."
     errorTab[10063] = "The name is too long."
     errorTab[10064] = "The host is down."
     errorTab[10065] = "The host is unreachable."
+    errorTab[10066] = "Directory not empty."
+    errorTab[10067] = "Too many processes."
+    errorTab[10068] = "User quota exceeded."
+    errorTab[10069] = "Disk quota exceeded."
+    errorTab[10070] = "Stale file handle reference."
+    errorTab[10071] = "Item is remote."
+    errorTab[10091] = "Network subsystem is unavailable."
+    errorTab[10092] = "Winsock.dll version out of range."
+    errorTab[10093] = "Successful WSAStartup not yet performed."
+    errorTab[10101] = "Graceful shutdown in progress."
+    errorTab[10102] = "No more results from WSALookupServiceNext."
+    errorTab[10103] = "Call has been canceled."
+    errorTab[10104] = "Procedure call table is invalid."
+    errorTab[10105] = "Service provider is invalid."
+    errorTab[10106] = "Service provider failed to initialize."
+    errorTab[10107] = "System call failure."
+    errorTab[10108] = "Service not found."
+    errorTab[10109] = "Class type not found."
+    errorTab[10110] = "No more results from WSALookupServiceNext."
+    errorTab[10111] = "Call was canceled."
+    errorTab[10112] = "Database query was refused."
+    errorTab[11001] = "Host not found."
+    errorTab[11002] = "Nonauthoritative host not found."
+    errorTab[11003] = "This is a nonrecoverable error."
+    errorTab[11004] = "Valid name, no data record requested type."
+    errorTab[11005] = "QoS receivers."
+    errorTab[11006] = "QoS senders."
+    errorTab[11007] = "No QoS senders."
+    errorTab[11008] = "QoS no receivers."
+    errorTab[11009] = "QoS request confirmed."
+    errorTab[11010] = "QoS admission error."
+    errorTab[11011] = "QoS policy failure."
+    errorTab[11012] = "QoS bad style."
+    errorTab[11013] = "QoS bad object."
+    errorTab[11014] = "QoS traffic control error."
+    errorTab[11015] = "QoS generic error."
+    errorTab[11016] = "QoS service type error."
+    errorTab[11017] = "QoS flowspec error."
+    errorTab[11018] = "Invalid QoS provider buffer."
+    errorTab[11019] = "Invalid QoS filter style."
+    errorTab[11020] = "Invalid QoS filter style."
+    errorTab[11021] = "Incorrect QoS filter count."
+    errorTab[11022] = "Invalid QoS object length."
+    errorTab[11023] = "Incorrect QoS flow count."
+    errorTab[11024] = "Unrecognized QoS object."
+    errorTab[11025] = "Invalid QoS policy object."
+    errorTab[11026] = "Invalid QoS flow descriptor."
+    errorTab[11027] = "Invalid QoS provider-specific flowspec."
+    errorTab[11028] = "Invalid QoS provider-specific filterspec."
+    errorTab[11029] = "Invalid QoS shape discard mode object."
+    errorTab[11030] = "Invalid QoS shaping rate object."
+    errorTab[11031] = "Reserved policy QoS element type."
     __all__.append("errorTab")
 
 
diff --git a/Misc/NEWS.d/next/Library/2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst b/Misc/NEWS.d/next/Library/2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst
new file mode 100644
index 000000000000..62f3c44c284e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst
@@ -0,0 +1 @@
+Completing WSA* error codes in :mod:`socket`.


From webhook-mailer at python.org  Mon Sep  9 06:37:59 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 10:37:59 -0000
Subject: [Python-checkins] bpo-37705: Improve the implementation of
 winerror_to_errno() (GH-15623)
Message-ID: <mailman.310.1568025480.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/68e401fa0b1a3407bce395ad893535f65107ee6e
commit: 68e401fa0b1a3407bce395ad893535f65107ee6e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T03:37:55-07:00
summary:

bpo-37705: Improve the implementation of winerror_to_errno() (GH-15623)


winerror_to_errno() is no longer automatically generated.
Do not rely on the old _dosmapperr() function.
Add ERROR_NO_UNICODE_TRANSLATION (1113) -> EILSEQ.
(cherry picked from commit 19052a11314e7be7ba003fd6cdbb5400a5d77d96)

Co-authored-by: Zackery Spytz <zspytz at gmail.com>

files:
A Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst
D PC/generrmap.c
M PC/errmap.h

diff --git a/Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst b/Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst
new file mode 100644
index 000000000000..a374c3a2965a
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst
@@ -0,0 +1 @@
+Improve the implementation of ``winerror_to_errno()``.
diff --git a/PC/errmap.h b/PC/errmap.h
index 985f673a4649..a7489ab75c65 100644
--- a/PC/errmap.h
+++ b/PC/errmap.h
@@ -1,80 +1,140 @@
-/* Generated file. Do not edit. */
-int winerror_to_errno(int winerror)
+int
+winerror_to_errno(int winerror)
 {
-    switch(winerror) {
-        case 2: return 2;
-        case 3: return 2;
-        case 4: return 24;
-        case 5: return 13;
-        case 6: return 9;
-        case 7: return 12;
-        case 8: return 12;
-        case 9: return 12;
-        case 10: return 7;
-        case 11: return 8;
-        case 15: return 2;
-        case 16: return 13;
-        case 17: return 18;
-        case 18: return 2;
-        case 19: return 13;
-        case 20: return 13;
-        case 21: return 13;
-        case 22: return 13;
-        case 23: return 13;
-        case 24: return 13;
-        case 25: return 13;
-        case 26: return 13;
-        case 27: return 13;
-        case 28: return 13;
-        case 29: return 13;
-        case 30: return 13;
-        case 31: return 13;
-        case 32: return 13;
-        case 33: return 13;
-        case 34: return 13;
-        case 35: return 13;
-        case 36: return 13;
-        case 53: return 2;
-        case 65: return 13;
-        case 67: return 2;
-        case 80: return 17;
-        case 82: return 13;
-        case 83: return 13;
-        case 89: return 11;
-        case 108: return 13;
-        case 109: return 32;
-        case 112: return 28;
-        case 114: return 9;
-        case 128: return 10;
-        case 129: return 10;
-        case 130: return 9;
-        case 132: return 13;
-        case 145: return 41;
-        case 158: return 13;
-        case 161: return 2;
-        case 164: return 11;
-        case 167: return 13;
-        case 183: return 17;
-        case 188: return 8;
-        case 189: return 8;
-        case 190: return 8;
-        case 191: return 8;
-        case 192: return 8;
-        case 193: return 8;
-        case 194: return 8;
-        case 195: return 8;
-        case 196: return 8;
-        case 197: return 8;
-        case 198: return 8;
-        case 199: return 8;
-        case 200: return 8;
-        case 201: return 8;
-        case 202: return 8;
-        case 206: return 2;
-        case 215: return 11;
-        case 232: return 32;
-        case 267: return 20;
-        case 1816: return 12;
-        default: return EINVAL;
+    // Unwrap FACILITY_WIN32 HRESULT errors.
+    if ((winerror & 0xFFFF0000) == 0x80070000) {
+        winerror &= 0x0000FFFF;
+    }
+
+    // Winsock error codes (10000-11999) are errno values.
+    if (winerror >= 10000 && winerror < 12000) {
+        switch (winerror) {
+        case WSAEINTR:
+        case WSAEBADF:
+        case WSAEACCES:
+        case WSAEFAULT:
+        case WSAEINVAL:
+        case WSAEMFILE:
+            // Winsock definitions of errno values. See WinSock2.h
+            return winerror - 10000;
+        default:
+            return winerror;
+        }
+    }
+
+    switch (winerror) {
+    case ERROR_FILE_NOT_FOUND:            //    2
+    case ERROR_PATH_NOT_FOUND:            //    3
+    case ERROR_INVALID_DRIVE:             //   15
+    case ERROR_NO_MORE_FILES:             //   18
+    case ERROR_BAD_NETPATH:               //   53
+    case ERROR_BAD_NET_NAME:              //   67
+    case ERROR_BAD_PATHNAME:              //  161
+    case ERROR_FILENAME_EXCED_RANGE:      //  206
+        return ENOENT;
+
+    case ERROR_BAD_ENVIRONMENT:           //   10
+        return E2BIG;
+
+    case ERROR_BAD_FORMAT:                //   11
+    case ERROR_INVALID_STARTING_CODESEG:  //  188
+    case ERROR_INVALID_STACKSEG:          //  189
+    case ERROR_INVALID_MODULETYPE:        //  190
+    case ERROR_INVALID_EXE_SIGNATURE:     //  191
+    case ERROR_EXE_MARKED_INVALID:        //  192
+    case ERROR_BAD_EXE_FORMAT:            //  193
+    case ERROR_ITERATED_DATA_EXCEEDS_64k: //  194
+    case ERROR_INVALID_MINALLOCSIZE:      //  195
+    case ERROR_DYNLINK_FROM_INVALID_RING: //  196
+    case ERROR_IOPL_NOT_ENABLED:          //  197
+    case ERROR_INVALID_SEGDPL:            //  198
+    case ERROR_AUTODATASEG_EXCEEDS_64k:   //  199
+    case ERROR_RING2SEG_MUST_BE_MOVABLE:  //  200
+    case ERROR_RELOC_CHAIN_XEEDS_SEGLIM:  //  201
+    case ERROR_INFLOOP_IN_RELOC_CHAIN:    //  202
+        return ENOEXEC;
+
+    case ERROR_INVALID_HANDLE:            //    6
+    case ERROR_INVALID_TARGET_HANDLE:     //  114
+    case ERROR_DIRECT_ACCESS_HANDLE:      //  130
+        return EBADF;
+
+    case ERROR_WAIT_NO_CHILDREN:          //  128
+    case ERROR_CHILD_NOT_COMPLETE:        //  129
+        return ECHILD;
+
+    case ERROR_NO_PROC_SLOTS:             //   89
+    case ERROR_MAX_THRDS_REACHED:         //  164
+    case ERROR_NESTING_NOT_ALLOWED:       //  215
+        return EAGAIN;
+
+    case ERROR_ARENA_TRASHED:             //    7
+    case ERROR_NOT_ENOUGH_MEMORY:         //    8
+    case ERROR_INVALID_BLOCK:             //    9
+    case ERROR_NOT_ENOUGH_QUOTA:          // 1816
+        return ENOMEM;
+
+    case ERROR_ACCESS_DENIED:             //    5
+    case ERROR_CURRENT_DIRECTORY:         //   16
+    case ERROR_WRITE_PROTECT:             //   19
+    case ERROR_BAD_UNIT:                  //   20
+    case ERROR_NOT_READY:                 //   21
+    case ERROR_BAD_COMMAND:               //   22
+    case ERROR_CRC:                       //   23
+    case ERROR_BAD_LENGTH:                //   24
+    case ERROR_SEEK:                      //   25
+    case ERROR_NOT_DOS_DISK:              //   26
+    case ERROR_SECTOR_NOT_FOUND:          //   27
+    case ERROR_OUT_OF_PAPER:              //   28
+    case ERROR_WRITE_FAULT:               //   29
+    case ERROR_READ_FAULT:                //   30
+    case ERROR_GEN_FAILURE:               //   31
+    case ERROR_SHARING_VIOLATION:         //   32
+    case ERROR_LOCK_VIOLATION:            //   33
+    case ERROR_WRONG_DISK:                //   34
+    case ERROR_SHARING_BUFFER_EXCEEDED:   //   36
+    case ERROR_NETWORK_ACCESS_DENIED:     //   65
+    case ERROR_CANNOT_MAKE:               //   82
+    case ERROR_FAIL_I24:                  //   83
+    case ERROR_DRIVE_LOCKED:              //  108
+    case ERROR_SEEK_ON_DEVICE:            //  132
+    case ERROR_NOT_LOCKED:                //  158
+    case ERROR_LOCK_FAILED:               //  167
+    case 35:                              //   35 (undefined)
+        return EACCES;
+
+    case ERROR_FILE_EXISTS:               //   80
+    case ERROR_ALREADY_EXISTS:            //  183
+        return EEXIST;
+
+    case ERROR_NOT_SAME_DEVICE:           //   17
+        return EXDEV;
+
+    case ERROR_DIRECTORY:                 //  267 (bpo-12802)
+        return ENOTDIR;
+
+    case ERROR_TOO_MANY_OPEN_FILES:       //    4
+        return EMFILE;
+
+    case ERROR_DISK_FULL:                 //  112
+        return ENOSPC;
+
+    case ERROR_BROKEN_PIPE:               //  109
+    case ERROR_NO_DATA:                   //  232 (bpo-13063)
+        return EPIPE;
+
+    case ERROR_DIR_NOT_EMPTY:             //  145
+        return ENOTEMPTY;
+
+    case ERROR_NO_UNICODE_TRANSLATION:    // 1113
+        return EILSEQ;
+
+    case ERROR_INVALID_FUNCTION:          //    1
+    case ERROR_INVALID_ACCESS:            //   12
+    case ERROR_INVALID_DATA:              //   13
+    case ERROR_INVALID_PARAMETER:         //   87
+    case ERROR_NEGATIVE_SEEK:             //  131
+    default:
+        return EINVAL;
     }
 }
diff --git a/PC/generrmap.c b/PC/generrmap.c
deleted file mode 100644
index 953344c0d795..000000000000
--- a/PC/generrmap.c
+++ /dev/null
@@ -1,32 +0,0 @@
-#include <windows.h>
-#include <fcntl.h>
-#include <io.h>
-#include <stdio.h>
-#include <errno.h>
-
-/* Extract the mapping of Win32 error codes to errno */
-
-int main()
-{
-    int i;
-    _setmode(fileno(stdout), O_BINARY);
-    printf("/* Generated file. Do not edit. */\n");
-    printf("int winerror_to_errno(int winerror)\n");
-    printf("{\n    switch(winerror) {\n");
-    for(i=1; i < 65000; i++) {
-        _dosmaperr(i);
-        if (errno == EINVAL) {
-            /* Issue #12802 */
-            if (i == ERROR_DIRECTORY)
-                errno = ENOTDIR;
-            /* Issue #13063 */
-            else if (i == ERROR_NO_DATA)
-                errno = EPIPE;
-            else
-                continue;
-        }
-        printf("        case %d: return %d;\n", i, errno);
-    }
-    printf("        default: return EINVAL;\n");
-    printf("    }\n}\n");
-}


From webhook-mailer at python.org  Mon Sep  9 06:39:26 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 10:39:26 -0000
Subject: [Python-checkins] bpo-37705: Improve the implementation of
 winerror_to_errno() (GH-15623)
Message-ID: <mailman.311.1568025567.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c25759187829d5732eea73f52a269c88aba28371
commit: c25759187829d5732eea73f52a269c88aba28371
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T03:39:19-07:00
summary:

bpo-37705: Improve the implementation of winerror_to_errno() (GH-15623)


winerror_to_errno() is no longer automatically generated.
Do not rely on the old _dosmapperr() function.
Add ERROR_NO_UNICODE_TRANSLATION (1113) -> EILSEQ.
(cherry picked from commit 19052a11314e7be7ba003fd6cdbb5400a5d77d96)

Co-authored-by: Zackery Spytz <zspytz at gmail.com>

files:
A Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst
D PC/generrmap.c
M PC/errmap.h

diff --git a/Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst b/Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst
new file mode 100644
index 000000000000..a374c3a2965a
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst
@@ -0,0 +1 @@
+Improve the implementation of ``winerror_to_errno()``.
diff --git a/PC/errmap.h b/PC/errmap.h
index 985f673a4649..a7489ab75c65 100644
--- a/PC/errmap.h
+++ b/PC/errmap.h
@@ -1,80 +1,140 @@
-/* Generated file. Do not edit. */
-int winerror_to_errno(int winerror)
+int
+winerror_to_errno(int winerror)
 {
-    switch(winerror) {
-        case 2: return 2;
-        case 3: return 2;
-        case 4: return 24;
-        case 5: return 13;
-        case 6: return 9;
-        case 7: return 12;
-        case 8: return 12;
-        case 9: return 12;
-        case 10: return 7;
-        case 11: return 8;
-        case 15: return 2;
-        case 16: return 13;
-        case 17: return 18;
-        case 18: return 2;
-        case 19: return 13;
-        case 20: return 13;
-        case 21: return 13;
-        case 22: return 13;
-        case 23: return 13;
-        case 24: return 13;
-        case 25: return 13;
-        case 26: return 13;
-        case 27: return 13;
-        case 28: return 13;
-        case 29: return 13;
-        case 30: return 13;
-        case 31: return 13;
-        case 32: return 13;
-        case 33: return 13;
-        case 34: return 13;
-        case 35: return 13;
-        case 36: return 13;
-        case 53: return 2;
-        case 65: return 13;
-        case 67: return 2;
-        case 80: return 17;
-        case 82: return 13;
-        case 83: return 13;
-        case 89: return 11;
-        case 108: return 13;
-        case 109: return 32;
-        case 112: return 28;
-        case 114: return 9;
-        case 128: return 10;
-        case 129: return 10;
-        case 130: return 9;
-        case 132: return 13;
-        case 145: return 41;
-        case 158: return 13;
-        case 161: return 2;
-        case 164: return 11;
-        case 167: return 13;
-        case 183: return 17;
-        case 188: return 8;
-        case 189: return 8;
-        case 190: return 8;
-        case 191: return 8;
-        case 192: return 8;
-        case 193: return 8;
-        case 194: return 8;
-        case 195: return 8;
-        case 196: return 8;
-        case 197: return 8;
-        case 198: return 8;
-        case 199: return 8;
-        case 200: return 8;
-        case 201: return 8;
-        case 202: return 8;
-        case 206: return 2;
-        case 215: return 11;
-        case 232: return 32;
-        case 267: return 20;
-        case 1816: return 12;
-        default: return EINVAL;
+    // Unwrap FACILITY_WIN32 HRESULT errors.
+    if ((winerror & 0xFFFF0000) == 0x80070000) {
+        winerror &= 0x0000FFFF;
+    }
+
+    // Winsock error codes (10000-11999) are errno values.
+    if (winerror >= 10000 && winerror < 12000) {
+        switch (winerror) {
+        case WSAEINTR:
+        case WSAEBADF:
+        case WSAEACCES:
+        case WSAEFAULT:
+        case WSAEINVAL:
+        case WSAEMFILE:
+            // Winsock definitions of errno values. See WinSock2.h
+            return winerror - 10000;
+        default:
+            return winerror;
+        }
+    }
+
+    switch (winerror) {
+    case ERROR_FILE_NOT_FOUND:            //    2
+    case ERROR_PATH_NOT_FOUND:            //    3
+    case ERROR_INVALID_DRIVE:             //   15
+    case ERROR_NO_MORE_FILES:             //   18
+    case ERROR_BAD_NETPATH:               //   53
+    case ERROR_BAD_NET_NAME:              //   67
+    case ERROR_BAD_PATHNAME:              //  161
+    case ERROR_FILENAME_EXCED_RANGE:      //  206
+        return ENOENT;
+
+    case ERROR_BAD_ENVIRONMENT:           //   10
+        return E2BIG;
+
+    case ERROR_BAD_FORMAT:                //   11
+    case ERROR_INVALID_STARTING_CODESEG:  //  188
+    case ERROR_INVALID_STACKSEG:          //  189
+    case ERROR_INVALID_MODULETYPE:        //  190
+    case ERROR_INVALID_EXE_SIGNATURE:     //  191
+    case ERROR_EXE_MARKED_INVALID:        //  192
+    case ERROR_BAD_EXE_FORMAT:            //  193
+    case ERROR_ITERATED_DATA_EXCEEDS_64k: //  194
+    case ERROR_INVALID_MINALLOCSIZE:      //  195
+    case ERROR_DYNLINK_FROM_INVALID_RING: //  196
+    case ERROR_IOPL_NOT_ENABLED:          //  197
+    case ERROR_INVALID_SEGDPL:            //  198
+    case ERROR_AUTODATASEG_EXCEEDS_64k:   //  199
+    case ERROR_RING2SEG_MUST_BE_MOVABLE:  //  200
+    case ERROR_RELOC_CHAIN_XEEDS_SEGLIM:  //  201
+    case ERROR_INFLOOP_IN_RELOC_CHAIN:    //  202
+        return ENOEXEC;
+
+    case ERROR_INVALID_HANDLE:            //    6
+    case ERROR_INVALID_TARGET_HANDLE:     //  114
+    case ERROR_DIRECT_ACCESS_HANDLE:      //  130
+        return EBADF;
+
+    case ERROR_WAIT_NO_CHILDREN:          //  128
+    case ERROR_CHILD_NOT_COMPLETE:        //  129
+        return ECHILD;
+
+    case ERROR_NO_PROC_SLOTS:             //   89
+    case ERROR_MAX_THRDS_REACHED:         //  164
+    case ERROR_NESTING_NOT_ALLOWED:       //  215
+        return EAGAIN;
+
+    case ERROR_ARENA_TRASHED:             //    7
+    case ERROR_NOT_ENOUGH_MEMORY:         //    8
+    case ERROR_INVALID_BLOCK:             //    9
+    case ERROR_NOT_ENOUGH_QUOTA:          // 1816
+        return ENOMEM;
+
+    case ERROR_ACCESS_DENIED:             //    5
+    case ERROR_CURRENT_DIRECTORY:         //   16
+    case ERROR_WRITE_PROTECT:             //   19
+    case ERROR_BAD_UNIT:                  //   20
+    case ERROR_NOT_READY:                 //   21
+    case ERROR_BAD_COMMAND:               //   22
+    case ERROR_CRC:                       //   23
+    case ERROR_BAD_LENGTH:                //   24
+    case ERROR_SEEK:                      //   25
+    case ERROR_NOT_DOS_DISK:              //   26
+    case ERROR_SECTOR_NOT_FOUND:          //   27
+    case ERROR_OUT_OF_PAPER:              //   28
+    case ERROR_WRITE_FAULT:               //   29
+    case ERROR_READ_FAULT:                //   30
+    case ERROR_GEN_FAILURE:               //   31
+    case ERROR_SHARING_VIOLATION:         //   32
+    case ERROR_LOCK_VIOLATION:            //   33
+    case ERROR_WRONG_DISK:                //   34
+    case ERROR_SHARING_BUFFER_EXCEEDED:   //   36
+    case ERROR_NETWORK_ACCESS_DENIED:     //   65
+    case ERROR_CANNOT_MAKE:               //   82
+    case ERROR_FAIL_I24:                  //   83
+    case ERROR_DRIVE_LOCKED:              //  108
+    case ERROR_SEEK_ON_DEVICE:            //  132
+    case ERROR_NOT_LOCKED:                //  158
+    case ERROR_LOCK_FAILED:               //  167
+    case 35:                              //   35 (undefined)
+        return EACCES;
+
+    case ERROR_FILE_EXISTS:               //   80
+    case ERROR_ALREADY_EXISTS:            //  183
+        return EEXIST;
+
+    case ERROR_NOT_SAME_DEVICE:           //   17
+        return EXDEV;
+
+    case ERROR_DIRECTORY:                 //  267 (bpo-12802)
+        return ENOTDIR;
+
+    case ERROR_TOO_MANY_OPEN_FILES:       //    4
+        return EMFILE;
+
+    case ERROR_DISK_FULL:                 //  112
+        return ENOSPC;
+
+    case ERROR_BROKEN_PIPE:               //  109
+    case ERROR_NO_DATA:                   //  232 (bpo-13063)
+        return EPIPE;
+
+    case ERROR_DIR_NOT_EMPTY:             //  145
+        return ENOTEMPTY;
+
+    case ERROR_NO_UNICODE_TRANSLATION:    // 1113
+        return EILSEQ;
+
+    case ERROR_INVALID_FUNCTION:          //    1
+    case ERROR_INVALID_ACCESS:            //   12
+    case ERROR_INVALID_DATA:              //   13
+    case ERROR_INVALID_PARAMETER:         //   87
+    case ERROR_NEGATIVE_SEEK:             //  131
+    default:
+        return EINVAL;
     }
 }
diff --git a/PC/generrmap.c b/PC/generrmap.c
deleted file mode 100644
index 953344c0d795..000000000000
--- a/PC/generrmap.c
+++ /dev/null
@@ -1,32 +0,0 @@
-#include <windows.h>
-#include <fcntl.h>
-#include <io.h>
-#include <stdio.h>
-#include <errno.h>
-
-/* Extract the mapping of Win32 error codes to errno */
-
-int main()
-{
-    int i;
-    _setmode(fileno(stdout), O_BINARY);
-    printf("/* Generated file. Do not edit. */\n");
-    printf("int winerror_to_errno(int winerror)\n");
-    printf("{\n    switch(winerror) {\n");
-    for(i=1; i < 65000; i++) {
-        _dosmaperr(i);
-        if (errno == EINVAL) {
-            /* Issue #12802 */
-            if (i == ERROR_DIRECTORY)
-                errno = ENOTDIR;
-            /* Issue #13063 */
-            else if (i == ERROR_NO_DATA)
-                errno = EPIPE;
-            else
-                continue;
-        }
-        printf("        case %d: return %d;\n", i, errno);
-    }
-    printf("        default: return EINVAL;\n");
-    printf("    }\n}\n");
-}


From webhook-mailer at python.org  Mon Sep  9 06:46:02 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Mon, 09 Sep 2019 10:46:02 -0000
Subject: [Python-checkins] bpo-36250: ignore ValueError from signal in
 non-main thread (GH-12251)
Message-ID: <mailman.312.1568025963.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8d64bfafdffd9f866bb6ac2e5b4c4bdfcb16aea0
commit: 8d64bfafdffd9f866bb6ac2e5b4c4bdfcb16aea0
branch: master
author: Daniel Hahler <github at thequod.de>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-09T05:45:58-05:00
summary:

bpo-36250: ignore ValueError from signal in non-main thread (GH-12251)

Authored-By: blueyed <github at thequod.de>

files:
A Misc/NEWS.d/next/Library/2019-03-09-16-04-12.bpo-36250.tSK4N1.rst
M Lib/pdb.py
M Lib/test/test_pdb.py

diff --git a/Lib/pdb.py b/Lib/pdb.py
index 69fd8bd6efb0..8c1c96163ed9 100755
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -340,8 +340,12 @@ def preloop(self):
     def interaction(self, frame, traceback):
         # Restore the previous signal handler at the Pdb prompt.
         if Pdb._previous_sigint_handler:
-            signal.signal(signal.SIGINT, Pdb._previous_sigint_handler)
-            Pdb._previous_sigint_handler = None
+            try:
+                signal.signal(signal.SIGINT, Pdb._previous_sigint_handler)
+            except ValueError:  # ValueError: signal only works in main thread
+                pass
+            else:
+                Pdb._previous_sigint_handler = None
         if self.setup(frame, traceback):
             # no interaction desired at this time (happens if .pdbrc contains
             # a command like "continue")
diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py
index 646bdb16e53b..fcb7e4e6072c 100644
--- a/Lib/test/test_pdb.py
+++ b/Lib/test/test_pdb.py
@@ -1333,6 +1333,35 @@ def start_pdb():
         self.assertNotIn('Error', stdout.decode(),
                          "Got an error running test script under PDB")
 
+    def test_issue36250(self):
+
+        with open(support.TESTFN, 'wb') as f:
+            f.write(textwrap.dedent("""
+                import threading
+                import pdb
+
+                evt = threading.Event()
+
+                def start_pdb():
+                    evt.wait()
+                    pdb.Pdb(readrc=False).set_trace()
+
+                t = threading.Thread(target=start_pdb)
+                t.start()
+                pdb.Pdb(readrc=False).set_trace()
+                evt.set()
+                t.join()""").encode('ascii'))
+        cmd = [sys.executable, '-u', support.TESTFN]
+        proc = subprocess.Popen(cmd,
+            stdout=subprocess.PIPE,
+            stdin=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            )
+        self.addCleanup(proc.stdout.close)
+        stdout, stderr = proc.communicate(b'cont\ncont\n')
+        self.assertNotIn('Error', stdout.decode(),
+                         "Got an error running test script under PDB")
+
     def test_issue16180(self):
         # A syntax error in the debuggee.
         script = "def f: pass\n"
diff --git a/Misc/NEWS.d/next/Library/2019-03-09-16-04-12.bpo-36250.tSK4N1.rst b/Misc/NEWS.d/next/Library/2019-03-09-16-04-12.bpo-36250.tSK4N1.rst
new file mode 100644
index 000000000000..8d9fbcbb1cba
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-03-09-16-04-12.bpo-36250.tSK4N1.rst
@@ -0,0 +1,2 @@
+Ignore ``ValueError`` from ``signal`` with ``interaction`` in non-main
+thread.


From webhook-mailer at python.org  Mon Sep  9 06:49:38 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Mon, 09 Sep 2019 10:49:38 -0000
Subject: [Python-checkins] Future-proof .travis.yml for Python 3.10
 (GH-14831)
Message-ID: <mailman.313.1568026179.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bba9207248e2a777086d8525739693e3e9595f15
commit: bba9207248e2a777086d8525739693e3e9595f15
branch: master
author: Anthony Sottile <asottile at umich.edu>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-09T05:49:33-05:00
summary:

Future-proof .travis.yml for Python 3.10 (GH-14831)

files:
M .travis.yml

diff --git a/.travis.yml b/.travis.yml
index cff401f5bcb2..ffc4d82c39d9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,7 +23,7 @@ env:
 branches:
   only:
     - master
-    - /^\d\.\d$/
+    - /^\d\.\d+$/
     - buildbot-custom
 
 matrix:


From webhook-mailer at python.org  Mon Sep  9 06:54:59 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Mon, 09 Sep 2019 10:54:59 -0000
Subject: [Python-checkins] Doc: Fix PDF build (NoUri). (GH-15739)
Message-ID: <mailman.314.1568026500.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/63c98ed2d21d22b46f3517fd7dfd88f0c1521299
commit: 63c98ed2d21d22b46f3517fd7dfd88f0c1521299
branch: master
author: Julien Palard <julien at palard.fr>
committer: GitHub <noreply at github.com>
date: 2019-09-09T12:54:56+02:00
summary:

Doc: Fix PDF build (NoUri). (GH-15739)

files:
M Doc/tools/extensions/pyspecific.py

diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index 8839033b983c..975c87d12383 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -22,6 +22,10 @@
 
 from sphinx import addnodes
 from sphinx.builders import Builder
+try:
+    from sphinx.errors import NoUri
+except ImportError:
+    from sphinx.environment import NoUri
 from sphinx.locale import translators
 from sphinx.util import status_iterator, logging
 from sphinx.util.nodes import split_explicit_title
@@ -569,10 +573,13 @@ def process_audit_events(app, doctree, fromdocname):
         for i, (doc, label) in backlinks:
             if isinstance(label, str):
                 ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True)
-                ref['refuri'] = "{}#{}".format(
-                    app.builder.get_relative_uri(fromdocname, doc),
-                    label,
-                )
+                try:
+                    ref['refuri'] = "{}#{}".format(
+                        app.builder.get_relative_uri(fromdocname, doc),
+                        label,
+                    )
+                except NoUri:
+                    continue
                 node += ref
         row += nodes.entry('', node)
 


From webhook-mailer at python.org  Mon Sep  9 06:55:26 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Mon, 09 Sep 2019 10:55:26 -0000
Subject: [Python-checkins] bpo-37212: Preserve keyword argument order in
 unittest.mock.call and error messages (GH-14310)
Message-ID: <mailman.315.1568026527.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9d607061c9c888913ae2c18543775cf360d55f27
commit: 9d607061c9c888913ae2c18543775cf360d55f27
branch: master
author: Xtreak <tir.karthi at gmail.com>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-09T05:55:22-05:00
summary:

bpo-37212: Preserve keyword argument order in unittest.mock.call and error messages (GH-14310)

files:
A Misc/NEWS.d/next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testmock.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 89312f18c967..c9194fff4386 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -2320,7 +2320,7 @@ def _format_call_signature(name, args, kwargs):
     formatted_args = ''
     args_string = ', '.join([repr(arg) for arg in args])
     kwargs_string = ', '.join([
-        '%s=%r' % (key, value) for key, value in sorted(kwargs.items())
+        '%s=%r' % (key, value) for key, value in kwargs.items()
     ])
     if args_string:
         formatted_args = args_string
diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py
index 265eb1bba56c..581afaaeb815 100644
--- a/Lib/unittest/test/testmock/testmock.py
+++ b/Lib/unittest/test/testmock/testmock.py
@@ -1571,11 +1571,11 @@ def test_assert_called_once_message_not_called(self):
             m.assert_called_once()
         self.assertNotIn("Calls:", str(e.exception))
 
-    #Issue21256 printout of keyword args should be in deterministic order
-    def test_sorted_call_signature(self):
+    #Issue37212 printout of keyword args now preserves the original order
+    def test_ordered_call_signature(self):
         m = Mock()
         m.hello(name='hello', daddy='hero')
-        text = "call(daddy='hero', name='hello')"
+        text = "call(name='hello', daddy='hero')"
         self.assertEqual(repr(m.hello.call_args), text)
 
     #Issue21270 overrides tuple methods for mock.call objects
diff --git a/Misc/NEWS.d/next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst b/Misc/NEWS.d/next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst
new file mode 100644
index 000000000000..520a0229aa9d
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst
@@ -0,0 +1,2 @@
+:func:`unittest.mock.call` now preserves the order of keyword arguments in
+repr output. Patch by Karthikeyan Singaravelan.


From webhook-mailer at python.org  Mon Sep  9 07:25:25 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 11:25:25 -0000
Subject: [Python-checkins] bpo-36250: ignore ValueError from signal in
 non-main thread (GH-12251)
Message-ID: <mailman.316.1568028325.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/87a5a331eab5a99538d60a6dab25bdf299a68e3e
commit: 87a5a331eab5a99538d60a6dab25bdf299a68e3e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T04:25:21-07:00
summary:

bpo-36250: ignore ValueError from signal in non-main thread (GH-12251)


Authored-By: blueyed <github at thequod.de>
(cherry picked from commit 8d64bfafdffd9f866bb6ac2e5b4c4bdfcb16aea0)

Co-authored-by: Daniel Hahler <github at thequod.de>

files:
A Misc/NEWS.d/next/Library/2019-03-09-16-04-12.bpo-36250.tSK4N1.rst
M Lib/pdb.py
M Lib/test/test_pdb.py

diff --git a/Lib/pdb.py b/Lib/pdb.py
index 69fd8bd6efb0..8c1c96163ed9 100755
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -340,8 +340,12 @@ def preloop(self):
     def interaction(self, frame, traceback):
         # Restore the previous signal handler at the Pdb prompt.
         if Pdb._previous_sigint_handler:
-            signal.signal(signal.SIGINT, Pdb._previous_sigint_handler)
-            Pdb._previous_sigint_handler = None
+            try:
+                signal.signal(signal.SIGINT, Pdb._previous_sigint_handler)
+            except ValueError:  # ValueError: signal only works in main thread
+                pass
+            else:
+                Pdb._previous_sigint_handler = None
         if self.setup(frame, traceback):
             # no interaction desired at this time (happens if .pdbrc contains
             # a command like "continue")
diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py
index 16d245a5602a..4c38e919a83b 100644
--- a/Lib/test/test_pdb.py
+++ b/Lib/test/test_pdb.py
@@ -1333,6 +1333,35 @@ def start_pdb():
         self.assertNotIn('Error', stdout.decode(),
                          "Got an error running test script under PDB")
 
+    def test_issue36250(self):
+
+        with open(support.TESTFN, 'wb') as f:
+            f.write(textwrap.dedent("""
+                import threading
+                import pdb
+
+                evt = threading.Event()
+
+                def start_pdb():
+                    evt.wait()
+                    pdb.Pdb(readrc=False).set_trace()
+
+                t = threading.Thread(target=start_pdb)
+                t.start()
+                pdb.Pdb(readrc=False).set_trace()
+                evt.set()
+                t.join()""").encode('ascii'))
+        cmd = [sys.executable, '-u', support.TESTFN]
+        proc = subprocess.Popen(cmd,
+            stdout=subprocess.PIPE,
+            stdin=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            )
+        self.addCleanup(proc.stdout.close)
+        stdout, stderr = proc.communicate(b'cont\ncont\n')
+        self.assertNotIn('Error', stdout.decode(),
+                         "Got an error running test script under PDB")
+
     def test_issue16180(self):
         # A syntax error in the debuggee.
         script = "def f: pass\n"
diff --git a/Misc/NEWS.d/next/Library/2019-03-09-16-04-12.bpo-36250.tSK4N1.rst b/Misc/NEWS.d/next/Library/2019-03-09-16-04-12.bpo-36250.tSK4N1.rst
new file mode 100644
index 000000000000..8d9fbcbb1cba
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-03-09-16-04-12.bpo-36250.tSK4N1.rst
@@ -0,0 +1,2 @@
+Ignore ``ValueError`` from ``signal`` with ``interaction`` in non-main
+thread.


From webhook-mailer at python.org  Mon Sep  9 07:29:15 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 11:29:15 -0000
Subject: [Python-checkins] [3.8] Doc: Fix PDF build (NoUri). (GH-15739)
 (GH-15754)
Message-ID: <mailman.317.1568028557.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/252267925d3e74cfaf5216ecb0839c89c2a1baa8
commit: 252267925d3e74cfaf5216ecb0839c89c2a1baa8
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T04:29:10-07:00
summary:

[3.8] Doc: Fix PDF build (NoUri). (GH-15739) (GH-15754)



(cherry picked from commit 63c98ed2d21d22b46f3517fd7dfd88f0c1521299)


Co-authored-by: Julien Palard <julien at palard.fr>

Automerge-Triggered-By: @JulienPalard

files:
M Doc/tools/extensions/pyspecific.py

diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index 28b8bda31146..f41077b07615 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -22,6 +22,10 @@
 
 from sphinx import addnodes
 from sphinx.builders import Builder
+try:
+    from sphinx.errors import NoUri
+except ImportError:
+    from sphinx.environment import NoUri
 from sphinx.locale import translators
 from sphinx.util import status_iterator, logging
 from sphinx.util.nodes import split_explicit_title
@@ -569,10 +573,13 @@ def process_audit_events(app, doctree, fromdocname):
         for i, (doc, label) in backlinks:
             if isinstance(label, str):
                 ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True)
-                ref['refuri'] = "{}#{}".format(
-                    app.builder.get_relative_uri(fromdocname, doc),
-                    label,
-                )
+                try:
+                    ref['refuri'] = "{}#{}".format(
+                        app.builder.get_relative_uri(fromdocname, doc),
+                        label,
+                    )
+                except NoUri:
+                    continue
                 node += ref
         row += nodes.entry('', node)
 


From webhook-mailer at python.org  Mon Sep  9 07:29:57 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Mon, 09 Sep 2019 11:29:57 -0000
Subject: [Python-checkins] logging.Formatter docs: Add missing `validate`
 parameter, clarify `style` parameter (GH-15222)
Message-ID: <mailman.318.1568028597.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4f0f9f066e72255c88d4ec9e9f8e510903577713
commit: 4f0f9f066e72255c88d4ec9e9f8e510903577713
branch: master
author: David R?thlisberger <david at stb-tester.com>
committer: Vinay Sajip <vinay_sajip at yahoo.co.uk>
date: 2019-09-09T12:29:54+01:00
summary:

logging.Formatter docs: Add missing `validate` parameter, clarify `style` parameter (GH-15222)

files:
M Doc/library/logging.rst

diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst
index cc611fc87086..dec27bff564e 100644
--- a/Doc/library/logging.rst
+++ b/Doc/library/logging.rst
@@ -528,7 +528,7 @@ The useful mapping keys in a :class:`LogRecord` are given in the section on
 :ref:`logrecord-attributes`.
 
 
-.. class:: Formatter(fmt=None, datefmt=None, style='%')
+.. class:: Formatter(fmt=None, datefmt=None, style='%', validate=True)
 
    Returns a new instance of the :class:`Formatter` class.  The instance is
    initialized with a format string for the message as a whole, as well as a
@@ -538,8 +538,11 @@ The useful mapping keys in a :class:`LogRecord` are given in the section on
 
    The *style* parameter can be one of '%', '{' or '$' and determines how
    the format string will be merged with its data: using one of %-formatting,
-   :meth:`str.format` or :class:`string.Template`. See :ref:`formatting-styles`
-   for more information on using {- and $-formatting for log messages.
+   :meth:`str.format` or :class:`string.Template`. This only applies to the
+   format string *fmt* (e.g. ``'%(message)s'`` or ``{message}``), not to the
+   actual log messages passed to ``Logger.debug`` etc; see
+   :ref:`formatting-styles` for more information on using {- and $-formatting
+   for log messages.
 
    .. versionchanged:: 3.2
       The *style* parameter was added.


From webhook-mailer at python.org  Mon Sep  9 07:41:32 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 11:41:32 -0000
Subject: [Python-checkins] bpo-37841: Remove python_uwp dependency on
 msvcp140.dll (GH-15253)
Message-ID: <mailman.319.1568029293.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/58a7f809ac978ed54034b7b48f2ea64000aa4661
commit: 58a7f809ac978ed54034b7b48f2ea64000aa4661
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T04:41:28-07:00
summary:

bpo-37841: Remove python_uwp dependency on msvcp140.dll (GH-15253)

(cherry picked from commit b0dace3e979381426385c551b116d0f1434096ee)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
M PCbuild/python_uwp.vcxproj

diff --git a/PCbuild/python_uwp.vcxproj b/PCbuild/python_uwp.vcxproj
index af187dd4df30..14e138cbed3e 100644
--- a/PCbuild/python_uwp.vcxproj
+++ b/PCbuild/python_uwp.vcxproj
@@ -65,6 +65,15 @@
       <SubSystem>Console</SubSystem>
     </Link>
   </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="$(Configuration) != 'Debug'">
+    <ClCompile>
+      <RuntimeLibrary>Multithreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>ucrt.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <IgnoreSpecificDefaultLibraries>libucrt;%(IgnoreSpecificDefaultLibraries)</IgnoreSpecificDefaultLibraries>
+    </Link>
+  </ItemDefinitionGroup>
   <ItemGroup>
     <None Include="..\PC\pycon.ico" />
   </ItemGroup>


From webhook-mailer at python.org  Mon Sep  9 07:42:47 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 11:42:47 -0000
Subject: [Python-checkins] bpo-37212: Preserve keyword argument order in
 unittest.mock.call and error messages (GH-14310)
Message-ID: <mailman.320.1568029368.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bee8bfe5f440c2dde7f5af189febdbf81b27abd5
commit: bee8bfe5f440c2dde7f5af189febdbf81b27abd5
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T04:42:43-07:00
summary:

bpo-37212: Preserve keyword argument order in unittest.mock.call and error messages (GH-14310)

(cherry picked from commit 9d607061c9c888913ae2c18543775cf360d55f27)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testmock.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 4c76f53f3870..7a4fcf4e3aa9 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -2320,7 +2320,7 @@ def _format_call_signature(name, args, kwargs):
     formatted_args = ''
     args_string = ', '.join([repr(arg) for arg in args])
     kwargs_string = ', '.join([
-        '%s=%r' % (key, value) for key, value in sorted(kwargs.items())
+        '%s=%r' % (key, value) for key, value in kwargs.items()
     ])
     if args_string:
         formatted_args = args_string
diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py
index d3a1e89da81a..413ee6895101 100644
--- a/Lib/unittest/test/testmock/testmock.py
+++ b/Lib/unittest/test/testmock/testmock.py
@@ -1563,11 +1563,11 @@ def test_assert_called_once_message_not_called(self):
             m.assert_called_once()
         self.assertNotIn("Calls:", str(e.exception))
 
-    #Issue21256 printout of keyword args should be in deterministic order
-    def test_sorted_call_signature(self):
+    #Issue37212 printout of keyword args now preserves the original order
+    def test_ordered_call_signature(self):
         m = Mock()
         m.hello(name='hello', daddy='hero')
-        text = "call(daddy='hero', name='hello')"
+        text = "call(name='hello', daddy='hero')"
         self.assertEqual(repr(m.hello.call_args), text)
 
     #Issue21270 overrides tuple methods for mock.call objects
diff --git a/Misc/NEWS.d/next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst b/Misc/NEWS.d/next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst
new file mode 100644
index 000000000000..520a0229aa9d
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst
@@ -0,0 +1,2 @@
+:func:`unittest.mock.call` now preserves the order of keyword arguments in
+repr output. Patch by Karthikeyan Singaravelan.


From webhook-mailer at python.org  Mon Sep  9 08:13:06 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Mon, 09 Sep 2019 12:13:06 -0000
Subject: [Python-checkins] bpo-34652 again: Remove lchmod from the default
 AC_CHECK_FUNCS list. (GH-15758)
Message-ID: <mailman.321.1568031187.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bed04b664729e3e6fcee42daa108936360bac6ea
commit: bed04b664729e3e6fcee42daa108936360bac6ea
branch: master
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-09T05:13:00-07:00
summary:

bpo-34652 again: Remove lchmod from the default AC_CHECK_FUNCS list. (GH-15758)

files:
M aclocal.m4
M configure
M configure.ac

diff --git a/aclocal.m4 b/aclocal.m4
index 3d6b1a375fdc..85f00dd5fac7 100644
--- a/aclocal.m4
+++ b/aclocal.m4
@@ -12,9 +12,9 @@
 # PARTICULAR PURPOSE.
 
 m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])])
-# pkg.m4 - Macros to locate and utilise pkg-config.   -*- Autoconf -*-
-# serial 11 (pkg-config-0.29.1)
-
+dnl pkg.m4 - Macros to locate and utilise pkg-config.   -*- Autoconf -*-
+dnl serial 11 (pkg-config-0.29.1)
+dnl
 dnl Copyright ? 2004 Scott James Remnant <scott at netsplit.com>.
 dnl Copyright ? 2012-2015 Dan Nicholson <dbn.lists at gmail.com>
 dnl
@@ -288,73 +288,5 @@ AS_VAR_COPY([$1], [pkg_cv_][$1])
 AS_VAR_IF([$1], [""], [$5], [$4])dnl
 ])dnl PKG_CHECK_VAR
 
-dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES,
-dnl   [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND],
-dnl   [DESCRIPTION], [DEFAULT])
-dnl ------------------------------------------
-dnl
-dnl Prepare a "--with-" configure option using the lowercase
-dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and
-dnl PKG_CHECK_MODULES in a single macro.
-AC_DEFUN([PKG_WITH_MODULES],
-[
-m4_pushdef([with_arg], m4_tolower([$1]))
-
-m4_pushdef([description],
-           [m4_default([$5], [build with ]with_arg[ support])])
-
-m4_pushdef([def_arg], [m4_default([$6], [auto])])
-m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes])
-m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no])
-
-m4_case(def_arg,
-            [yes],[m4_pushdef([with_without], [--without-]with_arg)],
-            [m4_pushdef([with_without],[--with-]with_arg)])
-
-AC_ARG_WITH(with_arg,
-     AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),,
-    [AS_TR_SH([with_]with_arg)=def_arg])
-
-AS_CASE([$AS_TR_SH([with_]with_arg)],
-            [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)],
-            [auto],[PKG_CHECK_MODULES([$1],[$2],
-                                        [m4_n([def_action_if_found]) $3],
-                                        [m4_n([def_action_if_not_found]) $4])])
-
-m4_popdef([with_arg])
-m4_popdef([description])
-m4_popdef([def_arg])
-
-])dnl PKG_WITH_MODULES
-
-dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES,
-dnl   [DESCRIPTION], [DEFAULT])
-dnl -----------------------------------------------
-dnl
-dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES
-dnl check._[VARIABLE-PREFIX] is exported as make variable.
-AC_DEFUN([PKG_HAVE_WITH_MODULES],
-[
-PKG_WITH_MODULES([$1],[$2],,,[$3],[$4])
-
-AM_CONDITIONAL([HAVE_][$1],
-               [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"])
-])dnl PKG_HAVE_WITH_MODULES
-
-dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES,
-dnl   [DESCRIPTION], [DEFAULT])
-dnl ------------------------------------------------------
-dnl
-dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after
-dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make
-dnl and preprocessor variable.
-AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES],
-[
-PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4])
-
-AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"],
-        [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])])
-])dnl PKG_HAVE_DEFINE_WITH_MODULES
-
 m4_include([m4/ax_c_float_words_bigendian.m4])
 m4_include([m4/ax_check_openssl.m4])
diff --git a/configure b/configure
index bffb849b797b..d4d59be8602a 100755
--- a/configure
+++ b/configure
@@ -786,6 +786,7 @@ infodir
 docdir
 oldincludedir
 includedir
+runstatedir
 localstatedir
 sharedstatedir
 sysconfdir
@@ -899,6 +900,7 @@ datadir='${datarootdir}'
 sysconfdir='${prefix}/etc'
 sharedstatedir='${prefix}/com'
 localstatedir='${prefix}/var'
+runstatedir='${localstatedir}/run'
 includedir='${prefix}/include'
 oldincludedir='/usr/include'
 docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@@ -1151,6 +1153,15 @@ do
   | -silent | --silent | --silen | --sile | --sil)
     silent=yes ;;
 
+  -runstatedir | --runstatedir | --runstatedi | --runstated \
+  | --runstate | --runstat | --runsta | --runst | --runs \
+  | --run | --ru | --r)
+    ac_prev=runstatedir ;;
+  -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
+  | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
+  | --run=* | --ru=* | --r=*)
+    runstatedir=$ac_optarg ;;
+
   -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
     ac_prev=sbindir ;;
   -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@@ -1288,7 +1299,7 @@ fi
 for ac_var in	exec_prefix prefix bindir sbindir libexecdir datarootdir \
 		datadir sysconfdir sharedstatedir localstatedir includedir \
 		oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
-		libdir localedir mandir
+		libdir localedir mandir runstatedir
 do
   eval ac_val=\$$ac_var
   # Remove trailing slashes.
@@ -1441,6 +1452,7 @@ Fine tuning of the installation directories:
   --sysconfdir=DIR        read-only single-machine data [PREFIX/etc]
   --sharedstatedir=DIR    modifiable architecture-independent data [PREFIX/com]
   --localstatedir=DIR     modifiable single-machine data [PREFIX/var]
+  --runstatedir=DIR       modifiable per-process data [LOCALSTATEDIR/run]
   --libdir=DIR            object code libraries [EPREFIX/lib]
   --includedir=DIR        C header files [PREFIX/include]
   --oldincludedir=DIR     C header files for non-gcc [/usr/include]
@@ -11494,7 +11506,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
  getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
  getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \
  if_nameindex \
- initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \
+ initgroups kill killpg lchown lockf linkat lstat lutimes mmap \
  memrchr mbrtowc mkdirat mkfifo \
  madvise mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
  posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \
diff --git a/configure.ac b/configure.ac
index 49ee3f6898cc..43514129dd5c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3544,7 +3544,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
  getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
  getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \
  if_nameindex \
- initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \
+ initgroups kill killpg lchown lockf linkat lstat lutimes mmap \
  memrchr mbrtowc mkdirat mkfifo \
  madvise mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
  posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \


From webhook-mailer at python.org  Mon Sep  9 08:15:10 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 12:15:10 -0000
Subject: [Python-checkins] bpo-37283: Ensure command-line and unattend.xml
 setting override previously detected states in Windows installer (GH-15759)
Message-ID: <mailman.322.1568031311.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3a0ddbcdfcbc0f4372905fabf81e093f1b043e99
commit: 3a0ddbcdfcbc0f4372905fabf81e093f1b043e99
branch: master
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-09T05:15:07-07:00
summary:

bpo-37283: Ensure command-line and unattend.xml setting override previously detected states in Windows installer (GH-15759)

files:
A Misc/NEWS.d/next/Windows/2019-09-09-12-22-23.bpo-37283.8NvOkU.rst
M Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
M Tools/msi/bundle/bundle.wxs

diff --git a/Misc/NEWS.d/next/Windows/2019-09-09-12-22-23.bpo-37283.8NvOkU.rst b/Misc/NEWS.d/next/Windows/2019-09-09-12-22-23.bpo-37283.8NvOkU.rst
new file mode 100644
index 000000000000..973047839301
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-09-12-22-23.bpo-37283.8NvOkU.rst
@@ -0,0 +1,2 @@
+Ensure command-line and unattend.xml setting override previously detected
+states in Windows installer.
diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
index 7cd8fb8e0583..570798de1aaf 100644
--- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
+++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
@@ -727,9 +727,13 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
                 BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr);
             }
 
-            _engine->SetVariableNumeric(L"Include_launcher", 1);
+            LONGLONG includeLauncher;
+            if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))
+                || includeLauncher == -1) {
+                _engine->SetVariableNumeric(L"Include_launcher", 1);
+                _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0);
+            }
             _engine->SetVariableNumeric(L"DetectedOldLauncher", 1);
-            _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0);
         }
         return CheckCanceled() ? IDCANCEL : IDNOACTION;
     }
@@ -796,6 +800,12 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
             }
         }
 
+        LONGLONG includeLauncher;
+        if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))
+            && includeLauncher != -1) {
+            detectedLauncher = FALSE;
+        }
+
         if (detectedLauncher) {
             /* When we detect the current version of the launcher. */
             _engine->SetVariableNumeric(L"Include_launcher", 1);
@@ -819,6 +829,14 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
             _baFunction->OnDetectComplete();
         }
 
+        if (SUCCEEDED(hrStatus)) {
+            LONGLONG includeLauncher;
+            if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))
+                && includeLauncher == -1) {
+                _engine->SetVariableNumeric(L"Include_launcher", 1);
+            }
+        }
+
         if (SUCCEEDED(hrStatus)) {
             hrStatus = EvaluateConditions();
         }
@@ -1451,6 +1469,10 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
         hr = ParseOverridableVariablesFromXml(pixdManifest);
         BalExitOnFailure(hr, "Failed to read overridable variables.");
 
+        if (_command.action == BOOTSTRAPPER_ACTION_MODIFY) {
+            LoadOptionalFeatureStates(_engine);
+        }
+
         hr = ParseVariablesFromUnattendXml();
         ExitOnFailure(hr, "Failed to read unattend.ini file.");
 
@@ -1478,10 +1500,6 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
         hr = UpdateUIStrings(_command.action);
         BalExitOnFailure(hr, "Failed to load UI strings.");
 
-        if (_command.action == BOOTSTRAPPER_ACTION_MODIFY) {
-            LoadOptionalFeatureStates(_engine);
-        }
-
         GetBundleFileVersion();
         // don't fail if we couldn't get the version info; best-effort only
     LExit:
diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs
index f6cff6fc351d..ddd6870f6255 100644
--- a/Tools/msi/bundle/bundle.wxs
+++ b/Tools/msi/bundle/bundle.wxs
@@ -71,11 +71,10 @@
     <Variable Name="Include_tools" Value="1" bal:Overridable="yes" />
     <Variable Name="Include_tcltk" Value="1" bal:Overridable="yes" />
     <Variable Name="Include_pip" Value="1" bal:Overridable="yes" />
+    <Variable Name="Include_launcher" Value="-1" bal:Overridable="yes" />
     <?if "$(var.PyTestExt)"="" ?>
-    <Variable Name="Include_launcher" Value="1" bal:Overridable="yes" />
     <Variable Name="Include_launcherState" Value="enabled" bal:Overridable="yes" />
     <?else ?>
-    <Variable Name="Include_launcher" Value="0" />
     <Variable Name="Include_launcherState" Value="disable" />
     <?endif ?>
     <Variable Name="Include_symbols" Value="0" bal:Overridable="yes" />


From webhook-mailer at python.org  Mon Sep  9 08:17:22 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Mon, 09 Sep 2019 12:17:22 -0000
Subject: [Python-checkins] bpo-37589: Add a few missing dependencies on .h
 files in the Makefile. (GH-15757)
Message-ID: <mailman.323.1568031443.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b4612f5d54aced5bc37f1b85bf50b4cafa2480f0
commit: b4612f5d54aced5bc37f1b85bf50b4cafa2480f0
branch: master
author: T. Wouters <thomas at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-09T05:17:18-07:00
summary:

bpo-37589: Add a few missing dependencies on .h files in the Makefile. (GH-15757)

The missing dependencies prevented incremental builds from working when you touched any
of these files. Based on GH-14758 by @vemakereporter.

files:
M Makefile.pre.in

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 363a4eb9d2cb..dbf95fdcd181 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -305,8 +305,9 @@ POBJS=		\
 PARSER_OBJS=	$(POBJS) Parser/myreadline.o Parser/parsetok.o Parser/tokenizer.o
 
 PARSER_HEADERS= \
-		$(srcdir)/Parser/parser.h \
+		$(srcdir)/Include/grammar.h \
 		$(srcdir)/Include/parsetok.h \
+		$(srcdir)/Parser/parser.h \
 		$(srcdir)/Parser/tokenizer.h
 
 ##########################################################################
@@ -865,7 +866,7 @@ regen-symbol: $(srcdir)/Include/graminit.h
 		$(srcdir)/Include/graminit.h \
 		$(srcdir)/Lib/symbol.py
 
-Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
+Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o Parser/parsetok.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
 
 Python/getplatform.o: $(srcdir)/Python/getplatform.c
 		$(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c
@@ -965,11 +966,11 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/abstract.h \
 		$(srcdir)/Include/asdl.h \
 		$(srcdir)/Include/ast.h \
-		$(srcdir)/Include/bltinmodule.h \
 		$(srcdir)/Include/bitset.h \
+		$(srcdir)/Include/bltinmodule.h \
 		$(srcdir)/Include/boolobject.h \
-		$(srcdir)/Include/bytes_methods.h \
 		$(srcdir)/Include/bytearrayobject.h \
+		$(srcdir)/Include/bytes_methods.h \
 		$(srcdir)/Include/bytesobject.h \
 		$(srcdir)/Include/cellobject.h \
 		$(srcdir)/Include/ceval.h \
@@ -978,6 +979,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/codecs.h \
 		$(srcdir)/Include/compile.h \
 		$(srcdir)/Include/complexobject.h \
+		$(srcdir)/Include/context.h \
 		$(srcdir)/Include/descrobject.h \
 		$(srcdir)/Include/dictobject.h \
 		$(srcdir)/Include/dtoa.h \
@@ -1007,6 +1009,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/node.h \
 		$(srcdir)/Include/object.h \
 		$(srcdir)/Include/objimpl.h \
+		$(srcdir)/Include/odictobject.h \
 		$(srcdir)/Include/opcode.h \
 		$(srcdir)/Include/osdefs.h \
 		$(srcdir)/Include/osmodule.h \
@@ -1021,15 +1024,15 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/pyfpe.h \
 		$(srcdir)/Include/pyhash.h \
 		$(srcdir)/Include/pylifecycle.h \
-		$(srcdir)/Include/pymath.h \
+		$(srcdir)/Include/pymacconfig.h \
 		$(srcdir)/Include/pymacro.h \
+		$(srcdir)/Include/pymath.h \
 		$(srcdir)/Include/pymem.h \
 		$(srcdir)/Include/pyport.h \
 		$(srcdir)/Include/pystate.h \
-		$(srcdir)/Include/context.h \
 		$(srcdir)/Include/pystrcmp.h \
-		$(srcdir)/Include/pystrtod.h \
 		$(srcdir)/Include/pystrhex.h \
+		$(srcdir)/Include/pystrtod.h \
 		$(srcdir)/Include/pythonrun.h \
 		$(srcdir)/Include/pythread.h \
 		$(srcdir)/Include/pytime.h \
@@ -1040,6 +1043,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/structseq.h \
 		$(srcdir)/Include/symtable.h \
 		$(srcdir)/Include/sysmodule.h \
+		$(srcdir)/Include/token.h \
 		$(srcdir)/Include/traceback.h \
 		$(srcdir)/Include/tracemalloc.h \
 		$(srcdir)/Include/tupleobject.h \


From webhook-mailer at python.org  Mon Sep  9 08:33:47 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 12:33:47 -0000
Subject: [Python-checkins] bpo-37702: Fix SSL's certificate-store leak on
 Windows (GH-15632)
Message-ID: <mailman.324.1568032428.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ed70129e15ea028469145111044a4349960a4e6f
commit: ed70129e15ea028469145111044a4349960a4e6f
branch: master
author: neonene <53406459+neonene at users.noreply.github.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-09T05:33:43-07:00
summary:

bpo-37702: Fix SSL's certificate-store leak on Windows (GH-15632)

ssl_collect_certificates function in _ssl.c has a memory leak.
Calling CertOpenStore() and CertAddStoreToCollection(), a store's refcnt gets incremented by 2.
But CertCloseStore() is called only once and the refcnt leaves 1.

files:
A Misc/NEWS.d/next/Windows/2019-07-29-16-49-31.bpo-37702.Lj2f5e.rst
M Modules/_ssl.c

diff --git a/Misc/NEWS.d/next/Windows/2019-07-29-16-49-31.bpo-37702.Lj2f5e.rst b/Misc/NEWS.d/next/Windows/2019-07-29-16-49-31.bpo-37702.Lj2f5e.rst
new file mode 100644
index 000000000000..67d53d4c4627
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-07-29-16-49-31.bpo-37702.Lj2f5e.rst
@@ -0,0 +1,2 @@
+Fix memory leak on Windows in creating an SSLContext object or
+running urllib.request.urlopen('https://...').
\ No newline at end of file
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index 3d63612168b2..d94efa98e951 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -5581,6 +5581,7 @@ ssl_collect_certificates(const char *store_name)
             if (result) {
                 ++storesAdded;
             }
+            CertCloseStore(hSystemStore, 0);  /* flag must be 0 */
         }
     }
     if (storesAdded == 0) {


From webhook-mailer at python.org  Mon Sep  9 08:42:31 2019
From: webhook-mailer at python.org (Pablo Galindo)
Date: Mon, 09 Sep 2019 12:42:31 -0000
Subject: [Python-checkins] Fix typo in math.prod example (GH-15614)
Message-ID: <mailman.325.1568032951.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1a8de82d3a30ecc7ed18a5ad51a0e17417ebfb89
commit: 1a8de82d3a30ecc7ed18a5ad51a0e17417ebfb89
branch: master
author: Ashwin Vishnu <9155111+ashwinvis at users.noreply.github.com>
committer: Pablo Galindo <Pablogsal at gmail.com>
date: 2019-09-09T13:42:27+01:00
summary:

Fix typo in math.prod example (GH-15614)

files:
M Doc/whatsnew/3.8.rst

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 4a1362d943c8..2b4eb63d61f6 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -767,7 +767,7 @@ numbers::
 
     >>> prior = 0.8
     >>> likelihoods = [0.625, 0.84, 0.30]
-    >>> (link: http://math.prod) math.prod(likelihoods, start=prior)
+    >>> math.prod(likelihoods, start=prior)
     0.126
 
 (Contributed by Pablo Galindo in :issue:`35606`)


From webhook-mailer at python.org  Mon Sep  9 09:01:51 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 13:01:51 -0000
Subject: [Python-checkins] bpo-34652 again: Remove lchmod from the default
 AC_CHECK_FUNCS list. (GH-15758)
Message-ID: <mailman.326.1568034112.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/12acb5b9c8c21e486083483bded422d8443c38a7
commit: 12acb5b9c8c21e486083483bded422d8443c38a7
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T06:01:41-07:00
summary:

bpo-34652 again: Remove lchmod from the default AC_CHECK_FUNCS list. (GH-15758)

(cherry picked from commit bed04b664729e3e6fcee42daa108936360bac6ea)

Co-authored-by: Benjamin Peterson <benjamin at python.org>

files:
M aclocal.m4
M configure
M configure.ac

diff --git a/aclocal.m4 b/aclocal.m4
index 3d6b1a375fdc..85f00dd5fac7 100644
--- a/aclocal.m4
+++ b/aclocal.m4
@@ -12,9 +12,9 @@
 # PARTICULAR PURPOSE.
 
 m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])])
-# pkg.m4 - Macros to locate and utilise pkg-config.   -*- Autoconf -*-
-# serial 11 (pkg-config-0.29.1)
-
+dnl pkg.m4 - Macros to locate and utilise pkg-config.   -*- Autoconf -*-
+dnl serial 11 (pkg-config-0.29.1)
+dnl
 dnl Copyright ? 2004 Scott James Remnant <scott at netsplit.com>.
 dnl Copyright ? 2012-2015 Dan Nicholson <dbn.lists at gmail.com>
 dnl
@@ -288,73 +288,5 @@ AS_VAR_COPY([$1], [pkg_cv_][$1])
 AS_VAR_IF([$1], [""], [$5], [$4])dnl
 ])dnl PKG_CHECK_VAR
 
-dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES,
-dnl   [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND],
-dnl   [DESCRIPTION], [DEFAULT])
-dnl ------------------------------------------
-dnl
-dnl Prepare a "--with-" configure option using the lowercase
-dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and
-dnl PKG_CHECK_MODULES in a single macro.
-AC_DEFUN([PKG_WITH_MODULES],
-[
-m4_pushdef([with_arg], m4_tolower([$1]))
-
-m4_pushdef([description],
-           [m4_default([$5], [build with ]with_arg[ support])])
-
-m4_pushdef([def_arg], [m4_default([$6], [auto])])
-m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes])
-m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no])
-
-m4_case(def_arg,
-            [yes],[m4_pushdef([with_without], [--without-]with_arg)],
-            [m4_pushdef([with_without],[--with-]with_arg)])
-
-AC_ARG_WITH(with_arg,
-     AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),,
-    [AS_TR_SH([with_]with_arg)=def_arg])
-
-AS_CASE([$AS_TR_SH([with_]with_arg)],
-            [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)],
-            [auto],[PKG_CHECK_MODULES([$1],[$2],
-                                        [m4_n([def_action_if_found]) $3],
-                                        [m4_n([def_action_if_not_found]) $4])])
-
-m4_popdef([with_arg])
-m4_popdef([description])
-m4_popdef([def_arg])
-
-])dnl PKG_WITH_MODULES
-
-dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES,
-dnl   [DESCRIPTION], [DEFAULT])
-dnl -----------------------------------------------
-dnl
-dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES
-dnl check._[VARIABLE-PREFIX] is exported as make variable.
-AC_DEFUN([PKG_HAVE_WITH_MODULES],
-[
-PKG_WITH_MODULES([$1],[$2],,,[$3],[$4])
-
-AM_CONDITIONAL([HAVE_][$1],
-               [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"])
-])dnl PKG_HAVE_WITH_MODULES
-
-dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES,
-dnl   [DESCRIPTION], [DEFAULT])
-dnl ------------------------------------------------------
-dnl
-dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after
-dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make
-dnl and preprocessor variable.
-AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES],
-[
-PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4])
-
-AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"],
-        [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])])
-])dnl PKG_HAVE_DEFINE_WITH_MODULES
-
 m4_include([m4/ax_c_float_words_bigendian.m4])
 m4_include([m4/ax_check_openssl.m4])
diff --git a/configure b/configure
index 3cd9b8866c72..59466c6123d6 100755
--- a/configure
+++ b/configure
@@ -786,6 +786,7 @@ infodir
 docdir
 oldincludedir
 includedir
+runstatedir
 localstatedir
 sharedstatedir
 sysconfdir
@@ -899,6 +900,7 @@ datadir='${datarootdir}'
 sysconfdir='${prefix}/etc'
 sharedstatedir='${prefix}/com'
 localstatedir='${prefix}/var'
+runstatedir='${localstatedir}/run'
 includedir='${prefix}/include'
 oldincludedir='/usr/include'
 docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@@ -1151,6 +1153,15 @@ do
   | -silent | --silent | --silen | --sile | --sil)
     silent=yes ;;
 
+  -runstatedir | --runstatedir | --runstatedi | --runstated \
+  | --runstate | --runstat | --runsta | --runst | --runs \
+  | --run | --ru | --r)
+    ac_prev=runstatedir ;;
+  -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
+  | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
+  | --run=* | --ru=* | --r=*)
+    runstatedir=$ac_optarg ;;
+
   -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
     ac_prev=sbindir ;;
   -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@@ -1288,7 +1299,7 @@ fi
 for ac_var in	exec_prefix prefix bindir sbindir libexecdir datarootdir \
 		datadir sysconfdir sharedstatedir localstatedir includedir \
 		oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
-		libdir localedir mandir
+		libdir localedir mandir runstatedir
 do
   eval ac_val=\$$ac_var
   # Remove trailing slashes.
@@ -1441,6 +1452,7 @@ Fine tuning of the installation directories:
   --sysconfdir=DIR        read-only single-machine data [PREFIX/etc]
   --sharedstatedir=DIR    modifiable architecture-independent data [PREFIX/com]
   --localstatedir=DIR     modifiable single-machine data [PREFIX/var]
+  --runstatedir=DIR       modifiable per-process data [LOCALSTATEDIR/run]
   --libdir=DIR            object code libraries [EPREFIX/lib]
   --includedir=DIR        C header files [PREFIX/include]
   --oldincludedir=DIR     C header files for non-gcc [/usr/include]
@@ -11494,7 +11506,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
  getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
  getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \
  if_nameindex \
- initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \
+ initgroups kill killpg lchown lockf linkat lstat lutimes mmap \
  memrchr mbrtowc mkdirat mkfifo \
  madvise mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
  posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \
diff --git a/configure.ac b/configure.ac
index 033a93cd3f3b..0689c70e1e99 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3544,7 +3544,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
  getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
  getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \
  if_nameindex \
- initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \
+ initgroups kill killpg lchown lockf linkat lstat lutimes mmap \
  memrchr mbrtowc mkdirat mkfifo \
  madvise mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
  posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \


From webhook-mailer at python.org  Mon Sep  9 09:02:47 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 13:02:47 -0000
Subject: [Python-checkins] bpo-37283: Ensure command-line and unattend.xml
 setting override previously detected states in Windows installer (GH-15759)
Message-ID: <mailman.327.1568034169.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2a4a982cbccf66dd8d29439dbd232c79fe3ec44e
commit: 2a4a982cbccf66dd8d29439dbd232c79fe3ec44e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T06:02:42-07:00
summary:

bpo-37283: Ensure command-line and unattend.xml setting override previously detected states in Windows installer (GH-15759)

(cherry picked from commit 3a0ddbcdfcbc0f4372905fabf81e093f1b043e99)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-09-12-22-23.bpo-37283.8NvOkU.rst
M Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
M Tools/msi/bundle/bundle.wxs

diff --git a/Misc/NEWS.d/next/Windows/2019-09-09-12-22-23.bpo-37283.8NvOkU.rst b/Misc/NEWS.d/next/Windows/2019-09-09-12-22-23.bpo-37283.8NvOkU.rst
new file mode 100644
index 000000000000..973047839301
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-09-12-22-23.bpo-37283.8NvOkU.rst
@@ -0,0 +1,2 @@
+Ensure command-line and unattend.xml setting override previously detected
+states in Windows installer.
diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
index 7cd8fb8e0583..570798de1aaf 100644
--- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
+++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
@@ -727,9 +727,13 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
                 BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr);
             }
 
-            _engine->SetVariableNumeric(L"Include_launcher", 1);
+            LONGLONG includeLauncher;
+            if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))
+                || includeLauncher == -1) {
+                _engine->SetVariableNumeric(L"Include_launcher", 1);
+                _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0);
+            }
             _engine->SetVariableNumeric(L"DetectedOldLauncher", 1);
-            _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0);
         }
         return CheckCanceled() ? IDCANCEL : IDNOACTION;
     }
@@ -796,6 +800,12 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
             }
         }
 
+        LONGLONG includeLauncher;
+        if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))
+            && includeLauncher != -1) {
+            detectedLauncher = FALSE;
+        }
+
         if (detectedLauncher) {
             /* When we detect the current version of the launcher. */
             _engine->SetVariableNumeric(L"Include_launcher", 1);
@@ -819,6 +829,14 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
             _baFunction->OnDetectComplete();
         }
 
+        if (SUCCEEDED(hrStatus)) {
+            LONGLONG includeLauncher;
+            if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))
+                && includeLauncher == -1) {
+                _engine->SetVariableNumeric(L"Include_launcher", 1);
+            }
+        }
+
         if (SUCCEEDED(hrStatus)) {
             hrStatus = EvaluateConditions();
         }
@@ -1451,6 +1469,10 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
         hr = ParseOverridableVariablesFromXml(pixdManifest);
         BalExitOnFailure(hr, "Failed to read overridable variables.");
 
+        if (_command.action == BOOTSTRAPPER_ACTION_MODIFY) {
+            LoadOptionalFeatureStates(_engine);
+        }
+
         hr = ParseVariablesFromUnattendXml();
         ExitOnFailure(hr, "Failed to read unattend.ini file.");
 
@@ -1478,10 +1500,6 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
         hr = UpdateUIStrings(_command.action);
         BalExitOnFailure(hr, "Failed to load UI strings.");
 
-        if (_command.action == BOOTSTRAPPER_ACTION_MODIFY) {
-            LoadOptionalFeatureStates(_engine);
-        }
-
         GetBundleFileVersion();
         // don't fail if we couldn't get the version info; best-effort only
     LExit:
diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs
index f6cff6fc351d..ddd6870f6255 100644
--- a/Tools/msi/bundle/bundle.wxs
+++ b/Tools/msi/bundle/bundle.wxs
@@ -71,11 +71,10 @@
     <Variable Name="Include_tools" Value="1" bal:Overridable="yes" />
     <Variable Name="Include_tcltk" Value="1" bal:Overridable="yes" />
     <Variable Name="Include_pip" Value="1" bal:Overridable="yes" />
+    <Variable Name="Include_launcher" Value="-1" bal:Overridable="yes" />
     <?if "$(var.PyTestExt)"="" ?>
-    <Variable Name="Include_launcher" Value="1" bal:Overridable="yes" />
     <Variable Name="Include_launcherState" Value="enabled" bal:Overridable="yes" />
     <?else ?>
-    <Variable Name="Include_launcher" Value="0" />
     <Variable Name="Include_launcherState" Value="disable" />
     <?endif ?>
     <Variable Name="Include_symbols" Value="0" bal:Overridable="yes" />


From webhook-mailer at python.org  Mon Sep  9 09:08:59 2019
From: webhook-mailer at python.org (Dino Viehland)
Date: Mon, 09 Sep 2019 13:08:59 -0000
Subject: [Python-checkins] bpo-26185: Fix repr() on empty ZipInfo object
 (#13441)
Message-ID: <mailman.328.1568034540.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/992347d7376765fe3f4fc958fb1be193ba21f6c3
commit: 992347d7376765fe3f4fc958fb1be193ba21f6c3
branch: master
author: Micka?l Schoentgen <contact at tiger-222.fr>
committer: Dino Viehland <dinoviehland at gmail.com>
date: 2019-09-09T06:08:54-07:00
summary:

bpo-26185: Fix repr() on empty ZipInfo object (#13441)

* bpo-26185: Fix repr() on empty ZipInfo object

It was failing on AttributeError due to inexistant
but required attributes file_size and compress_size.
They are now initialized to 0 in ZipInfo.__init__().

* Remove useless hasattr() in ZipInfo._open_to_write()

* Completely remove file_size setting in _open_to_write().

files:
A Misc/NEWS.d/next/Library/2019-05-20-14-17-29.bpo-26185.pQW4mI.rst
M Lib/test/test_zipfile.py
M Lib/zipfile.py

diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
index 8e437e5cb2b9..f9ee740eb6fa 100644
--- a/Lib/test/test_zipfile.py
+++ b/Lib/test/test_zipfile.py
@@ -1655,6 +1655,33 @@ def test_create_zipinfo_before_1980(self):
         self.assertRaises(ValueError,
                           zipfile.ZipInfo, 'seventies', (1979, 1, 1, 0, 0, 0))
 
+    def test_create_empty_zipinfo_repr(self):
+        """Before bpo-26185, repr() on empty ZipInfo object was failing."""
+        zi = zipfile.ZipInfo(filename="empty")
+        self.assertEqual(repr(zi), "<ZipInfo filename='empty' file_size=0>")
+
+    def test_create_empty_zipinfo_default_attributes(self):
+        """Ensure all required attributes are set."""
+        zi = zipfile.ZipInfo()
+        self.assertEqual(zi.orig_filename, "NoName")
+        self.assertEqual(zi.filename, "NoName")
+        self.assertEqual(zi.date_time, (1980, 1, 1, 0, 0, 0))
+        self.assertEqual(zi.compress_type, zipfile.ZIP_STORED)
+        self.assertEqual(zi.comment, b"")
+        self.assertEqual(zi.extra, b"")
+        self.assertIn(zi.create_system, (0, 3))
+        self.assertEqual(zi.create_version, zipfile.DEFAULT_VERSION)
+        self.assertEqual(zi.extract_version, zipfile.DEFAULT_VERSION)
+        self.assertEqual(zi.reserved, 0)
+        self.assertEqual(zi.flag_bits, 0)
+        self.assertEqual(zi.volume, 0)
+        self.assertEqual(zi.internal_attr, 0)
+        self.assertEqual(zi.external_attr, 0)
+
+        # Before bpo-26185, both were missing
+        self.assertEqual(zi.file_size, 0)
+        self.assertEqual(zi.compress_size, 0)
+
     def test_zipfile_with_short_extra_field(self):
         """If an extra field in the header is less than 4 bytes, skip it."""
         zipdata = (
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
index dfd090795019..c2c5b6a2a44a 100644
--- a/Lib/zipfile.py
+++ b/Lib/zipfile.py
@@ -376,11 +376,11 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)):
         self.volume = 0                 # Volume number of file header
         self.internal_attr = 0          # Internal attributes
         self.external_attr = 0          # External file attributes
+        self.compress_size = 0          # Size of the compressed file
+        self.file_size = 0              # Size of the uncompressed file
         # Other attributes are set by class ZipFile:
         # header_offset         Byte offset to the file header
         # CRC                   CRC-32 of the uncompressed file
-        # compress_size         Size of the compressed file
-        # file_size             Size of the uncompressed file
 
     def __repr__(self):
         result = ['<%s filename=%r' % (self.__class__.__name__, self.filename)]
@@ -1564,9 +1564,7 @@ def _open_to_write(self, zinfo, force_zip64=False):
                              "another write handle open on it. "
                              "Close the first handle before opening another.")
 
-        # Sizes and CRC are overwritten with correct data after processing the file
-        if not hasattr(zinfo, 'file_size'):
-            zinfo.file_size = 0
+        # Size and CRC are overwritten with correct data after processing the file
         zinfo.compress_size = 0
         zinfo.CRC = 0
 
diff --git a/Misc/NEWS.d/next/Library/2019-05-20-14-17-29.bpo-26185.pQW4mI.rst b/Misc/NEWS.d/next/Library/2019-05-20-14-17-29.bpo-26185.pQW4mI.rst
new file mode 100644
index 000000000000..dc71dae69246
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-20-14-17-29.bpo-26185.pQW4mI.rst
@@ -0,0 +1,2 @@
+Fix :func:`repr` on empty :class:`ZipInfo` object. Patch by Micka?l
+Schoentgen.


From webhook-mailer at python.org  Mon Sep  9 09:10:26 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Mon, 09 Sep 2019 13:10:26 -0000
Subject: [Python-checkins] bpo-37936: Avoid ignoring files that we actually
 do track. (GH-15451) (GH-15748)
Message-ID: <mailman.329.1568034627.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9ae9ac1061c76d637ebe9c1d6c6c802311d6ddba
commit: 9ae9ac1061c76d637ebe9c1d6c6c802311d6ddba
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-09T08:10:19-05:00
summary:

bpo-37936: Avoid ignoring files that we actually do track. (GH-15451) (GH-15748)

There were about 14 files that are actually in the repo but that are
covered by the rules in .gitignore.

Git itself takes no notice of what .gitignore says about files that
it's already tracking... but the discrepancy can be confusing to a
human that adds a new file unexpectedly covered by these rules, as
well as to non-Git software that looks at .gitignore but doesn't
implement this wrinkle in its semantics.  (E.g., `rg`.)

Several of these are from rules that apply more broadly than
intended: for example, `Makefile` applies to `Doc/Makefile` and
`Tools/freeze/test/Makefile`, whereas `/Makefile` means only the
`Makefile` at the repo's root.

https://bugs.python.org/issue37936
(cherry picked from commit 5e5e9515029f70836003a8cfb30433166fcc8db7)

Authored-by: Greg Price <gnprice at gmail.com>

files:
A Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst
M .gitignore

diff --git a/.gitignore b/.gitignore
index cdfe476088a8..6ebf6247f672 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,7 +27,8 @@ Include/pydtrace_probes.h
 Lib/distutils/command/*.pdb
 Lib/lib2to3/*.pickle
 Lib/test/data/*
-Makefile
+!Lib/test/data/README
+/Makefile
 Makefile.pre
 Misc/python.pc
 Misc/python-config.sh
@@ -77,6 +78,7 @@ config.log
 config.status
 config.status.lineno
 core
+!Tools/msi/core/
 db_home
 .hg/
 .idea/
@@ -87,7 +89,7 @@ libpython*.dylib
 libpython*.dll
 platform
 pybuilddir.txt
-pyconfig.h
+/pyconfig.h
 python-config
 python-config.py
 python.bat
diff --git a/Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst b/Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst
new file mode 100644
index 000000000000..4c6486103881
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst
@@ -0,0 +1,2 @@
+The :file:`.gitignore` file no longer applies to any files that are in fact
+tracked in the Git repository.  Patch by Greg Price.


From webhook-mailer at python.org  Mon Sep  9 09:15:31 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 13:15:31 -0000
Subject: [Python-checkins] bpo-37283: Ensure command-line and unattend.xml
 setting override previously detected states in Windows installer (GH-15759)
Message-ID: <mailman.330.1568034932.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d3b8a6bf7c5f7574a3256468c4d0a755ba7a9048
commit: d3b8a6bf7c5f7574a3256468c4d0a755ba7a9048
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T06:15:27-07:00
summary:

bpo-37283: Ensure command-line and unattend.xml setting override previously detected states in Windows installer (GH-15759)

(cherry picked from commit 3a0ddbcdfcbc0f4372905fabf81e093f1b043e99)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-09-12-22-23.bpo-37283.8NvOkU.rst
M Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
M Tools/msi/bundle/bundle.wxs

diff --git a/Misc/NEWS.d/next/Windows/2019-09-09-12-22-23.bpo-37283.8NvOkU.rst b/Misc/NEWS.d/next/Windows/2019-09-09-12-22-23.bpo-37283.8NvOkU.rst
new file mode 100644
index 000000000000..973047839301
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-09-12-22-23.bpo-37283.8NvOkU.rst
@@ -0,0 +1,2 @@
+Ensure command-line and unattend.xml setting override previously detected
+states in Windows installer.
diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
index 2e468b7e57b2..cd4a1f8feb1d 100644
--- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
+++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
@@ -727,9 +727,13 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
                 BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr);
             }
 
-            _engine->SetVariableNumeric(L"Include_launcher", 1);
+            LONGLONG includeLauncher;
+            if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))
+                || includeLauncher == -1) {
+                _engine->SetVariableNumeric(L"Include_launcher", 1);
+                _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0);
+            }
             _engine->SetVariableNumeric(L"DetectedOldLauncher", 1);
-            _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0);
         }
         return CheckCanceled() ? IDCANCEL : IDNOACTION;
     }
@@ -796,6 +800,12 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
             }
         }
 
+        LONGLONG includeLauncher;
+        if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))
+            && includeLauncher != -1) {
+            detectedLauncher = FALSE;
+        }
+
         if (detectedLauncher) {
             /* When we detect the current version of the launcher. */
             _engine->SetVariableNumeric(L"Include_launcher", 1);
@@ -819,6 +829,14 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
             _baFunction->OnDetectComplete();
         }
 
+        if (SUCCEEDED(hrStatus)) {
+            LONGLONG includeLauncher;
+            if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))
+                && includeLauncher == -1) {
+                _engine->SetVariableNumeric(L"Include_launcher", 1);
+            }
+        }
+
         if (SUCCEEDED(hrStatus)) {
             hrStatus = EvaluateConditions();
         }
@@ -1451,6 +1469,10 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
         hr = ParseOverridableVariablesFromXml(pixdManifest);
         BalExitOnFailure(hr, "Failed to read overridable variables.");
 
+        if (_command.action == BOOTSTRAPPER_ACTION_MODIFY) {
+            LoadOptionalFeatureStates(_engine);
+        }
+
         hr = ParseVariablesFromUnattendXml();
         ExitOnFailure(hr, "Failed to read unattend.ini file.");
 
@@ -1478,10 +1500,6 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
         hr = UpdateUIStrings(_command.action);
         BalExitOnFailure(hr, "Failed to load UI strings.");
 
-        if (_command.action == BOOTSTRAPPER_ACTION_MODIFY) {
-            LoadOptionalFeatureStates(_engine);
-        }
-
         GetBundleFileVersion();
         // don't fail if we couldn't get the version info; best-effort only
     LExit:
diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs
index f6cff6fc351d..ddd6870f6255 100644
--- a/Tools/msi/bundle/bundle.wxs
+++ b/Tools/msi/bundle/bundle.wxs
@@ -71,11 +71,10 @@
     <Variable Name="Include_tools" Value="1" bal:Overridable="yes" />
     <Variable Name="Include_tcltk" Value="1" bal:Overridable="yes" />
     <Variable Name="Include_pip" Value="1" bal:Overridable="yes" />
+    <Variable Name="Include_launcher" Value="-1" bal:Overridable="yes" />
     <?if "$(var.PyTestExt)"="" ?>
-    <Variable Name="Include_launcher" Value="1" bal:Overridable="yes" />
     <Variable Name="Include_launcherState" Value="enabled" bal:Overridable="yes" />
     <?else ?>
-    <Variable Name="Include_launcher" Value="0" />
     <Variable Name="Include_launcherState" Value="disable" />
     <?endif ?>
     <Variable Name="Include_symbols" Value="0" bal:Overridable="yes" />


From webhook-mailer at python.org  Mon Sep  9 09:24:19 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Mon, 09 Sep 2019 13:24:19 -0000
Subject: [Python-checkins] bpo-32587: Fixes unsafe downcast in PC/winreg.c
 (GH-15766)
Message-ID: <mailman.331.1568035460.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ef66f31ce21cd759cc0c618c5c42ba6da0a06834
commit: ef66f31ce21cd759cc0c618c5c42ba6da0a06834
branch: master
author: Steve Dower <steve.dower at python.org>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-09T14:24:15+01:00
summary:

bpo-32587: Fixes unsafe downcast in PC/winreg.c (GH-15766)

files:
M PC/winreg.c

diff --git a/PC/winreg.c b/PC/winreg.c
index 37bc2c72dcde..72a7c380beef 100644
--- a/PC/winreg.c
+++ b/PC/winreg.c
@@ -772,7 +772,7 @@ Reg2Py(BYTE *retDataBuf, DWORD retDataSize, DWORD typ)
                         return NULL;
                     }
                     PyList_SET_ITEM(obData, index, uni);
-                    len -= slen + 1;
+                    len -= Py_SAFE_DOWNCAST(slen + 1, size_t, int);
                 }
                 PyMem_Free(str);
 


From webhook-mailer at python.org  Mon Sep  9 09:28:11 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 13:28:11 -0000
Subject: [Python-checkins] bpo-37589: Add a few missing dependencies on .h
 files in the Makefile. (GH-15757)
Message-ID: <mailman.332.1568035692.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/248387f3f1182087da8f432c989ac6f18315b50f
commit: 248387f3f1182087da8f432c989ac6f18315b50f
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T06:28:07-07:00
summary:

bpo-37589: Add a few missing dependencies on .h files in the Makefile. (GH-15757)


The missing dependencies prevented incremental builds from working when you touched any
of these files. Based on GH-14758 by @vemakereporter.
(cherry picked from commit b4612f5d54aced5bc37f1b85bf50b4cafa2480f0)

Co-authored-by: T. Wouters <thomas at python.org>

files:
M Makefile.pre.in

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 6a9f4b52704d..b02f78412f16 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -305,8 +305,9 @@ POBJS=		\
 PARSER_OBJS=	$(POBJS) Parser/myreadline.o Parser/parsetok.o Parser/tokenizer.o
 
 PARSER_HEADERS= \
-		$(srcdir)/Parser/parser.h \
+		$(srcdir)/Include/grammar.h \
 		$(srcdir)/Include/parsetok.h \
+		$(srcdir)/Parser/parser.h \
 		$(srcdir)/Parser/tokenizer.h
 
 ##########################################################################
@@ -865,7 +866,7 @@ regen-symbol: $(srcdir)/Include/graminit.h
 		$(srcdir)/Include/graminit.h \
 		$(srcdir)/Lib/symbol.py
 
-Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
+Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o Parser/parsetok.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
 
 Python/getplatform.o: $(srcdir)/Python/getplatform.c
 		$(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c
@@ -965,11 +966,11 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/abstract.h \
 		$(srcdir)/Include/asdl.h \
 		$(srcdir)/Include/ast.h \
-		$(srcdir)/Include/bltinmodule.h \
 		$(srcdir)/Include/bitset.h \
+		$(srcdir)/Include/bltinmodule.h \
 		$(srcdir)/Include/boolobject.h \
-		$(srcdir)/Include/bytes_methods.h \
 		$(srcdir)/Include/bytearrayobject.h \
+		$(srcdir)/Include/bytes_methods.h \
 		$(srcdir)/Include/bytesobject.h \
 		$(srcdir)/Include/cellobject.h \
 		$(srcdir)/Include/ceval.h \
@@ -978,6 +979,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/codecs.h \
 		$(srcdir)/Include/compile.h \
 		$(srcdir)/Include/complexobject.h \
+		$(srcdir)/Include/context.h \
 		$(srcdir)/Include/descrobject.h \
 		$(srcdir)/Include/dictobject.h \
 		$(srcdir)/Include/dtoa.h \
@@ -1007,6 +1009,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/node.h \
 		$(srcdir)/Include/object.h \
 		$(srcdir)/Include/objimpl.h \
+		$(srcdir)/Include/odictobject.h \
 		$(srcdir)/Include/opcode.h \
 		$(srcdir)/Include/osdefs.h \
 		$(srcdir)/Include/osmodule.h \
@@ -1021,15 +1024,15 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/pyfpe.h \
 		$(srcdir)/Include/pyhash.h \
 		$(srcdir)/Include/pylifecycle.h \
-		$(srcdir)/Include/pymath.h \
+		$(srcdir)/Include/pymacconfig.h \
 		$(srcdir)/Include/pymacro.h \
+		$(srcdir)/Include/pymath.h \
 		$(srcdir)/Include/pymem.h \
 		$(srcdir)/Include/pyport.h \
 		$(srcdir)/Include/pystate.h \
-		$(srcdir)/Include/context.h \
 		$(srcdir)/Include/pystrcmp.h \
-		$(srcdir)/Include/pystrtod.h \
 		$(srcdir)/Include/pystrhex.h \
+		$(srcdir)/Include/pystrtod.h \
 		$(srcdir)/Include/pythonrun.h \
 		$(srcdir)/Include/pythread.h \
 		$(srcdir)/Include/pytime.h \
@@ -1040,6 +1043,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/structseq.h \
 		$(srcdir)/Include/symtable.h \
 		$(srcdir)/Include/sysmodule.h \
+		$(srcdir)/Include/token.h \
 		$(srcdir)/Include/traceback.h \
 		$(srcdir)/Include/tracemalloc.h \
 		$(srcdir)/Include/tupleobject.h \


From webhook-mailer at python.org  Mon Sep  9 09:38:42 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 13:38:42 -0000
Subject: [Python-checkins] [3.7] bpo-37589: Add a few missing dependencies
 on .h files in the Makefile. (GH-15757) (GH-15769)
Message-ID: <mailman.333.1568036323.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/64f7f6efa0b6ccd5f942b0dd7c63978957d1660e
commit: 64f7f6efa0b6ccd5f942b0dd7c63978957d1660e
branch: 3.7
author: T. Wouters <thomas at python.org>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-09T06:38:37-07:00
summary:

[3.7] bpo-37589: Add a few missing dependencies on .h files in the Makefile. (GH-15757) (GH-15769)



The missing dependencies prevented incremental builds from working when you touched any
of these files. Based on GH-14758 by @vemakereporter..
(cherry picked from commit b4612f5d54aced5bc37f1b85bf50b4cafa2480f0)




https://bugs.python.org/issue37589



Automerge-Triggered-By: @gpshead

files:
M Makefile.pre.in

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 1a084d01549f..beaccf589488 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -321,8 +321,9 @@ PGOBJS=		\
 		Parser/pgenmain.o
 
 PARSER_HEADERS= \
-		$(srcdir)/Parser/parser.h \
+		$(srcdir)/Include/grammar.h \
 		$(srcdir)/Include/parsetok.h \
+		$(srcdir)/Parser/parser.h \
 		$(srcdir)/Parser/tokenizer.h
 
 PGENOBJS=	$(POBJS) $(PGOBJS)
@@ -856,7 +857,7 @@ regen-opcode:
 		$(srcdir)/Include/opcode.h.new
 	$(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new
 
-Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
+Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o Parser/parsetok.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
 
 Python/getplatform.o: $(srcdir)/Python/getplatform.c
 		$(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c
@@ -951,11 +952,11 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/accu.h \
 		$(srcdir)/Include/asdl.h \
 		$(srcdir)/Include/ast.h \
-		$(srcdir)/Include/bltinmodule.h \
 		$(srcdir)/Include/bitset.h \
+		$(srcdir)/Include/bltinmodule.h \
 		$(srcdir)/Include/boolobject.h \
-		$(srcdir)/Include/bytes_methods.h \
 		$(srcdir)/Include/bytearrayobject.h \
+		$(srcdir)/Include/bytes_methods.h \
 		$(srcdir)/Include/bytesobject.h \
 		$(srcdir)/Include/cellobject.h \
 		$(srcdir)/Include/ceval.h \
@@ -964,6 +965,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/codecs.h \
 		$(srcdir)/Include/compile.h \
 		$(srcdir)/Include/complexobject.h \
+		$(srcdir)/Include/context.h \
 		$(srcdir)/Include/descrobject.h \
 		$(srcdir)/Include/dictobject.h \
 		$(srcdir)/Include/dtoa.h \
@@ -993,6 +995,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/node.h \
 		$(srcdir)/Include/object.h \
 		$(srcdir)/Include/objimpl.h \
+		$(srcdir)/Include/odictobject.h \
 		$(srcdir)/Include/opcode.h \
 		$(srcdir)/Include/osdefs.h \
 		$(srcdir)/Include/osmodule.h \
@@ -1009,15 +1012,15 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/pyfpe.h \
 		$(srcdir)/Include/pyhash.h \
 		$(srcdir)/Include/pylifecycle.h \
-		$(srcdir)/Include/pymath.h \
+		$(srcdir)/Include/pymacconfig.h \
 		$(srcdir)/Include/pymacro.h \
+		$(srcdir)/Include/pymath.h \
 		$(srcdir)/Include/pymem.h \
 		$(srcdir)/Include/pyport.h \
 		$(srcdir)/Include/pystate.h \
-		$(srcdir)/Include/context.h \
 		$(srcdir)/Include/pystrcmp.h \
-		$(srcdir)/Include/pystrtod.h \
 		$(srcdir)/Include/pystrhex.h \
+		$(srcdir)/Include/pystrtod.h \
 		$(srcdir)/Include/pythonrun.h \
 		$(srcdir)/Include/pythread.h \
 		$(srcdir)/Include/pytime.h \
@@ -1028,6 +1031,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/structseq.h \
 		$(srcdir)/Include/symtable.h \
 		$(srcdir)/Include/sysmodule.h \
+		$(srcdir)/Include/token.h \
 		$(srcdir)/Include/traceback.h \
 		$(srcdir)/Include/tupleobject.h \
 		$(srcdir)/Include/ucnhash.h \


From webhook-mailer at python.org  Mon Sep  9 09:40:48 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 13:40:48 -0000
Subject: [Python-checkins] bpo-37702: Fix SSL's certificate-store leak on
 Windows (GH-15632)
Message-ID: <mailman.334.1568036448.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/472f37ab9a33050d7d50d1ebe33ba324a51c52c2
commit: 472f37ab9a33050d7d50d1ebe33ba324a51c52c2
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T06:40:42-07:00
summary:

bpo-37702: Fix SSL's certificate-store leak on Windows (GH-15632)


ssl_collect_certificates function in _ssl.c has a memory leak.
Calling CertOpenStore() and CertAddStoreToCollection(), a store's refcnt gets incremented by 2.
But CertCloseStore() is called only once and the refcnt leaves 1.
(cherry picked from commit ed70129e15ea028469145111044a4349960a4e6f)

Co-authored-by: neonene <53406459+neonene at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Windows/2019-07-29-16-49-31.bpo-37702.Lj2f5e.rst
M Modules/_ssl.c

diff --git a/Misc/NEWS.d/next/Windows/2019-07-29-16-49-31.bpo-37702.Lj2f5e.rst b/Misc/NEWS.d/next/Windows/2019-07-29-16-49-31.bpo-37702.Lj2f5e.rst
new file mode 100644
index 000000000000..67d53d4c4627
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-07-29-16-49-31.bpo-37702.Lj2f5e.rst
@@ -0,0 +1,2 @@
+Fix memory leak on Windows in creating an SSLContext object or
+running urllib.request.urlopen('https://...').
\ No newline at end of file
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index b079663cc223..91820fe9529d 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -5448,6 +5448,7 @@ ssl_collect_certificates(const char *store_name)
             if (result) {
                 ++storesAdded;
             }
+            CertCloseStore(hSystemStore, 0);  /* flag must be 0 */
         }
     }
     if (storesAdded == 0) {


From webhook-mailer at python.org  Mon Sep  9 09:42:55 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 13:42:55 -0000
Subject: [Python-checkins] Fix typo in math.prod example (GH-15614)
Message-ID: <mailman.335.1568036575.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5e053eb98eb0d65a8e0f00b3641f9907198aace3
commit: 5e053eb98eb0d65a8e0f00b3641f9907198aace3
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T06:42:50-07:00
summary:

Fix typo in math.prod example (GH-15614)

(cherry picked from commit 1a8de82d3a30ecc7ed18a5ad51a0e17417ebfb89)

Co-authored-by: Ashwin Vishnu <9155111+ashwinvis at users.noreply.github.com>

files:
M Doc/whatsnew/3.8.rst

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 4a1362d943c8..2b4eb63d61f6 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -767,7 +767,7 @@ numbers::
 
     >>> prior = 0.8
     >>> likelihoods = [0.625, 0.84, 0.30]
-    >>> (link: http://math.prod) math.prod(likelihoods, start=prior)
+    >>> math.prod(likelihoods, start=prior)
     0.126
 
 (Contributed by Pablo Galindo in :issue:`35606`)


From webhook-mailer at python.org  Mon Sep  9 09:46:30 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 09 Sep 2019 13:46:30 -0000
Subject: [Python-checkins] bpo-38037: Fix reference counters in signal
 module (GH-15753)
Message-ID: <mailman.336.1568036791.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/77643c486fd22d8030e0d82c13012684b4ab6df1
commit: 77643c486fd22d8030e0d82c13012684b4ab6df1
branch: master
author: animalize <animalize at users.noreply.github.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-09T15:46:26+02:00
summary:

bpo-38037: Fix reference counters in signal module (GH-15753)

files:
A Misc/NEWS.d/next/Library/2019-09-09-18-39-23.bpo-38037.B0UgFU.rst
M Modules/signalmodule.c

diff --git a/Misc/NEWS.d/next/Library/2019-09-09-18-39-23.bpo-38037.B0UgFU.rst b/Misc/NEWS.d/next/Library/2019-09-09-18-39-23.bpo-38037.B0UgFU.rst
new file mode 100644
index 000000000000..607673bd9994
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-09-18-39-23.bpo-38037.B0UgFU.rst
@@ -0,0 +1 @@
+Fix reference counters in the :mod:`signal` module.
diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c
index 95569b931d60..1d99f877e162 100644
--- a/Modules/signalmodule.c
+++ b/Modules/signalmodule.c
@@ -1329,7 +1329,7 @@ static struct PyModuleDef signalmodule = {
 PyMODINIT_FUNC
 PyInit__signal(void)
 {
-    PyObject *m, *d, *x;
+    PyObject *m, *d;
     int i;
 
     /* Create the module and add the functions */
@@ -1350,13 +1350,17 @@ PyInit__signal(void)
     /* Add some symbolic constants to the module */
     d = PyModule_GetDict(m);
 
-    x = DefaultHandler = PyLong_FromVoidPtr((void *)SIG_DFL);
-    if (PyModule_AddObject(m, "SIG_DFL", x))
+    DefaultHandler = PyLong_FromVoidPtr((void *)SIG_DFL);
+    if (!DefaultHandler ||
+        PyDict_SetItemString(d, "SIG_DFL", DefaultHandler) < 0) {
         goto finally;
+    }
 
-    x = IgnoreHandler = PyLong_FromVoidPtr((void *)SIG_IGN);
-    if (PyModule_AddObject(m, "SIG_IGN", x))
+    IgnoreHandler = PyLong_FromVoidPtr((void *)SIG_IGN);
+    if (!IgnoreHandler ||
+        PyDict_SetItemString(d, "SIG_IGN", IgnoreHandler) < 0) {
         goto finally;
+    }
 
     if (PyModule_AddIntMacro(m, NSIG))
         goto finally;
@@ -1374,8 +1378,8 @@ PyInit__signal(void)
          goto finally;
 #endif
 
-    x = IntHandler = PyDict_GetItemString(d, "default_int_handler");
-    if (!x)
+    IntHandler = PyDict_GetItemString(d, "default_int_handler");
+    if (!IntHandler)
         goto finally;
     Py_INCREF(IntHandler);
 
@@ -1568,8 +1572,10 @@ PyInit__signal(void)
 #if defined (HAVE_SETITIMER) || defined (HAVE_GETITIMER)
     ItimerError = PyErr_NewException("signal.ItimerError",
             PyExc_OSError, NULL);
-    if (PyModule_AddObject(m, "ItimerError", ItimerError))
+    if (!ItimerError ||
+        PyDict_SetItemString(d, "ItimerError", ItimerError) < 0) {
         goto finally;
+    }
 #endif
 
 #ifdef CTRL_C_EVENT
@@ -1615,6 +1621,9 @@ finisignal(void)
     Py_CLEAR(IntHandler);
     Py_CLEAR(DefaultHandler);
     Py_CLEAR(IgnoreHandler);
+#ifdef HAVE_GETITIMER
+    Py_CLEAR(ItimerError);
+#endif
 }
 
 


From webhook-mailer at python.org  Mon Sep  9 09:48:28 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 13:48:28 -0000
Subject: [Python-checkins] bpo-37702: Fix SSL's certificate-store leak on
 Windows (GH-15632)
Message-ID: <mailman.337.1568036908.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5d695b6b7bcccf5f028cdacd986096de15bc0ca6
commit: 5d695b6b7bcccf5f028cdacd986096de15bc0ca6
branch: 3.8
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-09T06:48:22-07:00
summary:

bpo-37702: Fix SSL's certificate-store leak on Windows (GH-15632)

ssl_collect_certificates function in _ssl.c has a memory leak.
Calling CertOpenStore() and CertAddStoreToCollection(), a store's refcnt gets incremented by 2.
But CertCloseStore() is called only once and the refcnt leaves 1.

files:
A Misc/NEWS.d/next/Windows/2019-07-29-16-49-31.bpo-37702.Lj2f5e.rst
M Modules/_ssl.c

diff --git a/Misc/NEWS.d/next/Windows/2019-07-29-16-49-31.bpo-37702.Lj2f5e.rst b/Misc/NEWS.d/next/Windows/2019-07-29-16-49-31.bpo-37702.Lj2f5e.rst
new file mode 100644
index 000000000000..67d53d4c4627
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-07-29-16-49-31.bpo-37702.Lj2f5e.rst
@@ -0,0 +1,2 @@
+Fix memory leak on Windows in creating an SSLContext object or
+running urllib.request.urlopen('https://...').
\ No newline at end of file
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index 089aa3b24a02..6f91b4882528 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -5581,6 +5581,7 @@ ssl_collect_certificates(const char *store_name)
             if (result) {
                 ++storesAdded;
             }
+            CertCloseStore(hSystemStore, 0);  /* flag must be 0 */
         }
     }
     if (storesAdded == 0) {


From webhook-mailer at python.org  Mon Sep  9 09:50:45 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Mon, 09 Sep 2019 13:50:45 -0000
Subject: [Python-checkins] bpo-37662: Documented
 venv.EnvBuilder.upgrade_dependencies(). (GH-15768)
Message-ID: <mailman.338.1568037046.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/264e034f990240e2aa379d8484b15b9e70c1fad5
commit: 264e034f990240e2aa379d8484b15b9e70c1fad5
branch: master
author: Vinay Sajip <vinay_sajip at yahoo.co.uk>
committer: GitHub <noreply at github.com>
date: 2019-09-09T14:50:38+01:00
summary:

bpo-37662: Documented venv.EnvBuilder.upgrade_dependencies(). (GH-15768)

files:
M Doc/library/venv.rst

diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst
index 62732d224386..c674b02a7065 100644
--- a/Doc/library/venv.rst
+++ b/Doc/library/venv.rst
@@ -191,6 +191,14 @@ creation according to their needs, the :class:`EnvBuilder` class.
         Installs activation scripts appropriate to the platform into the virtual
         environment.
 
+    .. method:: upgrade_dependencies(context)
+
+       Upgrades the core venv dependency packages (currently ``pip`` and
+       ``setuptools``) in the environment. This is done by shelling out to the
+       ``pip`` executable in the environment.
+
+       .. versionadded:: 3.8
+
     .. method:: post_setup(context)
 
         A placeholder method which can be overridden in third party


From webhook-mailer at python.org  Mon Sep  9 10:06:57 2019
From: webhook-mailer at python.org (Michael Foord)
Date: Mon, 09 Sep 2019 14:06:57 -0000
Subject: [Python-checkins] bpo-34596: Fallback to a default reason when
 @unittest.skip is uncalled (#9082)
Message-ID: <mailman.339.1568038019.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d5fd75c53fad7049fc640c9a6162d35f0c5bea03
commit: d5fd75c53fad7049fc640c9a6162d35f0c5bea03
branch: master
author: Naitree Zhu <Naitreey at gmail.com>
committer: Michael Foord <voidspace at users.noreply.github.com>
date: 2019-09-09T16:06:48+02:00
summary:

bpo-34596: Fallback to a default reason when @unittest.skip is uncalled (#9082)

* bpo-34596: Fallback to a default reason when @unittest.skip is uncalled

* Change default reason to empty string

* Fix rst formatting of NEWS entry

files:
A Misc/NEWS.d/next/Tests/2018-09-07-01-18-27.bpo-34596.r2-EGd.rst
M Lib/unittest/case.py
M Lib/unittest/test/test_skipping.py

diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py
index 8afb84513590..bac9789c943f 100644
--- a/Lib/unittest/case.py
+++ b/Lib/unittest/case.py
@@ -10,6 +10,7 @@
 import collections
 import contextlib
 import traceback
+import types
 
 from . import result
 from .util import (strclass, safe_repr, _count_diff_all_purpose,
@@ -122,6 +123,10 @@ def skip_wrapper(*args, **kwargs):
         test_item.__unittest_skip__ = True
         test_item.__unittest_skip_why__ = reason
         return test_item
+    if isinstance(reason, types.FunctionType):
+        test_item = reason
+        reason = ''
+        return decorator(test_item)
     return decorator
 
 def skipIf(condition, reason):
diff --git a/Lib/unittest/test/test_skipping.py b/Lib/unittest/test/test_skipping.py
index 71f7b70e479d..1c178a95f750 100644
--- a/Lib/unittest/test/test_skipping.py
+++ b/Lib/unittest/test/test_skipping.py
@@ -255,6 +255,17 @@ def test_1(self):
         suite.run(result)
         self.assertEqual(result.skipped, [(test, "testing")])
 
+    def test_skip_without_reason(self):
+        class Foo(unittest.TestCase):
+            @unittest.skip
+            def test_1(self):
+                pass
+
+        result = unittest.TestResult()
+        test = Foo("test_1")
+        suite = unittest.TestSuite([test])
+        suite.run(result)
+        self.assertEqual(result.skipped, [(test, "")])
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Tests/2018-09-07-01-18-27.bpo-34596.r2-EGd.rst b/Misc/NEWS.d/next/Tests/2018-09-07-01-18-27.bpo-34596.r2-EGd.rst
new file mode 100644
index 000000000000..156e8aa8945d
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2018-09-07-01-18-27.bpo-34596.r2-EGd.rst
@@ -0,0 +1,2 @@
+Fallback to a default reason when :func:`unittest.skip` is uncalled. Patch by
+Naitree Zhu.


From webhook-mailer at python.org  Mon Sep  9 10:08:27 2019
From: webhook-mailer at python.org (Pablo Galindo)
Date: Mon, 09 Sep 2019 14:08:27 -0000
Subject: [Python-checkins] Fix typo in the algorithm description (GH-15774)
Message-ID: <mailman.340.1568038107.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c638521dbf8862d63151e4b984c9b586a954db4c
commit: c638521dbf8862d63151e4b984c9b586a954db4c
branch: master
author: Pablo Galindo <Pablogsal at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T15:08:23+01:00
summary:

Fix typo in the algorithm description (GH-15774)

files:
M Parser/pgen/automata.py

diff --git a/Parser/pgen/automata.py b/Parser/pgen/automata.py
index 3147d8636ff3..545a7370f7ee 100644
--- a/Parser/pgen/automata.py
+++ b/Parser/pgen/automata.py
@@ -242,7 +242,7 @@ def add_closure(nfa_state, base_nfa_set):
             # Now create new DFAs by visiting all posible transitions between
             # the current DFA state and the new power-set states (each nfa_set)
             # via the different labels. As the nodes are appended to *states* this
-            # is performing a deep-first search traversal over the power-set of
+            # is performing a breadth-first search traversal over the power-set of
             # the states of the original NFA.
             for label, nfa_set in sorted(arcs.items()):
                 for exisisting_state in states:


From webhook-mailer at python.org  Mon Sep  9 10:13:47 2019
From: webhook-mailer at python.org (Ronald Oussoren)
Date: Mon, 09 Sep 2019 14:13:47 -0000
Subject: [Python-checkins] bpo-38053 Update documentation for plistlib
 (GH-15727)
Message-ID: <mailman.341.1568038428.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/24b11b8c95e96cf9c93fb7fc253e6e96506f2d77
commit: 24b11b8c95e96cf9c93fb7fc253e6e96506f2d77
branch: master
author: Jon Janzen <jjjonjanzen at gmail.com>
committer: Ronald Oussoren <ronaldoussoren at mac.com>
date: 2019-09-09T16:13:43+02:00
summary:

bpo-38053 Update documentation for plistlib (GH-15727)

* Update documentation for plistlib

-  Update "Mac OS X" to "Apple" since plists are used more widely than just macOS
-  Re-add the UID class documentation (oops, removed in GH-15615)

files:
A Misc/NEWS.d/next/Documentation/2019-09-07-19-09-01.bpo-38053.lttibE.rst
M Doc/library/plistlib.rst

diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst
index f71aaa2354e9..68320f157771 100644
--- a/Doc/library/plistlib.rst
+++ b/Doc/library/plistlib.rst
@@ -1,8 +1,8 @@
-:mod:`plistlib` --- Generate and parse Mac OS X ``.plist`` files
-================================================================
+:mod:`plistlib` --- Generate and parse Apple ``.plist`` files
+=============================================================
 
 .. module:: plistlib
-   :synopsis: Generate and parse Mac OS X plist files.
+   :synopsis: Generate and parse Apple plist files.
 
 .. moduleauthor:: Jack Jansen
 .. sectionauthor:: Georg Brandl <georg at python.org>
@@ -17,7 +17,8 @@
 --------------
 
 This module provides an interface for reading and writing the "property list"
-files used mainly by Mac OS X and supports both binary and XML plist files.
+files used by Apple, primarily on macOS and iOS. This module supports both binary
+and XML plist files.
 
 The property list (``.plist``) file format is a simple serialization supporting
 basic object types, like dictionaries, lists, numbers and strings.  Usually the
@@ -124,6 +125,19 @@ This module defines the following functions:
    .. versionadded:: 3.4
 
 
+The following classes are available:
+
+.. class:: UID(data)
+
+   Wraps an :class:`int`.  This is used when reading or writing NSKeyedArchiver
+   encoded data, which contains UID (see PList manual).
+
+   It has one attribute, :attr:`data`, which can be used to retrieve the int value
+   of the UID.  :attr:`data` must be in the range `0 <= data <= 2**64`.
+
+   .. versionadded:: 3.8
+
+
 The following constants are available:
 
 .. data:: FMT_XML
diff --git a/Misc/NEWS.d/next/Documentation/2019-09-07-19-09-01.bpo-38053.lttibE.rst b/Misc/NEWS.d/next/Documentation/2019-09-07-19-09-01.bpo-38053.lttibE.rst
new file mode 100644
index 000000000000..bc0a98721c8f
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-09-07-19-09-01.bpo-38053.lttibE.rst
@@ -0,0 +1 @@
+Modernized the plistlib documentation
\ No newline at end of file


From webhook-mailer at python.org  Mon Sep  9 10:15:47 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 14:15:47 -0000
Subject: [Python-checkins] bpo-38059: Using sys.exit() over exit() in
 inspect.py (GH-15666)
Message-ID: <mailman.342.1568038547.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e3c59a75279b0df4e7553d6f0031e202de434e43
commit: e3c59a75279b0df4e7553d6f0031e202de434e43
branch: master
author: Alan Yee <alanyee at users.noreply.github.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-09T07:15:43-07:00
summary:

bpo-38059: Using sys.exit() over exit() in inspect.py (GH-15666)

Constants added by the site module like exit() "should not be used in programs"

files:
A Misc/NEWS.d/next/Library/2019-09-08-11-36-50.bpo-38059.8SA6co.rst
M Lib/inspect.py

diff --git a/Lib/inspect.py b/Lib/inspect.py
index a616f2d49b7d..0a57749ccdd4 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -3118,7 +3118,7 @@ def _main():
                                                     type(exc).__name__,
                                                     exc)
         print(msg, file=sys.stderr)
-        exit(2)
+        sys.exit(2)
 
     if has_attrs:
         parts = attrs.split(".")
@@ -3128,7 +3128,7 @@ def _main():
 
     if module.__name__ in sys.builtin_module_names:
         print("Can't get info for builtin modules.", file=sys.stderr)
-        exit(1)
+        sys.exit(1)
 
     if args.details:
         print('Target: {}'.format(target))
diff --git a/Misc/NEWS.d/next/Library/2019-09-08-11-36-50.bpo-38059.8SA6co.rst b/Misc/NEWS.d/next/Library/2019-09-08-11-36-50.bpo-38059.8SA6co.rst
new file mode 100644
index 000000000000..001952ae1261
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-08-11-36-50.bpo-38059.8SA6co.rst
@@ -0,0 +1 @@
+inspect.py now uses sys.exit() instead of exit()


From webhook-mailer at python.org  Mon Sep  9 10:16:37 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Mon, 09 Sep 2019 14:16:37 -0000
Subject: [Python-checkins] Mark files as executable that are meant as
 scripts. (GH-15354)
Message-ID: <mailman.343.1568038597.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/fa3a38d81faaf96d17b5a7f0248b9923e3a648cc
commit: fa3a38d81faaf96d17b5a7f0248b9923e3a648cc
branch: master
author: Greg Price <gnprice at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-09T07:16:33-07:00
summary:

Mark files as executable that are meant as scripts. (GH-15354)

This is the converse of GH-15353 -- in addition to plenty of
scripts in the tree that are marked with the executable bit
(and so can be directly executed), there are a few that have
a leading `#!` which could let them be executed, but it doesn't
do anything because they don't have the executable bit set.

Here's a command which finds such files and marks them.  The
first line finds files in the tree with a `#!` line *anywhere*;
the next-to-last step checks that the *first* line is actually of
that form.  In between we filter out files that already have the
bit set, and some files that are meant as fragments to be
consumed by one or another kind of preprocessor.

    $ git grep -l '^#!' \
      | grep -vxFf <( \
          git ls-files --stage \
          | perl -lane 'print $F[3] if (!/^100644/)' \
        ) \
      | grep -ve '\.in$' -e '^Doc/includes/' \
      | while read f; do
          head -c2 "$f" | grep -qxF '#!' \
          && chmod a+x "$f"; \
        done

files:
M Lib/pydoc.py
M Lib/turtledemo/__main__.py
M Lib/turtledemo/sorting_animate.py
M Misc/python-wing3.wpr
M Misc/python-wing4.wpr
M Misc/python-wing5.wpr
M PCbuild/fix_encoding.py
M PCbuild/get_external.py
M PCbuild/prepare_ssl.py
M Parser/asdl_c.py
M Tools/scripts/generate_token.py

diff --git a/Lib/pydoc.py b/Lib/pydoc.py
old mode 100644
new mode 100755
diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py
old mode 100644
new mode 100755
diff --git a/Lib/turtledemo/sorting_animate.py b/Lib/turtledemo/sorting_animate.py
old mode 100644
new mode 100755
diff --git a/Misc/python-wing3.wpr b/Misc/python-wing3.wpr
old mode 100644
new mode 100755
diff --git a/Misc/python-wing4.wpr b/Misc/python-wing4.wpr
old mode 100644
new mode 100755
diff --git a/Misc/python-wing5.wpr b/Misc/python-wing5.wpr
old mode 100644
new mode 100755
diff --git a/PCbuild/fix_encoding.py b/PCbuild/fix_encoding.py
old mode 100644
new mode 100755
diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py
old mode 100644
new mode 100755
diff --git a/PCbuild/prepare_ssl.py b/PCbuild/prepare_ssl.py
old mode 100644
new mode 100755
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
old mode 100644
new mode 100755
diff --git a/Tools/scripts/generate_token.py b/Tools/scripts/generate_token.py
old mode 100644
new mode 100755


From webhook-mailer at python.org  Mon Sep  9 10:18:10 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Mon, 09 Sep 2019 14:18:10 -0000
Subject: [Python-checkins] docs: Add references to AsyncMock in
 unittest.mock.patch (#13681)
Message-ID: <mailman.344.1568038690.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f5e7f39d2916ed150e80381faed125f405a11e11
commit: f5e7f39d2916ed150e80381faed125f405a11e11
branch: master
author: Mario Corchero <mcorcherojim at bloomberg.net>
committer: Lisa Roach <lisaroach14 at gmail.com>
date: 2019-09-09T15:18:06+01:00
summary:

docs: Add references to AsyncMock in unittest.mock.patch (#13681)

Update the docs as patch can now return an AsyncMock if the patched
object is an async function.

files:
M Doc/library/unittest.mock.rst
M Lib/unittest/mock.py

diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index 304ba532274d..9bb342773a97 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -1307,8 +1307,10 @@ patch
     is patched with a *new* object. When the function/with statement exits
     the patch is undone.
 
-    If *new* is omitted, then the target is replaced with a
-    :class:`MagicMock`. If :func:`patch` is used as a decorator and *new* is
+    If *new* is omitted, then the target is replaced with an
+    :class:`AsyncMock` if the patched object is an async function or
+    a :class:`MagicMock` otherwise.
+    If :func:`patch` is used as a decorator and *new* is
     omitted, the created mock is passed in as an extra argument to the
     decorated function. If :func:`patch` is used as a context manager the created
     mock is returned by the context manager.
@@ -1326,8 +1328,8 @@ patch
     patch to pass in the object being mocked as the spec/spec_set object.
 
     *new_callable* allows you to specify a different class, or callable object,
-    that will be called to create the *new* object. By default :class:`MagicMock` is
-    used.
+    that will be called to create the *new* object. By default :class:`AsyncMock`
+    is used for async functions and :class:`MagicMock` for the rest.
 
     A more powerful form of *spec* is *autospec*. If you set ``autospec=True``
     then the mock will be created with a spec from the object being replaced.
@@ -1491,6 +1493,10 @@ work as expected::
     ...
     >>> test()
 
+.. versionchanged:: 3.8
+
+    :func:`patch` now returns an :class:`AsyncMock` if the target is an async function.
+
 
 patch.object
 ~~~~~~~~~~~~
@@ -2275,6 +2281,12 @@ See :ref:`auto-speccing` for examples of how to use auto-speccing with
 :func:`create_autospec` and the *autospec* argument to :func:`patch`.
 
 
+.. versionchanged:: 3.8
+
+    :func:`create_autospec` now returns an :class:`AsyncMock` if the target is
+    an async function.
+
+
 ANY
 ~~~
 
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index c9194fff4386..c26b367a4dc2 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -1631,8 +1631,9 @@ def patch(
     is patched with a `new` object. When the function/with statement exits
     the patch is undone.
 
-    If `new` is omitted, then the target is replaced with a
-    `MagicMock`. If `patch` is used as a decorator and `new` is
+    If `new` is omitted, then the target is replaced with an
+    `AsyncMock if the patched object is an async function or a
+    `MagicMock` otherwise. If `patch` is used as a decorator and `new` is
     omitted, the created mock is passed in as an extra argument to the
     decorated function. If `patch` is used as a context manager the created
     mock is returned by the context manager.
@@ -1650,8 +1651,8 @@ def patch(
     patch to pass in the object being mocked as the spec/spec_set object.
 
     `new_callable` allows you to specify a different class, or callable object,
-    that will be called to create the `new` object. By default `MagicMock` is
-    used.
+    that will be called to create the `new` object. By default `AsyncMock` is
+    used for async functions and `MagicMock` for the rest.
 
     A more powerful form of `spec` is `autospec`. If you set `autospec=True`
     then the mock will be created with a spec from the object being replaced.


From webhook-mailer at python.org  Mon Sep  9 10:23:40 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 14:23:40 -0000
Subject: [Python-checkins] bpo-32587: Fixes unsafe downcast in PC/winreg.c
 (GH-15766)
Message-ID: <mailman.345.1568039021.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/44729c9f5198211faf533da49fa0fa26693a1993
commit: 44729c9f5198211faf533da49fa0fa26693a1993
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T07:23:37-07:00
summary:

bpo-32587: Fixes unsafe downcast in PC/winreg.c (GH-15766)

(cherry picked from commit ef66f31ce21cd759cc0c618c5c42ba6da0a06834)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
M PC/winreg.c

diff --git a/PC/winreg.c b/PC/winreg.c
index 37bc2c72dcde..72a7c380beef 100644
--- a/PC/winreg.c
+++ b/PC/winreg.c
@@ -772,7 +772,7 @@ Reg2Py(BYTE *retDataBuf, DWORD retDataSize, DWORD typ)
                         return NULL;
                     }
                     PyList_SET_ITEM(obData, index, uni);
-                    len -= slen + 1;
+                    len -= Py_SAFE_DOWNCAST(slen + 1, size_t, int);
                 }
                 PyMem_Free(str);
 


From webhook-mailer at python.org  Mon Sep  9 10:25:24 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 14:25:24 -0000
Subject: [Python-checkins] bpo-32587: Fixes unsafe downcast in PC/winreg.c
 (GH-15766)
Message-ID: <mailman.346.1568039125.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/55a6f73b49625ebff575521c3a0b919880f5f0ec
commit: 55a6f73b49625ebff575521c3a0b919880f5f0ec
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T07:25:20-07:00
summary:

bpo-32587: Fixes unsafe downcast in PC/winreg.c (GH-15766)

(cherry picked from commit ef66f31ce21cd759cc0c618c5c42ba6da0a06834)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
M PC/winreg.c

diff --git a/PC/winreg.c b/PC/winreg.c
index 0016b320cac0..16a10c7232a1 100644
--- a/PC/winreg.c
+++ b/PC/winreg.c
@@ -772,7 +772,7 @@ Reg2Py(BYTE *retDataBuf, DWORD retDataSize, DWORD typ)
                         return NULL;
                     }
                     PyList_SET_ITEM(obData, index, uni);
-                    len -= slen + 1;
+                    len -= Py_SAFE_DOWNCAST(slen + 1, size_t, int);
                 }
                 PyMem_Free(str);
 


From webhook-mailer at python.org  Mon Sep  9 10:42:39 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 14:42:39 -0000
Subject: [Python-checkins] bpo-38037: Fix reference counters in signal
 module (GH-15753)
Message-ID: <mailman.347.1568040160.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b150d0bf1bb4c3203bb3293625e32aed01b25887
commit: b150d0bf1bb4c3203bb3293625e32aed01b25887
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T07:42:35-07:00
summary:

bpo-38037: Fix reference counters in signal module (GH-15753)

(cherry picked from commit 77643c486fd22d8030e0d82c13012684b4ab6df1)

Co-authored-by: animalize <animalize at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Library/2019-09-09-18-39-23.bpo-38037.B0UgFU.rst
M Modules/signalmodule.c

diff --git a/Misc/NEWS.d/next/Library/2019-09-09-18-39-23.bpo-38037.B0UgFU.rst b/Misc/NEWS.d/next/Library/2019-09-09-18-39-23.bpo-38037.B0UgFU.rst
new file mode 100644
index 000000000000..607673bd9994
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-09-18-39-23.bpo-38037.B0UgFU.rst
@@ -0,0 +1 @@
+Fix reference counters in the :mod:`signal` module.
diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c
index 7698984ff3af..9aca70599689 100644
--- a/Modules/signalmodule.c
+++ b/Modules/signalmodule.c
@@ -1329,7 +1329,7 @@ static struct PyModuleDef signalmodule = {
 PyMODINIT_FUNC
 PyInit__signal(void)
 {
-    PyObject *m, *d, *x;
+    PyObject *m, *d;
     int i;
 
     /* Create the module and add the functions */
@@ -1350,13 +1350,17 @@ PyInit__signal(void)
     /* Add some symbolic constants to the module */
     d = PyModule_GetDict(m);
 
-    x = DefaultHandler = PyLong_FromVoidPtr((void *)SIG_DFL);
-    if (PyModule_AddObject(m, "SIG_DFL", x))
+    DefaultHandler = PyLong_FromVoidPtr((void *)SIG_DFL);
+    if (!DefaultHandler ||
+        PyDict_SetItemString(d, "SIG_DFL", DefaultHandler) < 0) {
         goto finally;
+    }
 
-    x = IgnoreHandler = PyLong_FromVoidPtr((void *)SIG_IGN);
-    if (PyModule_AddObject(m, "SIG_IGN", x))
+    IgnoreHandler = PyLong_FromVoidPtr((void *)SIG_IGN);
+    if (!IgnoreHandler ||
+        PyDict_SetItemString(d, "SIG_IGN", IgnoreHandler) < 0) {
         goto finally;
+    }
 
     if (PyModule_AddIntMacro(m, NSIG))
         goto finally;
@@ -1374,8 +1378,8 @@ PyInit__signal(void)
          goto finally;
 #endif
 
-    x = IntHandler = PyDict_GetItemString(d, "default_int_handler");
-    if (!x)
+    IntHandler = PyDict_GetItemString(d, "default_int_handler");
+    if (!IntHandler)
         goto finally;
     Py_INCREF(IntHandler);
 
@@ -1568,8 +1572,10 @@ PyInit__signal(void)
 #if defined (HAVE_SETITIMER) || defined (HAVE_GETITIMER)
     ItimerError = PyErr_NewException("signal.ItimerError",
             PyExc_OSError, NULL);
-    if (PyModule_AddObject(m, "ItimerError", ItimerError))
+    if (!ItimerError ||
+        PyDict_SetItemString(d, "ItimerError", ItimerError) < 0) {
         goto finally;
+    }
 #endif
 
 #ifdef CTRL_C_EVENT
@@ -1615,6 +1621,9 @@ finisignal(void)
     Py_CLEAR(IntHandler);
     Py_CLEAR(DefaultHandler);
     Py_CLEAR(IgnoreHandler);
+#ifdef HAVE_GETITIMER
+    Py_CLEAR(ItimerError);
+#endif
 }
 
 


From webhook-mailer at python.org  Mon Sep  9 10:43:30 2019
From: webhook-mailer at python.org (Larry Hastings)
Date: Mon, 09 Sep 2019 14:43:30 -0000
Subject: [Python-checkins] Blurb release and pydoc-topics for 3.5.8rc1.
Message-ID: <mailman.348.1568040211.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ee5b1a2199b6631a1d7187f121de8813047fdd37
commit: ee5b1a2199b6631a1d7187f121de8813047fdd37
branch: 3.5
author: Larry Hastings <larry at hastings.org>
committer: Larry Hastings <larry at hastings.org>
date: 2019-09-09T12:16:26+01:00
summary:

Blurb release and pydoc-topics for 3.5.8rc1.

files:
A Misc/NEWS.d/3.5.8rc1.rst
D Misc/NEWS.d/next/Build/2019-03-29-14-29-06.bpo-36478.hzyneF.rst
D Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst
D Misc/NEWS.d/next/Security/2019-04-29-15-34-59.bpo-36742.QCUY0i.rst
D Misc/NEWS.d/next/Security/2019-05-04-13-33-37.bpo-34155.MJll68.rst
D Misc/NEWS.d/next/Security/2019-05-21-23-20-18.bpo-35907.NC_zNK.rst
D Misc/NEWS.d/next/Security/2019-07-16-08-11-00.bpo-37461.1Ahz7O.rst
D Misc/NEWS.d/next/Tests/2019-04-05-10-34-29.bpo-36576.7Cp2kK.rst
D Misc/NEWS.d/next/Tests/2019-05-08-15-55-46.bpo-36816.WBKRGZ.rst
M Lib/pydoc_data/topics.py

diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py
index f46c421fb002..172b4f86ada3 100644
--- a/Lib/pydoc_data/topics.py
+++ b/Lib/pydoc_data/topics.py
@@ -1,79 +1,12423 @@
 # -*- coding: utf-8 -*-
-# Autogenerated by Sphinx on Sun Mar 17 16:30:44 2019
-topics = {'assert': u'\nThe "assert" statement\n**********************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n   assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, "assert expression", is equivalent to\n\n   if __debug__:\n       if not expression: raise AssertionError\n\nThe extended form, "assert expression1, expression2", is equivalent to\n\n   if __debug__:\n       if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that "__debug__" and "AssertionError" refer\nto the built-in variables with those names.  In the current\nimplementation, the built-in variable "__debug__" is "True" under\nnormal circumstances, "False" when optimization is requested (command\nline option -O).  The current code generator emits no code for an\nassert statement when optimization is requested at compile time.  Note\nthat it is unnecessary to include the source code for the expression\nthat failed in the error message; it will be displayed as part of the\nstack trace.\n\nAssignments to "__debug__" are illegal.  The value for the built-in\nvariable is determined when the interpreter starts.\n',
- 'assignment': u'\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n   assignment_stmt ::= (target_list "=")+ (starred_expression | yield_expression)\n   target_list     ::= target ("," target)* [","]\n   target          ::= identifier\n              | "(" target_list ")"\n              | "[" [target_list] "]"\n              | attributeref\n              | subscription\n              | slicing\n              | "*" target\n\n(See section *Primaries* for the syntax definitions for\n*attributeref*, *subscription*, and *slicing*.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable.  The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list, optionally enclosed in\nparentheses or square brackets, is recursively defined as follows.\n\n* If the target list is empty: The object must also be an empty\n  iterable.\n\n* If the target list is a single target in parentheses: The object\n  is assigned to that target.\n\n* If the target list is a comma-separated list of targets, or a\n  single target in square brackets: The object must be an iterable\n  with the same number of items as there are targets in the target\n  list, and the items are assigned, from left to right, to the\n  corresponding targets.\n\n  * If the target list contains one target prefixed with an\n    asterisk, called a "starred" target: The object must be an\n    iterable with at least as many items as there are targets in the\n    target list, minus one.  The first items of the iterable are\n    assigned, from left to right, to the targets before the starred\n    target.  The final items of the iterable are assigned to the\n    targets after the starred target.  A list of the remaining items\n    in the iterable is then assigned to the starred target (the list\n    can be empty).\n\n  * Else: The object must be an iterable with the same number of\n    items as there are targets in the target list, and the items are\n    assigned, from left to right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n  * If the name does not occur in a "global" or "nonlocal" statement\n    in the current code block: the name is bound to the object in the\n    current local namespace.\n\n  * Otherwise: the name is bound to the object in the global\n    namespace or the outer namespace determined by "nonlocal",\n    respectively.\n\n  The name is rebound if it was already bound.  This may cause the\n  reference count for the object previously bound to the name to reach\n  zero, causing the object to be deallocated and its destructor (if it\n  has one) to be called.\n\n* If the target is an attribute reference: The primary expression in\n  the reference is evaluated.  It should yield an object with\n  assignable attributes; if this is not the case, "TypeError" is\n  raised.  That object is then asked to assign the assigned object to\n  the given attribute; if it cannot perform the assignment, it raises\n  an exception (usually but not necessarily "AttributeError").\n\n  Note: If the object is a class instance and the attribute reference\n  occurs on both sides of the assignment operator, the RHS expression,\n  "a.x" can access either an instance attribute or (if no instance\n  attribute exists) a class attribute.  The LHS target "a.x" is always\n  set as an instance attribute, creating it if necessary.  Thus, the\n  two occurrences of "a.x" do not necessarily refer to the same\n  attribute: if the RHS expression refers to a class attribute, the\n  LHS creates a new instance attribute as the target of the\n  assignment:\n\n     class Cls:\n         x = 3             # class variable\n     inst = Cls()\n     inst.x = inst.x + 1   # writes inst.x as 4 leaving Cls.x as 3\n\n  This description does not necessarily apply to descriptor\n  attributes, such as properties created with "property()".\n\n* If the target is a subscription: The primary expression in the\n  reference is evaluated.  It should yield either a mutable sequence\n  object (such as a list) or a mapping object (such as a dictionary).\n  Next, the subscript expression is evaluated.\n\n  If the primary is a mutable sequence object (such as a list), the\n  subscript must yield an integer.  If it is negative, the sequence\'s\n  length is added to it.  The resulting value must be a nonnegative\n  integer less than the sequence\'s length, and the sequence is asked\n  to assign the assigned object to its item with that index.  If the\n  index is out of range, "IndexError" is raised (assignment to a\n  subscripted sequence cannot add new items to a list).\n\n  If the primary is a mapping object (such as a dictionary), the\n  subscript must have a type compatible with the mapping\'s key type,\n  and the mapping is then asked to create a key/datum pair which maps\n  the subscript to the assigned object.  This can either replace an\n  existing key/value pair with the same key value, or insert a new\n  key/value pair (if no key with the same value existed).\n\n  For user-defined objects, the "__setitem__()" method is called with\n  appropriate arguments.\n\n* If the target is a slicing: The primary expression in the\n  reference is evaluated.  It should yield a mutable sequence object\n  (such as a list).  The assigned object should be a sequence object\n  of the same type.  Next, the lower and upper bound expressions are\n  evaluated, insofar they are present; defaults are zero and the\n  sequence\'s length.  The bounds should evaluate to integers. If\n  either bound is negative, the sequence\'s length is added to it.  The\n  resulting bounds are clipped to lie between zero and the sequence\'s\n  length, inclusive.  Finally, the sequence object is asked to replace\n  the slice with the items of the assigned sequence.  The length of\n  the slice may be different from the length of the assigned sequence,\n  thus changing the length of the target sequence, if the target\n  sequence allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nAlthough the definition of assignment implies that overlaps between\nthe left-hand side and the right-hand side are \'simultaneous\' (for\nexample "a, b = b, a" swaps two variables), overlaps *within* the\ncollection of assigned-to variables occur left-to-right, sometimes\nresulting in confusion.  For instance, the following program prints\n"[0, 2]":\n\n   x = [0, 1]\n   i = 0\n   i, x[i] = 1, 2         # i is updated, then x[i] is updated\n   print(x)\n\nSee also: **PEP 3132** - Extended Iterable Unpacking\n\n     The specification for the "*target" feature.\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n   augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n   augtarget                 ::= identifier | attributeref | subscription | slicing\n   augop                     ::= "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**="\n             | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions of the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like "x += 1" can be rewritten as\n"x = x + 1" to achieve a similar, but not exactly equal effect. In the\naugmented version, "x" is only evaluated once. Also, when possible,\nthe actual operation is performed *in-place*, meaning that rather than\ncreating a new object and assigning that to the target, the old object\nis modified instead.\n\nUnlike normal assignments, augmented assignments evaluate the left-\nhand side *before* evaluating the right-hand side.  For example, "a[i]\n+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and performs\nthe addition, and lastly, it writes the result back to "a[i]".\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n',
- 'atom-identifiers': u'\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name.  See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a "NameError" exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them.  The transformation inserts the\nclass name, with leading underscores removed and a single underscore\ninserted, in front of the name.  For example, the identifier "__spam"\noccurring in a class named "Ham" will be transformed to "_Ham__spam".\nThis transformation is independent of the syntactical context in which\nthe identifier is used.  If the transformed name is extremely long\n(longer than 255 characters), implementation defined truncation may\nhappen. If the class name consists only of underscores, no\ntransformation is done.\n',
- 'atom-literals': u"\nLiterals\n********\n\nPython supports string and bytes literals and various numeric\nliterals:\n\n   literal ::= stringliteral | bytesliteral\n               | integer | floatnumber | imagnumber\n\nEvaluation of a literal yields an object of the given type (string,\nbytes, integer, floating point number, complex number) with the given\nvalue.  The value may be approximated in the case of floating point\nand imaginary (complex) literals.  See section *Literals* for details.\n\nAll literals correspond to immutable data types, and hence the\nobject's identity is less important than its value.  Multiple\nevaluations of literals with the same value (either the same\noccurrence in the program text or a different occurrence) may obtain\nthe same object or a different object with the same value.\n",
- 'attribute-access': u'\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of "x.name") for\nclass instances.\n\nobject.__getattr__(self, name)\n\n   Called when an attribute lookup has not found the attribute in the\n   usual places (i.e. it is not an instance attribute nor is it found\n   in the class tree for "self").  "name" is the attribute name. This\n   method should return the (computed) attribute value or raise an\n   "AttributeError" exception.\n\n   Note that if the attribute is found through the normal mechanism,\n   "__getattr__()" is not called.  (This is an intentional asymmetry\n   between "__getattr__()" and "__setattr__()".) This is done both for\n   efficiency reasons and because otherwise "__getattr__()" would have\n   no way to access other attributes of the instance.  Note that at\n   least for instance variables, you can fake total control by not\n   inserting any values in the instance attribute dictionary (but\n   instead inserting them in another object).  See the\n   "__getattribute__()" method below for a way to actually get total\n   control over attribute access.\n\nobject.__getattribute__(self, name)\n\n   Called unconditionally to implement attribute accesses for\n   instances of the class. If the class also defines "__getattr__()",\n   the latter will not be called unless "__getattribute__()" either\n   calls it explicitly or raises an "AttributeError". This method\n   should return the (computed) attribute value or raise an\n   "AttributeError" exception. In order to avoid infinite recursion in\n   this method, its implementation should always call the base class\n   method with the same name to access any attributes it needs, for\n   example, "object.__getattribute__(self, name)".\n\n   Note: This method may still be bypassed when looking up special\n     methods as the result of implicit invocation via language syntax\n     or built-in functions. See *Special method lookup*.\n\nobject.__setattr__(self, name, value)\n\n   Called when an attribute assignment is attempted.  This is called\n   instead of the normal mechanism (i.e. store the value in the\n   instance dictionary). *name* is the attribute name, *value* is the\n   value to be assigned to it.\n\n   If "__setattr__()" wants to assign to an instance attribute, it\n   should call the base class method with the same name, for example,\n   "object.__setattr__(self, name, value)".\n\nobject.__delattr__(self, name)\n\n   Like "__setattr__()" but for attribute deletion instead of\n   assignment.  This should only be implemented if "del obj.name" is\n   meaningful for the object.\n\nobject.__dir__(self)\n\n   Called when "dir()" is called on the object. A sequence must be\n   returned. "dir()" converts the returned sequence to a list and\n   sorts it.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents).  In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' "__dict__".\n\nobject.__get__(self, instance, owner)\n\n   Called to get the attribute of the owner class (class attribute\n   access) or of an instance of that class (instance attribute\n   access). *owner* is always the owner class, while *instance* is the\n   instance that the attribute was accessed through, or "None" when\n   the attribute is accessed through the *owner*.  This method should\n   return the (computed) attribute value or raise an "AttributeError"\n   exception.\n\nobject.__set__(self, instance, value)\n\n   Called to set the attribute on an instance *instance* of the owner\n   class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n   Called to delete the attribute on an instance *instance* of the\n   owner class.\n\nThe attribute "__objclass__" is interpreted by the "inspect" module as\nspecifying the class where this object was defined (setting this\nappropriately can assist in runtime introspection of dynamic class\nattributes). For callables, it may indicate that an instance of the\ngiven type (or a subclass) is expected or required as the first\npositional argument (for example, CPython sets this attribute for\nunbound methods that are implemented in C).\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol:  "__get__()", "__set__()", and\n"__delete__()". If any of those methods are defined for an object, it\nis said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, "a.x" has a\nlookup chain starting with "a.__dict__[\'x\']", then\n"type(a).__dict__[\'x\']", and continuing through the base classes of\n"type(a)" excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead.  Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.\n\nThe starting point for descriptor invocation is a binding, "a.x". How\nthe arguments are assembled depends on "a":\n\nDirect Call\n   The simplest and least common call is when user code directly\n   invokes a descriptor method:    "x.__get__(a)".\n\nInstance Binding\n   If binding to an object instance, "a.x" is transformed into the\n   call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n\nClass Binding\n   If binding to a class, "A.x" is transformed into the call:\n   "A.__dict__[\'x\'].__get__(None, A)".\n\nSuper Binding\n   If "a" is an instance of "super", then the binding "super(B,\n   obj).m()" searches "obj.__class__.__mro__" for the base class "A"\n   immediately preceding "B" and then invokes the descriptor with the\n   call: "A.__dict__[\'m\'].__get__(obj, obj.__class__)".\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined.  A descriptor can define\nany combination of "__get__()", "__set__()" and "__delete__()".  If it\ndoes not define "__get__()", then accessing the attribute will return\nthe descriptor object itself unless there is a value in the object\'s\ninstance dictionary.  If the descriptor defines "__set__()" and/or\n"__delete__()", it is a data descriptor; if it defines neither, it is\na non-data descriptor.  Normally, data descriptors define both\n"__get__()" and "__set__()", while non-data descriptors have just the\n"__get__()" method.  Data descriptors with "__set__()" and "__get__()"\ndefined always override a redefinition in an instance dictionary.  In\ncontrast, non-data descriptors can be overridden by instances.\n\nPython methods (including "staticmethod()" and "classmethod()") are\nimplemented as non-data descriptors.  Accordingly, instances can\nredefine and override methods.  This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe "property()" function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of classes have a dictionary for attribute\nstorage.  This wastes space for objects having very few instance\nvariables.  The space consumption can become acute when creating large\nnumbers of instances.\n\nThe default can be overridden by defining *__slots__* in a class\ndefinition. The *__slots__* declaration takes a sequence of instance\nvariables and reserves just enough space in each instance to hold a\nvalue for each variable.  Space is saved because *__dict__* is not\ncreated for each instance.\n\nobject.__slots__\n\n   This class variable can be assigned a string, iterable, or sequence\n   of strings with variable names used by instances.  *__slots__*\n   reserves space for the declared variables and prevents the\n   automatic creation of *__dict__* and *__weakref__* for each\n   instance.\n\n\nNotes on using *__slots__*\n--------------------------\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n  attribute of that class will always be accessible, so a *__slots__*\n  definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n  variables not listed in the *__slots__* definition.  Attempts to\n  assign to an unlisted variable name raises "AttributeError". If\n  dynamic assignment of new variables is desired, then add\n  "\'__dict__\'" to the sequence of strings in the *__slots__*\n  declaration.\n\n* Without a *__weakref__* variable for each instance, classes\n  defining *__slots__* do not support weak references to its\n  instances. If weak reference support is needed, then add\n  "\'__weakref__\'" to the sequence of strings in the *__slots__*\n  declaration.\n\n* *__slots__* are implemented at the class level by creating\n  descriptors (*Implementing Descriptors*) for each variable name.  As\n  a result, class attributes cannot be used to set default values for\n  instance variables defined by *__slots__*; otherwise, the class\n  attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n  where it is defined.  As a result, subclasses will have a *__dict__*\n  unless they also define *__slots__* (which must only contain names\n  of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the\n  instance variable defined by the base class slot is inaccessible\n  (except by retrieving its descriptor directly from the base class).\n  This renders the meaning of the program undefined.  In the future, a\n  check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n  "variable-length" built-in types such as "int", "bytes" and "tuple".\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings\n  may also be used; however, in the future, special meaning may be\n  assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n  *__slots__*.\n',
- 'attribute-references': u'\nAttribute references\n********************\n\nAn attribute reference is a primary followed by a period and a name:\n\n   attributeref ::= primary "." identifier\n\nThe primary must evaluate to an object of a type that supports\nattribute references, which most objects do.  This object is then\nasked to produce the attribute whose name is the identifier.  This\nproduction can be customized by overriding the "__getattr__()" method.\nIf this attribute is not available, the exception "AttributeError" is\nraised.  Otherwise, the type and value of the object produced is\ndetermined by the object.  Multiple evaluations of the same attribute\nreference may yield different objects.\n',
- 'augassign': u'\nAugmented assignment statements\n*******************************\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n   augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n   augtarget                 ::= identifier | attributeref | subscription | slicing\n   augop                     ::= "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**="\n             | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions of the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like "x += 1" can be rewritten as\n"x = x + 1" to achieve a similar, but not exactly equal effect. In the\naugmented version, "x" is only evaluated once. Also, when possible,\nthe actual operation is performed *in-place*, meaning that rather than\ncreating a new object and assigning that to the target, the old object\nis modified instead.\n\nUnlike normal assignments, augmented assignments evaluate the left-\nhand side *before* evaluating the right-hand side.  For example, "a[i]\n+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and performs\nthe addition, and lastly, it writes the result back to "a[i]".\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n',
- 'binary': u'\nBinary arithmetic operations\n****************************\n\nThe binary arithmetic operations have the conventional priority\nlevels.  Note that some of these operations also apply to certain non-\nnumeric types.  Apart from the power operator, there are only two\nlevels, one for multiplicative operators and one for additive\noperators:\n\n   m_expr ::= u_expr | m_expr "*" u_expr | m_expr "@" m_expr |\n              m_expr "//" u_expr| m_expr "/" u_expr |\n              m_expr "%" u_expr\n   a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n\nThe "*" (multiplication) operator yields the product of its arguments.\nThe arguments must either both be numbers, or one argument must be an\ninteger and the other must be a sequence. In the former case, the\nnumbers are converted to a common type and then multiplied together.\nIn the latter case, sequence repetition is performed; a negative\nrepetition factor yields an empty sequence.\n\nThe "@" (at) operator is intended to be used for matrix\nmultiplication.  No builtin Python types implement this operator.\n\nNew in version 3.5.\n\nThe "/" (division) and "//" (floor division) operators yield the\nquotient of their arguments.  The numeric arguments are first\nconverted to a common type. Division of integers yields a float, while\nfloor division of integers results in an integer; the result is that\nof mathematical division with the \'floor\' function applied to the\nresult.  Division by zero raises the "ZeroDivisionError" exception.\n\nThe "%" (modulo) operator yields the remainder from the division of\nthe first argument by the second.  The numeric arguments are first\nconverted to a common type.  A zero right argument raises the\n"ZeroDivisionError" exception.  The arguments may be floating point\nnumbers, e.g., "3.14%0.7" equals "0.34" (since "3.14" equals "4*0.7 +\n0.34".)  The modulo operator always yields a result with the same sign\nas its second operand (or zero); the absolute value of the result is\nstrictly smaller than the absolute value of the second operand [1].\n\nThe floor division and modulo operators are connected by the following\nidentity: "x == (x//y)*y + (x%y)".  Floor division and modulo are also\nconnected with the built-in function "divmod()": "divmod(x, y) ==\n(x//y, x%y)". [2].\n\nIn addition to performing the modulo operation on numbers, the "%"\noperator is also overloaded by string objects to perform old-style\nstring formatting (also known as interpolation).  The syntax for\nstring formatting is described in the Python Library Reference,\nsection *printf-style String Formatting*.\n\nThe floor division operator, the modulo operator, and the "divmod()"\nfunction are not defined for complex numbers.  Instead, convert to a\nfloating point number using the "abs()" function if appropriate.\n\nThe "+" (addition) operator yields the sum of its arguments.  The\narguments must either both be numbers or both be sequences of the same\ntype.  In the former case, the numbers are converted to a common type\nand then added together. In the latter case, the sequences are\nconcatenated.\n\nThe "-" (subtraction) operator yields the difference of its arguments.\nThe numeric arguments are first converted to a common type.\n',
- 'bitwise': u'\nBinary bitwise operations\n*************************\n\nEach of the three bitwise operations has a different priority level:\n\n   and_expr ::= shift_expr | and_expr "&" shift_expr\n   xor_expr ::= and_expr | xor_expr "^" and_expr\n   or_expr  ::= xor_expr | or_expr "|" xor_expr\n\nThe "&" operator yields the bitwise AND of its arguments, which must\nbe integers.\n\nThe "^" operator yields the bitwise XOR (exclusive OR) of its\narguments, which must be integers.\n\nThe "|" operator yields the bitwise (inclusive) OR of its arguments,\nwhich must be integers.\n',
- 'bltin-code-objects': u'\nCode Objects\n************\n\nCode objects are used by the implementation to represent "pseudo-\ncompiled" executable Python code such as a function body. They differ\nfrom function objects because they don\'t contain a reference to their\nglobal execution environment.  Code objects are returned by the built-\nin "compile()" function and can be extracted from function objects\nthrough their "__code__" attribute. See also the "code" module.\n\nA code object can be executed or evaluated by passing it (instead of a\nsource string) to the "exec()" or "eval()"  built-in functions.\n\nSee *The standard type hierarchy* for more information.\n',
- 'bltin-ellipsis-object': u'\nThe Ellipsis Object\n*******************\n\nThis object is commonly used by slicing (see *Slicings*).  It supports\nno special operations.  There is exactly one ellipsis object, named\n"Ellipsis" (a built-in name).  "type(Ellipsis)()" produces the\n"Ellipsis" singleton.\n\nIt is written as "Ellipsis" or "...".\n',
- 'bltin-null-object': u'\nThe Null Object\n***************\n\nThis object is returned by functions that don\'t explicitly return a\nvalue.  It supports no special operations.  There is exactly one null\nobject, named "None" (a built-in name).  "type(None)()" produces the\nsame singleton.\n\nIt is written as "None".\n',
- 'bltin-type-objects': u'\nType Objects\n************\n\nType objects represent the various object types.  An object\'s type is\naccessed by the built-in function "type()".  There are no special\noperations on types.  The standard module "types" defines names for\nall standard built-in types.\n\nTypes are written like this: "<class \'int\'>".\n',
- 'booleans': u'\nBoolean operations\n******************\n\n   or_test  ::= and_test | or_test "or" and_test\n   and_test ::= not_test | and_test "and" not_test\n   not_test ::= comparison | "not" not_test\n\nIn the context of Boolean operations, and also when expressions are\nused by control flow statements, the following values are interpreted\nas false: "False", "None", numeric zero of all types, and empty\nstrings and containers (including strings, tuples, lists,\ndictionaries, sets and frozensets).  All other values are interpreted\nas true.  User-defined objects can customize their truth value by\nproviding a "__bool__()" method.\n\nThe operator "not" yields "True" if its argument is false, "False"\notherwise.\n\nThe expression "x and y" first evaluates *x*; if *x* is false, its\nvalue is returned; otherwise, *y* is evaluated and the resulting value\nis returned.\n\nThe expression "x or y" first evaluates *x*; if *x* is true, its value\nis returned; otherwise, *y* is evaluated and the resulting value is\nreturned.\n\n(Note that neither "and" nor "or" restrict the value and type they\nreturn to "False" and "True", but rather return the last evaluated\nargument.  This is sometimes useful, e.g., if "s" is a string that\nshould be replaced by a default value if it is empty, the expression\n"s or \'foo\'" yields the desired value.  Because "not" has to create a\nnew value, it returns a boolean value regardless of the type of its\nargument (for example, "not \'foo\'" produces "False" rather than "\'\'".)\n',
- 'break': u'\nThe "break" statement\n*********************\n\n   break_stmt ::= "break"\n\n"break" may only occur syntactically nested in a "for" or "while"\nloop, but not nested in a function or class definition within that\nloop.\n\nIt terminates the nearest enclosing loop, skipping the optional "else"\nclause if the loop has one.\n\nIf a "for" loop is terminated by "break", the loop control target\nkeeps its current value.\n\nWhen "break" passes control out of a "try" statement with a "finally"\nclause, that "finally" clause is executed before really leaving the\nloop.\n',
- 'callable-types': u'\nEmulating callable objects\n**************************\n\nobject.__call__(self[, args...])\n\n   Called when the instance is "called" as a function; if this method\n   is defined, "x(arg1, arg2, ...)" is a shorthand for\n   "x.__call__(arg1, arg2, ...)".\n',
- 'calls': u'\nCalls\n*****\n\nA call calls a callable object (e.g., a *function*) with a possibly\nempty series of *arguments*:\n\n   call                 ::= primary "(" [argument_list [","] | comprehension] ")"\n   argument_list        ::= positional_arguments ["," starred_and_keywords]\n                       ["," keywords_arguments]\n                     | starred_and_keywords ["," keywords_arguments]\n                     | keywords_arguments\n   positional_arguments ::= ["*"] expression ("," ["*"] expression)*\n   starred_and_keywords ::= ("*" expression | keyword_item)\n                            ("," "*" expression | "," keyword_item)*\n   keywords_arguments   ::= (keyword_item | "**" expression)\n                          ("," keyword_item | "," "**" expression)*\n   keyword_item         ::= identifier "=" expression\n\nAn optional trailing comma may be present after the positional and\nkeyword arguments but does not affect the semantics.\n\nThe primary must evaluate to a callable object (user-defined\nfunctions, built-in functions, methods of built-in objects, class\nobjects, methods of class instances, and all objects having a\n"__call__()" method are callable).  All argument expressions are\nevaluated before the call is attempted.  Please refer to section\n*Function definitions* for the syntax of formal *parameter* lists.\n\nIf keyword arguments are present, they are first converted to\npositional arguments, as follows.  First, a list of unfilled slots is\ncreated for the formal parameters.  If there are N positional\narguments, they are placed in the first N slots.  Next, for each\nkeyword argument, the identifier is used to determine the\ncorresponding slot (if the identifier is the same as the first formal\nparameter name, the first slot is used, and so on).  If the slot is\nalready filled, a "TypeError" exception is raised. Otherwise, the\nvalue of the argument is placed in the slot, filling it (even if the\nexpression is "None", it fills the slot).  When all arguments have\nbeen processed, the slots that are still unfilled are filled with the\ncorresponding default value from the function definition.  (Default\nvalues are calculated, once, when the function is defined; thus, a\nmutable object such as a list or dictionary used as default value will\nbe shared by all calls that don\'t specify an argument value for the\ncorresponding slot; this should usually be avoided.)  If there are any\nunfilled slots for which no default value is specified, a "TypeError"\nexception is raised.  Otherwise, the list of filled slots is used as\nthe argument list for the call.\n\n**CPython implementation detail:** An implementation may provide\nbuilt-in functions whose positional parameters do not have names, even\nif they are \'named\' for the purpose of documentation, and which\ntherefore cannot be supplied by keyword.  In CPython, this is the case\nfor functions implemented in C that use "PyArg_ParseTuple()" to parse\ntheir arguments.\n\nIf there are more positional arguments than there are formal parameter\nslots, a "TypeError" exception is raised, unless a formal parameter\nusing the syntax "*identifier" is present; in this case, that formal\nparameter receives a tuple containing the excess positional arguments\n(or an empty tuple if there were no excess positional arguments).\n\nIf any keyword argument does not correspond to a formal parameter\nname, a "TypeError" exception is raised, unless a formal parameter\nusing the syntax "**identifier" is present; in this case, that formal\nparameter receives a dictionary containing the excess keyword\narguments (using the keywords as keys and the argument values as\ncorresponding values), or a (new) empty dictionary if there were no\nexcess keyword arguments.\n\nIf the syntax "*expression" appears in the function call, "expression"\nmust evaluate to an *iterable*.  Elements from these iterables are\ntreated as if they were additional positional arguments.  For the call\n"f(x1, x2, *y, x3, x4)", if *y* evaluates to a sequence *y1*, ...,\n*yM*, this is equivalent to a call with M+4 positional arguments *x1*,\n*x2*, *y1*, ..., *yM*, *x3*, *x4*.\n\nA consequence of this is that although the "*expression" syntax may\nappear *after* explicit keyword arguments, it is processed *before*\nthe keyword arguments (and any "**expression" arguments -- see below).\nSo:\n\n   >>> def f(a, b):\n   ...     print(a, b)\n   ...\n   >>> f(b=1, *(2,))\n   2 1\n   >>> f(a=1, *(2,))\n   Traceback (most recent call last):\n     File "<stdin>", line 1, in <module>\n   TypeError: f() got multiple values for keyword argument \'a\'\n   >>> f(1, *(2,))\n   1 2\n\nIt is unusual for both keyword arguments and the "*expression" syntax\nto be used in the same call, so in practice this confusion does not\narise.\n\nIf the syntax "**expression" appears in the function call,\n"expression" must evaluate to a *mapping*, the contents of which are\ntreated as additional keyword arguments.  If a keyword is already\npresent (as an explicit keyword argument, or from another unpacking),\na "TypeError" exception is raised.\n\nFormal parameters using the syntax "*identifier" or "**identifier"\ncannot be used as positional argument slots or as keyword argument\nnames.\n\nChanged in version 3.5: Function calls accept any number of "*" and\n"**" unpackings, positional arguments may follow iterable unpackings\n("*"), and keyword arguments may follow dictionary unpackings ("**").\nOriginally proposed by **PEP 448**.\n\nA call always returns some value, possibly "None", unless it raises an\nexception.  How this value is computed depends on the type of the\ncallable object.\n\nIf it is---\n\na user-defined function:\n   The code block for the function is executed, passing it the\n   argument list.  The first thing the code block will do is bind the\n   formal parameters to the arguments; this is described in section\n   *Function definitions*.  When the code block executes a "return"\n   statement, this specifies the return value of the function call.\n\na built-in function or method:\n   The result is up to the interpreter; see *Built-in Functions* for\n   the descriptions of built-in functions and methods.\n\na class object:\n   A new instance of that class is returned.\n\na class instance method:\n   The corresponding user-defined function is called, with an argument\n   list that is one longer than the argument list of the call: the\n   instance becomes the first argument.\n\na class instance:\n   The class must define a "__call__()" method; the effect is then the\n   same as if that method was called.\n',
- 'class': u'\nClass definitions\n*****************\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n   classdef    ::= [decorators] "class" classname [inheritance] ":" suite\n   inheritance ::= "(" [argument_list] ")"\n   classname   ::= identifier\n\nA class definition is an executable statement.  The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing.  Classes without an inheritance\nlist inherit, by default, from the base class "object"; hence,\n\n   class Foo:\n       pass\n\nis equivalent to\n\n   class Foo(object):\n       pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.)  When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n   @f1(arg)\n   @f2\n   class Foo: pass\n\nis roughly equivalent to\n\n   class Foo: pass\n   Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators.  The result is then bound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances.  Instance attributes\ncan be set in a method with "self.name = value".  Both class and\ninstance attributes are accessible through the notation ""self.name"",\nand an instance attribute hides a class attribute with the same name\nwhen accessed in this way.  Class attributes can be used as defaults\nfor instance attributes, but using mutable values there can lead to\nunexpected results.  *Descriptors* can be used to create instance\nvariables with different implementation details.\n\nSee also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n  Class Decorators\n',
- 'comparisons': u'\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation.  Also unlike C, expressions like "a < b < c" have the\ninterpretation that is conventional in mathematics:\n\n   comparison    ::= or_expr ( comp_operator or_expr )*\n   comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n                     | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: "True" or "False".\n\nComparisons can be chained arbitrarily, e.g., "x < y <= z" is\nequivalent to "x < y and y <= z", except that "y" is evaluated only\nonce (but in both cases "z" is not evaluated at all when "x < y" is\nfound to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then "a op1 b op2 c ... y\nopN z" is equivalent to "a op1 b and b op2 c and ... y opN z", except\nthat each expression is evaluated at most once.\n\nNote that "a op1 b op2 c" doesn\'t imply any kind of comparison between\n*a* and *c*, so that, e.g., "x < y > z" is perfectly legal (though\nperhaps not pretty).\n\n\nValue comparisons\n=================\n\nThe operators "<", ">", "==", ">=", "<=", and "!=" compare the values\nof two objects.  The objects do not need to have the same type.\n\nChapter *Objects, values and types* states that objects have a value\n(in addition to type and identity).  The value of an object is a\nrather abstract notion in Python: For example, there is no canonical\naccess method for an object\'s value.  Also, there is no requirement\nthat the value of an object should be constructed in a particular way,\ne.g. comprised of all its data attributes. Comparison operators\nimplement a particular notion of what the value of an object is.  One\ncan think of them as defining the value of an object indirectly, by\nmeans of their comparison implementation.\n\nBecause all types are (direct or indirect) subtypes of "object", they\ninherit the default comparison behavior from "object".  Types can\ncustomize their comparison behavior by implementing *rich comparison\nmethods* like "__lt__()", described in *Basic customization*.\n\nThe default behavior for equality comparison ("==" and "!=") is based\non the identity of the objects.  Hence, equality comparison of\ninstances with the same identity results in equality, and equality\ncomparison of instances with different identities results in\ninequality.  A motivation for this default behavior is the desire that\nall objects should be reflexive (i.e. "x is y" implies "x == y").\n\nA default order comparison ("<", ">", "<=", and ">=") is not provided;\nan attempt raises "TypeError".  A motivation for this default behavior\nis the lack of a similar invariant as for equality.\n\nThe behavior of the default equality comparison, that instances with\ndifferent identities are always unequal, may be in contrast to what\ntypes will need that have a sensible definition of object value and\nvalue-based equality.  Such types will need to customize their\ncomparison behavior, and in fact, a number of built-in types have done\nthat.\n\nThe following list describes the comparison behavior of the most\nimportant built-in types.\n\n* Numbers of built-in numeric types (*Numeric Types --- int, float,\n  complex*) and of the standard library types "fractions.Fraction" and\n  "decimal.Decimal" can be compared within and across their types,\n  with the restriction that complex numbers do not support order\n  comparison.  Within the limits of the types involved, they compare\n  mathematically (algorithmically) correct without loss of precision.\n\n  The not-a-number values "float(\'NaN\')" and "Decimal(\'NaN\')" are\n  special.  They are identical to themselves ("x is x" is true) but\n  are not equal to themselves ("x == x" is false).  Additionally,\n  comparing any number to a not-a-number value will return "False".\n  For example, both "3 < float(\'NaN\')" and "float(\'NaN\') < 3" will\n  return "False".\n\n* Binary sequences (instances of "bytes" or "bytearray") can be\n  compared within and across their types.  They compare\n  lexicographically using the numeric values of their elements.\n\n* Strings (instances of "str") compare lexicographically using the\n  numerical Unicode code points (the result of the built-in function\n  "ord()") of their characters. [3]\n\n  Strings and binary sequences cannot be directly compared.\n\n* Sequences (instances of "tuple", "list", or "range") can be\n  compared only within each of their types, with the restriction that\n  ranges do not support order comparison.  Equality comparison across\n  these types results in unequality, and ordering comparison across\n  these types raises "TypeError".\n\n  Sequences compare lexicographically using comparison of\n  corresponding elements, whereby reflexivity of the elements is\n  enforced.\n\n  In enforcing reflexivity of elements, the comparison of collections\n  assumes that for a collection element "x", "x == x" is always true.\n  Based on that assumption, element identity is compared first, and\n  element comparison is performed only for distinct elements.  This\n  approach yields the same result as a strict element comparison\n  would, if the compared elements are reflexive.  For non-reflexive\n  elements, the result is different than for strict element\n  comparison, and may be surprising:  The non-reflexive not-a-number\n  values for example result in the following comparison behavior when\n  used in a list:\n\n     >>> nan = float(\'NaN\')\n     >>> nan is nan\n     True\n     >>> nan == nan\n     False                 <-- the defined non-reflexive behavior of NaN\n     >>> [nan] == [nan]\n     True                  <-- list enforces reflexivity and tests identity first\n\n  Lexicographical comparison between built-in collections works as\n  follows:\n\n  * For two collections to compare equal, they must be of the same\n    type, have the same length, and each pair of corresponding\n    elements must compare equal (for example, "[1,2] == (1,2)" is\n    false because the type is not the same).\n\n  * Collections that support order comparison are ordered the same\n    as their first unequal elements (for example, "[1,2,x] <= [1,2,y]"\n    has the same value as "x <= y").  If a corresponding element does\n    not exist, the shorter collection is ordered first (for example,\n    "[1,2] < [1,2,3]" is true).\n\n* Mappings (instances of "dict") compare equal if and only if they\n  have equal *(key, value)* pairs. Equality comparison of the keys and\n  values enforces reflexivity.\n\n  Order comparisons ("<", ">", "<=", and ">=") raise "TypeError".\n\n* Sets (instances of "set" or "frozenset") can be compared within\n  and across their types.\n\n  They define order comparison operators to mean subset and superset\n  tests.  Those relations do not define total orderings (for example,\n  the two sets "{1,2}" and "{2,3}" are not equal, nor subsets of one\n  another, nor supersets of one another).  Accordingly, sets are not\n  appropriate arguments for functions which depend on total ordering\n  (for example, "min()", "max()", and "sorted()" produce undefined\n  results given a list of sets as inputs).\n\n  Comparison of sets enforces reflexivity of its elements.\n\n* Most other built-in types have no comparison methods implemented,\n  so they inherit the default comparison behavior.\n\nUser-defined classes that customize their comparison behavior should\nfollow some consistency rules, if possible:\n\n* Equality comparison should be reflexive. In other words, identical\n  objects should compare equal:\n\n     "x is y" implies "x == y"\n\n* Comparison should be symmetric. In other words, the following\n  expressions should have the same result:\n\n     "x == y" and "y == x"\n\n     "x != y" and "y != x"\n\n     "x < y" and "y > x"\n\n     "x <= y" and "y >= x"\n\n* Comparison should be transitive. The following (non-exhaustive)\n  examples illustrate that:\n\n     "x > y and y > z" implies "x > z"\n\n     "x < y and y <= z" implies "x < z"\n\n* Inverse comparison should result in the boolean negation. In other\n  words, the following expressions should have the same result:\n\n     "x == y" and "not x != y"\n\n     "x < y" and "not x >= y" (for total ordering)\n\n     "x > y" and "not x <= y" (for total ordering)\n\n  The last two expressions apply to totally ordered collections (e.g.\n  to sequences, but not to sets or mappings). See also the\n  "total_ordering()" decorator.\n\n* The "hash()" result should be consistent with equality. Objects\n  that are equal should either have the same hash value, or be marked\n  as unhashable.\n\nPython does not enforce these consistency rules. In fact, the\nnot-a-number values are an example for not following these rules.\n\n\nMembership test operations\n==========================\n\nThe operators "in" and "not in" test for membership.  "x in s"\nevaluates to "True" if *x* is a member of *s*, and "False" otherwise.\n"x not in s" returns the negation of "x in s".  All built-in sequences\nand set types support this as well as dictionary, for which "in" tests\nwhether the dictionary has a given key. For container types such as\nlist, tuple, set, frozenset, dict, or collections.deque, the\nexpression "x in y" is equivalent to "any(x is e or x == e for e in\ny)".\n\nFor the string and bytes types, "x in y" is "True" if and only if *x*\nis a substring of *y*.  An equivalent test is "y.find(x) != -1".\nEmpty strings are always considered to be a substring of any other\nstring, so """ in "abc"" will return "True".\n\nFor user-defined classes which define the "__contains__()" method, "x\nin y" returns "True" if "y.__contains__(x)" returns a true value, and\n"False" otherwise.\n\nFor user-defined classes which do not define "__contains__()" but do\ndefine "__iter__()", "x in y" is "True" if some value "z" with "x ==\nz" is produced while iterating over "y".  If an exception is raised\nduring the iteration, it is as if "in" raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n"__getitem__()", "x in y" is "True" if and only if there is a non-\nnegative integer index *i* such that "x == y[i]", and all lower\ninteger indices do not raise "IndexError" exception.  (If any other\nexception is raised, it is as if "in" raised that exception).\n\nThe operator "not in" is defined to have the inverse true value of\n"in".\n\n\nIdentity comparisons\n====================\n\nThe operators "is" and "is not" test for object identity: "x is y" is\ntrue if and only if *x* and *y* are the same object.  Object identity\nis determined using the "id()" function.  "x is not y" yields the\ninverse truth value. [4]\n',
- 'compound': u'\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way.  In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe "if", "while" and "for" statements implement traditional control\nflow constructs.  "try" specifies exception handlers and/or cleanup\ncode for a group of statements, while the "with" statement allows the\nexecution of initialization and finalization code around a block of\ncode.  Function and class definitions are also syntactically compound\nstatements.\n\nA compound statement consists of one or more \'clauses.\'  A clause\nconsists of a header and a \'suite.\'  The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon.  A suite is a group of statements controlled by a\nclause.  A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines.  Only the latter form of a suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which "if" clause a following "else" clause would belong:\n\n   if test1: if test2: print(x)\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n"print()" calls are executed:\n\n   if x < y < z: print(x); print(y); print(z)\n\nSummarizing:\n\n   compound_stmt ::= if_stmt\n                     | while_stmt\n                     | for_stmt\n                     | try_stmt\n                     | with_stmt\n                     | funcdef\n                     | classdef\n                     | async_with_stmt\n                     | async_for_stmt\n                     | async_funcdef\n   suite         ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n   statement     ::= stmt_list NEWLINE | compound_stmt\n   stmt_list     ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a "NEWLINE" possibly followed by a\n"DEDENT".  Also note that optional continuation clauses always begin\nwith a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling "else"\' problem is solved in Python by\nrequiring nested "if" statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe "if" statement\n==================\n\nThe "if" statement is used for conditional execution:\n\n   if_stmt ::= "if" expression ":" suite\n               ( "elif" expression ":" suite )*\n               ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the "if" statement is executed or evaluated).\nIf all expressions are false, the suite of the "else" clause, if\npresent, is executed.\n\n\nThe "while" statement\n=====================\n\nThe "while" statement is used for repeated execution as long as an\nexpression is true:\n\n   while_stmt ::= "while" expression ":" suite\n                  ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the "else" clause, if present, is executed\nand the loop terminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite.  A "continue" statement\nexecuted in the first suite skips the rest of the suite and goes back\nto testing the expression.\n\n\nThe "for" statement\n===================\n\nThe "for" statement is used to iterate over the elements of a sequence\n(such as a string, tuple or list) or other iterable object:\n\n   for_stmt ::= "for" target_list "in" expression_list ":" suite\n                ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject.  An iterator is created for the result of the\n"expression_list".  The suite is then executed once for each item\nprovided by the iterator, in the order returned by the iterator.  Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments (see *Assignment statements*), and then the suite is\nexecuted.  When the items are exhausted (which is immediately when the\nsequence is empty or an iterator raises a "StopIteration" exception),\nthe suite in the "else" clause, if present, is executed, and the loop\nterminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite.  A "continue" statement\nexecuted in the first suite skips the rest of the suite and continues\nwith the next item, or with the "else" clause if there is no next\nitem.\n\nThe for-loop makes assignments to the variables(s) in the target list.\nThis overwrites all previous assignments to those variables including\nthose made in the suite of the for-loop:\n\n   for i in range(10):\n       print(i)\n       i = 5             # this will not affect the for-loop\n                         # because i will be overwritten with the next\n                         # index in the range\n\nNames in the target list are not deleted when the loop is finished,\nbut if the sequence is empty, they will not have been assigned to at\nall by the loop.  Hint: the built-in function "range()" returns an\niterator of integers suitable to emulate the effect of Pascal\'s "for i\n:= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n\nNote: There is a subtlety when the sequence is being modified by the\n  loop (this can only occur for mutable sequences, i.e. lists).  An\n  internal counter is used to keep track of which item is used next,\n  and this is incremented on each iteration.  When this counter has\n  reached the length of the sequence the loop terminates.  This means\n  that if the suite deletes the current (or a previous) item from the\n  sequence, the next item will be skipped (since it gets the index of\n  the current item which has already been treated).  Likewise, if the\n  suite inserts an item in the sequence before the current item, the\n  current item will be treated again the next time through the loop.\n  This can lead to nasty bugs that can be avoided by making a\n  temporary copy using a slice of the whole sequence, e.g.,\n\n     for x in a[:]:\n         if x < 0: a.remove(x)\n\n\nThe "try" statement\n===================\n\nThe "try" statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n   try_stmt  ::= try1_stmt | try2_stmt\n   try1_stmt ::= "try" ":" suite\n                 ("except" [expression ["as" identifier]] ":" suite)+\n                 ["else" ":" suite]\n                 ["finally" ":" suite]\n   try2_stmt ::= "try" ":" suite\n                 "finally" ":" suite\n\nThe "except" clause(s) specify one or more exception handlers. When no\nexception occurs in the "try" clause, no exception handler is\nexecuted. When an exception occurs in the "try" suite, a search for an\nexception handler is started.  This search inspects the except clauses\nin turn until one is found that matches the exception.  An expression-\nless except clause, if present, must be last; it matches any\nexception.  For an except clause with an expression, that expression\nis evaluated, and the clause matches the exception if the resulting\nobject is "compatible" with the exception.  An object is compatible\nwith an exception if it is the class or a base class of the exception\nobject or a tuple containing an item compatible with the exception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire "try" statement raised\nthe exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the "as" keyword in that except clause, if\npresent, and the except clause\'s suite is executed.  All except\nclauses must have an executable block.  When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using "as target", it is cleared\nat the end of the except clause.  This is as if\n\n   except E as N:\n       foo\n\nwas translated to\n\n   except E as N:\n       try:\n           foo\n       finally:\n           del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause.  Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the "sys" module and can be accessed via\n"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of the\nexception class, the exception instance and a traceback object (see\nsection *The standard type hierarchy*) identifying the point in the\nprogram where the exception occurred.  "sys.exc_info()" values are\nrestored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional "else" clause is executed if and when control flows off\nthe end of the "try" clause. [2] Exceptions in the "else" clause are\nnot handled by the preceding "except" clauses.\n\nIf "finally" is present, it specifies a \'cleanup\' handler.  The "try"\nclause is executed, including any "except" and "else" clauses.  If an\nexception occurs in any of the clauses and is not handled, the\nexception is temporarily saved. The "finally" clause is executed.  If\nthere is a saved exception it is re-raised at the end of the "finally"\nclause.  If the "finally" clause raises another exception, the saved\nexception is set as the context of the new exception. If the "finally"\nclause executes a "return" or "break" statement, the saved exception\nis discarded:\n\n   >>> def f():\n   ...     try:\n   ...         1/0\n   ...     finally:\n   ...         return 42\n   ...\n   >>> f()\n   42\n\nThe exception information is not available to the program during\nexecution of the "finally" clause.\n\nWhen a "return", "break" or "continue" statement is executed in the\n"try" suite of a "try"..."finally" statement, the "finally" clause is\nalso executed \'on the way out.\' A "continue" statement is illegal in\nthe "finally" clause. (The reason is a problem with the current\nimplementation --- this restriction may be lifted in the future).\n\nThe return value of a function is determined by the last "return"\nstatement executed.  Since the "finally" clause always executes, a\n"return" statement executed in the "finally" clause will always be the\nlast one executed:\n\n   >>> def foo():\n   ...     try:\n   ...         return \'try\'\n   ...     finally:\n   ...         return \'finally\'\n   ...\n   >>> foo()\n   \'finally\'\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the "raise" statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe "with" statement\n====================\n\nThe "with" statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common "try"..."except"..."finally"\nusage patterns to be encapsulated for convenient reuse.\n\n   with_stmt ::= "with" with_item ("," with_item)* ":" suite\n   with_item ::= expression ["as" target]\n\nThe execution of the "with" statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the "with_item")\n   is evaluated to obtain a context manager.\n\n2. The context manager\'s "__exit__()" is loaded for later use.\n\n3. The context manager\'s "__enter__()" method is invoked.\n\n4. If a target was included in the "with" statement, the return\n   value from "__enter__()" is assigned to it.\n\n   Note: The "with" statement guarantees that if the "__enter__()"\n     method returns without an error, then "__exit__()" will always be\n     called. Thus, if an error occurs during the assignment to the\n     target list, it will be treated the same as an error occurring\n     within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s "__exit__()" method is invoked.  If an\n   exception caused the suite to be exited, its type, value, and\n   traceback are passed as arguments to "__exit__()". Otherwise, three\n   "None" arguments are supplied.\n\n   If the suite was exited due to an exception, and the return value\n   from the "__exit__()" method was false, the exception is reraised.\n   If the return value was true, the exception is suppressed, and\n   execution continues with the statement following the "with"\n   statement.\n\n   If the suite was exited for any reason other than an exception, the\n   return value from "__exit__()" is ignored, and execution proceeds\n   at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple "with" statements were nested:\n\n   with A() as a, B() as b:\n       suite\n\nis equivalent to\n\n   with A() as a:\n       with B() as b:\n           suite\n\nChanged in version 3.1: Support for multiple context expressions.\n\nSee also: **PEP 343** - The "with" statement\n\n     The specification, background, and examples for the Python "with"\n     statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n   funcdef        ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n   decorators     ::= decorator+\n   decorator      ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n   dotted_name    ::= identifier ("." identifier)*\n   parameter_list ::= (defparameter ",")*\n                      | "*" [parameter] ("," defparameter)* ["," "**" parameter]\n                      | "**" parameter\n                      | defparameter [","] )\n   parameter      ::= identifier [":" expression]\n   defparameter   ::= parameter ["=" expression]\n   funcname       ::= identifier\n\nA function definition is an executable statement.  Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function).  This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition.  The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object.  Multiple decorators are applied in\nnested fashion. For example, the following code\n\n   @f1(arg)\n   @f2\n   def func(): pass\n\nis roughly equivalent to\n\n   def func(): pass\n   func = f1(arg)(f2(func))\n\nexcept that the original function is not temporarily bound to the name\n"func".\n\nWhen one or more *parameters* have the form *parameter* "="\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding *argument* may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted.  If a parameter has a default value, all following\nparameters up until the ""*"" must also have a default value --- this\nis a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated from left to right when the\nfunction definition is executed.** This means that the expression is\nevaluated once, when the function is defined, and that the same "pre-\ncomputed" value is used for each call.  This is especially important\nto understand when a default parameter is a mutable object, such as a\nlist or a dictionary: if the function modifies the object (e.g. by\nappending an item to a list), the default value is in effect modified.\nThis is generally not what was intended.  A way around this is to use\n"None" as the default, and explicitly test for it in the body of the\nfunction, e.g.:\n\n   def whats_on_the_telly(penguin=None):\n       if penguin is None:\n           penguin = []\n       penguin.append("property of the zoo")\n       return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values.  If the form\n""*identifier"" is present, it is initialized to a tuple receiving any\nexcess positional parameters, defaulting to the empty tuple.  If the\nform ""**identifier"" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after ""*"" or ""*identifier"" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "": expression"" following\nthe parameter name.  Any parameter may have an annotation even those\nof the form "*identifier" or "**identifier".  Functions may have\n"return" annotation of the form ""-> expression"" after the parameter\nlist.  These annotations can be any valid Python expression and are\nevaluated when the function definition is executed.  Annotations may\nbe evaluated in a different order than they appear in the source code.\nThe presence of annotations does not change the semantics of a\nfunction.  The annotation values are available as values of a\ndictionary keyed by the parameters\' names in the "__annotations__"\nattribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions.  This uses lambda\nexpressions, described in section *Lambdas*.  Note that the lambda\nexpression is merely a shorthand for a simplified function definition;\na function defined in a ""def"" statement can be passed around or\nassigned to another name just like a function defined by a lambda\nexpression.  The ""def"" form is actually more powerful since it\nallows the execution of multiple statements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects.  A ""def""\nstatement executed inside a function definition defines a local\nfunction that can be returned or passed around.  Free variables used\nin the nested function can access the local variables of the function\ncontaining the def.  See section *Naming and binding* for details.\n\nSee also: **PEP 3107** - Function Annotations\n\n     The original specification for function annotations.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n   classdef    ::= [decorators] "class" classname [inheritance] ":" suite\n   inheritance ::= "(" [argument_list] ")"\n   classname   ::= identifier\n\nA class definition is an executable statement.  The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing.  Classes without an inheritance\nlist inherit, by default, from the base class "object"; hence,\n\n   class Foo:\n       pass\n\nis equivalent to\n\n   class Foo(object):\n       pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.)  When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n   @f1(arg)\n   @f2\n   class Foo: pass\n\nis roughly equivalent to\n\n   class Foo: pass\n   Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators.  The result is then bound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances.  Instance attributes\ncan be set in a method with "self.name = value".  Both class and\ninstance attributes are accessible through the notation ""self.name"",\nand an instance attribute hides a class attribute with the same name\nwhen accessed in this way.  Class attributes can be used as defaults\nfor instance attributes, but using mutable values there can lead to\nunexpected results.  *Descriptors* can be used to create instance\nvariables with different implementation details.\n\nSee also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n  Class Decorators\n\n\nCoroutines\n==========\n\nNew in version 3.5.\n\n\nCoroutine function definition\n-----------------------------\n\n   async_funcdef ::= [decorators] "async" "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n\nExecution of Python coroutines can be suspended and resumed at many\npoints (see *coroutine*).  In the body of a coroutine, any "await" and\n"async" identifiers become reserved keywords; "await" expressions,\n"async for" and "async with" can only be used in coroutine bodies.\n\nFunctions defined with "async def" syntax are always coroutine\nfunctions, even if they do not contain "await" or "async" keywords.\n\nIt is a "SyntaxError" to use "yield" expressions in "async def"\ncoroutines.\n\nAn example of a coroutine function:\n\n   async def func(param1, param2):\n       do_stuff()\n       await some_coroutine()\n\n\nThe "async for" statement\n-------------------------\n\n   async_for_stmt ::= "async" for_stmt\n\nAn *asynchronous iterable* is able to call asynchronous code in its\n*iter* implementation, and *asynchronous iterator* can call\nasynchronous code in its *next* method.\n\nThe "async for" statement allows convenient iteration over\nasynchronous iterators.\n\nThe following code:\n\n   async for TARGET in ITER:\n       BLOCK\n   else:\n       BLOCK2\n\nIs semantically equivalent to:\n\n   iter = (ITER)\n   iter = type(iter).__aiter__(iter)\n   running = True\n   while running:\n       try:\n           TARGET = await type(iter).__anext__(iter)\n       except StopAsyncIteration:\n           running = False\n       else:\n           BLOCK\n   else:\n       BLOCK2\n\nSee also "__aiter__()" and "__anext__()" for details.\n\nIt is a "SyntaxError" to use "async for" statement outside of an\n"async def" function.\n\n\nThe "async with" statement\n--------------------------\n\n   async_with_stmt ::= "async" with_stmt\n\nAn *asynchronous context manager* is a *context manager* that is able\nto suspend execution in its *enter* and *exit* methods.\n\nThe following code:\n\n   async with EXPR as VAR:\n       BLOCK\n\nIs semantically equivalent to:\n\n   mgr = (EXPR)\n   aexit = type(mgr).__aexit__\n   aenter = type(mgr).__aenter__(mgr)\n   exc = True\n\n   VAR = await aenter\n   try:\n       BLOCK\n   except:\n       if not await aexit(mgr, *sys.exc_info()):\n           raise\n   else:\n       await aexit(mgr, None, None, None)\n\nSee also "__aenter__()" and "__aexit__()" for details.\n\nIt is a "SyntaxError" to use "async with" statement outside of an\n"async def" function.\n\nSee also: **PEP 492** - Coroutines with async and await syntax\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack unless\n    there is a "finally" clause which happens to raise another\n    exception. That new exception causes the old one to be lost.\n\n[2] Currently, control "flows off the end" except in the case of\n    an exception or the execution of a "return", "continue", or\n    "break" statement.\n\n[3] A string literal appearing as the first statement in the\n    function body is transformed into the function\'s "__doc__"\n    attribute and therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n    body is transformed into the namespace\'s "__doc__" item and\n    therefore the class\'s *docstring*.\n',
- 'context-managers': u'\nWith Statement Context Managers\n*******************************\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a "with" statement. The context manager\nhandles the entry into, and the exit from, the desired runtime context\nfor the execution of the block of code.  Context managers are normally\ninvoked using the "with" statement (described in section *The with\nstatement*), but can also be used by directly invoking their methods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n   Enter the runtime context related to this object. The "with"\n   statement will bind this method\'s return value to the target(s)\n   specified in the "as" clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n   Exit the runtime context related to this object. The parameters\n   describe the exception that caused the context to be exited. If the\n   context was exited without an exception, all three arguments will\n   be "None".\n\n   If an exception is supplied, and the method wishes to suppress the\n   exception (i.e., prevent it from being propagated), it should\n   return a true value. Otherwise, the exception will be processed\n   normally upon exit from this method.\n\n   Note that "__exit__()" methods should not reraise the passed-in\n   exception; this is the caller\'s responsibility.\n\nSee also: **PEP 343** - The "with" statement\n\n     The specification, background, and examples for the Python "with"\n     statement.\n',
- 'continue': u'\nThe "continue" statement\n************************\n\n   continue_stmt ::= "continue"\n\n"continue" may only occur syntactically nested in a "for" or "while"\nloop, but not nested in a function or class definition or "finally"\nclause within that loop.  It continues with the next cycle of the\nnearest enclosing loop.\n\nWhen "continue" passes control out of a "try" statement with a\n"finally" clause, that "finally" clause is executed before really\nstarting the next loop cycle.\n',
- 'conversions': u'\nArithmetic conversions\n**********************\n\nWhen a description of an arithmetic operator below uses the phrase\n"the numeric arguments are converted to a common type," this means\nthat the operator implementation for built-in types works as follows:\n\n* If either argument is a complex number, the other is converted to\n  complex;\n\n* otherwise, if either argument is a floating point number, the\n  other is converted to floating point;\n\n* otherwise, both must be integers and no conversion is necessary.\n\nSome additional rules apply for certain operators (e.g., a string as a\nleft argument to the \'%\' operator).  Extensions must define their own\nconversion behavior.\n',
- 'customization': u'\nBasic customization\n*******************\n\nobject.__new__(cls[, ...])\n\n   Called to create a new instance of class *cls*.  "__new__()" is a\n   static method (special-cased so you need not declare it as such)\n   that takes the class of which an instance was requested as its\n   first argument.  The remaining arguments are those passed to the\n   object constructor expression (the call to the class).  The return\n   value of "__new__()" should be the new object instance (usually an\n   instance of *cls*).\n\n   Typical implementations create a new instance of the class by\n   invoking the superclass\'s "__new__()" method using\n   "super().__new__(cls[, ...])" with appropriate arguments and then\n   modifying the newly-created instance as necessary before returning\n   it.\n\n   If "__new__()" returns an instance of *cls*, then the new\n   instance\'s "__init__()" method will be invoked like\n   "__init__(self[, ...])", where *self* is the new instance and the\n   remaining arguments are the same as were passed to "__new__()".\n\n   If "__new__()" does not return an instance of *cls*, then the new\n   instance\'s "__init__()" method will not be invoked.\n\n   "__new__()" is intended mainly to allow subclasses of immutable\n   types (like int, str, or tuple) to customize instance creation.  It\n   is also commonly overridden in custom metaclasses in order to\n   customize class creation.\n\nobject.__init__(self[, ...])\n\n   Called after the instance has been created (by "__new__()"), but\n   before it is returned to the caller.  The arguments are those\n   passed to the class constructor expression.  If a base class has an\n   "__init__()" method, the derived class\'s "__init__()" method, if\n   any, must explicitly call it to ensure proper initialization of the\n   base class part of the instance; for example:\n   "super().__init__([args...])".\n\n   Because "__new__()" and "__init__()" work together in constructing\n   objects ("__new__()" to create it, and "__init__()" to customize\n   it), no non-"None" value may be returned by "__init__()"; doing so\n   will cause a "TypeError" to be raised at runtime.\n\nobject.__del__(self)\n\n   Called when the instance is about to be destroyed.  This is also\n   called a destructor.  If a base class has a "__del__()" method, the\n   derived class\'s "__del__()" method, if any, must explicitly call it\n   to ensure proper deletion of the base class part of the instance.\n   Note that it is possible (though not recommended!) for the\n   "__del__()" method to postpone destruction of the instance by\n   creating a new reference to it.  It may then be called at a later\n   time when this new reference is deleted.  It is not guaranteed that\n   "__del__()" methods are called for objects that still exist when\n   the interpreter exits.\n\n   Note: "del x" doesn\'t directly call "x.__del__()" --- the former\n     decrements the reference count for "x" by one, and the latter is\n     only called when "x"\'s reference count reaches zero.  Some common\n     situations that may prevent the reference count of an object from\n     going to zero include: circular references between objects (e.g.,\n     a doubly-linked list or a tree data structure with parent and\n     child pointers); a reference to the object on the stack frame of\n     a function that caught an exception (the traceback stored in\n     "sys.exc_info()[2]" keeps the stack frame alive); or a reference\n     to the object on the stack frame that raised an unhandled\n     exception in interactive mode (the traceback stored in\n     "sys.last_traceback" keeps the stack frame alive).  The first\n     situation can only be remedied by explicitly breaking the cycles;\n     the second can be resolved by freeing the reference to the\n     traceback object when it is no longer useful, and the third can\n     be resolved by storing "None" in "sys.last_traceback". Circular\n     references which are garbage are detected and cleaned up when the\n     cyclic garbage collector is enabled (it\'s on by default). Refer\n     to the documentation for the "gc" module for more information\n     about this topic.\n\n   Warning: Due to the precarious circumstances under which\n     "__del__()" methods are invoked, exceptions that occur during\n     their execution are ignored, and a warning is printed to\n     "sys.stderr" instead. Also, when "__del__()" is invoked in\n     response to a module being deleted (e.g., when execution of the\n     program is done), other globals referenced by the "__del__()"\n     method may already have been deleted or in the process of being\n     torn down (e.g. the import machinery shutting down).  For this\n     reason, "__del__()" methods should do the absolute minimum needed\n     to maintain external invariants.  Starting with version 1.5,\n     Python guarantees that globals whose name begins with a single\n     underscore are deleted from their module before other globals are\n     deleted; if no other references to such globals exist, this may\n     help in assuring that imported modules are still available at the\n     time when the "__del__()" method is called.\n\nobject.__repr__(self)\n\n   Called by the "repr()" built-in function to compute the "official"\n   string representation of an object.  If at all possible, this\n   should look like a valid Python expression that could be used to\n   recreate an object with the same value (given an appropriate\n   environment).  If this is not possible, a string of the form\n   "<...some useful description...>" should be returned. The return\n   value must be a string object. If a class defines "__repr__()" but\n   not "__str__()", then "__repr__()" is also used when an "informal"\n   string representation of instances of that class is required.\n\n   This is typically used for debugging, so it is important that the\n   representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n   Called by "str(object)" and the built-in functions "format()" and\n   "print()" to compute the "informal" or nicely printable string\n   representation of an object.  The return value must be a *string*\n   object.\n\n   This method differs from "object.__repr__()" in that there is no\n   expectation that "__str__()" return a valid Python expression: a\n   more convenient or concise representation can be used.\n\n   The default implementation defined by the built-in type "object"\n   calls "object.__repr__()".\n\nobject.__bytes__(self)\n\n   Called by "bytes()" to compute a byte-string representation of an\n   object. This should return a "bytes" object.\n\nobject.__format__(self, format_spec)\n\n   Called by the "format()" built-in function (and by extension, the\n   "str.format()" method of class "str") to produce a "formatted"\n   string representation of an object. The "format_spec" argument is a\n   string that contains a description of the formatting options\n   desired. The interpretation of the "format_spec" argument is up to\n   the type implementing "__format__()", however most classes will\n   either delegate formatting to one of the built-in types, or use a\n   similar formatting option syntax.\n\n   See *Format Specification Mini-Language* for a description of the\n   standard formatting syntax.\n\n   The return value must be a string object.\n\n   Changed in version 3.4: The __format__ method of "object" itself\n   raises a "TypeError" if passed any non-empty string.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n   These are the so-called "rich comparison" methods. The\n   correspondence between operator symbols and method names is as\n   follows: "x<y" calls "x.__lt__(y)", "x<=y" calls "x.__le__(y)",\n   "x==y" calls "x.__eq__(y)", "x!=y" calls "x.__ne__(y)", "x>y" calls\n   "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n\n   A rich comparison method may return the singleton "NotImplemented"\n   if it does not implement the operation for a given pair of\n   arguments. By convention, "False" and "True" are returned for a\n   successful comparison. However, these methods can return any value,\n   so if the comparison operator is used in a Boolean context (e.g.,\n   in the condition of an "if" statement), Python will call "bool()"\n   on the value to determine if the result is true or false.\n\n   By default, "__ne__()" delegates to "__eq__()" and inverts the\n   result unless it is "NotImplemented".  There are no other implied\n   relationships among the comparison operators, for example, the\n   truth of "(x<y or x==y)" does not imply "x<=y". To automatically\n   generate ordering operations from a single root operation, see\n   "functools.total_ordering()".\n\n   See the paragraph on "__hash__()" for some important notes on\n   creating *hashable* objects which support custom comparison\n   operations and are usable as dictionary keys.\n\n   There are no swapped-argument versions of these methods (to be used\n   when the left argument does not support the operation but the right\n   argument does); rather, "__lt__()" and "__gt__()" are each other\'s\n   reflection, "__le__()" and "__ge__()" are each other\'s reflection,\n   and "__eq__()" and "__ne__()" are their own reflection. If the\n   operands are of different types, and right operand\'s type is a\n   direct or indirect subclass of the left operand\'s type, the\n   reflected method of the right operand has priority, otherwise the\n   left operand\'s method has priority.  Virtual subclassing is not\n   considered.\n\nobject.__hash__(self)\n\n   Called by built-in function "hash()" and for operations on members\n   of hashed collections including "set", "frozenset", and "dict".\n   "__hash__()" should return an integer. The only required property\n   is that objects which compare equal have the same hash value; it is\n   advised to mix together the hash values of the components of the\n   object that also play a part in comparison of objects by packing\n   them into a tuple and hashing the tuple. Example:\n\n      def __hash__(self):\n          return hash((self.name, self.nick, self.color))\n\n   Note: "hash()" truncates the value returned from an object\'s\n     custom "__hash__()" method to the size of a "Py_ssize_t".  This\n     is typically 8 bytes on 64-bit builds and 4 bytes on 32-bit\n     builds. If an object\'s   "__hash__()" must interoperate on builds\n     of different bit sizes, be sure to check the width on all\n     supported builds.  An easy way to do this is with "python -c\n     "import sys; print(sys.hash_info.width)"".\n\n   If a class does not define an "__eq__()" method it should not\n   define a "__hash__()" operation either; if it defines "__eq__()"\n   but not "__hash__()", its instances will not be usable as items in\n   hashable collections.  If a class defines mutable objects and\n   implements an "__eq__()" method, it should not implement\n   "__hash__()", since the implementation of hashable collections\n   requires that a key\'s hash value is immutable (if the object\'s hash\n   value changes, it will be in the wrong hash bucket).\n\n   User-defined classes have "__eq__()" and "__hash__()" methods by\n   default; with them, all objects compare unequal (except with\n   themselves) and "x.__hash__()" returns an appropriate value such\n   that "x == y" implies both that "x is y" and "hash(x) == hash(y)".\n\n   A class that overrides "__eq__()" and does not define "__hash__()"\n   will have its "__hash__()" implicitly set to "None".  When the\n   "__hash__()" method of a class is "None", instances of the class\n   will raise an appropriate "TypeError" when a program attempts to\n   retrieve their hash value, and will also be correctly identified as\n   unhashable when checking "isinstance(obj, collections.Hashable)".\n\n   If a class that overrides "__eq__()" needs to retain the\n   implementation of "__hash__()" from a parent class, the interpreter\n   must be told this explicitly by setting "__hash__ =\n   <ParentClass>.__hash__".\n\n   If a class that does not override "__eq__()" wishes to suppress\n   hash support, it should include "__hash__ = None" in the class\n   definition. A class which defines its own "__hash__()" that\n   explicitly raises a "TypeError" would be incorrectly identified as\n   hashable by an "isinstance(obj, collections.Hashable)" call.\n\n   Note: By default, the "__hash__()" values of str, bytes and\n     datetime objects are "salted" with an unpredictable random value.\n     Although they remain constant within an individual Python\n     process, they are not predictable between repeated invocations of\n     Python.This is intended to provide protection against a denial-\n     of-service caused by carefully-chosen inputs that exploit the\n     worst case performance of a dict insertion, O(n^2) complexity.\n     See http://www.ocert.org/advisories/ocert-2011-003.html for\n     details.Changing hash values affects the iteration order of\n     dicts, sets and other mappings.  Python has never made guarantees\n     about this ordering (and it typically varies between 32-bit and\n     64-bit builds).See also "PYTHONHASHSEED".\n\n   Changed in version 3.3: Hash randomization is enabled by default.\n\nobject.__bool__(self)\n\n   Called to implement truth value testing and the built-in operation\n   "bool()"; should return "False" or "True".  When this method is not\n   defined, "__len__()" is called, if it is defined, and the object is\n   considered true if its result is nonzero.  If a class defines\n   neither "__len__()" nor "__bool__()", all its instances are\n   considered true.\n',
- 'debugger': u'\n"pdb" --- The Python Debugger\n*****************************\n\n**Source code:** Lib/pdb.py\n\n======================================================================\n\nThe module "pdb" defines an interactive source code debugger for\nPython programs.  It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame.  It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible -- it is actually defined as the class\n"Pdb". This is currently undocumented but easily understood by reading\nthe source.  The extension interface uses the modules "bdb" and "cmd".\n\nThe debugger\'s prompt is "(Pdb)". Typical usage to run a program under\ncontrol of the debugger is:\n\n   >>> import pdb\n   >>> import mymodule\n   >>> pdb.run(\'mymodule.test()\')\n   > <string>(0)?()\n   (Pdb) continue\n   > <string>(1)?()\n   (Pdb) continue\n   NameError: \'spam\'\n   > <string>(1)?()\n   (Pdb)\n\nChanged in version 3.3: Tab-completion via the "readline" module is\navailable for commands and command arguments, e.g. the current global\nand local names are offered as arguments of the "p" command.\n\n"pdb.py" can also be invoked as a script to debug other scripts.  For\nexample:\n\n   python3 -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally.  After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program.  Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 3.2: "pdb.py" now accepts a "-c" option that executes\ncommands as if given in a ".pdbrc" file, see *Debugger Commands*.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n   import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger.  You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the "continue" command.\n\nThe typical usage to inspect a crashed program is:\n\n   >>> import pdb\n   >>> import mymodule\n   >>> mymodule.test()\n   Traceback (most recent call last):\n     File "<stdin>", line 1, in <module>\n     File "./mymodule.py", line 4, in test\n       test2()\n     File "./mymodule.py", line 3, in test2\n       print(spam)\n   NameError: spam\n   >>> pdb.pm()\n   > ./mymodule.py(3)test2()\n   -> print(spam)\n   (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement, globals=None, locals=None)\n\n   Execute the *statement* (given as a string or a code object) under\n   debugger control.  The debugger prompt appears before any code is\n   executed; you can set breakpoints and type "continue", or you can\n   step through the statement using "step" or "next" (all these\n   commands are explained below).  The optional *globals* and *locals*\n   arguments specify the environment in which the code is executed; by\n   default the dictionary of the module "__main__" is used.  (See the\n   explanation of the built-in "exec()" or "eval()" functions.)\n\npdb.runeval(expression, globals=None, locals=None)\n\n   Evaluate the *expression* (given as a string or a code object)\n   under debugger control.  When "runeval()" returns, it returns the\n   value of the expression.  Otherwise this function is similar to\n   "run()".\n\npdb.runcall(function, *args, **kwds)\n\n   Call the *function* (a function or method object, not a string)\n   with the given arguments.  When "runcall()" returns, it returns\n   whatever the function call returned.  The debugger prompt appears\n   as soon as the function is entered.\n\npdb.set_trace()\n\n   Enter the debugger at the calling stack frame.  This is useful to\n   hard-code a breakpoint at a given point in a program, even if the\n   code is not otherwise being debugged (e.g. when an assertion\n   fails).\n\npdb.post_mortem(traceback=None)\n\n   Enter post-mortem debugging of the given *traceback* object.  If no\n   *traceback* is given, it uses the one of the exception that is\n   currently being handled (an exception must be being handled if the\n   default is to be used).\n\npdb.pm()\n\n   Enter post-mortem debugging of the traceback found in\n   "sys.last_traceback".\n\nThe "run*" functions and "set_trace()" are aliases for instantiating\nthe "Pdb" class and calling the method of the same name.  If you want\nto access further features, you have to do this yourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None, nosigint=False)\n\n   "Pdb" is the debugger class.\n\n   The *completekey*, *stdin* and *stdout* arguments are passed to the\n   underlying "cmd.Cmd" class; see the description there.\n\n   The *skip* argument, if given, must be an iterable of glob-style\n   module name patterns.  The debugger will not step into frames that\n   originate in a module that matches one of these patterns. [1]\n\n   By default, Pdb sets a handler for the SIGINT signal (which is sent\n   when the user presses "Ctrl-C" on the console) when you give a\n   "continue" command. This allows you to break into the debugger\n   again by pressing "Ctrl-C".  If you want Pdb not to touch the\n   SIGINT handler, set *nosigint* to true.\n\n   Example call to enable tracing with *skip*:\n\n      import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n   New in version 3.1: The *skip* argument.\n\n   New in version 3.2: The *nosigint* argument.  Previously, a SIGINT\n   handler was never set by Pdb.\n\n   run(statement, globals=None, locals=None)\n   runeval(expression, globals=None, locals=None)\n   runcall(function, *args, **kwds)\n   set_trace()\n\n      See the documentation for the functions explained above.\n\n\nDebugger Commands\n=================\n\nThe commands recognized by the debugger are listed below.  Most\ncommands can be abbreviated to one or two letters as indicated; e.g.\n"h(elp)" means that either "h" or "help" can be used to enter the help\ncommand (but not "he" or "hel", nor "H" or "Help" or "HELP").\nArguments to commands must be separated by whitespace (spaces or\ntabs).  Optional arguments are enclosed in square brackets ("[]") in\nthe command syntax; the square brackets must not be typed.\nAlternatives in the command syntax are separated by a vertical bar\n("|").\n\nEntering a blank line repeats the last command entered.  Exception: if\nthe last command was a "list" command, the next 11 lines are listed.\n\nCommands that the debugger doesn\'t recognize are assumed to be Python\nstatements and are executed in the context of the program being\ndebugged.  Python statements can also be prefixed with an exclamation\npoint ("!").  This is a powerful way to inspect the program being\ndebugged; it is even possible to change a variable or call a function.\nWhen an exception occurs in such a statement, the exception name is\nprinted but the debugger\'s state is not changed.\n\nThe debugger supports *aliases*.  Aliases can have parameters which\nallows one a certain level of adaptability to the context under\nexamination.\n\nMultiple commands may be entered on a single line, separated by ";;".\n(A single ";" is not used as it is the separator for multiple commands\nin a line that is passed to the Python parser.)  No intelligence is\napplied to separating the commands; the input is split at the first\n";;" pair, even if it is in the middle of a quoted string.\n\nIf a file ".pdbrc" exists in the user\'s home directory or in the\ncurrent directory, it is read in and executed as if it had been typed\nat the debugger prompt.  This is particularly useful for aliases.  If\nboth files exist, the one in the home directory is read first and\naliases defined there can be overridden by the local file.\n\nChanged in version 3.2: ".pdbrc" can now contain commands that\ncontinue debugging, such as "continue" or "next".  Previously, these\ncommands had no effect.\n\nh(elp) [command]\n\n   Without argument, print the list of available commands.  With a\n   *command* as argument, print help about that command.  "help pdb"\n   displays the full documentation (the docstring of the "pdb"\n   module).  Since the *command* argument must be an identifier, "help\n   exec" must be entered to get help on the "!" command.\n\nw(here)\n\n   Print a stack trace, with the most recent frame at the bottom.  An\n   arrow indicates the current frame, which determines the context of\n   most commands.\n\nd(own) [count]\n\n   Move the current frame *count* (default one) levels down in the\n   stack trace (to a newer frame).\n\nu(p) [count]\n\n   Move the current frame *count* (default one) levels up in the stack\n   trace (to an older frame).\n\nb(reak) [([filename:]lineno | function) [, condition]]\n\n   With a *lineno* argument, set a break there in the current file.\n   With a *function* argument, set a break at the first executable\n   statement within that function.  The line number may be prefixed\n   with a filename and a colon, to specify a breakpoint in another\n   file (probably one that hasn\'t been loaded yet).  The file is\n   searched on "sys.path".  Note that each breakpoint is assigned a\n   number to which all the other breakpoint commands refer.\n\n   If a second argument is present, it is an expression which must\n   evaluate to true before the breakpoint is honored.\n\n   Without argument, list all breaks, including for each breakpoint,\n   the number of times that breakpoint has been hit, the current\n   ignore count, and the associated condition if any.\n\ntbreak [([filename:]lineno | function) [, condition]]\n\n   Temporary breakpoint, which is removed automatically when it is\n   first hit. The arguments are the same as for "break".\n\ncl(ear) [filename:lineno | bpnumber [bpnumber ...]]\n\n   With a *filename:lineno* argument, clear all the breakpoints at\n   this line. With a space separated list of breakpoint numbers, clear\n   those breakpoints. Without argument, clear all breaks (but first\n   ask confirmation).\n\ndisable [bpnumber [bpnumber ...]]\n\n   Disable the breakpoints given as a space separated list of\n   breakpoint numbers.  Disabling a breakpoint means it cannot cause\n   the program to stop execution, but unlike clearing a breakpoint, it\n   remains in the list of breakpoints and can be (re-)enabled.\n\nenable [bpnumber [bpnumber ...]]\n\n   Enable the breakpoints specified.\n\nignore bpnumber [count]\n\n   Set the ignore count for the given breakpoint number.  If count is\n   omitted, the ignore count is set to 0.  A breakpoint becomes active\n   when the ignore count is zero.  When non-zero, the count is\n   decremented each time the breakpoint is reached and the breakpoint\n   is not disabled and any associated condition evaluates to true.\n\ncondition bpnumber [condition]\n\n   Set a new *condition* for the breakpoint, an expression which must\n   evaluate to true before the breakpoint is honored.  If *condition*\n   is absent, any existing condition is removed; i.e., the breakpoint\n   is made unconditional.\n\ncommands [bpnumber]\n\n   Specify a list of commands for breakpoint number *bpnumber*.  The\n   commands themselves appear on the following lines.  Type a line\n   containing just "end" to terminate the commands. An example:\n\n      (Pdb) commands 1\n      (com) p some_variable\n      (com) end\n      (Pdb)\n\n   To remove all commands from a breakpoint, type commands and follow\n   it immediately with "end"; that is, give no commands.\n\n   With no *bpnumber* argument, commands refers to the last breakpoint\n   set.\n\n   You can use breakpoint commands to start your program up again.\n   Simply use the continue command, or step, or any other command that\n   resumes execution.\n\n   Specifying any command resuming execution (currently continue,\n   step, next, return, jump, quit and their abbreviations) terminates\n   the command list (as if that command was immediately followed by\n   end). This is because any time you resume execution (even with a\n   simple next or step), you may encounter another breakpoint\u2014which\n   could have its own command list, leading to ambiguities about which\n   list to execute.\n\n   If you use the \'silent\' command in the command list, the usual\n   message about stopping at a breakpoint is not printed.  This may be\n   desirable for breakpoints that are to print a specific message and\n   then continue.  If none of the other commands print anything, you\n   see no sign that the breakpoint was reached.\n\ns(tep)\n\n   Execute the current line, stop at the first possible occasion\n   (either in a function that is called or on the next line in the\n   current function).\n\nn(ext)\n\n   Continue execution until the next line in the current function is\n   reached or it returns.  (The difference between "next" and "step"\n   is that "step" stops inside a called function, while "next"\n   executes called functions at (nearly) full speed, only stopping at\n   the next line in the current function.)\n\nunt(il) [lineno]\n\n   Without argument, continue execution until the line with a number\n   greater than the current one is reached.\n\n   With a line number, continue execution until a line with a number\n   greater or equal to that is reached.  In both cases, also stop when\n   the current frame returns.\n\n   Changed in version 3.2: Allow giving an explicit line number.\n\nr(eturn)\n\n   Continue execution until the current function returns.\n\nc(ont(inue))\n\n   Continue execution, only stop when a breakpoint is encountered.\n\nj(ump) lineno\n\n   Set the next line that will be executed.  Only available in the\n   bottom-most frame.  This lets you jump back and execute code again,\n   or jump forward to skip code that you don\'t want to run.\n\n   It should be noted that not all jumps are allowed -- for instance\n   it is not possible to jump into the middle of a "for" loop or out\n   of a "finally" clause.\n\nl(ist) [first[, last]]\n\n   List source code for the current file.  Without arguments, list 11\n   lines around the current line or continue the previous listing.\n   With "." as argument, list 11 lines around the current line.  With\n   one argument, list 11 lines around at that line.  With two\n   arguments, list the given range; if the second argument is less\n   than the first, it is interpreted as a count.\n\n   The current line in the current frame is indicated by "->".  If an\n   exception is being debugged, the line where the exception was\n   originally raised or propagated is indicated by ">>", if it differs\n   from the current line.\n\n   New in version 3.2: The ">>" marker.\n\nll | longlist\n\n   List all source code for the current function or frame.\n   Interesting lines are marked as for "list".\n\n   New in version 3.2.\n\na(rgs)\n\n   Print the argument list of the current function.\n\np expression\n\n   Evaluate the *expression* in the current context and print its\n   value.\n\n   Note: "print()" can also be used, but is not a debugger command\n     --- this executes the Python "print()" function.\n\npp expression\n\n   Like the "p" command, except the value of the expression is pretty-\n   printed using the "pprint" module.\n\nwhatis expression\n\n   Print the type of the *expression*.\n\nsource expression\n\n   Try to get source code for the given object and display it.\n\n   New in version 3.2.\n\ndisplay [expression]\n\n   Display the value of the expression if it changed, each time\n   execution stops in the current frame.\n\n   Without expression, list all display expressions for the current\n   frame.\n\n   New in version 3.2.\n\nundisplay [expression]\n\n   Do not display the expression any more in the current frame.\n   Without expression, clear all display expressions for the current\n   frame.\n\n   New in version 3.2.\n\ninteract\n\n   Start an interactive interpreter (using the "code" module) whose\n   global namespace contains all the (global and local) names found in\n   the current scope.\n\n   New in version 3.2.\n\nalias [name [command]]\n\n   Create an alias called *name* that executes *command*.  The command\n   must *not* be enclosed in quotes.  Replaceable parameters can be\n   indicated by "%1", "%2", and so on, while "%*" is replaced by all\n   the parameters. If no command is given, the current alias for\n   *name* is shown. If no arguments are given, all aliases are listed.\n\n   Aliases may be nested and can contain anything that can be legally\n   typed at the pdb prompt.  Note that internal pdb commands *can* be\n   overridden by aliases.  Such a command is then hidden until the\n   alias is removed.  Aliasing is recursively applied to the first\n   word of the command line; all other words in the line are left\n   alone.\n\n   As an example, here are two useful aliases (especially when placed\n   in the ".pdbrc" file):\n\n      # Print instance variables (usage "pi classInst")\n      alias pi for k in %1.__dict__.keys(): print("%1.",k,"=",%1.__dict__[k])\n      # Print instance variables in self\n      alias ps pi self\n\nunalias name\n\n   Delete the specified alias.\n\n! statement\n\n   Execute the (one-line) *statement* in the context of the current\n   stack frame. The exclamation point can be omitted unless the first\n   word of the statement resembles a debugger command.  To set a\n   global variable, you can prefix the assignment command with a\n   "global" statement on the same line, e.g.:\n\n      (Pdb) global list_options; list_options = [\'-l\']\n      (Pdb)\n\nrun [args ...]\nrestart [args ...]\n\n   Restart the debugged Python program.  If an argument is supplied,\n   it is split with "shlex" and the result is used as the new\n   "sys.argv". History, breakpoints, actions and debugger options are\n   preserved. "restart" is an alias for "run".\n\nq(uit)\n\n   Quit from the debugger.  The program being executed is aborted.\n\n-[ Footnotes ]-\n\n[1] Whether a frame is considered to originate in a certain module\n    is determined by the "__name__" in the frame globals.\n',
- 'del': u'\nThe "del" statement\n*******************\n\n   del_stmt ::= "del" target_list\n\nDeletion is recursively defined very similar to the way assignment is\ndefined. Rather than spelling it out in full details, here are some\nhints.\n\nDeletion of a target list recursively deletes each target, from left\nto right.\n\nDeletion of a name removes the binding of that name from the local or\nglobal namespace, depending on whether the name occurs in a "global"\nstatement in the same code block.  If the name is unbound, a\n"NameError" exception will be raised.\n\nDeletion of attribute references, subscriptions and slicings is passed\nto the primary object involved; deletion of a slicing is in general\nequivalent to assignment of an empty slice of the right type (but even\nthis is determined by the sliced object).\n\nChanged in version 3.2: Previously it was illegal to delete a name\nfrom the local namespace if it occurs as a free variable in a nested\nblock.\n',
- 'dict': u'\nDictionary displays\n*******************\n\nA dictionary display is a possibly empty series of key/datum pairs\nenclosed in curly braces:\n\n   dict_display       ::= "{" [key_datum_list | dict_comprehension] "}"\n   key_datum_list     ::= key_datum ("," key_datum)* [","]\n   key_datum          ::= expression ":" expression | "**" or_expr\n   dict_comprehension ::= expression ":" expression comp_for\n\nA dictionary display yields a new dictionary object.\n\nIf a comma-separated sequence of key/datum pairs is given, they are\nevaluated from left to right to define the entries of the dictionary:\neach key object is used as a key into the dictionary to store the\ncorresponding datum.  This means that you can specify the same key\nmultiple times in the key/datum list, and the final dictionary\'s value\nfor that key will be the last one given.\n\nA double asterisk "**" denotes *dictionary unpacking*. Its operand\nmust be a *mapping*.  Each mapping item is added to the new\ndictionary.  Later values replace values already set by earlier\nkey/datum pairs and earlier dictionary unpackings.\n\nNew in version 3.5: Unpacking into dictionary displays, originally\nproposed by **PEP 448**.\n\nA dict comprehension, in contrast to list and set comprehensions,\nneeds two expressions separated with a colon followed by the usual\n"for" and "if" clauses. When the comprehension is run, the resulting\nkey and value elements are inserted in the new dictionary in the order\nthey are produced.\n\nRestrictions on the types of the key values are listed earlier in\nsection *The standard type hierarchy*.  (To summarize, the key type\nshould be *hashable*, which excludes all mutable objects.)  Clashes\nbetween duplicate keys are not detected; the last datum (textually\nrightmost in the display) stored for a given key value prevails.\n',
- 'dynamic-features': u'\nInteraction with dynamic features\n*********************************\n\nName resolution of free variables occurs at runtime, not at compile\ntime. This means that the following code will print 42:\n\n   i = 10\n   def f():\n       print(i)\n   i = 42\n   f()\n\nThe "eval()" and "exec()" functions do not have access to the full\nenvironment for resolving names.  Names may be resolved in the local\nand global namespaces of the caller.  Free variables are not resolved\nin the nearest enclosing namespace, but in the global namespace.  [1]\nThe "exec()" and "eval()" functions have optional arguments to\noverride the global and local namespace.  If only one namespace is\nspecified, it is used for both.\n',
- 'else': u'\nThe "if" statement\n******************\n\nThe "if" statement is used for conditional execution:\n\n   if_stmt ::= "if" expression ":" suite\n               ( "elif" expression ":" suite )*\n               ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the "if" statement is executed or evaluated).\nIf all expressions are false, the suite of the "else" clause, if\npresent, is executed.\n',
- 'exceptions': u'\nExceptions\n**********\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions.  An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero).  A Python program can also\nexplicitly raise an exception with the "raise" statement. Exception\nhandlers are specified with the "try" ... "except" statement.  The\n"finally" clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop.  In\neither case, it prints a stack backtrace, except when the exception is\n"SystemExit".\n\nExceptions are identified by class instances.  The "except" clause is\nselected depending on the class of the instance: it must reference the\nclass of the instance or a base class thereof.  The instance can be\nreceived by the handler and can carry additional information about the\nexceptional condition.\n\nNote: Exception messages are not part of the Python API.  Their\n  contents may change from one version of Python to the next without\n  warning and should not be relied on by code which will run under\n  multiple versions of the interpreter.\n\nSee also the description of the "try" statement in section *The try\nstatement* and "raise" statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by\n    these operations is not available at the time the module is\n    compiled.\n',
- 'execmodel': u'\nExecution model\n***************\n\n\nStructure of a program\n======================\n\nA Python program is constructed from code blocks. A *block* is a piece\nof Python program text that is executed as a unit. The following are\nblocks: a module, a function body, and a class definition. Each\ncommand typed interactively is a block.  A script file (a file given\nas standard input to the interpreter or specified as a command line\nargument to the interpreter) is a code block.  A script command (a\ncommand specified on the interpreter command line with the \'**-c**\'\noption) is a code block.  The string argument passed to the built-in\nfunctions "eval()" and "exec()" is a code block.\n\nA code block is executed in an *execution frame*.  A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\n\nNaming and binding\n==================\n\n\nBinding of names\n----------------\n\n*Names* refer to objects.  Names are introduced by name binding\noperations.\n\nThe following constructs bind names: formal parameters to functions,\n"import" statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, "for" loop header, or after\n"as" in a "with" statement or "except" clause. The "import" statement\nof the form "from ... import *" binds all names defined in the\nimported module, except those beginning with an underscore.  This form\nmay only be used at the module level.\n\nA target occurring in a "del" statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name).\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name is bound in a block, it is a local variable of that block,\nunless declared as "nonlocal" or "global".  If a name is bound at the\nmodule level, it is a global variable.  (The variables of the module\ncode block are local and global.)  If a variable is used in a code\nblock but not defined there, it is a *free variable*.\n\nEach occurrence of a name in the program text refers to the *binding*\nof that name established by the following name resolution rules.\n\n\nResolution of names\n-------------------\n\nA *scope* defines the visibility of a name within a block.  If a local\nvariable is defined in a block, its scope includes that block.  If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name.\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope.  The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nWhen a name is not found at all, a "NameError" exception is raised. If\nthe current scope is a function scope, and the name refers to a local\nvariable that has not yet been bound to a value at the point where the\nname is used, an "UnboundLocalError" exception is raised.\n"UnboundLocalError" is a subclass of "NameError".\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block.  This can lead to errors when a name is used within a\nblock before it is bound.  This rule is subtle.  Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block.  The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the "global" statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace.  Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module "builtins".  The global namespace is searched first.  If\nthe name is not found there, the builtins namespace is searched.  The\n"global" statement must precede all uses of the name.\n\nThe "global" statement has the same scope as a name binding operation\nin the same block.  If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nThe "nonlocal" statement causes corresponding names to refer to\npreviously bound variables in the nearest enclosing function scope.\n"SyntaxError" is raised at compile time if the given name does not\nexist in any enclosing function scope.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported.  The main module for a script is always called\n"__main__".\n\nClass definition blocks and arguments to "exec()" and "eval()" are\nspecial in the context of name resolution. A class definition is an\nexecutable statement that may use and define names. These references\nfollow the normal rules for name resolution with an exception that\nunbound local variables are looked up in the global namespace. The\nnamespace of the class definition becomes the attribute dictionary of\nthe class. The scope of names defined in a class block is limited to\nthe class block; it does not extend to the code blocks of methods --\nthis includes comprehensions and generator expressions since they are\nimplemented using a function scope.  This means that the following\nwill fail:\n\n   class A:\n       a = 42\n       b = list(a + i for i in range(10))\n\n\nBuiltins and restricted execution\n---------------------------------\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name "__builtins__" in its global\nnamespace; this should be a dictionary or a module (in the latter case\nthe module\'s dictionary is used).  By default, when in the "__main__"\nmodule, "__builtins__" is the built-in module "builtins"; when in any\nother module, "__builtins__" is an alias for the dictionary of the\n"builtins" module itself.  "__builtins__" can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n"__builtins__"; it is strictly an implementation detail.  Users\nwanting to override values in the builtins namespace should "import"\nthe "builtins" module and modify its attributes appropriately.\n\n\nInteraction with dynamic features\n---------------------------------\n\nName resolution of free variables occurs at runtime, not at compile\ntime. This means that the following code will print 42:\n\n   i = 10\n   def f():\n       print(i)\n   i = 42\n   f()\n\nThe "eval()" and "exec()" functions do not have access to the full\nenvironment for resolving names.  Names may be resolved in the local\nand global namespaces of the caller.  Free variables are not resolved\nin the nearest enclosing namespace, but in the global namespace.  [1]\nThe "exec()" and "eval()" functions have optional arguments to\noverride the global and local namespace.  If only one namespace is\nspecified, it is used for both.\n\n\nExceptions\n==========\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions.  An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero).  A Python program can also\nexplicitly raise an exception with the "raise" statement. Exception\nhandlers are specified with the "try" ... "except" statement.  The\n"finally" clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop.  In\neither case, it prints a stack backtrace, except when the exception is\n"SystemExit".\n\nExceptions are identified by class instances.  The "except" clause is\nselected depending on the class of the instance: it must reference the\nclass of the instance or a base class thereof.  The instance can be\nreceived by the handler and can carry additional information about the\nexceptional condition.\n\nNote: Exception messages are not part of the Python API.  Their\n  contents may change from one version of Python to the next without\n  warning and should not be relied on by code which will run under\n  multiple versions of the interpreter.\n\nSee also the description of the "try" statement in section *The try\nstatement* and "raise" statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by\n    these operations is not available at the time the module is\n    compiled.\n',
- 'exprlists': u'\nExpression lists\n****************\n\n   expression_list    ::= expression ( "," expression )* [","]\n   starred_list       ::= starred_item ( "," starred_item )* [","]\n   starred_expression ::= expression | ( starred_item "," )* [starred_item]\n   starred_item       ::= expression | "*" or_expr\n\nExcept when part of a list or set display, an expression list\ncontaining at least one comma yields a tuple.  The length of the tuple\nis the number of expressions in the list.  The expressions are\nevaluated from left to right.\n\nAn asterisk "*" denotes *iterable unpacking*.  Its operand must be an\n*iterable*.  The iterable is expanded into a sequence of items, which\nare included in the new tuple, list, or set, at the site of the\nunpacking.\n\nNew in version 3.5: Iterable unpacking in expression lists, originally\nproposed by **PEP 448**.\n\nThe trailing comma is required only to create a single tuple (a.k.a. a\n*singleton*); it is optional in all other cases.  A single expression\nwithout a trailing comma doesn\'t create a tuple, but rather yields the\nvalue of that expression. (To create an empty tuple, use an empty pair\nof parentheses: "()".)\n',
- 'floating': u'\nFloating point literals\n***********************\n\nFloating point literals are described by the following lexical\ndefinitions:\n\n   floatnumber   ::= pointfloat | exponentfloat\n   pointfloat    ::= [intpart] fraction | intpart "."\n   exponentfloat ::= (intpart | pointfloat) exponent\n   intpart       ::= digit+\n   fraction      ::= "." digit+\n   exponent      ::= ("e" | "E") ["+" | "-"] digit+\n\nNote that the integer and exponent parts are always interpreted using\nradix 10. For example, "077e010" is legal, and denotes the same number\nas "77e10". The allowed range of floating point literals is\nimplementation-dependent. Some examples of floating point literals:\n\n   3.14    10.    .001    1e100    3.14e-10    0e0\n\nNote that numeric literals do not include a sign; a phrase like "-1"\nis actually an expression composed of the unary operator "-" and the\nliteral "1".\n',
- 'for': u'\nThe "for" statement\n*******************\n\nThe "for" statement is used to iterate over the elements of a sequence\n(such as a string, tuple or list) or other iterable object:\n\n   for_stmt ::= "for" target_list "in" expression_list ":" suite\n                ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject.  An iterator is created for the result of the\n"expression_list".  The suite is then executed once for each item\nprovided by the iterator, in the order returned by the iterator.  Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments (see *Assignment statements*), and then the suite is\nexecuted.  When the items are exhausted (which is immediately when the\nsequence is empty or an iterator raises a "StopIteration" exception),\nthe suite in the "else" clause, if present, is executed, and the loop\nterminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite.  A "continue" statement\nexecuted in the first suite skips the rest of the suite and continues\nwith the next item, or with the "else" clause if there is no next\nitem.\n\nThe for-loop makes assignments to the variables(s) in the target list.\nThis overwrites all previous assignments to those variables including\nthose made in the suite of the for-loop:\n\n   for i in range(10):\n       print(i)\n       i = 5             # this will not affect the for-loop\n                         # because i will be overwritten with the next\n                         # index in the range\n\nNames in the target list are not deleted when the loop is finished,\nbut if the sequence is empty, they will not have been assigned to at\nall by the loop.  Hint: the built-in function "range()" returns an\niterator of integers suitable to emulate the effect of Pascal\'s "for i\n:= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n\nNote: There is a subtlety when the sequence is being modified by the\n  loop (this can only occur for mutable sequences, i.e. lists).  An\n  internal counter is used to keep track of which item is used next,\n  and this is incremented on each iteration.  When this counter has\n  reached the length of the sequence the loop terminates.  This means\n  that if the suite deletes the current (or a previous) item from the\n  sequence, the next item will be skipped (since it gets the index of\n  the current item which has already been treated).  Likewise, if the\n  suite inserts an item in the sequence before the current item, the\n  current item will be treated again the next time through the loop.\n  This can lead to nasty bugs that can be avoided by making a\n  temporary copy using a slice of the whole sequence, e.g.,\n\n     for x in a[:]:\n         if x < 0: a.remove(x)\n',
- 'formatstrings': u'\nFormat String Syntax\n********************\n\nThe "str.format()" method and the "Formatter" class share the same\nsyntax for format strings (although in the case of "Formatter",\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n"{}". Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output.  If you need to include\na brace character in the literal text, it can be escaped by doubling:\n"{{" and "}}".\n\nThe grammar for a replacement field is as follows:\n\n      replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n      field_name        ::= arg_name ("." attribute_name | "[" element_index "]")*\n      arg_name          ::= [identifier | integer]\n      attribute_name    ::= identifier\n      element_index     ::= integer | index_string\n      index_string      ::= <any source character except "]"> +\n      conversion        ::= "r" | "s" | "a"\n      format_spec       ::= <described in the next section>\n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a  *conversion* field, which is\npreceded by an exclamation point "\'!\'", and a *format_spec*, which is\npreceded by a colon "\':\'".  These specify a non-default format for the\nreplacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either a\nnumber or a keyword.  If it\'s a number, it refers to a positional\nargument, and if it\'s a keyword, it refers to a named keyword\nargument.  If the numerical arg_names in a format string are 0, 1, 2,\n... in sequence, they can all be omitted (not just some) and the\nnumbers 0, 1, 2, ... will be automatically inserted in that order.\nBecause *arg_name* is not quote-delimited, it is not possible to\nspecify arbitrary dictionary keys (e.g., the strings "\'10\'" or\n"\':-]\'") within a format string. The *arg_name* can be followed by any\nnumber of index or attribute expressions. An expression of the form\n"\'.name\'" selects the named attribute using "getattr()", while an\nexpression of the form "\'[index]\'" does an index lookup using\n"__getitem__()".\n\nChanged in version 3.1: The positional argument specifiers can be\nomitted, so "\'{} {}\'" is equivalent to "\'{0} {1}\'".\n\nSome simple format string examples:\n\n   "First, thou shalt count to {0}"  # References first positional argument\n   "Bring me a {}"                   # Implicitly references the first positional argument\n   "From {} to {}"                   # Same as "From {0} to {1}"\n   "My quest is {name}"              # References keyword argument \'name\'\n   "Weight in tons {0.weight}"       # \'weight\' attribute of first positional arg\n   "Units destroyed: {players[0]}"   # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the "__format__()"\nmethod of the value itself.  However, in some cases it is desirable to\nforce a type to be formatted as a string, overriding its own\ndefinition of formatting.  By converting the value to a string before\ncalling "__format__()", the normal formatting logic is bypassed.\n\nThree conversion flags are currently supported: "\'!s\'" which calls\n"str()" on the value, "\'!r\'" which calls "repr()" and "\'!a\'" which\ncalls "ascii()".\n\nSome examples:\n\n   "Harold\'s a clever {0!s}"        # Calls str() on the argument first\n   "Bring out the holy {name!r}"    # Calls repr() on the argument first\n   "More {!a}"                      # Calls ascii() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on.  Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields may contain a field name,\nconversion flag and format specification, but deeper nesting is not\nallowed.  The replacement fields within the format_spec are\nsubstituted before the *format_spec* string is interpreted. This\nallows the formatting of a value to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*).  They can also be passed directly to the\nbuilt-in "format()" function.  Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string ("""") produces\nthe same result as if you had called "str()" on the value. A non-empty\nformat string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n   format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n   fill        ::= <any character>\n   align       ::= "<" | ">" | "=" | "^"\n   sign        ::= "+" | "-" | " "\n   width       ::= integer\n   precision   ::= integer\n   type        ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nIf a valid *align* value is specified, it can be preceded by a *fill*\ncharacter that can be any character and defaults to a space if\nomitted. It is not possible to use a literal curly brace (""{"" or\n""}"") as the *fill* character when using the "str.format()" method.\nHowever, it is possible to insert a curly brace with a nested\nreplacement field.  This limitation doesn\'t affect the "format()"\nfunction.\n\nThe meaning of the various alignment options is as follows:\n\n   +-----------+------------------------------------------------------------+\n   | Option    | Meaning                                                    |\n   +===========+============================================================+\n   | "\'<\'"     | Forces the field to be left-aligned within the available   |\n   |           | space (this is the default for most objects).              |\n   +-----------+------------------------------------------------------------+\n   | "\'>\'"     | Forces the field to be right-aligned within the available  |\n   |           | space (this is the default for numbers).                   |\n   +-----------+------------------------------------------------------------+\n   | "\'=\'"     | Forces the padding to be placed after the sign (if any)    |\n   |           | but before the digits.  This is used for printing fields   |\n   |           | in the form \'+000000120\'. This alignment option is only    |\n   |           | valid for numeric types.  It becomes the default when \'0\'  |\n   |           | immediately precedes the field width.                      |\n   +-----------+------------------------------------------------------------+\n   | "\'^\'"     | Forces the field to be centered within the available       |\n   |           | space.                                                     |\n   +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n   +-----------+------------------------------------------------------------+\n   | Option    | Meaning                                                    |\n   +===========+============================================================+\n   | "\'+\'"     | indicates that a sign should be used for both positive as  |\n   |           | well as negative numbers.                                  |\n   +-----------+------------------------------------------------------------+\n   | "\'-\'"     | indicates that a sign should be used only for negative     |\n   |           | numbers (this is the default behavior).                    |\n   +-----------+------------------------------------------------------------+\n   | space     | indicates that a leading space should be used on positive  |\n   |           | numbers, and a minus sign on negative numbers.             |\n   +-----------+------------------------------------------------------------+\n\nThe "\'#\'" option causes the "alternate form" to be used for the\nconversion.  The alternate form is defined differently for different\ntypes.  This option is only valid for integer, float, complex and\nDecimal types. For integers, when binary, octal, or hexadecimal output\nis used, this option adds the prefix respective "\'0b\'", "\'0o\'", or\n"\'0x\'" to the output value. For floats, complex and Decimal the\nalternate form causes the result of the conversion to always contain a\ndecimal-point character, even if no digits follow it. Normally, a\ndecimal-point character appears in the result of these conversions\nonly if a digit follows it. In addition, for "\'g\'" and "\'G\'"\nconversions, trailing zeros are not removed from the result.\n\nThe "\',\'" option signals the use of a comma for a thousands separator.\nFor a locale aware separator, use the "\'n\'" integer presentation type\ninstead.\n\nChanged in version 3.1: Added the "\',\'" option (see also **PEP 378**).\n\n*width* is a decimal integer defining the minimum field width.  If not\nspecified, then the field width will be determined by the content.\n\nWhen no explicit alignment is given, preceding the *width* field by a\nzero ("\'0\'") character enables sign-aware zero-padding for numeric\ntypes.  This is equivalent to a *fill* character of "\'0\'" with an\n*alignment* type of "\'=\'".\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with "\'f\'" and "\'F\'", or before and after the decimal point\nfor a floating point value formatted with "\'g\'" or "\'G\'".  For non-\nnumber types the field indicates the maximum field size - in other\nwords, how many characters will be used from the field content. The\n*precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | "\'s\'"     | String format. This is the default type for strings and    |\n   |           | may be omitted.                                            |\n   +-----------+------------------------------------------------------------+\n   | None      | The same as "\'s\'".                                         |\n   +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | "\'b\'"     | Binary format. Outputs the number in base 2.               |\n   +-----------+------------------------------------------------------------+\n   | "\'c\'"     | Character. Converts the integer to the corresponding       |\n   |           | unicode character before printing.                         |\n   +-----------+------------------------------------------------------------+\n   | "\'d\'"     | Decimal Integer. Outputs the number in base 10.            |\n   +-----------+------------------------------------------------------------+\n   | "\'o\'"     | Octal format. Outputs the number in base 8.                |\n   +-----------+------------------------------------------------------------+\n   | "\'x\'"     | Hex format. Outputs the number in base 16, using lower-    |\n   |           | case letters for the digits above 9.                       |\n   +-----------+------------------------------------------------------------+\n   | "\'X\'"     | Hex format. Outputs the number in base 16, using upper-    |\n   |           | case letters for the digits above 9.                       |\n   +-----------+------------------------------------------------------------+\n   | "\'n\'"     | Number. This is the same as "\'d\'", except that it uses the |\n   |           | current locale setting to insert the appropriate number    |\n   |           | separator characters.                                      |\n   +-----------+------------------------------------------------------------+\n   | None      | The same as "\'d\'".                                         |\n   +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except "\'n\'"\nand "None"). When doing so, "float()" is used to convert the integer\nto a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | "\'e\'"     | Exponent notation. Prints the number in scientific         |\n   |           | notation using the letter \'e\' to indicate the exponent.    |\n   |           | The default precision is "6".                              |\n   +-----------+------------------------------------------------------------+\n   | "\'E\'"     | Exponent notation. Same as "\'e\'" except it uses an upper   |\n   |           | case \'E\' as the separator character.                       |\n   +-----------+------------------------------------------------------------+\n   | "\'f\'"     | Fixed point. Displays the number as a fixed-point number.  |\n   |           | The default precision is "6".                              |\n   +-----------+------------------------------------------------------------+\n   | "\'F\'"     | Fixed point. Same as "\'f\'", but converts "nan" to "NAN"    |\n   |           | and "inf" to "INF".                                        |\n   +-----------+------------------------------------------------------------+\n   | "\'g\'"     | General format.  For a given precision "p >= 1", this      |\n   |           | rounds the number to "p" significant digits and then       |\n   |           | formats the result in either fixed-point format or in      |\n   |           | scientific notation, depending on its magnitude.  The      |\n   |           | precise rules are as follows: suppose that the result      |\n   |           | formatted with presentation type "\'e\'" and precision "p-1" |\n   |           | would have exponent "exp".  Then if "-4 <= exp < p", the   |\n   |           | number is formatted with presentation type "\'f\'" and       |\n   |           | precision "p-1-exp".  Otherwise, the number is formatted   |\n   |           | with presentation type "\'e\'" and precision "p-1". In both  |\n   |           | cases insignificant trailing zeros are removed from the    |\n   |           | significand, and the decimal point is also removed if      |\n   |           | there are no remaining digits following it.  Positive and  |\n   |           | negative infinity, positive and negative zero, and nans,   |\n   |           | are formatted as "inf", "-inf", "0", "-0" and "nan"        |\n   |           | respectively, regardless of the precision.  A precision of |\n   |           | "0" is treated as equivalent to a precision of "1". The    |\n   |           | default precision is "6".                                  |\n   +-----------+------------------------------------------------------------+\n   | "\'G\'"     | General format. Same as "\'g\'" except switches to "\'E\'" if  |\n   |           | the number gets too large. The representations of infinity |\n   |           | and NaN are uppercased, too.                               |\n   +-----------+------------------------------------------------------------+\n   | "\'n\'"     | Number. This is the same as "\'g\'", except that it uses the |\n   |           | current locale setting to insert the appropriate number    |\n   |           | separator characters.                                      |\n   +-----------+------------------------------------------------------------+\n   | "\'%\'"     | Percentage. Multiplies the number by 100 and displays in   |\n   |           | fixed ("\'f\'") format, followed by a percent sign.          |\n   +-----------+------------------------------------------------------------+\n   | None      | Similar to "\'g\'", except that fixed-point notation, when   |\n   |           | used, has at least one digit past the decimal point. The   |\n   |           | default precision is as high as needed to represent the    |\n   |           | particular value. The overall effect is to match the       |\n   |           | output of "str()" as altered by the other format           |\n   |           | modifiers.                                                 |\n   +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the "str.format()" syntax and\ncomparison with the old "%"-formatting.\n\nIn most of the cases the syntax is similar to the old "%"-formatting,\nwith the addition of the "{}" and with ":" used instead of "%". For\nexample, "\'%03.2f\'" can be translated to "\'{:03.2f}\'".\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n   >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n   \'a, b, c\'\n   >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\')  # 3.1+ only\n   \'a, b, c\'\n   >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n   \'c, b, a\'\n   >>> \'{2}, {1}, {0}\'.format(*\'abc\')      # unpacking argument sequence\n   \'c, b, a\'\n   >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\')   # arguments\' indices can be repeated\n   \'abracadabra\'\n\nAccessing arguments by name:\n\n   >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n   \'Coordinates: 37.24N, -115.81W\'\n   >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n   >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n   \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n   >>> c = 3-5j\n   >>> (\'The complex number {0} is formed from the real part {0.real} \'\n   ...  \'and the imaginary part {0.imag}.\').format(c)\n   \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n   >>> class Point:\n   ...     def __init__(self, x, y):\n   ...         self.x, self.y = x, y\n   ...     def __str__(self):\n   ...         return \'Point({self.x}, {self.y})\'.format(self=self)\n   ...\n   >>> str(Point(4, 2))\n   \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n   >>> coord = (3, 5)\n   >>> \'X: {0[0]};  Y: {0[1]}\'.format(coord)\n   \'X: 3;  Y: 5\'\n\nReplacing "%s" and "%r":\n\n   >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n   "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n   >>> \'{:<30}\'.format(\'left aligned\')\n   \'left aligned                  \'\n   >>> \'{:>30}\'.format(\'right aligned\')\n   \'                 right aligned\'\n   >>> \'{:^30}\'.format(\'centered\')\n   \'           centered           \'\n   >>> \'{:*^30}\'.format(\'centered\')  # use \'*\' as a fill char\n   \'***********centered***********\'\n\nReplacing "%+f", "%-f", and "% f" and specifying a sign:\n\n   >>> \'{:+f}; {:+f}\'.format(3.14, -3.14)  # show it always\n   \'+3.140000; -3.140000\'\n   >>> \'{: f}; {: f}\'.format(3.14, -3.14)  # show a space for positive numbers\n   \' 3.140000; -3.140000\'\n   >>> \'{:-f}; {:-f}\'.format(3.14, -3.14)  # show only the minus -- same as \'{:f}; {:f}\'\n   \'3.140000; -3.140000\'\n\nReplacing "%x" and "%o" and converting the value to different bases:\n\n   >>> # format also supports binary numbers\n   >>> "int: {0:d};  hex: {0:x};  oct: {0:o};  bin: {0:b}".format(42)\n   \'int: 42;  hex: 2a;  oct: 52;  bin: 101010\'\n   >>> # with 0x, 0o, or 0b as prefix:\n   >>> "int: {0:d};  hex: {0:#x};  oct: {0:#o};  bin: {0:#b}".format(42)\n   \'int: 42;  hex: 0x2a;  oct: 0o52;  bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n   >>> \'{:,}\'.format(1234567890)\n   \'1,234,567,890\'\n\nExpressing a percentage:\n\n   >>> points = 19\n   >>> total = 22\n   >>> \'Correct answers: {:.2%}\'.format(points/total)\n   \'Correct answers: 86.36%\'\n\nUsing type-specific formatting:\n\n   >>> import datetime\n   >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n   >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n   \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n   >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n   ...     \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n   ...\n   \'left<<<<<<<<<<<<\'\n   \'^^^^^center^^^^^\'\n   \'>>>>>>>>>>>right\'\n   >>>\n   >>> octets = [192, 168, 0, 1]\n   >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n   \'C0A80001\'\n   >>> int(_, 16)\n   3232235521\n   >>>\n   >>> width = 5\n   >>> for num in range(5,12): #doctest: +NORMALIZE_WHITESPACE\n   ...     for base in \'dXob\':\n   ...         print(\'{0:{width}{base}}\'.format(num, base=base, width=width), end=\' \')\n   ...     print()\n   ...\n       5     5     5   101\n       6     6     6   110\n       7     7     7   111\n       8     8    10  1000\n       9     9    11  1001\n      10     A    12  1010\n      11     B    13  1011\n',
- 'function': u'\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n   funcdef        ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n   decorators     ::= decorator+\n   decorator      ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n   dotted_name    ::= identifier ("." identifier)*\n   parameter_list ::= (defparameter ",")*\n                      | "*" [parameter] ("," defparameter)* ["," "**" parameter]\n                      | "**" parameter\n                      | defparameter [","] )\n   parameter      ::= identifier [":" expression]\n   defparameter   ::= parameter ["=" expression]\n   funcname       ::= identifier\n\nA function definition is an executable statement.  Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function).  This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition.  The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object.  Multiple decorators are applied in\nnested fashion. For example, the following code\n\n   @f1(arg)\n   @f2\n   def func(): pass\n\nis roughly equivalent to\n\n   def func(): pass\n   func = f1(arg)(f2(func))\n\nexcept that the original function is not temporarily bound to the name\n"func".\n\nWhen one or more *parameters* have the form *parameter* "="\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding *argument* may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted.  If a parameter has a default value, all following\nparameters up until the ""*"" must also have a default value --- this\nis a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated from left to right when the\nfunction definition is executed.** This means that the expression is\nevaluated once, when the function is defined, and that the same "pre-\ncomputed" value is used for each call.  This is especially important\nto understand when a default parameter is a mutable object, such as a\nlist or a dictionary: if the function modifies the object (e.g. by\nappending an item to a list), the default value is in effect modified.\nThis is generally not what was intended.  A way around this is to use\n"None" as the default, and explicitly test for it in the body of the\nfunction, e.g.:\n\n   def whats_on_the_telly(penguin=None):\n       if penguin is None:\n           penguin = []\n       penguin.append("property of the zoo")\n       return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values.  If the form\n""*identifier"" is present, it is initialized to a tuple receiving any\nexcess positional parameters, defaulting to the empty tuple.  If the\nform ""**identifier"" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after ""*"" or ""*identifier"" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "": expression"" following\nthe parameter name.  Any parameter may have an annotation even those\nof the form "*identifier" or "**identifier".  Functions may have\n"return" annotation of the form ""-> expression"" after the parameter\nlist.  These annotations can be any valid Python expression and are\nevaluated when the function definition is executed.  Annotations may\nbe evaluated in a different order than they appear in the source code.\nThe presence of annotations does not change the semantics of a\nfunction.  The annotation values are available as values of a\ndictionary keyed by the parameters\' names in the "__annotations__"\nattribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions.  This uses lambda\nexpressions, described in section *Lambdas*.  Note that the lambda\nexpression is merely a shorthand for a simplified function definition;\na function defined in a ""def"" statement can be passed around or\nassigned to another name just like a function defined by a lambda\nexpression.  The ""def"" form is actually more powerful since it\nallows the execution of multiple statements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects.  A ""def""\nstatement executed inside a function definition defines a local\nfunction that can be returned or passed around.  Free variables used\nin the nested function can access the local variables of the function\ncontaining the def.  See section *Naming and binding* for details.\n\nSee also: **PEP 3107** - Function Annotations\n\n     The original specification for function annotations.\n',
- 'global': u'\nThe "global" statement\n**********************\n\n   global_stmt ::= "global" identifier ("," identifier)*\n\nThe "global" statement is a declaration which holds for the entire\ncurrent code block.  It means that the listed identifiers are to be\ninterpreted as globals.  It would be impossible to assign to a global\nvariable without "global", although free variables may refer to\nglobals without being declared global.\n\nNames listed in a "global" statement must not be used in the same code\nblock textually preceding that "global" statement.\n\nNames listed in a "global" statement must not be defined as formal\nparameters or in a "for" loop control target, "class" definition,\nfunction definition, or "import" statement.\n\n**CPython implementation detail:** The current implementation does not\nenforce the two restrictions, but programs should not abuse this\nfreedom, as future implementations may enforce them or silently change\nthe meaning of the program.\n\n**Programmer\'s note:** "global" is a directive to the parser.  It\napplies only to code parsed at the same time as the "global"\nstatement. In particular, a "global" statement contained in a string\nor code object supplied to the built-in "exec()" function does not\naffect the code block *containing* the function call, and code\ncontained in such a string is unaffected by "global" statements in the\ncode containing the function call.  The same applies to the "eval()"\nand "compile()" functions.\n',
- 'id-classes': u'\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings.  These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n"_*"\n   Not imported by "from module import *".  The special identifier "_"\n   is used in the interactive interpreter to store the result of the\n   last evaluation; it is stored in the "builtins" module.  When not\n   in interactive mode, "_" has no special meaning and is not defined.\n   See section *The import statement*.\n\n   Note: The name "_" is often used in conjunction with\n     internationalization; refer to the documentation for the\n     "gettext" module for more information on this convention.\n\n"__*__"\n   System-defined names. These names are defined by the interpreter\n   and its implementation (including the standard library).  Current\n   system names are discussed in the *Special method names* section\n   and elsewhere.  More will likely be defined in future versions of\n   Python.  *Any* use of "__*__" names, in any context, that does not\n   follow explicitly documented use, is subject to breakage without\n   warning.\n\n"__*"\n   Class-private names.  Names in this category, when used within the\n   context of a class definition, are re-written to use a mangled form\n   to help avoid name clashes between "private" attributes of base and\n   derived classes. See section *Identifiers (Names)*.\n',
- 'identifiers': u'\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions.\n\nThe syntax of identifiers in Python is based on the Unicode standard\nannex UAX-31, with elaboration and changes as defined below; see also\n**PEP 3131** for further details.\n\nWithin the ASCII range (U+0001..U+007F), the valid characters for\nidentifiers are the same as in Python 2.x: the uppercase and lowercase\nletters "A" through "Z", the underscore "_" and, except for the first\ncharacter, the digits "0" through "9".\n\nPython 3.0 introduces additional characters from outside the ASCII\nrange (see **PEP 3131**).  For these characters, the classification\nuses the version of the Unicode Character Database as included in the\n"unicodedata" module.\n\nIdentifiers are unlimited in length.  Case is significant.\n\n   identifier   ::= xid_start xid_continue*\n   id_start     ::= <all characters in general categories Lu, Ll, Lt, Lm, Lo, Nl, the underscore, and characters with the Other_ID_Start property>\n   id_continue  ::= <all characters in id_start, plus characters in the categories Mn, Mc, Nd, Pc and others with the Other_ID_Continue property>\n   xid_start    ::= <all characters in id_start whose NFKC normalization is in "id_start xid_continue*">\n   xid_continue ::= <all characters in id_continue whose NFKC normalization is in "id_continue*">\n\nThe Unicode category codes mentioned above stand for:\n\n* *Lu* - uppercase letters\n\n* *Ll* - lowercase letters\n\n* *Lt* - titlecase letters\n\n* *Lm* - modifier letters\n\n* *Lo* - other letters\n\n* *Nl* - letter numbers\n\n* *Mn* - nonspacing marks\n\n* *Mc* - spacing combining marks\n\n* *Nd* - decimal numbers\n\n* *Pc* - connector punctuations\n\n* *Other_ID_Start* - explicit list of characters in PropList.txt to\n  support backwards compatibility\n\n* *Other_ID_Continue* - likewise\n\nAll identifiers are converted into the normal form NFKC while parsing;\ncomparison of identifiers is based on NFKC.\n\nA non-normative HTML file listing all valid identifier characters for\nUnicode 4.1 can be found at https://www.dcl.hpi.uni-\npotsdam.de/home/loewis/table-3131.html.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers.  They must\nbe spelled exactly as written here:\n\n   False      class      finally    is         return\n   None       continue   for        lambda     try\n   True       def        from       nonlocal   while\n   and        del        global     not        with\n   as         elif       if         or         yield\n   assert     else       import     pass\n   break      except     in         raise\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings.  These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n"_*"\n   Not imported by "from module import *".  The special identifier "_"\n   is used in the interactive interpreter to store the result of the\n   last evaluation; it is stored in the "builtins" module.  When not\n   in interactive mode, "_" has no special meaning and is not defined.\n   See section *The import statement*.\n\n   Note: The name "_" is often used in conjunction with\n     internationalization; refer to the documentation for the\n     "gettext" module for more information on this convention.\n\n"__*__"\n   System-defined names. These names are defined by the interpreter\n   and its implementation (including the standard library).  Current\n   system names are discussed in the *Special method names* section\n   and elsewhere.  More will likely be defined in future versions of\n   Python.  *Any* use of "__*__" names, in any context, that does not\n   follow explicitly documented use, is subject to breakage without\n   warning.\n\n"__*"\n   Class-private names.  Names in this category, when used within the\n   context of a class definition, are re-written to use a mangled form\n   to help avoid name clashes between "private" attributes of base and\n   derived classes. See section *Identifiers (Names)*.\n',
- 'if': u'\nThe "if" statement\n******************\n\nThe "if" statement is used for conditional execution:\n\n   if_stmt ::= "if" expression ":" suite\n               ( "elif" expression ":" suite )*\n               ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the "if" statement is executed or evaluated).\nIf all expressions are false, the suite of the "else" clause, if\npresent, is executed.\n',
- 'imaginary': u'\nImaginary literals\n******************\n\nImaginary literals are described by the following lexical definitions:\n\n   imagnumber ::= (floatnumber | intpart) ("j" | "J")\n\nAn imaginary literal yields a complex number with a real part of 0.0.\nComplex numbers are represented as a pair of floating point numbers\nand have the same restrictions on their range.  To create a complex\nnumber with a nonzero real part, add a floating point number to it,\ne.g., "(3+4j)".  Some examples of imaginary literals:\n\n   3.14j   10.j    10j     .001j   1e100j  3.14e-10j\n',
- 'import': u'\nThe "import" statement\n**********************\n\n   import_stmt     ::= "import" module ["as" name] ( "," module ["as" name] )*\n                   | "from" relative_module "import" identifier ["as" name]\n                   ( "," identifier ["as" name] )*\n                   | "from" relative_module "import" "(" identifier ["as" name]\n                   ( "," identifier ["as" name] )* [","] ")"\n                   | "from" module "import" "*"\n   module          ::= (identifier ".")* identifier\n   relative_module ::= "."* module | "."+\n   name            ::= identifier\n\nThe basic import statement (no "from" clause) is executed in two\nsteps:\n\n1. find a module, loading and initializing it if necessary\n\n2. define a name or names in the local namespace for the scope\n   where the "import" statement occurs.\n\nWhen the statement contains multiple clauses (separated by commas) the\ntwo steps are carried out separately for each clause, just as though\nthe clauses had been separated out into individual import statements.\n\nThe details of the first step, finding and loading modules are\ndescribed in greater detail in the section on the *import system*,\nwhich also describes the various types of packages and modules that\ncan be imported, as well as all the hooks that can be used to\ncustomize the import system. Note that failures in this step may\nindicate either that the module could not be located, *or* that an\nerror occurred while initializing the module, which includes execution\nof the module\'s code.\n\nIf the requested module is retrieved successfully, it will be made\navailable in the local namespace in one of three ways:\n\n* If the module name is followed by "as", then the name following\n  "as" is bound directly to the imported module.\n\n* If no other name is specified, and the module being imported is a\n  top level module, the module\'s name is bound in the local namespace\n  as a reference to the imported module\n\n* If the module being imported is *not* a top level module, then the\n  name of the top level package that contains the module is bound in\n  the local namespace as a reference to the top level package. The\n  imported module must be accessed using its full qualified name\n  rather than directly\n\nThe "from" form uses a slightly more complex process:\n\n1. find the module specified in the "from" clause, loading and\n   initializing it if necessary;\n\n2. for each of the identifiers specified in the "import" clauses:\n\n   1. check if the imported module has an attribute by that name\n\n   2. if not, attempt to import a submodule with that name and then\n      check the imported module again for that attribute\n\n   3. if the attribute is not found, "ImportError" is raised.\n\n   4. otherwise, a reference to that value is stored in the local\n      namespace, using the name in the "as" clause if it is present,\n      otherwise using the attribute name\n\nExamples:\n\n   import foo                 # foo imported and bound locally\n   import foo.bar.baz         # foo.bar.baz imported, foo bound locally\n   import foo.bar.baz as fbb  # foo.bar.baz imported and bound as fbb\n   from foo.bar import baz    # foo.bar.baz imported and bound as baz\n   from foo import attr       # foo imported and foo.attr bound as attr\n\nIf the list of identifiers is replaced by a star ("\'*\'"), all public\nnames defined in the module are bound in the local namespace for the\nscope where the "import" statement occurs.\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named "__all__"; if defined, it must\nbe a sequence of strings which are names defined or imported by that\nmodule.  The names given in "__all__" are all considered public and\nare required to exist.  If "__all__" is not defined, the set of public\nnames includes all names found in the module\'s namespace which do not\nbegin with an underscore character ("\'_\'").  "__all__" should contain\nthe entire public API. It is intended to avoid accidentally exporting\nitems that are not part of the API (such as library modules which were\nimported and used within the module).\n\nThe wild card form of import --- "from module import *" --- is only\nallowed at the module level.  Attempting to use it in class or\nfunction definitions will raise a "SyntaxError".\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after "from" you\ncan specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n"from . import mod" from a module in the "pkg" package then you will\nend up importing "pkg.mod". If you execute "from ..subpkg2 import mod"\nfrom within "pkg.subpkg1" you will import "pkg.subpkg2.mod". The\nspecification for relative imports is contained within **PEP 328**.\n\n"importlib.import_module()" is provided to support applications that\ndetermine dynamically the modules to be loaded.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python where the feature\nbecomes standard.\n\nThe future statement is intended to ease migration to future versions\nof Python that introduce incompatible changes to the language.  It\nallows use of the new features on a per-module basis before the\nrelease in which the feature becomes standard.\n\n   future_statement ::= "from" "__future__" "import" feature ["as" name]\n                        ("," feature ["as" name])*\n                        | "from" "__future__" "import" "(" feature ["as" name]\n                        ("," feature ["as" name])* [","] ")"\n   feature          ::= identifier\n   name             ::= identifier\n\nA future statement must appear near the top of the module.  The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 3.0 are "absolute_import",\n"division", "generators", "unicode_literals", "print_function",\n"nested_scopes" and "with_statement".  They are all redundant because\nthey are always enabled, and only kept for backwards compatibility.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code.  It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently.  Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module "__future__", described later, and it will\nbe imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n   import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by calls to the built-in functions "exec()" and\n"compile()" that occur in a module "M" containing a future statement\nwill, by default, use the new syntax or semantics associated with the\nfuture statement.  This can be controlled by optional arguments to\n"compile()" --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session.  If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also: **PEP 236** - Back to the __future__\n\n     The original proposal for the __future__ mechanism.\n',
- 'in': u'\nMembership test operations\n**************************\n\nThe operators "in" and "not in" test for membership.  "x in s"\nevaluates to "True" if *x* is a member of *s*, and "False" otherwise.\n"x not in s" returns the negation of "x in s".  All built-in sequences\nand set types support this as well as dictionary, for which "in" tests\nwhether the dictionary has a given key. For container types such as\nlist, tuple, set, frozenset, dict, or collections.deque, the\nexpression "x in y" is equivalent to "any(x is e or x == e for e in\ny)".\n\nFor the string and bytes types, "x in y" is "True" if and only if *x*\nis a substring of *y*.  An equivalent test is "y.find(x) != -1".\nEmpty strings are always considered to be a substring of any other\nstring, so """ in "abc"" will return "True".\n\nFor user-defined classes which define the "__contains__()" method, "x\nin y" returns "True" if "y.__contains__(x)" returns a true value, and\n"False" otherwise.\n\nFor user-defined classes which do not define "__contains__()" but do\ndefine "__iter__()", "x in y" is "True" if some value "z" with "x ==\nz" is produced while iterating over "y".  If an exception is raised\nduring the iteration, it is as if "in" raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n"__getitem__()", "x in y" is "True" if and only if there is a non-\nnegative integer index *i* such that "x == y[i]", and all lower\ninteger indices do not raise "IndexError" exception.  (If any other\nexception is raised, it is as if "in" raised that exception).\n\nThe operator "not in" is defined to have the inverse true value of\n"in".\n',
- 'integers': u'\nInteger literals\n****************\n\nInteger literals are described by the following lexical definitions:\n\n   integer        ::= decimalinteger | octinteger | hexinteger | bininteger\n   decimalinteger ::= nonzerodigit digit* | "0"+\n   nonzerodigit   ::= "1"..."9"\n   digit          ::= "0"..."9"\n   octinteger     ::= "0" ("o" | "O") octdigit+\n   hexinteger     ::= "0" ("x" | "X") hexdigit+\n   bininteger     ::= "0" ("b" | "B") bindigit+\n   octdigit       ::= "0"..."7"\n   hexdigit       ::= digit | "a"..."f" | "A"..."F"\n   bindigit       ::= "0" | "1"\n\nThere is no limit for the length of integer literals apart from what\ncan be stored in available memory.\n\nNote that leading zeros in a non-zero decimal number are not allowed.\nThis is for disambiguation with C-style octal literals, which Python\nused before version 3.0.\n\nSome examples of integer literals:\n\n   7     2147483647                        0o177    0b100110111\n   3     79228162514264337593543950336     0o377    0xdeadbeef\n',
- 'lambda': u'\nLambdas\n*******\n\n   lambda_expr        ::= "lambda" [parameter_list]: expression\n   lambda_expr_nocond ::= "lambda" [parameter_list]: expression_nocond\n\nLambda expressions (sometimes called lambda forms) are used to create\nanonymous functions. The expression "lambda arguments: expression"\nyields a function object.  The unnamed object behaves like a function\nobject defined with:\n\n   def <lambda>(arguments):\n       return expression\n\nSee section *Function definitions* for the syntax of parameter lists.\nNote that functions created with lambda expressions cannot contain\nstatements or annotations.\n',
- 'lists': u'\nList displays\n*************\n\nA list display is a possibly empty series of expressions enclosed in\nsquare brackets:\n\n   list_display ::= "[" [starred_list | comprehension] "]"\n\nA list display yields a new list object, the contents being specified\nby either a list of expressions or a comprehension.  When a comma-\nseparated list of expressions is supplied, its elements are evaluated\nfrom left to right and placed into the list object in that order.\nWhen a comprehension is supplied, the list is constructed from the\nelements resulting from the comprehension.\n',
- 'naming': u'\nNaming and binding\n******************\n\n\nBinding of names\n================\n\n*Names* refer to objects.  Names are introduced by name binding\noperations.\n\nThe following constructs bind names: formal parameters to functions,\n"import" statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, "for" loop header, or after\n"as" in a "with" statement or "except" clause. The "import" statement\nof the form "from ... import *" binds all names defined in the\nimported module, except those beginning with an underscore.  This form\nmay only be used at the module level.\n\nA target occurring in a "del" statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name).\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name is bound in a block, it is a local variable of that block,\nunless declared as "nonlocal" or "global".  If a name is bound at the\nmodule level, it is a global variable.  (The variables of the module\ncode block are local and global.)  If a variable is used in a code\nblock but not defined there, it is a *free variable*.\n\nEach occurrence of a name in the program text refers to the *binding*\nof that name established by the following name resolution rules.\n\n\nResolution of names\n===================\n\nA *scope* defines the visibility of a name within a block.  If a local\nvariable is defined in a block, its scope includes that block.  If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name.\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope.  The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nWhen a name is not found at all, a "NameError" exception is raised. If\nthe current scope is a function scope, and the name refers to a local\nvariable that has not yet been bound to a value at the point where the\nname is used, an "UnboundLocalError" exception is raised.\n"UnboundLocalError" is a subclass of "NameError".\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block.  This can lead to errors when a name is used within a\nblock before it is bound.  This rule is subtle.  Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block.  The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the "global" statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace.  Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module "builtins".  The global namespace is searched first.  If\nthe name is not found there, the builtins namespace is searched.  The\n"global" statement must precede all uses of the name.\n\nThe "global" statement has the same scope as a name binding operation\nin the same block.  If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nThe "nonlocal" statement causes corresponding names to refer to\npreviously bound variables in the nearest enclosing function scope.\n"SyntaxError" is raised at compile time if the given name does not\nexist in any enclosing function scope.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported.  The main module for a script is always called\n"__main__".\n\nClass definition blocks and arguments to "exec()" and "eval()" are\nspecial in the context of name resolution. A class definition is an\nexecutable statement that may use and define names. These references\nfollow the normal rules for name resolution with an exception that\nunbound local variables are looked up in the global namespace. The\nnamespace of the class definition becomes the attribute dictionary of\nthe class. The scope of names defined in a class block is limited to\nthe class block; it does not extend to the code blocks of methods --\nthis includes comprehensions and generator expressions since they are\nimplemented using a function scope.  This means that the following\nwill fail:\n\n   class A:\n       a = 42\n       b = list(a + i for i in range(10))\n\n\nBuiltins and restricted execution\n=================================\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name "__builtins__" in its global\nnamespace; this should be a dictionary or a module (in the latter case\nthe module\'s dictionary is used).  By default, when in the "__main__"\nmodule, "__builtins__" is the built-in module "builtins"; when in any\nother module, "__builtins__" is an alias for the dictionary of the\n"builtins" module itself.  "__builtins__" can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n"__builtins__"; it is strictly an implementation detail.  Users\nwanting to override values in the builtins namespace should "import"\nthe "builtins" module and modify its attributes appropriately.\n\n\nInteraction with dynamic features\n=================================\n\nName resolution of free variables occurs at runtime, not at compile\ntime. This means that the following code will print 42:\n\n   i = 10\n   def f():\n       print(i)\n   i = 42\n   f()\n\nThe "eval()" and "exec()" functions do not have access to the full\nenvironment for resolving names.  Names may be resolved in the local\nand global namespaces of the caller.  Free variables are not resolved\nin the nearest enclosing namespace, but in the global namespace.  [1]\nThe "exec()" and "eval()" functions have optional arguments to\noverride the global and local namespace.  If only one namespace is\nspecified, it is used for both.\n',
- 'nonlocal': u'\nThe "nonlocal" statement\n************************\n\n   nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n\nThe "nonlocal" statement causes the listed identifiers to refer to\npreviously bound variables in the nearest enclosing scope excluding\nglobals. This is important because the default behavior for binding is\nto search the local namespace first.  The statement allows\nencapsulated code to rebind variables outside of the local scope\nbesides the global (module) scope.\n\nNames listed in a "nonlocal" statement, unlike those listed in a\n"global" statement, must refer to pre-existing bindings in an\nenclosing scope (the scope in which a new binding should be created\ncannot be determined unambiguously).\n\nNames listed in a "nonlocal" statement must not collide with pre-\nexisting bindings in the local scope.\n\nSee also: **PEP 3104** - Access to Names in Outer Scopes\n\n     The specification for the "nonlocal" statement.\n',
- 'numbers': u'\nNumeric literals\n****************\n\nThere are three types of numeric literals: integers, floating point\nnumbers, and imaginary numbers.  There are no complex literals\n(complex numbers can be formed by adding a real number and an\nimaginary number).\n\nNote that numeric literals do not include a sign; a phrase like "-1"\nis actually an expression composed of the unary operator \'"-"\' and the\nliteral "1".\n',
- 'numeric-types': u'\nEmulating numeric types\n***********************\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__matmul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations ("+", "-", "*", "@", "/", "//", "%", "divmod()",\n   "pow()", "**", "<<", ">>", "&", "^", "|").  For instance, to\n   evaluate the expression "x + y", where *x* is an instance of a\n   class that has an "__add__()" method, "x.__add__(y)" is called.\n   The "__divmod__()" method should be the equivalent to using\n   "__floordiv__()" and "__mod__()"; it should not be related to\n   "__truediv__()".  Note that "__pow__()" should be defined to accept\n   an optional third argument if the ternary version of the built-in\n   "pow()" function is to be supported.\n\n   If one of those methods does not support the operation with the\n   supplied arguments, it should return "NotImplemented".\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rmatmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations ("+", "-", "*", "@", "/", "//", "%", "divmod()",\n   "pow()", "**", "<<", ">>", "&", "^", "|") with reflected (swapped)\n   operands.  These functions are only called if the left operand does\n   not support the corresponding operation and the operands are of\n   different types. [2] For instance, to evaluate the expression "x -\n   y", where *y* is an instance of a class that has an "__rsub__()"\n   method, "y.__rsub__(x)" is called if "x.__sub__(y)" returns\n   *NotImplemented*.\n\n   Note that ternary "pow()" will not try calling "__rpow__()" (the\n   coercion rules would become too complicated).\n\n   Note: If the right operand\'s type is a subclass of the left\n     operand\'s type and that subclass provides the reflected method\n     for the operation, this method will be called before the left\n     operand\'s non-reflected method.  This behavior allows subclasses\n     to override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__imatmul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n   These methods are called to implement the augmented arithmetic\n   assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", "**=",\n   "<<=", ">>=", "&=", "^=", "|=").  These methods should attempt to\n   do the operation in-place (modifying *self*) and return the result\n   (which could be, but does not have to be, *self*).  If a specific\n   method is not defined, the augmented assignment falls back to the\n   normal methods.  For instance, if *x* is an instance of a class\n   with an "__iadd__()" method, "x += y" is equivalent to "x =\n   x.__iadd__(y)" . Otherwise, "x.__add__(y)" and "y.__radd__(x)" are\n   considered, as with the evaluation of "x + y". In certain\n   situations, augmented assignment can result in unexpected errors\n   (see *Why does a_tuple[i] += [\'item\'] raise an exception when the\n   addition works?*), but this behavior is in fact part of the data\n   model.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n   Called to implement the unary arithmetic operations ("-", "+",\n   "abs()" and "~").\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n   Called to implement the built-in functions "complex()", "int()",\n   "float()" and "round()".  Should return a value of the appropriate\n   type.\n\nobject.__index__(self)\n\n   Called to implement "operator.index()", and whenever Python needs\n   to losslessly convert the numeric object to an integer object (such\n   as in slicing, or in the built-in "bin()", "hex()" and "oct()"\n   functions). Presence of this method indicates that the numeric\n   object is an integer type.  Must return an integer.\n\n   Note: In order to have a coherent integer type class, when\n     "__index__()" is defined "__int__()" should also be defined, and\n     both should return the same value.\n',
- 'objects': u'\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data.  All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value.  An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory.  The \'"is"\' operator compares the\nidentity of two objects; the "id()" function returns an integer\nrepresenting its identity.\n\n**CPython implementation detail:** For CPython, "id(x)" is the memory\naddress where "x" is stored.\n\nAn object\'s type determines the operations that the object supports\n(e.g., "does it have a length?") and also defines the possible values\nfor objects of that type.  The "type()" function returns an object\'s\ntype (which is an object itself).  Like its identity, an object\'s\n*type* is also unchangeable. [1]\n\nThe *value* of some objects can change.  Objects whose value can\nchange are said to be *mutable*; objects whose value is unchangeable\nonce they are created are called *immutable*. (The value of an\nimmutable container object that contains a reference to a mutable\nobject can change when the latter\'s value is changed; however the\ncontainer is still considered immutable, because the collection of\nobjects it contains cannot be changed.  So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected.  An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references.  See the documentation of the "gc" module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change. Do not depend\non immediate finalization of objects when they become unreachable (so\nyou should always close files explicitly).\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'"try"..."except"\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows.  It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a "close()" method. Programs\nare strongly recommended to explicitly close such objects.  The\n\'"try"..."finally"\' statement and the \'"with"\' statement provide\nconvenient ways to do this.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries.  The references are part of a container\'s value.  In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied.  So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior.  Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed.  E.g., after "a = 1; b = 1",\n"a" and "b" may or may not refer to the same object with the value\none, depending on the implementation, but after "c = []; d = []", "c"\nand "d" are guaranteed to refer to two different, unique, newly\ncreated empty lists. (Note that "c = d = []" assigns the same object\nto both "c" and "d".)\n',
- 'operator-summary': u'\nOperator precedence\n*******************\n\nThe following table summarizes the operator precedence in Python, from\nlowest precedence (least binding) to highest precedence (most\nbinding).  Operators in the same box have the same precedence.  Unless\nthe syntax is explicitly given, operators are binary.  Operators in\nthe same box group left to right (except for exponentiation, which\ngroups from right to left).\n\nNote that comparisons, membership tests, and identity tests, all have\nthe same precedence and have a left-to-right chaining feature as\ndescribed in the *Comparisons* section.\n\n+-------------------------------------------------+---------------------------------------+\n| Operator                                        | Description                           |\n+=================================================+=======================================+\n| "lambda"                                        | Lambda expression                     |\n+-------------------------------------------------+---------------------------------------+\n| "if" -- "else"                                  | Conditional expression                |\n+-------------------------------------------------+---------------------------------------+\n| "or"                                            | Boolean OR                            |\n+-------------------------------------------------+---------------------------------------+\n| "and"                                           | Boolean AND                           |\n+-------------------------------------------------+---------------------------------------+\n| "not" "x"                                       | Boolean NOT                           |\n+-------------------------------------------------+---------------------------------------+\n| "in", "not in", "is", "is not", "<", "<=", ">", | Comparisons, including membership     |\n| ">=", "!=", "=="                                | tests and identity tests              |\n+-------------------------------------------------+---------------------------------------+\n| "|"                                             | Bitwise OR                            |\n+-------------------------------------------------+---------------------------------------+\n| "^"                                             | Bitwise XOR                           |\n+-------------------------------------------------+---------------------------------------+\n| "&"                                             | Bitwise AND                           |\n+-------------------------------------------------+---------------------------------------+\n| "<<", ">>"                                      | Shifts                                |\n+-------------------------------------------------+---------------------------------------+\n| "+", "-"                                        | Addition and subtraction              |\n+-------------------------------------------------+---------------------------------------+\n| "*", "@", "/", "//", "%"                        | Multiplication, matrix multiplication |\n|                                                 | division, remainder [5]               |\n+-------------------------------------------------+---------------------------------------+\n| "+x", "-x", "~x"                                | Positive, negative, bitwise NOT       |\n+-------------------------------------------------+---------------------------------------+\n| "**"                                            | Exponentiation [6]                    |\n+-------------------------------------------------+---------------------------------------+\n| "await" "x"                                     | Await expression                      |\n+-------------------------------------------------+---------------------------------------+\n| "x[index]", "x[index:index]",                   | Subscription, slicing, call,          |\n| "x(arguments...)", "x.attribute"                | attribute reference                   |\n+-------------------------------------------------+---------------------------------------+\n| "(expressions...)", "[expressions...]", "{key:  | Binding or tuple display, list        |\n| value...}", "{expressions...}"                  | display, dictionary display, set      |\n|                                                 | display                               |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] While "abs(x%y) < abs(y)" is true mathematically, for floats\n    it may not be true numerically due to roundoff.  For example, and\n    assuming a platform on which a Python float is an IEEE 754 double-\n    precision number, in order that "-1e-100 % 1e100" have the same\n    sign as "1e100", the computed result is "-1e-100 + 1e100", which\n    is numerically exactly equal to "1e100".  The function\n    "math.fmod()" returns a result whose sign matches the sign of the\n    first argument instead, and so returns "-1e-100" in this case.\n    Which approach is more appropriate depends on the application.\n\n[2] If x is very close to an exact integer multiple of y, it\'s\n    possible for "x//y" to be one larger than "(x-x%y)//y" due to\n    rounding.  In such cases, Python returns the latter result, in\n    order to preserve that "divmod(x,y)[0] * y + x % y" be very close\n    to "x".\n\n[3] The Unicode standard distinguishes between *code points* (e.g.\n    U+0041) and *abstract characters* (e.g. "LATIN CAPITAL LETTER A").\n    While most abstract characters in Unicode are only represented\n    using one code point, there is a number of abstract characters\n    that can in addition be represented using a sequence of more than\n    one code point.  For example, the abstract character "LATIN\n    CAPITAL LETTER C WITH CEDILLA" can be represented as a single\n    *precomposed character* at code position U+00C7, or as a sequence\n    of a *base character* at code position U+0043 (LATIN CAPITAL\n    LETTER C), followed by a *combining character* at code position\n    U+0327 (COMBINING CEDILLA).\n\n    The comparison operators on strings compare at the level of\n    Unicode code points. This may be counter-intuitive to humans.  For\n    example, ""\\u00C7" == "\\u0043\\u0327"" is "False", even though both\n    strings represent the same abstract character "LATIN CAPITAL\n    LETTER C WITH CEDILLA".\n\n    To compare strings at the level of abstract characters (that is,\n    in a way intuitive to humans), use "unicodedata.normalize()".\n\n[4] Due to automatic garbage-collection, free lists, and the\n    dynamic nature of descriptors, you may notice seemingly unusual\n    behaviour in certain uses of the "is" operator, like those\n    involving comparisons between instance methods, or constants.\n    Check their documentation for more info.\n\n[5] The "%" operator is also used for string formatting; the same\n    precedence applies.\n\n[6] The power operator "**" binds less tightly than an arithmetic\n    or bitwise unary operator on its right, that is, "2**-1" is "0.5".\n',
- 'pass': u'\nThe "pass" statement\n********************\n\n   pass_stmt ::= "pass"\n\n"pass" is a null operation --- when it is executed, nothing happens.\nIt is useful as a placeholder when a statement is required\nsyntactically, but no code needs to be executed, for example:\n\n   def f(arg): pass    # a function that does nothing (yet)\n\n   class C: pass       # a class with no methods (yet)\n',
- 'power': u'\nThe power operator\n******************\n\nThe power operator binds more tightly than unary operators on its\nleft; it binds less tightly than unary operators on its right.  The\nsyntax is:\n\n   power ::= ( await_expr | primary ) ["**" u_expr]\n\nThus, in an unparenthesized sequence of power and unary operators, the\noperators are evaluated from right to left (this does not constrain\nthe evaluation order for the operands): "-1**2" results in "-1".\n\nThe power operator has the same semantics as the built-in "pow()"\nfunction, when called with two arguments: it yields its left argument\nraised to the power of its right argument.  The numeric arguments are\nfirst converted to a common type, and the result is of that type.\n\nFor int operands, the result has the same type as the operands unless\nthe second argument is negative; in that case, all arguments are\nconverted to float and a float result is delivered. For example,\n"10**2" returns "100", but "10**-2" returns "0.01".\n\nRaising "0.0" to a negative power results in a "ZeroDivisionError".\nRaising a negative number to a fractional power results in a "complex"\nnumber. (In earlier versions it raised a "ValueError".)\n',
- 'raise': u'\nThe "raise" statement\n*********************\n\n   raise_stmt ::= "raise" [expression ["from" expression]]\n\nIf no expressions are present, "raise" re-raises the last exception\nthat was active in the current scope.  If no exception is active in\nthe current scope, a "RuntimeError" exception is raised indicating\nthat this is an error.\n\nOtherwise, "raise" evaluates the first expression as the exception\nobject.  It must be either a subclass or an instance of\n"BaseException". If it is a class, the exception instance will be\nobtained when needed by instantiating the class with no arguments.\n\nThe *type* of the exception is the exception instance\'s class, the\n*value* is the instance itself.\n\nA traceback object is normally created automatically when an exception\nis raised and attached to it as the "__traceback__" attribute, which\nis writable. You can create an exception and set your own traceback in\none step using the "with_traceback()" exception method (which returns\nthe same exception instance, with its traceback set to its argument),\nlike so:\n\n   raise Exception("foo occurred").with_traceback(tracebackobj)\n\nThe "from" clause is used for exception chaining: if given, the second\n*expression* must be another exception class or instance, which will\nthen be attached to the raised exception as the "__cause__" attribute\n(which is writable).  If the raised exception is not handled, both\nexceptions will be printed:\n\n   >>> try:\n   ...     print(1 / 0)\n   ... except Exception as exc:\n   ...     raise RuntimeError("Something bad happened") from exc\n   ...\n   Traceback (most recent call last):\n     File "<stdin>", line 2, in <module>\n   ZeroDivisionError: int division or modulo by zero\n\n   The above exception was the direct cause of the following exception:\n\n   Traceback (most recent call last):\n     File "<stdin>", line 4, in <module>\n   RuntimeError: Something bad happened\n\nA similar mechanism works implicitly if an exception is raised inside\nan exception handler or a "finally" clause: the previous exception is\nthen attached as the new exception\'s "__context__" attribute:\n\n   >>> try:\n   ...     print(1 / 0)\n   ... except:\n   ...     raise RuntimeError("Something bad happened")\n   ...\n   Traceback (most recent call last):\n     File "<stdin>", line 2, in <module>\n   ZeroDivisionError: int division or modulo by zero\n\n   During handling of the above exception, another exception occurred:\n\n   Traceback (most recent call last):\n     File "<stdin>", line 4, in <module>\n   RuntimeError: Something bad happened\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information about handling exceptions is in section\n*The try statement*.\n',
- 'return': u'\nThe "return" statement\n**********************\n\n   return_stmt ::= "return" [expression_list]\n\n"return" may only occur syntactically nested in a function definition,\nnot within a nested class definition.\n\nIf an expression list is present, it is evaluated, else "None" is\nsubstituted.\n\n"return" leaves the current function call with the expression list (or\n"None") as return value.\n\nWhen "return" passes control out of a "try" statement with a "finally"\nclause, that "finally" clause is executed before really leaving the\nfunction.\n\nIn a generator function, the "return" statement indicates that the\ngenerator is done and will cause "StopIteration" to be raised. The\nreturned value (if any) is used as an argument to construct\n"StopIteration" and becomes the "StopIteration.value" attribute.\n',
- 'sequence-types': u'\nEmulating container types\n*************************\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well.  The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which "0 <= k < N" where\n*N* is the length of the sequence, or slice objects, which define a\nrange of items.  It is also recommended that mappings provide the\nmethods "keys()", "values()", "items()", "get()", "clear()",\n"setdefault()", "pop()", "popitem()", "copy()", and "update()"\nbehaving similar to those for Python\'s standard dictionary objects.\nThe "collections" module provides a "MutableMapping" abstract base\nclass to help create those methods from a base set of "__getitem__()",\n"__setitem__()", "__delitem__()", and "keys()". Mutable sequences\nshould provide methods "append()", "count()", "index()", "extend()",\n"insert()", "pop()", "remove()", "reverse()" and "sort()", like Python\nstandard list objects.  Finally, sequence types should implement\naddition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods "__add__()", "__radd__()",\n"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" described\nbelow; they should not define other numerical operators.  It is\nrecommended that both mappings and sequences implement the\n"__contains__()" method to allow efficient use of the "in" operator;\nfor mappings, "in" should search the mapping\'s keys; for sequences, it\nshould search through the values.  It is further recommended that both\nmappings and sequences implement the "__iter__()" method to allow\nefficient iteration through the container; for mappings, "__iter__()"\nshould be the same as "keys()"; for sequences, it should iterate\nthrough the values.\n\nobject.__len__(self)\n\n   Called to implement the built-in function "len()".  Should return\n   the length of the object, an integer ">=" 0.  Also, an object that\n   doesn\'t define a "__bool__()" method and whose "__len__()" method\n   returns zero is considered to be false in a Boolean context.\n\n   **CPython implementation detail:** In CPython, the length is\n   required to be at most "sys.maxsize". If the length is larger than\n   "sys.maxsize" some features (such as "len()") may raise\n   "OverflowError".  To prevent raising "OverflowError" by truth value\n   testing, an object must define a "__bool__()" method.\n\nobject.__length_hint__(self)\n\n   Called to implement "operator.length_hint()". Should return an\n   estimated length for the object (which may be greater or less than\n   the actual length). The length must be an integer ">=" 0. This\n   method is purely an optimization and is never required for\n   correctness.\n\n   New in version 3.4.\n\nNote: Slicing is done exclusively with the following three methods.\n  A call like\n\n     a[1:2] = b\n\n  is translated to\n\n     a[slice(1, 2, None)] = b\n\n  and so forth.  Missing slice items are always filled in with "None".\n\nobject.__getitem__(self, key)\n\n   Called to implement evaluation of "self[key]". For sequence types,\n   the accepted keys should be integers and slice objects.  Note that\n   the special interpretation of negative indexes (if the class wishes\n   to emulate a sequence type) is up to the "__getitem__()" method. If\n   *key* is of an inappropriate type, "TypeError" may be raised; if of\n   a value outside the set of indexes for the sequence (after any\n   special interpretation of negative values), "IndexError" should be\n   raised. For mapping types, if *key* is missing (not in the\n   container), "KeyError" should be raised.\n\n   Note: "for" loops expect that an "IndexError" will be raised for\n     illegal indexes to allow proper detection of the end of the\n     sequence.\n\nobject.__missing__(self, key)\n\n   Called by "dict"."__getitem__()" to implement "self[key]" for dict\n   subclasses when key is not in the dictionary.\n\nobject.__setitem__(self, key, value)\n\n   Called to implement assignment to "self[key]".  Same note as for\n   "__getitem__()".  This should only be implemented for mappings if\n   the objects support changes to the values for keys, or if new keys\n   can be added, or for sequences if elements can be replaced.  The\n   same exceptions should be raised for improper *key* values as for\n   the "__getitem__()" method.\n\nobject.__delitem__(self, key)\n\n   Called to implement deletion of "self[key]".  Same note as for\n   "__getitem__()".  This should only be implemented for mappings if\n   the objects support removal of keys, or for sequences if elements\n   can be removed from the sequence.  The same exceptions should be\n   raised for improper *key* values as for the "__getitem__()" method.\n\nobject.__iter__(self)\n\n   This method is called when an iterator is required for a container.\n   This method should return a new iterator object that can iterate\n   over all the objects in the container.  For mappings, it should\n   iterate over the keys of the container.\n\n   Iterator objects also need to implement this method; they are\n   required to return themselves.  For more information on iterator\n   objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n   Called (if present) by the "reversed()" built-in to implement\n   reverse iteration.  It should return a new iterator object that\n   iterates over all the objects in the container in reverse order.\n\n   If the "__reversed__()" method is not provided, the "reversed()"\n   built-in will fall back to using the sequence protocol ("__len__()"\n   and "__getitem__()").  Objects that support the sequence protocol\n   should only provide "__reversed__()" if they can provide an\n   implementation that is more efficient than the one provided by\n   "reversed()".\n\nThe membership test operators ("in" and "not in") are normally\nimplemented as an iteration through a sequence.  However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n   Called to implement membership test operators.  Should return true\n   if *item* is in *self*, false otherwise.  For mapping objects, this\n   should consider the keys of the mapping rather than the values or\n   the key-item pairs.\n\n   For objects that don\'t define "__contains__()", the membership test\n   first tries iteration via "__iter__()", then the old sequence\n   iteration protocol via "__getitem__()", see *this section in the\n   language reference*.\n',
- 'shifting': u'\nShifting operations\n*******************\n\nThe shifting operations have lower priority than the arithmetic\noperations:\n\n   shift_expr ::= a_expr | shift_expr ( "<<" | ">>" ) a_expr\n\nThese operators accept integers as arguments.  They shift the first\nargument to the left or right by the number of bits given by the\nsecond argument.\n\nA right shift by *n* bits is defined as floor division by "pow(2,n)".\nA left shift by *n* bits is defined as multiplication with "pow(2,n)".\n\nNote: In the current implementation, the right-hand operand is\n  required to be at most "sys.maxsize".  If the right-hand operand is\n  larger than "sys.maxsize" an "OverflowError" exception is raised.\n',
- 'slicings': u'\nSlicings\n********\n\nA slicing selects a range of items in a sequence object (e.g., a\nstring, tuple or list).  Slicings may be used as expressions or as\ntargets in assignment or "del" statements.  The syntax for a slicing:\n\n   slicing      ::= primary "[" slice_list "]"\n   slice_list   ::= slice_item ("," slice_item)* [","]\n   slice_item   ::= expression | proper_slice\n   proper_slice ::= [lower_bound] ":" [upper_bound] [ ":" [stride] ]\n   lower_bound  ::= expression\n   upper_bound  ::= expression\n   stride       ::= expression\n\nThere is ambiguity in the formal syntax here: anything that looks like\nan expression list also looks like a slice list, so any subscription\ncan be interpreted as a slicing.  Rather than further complicating the\nsyntax, this is disambiguated by defining that in this case the\ninterpretation as a subscription takes priority over the\ninterpretation as a slicing (this is the case if the slice list\ncontains no proper slice).\n\nThe semantics for a slicing are as follows.  The primary is indexed\n(using the same "__getitem__()" method as normal subscription) with a\nkey that is constructed from the slice list, as follows.  If the slice\nlist contains at least one comma, the key is a tuple containing the\nconversion of the slice items; otherwise, the conversion of the lone\nslice item is the key.  The conversion of a slice item that is an\nexpression is that expression.  The conversion of a proper slice is a\nslice object (see section *The standard type hierarchy*) whose\n"start", "stop" and "step" attributes are the values of the\nexpressions given as lower bound, upper bound and stride,\nrespectively, substituting "None" for missing expressions.\n',
- 'specialattrs': u'\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant.  Some of these are not reported\nby the "dir()" built-in function.\n\nobject.__dict__\n\n   A dictionary or other mapping object used to store an object\'s\n   (writable) attributes.\n\ninstance.__class__\n\n   The class to which a class instance belongs.\n\nclass.__bases__\n\n   The tuple of base classes of a class object.\n\ndefinition.__name__\n\n   The name of the class, function, method, descriptor, or generator\n   instance.\n\ndefinition.__qualname__\n\n   The *qualified name* of the class, function, method, descriptor, or\n   generator instance.\n\n   New in version 3.3.\n\nclass.__mro__\n\n   This attribute is a tuple of classes that are considered when\n   looking for base classes during method resolution.\n\nclass.mro()\n\n   This method can be overridden by a metaclass to customize the\n   method resolution order for its instances.  It is called at class\n   instantiation, and its result is stored in "__mro__".\n\nclass.__subclasses__()\n\n   Each class keeps a list of weak references to its immediate\n   subclasses.  This method returns a list of all those references\n   still alive. Example:\n\n      >>> int.__subclasses__()\n      [<class \'bool\'>]\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found\n    in the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list "[1, 2]" is considered equal to\n    "[1.0, 2.0]", and similarly for tuples.\n\n[3] They must have since the parser can\'t tell the type of the\n    operands.\n\n[4] Cased characters are those with general category property\n    being one of "Lu" (Letter, uppercase), "Ll" (Letter, lowercase),\n    or "Lt" (Letter, titlecase).\n\n[5] To format only a tuple you should therefore provide a\n    singleton tuple whose only element is the tuple to be formatted.\n',
- 'specialnames': u'\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators.  For instance, if a class defines\na method named "__getitem__()", and "x" is an instance of this class,\nthen "x[i]" is roughly equivalent to "type(x).__getitem__(x, i)".\nExcept where mentioned, attempts to execute an operation raise an\nexception when no appropriate method is defined (typically\n"AttributeError" or "TypeError").\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled.  For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense.  (One example of this is the\n"NodeList" interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n   Called to create a new instance of class *cls*.  "__new__()" is a\n   static method (special-cased so you need not declare it as such)\n   that takes the class of which an instance was requested as its\n   first argument.  The remaining arguments are those passed to the\n   object constructor expression (the call to the class).  The return\n   value of "__new__()" should be the new object instance (usually an\n   instance of *cls*).\n\n   Typical implementations create a new instance of the class by\n   invoking the superclass\'s "__new__()" method using\n   "super().__new__(cls[, ...])" with appropriate arguments and then\n   modifying the newly-created instance as necessary before returning\n   it.\n\n   If "__new__()" returns an instance of *cls*, then the new\n   instance\'s "__init__()" method will be invoked like\n   "__init__(self[, ...])", where *self* is the new instance and the\n   remaining arguments are the same as were passed to "__new__()".\n\n   If "__new__()" does not return an instance of *cls*, then the new\n   instance\'s "__init__()" method will not be invoked.\n\n   "__new__()" is intended mainly to allow subclasses of immutable\n   types (like int, str, or tuple) to customize instance creation.  It\n   is also commonly overridden in custom metaclasses in order to\n   customize class creation.\n\nobject.__init__(self[, ...])\n\n   Called after the instance has been created (by "__new__()"), but\n   before it is returned to the caller.  The arguments are those\n   passed to the class constructor expression.  If a base class has an\n   "__init__()" method, the derived class\'s "__init__()" method, if\n   any, must explicitly call it to ensure proper initialization of the\n   base class part of the instance; for example:\n   "super().__init__([args...])".\n\n   Because "__new__()" and "__init__()" work together in constructing\n   objects ("__new__()" to create it, and "__init__()" to customize\n   it), no non-"None" value may be returned by "__init__()"; doing so\n   will cause a "TypeError" to be raised at runtime.\n\nobject.__del__(self)\n\n   Called when the instance is about to be destroyed.  This is also\n   called a destructor.  If a base class has a "__del__()" method, the\n   derived class\'s "__del__()" method, if any, must explicitly call it\n   to ensure proper deletion of the base class part of the instance.\n   Note that it is possible (though not recommended!) for the\n   "__del__()" method to postpone destruction of the instance by\n   creating a new reference to it.  It may then be called at a later\n   time when this new reference is deleted.  It is not guaranteed that\n   "__del__()" methods are called for objects that still exist when\n   the interpreter exits.\n\n   Note: "del x" doesn\'t directly call "x.__del__()" --- the former\n     decrements the reference count for "x" by one, and the latter is\n     only called when "x"\'s reference count reaches zero.  Some common\n     situations that may prevent the reference count of an object from\n     going to zero include: circular references between objects (e.g.,\n     a doubly-linked list or a tree data structure with parent and\n     child pointers); a reference to the object on the stack frame of\n     a function that caught an exception (the traceback stored in\n     "sys.exc_info()[2]" keeps the stack frame alive); or a reference\n     to the object on the stack frame that raised an unhandled\n     exception in interactive mode (the traceback stored in\n     "sys.last_traceback" keeps the stack frame alive).  The first\n     situation can only be remedied by explicitly breaking the cycles;\n     the second can be resolved by freeing the reference to the\n     traceback object when it is no longer useful, and the third can\n     be resolved by storing "None" in "sys.last_traceback". Circular\n     references which are garbage are detected and cleaned up when the\n     cyclic garbage collector is enabled (it\'s on by default). Refer\n     to the documentation for the "gc" module for more information\n     about this topic.\n\n   Warning: Due to the precarious circumstances under which\n     "__del__()" methods are invoked, exceptions that occur during\n     their execution are ignored, and a warning is printed to\n     "sys.stderr" instead. Also, when "__del__()" is invoked in\n     response to a module being deleted (e.g., when execution of the\n     program is done), other globals referenced by the "__del__()"\n     method may already have been deleted or in the process of being\n     torn down (e.g. the import machinery shutting down).  For this\n     reason, "__del__()" methods should do the absolute minimum needed\n     to maintain external invariants.  Starting with version 1.5,\n     Python guarantees that globals whose name begins with a single\n     underscore are deleted from their module before other globals are\n     deleted; if no other references to such globals exist, this may\n     help in assuring that imported modules are still available at the\n     time when the "__del__()" method is called.\n\nobject.__repr__(self)\n\n   Called by the "repr()" built-in function to compute the "official"\n   string representation of an object.  If at all possible, this\n   should look like a valid Python expression that could be used to\n   recreate an object with the same value (given an appropriate\n   environment).  If this is not possible, a string of the form\n   "<...some useful description...>" should be returned. The return\n   value must be a string object. If a class defines "__repr__()" but\n   not "__str__()", then "__repr__()" is also used when an "informal"\n   string representation of instances of that class is required.\n\n   This is typically used for debugging, so it is important that the\n   representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n   Called by "str(object)" and the built-in functions "format()" and\n   "print()" to compute the "informal" or nicely printable string\n   representation of an object.  The return value must be a *string*\n   object.\n\n   This method differs from "object.__repr__()" in that there is no\n   expectation that "__str__()" return a valid Python expression: a\n   more convenient or concise representation can be used.\n\n   The default implementation defined by the built-in type "object"\n   calls "object.__repr__()".\n\nobject.__bytes__(self)\n\n   Called by "bytes()" to compute a byte-string representation of an\n   object. This should return a "bytes" object.\n\nobject.__format__(self, format_spec)\n\n   Called by the "format()" built-in function (and by extension, the\n   "str.format()" method of class "str") to produce a "formatted"\n   string representation of an object. The "format_spec" argument is a\n   string that contains a description of the formatting options\n   desired. The interpretation of the "format_spec" argument is up to\n   the type implementing "__format__()", however most classes will\n   either delegate formatting to one of the built-in types, or use a\n   similar formatting option syntax.\n\n   See *Format Specification Mini-Language* for a description of the\n   standard formatting syntax.\n\n   The return value must be a string object.\n\n   Changed in version 3.4: The __format__ method of "object" itself\n   raises a "TypeError" if passed any non-empty string.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n   These are the so-called "rich comparison" methods. The\n   correspondence between operator symbols and method names is as\n   follows: "x<y" calls "x.__lt__(y)", "x<=y" calls "x.__le__(y)",\n   "x==y" calls "x.__eq__(y)", "x!=y" calls "x.__ne__(y)", "x>y" calls\n   "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n\n   A rich comparison method may return the singleton "NotImplemented"\n   if it does not implement the operation for a given pair of\n   arguments. By convention, "False" and "True" are returned for a\n   successful comparison. However, these methods can return any value,\n   so if the comparison operator is used in a Boolean context (e.g.,\n   in the condition of an "if" statement), Python will call "bool()"\n   on the value to determine if the result is true or false.\n\n   By default, "__ne__()" delegates to "__eq__()" and inverts the\n   result unless it is "NotImplemented".  There are no other implied\n   relationships among the comparison operators, for example, the\n   truth of "(x<y or x==y)" does not imply "x<=y". To automatically\n   generate ordering operations from a single root operation, see\n   "functools.total_ordering()".\n\n   See the paragraph on "__hash__()" for some important notes on\n   creating *hashable* objects which support custom comparison\n   operations and are usable as dictionary keys.\n\n   There are no swapped-argument versions of these methods (to be used\n   when the left argument does not support the operation but the right\n   argument does); rather, "__lt__()" and "__gt__()" are each other\'s\n   reflection, "__le__()" and "__ge__()" are each other\'s reflection,\n   and "__eq__()" and "__ne__()" are their own reflection. If the\n   operands are of different types, and right operand\'s type is a\n   direct or indirect subclass of the left operand\'s type, the\n   reflected method of the right operand has priority, otherwise the\n   left operand\'s method has priority.  Virtual subclassing is not\n   considered.\n\nobject.__hash__(self)\n\n   Called by built-in function "hash()" and for operations on members\n   of hashed collections including "set", "frozenset", and "dict".\n   "__hash__()" should return an integer. The only required property\n   is that objects which compare equal have the same hash value; it is\n   advised to mix together the hash values of the components of the\n   object that also play a part in comparison of objects by packing\n   them into a tuple and hashing the tuple. Example:\n\n      def __hash__(self):\n          return hash((self.name, self.nick, self.color))\n\n   Note: "hash()" truncates the value returned from an object\'s\n     custom "__hash__()" method to the size of a "Py_ssize_t".  This\n     is typically 8 bytes on 64-bit builds and 4 bytes on 32-bit\n     builds. If an object\'s   "__hash__()" must interoperate on builds\n     of different bit sizes, be sure to check the width on all\n     supported builds.  An easy way to do this is with "python -c\n     "import sys; print(sys.hash_info.width)"".\n\n   If a class does not define an "__eq__()" method it should not\n   define a "__hash__()" operation either; if it defines "__eq__()"\n   but not "__hash__()", its instances will not be usable as items in\n   hashable collections.  If a class defines mutable objects and\n   implements an "__eq__()" method, it should not implement\n   "__hash__()", since the implementation of hashable collections\n   requires that a key\'s hash value is immutable (if the object\'s hash\n   value changes, it will be in the wrong hash bucket).\n\n   User-defined classes have "__eq__()" and "__hash__()" methods by\n   default; with them, all objects compare unequal (except with\n   themselves) and "x.__hash__()" returns an appropriate value such\n   that "x == y" implies both that "x is y" and "hash(x) == hash(y)".\n\n   A class that overrides "__eq__()" and does not define "__hash__()"\n   will have its "__hash__()" implicitly set to "None".  When the\n   "__hash__()" method of a class is "None", instances of the class\n   will raise an appropriate "TypeError" when a program attempts to\n   retrieve their hash value, and will also be correctly identified as\n   unhashable when checking "isinstance(obj, collections.Hashable)".\n\n   If a class that overrides "__eq__()" needs to retain the\n   implementation of "__hash__()" from a parent class, the interpreter\n   must be told this explicitly by setting "__hash__ =\n   <ParentClass>.__hash__".\n\n   If a class that does not override "__eq__()" wishes to suppress\n   hash support, it should include "__hash__ = None" in the class\n   definition. A class which defines its own "__hash__()" that\n   explicitly raises a "TypeError" would be incorrectly identified as\n   hashable by an "isinstance(obj, collections.Hashable)" call.\n\n   Note: By default, the "__hash__()" values of str, bytes and\n     datetime objects are "salted" with an unpredictable random value.\n     Although they remain constant within an individual Python\n     process, they are not predictable between repeated invocations of\n     Python.This is intended to provide protection against a denial-\n     of-service caused by carefully-chosen inputs that exploit the\n     worst case performance of a dict insertion, O(n^2) complexity.\n     See http://www.ocert.org/advisories/ocert-2011-003.html for\n     details.Changing hash values affects the iteration order of\n     dicts, sets and other mappings.  Python has never made guarantees\n     about this ordering (and it typically varies between 32-bit and\n     64-bit builds).See also "PYTHONHASHSEED".\n\n   Changed in version 3.3: Hash randomization is enabled by default.\n\nobject.__bool__(self)\n\n   Called to implement truth value testing and the built-in operation\n   "bool()"; should return "False" or "True".  When this method is not\n   defined, "__len__()" is called, if it is defined, and the object is\n   considered true if its result is nonzero.  If a class defines\n   neither "__len__()" nor "__bool__()", all its instances are\n   considered true.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of "x.name") for\nclass instances.\n\nobject.__getattr__(self, name)\n\n   Called when an attribute lookup has not found the attribute in the\n   usual places (i.e. it is not an instance attribute nor is it found\n   in the class tree for "self").  "name" is the attribute name. This\n   method should return the (computed) attribute value or raise an\n   "AttributeError" exception.\n\n   Note that if the attribute is found through the normal mechanism,\n   "__getattr__()" is not called.  (This is an intentional asymmetry\n   between "__getattr__()" and "__setattr__()".) This is done both for\n   efficiency reasons and because otherwise "__getattr__()" would have\n   no way to access other attributes of the instance.  Note that at\n   least for instance variables, you can fake total control by not\n   inserting any values in the instance attribute dictionary (but\n   instead inserting them in another object).  See the\n   "__getattribute__()" method below for a way to actually get total\n   control over attribute access.\n\nobject.__getattribute__(self, name)\n\n   Called unconditionally to implement attribute accesses for\n   instances of the class. If the class also defines "__getattr__()",\n   the latter will not be called unless "__getattribute__()" either\n   calls it explicitly or raises an "AttributeError". This method\n   should return the (computed) attribute value or raise an\n   "AttributeError" exception. In order to avoid infinite recursion in\n   this method, its implementation should always call the base class\n   method with the same name to access any attributes it needs, for\n   example, "object.__getattribute__(self, name)".\n\n   Note: This method may still be bypassed when looking up special\n     methods as the result of implicit invocation via language syntax\n     or built-in functions. See *Special method lookup*.\n\nobject.__setattr__(self, name, value)\n\n   Called when an attribute assignment is attempted.  This is called\n   instead of the normal mechanism (i.e. store the value in the\n   instance dictionary). *name* is the attribute name, *value* is the\n   value to be assigned to it.\n\n   If "__setattr__()" wants to assign to an instance attribute, it\n   should call the base class method with the same name, for example,\n   "object.__setattr__(self, name, value)".\n\nobject.__delattr__(self, name)\n\n   Like "__setattr__()" but for attribute deletion instead of\n   assignment.  This should only be implemented if "del obj.name" is\n   meaningful for the object.\n\nobject.__dir__(self)\n\n   Called when "dir()" is called on the object. A sequence must be\n   returned. "dir()" converts the returned sequence to a list and\n   sorts it.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents).  In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' "__dict__".\n\nobject.__get__(self, instance, owner)\n\n   Called to get the attribute of the owner class (class attribute\n   access) or of an instance of that class (instance attribute\n   access). *owner* is always the owner class, while *instance* is the\n   instance that the attribute was accessed through, or "None" when\n   the attribute is accessed through the *owner*.  This method should\n   return the (computed) attribute value or raise an "AttributeError"\n   exception.\n\nobject.__set__(self, instance, value)\n\n   Called to set the attribute on an instance *instance* of the owner\n   class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n   Called to delete the attribute on an instance *instance* of the\n   owner class.\n\nThe attribute "__objclass__" is interpreted by the "inspect" module as\nspecifying the class where this object was defined (setting this\nappropriately can assist in runtime introspection of dynamic class\nattributes). For callables, it may indicate that an instance of the\ngiven type (or a subclass) is expected or required as the first\npositional argument (for example, CPython sets this attribute for\nunbound methods that are implemented in C).\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol:  "__get__()", "__set__()", and\n"__delete__()". If any of those methods are defined for an object, it\nis said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, "a.x" has a\nlookup chain starting with "a.__dict__[\'x\']", then\n"type(a).__dict__[\'x\']", and continuing through the base classes of\n"type(a)" excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead.  Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.\n\nThe starting point for descriptor invocation is a binding, "a.x". How\nthe arguments are assembled depends on "a":\n\nDirect Call\n   The simplest and least common call is when user code directly\n   invokes a descriptor method:    "x.__get__(a)".\n\nInstance Binding\n   If binding to an object instance, "a.x" is transformed into the\n   call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n\nClass Binding\n   If binding to a class, "A.x" is transformed into the call:\n   "A.__dict__[\'x\'].__get__(None, A)".\n\nSuper Binding\n   If "a" is an instance of "super", then the binding "super(B,\n   obj).m()" searches "obj.__class__.__mro__" for the base class "A"\n   immediately preceding "B" and then invokes the descriptor with the\n   call: "A.__dict__[\'m\'].__get__(obj, obj.__class__)".\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined.  A descriptor can define\nany combination of "__get__()", "__set__()" and "__delete__()".  If it\ndoes not define "__get__()", then accessing the attribute will return\nthe descriptor object itself unless there is a value in the object\'s\ninstance dictionary.  If the descriptor defines "__set__()" and/or\n"__delete__()", it is a data descriptor; if it defines neither, it is\na non-data descriptor.  Normally, data descriptors define both\n"__get__()" and "__set__()", while non-data descriptors have just the\n"__get__()" method.  Data descriptors with "__set__()" and "__get__()"\ndefined always override a redefinition in an instance dictionary.  In\ncontrast, non-data descriptors can be overridden by instances.\n\nPython methods (including "staticmethod()" and "classmethod()") are\nimplemented as non-data descriptors.  Accordingly, instances can\nredefine and override methods.  This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe "property()" function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of classes have a dictionary for attribute\nstorage.  This wastes space for objects having very few instance\nvariables.  The space consumption can become acute when creating large\nnumbers of instances.\n\nThe default can be overridden by defining *__slots__* in a class\ndefinition. The *__slots__* declaration takes a sequence of instance\nvariables and reserves just enough space in each instance to hold a\nvalue for each variable.  Space is saved because *__dict__* is not\ncreated for each instance.\n\nobject.__slots__\n\n   This class variable can be assigned a string, iterable, or sequence\n   of strings with variable names used by instances.  *__slots__*\n   reserves space for the declared variables and prevents the\n   automatic creation of *__dict__* and *__weakref__* for each\n   instance.\n\n\nNotes on using *__slots__*\n~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n  attribute of that class will always be accessible, so a *__slots__*\n  definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n  variables not listed in the *__slots__* definition.  Attempts to\n  assign to an unlisted variable name raises "AttributeError". If\n  dynamic assignment of new variables is desired, then add\n  "\'__dict__\'" to the sequence of strings in the *__slots__*\n  declaration.\n\n* Without a *__weakref__* variable for each instance, classes\n  defining *__slots__* do not support weak references to its\n  instances. If weak reference support is needed, then add\n  "\'__weakref__\'" to the sequence of strings in the *__slots__*\n  declaration.\n\n* *__slots__* are implemented at the class level by creating\n  descriptors (*Implementing Descriptors*) for each variable name.  As\n  a result, class attributes cannot be used to set default values for\n  instance variables defined by *__slots__*; otherwise, the class\n  attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n  where it is defined.  As a result, subclasses will have a *__dict__*\n  unless they also define *__slots__* (which must only contain names\n  of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the\n  instance variable defined by the base class slot is inaccessible\n  (except by retrieving its descriptor directly from the base class).\n  This renders the meaning of the program undefined.  In the future, a\n  check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n  "variable-length" built-in types such as "int", "bytes" and "tuple".\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings\n  may also be used; however, in the future, special meaning may be\n  assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n  *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, classes are constructed using "type()". The class body is\nexecuted in a new namespace and the class name is bound locally to the\nresult of "type(name, bases, namespace)".\n\nThe class creation process can be customized by passing the\n"metaclass" keyword argument in the class definition line, or by\ninheriting from an existing class that included such an argument. In\nthe following example, both "MyClass" and "MySubclass" are instances\nof "Meta":\n\n   class Meta(type):\n       pass\n\n   class MyClass(metaclass=Meta):\n       pass\n\n   class MySubclass(MyClass):\n       pass\n\nAny other keyword arguments that are specified in the class definition\nare passed through to all metaclass operations described below.\n\nWhen a class definition is executed, the following steps occur:\n\n* the appropriate metaclass is determined\n\n* the class namespace is prepared\n\n* the class body is executed\n\n* the class object is created\n\n\nDetermining the appropriate metaclass\n-------------------------------------\n\nThe appropriate metaclass for a class definition is determined as\nfollows:\n\n* if no bases and no explicit metaclass are given, then "type()" is\n  used\n\n* if an explicit metaclass is given and it is *not* an instance of\n  "type()", then it is used directly as the metaclass\n\n* if an instance of "type()" is given as the explicit metaclass, or\n  bases are defined, then the most derived metaclass is used\n\nThe most derived metaclass is selected from the explicitly specified\nmetaclass (if any) and the metaclasses (i.e. "type(cls)") of all\nspecified base classes. The most derived metaclass is one which is a\nsubtype of *all* of these candidate metaclasses. If none of the\ncandidate metaclasses meets that criterion, then the class definition\nwill fail with "TypeError".\n\n\nPreparing the class namespace\n-----------------------------\n\nOnce the appropriate metaclass has been identified, then the class\nnamespace is prepared. If the metaclass has a "__prepare__" attribute,\nit is called as "namespace = metaclass.__prepare__(name, bases,\n**kwds)" (where the additional keyword arguments, if any, come from\nthe class definition).\n\nIf the metaclass has no "__prepare__" attribute, then the class\nnamespace is initialised as an empty "dict()" instance.\n\nSee also: **PEP 3115** - Metaclasses in Python 3000\n\n     Introduced the "__prepare__" namespace hook\n\n\nExecuting the class body\n------------------------\n\nThe class body is executed (approximately) as "exec(body, globals(),\nnamespace)". The key difference from a normal call to "exec()" is that\nlexical scoping allows the class body (including any methods) to\nreference names from the current and outer scopes when the class\ndefinition occurs inside a function.\n\nHowever, even when the class definition occurs inside the function,\nmethods defined inside the class still cannot see names defined at the\nclass scope. Class variables must be accessed through the first\nparameter of instance or class methods, and cannot be accessed at all\nfrom static methods.\n\n\nCreating the class object\n-------------------------\n\nOnce the class namespace has been populated by executing the class\nbody, the class object is created by calling "metaclass(name, bases,\nnamespace, **kwds)" (the additional keywords passed here are the same\nas those passed to "__prepare__").\n\nThis class object is the one that will be referenced by the zero-\nargument form of "super()". "__class__" is an implicit closure\nreference created by the compiler if any methods in a class body refer\nto either "__class__" or "super". This allows the zero argument form\nof "super()" to correctly identify the class being defined based on\nlexical scoping, while the class or instance that was used to make the\ncurrent call is identified based on the first argument passed to the\nmethod.\n\nAfter the class object is created, it is passed to the class\ndecorators included in the class definition (if any) and the resulting\nobject is bound in the local namespace as the defined class.\n\nWhen a new class is created by "type.__new__", the object provided as\nthe namespace parameter is copied to a standard Python dictionary and\nthe original object is discarded. The new copy becomes the "__dict__"\nattribute of the class object.\n\nSee also: **PEP 3135** - New super\n\n     Describes the implicit "__class__" closure reference\n\n\nMetaclass example\n-----------------\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored include logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\nHere is an example of a metaclass that uses an\n"collections.OrderedDict" to remember the order that class variables\nare defined:\n\n   class OrderedClass(type):\n\n       @classmethod\n       def __prepare__(metacls, name, bases, **kwds):\n           return collections.OrderedDict()\n\n       def __new__(cls, name, bases, namespace, **kwds):\n           result = type.__new__(cls, name, bases, dict(namespace))\n           result.members = tuple(namespace)\n           return result\n\n   class A(metaclass=OrderedClass):\n       def one(self): pass\n       def two(self): pass\n       def three(self): pass\n       def four(self): pass\n\n   >>> A.members\n   (\'__module__\', \'one\', \'two\', \'three\', \'four\')\n\nWhen the class definition for *A* gets executed, the process begins\nwith calling the metaclass\'s "__prepare__()" method which returns an\nempty "collections.OrderedDict".  That mapping records the methods and\nattributes of *A* as they are defined within the body of the class\nstatement. Once those definitions are executed, the ordered dictionary\nis fully populated and the metaclass\'s "__new__()" method gets\ninvoked.  That method builds the new type and it saves the ordered\ndictionary keys in an attribute called "members".\n\n\nCustomizing instance and subclass checks\n========================================\n\nThe following methods are used to override the default behavior of the\n"isinstance()" and "issubclass()" built-in functions.\n\nIn particular, the metaclass "abc.ABCMeta" implements these methods in\norder to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n   Return true if *instance* should be considered a (direct or\n   indirect) instance of *class*. If defined, called to implement\n   "isinstance(instance, class)".\n\nclass.__subclasscheck__(self, subclass)\n\n   Return true if *subclass* should be considered a (direct or\n   indirect) subclass of *class*.  If defined, called to implement\n   "issubclass(subclass, class)".\n\nNote that these methods are looked up on the type (metaclass) of a\nclass.  They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also: **PEP 3119** - Introducing Abstract Base Classes\n\n     Includes the specification for customizing "isinstance()" and\n     "issubclass()" behavior through "__instancecheck__()" and\n     "__subclasscheck__()", with motivation for this functionality in\n     the context of adding Abstract Base Classes (see the "abc"\n     module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n   Called when the instance is "called" as a function; if this method\n   is defined, "x(arg1, arg2, ...)" is a shorthand for\n   "x.__call__(arg1, arg2, ...)".\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well.  The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which "0 <= k < N" where\n*N* is the length of the sequence, or slice objects, which define a\nrange of items.  It is also recommended that mappings provide the\nmethods "keys()", "values()", "items()", "get()", "clear()",\n"setdefault()", "pop()", "popitem()", "copy()", and "update()"\nbehaving similar to those for Python\'s standard dictionary objects.\nThe "collections" module provides a "MutableMapping" abstract base\nclass to help create those methods from a base set of "__getitem__()",\n"__setitem__()", "__delitem__()", and "keys()". Mutable sequences\nshould provide methods "append()", "count()", "index()", "extend()",\n"insert()", "pop()", "remove()", "reverse()" and "sort()", like Python\nstandard list objects.  Finally, sequence types should implement\naddition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods "__add__()", "__radd__()",\n"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" described\nbelow; they should not define other numerical operators.  It is\nrecommended that both mappings and sequences implement the\n"__contains__()" method to allow efficient use of the "in" operator;\nfor mappings, "in" should search the mapping\'s keys; for sequences, it\nshould search through the values.  It is further recommended that both\nmappings and sequences implement the "__iter__()" method to allow\nefficient iteration through the container; for mappings, "__iter__()"\nshould be the same as "keys()"; for sequences, it should iterate\nthrough the values.\n\nobject.__len__(self)\n\n   Called to implement the built-in function "len()".  Should return\n   the length of the object, an integer ">=" 0.  Also, an object that\n   doesn\'t define a "__bool__()" method and whose "__len__()" method\n   returns zero is considered to be false in a Boolean context.\n\n   **CPython implementation detail:** In CPython, the length is\n   required to be at most "sys.maxsize". If the length is larger than\n   "sys.maxsize" some features (such as "len()") may raise\n   "OverflowError".  To prevent raising "OverflowError" by truth value\n   testing, an object must define a "__bool__()" method.\n\nobject.__length_hint__(self)\n\n   Called to implement "operator.length_hint()". Should return an\n   estimated length for the object (which may be greater or less than\n   the actual length). The length must be an integer ">=" 0. This\n   method is purely an optimization and is never required for\n   correctness.\n\n   New in version 3.4.\n\nNote: Slicing is done exclusively with the following three methods.\n  A call like\n\n     a[1:2] = b\n\n  is translated to\n\n     a[slice(1, 2, None)] = b\n\n  and so forth.  Missing slice items are always filled in with "None".\n\nobject.__getitem__(self, key)\n\n   Called to implement evaluation of "self[key]". For sequence types,\n   the accepted keys should be integers and slice objects.  Note that\n   the special interpretation of negative indexes (if the class wishes\n   to emulate a sequence type) is up to the "__getitem__()" method. If\n   *key* is of an inappropriate type, "TypeError" may be raised; if of\n   a value outside the set of indexes for the sequence (after any\n   special interpretation of negative values), "IndexError" should be\n   raised. For mapping types, if *key* is missing (not in the\n   container), "KeyError" should be raised.\n\n   Note: "for" loops expect that an "IndexError" will be raised for\n     illegal indexes to allow proper detection of the end of the\n     sequence.\n\nobject.__missing__(self, key)\n\n   Called by "dict"."__getitem__()" to implement "self[key]" for dict\n   subclasses when key is not in the dictionary.\n\nobject.__setitem__(self, key, value)\n\n   Called to implement assignment to "self[key]".  Same note as for\n   "__getitem__()".  This should only be implemented for mappings if\n   the objects support changes to the values for keys, or if new keys\n   can be added, or for sequences if elements can be replaced.  The\n   same exceptions should be raised for improper *key* values as for\n   the "__getitem__()" method.\n\nobject.__delitem__(self, key)\n\n   Called to implement deletion of "self[key]".  Same note as for\n   "__getitem__()".  This should only be implemented for mappings if\n   the objects support removal of keys, or for sequences if elements\n   can be removed from the sequence.  The same exceptions should be\n   raised for improper *key* values as for the "__getitem__()" method.\n\nobject.__iter__(self)\n\n   This method is called when an iterator is required for a container.\n   This method should return a new iterator object that can iterate\n   over all the objects in the container.  For mappings, it should\n   iterate over the keys of the container.\n\n   Iterator objects also need to implement this method; they are\n   required to return themselves.  For more information on iterator\n   objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n   Called (if present) by the "reversed()" built-in to implement\n   reverse iteration.  It should return a new iterator object that\n   iterates over all the objects in the container in reverse order.\n\n   If the "__reversed__()" method is not provided, the "reversed()"\n   built-in will fall back to using the sequence protocol ("__len__()"\n   and "__getitem__()").  Objects that support the sequence protocol\n   should only provide "__reversed__()" if they can provide an\n   implementation that is more efficient than the one provided by\n   "reversed()".\n\nThe membership test operators ("in" and "not in") are normally\nimplemented as an iteration through a sequence.  However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n   Called to implement membership test operators.  Should return true\n   if *item* is in *self*, false otherwise.  For mapping objects, this\n   should consider the keys of the mapping rather than the values or\n   the key-item pairs.\n\n   For objects that don\'t define "__contains__()", the membership test\n   first tries iteration via "__iter__()", then the old sequence\n   iteration protocol via "__getitem__()", see *this section in the\n   language reference*.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__matmul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations ("+", "-", "*", "@", "/", "//", "%", "divmod()",\n   "pow()", "**", "<<", ">>", "&", "^", "|").  For instance, to\n   evaluate the expression "x + y", where *x* is an instance of a\n   class that has an "__add__()" method, "x.__add__(y)" is called.\n   The "__divmod__()" method should be the equivalent to using\n   "__floordiv__()" and "__mod__()"; it should not be related to\n   "__truediv__()".  Note that "__pow__()" should be defined to accept\n   an optional third argument if the ternary version of the built-in\n   "pow()" function is to be supported.\n\n   If one of those methods does not support the operation with the\n   supplied arguments, it should return "NotImplemented".\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rmatmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations ("+", "-", "*", "@", "/", "//", "%", "divmod()",\n   "pow()", "**", "<<", ">>", "&", "^", "|") with reflected (swapped)\n   operands.  These functions are only called if the left operand does\n   not support the corresponding operation and the operands are of\n   different types. [2] For instance, to evaluate the expression "x -\n   y", where *y* is an instance of a class that has an "__rsub__()"\n   method, "y.__rsub__(x)" is called if "x.__sub__(y)" returns\n   *NotImplemented*.\n\n   Note that ternary "pow()" will not try calling "__rpow__()" (the\n   coercion rules would become too complicated).\n\n   Note: If the right operand\'s type is a subclass of the left\n     operand\'s type and that subclass provides the reflected method\n     for the operation, this method will be called before the left\n     operand\'s non-reflected method.  This behavior allows subclasses\n     to override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__imatmul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n   These methods are called to implement the augmented arithmetic\n   assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", "**=",\n   "<<=", ">>=", "&=", "^=", "|=").  These methods should attempt to\n   do the operation in-place (modifying *self*) and return the result\n   (which could be, but does not have to be, *self*).  If a specific\n   method is not defined, the augmented assignment falls back to the\n   normal methods.  For instance, if *x* is an instance of a class\n   with an "__iadd__()" method, "x += y" is equivalent to "x =\n   x.__iadd__(y)" . Otherwise, "x.__add__(y)" and "y.__radd__(x)" are\n   considered, as with the evaluation of "x + y". In certain\n   situations, augmented assignment can result in unexpected errors\n   (see *Why does a_tuple[i] += [\'item\'] raise an exception when the\n   addition works?*), but this behavior is in fact part of the data\n   model.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n   Called to implement the unary arithmetic operations ("-", "+",\n   "abs()" and "~").\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n   Called to implement the built-in functions "complex()", "int()",\n   "float()" and "round()".  Should return a value of the appropriate\n   type.\n\nobject.__index__(self)\n\n   Called to implement "operator.index()", and whenever Python needs\n   to losslessly convert the numeric object to an integer object (such\n   as in slicing, or in the built-in "bin()", "hex()" and "oct()"\n   functions). Presence of this method indicates that the numeric\n   object is an integer type.  Must return an integer.\n\n   Note: In order to have a coherent integer type class, when\n     "__index__()" is defined "__int__()" should also be defined, and\n     both should return the same value.\n\n\nWith Statement Context Managers\n===============================\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a "with" statement. The context manager\nhandles the entry into, and the exit from, the desired runtime context\nfor the execution of the block of code.  Context managers are normally\ninvoked using the "with" statement (described in section *The with\nstatement*), but can also be used by directly invoking their methods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n   Enter the runtime context related to this object. The "with"\n   statement will bind this method\'s return value to the target(s)\n   specified in the "as" clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n   Exit the runtime context related to this object. The parameters\n   describe the exception that caused the context to be exited. If the\n   context was exited without an exception, all three arguments will\n   be "None".\n\n   If an exception is supplied, and the method wishes to suppress the\n   exception (i.e., prevent it from being propagated), it should\n   return a true value. Otherwise, the exception will be processed\n   normally upon exit from this method.\n\n   Note that "__exit__()" methods should not reraise the passed-in\n   exception; this is the caller\'s responsibility.\n\nSee also: **PEP 343** - The "with" statement\n\n     The specification, background, and examples for the Python "with"\n     statement.\n\n\nSpecial method lookup\n=====================\n\nFor custom classes, implicit invocations of special methods are only\nguaranteed to work correctly if defined on an object\'s type, not in\nthe object\'s instance dictionary.  That behaviour is the reason why\nthe following code raises an exception:\n\n   >>> class C:\n   ...     pass\n   ...\n   >>> c = C()\n   >>> c.__len__ = lambda: 5\n   >>> len(c)\n   Traceback (most recent call last):\n     File "<stdin>", line 1, in <module>\n   TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as "__hash__()" and "__repr__()" that are implemented by\nall objects, including type objects. If the implicit lookup of these\nmethods used the conventional lookup process, they would fail when\ninvoked on the type object itself:\n\n   >>> 1 .__hash__() == hash(1)\n   True\n   >>> int.__hash__() == hash(int)\n   Traceback (most recent call last):\n     File "<stdin>", line 1, in <module>\n   TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n   >>> type(1).__hash__(1) == hash(1)\n   True\n   >>> type(int).__hash__(int) == hash(int)\n   True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe "__getattribute__()" method even of the object\'s metaclass:\n\n   >>> class Meta(type):\n   ...     def __getattribute__(*args):\n   ...         print("Metaclass getattribute invoked")\n   ...         return type.__getattribute__(*args)\n   ...\n   >>> class C(object, metaclass=Meta):\n   ...     def __len__(self):\n   ...         return 10\n   ...     def __getattribute__(*args):\n   ...         print("Class getattribute invoked")\n   ...         return object.__getattribute__(*args)\n   ...\n   >>> c = C()\n   >>> c.__len__()                 # Explicit lookup via instance\n   Class getattribute invoked\n   10\n   >>> type(c).__len__(c)          # Explicit lookup via type\n   Metaclass getattribute invoked\n   10\n   >>> len(c)                      # Implicit lookup\n   10\n\nBypassing the "__getattribute__()" machinery in this fashion provides\nsignificant scope for speed optimisations within the interpreter, at\nthe cost of some flexibility in the handling of special methods (the\nspecial method *must* be set on the class object itself in order to be\nconsistently invoked by the interpreter).\n',
- 'string-methods': u'\nString Methods\n**************\n\nStrings implement all of the *common* sequence operations, along with\nthe additional methods described below.\n\nStrings also support two styles of string formatting, one providing a\nlarge degree of flexibility and customization (see "str.format()",\n*Format String Syntax* and *Custom String Formatting*) and the other\nbased on C "printf" style formatting that handles a narrower range of\ntypes and is slightly harder to use correctly, but is often faster for\nthe cases it can handle (*printf-style String Formatting*).\n\nThe *Text Processing Services* section of the standard library covers\na number of other modules that provide various text related utilities\n(including regular expression support in the "re" module).\n\nstr.capitalize()\n\n   Return a copy of the string with its first character capitalized\n   and the rest lowercased.\n\nstr.casefold()\n\n   Return a casefolded copy of the string. Casefolded strings may be\n   used for caseless matching.\n\n   Casefolding is similar to lowercasing but more aggressive because\n   it is intended to remove all case distinctions in a string. For\n   example, the German lowercase letter "\'\xdf\'" is equivalent to ""ss"".\n   Since it is already lowercase, "lower()" would do nothing to "\'\xdf\'";\n   "casefold()" converts it to ""ss"".\n\n   The casefolding algorithm is described in section 3.13 of the\n   Unicode Standard.\n\n   New in version 3.3.\n\nstr.center(width[, fillchar])\n\n   Return centered in a string of length *width*. Padding is done\n   using the specified *fillchar* (default is an ASCII space). The\n   original string is returned if *width* is less than or equal to\n   "len(s)".\n\nstr.count(sub[, start[, end]])\n\n   Return the number of non-overlapping occurrences of substring *sub*\n   in the range [*start*, *end*].  Optional arguments *start* and\n   *end* are interpreted as in slice notation.\n\nstr.encode(encoding="utf-8", errors="strict")\n\n   Return an encoded version of the string as a bytes object. Default\n   encoding is "\'utf-8\'". *errors* may be given to set a different\n   error handling scheme. The default for *errors* is "\'strict\'",\n   meaning that encoding errors raise a "UnicodeError". Other possible\n   values are "\'ignore\'", "\'replace\'", "\'xmlcharrefreplace\'",\n   "\'backslashreplace\'" and any other name registered via\n   "codecs.register_error()", see section *Error Handlers*. For a list\n   of possible encodings, see section *Standard Encodings*.\n\n   Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n   Return "True" if the string ends with the specified *suffix*,\n   otherwise return "False".  *suffix* can also be a tuple of suffixes\n   to look for.  With optional *start*, test beginning at that\n   position.  With optional *end*, stop comparing at that position.\n\nstr.expandtabs(tabsize=8)\n\n   Return a copy of the string where all tab characters are replaced\n   by one or more spaces, depending on the current column and the\n   given tab size.  Tab positions occur every *tabsize* characters\n   (default is 8, giving tab positions at columns 0, 8, 16 and so on).\n   To expand the string, the current column is set to zero and the\n   string is examined character by character.  If the character is a\n   tab ("\\t"), one or more space characters are inserted in the result\n   until the current column is equal to the next tab position. (The\n   tab character itself is not copied.)  If the character is a newline\n   ("\\n") or return ("\\r"), it is copied and the current column is\n   reset to zero.  Any other character is copied unchanged and the\n   current column is incremented by one regardless of how the\n   character is represented when printed.\n\n   >>> \'01\\t012\\t0123\\t01234\'.expandtabs()\n   \'01      012     0123    01234\'\n   >>> \'01\\t012\\t0123\\t01234\'.expandtabs(4)\n   \'01  012 0123    01234\'\n\nstr.find(sub[, start[, end]])\n\n   Return the lowest index in the string where substring *sub* is\n   found within the slice "s[start:end]".  Optional arguments *start*\n   and *end* are interpreted as in slice notation.  Return "-1" if\n   *sub* is not found.\n\n   Note: The "find()" method should be used only if you need to know\n     the position of *sub*.  To check if *sub* is a substring or not,\n     use the "in" operator:\n\n        >>> \'Py\' in \'Python\'\n        True\n\nstr.format(*args, **kwargs)\n\n   Perform a string formatting operation.  The string on which this\n   method is called can contain literal text or replacement fields\n   delimited by braces "{}".  Each replacement field contains either\n   the numeric index of a positional argument, or the name of a\n   keyword argument.  Returns a copy of the string where each\n   replacement field is replaced with the string value of the\n   corresponding argument.\n\n   >>> "The sum of 1 + 2 is {0}".format(1+2)\n   \'The sum of 1 + 2 is 3\'\n\n   See *Format String Syntax* for a description of the various\n   formatting options that can be specified in format strings.\n\nstr.format_map(mapping)\n\n   Similar to "str.format(**mapping)", except that "mapping" is used\n   directly and not copied to a "dict".  This is useful if for example\n   "mapping" is a dict subclass:\n\n   >>> class Default(dict):\n   ...     def __missing__(self, key):\n   ...         return key\n   ...\n   >>> \'{name} was born in {country}\'.format_map(Default(name=\'Guido\'))\n   \'Guido was born in country\'\n\n   New in version 3.2.\n\nstr.index(sub[, start[, end]])\n\n   Like "find()", but raise "ValueError" when the substring is not\n   found.\n\nstr.isalnum()\n\n   Return true if all characters in the string are alphanumeric and\n   there is at least one character, false otherwise.  A character "c"\n   is alphanumeric if one of the following returns "True":\n   "c.isalpha()", "c.isdecimal()", "c.isdigit()", or "c.isnumeric()".\n\nstr.isalpha()\n\n   Return true if all characters in the string are alphabetic and\n   there is at least one character, false otherwise.  Alphabetic\n   characters are those characters defined in the Unicode character\n   database as "Letter", i.e., those with general category property\n   being one of "Lm", "Lt", "Lu", "Ll", or "Lo".  Note that this is\n   different from the "Alphabetic" property defined in the Unicode\n   Standard.\n\nstr.isdecimal()\n\n   Return true if all characters in the string are decimal characters\n   and there is at least one character, false otherwise. Decimal\n   characters are those that can be used to form numbers in base 10,\n   e.g. U+0660, ARABIC-INDIC DIGIT ZERO.  Formally a decimal character\n   is a character in the Unicode General Category "Nd".\n\nstr.isdigit()\n\n   Return true if all characters in the string are digits and there is\n   at least one character, false otherwise.  Digits include decimal\n   characters and digits that need special handling, such as the\n   compatibility superscript digits. This covers digits which cannot\n   be used to form numbers in base 10, like the Kharosthi numbers.\n   Formally, a digit is a character that has the property value\n   Numeric_Type=Digit or Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n   Return true if the string is a valid identifier according to the\n   language definition, section *Identifiers and keywords*.\n\n   Use "keyword.iskeyword()" to test for reserved identifiers such as\n   "def" and "class".\n\nstr.islower()\n\n   Return true if all cased characters [4] in the string are lowercase\n   and there is at least one cased character, false otherwise.\n\nstr.isnumeric()\n\n   Return true if all characters in the string are numeric characters,\n   and there is at least one character, false otherwise. Numeric\n   characters include digit characters, and all characters that have\n   the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n   ONE FIFTH.  Formally, numeric characters are those with the\n   property value Numeric_Type=Digit, Numeric_Type=Decimal or\n   Numeric_Type=Numeric.\n\nstr.isprintable()\n\n   Return true if all characters in the string are printable or the\n   string is empty, false otherwise.  Nonprintable characters are\n   those characters defined in the Unicode character database as\n   "Other" or "Separator", excepting the ASCII space (0x20) which is\n   considered printable.  (Note that printable characters in this\n   context are those which should not be escaped when "repr()" is\n   invoked on a string.  It has no bearing on the handling of strings\n   written to "sys.stdout" or "sys.stderr".)\n\nstr.isspace()\n\n   Return true if there are only whitespace characters in the string\n   and there is at least one character, false otherwise.  Whitespace\n   characters  are those characters defined in the Unicode character\n   database as "Other" or "Separator" and those with bidirectional\n   property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n   Return true if the string is a titlecased string and there is at\n   least one character, for example uppercase characters may only\n   follow uncased characters and lowercase characters only cased ones.\n   Return false otherwise.\n\nstr.isupper()\n\n   Return true if all cased characters [4] in the string are uppercase\n   and there is at least one cased character, false otherwise.\n\nstr.join(iterable)\n\n   Return a string which is the concatenation of the strings in\n   *iterable*. A "TypeError" will be raised if there are any non-\n   string values in *iterable*, including "bytes" objects.  The\n   separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n   Return the string left justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is an ASCII\n   space). The original string is returned if *width* is less than or\n   equal to "len(s)".\n\nstr.lower()\n\n   Return a copy of the string with all the cased characters [4]\n   converted to lowercase.\n\n   The lowercasing algorithm used is described in section 3.13 of the\n   Unicode Standard.\n\nstr.lstrip([chars])\n\n   Return a copy of the string with leading characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or "None", the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a prefix; rather,\n   all combinations of its values are stripped:\n\n      >>> \'   spacious   \'.lstrip()\n      \'spacious   \'\n      >>> \'www.example.com\'.lstrip(\'cmowz.\')\n      \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n   This static method returns a translation table usable for\n   "str.translate()".\n\n   If there is only one argument, it must be a dictionary mapping\n   Unicode ordinals (integers) or characters (strings of length 1) to\n   Unicode ordinals, strings (of arbitrary lengths) or "None".\n   Character keys will then be converted to ordinals.\n\n   If there are two arguments, they must be strings of equal length,\n   and in the resulting dictionary, each character in x will be mapped\n   to the character at the same position in y.  If there is a third\n   argument, it must be a string, whose characters will be mapped to\n   "None" in the result.\n\nstr.partition(sep)\n\n   Split the string at the first occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing the string itself, followed by\n   two empty strings.\n\nstr.replace(old, new[, count])\n\n   Return a copy of the string with all occurrences of substring *old*\n   replaced by *new*.  If the optional argument *count* is given, only\n   the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n   Return the highest index in the string where substring *sub* is\n   found, such that *sub* is contained within "s[start:end]".\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return "-1" on failure.\n\nstr.rindex(sub[, start[, end]])\n\n   Like "rfind()" but raises "ValueError" when the substring *sub* is\n   not found.\n\nstr.rjust(width[, fillchar])\n\n   Return the string right justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is an ASCII\n   space). The original string is returned if *width* is less than or\n   equal to "len(s)".\n\nstr.rpartition(sep)\n\n   Split the string at the last occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing two empty strings, followed by\n   the string itself.\n\nstr.rsplit(sep=None, maxsplit=-1)\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n   are done, the *rightmost* ones.  If *sep* is not specified or\n   "None", any whitespace string is a separator.  Except for splitting\n   from the right, "rsplit()" behaves like "split()" which is\n   described in detail below.\n\nstr.rstrip([chars])\n\n   Return a copy of the string with trailing characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or "None", the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a suffix; rather,\n   all combinations of its values are stripped:\n\n      >>> \'   spacious   \'.rstrip()\n      \'   spacious\'\n      >>> \'mississippi\'.rstrip(\'ipz\')\n      \'mississ\'\n\nstr.split(sep=None, maxsplit=-1)\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string.  If *maxsplit* is given, at most *maxsplit*\n   splits are done (thus, the list will have at most "maxsplit+1"\n   elements).  If *maxsplit* is not specified or "-1", then there is\n   no limit on the number of splits (all possible splits are made).\n\n   If *sep* is given, consecutive delimiters are not grouped together\n   and are deemed to delimit empty strings (for example,\n   "\'1,,2\'.split(\',\')" returns "[\'1\', \'\', \'2\']").  The *sep* argument\n   may consist of multiple characters (for example,\n   "\'1<>2<>3\'.split(\'<>\')" returns "[\'1\', \'2\', \'3\']"). Splitting an\n   empty string with a specified separator returns "[\'\']".\n\n   For example:\n\n      >>> \'1,2,3\'.split(\',\')\n      [\'1\', \'2\', \'3\']\n      >>> \'1,2,3\'.split(\',\', maxsplit=1)\n      [\'1\', \'2,3\']\n      >>> \'1,2,,3,\'.split(\',\')\n      [\'1\', \'2\', \'\', \'3\', \'\']\n\n   If *sep* is not specified or is "None", a different splitting\n   algorithm is applied: runs of consecutive whitespace are regarded\n   as a single separator, and the result will contain no empty strings\n   at the start or end if the string has leading or trailing\n   whitespace.  Consequently, splitting an empty string or a string\n   consisting of just whitespace with a "None" separator returns "[]".\n\n   For example:\n\n      >>> \'1 2 3\'.split()\n      [\'1\', \'2\', \'3\']\n      >>> \'1 2 3\'.split(maxsplit=1)\n      [\'1\', \'2 3\']\n      >>> \'   1   2   3   \'.split()\n      [\'1\', \'2\', \'3\']\n\nstr.splitlines([keepends])\n\n   Return a list of the lines in the string, breaking at line\n   boundaries.  Line breaks are not included in the resulting list\n   unless *keepends* is given and true.\n\n   This method splits on the following line boundaries.  In\n   particular, the boundaries are a superset of *universal newlines*.\n\n   +-------------------------+-------------------------------+\n   | Representation          | Description                   |\n   +=========================+===============================+\n   | "\\n"                    | Line Feed                     |\n   +-------------------------+-------------------------------+\n   | "\\r"                    | Carriage Return               |\n   +-------------------------+-------------------------------+\n   | "\\r\\n"                  | Carriage Return + Line Feed   |\n   +-------------------------+-------------------------------+\n   | "\\v" or "\\x0b"          | Line Tabulation               |\n   +-------------------------+-------------------------------+\n   | "\\f" or "\\x0c"          | Form Feed                     |\n   +-------------------------+-------------------------------+\n   | "\\x1c"                  | File Separator                |\n   +-------------------------+-------------------------------+\n   | "\\x1d"                  | Group Separator               |\n   +-------------------------+-------------------------------+\n   | "\\x1e"                  | Record Separator              |\n   +-------------------------+-------------------------------+\n   | "\\x85"                  | Next Line (C1 Control Code)   |\n   +-------------------------+-------------------------------+\n   | "\\u2028"                | Line Separator                |\n   +-------------------------+-------------------------------+\n   | "\\u2029"                | Paragraph Separator           |\n   +-------------------------+-------------------------------+\n\n   Changed in version 3.2: "\\v" and "\\f" added to list of line\n   boundaries.\n\n   For example:\n\n      >>> \'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()\n      [\'ab c\', \'\', \'de fg\', \'kl\']\n      >>> \'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines(keepends=True)\n      [\'ab c\\n\', \'\\n\', \'de fg\\r\', \'kl\\r\\n\']\n\n   Unlike "split()" when a delimiter string *sep* is given, this\n   method returns an empty list for the empty string, and a terminal\n   line break does not result in an extra line:\n\n      >>> "".splitlines()\n      []\n      >>> "One line\\n".splitlines()\n      [\'One line\']\n\n   For comparison, "split(\'\\n\')" gives:\n\n      >>> \'\'.split(\'\\n\')\n      [\'\']\n      >>> \'Two lines\\n\'.split(\'\\n\')\n      [\'Two lines\', \'\']\n\nstr.startswith(prefix[, start[, end]])\n\n   Return "True" if string starts with the *prefix*, otherwise return\n   "False". *prefix* can also be a tuple of prefixes to look for.\n   With optional *start*, test string beginning at that position.\n   With optional *end*, stop comparing string at that position.\n\nstr.strip([chars])\n\n   Return a copy of the string with the leading and trailing\n   characters removed. The *chars* argument is a string specifying the\n   set of characters to be removed. If omitted or "None", the *chars*\n   argument defaults to removing whitespace. The *chars* argument is\n   not a prefix or suffix; rather, all combinations of its values are\n   stripped:\n\n      >>> \'   spacious   \'.strip()\n      \'spacious\'\n      >>> \'www.example.com\'.strip(\'cmowz.\')\n      \'example\'\n\n   The outermost leading and trailing *chars* argument values are\n   stripped from the string. Characters are removed from the leading\n   end until reaching a string character that is not contained in the\n   set of characters in *chars*. A similar action takes place on the\n   trailing end. For example:\n\n      >>> comment_string = \'#....... Section 3.2.1 Issue #32 .......\'\n      >>> comment_string.strip(\'.#! \')\n      \'Section 3.2.1 Issue #32\'\n\nstr.swapcase()\n\n   Return a copy of the string with uppercase characters converted to\n   lowercase and vice versa. Note that it is not necessarily true that\n   "s.swapcase().swapcase() == s".\n\nstr.title()\n\n   Return a titlecased version of the string where words start with an\n   uppercase character and the remaining characters are lowercase.\n\n   For example:\n\n      >>> \'Hello world\'.title()\n      \'Hello World\'\n\n   The algorithm uses a simple language-independent definition of a\n   word as groups of consecutive letters.  The definition works in\n   many contexts but it means that apostrophes in contractions and\n   possessives form word boundaries, which may not be the desired\n   result:\n\n      >>> "they\'re bill\'s friends from the UK".title()\n      "They\'Re Bill\'S Friends From The Uk"\n\n   A workaround for apostrophes can be constructed using regular\n   expressions:\n\n      >>> import re\n      >>> def titlecase(s):\n      ...     return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n      ...                   lambda mo: mo.group(0)[0].upper() +\n      ...                              mo.group(0)[1:].lower(),\n      ...                   s)\n      ...\n      >>> titlecase("they\'re bill\'s friends.")\n      "They\'re Bill\'s Friends."\n\nstr.translate(table)\n\n   Return a copy of the string in which each character has been mapped\n   through the given translation table.  The table must be an object\n   that implements indexing via "__getitem__()", typically a *mapping*\n   or *sequence*.  When indexed by a Unicode ordinal (an integer), the\n   table object can do any of the following: return a Unicode ordinal\n   or a string, to map the character to one or more other characters;\n   return "None", to delete the character from the return string; or\n   raise a "LookupError" exception, to map the character to itself.\n\n   You can use "str.maketrans()" to create a translation map from\n   character-to-character mappings in different formats.\n\n   See also the "codecs" module for a more flexible approach to custom\n   character mappings.\n\nstr.upper()\n\n   Return a copy of the string with all the cased characters [4]\n   converted to uppercase.  Note that "str.upper().isupper()" might be\n   "False" if "s" contains uncased characters or if the Unicode\n   category of the resulting character(s) is not "Lu" (Letter,\n   uppercase), but e.g. "Lt" (Letter, titlecase).\n\n   The uppercasing algorithm used is described in section 3.13 of the\n   Unicode Standard.\n\nstr.zfill(width)\n\n   Return a copy of the string left filled with ASCII "\'0\'" digits to\n   make a string of length *width*. A leading sign prefix\n   ("\'+\'"/"\'-\'") is handled by inserting the padding *after* the sign\n   character rather than before. The original string is returned if\n   *width* is less than or equal to "len(s)".\n\n   For example:\n\n      >>> "42".zfill(5)\n      \'00042\'\n      >>> "-42".zfill(5)\n      \'-0042\'\n',
- 'strings': u'\nString and Bytes literals\n*************************\n\nString literals are described by the following lexical definitions:\n\n   stringliteral   ::= [stringprefix](shortstring | longstring)\n   stringprefix    ::= "r" | "u" | "R" | "U"\n   shortstring     ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n   longstring      ::= "\'\'\'" longstringitem* "\'\'\'" | \'"""\' longstringitem* \'"""\'\n   shortstringitem ::= shortstringchar | stringescapeseq\n   longstringitem  ::= longstringchar | stringescapeseq\n   shortstringchar ::= <any source character except "\\" or newline or the quote>\n   longstringchar  ::= <any source character except "\\">\n   stringescapeseq ::= "\\" <any source character>\n\n   bytesliteral   ::= bytesprefix(shortbytes | longbytes)\n   bytesprefix    ::= "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB"\n   shortbytes     ::= "\'" shortbytesitem* "\'" | \'"\' shortbytesitem* \'"\'\n   longbytes      ::= "\'\'\'" longbytesitem* "\'\'\'" | \'"""\' longbytesitem* \'"""\'\n   shortbytesitem ::= shortbyteschar | bytesescapeseq\n   longbytesitem  ::= longbyteschar | bytesescapeseq\n   shortbyteschar ::= <any ASCII character except "\\" or newline or the quote>\n   longbyteschar  ::= <any ASCII character except "\\">\n   bytesescapeseq ::= "\\" <any ASCII character>\n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the "stringprefix" or "bytesprefix"\nand the rest of the literal. The source character set is defined by\nthe encoding declaration; it is UTF-8 if no encoding declaration is\ngiven in the source file; see section *Encoding declarations*.\n\nIn plain English: Both types of literals can be enclosed in matching\nsingle quotes ("\'") or double quotes (""").  They can also be enclosed\nin matching groups of three single or double quotes (these are\ngenerally referred to as *triple-quoted strings*).  The backslash\n("\\") character is used to escape characters that otherwise have a\nspecial meaning, such as newline, backslash itself, or the quote\ncharacter.\n\nBytes literals are always prefixed with "\'b\'" or "\'B\'"; they produce\nan instance of the "bytes" type instead of the "str" type.  They may\nonly contain ASCII characters; bytes with a numeric value of 128 or\ngreater must be expressed with escapes.\n\nAs of Python 3.3 it is possible again to prefix string literals with a\n"u" prefix to simplify maintenance of dual 2.x and 3.x codebases.\n\nBoth string and bytes literals may optionally be prefixed with a\nletter "\'r\'" or "\'R\'"; such strings are called *raw strings* and treat\nbackslashes as literal characters.  As a result, in string literals,\n"\'\\U\'" and "\'\\u\'" escapes in raw strings are not treated specially.\nGiven that Python 2.x\'s raw unicode literals behave differently than\nPython 3.x\'s the "\'ur\'" syntax is not supported.\n\nNew in version 3.3: The "\'rb\'" prefix of raw bytes literals has been\nadded as a synonym of "\'br\'".\n\nNew in version 3.3: Support for the unicode legacy literal\n("u\'value\'") was reintroduced to simplify the maintenance of dual\nPython 2.x and 3.x codebases. See **PEP 414** for more information.\n\nIn triple-quoted literals, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the literal.  (A "quote" is the character used to open the\nliteral, i.e. either "\'" or """.)\n\nUnless an "\'r\'" or "\'R\'" prefix is present, escape sequences in string\nand bytes literals are interpreted according to rules similar to those\nused by Standard C.  The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence   | Meaning                           | Notes   |\n+===================+===================================+=========+\n| "\\newline"        | Backslash and newline ignored     |         |\n+-------------------+-----------------------------------+---------+\n| "\\\\"              | Backslash ("\\")                   |         |\n+-------------------+-----------------------------------+---------+\n| "\\\'"              | Single quote ("\'")                |         |\n+-------------------+-----------------------------------+---------+\n| "\\""              | Double quote (""")                |         |\n+-------------------+-----------------------------------+---------+\n| "\\a"              | ASCII Bell (BEL)                  |         |\n+-------------------+-----------------------------------+---------+\n| "\\b"              | ASCII Backspace (BS)              |         |\n+-------------------+-----------------------------------+---------+\n| "\\f"              | ASCII Formfeed (FF)               |         |\n+-------------------+-----------------------------------+---------+\n| "\\n"              | ASCII Linefeed (LF)               |         |\n+-------------------+-----------------------------------+---------+\n| "\\r"              | ASCII Carriage Return (CR)        |         |\n+-------------------+-----------------------------------+---------+\n| "\\t"              | ASCII Horizontal Tab (TAB)        |         |\n+-------------------+-----------------------------------+---------+\n| "\\v"              | ASCII Vertical Tab (VT)           |         |\n+-------------------+-----------------------------------+---------+\n| "\\ooo"            | Character with octal value *ooo*  | (1,3)   |\n+-------------------+-----------------------------------+---------+\n| "\\xhh"            | Character with hex value *hh*     | (2,3)   |\n+-------------------+-----------------------------------+---------+\n\nEscape sequences only recognized in string literals are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence   | Meaning                           | Notes   |\n+===================+===================================+=========+\n| "\\N{name}"        | Character named *name* in the     | (4)     |\n|                   | Unicode database                  |         |\n+-------------------+-----------------------------------+---------+\n| "\\uxxxx"          | Character with 16-bit hex value   | (5)     |\n|                   | *xxxx*                            |         |\n+-------------------+-----------------------------------+---------+\n| "\\Uxxxxxxxx"      | Character with 32-bit hex value   | (6)     |\n|                   | *xxxxxxxx*                        |         |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. As in Standard C, up to three octal digits are accepted.\n\n2. Unlike in Standard C, exactly two hex digits are required.\n\n3. In a bytes literal, hexadecimal and octal escapes denote the\n   byte with the given value. In a string literal, these escapes\n   denote a Unicode character with the given value.\n\n4. Changed in version 3.3: Support for name aliases [1] has been\n   added.\n\n5. Exactly four hex digits are required.\n\n6. Any Unicode character can be encoded this way.  Exactly eight\n   hex digits are required.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the result*.  (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.)  It is also\nimportant to note that the escape sequences only recognized in string\nliterals fall into the category of unrecognized escapes for bytes\nliterals.\n\nEven in a raw literal, quotes can be escaped with a backslash, but the\nbackslash remains in the result; for example, "r"\\""" is a valid\nstring literal consisting of two characters: a backslash and a double\nquote; "r"\\"" is not a valid string literal (even a raw string cannot\nend in an odd number of backslashes).  Specifically, *a raw literal\ncannot end in a single backslash* (since the backslash would escape\nthe following quote character).  Note also that a single backslash\nfollowed by a newline is interpreted as those two characters as part\nof the literal, *not* as a line continuation.\n',
- 'subscriptions': u'\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n   subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object that supports subscription\n(lists or dictionaries for example).  User-defined objects can support\nsubscription by defining a "__getitem__()" method.\n\nFor built-in objects, there are two types of objects that support\nsubscription:\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey.  (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to\nan integer or a slice (as discussed in the following section).\n\nThe formal syntax makes no special provision for negative indices in\nsequences; however, built-in sequences all provide a "__getitem__()"\nmethod that interprets negative indices by adding the length of the\nsequence to the index (so that "x[-1]" selects the last item of "x").\nThe resulting value must be a nonnegative integer less than the number\nof items in the sequence, and the subscription selects the item whose\nindex is that value (counting from zero). Since the support for\nnegative indices and slicing occurs in the object\'s "__getitem__()"\nmethod, subclasses overriding this method will need to explicitly add\nthat support.\n\nA string\'s items are characters.  A character is not a separate data\ntype but a string of exactly one character.\n',
- 'truth': u'\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an "if" or\n"while" condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* "None"\n\n* "False"\n\n* zero of any numeric type, for example, "0", "0.0", "0j".\n\n* any empty sequence, for example, "\'\'", "()", "[]".\n\n* any empty mapping, for example, "{}".\n\n* instances of user-defined classes, if the class defines a\n  "__bool__()" or "__len__()" method, when that method returns the\n  integer zero or "bool" value "False". [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn "0" or "False" for false and "1" or "True" for true, unless\notherwise stated. (Important exception: the Boolean operations "or"\nand "and" always return one of their operands.)\n',
- 'try': u'\nThe "try" statement\n*******************\n\nThe "try" statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n   try_stmt  ::= try1_stmt | try2_stmt\n   try1_stmt ::= "try" ":" suite\n                 ("except" [expression ["as" identifier]] ":" suite)+\n                 ["else" ":" suite]\n                 ["finally" ":" suite]\n   try2_stmt ::= "try" ":" suite\n                 "finally" ":" suite\n\nThe "except" clause(s) specify one or more exception handlers. When no\nexception occurs in the "try" clause, no exception handler is\nexecuted. When an exception occurs in the "try" suite, a search for an\nexception handler is started.  This search inspects the except clauses\nin turn until one is found that matches the exception.  An expression-\nless except clause, if present, must be last; it matches any\nexception.  For an except clause with an expression, that expression\nis evaluated, and the clause matches the exception if the resulting\nobject is "compatible" with the exception.  An object is compatible\nwith an exception if it is the class or a base class of the exception\nobject or a tuple containing an item compatible with the exception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire "try" statement raised\nthe exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the "as" keyword in that except clause, if\npresent, and the except clause\'s suite is executed.  All except\nclauses must have an executable block.  When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using "as target", it is cleared\nat the end of the except clause.  This is as if\n\n   except E as N:\n       foo\n\nwas translated to\n\n   except E as N:\n       try:\n           foo\n       finally:\n           del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause.  Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the "sys" module and can be accessed via\n"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of the\nexception class, the exception instance and a traceback object (see\nsection *The standard type hierarchy*) identifying the point in the\nprogram where the exception occurred.  "sys.exc_info()" values are\nrestored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional "else" clause is executed if and when control flows off\nthe end of the "try" clause. [2] Exceptions in the "else" clause are\nnot handled by the preceding "except" clauses.\n\nIf "finally" is present, it specifies a \'cleanup\' handler.  The "try"\nclause is executed, including any "except" and "else" clauses.  If an\nexception occurs in any of the clauses and is not handled, the\nexception is temporarily saved. The "finally" clause is executed.  If\nthere is a saved exception it is re-raised at the end of the "finally"\nclause.  If the "finally" clause raises another exception, the saved\nexception is set as the context of the new exception. If the "finally"\nclause executes a "return" or "break" statement, the saved exception\nis discarded:\n\n   >>> def f():\n   ...     try:\n   ...         1/0\n   ...     finally:\n   ...         return 42\n   ...\n   >>> f()\n   42\n\nThe exception information is not available to the program during\nexecution of the "finally" clause.\n\nWhen a "return", "break" or "continue" statement is executed in the\n"try" suite of a "try"..."finally" statement, the "finally" clause is\nalso executed \'on the way out.\' A "continue" statement is illegal in\nthe "finally" clause. (The reason is a problem with the current\nimplementation --- this restriction may be lifted in the future).\n\nThe return value of a function is determined by the last "return"\nstatement executed.  Since the "finally" clause always executes, a\n"return" statement executed in the "finally" clause will always be the\nlast one executed:\n\n   >>> def foo():\n   ...     try:\n   ...         return \'try\'\n   ...     finally:\n   ...         return \'finally\'\n   ...\n   >>> foo()\n   \'finally\'\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the "raise" statement to\ngenerate exceptions may be found in section *The raise statement*.\n',
- 'types': u'\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python.  Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types.  Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.), although such additions\nwill often be provided via the standard library instead.\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\'  These are attributes that provide access to the\nimplementation and are not intended for general use.  Their definition\nmay change in the future.\n\nNone\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the built-in name "None". It\n   is used to signify the absence of a value in many situations, e.g.,\n   it is returned from functions that don\'t explicitly return\n   anything. Its truth value is false.\n\nNotImplemented\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the built-in name\n   "NotImplemented". Numeric methods and rich comparison methods\n   should return this value if they do not implement the operation for\n   the operands provided.  (The interpreter will then try the\n   reflected operation, or some other fallback, depending on the\n   operator.)  Its truth value is true.\n\n   See *Implementing the arithmetic operations* for more details.\n\nEllipsis\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the literal "..." or the\n   built-in name "Ellipsis".  Its truth value is true.\n\n"numbers.Number"\n   These are created by numeric literals and returned as results by\n   arithmetic operators and arithmetic built-in functions.  Numeric\n   objects are immutable; once created their value never changes.\n   Python numbers are of course strongly related to mathematical\n   numbers, but subject to the limitations of numerical representation\n   in computers.\n\n   Python distinguishes between integers, floating point numbers, and\n   complex numbers:\n\n   "numbers.Integral"\n      These represent elements from the mathematical set of integers\n      (positive and negative).\n\n      There are two types of integers:\n\n      Integers ("int")\n\n         These represent numbers in an unlimited range, subject to\n         available (virtual) memory only.  For the purpose of shift\n         and mask operations, a binary representation is assumed, and\n         negative numbers are represented in a variant of 2\'s\n         complement which gives the illusion of an infinite string of\n         sign bits extending to the left.\n\n      Booleans ("bool")\n         These represent the truth values False and True.  The two\n         objects representing the values "False" and "True" are the\n         only Boolean objects. The Boolean type is a subtype of the\n         integer type, and Boolean values behave like the values 0 and\n         1, respectively, in almost all contexts, the exception being\n         that when converted to a string, the strings ""False"" or\n         ""True"" are returned, respectively.\n\n      The rules for integer representation are intended to give the\n      most meaningful interpretation of shift and mask operations\n      involving negative integers.\n\n   "numbers.Real" ("float")\n      These represent machine-level double precision floating point\n      numbers. You are at the mercy of the underlying machine\n      architecture (and C or Java implementation) for the accepted\n      range and handling of overflow. Python does not support single-\n      precision floating point numbers; the savings in processor and\n      memory usage that are usually the reason for using these are\n      dwarfed by the overhead of using objects in Python, so there is\n      no reason to complicate the language with two kinds of floating\n      point numbers.\n\n   "numbers.Complex" ("complex")\n      These represent complex numbers as a pair of machine-level\n      double precision floating point numbers.  The same caveats apply\n      as for floating point numbers. The real and imaginary parts of a\n      complex number "z" can be retrieved through the read-only\n      attributes "z.real" and "z.imag".\n\nSequences\n   These represent finite ordered sets indexed by non-negative\n   numbers. The built-in function "len()" returns the number of items\n   of a sequence. When the length of a sequence is *n*, the index set\n   contains the numbers 0, 1, ..., *n*-1.  Item *i* of sequence *a* is\n   selected by "a[i]".\n\n   Sequences also support slicing: "a[i:j]" selects all items with\n   index *k* such that *i* "<=" *k* "<" *j*.  When used as an\n   expression, a slice is a sequence of the same type.  This implies\n   that the index set is renumbered so that it starts at 0.\n\n   Some sequences also support "extended slicing" with a third "step"\n   parameter: "a[i:j:k]" selects all items of *a* with index *x* where\n   "x = i + n*k", *n* ">=" "0" and *i* "<=" *x* "<" *j*.\n\n   Sequences are distinguished according to their mutability:\n\n   Immutable sequences\n      An object of an immutable sequence type cannot change once it is\n      created.  (If the object contains references to other objects,\n      these other objects may be mutable and may be changed; however,\n      the collection of objects directly referenced by an immutable\n      object cannot change.)\n\n      The following types are immutable sequences:\n\n      Strings\n         A string is a sequence of values that represent Unicode code\n         points. All the code points in the range "U+0000 - U+10FFFF"\n         can be represented in a string.  Python doesn\'t have a "char"\n         type; instead, every code point in the string is represented\n         as a string object with length "1".  The built-in function\n         "ord()" converts a code point from its string form to an\n         integer in the range "0 - 10FFFF"; "chr()" converts an\n         integer in the range "0 - 10FFFF" to the corresponding length\n         "1" string object. "str.encode()" can be used to convert a\n         "str" to "bytes" using the given text encoding, and\n         "bytes.decode()" can be used to achieve the opposite.\n\n      Tuples\n         The items of a tuple are arbitrary Python objects. Tuples of\n         two or more items are formed by comma-separated lists of\n         expressions.  A tuple of one item (a \'singleton\') can be\n         formed by affixing a comma to an expression (an expression by\n         itself does not create a tuple, since parentheses must be\n         usable for grouping of expressions).  An empty tuple can be\n         formed by an empty pair of parentheses.\n\n      Bytes\n         A bytes object is an immutable array.  The items are 8-bit\n         bytes, represented by integers in the range 0 <= x < 256.\n         Bytes literals (like "b\'abc\'") and the built-in function\n         "bytes()" can be used to construct bytes objects.  Also,\n         bytes objects can be decoded to strings via the "decode()"\n         method.\n\n   Mutable sequences\n      Mutable sequences can be changed after they are created.  The\n      subscription and slicing notations can be used as the target of\n      assignment and "del" (delete) statements.\n\n      There are currently two intrinsic mutable sequence types:\n\n      Lists\n         The items of a list are arbitrary Python objects.  Lists are\n         formed by placing a comma-separated list of expressions in\n         square brackets. (Note that there are no special cases needed\n         to form lists of length 0 or 1.)\n\n      Byte Arrays\n         A bytearray object is a mutable array. They are created by\n         the built-in "bytearray()" constructor.  Aside from being\n         mutable (and hence unhashable), byte arrays otherwise provide\n         the same interface and functionality as immutable bytes\n         objects.\n\n      The extension module "array" provides an additional example of a\n      mutable sequence type, as does the "collections" module.\n\nSet types\n   These represent unordered, finite sets of unique, immutable\n   objects. As such, they cannot be indexed by any subscript. However,\n   they can be iterated over, and the built-in function "len()"\n   returns the number of items in a set. Common uses for sets are fast\n   membership testing, removing duplicates from a sequence, and\n   computing mathematical operations such as intersection, union,\n   difference, and symmetric difference.\n\n   For set elements, the same immutability rules apply as for\n   dictionary keys. Note that numeric types obey the normal rules for\n   numeric comparison: if two numbers compare equal (e.g., "1" and\n   "1.0"), only one of them can be contained in a set.\n\n   There are currently two intrinsic set types:\n\n   Sets\n      These represent a mutable set. They are created by the built-in\n      "set()" constructor and can be modified afterwards by several\n      methods, such as "add()".\n\n   Frozen sets\n      These represent an immutable set.  They are created by the\n      built-in "frozenset()" constructor.  As a frozenset is immutable\n      and *hashable*, it can be used again as an element of another\n      set, or as a dictionary key.\n\nMappings\n   These represent finite sets of objects indexed by arbitrary index\n   sets. The subscript notation "a[k]" selects the item indexed by "k"\n   from the mapping "a"; this can be used in expressions and as the\n   target of assignments or "del" statements. The built-in function\n   "len()" returns the number of items in a mapping.\n\n   There is currently a single intrinsic mapping type:\n\n   Dictionaries\n      These represent finite sets of objects indexed by nearly\n      arbitrary values.  The only types of values not acceptable as\n      keys are values containing lists or dictionaries or other\n      mutable types that are compared by value rather than by object\n      identity, the reason being that the efficient implementation of\n      dictionaries requires a key\'s hash value to remain constant.\n      Numeric types used for keys obey the normal rules for numeric\n      comparison: if two numbers compare equal (e.g., "1" and "1.0")\n      then they can be used interchangeably to index the same\n      dictionary entry.\n\n      Dictionaries are mutable; they can be created by the "{...}"\n      notation (see section *Dictionary displays*).\n\n      The extension modules "dbm.ndbm" and "dbm.gnu" provide\n      additional examples of mapping types, as does the "collections"\n      module.\n\nCallable types\n   These are the types to which the function call operation (see\n   section *Calls*) can be applied:\n\n   User-defined functions\n      A user-defined function object is created by a function\n      definition (see section *Function definitions*).  It should be\n      called with an argument list containing the same number of items\n      as the function\'s formal parameter list.\n\n      Special attributes:\n\n      +---------------------------+---------------------------------+-------------+\n      | Attribute                 | Meaning                         |             |\n      +===========================+=================================+=============+\n      | "__doc__"                 | The function\'s documentation    | Writable    |\n      |                           | string, or "None" if            |             |\n      |                           | unavailable; not inherited by   |             |\n      |                           | subclasses                      |             |\n      +---------------------------+---------------------------------+-------------+\n      | "__name__"                | The function\'s name             | Writable    |\n      +---------------------------+---------------------------------+-------------+\n      | "__qualname__"            | The function\'s *qualified name* | Writable    |\n      |                           | New in version 3.3.             |             |\n      +---------------------------+---------------------------------+-------------+\n      | "__module__"              | The name of the module the      | Writable    |\n      |                           | function was defined in, or     |             |\n      |                           | "None" if unavailable.          |             |\n      +---------------------------+---------------------------------+-------------+\n      | "__defaults__"            | A tuple containing default      | Writable    |\n      |                           | argument values for those       |             |\n      |                           | arguments that have defaults,   |             |\n      |                           | or "None" if no arguments have  |             |\n      |                           | a default value                 |             |\n      +---------------------------+---------------------------------+-------------+\n      | "__code__"                | The code object representing    | Writable    |\n      |                           | the compiled function body.     |             |\n      +---------------------------+---------------------------------+-------------+\n      | "__globals__"             | A reference to the dictionary   | Read-only   |\n      |                           | that holds the function\'s       |             |\n      |                           | global variables --- the global |             |\n      |                           | namespace of the module in      |             |\n      |                           | which the function was defined. |             |\n      +---------------------------+---------------------------------+-------------+\n      | "__dict__"                | The namespace supporting        | Writable    |\n      |                           | arbitrary function attributes.  |             |\n      +---------------------------+---------------------------------+-------------+\n      | "__closure__"             | "None" or a tuple of cells that | Read-only   |\n      |                           | contain bindings for the        |             |\n      |                           | function\'s free variables.      |             |\n      +---------------------------+---------------------------------+-------------+\n      | "__annotations__"         | A dict containing annotations   | Writable    |\n      |                           | of parameters.  The keys of the |             |\n      |                           | dict are the parameter names,   |             |\n      |                           | and "\'return\'" for the return   |             |\n      |                           | annotation, if provided.        |             |\n      +---------------------------+---------------------------------+-------------+\n      | "__kwdefaults__"          | A dict containing defaults for  | Writable    |\n      |                           | keyword-only parameters.        |             |\n      +---------------------------+---------------------------------+-------------+\n\n      Most of the attributes labelled "Writable" check the type of the\n      assigned value.\n\n      Function objects also support getting and setting arbitrary\n      attributes, which can be used, for example, to attach metadata\n      to functions.  Regular attribute dot-notation is used to get and\n      set such attributes. *Note that the current implementation only\n      supports function attributes on user-defined functions. Function\n      attributes on built-in functions may be supported in the\n      future.*\n\n      Additional information about a function\'s definition can be\n      retrieved from its code object; see the description of internal\n      types below.\n\n   Instance methods\n      An instance method object combines a class, a class instance and\n      any callable object (normally a user-defined function).\n\n      Special read-only attributes: "__self__" is the class instance\n      object, "__func__" is the function object; "__doc__" is the\n      method\'s documentation (same as "__func__.__doc__"); "__name__"\n      is the method name (same as "__func__.__name__"); "__module__"\n      is the name of the module the method was defined in, or "None"\n      if unavailable.\n\n      Methods also support accessing (but not setting) the arbitrary\n      function attributes on the underlying function object.\n\n      User-defined method objects may be created when getting an\n      attribute of a class (perhaps via an instance of that class), if\n      that attribute is a user-defined function object or a class\n      method object.\n\n      When an instance method object is created by retrieving a user-\n      defined function object from a class via one of its instances,\n      its "__self__" attribute is the instance, and the method object\n      is said to be bound.  The new method\'s "__func__" attribute is\n      the original function object.\n\n      When a user-defined method object is created by retrieving\n      another method object from a class or instance, the behaviour is\n      the same as for a function object, except that the "__func__"\n      attribute of the new instance is not the original method object\n      but its "__func__" attribute.\n\n      When an instance method object is created by retrieving a class\n      method object from a class or instance, its "__self__" attribute\n      is the class itself, and its "__func__" attribute is the\n      function object underlying the class method.\n\n      When an instance method object is called, the underlying\n      function ("__func__") is called, inserting the class instance\n      ("__self__") in front of the argument list.  For instance, when\n      "C" is a class which contains a definition for a function "f()",\n      and "x" is an instance of "C", calling "x.f(1)" is equivalent to\n      calling "C.f(x, 1)".\n\n      When an instance method object is derived from a class method\n      object, the "class instance" stored in "__self__" will actually\n      be the class itself, so that calling either "x.f(1)" or "C.f(1)"\n      is equivalent to calling "f(C,1)" where "f" is the underlying\n      function.\n\n      Note that the transformation from function object to instance\n      method object happens each time the attribute is retrieved from\n      the instance.  In some cases, a fruitful optimization is to\n      assign the attribute to a local variable and call that local\n      variable. Also notice that this transformation only happens for\n      user-defined functions; other callable objects (and all non-\n      callable objects) are retrieved without transformation.  It is\n      also important to note that user-defined functions which are\n      attributes of a class instance are not converted to bound\n      methods; this *only* happens when the function is an attribute\n      of the class.\n\n   Generator functions\n      A function or method which uses the "yield" statement (see\n      section *The yield statement*) is called a *generator function*.\n      Such a function, when called, always returns an iterator object\n      which can be used to execute the body of the function:  calling\n      the iterator\'s "iterator.__next__()" method will cause the\n      function to execute until it provides a value using the "yield"\n      statement.  When the function executes a "return" statement or\n      falls off the end, a "StopIteration" exception is raised and the\n      iterator will have reached the end of the set of values to be\n      returned.\n\n   Coroutine functions\n      A function or method which is defined using "async def" is\n      called a *coroutine function*.  Such a function, when called,\n      returns a *coroutine* object.  It may contain "await"\n      expressions, as well as "async with" and "async for" statements.\n      See also the *Coroutine Objects* section.\n\n   Built-in functions\n      A built-in function object is a wrapper around a C function.\n      Examples of built-in functions are "len()" and "math.sin()"\n      ("math" is a standard built-in module). The number and type of\n      the arguments are determined by the C function. Special read-\n      only attributes: "__doc__" is the function\'s documentation\n      string, or "None" if unavailable; "__name__" is the function\'s\n      name; "__self__" is set to "None" (but see the next item);\n      "__module__" is the name of the module the function was defined\n      in or "None" if unavailable.\n\n   Built-in methods\n      This is really a different disguise of a built-in function, this\n      time containing an object passed to the C function as an\n      implicit extra argument.  An example of a built-in method is\n      "alist.append()", assuming *alist* is a list object. In this\n      case, the special read-only attribute "__self__" is set to the\n      object denoted by *alist*.\n\n   Classes\n      Classes are callable.  These objects normally act as factories\n      for new instances of themselves, but variations are possible for\n      class types that override "__new__()".  The arguments of the\n      call are passed to "__new__()" and, in the typical case, to\n      "__init__()" to initialize the new instance.\n\n   Class Instances\n      Instances of arbitrary classes can be made callable by defining\n      a "__call__()" method in their class.\n\nModules\n   Modules are a basic organizational unit of Python code, and are\n   created by the *import system* as invoked either by the "import"\n   statement (see "import"), or by calling functions such as\n   "importlib.import_module()" and built-in "__import__()".  A module\n   object has a namespace implemented by a dictionary object (this is\n   the dictionary referenced by the "__globals__" attribute of\n   functions defined in the module).  Attribute references are\n   translated to lookups in this dictionary, e.g., "m.x" is equivalent\n   to "m.__dict__["x"]". A module object does not contain the code\n   object used to initialize the module (since it isn\'t needed once\n   the initialization is done).\n\n   Attribute assignment updates the module\'s namespace dictionary,\n   e.g., "m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n\n   Special read-only attribute: "__dict__" is the module\'s namespace\n   as a dictionary object.\n\n   **CPython implementation detail:** Because of the way CPython\n   clears module dictionaries, the module dictionary will be cleared\n   when the module falls out of scope even if the dictionary still has\n   live references.  To avoid this, copy the dictionary or keep the\n   module around while using its dictionary directly.\n\n   Predefined (writable) attributes: "__name__" is the module\'s name;\n   "__doc__" is the module\'s documentation string, or "None" if\n   unavailable; "__file__" is the pathname of the file from which the\n   module was loaded, if it was loaded from a file. The "__file__"\n   attribute may be missing for certain types of modules, such as C\n   modules that are statically linked into the interpreter; for\n   extension modules loaded dynamically from a shared library, it is\n   the pathname of the shared library file.\n\nCustom classes\n   Custom class types are typically created by class definitions (see\n   section *Class definitions*).  A class has a namespace implemented\n   by a dictionary object. Class attribute references are translated\n   to lookups in this dictionary, e.g., "C.x" is translated to\n   "C.__dict__["x"]" (although there are a number of hooks which allow\n   for other means of locating attributes). When the attribute name is\n   not found there, the attribute search continues in the base\n   classes. This search of the base classes uses the C3 method\n   resolution order which behaves correctly even in the presence of\n   \'diamond\' inheritance structures where there are multiple\n   inheritance paths leading back to a common ancestor. Additional\n   details on the C3 MRO used by Python can be found in the\n   documentation accompanying the 2.3 release at\n   https://www.python.org/download/releases/2.3/mro/.\n\n   When a class attribute reference (for class "C", say) would yield a\n   class method object, it is transformed into an instance method\n   object whose "__self__" attributes is "C".  When it would yield a\n   static method object, it is transformed into the object wrapped by\n   the static method object. See section *Implementing Descriptors*\n   for another way in which attributes retrieved from a class may\n   differ from those actually contained in its "__dict__".\n\n   Class attribute assignments update the class\'s dictionary, never\n   the dictionary of a base class.\n\n   A class object can be called (see above) to yield a class instance\n   (see below).\n\n   Special attributes: "__name__" is the class name; "__module__" is\n   the module name in which the class was defined; "__dict__" is the\n   dictionary containing the class\'s namespace; "__bases__" is a tuple\n   containing the base classes, in the order of their occurrence in\n   the base class list; "__doc__" is the class\'s documentation string,\n   or "None" if undefined.\n\nClass instances\n   A class instance is created by calling a class object (see above).\n   A class instance has a namespace implemented as a dictionary which\n   is the first place in which attribute references are searched.\n   When an attribute is not found there, and the instance\'s class has\n   an attribute by that name, the search continues with the class\n   attributes.  If a class attribute is found that is a user-defined\n   function object, it is transformed into an instance method object\n   whose "__self__" attribute is the instance.  Static method and\n   class method objects are also transformed; see above under\n   "Classes".  See section *Implementing Descriptors* for another way\n   in which attributes of a class retrieved via its instances may\n   differ from the objects actually stored in the class\'s "__dict__".\n   If no class attribute is found, and the object\'s class has a\n   "__getattr__()" method, that is called to satisfy the lookup.\n\n   Attribute assignments and deletions update the instance\'s\n   dictionary, never a class\'s dictionary.  If the class has a\n   "__setattr__()" or "__delattr__()" method, this is called instead\n   of updating the instance dictionary directly.\n\n   Class instances can pretend to be numbers, sequences, or mappings\n   if they have methods with certain special names.  See section\n   *Special method names*.\n\n   Special attributes: "__dict__" is the attribute dictionary;\n   "__class__" is the instance\'s class.\n\nI/O objects (also known as file objects)\n   A *file object* represents an open file.  Various shortcuts are\n   available to create file objects: the "open()" built-in function,\n   and also "os.popen()", "os.fdopen()", and the "makefile()" method\n   of socket objects (and perhaps by other functions or methods\n   provided by extension modules).\n\n   The objects "sys.stdin", "sys.stdout" and "sys.stderr" are\n   initialized to file objects corresponding to the interpreter\'s\n   standard input, output and error streams; they are all open in text\n   mode and therefore follow the interface defined by the\n   "io.TextIOBase" abstract class.\n\nInternal types\n   A few types used internally by the interpreter are exposed to the\n   user. Their definitions may change with future versions of the\n   interpreter, but they are mentioned here for completeness.\n\n   Code objects\n      Code objects represent *byte-compiled* executable Python code,\n      or *bytecode*. The difference between a code object and a\n      function object is that the function object contains an explicit\n      reference to the function\'s globals (the module in which it was\n      defined), while a code object contains no context; also the\n      default argument values are stored in the function object, not\n      in the code object (because they represent values calculated at\n      run-time).  Unlike function objects, code objects are immutable\n      and contain no references (directly or indirectly) to mutable\n      objects.\n\n      Special read-only attributes: "co_name" gives the function name;\n      "co_argcount" is the number of positional arguments (including\n      arguments with default values); "co_nlocals" is the number of\n      local variables used by the function (including arguments);\n      "co_varnames" is a tuple containing the names of the local\n      variables (starting with the argument names); "co_cellvars" is a\n      tuple containing the names of local variables that are\n      referenced by nested functions; "co_freevars" is a tuple\n      containing the names of free variables; "co_code" is a string\n      representing the sequence of bytecode instructions; "co_consts"\n      is a tuple containing the literals used by the bytecode;\n      "co_names" is a tuple containing the names used by the bytecode;\n      "co_filename" is the filename from which the code was compiled;\n      "co_firstlineno" is the first line number of the function;\n      "co_lnotab" is a string encoding the mapping from bytecode\n      offsets to line numbers (for details see the source code of the\n      interpreter); "co_stacksize" is the required stack size\n      (including local variables); "co_flags" is an integer encoding a\n      number of flags for the interpreter.\n\n      The following flag bits are defined for "co_flags": bit "0x04"\n      is set if the function uses the "*arguments" syntax to accept an\n      arbitrary number of positional arguments; bit "0x08" is set if\n      the function uses the "**keywords" syntax to accept arbitrary\n      keyword arguments; bit "0x20" is set if the function is a\n      generator.\n\n      Future feature declarations ("from __future__ import division")\n      also use bits in "co_flags" to indicate whether a code object\n      was compiled with a particular feature enabled: bit "0x2000" is\n      set if the function was compiled with future division enabled;\n      bits "0x10" and "0x1000" were used in earlier versions of\n      Python.\n\n      Other bits in "co_flags" are reserved for internal use.\n\n      If a code object represents a function, the first item in\n      "co_consts" is the documentation string of the function, or\n      "None" if undefined.\n\n   Frame objects\n      Frame objects represent execution frames.  They may occur in\n      traceback objects (see below).\n\n      Special read-only attributes: "f_back" is to the previous stack\n      frame (towards the caller), or "None" if this is the bottom\n      stack frame; "f_code" is the code object being executed in this\n      frame; "f_locals" is the dictionary used to look up local\n      variables; "f_globals" is used for global variables;\n      "f_builtins" is used for built-in (intrinsic) names; "f_lasti"\n      gives the precise instruction (this is an index into the\n      bytecode string of the code object).\n\n      Special writable attributes: "f_trace", if not "None", is a\n      function called at the start of each source code line (this is\n      used by the debugger); "f_lineno" is the current line number of\n      the frame --- writing to this from within a trace function jumps\n      to the given line (only for the bottom-most frame).  A debugger\n      can implement a Jump command (aka Set Next Statement) by writing\n      to f_lineno.\n\n      Frame objects support one method:\n\n      frame.clear()\n\n         This method clears all references to local variables held by\n         the frame.  Also, if the frame belonged to a generator, the\n         generator is finalized.  This helps break reference cycles\n         involving frame objects (for example when catching an\n         exception and storing its traceback for later use).\n\n         "RuntimeError" is raised if the frame is currently executing.\n\n         New in version 3.4.\n\n   Traceback objects\n      Traceback objects represent a stack trace of an exception.  A\n      traceback object is created when an exception occurs.  When the\n      search for an exception handler unwinds the execution stack, at\n      each unwound level a traceback object is inserted in front of\n      the current traceback.  When an exception handler is entered,\n      the stack trace is made available to the program. (See section\n      *The try statement*.) It is accessible as the third item of the\n      tuple returned by "sys.exc_info()". When the program contains no\n      suitable handler, the stack trace is written (nicely formatted)\n      to the standard error stream; if the interpreter is interactive,\n      it is also made available to the user as "sys.last_traceback".\n\n      Special read-only attributes: "tb_next" is the next level in the\n      stack trace (towards the frame where the exception occurred), or\n      "None" if there is no next level; "tb_frame" points to the\n      execution frame of the current level; "tb_lineno" gives the line\n      number where the exception occurred; "tb_lasti" indicates the\n      precise instruction.  The line number and last instruction in\n      the traceback may differ from the line number of its frame\n      object if the exception occurred in a "try" statement with no\n      matching except clause or with a finally clause.\n\n   Slice objects\n      Slice objects are used to represent slices for "__getitem__()"\n      methods.  They are also created by the built-in "slice()"\n      function.\n\n      Special read-only attributes: "start" is the lower bound; "stop"\n      is the upper bound; "step" is the step value; each is "None" if\n      omitted.  These attributes can have any type.\n\n      Slice objects support one method:\n\n      slice.indices(self, length)\n\n         This method takes a single integer argument *length* and\n         computes information about the slice that the slice object\n         would describe if applied to a sequence of *length* items.\n         It returns a tuple of three integers; respectively these are\n         the *start* and *stop* indices and the *step* or stride\n         length of the slice. Missing or out-of-bounds indices are\n         handled in a manner consistent with regular slices.\n\n   Static method objects\n      Static method objects provide a way of defeating the\n      transformation of function objects to method objects described\n      above. A static method object is a wrapper around any other\n      object, usually a user-defined method object. When a static\n      method object is retrieved from a class or a class instance, the\n      object actually returned is the wrapped object, which is not\n      subject to any further transformation. Static method objects are\n      not themselves callable, although the objects they wrap usually\n      are. Static method objects are created by the built-in\n      "staticmethod()" constructor.\n\n   Class method objects\n      A class method object, like a static method object, is a wrapper\n      around another object that alters the way in which that object\n      is retrieved from classes and class instances. The behaviour of\n      class method objects upon such retrieval is described above,\n      under "User-defined methods". Class method objects are created\n      by the built-in "classmethod()" constructor.\n',
- 'typesfunctions': u'\nFunctions\n*********\n\nFunction objects are created by function definitions.  The only\noperation on a function object is to call it: "func(argument-list)".\n\nThere are really two flavors of function objects: built-in functions\nand user-defined functions.  Both support the same operation (to call\nthe function), but the implementation is different, hence the\ndifferent object types.\n\nSee *Function definitions* for more information.\n',
- 'typesmapping': u'\nMapping Types --- "dict"\n************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects.  There is currently only one standard\nmapping type, the *dictionary*.  (For other containers see the built-\nin "list", "set", and "tuple" classes, and the "collections" module.)\n\nA dictionary\'s keys are *almost* arbitrary values.  Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys.  Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as "1" and "1.0") then they can be used interchangeably to index\nthe same dictionary entry.  (Note however, that since computers store\nfloating-point numbers as approximations it is usually unwise to use\nthem as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of "key:\nvalue" pairs within braces, for example: "{\'jack\': 4098, \'sjoerd\':\n4127}" or "{4098: \'jack\', 4127: \'sjoerd\'}", or by the "dict"\nconstructor.\n\nclass class dict(**kwarg)\nclass class dict(mapping, **kwarg)\nclass class dict(iterable, **kwarg)\n\n   Return a new dictionary initialized from an optional positional\n   argument and a possibly empty set of keyword arguments.\n\n   If no positional argument is given, an empty dictionary is created.\n   If a positional argument is given and it is a mapping object, a\n   dictionary is created with the same key-value pairs as the mapping\n   object.  Otherwise, the positional argument must be an *iterable*\n   object.  Each item in the iterable must itself be an iterable with\n   exactly two objects.  The first object of each item becomes a key\n   in the new dictionary, and the second object the corresponding\n   value.  If a key occurs more than once, the last value for that key\n   becomes the corresponding value in the new dictionary.\n\n   If keyword arguments are given, the keyword arguments and their\n   values are added to the dictionary created from the positional\n   argument.  If a key being added is already present, the value from\n   the keyword argument replaces the value from the positional\n   argument.\n\n   To illustrate, the following examples all return a dictionary equal\n   to "{"one": 1, "two": 2, "three": 3}":\n\n      >>> a = dict(one=1, two=2, three=3)\n      >>> b = {\'one\': 1, \'two\': 2, \'three\': 3}\n      >>> c = dict(zip([\'one\', \'two\', \'three\'], [1, 2, 3]))\n      >>> d = dict([(\'two\', 2), (\'one\', 1), (\'three\', 3)])\n      >>> e = dict({\'three\': 3, \'one\': 1, \'two\': 2})\n      >>> a == b == c == d == e\n      True\n\n   Providing keyword arguments as in the first example only works for\n   keys that are valid Python identifiers.  Otherwise, any valid keys\n   can be used.\n\n   These are the operations that dictionaries support (and therefore,\n   custom mapping types should support too):\n\n   len(d)\n\n      Return the number of items in the dictionary *d*.\n\n   d[key]\n\n      Return the item of *d* with key *key*.  Raises a "KeyError" if\n      *key* is not in the map.\n\n      If a subclass of dict defines a method "__missing__()" and *key*\n      is not present, the "d[key]" operation calls that method with\n      the key *key* as argument.  The "d[key]" operation then returns\n      or raises whatever is returned or raised by the\n      "__missing__(key)" call. No other operations or methods invoke\n      "__missing__()". If "__missing__()" is not defined, "KeyError"\n      is raised. "__missing__()" must be a method; it cannot be an\n      instance variable:\n\n         >>> class Counter(dict):\n         ...     def __missing__(self, key):\n         ...         return 0\n         >>> c = Counter()\n         >>> c[\'red\']\n         0\n         >>> c[\'red\'] += 1\n         >>> c[\'red\']\n         1\n\n      The example above shows part of the implementation of\n      "collections.Counter".  A different "__missing__" method is used\n      by "collections.defaultdict".\n\n   d[key] = value\n\n      Set "d[key]" to *value*.\n\n   del d[key]\n\n      Remove "d[key]" from *d*.  Raises a "KeyError" if *key* is not\n      in the map.\n\n   key in d\n\n      Return "True" if *d* has a key *key*, else "False".\n\n   key not in d\n\n      Equivalent to "not key in d".\n\n   iter(d)\n\n      Return an iterator over the keys of the dictionary.  This is a\n      shortcut for "iter(d.keys())".\n\n   clear()\n\n      Remove all items from the dictionary.\n\n   copy()\n\n      Return a shallow copy of the dictionary.\n\n   classmethod fromkeys(seq[, value])\n\n      Create a new dictionary with keys from *seq* and values set to\n      *value*.\n\n      "fromkeys()" is a class method that returns a new dictionary.\n      *value* defaults to "None".\n\n   get(key[, default])\n\n      Return the value for *key* if *key* is in the dictionary, else\n      *default*. If *default* is not given, it defaults to "None", so\n      that this method never raises a "KeyError".\n\n   items()\n\n      Return a new view of the dictionary\'s items ("(key, value)"\n      pairs). See the *documentation of view objects*.\n\n   keys()\n\n      Return a new view of the dictionary\'s keys.  See the\n      *documentation of view objects*.\n\n   pop(key[, default])\n\n      If *key* is in the dictionary, remove it and return its value,\n      else return *default*.  If *default* is not given and *key* is\n      not in the dictionary, a "KeyError" is raised.\n\n   popitem()\n\n      Remove and return an arbitrary "(key, value)" pair from the\n      dictionary.\n\n      "popitem()" is useful to destructively iterate over a\n      dictionary, as often used in set algorithms.  If the dictionary\n      is empty, calling "popitem()" raises a "KeyError".\n\n   setdefault(key[, default])\n\n      If *key* is in the dictionary, return its value.  If not, insert\n      *key* with a value of *default* and return *default*.  *default*\n      defaults to "None".\n\n   update([other])\n\n      Update the dictionary with the key/value pairs from *other*,\n      overwriting existing keys.  Return "None".\n\n      "update()" accepts either another dictionary object or an\n      iterable of key/value pairs (as tuples or other iterables of\n      length two).  If keyword arguments are specified, the dictionary\n      is then updated with those key/value pairs: "d.update(red=1,\n      blue=2)".\n\n   values()\n\n      Return a new view of the dictionary\'s values.  See the\n      *documentation of view objects*.\n\n   Dictionaries compare equal if and only if they have the same "(key,\n   value)" pairs. Order comparisons (\'<\', \'<=\', \'>=\', \'>\') raise\n   "TypeError".\n\nSee also: "types.MappingProxyType" can be used to create a read-only\n  view of a "dict".\n\n\nDictionary view objects\n=======================\n\nThe objects returned by "dict.keys()", "dict.values()" and\n"dict.items()" are *view objects*.  They provide a dynamic view on the\ndictionary\'s entries, which means that when the dictionary changes,\nthe view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n   Return the number of entries in the dictionary.\n\niter(dictview)\n\n   Return an iterator over the keys, values or items (represented as\n   tuples of "(key, value)") in the dictionary.\n\n   Keys and values are iterated over in an arbitrary order which is\n   non-random, varies across Python implementations, and depends on\n   the dictionary\'s history of insertions and deletions. If keys,\n   values and items views are iterated over with no intervening\n   modifications to the dictionary, the order of items will directly\n   correspond.  This allows the creation of "(value, key)" pairs using\n   "zip()": "pairs = zip(d.values(), d.keys())".  Another way to\n   create the same list is "pairs = [(v, k) for (k, v) in d.items()]".\n\n   Iterating views while adding or deleting entries in the dictionary\n   may raise a "RuntimeError" or fail to iterate over all entries.\n\nx in dictview\n\n   Return "True" if *x* is in the underlying dictionary\'s keys, values\n   or items (in the latter case, *x* should be a "(key, value)"\n   tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that "(key, value)" pairs are unique\nand hashable, then the items view is also set-like.  (Values views are\nnot treated as set-like since the entries are generally not unique.)\nFor set-like views, all of the operations defined for the abstract\nbase class "collections.abc.Set" are available (for example, "==",\n"<", or "^").\n\nAn example of dictionary view usage:\n\n   >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n   >>> keys = dishes.keys()\n   >>> values = dishes.values()\n\n   >>> # iteration\n   >>> n = 0\n   >>> for val in values:\n   ...     n += val\n   >>> print(n)\n   504\n\n   >>> # keys and values are iterated over in the same order\n   >>> list(keys)\n   [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n   >>> list(values)\n   [2, 1, 1, 500]\n\n   >>> # view objects are dynamic and reflect dict changes\n   >>> del dishes[\'eggs\']\n   >>> del dishes[\'sausage\']\n   >>> list(keys)\n   [\'spam\', \'bacon\']\n\n   >>> # set operations\n   >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n   {\'bacon\'}\n   >>> keys ^ {\'sausage\', \'juice\'}\n   {\'juice\', \'sausage\', \'bacon\', \'spam\'}\n',
- 'typesmethods': u'\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as "append()" on lists)\nand class instance methods.  Built-in methods are described with the\ntypes that support them.\n\nIf you access a method (a function defined in a class namespace)\nthrough an instance, you get a special object: a *bound method* (also\ncalled *instance method*) object. When called, it will add the "self"\nargument to the argument list.  Bound methods have two special read-\nonly attributes: "m.__self__" is the object on which the method\noperates, and "m.__func__" is the function implementing the method.\nCalling "m(arg-1, arg-2, ..., arg-n)" is completely equivalent to\ncalling "m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)".\n\nLike function objects, bound method objects support getting arbitrary\nattributes.  However, since method attributes are actually stored on\nthe underlying function object ("meth.__func__"), setting method\nattributes on bound methods is disallowed.  Attempting to set an\nattribute on a method results in an "AttributeError" being raised.  In\norder to set a method attribute, you need to explicitly set it on the\nunderlying function object:\n\n   >>> class C:\n   ...     def method(self):\n   ...         pass\n   ...\n   >>> c = C()\n   >>> c.method.whoami = \'my name is method\'  # can\'t set on the method\n   Traceback (most recent call last):\n     File "<stdin>", line 1, in <module>\n   AttributeError: \'method\' object has no attribute \'whoami\'\n   >>> c.method.__func__.whoami = \'my name is method\'\n   >>> c.method.whoami\n   \'my name is method\'\n\nSee *The standard type hierarchy* for more information.\n',
- 'typesmodules': u'\nModules\n*******\n\nThe only special operation on a module is attribute access: "m.name",\nwhere *m* is a module and *name* accesses a name defined in *m*\'s\nsymbol table. Module attributes can be assigned to.  (Note that the\n"import" statement is not, strictly speaking, an operation on a module\nobject; "import foo" does not require a module object named *foo* to\nexist, rather it requires an (external) *definition* for a module\nnamed *foo* somewhere.)\n\nA special attribute of every module is "__dict__". This is the\ndictionary containing the module\'s symbol table. Modifying this\ndictionary will actually change the module\'s symbol table, but direct\nassignment to the "__dict__" attribute is not possible (you can write\n"m.__dict__[\'a\'] = 1", which defines "m.a" to be "1", but you can\'t\nwrite "m.__dict__ = {}").  Modifying "__dict__" directly is not\nrecommended.\n\nModules built into the interpreter are written like this: "<module\n\'sys\' (built-in)>".  If loaded from a file, they are written as\n"<module \'os\' from \'/usr/local/lib/pythonX.Y/os.pyc\'>".\n',
- 'typesseq': u'\nSequence Types --- "list", "tuple", "range"\n*******************************************\n\nThere are three basic sequence types: lists, tuples, and range\nobjects. Additional sequence types tailored for processing of *binary\ndata* and *text strings* are described in dedicated sections.\n\n\nCommon Sequence Operations\n==========================\n\nThe operations in the following table are supported by most sequence\ntypes, both mutable and immutable. The "collections.abc.Sequence" ABC\nis provided to make it easier to correctly implement these operations\non custom sequence types.\n\nThis table lists the sequence operations sorted in ascending priority.\nIn the table, *s* and *t* are sequences of the same type, *n*, *i*,\n*j* and *k* are integers and *x* is an arbitrary object that meets any\ntype and value restrictions imposed by *s*.\n\nThe "in" and "not in" operations have the same priorities as the\ncomparison operations. The "+" (concatenation) and "*" (repetition)\noperations have the same priority as the corresponding numeric\noperations. [3]\n\n+----------------------------+----------------------------------+------------+\n| Operation                  | Result                           | Notes      |\n+============================+==================================+============+\n| "x in s"                   | "True" if an item of *s* is      | (1)        |\n|                            | equal to *x*, else "False"       |            |\n+----------------------------+----------------------------------+------------+\n| "x not in s"               | "False" if an item of *s* is     | (1)        |\n|                            | equal to *x*, else "True"        |            |\n+----------------------------+----------------------------------+------------+\n| "s + t"                    | the concatenation of *s* and *t* | (6)(7)     |\n+----------------------------+----------------------------------+------------+\n| "s * n" or "n * s"         | equivalent to adding *s* to      | (2)(7)     |\n|                            | itself *n* times                 |            |\n+----------------------------+----------------------------------+------------+\n| "s[i]"                     | *i*th item of *s*, origin 0      | (3)        |\n+----------------------------+----------------------------------+------------+\n| "s[i:j]"                   | slice of *s* from *i* to *j*     | (3)(4)     |\n+----------------------------+----------------------------------+------------+\n| "s[i:j:k]"                 | slice of *s* from *i* to *j*     | (3)(5)     |\n|                            | with step *k*                    |            |\n+----------------------------+----------------------------------+------------+\n| "len(s)"                   | length of *s*                    |            |\n+----------------------------+----------------------------------+------------+\n| "min(s)"                   | smallest item of *s*             |            |\n+----------------------------+----------------------------------+------------+\n| "max(s)"                   | largest item of *s*              |            |\n+----------------------------+----------------------------------+------------+\n| "s.index(x[, i[, j]])"     | index of the first occurrence of | (8)        |\n|                            | *x* in *s* (at or after index    |            |\n|                            | *i* and before index *j*)        |            |\n+----------------------------+----------------------------------+------------+\n| "s.count(x)"               | total number of occurrences of   |            |\n|                            | *x* in *s*                       |            |\n+----------------------------+----------------------------------+------------+\n\nSequences of the same type also support comparisons.  In particular,\ntuples and lists are compared lexicographically by comparing\ncorresponding elements. This means that to compare equal, every\nelement must compare equal and the two sequences must be of the same\ntype and have the same length.  (For full details see *Comparisons* in\nthe language reference.)\n\nNotes:\n\n1. While the "in" and "not in" operations are used only for simple\n   containment testing in the general case, some specialised sequences\n   (such as "str", "bytes" and "bytearray") also use them for\n   subsequence testing:\n\n      >>> "gg" in "eggs"\n      True\n\n2. Values of *n* less than "0" are treated as "0" (which yields an\n   empty sequence of the same type as *s*).  Note that items in the\n   sequence *s* are not copied; they are referenced multiple times.\n   This often haunts new Python programmers; consider:\n\n      >>> lists = [[]] * 3\n      >>> lists\n      [[], [], []]\n      >>> lists[0].append(3)\n      >>> lists\n      [[3], [3], [3]]\n\n   What has happened is that "[[]]" is a one-element list containing\n   an empty list, so all three elements of "[[]] * 3" are references\n   to this single empty list.  Modifying any of the elements of\n   "lists" modifies this single list. You can create a list of\n   different lists this way:\n\n      >>> lists = [[] for i in range(3)]\n      >>> lists[0].append(3)\n      >>> lists[1].append(5)\n      >>> lists[2].append(7)\n      >>> lists\n      [[3], [5], [7]]\n\n   Further explanation is available in the FAQ entry *How do I create\n   a multidimensional list?*.\n\n3. If *i* or *j* is negative, the index is relative to the end of\n   sequence *s*: "len(s) + i" or "len(s) + j" is substituted.  But\n   note that "-0" is still "0".\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n   items with index *k* such that "i <= k < j".  If *i* or *j* is\n   greater than "len(s)", use "len(s)".  If *i* is omitted or "None",\n   use "0".  If *j* is omitted or "None", use "len(s)".  If *i* is\n   greater than or equal to *j*, the slice is empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n   sequence of items with index  "x = i + n*k" such that "0 <= n <\n   (j-i)/k".  In other words, the indices are "i", "i+k", "i+2*k",\n   "i+3*k" and so on, stopping when *j* is reached (but never\n   including *j*).  When *k* is positive, *i* and *j* are reduced to\n   "len(s)" if they are greater. When *k* is negative, *i* and *j* are\n   reduced to "len(s) - 1" if they are greater.  If *i* or *j* are\n   omitted or "None", they become "end" values (which end depends on\n   the sign of *k*).  Note, *k* cannot be zero. If *k* is "None", it\n   is treated like "1".\n\n6. Concatenating immutable sequences always results in a new\n   object. This means that building up a sequence by repeated\n   concatenation will have a quadratic runtime cost in the total\n   sequence length. To get a linear runtime cost, you must switch to\n   one of the alternatives below:\n\n   * if concatenating "str" objects, you can build a list and use\n     "str.join()" at the end or else write to an "io.StringIO"\n     instance and retrieve its value when complete\n\n   * if concatenating "bytes" objects, you can similarly use\n     "bytes.join()" or "io.BytesIO", or you can do in-place\n     concatenation with a "bytearray" object.  "bytearray" objects are\n     mutable and have an efficient overallocation mechanism\n\n   * if concatenating "tuple" objects, extend a "list" instead\n\n   * for other types, investigate the relevant class documentation\n\n7. Some sequence types (such as "range") only support item\n   sequences that follow specific patterns, and hence don\'t support\n   sequence concatenation or repetition.\n\n8. "index" raises "ValueError" when *x* is not found in *s*. When\n   supported, the additional arguments to the index method allow\n   efficient searching of subsections of the sequence. Passing the\n   extra arguments is roughly equivalent to using "s[i:j].index(x)",\n   only without copying any data and with the returned index being\n   relative to the start of the sequence rather than the start of the\n   slice.\n\n\nImmutable Sequence Types\n========================\n\nThe only operation that immutable sequence types generally implement\nthat is not also implemented by mutable sequence types is support for\nthe "hash()" built-in.\n\nThis support allows immutable sequences, such as "tuple" instances, to\nbe used as "dict" keys and stored in "set" and "frozenset" instances.\n\nAttempting to hash an immutable sequence that contains unhashable\nvalues will result in "TypeError".\n\n\nMutable Sequence Types\n======================\n\nThe operations in the following table are defined on mutable sequence\ntypes. The "collections.abc.MutableSequence" ABC is provided to make\nit easier to correctly implement these operations on custom sequence\ntypes.\n\nIn the table *s* is an instance of a mutable sequence type, *t* is any\niterable object and *x* is an arbitrary object that meets any type and\nvalue restrictions imposed by *s* (for example, "bytearray" only\naccepts integers that meet the value restriction "0 <= x <= 255").\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation                      | Result                           | Notes                 |\n+================================+==================================+=======================+\n| "s[i] = x"                     | item *i* of *s* is replaced by   |                       |\n|                                | *x*                              |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s[i:j] = t"                   | slice of *s* from *i* to *j* is  |                       |\n|                                | replaced by the contents of the  |                       |\n|                                | iterable *t*                     |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "del s[i:j]"                   | same as "s[i:j] = []"            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s[i:j:k] = t"                 | the elements of "s[i:j:k]" are   | (1)                   |\n|                                | replaced by those of *t*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "del s[i:j:k]"                 | removes the elements of          |                       |\n|                                | "s[i:j:k]" from the list         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.append(x)"                  | appends *x* to the end of the    |                       |\n|                                | sequence (same as                |                       |\n|                                | "s[len(s):len(s)] = [x]")        |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.clear()"                    | removes all items from "s" (same | (5)                   |\n|                                | as "del s[:]")                   |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.copy()"                     | creates a shallow copy of "s"    | (5)                   |\n|                                | (same as "s[:]")                 |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.extend(t)" or "s += t"      | extends *s* with the contents of |                       |\n|                                | *t* (for the most part the same  |                       |\n|                                | as "s[len(s):len(s)] = t")       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s *= n"                       | updates *s* with its contents    | (6)                   |\n|                                | repeated *n* times               |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.insert(i, x)"               | inserts *x* into *s* at the      |                       |\n|                                | index given by *i* (same as      |                       |\n|                                | "s[i:i] = [x]")                  |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.pop([i])"                   | retrieves the item at *i* and    | (2)                   |\n|                                | also removes it from *s*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.remove(x)"                  | remove the first item from *s*   | (3)                   |\n|                                | where "s[i] == x"                |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.reverse()"                  | reverses the items of *s* in     | (4)                   |\n|                                | place                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The optional argument *i* defaults to "-1", so that by default\n   the last item is removed and returned.\n\n3. "remove" raises "ValueError" when *x* is not found in *s*.\n\n4. The "reverse()" method modifies the sequence in place for\n   economy of space when reversing a large sequence.  To remind users\n   that it operates by side effect, it does not return the reversed\n   sequence.\n\n5. "clear()" and "copy()" are included for consistency with the\n   interfaces of mutable containers that don\'t support slicing\n   operations (such as "dict" and "set")\n\n   New in version 3.3: "clear()" and "copy()" methods.\n\n6. The value *n* is an integer, or an object implementing\n   "__index__()".  Zero and negative values of *n* clear the sequence.\n   Items in the sequence are not copied; they are referenced multiple\n   times, as explained for "s * n" under *Common Sequence Operations*.\n\n\nLists\n=====\n\nLists are mutable sequences, typically used to store collections of\nhomogeneous items (where the precise degree of similarity will vary by\napplication).\n\nclass class list([iterable])\n\n   Lists may be constructed in several ways:\n\n   * Using a pair of square brackets to denote the empty list: "[]"\n\n   * Using square brackets, separating items with commas: "[a]",\n     "[a, b, c]"\n\n   * Using a list comprehension: "[x for x in iterable]"\n\n   * Using the type constructor: "list()" or "list(iterable)"\n\n   The constructor builds a list whose items are the same and in the\n   same order as *iterable*\'s items.  *iterable* may be either a\n   sequence, a container that supports iteration, or an iterator\n   object.  If *iterable* is already a list, a copy is made and\n   returned, similar to "iterable[:]". For example, "list(\'abc\')"\n   returns "[\'a\', \'b\', \'c\']" and "list( (1, 2, 3) )" returns "[1, 2,\n   3]". If no argument is given, the constructor creates a new empty\n   list, "[]".\n\n   Many other operations also produce lists, including the "sorted()"\n   built-in.\n\n   Lists implement all of the *common* and *mutable* sequence\n   operations. Lists also provide the following additional method:\n\n   sort(*, key=None, reverse=None)\n\n      This method sorts the list in place, using only "<" comparisons\n      between items. Exceptions are not suppressed - if any comparison\n      operations fail, the entire sort operation will fail (and the\n      list will likely be left in a partially modified state).\n\n      "sort()" accepts two arguments that can only be passed by\n      keyword (*keyword-only arguments*):\n\n      *key* specifies a function of one argument that is used to\n      extract a comparison key from each list element (for example,\n      "key=str.lower"). The key corresponding to each item in the list\n      is calculated once and then used for the entire sorting process.\n      The default value of "None" means that list items are sorted\n      directly without calculating a separate key value.\n\n      The "functools.cmp_to_key()" utility is available to convert a\n      2.x style *cmp* function to a *key* function.\n\n      *reverse* is a boolean value.  If set to "True", then the list\n      elements are sorted as if each comparison were reversed.\n\n      This method modifies the sequence in place for economy of space\n      when sorting a large sequence.  To remind users that it operates\n      by side effect, it does not return the sorted sequence (use\n      "sorted()" to explicitly request a new sorted list instance).\n\n      The "sort()" method is guaranteed to be stable.  A sort is\n      stable if it guarantees not to change the relative order of\n      elements that compare equal --- this is helpful for sorting in\n      multiple passes (for example, sort by department, then by salary\n      grade).\n\n      **CPython implementation detail:** While a list is being sorted,\n      the effect of attempting to mutate, or even inspect, the list is\n      undefined.  The C implementation of Python makes the list appear\n      empty for the duration, and raises "ValueError" if it can detect\n      that the list has been mutated during a sort.\n\n\nTuples\n======\n\nTuples are immutable sequences, typically used to store collections of\nheterogeneous data (such as the 2-tuples produced by the "enumerate()"\nbuilt-in). Tuples are also used for cases where an immutable sequence\nof homogeneous data is needed (such as allowing storage in a "set" or\n"dict" instance).\n\nclass class tuple([iterable])\n\n   Tuples may be constructed in a number of ways:\n\n   * Using a pair of parentheses to denote the empty tuple: "()"\n\n   * Using a trailing comma for a singleton tuple: "a," or "(a,)"\n\n   * Separating items with commas: "a, b, c" or "(a, b, c)"\n\n   * Using the "tuple()" built-in: "tuple()" or "tuple(iterable)"\n\n   The constructor builds a tuple whose items are the same and in the\n   same order as *iterable*\'s items.  *iterable* may be either a\n   sequence, a container that supports iteration, or an iterator\n   object.  If *iterable* is already a tuple, it is returned\n   unchanged. For example, "tuple(\'abc\')" returns "(\'a\', \'b\', \'c\')"\n   and "tuple( [1, 2, 3] )" returns "(1, 2, 3)". If no argument is\n   given, the constructor creates a new empty tuple, "()".\n\n   Note that it is actually the comma which makes a tuple, not the\n   parentheses. The parentheses are optional, except in the empty\n   tuple case, or when they are needed to avoid syntactic ambiguity.\n   For example, "f(a, b, c)" is a function call with three arguments,\n   while "f((a, b, c))" is a function call with a 3-tuple as the sole\n   argument.\n\n   Tuples implement all of the *common* sequence operations.\n\nFor heterogeneous collections of data where access by name is clearer\nthan access by index, "collections.namedtuple()" may be a more\nappropriate choice than a simple tuple object.\n\n\nRanges\n======\n\nThe "range" type represents an immutable sequence of numbers and is\ncommonly used for looping a specific number of times in "for" loops.\n\nclass class range(stop)\nclass class range(start, stop[, step])\n\n   The arguments to the range constructor must be integers (either\n   built-in "int" or any object that implements the "__index__"\n   special method).  If the *step* argument is omitted, it defaults to\n   "1". If the *start* argument is omitted, it defaults to "0". If\n   *step* is zero, "ValueError" is raised.\n\n   For a positive *step*, the contents of a range "r" are determined\n   by the formula "r[i] = start + step*i" where "i >= 0" and "r[i] <\n   stop".\n\n   For a negative *step*, the contents of the range are still\n   determined by the formula "r[i] = start + step*i", but the\n   constraints are "i >= 0" and "r[i] > stop".\n\n   A range object will be empty if "r[0]" does not meet the value\n   constraint. Ranges do support negative indices, but these are\n   interpreted as indexing from the end of the sequence determined by\n   the positive indices.\n\n   Ranges containing absolute values larger than "sys.maxsize" are\n   permitted but some features (such as "len()") may raise\n   "OverflowError".\n\n   Range examples:\n\n      >>> list(range(10))\n      [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n      >>> list(range(1, 11))\n      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n      >>> list(range(0, 30, 5))\n      [0, 5, 10, 15, 20, 25]\n      >>> list(range(0, 10, 3))\n      [0, 3, 6, 9]\n      >>> list(range(0, -10, -1))\n      [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]\n      >>> list(range(0))\n      []\n      >>> list(range(1, 0))\n      []\n\n   Ranges implement all of the *common* sequence operations except\n   concatenation and repetition (due to the fact that range objects\n   can only represent sequences that follow a strict pattern and\n   repetition and concatenation will usually violate that pattern).\n\n   start\n\n      The value of the *start* parameter (or "0" if the parameter was\n      not supplied)\n\n   stop\n\n      The value of the *stop* parameter\n\n   step\n\n      The value of the *step* parameter (or "1" if the parameter was\n      not supplied)\n\nThe advantage of the "range" type over a regular "list" or "tuple" is\nthat a "range" object will always take the same (small) amount of\nmemory, no matter the size of the range it represents (as it only\nstores the "start", "stop" and "step" values, calculating individual\nitems and subranges as needed).\n\nRange objects implement the "collections.abc.Sequence" ABC, and\nprovide features such as containment tests, element index lookup,\nslicing and support for negative indices (see *Sequence Types ---\nlist, tuple, range*):\n\n>>> r = range(0, 20, 2)\n>>> r\nrange(0, 20, 2)\n>>> 11 in r\nFalse\n>>> 10 in r\nTrue\n>>> r.index(10)\n5\n>>> r[5]\n10\n>>> r[:5]\nrange(0, 10, 2)\n>>> r[-1]\n18\n\nTesting range objects for equality with "==" and "!=" compares them as\nsequences.  That is, two range objects are considered equal if they\nrepresent the same sequence of values.  (Note that two range objects\nthat compare equal might have different "start", "stop" and "step"\nattributes, for example "range(0) == range(2, 1, 3)" or "range(0, 3,\n2) == range(0, 4, 2)".)\n\nChanged in version 3.2: Implement the Sequence ABC. Support slicing\nand negative indices. Test "int" objects for membership in constant\ntime instead of iterating through all items.\n\nChanged in version 3.3: Define \'==\' and \'!=\' to compare range objects\nbased on the sequence of values they define (instead of comparing\nbased on object identity).\n\nNew in version 3.3: The "start", "stop" and "step" attributes.\n',
- 'typesseq-mutable': u'\nMutable Sequence Types\n**********************\n\nThe operations in the following table are defined on mutable sequence\ntypes. The "collections.abc.MutableSequence" ABC is provided to make\nit easier to correctly implement these operations on custom sequence\ntypes.\n\nIn the table *s* is an instance of a mutable sequence type, *t* is any\niterable object and *x* is an arbitrary object that meets any type and\nvalue restrictions imposed by *s* (for example, "bytearray" only\naccepts integers that meet the value restriction "0 <= x <= 255").\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation                      | Result                           | Notes                 |\n+================================+==================================+=======================+\n| "s[i] = x"                     | item *i* of *s* is replaced by   |                       |\n|                                | *x*                              |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s[i:j] = t"                   | slice of *s* from *i* to *j* is  |                       |\n|                                | replaced by the contents of the  |                       |\n|                                | iterable *t*                     |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "del s[i:j]"                   | same as "s[i:j] = []"            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s[i:j:k] = t"                 | the elements of "s[i:j:k]" are   | (1)                   |\n|                                | replaced by those of *t*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "del s[i:j:k]"                 | removes the elements of          |                       |\n|                                | "s[i:j:k]" from the list         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.append(x)"                  | appends *x* to the end of the    |                       |\n|                                | sequence (same as                |                       |\n|                                | "s[len(s):len(s)] = [x]")        |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.clear()"                    | removes all items from "s" (same | (5)                   |\n|                                | as "del s[:]")                   |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.copy()"                     | creates a shallow copy of "s"    | (5)                   |\n|                                | (same as "s[:]")                 |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.extend(t)" or "s += t"      | extends *s* with the contents of |                       |\n|                                | *t* (for the most part the same  |                       |\n|                                | as "s[len(s):len(s)] = t")       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s *= n"                       | updates *s* with its contents    | (6)                   |\n|                                | repeated *n* times               |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.insert(i, x)"               | inserts *x* into *s* at the      |                       |\n|                                | index given by *i* (same as      |                       |\n|                                | "s[i:i] = [x]")                  |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.pop([i])"                   | retrieves the item at *i* and    | (2)                   |\n|                                | also removes it from *s*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.remove(x)"                  | remove the first item from *s*   | (3)                   |\n|                                | where "s[i] == x"                |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| "s.reverse()"                  | reverses the items of *s* in     | (4)                   |\n|                                | place                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The optional argument *i* defaults to "-1", so that by default\n   the last item is removed and returned.\n\n3. "remove" raises "ValueError" when *x* is not found in *s*.\n\n4. The "reverse()" method modifies the sequence in place for\n   economy of space when reversing a large sequence.  To remind users\n   that it operates by side effect, it does not return the reversed\n   sequence.\n\n5. "clear()" and "copy()" are included for consistency with the\n   interfaces of mutable containers that don\'t support slicing\n   operations (such as "dict" and "set")\n\n   New in version 3.3: "clear()" and "copy()" methods.\n\n6. The value *n* is an integer, or an object implementing\n   "__index__()".  Zero and negative values of *n* clear the sequence.\n   Items in the sequence are not copied; they are referenced multiple\n   times, as explained for "s * n" under *Common Sequence Operations*.\n',
- 'unary': u'\nUnary arithmetic and bitwise operations\n***************************************\n\nAll unary arithmetic and bitwise operations have the same priority:\n\n   u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n\nThe unary "-" (minus) operator yields the negation of its numeric\nargument.\n\nThe unary "+" (plus) operator yields its numeric argument unchanged.\n\nThe unary "~" (invert) operator yields the bitwise inversion of its\ninteger argument.  The bitwise inversion of "x" is defined as\n"-(x+1)".  It only applies to integral numbers.\n\nIn all three cases, if the argument does not have the proper type, a\n"TypeError" exception is raised.\n',
- 'while': u'\nThe "while" statement\n*********************\n\nThe "while" statement is used for repeated execution as long as an\nexpression is true:\n\n   while_stmt ::= "while" expression ":" suite\n                  ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the "else" clause, if present, is executed\nand the loop terminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite.  A "continue" statement\nexecuted in the first suite skips the rest of the suite and goes back\nto testing the expression.\n',
- 'with': u'\nThe "with" statement\n********************\n\nThe "with" statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common "try"..."except"..."finally"\nusage patterns to be encapsulated for convenient reuse.\n\n   with_stmt ::= "with" with_item ("," with_item)* ":" suite\n   with_item ::= expression ["as" target]\n\nThe execution of the "with" statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the "with_item")\n   is evaluated to obtain a context manager.\n\n2. The context manager\'s "__exit__()" is loaded for later use.\n\n3. The context manager\'s "__enter__()" method is invoked.\n\n4. If a target was included in the "with" statement, the return\n   value from "__enter__()" is assigned to it.\n\n   Note: The "with" statement guarantees that if the "__enter__()"\n     method returns without an error, then "__exit__()" will always be\n     called. Thus, if an error occurs during the assignment to the\n     target list, it will be treated the same as an error occurring\n     within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s "__exit__()" method is invoked.  If an\n   exception caused the suite to be exited, its type, value, and\n   traceback are passed as arguments to "__exit__()". Otherwise, three\n   "None" arguments are supplied.\n\n   If the suite was exited due to an exception, and the return value\n   from the "__exit__()" method was false, the exception is reraised.\n   If the return value was true, the exception is suppressed, and\n   execution continues with the statement following the "with"\n   statement.\n\n   If the suite was exited for any reason other than an exception, the\n   return value from "__exit__()" is ignored, and execution proceeds\n   at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple "with" statements were nested:\n\n   with A() as a, B() as b:\n       suite\n\nis equivalent to\n\n   with A() as a:\n       with B() as b:\n           suite\n\nChanged in version 3.1: Support for multiple context expressions.\n\nSee also: **PEP 343** - The "with" statement\n\n     The specification, background, and examples for the Python "with"\n     statement.\n',
- 'yield': u'\nThe "yield" statement\n*********************\n\n   yield_stmt ::= yield_expression\n\nA "yield" statement is semantically equivalent to a *yield\nexpression*. The yield statement can be used to omit the parentheses\nthat would otherwise be required in the equivalent yield expression\nstatement. For example, the yield statements\n\n   yield <expr>\n   yield from <expr>\n\nare equivalent to the yield expression statements\n\n   (yield <expr>)\n   (yield from <expr>)\n\nYield expressions and statements are only used when defining a\n*generator* function, and are only used in the body of the generator\nfunction.  Using yield in a function definition is sufficient to cause\nthat definition to create a generator function instead of a normal\nfunction.\n\nFor full details of "yield" semantics, refer to the *Yield\nexpressions* section.\n'}
+# Autogenerated by Sphinx on Mon Sep  9 12:15:17 2019
+topics = {'assert': '\n'
+           'The "assert" statement\n'
+           '**********************\n'
+           '\n'
+           'Assert statements are a convenient way to insert debugging '
+           'assertions\n'
+           'into a program:\n'
+           '\n'
+           '   assert_stmt ::= "assert" expression ["," expression]\n'
+           '\n'
+           'The simple form, "assert expression", is equivalent to\n'
+           '\n'
+           '   if __debug__:\n'
+           '       if not expression: raise AssertionError\n'
+           '\n'
+           'The extended form, "assert expression1, expression2", is '
+           'equivalent to\n'
+           '\n'
+           '   if __debug__:\n'
+           '       if not expression1: raise AssertionError(expression2)\n'
+           '\n'
+           'These equivalences assume that "__debug__" and "AssertionError" '
+           'refer\n'
+           'to the built-in variables with those names.  In the current\n'
+           'implementation, the built-in variable "__debug__" is "True" under\n'
+           'normal circumstances, "False" when optimization is requested '
+           '(command\n'
+           'line option -O).  The current code generator emits no code for an\n'
+           'assert statement when optimization is requested at compile time.  '
+           'Note\n'
+           'that it is unnecessary to include the source code for the '
+           'expression\n'
+           'that failed in the error message; it will be displayed as part of '
+           'the\n'
+           'stack trace.\n'
+           '\n'
+           'Assignments to "__debug__" are illegal.  The value for the '
+           'built-in\n'
+           'variable is determined when the interpreter starts.\n',
+ 'assignment': '\n'
+               'Assignment statements\n'
+               '*********************\n'
+               '\n'
+               'Assignment statements are used to (re)bind names to values and '
+               'to\n'
+               'modify attributes or items of mutable objects:\n'
+               '\n'
+               '   assignment_stmt ::= (target_list "=")+ (starred_expression '
+               '| yield_expression)\n'
+               '   target_list     ::= target ("," target)* [","]\n'
+               '   target          ::= identifier\n'
+               '              | "(" target_list ")"\n'
+               '              | "[" [target_list] "]"\n'
+               '              | attributeref\n'
+               '              | subscription\n'
+               '              | slicing\n'
+               '              | "*" target\n'
+               '\n'
+               '(See section *Primaries* for the syntax definitions for\n'
+               '*attributeref*, *subscription*, and *slicing*.)\n'
+               '\n'
+               'An assignment statement evaluates the expression list '
+               '(remember that\n'
+               'this can be a single expression or a comma-separated list, the '
+               'latter\n'
+               'yielding a tuple) and assigns the single resulting object to '
+               'each of\n'
+               'the target lists, from left to right.\n'
+               '\n'
+               'Assignment is defined recursively depending on the form of the '
+               'target\n'
+               '(list). When a target is part of a mutable object (an '
+               'attribute\n'
+               'reference, subscription or slicing), the mutable object must\n'
+               'ultimately perform the assignment and decide about its '
+               'validity, and\n'
+               'may raise an exception if the assignment is unacceptable.  The '
+               'rules\n'
+               'observed by various types and the exceptions raised are given '
+               'with the\n'
+               'definition of the object types (see section *The standard '
+               'type\n'
+               'hierarchy*).\n'
+               '\n'
+               'Assignment of an object to a target list, optionally enclosed '
+               'in\n'
+               'parentheses or square brackets, is recursively defined as '
+               'follows.\n'
+               '\n'
+               '* If the target list is empty: The object must also be an '
+               'empty\n'
+               '  iterable.\n'
+               '\n'
+               '* If the target list is a single target in parentheses: The '
+               'object\n'
+               '  is assigned to that target.\n'
+               '\n'
+               '* If the target list is a comma-separated list of targets, or '
+               'a\n'
+               '  single target in square brackets: The object must be an '
+               'iterable\n'
+               '  with the same number of items as there are targets in the '
+               'target\n'
+               '  list, and the items are assigned, from left to right, to '
+               'the\n'
+               '  corresponding targets.\n'
+               '\n'
+               '  * If the target list contains one target prefixed with an\n'
+               '    asterisk, called a "starred" target: The object must be '
+               'an\n'
+               '    iterable with at least as many items as there are targets '
+               'in the\n'
+               '    target list, minus one.  The first items of the iterable '
+               'are\n'
+               '    assigned, from left to right, to the targets before the '
+               'starred\n'
+               '    target.  The final items of the iterable are assigned to '
+               'the\n'
+               '    targets after the starred target.  A list of the remaining '
+               'items\n'
+               '    in the iterable is then assigned to the starred target '
+               '(the list\n'
+               '    can be empty).\n'
+               '\n'
+               '  * Else: The object must be an iterable with the same number '
+               'of\n'
+               '    items as there are targets in the target list, and the '
+               'items are\n'
+               '    assigned, from left to right, to the corresponding '
+               'targets.\n'
+               '\n'
+               'Assignment of an object to a single target is recursively '
+               'defined as\n'
+               'follows.\n'
+               '\n'
+               '* If the target is an identifier (name):\n'
+               '\n'
+               '  * If the name does not occur in a "global" or "nonlocal" '
+               'statement\n'
+               '    in the current code block: the name is bound to the object '
+               'in the\n'
+               '    current local namespace.\n'
+               '\n'
+               '  * Otherwise: the name is bound to the object in the global\n'
+               '    namespace or the outer namespace determined by '
+               '"nonlocal",\n'
+               '    respectively.\n'
+               '\n'
+               '  The name is rebound if it was already bound.  This may cause '
+               'the\n'
+               '  reference count for the object previously bound to the name '
+               'to reach\n'
+               '  zero, causing the object to be deallocated and its '
+               'destructor (if it\n'
+               '  has one) to be called.\n'
+               '\n'
+               '* If the target is an attribute reference: The primary '
+               'expression in\n'
+               '  the reference is evaluated.  It should yield an object with\n'
+               '  assignable attributes; if this is not the case, "TypeError" '
+               'is\n'
+               '  raised.  That object is then asked to assign the assigned '
+               'object to\n'
+               '  the given attribute; if it cannot perform the assignment, it '
+               'raises\n'
+               '  an exception (usually but not necessarily '
+               '"AttributeError").\n'
+               '\n'
+               '  Note: If the object is a class instance and the attribute '
+               'reference\n'
+               '  occurs on both sides of the assignment operator, the RHS '
+               'expression,\n'
+               '  "a.x" can access either an instance attribute or (if no '
+               'instance\n'
+               '  attribute exists) a class attribute.  The LHS target "a.x" '
+               'is always\n'
+               '  set as an instance attribute, creating it if necessary.  '
+               'Thus, the\n'
+               '  two occurrences of "a.x" do not necessarily refer to the '
+               'same\n'
+               '  attribute: if the RHS expression refers to a class '
+               'attribute, the\n'
+               '  LHS creates a new instance attribute as the target of the\n'
+               '  assignment:\n'
+               '\n'
+               '     class Cls:\n'
+               '         x = 3             # class variable\n'
+               '     inst = Cls()\n'
+               '     inst.x = inst.x + 1   # writes inst.x as 4 leaving Cls.x '
+               'as 3\n'
+               '\n'
+               '  This description does not necessarily apply to descriptor\n'
+               '  attributes, such as properties created with "property()".\n'
+               '\n'
+               '* If the target is a subscription: The primary expression in '
+               'the\n'
+               '  reference is evaluated.  It should yield either a mutable '
+               'sequence\n'
+               '  object (such as a list) or a mapping object (such as a '
+               'dictionary).\n'
+               '  Next, the subscript expression is evaluated.\n'
+               '\n'
+               '  If the primary is a mutable sequence object (such as a '
+               'list), the\n'
+               '  subscript must yield an integer.  If it is negative, the '
+               "sequence's\n"
+               '  length is added to it.  The resulting value must be a '
+               'nonnegative\n'
+               "  integer less than the sequence's length, and the sequence is "
+               'asked\n'
+               '  to assign the assigned object to its item with that index.  '
+               'If the\n'
+               '  index is out of range, "IndexError" is raised (assignment to '
+               'a\n'
+               '  subscripted sequence cannot add new items to a list).\n'
+               '\n'
+               '  If the primary is a mapping object (such as a dictionary), '
+               'the\n'
+               "  subscript must have a type compatible with the mapping's key "
+               'type,\n'
+               '  and the mapping is then asked to create a key/datum pair '
+               'which maps\n'
+               '  the subscript to the assigned object.  This can either '
+               'replace an\n'
+               '  existing key/value pair with the same key value, or insert a '
+               'new\n'
+               '  key/value pair (if no key with the same value existed).\n'
+               '\n'
+               '  For user-defined objects, the "__setitem__()" method is '
+               'called with\n'
+               '  appropriate arguments.\n'
+               '\n'
+               '* If the target is a slicing: The primary expression in the\n'
+               '  reference is evaluated.  It should yield a mutable sequence '
+               'object\n'
+               '  (such as a list).  The assigned object should be a sequence '
+               'object\n'
+               '  of the same type.  Next, the lower and upper bound '
+               'expressions are\n'
+               '  evaluated, insofar they are present; defaults are zero and '
+               'the\n'
+               "  sequence's length.  The bounds should evaluate to integers. "
+               'If\n'
+               "  either bound is negative, the sequence's length is added to "
+               'it.  The\n'
+               '  resulting bounds are clipped to lie between zero and the '
+               "sequence's\n"
+               '  length, inclusive.  Finally, the sequence object is asked to '
+               'replace\n'
+               '  the slice with the items of the assigned sequence.  The '
+               'length of\n'
+               '  the slice may be different from the length of the assigned '
+               'sequence,\n'
+               '  thus changing the length of the target sequence, if the '
+               'target\n'
+               '  sequence allows it.\n'
+               '\n'
+               '**CPython implementation detail:** In the current '
+               'implementation, the\n'
+               'syntax for targets is taken to be the same as for expressions, '
+               'and\n'
+               'invalid syntax is rejected during the code generation phase, '
+               'causing\n'
+               'less detailed error messages.\n'
+               '\n'
+               'Although the definition of assignment implies that overlaps '
+               'between\n'
+               "the left-hand side and the right-hand side are 'simultaneous' "
+               '(for\n'
+               'example "a, b = b, a" swaps two variables), overlaps *within* '
+               'the\n'
+               'collection of assigned-to variables occur left-to-right, '
+               'sometimes\n'
+               'resulting in confusion.  For instance, the following program '
+               'prints\n'
+               '"[0, 2]":\n'
+               '\n'
+               '   x = [0, 1]\n'
+               '   i = 0\n'
+               '   i, x[i] = 1, 2         # i is updated, then x[i] is '
+               'updated\n'
+               '   print(x)\n'
+               '\n'
+               'See also: **PEP 3132** - Extended Iterable Unpacking\n'
+               '\n'
+               '     The specification for the "*target" feature.\n'
+               '\n'
+               '\n'
+               'Augmented assignment statements\n'
+               '===============================\n'
+               '\n'
+               'Augmented assignment is the combination, in a single '
+               'statement, of a\n'
+               'binary operation and an assignment statement:\n'
+               '\n'
+               '   augmented_assignment_stmt ::= augtarget augop '
+               '(expression_list | yield_expression)\n'
+               '   augtarget                 ::= identifier | attributeref | '
+               'subscription | slicing\n'
+               '   augop                     ::= "+=" | "-=" | "*=" | "@=" | '
+               '"/=" | "//=" | "%=" | "**="\n'
+               '             | ">>=" | "<<=" | "&=" | "^=" | "|="\n'
+               '\n'
+               '(See section *Primaries* for the syntax definitions of the '
+               'last three\n'
+               'symbols.)\n'
+               '\n'
+               'An augmented assignment evaluates the target (which, unlike '
+               'normal\n'
+               'assignment statements, cannot be an unpacking) and the '
+               'expression\n'
+               'list, performs the binary operation specific to the type of '
+               'assignment\n'
+               'on the two operands, and assigns the result to the original '
+               'target.\n'
+               'The target is only evaluated once.\n'
+               '\n'
+               'An augmented assignment expression like "x += 1" can be '
+               'rewritten as\n'
+               '"x = x + 1" to achieve a similar, but not exactly equal '
+               'effect. In the\n'
+               'augmented version, "x" is only evaluated once. Also, when '
+               'possible,\n'
+               'the actual operation is performed *in-place*, meaning that '
+               'rather than\n'
+               'creating a new object and assigning that to the target, the '
+               'old object\n'
+               'is modified instead.\n'
+               '\n'
+               'Unlike normal assignments, augmented assignments evaluate the '
+               'left-\n'
+               'hand side *before* evaluating the right-hand side.  For '
+               'example, "a[i]\n'
+               '+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and '
+               'performs\n'
+               'the addition, and lastly, it writes the result back to '
+               '"a[i]".\n'
+               '\n'
+               'With the exception of assigning to tuples and multiple targets '
+               'in a\n'
+               'single statement, the assignment done by augmented assignment\n'
+               'statements is handled the same way as normal assignments. '
+               'Similarly,\n'
+               'with the exception of the possible *in-place* behavior, the '
+               'binary\n'
+               'operation performed by augmented assignment is the same as the '
+               'normal\n'
+               'binary operations.\n'
+               '\n'
+               'For targets which are attribute references, the same *caveat '
+               'about\n'
+               'class and instance attributes* applies as for regular '
+               'assignments.\n',
+ 'atom-identifiers': '\n'
+                     'Identifiers (Names)\n'
+                     '*******************\n'
+                     '\n'
+                     'An identifier occurring as an atom is a name.  See '
+                     'section\n'
+                     '*Identifiers and keywords* for lexical definition and '
+                     'section *Naming\n'
+                     'and binding* for documentation of naming and binding.\n'
+                     '\n'
+                     'When the name is bound to an object, evaluation of the '
+                     'atom yields\n'
+                     'that object. When a name is not bound, an attempt to '
+                     'evaluate it\n'
+                     'raises a "NameError" exception.\n'
+                     '\n'
+                     '**Private name mangling:** When an identifier that '
+                     'textually occurs in\n'
+                     'a class definition begins with two or more underscore '
+                     'characters and\n'
+                     'does not end in two or more underscores, it is '
+                     'considered a *private\n'
+                     'name* of that class. Private names are transformed to a '
+                     'longer form\n'
+                     'before code is generated for them.  The transformation '
+                     'inserts the\n'
+                     'class name, with leading underscores removed and a '
+                     'single underscore\n'
+                     'inserted, in front of the name.  For example, the '
+                     'identifier "__spam"\n'
+                     'occurring in a class named "Ham" will be transformed to '
+                     '"_Ham__spam".\n'
+                     'This transformation is independent of the syntactical '
+                     'context in which\n'
+                     'the identifier is used.  If the transformed name is '
+                     'extremely long\n'
+                     '(longer than 255 characters), implementation defined '
+                     'truncation may\n'
+                     'happen. If the class name consists only of underscores, '
+                     'no\n'
+                     'transformation is done.\n',
+ 'atom-literals': '\n'
+                  'Literals\n'
+                  '********\n'
+                  '\n'
+                  'Python supports string and bytes literals and various '
+                  'numeric\n'
+                  'literals:\n'
+                  '\n'
+                  '   literal ::= stringliteral | bytesliteral\n'
+                  '               | integer | floatnumber | imagnumber\n'
+                  '\n'
+                  'Evaluation of a literal yields an object of the given type '
+                  '(string,\n'
+                  'bytes, integer, floating point number, complex number) with '
+                  'the given\n'
+                  'value.  The value may be approximated in the case of '
+                  'floating point\n'
+                  'and imaginary (complex) literals.  See section *Literals* '
+                  'for details.\n'
+                  '\n'
+                  'All literals correspond to immutable data types, and hence '
+                  'the\n'
+                  "object's identity is less important than its value.  "
+                  'Multiple\n'
+                  'evaluations of literals with the same value (either the '
+                  'same\n'
+                  'occurrence in the program text or a different occurrence) '
+                  'may obtain\n'
+                  'the same object or a different object with the same '
+                  'value.\n',
+ 'attribute-access': '\n'
+                     'Customizing attribute access\n'
+                     '****************************\n'
+                     '\n'
+                     'The following methods can be defined to customize the '
+                     'meaning of\n'
+                     'attribute access (use of, assignment to, or deletion of '
+                     '"x.name") for\n'
+                     'class instances.\n'
+                     '\n'
+                     'object.__getattr__(self, name)\n'
+                     '\n'
+                     '   Called when an attribute lookup has not found the '
+                     'attribute in the\n'
+                     '   usual places (i.e. it is not an instance attribute '
+                     'nor is it found\n'
+                     '   in the class tree for "self").  "name" is the '
+                     'attribute name. This\n'
+                     '   method should return the (computed) attribute value '
+                     'or raise an\n'
+                     '   "AttributeError" exception.\n'
+                     '\n'
+                     '   Note that if the attribute is found through the '
+                     'normal mechanism,\n'
+                     '   "__getattr__()" is not called.  (This is an '
+                     'intentional asymmetry\n'
+                     '   between "__getattr__()" and "__setattr__()".) This is '
+                     'done both for\n'
+                     '   efficiency reasons and because otherwise '
+                     '"__getattr__()" would have\n'
+                     '   no way to access other attributes of the instance.  '
+                     'Note that at\n'
+                     '   least for instance variables, you can fake total '
+                     'control by not\n'
+                     '   inserting any values in the instance attribute '
+                     'dictionary (but\n'
+                     '   instead inserting them in another object).  See the\n'
+                     '   "__getattribute__()" method below for a way to '
+                     'actually get total\n'
+                     '   control over attribute access.\n'
+                     '\n'
+                     'object.__getattribute__(self, name)\n'
+                     '\n'
+                     '   Called unconditionally to implement attribute '
+                     'accesses for\n'
+                     '   instances of the class. If the class also defines '
+                     '"__getattr__()",\n'
+                     '   the latter will not be called unless '
+                     '"__getattribute__()" either\n'
+                     '   calls it explicitly or raises an "AttributeError". '
+                     'This method\n'
+                     '   should return the (computed) attribute value or raise '
+                     'an\n'
+                     '   "AttributeError" exception. In order to avoid '
+                     'infinite recursion in\n'
+                     '   this method, its implementation should always call '
+                     'the base class\n'
+                     '   method with the same name to access any attributes it '
+                     'needs, for\n'
+                     '   example, "object.__getattribute__(self, name)".\n'
+                     '\n'
+                     '   Note: This method may still be bypassed when looking '
+                     'up special\n'
+                     '     methods as the result of implicit invocation via '
+                     'language syntax\n'
+                     '     or built-in functions. See *Special method '
+                     'lookup*.\n'
+                     '\n'
+                     'object.__setattr__(self, name, value)\n'
+                     '\n'
+                     '   Called when an attribute assignment is attempted.  '
+                     'This is called\n'
+                     '   instead of the normal mechanism (i.e. store the value '
+                     'in the\n'
+                     '   instance dictionary). *name* is the attribute name, '
+                     '*value* is the\n'
+                     '   value to be assigned to it.\n'
+                     '\n'
+                     '   If "__setattr__()" wants to assign to an instance '
+                     'attribute, it\n'
+                     '   should call the base class method with the same name, '
+                     'for example,\n'
+                     '   "object.__setattr__(self, name, value)".\n'
+                     '\n'
+                     'object.__delattr__(self, name)\n'
+                     '\n'
+                     '   Like "__setattr__()" but for attribute deletion '
+                     'instead of\n'
+                     '   assignment.  This should only be implemented if "del '
+                     'obj.name" is\n'
+                     '   meaningful for the object.\n'
+                     '\n'
+                     'object.__dir__(self)\n'
+                     '\n'
+                     '   Called when "dir()" is called on the object. A '
+                     'sequence must be\n'
+                     '   returned. "dir()" converts the returned sequence to a '
+                     'list and\n'
+                     '   sorts it.\n'
+                     '\n'
+                     '\n'
+                     'Implementing Descriptors\n'
+                     '========================\n'
+                     '\n'
+                     'The following methods only apply when an instance of the '
+                     'class\n'
+                     'containing the method (a so-called *descriptor* class) '
+                     'appears in an\n'
+                     '*owner* class (the descriptor must be in either the '
+                     "owner's class\n"
+                     'dictionary or in the class dictionary for one of its '
+                     'parents).  In the\n'
+                     'examples below, "the attribute" refers to the attribute '
+                     'whose name is\n'
+                     "the key of the property in the owner class' "
+                     '"__dict__".\n'
+                     '\n'
+                     'object.__get__(self, instance, owner)\n'
+                     '\n'
+                     '   Called to get the attribute of the owner class (class '
+                     'attribute\n'
+                     '   access) or of an instance of that class (instance '
+                     'attribute\n'
+                     '   access). *owner* is always the owner class, while '
+                     '*instance* is the\n'
+                     '   instance that the attribute was accessed through, or '
+                     '"None" when\n'
+                     '   the attribute is accessed through the *owner*.  This '
+                     'method should\n'
+                     '   return the (computed) attribute value or raise an '
+                     '"AttributeError"\n'
+                     '   exception.\n'
+                     '\n'
+                     'object.__set__(self, instance, value)\n'
+                     '\n'
+                     '   Called to set the attribute on an instance *instance* '
+                     'of the owner\n'
+                     '   class to a new value, *value*.\n'
+                     '\n'
+                     'object.__delete__(self, instance)\n'
+                     '\n'
+                     '   Called to delete the attribute on an instance '
+                     '*instance* of the\n'
+                     '   owner class.\n'
+                     '\n'
+                     'The attribute "__objclass__" is interpreted by the '
+                     '"inspect" module as\n'
+                     'specifying the class where this object was defined '
+                     '(setting this\n'
+                     'appropriately can assist in runtime introspection of '
+                     'dynamic class\n'
+                     'attributes). For callables, it may indicate that an '
+                     'instance of the\n'
+                     'given type (or a subclass) is expected or required as '
+                     'the first\n'
+                     'positional argument (for example, CPython sets this '
+                     'attribute for\n'
+                     'unbound methods that are implemented in C).\n'
+                     '\n'
+                     '\n'
+                     'Invoking Descriptors\n'
+                     '====================\n'
+                     '\n'
+                     'In general, a descriptor is an object attribute with '
+                     '"binding\n'
+                     'behavior", one whose attribute access has been '
+                     'overridden by methods\n'
+                     'in the descriptor protocol:  "__get__()", "__set__()", '
+                     'and\n'
+                     '"__delete__()". If any of those methods are defined for '
+                     'an object, it\n'
+                     'is said to be a descriptor.\n'
+                     '\n'
+                     'The default behavior for attribute access is to get, '
+                     'set, or delete\n'
+                     "the attribute from an object's dictionary. For instance, "
+                     '"a.x" has a\n'
+                     'lookup chain starting with "a.__dict__[\'x\']", then\n'
+                     '"type(a).__dict__[\'x\']", and continuing through the '
+                     'base classes of\n'
+                     '"type(a)" excluding metaclasses.\n'
+                     '\n'
+                     'However, if the looked-up value is an object defining '
+                     'one of the\n'
+                     'descriptor methods, then Python may override the default '
+                     'behavior and\n'
+                     'invoke the descriptor method instead.  Where this occurs '
+                     'in the\n'
+                     'precedence chain depends on which descriptor methods '
+                     'were defined and\n'
+                     'how they were called.\n'
+                     '\n'
+                     'The starting point for descriptor invocation is a '
+                     'binding, "a.x". How\n'
+                     'the arguments are assembled depends on "a":\n'
+                     '\n'
+                     'Direct Call\n'
+                     '   The simplest and least common call is when user code '
+                     'directly\n'
+                     '   invokes a descriptor method:    "x.__get__(a)".\n'
+                     '\n'
+                     'Instance Binding\n'
+                     '   If binding to an object instance, "a.x" is '
+                     'transformed into the\n'
+                     '   call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n'
+                     '\n'
+                     'Class Binding\n'
+                     '   If binding to a class, "A.x" is transformed into the '
+                     'call:\n'
+                     '   "A.__dict__[\'x\'].__get__(None, A)".\n'
+                     '\n'
+                     'Super Binding\n'
+                     '   If "a" is an instance of "super", then the binding '
+                     '"super(B,\n'
+                     '   obj).m()" searches "obj.__class__.__mro__" for the '
+                     'base class "A"\n'
+                     '   immediately preceding "B" and then invokes the '
+                     'descriptor with the\n'
+                     '   call: "A.__dict__[\'m\'].__get__(obj, '
+                     'obj.__class__)".\n'
+                     '\n'
+                     'For instance bindings, the precedence of descriptor '
+                     'invocation depends\n'
+                     'on the which descriptor methods are defined.  A '
+                     'descriptor can define\n'
+                     'any combination of "__get__()", "__set__()" and '
+                     '"__delete__()".  If it\n'
+                     'does not define "__get__()", then accessing the '
+                     'attribute will return\n'
+                     'the descriptor object itself unless there is a value in '
+                     "the object's\n"
+                     'instance dictionary.  If the descriptor defines '
+                     '"__set__()" and/or\n'
+                     '"__delete__()", it is a data descriptor; if it defines '
+                     'neither, it is\n'
+                     'a non-data descriptor.  Normally, data descriptors '
+                     'define both\n'
+                     '"__get__()" and "__set__()", while non-data descriptors '
+                     'have just the\n'
+                     '"__get__()" method.  Data descriptors with "__set__()" '
+                     'and "__get__()"\n'
+                     'defined always override a redefinition in an instance '
+                     'dictionary.  In\n'
+                     'contrast, non-data descriptors can be overridden by '
+                     'instances.\n'
+                     '\n'
+                     'Python methods (including "staticmethod()" and '
+                     '"classmethod()") are\n'
+                     'implemented as non-data descriptors.  Accordingly, '
+                     'instances can\n'
+                     'redefine and override methods.  This allows individual '
+                     'instances to\n'
+                     'acquire behaviors that differ from other instances of '
+                     'the same class.\n'
+                     '\n'
+                     'The "property()" function is implemented as a data '
+                     'descriptor.\n'
+                     'Accordingly, instances cannot override the behavior of a '
+                     'property.\n'
+                     '\n'
+                     '\n'
+                     '__slots__\n'
+                     '=========\n'
+                     '\n'
+                     'By default, instances of classes have a dictionary for '
+                     'attribute\n'
+                     'storage.  This wastes space for objects having very few '
+                     'instance\n'
+                     'variables.  The space consumption can become acute when '
+                     'creating large\n'
+                     'numbers of instances.\n'
+                     '\n'
+                     'The default can be overridden by defining *__slots__* in '
+                     'a class\n'
+                     'definition. The *__slots__* declaration takes a sequence '
+                     'of instance\n'
+                     'variables and reserves just enough space in each '
+                     'instance to hold a\n'
+                     'value for each variable.  Space is saved because '
+                     '*__dict__* is not\n'
+                     'created for each instance.\n'
+                     '\n'
+                     'object.__slots__\n'
+                     '\n'
+                     '   This class variable can be assigned a string, '
+                     'iterable, or sequence\n'
+                     '   of strings with variable names used by instances.  '
+                     '*__slots__*\n'
+                     '   reserves space for the declared variables and '
+                     'prevents the\n'
+                     '   automatic creation of *__dict__* and *__weakref__* '
+                     'for each\n'
+                     '   instance.\n'
+                     '\n'
+                     '\n'
+                     'Notes on using *__slots__*\n'
+                     '--------------------------\n'
+                     '\n'
+                     '* When inheriting from a class without *__slots__*, the '
+                     '*__dict__*\n'
+                     '  attribute of that class will always be accessible, so '
+                     'a *__slots__*\n'
+                     '  definition in the subclass is meaningless.\n'
+                     '\n'
+                     '* Without a *__dict__* variable, instances cannot be '
+                     'assigned new\n'
+                     '  variables not listed in the *__slots__* definition.  '
+                     'Attempts to\n'
+                     '  assign to an unlisted variable name raises '
+                     '"AttributeError". If\n'
+                     '  dynamic assignment of new variables is desired, then '
+                     'add\n'
+                     '  "\'__dict__\'" to the sequence of strings in the '
+                     '*__slots__*\n'
+                     '  declaration.\n'
+                     '\n'
+                     '* Without a *__weakref__* variable for each instance, '
+                     'classes\n'
+                     '  defining *__slots__* do not support weak references to '
+                     'its\n'
+                     '  instances. If weak reference support is needed, then '
+                     'add\n'
+                     '  "\'__weakref__\'" to the sequence of strings in the '
+                     '*__slots__*\n'
+                     '  declaration.\n'
+                     '\n'
+                     '* *__slots__* are implemented at the class level by '
+                     'creating\n'
+                     '  descriptors (*Implementing Descriptors*) for each '
+                     'variable name.  As\n'
+                     '  a result, class attributes cannot be used to set '
+                     'default values for\n'
+                     '  instance variables defined by *__slots__*; otherwise, '
+                     'the class\n'
+                     '  attribute would overwrite the descriptor assignment.\n'
+                     '\n'
+                     '* The action of a *__slots__* declaration is limited to '
+                     'the class\n'
+                     '  where it is defined.  As a result, subclasses will '
+                     'have a *__dict__*\n'
+                     '  unless they also define *__slots__* (which must only '
+                     'contain names\n'
+                     '  of any *additional* slots).\n'
+                     '\n'
+                     '* If a class defines a slot also defined in a base '
+                     'class, the\n'
+                     '  instance variable defined by the base class slot is '
+                     'inaccessible\n'
+                     '  (except by retrieving its descriptor directly from the '
+                     'base class).\n'
+                     '  This renders the meaning of the program undefined.  In '
+                     'the future, a\n'
+                     '  check may be added to prevent this.\n'
+                     '\n'
+                     '* Nonempty *__slots__* does not work for classes derived '
+                     'from\n'
+                     '  "variable-length" built-in types such as "int", '
+                     '"bytes" and "tuple".\n'
+                     '\n'
+                     '* Any non-string iterable may be assigned to '
+                     '*__slots__*. Mappings\n'
+                     '  may also be used; however, in the future, special '
+                     'meaning may be\n'
+                     '  assigned to the values corresponding to each key.\n'
+                     '\n'
+                     '* *__class__* assignment works only if both classes have '
+                     'the same\n'
+                     '  *__slots__*.\n',
+ 'attribute-references': '\n'
+                         'Attribute references\n'
+                         '********************\n'
+                         '\n'
+                         'An attribute reference is a primary followed by a '
+                         'period and a name:\n'
+                         '\n'
+                         '   attributeref ::= primary "." identifier\n'
+                         '\n'
+                         'The primary must evaluate to an object of a type '
+                         'that supports\n'
+                         'attribute references, which most objects do.  This '
+                         'object is then\n'
+                         'asked to produce the attribute whose name is the '
+                         'identifier.  This\n'
+                         'production can be customized by overriding the '
+                         '"__getattr__()" method.\n'
+                         'If this attribute is not available, the exception '
+                         '"AttributeError" is\n'
+                         'raised.  Otherwise, the type and value of the object '
+                         'produced is\n'
+                         'determined by the object.  Multiple evaluations of '
+                         'the same attribute\n'
+                         'reference may yield different objects.\n',
+ 'augassign': '\n'
+              'Augmented assignment statements\n'
+              '*******************************\n'
+              '\n'
+              'Augmented assignment is the combination, in a single statement, '
+              'of a\n'
+              'binary operation and an assignment statement:\n'
+              '\n'
+              '   augmented_assignment_stmt ::= augtarget augop '
+              '(expression_list | yield_expression)\n'
+              '   augtarget                 ::= identifier | attributeref | '
+              'subscription | slicing\n'
+              '   augop                     ::= "+=" | "-=" | "*=" | "@=" | '
+              '"/=" | "//=" | "%=" | "**="\n'
+              '             | ">>=" | "<<=" | "&=" | "^=" | "|="\n'
+              '\n'
+              '(See section *Primaries* for the syntax definitions of the last '
+              'three\n'
+              'symbols.)\n'
+              '\n'
+              'An augmented assignment evaluates the target (which, unlike '
+              'normal\n'
+              'assignment statements, cannot be an unpacking) and the '
+              'expression\n'
+              'list, performs the binary operation specific to the type of '
+              'assignment\n'
+              'on the two operands, and assigns the result to the original '
+              'target.\n'
+              'The target is only evaluated once.\n'
+              '\n'
+              'An augmented assignment expression like "x += 1" can be '
+              'rewritten as\n'
+              '"x = x + 1" to achieve a similar, but not exactly equal effect. '
+              'In the\n'
+              'augmented version, "x" is only evaluated once. Also, when '
+              'possible,\n'
+              'the actual operation is performed *in-place*, meaning that '
+              'rather than\n'
+              'creating a new object and assigning that to the target, the old '
+              'object\n'
+              'is modified instead.\n'
+              '\n'
+              'Unlike normal assignments, augmented assignments evaluate the '
+              'left-\n'
+              'hand side *before* evaluating the right-hand side.  For '
+              'example, "a[i]\n'
+              '+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and '
+              'performs\n'
+              'the addition, and lastly, it writes the result back to "a[i]".\n'
+              '\n'
+              'With the exception of assigning to tuples and multiple targets '
+              'in a\n'
+              'single statement, the assignment done by augmented assignment\n'
+              'statements is handled the same way as normal assignments. '
+              'Similarly,\n'
+              'with the exception of the possible *in-place* behavior, the '
+              'binary\n'
+              'operation performed by augmented assignment is the same as the '
+              'normal\n'
+              'binary operations.\n'
+              '\n'
+              'For targets which are attribute references, the same *caveat '
+              'about\n'
+              'class and instance attributes* applies as for regular '
+              'assignments.\n',
+ 'binary': '\n'
+           'Binary arithmetic operations\n'
+           '****************************\n'
+           '\n'
+           'The binary arithmetic operations have the conventional priority\n'
+           'levels.  Note that some of these operations also apply to certain '
+           'non-\n'
+           'numeric types.  Apart from the power operator, there are only two\n'
+           'levels, one for multiplicative operators and one for additive\n'
+           'operators:\n'
+           '\n'
+           '   m_expr ::= u_expr | m_expr "*" u_expr | m_expr "@" m_expr |\n'
+           '              m_expr "//" u_expr| m_expr "/" u_expr |\n'
+           '              m_expr "%" u_expr\n'
+           '   a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n'
+           '\n'
+           'The "*" (multiplication) operator yields the product of its '
+           'arguments.\n'
+           'The arguments must either both be numbers, or one argument must be '
+           'an\n'
+           'integer and the other must be a sequence. In the former case, the\n'
+           'numbers are converted to a common type and then multiplied '
+           'together.\n'
+           'In the latter case, sequence repetition is performed; a negative\n'
+           'repetition factor yields an empty sequence.\n'
+           '\n'
+           'The "@" (at) operator is intended to be used for matrix\n'
+           'multiplication.  No builtin Python types implement this operator.\n'
+           '\n'
+           'New in version 3.5.\n'
+           '\n'
+           'The "/" (division) and "//" (floor division) operators yield the\n'
+           'quotient of their arguments.  The numeric arguments are first\n'
+           'converted to a common type. Division of integers yields a float, '
+           'while\n'
+           'floor division of integers results in an integer; the result is '
+           'that\n'
+           "of mathematical division with the 'floor' function applied to the\n"
+           'result.  Division by zero raises the "ZeroDivisionError" '
+           'exception.\n'
+           '\n'
+           'The "%" (modulo) operator yields the remainder from the division '
+           'of\n'
+           'the first argument by the second.  The numeric arguments are '
+           'first\n'
+           'converted to a common type.  A zero right argument raises the\n'
+           '"ZeroDivisionError" exception.  The arguments may be floating '
+           'point\n'
+           'numbers, e.g., "3.14%0.7" equals "0.34" (since "3.14" equals '
+           '"4*0.7 +\n'
+           '0.34".)  The modulo operator always yields a result with the same '
+           'sign\n'
+           'as its second operand (or zero); the absolute value of the result '
+           'is\n'
+           'strictly smaller than the absolute value of the second operand '
+           '[1].\n'
+           '\n'
+           'The floor division and modulo operators are connected by the '
+           'following\n'
+           'identity: "x == (x//y)*y + (x%y)".  Floor division and modulo are '
+           'also\n'
+           'connected with the built-in function "divmod()": "divmod(x, y) ==\n'
+           '(x//y, x%y)". [2].\n'
+           '\n'
+           'In addition to performing the modulo operation on numbers, the '
+           '"%"\n'
+           'operator is also overloaded by string objects to perform '
+           'old-style\n'
+           'string formatting (also known as interpolation).  The syntax for\n'
+           'string formatting is described in the Python Library Reference,\n'
+           'section *printf-style String Formatting*.\n'
+           '\n'
+           'The floor division operator, the modulo operator, and the '
+           '"divmod()"\n'
+           'function are not defined for complex numbers.  Instead, convert to '
+           'a\n'
+           'floating point number using the "abs()" function if appropriate.\n'
+           '\n'
+           'The "+" (addition) operator yields the sum of its arguments.  The\n'
+           'arguments must either both be numbers or both be sequences of the '
+           'same\n'
+           'type.  In the former case, the numbers are converted to a common '
+           'type\n'
+           'and then added together. In the latter case, the sequences are\n'
+           'concatenated.\n'
+           '\n'
+           'The "-" (subtraction) operator yields the difference of its '
+           'arguments.\n'
+           'The numeric arguments are first converted to a common type.\n',
+ 'bitwise': '\n'
+            'Binary bitwise operations\n'
+            '*************************\n'
+            '\n'
+            'Each of the three bitwise operations has a different priority '
+            'level:\n'
+            '\n'
+            '   and_expr ::= shift_expr | and_expr "&" shift_expr\n'
+            '   xor_expr ::= and_expr | xor_expr "^" and_expr\n'
+            '   or_expr  ::= xor_expr | or_expr "|" xor_expr\n'
+            '\n'
+            'The "&" operator yields the bitwise AND of its arguments, which '
+            'must\n'
+            'be integers.\n'
+            '\n'
+            'The "^" operator yields the bitwise XOR (exclusive OR) of its\n'
+            'arguments, which must be integers.\n'
+            '\n'
+            'The "|" operator yields the bitwise (inclusive) OR of its '
+            'arguments,\n'
+            'which must be integers.\n',
+ 'bltin-code-objects': '\n'
+                       'Code Objects\n'
+                       '************\n'
+                       '\n'
+                       'Code objects are used by the implementation to '
+                       'represent "pseudo-\n'
+                       'compiled" executable Python code such as a function '
+                       'body. They differ\n'
+                       "from function objects because they don't contain a "
+                       'reference to their\n'
+                       'global execution environment.  Code objects are '
+                       'returned by the built-\n'
+                       'in "compile()" function and can be extracted from '
+                       'function objects\n'
+                       'through their "__code__" attribute. See also the '
+                       '"code" module.\n'
+                       '\n'
+                       'A code object can be executed or evaluated by passing '
+                       'it (instead of a\n'
+                       'source string) to the "exec()" or "eval()"  built-in '
+                       'functions.\n'
+                       '\n'
+                       'See *The standard type hierarchy* for more '
+                       'information.\n',
+ 'bltin-ellipsis-object': '\n'
+                          'The Ellipsis Object\n'
+                          '*******************\n'
+                          '\n'
+                          'This object is commonly used by slicing (see '
+                          '*Slicings*).  It supports\n'
+                          'no special operations.  There is exactly one '
+                          'ellipsis object, named\n'
+                          '"Ellipsis" (a built-in name).  "type(Ellipsis)()" '
+                          'produces the\n'
+                          '"Ellipsis" singleton.\n'
+                          '\n'
+                          'It is written as "Ellipsis" or "...".\n',
+ 'bltin-null-object': '\n'
+                      'The Null Object\n'
+                      '***************\n'
+                      '\n'
+                      "This object is returned by functions that don't "
+                      'explicitly return a\n'
+                      'value.  It supports no special operations.  There is '
+                      'exactly one null\n'
+                      'object, named "None" (a built-in name).  "type(None)()" '
+                      'produces the\n'
+                      'same singleton.\n'
+                      '\n'
+                      'It is written as "None".\n',
+ 'bltin-type-objects': '\n'
+                       'Type Objects\n'
+                       '************\n'
+                       '\n'
+                       'Type objects represent the various object types.  An '
+                       "object's type is\n"
+                       'accessed by the built-in function "type()".  There are '
+                       'no special\n'
+                       'operations on types.  The standard module "types" '
+                       'defines names for\n'
+                       'all standard built-in types.\n'
+                       '\n'
+                       'Types are written like this: "<class \'int\'>".\n',
+ 'booleans': '\n'
+             'Boolean operations\n'
+             '******************\n'
+             '\n'
+             '   or_test  ::= and_test | or_test "or" and_test\n'
+             '   and_test ::= not_test | and_test "and" not_test\n'
+             '   not_test ::= comparison | "not" not_test\n'
+             '\n'
+             'In the context of Boolean operations, and also when expressions '
+             'are\n'
+             'used by control flow statements, the following values are '
+             'interpreted\n'
+             'as false: "False", "None", numeric zero of all types, and empty\n'
+             'strings and containers (including strings, tuples, lists,\n'
+             'dictionaries, sets and frozensets).  All other values are '
+             'interpreted\n'
+             'as true.  User-defined objects can customize their truth value '
+             'by\n'
+             'providing a "__bool__()" method.\n'
+             '\n'
+             'The operator "not" yields "True" if its argument is false, '
+             '"False"\n'
+             'otherwise.\n'
+             '\n'
+             'The expression "x and y" first evaluates *x*; if *x* is false, '
+             'its\n'
+             'value is returned; otherwise, *y* is evaluated and the resulting '
+             'value\n'
+             'is returned.\n'
+             '\n'
+             'The expression "x or y" first evaluates *x*; if *x* is true, its '
+             'value\n'
+             'is returned; otherwise, *y* is evaluated and the resulting value '
+             'is\n'
+             'returned.\n'
+             '\n'
+             '(Note that neither "and" nor "or" restrict the value and type '
+             'they\n'
+             'return to "False" and "True", but rather return the last '
+             'evaluated\n'
+             'argument.  This is sometimes useful, e.g., if "s" is a string '
+             'that\n'
+             'should be replaced by a default value if it is empty, the '
+             'expression\n'
+             '"s or \'foo\'" yields the desired value.  Because "not" has to '
+             'create a\n'
+             'new value, it returns a boolean value regardless of the type of '
+             'its\n'
+             'argument (for example, "not \'foo\'" produces "False" rather '
+             'than "\'\'".)\n',
+ 'break': '\n'
+          'The "break" statement\n'
+          '*********************\n'
+          '\n'
+          '   break_stmt ::= "break"\n'
+          '\n'
+          '"break" may only occur syntactically nested in a "for" or "while"\n'
+          'loop, but not nested in a function or class definition within that\n'
+          'loop.\n'
+          '\n'
+          'It terminates the nearest enclosing loop, skipping the optional '
+          '"else"\n'
+          'clause if the loop has one.\n'
+          '\n'
+          'If a "for" loop is terminated by "break", the loop control target\n'
+          'keeps its current value.\n'
+          '\n'
+          'When "break" passes control out of a "try" statement with a '
+          '"finally"\n'
+          'clause, that "finally" clause is executed before really leaving '
+          'the\n'
+          'loop.\n',
+ 'callable-types': '\n'
+                   'Emulating callable objects\n'
+                   '**************************\n'
+                   '\n'
+                   'object.__call__(self[, args...])\n'
+                   '\n'
+                   '   Called when the instance is "called" as a function; if '
+                   'this method\n'
+                   '   is defined, "x(arg1, arg2, ...)" is a shorthand for\n'
+                   '   "x.__call__(arg1, arg2, ...)".\n',
+ 'calls': '\n'
+          'Calls\n'
+          '*****\n'
+          '\n'
+          'A call calls a callable object (e.g., a *function*) with a '
+          'possibly\n'
+          'empty series of *arguments*:\n'
+          '\n'
+          '   call                 ::= primary "(" [argument_list [","] | '
+          'comprehension] ")"\n'
+          '   argument_list        ::= positional_arguments ["," '
+          'starred_and_keywords]\n'
+          '                       ["," keywords_arguments]\n'
+          '                     | starred_and_keywords ["," '
+          'keywords_arguments]\n'
+          '                     | keywords_arguments\n'
+          '   positional_arguments ::= ["*"] expression ("," ["*"] '
+          'expression)*\n'
+          '   starred_and_keywords ::= ("*" expression | keyword_item)\n'
+          '                            ("," "*" expression | "," '
+          'keyword_item)*\n'
+          '   keywords_arguments   ::= (keyword_item | "**" expression)\n'
+          '                          ("," keyword_item | "," "**" '
+          'expression)*\n'
+          '   keyword_item         ::= identifier "=" expression\n'
+          '\n'
+          'An optional trailing comma may be present after the positional and\n'
+          'keyword arguments but does not affect the semantics.\n'
+          '\n'
+          'The primary must evaluate to a callable object (user-defined\n'
+          'functions, built-in functions, methods of built-in objects, class\n'
+          'objects, methods of class instances, and all objects having a\n'
+          '"__call__()" method are callable).  All argument expressions are\n'
+          'evaluated before the call is attempted.  Please refer to section\n'
+          '*Function definitions* for the syntax of formal *parameter* lists.\n'
+          '\n'
+          'If keyword arguments are present, they are first converted to\n'
+          'positional arguments, as follows.  First, a list of unfilled slots '
+          'is\n'
+          'created for the formal parameters.  If there are N positional\n'
+          'arguments, they are placed in the first N slots.  Next, for each\n'
+          'keyword argument, the identifier is used to determine the\n'
+          'corresponding slot (if the identifier is the same as the first '
+          'formal\n'
+          'parameter name, the first slot is used, and so on).  If the slot '
+          'is\n'
+          'already filled, a "TypeError" exception is raised. Otherwise, the\n'
+          'value of the argument is placed in the slot, filling it (even if '
+          'the\n'
+          'expression is "None", it fills the slot).  When all arguments have\n'
+          'been processed, the slots that are still unfilled are filled with '
+          'the\n'
+          'corresponding default value from the function definition.  '
+          '(Default\n'
+          'values are calculated, once, when the function is defined; thus, a\n'
+          'mutable object such as a list or dictionary used as default value '
+          'will\n'
+          "be shared by all calls that don't specify an argument value for "
+          'the\n'
+          'corresponding slot; this should usually be avoided.)  If there are '
+          'any\n'
+          'unfilled slots for which no default value is specified, a '
+          '"TypeError"\n'
+          'exception is raised.  Otherwise, the list of filled slots is used '
+          'as\n'
+          'the argument list for the call.\n'
+          '\n'
+          '**CPython implementation detail:** An implementation may provide\n'
+          'built-in functions whose positional parameters do not have names, '
+          'even\n'
+          "if they are 'named' for the purpose of documentation, and which\n"
+          'therefore cannot be supplied by keyword.  In CPython, this is the '
+          'case\n'
+          'for functions implemented in C that use "PyArg_ParseTuple()" to '
+          'parse\n'
+          'their arguments.\n'
+          '\n'
+          'If there are more positional arguments than there are formal '
+          'parameter\n'
+          'slots, a "TypeError" exception is raised, unless a formal '
+          'parameter\n'
+          'using the syntax "*identifier" is present; in this case, that '
+          'formal\n'
+          'parameter receives a tuple containing the excess positional '
+          'arguments\n'
+          '(or an empty tuple if there were no excess positional arguments).\n'
+          '\n'
+          'If any keyword argument does not correspond to a formal parameter\n'
+          'name, a "TypeError" exception is raised, unless a formal parameter\n'
+          'using the syntax "**identifier" is present; in this case, that '
+          'formal\n'
+          'parameter receives a dictionary containing the excess keyword\n'
+          'arguments (using the keywords as keys and the argument values as\n'
+          'corresponding values), or a (new) empty dictionary if there were '
+          'no\n'
+          'excess keyword arguments.\n'
+          '\n'
+          'If the syntax "*expression" appears in the function call, '
+          '"expression"\n'
+          'must evaluate to an *iterable*.  Elements from these iterables are\n'
+          'treated as if they were additional positional arguments.  For the '
+          'call\n'
+          '"f(x1, x2, *y, x3, x4)", if *y* evaluates to a sequence *y1*, ...,\n'
+          '*yM*, this is equivalent to a call with M+4 positional arguments '
+          '*x1*,\n'
+          '*x2*, *y1*, ..., *yM*, *x3*, *x4*.\n'
+          '\n'
+          'A consequence of this is that although the "*expression" syntax '
+          'may\n'
+          'appear *after* explicit keyword arguments, it is processed '
+          '*before*\n'
+          'the keyword arguments (and any "**expression" arguments -- see '
+          'below).\n'
+          'So:\n'
+          '\n'
+          '   >>> def f(a, b):\n'
+          '   ...     print(a, b)\n'
+          '   ...\n'
+          '   >>> f(b=1, *(2,))\n'
+          '   2 1\n'
+          '   >>> f(a=1, *(2,))\n'
+          '   Traceback (most recent call last):\n'
+          '     File "<stdin>", line 1, in <module>\n'
+          "   TypeError: f() got multiple values for keyword argument 'a'\n"
+          '   >>> f(1, *(2,))\n'
+          '   1 2\n'
+          '\n'
+          'It is unusual for both keyword arguments and the "*expression" '
+          'syntax\n'
+          'to be used in the same call, so in practice this confusion does '
+          'not\n'
+          'arise.\n'
+          '\n'
+          'If the syntax "**expression" appears in the function call,\n'
+          '"expression" must evaluate to a *mapping*, the contents of which '
+          'are\n'
+          'treated as additional keyword arguments.  If a keyword is already\n'
+          'present (as an explicit keyword argument, or from another '
+          'unpacking),\n'
+          'a "TypeError" exception is raised.\n'
+          '\n'
+          'Formal parameters using the syntax "*identifier" or "**identifier"\n'
+          'cannot be used as positional argument slots or as keyword argument\n'
+          'names.\n'
+          '\n'
+          'Changed in version 3.5: Function calls accept any number of "*" '
+          'and\n'
+          '"**" unpackings, positional arguments may follow iterable '
+          'unpackings\n'
+          '("*"), and keyword arguments may follow dictionary unpackings '
+          '("**").\n'
+          'Originally proposed by **PEP 448**.\n'
+          '\n'
+          'A call always returns some value, possibly "None", unless it raises '
+          'an\n'
+          'exception.  How this value is computed depends on the type of the\n'
+          'callable object.\n'
+          '\n'
+          'If it is---\n'
+          '\n'
+          'a user-defined function:\n'
+          '   The code block for the function is executed, passing it the\n'
+          '   argument list.  The first thing the code block will do is bind '
+          'the\n'
+          '   formal parameters to the arguments; this is described in '
+          'section\n'
+          '   *Function definitions*.  When the code block executes a '
+          '"return"\n'
+          '   statement, this specifies the return value of the function '
+          'call.\n'
+          '\n'
+          'a built-in function or method:\n'
+          '   The result is up to the interpreter; see *Built-in Functions* '
+          'for\n'
+          '   the descriptions of built-in functions and methods.\n'
+          '\n'
+          'a class object:\n'
+          '   A new instance of that class is returned.\n'
+          '\n'
+          'a class instance method:\n'
+          '   The corresponding user-defined function is called, with an '
+          'argument\n'
+          '   list that is one longer than the argument list of the call: the\n'
+          '   instance becomes the first argument.\n'
+          '\n'
+          'a class instance:\n'
+          '   The class must define a "__call__()" method; the effect is then '
+          'the\n'
+          '   same as if that method was called.\n',
+ 'class': '\n'
+          'Class definitions\n'
+          '*****************\n'
+          '\n'
+          'A class definition defines a class object (see section *The '
+          'standard\n'
+          'type hierarchy*):\n'
+          '\n'
+          '   classdef    ::= [decorators] "class" classname [inheritance] ":" '
+          'suite\n'
+          '   inheritance ::= "(" [argument_list] ")"\n'
+          '   classname   ::= identifier\n'
+          '\n'
+          'A class definition is an executable statement.  The inheritance '
+          'list\n'
+          'usually gives a list of base classes (see *Customizing class '
+          'creation*\n'
+          'for more advanced uses), so each item in the list should evaluate '
+          'to a\n'
+          'class object which allows subclassing.  Classes without an '
+          'inheritance\n'
+          'list inherit, by default, from the base class "object"; hence,\n'
+          '\n'
+          '   class Foo:\n'
+          '       pass\n'
+          '\n'
+          'is equivalent to\n'
+          '\n'
+          '   class Foo(object):\n'
+          '       pass\n'
+          '\n'
+          "The class's suite is then executed in a new execution frame (see\n"
+          '*Naming and binding*), using a newly created local namespace and '
+          'the\n'
+          'original global namespace. (Usually, the suite contains mostly\n'
+          "function definitions.)  When the class's suite finishes execution, "
+          'its\n'
+          'execution frame is discarded but its local namespace is saved. [4] '
+          'A\n'
+          'class object is then created using the inheritance list for the '
+          'base\n'
+          'classes and the saved local namespace for the attribute '
+          'dictionary.\n'
+          'The class name is bound to this class object in the original local\n'
+          'namespace.\n'
+          '\n'
+          'Class creation can be customized heavily using *metaclasses*.\n'
+          '\n'
+          'Classes can also be decorated: just like when decorating '
+          'functions,\n'
+          '\n'
+          '   @f1(arg)\n'
+          '   @f2\n'
+          '   class Foo: pass\n'
+          '\n'
+          'is roughly equivalent to\n'
+          '\n'
+          '   class Foo: pass\n'
+          '   Foo = f1(arg)(f2(Foo))\n'
+          '\n'
+          'The evaluation rules for the decorator expressions are the same as '
+          'for\n'
+          'function decorators.  The result is then bound to the class name.\n'
+          '\n'
+          "**Programmer's note:** Variables defined in the class definition "
+          'are\n'
+          'class attributes; they are shared by instances.  Instance '
+          'attributes\n'
+          'can be set in a method with "self.name = value".  Both class and\n'
+          'instance attributes are accessible through the notation '
+          '""self.name"",\n'
+          'and an instance attribute hides a class attribute with the same '
+          'name\n'
+          'when accessed in this way.  Class attributes can be used as '
+          'defaults\n'
+          'for instance attributes, but using mutable values there can lead '
+          'to\n'
+          'unexpected results.  *Descriptors* can be used to create instance\n'
+          'variables with different implementation details.\n'
+          '\n'
+          'See also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n'
+          '  Class Decorators\n',
+ 'comparisons': '\n'
+                'Comparisons\n'
+                '***********\n'
+                '\n'
+                'Unlike C, all comparison operations in Python have the same '
+                'priority,\n'
+                'which is lower than that of any arithmetic, shifting or '
+                'bitwise\n'
+                'operation.  Also unlike C, expressions like "a < b < c" have '
+                'the\n'
+                'interpretation that is conventional in mathematics:\n'
+                '\n'
+                '   comparison    ::= or_expr ( comp_operator or_expr )*\n'
+                '   comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n'
+                '                     | "is" ["not"] | ["not"] "in"\n'
+                '\n'
+                'Comparisons yield boolean values: "True" or "False".\n'
+                '\n'
+                'Comparisons can be chained arbitrarily, e.g., "x < y <= z" '
+                'is\n'
+                'equivalent to "x < y and y <= z", except that "y" is '
+                'evaluated only\n'
+                'once (but in both cases "z" is not evaluated at all when "x < '
+                'y" is\n'
+                'found to be false).\n'
+                '\n'
+                'Formally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and '
+                '*op1*,\n'
+                '*op2*, ..., *opN* are comparison operators, then "a op1 b op2 '
+                'c ... y\n'
+                'opN z" is equivalent to "a op1 b and b op2 c and ... y opN '
+                'z", except\n'
+                'that each expression is evaluated at most once.\n'
+                '\n'
+                'Note that "a op1 b op2 c" doesn\'t imply any kind of '
+                'comparison between\n'
+                '*a* and *c*, so that, e.g., "x < y > z" is perfectly legal '
+                '(though\n'
+                'perhaps not pretty).\n'
+                '\n'
+                '\n'
+                'Value comparisons\n'
+                '=================\n'
+                '\n'
+                'The operators "<", ">", "==", ">=", "<=", and "!=" compare '
+                'the values\n'
+                'of two objects.  The objects do not need to have the same '
+                'type.\n'
+                '\n'
+                'Chapter *Objects, values and types* states that objects have '
+                'a value\n'
+                '(in addition to type and identity).  The value of an object '
+                'is a\n'
+                'rather abstract notion in Python: For example, there is no '
+                'canonical\n'
+                "access method for an object's value.  Also, there is no "
+                'requirement\n'
+                'that the value of an object should be constructed in a '
+                'particular way,\n'
+                'e.g. comprised of all its data attributes. Comparison '
+                'operators\n'
+                'implement a particular notion of what the value of an object '
+                'is.  One\n'
+                'can think of them as defining the value of an object '
+                'indirectly, by\n'
+                'means of their comparison implementation.\n'
+                '\n'
+                'Because all types are (direct or indirect) subtypes of '
+                '"object", they\n'
+                'inherit the default comparison behavior from "object".  Types '
+                'can\n'
+                'customize their comparison behavior by implementing *rich '
+                'comparison\n'
+                'methods* like "__lt__()", described in *Basic '
+                'customization*.\n'
+                '\n'
+                'The default behavior for equality comparison ("==" and "!=") '
+                'is based\n'
+                'on the identity of the objects.  Hence, equality comparison '
+                'of\n'
+                'instances with the same identity results in equality, and '
+                'equality\n'
+                'comparison of instances with different identities results in\n'
+                'inequality.  A motivation for this default behavior is the '
+                'desire that\n'
+                'all objects should be reflexive (i.e. "x is y" implies "x == '
+                'y").\n'
+                '\n'
+                'A default order comparison ("<", ">", "<=", and ">=") is not '
+                'provided;\n'
+                'an attempt raises "TypeError".  A motivation for this default '
+                'behavior\n'
+                'is the lack of a similar invariant as for equality.\n'
+                '\n'
+                'The behavior of the default equality comparison, that '
+                'instances with\n'
+                'different identities are always unequal, may be in contrast '
+                'to what\n'
+                'types will need that have a sensible definition of object '
+                'value and\n'
+                'value-based equality.  Such types will need to customize '
+                'their\n'
+                'comparison behavior, and in fact, a number of built-in types '
+                'have done\n'
+                'that.\n'
+                '\n'
+                'The following list describes the comparison behavior of the '
+                'most\n'
+                'important built-in types.\n'
+                '\n'
+                '* Numbers of built-in numeric types (*Numeric Types --- int, '
+                'float,\n'
+                '  complex*) and of the standard library types '
+                '"fractions.Fraction" and\n'
+                '  "decimal.Decimal" can be compared within and across their '
+                'types,\n'
+                '  with the restriction that complex numbers do not support '
+                'order\n'
+                '  comparison.  Within the limits of the types involved, they '
+                'compare\n'
+                '  mathematically (algorithmically) correct without loss of '
+                'precision.\n'
+                '\n'
+                '  The not-a-number values "float(\'NaN\')" and '
+                '"Decimal(\'NaN\')" are\n'
+                '  special.  They are identical to themselves ("x is x" is '
+                'true) but\n'
+                '  are not equal to themselves ("x == x" is false).  '
+                'Additionally,\n'
+                '  comparing any number to a not-a-number value will return '
+                '"False".\n'
+                '  For example, both "3 < float(\'NaN\')" and "float(\'NaN\') '
+                '< 3" will\n'
+                '  return "False".\n'
+                '\n'
+                '* Binary sequences (instances of "bytes" or "bytearray") can '
+                'be\n'
+                '  compared within and across their types.  They compare\n'
+                '  lexicographically using the numeric values of their '
+                'elements.\n'
+                '\n'
+                '* Strings (instances of "str") compare lexicographically '
+                'using the\n'
+                '  numerical Unicode code points (the result of the built-in '
+                'function\n'
+                '  "ord()") of their characters. [3]\n'
+                '\n'
+                '  Strings and binary sequences cannot be directly compared.\n'
+                '\n'
+                '* Sequences (instances of "tuple", "list", or "range") can '
+                'be\n'
+                '  compared only within each of their types, with the '
+                'restriction that\n'
+                '  ranges do not support order comparison.  Equality '
+                'comparison across\n'
+                '  these types results in unequality, and ordering comparison '
+                'across\n'
+                '  these types raises "TypeError".\n'
+                '\n'
+                '  Sequences compare lexicographically using comparison of\n'
+                '  corresponding elements, whereby reflexivity of the elements '
+                'is\n'
+                '  enforced.\n'
+                '\n'
+                '  In enforcing reflexivity of elements, the comparison of '
+                'collections\n'
+                '  assumes that for a collection element "x", "x == x" is '
+                'always true.\n'
+                '  Based on that assumption, element identity is compared '
+                'first, and\n'
+                '  element comparison is performed only for distinct '
+                'elements.  This\n'
+                '  approach yields the same result as a strict element '
+                'comparison\n'
+                '  would, if the compared elements are reflexive.  For '
+                'non-reflexive\n'
+                '  elements, the result is different than for strict element\n'
+                '  comparison, and may be surprising:  The non-reflexive '
+                'not-a-number\n'
+                '  values for example result in the following comparison '
+                'behavior when\n'
+                '  used in a list:\n'
+                '\n'
+                "     >>> nan = float('NaN')\n"
+                '     >>> nan is nan\n'
+                '     True\n'
+                '     >>> nan == nan\n'
+                '     False                 <-- the defined non-reflexive '
+                'behavior of NaN\n'
+                '     >>> [nan] == [nan]\n'
+                '     True                  <-- list enforces reflexivity and '
+                'tests identity first\n'
+                '\n'
+                '  Lexicographical comparison between built-in collections '
+                'works as\n'
+                '  follows:\n'
+                '\n'
+                '  * For two collections to compare equal, they must be of the '
+                'same\n'
+                '    type, have the same length, and each pair of '
+                'corresponding\n'
+                '    elements must compare equal (for example, "[1,2] == '
+                '(1,2)" is\n'
+                '    false because the type is not the same).\n'
+                '\n'
+                '  * Collections that support order comparison are ordered the '
+                'same\n'
+                '    as their first unequal elements (for example, "[1,2,x] <= '
+                '[1,2,y]"\n'
+                '    has the same value as "x <= y").  If a corresponding '
+                'element does\n'
+                '    not exist, the shorter collection is ordered first (for '
+                'example,\n'
+                '    "[1,2] < [1,2,3]" is true).\n'
+                '\n'
+                '* Mappings (instances of "dict") compare equal if and only if '
+                'they\n'
+                '  have equal *(key, value)* pairs. Equality comparison of the '
+                'keys and\n'
+                '  values enforces reflexivity.\n'
+                '\n'
+                '  Order comparisons ("<", ">", "<=", and ">=") raise '
+                '"TypeError".\n'
+                '\n'
+                '* Sets (instances of "set" or "frozenset") can be compared '
+                'within\n'
+                '  and across their types.\n'
+                '\n'
+                '  They define order comparison operators to mean subset and '
+                'superset\n'
+                '  tests.  Those relations do not define total orderings (for '
+                'example,\n'
+                '  the two sets "{1,2}" and "{2,3}" are not equal, nor subsets '
+                'of one\n'
+                '  another, nor supersets of one another).  Accordingly, sets '
+                'are not\n'
+                '  appropriate arguments for functions which depend on total '
+                'ordering\n'
+                '  (for example, "min()", "max()", and "sorted()" produce '
+                'undefined\n'
+                '  results given a list of sets as inputs).\n'
+                '\n'
+                '  Comparison of sets enforces reflexivity of its elements.\n'
+                '\n'
+                '* Most other built-in types have no comparison methods '
+                'implemented,\n'
+                '  so they inherit the default comparison behavior.\n'
+                '\n'
+                'User-defined classes that customize their comparison behavior '
+                'should\n'
+                'follow some consistency rules, if possible:\n'
+                '\n'
+                '* Equality comparison should be reflexive. In other words, '
+                'identical\n'
+                '  objects should compare equal:\n'
+                '\n'
+                '     "x is y" implies "x == y"\n'
+                '\n'
+                '* Comparison should be symmetric. In other words, the '
+                'following\n'
+                '  expressions should have the same result:\n'
+                '\n'
+                '     "x == y" and "y == x"\n'
+                '\n'
+                '     "x != y" and "y != x"\n'
+                '\n'
+                '     "x < y" and "y > x"\n'
+                '\n'
+                '     "x <= y" and "y >= x"\n'
+                '\n'
+                '* Comparison should be transitive. The following '
+                '(non-exhaustive)\n'
+                '  examples illustrate that:\n'
+                '\n'
+                '     "x > y and y > z" implies "x > z"\n'
+                '\n'
+                '     "x < y and y <= z" implies "x < z"\n'
+                '\n'
+                '* Inverse comparison should result in the boolean negation. '
+                'In other\n'
+                '  words, the following expressions should have the same '
+                'result:\n'
+                '\n'
+                '     "x == y" and "not x != y"\n'
+                '\n'
+                '     "x < y" and "not x >= y" (for total ordering)\n'
+                '\n'
+                '     "x > y" and "not x <= y" (for total ordering)\n'
+                '\n'
+                '  The last two expressions apply to totally ordered '
+                'collections (e.g.\n'
+                '  to sequences, but not to sets or mappings). See also the\n'
+                '  "total_ordering()" decorator.\n'
+                '\n'
+                '* The "hash()" result should be consistent with equality. '
+                'Objects\n'
+                '  that are equal should either have the same hash value, or '
+                'be marked\n'
+                '  as unhashable.\n'
+                '\n'
+                'Python does not enforce these consistency rules. In fact, '
+                'the\n'
+                'not-a-number values are an example for not following these '
+                'rules.\n'
+                '\n'
+                '\n'
+                'Membership test operations\n'
+                '==========================\n'
+                '\n'
+                'The operators "in" and "not in" test for membership.  "x in '
+                's"\n'
+                'evaluates to "True" if *x* is a member of *s*, and "False" '
+                'otherwise.\n'
+                '"x not in s" returns the negation of "x in s".  All built-in '
+                'sequences\n'
+                'and set types support this as well as dictionary, for which '
+                '"in" tests\n'
+                'whether the dictionary has a given key. For container types '
+                'such as\n'
+                'list, tuple, set, frozenset, dict, or collections.deque, the\n'
+                'expression "x in y" is equivalent to "any(x is e or x == e '
+                'for e in\n'
+                'y)".\n'
+                '\n'
+                'For the string and bytes types, "x in y" is "True" if and '
+                'only if *x*\n'
+                'is a substring of *y*.  An equivalent test is "y.find(x) != '
+                '-1".\n'
+                'Empty strings are always considered to be a substring of any '
+                'other\n'
+                'string, so """ in "abc"" will return "True".\n'
+                '\n'
+                'For user-defined classes which define the "__contains__()" '
+                'method, "x\n'
+                'in y" returns "True" if "y.__contains__(x)" returns a true '
+                'value, and\n'
+                '"False" otherwise.\n'
+                '\n'
+                'For user-defined classes which do not define "__contains__()" '
+                'but do\n'
+                'define "__iter__()", "x in y" is "True" if some value "z" '
+                'with "x ==\n'
+                'z" is produced while iterating over "y".  If an exception is '
+                'raised\n'
+                'during the iteration, it is as if "in" raised that '
+                'exception.\n'
+                '\n'
+                'Lastly, the old-style iteration protocol is tried: if a class '
+                'defines\n'
+                '"__getitem__()", "x in y" is "True" if and only if there is a '
+                'non-\n'
+                'negative integer index *i* such that "x == y[i]", and all '
+                'lower\n'
+                'integer indices do not raise "IndexError" exception.  (If any '
+                'other\n'
+                'exception is raised, it is as if "in" raised that '
+                'exception).\n'
+                '\n'
+                'The operator "not in" is defined to have the inverse true '
+                'value of\n'
+                '"in".\n'
+                '\n'
+                '\n'
+                'Identity comparisons\n'
+                '====================\n'
+                '\n'
+                'The operators "is" and "is not" test for object identity: "x '
+                'is y" is\n'
+                'true if and only if *x* and *y* are the same object.  Object '
+                'identity\n'
+                'is determined using the "id()" function.  "x is not y" yields '
+                'the\n'
+                'inverse truth value. [4]\n',
+ 'compound': '\n'
+             'Compound statements\n'
+             '*******************\n'
+             '\n'
+             'Compound statements contain (groups of) other statements; they '
+             'affect\n'
+             'or control the execution of those other statements in some way.  '
+             'In\n'
+             'general, compound statements span multiple lines, although in '
+             'simple\n'
+             'incarnations a whole compound statement may be contained in one '
+             'line.\n'
+             '\n'
+             'The "if", "while" and "for" statements implement traditional '
+             'control\n'
+             'flow constructs.  "try" specifies exception handlers and/or '
+             'cleanup\n'
+             'code for a group of statements, while the "with" statement '
+             'allows the\n'
+             'execution of initialization and finalization code around a block '
+             'of\n'
+             'code.  Function and class definitions are also syntactically '
+             'compound\n'
+             'statements.\n'
+             '\n'
+             "A compound statement consists of one or more 'clauses.'  A "
+             'clause\n'
+             "consists of a header and a 'suite.'  The clause headers of a\n"
+             'particular compound statement are all at the same indentation '
+             'level.\n'
+             'Each clause header begins with a uniquely identifying keyword '
+             'and ends\n'
+             'with a colon.  A suite is a group of statements controlled by a\n'
+             'clause.  A suite can be one or more semicolon-separated simple\n'
+             'statements on the same line as the header, following the '
+             "header's\n"
+             'colon, or it can be one or more indented statements on '
+             'subsequent\n'
+             'lines.  Only the latter form of a suite can contain nested '
+             'compound\n'
+             "statements; the following is illegal, mostly because it wouldn't "
+             'be\n'
+             'clear to which "if" clause a following "else" clause would '
+             'belong:\n'
+             '\n'
+             '   if test1: if test2: print(x)\n'
+             '\n'
+             'Also note that the semicolon binds tighter than the colon in '
+             'this\n'
+             'context, so that in the following example, either all or none of '
+             'the\n'
+             '"print()" calls are executed:\n'
+             '\n'
+             '   if x < y < z: print(x); print(y); print(z)\n'
+             '\n'
+             'Summarizing:\n'
+             '\n'
+             '   compound_stmt ::= if_stmt\n'
+             '                     | while_stmt\n'
+             '                     | for_stmt\n'
+             '                     | try_stmt\n'
+             '                     | with_stmt\n'
+             '                     | funcdef\n'
+             '                     | classdef\n'
+             '                     | async_with_stmt\n'
+             '                     | async_for_stmt\n'
+             '                     | async_funcdef\n'
+             '   suite         ::= stmt_list NEWLINE | NEWLINE INDENT '
+             'statement+ DEDENT\n'
+             '   statement     ::= stmt_list NEWLINE | compound_stmt\n'
+             '   stmt_list     ::= simple_stmt (";" simple_stmt)* [";"]\n'
+             '\n'
+             'Note that statements always end in a "NEWLINE" possibly followed '
+             'by a\n'
+             '"DEDENT".  Also note that optional continuation clauses always '
+             'begin\n'
+             'with a keyword that cannot start a statement, thus there are no\n'
+             'ambiguities (the \'dangling "else"\' problem is solved in Python '
+             'by\n'
+             'requiring nested "if" statements to be indented).\n'
+             '\n'
+             'The formatting of the grammar rules in the following sections '
+             'places\n'
+             'each clause on a separate line for clarity.\n'
+             '\n'
+             '\n'
+             'The "if" statement\n'
+             '==================\n'
+             '\n'
+             'The "if" statement is used for conditional execution:\n'
+             '\n'
+             '   if_stmt ::= "if" expression ":" suite\n'
+             '               ( "elif" expression ":" suite )*\n'
+             '               ["else" ":" suite]\n'
+             '\n'
+             'It selects exactly one of the suites by evaluating the '
+             'expressions one\n'
+             'by one until one is found to be true (see section *Boolean '
+             'operations*\n'
+             'for the definition of true and false); then that suite is '
+             'executed\n'
+             '(and no other part of the "if" statement is executed or '
+             'evaluated).\n'
+             'If all expressions are false, the suite of the "else" clause, '
+             'if\n'
+             'present, is executed.\n'
+             '\n'
+             '\n'
+             'The "while" statement\n'
+             '=====================\n'
+             '\n'
+             'The "while" statement is used for repeated execution as long as '
+             'an\n'
+             'expression is true:\n'
+             '\n'
+             '   while_stmt ::= "while" expression ":" suite\n'
+             '                  ["else" ":" suite]\n'
+             '\n'
+             'This repeatedly tests the expression and, if it is true, '
+             'executes the\n'
+             'first suite; if the expression is false (which may be the first '
+             'time\n'
+             'it is tested) the suite of the "else" clause, if present, is '
+             'executed\n'
+             'and the loop terminates.\n'
+             '\n'
+             'A "break" statement executed in the first suite terminates the '
+             'loop\n'
+             'without executing the "else" clause\'s suite.  A "continue" '
+             'statement\n'
+             'executed in the first suite skips the rest of the suite and goes '
+             'back\n'
+             'to testing the expression.\n'
+             '\n'
+             '\n'
+             'The "for" statement\n'
+             '===================\n'
+             '\n'
+             'The "for" statement is used to iterate over the elements of a '
+             'sequence\n'
+             '(such as a string, tuple or list) or other iterable object:\n'
+             '\n'
+             '   for_stmt ::= "for" target_list "in" expression_list ":" '
+             'suite\n'
+             '                ["else" ":" suite]\n'
+             '\n'
+             'The expression list is evaluated once; it should yield an '
+             'iterable\n'
+             'object.  An iterator is created for the result of the\n'
+             '"expression_list".  The suite is then executed once for each '
+             'item\n'
+             'provided by the iterator, in the order returned by the '
+             'iterator.  Each\n'
+             'item in turn is assigned to the target list using the standard '
+             'rules\n'
+             'for assignments (see *Assignment statements*), and then the '
+             'suite is\n'
+             'executed.  When the items are exhausted (which is immediately '
+             'when the\n'
+             'sequence is empty or an iterator raises a "StopIteration" '
+             'exception),\n'
+             'the suite in the "else" clause, if present, is executed, and the '
+             'loop\n'
+             'terminates.\n'
+             '\n'
+             'A "break" statement executed in the first suite terminates the '
+             'loop\n'
+             'without executing the "else" clause\'s suite.  A "continue" '
+             'statement\n'
+             'executed in the first suite skips the rest of the suite and '
+             'continues\n'
+             'with the next item, or with the "else" clause if there is no '
+             'next\n'
+             'item.\n'
+             '\n'
+             'The for-loop makes assignments to the variables(s) in the target '
+             'list.\n'
+             'This overwrites all previous assignments to those variables '
+             'including\n'
+             'those made in the suite of the for-loop:\n'
+             '\n'
+             '   for i in range(10):\n'
+             '       print(i)\n'
+             '       i = 5             # this will not affect the for-loop\n'
+             '                         # because i will be overwritten with '
+             'the next\n'
+             '                         # index in the range\n'
+             '\n'
+             'Names in the target list are not deleted when the loop is '
+             'finished,\n'
+             'but if the sequence is empty, they will not have been assigned '
+             'to at\n'
+             'all by the loop.  Hint: the built-in function "range()" returns '
+             'an\n'
+             "iterator of integers suitable to emulate the effect of Pascal's "
+             '"for i\n'
+             ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, '
+             '2]".\n'
+             '\n'
+             'Note: There is a subtlety when the sequence is being modified by '
+             'the\n'
+             '  loop (this can only occur for mutable sequences, i.e. lists).  '
+             'An\n'
+             '  internal counter is used to keep track of which item is used '
+             'next,\n'
+             '  and this is incremented on each iteration.  When this counter '
+             'has\n'
+             '  reached the length of the sequence the loop terminates.  This '
+             'means\n'
+             '  that if the suite deletes the current (or a previous) item '
+             'from the\n'
+             '  sequence, the next item will be skipped (since it gets the '
+             'index of\n'
+             '  the current item which has already been treated).  Likewise, '
+             'if the\n'
+             '  suite inserts an item in the sequence before the current item, '
+             'the\n'
+             '  current item will be treated again the next time through the '
+             'loop.\n'
+             '  This can lead to nasty bugs that can be avoided by making a\n'
+             '  temporary copy using a slice of the whole sequence, e.g.,\n'
+             '\n'
+             '     for x in a[:]:\n'
+             '         if x < 0: a.remove(x)\n'
+             '\n'
+             '\n'
+             'The "try" statement\n'
+             '===================\n'
+             '\n'
+             'The "try" statement specifies exception handlers and/or cleanup '
+             'code\n'
+             'for a group of statements:\n'
+             '\n'
+             '   try_stmt  ::= try1_stmt | try2_stmt\n'
+             '   try1_stmt ::= "try" ":" suite\n'
+             '                 ("except" [expression ["as" identifier]] ":" '
+             'suite)+\n'
+             '                 ["else" ":" suite]\n'
+             '                 ["finally" ":" suite]\n'
+             '   try2_stmt ::= "try" ":" suite\n'
+             '                 "finally" ":" suite\n'
+             '\n'
+             'The "except" clause(s) specify one or more exception handlers. '
+             'When no\n'
+             'exception occurs in the "try" clause, no exception handler is\n'
+             'executed. When an exception occurs in the "try" suite, a search '
+             'for an\n'
+             'exception handler is started.  This search inspects the except '
+             'clauses\n'
+             'in turn until one is found that matches the exception.  An '
+             'expression-\n'
+             'less except clause, if present, must be last; it matches any\n'
+             'exception.  For an except clause with an expression, that '
+             'expression\n'
+             'is evaluated, and the clause matches the exception if the '
+             'resulting\n'
+             'object is "compatible" with the exception.  An object is '
+             'compatible\n'
+             'with an exception if it is the class or a base class of the '
+             'exception\n'
+             'object or a tuple containing an item compatible with the '
+             'exception.\n'
+             '\n'
+             'If no except clause matches the exception, the search for an '
+             'exception\n'
+             'handler continues in the surrounding code and on the invocation '
+             'stack.\n'
+             '[1]\n'
+             '\n'
+             'If the evaluation of an expression in the header of an except '
+             'clause\n'
+             'raises an exception, the original search for a handler is '
+             'canceled and\n'
+             'a search starts for the new exception in the surrounding code '
+             'and on\n'
+             'the call stack (it is treated as if the entire "try" statement '
+             'raised\n'
+             'the exception).\n'
+             '\n'
+             'When a matching except clause is found, the exception is '
+             'assigned to\n'
+             'the target specified after the "as" keyword in that except '
+             'clause, if\n'
+             "present, and the except clause's suite is executed.  All except\n"
+             'clauses must have an executable block.  When the end of this '
+             'block is\n'
+             'reached, execution continues normally after the entire try '
+             'statement.\n'
+             '(This means that if two nested handlers exist for the same '
+             'exception,\n'
+             'and the exception occurs in the try clause of the inner handler, '
+             'the\n'
+             'outer handler will not handle the exception.)\n'
+             '\n'
+             'When an exception has been assigned using "as target", it is '
+             'cleared\n'
+             'at the end of the except clause.  This is as if\n'
+             '\n'
+             '   except E as N:\n'
+             '       foo\n'
+             '\n'
+             'was translated to\n'
+             '\n'
+             '   except E as N:\n'
+             '       try:\n'
+             '           foo\n'
+             '       finally:\n'
+             '           del N\n'
+             '\n'
+             'This means the exception must be assigned to a different name to '
+             'be\n'
+             'able to refer to it after the except clause.  Exceptions are '
+             'cleared\n'
+             'because with the traceback attached to them, they form a '
+             'reference\n'
+             'cycle with the stack frame, keeping all locals in that frame '
+             'alive\n'
+             'until the next garbage collection occurs.\n'
+             '\n'
+             "Before an except clause's suite is executed, details about the\n"
+             'exception are stored in the "sys" module and can be accessed '
+             'via\n'
+             '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting '
+             'of the\n'
+             'exception class, the exception instance and a traceback object '
+             '(see\n'
+             'section *The standard type hierarchy*) identifying the point in '
+             'the\n'
+             'program where the exception occurred.  "sys.exc_info()" values '
+             'are\n'
+             'restored to their previous values (before the call) when '
+             'returning\n'
+             'from a function that handled an exception.\n'
+             '\n'
+             'The optional "else" clause is executed if and when control flows '
+             'off\n'
+             'the end of the "try" clause. [2] Exceptions in the "else" clause '
+             'are\n'
+             'not handled by the preceding "except" clauses.\n'
+             '\n'
+             'If "finally" is present, it specifies a \'cleanup\' handler.  '
+             'The "try"\n'
+             'clause is executed, including any "except" and "else" clauses.  '
+             'If an\n'
+             'exception occurs in any of the clauses and is not handled, the\n'
+             'exception is temporarily saved. The "finally" clause is '
+             'executed.  If\n'
+             'there is a saved exception it is re-raised at the end of the '
+             '"finally"\n'
+             'clause.  If the "finally" clause raises another exception, the '
+             'saved\n'
+             'exception is set as the context of the new exception. If the '
+             '"finally"\n'
+             'clause executes a "return" or "break" statement, the saved '
+             'exception\n'
+             'is discarded:\n'
+             '\n'
+             '   >>> def f():\n'
+             '   ...     try:\n'
+             '   ...         1/0\n'
+             '   ...     finally:\n'
+             '   ...         return 42\n'
+             '   ...\n'
+             '   >>> f()\n'
+             '   42\n'
+             '\n'
+             'The exception information is not available to the program '
+             'during\n'
+             'execution of the "finally" clause.\n'
+             '\n'
+             'When a "return", "break" or "continue" statement is executed in '
+             'the\n'
+             '"try" suite of a "try"..."finally" statement, the "finally" '
+             'clause is\n'
+             'also executed \'on the way out.\' A "continue" statement is '
+             'illegal in\n'
+             'the "finally" clause. (The reason is a problem with the current\n'
+             'implementation --- this restriction may be lifted in the '
+             'future).\n'
+             '\n'
+             'The return value of a function is determined by the last '
+             '"return"\n'
+             'statement executed.  Since the "finally" clause always executes, '
+             'a\n'
+             '"return" statement executed in the "finally" clause will always '
+             'be the\n'
+             'last one executed:\n'
+             '\n'
+             '   >>> def foo():\n'
+             '   ...     try:\n'
+             "   ...         return 'try'\n"
+             '   ...     finally:\n'
+             "   ...         return 'finally'\n"
+             '   ...\n'
+             '   >>> foo()\n'
+             "   'finally'\n"
+             '\n'
+             'Additional information on exceptions can be found in section\n'
+             '*Exceptions*, and information on using the "raise" statement to\n'
+             'generate exceptions may be found in section *The raise '
+             'statement*.\n'
+             '\n'
+             '\n'
+             'The "with" statement\n'
+             '====================\n'
+             '\n'
+             'The "with" statement is used to wrap the execution of a block '
+             'with\n'
+             'methods defined by a context manager (see section *With '
+             'Statement\n'
+             'Context Managers*). This allows common '
+             '"try"..."except"..."finally"\n'
+             'usage patterns to be encapsulated for convenient reuse.\n'
+             '\n'
+             '   with_stmt ::= "with" with_item ("," with_item)* ":" suite\n'
+             '   with_item ::= expression ["as" target]\n'
+             '\n'
+             'The execution of the "with" statement with one "item" proceeds '
+             'as\n'
+             'follows:\n'
+             '\n'
+             '1. The context expression (the expression given in the '
+             '"with_item")\n'
+             '   is evaluated to obtain a context manager.\n'
+             '\n'
+             '2. The context manager\'s "__exit__()" is loaded for later use.\n'
+             '\n'
+             '3. The context manager\'s "__enter__()" method is invoked.\n'
+             '\n'
+             '4. If a target was included in the "with" statement, the return\n'
+             '   value from "__enter__()" is assigned to it.\n'
+             '\n'
+             '   Note: The "with" statement guarantees that if the '
+             '"__enter__()"\n'
+             '     method returns without an error, then "__exit__()" will '
+             'always be\n'
+             '     called. Thus, if an error occurs during the assignment to '
+             'the\n'
+             '     target list, it will be treated the same as an error '
+             'occurring\n'
+             '     within the suite would be. See step 6 below.\n'
+             '\n'
+             '5. The suite is executed.\n'
+             '\n'
+             '6. The context manager\'s "__exit__()" method is invoked.  If '
+             'an\n'
+             '   exception caused the suite to be exited, its type, value, '
+             'and\n'
+             '   traceback are passed as arguments to "__exit__()". Otherwise, '
+             'three\n'
+             '   "None" arguments are supplied.\n'
+             '\n'
+             '   If the suite was exited due to an exception, and the return '
+             'value\n'
+             '   from the "__exit__()" method was false, the exception is '
+             'reraised.\n'
+             '   If the return value was true, the exception is suppressed, '
+             'and\n'
+             '   execution continues with the statement following the "with"\n'
+             '   statement.\n'
+             '\n'
+             '   If the suite was exited for any reason other than an '
+             'exception, the\n'
+             '   return value from "__exit__()" is ignored, and execution '
+             'proceeds\n'
+             '   at the normal location for the kind of exit that was taken.\n'
+             '\n'
+             'With more than one item, the context managers are processed as '
+             'if\n'
+             'multiple "with" statements were nested:\n'
+             '\n'
+             '   with A() as a, B() as b:\n'
+             '       suite\n'
+             '\n'
+             'is equivalent to\n'
+             '\n'
+             '   with A() as a:\n'
+             '       with B() as b:\n'
+             '           suite\n'
+             '\n'
+             'Changed in version 3.1: Support for multiple context '
+             'expressions.\n'
+             '\n'
+             'See also: **PEP 343** - The "with" statement\n'
+             '\n'
+             '     The specification, background, and examples for the Python '
+             '"with"\n'
+             '     statement.\n'
+             '\n'
+             '\n'
+             'Function definitions\n'
+             '====================\n'
+             '\n'
+             'A function definition defines a user-defined function object '
+             '(see\n'
+             'section *The standard type hierarchy*):\n'
+             '\n'
+             '   funcdef        ::= [decorators] "def" funcname "(" '
+             '[parameter_list] ")" ["->" expression] ":" suite\n'
+             '   decorators     ::= decorator+\n'
+             '   decorator      ::= "@" dotted_name ["(" [argument_list [","]] '
+             '")"] NEWLINE\n'
+             '   dotted_name    ::= identifier ("." identifier)*\n'
+             '   parameter_list ::= (defparameter ",")*\n'
+             '                      | "*" [parameter] ("," defparameter)* ["," '
+             '"**" parameter]\n'
+             '                      | "**" parameter\n'
+             '                      | defparameter [","] )\n'
+             '   parameter      ::= identifier [":" expression]\n'
+             '   defparameter   ::= parameter ["=" expression]\n'
+             '   funcname       ::= identifier\n'
+             '\n'
+             'A function definition is an executable statement.  Its execution '
+             'binds\n'
+             'the function name in the current local namespace to a function '
+             'object\n'
+             '(a wrapper around the executable code for the function).  This\n'
+             'function object contains a reference to the current global '
+             'namespace\n'
+             'as the global namespace to be used when the function is called.\n'
+             '\n'
+             'The function definition does not execute the function body; this '
+             'gets\n'
+             'executed only when the function is called. [3]\n'
+             '\n'
+             'A function definition may be wrapped by one or more *decorator*\n'
+             'expressions. Decorator expressions are evaluated when the '
+             'function is\n'
+             'defined, in the scope that contains the function definition.  '
+             'The\n'
+             'result must be a callable, which is invoked with the function '
+             'object\n'
+             'as the only argument. The returned value is bound to the '
+             'function name\n'
+             'instead of the function object.  Multiple decorators are applied '
+             'in\n'
+             'nested fashion. For example, the following code\n'
+             '\n'
+             '   @f1(arg)\n'
+             '   @f2\n'
+             '   def func(): pass\n'
+             '\n'
+             'is roughly equivalent to\n'
+             '\n'
+             '   def func(): pass\n'
+             '   func = f1(arg)(f2(func))\n'
+             '\n'
+             'except that the original function is not temporarily bound to '
+             'the name\n'
+             '"func".\n'
+             '\n'
+             'When one or more *parameters* have the form *parameter* "="\n'
+             '*expression*, the function is said to have "default parameter '
+             'values."\n'
+             'For a parameter with a default value, the corresponding '
+             '*argument* may\n'
+             "be omitted from a call, in which case the parameter's default "
+             'value is\n'
+             'substituted.  If a parameter has a default value, all following\n'
+             'parameters up until the ""*"" must also have a default value --- '
+             'this\n'
+             'is a syntactic restriction that is not expressed by the '
+             'grammar.\n'
+             '\n'
+             '**Default parameter values are evaluated from left to right when '
+             'the\n'
+             'function definition is executed.** This means that the '
+             'expression is\n'
+             'evaluated once, when the function is defined, and that the same '
+             '"pre-\n'
+             'computed" value is used for each call.  This is especially '
+             'important\n'
+             'to understand when a default parameter is a mutable object, such '
+             'as a\n'
+             'list or a dictionary: if the function modifies the object (e.g. '
+             'by\n'
+             'appending an item to a list), the default value is in effect '
+             'modified.\n'
+             'This is generally not what was intended.  A way around this is '
+             'to use\n'
+             '"None" as the default, and explicitly test for it in the body of '
+             'the\n'
+             'function, e.g.:\n'
+             '\n'
+             '   def whats_on_the_telly(penguin=None):\n'
+             '       if penguin is None:\n'
+             '           penguin = []\n'
+             '       penguin.append("property of the zoo")\n'
+             '       return penguin\n'
+             '\n'
+             'Function call semantics are described in more detail in section\n'
+             '*Calls*. A function call always assigns values to all '
+             'parameters\n'
+             'mentioned in the parameter list, either from position arguments, '
+             'from\n'
+             'keyword arguments, or from default values.  If the form\n'
+             '""*identifier"" is present, it is initialized to a tuple '
+             'receiving any\n'
+             'excess positional parameters, defaulting to the empty tuple.  If '
+             'the\n'
+             'form ""**identifier"" is present, it is initialized to a new\n'
+             'dictionary receiving any excess keyword arguments, defaulting to '
+             'a new\n'
+             'empty dictionary. Parameters after ""*"" or ""*identifier"" are\n'
+             'keyword-only parameters and may only be passed used keyword '
+             'arguments.\n'
+             '\n'
+             'Parameters may have annotations of the form "": expression"" '
+             'following\n'
+             'the parameter name.  Any parameter may have an annotation even '
+             'those\n'
+             'of the form "*identifier" or "**identifier".  Functions may '
+             'have\n'
+             '"return" annotation of the form ""-> expression"" after the '
+             'parameter\n'
+             'list.  These annotations can be any valid Python expression and '
+             'are\n'
+             'evaluated when the function definition is executed.  Annotations '
+             'may\n'
+             'be evaluated in a different order than they appear in the source '
+             'code.\n'
+             'The presence of annotations does not change the semantics of a\n'
+             'function.  The annotation values are available as values of a\n'
+             "dictionary keyed by the parameters' names in the "
+             '"__annotations__"\n'
+             'attribute of the function object.\n'
+             '\n'
+             'It is also possible to create anonymous functions (functions not '
+             'bound\n'
+             'to a name), for immediate use in expressions.  This uses lambda\n'
+             'expressions, described in section *Lambdas*.  Note that the '
+             'lambda\n'
+             'expression is merely a shorthand for a simplified function '
+             'definition;\n'
+             'a function defined in a ""def"" statement can be passed around '
+             'or\n'
+             'assigned to another name just like a function defined by a '
+             'lambda\n'
+             'expression.  The ""def"" form is actually more powerful since '
+             'it\n'
+             'allows the execution of multiple statements and annotations.\n'
+             '\n'
+             "**Programmer's note:** Functions are first-class objects.  A "
+             '""def""\n'
+             'statement executed inside a function definition defines a local\n'
+             'function that can be returned or passed around.  Free variables '
+             'used\n'
+             'in the nested function can access the local variables of the '
+             'function\n'
+             'containing the def.  See section *Naming and binding* for '
+             'details.\n'
+             '\n'
+             'See also: **PEP 3107** - Function Annotations\n'
+             '\n'
+             '     The original specification for function annotations.\n'
+             '\n'
+             '\n'
+             'Class definitions\n'
+             '=================\n'
+             '\n'
+             'A class definition defines a class object (see section *The '
+             'standard\n'
+             'type hierarchy*):\n'
+             '\n'
+             '   classdef    ::= [decorators] "class" classname [inheritance] '
+             '":" suite\n'
+             '   inheritance ::= "(" [argument_list] ")"\n'
+             '   classname   ::= identifier\n'
+             '\n'
+             'A class definition is an executable statement.  The inheritance '
+             'list\n'
+             'usually gives a list of base classes (see *Customizing class '
+             'creation*\n'
+             'for more advanced uses), so each item in the list should '
+             'evaluate to a\n'
+             'class object which allows subclassing.  Classes without an '
+             'inheritance\n'
+             'list inherit, by default, from the base class "object"; hence,\n'
+             '\n'
+             '   class Foo:\n'
+             '       pass\n'
+             '\n'
+             'is equivalent to\n'
+             '\n'
+             '   class Foo(object):\n'
+             '       pass\n'
+             '\n'
+             "The class's suite is then executed in a new execution frame "
+             '(see\n'
+             '*Naming and binding*), using a newly created local namespace and '
+             'the\n'
+             'original global namespace. (Usually, the suite contains mostly\n'
+             "function definitions.)  When the class's suite finishes "
+             'execution, its\n'
+             'execution frame is discarded but its local namespace is saved. '
+             '[4] A\n'
+             'class object is then created using the inheritance list for the '
+             'base\n'
+             'classes and the saved local namespace for the attribute '
+             'dictionary.\n'
+             'The class name is bound to this class object in the original '
+             'local\n'
+             'namespace.\n'
+             '\n'
+             'Class creation can be customized heavily using *metaclasses*.\n'
+             '\n'
+             'Classes can also be decorated: just like when decorating '
+             'functions,\n'
+             '\n'
+             '   @f1(arg)\n'
+             '   @f2\n'
+             '   class Foo: pass\n'
+             '\n'
+             'is roughly equivalent to\n'
+             '\n'
+             '   class Foo: pass\n'
+             '   Foo = f1(arg)(f2(Foo))\n'
+             '\n'
+             'The evaluation rules for the decorator expressions are the same '
+             'as for\n'
+             'function decorators.  The result is then bound to the class '
+             'name.\n'
+             '\n'
+             "**Programmer's note:** Variables defined in the class definition "
+             'are\n'
+             'class attributes; they are shared by instances.  Instance '
+             'attributes\n'
+             'can be set in a method with "self.name = value".  Both class '
+             'and\n'
+             'instance attributes are accessible through the notation '
+             '""self.name"",\n'
+             'and an instance attribute hides a class attribute with the same '
+             'name\n'
+             'when accessed in this way.  Class attributes can be used as '
+             'defaults\n'
+             'for instance attributes, but using mutable values there can lead '
+             'to\n'
+             'unexpected results.  *Descriptors* can be used to create '
+             'instance\n'
+             'variables with different implementation details.\n'
+             '\n'
+             'See also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n'
+             '  Class Decorators\n'
+             '\n'
+             '\n'
+             'Coroutines\n'
+             '==========\n'
+             '\n'
+             'New in version 3.5.\n'
+             '\n'
+             '\n'
+             'Coroutine function definition\n'
+             '-----------------------------\n'
+             '\n'
+             '   async_funcdef ::= [decorators] "async" "def" funcname "(" '
+             '[parameter_list] ")" ["->" expression] ":" suite\n'
+             '\n'
+             'Execution of Python coroutines can be suspended and resumed at '
+             'many\n'
+             'points (see *coroutine*).  In the body of a coroutine, any '
+             '"await" and\n'
+             '"async" identifiers become reserved keywords; "await" '
+             'expressions,\n'
+             '"async for" and "async with" can only be used in coroutine '
+             'bodies.\n'
+             '\n'
+             'Functions defined with "async def" syntax are always coroutine\n'
+             'functions, even if they do not contain "await" or "async" '
+             'keywords.\n'
+             '\n'
+             'It is a "SyntaxError" to use "yield" expressions in "async def"\n'
+             'coroutines.\n'
+             '\n'
+             'An example of a coroutine function:\n'
+             '\n'
+             '   async def func(param1, param2):\n'
+             '       do_stuff()\n'
+             '       await some_coroutine()\n'
+             '\n'
+             '\n'
+             'The "async for" statement\n'
+             '-------------------------\n'
+             '\n'
+             '   async_for_stmt ::= "async" for_stmt\n'
+             '\n'
+             'An *asynchronous iterable* is able to call asynchronous code in '
+             'its\n'
+             '*iter* implementation, and *asynchronous iterator* can call\n'
+             'asynchronous code in its *next* method.\n'
+             '\n'
+             'The "async for" statement allows convenient iteration over\n'
+             'asynchronous iterators.\n'
+             '\n'
+             'The following code:\n'
+             '\n'
+             '   async for TARGET in ITER:\n'
+             '       BLOCK\n'
+             '   else:\n'
+             '       BLOCK2\n'
+             '\n'
+             'Is semantically equivalent to:\n'
+             '\n'
+             '   iter = (ITER)\n'
+             '   iter = type(iter).__aiter__(iter)\n'
+             '   running = True\n'
+             '   while running:\n'
+             '       try:\n'
+             '           TARGET = await type(iter).__anext__(iter)\n'
+             '       except StopAsyncIteration:\n'
+             '           running = False\n'
+             '       else:\n'
+             '           BLOCK\n'
+             '   else:\n'
+             '       BLOCK2\n'
+             '\n'
+             'See also "__aiter__()" and "__anext__()" for details.\n'
+             '\n'
+             'It is a "SyntaxError" to use "async for" statement outside of '
+             'an\n'
+             '"async def" function.\n'
+             '\n'
+             '\n'
+             'The "async with" statement\n'
+             '--------------------------\n'
+             '\n'
+             '   async_with_stmt ::= "async" with_stmt\n'
+             '\n'
+             'An *asynchronous context manager* is a *context manager* that is '
+             'able\n'
+             'to suspend execution in its *enter* and *exit* methods.\n'
+             '\n'
+             'The following code:\n'
+             '\n'
+             '   async with EXPR as VAR:\n'
+             '       BLOCK\n'
+             '\n'
+             'Is semantically equivalent to:\n'
+             '\n'
+             '   mgr = (EXPR)\n'
+             '   aexit = type(mgr).__aexit__\n'
+             '   aenter = type(mgr).__aenter__(mgr)\n'
+             '   exc = True\n'
+             '\n'
+             '   VAR = await aenter\n'
+             '   try:\n'
+             '       BLOCK\n'
+             '   except:\n'
+             '       if not await aexit(mgr, *sys.exc_info()):\n'
+             '           raise\n'
+             '   else:\n'
+             '       await aexit(mgr, None, None, None)\n'
+             '\n'
+             'See also "__aenter__()" and "__aexit__()" for details.\n'
+             '\n'
+             'It is a "SyntaxError" to use "async with" statement outside of '
+             'an\n'
+             '"async def" function.\n'
+             '\n'
+             'See also: **PEP 492** - Coroutines with async and await syntax\n'
+             '\n'
+             '-[ Footnotes ]-\n'
+             '\n'
+             '[1] The exception is propagated to the invocation stack unless\n'
+             '    there is a "finally" clause which happens to raise another\n'
+             '    exception. That new exception causes the old one to be '
+             'lost.\n'
+             '\n'
+             '[2] Currently, control "flows off the end" except in the case '
+             'of\n'
+             '    an exception or the execution of a "return", "continue", or\n'
+             '    "break" statement.\n'
+             '\n'
+             '[3] A string literal appearing as the first statement in the\n'
+             '    function body is transformed into the function\'s "__doc__"\n'
+             "    attribute and therefore the function's *docstring*.\n"
+             '\n'
+             '[4] A string literal appearing as the first statement in the '
+             'class\n'
+             '    body is transformed into the namespace\'s "__doc__" item '
+             'and\n'
+             "    therefore the class's *docstring*.\n",
+ 'context-managers': '\n'
+                     'With Statement Context Managers\n'
+                     '*******************************\n'
+                     '\n'
+                     'A *context manager* is an object that defines the '
+                     'runtime context to\n'
+                     'be established when executing a "with" statement. The '
+                     'context manager\n'
+                     'handles the entry into, and the exit from, the desired '
+                     'runtime context\n'
+                     'for the execution of the block of code.  Context '
+                     'managers are normally\n'
+                     'invoked using the "with" statement (described in section '
+                     '*The with\n'
+                     'statement*), but can also be used by directly invoking '
+                     'their methods.\n'
+                     '\n'
+                     'Typical uses of context managers include saving and '
+                     'restoring various\n'
+                     'kinds of global state, locking and unlocking resources, '
+                     'closing opened\n'
+                     'files, etc.\n'
+                     '\n'
+                     'For more information on context managers, see *Context '
+                     'Manager Types*.\n'
+                     '\n'
+                     'object.__enter__(self)\n'
+                     '\n'
+                     '   Enter the runtime context related to this object. The '
+                     '"with"\n'
+                     "   statement will bind this method's return value to the "
+                     'target(s)\n'
+                     '   specified in the "as" clause of the statement, if '
+                     'any.\n'
+                     '\n'
+                     'object.__exit__(self, exc_type, exc_value, traceback)\n'
+                     '\n'
+                     '   Exit the runtime context related to this object. The '
+                     'parameters\n'
+                     '   describe the exception that caused the context to be '
+                     'exited. If the\n'
+                     '   context was exited without an exception, all three '
+                     'arguments will\n'
+                     '   be "None".\n'
+                     '\n'
+                     '   If an exception is supplied, and the method wishes to '
+                     'suppress the\n'
+                     '   exception (i.e., prevent it from being propagated), '
+                     'it should\n'
+                     '   return a true value. Otherwise, the exception will be '
+                     'processed\n'
+                     '   normally upon exit from this method.\n'
+                     '\n'
+                     '   Note that "__exit__()" methods should not reraise the '
+                     'passed-in\n'
+                     "   exception; this is the caller's responsibility.\n"
+                     '\n'
+                     'See also: **PEP 343** - The "with" statement\n'
+                     '\n'
+                     '     The specification, background, and examples for the '
+                     'Python "with"\n'
+                     '     statement.\n',
+ 'continue': '\n'
+             'The "continue" statement\n'
+             '************************\n'
+             '\n'
+             '   continue_stmt ::= "continue"\n'
+             '\n'
+             '"continue" may only occur syntactically nested in a "for" or '
+             '"while"\n'
+             'loop, but not nested in a function or class definition or '
+             '"finally"\n'
+             'clause within that loop.  It continues with the next cycle of '
+             'the\n'
+             'nearest enclosing loop.\n'
+             '\n'
+             'When "continue" passes control out of a "try" statement with a\n'
+             '"finally" clause, that "finally" clause is executed before '
+             'really\n'
+             'starting the next loop cycle.\n',
+ 'conversions': '\n'
+                'Arithmetic conversions\n'
+                '**********************\n'
+                '\n'
+                'When a description of an arithmetic operator below uses the '
+                'phrase\n'
+                '"the numeric arguments are converted to a common type," this '
+                'means\n'
+                'that the operator implementation for built-in types works as '
+                'follows:\n'
+                '\n'
+                '* If either argument is a complex number, the other is '
+                'converted to\n'
+                '  complex;\n'
+                '\n'
+                '* otherwise, if either argument is a floating point number, '
+                'the\n'
+                '  other is converted to floating point;\n'
+                '\n'
+                '* otherwise, both must be integers and no conversion is '
+                'necessary.\n'
+                '\n'
+                'Some additional rules apply for certain operators (e.g., a '
+                'string as a\n'
+                "left argument to the '%' operator).  Extensions must define "
+                'their own\n'
+                'conversion behavior.\n',
+ 'customization': '\n'
+                  'Basic customization\n'
+                  '*******************\n'
+                  '\n'
+                  'object.__new__(cls[, ...])\n'
+                  '\n'
+                  '   Called to create a new instance of class *cls*.  '
+                  '"__new__()" is a\n'
+                  '   static method (special-cased so you need not declare it '
+                  'as such)\n'
+                  '   that takes the class of which an instance was requested '
+                  'as its\n'
+                  '   first argument.  The remaining arguments are those '
+                  'passed to the\n'
+                  '   object constructor expression (the call to the class).  '
+                  'The return\n'
+                  '   value of "__new__()" should be the new object instance '
+                  '(usually an\n'
+                  '   instance of *cls*).\n'
+                  '\n'
+                  '   Typical implementations create a new instance of the '
+                  'class by\n'
+                  '   invoking the superclass\'s "__new__()" method using\n'
+                  '   "super().__new__(cls[, ...])" with appropriate arguments '
+                  'and then\n'
+                  '   modifying the newly-created instance as necessary before '
+                  'returning\n'
+                  '   it.\n'
+                  '\n'
+                  '   If "__new__()" returns an instance of *cls*, then the '
+                  'new\n'
+                  '   instance\'s "__init__()" method will be invoked like\n'
+                  '   "__init__(self[, ...])", where *self* is the new '
+                  'instance and the\n'
+                  '   remaining arguments are the same as were passed to '
+                  '"__new__()".\n'
+                  '\n'
+                  '   If "__new__()" does not return an instance of *cls*, '
+                  'then the new\n'
+                  '   instance\'s "__init__()" method will not be invoked.\n'
+                  '\n'
+                  '   "__new__()" is intended mainly to allow subclasses of '
+                  'immutable\n'
+                  '   types (like int, str, or tuple) to customize instance '
+                  'creation.  It\n'
+                  '   is also commonly overridden in custom metaclasses in '
+                  'order to\n'
+                  '   customize class creation.\n'
+                  '\n'
+                  'object.__init__(self[, ...])\n'
+                  '\n'
+                  '   Called after the instance has been created (by '
+                  '"__new__()"), but\n'
+                  '   before it is returned to the caller.  The arguments are '
+                  'those\n'
+                  '   passed to the class constructor expression.  If a base '
+                  'class has an\n'
+                  '   "__init__()" method, the derived class\'s "__init__()" '
+                  'method, if\n'
+                  '   any, must explicitly call it to ensure proper '
+                  'initialization of the\n'
+                  '   base class part of the instance; for example:\n'
+                  '   "super().__init__([args...])".\n'
+                  '\n'
+                  '   Because "__new__()" and "__init__()" work together in '
+                  'constructing\n'
+                  '   objects ("__new__()" to create it, and "__init__()" to '
+                  'customize\n'
+                  '   it), no non-"None" value may be returned by '
+                  '"__init__()"; doing so\n'
+                  '   will cause a "TypeError" to be raised at runtime.\n'
+                  '\n'
+                  'object.__del__(self)\n'
+                  '\n'
+                  '   Called when the instance is about to be destroyed.  This '
+                  'is also\n'
+                  '   called a destructor.  If a base class has a "__del__()" '
+                  'method, the\n'
+                  '   derived class\'s "__del__()" method, if any, must '
+                  'explicitly call it\n'
+                  '   to ensure proper deletion of the base class part of the '
+                  'instance.\n'
+                  '   Note that it is possible (though not recommended!) for '
+                  'the\n'
+                  '   "__del__()" method to postpone destruction of the '
+                  'instance by\n'
+                  '   creating a new reference to it.  It may then be called '
+                  'at a later\n'
+                  '   time when this new reference is deleted.  It is not '
+                  'guaranteed that\n'
+                  '   "__del__()" methods are called for objects that still '
+                  'exist when\n'
+                  '   the interpreter exits.\n'
+                  '\n'
+                  '   Note: "del x" doesn\'t directly call "x.__del__()" --- '
+                  'the former\n'
+                  '     decrements the reference count for "x" by one, and the '
+                  'latter is\n'
+                  '     only called when "x"\'s reference count reaches zero.  '
+                  'Some common\n'
+                  '     situations that may prevent the reference count of an '
+                  'object from\n'
+                  '     going to zero include: circular references between '
+                  'objects (e.g.,\n'
+                  '     a doubly-linked list or a tree data structure with '
+                  'parent and\n'
+                  '     child pointers); a reference to the object on the '
+                  'stack frame of\n'
+                  '     a function that caught an exception (the traceback '
+                  'stored in\n'
+                  '     "sys.exc_info()[2]" keeps the stack frame alive); or a '
+                  'reference\n'
+                  '     to the object on the stack frame that raised an '
+                  'unhandled\n'
+                  '     exception in interactive mode (the traceback stored '
+                  'in\n'
+                  '     "sys.last_traceback" keeps the stack frame alive).  '
+                  'The first\n'
+                  '     situation can only be remedied by explicitly breaking '
+                  'the cycles;\n'
+                  '     the second can be resolved by freeing the reference to '
+                  'the\n'
+                  '     traceback object when it is no longer useful, and the '
+                  'third can\n'
+                  '     be resolved by storing "None" in "sys.last_traceback". '
+                  'Circular\n'
+                  '     references which are garbage are detected and cleaned '
+                  'up when the\n'
+                  "     cyclic garbage collector is enabled (it's on by "
+                  'default). Refer\n'
+                  '     to the documentation for the "gc" module for more '
+                  'information\n'
+                  '     about this topic.\n'
+                  '\n'
+                  '   Warning: Due to the precarious circumstances under '
+                  'which\n'
+                  '     "__del__()" methods are invoked, exceptions that occur '
+                  'during\n'
+                  '     their execution are ignored, and a warning is printed '
+                  'to\n'
+                  '     "sys.stderr" instead. Also, when "__del__()" is '
+                  'invoked in\n'
+                  '     response to a module being deleted (e.g., when '
+                  'execution of the\n'
+                  '     program is done), other globals referenced by the '
+                  '"__del__()"\n'
+                  '     method may already have been deleted or in the process '
+                  'of being\n'
+                  '     torn down (e.g. the import machinery shutting down).  '
+                  'For this\n'
+                  '     reason, "__del__()" methods should do the absolute '
+                  'minimum needed\n'
+                  '     to maintain external invariants.  Starting with '
+                  'version 1.5,\n'
+                  '     Python guarantees that globals whose name begins with '
+                  'a single\n'
+                  '     underscore are deleted from their module before other '
+                  'globals are\n'
+                  '     deleted; if no other references to such globals exist, '
+                  'this may\n'
+                  '     help in assuring that imported modules are still '
+                  'available at the\n'
+                  '     time when the "__del__()" method is called.\n'
+                  '\n'
+                  'object.__repr__(self)\n'
+                  '\n'
+                  '   Called by the "repr()" built-in function to compute the '
+                  '"official"\n'
+                  '   string representation of an object.  If at all possible, '
+                  'this\n'
+                  '   should look like a valid Python expression that could be '
+                  'used to\n'
+                  '   recreate an object with the same value (given an '
+                  'appropriate\n'
+                  '   environment).  If this is not possible, a string of the '
+                  'form\n'
+                  '   "<...some useful description...>" should be returned. '
+                  'The return\n'
+                  '   value must be a string object. If a class defines '
+                  '"__repr__()" but\n'
+                  '   not "__str__()", then "__repr__()" is also used when an '
+                  '"informal"\n'
+                  '   string representation of instances of that class is '
+                  'required.\n'
+                  '\n'
+                  '   This is typically used for debugging, so it is important '
+                  'that the\n'
+                  '   representation is information-rich and unambiguous.\n'
+                  '\n'
+                  'object.__str__(self)\n'
+                  '\n'
+                  '   Called by "str(object)" and the built-in functions '
+                  '"format()" and\n'
+                  '   "print()" to compute the "informal" or nicely printable '
+                  'string\n'
+                  '   representation of an object.  The return value must be a '
+                  '*string*\n'
+                  '   object.\n'
+                  '\n'
+                  '   This method differs from "object.__repr__()" in that '
+                  'there is no\n'
+                  '   expectation that "__str__()" return a valid Python '
+                  'expression: a\n'
+                  '   more convenient or concise representation can be used.\n'
+                  '\n'
+                  '   The default implementation defined by the built-in type '
+                  '"object"\n'
+                  '   calls "object.__repr__()".\n'
+                  '\n'
+                  'object.__bytes__(self)\n'
+                  '\n'
+                  '   Called by "bytes()" to compute a byte-string '
+                  'representation of an\n'
+                  '   object. This should return a "bytes" object.\n'
+                  '\n'
+                  'object.__format__(self, format_spec)\n'
+                  '\n'
+                  '   Called by the "format()" built-in function (and by '
+                  'extension, the\n'
+                  '   "str.format()" method of class "str") to produce a '
+                  '"formatted"\n'
+                  '   string representation of an object. The "format_spec" '
+                  'argument is a\n'
+                  '   string that contains a description of the formatting '
+                  'options\n'
+                  '   desired. The interpretation of the "format_spec" '
+                  'argument is up to\n'
+                  '   the type implementing "__format__()", however most '
+                  'classes will\n'
+                  '   either delegate formatting to one of the built-in types, '
+                  'or use a\n'
+                  '   similar formatting option syntax.\n'
+                  '\n'
+                  '   See *Format Specification Mini-Language* for a '
+                  'description of the\n'
+                  '   standard formatting syntax.\n'
+                  '\n'
+                  '   The return value must be a string object.\n'
+                  '\n'
+                  '   Changed in version 3.4: The __format__ method of '
+                  '"object" itself\n'
+                  '   raises a "TypeError" if passed any non-empty string.\n'
+                  '\n'
+                  'object.__lt__(self, other)\n'
+                  'object.__le__(self, other)\n'
+                  'object.__eq__(self, other)\n'
+                  'object.__ne__(self, other)\n'
+                  'object.__gt__(self, other)\n'
+                  'object.__ge__(self, other)\n'
+                  '\n'
+                  '   These are the so-called "rich comparison" methods. The\n'
+                  '   correspondence between operator symbols and method names '
+                  'is as\n'
+                  '   follows: "x<y" calls "x.__lt__(y)", "x<=y" calls '
+                  '"x.__le__(y)",\n'
+                  '   "x==y" calls "x.__eq__(y)", "x!=y" calls "x.__ne__(y)", '
+                  '"x>y" calls\n'
+                  '   "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n'
+                  '\n'
+                  '   A rich comparison method may return the singleton '
+                  '"NotImplemented"\n'
+                  '   if it does not implement the operation for a given pair '
+                  'of\n'
+                  '   arguments. By convention, "False" and "True" are '
+                  'returned for a\n'
+                  '   successful comparison. However, these methods can return '
+                  'any value,\n'
+                  '   so if the comparison operator is used in a Boolean '
+                  'context (e.g.,\n'
+                  '   in the condition of an "if" statement), Python will call '
+                  '"bool()"\n'
+                  '   on the value to determine if the result is true or '
+                  'false.\n'
+                  '\n'
+                  '   By default, "__ne__()" delegates to "__eq__()" and '
+                  'inverts the\n'
+                  '   result unless it is "NotImplemented".  There are no '
+                  'other implied\n'
+                  '   relationships among the comparison operators, for '
+                  'example, the\n'
+                  '   truth of "(x<y or x==y)" does not imply "x<=y". To '
+                  'automatically\n'
+                  '   generate ordering operations from a single root '
+                  'operation, see\n'
+                  '   "functools.total_ordering()".\n'
+                  '\n'
+                  '   See the paragraph on "__hash__()" for some important '
+                  'notes on\n'
+                  '   creating *hashable* objects which support custom '
+                  'comparison\n'
+                  '   operations and are usable as dictionary keys.\n'
+                  '\n'
+                  '   There are no swapped-argument versions of these methods '
+                  '(to be used\n'
+                  '   when the left argument does not support the operation '
+                  'but the right\n'
+                  '   argument does); rather, "__lt__()" and "__gt__()" are '
+                  "each other's\n"
+                  '   reflection, "__le__()" and "__ge__()" are each other\'s '
+                  'reflection,\n'
+                  '   and "__eq__()" and "__ne__()" are their own reflection. '
+                  'If the\n'
+                  "   operands are of different types, and right operand's "
+                  'type is a\n'
+                  "   direct or indirect subclass of the left operand's type, "
+                  'the\n'
+                  '   reflected method of the right operand has priority, '
+                  'otherwise the\n'
+                  "   left operand's method has priority.  Virtual subclassing "
+                  'is not\n'
+                  '   considered.\n'
+                  '\n'
+                  'object.__hash__(self)\n'
+                  '\n'
+                  '   Called by built-in function "hash()" and for operations '
+                  'on members\n'
+                  '   of hashed collections including "set", "frozenset", and '
+                  '"dict".\n'
+                  '   "__hash__()" should return an integer. The only required '
+                  'property\n'
+                  '   is that objects which compare equal have the same hash '
+                  'value; it is\n'
+                  '   advised to mix together the hash values of the '
+                  'components of the\n'
+                  '   object that also play a part in comparison of objects by '
+                  'packing\n'
+                  '   them into a tuple and hashing the tuple. Example:\n'
+                  '\n'
+                  '      def __hash__(self):\n'
+                  '          return hash((self.name, self.nick, self.color))\n'
+                  '\n'
+                  '   Note: "hash()" truncates the value returned from an '
+                  "object's\n"
+                  '     custom "__hash__()" method to the size of a '
+                  '"Py_ssize_t".  This\n'
+                  '     is typically 8 bytes on 64-bit builds and 4 bytes on '
+                  '32-bit\n'
+                  '     builds. If an object\'s   "__hash__()" must '
+                  'interoperate on builds\n'
+                  '     of different bit sizes, be sure to check the width on '
+                  'all\n'
+                  '     supported builds.  An easy way to do this is with '
+                  '"python -c\n'
+                  '     "import sys; print(sys.hash_info.width)"".\n'
+                  '\n'
+                  '   If a class does not define an "__eq__()" method it '
+                  'should not\n'
+                  '   define a "__hash__()" operation either; if it defines '
+                  '"__eq__()"\n'
+                  '   but not "__hash__()", its instances will not be usable '
+                  'as items in\n'
+                  '   hashable collections.  If a class defines mutable '
+                  'objects and\n'
+                  '   implements an "__eq__()" method, it should not '
+                  'implement\n'
+                  '   "__hash__()", since the implementation of hashable '
+                  'collections\n'
+                  "   requires that a key's hash value is immutable (if the "
+                  "object's hash\n"
+                  '   value changes, it will be in the wrong hash bucket).\n'
+                  '\n'
+                  '   User-defined classes have "__eq__()" and "__hash__()" '
+                  'methods by\n'
+                  '   default; with them, all objects compare unequal (except '
+                  'with\n'
+                  '   themselves) and "x.__hash__()" returns an appropriate '
+                  'value such\n'
+                  '   that "x == y" implies both that "x is y" and "hash(x) == '
+                  'hash(y)".\n'
+                  '\n'
+                  '   A class that overrides "__eq__()" and does not define '
+                  '"__hash__()"\n'
+                  '   will have its "__hash__()" implicitly set to "None".  '
+                  'When the\n'
+                  '   "__hash__()" method of a class is "None", instances of '
+                  'the class\n'
+                  '   will raise an appropriate "TypeError" when a program '
+                  'attempts to\n'
+                  '   retrieve their hash value, and will also be correctly '
+                  'identified as\n'
+                  '   unhashable when checking "isinstance(obj, '
+                  'collections.Hashable)".\n'
+                  '\n'
+                  '   If a class that overrides "__eq__()" needs to retain '
+                  'the\n'
+                  '   implementation of "__hash__()" from a parent class, the '
+                  'interpreter\n'
+                  '   must be told this explicitly by setting "__hash__ =\n'
+                  '   <ParentClass>.__hash__".\n'
+                  '\n'
+                  '   If a class that does not override "__eq__()" wishes to '
+                  'suppress\n'
+                  '   hash support, it should include "__hash__ = None" in the '
+                  'class\n'
+                  '   definition. A class which defines its own "__hash__()" '
+                  'that\n'
+                  '   explicitly raises a "TypeError" would be incorrectly '
+                  'identified as\n'
+                  '   hashable by an "isinstance(obj, collections.Hashable)" '
+                  'call.\n'
+                  '\n'
+                  '   Note: By default, the "__hash__()" values of str, bytes '
+                  'and\n'
+                  '     datetime objects are "salted" with an unpredictable '
+                  'random value.\n'
+                  '     Although they remain constant within an individual '
+                  'Python\n'
+                  '     process, they are not predictable between repeated '
+                  'invocations of\n'
+                  '     Python.This is intended to provide protection against '
+                  'a denial-\n'
+                  '     of-service caused by carefully-chosen inputs that '
+                  'exploit the\n'
+                  '     worst case performance of a dict insertion, O(n^2) '
+                  'complexity.\n'
+                  '     See '
+                  'http://www.ocert.org/advisories/ocert-2011-003.html for\n'
+                  '     details.Changing hash values affects the iteration '
+                  'order of\n'
+                  '     dicts, sets and other mappings.  Python has never made '
+                  'guarantees\n'
+                  '     about this ordering (and it typically varies between '
+                  '32-bit and\n'
+                  '     64-bit builds).See also "PYTHONHASHSEED".\n'
+                  '\n'
+                  '   Changed in version 3.3: Hash randomization is enabled by '
+                  'default.\n'
+                  '\n'
+                  'object.__bool__(self)\n'
+                  '\n'
+                  '   Called to implement truth value testing and the built-in '
+                  'operation\n'
+                  '   "bool()"; should return "False" or "True".  When this '
+                  'method is not\n'
+                  '   defined, "__len__()" is called, if it is defined, and '
+                  'the object is\n'
+                  '   considered true if its result is nonzero.  If a class '
+                  'defines\n'
+                  '   neither "__len__()" nor "__bool__()", all its instances '
+                  'are\n'
+                  '   considered true.\n',
+ 'debugger': '\n'
+             '"pdb" --- The Python Debugger\n'
+             '*****************************\n'
+             '\n'
+             '**Source code:** Lib/pdb.py\n'
+             '\n'
+             '======================================================================\n'
+             '\n'
+             'The module "pdb" defines an interactive source code debugger '
+             'for\n'
+             'Python programs.  It supports setting (conditional) breakpoints '
+             'and\n'
+             'single stepping at the source line level, inspection of stack '
+             'frames,\n'
+             'source code listing, and evaluation of arbitrary Python code in '
+             'the\n'
+             'context of any stack frame.  It also supports post-mortem '
+             'debugging\n'
+             'and can be called under program control.\n'
+             '\n'
+             'The debugger is extensible -- it is actually defined as the '
+             'class\n'
+             '"Pdb". This is currently undocumented but easily understood by '
+             'reading\n'
+             'the source.  The extension interface uses the modules "bdb" and '
+             '"cmd".\n'
+             '\n'
+             'The debugger\'s prompt is "(Pdb)". Typical usage to run a '
+             'program under\n'
+             'control of the debugger is:\n'
+             '\n'
+             '   >>> import pdb\n'
+             '   >>> import mymodule\n'
+             "   >>> pdb.run('mymodule.test()')\n"
+             '   > <string>(0)?()\n'
+             '   (Pdb) continue\n'
+             '   > <string>(1)?()\n'
+             '   (Pdb) continue\n'
+             "   NameError: 'spam'\n"
+             '   > <string>(1)?()\n'
+             '   (Pdb)\n'
+             '\n'
+             'Changed in version 3.3: Tab-completion via the "readline" module '
+             'is\n'
+             'available for commands and command arguments, e.g. the current '
+             'global\n'
+             'and local names are offered as arguments of the "p" command.\n'
+             '\n'
+             '"pdb.py" can also be invoked as a script to debug other '
+             'scripts.  For\n'
+             'example:\n'
+             '\n'
+             '   python3 -m pdb myscript.py\n'
+             '\n'
+             'When invoked as a script, pdb will automatically enter '
+             'post-mortem\n'
+             'debugging if the program being debugged exits abnormally.  After '
+             'post-\n'
+             'mortem debugging (or after normal exit of the program), pdb '
+             'will\n'
+             "restart the program.  Automatic restarting preserves pdb's state "
+             '(such\n'
+             'as breakpoints) and in most cases is more useful than quitting '
+             'the\n'
+             "debugger upon program's exit.\n"
+             '\n'
+             'New in version 3.2: "pdb.py" now accepts a "-c" option that '
+             'executes\n'
+             'commands as if given in a ".pdbrc" file, see *Debugger '
+             'Commands*.\n'
+             '\n'
+             'The typical usage to break into the debugger from a running '
+             'program is\n'
+             'to insert\n'
+             '\n'
+             '   import pdb; pdb.set_trace()\n'
+             '\n'
+             'at the location you want to break into the debugger.  You can '
+             'then\n'
+             'step through the code following this statement, and continue '
+             'running\n'
+             'without the debugger using the "continue" command.\n'
+             '\n'
+             'The typical usage to inspect a crashed program is:\n'
+             '\n'
+             '   >>> import pdb\n'
+             '   >>> import mymodule\n'
+             '   >>> mymodule.test()\n'
+             '   Traceback (most recent call last):\n'
+             '     File "<stdin>", line 1, in <module>\n'
+             '     File "./mymodule.py", line 4, in test\n'
+             '       test2()\n'
+             '     File "./mymodule.py", line 3, in test2\n'
+             '       print(spam)\n'
+             '   NameError: spam\n'
+             '   >>> pdb.pm()\n'
+             '   > ./mymodule.py(3)test2()\n'
+             '   -> print(spam)\n'
+             '   (Pdb)\n'
+             '\n'
+             'The module defines the following functions; each enters the '
+             'debugger\n'
+             'in a slightly different way:\n'
+             '\n'
+             'pdb.run(statement, globals=None, locals=None)\n'
+             '\n'
+             '   Execute the *statement* (given as a string or a code object) '
+             'under\n'
+             '   debugger control.  The debugger prompt appears before any '
+             'code is\n'
+             '   executed; you can set breakpoints and type "continue", or you '
+             'can\n'
+             '   step through the statement using "step" or "next" (all these\n'
+             '   commands are explained below).  The optional *globals* and '
+             '*locals*\n'
+             '   arguments specify the environment in which the code is '
+             'executed; by\n'
+             '   default the dictionary of the module "__main__" is used.  '
+             '(See the\n'
+             '   explanation of the built-in "exec()" or "eval()" functions.)\n'
+             '\n'
+             'pdb.runeval(expression, globals=None, locals=None)\n'
+             '\n'
+             '   Evaluate the *expression* (given as a string or a code '
+             'object)\n'
+             '   under debugger control.  When "runeval()" returns, it returns '
+             'the\n'
+             '   value of the expression.  Otherwise this function is similar '
+             'to\n'
+             '   "run()".\n'
+             '\n'
+             'pdb.runcall(function, *args, **kwds)\n'
+             '\n'
+             '   Call the *function* (a function or method object, not a '
+             'string)\n'
+             '   with the given arguments.  When "runcall()" returns, it '
+             'returns\n'
+             '   whatever the function call returned.  The debugger prompt '
+             'appears\n'
+             '   as soon as the function is entered.\n'
+             '\n'
+             'pdb.set_trace()\n'
+             '\n'
+             '   Enter the debugger at the calling stack frame.  This is '
+             'useful to\n'
+             '   hard-code a breakpoint at a given point in a program, even if '
+             'the\n'
+             '   code is not otherwise being debugged (e.g. when an assertion\n'
+             '   fails).\n'
+             '\n'
+             'pdb.post_mortem(traceback=None)\n'
+             '\n'
+             '   Enter post-mortem debugging of the given *traceback* object.  '
+             'If no\n'
+             '   *traceback* is given, it uses the one of the exception that '
+             'is\n'
+             '   currently being handled (an exception must be being handled '
+             'if the\n'
+             '   default is to be used).\n'
+             '\n'
+             'pdb.pm()\n'
+             '\n'
+             '   Enter post-mortem debugging of the traceback found in\n'
+             '   "sys.last_traceback".\n'
+             '\n'
+             'The "run*" functions and "set_trace()" are aliases for '
+             'instantiating\n'
+             'the "Pdb" class and calling the method of the same name.  If you '
+             'want\n'
+             'to access further features, you have to do this yourself:\n'
+             '\n'
+             "class class pdb.Pdb(completekey='tab', stdin=None, stdout=None, "
+             'skip=None, nosigint=False)\n'
+             '\n'
+             '   "Pdb" is the debugger class.\n'
+             '\n'
+             '   The *completekey*, *stdin* and *stdout* arguments are passed '
+             'to the\n'
+             '   underlying "cmd.Cmd" class; see the description there.\n'
+             '\n'
+             '   The *skip* argument, if given, must be an iterable of '
+             'glob-style\n'
+             '   module name patterns.  The debugger will not step into frames '
+             'that\n'
+             '   originate in a module that matches one of these patterns. '
+             '[1]\n'
+             '\n'
+             '   By default, Pdb sets a handler for the SIGINT signal (which '
+             'is sent\n'
+             '   when the user presses "Ctrl-C" on the console) when you give '
+             'a\n'
+             '   "continue" command. This allows you to break into the '
+             'debugger\n'
+             '   again by pressing "Ctrl-C".  If you want Pdb not to touch '
+             'the\n'
+             '   SIGINT handler, set *nosigint* to true.\n'
+             '\n'
+             '   Example call to enable tracing with *skip*:\n'
+             '\n'
+             "      import pdb; pdb.Pdb(skip=['django.*']).set_trace()\n"
+             '\n'
+             '   New in version 3.1: The *skip* argument.\n'
+             '\n'
+             '   New in version 3.2: The *nosigint* argument.  Previously, a '
+             'SIGINT\n'
+             '   handler was never set by Pdb.\n'
+             '\n'
+             '   run(statement, globals=None, locals=None)\n'
+             '   runeval(expression, globals=None, locals=None)\n'
+             '   runcall(function, *args, **kwds)\n'
+             '   set_trace()\n'
+             '\n'
+             '      See the documentation for the functions explained above.\n'
+             '\n'
+             '\n'
+             'Debugger Commands\n'
+             '=================\n'
+             '\n'
+             'The commands recognized by the debugger are listed below.  Most\n'
+             'commands can be abbreviated to one or two letters as indicated; '
+             'e.g.\n'
+             '"h(elp)" means that either "h" or "help" can be used to enter '
+             'the help\n'
+             'command (but not "he" or "hel", nor "H" or "Help" or "HELP").\n'
+             'Arguments to commands must be separated by whitespace (spaces '
+             'or\n'
+             'tabs).  Optional arguments are enclosed in square brackets '
+             '("[]") in\n'
+             'the command syntax; the square brackets must not be typed.\n'
+             'Alternatives in the command syntax are separated by a vertical '
+             'bar\n'
+             '("|").\n'
+             '\n'
+             'Entering a blank line repeats the last command entered.  '
+             'Exception: if\n'
+             'the last command was a "list" command, the next 11 lines are '
+             'listed.\n'
+             '\n'
+             "Commands that the debugger doesn't recognize are assumed to be "
+             'Python\n'
+             'statements and are executed in the context of the program being\n'
+             'debugged.  Python statements can also be prefixed with an '
+             'exclamation\n'
+             'point ("!").  This is a powerful way to inspect the program '
+             'being\n'
+             'debugged; it is even possible to change a variable or call a '
+             'function.\n'
+             'When an exception occurs in such a statement, the exception name '
+             'is\n'
+             "printed but the debugger's state is not changed.\n"
+             '\n'
+             'The debugger supports *aliases*.  Aliases can have parameters '
+             'which\n'
+             'allows one a certain level of adaptability to the context under\n'
+             'examination.\n'
+             '\n'
+             'Multiple commands may be entered on a single line, separated by '
+             '";;".\n'
+             '(A single ";" is not used as it is the separator for multiple '
+             'commands\n'
+             'in a line that is passed to the Python parser.)  No intelligence '
+             'is\n'
+             'applied to separating the commands; the input is split at the '
+             'first\n'
+             '";;" pair, even if it is in the middle of a quoted string.\n'
+             '\n'
+             'If a file ".pdbrc" exists in the user\'s home directory or in '
+             'the\n'
+             'current directory, it is read in and executed as if it had been '
+             'typed\n'
+             'at the debugger prompt.  This is particularly useful for '
+             'aliases.  If\n'
+             'both files exist, the one in the home directory is read first '
+             'and\n'
+             'aliases defined there can be overridden by the local file.\n'
+             '\n'
+             'Changed in version 3.2: ".pdbrc" can now contain commands that\n'
+             'continue debugging, such as "continue" or "next".  Previously, '
+             'these\n'
+             'commands had no effect.\n'
+             '\n'
+             'h(elp) [command]\n'
+             '\n'
+             '   Without argument, print the list of available commands.  With '
+             'a\n'
+             '   *command* as argument, print help about that command.  "help '
+             'pdb"\n'
+             '   displays the full documentation (the docstring of the "pdb"\n'
+             '   module).  Since the *command* argument must be an identifier, '
+             '"help\n'
+             '   exec" must be entered to get help on the "!" command.\n'
+             '\n'
+             'w(here)\n'
+             '\n'
+             '   Print a stack trace, with the most recent frame at the '
+             'bottom.  An\n'
+             '   arrow indicates the current frame, which determines the '
+             'context of\n'
+             '   most commands.\n'
+             '\n'
+             'd(own) [count]\n'
+             '\n'
+             '   Move the current frame *count* (default one) levels down in '
+             'the\n'
+             '   stack trace (to a newer frame).\n'
+             '\n'
+             'u(p) [count]\n'
+             '\n'
+             '   Move the current frame *count* (default one) levels up in the '
+             'stack\n'
+             '   trace (to an older frame).\n'
+             '\n'
+             'b(reak) [([filename:]lineno | function) [, condition]]\n'
+             '\n'
+             '   With a *lineno* argument, set a break there in the current '
+             'file.\n'
+             '   With a *function* argument, set a break at the first '
+             'executable\n'
+             '   statement within that function.  The line number may be '
+             'prefixed\n'
+             '   with a filename and a colon, to specify a breakpoint in '
+             'another\n'
+             "   file (probably one that hasn't been loaded yet).  The file "
+             'is\n'
+             '   searched on "sys.path".  Note that each breakpoint is '
+             'assigned a\n'
+             '   number to which all the other breakpoint commands refer.\n'
+             '\n'
+             '   If a second argument is present, it is an expression which '
+             'must\n'
+             '   evaluate to true before the breakpoint is honored.\n'
+             '\n'
+             '   Without argument, list all breaks, including for each '
+             'breakpoint,\n'
+             '   the number of times that breakpoint has been hit, the '
+             'current\n'
+             '   ignore count, and the associated condition if any.\n'
+             '\n'
+             'tbreak [([filename:]lineno | function) [, condition]]\n'
+             '\n'
+             '   Temporary breakpoint, which is removed automatically when it '
+             'is\n'
+             '   first hit. The arguments are the same as for "break".\n'
+             '\n'
+             'cl(ear) [filename:lineno | bpnumber [bpnumber ...]]\n'
+             '\n'
+             '   With a *filename:lineno* argument, clear all the breakpoints '
+             'at\n'
+             '   this line. With a space separated list of breakpoint numbers, '
+             'clear\n'
+             '   those breakpoints. Without argument, clear all breaks (but '
+             'first\n'
+             '   ask confirmation).\n'
+             '\n'
+             'disable [bpnumber [bpnumber ...]]\n'
+             '\n'
+             '   Disable the breakpoints given as a space separated list of\n'
+             '   breakpoint numbers.  Disabling a breakpoint means it cannot '
+             'cause\n'
+             '   the program to stop execution, but unlike clearing a '
+             'breakpoint, it\n'
+             '   remains in the list of breakpoints and can be (re-)enabled.\n'
+             '\n'
+             'enable [bpnumber [bpnumber ...]]\n'
+             '\n'
+             '   Enable the breakpoints specified.\n'
+             '\n'
+             'ignore bpnumber [count]\n'
+             '\n'
+             '   Set the ignore count for the given breakpoint number.  If '
+             'count is\n'
+             '   omitted, the ignore count is set to 0.  A breakpoint becomes '
+             'active\n'
+             '   when the ignore count is zero.  When non-zero, the count is\n'
+             '   decremented each time the breakpoint is reached and the '
+             'breakpoint\n'
+             '   is not disabled and any associated condition evaluates to '
+             'true.\n'
+             '\n'
+             'condition bpnumber [condition]\n'
+             '\n'
+             '   Set a new *condition* for the breakpoint, an expression which '
+             'must\n'
+             '   evaluate to true before the breakpoint is honored.  If '
+             '*condition*\n'
+             '   is absent, any existing condition is removed; i.e., the '
+             'breakpoint\n'
+             '   is made unconditional.\n'
+             '\n'
+             'commands [bpnumber]\n'
+             '\n'
+             '   Specify a list of commands for breakpoint number *bpnumber*.  '
+             'The\n'
+             '   commands themselves appear on the following lines.  Type a '
+             'line\n'
+             '   containing just "end" to terminate the commands. An example:\n'
+             '\n'
+             '      (Pdb) commands 1\n'
+             '      (com) p some_variable\n'
+             '      (com) end\n'
+             '      (Pdb)\n'
+             '\n'
+             '   To remove all commands from a breakpoint, type commands and '
+             'follow\n'
+             '   it immediately with "end"; that is, give no commands.\n'
+             '\n'
+             '   With no *bpnumber* argument, commands refers to the last '
+             'breakpoint\n'
+             '   set.\n'
+             '\n'
+             '   You can use breakpoint commands to start your program up '
+             'again.\n'
+             '   Simply use the continue command, or step, or any other '
+             'command that\n'
+             '   resumes execution.\n'
+             '\n'
+             '   Specifying any command resuming execution (currently '
+             'continue,\n'
+             '   step, next, return, jump, quit and their abbreviations) '
+             'terminates\n'
+             '   the command list (as if that command was immediately followed '
+             'by\n'
+             '   end). This is because any time you resume execution (even '
+             'with a\n'
+             '   simple next or step), you may encounter another '
+             'breakpoint?which\n'
+             '   could have its own command list, leading to ambiguities about '
+             'which\n'
+             '   list to execute.\n'
+             '\n'
+             "   If you use the 'silent' command in the command list, the "
+             'usual\n'
+             '   message about stopping at a breakpoint is not printed.  This '
+             'may be\n'
+             '   desirable for breakpoints that are to print a specific '
+             'message and\n'
+             '   then continue.  If none of the other commands print anything, '
+             'you\n'
+             '   see no sign that the breakpoint was reached.\n'
+             '\n'
+             's(tep)\n'
+             '\n'
+             '   Execute the current line, stop at the first possible '
+             'occasion\n'
+             '   (either in a function that is called or on the next line in '
+             'the\n'
+             '   current function).\n'
+             '\n'
+             'n(ext)\n'
+             '\n'
+             '   Continue execution until the next line in the current '
+             'function is\n'
+             '   reached or it returns.  (The difference between "next" and '
+             '"step"\n'
+             '   is that "step" stops inside a called function, while "next"\n'
+             '   executes called functions at (nearly) full speed, only '
+             'stopping at\n'
+             '   the next line in the current function.)\n'
+             '\n'
+             'unt(il) [lineno]\n'
+             '\n'
+             '   Without argument, continue execution until the line with a '
+             'number\n'
+             '   greater than the current one is reached.\n'
+             '\n'
+             '   With a line number, continue execution until a line with a '
+             'number\n'
+             '   greater or equal to that is reached.  In both cases, also '
+             'stop when\n'
+             '   the current frame returns.\n'
+             '\n'
+             '   Changed in version 3.2: Allow giving an explicit line '
+             'number.\n'
+             '\n'
+             'r(eturn)\n'
+             '\n'
+             '   Continue execution until the current function returns.\n'
+             '\n'
+             'c(ont(inue))\n'
+             '\n'
+             '   Continue execution, only stop when a breakpoint is '
+             'encountered.\n'
+             '\n'
+             'j(ump) lineno\n'
+             '\n'
+             '   Set the next line that will be executed.  Only available in '
+             'the\n'
+             '   bottom-most frame.  This lets you jump back and execute code '
+             'again,\n'
+             "   or jump forward to skip code that you don't want to run.\n"
+             '\n'
+             '   It should be noted that not all jumps are allowed -- for '
+             'instance\n'
+             '   it is not possible to jump into the middle of a "for" loop or '
+             'out\n'
+             '   of a "finally" clause.\n'
+             '\n'
+             'l(ist) [first[, last]]\n'
+             '\n'
+             '   List source code for the current file.  Without arguments, '
+             'list 11\n'
+             '   lines around the current line or continue the previous '
+             'listing.\n'
+             '   With "." as argument, list 11 lines around the current line.  '
+             'With\n'
+             '   one argument, list 11 lines around at that line.  With two\n'
+             '   arguments, list the given range; if the second argument is '
+             'less\n'
+             '   than the first, it is interpreted as a count.\n'
+             '\n'
+             '   The current line in the current frame is indicated by "->".  '
+             'If an\n'
+             '   exception is being debugged, the line where the exception '
+             'was\n'
+             '   originally raised or propagated is indicated by ">>", if it '
+             'differs\n'
+             '   from the current line.\n'
+             '\n'
+             '   New in version 3.2: The ">>" marker.\n'
+             '\n'
+             'll | longlist\n'
+             '\n'
+             '   List all source code for the current function or frame.\n'
+             '   Interesting lines are marked as for "list".\n'
+             '\n'
+             '   New in version 3.2.\n'
+             '\n'
+             'a(rgs)\n'
+             '\n'
+             '   Print the argument list of the current function.\n'
+             '\n'
+             'p expression\n'
+             '\n'
+             '   Evaluate the *expression* in the current context and print '
+             'its\n'
+             '   value.\n'
+             '\n'
+             '   Note: "print()" can also be used, but is not a debugger '
+             'command\n'
+             '     --- this executes the Python "print()" function.\n'
+             '\n'
+             'pp expression\n'
+             '\n'
+             '   Like the "p" command, except the value of the expression is '
+             'pretty-\n'
+             '   printed using the "pprint" module.\n'
+             '\n'
+             'whatis expression\n'
+             '\n'
+             '   Print the type of the *expression*.\n'
+             '\n'
+             'source expression\n'
+             '\n'
+             '   Try to get source code for the given object and display it.\n'
+             '\n'
+             '   New in version 3.2.\n'
+             '\n'
+             'display [expression]\n'
+             '\n'
+             '   Display the value of the expression if it changed, each time\n'
+             '   execution stops in the current frame.\n'
+             '\n'
+             '   Without expression, list all display expressions for the '
+             'current\n'
+             '   frame.\n'
+             '\n'
+             '   New in version 3.2.\n'
+             '\n'
+             'undisplay [expression]\n'
+             '\n'
+             '   Do not display the expression any more in the current frame.\n'
+             '   Without expression, clear all display expressions for the '
+             'current\n'
+             '   frame.\n'
+             '\n'
+             '   New in version 3.2.\n'
+             '\n'
+             'interact\n'
+             '\n'
+             '   Start an interactive interpreter (using the "code" module) '
+             'whose\n'
+             '   global namespace contains all the (global and local) names '
+             'found in\n'
+             '   the current scope.\n'
+             '\n'
+             '   New in version 3.2.\n'
+             '\n'
+             'alias [name [command]]\n'
+             '\n'
+             '   Create an alias called *name* that executes *command*.  The '
+             'command\n'
+             '   must *not* be enclosed in quotes.  Replaceable parameters can '
+             'be\n'
+             '   indicated by "%1", "%2", and so on, while "%*" is replaced by '
+             'all\n'
+             '   the parameters. If no command is given, the current alias '
+             'for\n'
+             '   *name* is shown. If no arguments are given, all aliases are '
+             'listed.\n'
+             '\n'
+             '   Aliases may be nested and can contain anything that can be '
+             'legally\n'
+             '   typed at the pdb prompt.  Note that internal pdb commands '
+             '*can* be\n'
+             '   overridden by aliases.  Such a command is then hidden until '
+             'the\n'
+             '   alias is removed.  Aliasing is recursively applied to the '
+             'first\n'
+             '   word of the command line; all other words in the line are '
+             'left\n'
+             '   alone.\n'
+             '\n'
+             '   As an example, here are two useful aliases (especially when '
+             'placed\n'
+             '   in the ".pdbrc" file):\n'
+             '\n'
+             '      # Print instance variables (usage "pi classInst")\n'
+             '      alias pi for k in %1.__dict__.keys(): '
+             'print("%1.",k,"=",%1.__dict__[k])\n'
+             '      # Print instance variables in self\n'
+             '      alias ps pi self\n'
+             '\n'
+             'unalias name\n'
+             '\n'
+             '   Delete the specified alias.\n'
+             '\n'
+             '! statement\n'
+             '\n'
+             '   Execute the (one-line) *statement* in the context of the '
+             'current\n'
+             '   stack frame. The exclamation point can be omitted unless the '
+             'first\n'
+             '   word of the statement resembles a debugger command.  To set '
+             'a\n'
+             '   global variable, you can prefix the assignment command with '
+             'a\n'
+             '   "global" statement on the same line, e.g.:\n'
+             '\n'
+             "      (Pdb) global list_options; list_options = ['-l']\n"
+             '      (Pdb)\n'
+             '\n'
+             'run [args ...]\n'
+             'restart [args ...]\n'
+             '\n'
+             '   Restart the debugged Python program.  If an argument is '
+             'supplied,\n'
+             '   it is split with "shlex" and the result is used as the new\n'
+             '   "sys.argv". History, breakpoints, actions and debugger '
+             'options are\n'
+             '   preserved. "restart" is an alias for "run".\n'
+             '\n'
+             'q(uit)\n'
+             '\n'
+             '   Quit from the debugger.  The program being executed is '
+             'aborted.\n'
+             '\n'
+             '-[ Footnotes ]-\n'
+             '\n'
+             '[1] Whether a frame is considered to originate in a certain '
+             'module\n'
+             '    is determined by the "__name__" in the frame globals.\n',
+ 'del': '\n'
+        'The "del" statement\n'
+        '*******************\n'
+        '\n'
+        '   del_stmt ::= "del" target_list\n'
+        '\n'
+        'Deletion is recursively defined very similar to the way assignment '
+        'is\n'
+        'defined. Rather than spelling it out in full details, here are some\n'
+        'hints.\n'
+        '\n'
+        'Deletion of a target list recursively deletes each target, from left\n'
+        'to right.\n'
+        '\n'
+        'Deletion of a name removes the binding of that name from the local '
+        'or\n'
+        'global namespace, depending on whether the name occurs in a "global"\n'
+        'statement in the same code block.  If the name is unbound, a\n'
+        '"NameError" exception will be raised.\n'
+        '\n'
+        'Deletion of attribute references, subscriptions and slicings is '
+        'passed\n'
+        'to the primary object involved; deletion of a slicing is in general\n'
+        'equivalent to assignment of an empty slice of the right type (but '
+        'even\n'
+        'this is determined by the sliced object).\n'
+        '\n'
+        'Changed in version 3.2: Previously it was illegal to delete a name\n'
+        'from the local namespace if it occurs as a free variable in a nested\n'
+        'block.\n',
+ 'dict': '\n'
+         'Dictionary displays\n'
+         '*******************\n'
+         '\n'
+         'A dictionary display is a possibly empty series of key/datum pairs\n'
+         'enclosed in curly braces:\n'
+         '\n'
+         '   dict_display       ::= "{" [key_datum_list | dict_comprehension] '
+         '"}"\n'
+         '   key_datum_list     ::= key_datum ("," key_datum)* [","]\n'
+         '   key_datum          ::= expression ":" expression | "**" or_expr\n'
+         '   dict_comprehension ::= expression ":" expression comp_for\n'
+         '\n'
+         'A dictionary display yields a new dictionary object.\n'
+         '\n'
+         'If a comma-separated sequence of key/datum pairs is given, they are\n'
+         'evaluated from left to right to define the entries of the '
+         'dictionary:\n'
+         'each key object is used as a key into the dictionary to store the\n'
+         'corresponding datum.  This means that you can specify the same key\n'
+         "multiple times in the key/datum list, and the final dictionary's "
+         'value\n'
+         'for that key will be the last one given.\n'
+         '\n'
+         'A double asterisk "**" denotes *dictionary unpacking*. Its operand\n'
+         'must be a *mapping*.  Each mapping item is added to the new\n'
+         'dictionary.  Later values replace values already set by earlier\n'
+         'key/datum pairs and earlier dictionary unpackings.\n'
+         '\n'
+         'New in version 3.5: Unpacking into dictionary displays, originally\n'
+         'proposed by **PEP 448**.\n'
+         '\n'
+         'A dict comprehension, in contrast to list and set comprehensions,\n'
+         'needs two expressions separated with a colon followed by the usual\n'
+         '"for" and "if" clauses. When the comprehension is run, the '
+         'resulting\n'
+         'key and value elements are inserted in the new dictionary in the '
+         'order\n'
+         'they are produced.\n'
+         '\n'
+         'Restrictions on the types of the key values are listed earlier in\n'
+         'section *The standard type hierarchy*.  (To summarize, the key type\n'
+         'should be *hashable*, which excludes all mutable objects.)  Clashes\n'
+         'between duplicate keys are not detected; the last datum (textually\n'
+         'rightmost in the display) stored for a given key value prevails.\n',
+ 'dynamic-features': '\n'
+                     'Interaction with dynamic features\n'
+                     '*********************************\n'
+                     '\n'
+                     'Name resolution of free variables occurs at runtime, not '
+                     'at compile\n'
+                     'time. This means that the following code will print 42:\n'
+                     '\n'
+                     '   i = 10\n'
+                     '   def f():\n'
+                     '       print(i)\n'
+                     '   i = 42\n'
+                     '   f()\n'
+                     '\n'
+                     'The "eval()" and "exec()" functions do not have access '
+                     'to the full\n'
+                     'environment for resolving names.  Names may be resolved '
+                     'in the local\n'
+                     'and global namespaces of the caller.  Free variables are '
+                     'not resolved\n'
+                     'in the nearest enclosing namespace, but in the global '
+                     'namespace.  [1]\n'
+                     'The "exec()" and "eval()" functions have optional '
+                     'arguments to\n'
+                     'override the global and local namespace.  If only one '
+                     'namespace is\n'
+                     'specified, it is used for both.\n',
+ 'else': '\n'
+         'The "if" statement\n'
+         '******************\n'
+         '\n'
+         'The "if" statement is used for conditional execution:\n'
+         '\n'
+         '   if_stmt ::= "if" expression ":" suite\n'
+         '               ( "elif" expression ":" suite )*\n'
+         '               ["else" ":" suite]\n'
+         '\n'
+         'It selects exactly one of the suites by evaluating the expressions '
+         'one\n'
+         'by one until one is found to be true (see section *Boolean '
+         'operations*\n'
+         'for the definition of true and false); then that suite is executed\n'
+         '(and no other part of the "if" statement is executed or evaluated).\n'
+         'If all expressions are false, the suite of the "else" clause, if\n'
+         'present, is executed.\n',
+ 'exceptions': '\n'
+               'Exceptions\n'
+               '**********\n'
+               '\n'
+               'Exceptions are a means of breaking out of the normal flow of '
+               'control\n'
+               'of a code block in order to handle errors or other '
+               'exceptional\n'
+               'conditions.  An exception is *raised* at the point where the '
+               'error is\n'
+               'detected; it may be *handled* by the surrounding code block or '
+               'by any\n'
+               'code block that directly or indirectly invoked the code block '
+               'where\n'
+               'the error occurred.\n'
+               '\n'
+               'The Python interpreter raises an exception when it detects a '
+               'run-time\n'
+               'error (such as division by zero).  A Python program can also\n'
+               'explicitly raise an exception with the "raise" statement. '
+               'Exception\n'
+               'handlers are specified with the "try" ... "except" statement.  '
+               'The\n'
+               '"finally" clause of such a statement can be used to specify '
+               'cleanup\n'
+               'code which does not handle the exception, but is executed '
+               'whether an\n'
+               'exception occurred or not in the preceding code.\n'
+               '\n'
+               'Python uses the "termination" model of error handling: an '
+               'exception\n'
+               'handler can find out what happened and continue execution at '
+               'an outer\n'
+               'level, but it cannot repair the cause of the error and retry '
+               'the\n'
+               'failing operation (except by re-entering the offending piece '
+               'of code\n'
+               'from the top).\n'
+               '\n'
+               'When an exception is not handled at all, the interpreter '
+               'terminates\n'
+               'execution of the program, or returns to its interactive main '
+               'loop.  In\n'
+               'either case, it prints a stack backtrace, except when the '
+               'exception is\n'
+               '"SystemExit".\n'
+               '\n'
+               'Exceptions are identified by class instances.  The "except" '
+               'clause is\n'
+               'selected depending on the class of the instance: it must '
+               'reference the\n'
+               'class of the instance or a base class thereof.  The instance '
+               'can be\n'
+               'received by the handler and can carry additional information '
+               'about the\n'
+               'exceptional condition.\n'
+               '\n'
+               'Note: Exception messages are not part of the Python API.  '
+               'Their\n'
+               '  contents may change from one version of Python to the next '
+               'without\n'
+               '  warning and should not be relied on by code which will run '
+               'under\n'
+               '  multiple versions of the interpreter.\n'
+               '\n'
+               'See also the description of the "try" statement in section '
+               '*The try\n'
+               'statement* and "raise" statement in section *The raise '
+               'statement*.\n'
+               '\n'
+               '-[ Footnotes ]-\n'
+               '\n'
+               '[1] This limitation occurs because the code that is executed '
+               'by\n'
+               '    these operations is not available at the time the module '
+               'is\n'
+               '    compiled.\n',
+ 'execmodel': '\n'
+              'Execution model\n'
+              '***************\n'
+              '\n'
+              '\n'
+              'Structure of a program\n'
+              '======================\n'
+              '\n'
+              'A Python program is constructed from code blocks. A *block* is '
+              'a piece\n'
+              'of Python program text that is executed as a unit. The '
+              'following are\n'
+              'blocks: a module, a function body, and a class definition. '
+              'Each\n'
+              'command typed interactively is a block.  A script file (a file '
+              'given\n'
+              'as standard input to the interpreter or specified as a command '
+              'line\n'
+              'argument to the interpreter) is a code block.  A script command '
+              '(a\n'
+              'command specified on the interpreter command line with the '
+              "'**-c**'\n"
+              'option) is a code block.  The string argument passed to the '
+              'built-in\n'
+              'functions "eval()" and "exec()" is a code block.\n'
+              '\n'
+              'A code block is executed in an *execution frame*.  A frame '
+              'contains\n'
+              'some administrative information (used for debugging) and '
+              'determines\n'
+              "where and how execution continues after the code block's "
+              'execution has\n'
+              'completed.\n'
+              '\n'
+              '\n'
+              'Naming and binding\n'
+              '==================\n'
+              '\n'
+              '\n'
+              'Binding of names\n'
+              '----------------\n'
+              '\n'
+              '*Names* refer to objects.  Names are introduced by name '
+              'binding\n'
+              'operations.\n'
+              '\n'
+              'The following constructs bind names: formal parameters to '
+              'functions,\n'
+              '"import" statements, class and function definitions (these bind '
+              'the\n'
+              'class or function name in the defining block), and targets that '
+              'are\n'
+              'identifiers if occurring in an assignment, "for" loop header, '
+              'or after\n'
+              '"as" in a "with" statement or "except" clause. The "import" '
+              'statement\n'
+              'of the form "from ... import *" binds all names defined in the\n'
+              'imported module, except those beginning with an underscore.  '
+              'This form\n'
+              'may only be used at the module level.\n'
+              '\n'
+              'A target occurring in a "del" statement is also considered '
+              'bound for\n'
+              'this purpose (though the actual semantics are to unbind the '
+              'name).\n'
+              '\n'
+              'Each assignment or import statement occurs within a block '
+              'defined by a\n'
+              'class or function definition or at the module level (the '
+              'top-level\n'
+              'code block).\n'
+              '\n'
+              'If a name is bound in a block, it is a local variable of that '
+              'block,\n'
+              'unless declared as "nonlocal" or "global".  If a name is bound '
+              'at the\n'
+              'module level, it is a global variable.  (The variables of the '
+              'module\n'
+              'code block are local and global.)  If a variable is used in a '
+              'code\n'
+              'block but not defined there, it is a *free variable*.\n'
+              '\n'
+              'Each occurrence of a name in the program text refers to the '
+              '*binding*\n'
+              'of that name established by the following name resolution '
+              'rules.\n'
+              '\n'
+              '\n'
+              'Resolution of names\n'
+              '-------------------\n'
+              '\n'
+              'A *scope* defines the visibility of a name within a block.  If '
+              'a local\n'
+              'variable is defined in a block, its scope includes that block.  '
+              'If the\n'
+              'definition occurs in a function block, the scope extends to any '
+              'blocks\n'
+              'contained within the defining one, unless a contained block '
+              'introduces\n'
+              'a different binding for the name.\n'
+              '\n'
+              'When a name is used in a code block, it is resolved using the '
+              'nearest\n'
+              'enclosing scope.  The set of all such scopes visible to a code '
+              'block\n'
+              "is called the block's *environment*.\n"
+              '\n'
+              'When a name is not found at all, a "NameError" exception is '
+              'raised. If\n'
+              'the current scope is a function scope, and the name refers to a '
+              'local\n'
+              'variable that has not yet been bound to a value at the point '
+              'where the\n'
+              'name is used, an "UnboundLocalError" exception is raised.\n'
+              '"UnboundLocalError" is a subclass of "NameError".\n'
+              '\n'
+              'If a name binding operation occurs anywhere within a code '
+              'block, all\n'
+              'uses of the name within the block are treated as references to '
+              'the\n'
+              'current block.  This can lead to errors when a name is used '
+              'within a\n'
+              'block before it is bound.  This rule is subtle.  Python lacks\n'
+              'declarations and allows name binding operations to occur '
+              'anywhere\n'
+              'within a code block.  The local variables of a code block can '
+              'be\n'
+              'determined by scanning the entire text of the block for name '
+              'binding\n'
+              'operations.\n'
+              '\n'
+              'If the "global" statement occurs within a block, all uses of '
+              'the name\n'
+              'specified in the statement refer to the binding of that name in '
+              'the\n'
+              'top-level namespace.  Names are resolved in the top-level '
+              'namespace by\n'
+              'searching the global namespace, i.e. the namespace of the '
+              'module\n'
+              'containing the code block, and the builtins namespace, the '
+              'namespace\n'
+              'of the module "builtins".  The global namespace is searched '
+              'first.  If\n'
+              'the name is not found there, the builtins namespace is '
+              'searched.  The\n'
+              '"global" statement must precede all uses of the name.\n'
+              '\n'
+              'The "global" statement has the same scope as a name binding '
+              'operation\n'
+              'in the same block.  If the nearest enclosing scope for a free '
+              'variable\n'
+              'contains a global statement, the free variable is treated as a '
+              'global.\n'
+              '\n'
+              'The "nonlocal" statement causes corresponding names to refer '
+              'to\n'
+              'previously bound variables in the nearest enclosing function '
+              'scope.\n'
+              '"SyntaxError" is raised at compile time if the given name does '
+              'not\n'
+              'exist in any enclosing function scope.\n'
+              '\n'
+              'The namespace for a module is automatically created the first '
+              'time a\n'
+              'module is imported.  The main module for a script is always '
+              'called\n'
+              '"__main__".\n'
+              '\n'
+              'Class definition blocks and arguments to "exec()" and "eval()" '
+              'are\n'
+              'special in the context of name resolution. A class definition '
+              'is an\n'
+              'executable statement that may use and define names. These '
+              'references\n'
+              'follow the normal rules for name resolution with an exception '
+              'that\n'
+              'unbound local variables are looked up in the global namespace. '
+              'The\n'
+              'namespace of the class definition becomes the attribute '
+              'dictionary of\n'
+              'the class. The scope of names defined in a class block is '
+              'limited to\n'
+              'the class block; it does not extend to the code blocks of '
+              'methods --\n'
+              'this includes comprehensions and generator expressions since '
+              'they are\n'
+              'implemented using a function scope.  This means that the '
+              'following\n'
+              'will fail:\n'
+              '\n'
+              '   class A:\n'
+              '       a = 42\n'
+              '       b = list(a + i for i in range(10))\n'
+              '\n'
+              '\n'
+              'Builtins and restricted execution\n'
+              '---------------------------------\n'
+              '\n'
+              'The builtins namespace associated with the execution of a code '
+              'block\n'
+              'is actually found by looking up the name "__builtins__" in its '
+              'global\n'
+              'namespace; this should be a dictionary or a module (in the '
+              'latter case\n'
+              "the module's dictionary is used).  By default, when in the "
+              '"__main__"\n'
+              'module, "__builtins__" is the built-in module "builtins"; when '
+              'in any\n'
+              'other module, "__builtins__" is an alias for the dictionary of '
+              'the\n'
+              '"builtins" module itself.  "__builtins__" can be set to a '
+              'user-created\n'
+              'dictionary to create a weak form of restricted execution.\n'
+              '\n'
+              '**CPython implementation detail:** Users should not touch\n'
+              '"__builtins__"; it is strictly an implementation detail.  '
+              'Users\n'
+              'wanting to override values in the builtins namespace should '
+              '"import"\n'
+              'the "builtins" module and modify its attributes appropriately.\n'
+              '\n'
+              '\n'
+              'Interaction with dynamic features\n'
+              '---------------------------------\n'
+              '\n'
+              'Name resolution of free variables occurs at runtime, not at '
+              'compile\n'
+              'time. This means that the following code will print 42:\n'
+              '\n'
+              '   i = 10\n'
+              '   def f():\n'
+              '       print(i)\n'
+              '   i = 42\n'
+              '   f()\n'
+              '\n'
+              'The "eval()" and "exec()" functions do not have access to the '
+              'full\n'
+              'environment for resolving names.  Names may be resolved in the '
+              'local\n'
+              'and global namespaces of the caller.  Free variables are not '
+              'resolved\n'
+              'in the nearest enclosing namespace, but in the global '
+              'namespace.  [1]\n'
+              'The "exec()" and "eval()" functions have optional arguments to\n'
+              'override the global and local namespace.  If only one namespace '
+              'is\n'
+              'specified, it is used for both.\n'
+              '\n'
+              '\n'
+              'Exceptions\n'
+              '==========\n'
+              '\n'
+              'Exceptions are a means of breaking out of the normal flow of '
+              'control\n'
+              'of a code block in order to handle errors or other exceptional\n'
+              'conditions.  An exception is *raised* at the point where the '
+              'error is\n'
+              'detected; it may be *handled* by the surrounding code block or '
+              'by any\n'
+              'code block that directly or indirectly invoked the code block '
+              'where\n'
+              'the error occurred.\n'
+              '\n'
+              'The Python interpreter raises an exception when it detects a '
+              'run-time\n'
+              'error (such as division by zero).  A Python program can also\n'
+              'explicitly raise an exception with the "raise" statement. '
+              'Exception\n'
+              'handlers are specified with the "try" ... "except" statement.  '
+              'The\n'
+              '"finally" clause of such a statement can be used to specify '
+              'cleanup\n'
+              'code which does not handle the exception, but is executed '
+              'whether an\n'
+              'exception occurred or not in the preceding code.\n'
+              '\n'
+              'Python uses the "termination" model of error handling: an '
+              'exception\n'
+              'handler can find out what happened and continue execution at an '
+              'outer\n'
+              'level, but it cannot repair the cause of the error and retry '
+              'the\n'
+              'failing operation (except by re-entering the offending piece of '
+              'code\n'
+              'from the top).\n'
+              '\n'
+              'When an exception is not handled at all, the interpreter '
+              'terminates\n'
+              'execution of the program, or returns to its interactive main '
+              'loop.  In\n'
+              'either case, it prints a stack backtrace, except when the '
+              'exception is\n'
+              '"SystemExit".\n'
+              '\n'
+              'Exceptions are identified by class instances.  The "except" '
+              'clause is\n'
+              'selected depending on the class of the instance: it must '
+              'reference the\n'
+              'class of the instance or a base class thereof.  The instance '
+              'can be\n'
+              'received by the handler and can carry additional information '
+              'about the\n'
+              'exceptional condition.\n'
+              '\n'
+              'Note: Exception messages are not part of the Python API.  '
+              'Their\n'
+              '  contents may change from one version of Python to the next '
+              'without\n'
+              '  warning and should not be relied on by code which will run '
+              'under\n'
+              '  multiple versions of the interpreter.\n'
+              '\n'
+              'See also the description of the "try" statement in section *The '
+              'try\n'
+              'statement* and "raise" statement in section *The raise '
+              'statement*.\n'
+              '\n'
+              '-[ Footnotes ]-\n'
+              '\n'
+              '[1] This limitation occurs because the code that is executed '
+              'by\n'
+              '    these operations is not available at the time the module '
+              'is\n'
+              '    compiled.\n',
+ 'exprlists': '\n'
+              'Expression lists\n'
+              '****************\n'
+              '\n'
+              '   expression_list    ::= expression ( "," expression )* [","]\n'
+              '   starred_list       ::= starred_item ( "," starred_item )* '
+              '[","]\n'
+              '   starred_expression ::= expression | ( starred_item "," )* '
+              '[starred_item]\n'
+              '   starred_item       ::= expression | "*" or_expr\n'
+              '\n'
+              'Except when part of a list or set display, an expression list\n'
+              'containing at least one comma yields a tuple.  The length of '
+              'the tuple\n'
+              'is the number of expressions in the list.  The expressions are\n'
+              'evaluated from left to right.\n'
+              '\n'
+              'An asterisk "*" denotes *iterable unpacking*.  Its operand must '
+              'be an\n'
+              '*iterable*.  The iterable is expanded into a sequence of items, '
+              'which\n'
+              'are included in the new tuple, list, or set, at the site of '
+              'the\n'
+              'unpacking.\n'
+              '\n'
+              'New in version 3.5: Iterable unpacking in expression lists, '
+              'originally\n'
+              'proposed by **PEP 448**.\n'
+              '\n'
+              'The trailing comma is required only to create a single tuple '
+              '(a.k.a. a\n'
+              '*singleton*); it is optional in all other cases.  A single '
+              'expression\n'
+              "without a trailing comma doesn't create a tuple, but rather "
+              'yields the\n'
+              'value of that expression. (To create an empty tuple, use an '
+              'empty pair\n'
+              'of parentheses: "()".)\n',
+ 'floating': '\n'
+             'Floating point literals\n'
+             '***********************\n'
+             '\n'
+             'Floating point literals are described by the following lexical\n'
+             'definitions:\n'
+             '\n'
+             '   floatnumber   ::= pointfloat | exponentfloat\n'
+             '   pointfloat    ::= [intpart] fraction | intpart "."\n'
+             '   exponentfloat ::= (intpart | pointfloat) exponent\n'
+             '   intpart       ::= digit+\n'
+             '   fraction      ::= "." digit+\n'
+             '   exponent      ::= ("e" | "E") ["+" | "-"] digit+\n'
+             '\n'
+             'Note that the integer and exponent parts are always interpreted '
+             'using\n'
+             'radix 10. For example, "077e010" is legal, and denotes the same '
+             'number\n'
+             'as "77e10". The allowed range of floating point literals is\n'
+             'implementation-dependent. Some examples of floating point '
+             'literals:\n'
+             '\n'
+             '   3.14    10.    .001    1e100    3.14e-10    0e0\n'
+             '\n'
+             'Note that numeric literals do not include a sign; a phrase like '
+             '"-1"\n'
+             'is actually an expression composed of the unary operator "-" and '
+             'the\n'
+             'literal "1".\n',
+ 'for': '\n'
+        'The "for" statement\n'
+        '*******************\n'
+        '\n'
+        'The "for" statement is used to iterate over the elements of a '
+        'sequence\n'
+        '(such as a string, tuple or list) or other iterable object:\n'
+        '\n'
+        '   for_stmt ::= "for" target_list "in" expression_list ":" suite\n'
+        '                ["else" ":" suite]\n'
+        '\n'
+        'The expression list is evaluated once; it should yield an iterable\n'
+        'object.  An iterator is created for the result of the\n'
+        '"expression_list".  The suite is then executed once for each item\n'
+        'provided by the iterator, in the order returned by the iterator.  '
+        'Each\n'
+        'item in turn is assigned to the target list using the standard rules\n'
+        'for assignments (see *Assignment statements*), and then the suite is\n'
+        'executed.  When the items are exhausted (which is immediately when '
+        'the\n'
+        'sequence is empty or an iterator raises a "StopIteration" '
+        'exception),\n'
+        'the suite in the "else" clause, if present, is executed, and the '
+        'loop\n'
+        'terminates.\n'
+        '\n'
+        'A "break" statement executed in the first suite terminates the loop\n'
+        'without executing the "else" clause\'s suite.  A "continue" '
+        'statement\n'
+        'executed in the first suite skips the rest of the suite and '
+        'continues\n'
+        'with the next item, or with the "else" clause if there is no next\n'
+        'item.\n'
+        '\n'
+        'The for-loop makes assignments to the variables(s) in the target '
+        'list.\n'
+        'This overwrites all previous assignments to those variables '
+        'including\n'
+        'those made in the suite of the for-loop:\n'
+        '\n'
+        '   for i in range(10):\n'
+        '       print(i)\n'
+        '       i = 5             # this will not affect the for-loop\n'
+        '                         # because i will be overwritten with the '
+        'next\n'
+        '                         # index in the range\n'
+        '\n'
+        'Names in the target list are not deleted when the loop is finished,\n'
+        'but if the sequence is empty, they will not have been assigned to at\n'
+        'all by the loop.  Hint: the built-in function "range()" returns an\n'
+        'iterator of integers suitable to emulate the effect of Pascal\'s "for '
+        'i\n'
+        ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n'
+        '\n'
+        'Note: There is a subtlety when the sequence is being modified by the\n'
+        '  loop (this can only occur for mutable sequences, i.e. lists).  An\n'
+        '  internal counter is used to keep track of which item is used next,\n'
+        '  and this is incremented on each iteration.  When this counter has\n'
+        '  reached the length of the sequence the loop terminates.  This '
+        'means\n'
+        '  that if the suite deletes the current (or a previous) item from '
+        'the\n'
+        '  sequence, the next item will be skipped (since it gets the index '
+        'of\n'
+        '  the current item which has already been treated).  Likewise, if '
+        'the\n'
+        '  suite inserts an item in the sequence before the current item, the\n'
+        '  current item will be treated again the next time through the loop.\n'
+        '  This can lead to nasty bugs that can be avoided by making a\n'
+        '  temporary copy using a slice of the whole sequence, e.g.,\n'
+        '\n'
+        '     for x in a[:]:\n'
+        '         if x < 0: a.remove(x)\n',
+ 'formatstrings': '\n'
+                  'Format String Syntax\n'
+                  '********************\n'
+                  '\n'
+                  'The "str.format()" method and the "Formatter" class share '
+                  'the same\n'
+                  'syntax for format strings (although in the case of '
+                  '"Formatter",\n'
+                  'subclasses can define their own format string syntax).\n'
+                  '\n'
+                  'Format strings contain "replacement fields" surrounded by '
+                  'curly braces\n'
+                  '"{}". Anything that is not contained in braces is '
+                  'considered literal\n'
+                  'text, which is copied unchanged to the output.  If you need '
+                  'to include\n'
+                  'a brace character in the literal text, it can be escaped by '
+                  'doubling:\n'
+                  '"{{" and "}}".\n'
+                  '\n'
+                  'The grammar for a replacement field is as follows:\n'
+                  '\n'
+                  '      replacement_field ::= "{" [field_name] ["!" '
+                  'conversion] [":" format_spec] "}"\n'
+                  '      field_name        ::= arg_name ("." attribute_name | '
+                  '"[" element_index "]")*\n'
+                  '      arg_name          ::= [identifier | integer]\n'
+                  '      attribute_name    ::= identifier\n'
+                  '      element_index     ::= integer | index_string\n'
+                  '      index_string      ::= <any source character except '
+                  '"]"> +\n'
+                  '      conversion        ::= "r" | "s" | "a"\n'
+                  '      format_spec       ::= <described in the next '
+                  'section>\n'
+                  '\n'
+                  'In less formal terms, the replacement field can start with '
+                  'a\n'
+                  '*field_name* that specifies the object whose value is to be '
+                  'formatted\n'
+                  'and inserted into the output instead of the replacement '
+                  'field. The\n'
+                  '*field_name* is optionally followed by a  *conversion* '
+                  'field, which is\n'
+                  'preceded by an exclamation point "\'!\'", and a '
+                  '*format_spec*, which is\n'
+                  'preceded by a colon "\':\'".  These specify a non-default '
+                  'format for the\n'
+                  'replacement value.\n'
+                  '\n'
+                  'See also the *Format Specification Mini-Language* section.\n'
+                  '\n'
+                  'The *field_name* itself begins with an *arg_name* that is '
+                  'either a\n'
+                  "number or a keyword.  If it's a number, it refers to a "
+                  'positional\n'
+                  "argument, and if it's a keyword, it refers to a named "
+                  'keyword\n'
+                  'argument.  If the numerical arg_names in a format string '
+                  'are 0, 1, 2,\n'
+                  '... in sequence, they can all be omitted (not just some) '
+                  'and the\n'
+                  'numbers 0, 1, 2, ... will be automatically inserted in that '
+                  'order.\n'
+                  'Because *arg_name* is not quote-delimited, it is not '
+                  'possible to\n'
+                  'specify arbitrary dictionary keys (e.g., the strings '
+                  '"\'10\'" or\n'
+                  '"\':-]\'") within a format string. The *arg_name* can be '
+                  'followed by any\n'
+                  'number of index or attribute expressions. An expression of '
+                  'the form\n'
+                  '"\'.name\'" selects the named attribute using "getattr()", '
+                  'while an\n'
+                  'expression of the form "\'[index]\'" does an index lookup '
+                  'using\n'
+                  '"__getitem__()".\n'
+                  '\n'
+                  'Changed in version 3.1: The positional argument specifiers '
+                  'can be\n'
+                  'omitted, so "\'{} {}\'" is equivalent to "\'{0} {1}\'".\n'
+                  '\n'
+                  'Some simple format string examples:\n'
+                  '\n'
+                  '   "First, thou shalt count to {0}"  # References first '
+                  'positional argument\n'
+                  '   "Bring me a {}"                   # Implicitly '
+                  'references the first positional argument\n'
+                  '   "From {} to {}"                   # Same as "From {0} to '
+                  '{1}"\n'
+                  '   "My quest is {name}"              # References keyword '
+                  "argument 'name'\n"
+                  '   "Weight in tons {0.weight}"       # \'weight\' attribute '
+                  'of first positional arg\n'
+                  '   "Units destroyed: {players[0]}"   # First element of '
+                  "keyword argument 'players'.\n"
+                  '\n'
+                  'The *conversion* field causes a type coercion before '
+                  'formatting.\n'
+                  'Normally, the job of formatting a value is done by the '
+                  '"__format__()"\n'
+                  'method of the value itself.  However, in some cases it is '
+                  'desirable to\n'
+                  'force a type to be formatted as a string, overriding its '
+                  'own\n'
+                  'definition of formatting.  By converting the value to a '
+                  'string before\n'
+                  'calling "__format__()", the normal formatting logic is '
+                  'bypassed.\n'
+                  '\n'
+                  'Three conversion flags are currently supported: "\'!s\'" '
+                  'which calls\n'
+                  '"str()" on the value, "\'!r\'" which calls "repr()" and '
+                  '"\'!a\'" which\n'
+                  'calls "ascii()".\n'
+                  '\n'
+                  'Some examples:\n'
+                  '\n'
+                  '   "Harold\'s a clever {0!s}"        # Calls str() on the '
+                  'argument first\n'
+                  '   "Bring out the holy {name!r}"    # Calls repr() on the '
+                  'argument first\n'
+                  '   "More {!a}"                      # Calls ascii() on the '
+                  'argument first\n'
+                  '\n'
+                  'The *format_spec* field contains a specification of how the '
+                  'value\n'
+                  'should be presented, including such details as field width, '
+                  'alignment,\n'
+                  'padding, decimal precision and so on.  Each value type can '
+                  'define its\n'
+                  'own "formatting mini-language" or interpretation of the '
+                  '*format_spec*.\n'
+                  '\n'
+                  'Most built-in types support a common formatting '
+                  'mini-language, which\n'
+                  'is described in the next section.\n'
+                  '\n'
+                  'A *format_spec* field can also include nested replacement '
+                  'fields\n'
+                  'within it. These nested replacement fields may contain a '
+                  'field name,\n'
+                  'conversion flag and format specification, but deeper '
+                  'nesting is not\n'
+                  'allowed.  The replacement fields within the format_spec '
+                  'are\n'
+                  'substituted before the *format_spec* string is interpreted. '
+                  'This\n'
+                  'allows the formatting of a value to be dynamically '
+                  'specified.\n'
+                  '\n'
+                  'See the *Format examples* section for some examples.\n'
+                  '\n'
+                  '\n'
+                  'Format Specification Mini-Language\n'
+                  '==================================\n'
+                  '\n'
+                  '"Format specifications" are used within replacement fields '
+                  'contained\n'
+                  'within a format string to define how individual values are '
+                  'presented\n'
+                  '(see *Format String Syntax*).  They can also be passed '
+                  'directly to the\n'
+                  'built-in "format()" function.  Each formattable type may '
+                  'define how\n'
+                  'the format specification is to be interpreted.\n'
+                  '\n'
+                  'Most built-in types implement the following options for '
+                  'format\n'
+                  'specifications, although some of the formatting options are '
+                  'only\n'
+                  'supported by the numeric types.\n'
+                  '\n'
+                  'A general convention is that an empty format string ("""") '
+                  'produces\n'
+                  'the same result as if you had called "str()" on the value. '
+                  'A non-empty\n'
+                  'format string typically modifies the result.\n'
+                  '\n'
+                  'The general form of a *standard format specifier* is:\n'
+                  '\n'
+                  '   format_spec ::= '
+                  '[[fill]align][sign][#][0][width][,][.precision][type]\n'
+                  '   fill        ::= <any character>\n'
+                  '   align       ::= "<" | ">" | "=" | "^"\n'
+                  '   sign        ::= "+" | "-" | " "\n'
+                  '   width       ::= integer\n'
+                  '   precision   ::= integer\n'
+                  '   type        ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" '
+                  '| "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n'
+                  '\n'
+                  'If a valid *align* value is specified, it can be preceded '
+                  'by a *fill*\n'
+                  'character that can be any character and defaults to a space '
+                  'if\n'
+                  'omitted. It is not possible to use a literal curly brace '
+                  '(""{"" or\n'
+                  '""}"") as the *fill* character when using the '
+                  '"str.format()" method.\n'
+                  'However, it is possible to insert a curly brace with a '
+                  'nested\n'
+                  "replacement field.  This limitation doesn't affect the "
+                  '"format()"\n'
+                  'function.\n'
+                  '\n'
+                  'The meaning of the various alignment options is as '
+                  'follows:\n'
+                  '\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | Option    | '
+                  'Meaning                                                    '
+                  '|\n'
+                  '   '
+                  '+===========+============================================================+\n'
+                  '   | "\'<\'"     | Forces the field to be left-aligned '
+                  'within the available   |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'>\'"     | Forces the field to be right-aligned '
+                  'within the available  |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'=\'"     | Forces the padding to be placed after '
+                  'the sign (if any)    |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'^\'"     | Forces the field to be centered within '
+                  'the available       |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '\n'
+                  'Note that unless a minimum field width is defined, the '
+                  'field width\n'
+                  'will always be the same size as the data to fill it, so '
+                  'that the\n'
+                  'alignment option has no meaning in this case.\n'
+                  '\n'
+                  'The *sign* option is only valid for number types, and can '
+                  'be one of\n'
+                  'the following:\n'
+                  '\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | Option    | '
+                  'Meaning                                                    '
+                  '|\n'
+                  '   '
+                  '+===========+============================================================+\n'
+                  '   | "\'+\'"     | indicates that a sign should be used for '
+                  'both positive as  |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'-\'"     | indicates that a sign should be used '
+                  'only for negative     |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | space     | indicates that a leading space should be '
+                  'used on positive  |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '\n'
+                  'The "\'#\'" option causes the "alternate form" to be used '
+                  'for the\n'
+                  'conversion.  The alternate form is defined differently for '
+                  'different\n'
+                  'types.  This option is only valid for integer, float, '
+                  'complex and\n'
+                  'Decimal types. For integers, when binary, octal, or '
+                  'hexadecimal output\n'
+                  'is used, this option adds the prefix respective "\'0b\'", '
+                  '"\'0o\'", or\n'
+                  '"\'0x\'" to the output value. For floats, complex and '
+                  'Decimal the\n'
+                  'alternate form causes the result of the conversion to '
+                  'always contain a\n'
+                  'decimal-point character, even if no digits follow it. '
+                  'Normally, a\n'
+                  'decimal-point character appears in the result of these '
+                  'conversions\n'
+                  'only if a digit follows it. In addition, for "\'g\'" and '
+                  '"\'G\'"\n'
+                  'conversions, trailing zeros are not removed from the '
+                  'result.\n'
+                  '\n'
+                  'The "\',\'" option signals the use of a comma for a '
+                  'thousands separator.\n'
+                  'For a locale aware separator, use the "\'n\'" integer '
+                  'presentation type\n'
+                  'instead.\n'
+                  '\n'
+                  'Changed in version 3.1: Added the "\',\'" option (see also '
+                  '**PEP 378**).\n'
+                  '\n'
+                  '*width* is a decimal integer defining the minimum field '
+                  'width.  If not\n'
+                  'specified, then the field width will be determined by the '
+                  'content.\n'
+                  '\n'
+                  'When no explicit alignment is given, preceding the *width* '
+                  'field by a\n'
+                  'zero ("\'0\'") character enables sign-aware zero-padding '
+                  'for numeric\n'
+                  'types.  This is equivalent to a *fill* character of "\'0\'" '
+                  'with an\n'
+                  '*alignment* type of "\'=\'".\n'
+                  '\n'
+                  'The *precision* is a decimal number indicating how many '
+                  'digits should\n'
+                  'be displayed after the decimal point for a floating point '
+                  'value\n'
+                  'formatted with "\'f\'" and "\'F\'", or before and after the '
+                  'decimal point\n'
+                  'for a floating point value formatted with "\'g\'" or '
+                  '"\'G\'".  For non-\n'
+                  'number types the field indicates the maximum field size - '
+                  'in other\n'
+                  'words, how many characters will be used from the field '
+                  'content. The\n'
+                  '*precision* is not allowed for integer values.\n'
+                  '\n'
+                  'Finally, the *type* determines how the data should be '
+                  'presented.\n'
+                  '\n'
+                  'The available string presentation types are:\n'
+                  '\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | Type      | '
+                  'Meaning                                                    '
+                  '|\n'
+                  '   '
+                  '+===========+============================================================+\n'
+                  '   | "\'s\'"     | String format. This is the default type '
+                  'for strings and    |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | None      | The same as '
+                  '"\'s\'".                                         |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '\n'
+                  'The available integer presentation types are:\n'
+                  '\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | Type      | '
+                  'Meaning                                                    '
+                  '|\n'
+                  '   '
+                  '+===========+============================================================+\n'
+                  '   | "\'b\'"     | Binary format. Outputs the number in '
+                  'base 2.               |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'c\'"     | Character. Converts the integer to the '
+                  'corresponding       |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'d\'"     | Decimal Integer. Outputs the number in '
+                  'base 10.            |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'o\'"     | Octal format. Outputs the number in base '
+                  '8.                |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'x\'"     | Hex format. Outputs the number in base '
+                  '16, using lower-    |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'X\'"     | Hex format. Outputs the number in base '
+                  '16, using upper-    |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'n\'"     | Number. This is the same as "\'d\'", '
+                  'except that it uses the |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | None      | The same as '
+                  '"\'d\'".                                         |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '\n'
+                  'In addition to the above presentation types, integers can '
+                  'be formatted\n'
+                  'with the floating point presentation types listed below '
+                  '(except "\'n\'"\n'
+                  'and "None"). When doing so, "float()" is used to convert '
+                  'the integer\n'
+                  'to a floating point number before formatting.\n'
+                  '\n'
+                  'The available presentation types for floating point and '
+                  'decimal values\n'
+                  'are:\n'
+                  '\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | Type      | '
+                  'Meaning                                                    '
+                  '|\n'
+                  '   '
+                  '+===========+============================================================+\n'
+                  '   | "\'e\'"     | Exponent notation. Prints the number in '
+                  'scientific         |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'E\'"     | Exponent notation. Same as "\'e\'" '
+                  'except it uses an upper   |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'f\'"     | Fixed point. Displays the number as a '
+                  'fixed-point number.  |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'F\'"     | Fixed point. Same as "\'f\'", but '
+                  'converts "nan" to "NAN"    |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'g\'"     | General format.  For a given precision '
+                  '"p >= 1", this      |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'G\'"     | General format. Same as "\'g\'" except '
+                  'switches to "\'E\'" if  |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'n\'"     | Number. This is the same as "\'g\'", '
+                  'except that it uses the |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | "\'%\'"     | Percentage. Multiplies the number by 100 '
+                  'and displays in   |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '   | None      | Similar to "\'g\'", except that '
+                  'fixed-point notation, when   |\n'
+                  '   '
+                  '+-----------+------------------------------------------------------------+\n'
+                  '\n'
+                  '\n'
+                  'Format examples\n'
+                  '===============\n'
+                  '\n'
+                  'This section contains examples of the "str.format()" syntax '
+                  'and\n'
+                  'comparison with the old "%"-formatting.\n'
+                  '\n'
+                  'In most of the cases the syntax is similar to the old '
+                  '"%"-formatting,\n'
+                  'with the addition of the "{}" and with ":" used instead of '
+                  '"%". For\n'
+                  'example, "\'%03.2f\'" can be translated to "\'{:03.2f}\'".\n'
+                  '\n'
+                  'The new format syntax also supports new and different '
+                  'options, shown\n'
+                  'in the follow examples.\n'
+                  '\n'
+                  'Accessing arguments by position:\n'
+                  '\n'
+                  "   >>> '{0}, {1}, {2}'.format('a', 'b', 'c')\n"
+                  "   'a, b, c'\n"
+                  "   >>> '{}, {}, {}'.format('a', 'b', 'c')  # 3.1+ only\n"
+                  "   'a, b, c'\n"
+                  "   >>> '{2}, {1}, {0}'.format('a', 'b', 'c')\n"
+                  "   'c, b, a'\n"
+                  "   >>> '{2}, {1}, {0}'.format(*'abc')      # unpacking "
+                  'argument sequence\n'
+                  "   'c, b, a'\n"
+                  "   >>> '{0}{1}{0}'.format('abra', 'cad')   # arguments' "
+                  'indices can be repeated\n'
+                  "   'abracadabra'\n"
+                  '\n'
+                  'Accessing arguments by name:\n'
+                  '\n'
+                  "   >>> 'Coordinates: {latitude}, "
+                  "{longitude}'.format(latitude='37.24N', "
+                  "longitude='-115.81W')\n"
+                  "   'Coordinates: 37.24N, -115.81W'\n"
+                  "   >>> coord = {'latitude': '37.24N', 'longitude': "
+                  "'-115.81W'}\n"
+                  "   >>> 'Coordinates: {latitude}, "
+                  "{longitude}'.format(**coord)\n"
+                  "   'Coordinates: 37.24N, -115.81W'\n"
+                  '\n'
+                  "Accessing arguments' attributes:\n"
+                  '\n'
+                  '   >>> c = 3-5j\n'
+                  "   >>> ('The complex number {0} is formed from the real "
+                  "part {0.real} '\n"
+                  "   ...  'and the imaginary part {0.imag}.').format(c)\n"
+                  "   'The complex number (3-5j) is formed from the real part "
+                  "3.0 and the imaginary part -5.0.'\n"
+                  '   >>> class Point:\n'
+                  '   ...     def __init__(self, x, y):\n'
+                  '   ...         self.x, self.y = x, y\n'
+                  '   ...     def __str__(self):\n'
+                  "   ...         return 'Point({self.x}, "
+                  "{self.y})'.format(self=self)\n"
+                  '   ...\n'
+                  '   >>> str(Point(4, 2))\n'
+                  "   'Point(4, 2)'\n"
+                  '\n'
+                  "Accessing arguments' items:\n"
+                  '\n'
+                  '   >>> coord = (3, 5)\n'
+                  "   >>> 'X: {0[0]};  Y: {0[1]}'.format(coord)\n"
+                  "   'X: 3;  Y: 5'\n"
+                  '\n'
+                  'Replacing "%s" and "%r":\n'
+                  '\n'
+                  '   >>> "repr() shows quotes: {!r}; str() doesn\'t: '
+                  '{!s}".format(\'test1\', \'test2\')\n'
+                  '   "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n'
+                  '\n'
+                  'Aligning the text and specifying a width:\n'
+                  '\n'
+                  "   >>> '{:<30}'.format('left aligned')\n"
+                  "   'left aligned                  '\n"
+                  "   >>> '{:>30}'.format('right aligned')\n"
+                  "   '                 right aligned'\n"
+                  "   >>> '{:^30}'.format('centered')\n"
+                  "   '           centered           '\n"
+                  "   >>> '{:*^30}'.format('centered')  # use '*' as a fill "
+                  'char\n'
+                  "   '***********centered***********'\n"
+                  '\n'
+                  'Replacing "%+f", "%-f", and "% f" and specifying a sign:\n'
+                  '\n'
+                  "   >>> '{:+f}; {:+f}'.format(3.14, -3.14)  # show it "
+                  'always\n'
+                  "   '+3.140000; -3.140000'\n"
+                  "   >>> '{: f}; {: f}'.format(3.14, -3.14)  # show a space "
+                  'for positive numbers\n'
+                  "   ' 3.140000; -3.140000'\n"
+                  "   >>> '{:-f}; {:-f}'.format(3.14, -3.14)  # show only the "
+                  "minus -- same as '{:f}; {:f}'\n"
+                  "   '3.140000; -3.140000'\n"
+                  '\n'
+                  'Replacing "%x" and "%o" and converting the value to '
+                  'different bases:\n'
+                  '\n'
+                  '   >>> # format also supports binary numbers\n'
+                  '   >>> "int: {0:d};  hex: {0:x};  oct: {0:o};  bin: '
+                  '{0:b}".format(42)\n'
+                  "   'int: 42;  hex: 2a;  oct: 52;  bin: 101010'\n"
+                  '   >>> # with 0x, 0o, or 0b as prefix:\n'
+                  '   >>> "int: {0:d};  hex: {0:#x};  oct: {0:#o};  bin: '
+                  '{0:#b}".format(42)\n'
+                  "   'int: 42;  hex: 0x2a;  oct: 0o52;  bin: 0b101010'\n"
+                  '\n'
+                  'Using the comma as a thousands separator:\n'
+                  '\n'
+                  "   >>> '{:,}'.format(1234567890)\n"
+                  "   '1,234,567,890'\n"
+                  '\n'
+                  'Expressing a percentage:\n'
+                  '\n'
+                  '   >>> points = 19\n'
+                  '   >>> total = 22\n'
+                  "   >>> 'Correct answers: {:.2%}'.format(points/total)\n"
+                  "   'Correct answers: 86.36%'\n"
+                  '\n'
+                  'Using type-specific formatting:\n'
+                  '\n'
+                  '   >>> import datetime\n'
+                  '   >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n'
+                  "   >>> '{:%Y-%m-%d %H:%M:%S}'.format(d)\n"
+                  "   '2010-07-04 12:15:58'\n"
+                  '\n'
+                  'Nesting arguments and more complex examples:\n'
+                  '\n'
+                  "   >>> for align, text in zip('<^>', ['left', 'center', "
+                  "'right']):\n"
+                  "   ...     '{0:{fill}{align}16}'.format(text, fill=align, "
+                  'align=align)\n'
+                  '   ...\n'
+                  "   'left<<<<<<<<<<<<'\n"
+                  "   '^^^^^center^^^^^'\n"
+                  "   '>>>>>>>>>>>right'\n"
+                  '   >>>\n'
+                  '   >>> octets = [192, 168, 0, 1]\n'
+                  "   >>> '{:02X}{:02X}{:02X}{:02X}'.format(*octets)\n"
+                  "   'C0A80001'\n"
+                  '   >>> int(_, 16)\n'
+                  '   3232235521\n'
+                  '   >>>\n'
+                  '   >>> width = 5\n'
+                  '   >>> for num in range(5,12): #doctest: '
+                  '+NORMALIZE_WHITESPACE\n'
+                  "   ...     for base in 'dXob':\n"
+                  "   ...         print('{0:{width}{base}}'.format(num, "
+                  "base=base, width=width), end=' ')\n"
+                  '   ...     print()\n'
+                  '   ...\n'
+                  '       5     5     5   101\n'
+                  '       6     6     6   110\n'
+                  '       7     7     7   111\n'
+                  '       8     8    10  1000\n'
+                  '       9     9    11  1001\n'
+                  '      10     A    12  1010\n'
+                  '      11     B    13  1011\n',
+ 'function': '\n'
+             'Function definitions\n'
+             '********************\n'
+             '\n'
+             'A function definition defines a user-defined function object '
+             '(see\n'
+             'section *The standard type hierarchy*):\n'
+             '\n'
+             '   funcdef        ::= [decorators] "def" funcname "(" '
+             '[parameter_list] ")" ["->" expression] ":" suite\n'
+             '   decorators     ::= decorator+\n'
+             '   decorator      ::= "@" dotted_name ["(" [argument_list [","]] '
+             '")"] NEWLINE\n'
+             '   dotted_name    ::= identifier ("." identifier)*\n'
+             '   parameter_list ::= (defparameter ",")*\n'
+             '                      | "*" [parameter] ("," defparameter)* ["," '
+             '"**" parameter]\n'
+             '                      | "**" parameter\n'
+             '                      | defparameter [","] )\n'
+             '   parameter      ::= identifier [":" expression]\n'
+             '   defparameter   ::= parameter ["=" expression]\n'
+             '   funcname       ::= identifier\n'
+             '\n'
+             'A function definition is an executable statement.  Its execution '
+             'binds\n'
+             'the function name in the current local namespace to a function '
+             'object\n'
+             '(a wrapper around the executable code for the function).  This\n'
+             'function object contains a reference to the current global '
+             'namespace\n'
+             'as the global namespace to be used when the function is called.\n'
+             '\n'
+             'The function definition does not execute the function body; this '
+             'gets\n'
+             'executed only when the function is called. [3]\n'
+             '\n'
+             'A function definition may be wrapped by one or more *decorator*\n'
+             'expressions. Decorator expressions are evaluated when the '
+             'function is\n'
+             'defined, in the scope that contains the function definition.  '
+             'The\n'
+             'result must be a callable, which is invoked with the function '
+             'object\n'
+             'as the only argument. The returned value is bound to the '
+             'function name\n'
+             'instead of the function object.  Multiple decorators are applied '
+             'in\n'
+             'nested fashion. For example, the following code\n'
+             '\n'
+             '   @f1(arg)\n'
+             '   @f2\n'
+             '   def func(): pass\n'
+             '\n'
+             'is roughly equivalent to\n'
+             '\n'
+             '   def func(): pass\n'
+             '   func = f1(arg)(f2(func))\n'
+             '\n'
+             'except that the original function is not temporarily bound to '
+             'the name\n'
+             '"func".\n'
+             '\n'
+             'When one or more *parameters* have the form *parameter* "="\n'
+             '*expression*, the function is said to have "default parameter '
+             'values."\n'
+             'For a parameter with a default value, the corresponding '
+             '*argument* may\n'
+             "be omitted from a call, in which case the parameter's default "
+             'value is\n'
+             'substituted.  If a parameter has a default value, all following\n'
+             'parameters up until the ""*"" must also have a default value --- '
+             'this\n'
+             'is a syntactic restriction that is not expressed by the '
+             'grammar.\n'
+             '\n'
+             '**Default parameter values are evaluated from left to right when '
+             'the\n'
+             'function definition is executed.** This means that the '
+             'expression is\n'
+             'evaluated once, when the function is defined, and that the same '
+             '"pre-\n'
+             'computed" value is used for each call.  This is especially '
+             'important\n'
+             'to understand when a default parameter is a mutable object, such '
+             'as a\n'
+             'list or a dictionary: if the function modifies the object (e.g. '
+             'by\n'
+             'appending an item to a list), the default value is in effect '
+             'modified.\n'
+             'This is generally not what was intended.  A way around this is '
+             'to use\n'
+             '"None" as the default, and explicitly test for it in the body of '
+             'the\n'
+             'function, e.g.:\n'
+             '\n'
+             '   def whats_on_the_telly(penguin=None):\n'
+             '       if penguin is None:\n'
+             '           penguin = []\n'
+             '       penguin.append("property of the zoo")\n'
+             '       return penguin\n'
+             '\n'
+             'Function call semantics are described in more detail in section\n'
+             '*Calls*. A function call always assigns values to all '
+             'parameters\n'
+             'mentioned in the parameter list, either from position arguments, '
+             'from\n'
+             'keyword arguments, or from default values.  If the form\n'
+             '""*identifier"" is present, it is initialized to a tuple '
+             'receiving any\n'
+             'excess positional parameters, defaulting to the empty tuple.  If '
+             'the\n'
+             'form ""**identifier"" is present, it is initialized to a new\n'
+             'dictionary receiving any excess keyword arguments, defaulting to '
+             'a new\n'
+             'empty dictionary. Parameters after ""*"" or ""*identifier"" are\n'
+             'keyword-only parameters and may only be passed used keyword '
+             'arguments.\n'
+             '\n'
+             'Parameters may have annotations of the form "": expression"" '
+             'following\n'
+             'the parameter name.  Any parameter may have an annotation even '
+             'those\n'
+             'of the form "*identifier" or "**identifier".  Functions may '
+             'have\n'
+             '"return" annotation of the form ""-> expression"" after the '
+             'parameter\n'
+             'list.  These annotations can be any valid Python expression and '
+             'are\n'
+             'evaluated when the function definition is executed.  Annotations '
+             'may\n'
+             'be evaluated in a different order than they appear in the source '
+             'code.\n'
+             'The presence of annotations does not change the semantics of a\n'
+             'function.  The annotation values are available as values of a\n'
+             "dictionary keyed by the parameters' names in the "
+             '"__annotations__"\n'
+             'attribute of the function object.\n'
+             '\n'
+             'It is also possible to create anonymous functions (functions not '
+             'bound\n'
+             'to a name), for immediate use in expressions.  This uses lambda\n'
+             'expressions, described in section *Lambdas*.  Note that the '
+             'lambda\n'
+             'expression is merely a shorthand for a simplified function '
+             'definition;\n'
+             'a function defined in a ""def"" statement can be passed around '
+             'or\n'
+             'assigned to another name just like a function defined by a '
+             'lambda\n'
+             'expression.  The ""def"" form is actually more powerful since '
+             'it\n'
+             'allows the execution of multiple statements and annotations.\n'
+             '\n'
+             "**Programmer's note:** Functions are first-class objects.  A "
+             '""def""\n'
+             'statement executed inside a function definition defines a local\n'
+             'function that can be returned or passed around.  Free variables '
+             'used\n'
+             'in the nested function can access the local variables of the '
+             'function\n'
+             'containing the def.  See section *Naming and binding* for '
+             'details.\n'
+             '\n'
+             'See also: **PEP 3107** - Function Annotations\n'
+             '\n'
+             '     The original specification for function annotations.\n',
+ 'global': '\n'
+           'The "global" statement\n'
+           '**********************\n'
+           '\n'
+           '   global_stmt ::= "global" identifier ("," identifier)*\n'
+           '\n'
+           'The "global" statement is a declaration which holds for the '
+           'entire\n'
+           'current code block.  It means that the listed identifiers are to '
+           'be\n'
+           'interpreted as globals.  It would be impossible to assign to a '
+           'global\n'
+           'variable without "global", although free variables may refer to\n'
+           'globals without being declared global.\n'
+           '\n'
+           'Names listed in a "global" statement must not be used in the same '
+           'code\n'
+           'block textually preceding that "global" statement.\n'
+           '\n'
+           'Names listed in a "global" statement must not be defined as '
+           'formal\n'
+           'parameters or in a "for" loop control target, "class" definition,\n'
+           'function definition, or "import" statement.\n'
+           '\n'
+           '**CPython implementation detail:** The current implementation does '
+           'not\n'
+           'enforce the two restrictions, but programs should not abuse this\n'
+           'freedom, as future implementations may enforce them or silently '
+           'change\n'
+           'the meaning of the program.\n'
+           '\n'
+           '**Programmer\'s note:** "global" is a directive to the parser.  '
+           'It\n'
+           'applies only to code parsed at the same time as the "global"\n'
+           'statement. In particular, a "global" statement contained in a '
+           'string\n'
+           'or code object supplied to the built-in "exec()" function does '
+           'not\n'
+           'affect the code block *containing* the function call, and code\n'
+           'contained in such a string is unaffected by "global" statements in '
+           'the\n'
+           'code containing the function call.  The same applies to the '
+           '"eval()"\n'
+           'and "compile()" functions.\n',
+ 'id-classes': '\n'
+               'Reserved classes of identifiers\n'
+               '*******************************\n'
+               '\n'
+               'Certain classes of identifiers (besides keywords) have '
+               'special\n'
+               'meanings.  These classes are identified by the patterns of '
+               'leading and\n'
+               'trailing underscore characters:\n'
+               '\n'
+               '"_*"\n'
+               '   Not imported by "from module import *".  The special '
+               'identifier "_"\n'
+               '   is used in the interactive interpreter to store the result '
+               'of the\n'
+               '   last evaluation; it is stored in the "builtins" module.  '
+               'When not\n'
+               '   in interactive mode, "_" has no special meaning and is not '
+               'defined.\n'
+               '   See section *The import statement*.\n'
+               '\n'
+               '   Note: The name "_" is often used in conjunction with\n'
+               '     internationalization; refer to the documentation for the\n'
+               '     "gettext" module for more information on this '
+               'convention.\n'
+               '\n'
+               '"__*__"\n'
+               '   System-defined names. These names are defined by the '
+               'interpreter\n'
+               '   and its implementation (including the standard library).  '
+               'Current\n'
+               '   system names are discussed in the *Special method names* '
+               'section\n'
+               '   and elsewhere.  More will likely be defined in future '
+               'versions of\n'
+               '   Python.  *Any* use of "__*__" names, in any context, that '
+               'does not\n'
+               '   follow explicitly documented use, is subject to breakage '
+               'without\n'
+               '   warning.\n'
+               '\n'
+               '"__*"\n'
+               '   Class-private names.  Names in this category, when used '
+               'within the\n'
+               '   context of a class definition, are re-written to use a '
+               'mangled form\n'
+               '   to help avoid name clashes between "private" attributes of '
+               'base and\n'
+               '   derived classes. See section *Identifiers (Names)*.\n',
+ 'identifiers': '\n'
+                'Identifiers and keywords\n'
+                '************************\n'
+                '\n'
+                'Identifiers (also referred to as *names*) are described by '
+                'the\n'
+                'following lexical definitions.\n'
+                '\n'
+                'The syntax of identifiers in Python is based on the Unicode '
+                'standard\n'
+                'annex UAX-31, with elaboration and changes as defined below; '
+                'see also\n'
+                '**PEP 3131** for further details.\n'
+                '\n'
+                'Within the ASCII range (U+0001..U+007F), the valid characters '
+                'for\n'
+                'identifiers are the same as in Python 2.x: the uppercase and '
+                'lowercase\n'
+                'letters "A" through "Z", the underscore "_" and, except for '
+                'the first\n'
+                'character, the digits "0" through "9".\n'
+                '\n'
+                'Python 3.0 introduces additional characters from outside the '
+                'ASCII\n'
+                'range (see **PEP 3131**).  For these characters, the '
+                'classification\n'
+                'uses the version of the Unicode Character Database as '
+                'included in the\n'
+                '"unicodedata" module.\n'
+                '\n'
+                'Identifiers are unlimited in length.  Case is significant.\n'
+                '\n'
+                '   identifier   ::= xid_start xid_continue*\n'
+                '   id_start     ::= <all characters in general categories Lu, '
+                'Ll, Lt, Lm, Lo, Nl, the underscore, and characters with the '
+                'Other_ID_Start property>\n'
+                '   id_continue  ::= <all characters in id_start, plus '
+                'characters in the categories Mn, Mc, Nd, Pc and others with '
+                'the Other_ID_Continue property>\n'
+                '   xid_start    ::= <all characters in id_start whose NFKC '
+                'normalization is in "id_start xid_continue*">\n'
+                '   xid_continue ::= <all characters in id_continue whose NFKC '
+                'normalization is in "id_continue*">\n'
+                '\n'
+                'The Unicode category codes mentioned above stand for:\n'
+                '\n'
+                '* *Lu* - uppercase letters\n'
+                '\n'
+                '* *Ll* - lowercase letters\n'
+                '\n'
+                '* *Lt* - titlecase letters\n'
+                '\n'
+                '* *Lm* - modifier letters\n'
+                '\n'
+                '* *Lo* - other letters\n'
+                '\n'
+                '* *Nl* - letter numbers\n'
+                '\n'
+                '* *Mn* - nonspacing marks\n'
+                '\n'
+                '* *Mc* - spacing combining marks\n'
+                '\n'
+                '* *Nd* - decimal numbers\n'
+                '\n'
+                '* *Pc* - connector punctuations\n'
+                '\n'
+                '* *Other_ID_Start* - explicit list of characters in '
+                'PropList.txt to\n'
+                '  support backwards compatibility\n'
+                '\n'
+                '* *Other_ID_Continue* - likewise\n'
+                '\n'
+                'All identifiers are converted into the normal form NFKC while '
+                'parsing;\n'
+                'comparison of identifiers is based on NFKC.\n'
+                '\n'
+                'A non-normative HTML file listing all valid identifier '
+                'characters for\n'
+                'Unicode 4.1 can be found at https://www.dcl.hpi.uni-\n'
+                'potsdam.de/home/loewis/table-3131.html.\n'
+                '\n'
+                '\n'
+                'Keywords\n'
+                '========\n'
+                '\n'
+                'The following identifiers are used as reserved words, or '
+                '*keywords* of\n'
+                'the language, and cannot be used as ordinary identifiers.  '
+                'They must\n'
+                'be spelled exactly as written here:\n'
+                '\n'
+                '   False      class      finally    is         return\n'
+                '   None       continue   for        lambda     try\n'
+                '   True       def        from       nonlocal   while\n'
+                '   and        del        global     not        with\n'
+                '   as         elif       if         or         yield\n'
+                '   assert     else       import     pass\n'
+                '   break      except     in         raise\n'
+                '\n'
+                '\n'
+                'Reserved classes of identifiers\n'
+                '===============================\n'
+                '\n'
+                'Certain classes of identifiers (besides keywords) have '
+                'special\n'
+                'meanings.  These classes are identified by the patterns of '
+                'leading and\n'
+                'trailing underscore characters:\n'
+                '\n'
+                '"_*"\n'
+                '   Not imported by "from module import *".  The special '
+                'identifier "_"\n'
+                '   is used in the interactive interpreter to store the result '
+                'of the\n'
+                '   last evaluation; it is stored in the "builtins" module.  '
+                'When not\n'
+                '   in interactive mode, "_" has no special meaning and is not '
+                'defined.\n'
+                '   See section *The import statement*.\n'
+                '\n'
+                '   Note: The name "_" is often used in conjunction with\n'
+                '     internationalization; refer to the documentation for '
+                'the\n'
+                '     "gettext" module for more information on this '
+                'convention.\n'
+                '\n'
+                '"__*__"\n'
+                '   System-defined names. These names are defined by the '
+                'interpreter\n'
+                '   and its implementation (including the standard library).  '
+                'Current\n'
+                '   system names are discussed in the *Special method names* '
+                'section\n'
+                '   and elsewhere.  More will likely be defined in future '
+                'versions of\n'
+                '   Python.  *Any* use of "__*__" names, in any context, that '
+                'does not\n'
+                '   follow explicitly documented use, is subject to breakage '
+                'without\n'
+                '   warning.\n'
+                '\n'
+                '"__*"\n'
+                '   Class-private names.  Names in this category, when used '
+                'within the\n'
+                '   context of a class definition, are re-written to use a '
+                'mangled form\n'
+                '   to help avoid name clashes between "private" attributes of '
+                'base and\n'
+                '   derived classes. See section *Identifiers (Names)*.\n',
+ 'if': '\n'
+       'The "if" statement\n'
+       '******************\n'
+       '\n'
+       'The "if" statement is used for conditional execution:\n'
+       '\n'
+       '   if_stmt ::= "if" expression ":" suite\n'
+       '               ( "elif" expression ":" suite )*\n'
+       '               ["else" ":" suite]\n'
+       '\n'
+       'It selects exactly one of the suites by evaluating the expressions '
+       'one\n'
+       'by one until one is found to be true (see section *Boolean '
+       'operations*\n'
+       'for the definition of true and false); then that suite is executed\n'
+       '(and no other part of the "if" statement is executed or evaluated).\n'
+       'If all expressions are false, the suite of the "else" clause, if\n'
+       'present, is executed.\n',
+ 'imaginary': '\n'
+              'Imaginary literals\n'
+              '******************\n'
+              '\n'
+              'Imaginary literals are described by the following lexical '
+              'definitions:\n'
+              '\n'
+              '   imagnumber ::= (floatnumber | intpart) ("j" | "J")\n'
+              '\n'
+              'An imaginary literal yields a complex number with a real part '
+              'of 0.0.\n'
+              'Complex numbers are represented as a pair of floating point '
+              'numbers\n'
+              'and have the same restrictions on their range.  To create a '
+              'complex\n'
+              'number with a nonzero real part, add a floating point number to '
+              'it,\n'
+              'e.g., "(3+4j)".  Some examples of imaginary literals:\n'
+              '\n'
+              '   3.14j   10.j    10j     .001j   1e100j  3.14e-10j\n',
+ 'import': '\n'
+           'The "import" statement\n'
+           '**********************\n'
+           '\n'
+           '   import_stmt     ::= "import" module ["as" name] ( "," module '
+           '["as" name] )*\n'
+           '                   | "from" relative_module "import" identifier '
+           '["as" name]\n'
+           '                   ( "," identifier ["as" name] )*\n'
+           '                   | "from" relative_module "import" "(" '
+           'identifier ["as" name]\n'
+           '                   ( "," identifier ["as" name] )* [","] ")"\n'
+           '                   | "from" module "import" "*"\n'
+           '   module          ::= (identifier ".")* identifier\n'
+           '   relative_module ::= "."* module | "."+\n'
+           '   name            ::= identifier\n'
+           '\n'
+           'The basic import statement (no "from" clause) is executed in two\n'
+           'steps:\n'
+           '\n'
+           '1. find a module, loading and initializing it if necessary\n'
+           '\n'
+           '2. define a name or names in the local namespace for the scope\n'
+           '   where the "import" statement occurs.\n'
+           '\n'
+           'When the statement contains multiple clauses (separated by commas) '
+           'the\n'
+           'two steps are carried out separately for each clause, just as '
+           'though\n'
+           'the clauses had been separated out into individual import '
+           'statements.\n'
+           '\n'
+           'The details of the first step, finding and loading modules are\n'
+           'described in greater detail in the section on the *import '
+           'system*,\n'
+           'which also describes the various types of packages and modules '
+           'that\n'
+           'can be imported, as well as all the hooks that can be used to\n'
+           'customize the import system. Note that failures in this step may\n'
+           'indicate either that the module could not be located, *or* that '
+           'an\n'
+           'error occurred while initializing the module, which includes '
+           'execution\n'
+           "of the module's code.\n"
+           '\n'
+           'If the requested module is retrieved successfully, it will be '
+           'made\n'
+           'available in the local namespace in one of three ways:\n'
+           '\n'
+           '* If the module name is followed by "as", then the name following\n'
+           '  "as" is bound directly to the imported module.\n'
+           '\n'
+           '* If no other name is specified, and the module being imported is '
+           'a\n'
+           "  top level module, the module's name is bound in the local "
+           'namespace\n'
+           '  as a reference to the imported module\n'
+           '\n'
+           '* If the module being imported is *not* a top level module, then '
+           'the\n'
+           '  name of the top level package that contains the module is bound '
+           'in\n'
+           '  the local namespace as a reference to the top level package. '
+           'The\n'
+           '  imported module must be accessed using its full qualified name\n'
+           '  rather than directly\n'
+           '\n'
+           'The "from" form uses a slightly more complex process:\n'
+           '\n'
+           '1. find the module specified in the "from" clause, loading and\n'
+           '   initializing it if necessary;\n'
+           '\n'
+           '2. for each of the identifiers specified in the "import" clauses:\n'
+           '\n'
+           '   1. check if the imported module has an attribute by that name\n'
+           '\n'
+           '   2. if not, attempt to import a submodule with that name and '
+           'then\n'
+           '      check the imported module again for that attribute\n'
+           '\n'
+           '   3. if the attribute is not found, "ImportError" is raised.\n'
+           '\n'
+           '   4. otherwise, a reference to that value is stored in the local\n'
+           '      namespace, using the name in the "as" clause if it is '
+           'present,\n'
+           '      otherwise using the attribute name\n'
+           '\n'
+           'Examples:\n'
+           '\n'
+           '   import foo                 # foo imported and bound locally\n'
+           '   import foo.bar.baz         # foo.bar.baz imported, foo bound '
+           'locally\n'
+           '   import foo.bar.baz as fbb  # foo.bar.baz imported and bound as '
+           'fbb\n'
+           '   from foo.bar import baz    # foo.bar.baz imported and bound as '
+           'baz\n'
+           '   from foo import attr       # foo imported and foo.attr bound as '
+           'attr\n'
+           '\n'
+           'If the list of identifiers is replaced by a star ("\'*\'"), all '
+           'public\n'
+           'names defined in the module are bound in the local namespace for '
+           'the\n'
+           'scope where the "import" statement occurs.\n'
+           '\n'
+           'The *public names* defined by a module are determined by checking '
+           'the\n'
+           'module\'s namespace for a variable named "__all__"; if defined, it '
+           'must\n'
+           'be a sequence of strings which are names defined or imported by '
+           'that\n'
+           'module.  The names given in "__all__" are all considered public '
+           'and\n'
+           'are required to exist.  If "__all__" is not defined, the set of '
+           'public\n'
+           "names includes all names found in the module's namespace which do "
+           'not\n'
+           'begin with an underscore character ("\'_\'").  "__all__" should '
+           'contain\n'
+           'the entire public API. It is intended to avoid accidentally '
+           'exporting\n'
+           'items that are not part of the API (such as library modules which '
+           'were\n'
+           'imported and used within the module).\n'
+           '\n'
+           'The wild card form of import --- "from module import *" --- is '
+           'only\n'
+           'allowed at the module level.  Attempting to use it in class or\n'
+           'function definitions will raise a "SyntaxError".\n'
+           '\n'
+           'When specifying what module to import you do not have to specify '
+           'the\n'
+           'absolute name of the module. When a module or package is '
+           'contained\n'
+           'within another package it is possible to make a relative import '
+           'within\n'
+           'the same top package without having to mention the package name. '
+           'By\n'
+           'using leading dots in the specified module or package after "from" '
+           'you\n'
+           'can specify how high to traverse up the current package hierarchy\n'
+           'without specifying exact names. One leading dot means the current\n'
+           'package where the module making the import exists. Two dots means '
+           'up\n'
+           'one package level. Three dots is up two levels, etc. So if you '
+           'execute\n'
+           '"from . import mod" from a module in the "pkg" package then you '
+           'will\n'
+           'end up importing "pkg.mod". If you execute "from ..subpkg2 import '
+           'mod"\n'
+           'from within "pkg.subpkg1" you will import "pkg.subpkg2.mod". The\n'
+           'specification for relative imports is contained within **PEP '
+           '328**.\n'
+           '\n'
+           '"importlib.import_module()" is provided to support applications '
+           'that\n'
+           'determine dynamically the modules to be loaded.\n'
+           '\n'
+           '\n'
+           'Future statements\n'
+           '=================\n'
+           '\n'
+           'A *future statement* is a directive to the compiler that a '
+           'particular\n'
+           'module should be compiled using syntax or semantics that will be\n'
+           'available in a specified future release of Python where the '
+           'feature\n'
+           'becomes standard.\n'
+           '\n'
+           'The future statement is intended to ease migration to future '
+           'versions\n'
+           'of Python that introduce incompatible changes to the language.  '
+           'It\n'
+           'allows use of the new features on a per-module basis before the\n'
+           'release in which the feature becomes standard.\n'
+           '\n'
+           '   future_statement ::= "from" "__future__" "import" feature ["as" '
+           'name]\n'
+           '                        ("," feature ["as" name])*\n'
+           '                        | "from" "__future__" "import" "(" feature '
+           '["as" name]\n'
+           '                        ("," feature ["as" name])* [","] ")"\n'
+           '   feature          ::= identifier\n'
+           '   name             ::= identifier\n'
+           '\n'
+           'A future statement must appear near the top of the module.  The '
+           'only\n'
+           'lines that can appear before a future statement are:\n'
+           '\n'
+           '* the module docstring (if any),\n'
+           '\n'
+           '* comments,\n'
+           '\n'
+           '* blank lines, and\n'
+           '\n'
+           '* other future statements.\n'
+           '\n'
+           'The features recognized by Python 3.0 are "absolute_import",\n'
+           '"division", "generators", "unicode_literals", "print_function",\n'
+           '"nested_scopes" and "with_statement".  They are all redundant '
+           'because\n'
+           'they are always enabled, and only kept for backwards '
+           'compatibility.\n'
+           '\n'
+           'A future statement is recognized and treated specially at compile\n'
+           'time: Changes to the semantics of core constructs are often\n'
+           'implemented by generating different code.  It may even be the '
+           'case\n'
+           'that a new feature introduces new incompatible syntax (such as a '
+           'new\n'
+           'reserved word), in which case the compiler may need to parse the\n'
+           'module differently.  Such decisions cannot be pushed off until\n'
+           'runtime.\n'
+           '\n'
+           'For any given release, the compiler knows which feature names '
+           'have\n'
+           'been defined, and raises a compile-time error if a future '
+           'statement\n'
+           'contains a feature not known to it.\n'
+           '\n'
+           'The direct runtime semantics are the same as for any import '
+           'statement:\n'
+           'there is a standard module "__future__", described later, and it '
+           'will\n'
+           'be imported in the usual way at the time the future statement is\n'
+           'executed.\n'
+           '\n'
+           'The interesting runtime semantics depend on the specific feature\n'
+           'enabled by the future statement.\n'
+           '\n'
+           'Note that there is nothing special about the statement:\n'
+           '\n'
+           '   import __future__ [as name]\n'
+           '\n'
+           "That is not a future statement; it's an ordinary import statement "
+           'with\n'
+           'no special semantics or syntax restrictions.\n'
+           '\n'
+           'Code compiled by calls to the built-in functions "exec()" and\n'
+           '"compile()" that occur in a module "M" containing a future '
+           'statement\n'
+           'will, by default, use the new syntax or semantics associated with '
+           'the\n'
+           'future statement.  This can be controlled by optional arguments '
+           'to\n'
+           '"compile()" --- see the documentation of that function for '
+           'details.\n'
+           '\n'
+           'A future statement typed at an interactive interpreter prompt '
+           'will\n'
+           'take effect for the rest of the interpreter session.  If an\n'
+           'interpreter is started with the *-i* option, is passed a script '
+           'name\n'
+           'to execute, and the script includes a future statement, it will be '
+           'in\n'
+           'effect in the interactive session started after the script is\n'
+           'executed.\n'
+           '\n'
+           'See also: **PEP 236** - Back to the __future__\n'
+           '\n'
+           '     The original proposal for the __future__ mechanism.\n',
+ 'in': '\n'
+       'Membership test operations\n'
+       '**************************\n'
+       '\n'
+       'The operators "in" and "not in" test for membership.  "x in s"\n'
+       'evaluates to "True" if *x* is a member of *s*, and "False" otherwise.\n'
+       '"x not in s" returns the negation of "x in s".  All built-in '
+       'sequences\n'
+       'and set types support this as well as dictionary, for which "in" '
+       'tests\n'
+       'whether the dictionary has a given key. For container types such as\n'
+       'list, tuple, set, frozenset, dict, or collections.deque, the\n'
+       'expression "x in y" is equivalent to "any(x is e or x == e for e in\n'
+       'y)".\n'
+       '\n'
+       'For the string and bytes types, "x in y" is "True" if and only if *x*\n'
+       'is a substring of *y*.  An equivalent test is "y.find(x) != -1".\n'
+       'Empty strings are always considered to be a substring of any other\n'
+       'string, so """ in "abc"" will return "True".\n'
+       '\n'
+       'For user-defined classes which define the "__contains__()" method, "x\n'
+       'in y" returns "True" if "y.__contains__(x)" returns a true value, and\n'
+       '"False" otherwise.\n'
+       '\n'
+       'For user-defined classes which do not define "__contains__()" but do\n'
+       'define "__iter__()", "x in y" is "True" if some value "z" with "x ==\n'
+       'z" is produced while iterating over "y".  If an exception is raised\n'
+       'during the iteration, it is as if "in" raised that exception.\n'
+       '\n'
+       'Lastly, the old-style iteration protocol is tried: if a class defines\n'
+       '"__getitem__()", "x in y" is "True" if and only if there is a non-\n'
+       'negative integer index *i* such that "x == y[i]", and all lower\n'
+       'integer indices do not raise "IndexError" exception.  (If any other\n'
+       'exception is raised, it is as if "in" raised that exception).\n'
+       '\n'
+       'The operator "not in" is defined to have the inverse true value of\n'
+       '"in".\n',
+ 'integers': '\n'
+             'Integer literals\n'
+             '****************\n'
+             '\n'
+             'Integer literals are described by the following lexical '
+             'definitions:\n'
+             '\n'
+             '   integer        ::= decimalinteger | octinteger | hexinteger | '
+             'bininteger\n'
+             '   decimalinteger ::= nonzerodigit digit* | "0"+\n'
+             '   nonzerodigit   ::= "1"..."9"\n'
+             '   digit          ::= "0"..."9"\n'
+             '   octinteger     ::= "0" ("o" | "O") octdigit+\n'
+             '   hexinteger     ::= "0" ("x" | "X") hexdigit+\n'
+             '   bininteger     ::= "0" ("b" | "B") bindigit+\n'
+             '   octdigit       ::= "0"..."7"\n'
+             '   hexdigit       ::= digit | "a"..."f" | "A"..."F"\n'
+             '   bindigit       ::= "0" | "1"\n'
+             '\n'
+             'There is no limit for the length of integer literals apart from '
+             'what\n'
+             'can be stored in available memory.\n'
+             '\n'
+             'Note that leading zeros in a non-zero decimal number are not '
+             'allowed.\n'
+             'This is for disambiguation with C-style octal literals, which '
+             'Python\n'
+             'used before version 3.0.\n'
+             '\n'
+             'Some examples of integer literals:\n'
+             '\n'
+             '   7     2147483647                        0o177    0b100110111\n'
+             '   3     79228162514264337593543950336     0o377    0xdeadbeef\n',
+ 'lambda': '\n'
+           'Lambdas\n'
+           '*******\n'
+           '\n'
+           '   lambda_expr        ::= "lambda" [parameter_list]: expression\n'
+           '   lambda_expr_nocond ::= "lambda" [parameter_list]: '
+           'expression_nocond\n'
+           '\n'
+           'Lambda expressions (sometimes called lambda forms) are used to '
+           'create\n'
+           'anonymous functions. The expression "lambda arguments: '
+           'expression"\n'
+           'yields a function object.  The unnamed object behaves like a '
+           'function\n'
+           'object defined with:\n'
+           '\n'
+           '   def <lambda>(arguments):\n'
+           '       return expression\n'
+           '\n'
+           'See section *Function definitions* for the syntax of parameter '
+           'lists.\n'
+           'Note that functions created with lambda expressions cannot '
+           'contain\n'
+           'statements or annotations.\n',
+ 'lists': '\n'
+          'List displays\n'
+          '*************\n'
+          '\n'
+          'A list display is a possibly empty series of expressions enclosed '
+          'in\n'
+          'square brackets:\n'
+          '\n'
+          '   list_display ::= "[" [starred_list | comprehension] "]"\n'
+          '\n'
+          'A list display yields a new list object, the contents being '
+          'specified\n'
+          'by either a list of expressions or a comprehension.  When a comma-\n'
+          'separated list of expressions is supplied, its elements are '
+          'evaluated\n'
+          'from left to right and placed into the list object in that order.\n'
+          'When a comprehension is supplied, the list is constructed from the\n'
+          'elements resulting from the comprehension.\n',
+ 'naming': '\n'
+           'Naming and binding\n'
+           '******************\n'
+           '\n'
+           '\n'
+           'Binding of names\n'
+           '================\n'
+           '\n'
+           '*Names* refer to objects.  Names are introduced by name binding\n'
+           'operations.\n'
+           '\n'
+           'The following constructs bind names: formal parameters to '
+           'functions,\n'
+           '"import" statements, class and function definitions (these bind '
+           'the\n'
+           'class or function name in the defining block), and targets that '
+           'are\n'
+           'identifiers if occurring in an assignment, "for" loop header, or '
+           'after\n'
+           '"as" in a "with" statement or "except" clause. The "import" '
+           'statement\n'
+           'of the form "from ... import *" binds all names defined in the\n'
+           'imported module, except those beginning with an underscore.  This '
+           'form\n'
+           'may only be used at the module level.\n'
+           '\n'
+           'A target occurring in a "del" statement is also considered bound '
+           'for\n'
+           'this purpose (though the actual semantics are to unbind the '
+           'name).\n'
+           '\n'
+           'Each assignment or import statement occurs within a block defined '
+           'by a\n'
+           'class or function definition or at the module level (the '
+           'top-level\n'
+           'code block).\n'
+           '\n'
+           'If a name is bound in a block, it is a local variable of that '
+           'block,\n'
+           'unless declared as "nonlocal" or "global".  If a name is bound at '
+           'the\n'
+           'module level, it is a global variable.  (The variables of the '
+           'module\n'
+           'code block are local and global.)  If a variable is used in a '
+           'code\n'
+           'block but not defined there, it is a *free variable*.\n'
+           '\n'
+           'Each occurrence of a name in the program text refers to the '
+           '*binding*\n'
+           'of that name established by the following name resolution rules.\n'
+           '\n'
+           '\n'
+           'Resolution of names\n'
+           '===================\n'
+           '\n'
+           'A *scope* defines the visibility of a name within a block.  If a '
+           'local\n'
+           'variable is defined in a block, its scope includes that block.  If '
+           'the\n'
+           'definition occurs in a function block, the scope extends to any '
+           'blocks\n'
+           'contained within the defining one, unless a contained block '
+           'introduces\n'
+           'a different binding for the name.\n'
+           '\n'
+           'When a name is used in a code block, it is resolved using the '
+           'nearest\n'
+           'enclosing scope.  The set of all such scopes visible to a code '
+           'block\n'
+           "is called the block's *environment*.\n"
+           '\n'
+           'When a name is not found at all, a "NameError" exception is '
+           'raised. If\n'
+           'the current scope is a function scope, and the name refers to a '
+           'local\n'
+           'variable that has not yet been bound to a value at the point where '
+           'the\n'
+           'name is used, an "UnboundLocalError" exception is raised.\n'
+           '"UnboundLocalError" is a subclass of "NameError".\n'
+           '\n'
+           'If a name binding operation occurs anywhere within a code block, '
+           'all\n'
+           'uses of the name within the block are treated as references to '
+           'the\n'
+           'current block.  This can lead to errors when a name is used within '
+           'a\n'
+           'block before it is bound.  This rule is subtle.  Python lacks\n'
+           'declarations and allows name binding operations to occur anywhere\n'
+           'within a code block.  The local variables of a code block can be\n'
+           'determined by scanning the entire text of the block for name '
+           'binding\n'
+           'operations.\n'
+           '\n'
+           'If the "global" statement occurs within a block, all uses of the '
+           'name\n'
+           'specified in the statement refer to the binding of that name in '
+           'the\n'
+           'top-level namespace.  Names are resolved in the top-level '
+           'namespace by\n'
+           'searching the global namespace, i.e. the namespace of the module\n'
+           'containing the code block, and the builtins namespace, the '
+           'namespace\n'
+           'of the module "builtins".  The global namespace is searched '
+           'first.  If\n'
+           'the name is not found there, the builtins namespace is searched.  '
+           'The\n'
+           '"global" statement must precede all uses of the name.\n'
+           '\n'
+           'The "global" statement has the same scope as a name binding '
+           'operation\n'
+           'in the same block.  If the nearest enclosing scope for a free '
+           'variable\n'
+           'contains a global statement, the free variable is treated as a '
+           'global.\n'
+           '\n'
+           'The "nonlocal" statement causes corresponding names to refer to\n'
+           'previously bound variables in the nearest enclosing function '
+           'scope.\n'
+           '"SyntaxError" is raised at compile time if the given name does '
+           'not\n'
+           'exist in any enclosing function scope.\n'
+           '\n'
+           'The namespace for a module is automatically created the first time '
+           'a\n'
+           'module is imported.  The main module for a script is always '
+           'called\n'
+           '"__main__".\n'
+           '\n'
+           'Class definition blocks and arguments to "exec()" and "eval()" '
+           'are\n'
+           'special in the context of name resolution. A class definition is '
+           'an\n'
+           'executable statement that may use and define names. These '
+           'references\n'
+           'follow the normal rules for name resolution with an exception '
+           'that\n'
+           'unbound local variables are looked up in the global namespace. '
+           'The\n'
+           'namespace of the class definition becomes the attribute dictionary '
+           'of\n'
+           'the class. The scope of names defined in a class block is limited '
+           'to\n'
+           'the class block; it does not extend to the code blocks of methods '
+           '--\n'
+           'this includes comprehensions and generator expressions since they '
+           'are\n'
+           'implemented using a function scope.  This means that the '
+           'following\n'
+           'will fail:\n'
+           '\n'
+           '   class A:\n'
+           '       a = 42\n'
+           '       b = list(a + i for i in range(10))\n'
+           '\n'
+           '\n'
+           'Builtins and restricted execution\n'
+           '=================================\n'
+           '\n'
+           'The builtins namespace associated with the execution of a code '
+           'block\n'
+           'is actually found by looking up the name "__builtins__" in its '
+           'global\n'
+           'namespace; this should be a dictionary or a module (in the latter '
+           'case\n'
+           "the module's dictionary is used).  By default, when in the "
+           '"__main__"\n'
+           'module, "__builtins__" is the built-in module "builtins"; when in '
+           'any\n'
+           'other module, "__builtins__" is an alias for the dictionary of '
+           'the\n'
+           '"builtins" module itself.  "__builtins__" can be set to a '
+           'user-created\n'
+           'dictionary to create a weak form of restricted execution.\n'
+           '\n'
+           '**CPython implementation detail:** Users should not touch\n'
+           '"__builtins__"; it is strictly an implementation detail.  Users\n'
+           'wanting to override values in the builtins namespace should '
+           '"import"\n'
+           'the "builtins" module and modify its attributes appropriately.\n'
+           '\n'
+           '\n'
+           'Interaction with dynamic features\n'
+           '=================================\n'
+           '\n'
+           'Name resolution of free variables occurs at runtime, not at '
+           'compile\n'
+           'time. This means that the following code will print 42:\n'
+           '\n'
+           '   i = 10\n'
+           '   def f():\n'
+           '       print(i)\n'
+           '   i = 42\n'
+           '   f()\n'
+           '\n'
+           'The "eval()" and "exec()" functions do not have access to the '
+           'full\n'
+           'environment for resolving names.  Names may be resolved in the '
+           'local\n'
+           'and global namespaces of the caller.  Free variables are not '
+           'resolved\n'
+           'in the nearest enclosing namespace, but in the global namespace.  '
+           '[1]\n'
+           'The "exec()" and "eval()" functions have optional arguments to\n'
+           'override the global and local namespace.  If only one namespace '
+           'is\n'
+           'specified, it is used for both.\n',
+ 'nonlocal': '\n'
+             'The "nonlocal" statement\n'
+             '************************\n'
+             '\n'
+             '   nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n'
+             '\n'
+             'The "nonlocal" statement causes the listed identifiers to refer '
+             'to\n'
+             'previously bound variables in the nearest enclosing scope '
+             'excluding\n'
+             'globals. This is important because the default behavior for '
+             'binding is\n'
+             'to search the local namespace first.  The statement allows\n'
+             'encapsulated code to rebind variables outside of the local '
+             'scope\n'
+             'besides the global (module) scope.\n'
+             '\n'
+             'Names listed in a "nonlocal" statement, unlike those listed in '
+             'a\n'
+             '"global" statement, must refer to pre-existing bindings in an\n'
+             'enclosing scope (the scope in which a new binding should be '
+             'created\n'
+             'cannot be determined unambiguously).\n'
+             '\n'
+             'Names listed in a "nonlocal" statement must not collide with '
+             'pre-\n'
+             'existing bindings in the local scope.\n'
+             '\n'
+             'See also: **PEP 3104** - Access to Names in Outer Scopes\n'
+             '\n'
+             '     The specification for the "nonlocal" statement.\n',
+ 'numbers': '\n'
+            'Numeric literals\n'
+            '****************\n'
+            '\n'
+            'There are three types of numeric literals: integers, floating '
+            'point\n'
+            'numbers, and imaginary numbers.  There are no complex literals\n'
+            '(complex numbers can be formed by adding a real number and an\n'
+            'imaginary number).\n'
+            '\n'
+            'Note that numeric literals do not include a sign; a phrase like '
+            '"-1"\n'
+            'is actually an expression composed of the unary operator \'"-"\' '
+            'and the\n'
+            'literal "1".\n',
+ 'numeric-types': '\n'
+                  'Emulating numeric types\n'
+                  '***********************\n'
+                  '\n'
+                  'The following methods can be defined to emulate numeric '
+                  'objects.\n'
+                  'Methods corresponding to operations that are not supported '
+                  'by the\n'
+                  'particular kind of number implemented (e.g., bitwise '
+                  'operations for\n'
+                  'non-integral numbers) should be left undefined.\n'
+                  '\n'
+                  'object.__add__(self, other)\n'
+                  'object.__sub__(self, other)\n'
+                  'object.__mul__(self, other)\n'
+                  'object.__matmul__(self, other)\n'
+                  'object.__truediv__(self, other)\n'
+                  'object.__floordiv__(self, other)\n'
+                  'object.__mod__(self, other)\n'
+                  'object.__divmod__(self, other)\n'
+                  'object.__pow__(self, other[, modulo])\n'
+                  'object.__lshift__(self, other)\n'
+                  'object.__rshift__(self, other)\n'
+                  'object.__and__(self, other)\n'
+                  'object.__xor__(self, other)\n'
+                  'object.__or__(self, other)\n'
+                  '\n'
+                  '   These methods are called to implement the binary '
+                  'arithmetic\n'
+                  '   operations ("+", "-", "*", "@", "/", "//", "%", '
+                  '"divmod()",\n'
+                  '   "pow()", "**", "<<", ">>", "&", "^", "|").  For '
+                  'instance, to\n'
+                  '   evaluate the expression "x + y", where *x* is an '
+                  'instance of a\n'
+                  '   class that has an "__add__()" method, "x.__add__(y)" is '
+                  'called.\n'
+                  '   The "__divmod__()" method should be the equivalent to '
+                  'using\n'
+                  '   "__floordiv__()" and "__mod__()"; it should not be '
+                  'related to\n'
+                  '   "__truediv__()".  Note that "__pow__()" should be '
+                  'defined to accept\n'
+                  '   an optional third argument if the ternary version of the '
+                  'built-in\n'
+                  '   "pow()" function is to be supported.\n'
+                  '\n'
+                  '   If one of those methods does not support the operation '
+                  'with the\n'
+                  '   supplied arguments, it should return "NotImplemented".\n'
+                  '\n'
+                  'object.__radd__(self, other)\n'
+                  'object.__rsub__(self, other)\n'
+                  'object.__rmul__(self, other)\n'
+                  'object.__rmatmul__(self, other)\n'
+                  'object.__rtruediv__(self, other)\n'
+                  'object.__rfloordiv__(self, other)\n'
+                  'object.__rmod__(self, other)\n'
+                  'object.__rdivmod__(self, other)\n'
+                  'object.__rpow__(self, other)\n'
+                  'object.__rlshift__(self, other)\n'
+                  'object.__rrshift__(self, other)\n'
+                  'object.__rand__(self, other)\n'
+                  'object.__rxor__(self, other)\n'
+                  'object.__ror__(self, other)\n'
+                  '\n'
+                  '   These methods are called to implement the binary '
+                  'arithmetic\n'
+                  '   operations ("+", "-", "*", "@", "/", "//", "%", '
+                  '"divmod()",\n'
+                  '   "pow()", "**", "<<", ">>", "&", "^", "|") with reflected '
+                  '(swapped)\n'
+                  '   operands.  These functions are only called if the left '
+                  'operand does\n'
+                  '   not support the corresponding operation and the operands '
+                  'are of\n'
+                  '   different types. [2] For instance, to evaluate the '
+                  'expression "x -\n'
+                  '   y", where *y* is an instance of a class that has an '
+                  '"__rsub__()"\n'
+                  '   method, "y.__rsub__(x)" is called if "x.__sub__(y)" '
+                  'returns\n'
+                  '   *NotImplemented*.\n'
+                  '\n'
+                  '   Note that ternary "pow()" will not try calling '
+                  '"__rpow__()" (the\n'
+                  '   coercion rules would become too complicated).\n'
+                  '\n'
+                  "   Note: If the right operand's type is a subclass of the "
+                  'left\n'
+                  "     operand's type and that subclass provides the "
+                  'reflected method\n'
+                  '     for the operation, this method will be called before '
+                  'the left\n'
+                  "     operand's non-reflected method.  This behavior allows "
+                  'subclasses\n'
+                  "     to override their ancestors' operations.\n"
+                  '\n'
+                  'object.__iadd__(self, other)\n'
+                  'object.__isub__(self, other)\n'
+                  'object.__imul__(self, other)\n'
+                  'object.__imatmul__(self, other)\n'
+                  'object.__itruediv__(self, other)\n'
+                  'object.__ifloordiv__(self, other)\n'
+                  'object.__imod__(self, other)\n'
+                  'object.__ipow__(self, other[, modulo])\n'
+                  'object.__ilshift__(self, other)\n'
+                  'object.__irshift__(self, other)\n'
+                  'object.__iand__(self, other)\n'
+                  'object.__ixor__(self, other)\n'
+                  'object.__ior__(self, other)\n'
+                  '\n'
+                  '   These methods are called to implement the augmented '
+                  'arithmetic\n'
+                  '   assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", '
+                  '"**=",\n'
+                  '   "<<=", ">>=", "&=", "^=", "|=").  These methods should '
+                  'attempt to\n'
+                  '   do the operation in-place (modifying *self*) and return '
+                  'the result\n'
+                  '   (which could be, but does not have to be, *self*).  If a '
+                  'specific\n'
+                  '   method is not defined, the augmented assignment falls '
+                  'back to the\n'
+                  '   normal methods.  For instance, if *x* is an instance of '
+                  'a class\n'
+                  '   with an "__iadd__()" method, "x += y" is equivalent to '
+                  '"x =\n'
+                  '   x.__iadd__(y)" . Otherwise, "x.__add__(y)" and '
+                  '"y.__radd__(x)" are\n'
+                  '   considered, as with the evaluation of "x + y". In '
+                  'certain\n'
+                  '   situations, augmented assignment can result in '
+                  'unexpected errors\n'
+                  "   (see *Why does a_tuple[i] += ['item'] raise an exception "
+                  'when the\n'
+                  '   addition works?*), but this behavior is in fact part of '
+                  'the data\n'
+                  '   model.\n'
+                  '\n'
+                  'object.__neg__(self)\n'
+                  'object.__pos__(self)\n'
+                  'object.__abs__(self)\n'
+                  'object.__invert__(self)\n'
+                  '\n'
+                  '   Called to implement the unary arithmetic operations '
+                  '("-", "+",\n'
+                  '   "abs()" and "~").\n'
+                  '\n'
+                  'object.__complex__(self)\n'
+                  'object.__int__(self)\n'
+                  'object.__float__(self)\n'
+                  'object.__round__(self[, n])\n'
+                  '\n'
+                  '   Called to implement the built-in functions "complex()", '
+                  '"int()",\n'
+                  '   "float()" and "round()".  Should return a value of the '
+                  'appropriate\n'
+                  '   type.\n'
+                  '\n'
+                  'object.__index__(self)\n'
+                  '\n'
+                  '   Called to implement "operator.index()", and whenever '
+                  'Python needs\n'
+                  '   to losslessly convert the numeric object to an integer '
+                  'object (such\n'
+                  '   as in slicing, or in the built-in "bin()", "hex()" and '
+                  '"oct()"\n'
+                  '   functions). Presence of this method indicates that the '
+                  'numeric\n'
+                  '   object is an integer type.  Must return an integer.\n'
+                  '\n'
+                  '   Note: In order to have a coherent integer type class, '
+                  'when\n'
+                  '     "__index__()" is defined "__int__()" should also be '
+                  'defined, and\n'
+                  '     both should return the same value.\n',
+ 'objects': '\n'
+            'Objects, values and types\n'
+            '*************************\n'
+            '\n'
+            "*Objects* are Python's abstraction for data.  All data in a "
+            'Python\n'
+            'program is represented by objects or by relations between '
+            'objects. (In\n'
+            'a sense, and in conformance to Von Neumann\'s model of a "stored\n'
+            'program computer," code is also represented by objects.)\n'
+            '\n'
+            "Every object has an identity, a type and a value.  An object's\n"
+            '*identity* never changes once it has been created; you may think '
+            'of it\n'
+            'as the object\'s address in memory.  The \'"is"\' operator '
+            'compares the\n'
+            'identity of two objects; the "id()" function returns an integer\n'
+            'representing its identity.\n'
+            '\n'
+            '**CPython implementation detail:** For CPython, "id(x)" is the '
+            'memory\n'
+            'address where "x" is stored.\n'
+            '\n'
+            "An object's type determines the operations that the object "
+            'supports\n'
+            '(e.g., "does it have a length?") and also defines the possible '
+            'values\n'
+            'for objects of that type.  The "type()" function returns an '
+            "object's\n"
+            'type (which is an object itself).  Like its identity, an '
+            "object's\n"
+            '*type* is also unchangeable. [1]\n'
+            '\n'
+            'The *value* of some objects can change.  Objects whose value can\n'
+            'change are said to be *mutable*; objects whose value is '
+            'unchangeable\n'
+            'once they are created are called *immutable*. (The value of an\n'
+            'immutable container object that contains a reference to a '
+            'mutable\n'
+            "object can change when the latter's value is changed; however "
+            'the\n'
+            'container is still considered immutable, because the collection '
+            'of\n'
+            'objects it contains cannot be changed.  So, immutability is not\n'
+            'strictly the same as having an unchangeable value, it is more '
+            'subtle.)\n'
+            "An object's mutability is determined by its type; for instance,\n"
+            'numbers, strings and tuples are immutable, while dictionaries '
+            'and\n'
+            'lists are mutable.\n'
+            '\n'
+            'Objects are never explicitly destroyed; however, when they '
+            'become\n'
+            'unreachable they may be garbage-collected.  An implementation is\n'
+            'allowed to postpone garbage collection or omit it altogether --- '
+            'it is\n'
+            'a matter of implementation quality how garbage collection is\n'
+            'implemented, as long as no objects are collected that are still\n'
+            'reachable.\n'
+            '\n'
+            '**CPython implementation detail:** CPython currently uses a '
+            'reference-\n'
+            'counting scheme with (optional) delayed detection of cyclically '
+            'linked\n'
+            'garbage, which collects most objects as soon as they become\n'
+            'unreachable, but is not guaranteed to collect garbage containing\n'
+            'circular references.  See the documentation of the "gc" module '
+            'for\n'
+            'information on controlling the collection of cyclic garbage. '
+            'Other\n'
+            'implementations act differently and CPython may change. Do not '
+            'depend\n'
+            'on immediate finalization of objects when they become unreachable '
+            '(so\n'
+            'you should always close files explicitly).\n'
+            '\n'
+            "Note that the use of the implementation's tracing or debugging\n"
+            'facilities may keep objects alive that would normally be '
+            'collectable.\n'
+            'Also note that catching an exception with a \'"try"..."except"\'\n'
+            'statement may keep objects alive.\n'
+            '\n'
+            'Some objects contain references to "external" resources such as '
+            'open\n'
+            'files or windows.  It is understood that these resources are '
+            'freed\n'
+            'when the object is garbage-collected, but since garbage '
+            'collection is\n'
+            'not guaranteed to happen, such objects also provide an explicit '
+            'way to\n'
+            'release the external resource, usually a "close()" method. '
+            'Programs\n'
+            'are strongly recommended to explicitly close such objects.  The\n'
+            '\'"try"..."finally"\' statement and the \'"with"\' statement '
+            'provide\n'
+            'convenient ways to do this.\n'
+            '\n'
+            'Some objects contain references to other objects; these are '
+            'called\n'
+            '*containers*. Examples of containers are tuples, lists and\n'
+            "dictionaries.  The references are part of a container's value.  "
+            'In\n'
+            'most cases, when we talk about the value of a container, we imply '
+            'the\n'
+            'values, not the identities of the contained objects; however, '
+            'when we\n'
+            'talk about the mutability of a container, only the identities of '
+            'the\n'
+            'immediately contained objects are implied.  So, if an immutable\n'
+            'container (like a tuple) contains a reference to a mutable '
+            'object, its\n'
+            'value changes if that mutable object is changed.\n'
+            '\n'
+            'Types affect almost all aspects of object behavior.  Even the\n'
+            'importance of object identity is affected in some sense: for '
+            'immutable\n'
+            'types, operations that compute new values may actually return a\n'
+            'reference to any existing object with the same type and value, '
+            'while\n'
+            'for mutable objects this is not allowed.  E.g., after "a = 1; b = '
+            '1",\n'
+            '"a" and "b" may or may not refer to the same object with the '
+            'value\n'
+            'one, depending on the implementation, but after "c = []; d = []", '
+            '"c"\n'
+            'and "d" are guaranteed to refer to two different, unique, newly\n'
+            'created empty lists. (Note that "c = d = []" assigns the same '
+            'object\n'
+            'to both "c" and "d".)\n',
+ 'operator-summary': '\n'
+                     'Operator precedence\n'
+                     '*******************\n'
+                     '\n'
+                     'The following table summarizes the operator precedence '
+                     'in Python, from\n'
+                     'lowest precedence (least binding) to highest precedence '
+                     '(most\n'
+                     'binding).  Operators in the same box have the same '
+                     'precedence.  Unless\n'
+                     'the syntax is explicitly given, operators are binary.  '
+                     'Operators in\n'
+                     'the same box group left to right (except for '
+                     'exponentiation, which\n'
+                     'groups from right to left).\n'
+                     '\n'
+                     'Note that comparisons, membership tests, and identity '
+                     'tests, all have\n'
+                     'the same precedence and have a left-to-right chaining '
+                     'feature as\n'
+                     'described in the *Comparisons* section.\n'
+                     '\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| Operator                                        | '
+                     'Description                           |\n'
+                     '+=================================================+=======================================+\n'
+                     '| "lambda"                                        | '
+                     'Lambda expression                     |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "if" -- "else"                                  | '
+                     'Conditional expression                |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "or"                                            | '
+                     'Boolean OR                            |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "and"                                           | '
+                     'Boolean AND                           |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "not" "x"                                       | '
+                     'Boolean NOT                           |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "in", "not in", "is", "is not", "<", "<=", ">", | '
+                     'Comparisons, including membership     |\n'
+                     '| ">=", "!=", "=="                                | '
+                     'tests and identity tests              |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "|"                                             | '
+                     'Bitwise OR                            |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "^"                                             | '
+                     'Bitwise XOR                           |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "&"                                             | '
+                     'Bitwise AND                           |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "<<", ">>"                                      | '
+                     'Shifts                                |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "+", "-"                                        | '
+                     'Addition and subtraction              |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "*", "@", "/", "//", "%"                        | '
+                     'Multiplication, matrix multiplication |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "+x", "-x", "~x"                                | '
+                     'Positive, negative, bitwise NOT       |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "**"                                            | '
+                     'Exponentiation [6]                    |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "await" "x"                                     | '
+                     'Await expression                      |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "x[index]", "x[index:index]",                   | '
+                     'Subscription, slicing, call,          |\n'
+                     '| "x(arguments...)", "x.attribute"                | '
+                     'attribute reference                   |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '| "(expressions...)", "[expressions...]", "{key:  | '
+                     'Binding or tuple display, list        |\n'
+                     '| value...}", "{expressions...}"                  | '
+                     'display, dictionary display, set      |\n'
+                     '+-------------------------------------------------+---------------------------------------+\n'
+                     '\n'
+                     '-[ Footnotes ]-\n'
+                     '\n'
+                     '[1] While "abs(x%y) < abs(y)" is true mathematically, '
+                     'for floats\n'
+                     '    it may not be true numerically due to roundoff.  For '
+                     'example, and\n'
+                     '    assuming a platform on which a Python float is an '
+                     'IEEE 754 double-\n'
+                     '    precision number, in order that "-1e-100 % 1e100" '
+                     'have the same\n'
+                     '    sign as "1e100", the computed result is "-1e-100 + '
+                     '1e100", which\n'
+                     '    is numerically exactly equal to "1e100".  The '
+                     'function\n'
+                     '    "math.fmod()" returns a result whose sign matches '
+                     'the sign of the\n'
+                     '    first argument instead, and so returns "-1e-100" in '
+                     'this case.\n'
+                     '    Which approach is more appropriate depends on the '
+                     'application.\n'
+                     '\n'
+                     '[2] If x is very close to an exact integer multiple of '
+                     "y, it's\n"
+                     '    possible for "x//y" to be one larger than '
+                     '"(x-x%y)//y" due to\n'
+                     '    rounding.  In such cases, Python returns the latter '
+                     'result, in\n'
+                     '    order to preserve that "divmod(x,y)[0] * y + x % y" '
+                     'be very close\n'
+                     '    to "x".\n'
+                     '\n'
+                     '[3] The Unicode standard distinguishes between *code '
+                     'points* (e.g.\n'
+                     '    U+0041) and *abstract characters* (e.g. "LATIN '
+                     'CAPITAL LETTER A").\n'
+                     '    While most abstract characters in Unicode are only '
+                     'represented\n'
+                     '    using one code point, there is a number of abstract '
+                     'characters\n'
+                     '    that can in addition be represented using a sequence '
+                     'of more than\n'
+                     '    one code point.  For example, the abstract character '
+                     '"LATIN\n'
+                     '    CAPITAL LETTER C WITH CEDILLA" can be represented as '
+                     'a single\n'
+                     '    *precomposed character* at code position U+00C7, or '
+                     'as a sequence\n'
+                     '    of a *base character* at code position U+0043 (LATIN '
+                     'CAPITAL\n'
+                     '    LETTER C), followed by a *combining character* at '
+                     'code position\n'
+                     '    U+0327 (COMBINING CEDILLA).\n'
+                     '\n'
+                     '    The comparison operators on strings compare at the '
+                     'level of\n'
+                     '    Unicode code points. This may be counter-intuitive '
+                     'to humans.  For\n'
+                     '    example, ""\\u00C7" == "\\u0043\\u0327"" is "False", '
+                     'even though both\n'
+                     '    strings represent the same abstract character "LATIN '
+                     'CAPITAL\n'
+                     '    LETTER C WITH CEDILLA".\n'
+                     '\n'
+                     '    To compare strings at the level of abstract '
+                     'characters (that is,\n'
+                     '    in a way intuitive to humans), use '
+                     '"unicodedata.normalize()".\n'
+                     '\n'
+                     '[4] Due to automatic garbage-collection, free lists, and '
+                     'the\n'
+                     '    dynamic nature of descriptors, you may notice '
+                     'seemingly unusual\n'
+                     '    behaviour in certain uses of the "is" operator, like '
+                     'those\n'
+                     '    involving comparisons between instance methods, or '
+                     'constants.\n'
+                     '    Check their documentation for more info.\n'
+                     '\n'
+                     '[5] The "%" operator is also used for string formatting; '
+                     'the same\n'
+                     '    precedence applies.\n'
+                     '\n'
+                     '[6] The power operator "**" binds less tightly than an '
+                     'arithmetic\n'
+                     '    or bitwise unary operator on its right, that is, '
+                     '"2**-1" is "0.5".\n',
+ 'pass': '\n'
+         'The "pass" statement\n'
+         '********************\n'
+         '\n'
+         '   pass_stmt ::= "pass"\n'
+         '\n'
+         '"pass" is a null operation --- when it is executed, nothing '
+         'happens.\n'
+         'It is useful as a placeholder when a statement is required\n'
+         'syntactically, but no code needs to be executed, for example:\n'
+         '\n'
+         '   def f(arg): pass    # a function that does nothing (yet)\n'
+         '\n'
+         '   class C: pass       # a class with no methods (yet)\n',
+ 'power': '\n'
+          'The power operator\n'
+          '******************\n'
+          '\n'
+          'The power operator binds more tightly than unary operators on its\n'
+          'left; it binds less tightly than unary operators on its right.  '
+          'The\n'
+          'syntax is:\n'
+          '\n'
+          '   power ::= ( await_expr | primary ) ["**" u_expr]\n'
+          '\n'
+          'Thus, in an unparenthesized sequence of power and unary operators, '
+          'the\n'
+          'operators are evaluated from right to left (this does not '
+          'constrain\n'
+          'the evaluation order for the operands): "-1**2" results in "-1".\n'
+          '\n'
+          'The power operator has the same semantics as the built-in "pow()"\n'
+          'function, when called with two arguments: it yields its left '
+          'argument\n'
+          'raised to the power of its right argument.  The numeric arguments '
+          'are\n'
+          'first converted to a common type, and the result is of that type.\n'
+          '\n'
+          'For int operands, the result has the same type as the operands '
+          'unless\n'
+          'the second argument is negative; in that case, all arguments are\n'
+          'converted to float and a float result is delivered. For example,\n'
+          '"10**2" returns "100", but "10**-2" returns "0.01".\n'
+          '\n'
+          'Raising "0.0" to a negative power results in a '
+          '"ZeroDivisionError".\n'
+          'Raising a negative number to a fractional power results in a '
+          '"complex"\n'
+          'number. (In earlier versions it raised a "ValueError".)\n',
+ 'raise': '\n'
+          'The "raise" statement\n'
+          '*********************\n'
+          '\n'
+          '   raise_stmt ::= "raise" [expression ["from" expression]]\n'
+          '\n'
+          'If no expressions are present, "raise" re-raises the last '
+          'exception\n'
+          'that was active in the current scope.  If no exception is active '
+          'in\n'
+          'the current scope, a "RuntimeError" exception is raised indicating\n'
+          'that this is an error.\n'
+          '\n'
+          'Otherwise, "raise" evaluates the first expression as the exception\n'
+          'object.  It must be either a subclass or an instance of\n'
+          '"BaseException". If it is a class, the exception instance will be\n'
+          'obtained when needed by instantiating the class with no arguments.\n'
+          '\n'
+          "The *type* of the exception is the exception instance's class, the\n"
+          '*value* is the instance itself.\n'
+          '\n'
+          'A traceback object is normally created automatically when an '
+          'exception\n'
+          'is raised and attached to it as the "__traceback__" attribute, '
+          'which\n'
+          'is writable. You can create an exception and set your own traceback '
+          'in\n'
+          'one step using the "with_traceback()" exception method (which '
+          'returns\n'
+          'the same exception instance, with its traceback set to its '
+          'argument),\n'
+          'like so:\n'
+          '\n'
+          '   raise Exception("foo occurred").with_traceback(tracebackobj)\n'
+          '\n'
+          'The "from" clause is used for exception chaining: if given, the '
+          'second\n'
+          '*expression* must be another exception class or instance, which '
+          'will\n'
+          'then be attached to the raised exception as the "__cause__" '
+          'attribute\n'
+          '(which is writable).  If the raised exception is not handled, both\n'
+          'exceptions will be printed:\n'
+          '\n'
+          '   >>> try:\n'
+          '   ...     print(1 / 0)\n'
+          '   ... except Exception as exc:\n'
+          '   ...     raise RuntimeError("Something bad happened") from exc\n'
+          '   ...\n'
+          '   Traceback (most recent call last):\n'
+          '     File "<stdin>", line 2, in <module>\n'
+          '   ZeroDivisionError: int division or modulo by zero\n'
+          '\n'
+          '   The above exception was the direct cause of the following '
+          'exception:\n'
+          '\n'
+          '   Traceback (most recent call last):\n'
+          '     File "<stdin>", line 4, in <module>\n'
+          '   RuntimeError: Something bad happened\n'
+          '\n'
+          'A similar mechanism works implicitly if an exception is raised '
+          'inside\n'
+          'an exception handler or a "finally" clause: the previous exception '
+          'is\n'
+          'then attached as the new exception\'s "__context__" attribute:\n'
+          '\n'
+          '   >>> try:\n'
+          '   ...     print(1 / 0)\n'
+          '   ... except:\n'
+          '   ...     raise RuntimeError("Something bad happened")\n'
+          '   ...\n'
+          '   Traceback (most recent call last):\n'
+          '     File "<stdin>", line 2, in <module>\n'
+          '   ZeroDivisionError: int division or modulo by zero\n'
+          '\n'
+          '   During handling of the above exception, another exception '
+          'occurred:\n'
+          '\n'
+          '   Traceback (most recent call last):\n'
+          '     File "<stdin>", line 4, in <module>\n'
+          '   RuntimeError: Something bad happened\n'
+          '\n'
+          'Additional information on exceptions can be found in section\n'
+          '*Exceptions*, and information about handling exceptions is in '
+          'section\n'
+          '*The try statement*.\n',
+ 'return': '\n'
+           'The "return" statement\n'
+           '**********************\n'
+           '\n'
+           '   return_stmt ::= "return" [expression_list]\n'
+           '\n'
+           '"return" may only occur syntactically nested in a function '
+           'definition,\n'
+           'not within a nested class definition.\n'
+           '\n'
+           'If an expression list is present, it is evaluated, else "None" is\n'
+           'substituted.\n'
+           '\n'
+           '"return" leaves the current function call with the expression list '
+           '(or\n'
+           '"None") as return value.\n'
+           '\n'
+           'When "return" passes control out of a "try" statement with a '
+           '"finally"\n'
+           'clause, that "finally" clause is executed before really leaving '
+           'the\n'
+           'function.\n'
+           '\n'
+           'In a generator function, the "return" statement indicates that '
+           'the\n'
+           'generator is done and will cause "StopIteration" to be raised. '
+           'The\n'
+           'returned value (if any) is used as an argument to construct\n'
+           '"StopIteration" and becomes the "StopIteration.value" attribute.\n',
+ 'sequence-types': '\n'
+                   'Emulating container types\n'
+                   '*************************\n'
+                   '\n'
+                   'The following methods can be defined to implement '
+                   'container objects.\n'
+                   'Containers usually are sequences (such as lists or tuples) '
+                   'or mappings\n'
+                   '(like dictionaries), but can represent other containers as '
+                   'well.  The\n'
+                   'first set of methods is used either to emulate a sequence '
+                   'or to\n'
+                   'emulate a mapping; the difference is that for a sequence, '
+                   'the\n'
+                   'allowable keys should be the integers *k* for which "0 <= '
+                   'k < N" where\n'
+                   '*N* is the length of the sequence, or slice objects, which '
+                   'define a\n'
+                   'range of items.  It is also recommended that mappings '
+                   'provide the\n'
+                   'methods "keys()", "values()", "items()", "get()", '
+                   '"clear()",\n'
+                   '"setdefault()", "pop()", "popitem()", "copy()", and '
+                   '"update()"\n'
+                   "behaving similar to those for Python's standard dictionary "
+                   'objects.\n'
+                   'The "collections" module provides a "MutableMapping" '
+                   'abstract base\n'
+                   'class to help create those methods from a base set of '
+                   '"__getitem__()",\n'
+                   '"__setitem__()", "__delitem__()", and "keys()". Mutable '
+                   'sequences\n'
+                   'should provide methods "append()", "count()", "index()", '
+                   '"extend()",\n'
+                   '"insert()", "pop()", "remove()", "reverse()" and "sort()", '
+                   'like Python\n'
+                   'standard list objects.  Finally, sequence types should '
+                   'implement\n'
+                   'addition (meaning concatenation) and multiplication '
+                   '(meaning\n'
+                   'repetition) by defining the methods "__add__()", '
+                   '"__radd__()",\n'
+                   '"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" '
+                   'described\n'
+                   'below; they should not define other numerical operators.  '
+                   'It is\n'
+                   'recommended that both mappings and sequences implement '
+                   'the\n'
+                   '"__contains__()" method to allow efficient use of the "in" '
+                   'operator;\n'
+                   'for mappings, "in" should search the mapping\'s keys; for '
+                   'sequences, it\n'
+                   'should search through the values.  It is further '
+                   'recommended that both\n'
+                   'mappings and sequences implement the "__iter__()" method '
+                   'to allow\n'
+                   'efficient iteration through the container; for mappings, '
+                   '"__iter__()"\n'
+                   'should be the same as "keys()"; for sequences, it should '
+                   'iterate\n'
+                   'through the values.\n'
+                   '\n'
+                   'object.__len__(self)\n'
+                   '\n'
+                   '   Called to implement the built-in function "len()".  '
+                   'Should return\n'
+                   '   the length of the object, an integer ">=" 0.  Also, an '
+                   'object that\n'
+                   '   doesn\'t define a "__bool__()" method and whose '
+                   '"__len__()" method\n'
+                   '   returns zero is considered to be false in a Boolean '
+                   'context.\n'
+                   '\n'
+                   '   **CPython implementation detail:** In CPython, the '
+                   'length is\n'
+                   '   required to be at most "sys.maxsize". If the length is '
+                   'larger than\n'
+                   '   "sys.maxsize" some features (such as "len()") may '
+                   'raise\n'
+                   '   "OverflowError".  To prevent raising "OverflowError" by '
+                   'truth value\n'
+                   '   testing, an object must define a "__bool__()" method.\n'
+                   '\n'
+                   'object.__length_hint__(self)\n'
+                   '\n'
+                   '   Called to implement "operator.length_hint()". Should '
+                   'return an\n'
+                   '   estimated length for the object (which may be greater '
+                   'or less than\n'
+                   '   the actual length). The length must be an integer ">=" '
+                   '0. This\n'
+                   '   method is purely an optimization and is never required '
+                   'for\n'
+                   '   correctness.\n'
+                   '\n'
+                   '   New in version 3.4.\n'
+                   '\n'
+                   'Note: Slicing is done exclusively with the following three '
+                   'methods.\n'
+                   '  A call like\n'
+                   '\n'
+                   '     a[1:2] = b\n'
+                   '\n'
+                   '  is translated to\n'
+                   '\n'
+                   '     a[slice(1, 2, None)] = b\n'
+                   '\n'
+                   '  and so forth.  Missing slice items are always filled in '
+                   'with "None".\n'
+                   '\n'
+                   'object.__getitem__(self, key)\n'
+                   '\n'
+                   '   Called to implement evaluation of "self[key]". For '
+                   'sequence types,\n'
+                   '   the accepted keys should be integers and slice '
+                   'objects.  Note that\n'
+                   '   the special interpretation of negative indexes (if the '
+                   'class wishes\n'
+                   '   to emulate a sequence type) is up to the '
+                   '"__getitem__()" method. If\n'
+                   '   *key* is of an inappropriate type, "TypeError" may be '
+                   'raised; if of\n'
+                   '   a value outside the set of indexes for the sequence '
+                   '(after any\n'
+                   '   special interpretation of negative values), '
+                   '"IndexError" should be\n'
+                   '   raised. For mapping types, if *key* is missing (not in '
+                   'the\n'
+                   '   container), "KeyError" should be raised.\n'
+                   '\n'
+                   '   Note: "for" loops expect that an "IndexError" will be '
+                   'raised for\n'
+                   '     illegal indexes to allow proper detection of the end '
+                   'of the\n'
+                   '     sequence.\n'
+                   '\n'
+                   'object.__missing__(self, key)\n'
+                   '\n'
+                   '   Called by "dict"."__getitem__()" to implement '
+                   '"self[key]" for dict\n'
+                   '   subclasses when key is not in the dictionary.\n'
+                   '\n'
+                   'object.__setitem__(self, key, value)\n'
+                   '\n'
+                   '   Called to implement assignment to "self[key]".  Same '
+                   'note as for\n'
+                   '   "__getitem__()".  This should only be implemented for '
+                   'mappings if\n'
+                   '   the objects support changes to the values for keys, or '
+                   'if new keys\n'
+                   '   can be added, or for sequences if elements can be '
+                   'replaced.  The\n'
+                   '   same exceptions should be raised for improper *key* '
+                   'values as for\n'
+                   '   the "__getitem__()" method.\n'
+                   '\n'
+                   'object.__delitem__(self, key)\n'
+                   '\n'
+                   '   Called to implement deletion of "self[key]".  Same note '
+                   'as for\n'
+                   '   "__getitem__()".  This should only be implemented for '
+                   'mappings if\n'
+                   '   the objects support removal of keys, or for sequences '
+                   'if elements\n'
+                   '   can be removed from the sequence.  The same exceptions '
+                   'should be\n'
+                   '   raised for improper *key* values as for the '
+                   '"__getitem__()" method.\n'
+                   '\n'
+                   'object.__iter__(self)\n'
+                   '\n'
+                   '   This method is called when an iterator is required for '
+                   'a container.\n'
+                   '   This method should return a new iterator object that '
+                   'can iterate\n'
+                   '   over all the objects in the container.  For mappings, '
+                   'it should\n'
+                   '   iterate over the keys of the container.\n'
+                   '\n'
+                   '   Iterator objects also need to implement this method; '
+                   'they are\n'
+                   '   required to return themselves.  For more information on '
+                   'iterator\n'
+                   '   objects, see *Iterator Types*.\n'
+                   '\n'
+                   'object.__reversed__(self)\n'
+                   '\n'
+                   '   Called (if present) by the "reversed()" built-in to '
+                   'implement\n'
+                   '   reverse iteration.  It should return a new iterator '
+                   'object that\n'
+                   '   iterates over all the objects in the container in '
+                   'reverse order.\n'
+                   '\n'
+                   '   If the "__reversed__()" method is not provided, the '
+                   '"reversed()"\n'
+                   '   built-in will fall back to using the sequence protocol '
+                   '("__len__()"\n'
+                   '   and "__getitem__()").  Objects that support the '
+                   'sequence protocol\n'
+                   '   should only provide "__reversed__()" if they can '
+                   'provide an\n'
+                   '   implementation that is more efficient than the one '
+                   'provided by\n'
+                   '   "reversed()".\n'
+                   '\n'
+                   'The membership test operators ("in" and "not in") are '
+                   'normally\n'
+                   'implemented as an iteration through a sequence.  However, '
+                   'container\n'
+                   'objects can supply the following special method with a '
+                   'more efficient\n'
+                   'implementation, which also does not require the object be '
+                   'a sequence.\n'
+                   '\n'
+                   'object.__contains__(self, item)\n'
+                   '\n'
+                   '   Called to implement membership test operators.  Should '
+                   'return true\n'
+                   '   if *item* is in *self*, false otherwise.  For mapping '
+                   'objects, this\n'
+                   '   should consider the keys of the mapping rather than the '
+                   'values or\n'
+                   '   the key-item pairs.\n'
+                   '\n'
+                   '   For objects that don\'t define "__contains__()", the '
+                   'membership test\n'
+                   '   first tries iteration via "__iter__()", then the old '
+                   'sequence\n'
+                   '   iteration protocol via "__getitem__()", see *this '
+                   'section in the\n'
+                   '   language reference*.\n',
+ 'shifting': '\n'
+             'Shifting operations\n'
+             '*******************\n'
+             '\n'
+             'The shifting operations have lower priority than the arithmetic\n'
+             'operations:\n'
+             '\n'
+             '   shift_expr ::= a_expr | shift_expr ( "<<" | ">>" ) a_expr\n'
+             '\n'
+             'These operators accept integers as arguments.  They shift the '
+             'first\n'
+             'argument to the left or right by the number of bits given by '
+             'the\n'
+             'second argument.\n'
+             '\n'
+             'A right shift by *n* bits is defined as floor division by '
+             '"pow(2,n)".\n'
+             'A left shift by *n* bits is defined as multiplication with '
+             '"pow(2,n)".\n'
+             '\n'
+             'Note: In the current implementation, the right-hand operand is\n'
+             '  required to be at most "sys.maxsize".  If the right-hand '
+             'operand is\n'
+             '  larger than "sys.maxsize" an "OverflowError" exception is '
+             'raised.\n',
+ 'slicings': '\n'
+             'Slicings\n'
+             '********\n'
+             '\n'
+             'A slicing selects a range of items in a sequence object (e.g., '
+             'a\n'
+             'string, tuple or list).  Slicings may be used as expressions or '
+             'as\n'
+             'targets in assignment or "del" statements.  The syntax for a '
+             'slicing:\n'
+             '\n'
+             '   slicing      ::= primary "[" slice_list "]"\n'
+             '   slice_list   ::= slice_item ("," slice_item)* [","]\n'
+             '   slice_item   ::= expression | proper_slice\n'
+             '   proper_slice ::= [lower_bound] ":" [upper_bound] [ ":" '
+             '[stride] ]\n'
+             '   lower_bound  ::= expression\n'
+             '   upper_bound  ::= expression\n'
+             '   stride       ::= expression\n'
+             '\n'
+             'There is ambiguity in the formal syntax here: anything that '
+             'looks like\n'
+             'an expression list also looks like a slice list, so any '
+             'subscription\n'
+             'can be interpreted as a slicing.  Rather than further '
+             'complicating the\n'
+             'syntax, this is disambiguated by defining that in this case the\n'
+             'interpretation as a subscription takes priority over the\n'
+             'interpretation as a slicing (this is the case if the slice list\n'
+             'contains no proper slice).\n'
+             '\n'
+             'The semantics for a slicing are as follows.  The primary is '
+             'indexed\n'
+             '(using the same "__getitem__()" method as normal subscription) '
+             'with a\n'
+             'key that is constructed from the slice list, as follows.  If the '
+             'slice\n'
+             'list contains at least one comma, the key is a tuple containing '
+             'the\n'
+             'conversion of the slice items; otherwise, the conversion of the '
+             'lone\n'
+             'slice item is the key.  The conversion of a slice item that is '
+             'an\n'
+             'expression is that expression.  The conversion of a proper slice '
+             'is a\n'
+             'slice object (see section *The standard type hierarchy*) whose\n'
+             '"start", "stop" and "step" attributes are the values of the\n'
+             'expressions given as lower bound, upper bound and stride,\n'
+             'respectively, substituting "None" for missing expressions.\n',
+ 'specialattrs': '\n'
+                 'Special Attributes\n'
+                 '******************\n'
+                 '\n'
+                 'The implementation adds a few special read-only attributes '
+                 'to several\n'
+                 'object types, where they are relevant.  Some of these are '
+                 'not reported\n'
+                 'by the "dir()" built-in function.\n'
+                 '\n'
+                 'object.__dict__\n'
+                 '\n'
+                 '   A dictionary or other mapping object used to store an '
+                 "object's\n"
+                 '   (writable) attributes.\n'
+                 '\n'
+                 'instance.__class__\n'
+                 '\n'
+                 '   The class to which a class instance belongs.\n'
+                 '\n'
+                 'class.__bases__\n'
+                 '\n'
+                 '   The tuple of base classes of a class object.\n'
+                 '\n'
+                 'definition.__name__\n'
+                 '\n'
+                 '   The name of the class, function, method, descriptor, or '
+                 'generator\n'
+                 '   instance.\n'
+                 '\n'
+                 'definition.__qualname__\n'
+                 '\n'
+                 '   The *qualified name* of the class, function, method, '
+                 'descriptor, or\n'
+                 '   generator instance.\n'
+                 '\n'
+                 '   New in version 3.3.\n'
+                 '\n'
+                 'class.__mro__\n'
+                 '\n'
+                 '   This attribute is a tuple of classes that are considered '
+                 'when\n'
+                 '   looking for base classes during method resolution.\n'
+                 '\n'
+                 'class.mro()\n'
+                 '\n'
+                 '   This method can be overridden by a metaclass to customize '
+                 'the\n'
+                 '   method resolution order for its instances.  It is called '
+                 'at class\n'
+                 '   instantiation, and its result is stored in "__mro__".\n'
+                 '\n'
+                 'class.__subclasses__()\n'
+                 '\n'
+                 '   Each class keeps a list of weak references to its '
+                 'immediate\n'
+                 '   subclasses.  This method returns a list of all those '
+                 'references\n'
+                 '   still alive. Example:\n'
+                 '\n'
+                 '      >>> int.__subclasses__()\n'
+                 "      [<class 'bool'>]\n"
+                 '\n'
+                 '-[ Footnotes ]-\n'
+                 '\n'
+                 '[1] Additional information on these special methods may be '
+                 'found\n'
+                 '    in the Python Reference Manual (*Basic customization*).\n'
+                 '\n'
+                 '[2] As a consequence, the list "[1, 2]" is considered equal '
+                 'to\n'
+                 '    "[1.0, 2.0]", and similarly for tuples.\n'
+                 '\n'
+                 "[3] They must have since the parser can't tell the type of "
+                 'the\n'
+                 '    operands.\n'
+                 '\n'
+                 '[4] Cased characters are those with general category '
+                 'property\n'
+                 '    being one of "Lu" (Letter, uppercase), "Ll" (Letter, '
+                 'lowercase),\n'
+                 '    or "Lt" (Letter, titlecase).\n'
+                 '\n'
+                 '[5] To format only a tuple you should therefore provide a\n'
+                 '    singleton tuple whose only element is the tuple to be '
+                 'formatted.\n',
+ 'specialnames': '\n'
+                 'Special method names\n'
+                 '********************\n'
+                 '\n'
+                 'A class can implement certain operations that are invoked by '
+                 'special\n'
+                 'syntax (such as arithmetic operations or subscripting and '
+                 'slicing) by\n'
+                 "defining methods with special names. This is Python's "
+                 'approach to\n'
+                 '*operator overloading*, allowing classes to define their own '
+                 'behavior\n'
+                 'with respect to language operators.  For instance, if a '
+                 'class defines\n'
+                 'a method named "__getitem__()", and "x" is an instance of '
+                 'this class,\n'
+                 'then "x[i]" is roughly equivalent to "type(x).__getitem__(x, '
+                 'i)".\n'
+                 'Except where mentioned, attempts to execute an operation '
+                 'raise an\n'
+                 'exception when no appropriate method is defined (typically\n'
+                 '"AttributeError" or "TypeError").\n'
+                 '\n'
+                 'When implementing a class that emulates any built-in type, '
+                 'it is\n'
+                 'important that the emulation only be implemented to the '
+                 'degree that it\n'
+                 'makes sense for the object being modelled.  For example, '
+                 'some\n'
+                 'sequences may work well with retrieval of individual '
+                 'elements, but\n'
+                 'extracting a slice may not make sense.  (One example of this '
+                 'is the\n'
+                 '"NodeList" interface in the W3C\'s Document Object Model.)\n'
+                 '\n'
+                 '\n'
+                 'Basic customization\n'
+                 '===================\n'
+                 '\n'
+                 'object.__new__(cls[, ...])\n'
+                 '\n'
+                 '   Called to create a new instance of class *cls*.  '
+                 '"__new__()" is a\n'
+                 '   static method (special-cased so you need not declare it '
+                 'as such)\n'
+                 '   that takes the class of which an instance was requested '
+                 'as its\n'
+                 '   first argument.  The remaining arguments are those passed '
+                 'to the\n'
+                 '   object constructor expression (the call to the class).  '
+                 'The return\n'
+                 '   value of "__new__()" should be the new object instance '
+                 '(usually an\n'
+                 '   instance of *cls*).\n'
+                 '\n'
+                 '   Typical implementations create a new instance of the '
+                 'class by\n'
+                 '   invoking the superclass\'s "__new__()" method using\n'
+                 '   "super().__new__(cls[, ...])" with appropriate arguments '
+                 'and then\n'
+                 '   modifying the newly-created instance as necessary before '
+                 'returning\n'
+                 '   it.\n'
+                 '\n'
+                 '   If "__new__()" returns an instance of *cls*, then the '
+                 'new\n'
+                 '   instance\'s "__init__()" method will be invoked like\n'
+                 '   "__init__(self[, ...])", where *self* is the new instance '
+                 'and the\n'
+                 '   remaining arguments are the same as were passed to '
+                 '"__new__()".\n'
+                 '\n'
+                 '   If "__new__()" does not return an instance of *cls*, then '
+                 'the new\n'
+                 '   instance\'s "__init__()" method will not be invoked.\n'
+                 '\n'
+                 '   "__new__()" is intended mainly to allow subclasses of '
+                 'immutable\n'
+                 '   types (like int, str, or tuple) to customize instance '
+                 'creation.  It\n'
+                 '   is also commonly overridden in custom metaclasses in '
+                 'order to\n'
+                 '   customize class creation.\n'
+                 '\n'
+                 'object.__init__(self[, ...])\n'
+                 '\n'
+                 '   Called after the instance has been created (by '
+                 '"__new__()"), but\n'
+                 '   before it is returned to the caller.  The arguments are '
+                 'those\n'
+                 '   passed to the class constructor expression.  If a base '
+                 'class has an\n'
+                 '   "__init__()" method, the derived class\'s "__init__()" '
+                 'method, if\n'
+                 '   any, must explicitly call it to ensure proper '
+                 'initialization of the\n'
+                 '   base class part of the instance; for example:\n'
+                 '   "super().__init__([args...])".\n'
+                 '\n'
+                 '   Because "__new__()" and "__init__()" work together in '
+                 'constructing\n'
+                 '   objects ("__new__()" to create it, and "__init__()" to '
+                 'customize\n'
+                 '   it), no non-"None" value may be returned by "__init__()"; '
+                 'doing so\n'
+                 '   will cause a "TypeError" to be raised at runtime.\n'
+                 '\n'
+                 'object.__del__(self)\n'
+                 '\n'
+                 '   Called when the instance is about to be destroyed.  This '
+                 'is also\n'
+                 '   called a destructor.  If a base class has a "__del__()" '
+                 'method, the\n'
+                 '   derived class\'s "__del__()" method, if any, must '
+                 'explicitly call it\n'
+                 '   to ensure proper deletion of the base class part of the '
+                 'instance.\n'
+                 '   Note that it is possible (though not recommended!) for '
+                 'the\n'
+                 '   "__del__()" method to postpone destruction of the '
+                 'instance by\n'
+                 '   creating a new reference to it.  It may then be called at '
+                 'a later\n'
+                 '   time when this new reference is deleted.  It is not '
+                 'guaranteed that\n'
+                 '   "__del__()" methods are called for objects that still '
+                 'exist when\n'
+                 '   the interpreter exits.\n'
+                 '\n'
+                 '   Note: "del x" doesn\'t directly call "x.__del__()" --- '
+                 'the former\n'
+                 '     decrements the reference count for "x" by one, and the '
+                 'latter is\n'
+                 '     only called when "x"\'s reference count reaches zero.  '
+                 'Some common\n'
+                 '     situations that may prevent the reference count of an '
+                 'object from\n'
+                 '     going to zero include: circular references between '
+                 'objects (e.g.,\n'
+                 '     a doubly-linked list or a tree data structure with '
+                 'parent and\n'
+                 '     child pointers); a reference to the object on the stack '
+                 'frame of\n'
+                 '     a function that caught an exception (the traceback '
+                 'stored in\n'
+                 '     "sys.exc_info()[2]" keeps the stack frame alive); or a '
+                 'reference\n'
+                 '     to the object on the stack frame that raised an '
+                 'unhandled\n'
+                 '     exception in interactive mode (the traceback stored in\n'
+                 '     "sys.last_traceback" keeps the stack frame alive).  The '
+                 'first\n'
+                 '     situation can only be remedied by explicitly breaking '
+                 'the cycles;\n'
+                 '     the second can be resolved by freeing the reference to '
+                 'the\n'
+                 '     traceback object when it is no longer useful, and the '
+                 'third can\n'
+                 '     be resolved by storing "None" in "sys.last_traceback". '
+                 'Circular\n'
+                 '     references which are garbage are detected and cleaned '
+                 'up when the\n'
+                 "     cyclic garbage collector is enabled (it's on by "
+                 'default). Refer\n'
+                 '     to the documentation for the "gc" module for more '
+                 'information\n'
+                 '     about this topic.\n'
+                 '\n'
+                 '   Warning: Due to the precarious circumstances under which\n'
+                 '     "__del__()" methods are invoked, exceptions that occur '
+                 'during\n'
+                 '     their execution are ignored, and a warning is printed '
+                 'to\n'
+                 '     "sys.stderr" instead. Also, when "__del__()" is invoked '
+                 'in\n'
+                 '     response to a module being deleted (e.g., when '
+                 'execution of the\n'
+                 '     program is done), other globals referenced by the '
+                 '"__del__()"\n'
+                 '     method may already have been deleted or in the process '
+                 'of being\n'
+                 '     torn down (e.g. the import machinery shutting down).  '
+                 'For this\n'
+                 '     reason, "__del__()" methods should do the absolute '
+                 'minimum needed\n'
+                 '     to maintain external invariants.  Starting with version '
+                 '1.5,\n'
+                 '     Python guarantees that globals whose name begins with a '
+                 'single\n'
+                 '     underscore are deleted from their module before other '
+                 'globals are\n'
+                 '     deleted; if no other references to such globals exist, '
+                 'this may\n'
+                 '     help in assuring that imported modules are still '
+                 'available at the\n'
+                 '     time when the "__del__()" method is called.\n'
+                 '\n'
+                 'object.__repr__(self)\n'
+                 '\n'
+                 '   Called by the "repr()" built-in function to compute the '
+                 '"official"\n'
+                 '   string representation of an object.  If at all possible, '
+                 'this\n'
+                 '   should look like a valid Python expression that could be '
+                 'used to\n'
+                 '   recreate an object with the same value (given an '
+                 'appropriate\n'
+                 '   environment).  If this is not possible, a string of the '
+                 'form\n'
+                 '   "<...some useful description...>" should be returned. The '
+                 'return\n'
+                 '   value must be a string object. If a class defines '
+                 '"__repr__()" but\n'
+                 '   not "__str__()", then "__repr__()" is also used when an '
+                 '"informal"\n'
+                 '   string representation of instances of that class is '
+                 'required.\n'
+                 '\n'
+                 '   This is typically used for debugging, so it is important '
+                 'that the\n'
+                 '   representation is information-rich and unambiguous.\n'
+                 '\n'
+                 'object.__str__(self)\n'
+                 '\n'
+                 '   Called by "str(object)" and the built-in functions '
+                 '"format()" and\n'
+                 '   "print()" to compute the "informal" or nicely printable '
+                 'string\n'
+                 '   representation of an object.  The return value must be a '
+                 '*string*\n'
+                 '   object.\n'
+                 '\n'
+                 '   This method differs from "object.__repr__()" in that '
+                 'there is no\n'
+                 '   expectation that "__str__()" return a valid Python '
+                 'expression: a\n'
+                 '   more convenient or concise representation can be used.\n'
+                 '\n'
+                 '   The default implementation defined by the built-in type '
+                 '"object"\n'
+                 '   calls "object.__repr__()".\n'
+                 '\n'
+                 'object.__bytes__(self)\n'
+                 '\n'
+                 '   Called by "bytes()" to compute a byte-string '
+                 'representation of an\n'
+                 '   object. This should return a "bytes" object.\n'
+                 '\n'
+                 'object.__format__(self, format_spec)\n'
+                 '\n'
+                 '   Called by the "format()" built-in function (and by '
+                 'extension, the\n'
+                 '   "str.format()" method of class "str") to produce a '
+                 '"formatted"\n'
+                 '   string representation of an object. The "format_spec" '
+                 'argument is a\n'
+                 '   string that contains a description of the formatting '
+                 'options\n'
+                 '   desired. The interpretation of the "format_spec" argument '
+                 'is up to\n'
+                 '   the type implementing "__format__()", however most '
+                 'classes will\n'
+                 '   either delegate formatting to one of the built-in types, '
+                 'or use a\n'
+                 '   similar formatting option syntax.\n'
+                 '\n'
+                 '   See *Format Specification Mini-Language* for a '
+                 'description of the\n'
+                 '   standard formatting syntax.\n'
+                 '\n'
+                 '   The return value must be a string object.\n'
+                 '\n'
+                 '   Changed in version 3.4: The __format__ method of "object" '
+                 'itself\n'
+                 '   raises a "TypeError" if passed any non-empty string.\n'
+                 '\n'
+                 'object.__lt__(self, other)\n'
+                 'object.__le__(self, other)\n'
+                 'object.__eq__(self, other)\n'
+                 'object.__ne__(self, other)\n'
+                 'object.__gt__(self, other)\n'
+                 'object.__ge__(self, other)\n'
+                 '\n'
+                 '   These are the so-called "rich comparison" methods. The\n'
+                 '   correspondence between operator symbols and method names '
+                 'is as\n'
+                 '   follows: "x<y" calls "x.__lt__(y)", "x<=y" calls '
+                 '"x.__le__(y)",\n'
+                 '   "x==y" calls "x.__eq__(y)", "x!=y" calls "x.__ne__(y)", '
+                 '"x>y" calls\n'
+                 '   "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n'
+                 '\n'
+                 '   A rich comparison method may return the singleton '
+                 '"NotImplemented"\n'
+                 '   if it does not implement the operation for a given pair '
+                 'of\n'
+                 '   arguments. By convention, "False" and "True" are returned '
+                 'for a\n'
+                 '   successful comparison. However, these methods can return '
+                 'any value,\n'
+                 '   so if the comparison operator is used in a Boolean '
+                 'context (e.g.,\n'
+                 '   in the condition of an "if" statement), Python will call '
+                 '"bool()"\n'
+                 '   on the value to determine if the result is true or '
+                 'false.\n'
+                 '\n'
+                 '   By default, "__ne__()" delegates to "__eq__()" and '
+                 'inverts the\n'
+                 '   result unless it is "NotImplemented".  There are no other '
+                 'implied\n'
+                 '   relationships among the comparison operators, for '
+                 'example, the\n'
+                 '   truth of "(x<y or x==y)" does not imply "x<=y". To '
+                 'automatically\n'
+                 '   generate ordering operations from a single root '
+                 'operation, see\n'
+                 '   "functools.total_ordering()".\n'
+                 '\n'
+                 '   See the paragraph on "__hash__()" for some important '
+                 'notes on\n'
+                 '   creating *hashable* objects which support custom '
+                 'comparison\n'
+                 '   operations and are usable as dictionary keys.\n'
+                 '\n'
+                 '   There are no swapped-argument versions of these methods '
+                 '(to be used\n'
+                 '   when the left argument does not support the operation but '
+                 'the right\n'
+                 '   argument does); rather, "__lt__()" and "__gt__()" are '
+                 "each other's\n"
+                 '   reflection, "__le__()" and "__ge__()" are each other\'s '
+                 'reflection,\n'
+                 '   and "__eq__()" and "__ne__()" are their own reflection. '
+                 'If the\n'
+                 "   operands are of different types, and right operand's type "
+                 'is a\n'
+                 "   direct or indirect subclass of the left operand's type, "
+                 'the\n'
+                 '   reflected method of the right operand has priority, '
+                 'otherwise the\n'
+                 "   left operand's method has priority.  Virtual subclassing "
+                 'is not\n'
+                 '   considered.\n'
+                 '\n'
+                 'object.__hash__(self)\n'
+                 '\n'
+                 '   Called by built-in function "hash()" and for operations '
+                 'on members\n'
+                 '   of hashed collections including "set", "frozenset", and '
+                 '"dict".\n'
+                 '   "__hash__()" should return an integer. The only required '
+                 'property\n'
+                 '   is that objects which compare equal have the same hash '
+                 'value; it is\n'
+                 '   advised to mix together the hash values of the components '
+                 'of the\n'
+                 '   object that also play a part in comparison of objects by '
+                 'packing\n'
+                 '   them into a tuple and hashing the tuple. Example:\n'
+                 '\n'
+                 '      def __hash__(self):\n'
+                 '          return hash((self.name, self.nick, self.color))\n'
+                 '\n'
+                 '   Note: "hash()" truncates the value returned from an '
+                 "object's\n"
+                 '     custom "__hash__()" method to the size of a '
+                 '"Py_ssize_t".  This\n'
+                 '     is typically 8 bytes on 64-bit builds and 4 bytes on '
+                 '32-bit\n'
+                 '     builds. If an object\'s   "__hash__()" must '
+                 'interoperate on builds\n'
+                 '     of different bit sizes, be sure to check the width on '
+                 'all\n'
+                 '     supported builds.  An easy way to do this is with '
+                 '"python -c\n'
+                 '     "import sys; print(sys.hash_info.width)"".\n'
+                 '\n'
+                 '   If a class does not define an "__eq__()" method it should '
+                 'not\n'
+                 '   define a "__hash__()" operation either; if it defines '
+                 '"__eq__()"\n'
+                 '   but not "__hash__()", its instances will not be usable as '
+                 'items in\n'
+                 '   hashable collections.  If a class defines mutable objects '
+                 'and\n'
+                 '   implements an "__eq__()" method, it should not implement\n'
+                 '   "__hash__()", since the implementation of hashable '
+                 'collections\n'
+                 "   requires that a key's hash value is immutable (if the "
+                 "object's hash\n"
+                 '   value changes, it will be in the wrong hash bucket).\n'
+                 '\n'
+                 '   User-defined classes have "__eq__()" and "__hash__()" '
+                 'methods by\n'
+                 '   default; with them, all objects compare unequal (except '
+                 'with\n'
+                 '   themselves) and "x.__hash__()" returns an appropriate '
+                 'value such\n'
+                 '   that "x == y" implies both that "x is y" and "hash(x) == '
+                 'hash(y)".\n'
+                 '\n'
+                 '   A class that overrides "__eq__()" and does not define '
+                 '"__hash__()"\n'
+                 '   will have its "__hash__()" implicitly set to "None".  '
+                 'When the\n'
+                 '   "__hash__()" method of a class is "None", instances of '
+                 'the class\n'
+                 '   will raise an appropriate "TypeError" when a program '
+                 'attempts to\n'
+                 '   retrieve their hash value, and will also be correctly '
+                 'identified as\n'
+                 '   unhashable when checking "isinstance(obj, '
+                 'collections.Hashable)".\n'
+                 '\n'
+                 '   If a class that overrides "__eq__()" needs to retain the\n'
+                 '   implementation of "__hash__()" from a parent class, the '
+                 'interpreter\n'
+                 '   must be told this explicitly by setting "__hash__ =\n'
+                 '   <ParentClass>.__hash__".\n'
+                 '\n'
+                 '   If a class that does not override "__eq__()" wishes to '
+                 'suppress\n'
+                 '   hash support, it should include "__hash__ = None" in the '
+                 'class\n'
+                 '   definition. A class which defines its own "__hash__()" '
+                 'that\n'
+                 '   explicitly raises a "TypeError" would be incorrectly '
+                 'identified as\n'
+                 '   hashable by an "isinstance(obj, collections.Hashable)" '
+                 'call.\n'
+                 '\n'
+                 '   Note: By default, the "__hash__()" values of str, bytes '
+                 'and\n'
+                 '     datetime objects are "salted" with an unpredictable '
+                 'random value.\n'
+                 '     Although they remain constant within an individual '
+                 'Python\n'
+                 '     process, they are not predictable between repeated '
+                 'invocations of\n'
+                 '     Python.This is intended to provide protection against a '
+                 'denial-\n'
+                 '     of-service caused by carefully-chosen inputs that '
+                 'exploit the\n'
+                 '     worst case performance of a dict insertion, O(n^2) '
+                 'complexity.\n'
+                 '     See http://www.ocert.org/advisories/ocert-2011-003.html '
+                 'for\n'
+                 '     details.Changing hash values affects the iteration '
+                 'order of\n'
+                 '     dicts, sets and other mappings.  Python has never made '
+                 'guarantees\n'
+                 '     about this ordering (and it typically varies between '
+                 '32-bit and\n'
+                 '     64-bit builds).See also "PYTHONHASHSEED".\n'
+                 '\n'
+                 '   Changed in version 3.3: Hash randomization is enabled by '
+                 'default.\n'
+                 '\n'
+                 'object.__bool__(self)\n'
+                 '\n'
+                 '   Called to implement truth value testing and the built-in '
+                 'operation\n'
+                 '   "bool()"; should return "False" or "True".  When this '
+                 'method is not\n'
+                 '   defined, "__len__()" is called, if it is defined, and the '
+                 'object is\n'
+                 '   considered true if its result is nonzero.  If a class '
+                 'defines\n'
+                 '   neither "__len__()" nor "__bool__()", all its instances '
+                 'are\n'
+                 '   considered true.\n'
+                 '\n'
+                 '\n'
+                 'Customizing attribute access\n'
+                 '============================\n'
+                 '\n'
+                 'The following methods can be defined to customize the '
+                 'meaning of\n'
+                 'attribute access (use of, assignment to, or deletion of '
+                 '"x.name") for\n'
+                 'class instances.\n'
+                 '\n'
+                 'object.__getattr__(self, name)\n'
+                 '\n'
+                 '   Called when an attribute lookup has not found the '
+                 'attribute in the\n'
+                 '   usual places (i.e. it is not an instance attribute nor is '
+                 'it found\n'
+                 '   in the class tree for "self").  "name" is the attribute '
+                 'name. This\n'
+                 '   method should return the (computed) attribute value or '
+                 'raise an\n'
+                 '   "AttributeError" exception.\n'
+                 '\n'
+                 '   Note that if the attribute is found through the normal '
+                 'mechanism,\n'
+                 '   "__getattr__()" is not called.  (This is an intentional '
+                 'asymmetry\n'
+                 '   between "__getattr__()" and "__setattr__()".) This is '
+                 'done both for\n'
+                 '   efficiency reasons and because otherwise "__getattr__()" '
+                 'would have\n'
+                 '   no way to access other attributes of the instance.  Note '
+                 'that at\n'
+                 '   least for instance variables, you can fake total control '
+                 'by not\n'
+                 '   inserting any values in the instance attribute dictionary '
+                 '(but\n'
+                 '   instead inserting them in another object).  See the\n'
+                 '   "__getattribute__()" method below for a way to actually '
+                 'get total\n'
+                 '   control over attribute access.\n'
+                 '\n'
+                 'object.__getattribute__(self, name)\n'
+                 '\n'
+                 '   Called unconditionally to implement attribute accesses '
+                 'for\n'
+                 '   instances of the class. If the class also defines '
+                 '"__getattr__()",\n'
+                 '   the latter will not be called unless "__getattribute__()" '
+                 'either\n'
+                 '   calls it explicitly or raises an "AttributeError". This '
+                 'method\n'
+                 '   should return the (computed) attribute value or raise an\n'
+                 '   "AttributeError" exception. In order to avoid infinite '
+                 'recursion in\n'
+                 '   this method, its implementation should always call the '
+                 'base class\n'
+                 '   method with the same name to access any attributes it '
+                 'needs, for\n'
+                 '   example, "object.__getattribute__(self, name)".\n'
+                 '\n'
+                 '   Note: This method may still be bypassed when looking up '
+                 'special\n'
+                 '     methods as the result of implicit invocation via '
+                 'language syntax\n'
+                 '     or built-in functions. See *Special method lookup*.\n'
+                 '\n'
+                 'object.__setattr__(self, name, value)\n'
+                 '\n'
+                 '   Called when an attribute assignment is attempted.  This '
+                 'is called\n'
+                 '   instead of the normal mechanism (i.e. store the value in '
+                 'the\n'
+                 '   instance dictionary). *name* is the attribute name, '
+                 '*value* is the\n'
+                 '   value to be assigned to it.\n'
+                 '\n'
+                 '   If "__setattr__()" wants to assign to an instance '
+                 'attribute, it\n'
+                 '   should call the base class method with the same name, for '
+                 'example,\n'
+                 '   "object.__setattr__(self, name, value)".\n'
+                 '\n'
+                 'object.__delattr__(self, name)\n'
+                 '\n'
+                 '   Like "__setattr__()" but for attribute deletion instead '
+                 'of\n'
+                 '   assignment.  This should only be implemented if "del '
+                 'obj.name" is\n'
+                 '   meaningful for the object.\n'
+                 '\n'
+                 'object.__dir__(self)\n'
+                 '\n'
+                 '   Called when "dir()" is called on the object. A sequence '
+                 'must be\n'
+                 '   returned. "dir()" converts the returned sequence to a '
+                 'list and\n'
+                 '   sorts it.\n'
+                 '\n'
+                 '\n'
+                 'Implementing Descriptors\n'
+                 '------------------------\n'
+                 '\n'
+                 'The following methods only apply when an instance of the '
+                 'class\n'
+                 'containing the method (a so-called *descriptor* class) '
+                 'appears in an\n'
+                 "*owner* class (the descriptor must be in either the owner's "
+                 'class\n'
+                 'dictionary or in the class dictionary for one of its '
+                 'parents).  In the\n'
+                 'examples below, "the attribute" refers to the attribute '
+                 'whose name is\n'
+                 'the key of the property in the owner class\' "__dict__".\n'
+                 '\n'
+                 'object.__get__(self, instance, owner)\n'
+                 '\n'
+                 '   Called to get the attribute of the owner class (class '
+                 'attribute\n'
+                 '   access) or of an instance of that class (instance '
+                 'attribute\n'
+                 '   access). *owner* is always the owner class, while '
+                 '*instance* is the\n'
+                 '   instance that the attribute was accessed through, or '
+                 '"None" when\n'
+                 '   the attribute is accessed through the *owner*.  This '
+                 'method should\n'
+                 '   return the (computed) attribute value or raise an '
+                 '"AttributeError"\n'
+                 '   exception.\n'
+                 '\n'
+                 'object.__set__(self, instance, value)\n'
+                 '\n'
+                 '   Called to set the attribute on an instance *instance* of '
+                 'the owner\n'
+                 '   class to a new value, *value*.\n'
+                 '\n'
+                 'object.__delete__(self, instance)\n'
+                 '\n'
+                 '   Called to delete the attribute on an instance *instance* '
+                 'of the\n'
+                 '   owner class.\n'
+                 '\n'
+                 'The attribute "__objclass__" is interpreted by the "inspect" '
+                 'module as\n'
+                 'specifying the class where this object was defined (setting '
+                 'this\n'
+                 'appropriately can assist in runtime introspection of dynamic '
+                 'class\n'
+                 'attributes). For callables, it may indicate that an instance '
+                 'of the\n'
+                 'given type (or a subclass) is expected or required as the '
+                 'first\n'
+                 'positional argument (for example, CPython sets this '
+                 'attribute for\n'
+                 'unbound methods that are implemented in C).\n'
+                 '\n'
+                 '\n'
+                 'Invoking Descriptors\n'
+                 '--------------------\n'
+                 '\n'
+                 'In general, a descriptor is an object attribute with '
+                 '"binding\n'
+                 'behavior", one whose attribute access has been overridden by '
+                 'methods\n'
+                 'in the descriptor protocol:  "__get__()", "__set__()", and\n'
+                 '"__delete__()". If any of those methods are defined for an '
+                 'object, it\n'
+                 'is said to be a descriptor.\n'
+                 '\n'
+                 'The default behavior for attribute access is to get, set, or '
+                 'delete\n'
+                 "the attribute from an object's dictionary. For instance, "
+                 '"a.x" has a\n'
+                 'lookup chain starting with "a.__dict__[\'x\']", then\n'
+                 '"type(a).__dict__[\'x\']", and continuing through the base '
+                 'classes of\n'
+                 '"type(a)" excluding metaclasses.\n'
+                 '\n'
+                 'However, if the looked-up value is an object defining one of '
+                 'the\n'
+                 'descriptor methods, then Python may override the default '
+                 'behavior and\n'
+                 'invoke the descriptor method instead.  Where this occurs in '
+                 'the\n'
+                 'precedence chain depends on which descriptor methods were '
+                 'defined and\n'
+                 'how they were called.\n'
+                 '\n'
+                 'The starting point for descriptor invocation is a binding, '
+                 '"a.x". How\n'
+                 'the arguments are assembled depends on "a":\n'
+                 '\n'
+                 'Direct Call\n'
+                 '   The simplest and least common call is when user code '
+                 'directly\n'
+                 '   invokes a descriptor method:    "x.__get__(a)".\n'
+                 '\n'
+                 'Instance Binding\n'
+                 '   If binding to an object instance, "a.x" is transformed '
+                 'into the\n'
+                 '   call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n'
+                 '\n'
+                 'Class Binding\n'
+                 '   If binding to a class, "A.x" is transformed into the '
+                 'call:\n'
+                 '   "A.__dict__[\'x\'].__get__(None, A)".\n'
+                 '\n'
+                 'Super Binding\n'
+                 '   If "a" is an instance of "super", then the binding '
+                 '"super(B,\n'
+                 '   obj).m()" searches "obj.__class__.__mro__" for the base '
+                 'class "A"\n'
+                 '   immediately preceding "B" and then invokes the descriptor '
+                 'with the\n'
+                 '   call: "A.__dict__[\'m\'].__get__(obj, obj.__class__)".\n'
+                 '\n'
+                 'For instance bindings, the precedence of descriptor '
+                 'invocation depends\n'
+                 'on the which descriptor methods are defined.  A descriptor '
+                 'can define\n'
+                 'any combination of "__get__()", "__set__()" and '
+                 '"__delete__()".  If it\n'
+                 'does not define "__get__()", then accessing the attribute '
+                 'will return\n'
+                 'the descriptor object itself unless there is a value in the '
+                 "object's\n"
+                 'instance dictionary.  If the descriptor defines "__set__()" '
+                 'and/or\n'
+                 '"__delete__()", it is a data descriptor; if it defines '
+                 'neither, it is\n'
+                 'a non-data descriptor.  Normally, data descriptors define '
+                 'both\n'
+                 '"__get__()" and "__set__()", while non-data descriptors have '
+                 'just the\n'
+                 '"__get__()" method.  Data descriptors with "__set__()" and '
+                 '"__get__()"\n'
+                 'defined always override a redefinition in an instance '
+                 'dictionary.  In\n'
+                 'contrast, non-data descriptors can be overridden by '
+                 'instances.\n'
+                 '\n'
+                 'Python methods (including "staticmethod()" and '
+                 '"classmethod()") are\n'
+                 'implemented as non-data descriptors.  Accordingly, instances '
+                 'can\n'
+                 'redefine and override methods.  This allows individual '
+                 'instances to\n'
+                 'acquire behaviors that differ from other instances of the '
+                 'same class.\n'
+                 '\n'
+                 'The "property()" function is implemented as a data '
+                 'descriptor.\n'
+                 'Accordingly, instances cannot override the behavior of a '
+                 'property.\n'
+                 '\n'
+                 '\n'
+                 '__slots__\n'
+                 '---------\n'
+                 '\n'
+                 'By default, instances of classes have a dictionary for '
+                 'attribute\n'
+                 'storage.  This wastes space for objects having very few '
+                 'instance\n'
+                 'variables.  The space consumption can become acute when '
+                 'creating large\n'
+                 'numbers of instances.\n'
+                 '\n'
+                 'The default can be overridden by defining *__slots__* in a '
+                 'class\n'
+                 'definition. The *__slots__* declaration takes a sequence of '
+                 'instance\n'
+                 'variables and reserves just enough space in each instance to '
+                 'hold a\n'
+                 'value for each variable.  Space is saved because *__dict__* '
+                 'is not\n'
+                 'created for each instance.\n'
+                 '\n'
+                 'object.__slots__\n'
+                 '\n'
+                 '   This class variable can be assigned a string, iterable, '
+                 'or sequence\n'
+                 '   of strings with variable names used by instances.  '
+                 '*__slots__*\n'
+                 '   reserves space for the declared variables and prevents '
+                 'the\n'
+                 '   automatic creation of *__dict__* and *__weakref__* for '
+                 'each\n'
+                 '   instance.\n'
+                 '\n'
+                 '\n'
+                 'Notes on using *__slots__*\n'
+                 '~~~~~~~~~~~~~~~~~~~~~~~~~~\n'
+                 '\n'
+                 '* When inheriting from a class without *__slots__*, the '
+                 '*__dict__*\n'
+                 '  attribute of that class will always be accessible, so a '
+                 '*__slots__*\n'
+                 '  definition in the subclass is meaningless.\n'
+                 '\n'
+                 '* Without a *__dict__* variable, instances cannot be '
+                 'assigned new\n'
+                 '  variables not listed in the *__slots__* definition.  '
+                 'Attempts to\n'
+                 '  assign to an unlisted variable name raises '
+                 '"AttributeError". If\n'
+                 '  dynamic assignment of new variables is desired, then add\n'
+                 '  "\'__dict__\'" to the sequence of strings in the '
+                 '*__slots__*\n'
+                 '  declaration.\n'
+                 '\n'
+                 '* Without a *__weakref__* variable for each instance, '
+                 'classes\n'
+                 '  defining *__slots__* do not support weak references to '
+                 'its\n'
+                 '  instances. If weak reference support is needed, then add\n'
+                 '  "\'__weakref__\'" to the sequence of strings in the '
+                 '*__slots__*\n'
+                 '  declaration.\n'
+                 '\n'
+                 '* *__slots__* are implemented at the class level by '
+                 'creating\n'
+                 '  descriptors (*Implementing Descriptors*) for each variable '
+                 'name.  As\n'
+                 '  a result, class attributes cannot be used to set default '
+                 'values for\n'
+                 '  instance variables defined by *__slots__*; otherwise, the '
+                 'class\n'
+                 '  attribute would overwrite the descriptor assignment.\n'
+                 '\n'
+                 '* The action of a *__slots__* declaration is limited to the '
+                 'class\n'
+                 '  where it is defined.  As a result, subclasses will have a '
+                 '*__dict__*\n'
+                 '  unless they also define *__slots__* (which must only '
+                 'contain names\n'
+                 '  of any *additional* slots).\n'
+                 '\n'
+                 '* If a class defines a slot also defined in a base class, '
+                 'the\n'
+                 '  instance variable defined by the base class slot is '
+                 'inaccessible\n'
+                 '  (except by retrieving its descriptor directly from the '
+                 'base class).\n'
+                 '  This renders the meaning of the program undefined.  In the '
+                 'future, a\n'
+                 '  check may be added to prevent this.\n'
+                 '\n'
+                 '* Nonempty *__slots__* does not work for classes derived '
+                 'from\n'
+                 '  "variable-length" built-in types such as "int", "bytes" '
+                 'and "tuple".\n'
+                 '\n'
+                 '* Any non-string iterable may be assigned to *__slots__*. '
+                 'Mappings\n'
+                 '  may also be used; however, in the future, special meaning '
+                 'may be\n'
+                 '  assigned to the values corresponding to each key.\n'
+                 '\n'
+                 '* *__class__* assignment works only if both classes have the '
+                 'same\n'
+                 '  *__slots__*.\n'
+                 '\n'
+                 '\n'
+                 'Customizing class creation\n'
+                 '==========================\n'
+                 '\n'
+                 'By default, classes are constructed using "type()". The '
+                 'class body is\n'
+                 'executed in a new namespace and the class name is bound '
+                 'locally to the\n'
+                 'result of "type(name, bases, namespace)".\n'
+                 '\n'
+                 'The class creation process can be customized by passing the\n'
+                 '"metaclass" keyword argument in the class definition line, '
+                 'or by\n'
+                 'inheriting from an existing class that included such an '
+                 'argument. In\n'
+                 'the following example, both "MyClass" and "MySubclass" are '
+                 'instances\n'
+                 'of "Meta":\n'
+                 '\n'
+                 '   class Meta(type):\n'
+                 '       pass\n'
+                 '\n'
+                 '   class MyClass(metaclass=Meta):\n'
+                 '       pass\n'
+                 '\n'
+                 '   class MySubclass(MyClass):\n'
+                 '       pass\n'
+                 '\n'
+                 'Any other keyword arguments that are specified in the class '
+                 'definition\n'
+                 'are passed through to all metaclass operations described '
+                 'below.\n'
+                 '\n'
+                 'When a class definition is executed, the following steps '
+                 'occur:\n'
+                 '\n'
+                 '* the appropriate metaclass is determined\n'
+                 '\n'
+                 '* the class namespace is prepared\n'
+                 '\n'
+                 '* the class body is executed\n'
+                 '\n'
+                 '* the class object is created\n'
+                 '\n'
+                 '\n'
+                 'Determining the appropriate metaclass\n'
+                 '-------------------------------------\n'
+                 '\n'
+                 'The appropriate metaclass for a class definition is '
+                 'determined as\n'
+                 'follows:\n'
+                 '\n'
+                 '* if no bases and no explicit metaclass are given, then '
+                 '"type()" is\n'
+                 '  used\n'
+                 '\n'
+                 '* if an explicit metaclass is given and it is *not* an '
+                 'instance of\n'
+                 '  "type()", then it is used directly as the metaclass\n'
+                 '\n'
+                 '* if an instance of "type()" is given as the explicit '
+                 'metaclass, or\n'
+                 '  bases are defined, then the most derived metaclass is '
+                 'used\n'
+                 '\n'
+                 'The most derived metaclass is selected from the explicitly '
+                 'specified\n'
+                 'metaclass (if any) and the metaclasses (i.e. "type(cls)") of '
+                 'all\n'
+                 'specified base classes. The most derived metaclass is one '
+                 'which is a\n'
+                 'subtype of *all* of these candidate metaclasses. If none of '
+                 'the\n'
+                 'candidate metaclasses meets that criterion, then the class '
+                 'definition\n'
+                 'will fail with "TypeError".\n'
+                 '\n'
+                 '\n'
+                 'Preparing the class namespace\n'
+                 '-----------------------------\n'
+                 '\n'
+                 'Once the appropriate metaclass has been identified, then the '
+                 'class\n'
+                 'namespace is prepared. If the metaclass has a "__prepare__" '
+                 'attribute,\n'
+                 'it is called as "namespace = metaclass.__prepare__(name, '
+                 'bases,\n'
+                 '**kwds)" (where the additional keyword arguments, if any, '
+                 'come from\n'
+                 'the class definition).\n'
+                 '\n'
+                 'If the metaclass has no "__prepare__" attribute, then the '
+                 'class\n'
+                 'namespace is initialised as an empty "dict()" instance.\n'
+                 '\n'
+                 'See also: **PEP 3115** - Metaclasses in Python 3000\n'
+                 '\n'
+                 '     Introduced the "__prepare__" namespace hook\n'
+                 '\n'
+                 '\n'
+                 'Executing the class body\n'
+                 '------------------------\n'
+                 '\n'
+                 'The class body is executed (approximately) as "exec(body, '
+                 'globals(),\n'
+                 'namespace)". The key difference from a normal call to '
+                 '"exec()" is that\n'
+                 'lexical scoping allows the class body (including any '
+                 'methods) to\n'
+                 'reference names from the current and outer scopes when the '
+                 'class\n'
+                 'definition occurs inside a function.\n'
+                 '\n'
+                 'However, even when the class definition occurs inside the '
+                 'function,\n'
+                 'methods defined inside the class still cannot see names '
+                 'defined at the\n'
+                 'class scope. Class variables must be accessed through the '
+                 'first\n'
+                 'parameter of instance or class methods, and cannot be '
+                 'accessed at all\n'
+                 'from static methods.\n'
+                 '\n'
+                 '\n'
+                 'Creating the class object\n'
+                 '-------------------------\n'
+                 '\n'
+                 'Once the class namespace has been populated by executing the '
+                 'class\n'
+                 'body, the class object is created by calling '
+                 '"metaclass(name, bases,\n'
+                 'namespace, **kwds)" (the additional keywords passed here are '
+                 'the same\n'
+                 'as those passed to "__prepare__").\n'
+                 '\n'
+                 'This class object is the one that will be referenced by the '
+                 'zero-\n'
+                 'argument form of "super()". "__class__" is an implicit '
+                 'closure\n'
+                 'reference created by the compiler if any methods in a class '
+                 'body refer\n'
+                 'to either "__class__" or "super". This allows the zero '
+                 'argument form\n'
+                 'of "super()" to correctly identify the class being defined '
+                 'based on\n'
+                 'lexical scoping, while the class or instance that was used '
+                 'to make the\n'
+                 'current call is identified based on the first argument '
+                 'passed to the\n'
+                 'method.\n'
+                 '\n'
+                 'After the class object is created, it is passed to the '
+                 'class\n'
+                 'decorators included in the class definition (if any) and the '
+                 'resulting\n'
+                 'object is bound in the local namespace as the defined '
+                 'class.\n'
+                 '\n'
+                 'When a new class is created by "type.__new__", the object '
+                 'provided as\n'
+                 'the namespace parameter is copied to a standard Python '
+                 'dictionary and\n'
+                 'the original object is discarded. The new copy becomes the '
+                 '"__dict__"\n'
+                 'attribute of the class object.\n'
+                 '\n'
+                 'See also: **PEP 3135** - New super\n'
+                 '\n'
+                 '     Describes the implicit "__class__" closure reference\n'
+                 '\n'
+                 '\n'
+                 'Metaclass example\n'
+                 '-----------------\n'
+                 '\n'
+                 'The potential uses for metaclasses are boundless. Some ideas '
+                 'that have\n'
+                 'been explored include logging, interface checking, '
+                 'automatic\n'
+                 'delegation, automatic property creation, proxies, '
+                 'frameworks, and\n'
+                 'automatic resource locking/synchronization.\n'
+                 '\n'
+                 'Here is an example of a metaclass that uses an\n'
+                 '"collections.OrderedDict" to remember the order that class '
+                 'variables\n'
+                 'are defined:\n'
+                 '\n'
+                 '   class OrderedClass(type):\n'
+                 '\n'
+                 '       @classmethod\n'
+                 '       def __prepare__(metacls, name, bases, **kwds):\n'
+                 '           return collections.OrderedDict()\n'
+                 '\n'
+                 '       def __new__(cls, name, bases, namespace, **kwds):\n'
+                 '           result = type.__new__(cls, name, bases, '
+                 'dict(namespace))\n'
+                 '           result.members = tuple(namespace)\n'
+                 '           return result\n'
+                 '\n'
+                 '   class A(metaclass=OrderedClass):\n'
+                 '       def one(self): pass\n'
+                 '       def two(self): pass\n'
+                 '       def three(self): pass\n'
+                 '       def four(self): pass\n'
+                 '\n'
+                 '   >>> A.members\n'
+                 "   ('__module__', 'one', 'two', 'three', 'four')\n"
+                 '\n'
+                 'When the class definition for *A* gets executed, the process '
+                 'begins\n'
+                 'with calling the metaclass\'s "__prepare__()" method which '
+                 'returns an\n'
+                 'empty "collections.OrderedDict".  That mapping records the '
+                 'methods and\n'
+                 'attributes of *A* as they are defined within the body of the '
+                 'class\n'
+                 'statement. Once those definitions are executed, the ordered '
+                 'dictionary\n'
+                 'is fully populated and the metaclass\'s "__new__()" method '
+                 'gets\n'
+                 'invoked.  That method builds the new type and it saves the '
+                 'ordered\n'
+                 'dictionary keys in an attribute called "members".\n'
+                 '\n'
+                 '\n'
+                 'Customizing instance and subclass checks\n'
+                 '========================================\n'
+                 '\n'
+                 'The following methods are used to override the default '
+                 'behavior of the\n'
+                 '"isinstance()" and "issubclass()" built-in functions.\n'
+                 '\n'
+                 'In particular, the metaclass "abc.ABCMeta" implements these '
+                 'methods in\n'
+                 'order to allow the addition of Abstract Base Classes (ABCs) '
+                 'as\n'
+                 '"virtual base classes" to any class or type (including '
+                 'built-in\n'
+                 'types), including other ABCs.\n'
+                 '\n'
+                 'class.__instancecheck__(self, instance)\n'
+                 '\n'
+                 '   Return true if *instance* should be considered a (direct '
+                 'or\n'
+                 '   indirect) instance of *class*. If defined, called to '
+                 'implement\n'
+                 '   "isinstance(instance, class)".\n'
+                 '\n'
+                 'class.__subclasscheck__(self, subclass)\n'
+                 '\n'
+                 '   Return true if *subclass* should be considered a (direct '
+                 'or\n'
+                 '   indirect) subclass of *class*.  If defined, called to '
+                 'implement\n'
+                 '   "issubclass(subclass, class)".\n'
+                 '\n'
+                 'Note that these methods are looked up on the type '
+                 '(metaclass) of a\n'
+                 'class.  They cannot be defined as class methods in the '
+                 'actual class.\n'
+                 'This is consistent with the lookup of special methods that '
+                 'are called\n'
+                 'on instances, only in this case the instance is itself a '
+                 'class.\n'
+                 '\n'
+                 'See also: **PEP 3119** - Introducing Abstract Base Classes\n'
+                 '\n'
+                 '     Includes the specification for customizing '
+                 '"isinstance()" and\n'
+                 '     "issubclass()" behavior through "__instancecheck__()" '
+                 'and\n'
+                 '     "__subclasscheck__()", with motivation for this '
+                 'functionality in\n'
+                 '     the context of adding Abstract Base Classes (see the '
+                 '"abc"\n'
+                 '     module) to the language.\n'
+                 '\n'
+                 '\n'
+                 'Emulating callable objects\n'
+                 '==========================\n'
+                 '\n'
+                 'object.__call__(self[, args...])\n'
+                 '\n'
+                 '   Called when the instance is "called" as a function; if '
+                 'this method\n'
+                 '   is defined, "x(arg1, arg2, ...)" is a shorthand for\n'
+                 '   "x.__call__(arg1, arg2, ...)".\n'
+                 '\n'
+                 '\n'
+                 'Emulating container types\n'
+                 '=========================\n'
+                 '\n'
+                 'The following methods can be defined to implement container '
+                 'objects.\n'
+                 'Containers usually are sequences (such as lists or tuples) '
+                 'or mappings\n'
+                 '(like dictionaries), but can represent other containers as '
+                 'well.  The\n'
+                 'first set of methods is used either to emulate a sequence or '
+                 'to\n'
+                 'emulate a mapping; the difference is that for a sequence, '
+                 'the\n'
+                 'allowable keys should be the integers *k* for which "0 <= k '
+                 '< N" where\n'
+                 '*N* is the length of the sequence, or slice objects, which '
+                 'define a\n'
+                 'range of items.  It is also recommended that mappings '
+                 'provide the\n'
+                 'methods "keys()", "values()", "items()", "get()", '
+                 '"clear()",\n'
+                 '"setdefault()", "pop()", "popitem()", "copy()", and '
+                 '"update()"\n'
+                 "behaving similar to those for Python's standard dictionary "
+                 'objects.\n'
+                 'The "collections" module provides a "MutableMapping" '
+                 'abstract base\n'
+                 'class to help create those methods from a base set of '
+                 '"__getitem__()",\n'
+                 '"__setitem__()", "__delitem__()", and "keys()". Mutable '
+                 'sequences\n'
+                 'should provide methods "append()", "count()", "index()", '
+                 '"extend()",\n'
+                 '"insert()", "pop()", "remove()", "reverse()" and "sort()", '
+                 'like Python\n'
+                 'standard list objects.  Finally, sequence types should '
+                 'implement\n'
+                 'addition (meaning concatenation) and multiplication '
+                 '(meaning\n'
+                 'repetition) by defining the methods "__add__()", '
+                 '"__radd__()",\n'
+                 '"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" '
+                 'described\n'
+                 'below; they should not define other numerical operators.  It '
+                 'is\n'
+                 'recommended that both mappings and sequences implement the\n'
+                 '"__contains__()" method to allow efficient use of the "in" '
+                 'operator;\n'
+                 'for mappings, "in" should search the mapping\'s keys; for '
+                 'sequences, it\n'
+                 'should search through the values.  It is further recommended '
+                 'that both\n'
+                 'mappings and sequences implement the "__iter__()" method to '
+                 'allow\n'
+                 'efficient iteration through the container; for mappings, '
+                 '"__iter__()"\n'
+                 'should be the same as "keys()"; for sequences, it should '
+                 'iterate\n'
+                 'through the values.\n'
+                 '\n'
+                 'object.__len__(self)\n'
+                 '\n'
+                 '   Called to implement the built-in function "len()".  '
+                 'Should return\n'
+                 '   the length of the object, an integer ">=" 0.  Also, an '
+                 'object that\n'
+                 '   doesn\'t define a "__bool__()" method and whose '
+                 '"__len__()" method\n'
+                 '   returns zero is considered to be false in a Boolean '
+                 'context.\n'
+                 '\n'
+                 '   **CPython implementation detail:** In CPython, the length '
+                 'is\n'
+                 '   required to be at most "sys.maxsize". If the length is '
+                 'larger than\n'
+                 '   "sys.maxsize" some features (such as "len()") may raise\n'
+                 '   "OverflowError".  To prevent raising "OverflowError" by '
+                 'truth value\n'
+                 '   testing, an object must define a "__bool__()" method.\n'
+                 '\n'
+                 'object.__length_hint__(self)\n'
+                 '\n'
+                 '   Called to implement "operator.length_hint()". Should '
+                 'return an\n'
+                 '   estimated length for the object (which may be greater or '
+                 'less than\n'
+                 '   the actual length). The length must be an integer ">=" 0. '
+                 'This\n'
+                 '   method is purely an optimization and is never required '
+                 'for\n'
+                 '   correctness.\n'
+                 '\n'
+                 '   New in version 3.4.\n'
+                 '\n'
+                 'Note: Slicing is done exclusively with the following three '
+                 'methods.\n'
+                 '  A call like\n'
+                 '\n'
+                 '     a[1:2] = b\n'
+                 '\n'
+                 '  is translated to\n'
+                 '\n'
+                 '     a[slice(1, 2, None)] = b\n'
+                 '\n'
+                 '  and so forth.  Missing slice items are always filled in '
+                 'with "None".\n'
+                 '\n'
+                 'object.__getitem__(self, key)\n'
+                 '\n'
+                 '   Called to implement evaluation of "self[key]". For '
+                 'sequence types,\n'
+                 '   the accepted keys should be integers and slice objects.  '
+                 'Note that\n'
+                 '   the special interpretation of negative indexes (if the '
+                 'class wishes\n'
+                 '   to emulate a sequence type) is up to the "__getitem__()" '
+                 'method. If\n'
+                 '   *key* is of an inappropriate type, "TypeError" may be '
+                 'raised; if of\n'
+                 '   a value outside the set of indexes for the sequence '
+                 '(after any\n'
+                 '   special interpretation of negative values), "IndexError" '
+                 'should be\n'
+                 '   raised. For mapping types, if *key* is missing (not in '
+                 'the\n'
+                 '   container), "KeyError" should be raised.\n'
+                 '\n'
+                 '   Note: "for" loops expect that an "IndexError" will be '
+                 'raised for\n'
+                 '     illegal indexes to allow proper detection of the end of '
+                 'the\n'
+                 '     sequence.\n'
+                 '\n'
+                 'object.__missing__(self, key)\n'
+                 '\n'
+                 '   Called by "dict"."__getitem__()" to implement "self[key]" '
+                 'for dict\n'
+                 '   subclasses when key is not in the dictionary.\n'
+                 '\n'
+                 'object.__setitem__(self, key, value)\n'
+                 '\n'
+                 '   Called to implement assignment to "self[key]".  Same note '
+                 'as for\n'
+                 '   "__getitem__()".  This should only be implemented for '
+                 'mappings if\n'
+                 '   the objects support changes to the values for keys, or if '
+                 'new keys\n'
+                 '   can be added, or for sequences if elements can be '
+                 'replaced.  The\n'
+                 '   same exceptions should be raised for improper *key* '
+                 'values as for\n'
+                 '   the "__getitem__()" method.\n'
+                 '\n'
+                 'object.__delitem__(self, key)\n'
+                 '\n'
+                 '   Called to implement deletion of "self[key]".  Same note '
+                 'as for\n'
+                 '   "__getitem__()".  This should only be implemented for '
+                 'mappings if\n'
+                 '   the objects support removal of keys, or for sequences if '
+                 'elements\n'
+                 '   can be removed from the sequence.  The same exceptions '
+                 'should be\n'
+                 '   raised for improper *key* values as for the '
+                 '"__getitem__()" method.\n'
+                 '\n'
+                 'object.__iter__(self)\n'
+                 '\n'
+                 '   This method is called when an iterator is required for a '
+                 'container.\n'
+                 '   This method should return a new iterator object that can '
+                 'iterate\n'
+                 '   over all the objects in the container.  For mappings, it '
+                 'should\n'
+                 '   iterate over the keys of the container.\n'
+                 '\n'
+                 '   Iterator objects also need to implement this method; they '
+                 'are\n'
+                 '   required to return themselves.  For more information on '
+                 'iterator\n'
+                 '   objects, see *Iterator Types*.\n'
+                 '\n'
+                 'object.__reversed__(self)\n'
+                 '\n'
+                 '   Called (if present) by the "reversed()" built-in to '
+                 'implement\n'
+                 '   reverse iteration.  It should return a new iterator '
+                 'object that\n'
+                 '   iterates over all the objects in the container in reverse '
+                 'order.\n'
+                 '\n'
+                 '   If the "__reversed__()" method is not provided, the '
+                 '"reversed()"\n'
+                 '   built-in will fall back to using the sequence protocol '
+                 '("__len__()"\n'
+                 '   and "__getitem__()").  Objects that support the sequence '
+                 'protocol\n'
+                 '   should only provide "__reversed__()" if they can provide '
+                 'an\n'
+                 '   implementation that is more efficient than the one '
+                 'provided by\n'
+                 '   "reversed()".\n'
+                 '\n'
+                 'The membership test operators ("in" and "not in") are '
+                 'normally\n'
+                 'implemented as an iteration through a sequence.  However, '
+                 'container\n'
+                 'objects can supply the following special method with a more '
+                 'efficient\n'
+                 'implementation, which also does not require the object be a '
+                 'sequence.\n'
+                 '\n'
+                 'object.__contains__(self, item)\n'
+                 '\n'
+                 '   Called to implement membership test operators.  Should '
+                 'return true\n'
+                 '   if *item* is in *self*, false otherwise.  For mapping '
+                 'objects, this\n'
+                 '   should consider the keys of the mapping rather than the '
+                 'values or\n'
+                 '   the key-item pairs.\n'
+                 '\n'
+                 '   For objects that don\'t define "__contains__()", the '
+                 'membership test\n'
+                 '   first tries iteration via "__iter__()", then the old '
+                 'sequence\n'
+                 '   iteration protocol via "__getitem__()", see *this section '
+                 'in the\n'
+                 '   language reference*.\n'
+                 '\n'
+                 '\n'
+                 'Emulating numeric types\n'
+                 '=======================\n'
+                 '\n'
+                 'The following methods can be defined to emulate numeric '
+                 'objects.\n'
+                 'Methods corresponding to operations that are not supported '
+                 'by the\n'
+                 'particular kind of number implemented (e.g., bitwise '
+                 'operations for\n'
+                 'non-integral numbers) should be left undefined.\n'
+                 '\n'
+                 'object.__add__(self, other)\n'
+                 'object.__sub__(self, other)\n'
+                 'object.__mul__(self, other)\n'
+                 'object.__matmul__(self, other)\n'
+                 'object.__truediv__(self, other)\n'
+                 'object.__floordiv__(self, other)\n'
+                 'object.__mod__(self, other)\n'
+                 'object.__divmod__(self, other)\n'
+                 'object.__pow__(self, other[, modulo])\n'
+                 'object.__lshift__(self, other)\n'
+                 'object.__rshift__(self, other)\n'
+                 'object.__and__(self, other)\n'
+                 'object.__xor__(self, other)\n'
+                 'object.__or__(self, other)\n'
+                 '\n'
+                 '   These methods are called to implement the binary '
+                 'arithmetic\n'
+                 '   operations ("+", "-", "*", "@", "/", "//", "%", '
+                 '"divmod()",\n'
+                 '   "pow()", "**", "<<", ">>", "&", "^", "|").  For instance, '
+                 'to\n'
+                 '   evaluate the expression "x + y", where *x* is an instance '
+                 'of a\n'
+                 '   class that has an "__add__()" method, "x.__add__(y)" is '
+                 'called.\n'
+                 '   The "__divmod__()" method should be the equivalent to '
+                 'using\n'
+                 '   "__floordiv__()" and "__mod__()"; it should not be '
+                 'related to\n'
+                 '   "__truediv__()".  Note that "__pow__()" should be defined '
+                 'to accept\n'
+                 '   an optional third argument if the ternary version of the '
+                 'built-in\n'
+                 '   "pow()" function is to be supported.\n'
+                 '\n'
+                 '   If one of those methods does not support the operation '
+                 'with the\n'
+                 '   supplied arguments, it should return "NotImplemented".\n'
+                 '\n'
+                 'object.__radd__(self, other)\n'
+                 'object.__rsub__(self, other)\n'
+                 'object.__rmul__(self, other)\n'
+                 'object.__rmatmul__(self, other)\n'
+                 'object.__rtruediv__(self, other)\n'
+                 'object.__rfloordiv__(self, other)\n'
+                 'object.__rmod__(self, other)\n'
+                 'object.__rdivmod__(self, other)\n'
+                 'object.__rpow__(self, other)\n'
+                 'object.__rlshift__(self, other)\n'
+                 'object.__rrshift__(self, other)\n'
+                 'object.__rand__(self, other)\n'
+                 'object.__rxor__(self, other)\n'
+                 'object.__ror__(self, other)\n'
+                 '\n'
+                 '   These methods are called to implement the binary '
+                 'arithmetic\n'
+                 '   operations ("+", "-", "*", "@", "/", "//", "%", '
+                 '"divmod()",\n'
+                 '   "pow()", "**", "<<", ">>", "&", "^", "|") with reflected '
+                 '(swapped)\n'
+                 '   operands.  These functions are only called if the left '
+                 'operand does\n'
+                 '   not support the corresponding operation and the operands '
+                 'are of\n'
+                 '   different types. [2] For instance, to evaluate the '
+                 'expression "x -\n'
+                 '   y", where *y* is an instance of a class that has an '
+                 '"__rsub__()"\n'
+                 '   method, "y.__rsub__(x)" is called if "x.__sub__(y)" '
+                 'returns\n'
+                 '   *NotImplemented*.\n'
+                 '\n'
+                 '   Note that ternary "pow()" will not try calling '
+                 '"__rpow__()" (the\n'
+                 '   coercion rules would become too complicated).\n'
+                 '\n'
+                 "   Note: If the right operand's type is a subclass of the "
+                 'left\n'
+                 "     operand's type and that subclass provides the reflected "
+                 'method\n'
+                 '     for the operation, this method will be called before '
+                 'the left\n'
+                 "     operand's non-reflected method.  This behavior allows "
+                 'subclasses\n'
+                 "     to override their ancestors' operations.\n"
+                 '\n'
+                 'object.__iadd__(self, other)\n'
+                 'object.__isub__(self, other)\n'
+                 'object.__imul__(self, other)\n'
+                 'object.__imatmul__(self, other)\n'
+                 'object.__itruediv__(self, other)\n'
+                 'object.__ifloordiv__(self, other)\n'
+                 'object.__imod__(self, other)\n'
+                 'object.__ipow__(self, other[, modulo])\n'
+                 'object.__ilshift__(self, other)\n'
+                 'object.__irshift__(self, other)\n'
+                 'object.__iand__(self, other)\n'
+                 'object.__ixor__(self, other)\n'
+                 'object.__ior__(self, other)\n'
+                 '\n'
+                 '   These methods are called to implement the augmented '
+                 'arithmetic\n'
+                 '   assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", '
+                 '"**=",\n'
+                 '   "<<=", ">>=", "&=", "^=", "|=").  These methods should '
+                 'attempt to\n'
+                 '   do the operation in-place (modifying *self*) and return '
+                 'the result\n'
+                 '   (which could be, but does not have to be, *self*).  If a '
+                 'specific\n'
+                 '   method is not defined, the augmented assignment falls '
+                 'back to the\n'
+                 '   normal methods.  For instance, if *x* is an instance of a '
+                 'class\n'
+                 '   with an "__iadd__()" method, "x += y" is equivalent to "x '
+                 '=\n'
+                 '   x.__iadd__(y)" . Otherwise, "x.__add__(y)" and '
+                 '"y.__radd__(x)" are\n'
+                 '   considered, as with the evaluation of "x + y". In '
+                 'certain\n'
+                 '   situations, augmented assignment can result in unexpected '
+                 'errors\n'
+                 "   (see *Why does a_tuple[i] += ['item'] raise an exception "
+                 'when the\n'
+                 '   addition works?*), but this behavior is in fact part of '
+                 'the data\n'
+                 '   model.\n'
+                 '\n'
+                 'object.__neg__(self)\n'
+                 'object.__pos__(self)\n'
+                 'object.__abs__(self)\n'
+                 'object.__invert__(self)\n'
+                 '\n'
+                 '   Called to implement the unary arithmetic operations ("-", '
+                 '"+",\n'
+                 '   "abs()" and "~").\n'
+                 '\n'
+                 'object.__complex__(self)\n'
+                 'object.__int__(self)\n'
+                 'object.__float__(self)\n'
+                 'object.__round__(self[, n])\n'
+                 '\n'
+                 '   Called to implement the built-in functions "complex()", '
+                 '"int()",\n'
+                 '   "float()" and "round()".  Should return a value of the '
+                 'appropriate\n'
+                 '   type.\n'
+                 '\n'
+                 'object.__index__(self)\n'
+                 '\n'
+                 '   Called to implement "operator.index()", and whenever '
+                 'Python needs\n'
+                 '   to losslessly convert the numeric object to an integer '
+                 'object (such\n'
+                 '   as in slicing, or in the built-in "bin()", "hex()" and '
+                 '"oct()"\n'
+                 '   functions). Presence of this method indicates that the '
+                 'numeric\n'
+                 '   object is an integer type.  Must return an integer.\n'
+                 '\n'
+                 '   Note: In order to have a coherent integer type class, '
+                 'when\n'
+                 '     "__index__()" is defined "__int__()" should also be '
+                 'defined, and\n'
+                 '     both should return the same value.\n'
+                 '\n'
+                 '\n'
+                 'With Statement Context Managers\n'
+                 '===============================\n'
+                 '\n'
+                 'A *context manager* is an object that defines the runtime '
+                 'context to\n'
+                 'be established when executing a "with" statement. The '
+                 'context manager\n'
+                 'handles the entry into, and the exit from, the desired '
+                 'runtime context\n'
+                 'for the execution of the block of code.  Context managers '
+                 'are normally\n'
+                 'invoked using the "with" statement (described in section '
+                 '*The with\n'
+                 'statement*), but can also be used by directly invoking their '
+                 'methods.\n'
+                 '\n'
+                 'Typical uses of context managers include saving and '
+                 'restoring various\n'
+                 'kinds of global state, locking and unlocking resources, '
+                 'closing opened\n'
+                 'files, etc.\n'
+                 '\n'
+                 'For more information on context managers, see *Context '
+                 'Manager Types*.\n'
+                 '\n'
+                 'object.__enter__(self)\n'
+                 '\n'
+                 '   Enter the runtime context related to this object. The '
+                 '"with"\n'
+                 "   statement will bind this method's return value to the "
+                 'target(s)\n'
+                 '   specified in the "as" clause of the statement, if any.\n'
+                 '\n'
+                 'object.__exit__(self, exc_type, exc_value, traceback)\n'
+                 '\n'
+                 '   Exit the runtime context related to this object. The '
+                 'parameters\n'
+                 '   describe the exception that caused the context to be '
+                 'exited. If the\n'
+                 '   context was exited without an exception, all three '
+                 'arguments will\n'
+                 '   be "None".\n'
+                 '\n'
+                 '   If an exception is supplied, and the method wishes to '
+                 'suppress the\n'
+                 '   exception (i.e., prevent it from being propagated), it '
+                 'should\n'
+                 '   return a true value. Otherwise, the exception will be '
+                 'processed\n'
+                 '   normally upon exit from this method.\n'
+                 '\n'
+                 '   Note that "__exit__()" methods should not reraise the '
+                 'passed-in\n'
+                 "   exception; this is the caller's responsibility.\n"
+                 '\n'
+                 'See also: **PEP 343** - The "with" statement\n'
+                 '\n'
+                 '     The specification, background, and examples for the '
+                 'Python "with"\n'
+                 '     statement.\n'
+                 '\n'
+                 '\n'
+                 'Special method lookup\n'
+                 '=====================\n'
+                 '\n'
+                 'For custom classes, implicit invocations of special methods '
+                 'are only\n'
+                 "guaranteed to work correctly if defined on an object's type, "
+                 'not in\n'
+                 "the object's instance dictionary.  That behaviour is the "
+                 'reason why\n'
+                 'the following code raises an exception:\n'
+                 '\n'
+                 '   >>> class C:\n'
+                 '   ...     pass\n'
+                 '   ...\n'
+                 '   >>> c = C()\n'
+                 '   >>> c.__len__ = lambda: 5\n'
+                 '   >>> len(c)\n'
+                 '   Traceback (most recent call last):\n'
+                 '     File "<stdin>", line 1, in <module>\n'
+                 "   TypeError: object of type 'C' has no len()\n"
+                 '\n'
+                 'The rationale behind this behaviour lies with a number of '
+                 'special\n'
+                 'methods such as "__hash__()" and "__repr__()" that are '
+                 'implemented by\n'
+                 'all objects, including type objects. If the implicit lookup '
+                 'of these\n'
+                 'methods used the conventional lookup process, they would '
+                 'fail when\n'
+                 'invoked on the type object itself:\n'
+                 '\n'
+                 '   >>> 1 .__hash__() == hash(1)\n'
+                 '   True\n'
+                 '   >>> int.__hash__() == hash(int)\n'
+                 '   Traceback (most recent call last):\n'
+                 '     File "<stdin>", line 1, in <module>\n'
+                 "   TypeError: descriptor '__hash__' of 'int' object needs an "
+                 'argument\n'
+                 '\n'
+                 'Incorrectly attempting to invoke an unbound method of a '
+                 'class in this\n'
+                 "way is sometimes referred to as 'metaclass confusion', and "
+                 'is avoided\n'
+                 'by bypassing the instance when looking up special methods:\n'
+                 '\n'
+                 '   >>> type(1).__hash__(1) == hash(1)\n'
+                 '   True\n'
+                 '   >>> type(int).__hash__(int) == hash(int)\n'
+                 '   True\n'
+                 '\n'
+                 'In addition to bypassing any instance attributes in the '
+                 'interest of\n'
+                 'correctness, implicit special method lookup generally also '
+                 'bypasses\n'
+                 'the "__getattribute__()" method even of the object\'s '
+                 'metaclass:\n'
+                 '\n'
+                 '   >>> class Meta(type):\n'
+                 '   ...     def __getattribute__(*args):\n'
+                 '   ...         print("Metaclass getattribute invoked")\n'
+                 '   ...         return type.__getattribute__(*args)\n'
+                 '   ...\n'
+                 '   >>> class C(object, metaclass=Meta):\n'
+                 '   ...     def __len__(self):\n'
+                 '   ...         return 10\n'
+                 '   ...     def __getattribute__(*args):\n'
+                 '   ...         print("Class getattribute invoked")\n'
+                 '   ...         return object.__getattribute__(*args)\n'
+                 '   ...\n'
+                 '   >>> c = C()\n'
+                 '   >>> c.__len__()                 # Explicit lookup via '
+                 'instance\n'
+                 '   Class getattribute invoked\n'
+                 '   10\n'
+                 '   >>> type(c).__len__(c)          # Explicit lookup via '
+                 'type\n'
+                 '   Metaclass getattribute invoked\n'
+                 '   10\n'
+                 '   >>> len(c)                      # Implicit lookup\n'
+                 '   10\n'
+                 '\n'
+                 'Bypassing the "__getattribute__()" machinery in this fashion '
+                 'provides\n'
+                 'significant scope for speed optimisations within the '
+                 'interpreter, at\n'
+                 'the cost of some flexibility in the handling of special '
+                 'methods (the\n'
+                 'special method *must* be set on the class object itself in '
+                 'order to be\n'
+                 'consistently invoked by the interpreter).\n',
+ 'string-methods': '\n'
+                   'String Methods\n'
+                   '**************\n'
+                   '\n'
+                   'Strings implement all of the *common* sequence operations, '
+                   'along with\n'
+                   'the additional methods described below.\n'
+                   '\n'
+                   'Strings also support two styles of string formatting, one '
+                   'providing a\n'
+                   'large degree of flexibility and customization (see '
+                   '"str.format()",\n'
+                   '*Format String Syntax* and *Custom String Formatting*) and '
+                   'the other\n'
+                   'based on C "printf" style formatting that handles a '
+                   'narrower range of\n'
+                   'types and is slightly harder to use correctly, but is '
+                   'often faster for\n'
+                   'the cases it can handle (*printf-style String '
+                   'Formatting*).\n'
+                   '\n'
+                   'The *Text Processing Services* section of the standard '
+                   'library covers\n'
+                   'a number of other modules that provide various text '
+                   'related utilities\n'
+                   '(including regular expression support in the "re" '
+                   'module).\n'
+                   '\n'
+                   'str.capitalize()\n'
+                   '\n'
+                   '   Return a copy of the string with its first character '
+                   'capitalized\n'
+                   '   and the rest lowercased.\n'
+                   '\n'
+                   'str.casefold()\n'
+                   '\n'
+                   '   Return a casefolded copy of the string. Casefolded '
+                   'strings may be\n'
+                   '   used for caseless matching.\n'
+                   '\n'
+                   '   Casefolding is similar to lowercasing but more '
+                   'aggressive because\n'
+                   '   it is intended to remove all case distinctions in a '
+                   'string. For\n'
+                   '   example, the German lowercase letter "\'?\'" is '
+                   'equivalent to ""ss"".\n'
+                   '   Since it is already lowercase, "lower()" would do '
+                   'nothing to "\'?\'";\n'
+                   '   "casefold()" converts it to ""ss"".\n'
+                   '\n'
+                   '   The casefolding algorithm is described in section 3.13 '
+                   'of the\n'
+                   '   Unicode Standard.\n'
+                   '\n'
+                   '   New in version 3.3.\n'
+                   '\n'
+                   'str.center(width[, fillchar])\n'
+                   '\n'
+                   '   Return centered in a string of length *width*. Padding '
+                   'is done\n'
+                   '   using the specified *fillchar* (default is an ASCII '
+                   'space). The\n'
+                   '   original string is returned if *width* is less than or '
+                   'equal to\n'
+                   '   "len(s)".\n'
+                   '\n'
+                   'str.count(sub[, start[, end]])\n'
+                   '\n'
+                   '   Return the number of non-overlapping occurrences of '
+                   'substring *sub*\n'
+                   '   in the range [*start*, *end*].  Optional arguments '
+                   '*start* and\n'
+                   '   *end* are interpreted as in slice notation.\n'
+                   '\n'
+                   'str.encode(encoding="utf-8", errors="strict")\n'
+                   '\n'
+                   '   Return an encoded version of the string as a bytes '
+                   'object. Default\n'
+                   '   encoding is "\'utf-8\'". *errors* may be given to set a '
+                   'different\n'
+                   '   error handling scheme. The default for *errors* is '
+                   '"\'strict\'",\n'
+                   '   meaning that encoding errors raise a "UnicodeError". '
+                   'Other possible\n'
+                   '   values are "\'ignore\'", "\'replace\'", '
+                   '"\'xmlcharrefreplace\'",\n'
+                   '   "\'backslashreplace\'" and any other name registered '
+                   'via\n'
+                   '   "codecs.register_error()", see section *Error '
+                   'Handlers*. For a list\n'
+                   '   of possible encodings, see section *Standard '
+                   'Encodings*.\n'
+                   '\n'
+                   '   Changed in version 3.1: Support for keyword arguments '
+                   'added.\n'
+                   '\n'
+                   'str.endswith(suffix[, start[, end]])\n'
+                   '\n'
+                   '   Return "True" if the string ends with the specified '
+                   '*suffix*,\n'
+                   '   otherwise return "False".  *suffix* can also be a tuple '
+                   'of suffixes\n'
+                   '   to look for.  With optional *start*, test beginning at '
+                   'that\n'
+                   '   position.  With optional *end*, stop comparing at that '
+                   'position.\n'
+                   '\n'
+                   'str.expandtabs(tabsize=8)\n'
+                   '\n'
+                   '   Return a copy of the string where all tab characters '
+                   'are replaced\n'
+                   '   by one or more spaces, depending on the current column '
+                   'and the\n'
+                   '   given tab size.  Tab positions occur every *tabsize* '
+                   'characters\n'
+                   '   (default is 8, giving tab positions at columns 0, 8, 16 '
+                   'and so on).\n'
+                   '   To expand the string, the current column is set to zero '
+                   'and the\n'
+                   '   string is examined character by character.  If the '
+                   'character is a\n'
+                   '   tab ("\\t"), one or more space characters are inserted '
+                   'in the result\n'
+                   '   until the current column is equal to the next tab '
+                   'position. (The\n'
+                   '   tab character itself is not copied.)  If the character '
+                   'is a newline\n'
+                   '   ("\\n") or return ("\\r"), it is copied and the current '
+                   'column is\n'
+                   '   reset to zero.  Any other character is copied unchanged '
+                   'and the\n'
+                   '   current column is incremented by one regardless of how '
+                   'the\n'
+                   '   character is represented when printed.\n'
+                   '\n'
+                   "   >>> '01\\t012\\t0123\\t01234'.expandtabs()\n"
+                   "   '01      012     0123    01234'\n"
+                   "   >>> '01\\t012\\t0123\\t01234'.expandtabs(4)\n"
+                   "   '01  012 0123    01234'\n"
+                   '\n'
+                   'str.find(sub[, start[, end]])\n'
+                   '\n'
+                   '   Return the lowest index in the string where substring '
+                   '*sub* is\n'
+                   '   found within the slice "s[start:end]".  Optional '
+                   'arguments *start*\n'
+                   '   and *end* are interpreted as in slice notation.  Return '
+                   '"-1" if\n'
+                   '   *sub* is not found.\n'
+                   '\n'
+                   '   Note: The "find()" method should be used only if you '
+                   'need to know\n'
+                   '     the position of *sub*.  To check if *sub* is a '
+                   'substring or not,\n'
+                   '     use the "in" operator:\n'
+                   '\n'
+                   "        >>> 'Py' in 'Python'\n"
+                   '        True\n'
+                   '\n'
+                   'str.format(*args, **kwargs)\n'
+                   '\n'
+                   '   Perform a string formatting operation.  The string on '
+                   'which this\n'
+                   '   method is called can contain literal text or '
+                   'replacement fields\n'
+                   '   delimited by braces "{}".  Each replacement field '
+                   'contains either\n'
+                   '   the numeric index of a positional argument, or the name '
+                   'of a\n'
+                   '   keyword argument.  Returns a copy of the string where '
+                   'each\n'
+                   '   replacement field is replaced with the string value of '
+                   'the\n'
+                   '   corresponding argument.\n'
+                   '\n'
+                   '   >>> "The sum of 1 + 2 is {0}".format(1+2)\n'
+                   "   'The sum of 1 + 2 is 3'\n"
+                   '\n'
+                   '   See *Format String Syntax* for a description of the '
+                   'various\n'
+                   '   formatting options that can be specified in format '
+                   'strings.\n'
+                   '\n'
+                   'str.format_map(mapping)\n'
+                   '\n'
+                   '   Similar to "str.format(**mapping)", except that '
+                   '"mapping" is used\n'
+                   '   directly and not copied to a "dict".  This is useful if '
+                   'for example\n'
+                   '   "mapping" is a dict subclass:\n'
+                   '\n'
+                   '   >>> class Default(dict):\n'
+                   '   ...     def __missing__(self, key):\n'
+                   '   ...         return key\n'
+                   '   ...\n'
+                   "   >>> '{name} was born in "
+                   "{country}'.format_map(Default(name='Guido'))\n"
+                   "   'Guido was born in country'\n"
+                   '\n'
+                   '   New in version 3.2.\n'
+                   '\n'
+                   'str.index(sub[, start[, end]])\n'
+                   '\n'
+                   '   Like "find()", but raise "ValueError" when the '
+                   'substring is not\n'
+                   '   found.\n'
+                   '\n'
+                   'str.isalnum()\n'
+                   '\n'
+                   '   Return true if all characters in the string are '
+                   'alphanumeric and\n'
+                   '   there is at least one character, false otherwise.  A '
+                   'character "c"\n'
+                   '   is alphanumeric if one of the following returns '
+                   '"True":\n'
+                   '   "c.isalpha()", "c.isdecimal()", "c.isdigit()", or '
+                   '"c.isnumeric()".\n'
+                   '\n'
+                   'str.isalpha()\n'
+                   '\n'
+                   '   Return true if all characters in the string are '
+                   'alphabetic and\n'
+                   '   there is at least one character, false otherwise.  '
+                   'Alphabetic\n'
+                   '   characters are those characters defined in the Unicode '
+                   'character\n'
+                   '   database as "Letter", i.e., those with general category '
+                   'property\n'
+                   '   being one of "Lm", "Lt", "Lu", "Ll", or "Lo".  Note '
+                   'that this is\n'
+                   '   different from the "Alphabetic" property defined in the '
+                   'Unicode\n'
+                   '   Standard.\n'
+                   '\n'
+                   'str.isdecimal()\n'
+                   '\n'
+                   '   Return true if all characters in the string are decimal '
+                   'characters\n'
+                   '   and there is at least one character, false otherwise. '
+                   'Decimal\n'
+                   '   characters are those that can be used to form numbers '
+                   'in base 10,\n'
+                   '   e.g. U+0660, ARABIC-INDIC DIGIT ZERO.  Formally a '
+                   'decimal character\n'
+                   '   is a character in the Unicode General Category "Nd".\n'
+                   '\n'
+                   'str.isdigit()\n'
+                   '\n'
+                   '   Return true if all characters in the string are digits '
+                   'and there is\n'
+                   '   at least one character, false otherwise.  Digits '
+                   'include decimal\n'
+                   '   characters and digits that need special handling, such '
+                   'as the\n'
+                   '   compatibility superscript digits. This covers digits '
+                   'which cannot\n'
+                   '   be used to form numbers in base 10, like the Kharosthi '
+                   'numbers.\n'
+                   '   Formally, a digit is a character that has the property '
+                   'value\n'
+                   '   Numeric_Type=Digit or Numeric_Type=Decimal.\n'
+                   '\n'
+                   'str.isidentifier()\n'
+                   '\n'
+                   '   Return true if the string is a valid identifier '
+                   'according to the\n'
+                   '   language definition, section *Identifiers and '
+                   'keywords*.\n'
+                   '\n'
+                   '   Use "keyword.iskeyword()" to test for reserved '
+                   'identifiers such as\n'
+                   '   "def" and "class".\n'
+                   '\n'
+                   'str.islower()\n'
+                   '\n'
+                   '   Return true if all cased characters [4] in the string '
+                   'are lowercase\n'
+                   '   and there is at least one cased character, false '
+                   'otherwise.\n'
+                   '\n'
+                   'str.isnumeric()\n'
+                   '\n'
+                   '   Return true if all characters in the string are numeric '
+                   'characters,\n'
+                   '   and there is at least one character, false otherwise. '
+                   'Numeric\n'
+                   '   characters include digit characters, and all characters '
+                   'that have\n'
+                   '   the Unicode numeric value property, e.g. U+2155, VULGAR '
+                   'FRACTION\n'
+                   '   ONE FIFTH.  Formally, numeric characters are those with '
+                   'the\n'
+                   '   property value Numeric_Type=Digit, Numeric_Type=Decimal '
+                   'or\n'
+                   '   Numeric_Type=Numeric.\n'
+                   '\n'
+                   'str.isprintable()\n'
+                   '\n'
+                   '   Return true if all characters in the string are '
+                   'printable or the\n'
+                   '   string is empty, false otherwise.  Nonprintable '
+                   'characters are\n'
+                   '   those characters defined in the Unicode character '
+                   'database as\n'
+                   '   "Other" or "Separator", excepting the ASCII space '
+                   '(0x20) which is\n'
+                   '   considered printable.  (Note that printable characters '
+                   'in this\n'
+                   '   context are those which should not be escaped when '
+                   '"repr()" is\n'
+                   '   invoked on a string.  It has no bearing on the handling '
+                   'of strings\n'
+                   '   written to "sys.stdout" or "sys.stderr".)\n'
+                   '\n'
+                   'str.isspace()\n'
+                   '\n'
+                   '   Return true if there are only whitespace characters in '
+                   'the string\n'
+                   '   and there is at least one character, false otherwise.  '
+                   'Whitespace\n'
+                   '   characters  are those characters defined in the Unicode '
+                   'character\n'
+                   '   database as "Other" or "Separator" and those with '
+                   'bidirectional\n'
+                   '   property being one of "WS", "B", or "S".\n'
+                   '\n'
+                   'str.istitle()\n'
+                   '\n'
+                   '   Return true if the string is a titlecased string and '
+                   'there is at\n'
+                   '   least one character, for example uppercase characters '
+                   'may only\n'
+                   '   follow uncased characters and lowercase characters only '
+                   'cased ones.\n'
+                   '   Return false otherwise.\n'
+                   '\n'
+                   'str.isupper()\n'
+                   '\n'
+                   '   Return true if all cased characters [4] in the string '
+                   'are uppercase\n'
+                   '   and there is at least one cased character, false '
+                   'otherwise.\n'
+                   '\n'
+                   'str.join(iterable)\n'
+                   '\n'
+                   '   Return a string which is the concatenation of the '
+                   'strings in\n'
+                   '   *iterable*. A "TypeError" will be raised if there are '
+                   'any non-\n'
+                   '   string values in *iterable*, including "bytes" '
+                   'objects.  The\n'
+                   '   separator between elements is the string providing this '
+                   'method.\n'
+                   '\n'
+                   'str.ljust(width[, fillchar])\n'
+                   '\n'
+                   '   Return the string left justified in a string of length '
+                   '*width*.\n'
+                   '   Padding is done using the specified *fillchar* (default '
+                   'is an ASCII\n'
+                   '   space). The original string is returned if *width* is '
+                   'less than or\n'
+                   '   equal to "len(s)".\n'
+                   '\n'
+                   'str.lower()\n'
+                   '\n'
+                   '   Return a copy of the string with all the cased '
+                   'characters [4]\n'
+                   '   converted to lowercase.\n'
+                   '\n'
+                   '   The lowercasing algorithm used is described in section '
+                   '3.13 of the\n'
+                   '   Unicode Standard.\n'
+                   '\n'
+                   'str.lstrip([chars])\n'
+                   '\n'
+                   '   Return a copy of the string with leading characters '
+                   'removed.  The\n'
+                   '   *chars* argument is a string specifying the set of '
+                   'characters to be\n'
+                   '   removed.  If omitted or "None", the *chars* argument '
+                   'defaults to\n'
+                   '   removing whitespace.  The *chars* argument is not a '
+                   'prefix; rather,\n'
+                   '   all combinations of its values are stripped:\n'
+                   '\n'
+                   "      >>> '   spacious   '.lstrip()\n"
+                   "      'spacious   '\n"
+                   "      >>> 'www.example.com'.lstrip('cmowz.')\n"
+                   "      'example.com'\n"
+                   '\n'
+                   'static str.maketrans(x[, y[, z]])\n'
+                   '\n'
+                   '   This static method returns a translation table usable '
+                   'for\n'
+                   '   "str.translate()".\n'
+                   '\n'
+                   '   If there is only one argument, it must be a dictionary '
+                   'mapping\n'
+                   '   Unicode ordinals (integers) or characters (strings of '
+                   'length 1) to\n'
+                   '   Unicode ordinals, strings (of arbitrary lengths) or '
+                   '"None".\n'
+                   '   Character keys will then be converted to ordinals.\n'
+                   '\n'
+                   '   If there are two arguments, they must be strings of '
+                   'equal length,\n'
+                   '   and in the resulting dictionary, each character in x '
+                   'will be mapped\n'
+                   '   to the character at the same position in y.  If there '
+                   'is a third\n'
+                   '   argument, it must be a string, whose characters will be '
+                   'mapped to\n'
+                   '   "None" in the result.\n'
+                   '\n'
+                   'str.partition(sep)\n'
+                   '\n'
+                   '   Split the string at the first occurrence of *sep*, and '
+                   'return a\n'
+                   '   3-tuple containing the part before the separator, the '
+                   'separator\n'
+                   '   itself, and the part after the separator.  If the '
+                   'separator is not\n'
+                   '   found, return a 3-tuple containing the string itself, '
+                   'followed by\n'
+                   '   two empty strings.\n'
+                   '\n'
+                   'str.replace(old, new[, count])\n'
+                   '\n'
+                   '   Return a copy of the string with all occurrences of '
+                   'substring *old*\n'
+                   '   replaced by *new*.  If the optional argument *count* is '
+                   'given, only\n'
+                   '   the first *count* occurrences are replaced.\n'
+                   '\n'
+                   'str.rfind(sub[, start[, end]])\n'
+                   '\n'
+                   '   Return the highest index in the string where substring '
+                   '*sub* is\n'
+                   '   found, such that *sub* is contained within '
+                   '"s[start:end]".\n'
+                   '   Optional arguments *start* and *end* are interpreted as '
+                   'in slice\n'
+                   '   notation.  Return "-1" on failure.\n'
+                   '\n'
+                   'str.rindex(sub[, start[, end]])\n'
+                   '\n'
+                   '   Like "rfind()" but raises "ValueError" when the '
+                   'substring *sub* is\n'
+                   '   not found.\n'
+                   '\n'
+                   'str.rjust(width[, fillchar])\n'
+                   '\n'
+                   '   Return the string right justified in a string of length '
+                   '*width*.\n'
+                   '   Padding is done using the specified *fillchar* (default '
+                   'is an ASCII\n'
+                   '   space). The original string is returned if *width* is '
+                   'less than or\n'
+                   '   equal to "len(s)".\n'
+                   '\n'
+                   'str.rpartition(sep)\n'
+                   '\n'
+                   '   Split the string at the last occurrence of *sep*, and '
+                   'return a\n'
+                   '   3-tuple containing the part before the separator, the '
+                   'separator\n'
+                   '   itself, and the part after the separator.  If the '
+                   'separator is not\n'
+                   '   found, return a 3-tuple containing two empty strings, '
+                   'followed by\n'
+                   '   the string itself.\n'
+                   '\n'
+                   'str.rsplit(sep=None, maxsplit=-1)\n'
+                   '\n'
+                   '   Return a list of the words in the string, using *sep* '
+                   'as the\n'
+                   '   delimiter string. If *maxsplit* is given, at most '
+                   '*maxsplit* splits\n'
+                   '   are done, the *rightmost* ones.  If *sep* is not '
+                   'specified or\n'
+                   '   "None", any whitespace string is a separator.  Except '
+                   'for splitting\n'
+                   '   from the right, "rsplit()" behaves like "split()" which '
+                   'is\n'
+                   '   described in detail below.\n'
+                   '\n'
+                   'str.rstrip([chars])\n'
+                   '\n'
+                   '   Return a copy of the string with trailing characters '
+                   'removed.  The\n'
+                   '   *chars* argument is a string specifying the set of '
+                   'characters to be\n'
+                   '   removed.  If omitted or "None", the *chars* argument '
+                   'defaults to\n'
+                   '   removing whitespace.  The *chars* argument is not a '
+                   'suffix; rather,\n'
+                   '   all combinations of its values are stripped:\n'
+                   '\n'
+                   "      >>> '   spacious   '.rstrip()\n"
+                   "      '   spacious'\n"
+                   "      >>> 'mississippi'.rstrip('ipz')\n"
+                   "      'mississ'\n"
+                   '\n'
+                   'str.split(sep=None, maxsplit=-1)\n'
+                   '\n'
+                   '   Return a list of the words in the string, using *sep* '
+                   'as the\n'
+                   '   delimiter string.  If *maxsplit* is given, at most '
+                   '*maxsplit*\n'
+                   '   splits are done (thus, the list will have at most '
+                   '"maxsplit+1"\n'
+                   '   elements).  If *maxsplit* is not specified or "-1", '
+                   'then there is\n'
+                   '   no limit on the number of splits (all possible splits '
+                   'are made).\n'
+                   '\n'
+                   '   If *sep* is given, consecutive delimiters are not '
+                   'grouped together\n'
+                   '   and are deemed to delimit empty strings (for example,\n'
+                   '   "\'1,,2\'.split(\',\')" returns "[\'1\', \'\', '
+                   '\'2\']").  The *sep* argument\n'
+                   '   may consist of multiple characters (for example,\n'
+                   '   "\'1<>2<>3\'.split(\'<>\')" returns "[\'1\', \'2\', '
+                   '\'3\']"). Splitting an\n'
+                   '   empty string with a specified separator returns '
+                   '"[\'\']".\n'
+                   '\n'
+                   '   For example:\n'
+                   '\n'
+                   "      >>> '1,2,3'.split(',')\n"
+                   "      ['1', '2', '3']\n"
+                   "      >>> '1,2,3'.split(',', maxsplit=1)\n"
+                   "      ['1', '2,3']\n"
+                   "      >>> '1,2,,3,'.split(',')\n"
+                   "      ['1', '2', '', '3', '']\n"
+                   '\n'
+                   '   If *sep* is not specified or is "None", a different '
+                   'splitting\n'
+                   '   algorithm is applied: runs of consecutive whitespace '
+                   'are regarded\n'
+                   '   as a single separator, and the result will contain no '
+                   'empty strings\n'
+                   '   at the start or end if the string has leading or '
+                   'trailing\n'
+                   '   whitespace.  Consequently, splitting an empty string or '
+                   'a string\n'
+                   '   consisting of just whitespace with a "None" separator '
+                   'returns "[]".\n'
+                   '\n'
+                   '   For example:\n'
+                   '\n'
+                   "      >>> '1 2 3'.split()\n"
+                   "      ['1', '2', '3']\n"
+                   "      >>> '1 2 3'.split(maxsplit=1)\n"
+                   "      ['1', '2 3']\n"
+                   "      >>> '   1   2   3   '.split()\n"
+                   "      ['1', '2', '3']\n"
+                   '\n'
+                   'str.splitlines([keepends])\n'
+                   '\n'
+                   '   Return a list of the lines in the string, breaking at '
+                   'line\n'
+                   '   boundaries.  Line breaks are not included in the '
+                   'resulting list\n'
+                   '   unless *keepends* is given and true.\n'
+                   '\n'
+                   '   This method splits on the following line boundaries.  '
+                   'In\n'
+                   '   particular, the boundaries are a superset of *universal '
+                   'newlines*.\n'
+                   '\n'
+                   '   '
+                   '+-------------------------+-------------------------------+\n'
+                   '   | Representation          | '
+                   'Description                   |\n'
+                   '   '
+                   '+=========================+===============================+\n'
+                   '   | "\\n"                    | Line '
+                   'Feed                     |\n'
+                   '   '
+                   '+-------------------------+-------------------------------+\n'
+                   '   | "\\r"                    | Carriage '
+                   'Return               |\n'
+                   '   '
+                   '+-------------------------+-------------------------------+\n'
+                   '   | "\\r\\n"                  | Carriage Return + Line '
+                   'Feed   |\n'
+                   '   '
+                   '+-------------------------+-------------------------------+\n'
+                   '   | "\\v" or "\\x0b"          | Line '
+                   'Tabulation               |\n'
+                   '   '
+                   '+-------------------------+-------------------------------+\n'
+                   '   | "\\f" or "\\x0c"          | Form '
+                   'Feed                     |\n'
+                   '   '
+                   '+-------------------------+-------------------------------+\n'
+                   '   | "\\x1c"                  | File '
+                   'Separator                |\n'
+                   '   '
+                   '+-------------------------+-------------------------------+\n'
+                   '   | "\\x1d"                  | Group '
+                   'Separator               |\n'
+                   '   '
+                   '+-------------------------+-------------------------------+\n'
+                   '   | "\\x1e"                  | Record '
+                   'Separator              |\n'
+                   '   '
+                   '+-------------------------+-------------------------------+\n'
+                   '   | "\\x85"                  | Next Line (C1 Control '
+                   'Code)   |\n'
+                   '   '
+                   '+-------------------------+-------------------------------+\n'
+                   '   | "\\u2028"                | Line '
+                   'Separator                |\n'
+                   '   '
+                   '+-------------------------+-------------------------------+\n'
+                   '   | "\\u2029"                | Paragraph '
+                   'Separator           |\n'
+                   '   '
+                   '+-------------------------+-------------------------------+\n'
+                   '\n'
+                   '   Changed in version 3.2: "\\v" and "\\f" added to list '
+                   'of line\n'
+                   '   boundaries.\n'
+                   '\n'
+                   '   For example:\n'
+                   '\n'
+                   "      >>> 'ab c\\n\\nde fg\\rkl\\r\\n'.splitlines()\n"
+                   "      ['ab c', '', 'de fg', 'kl']\n"
+                   "      >>> 'ab c\\n\\nde "
+                   "fg\\rkl\\r\\n'.splitlines(keepends=True)\n"
+                   "      ['ab c\\n', '\\n', 'de fg\\r', 'kl\\r\\n']\n"
+                   '\n'
+                   '   Unlike "split()" when a delimiter string *sep* is '
+                   'given, this\n'
+                   '   method returns an empty list for the empty string, and '
+                   'a terminal\n'
+                   '   line break does not result in an extra line:\n'
+                   '\n'
+                   '      >>> "".splitlines()\n'
+                   '      []\n'
+                   '      >>> "One line\\n".splitlines()\n'
+                   "      ['One line']\n"
+                   '\n'
+                   '   For comparison, "split(\'\\n\')" gives:\n'
+                   '\n'
+                   "      >>> ''.split('\\n')\n"
+                   "      ['']\n"
+                   "      >>> 'Two lines\\n'.split('\\n')\n"
+                   "      ['Two lines', '']\n"
+                   '\n'
+                   'str.startswith(prefix[, start[, end]])\n'
+                   '\n'
+                   '   Return "True" if string starts with the *prefix*, '
+                   'otherwise return\n'
+                   '   "False". *prefix* can also be a tuple of prefixes to '
+                   'look for.\n'
+                   '   With optional *start*, test string beginning at that '
+                   'position.\n'
+                   '   With optional *end*, stop comparing string at that '
+                   'position.\n'
+                   '\n'
+                   'str.strip([chars])\n'
+                   '\n'
+                   '   Return a copy of the string with the leading and '
+                   'trailing\n'
+                   '   characters removed. The *chars* argument is a string '
+                   'specifying the\n'
+                   '   set of characters to be removed. If omitted or "None", '
+                   'the *chars*\n'
+                   '   argument defaults to removing whitespace. The *chars* '
+                   'argument is\n'
+                   '   not a prefix or suffix; rather, all combinations of its '
+                   'values are\n'
+                   '   stripped:\n'
+                   '\n'
+                   "      >>> '   spacious   '.strip()\n"
+                   "      'spacious'\n"
+                   "      >>> 'www.example.com'.strip('cmowz.')\n"
+                   "      'example'\n"
+                   '\n'
+                   '   The outermost leading and trailing *chars* argument '
+                   'values are\n'
+                   '   stripped from the string. Characters are removed from '
+                   'the leading\n'
+                   '   end until reaching a string character that is not '
+                   'contained in the\n'
+                   '   set of characters in *chars*. A similar action takes '
+                   'place on the\n'
+                   '   trailing end. For example:\n'
+                   '\n'
+                   "      >>> comment_string = '#....... Section 3.2.1 Issue "
+                   "#32 .......'\n"
+                   "      >>> comment_string.strip('.#! ')\n"
+                   "      'Section 3.2.1 Issue #32'\n"
+                   '\n'
+                   'str.swapcase()\n'
+                   '\n'
+                   '   Return a copy of the string with uppercase characters '
+                   'converted to\n'
+                   '   lowercase and vice versa. Note that it is not '
+                   'necessarily true that\n'
+                   '   "s.swapcase().swapcase() == s".\n'
+                   '\n'
+                   'str.title()\n'
+                   '\n'
+                   '   Return a titlecased version of the string where words '
+                   'start with an\n'
+                   '   uppercase character and the remaining characters are '
+                   'lowercase.\n'
+                   '\n'
+                   '   For example:\n'
+                   '\n'
+                   "      >>> 'Hello world'.title()\n"
+                   "      'Hello World'\n"
+                   '\n'
+                   '   The algorithm uses a simple language-independent '
+                   'definition of a\n'
+                   '   word as groups of consecutive letters.  The definition '
+                   'works in\n'
+                   '   many contexts but it means that apostrophes in '
+                   'contractions and\n'
+                   '   possessives form word boundaries, which may not be the '
+                   'desired\n'
+                   '   result:\n'
+                   '\n'
+                   '      >>> "they\'re bill\'s friends from the UK".title()\n'
+                   '      "They\'Re Bill\'S Friends From The Uk"\n'
+                   '\n'
+                   '   A workaround for apostrophes can be constructed using '
+                   'regular\n'
+                   '   expressions:\n'
+                   '\n'
+                   '      >>> import re\n'
+                   '      >>> def titlecase(s):\n'
+                   '      ...     return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n'
+                   '      ...                   lambda mo: '
+                   'mo.group(0)[0].upper() +\n'
+                   '      ...                              '
+                   'mo.group(0)[1:].lower(),\n'
+                   '      ...                   s)\n'
+                   '      ...\n'
+                   '      >>> titlecase("they\'re bill\'s friends.")\n'
+                   '      "They\'re Bill\'s Friends."\n'
+                   '\n'
+                   'str.translate(table)\n'
+                   '\n'
+                   '   Return a copy of the string in which each character has '
+                   'been mapped\n'
+                   '   through the given translation table.  The table must be '
+                   'an object\n'
+                   '   that implements indexing via "__getitem__()", typically '
+                   'a *mapping*\n'
+                   '   or *sequence*.  When indexed by a Unicode ordinal (an '
+                   'integer), the\n'
+                   '   table object can do any of the following: return a '
+                   'Unicode ordinal\n'
+                   '   or a string, to map the character to one or more other '
+                   'characters;\n'
+                   '   return "None", to delete the character from the return '
+                   'string; or\n'
+                   '   raise a "LookupError" exception, to map the character '
+                   'to itself.\n'
+                   '\n'
+                   '   You can use "str.maketrans()" to create a translation '
+                   'map from\n'
+                   '   character-to-character mappings in different formats.\n'
+                   '\n'
+                   '   See also the "codecs" module for a more flexible '
+                   'approach to custom\n'
+                   '   character mappings.\n'
+                   '\n'
+                   'str.upper()\n'
+                   '\n'
+                   '   Return a copy of the string with all the cased '
+                   'characters [4]\n'
+                   '   converted to uppercase.  Note that '
+                   '"str.upper().isupper()" might be\n'
+                   '   "False" if "s" contains uncased characters or if the '
+                   'Unicode\n'
+                   '   category of the resulting character(s) is not "Lu" '
+                   '(Letter,\n'
+                   '   uppercase), but e.g. "Lt" (Letter, titlecase).\n'
+                   '\n'
+                   '   The uppercasing algorithm used is described in section '
+                   '3.13 of the\n'
+                   '   Unicode Standard.\n'
+                   '\n'
+                   'str.zfill(width)\n'
+                   '\n'
+                   '   Return a copy of the string left filled with ASCII '
+                   '"\'0\'" digits to\n'
+                   '   make a string of length *width*. A leading sign prefix\n'
+                   '   ("\'+\'"/"\'-\'") is handled by inserting the padding '
+                   '*after* the sign\n'
+                   '   character rather than before. The original string is '
+                   'returned if\n'
+                   '   *width* is less than or equal to "len(s)".\n'
+                   '\n'
+                   '   For example:\n'
+                   '\n'
+                   '      >>> "42".zfill(5)\n'
+                   "      '00042'\n"
+                   '      >>> "-42".zfill(5)\n'
+                   "      '-0042'\n",
+ 'strings': '\n'
+            'String and Bytes literals\n'
+            '*************************\n'
+            '\n'
+            'String literals are described by the following lexical '
+            'definitions:\n'
+            '\n'
+            '   stringliteral   ::= [stringprefix](shortstring | longstring)\n'
+            '   stringprefix    ::= "r" | "u" | "R" | "U"\n'
+            '   shortstring     ::= "\'" shortstringitem* "\'" | \'"\' '
+            'shortstringitem* \'"\'\n'
+            '   longstring      ::= "\'\'\'" longstringitem* "\'\'\'" | '
+            '\'"""\' longstringitem* \'"""\'\n'
+            '   shortstringitem ::= shortstringchar | stringescapeseq\n'
+            '   longstringitem  ::= longstringchar | stringescapeseq\n'
+            '   shortstringchar ::= <any source character except "\\" or '
+            'newline or the quote>\n'
+            '   longstringchar  ::= <any source character except "\\">\n'
+            '   stringescapeseq ::= "\\" <any source character>\n'
+            '\n'
+            '   bytesliteral   ::= bytesprefix(shortbytes | longbytes)\n'
+            '   bytesprefix    ::= "b" | "B" | "br" | "Br" | "bR" | "BR" | '
+            '"rb" | "rB" | "Rb" | "RB"\n'
+            '   shortbytes     ::= "\'" shortbytesitem* "\'" | \'"\' '
+            'shortbytesitem* \'"\'\n'
+            '   longbytes      ::= "\'\'\'" longbytesitem* "\'\'\'" | \'"""\' '
+            'longbytesitem* \'"""\'\n'
+            '   shortbytesitem ::= shortbyteschar | bytesescapeseq\n'
+            '   longbytesitem  ::= longbyteschar | bytesescapeseq\n'
+            '   shortbyteschar ::= <any ASCII character except "\\" or newline '
+            'or the quote>\n'
+            '   longbyteschar  ::= <any ASCII character except "\\">\n'
+            '   bytesescapeseq ::= "\\" <any ASCII character>\n'
+            '\n'
+            'One syntactic restriction not indicated by these productions is '
+            'that\n'
+            'whitespace is not allowed between the "stringprefix" or '
+            '"bytesprefix"\n'
+            'and the rest of the literal. The source character set is defined '
+            'by\n'
+            'the encoding declaration; it is UTF-8 if no encoding declaration '
+            'is\n'
+            'given in the source file; see section *Encoding declarations*.\n'
+            '\n'
+            'In plain English: Both types of literals can be enclosed in '
+            'matching\n'
+            'single quotes ("\'") or double quotes (""").  They can also be '
+            'enclosed\n'
+            'in matching groups of three single or double quotes (these are\n'
+            'generally referred to as *triple-quoted strings*).  The '
+            'backslash\n'
+            '("\\") character is used to escape characters that otherwise have '
+            'a\n'
+            'special meaning, such as newline, backslash itself, or the quote\n'
+            'character.\n'
+            '\n'
+            'Bytes literals are always prefixed with "\'b\'" or "\'B\'"; they '
+            'produce\n'
+            'an instance of the "bytes" type instead of the "str" type.  They '
+            'may\n'
+            'only contain ASCII characters; bytes with a numeric value of 128 '
+            'or\n'
+            'greater must be expressed with escapes.\n'
+            '\n'
+            'As of Python 3.3 it is possible again to prefix string literals '
+            'with a\n'
+            '"u" prefix to simplify maintenance of dual 2.x and 3.x '
+            'codebases.\n'
+            '\n'
+            'Both string and bytes literals may optionally be prefixed with a\n'
+            'letter "\'r\'" or "\'R\'"; such strings are called *raw strings* '
+            'and treat\n'
+            'backslashes as literal characters.  As a result, in string '
+            'literals,\n'
+            '"\'\\U\'" and "\'\\u\'" escapes in raw strings are not treated '
+            'specially.\n'
+            "Given that Python 2.x's raw unicode literals behave differently "
+            'than\n'
+            'Python 3.x\'s the "\'ur\'" syntax is not supported.\n'
+            '\n'
+            'New in version 3.3: The "\'rb\'" prefix of raw bytes literals has '
+            'been\n'
+            'added as a synonym of "\'br\'".\n'
+            '\n'
+            'New in version 3.3: Support for the unicode legacy literal\n'
+            '("u\'value\'") was reintroduced to simplify the maintenance of '
+            'dual\n'
+            'Python 2.x and 3.x codebases. See **PEP 414** for more '
+            'information.\n'
+            '\n'
+            'In triple-quoted literals, unescaped newlines and quotes are '
+            'allowed\n'
+            '(and are retained), except that three unescaped quotes in a row\n'
+            'terminate the literal.  (A "quote" is the character used to open '
+            'the\n'
+            'literal, i.e. either "\'" or """.)\n'
+            '\n'
+            'Unless an "\'r\'" or "\'R\'" prefix is present, escape sequences '
+            'in string\n'
+            'and bytes literals are interpreted according to rules similar to '
+            'those\n'
+            'used by Standard C.  The recognized escape sequences are:\n'
+            '\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '| Escape Sequence   | Meaning                           | Notes   '
+            '|\n'
+            '+===================+===================================+=========+\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '| "\\ooo"            | Character with octal value *ooo*  | '
+            '(1,3)   |\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '| "\\xhh"            | Character with hex value *hh*     | '
+            '(2,3)   |\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '\n'
+            'Escape sequences only recognized in string literals are:\n'
+            '\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '| Escape Sequence   | Meaning                           | Notes   '
+            '|\n'
+            '+===================+===================================+=========+\n'
+            '| "\\N{name}"        | Character named *name* in the     | '
+            '(4)     |\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '| "\\uxxxx"          | Character with 16-bit hex value   | '
+            '(5)     |\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '| "\\Uxxxxxxxx"      | Character with 32-bit hex value   | '
+            '(6)     |\n'
+            '+-------------------+-----------------------------------+---------+\n'
+            '\n'
+            'Notes:\n'
+            '\n'
+            '1. As in Standard C, up to three octal digits are accepted.\n'
+            '\n'
+            '2. Unlike in Standard C, exactly two hex digits are required.\n'
+            '\n'
+            '3. In a bytes literal, hexadecimal and octal escapes denote the\n'
+            '   byte with the given value. In a string literal, these escapes\n'
+            '   denote a Unicode character with the given value.\n'
+            '\n'
+            '4. Changed in version 3.3: Support for name aliases [1] has been\n'
+            '   added.\n'
+            '\n'
+            '5. Exactly four hex digits are required.\n'
+            '\n'
+            '6. Any Unicode character can be encoded this way.  Exactly eight\n'
+            '   hex digits are required.\n'
+            '\n'
+            'Unlike Standard C, all unrecognized escape sequences are left in '
+            'the\n'
+            'string unchanged, i.e., *the backslash is left in the result*.  '
+            '(This\n'
+            'behavior is useful when debugging: if an escape sequence is '
+            'mistyped,\n'
+            'the resulting output is more easily recognized as broken.)  It is '
+            'also\n'
+            'important to note that the escape sequences only recognized in '
+            'string\n'
+            'literals fall into the category of unrecognized escapes for '
+            'bytes\n'
+            'literals.\n'
+            '\n'
+            'Even in a raw literal, quotes can be escaped with a backslash, '
+            'but the\n'
+            'backslash remains in the result; for example, "r"\\""" is a '
+            'valid\n'
+            'string literal consisting of two characters: a backslash and a '
+            'double\n'
+            'quote; "r"\\"" is not a valid string literal (even a raw string '
+            'cannot\n'
+            'end in an odd number of backslashes).  Specifically, *a raw '
+            'literal\n'
+            'cannot end in a single backslash* (since the backslash would '
+            'escape\n'
+            'the following quote character).  Note also that a single '
+            'backslash\n'
+            'followed by a newline is interpreted as those two characters as '
+            'part\n'
+            'of the literal, *not* as a line continuation.\n',
+ 'subscriptions': '\n'
+                  'Subscriptions\n'
+                  '*************\n'
+                  '\n'
+                  'A subscription selects an item of a sequence (string, tuple '
+                  'or list)\n'
+                  'or mapping (dictionary) object:\n'
+                  '\n'
+                  '   subscription ::= primary "[" expression_list "]"\n'
+                  '\n'
+                  'The primary must evaluate to an object that supports '
+                  'subscription\n'
+                  '(lists or dictionaries for example).  User-defined objects '
+                  'can support\n'
+                  'subscription by defining a "__getitem__()" method.\n'
+                  '\n'
+                  'For built-in objects, there are two types of objects that '
+                  'support\n'
+                  'subscription:\n'
+                  '\n'
+                  'If the primary is a mapping, the expression list must '
+                  'evaluate to an\n'
+                  'object whose value is one of the keys of the mapping, and '
+                  'the\n'
+                  'subscription selects the value in the mapping that '
+                  'corresponds to that\n'
+                  'key.  (The expression list is a tuple except if it has '
+                  'exactly one\n'
+                  'item.)\n'
+                  '\n'
+                  'If the primary is a sequence, the expression (list) must '
+                  'evaluate to\n'
+                  'an integer or a slice (as discussed in the following '
+                  'section).\n'
+                  '\n'
+                  'The formal syntax makes no special provision for negative '
+                  'indices in\n'
+                  'sequences; however, built-in sequences all provide a '
+                  '"__getitem__()"\n'
+                  'method that interprets negative indices by adding the '
+                  'length of the\n'
+                  'sequence to the index (so that "x[-1]" selects the last '
+                  'item of "x").\n'
+                  'The resulting value must be a nonnegative integer less than '
+                  'the number\n'
+                  'of items in the sequence, and the subscription selects the '
+                  'item whose\n'
+                  'index is that value (counting from zero). Since the support '
+                  'for\n'
+                  "negative indices and slicing occurs in the object's "
+                  '"__getitem__()"\n'
+                  'method, subclasses overriding this method will need to '
+                  'explicitly add\n'
+                  'that support.\n'
+                  '\n'
+                  "A string's items are characters.  A character is not a "
+                  'separate data\n'
+                  'type but a string of exactly one character.\n',
+ 'truth': '\n'
+          'Truth Value Testing\n'
+          '*******************\n'
+          '\n'
+          'Any object can be tested for truth value, for use in an "if" or\n'
+          '"while" condition or as operand of the Boolean operations below. '
+          'The\n'
+          'following values are considered false:\n'
+          '\n'
+          '* "None"\n'
+          '\n'
+          '* "False"\n'
+          '\n'
+          '* zero of any numeric type, for example, "0", "0.0", "0j".\n'
+          '\n'
+          '* any empty sequence, for example, "\'\'", "()", "[]".\n'
+          '\n'
+          '* any empty mapping, for example, "{}".\n'
+          '\n'
+          '* instances of user-defined classes, if the class defines a\n'
+          '  "__bool__()" or "__len__()" method, when that method returns the\n'
+          '  integer zero or "bool" value "False". [1]\n'
+          '\n'
+          'All other values are considered true --- so objects of many types '
+          'are\n'
+          'always true.\n'
+          '\n'
+          'Operations and built-in functions that have a Boolean result '
+          'always\n'
+          'return "0" or "False" for false and "1" or "True" for true, unless\n'
+          'otherwise stated. (Important exception: the Boolean operations '
+          '"or"\n'
+          'and "and" always return one of their operands.)\n',
+ 'try': '\n'
+        'The "try" statement\n'
+        '*******************\n'
+        '\n'
+        'The "try" statement specifies exception handlers and/or cleanup code\n'
+        'for a group of statements:\n'
+        '\n'
+        '   try_stmt  ::= try1_stmt | try2_stmt\n'
+        '   try1_stmt ::= "try" ":" suite\n'
+        '                 ("except" [expression ["as" identifier]] ":" '
+        'suite)+\n'
+        '                 ["else" ":" suite]\n'
+        '                 ["finally" ":" suite]\n'
+        '   try2_stmt ::= "try" ":" suite\n'
+        '                 "finally" ":" suite\n'
+        '\n'
+        'The "except" clause(s) specify one or more exception handlers. When '
+        'no\n'
+        'exception occurs in the "try" clause, no exception handler is\n'
+        'executed. When an exception occurs in the "try" suite, a search for '
+        'an\n'
+        'exception handler is started.  This search inspects the except '
+        'clauses\n'
+        'in turn until one is found that matches the exception.  An '
+        'expression-\n'
+        'less except clause, if present, must be last; it matches any\n'
+        'exception.  For an except clause with an expression, that expression\n'
+        'is evaluated, and the clause matches the exception if the resulting\n'
+        'object is "compatible" with the exception.  An object is compatible\n'
+        'with an exception if it is the class or a base class of the '
+        'exception\n'
+        'object or a tuple containing an item compatible with the exception.\n'
+        '\n'
+        'If no except clause matches the exception, the search for an '
+        'exception\n'
+        'handler continues in the surrounding code and on the invocation '
+        'stack.\n'
+        '[1]\n'
+        '\n'
+        'If the evaluation of an expression in the header of an except clause\n'
+        'raises an exception, the original search for a handler is canceled '
+        'and\n'
+        'a search starts for the new exception in the surrounding code and on\n'
+        'the call stack (it is treated as if the entire "try" statement '
+        'raised\n'
+        'the exception).\n'
+        '\n'
+        'When a matching except clause is found, the exception is assigned to\n'
+        'the target specified after the "as" keyword in that except clause, '
+        'if\n'
+        "present, and the except clause's suite is executed.  All except\n"
+        'clauses must have an executable block.  When the end of this block '
+        'is\n'
+        'reached, execution continues normally after the entire try '
+        'statement.\n'
+        '(This means that if two nested handlers exist for the same '
+        'exception,\n'
+        'and the exception occurs in the try clause of the inner handler, the\n'
+        'outer handler will not handle the exception.)\n'
+        '\n'
+        'When an exception has been assigned using "as target", it is cleared\n'
+        'at the end of the except clause.  This is as if\n'
+        '\n'
+        '   except E as N:\n'
+        '       foo\n'
+        '\n'
+        'was translated to\n'
+        '\n'
+        '   except E as N:\n'
+        '       try:\n'
+        '           foo\n'
+        '       finally:\n'
+        '           del N\n'
+        '\n'
+        'This means the exception must be assigned to a different name to be\n'
+        'able to refer to it after the except clause.  Exceptions are cleared\n'
+        'because with the traceback attached to them, they form a reference\n'
+        'cycle with the stack frame, keeping all locals in that frame alive\n'
+        'until the next garbage collection occurs.\n'
+        '\n'
+        "Before an except clause's suite is executed, details about the\n"
+        'exception are stored in the "sys" module and can be accessed via\n'
+        '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of '
+        'the\n'
+        'exception class, the exception instance and a traceback object (see\n'
+        'section *The standard type hierarchy*) identifying the point in the\n'
+        'program where the exception occurred.  "sys.exc_info()" values are\n'
+        'restored to their previous values (before the call) when returning\n'
+        'from a function that handled an exception.\n'
+        '\n'
+        'The optional "else" clause is executed if and when control flows off\n'
+        'the end of the "try" clause. [2] Exceptions in the "else" clause are\n'
+        'not handled by the preceding "except" clauses.\n'
+        '\n'
+        'If "finally" is present, it specifies a \'cleanup\' handler.  The '
+        '"try"\n'
+        'clause is executed, including any "except" and "else" clauses.  If '
+        'an\n'
+        'exception occurs in any of the clauses and is not handled, the\n'
+        'exception is temporarily saved. The "finally" clause is executed.  '
+        'If\n'
+        'there is a saved exception it is re-raised at the end of the '
+        '"finally"\n'
+        'clause.  If the "finally" clause raises another exception, the saved\n'
+        'exception is set as the context of the new exception. If the '
+        '"finally"\n'
+        'clause executes a "return" or "break" statement, the saved exception\n'
+        'is discarded:\n'
+        '\n'
+        '   >>> def f():\n'
+        '   ...     try:\n'
+        '   ...         1/0\n'
+        '   ...     finally:\n'
+        '   ...         return 42\n'
+        '   ...\n'
+        '   >>> f()\n'
+        '   42\n'
+        '\n'
+        'The exception information is not available to the program during\n'
+        'execution of the "finally" clause.\n'
+        '\n'
+        'When a "return", "break" or "continue" statement is executed in the\n'
+        '"try" suite of a "try"..."finally" statement, the "finally" clause '
+        'is\n'
+        'also executed \'on the way out.\' A "continue" statement is illegal '
+        'in\n'
+        'the "finally" clause. (The reason is a problem with the current\n'
+        'implementation --- this restriction may be lifted in the future).\n'
+        '\n'
+        'The return value of a function is determined by the last "return"\n'
+        'statement executed.  Since the "finally" clause always executes, a\n'
+        '"return" statement executed in the "finally" clause will always be '
+        'the\n'
+        'last one executed:\n'
+        '\n'
+        '   >>> def foo():\n'
+        '   ...     try:\n'
+        "   ...         return 'try'\n"
+        '   ...     finally:\n'
+        "   ...         return 'finally'\n"
+        '   ...\n'
+        '   >>> foo()\n'
+        "   'finally'\n"
+        '\n'
+        'Additional information on exceptions can be found in section\n'
+        '*Exceptions*, and information on using the "raise" statement to\n'
+        'generate exceptions may be found in section *The raise statement*.\n',
+ 'types': '\n'
+          'The standard type hierarchy\n'
+          '***************************\n'
+          '\n'
+          'Below is a list of the types that are built into Python.  '
+          'Extension\n'
+          'modules (written in C, Java, or other languages, depending on the\n'
+          'implementation) can define additional types.  Future versions of\n'
+          'Python may add types to the type hierarchy (e.g., rational '
+          'numbers,\n'
+          'efficiently stored arrays of integers, etc.), although such '
+          'additions\n'
+          'will often be provided via the standard library instead.\n'
+          '\n'
+          'Some of the type descriptions below contain a paragraph listing\n'
+          "'special attributes.'  These are attributes that provide access to "
+          'the\n'
+          'implementation and are not intended for general use.  Their '
+          'definition\n'
+          'may change in the future.\n'
+          '\n'
+          'None\n'
+          '   This type has a single value.  There is a single object with '
+          'this\n'
+          '   value. This object is accessed through the built-in name "None". '
+          'It\n'
+          '   is used to signify the absence of a value in many situations, '
+          'e.g.,\n'
+          "   it is returned from functions that don't explicitly return\n"
+          '   anything. Its truth value is false.\n'
+          '\n'
+          'NotImplemented\n'
+          '   This type has a single value.  There is a single object with '
+          'this\n'
+          '   value. This object is accessed through the built-in name\n'
+          '   "NotImplemented". Numeric methods and rich comparison methods\n'
+          '   should return this value if they do not implement the operation '
+          'for\n'
+          '   the operands provided.  (The interpreter will then try the\n'
+          '   reflected operation, or some other fallback, depending on the\n'
+          '   operator.)  Its truth value is true.\n'
+          '\n'
+          '   See *Implementing the arithmetic operations* for more details.\n'
+          '\n'
+          'Ellipsis\n'
+          '   This type has a single value.  There is a single object with '
+          'this\n'
+          '   value. This object is accessed through the literal "..." or the\n'
+          '   built-in name "Ellipsis".  Its truth value is true.\n'
+          '\n'
+          '"numbers.Number"\n'
+          '   These are created by numeric literals and returned as results '
+          'by\n'
+          '   arithmetic operators and arithmetic built-in functions.  '
+          'Numeric\n'
+          '   objects are immutable; once created their value never changes.\n'
+          '   Python numbers are of course strongly related to mathematical\n'
+          '   numbers, but subject to the limitations of numerical '
+          'representation\n'
+          '   in computers.\n'
+          '\n'
+          '   Python distinguishes between integers, floating point numbers, '
+          'and\n'
+          '   complex numbers:\n'
+          '\n'
+          '   "numbers.Integral"\n'
+          '      These represent elements from the mathematical set of '
+          'integers\n'
+          '      (positive and negative).\n'
+          '\n'
+          '      There are two types of integers:\n'
+          '\n'
+          '      Integers ("int")\n'
+          '\n'
+          '         These represent numbers in an unlimited range, subject to\n'
+          '         available (virtual) memory only.  For the purpose of '
+          'shift\n'
+          '         and mask operations, a binary representation is assumed, '
+          'and\n'
+          "         negative numbers are represented in a variant of 2's\n"
+          '         complement which gives the illusion of an infinite string '
+          'of\n'
+          '         sign bits extending to the left.\n'
+          '\n'
+          '      Booleans ("bool")\n'
+          '         These represent the truth values False and True.  The two\n'
+          '         objects representing the values "False" and "True" are '
+          'the\n'
+          '         only Boolean objects. The Boolean type is a subtype of '
+          'the\n'
+          '         integer type, and Boolean values behave like the values 0 '
+          'and\n'
+          '         1, respectively, in almost all contexts, the exception '
+          'being\n'
+          '         that when converted to a string, the strings ""False"" or\n'
+          '         ""True"" are returned, respectively.\n'
+          '\n'
+          '      The rules for integer representation are intended to give '
+          'the\n'
+          '      most meaningful interpretation of shift and mask operations\n'
+          '      involving negative integers.\n'
+          '\n'
+          '   "numbers.Real" ("float")\n'
+          '      These represent machine-level double precision floating '
+          'point\n'
+          '      numbers. You are at the mercy of the underlying machine\n'
+          '      architecture (and C or Java implementation) for the accepted\n'
+          '      range and handling of overflow. Python does not support '
+          'single-\n'
+          '      precision floating point numbers; the savings in processor '
+          'and\n'
+          '      memory usage that are usually the reason for using these are\n'
+          '      dwarfed by the overhead of using objects in Python, so there '
+          'is\n'
+          '      no reason to complicate the language with two kinds of '
+          'floating\n'
+          '      point numbers.\n'
+          '\n'
+          '   "numbers.Complex" ("complex")\n'
+          '      These represent complex numbers as a pair of machine-level\n'
+          '      double precision floating point numbers.  The same caveats '
+          'apply\n'
+          '      as for floating point numbers. The real and imaginary parts '
+          'of a\n'
+          '      complex number "z" can be retrieved through the read-only\n'
+          '      attributes "z.real" and "z.imag".\n'
+          '\n'
+          'Sequences\n'
+          '   These represent finite ordered sets indexed by non-negative\n'
+          '   numbers. The built-in function "len()" returns the number of '
+          'items\n'
+          '   of a sequence. When the length of a sequence is *n*, the index '
+          'set\n'
+          '   contains the numbers 0, 1, ..., *n*-1.  Item *i* of sequence *a* '
+          'is\n'
+          '   selected by "a[i]".\n'
+          '\n'
+          '   Sequences also support slicing: "a[i:j]" selects all items with\n'
+          '   index *k* such that *i* "<=" *k* "<" *j*.  When used as an\n'
+          '   expression, a slice is a sequence of the same type.  This '
+          'implies\n'
+          '   that the index set is renumbered so that it starts at 0.\n'
+          '\n'
+          '   Some sequences also support "extended slicing" with a third '
+          '"step"\n'
+          '   parameter: "a[i:j:k]" selects all items of *a* with index *x* '
+          'where\n'
+          '   "x = i + n*k", *n* ">=" "0" and *i* "<=" *x* "<" *j*.\n'
+          '\n'
+          '   Sequences are distinguished according to their mutability:\n'
+          '\n'
+          '   Immutable sequences\n'
+          '      An object of an immutable sequence type cannot change once it '
+          'is\n'
+          '      created.  (If the object contains references to other '
+          'objects,\n'
+          '      these other objects may be mutable and may be changed; '
+          'however,\n'
+          '      the collection of objects directly referenced by an '
+          'immutable\n'
+          '      object cannot change.)\n'
+          '\n'
+          '      The following types are immutable sequences:\n'
+          '\n'
+          '      Strings\n'
+          '         A string is a sequence of values that represent Unicode '
+          'code\n'
+          '         points. All the code points in the range "U+0000 - '
+          'U+10FFFF"\n'
+          "         can be represented in a string.  Python doesn't have a "
+          '"char"\n'
+          '         type; instead, every code point in the string is '
+          'represented\n'
+          '         as a string object with length "1".  The built-in '
+          'function\n'
+          '         "ord()" converts a code point from its string form to an\n'
+          '         integer in the range "0 - 10FFFF"; "chr()" converts an\n'
+          '         integer in the range "0 - 10FFFF" to the corresponding '
+          'length\n'
+          '         "1" string object. "str.encode()" can be used to convert '
+          'a\n'
+          '         "str" to "bytes" using the given text encoding, and\n'
+          '         "bytes.decode()" can be used to achieve the opposite.\n'
+          '\n'
+          '      Tuples\n'
+          '         The items of a tuple are arbitrary Python objects. Tuples '
+          'of\n'
+          '         two or more items are formed by comma-separated lists of\n'
+          "         expressions.  A tuple of one item (a 'singleton') can be\n"
+          '         formed by affixing a comma to an expression (an expression '
+          'by\n'
+          '         itself does not create a tuple, since parentheses must be\n'
+          '         usable for grouping of expressions).  An empty tuple can '
+          'be\n'
+          '         formed by an empty pair of parentheses.\n'
+          '\n'
+          '      Bytes\n'
+          '         A bytes object is an immutable array.  The items are '
+          '8-bit\n'
+          '         bytes, represented by integers in the range 0 <= x < 256.\n'
+          '         Bytes literals (like "b\'abc\'") and the built-in '
+          'function\n'
+          '         "bytes()" can be used to construct bytes objects.  Also,\n'
+          '         bytes objects can be decoded to strings via the '
+          '"decode()"\n'
+          '         method.\n'
+          '\n'
+          '   Mutable sequences\n'
+          '      Mutable sequences can be changed after they are created.  '
+          'The\n'
+          '      subscription and slicing notations can be used as the target '
+          'of\n'
+          '      assignment and "del" (delete) statements.\n'
+          '\n'
+          '      There are currently two intrinsic mutable sequence types:\n'
+          '\n'
+          '      Lists\n'
+          '         The items of a list are arbitrary Python objects.  Lists '
+          'are\n'
+          '         formed by placing a comma-separated list of expressions '
+          'in\n'
+          '         square brackets. (Note that there are no special cases '
+          'needed\n'
+          '         to form lists of length 0 or 1.)\n'
+          '\n'
+          '      Byte Arrays\n'
+          '         A bytearray object is a mutable array. They are created '
+          'by\n'
+          '         the built-in "bytearray()" constructor.  Aside from being\n'
+          '         mutable (and hence unhashable), byte arrays otherwise '
+          'provide\n'
+          '         the same interface and functionality as immutable bytes\n'
+          '         objects.\n'
+          '\n'
+          '      The extension module "array" provides an additional example '
+          'of a\n'
+          '      mutable sequence type, as does the "collections" module.\n'
+          '\n'
+          'Set types\n'
+          '   These represent unordered, finite sets of unique, immutable\n'
+          '   objects. As such, they cannot be indexed by any subscript. '
+          'However,\n'
+          '   they can be iterated over, and the built-in function "len()"\n'
+          '   returns the number of items in a set. Common uses for sets are '
+          'fast\n'
+          '   membership testing, removing duplicates from a sequence, and\n'
+          '   computing mathematical operations such as intersection, union,\n'
+          '   difference, and symmetric difference.\n'
+          '\n'
+          '   For set elements, the same immutability rules apply as for\n'
+          '   dictionary keys. Note that numeric types obey the normal rules '
+          'for\n'
+          '   numeric comparison: if two numbers compare equal (e.g., "1" and\n'
+          '   "1.0"), only one of them can be contained in a set.\n'
+          '\n'
+          '   There are currently two intrinsic set types:\n'
+          '\n'
+          '   Sets\n'
+          '      These represent a mutable set. They are created by the '
+          'built-in\n'
+          '      "set()" constructor and can be modified afterwards by '
+          'several\n'
+          '      methods, such as "add()".\n'
+          '\n'
+          '   Frozen sets\n'
+          '      These represent an immutable set.  They are created by the\n'
+          '      built-in "frozenset()" constructor.  As a frozenset is '
+          'immutable\n'
+          '      and *hashable*, it can be used again as an element of '
+          'another\n'
+          '      set, or as a dictionary key.\n'
+          '\n'
+          'Mappings\n'
+          '   These represent finite sets of objects indexed by arbitrary '
+          'index\n'
+          '   sets. The subscript notation "a[k]" selects the item indexed by '
+          '"k"\n'
+          '   from the mapping "a"; this can be used in expressions and as '
+          'the\n'
+          '   target of assignments or "del" statements. The built-in '
+          'function\n'
+          '   "len()" returns the number of items in a mapping.\n'
+          '\n'
+          '   There is currently a single intrinsic mapping type:\n'
+          '\n'
+          '   Dictionaries\n'
+          '      These represent finite sets of objects indexed by nearly\n'
+          '      arbitrary values.  The only types of values not acceptable '
+          'as\n'
+          '      keys are values containing lists or dictionaries or other\n'
+          '      mutable types that are compared by value rather than by '
+          'object\n'
+          '      identity, the reason being that the efficient implementation '
+          'of\n'
+          "      dictionaries requires a key's hash value to remain constant.\n"
+          '      Numeric types used for keys obey the normal rules for '
+          'numeric\n'
+          '      comparison: if two numbers compare equal (e.g., "1" and '
+          '"1.0")\n'
+          '      then they can be used interchangeably to index the same\n'
+          '      dictionary entry.\n'
+          '\n'
+          '      Dictionaries are mutable; they can be created by the "{...}"\n'
+          '      notation (see section *Dictionary displays*).\n'
+          '\n'
+          '      The extension modules "dbm.ndbm" and "dbm.gnu" provide\n'
+          '      additional examples of mapping types, as does the '
+          '"collections"\n'
+          '      module.\n'
+          '\n'
+          'Callable types\n'
+          '   These are the types to which the function call operation (see\n'
+          '   section *Calls*) can be applied:\n'
+          '\n'
+          '   User-defined functions\n'
+          '      A user-defined function object is created by a function\n'
+          '      definition (see section *Function definitions*).  It should '
+          'be\n'
+          '      called with an argument list containing the same number of '
+          'items\n'
+          "      as the function's formal parameter list.\n"
+          '\n'
+          '      Special attributes:\n'
+          '\n'
+          '      '
+          '+---------------------------+---------------------------------+-------------+\n'
+          '      '
+          '+===========================+=================================+=============+\n'
+          '      | "__doc__"                 | The function\'s '
+          'documentation    | Writable    |\n'
+          '      '
+          '+---------------------------+---------------------------------+-------------+\n'
+          '      | "__name__"                | The function\'s '
+          'name             | Writable    |\n'
+          '      '
+          '+---------------------------+---------------------------------+-------------+\n'
+          '      | "__qualname__"            | The function\'s *qualified '
+          'name* | Writable    |\n'
+          '      '
+          '+---------------------------+---------------------------------+-------------+\n'
+          '      | "__module__"              | The name of the module the      '
+          '| Writable    |\n'
+          '      '
+          '+---------------------------+---------------------------------+-------------+\n'
+          '      | "__defaults__"            | A tuple containing default      '
+          '| Writable    |\n'
+          '      '
+          '+---------------------------+---------------------------------+-------------+\n'
+          '      | "__code__"                | The code object representing    '
+          '| Writable    |\n'
+          '      '
+          '+---------------------------+---------------------------------+-------------+\n'
+          '      | "__globals__"             | A reference to the dictionary   '
+          '| Read-only   |\n'
+          '      '
+          '+---------------------------+---------------------------------+-------------+\n'
+          '      | "__dict__"                | The namespace supporting        '
+          '| Writable    |\n'
+          '      '
+          '+---------------------------+---------------------------------+-------------+\n'
+          '      | "__closure__"             | "None" or a tuple of cells that '
+          '| Read-only   |\n'
+          '      '
+          '+---------------------------+---------------------------------+-------------+\n'
+          '      | "__annotations__"         | A dict containing annotations   '
+          '| Writable    |\n'
+          '      '
+          '+---------------------------+---------------------------------+-------------+\n'
+          '      | "__kwdefaults__"          | A dict containing defaults for  '
+          '| Writable    |\n'
+          '      '
+          '+---------------------------+---------------------------------+-------------+\n'
+          '\n'
+          '      Most of the attributes labelled "Writable" check the type of '
+          'the\n'
+          '      assigned value.\n'
+          '\n'
+          '      Function objects also support getting and setting arbitrary\n'
+          '      attributes, which can be used, for example, to attach '
+          'metadata\n'
+          '      to functions.  Regular attribute dot-notation is used to get '
+          'and\n'
+          '      set such attributes. *Note that the current implementation '
+          'only\n'
+          '      supports function attributes on user-defined functions. '
+          'Function\n'
+          '      attributes on built-in functions may be supported in the\n'
+          '      future.*\n'
+          '\n'
+          "      Additional information about a function's definition can be\n"
+          '      retrieved from its code object; see the description of '
+          'internal\n'
+          '      types below.\n'
+          '\n'
+          '   Instance methods\n'
+          '      An instance method object combines a class, a class instance '
+          'and\n'
+          '      any callable object (normally a user-defined function).\n'
+          '\n'
+          '      Special read-only attributes: "__self__" is the class '
+          'instance\n'
+          '      object, "__func__" is the function object; "__doc__" is the\n'
+          '      method\'s documentation (same as "__func__.__doc__"); '
+          '"__name__"\n'
+          '      is the method name (same as "__func__.__name__"); '
+          '"__module__"\n'
+          '      is the name of the module the method was defined in, or '
+          '"None"\n'
+          '      if unavailable.\n'
+          '\n'
+          '      Methods also support accessing (but not setting) the '
+          'arbitrary\n'
+          '      function attributes on the underlying function object.\n'
+          '\n'
+          '      User-defined method objects may be created when getting an\n'
+          '      attribute of a class (perhaps via an instance of that class), '
+          'if\n'
+          '      that attribute is a user-defined function object or a class\n'
+          '      method object.\n'
+          '\n'
+          '      When an instance method object is created by retrieving a '
+          'user-\n'
+          '      defined function object from a class via one of its '
+          'instances,\n'
+          '      its "__self__" attribute is the instance, and the method '
+          'object\n'
+          '      is said to be bound.  The new method\'s "__func__" attribute '
+          'is\n'
+          '      the original function object.\n'
+          '\n'
+          '      When a user-defined method object is created by retrieving\n'
+          '      another method object from a class or instance, the behaviour '
+          'is\n'
+          '      the same as for a function object, except that the '
+          '"__func__"\n'
+          '      attribute of the new instance is not the original method '
+          'object\n'
+          '      but its "__func__" attribute.\n'
+          '\n'
+          '      When an instance method object is created by retrieving a '
+          'class\n'
+          '      method object from a class or instance, its "__self__" '
+          'attribute\n'
+          '      is the class itself, and its "__func__" attribute is the\n'
+          '      function object underlying the class method.\n'
+          '\n'
+          '      When an instance method object is called, the underlying\n'
+          '      function ("__func__") is called, inserting the class '
+          'instance\n'
+          '      ("__self__") in front of the argument list.  For instance, '
+          'when\n'
+          '      "C" is a class which contains a definition for a function '
+          '"f()",\n'
+          '      and "x" is an instance of "C", calling "x.f(1)" is equivalent '
+          'to\n'
+          '      calling "C.f(x, 1)".\n'
+          '\n'
+          '      When an instance method object is derived from a class '
+          'method\n'
+          '      object, the "class instance" stored in "__self__" will '
+          'actually\n'
+          '      be the class itself, so that calling either "x.f(1)" or '
+          '"C.f(1)"\n'
+          '      is equivalent to calling "f(C,1)" where "f" is the '
+          'underlying\n'
+          '      function.\n'
+          '\n'
+          '      Note that the transformation from function object to '
+          'instance\n'
+          '      method object happens each time the attribute is retrieved '
+          'from\n'
+          '      the instance.  In some cases, a fruitful optimization is to\n'
+          '      assign the attribute to a local variable and call that local\n'
+          '      variable. Also notice that this transformation only happens '
+          'for\n'
+          '      user-defined functions; other callable objects (and all non-\n'
+          '      callable objects) are retrieved without transformation.  It '
+          'is\n'
+          '      also important to note that user-defined functions which are\n'
+          '      attributes of a class instance are not converted to bound\n'
+          '      methods; this *only* happens when the function is an '
+          'attribute\n'
+          '      of the class.\n'
+          '\n'
+          '   Generator functions\n'
+          '      A function or method which uses the "yield" statement (see\n'
+          '      section *The yield statement*) is called a *generator '
+          'function*.\n'
+          '      Such a function, when called, always returns an iterator '
+          'object\n'
+          '      which can be used to execute the body of the function:  '
+          'calling\n'
+          '      the iterator\'s "iterator.__next__()" method will cause the\n'
+          '      function to execute until it provides a value using the '
+          '"yield"\n'
+          '      statement.  When the function executes a "return" statement '
+          'or\n'
+          '      falls off the end, a "StopIteration" exception is raised and '
+          'the\n'
+          '      iterator will have reached the end of the set of values to '
+          'be\n'
+          '      returned.\n'
+          '\n'
+          '   Coroutine functions\n'
+          '      A function or method which is defined using "async def" is\n'
+          '      called a *coroutine function*.  Such a function, when '
+          'called,\n'
+          '      returns a *coroutine* object.  It may contain "await"\n'
+          '      expressions, as well as "async with" and "async for" '
+          'statements.\n'
+          '      See also the *Coroutine Objects* section.\n'
+          '\n'
+          '   Built-in functions\n'
+          '      A built-in function object is a wrapper around a C function.\n'
+          '      Examples of built-in functions are "len()" and "math.sin()"\n'
+          '      ("math" is a standard built-in module). The number and type '
+          'of\n'
+          '      the arguments are determined by the C function. Special '
+          'read-\n'
+          '      only attributes: "__doc__" is the function\'s documentation\n'
+          '      string, or "None" if unavailable; "__name__" is the '
+          "function's\n"
+          '      name; "__self__" is set to "None" (but see the next item);\n'
+          '      "__module__" is the name of the module the function was '
+          'defined\n'
+          '      in or "None" if unavailable.\n'
+          '\n'
+          '   Built-in methods\n'
+          '      This is really a different disguise of a built-in function, '
+          'this\n'
+          '      time containing an object passed to the C function as an\n'
+          '      implicit extra argument.  An example of a built-in method is\n'
+          '      "alist.append()", assuming *alist* is a list object. In this\n'
+          '      case, the special read-only attribute "__self__" is set to '
+          'the\n'
+          '      object denoted by *alist*.\n'
+          '\n'
+          '   Classes\n'
+          '      Classes are callable.  These objects normally act as '
+          'factories\n'
+          '      for new instances of themselves, but variations are possible '
+          'for\n'
+          '      class types that override "__new__()".  The arguments of the\n'
+          '      call are passed to "__new__()" and, in the typical case, to\n'
+          '      "__init__()" to initialize the new instance.\n'
+          '\n'
+          '   Class Instances\n'
+          '      Instances of arbitrary classes can be made callable by '
+          'defining\n'
+          '      a "__call__()" method in their class.\n'
+          '\n'
+          'Modules\n'
+          '   Modules are a basic organizational unit of Python code, and are\n'
+          '   created by the *import system* as invoked either by the '
+          '"import"\n'
+          '   statement (see "import"), or by calling functions such as\n'
+          '   "importlib.import_module()" and built-in "__import__()".  A '
+          'module\n'
+          '   object has a namespace implemented by a dictionary object (this '
+          'is\n'
+          '   the dictionary referenced by the "__globals__" attribute of\n'
+          '   functions defined in the module).  Attribute references are\n'
+          '   translated to lookups in this dictionary, e.g., "m.x" is '
+          'equivalent\n'
+          '   to "m.__dict__["x"]". A module object does not contain the code\n'
+          "   object used to initialize the module (since it isn't needed "
+          'once\n'
+          '   the initialization is done).\n'
+          '\n'
+          "   Attribute assignment updates the module's namespace dictionary,\n"
+          '   e.g., "m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n'
+          '\n'
+          '   Special read-only attribute: "__dict__" is the module\'s '
+          'namespace\n'
+          '   as a dictionary object.\n'
+          '\n'
+          '   **CPython implementation detail:** Because of the way CPython\n'
+          '   clears module dictionaries, the module dictionary will be '
+          'cleared\n'
+          '   when the module falls out of scope even if the dictionary still '
+          'has\n'
+          '   live references.  To avoid this, copy the dictionary or keep '
+          'the\n'
+          '   module around while using its dictionary directly.\n'
+          '\n'
+          '   Predefined (writable) attributes: "__name__" is the module\'s '
+          'name;\n'
+          '   "__doc__" is the module\'s documentation string, or "None" if\n'
+          '   unavailable; "__file__" is the pathname of the file from which '
+          'the\n'
+          '   module was loaded, if it was loaded from a file. The "__file__"\n'
+          '   attribute may be missing for certain types of modules, such as '
+          'C\n'
+          '   modules that are statically linked into the interpreter; for\n'
+          '   extension modules loaded dynamically from a shared library, it '
+          'is\n'
+          '   the pathname of the shared library file.\n'
+          '\n'
+          'Custom classes\n'
+          '   Custom class types are typically created by class definitions '
+          '(see\n'
+          '   section *Class definitions*).  A class has a namespace '
+          'implemented\n'
+          '   by a dictionary object. Class attribute references are '
+          'translated\n'
+          '   to lookups in this dictionary, e.g., "C.x" is translated to\n'
+          '   "C.__dict__["x"]" (although there are a number of hooks which '
+          'allow\n'
+          '   for other means of locating attributes). When the attribute name '
+          'is\n'
+          '   not found there, the attribute search continues in the base\n'
+          '   classes. This search of the base classes uses the C3 method\n'
+          '   resolution order which behaves correctly even in the presence '
+          'of\n'
+          "   'diamond' inheritance structures where there are multiple\n"
+          '   inheritance paths leading back to a common ancestor. Additional\n'
+          '   details on the C3 MRO used by Python can be found in the\n'
+          '   documentation accompanying the 2.3 release at\n'
+          '   https://www.python.org/download/releases/2.3/mro/.\n'
+          '\n'
+          '   When a class attribute reference (for class "C", say) would '
+          'yield a\n'
+          '   class method object, it is transformed into an instance method\n'
+          '   object whose "__self__" attributes is "C".  When it would yield '
+          'a\n'
+          '   static method object, it is transformed into the object wrapped '
+          'by\n'
+          '   the static method object. See section *Implementing '
+          'Descriptors*\n'
+          '   for another way in which attributes retrieved from a class may\n'
+          '   differ from those actually contained in its "__dict__".\n'
+          '\n'
+          "   Class attribute assignments update the class's dictionary, "
+          'never\n'
+          '   the dictionary of a base class.\n'
+          '\n'
+          '   A class object can be called (see above) to yield a class '
+          'instance\n'
+          '   (see below).\n'
+          '\n'
+          '   Special attributes: "__name__" is the class name; "__module__" '
+          'is\n'
+          '   the module name in which the class was defined; "__dict__" is '
+          'the\n'
+          '   dictionary containing the class\'s namespace; "__bases__" is a '
+          'tuple\n'
+          '   containing the base classes, in the order of their occurrence '
+          'in\n'
+          '   the base class list; "__doc__" is the class\'s documentation '
+          'string,\n'
+          '   or "None" if undefined.\n'
+          '\n'
+          'Class instances\n'
+          '   A class instance is created by calling a class object (see '
+          'above).\n'
+          '   A class instance has a namespace implemented as a dictionary '
+          'which\n'
+          '   is the first place in which attribute references are searched.\n'
+          "   When an attribute is not found there, and the instance's class "
+          'has\n'
+          '   an attribute by that name, the search continues with the class\n'
+          '   attributes.  If a class attribute is found that is a '
+          'user-defined\n'
+          '   function object, it is transformed into an instance method '
+          'object\n'
+          '   whose "__self__" attribute is the instance.  Static method and\n'
+          '   class method objects are also transformed; see above under\n'
+          '   "Classes".  See section *Implementing Descriptors* for another '
+          'way\n'
+          '   in which attributes of a class retrieved via its instances may\n'
+          "   differ from the objects actually stored in the class's "
+          '"__dict__".\n'
+          "   If no class attribute is found, and the object's class has a\n"
+          '   "__getattr__()" method, that is called to satisfy the lookup.\n'
+          '\n'
+          "   Attribute assignments and deletions update the instance's\n"
+          "   dictionary, never a class's dictionary.  If the class has a\n"
+          '   "__setattr__()" or "__delattr__()" method, this is called '
+          'instead\n'
+          '   of updating the instance dictionary directly.\n'
+          '\n'
+          '   Class instances can pretend to be numbers, sequences, or '
+          'mappings\n'
+          '   if they have methods with certain special names.  See section\n'
+          '   *Special method names*.\n'
+          '\n'
+          '   Special attributes: "__dict__" is the attribute dictionary;\n'
+          '   "__class__" is the instance\'s class.\n'
+          '\n'
+          'I/O objects (also known as file objects)\n'
+          '   A *file object* represents an open file.  Various shortcuts are\n'
+          '   available to create file objects: the "open()" built-in '
+          'function,\n'
+          '   and also "os.popen()", "os.fdopen()", and the "makefile()" '
+          'method\n'
+          '   of socket objects (and perhaps by other functions or methods\n'
+          '   provided by extension modules).\n'
+          '\n'
+          '   The objects "sys.stdin", "sys.stdout" and "sys.stderr" are\n'
+          "   initialized to file objects corresponding to the interpreter's\n"
+          '   standard input, output and error streams; they are all open in '
+          'text\n'
+          '   mode and therefore follow the interface defined by the\n'
+          '   "io.TextIOBase" abstract class.\n'
+          '\n'
+          'Internal types\n'
+          '   A few types used internally by the interpreter are exposed to '
+          'the\n'
+          '   user. Their definitions may change with future versions of the\n'
+          '   interpreter, but they are mentioned here for completeness.\n'
+          '\n'
+          '   Code objects\n'
+          '      Code objects represent *byte-compiled* executable Python '
+          'code,\n'
+          '      or *bytecode*. The difference between a code object and a\n'
+          '      function object is that the function object contains an '
+          'explicit\n'
+          "      reference to the function's globals (the module in which it "
+          'was\n'
+          '      defined), while a code object contains no context; also the\n'
+          '      default argument values are stored in the function object, '
+          'not\n'
+          '      in the code object (because they represent values calculated '
+          'at\n'
+          '      run-time).  Unlike function objects, code objects are '
+          'immutable\n'
+          '      and contain no references (directly or indirectly) to '
+          'mutable\n'
+          '      objects.\n'
+          '\n'
+          '      Special read-only attributes: "co_name" gives the function '
+          'name;\n'
+          '      "co_argcount" is the number of positional arguments '
+          '(including\n'
+          '      arguments with default values); "co_nlocals" is the number '
+          'of\n'
+          '      local variables used by the function (including arguments);\n'
+          '      "co_varnames" is a tuple containing the names of the local\n'
+          '      variables (starting with the argument names); "co_cellvars" '
+          'is a\n'
+          '      tuple containing the names of local variables that are\n'
+          '      referenced by nested functions; "co_freevars" is a tuple\n'
+          '      containing the names of free variables; "co_code" is a '
+          'string\n'
+          '      representing the sequence of bytecode instructions; '
+          '"co_consts"\n'
+          '      is a tuple containing the literals used by the bytecode;\n'
+          '      "co_names" is a tuple containing the names used by the '
+          'bytecode;\n'
+          '      "co_filename" is the filename from which the code was '
+          'compiled;\n'
+          '      "co_firstlineno" is the first line number of the function;\n'
+          '      "co_lnotab" is a string encoding the mapping from bytecode\n'
+          '      offsets to line numbers (for details see the source code of '
+          'the\n'
+          '      interpreter); "co_stacksize" is the required stack size\n'
+          '      (including local variables); "co_flags" is an integer '
+          'encoding a\n'
+          '      number of flags for the interpreter.\n'
+          '\n'
+          '      The following flag bits are defined for "co_flags": bit '
+          '"0x04"\n'
+          '      is set if the function uses the "*arguments" syntax to accept '
+          'an\n'
+          '      arbitrary number of positional arguments; bit "0x08" is set '
+          'if\n'
+          '      the function uses the "**keywords" syntax to accept '
+          'arbitrary\n'
+          '      keyword arguments; bit "0x20" is set if the function is a\n'
+          '      generator.\n'
+          '\n'
+          '      Future feature declarations ("from __future__ import '
+          'division")\n'
+          '      also use bits in "co_flags" to indicate whether a code '
+          'object\n'
+          '      was compiled with a particular feature enabled: bit "0x2000" '
+          'is\n'
+          '      set if the function was compiled with future division '
+          'enabled;\n'
+          '      bits "0x10" and "0x1000" were used in earlier versions of\n'
+          '      Python.\n'
+          '\n'
+          '      Other bits in "co_flags" are reserved for internal use.\n'
+          '\n'
+          '      If a code object represents a function, the first item in\n'
+          '      "co_consts" is the documentation string of the function, or\n'
+          '      "None" if undefined.\n'
+          '\n'
+          '   Frame objects\n'
+          '      Frame objects represent execution frames.  They may occur in\n'
+          '      traceback objects (see below).\n'
+          '\n'
+          '      Special read-only attributes: "f_back" is to the previous '
+          'stack\n'
+          '      frame (towards the caller), or "None" if this is the bottom\n'
+          '      stack frame; "f_code" is the code object being executed in '
+          'this\n'
+          '      frame; "f_locals" is the dictionary used to look up local\n'
+          '      variables; "f_globals" is used for global variables;\n'
+          '      "f_builtins" is used for built-in (intrinsic) names; '
+          '"f_lasti"\n'
+          '      gives the precise instruction (this is an index into the\n'
+          '      bytecode string of the code object).\n'
+          '\n'
+          '      Special writable attributes: "f_trace", if not "None", is a\n'
+          '      function called at the start of each source code line (this '
+          'is\n'
+          '      used by the debugger); "f_lineno" is the current line number '
+          'of\n'
+          '      the frame --- writing to this from within a trace function '
+          'jumps\n'
+          '      to the given line (only for the bottom-most frame).  A '
+          'debugger\n'
+          '      can implement a Jump command (aka Set Next Statement) by '
+          'writing\n'
+          '      to f_lineno.\n'
+          '\n'
+          '      Frame objects support one method:\n'
+          '\n'
+          '      frame.clear()\n'
+          '\n'
+          '         This method clears all references to local variables held '
+          'by\n'
+          '         the frame.  Also, if the frame belonged to a generator, '
+          'the\n'
+          '         generator is finalized.  This helps break reference '
+          'cycles\n'
+          '         involving frame objects (for example when catching an\n'
+          '         exception and storing its traceback for later use).\n'
+          '\n'
+          '         "RuntimeError" is raised if the frame is currently '
+          'executing.\n'
+          '\n'
+          '         New in version 3.4.\n'
+          '\n'
+          '   Traceback objects\n'
+          '      Traceback objects represent a stack trace of an exception.  '
+          'A\n'
+          '      traceback object is created when an exception occurs.  When '
+          'the\n'
+          '      search for an exception handler unwinds the execution stack, '
+          'at\n'
+          '      each unwound level a traceback object is inserted in front '
+          'of\n'
+          '      the current traceback.  When an exception handler is '
+          'entered,\n'
+          '      the stack trace is made available to the program. (See '
+          'section\n'
+          '      *The try statement*.) It is accessible as the third item of '
+          'the\n'
+          '      tuple returned by "sys.exc_info()". When the program contains '
+          'no\n'
+          '      suitable handler, the stack trace is written (nicely '
+          'formatted)\n'
+          '      to the standard error stream; if the interpreter is '
+          'interactive,\n'
+          '      it is also made available to the user as '
+          '"sys.last_traceback".\n'
+          '\n'
+          '      Special read-only attributes: "tb_next" is the next level in '
+          'the\n'
+          '      stack trace (towards the frame where the exception occurred), '
+          'or\n'
+          '      "None" if there is no next level; "tb_frame" points to the\n'
+          '      execution frame of the current level; "tb_lineno" gives the '
+          'line\n'
+          '      number where the exception occurred; "tb_lasti" indicates '
+          'the\n'
+          '      precise instruction.  The line number and last instruction '
+          'in\n'
+          '      the traceback may differ from the line number of its frame\n'
+          '      object if the exception occurred in a "try" statement with '
+          'no\n'
+          '      matching except clause or with a finally clause.\n'
+          '\n'
+          '   Slice objects\n'
+          '      Slice objects are used to represent slices for '
+          '"__getitem__()"\n'
+          '      methods.  They are also created by the built-in "slice()"\n'
+          '      function.\n'
+          '\n'
+          '      Special read-only attributes: "start" is the lower bound; '
+          '"stop"\n'
+          '      is the upper bound; "step" is the step value; each is "None" '
+          'if\n'
+          '      omitted.  These attributes can have any type.\n'
+          '\n'
+          '      Slice objects support one method:\n'
+          '\n'
+          '      slice.indices(self, length)\n'
+          '\n'
+          '         This method takes a single integer argument *length* and\n'
+          '         computes information about the slice that the slice '
+          'object\n'
+          '         would describe if applied to a sequence of *length* '
+          'items.\n'
+          '         It returns a tuple of three integers; respectively these '
+          'are\n'
+          '         the *start* and *stop* indices and the *step* or stride\n'
+          '         length of the slice. Missing or out-of-bounds indices are\n'
+          '         handled in a manner consistent with regular slices.\n'
+          '\n'
+          '   Static method objects\n'
+          '      Static method objects provide a way of defeating the\n'
+          '      transformation of function objects to method objects '
+          'described\n'
+          '      above. A static method object is a wrapper around any other\n'
+          '      object, usually a user-defined method object. When a static\n'
+          '      method object is retrieved from a class or a class instance, '
+          'the\n'
+          '      object actually returned is the wrapped object, which is not\n'
+          '      subject to any further transformation. Static method objects '
+          'are\n'
+          '      not themselves callable, although the objects they wrap '
+          'usually\n'
+          '      are. Static method objects are created by the built-in\n'
+          '      "staticmethod()" constructor.\n'
+          '\n'
+          '   Class method objects\n'
+          '      A class method object, like a static method object, is a '
+          'wrapper\n'
+          '      around another object that alters the way in which that '
+          'object\n'
+          '      is retrieved from classes and class instances. The behaviour '
+          'of\n'
+          '      class method objects upon such retrieval is described above,\n'
+          '      under "User-defined methods". Class method objects are '
+          'created\n'
+          '      by the built-in "classmethod()" constructor.\n',
+ 'typesfunctions': '\n'
+                   'Functions\n'
+                   '*********\n'
+                   '\n'
+                   'Function objects are created by function definitions.  The '
+                   'only\n'
+                   'operation on a function object is to call it: '
+                   '"func(argument-list)".\n'
+                   '\n'
+                   'There are really two flavors of function objects: built-in '
+                   'functions\n'
+                   'and user-defined functions.  Both support the same '
+                   'operation (to call\n'
+                   'the function), but the implementation is different, hence '
+                   'the\n'
+                   'different object types.\n'
+                   '\n'
+                   'See *Function definitions* for more information.\n',
+ 'typesmapping': '\n'
+                 'Mapping Types --- "dict"\n'
+                 '************************\n'
+                 '\n'
+                 'A *mapping* object maps *hashable* values to arbitrary '
+                 'objects.\n'
+                 'Mappings are mutable objects.  There is currently only one '
+                 'standard\n'
+                 'mapping type, the *dictionary*.  (For other containers see '
+                 'the built-\n'
+                 'in "list", "set", and "tuple" classes, and the "collections" '
+                 'module.)\n'
+                 '\n'
+                 "A dictionary's keys are *almost* arbitrary values.  Values "
+                 'that are\n'
+                 'not *hashable*, that is, values containing lists, '
+                 'dictionaries or\n'
+                 'other mutable types (that are compared by value rather than '
+                 'by object\n'
+                 'identity) may not be used as keys.  Numeric types used for '
+                 'keys obey\n'
+                 'the normal rules for numeric comparison: if two numbers '
+                 'compare equal\n'
+                 '(such as "1" and "1.0") then they can be used '
+                 'interchangeably to index\n'
+                 'the same dictionary entry.  (Note however, that since '
+                 'computers store\n'
+                 'floating-point numbers as approximations it is usually '
+                 'unwise to use\n'
+                 'them as dictionary keys.)\n'
+                 '\n'
+                 'Dictionaries can be created by placing a comma-separated '
+                 'list of "key:\n'
+                 'value" pairs within braces, for example: "{\'jack\': 4098, '
+                 "'sjoerd':\n"
+                 '4127}" or "{4098: \'jack\', 4127: \'sjoerd\'}", or by the '
+                 '"dict"\n'
+                 'constructor.\n'
+                 '\n'
+                 'class class dict(**kwarg)\n'
+                 'class class dict(mapping, **kwarg)\n'
+                 'class class dict(iterable, **kwarg)\n'
+                 '\n'
+                 '   Return a new dictionary initialized from an optional '
+                 'positional\n'
+                 '   argument and a possibly empty set of keyword arguments.\n'
+                 '\n'
+                 '   If no positional argument is given, an empty dictionary '
+                 'is created.\n'
+                 '   If a positional argument is given and it is a mapping '
+                 'object, a\n'
+                 '   dictionary is created with the same key-value pairs as '
+                 'the mapping\n'
+                 '   object.  Otherwise, the positional argument must be an '
+                 '*iterable*\n'
+                 '   object.  Each item in the iterable must itself be an '
+                 'iterable with\n'
+                 '   exactly two objects.  The first object of each item '
+                 'becomes a key\n'
+                 '   in the new dictionary, and the second object the '
+                 'corresponding\n'
+                 '   value.  If a key occurs more than once, the last value '
+                 'for that key\n'
+                 '   becomes the corresponding value in the new dictionary.\n'
+                 '\n'
+                 '   If keyword arguments are given, the keyword arguments and '
+                 'their\n'
+                 '   values are added to the dictionary created from the '
+                 'positional\n'
+                 '   argument.  If a key being added is already present, the '
+                 'value from\n'
+                 '   the keyword argument replaces the value from the '
+                 'positional\n'
+                 '   argument.\n'
+                 '\n'
+                 '   To illustrate, the following examples all return a '
+                 'dictionary equal\n'
+                 '   to "{"one": 1, "two": 2, "three": 3}":\n'
+                 '\n'
+                 '      >>> a = dict(one=1, two=2, three=3)\n'
+                 "      >>> b = {'one': 1, 'two': 2, 'three': 3}\n"
+                 "      >>> c = dict(zip(['one', 'two', 'three'], [1, 2, 3]))\n"
+                 "      >>> d = dict([('two', 2), ('one', 1), ('three', 3)])\n"
+                 "      >>> e = dict({'three': 3, 'one': 1, 'two': 2})\n"
+                 '      >>> a == b == c == d == e\n'
+                 '      True\n'
+                 '\n'
+                 '   Providing keyword arguments as in the first example only '
+                 'works for\n'
+                 '   keys that are valid Python identifiers.  Otherwise, any '
+                 'valid keys\n'
+                 '   can be used.\n'
+                 '\n'
+                 '   These are the operations that dictionaries support (and '
+                 'therefore,\n'
+                 '   custom mapping types should support too):\n'
+                 '\n'
+                 '   len(d)\n'
+                 '\n'
+                 '      Return the number of items in the dictionary *d*.\n'
+                 '\n'
+                 '   d[key]\n'
+                 '\n'
+                 '      Return the item of *d* with key *key*.  Raises a '
+                 '"KeyError" if\n'
+                 '      *key* is not in the map.\n'
+                 '\n'
+                 '      If a subclass of dict defines a method "__missing__()" '
+                 'and *key*\n'
+                 '      is not present, the "d[key]" operation calls that '
+                 'method with\n'
+                 '      the key *key* as argument.  The "d[key]" operation '
+                 'then returns\n'
+                 '      or raises whatever is returned or raised by the\n'
+                 '      "__missing__(key)" call. No other operations or '
+                 'methods invoke\n'
+                 '      "__missing__()". If "__missing__()" is not defined, '
+                 '"KeyError"\n'
+                 '      is raised. "__missing__()" must be a method; it cannot '
+                 'be an\n'
+                 '      instance variable:\n'
+                 '\n'
+                 '         >>> class Counter(dict):\n'
+                 '         ...     def __missing__(self, key):\n'
+                 '         ...         return 0\n'
+                 '         >>> c = Counter()\n'
+                 "         >>> c['red']\n"
+                 '         0\n'
+                 "         >>> c['red'] += 1\n"
+                 "         >>> c['red']\n"
+                 '         1\n'
+                 '\n'
+                 '      The example above shows part of the implementation of\n'
+                 '      "collections.Counter".  A different "__missing__" '
+                 'method is used\n'
+                 '      by "collections.defaultdict".\n'
+                 '\n'
+                 '   d[key] = value\n'
+                 '\n'
+                 '      Set "d[key]" to *value*.\n'
+                 '\n'
+                 '   del d[key]\n'
+                 '\n'
+                 '      Remove "d[key]" from *d*.  Raises a "KeyError" if '
+                 '*key* is not\n'
+                 '      in the map.\n'
+                 '\n'
+                 '   key in d\n'
+                 '\n'
+                 '      Return "True" if *d* has a key *key*, else "False".\n'
+                 '\n'
+                 '   key not in d\n'
+                 '\n'
+                 '      Equivalent to "not key in d".\n'
+                 '\n'
+                 '   iter(d)\n'
+                 '\n'
+                 '      Return an iterator over the keys of the dictionary.  '
+                 'This is a\n'
+                 '      shortcut for "iter(d.keys())".\n'
+                 '\n'
+                 '   clear()\n'
+                 '\n'
+                 '      Remove all items from the dictionary.\n'
+                 '\n'
+                 '   copy()\n'
+                 '\n'
+                 '      Return a shallow copy of the dictionary.\n'
+                 '\n'
+                 '   classmethod fromkeys(seq[, value])\n'
+                 '\n'
+                 '      Create a new dictionary with keys from *seq* and '
+                 'values set to\n'
+                 '      *value*.\n'
+                 '\n'
+                 '      "fromkeys()" is a class method that returns a new '
+                 'dictionary.\n'
+                 '      *value* defaults to "None".\n'
+                 '\n'
+                 '   get(key[, default])\n'
+                 '\n'
+                 '      Return the value for *key* if *key* is in the '
+                 'dictionary, else\n'
+                 '      *default*. If *default* is not given, it defaults to '
+                 '"None", so\n'
+                 '      that this method never raises a "KeyError".\n'
+                 '\n'
+                 '   items()\n'
+                 '\n'
+                 '      Return a new view of the dictionary\'s items ("(key, '
+                 'value)"\n'
+                 '      pairs). See the *documentation of view objects*.\n'
+                 '\n'
+                 '   keys()\n'
+                 '\n'
+                 "      Return a new view of the dictionary's keys.  See the\n"
+                 '      *documentation of view objects*.\n'
+                 '\n'
+                 '   pop(key[, default])\n'
+                 '\n'
+                 '      If *key* is in the dictionary, remove it and return '
+                 'its value,\n'
+                 '      else return *default*.  If *default* is not given and '
+                 '*key* is\n'
+                 '      not in the dictionary, a "KeyError" is raised.\n'
+                 '\n'
+                 '   popitem()\n'
+                 '\n'
+                 '      Remove and return an arbitrary "(key, value)" pair '
+                 'from the\n'
+                 '      dictionary.\n'
+                 '\n'
+                 '      "popitem()" is useful to destructively iterate over a\n'
+                 '      dictionary, as often used in set algorithms.  If the '
+                 'dictionary\n'
+                 '      is empty, calling "popitem()" raises a "KeyError".\n'
+                 '\n'
+                 '   setdefault(key[, default])\n'
+                 '\n'
+                 '      If *key* is in the dictionary, return its value.  If '
+                 'not, insert\n'
+                 '      *key* with a value of *default* and return *default*.  '
+                 '*default*\n'
+                 '      defaults to "None".\n'
+                 '\n'
+                 '   update([other])\n'
+                 '\n'
+                 '      Update the dictionary with the key/value pairs from '
+                 '*other*,\n'
+                 '      overwriting existing keys.  Return "None".\n'
+                 '\n'
+                 '      "update()" accepts either another dictionary object or '
+                 'an\n'
+                 '      iterable of key/value pairs (as tuples or other '
+                 'iterables of\n'
+                 '      length two).  If keyword arguments are specified, the '
+                 'dictionary\n'
+                 '      is then updated with those key/value pairs: '
+                 '"d.update(red=1,\n'
+                 '      blue=2)".\n'
+                 '\n'
+                 '   values()\n'
+                 '\n'
+                 "      Return a new view of the dictionary's values.  See "
+                 'the\n'
+                 '      *documentation of view objects*.\n'
+                 '\n'
+                 '   Dictionaries compare equal if and only if they have the '
+                 'same "(key,\n'
+                 '   value)" pairs. Order comparisons (\'<\', \'<=\', \'>=\', '
+                 "'>') raise\n"
+                 '   "TypeError".\n'
+                 '\n'
+                 'See also: "types.MappingProxyType" can be used to create a '
+                 'read-only\n'
+                 '  view of a "dict".\n'
+                 '\n'
+                 '\n'
+                 'Dictionary view objects\n'
+                 '=======================\n'
+                 '\n'
+                 'The objects returned by "dict.keys()", "dict.values()" and\n'
+                 '"dict.items()" are *view objects*.  They provide a dynamic '
+                 'view on the\n'
+                 "dictionary's entries, which means that when the dictionary "
+                 'changes,\n'
+                 'the view reflects these changes.\n'
+                 '\n'
+                 'Dictionary views can be iterated over to yield their '
+                 'respective data,\n'
+                 'and support membership tests:\n'
+                 '\n'
+                 'len(dictview)\n'
+                 '\n'
+                 '   Return the number of entries in the dictionary.\n'
+                 '\n'
+                 'iter(dictview)\n'
+                 '\n'
+                 '   Return an iterator over the keys, values or items '
+                 '(represented as\n'
+                 '   tuples of "(key, value)") in the dictionary.\n'
+                 '\n'
+                 '   Keys and values are iterated over in an arbitrary order '
+                 'which is\n'
+                 '   non-random, varies across Python implementations, and '
+                 'depends on\n'
+                 "   the dictionary's history of insertions and deletions. If "
+                 'keys,\n'
+                 '   values and items views are iterated over with no '
+                 'intervening\n'
+                 '   modifications to the dictionary, the order of items will '
+                 'directly\n'
+                 '   correspond.  This allows the creation of "(value, key)" '
+                 'pairs using\n'
+                 '   "zip()": "pairs = zip(d.values(), d.keys())".  Another '
+                 'way to\n'
+                 '   create the same list is "pairs = [(v, k) for (k, v) in '
+                 'd.items()]".\n'
+                 '\n'
+                 '   Iterating views while adding or deleting entries in the '
+                 'dictionary\n'
+                 '   may raise a "RuntimeError" or fail to iterate over all '
+                 'entries.\n'
+                 '\n'
+                 'x in dictview\n'
+                 '\n'
+                 '   Return "True" if *x* is in the underlying dictionary\'s '
+                 'keys, values\n'
+                 '   or items (in the latter case, *x* should be a "(key, '
+                 'value)"\n'
+                 '   tuple).\n'
+                 '\n'
+                 'Keys views are set-like since their entries are unique and '
+                 'hashable.\n'
+                 'If all values are hashable, so that "(key, value)" pairs are '
+                 'unique\n'
+                 'and hashable, then the items view is also set-like.  (Values '
+                 'views are\n'
+                 'not treated as set-like since the entries are generally not '
+                 'unique.)\n'
+                 'For set-like views, all of the operations defined for the '
+                 'abstract\n'
+                 'base class "collections.abc.Set" are available (for example, '
+                 '"==",\n'
+                 '"<", or "^").\n'
+                 '\n'
+                 'An example of dictionary view usage:\n'
+                 '\n'
+                 "   >>> dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, "
+                 "'spam': 500}\n"
+                 '   >>> keys = dishes.keys()\n'
+                 '   >>> values = dishes.values()\n'
+                 '\n'
+                 '   >>> # iteration\n'
+                 '   >>> n = 0\n'
+                 '   >>> for val in values:\n'
+                 '   ...     n += val\n'
+                 '   >>> print(n)\n'
+                 '   504\n'
+                 '\n'
+                 '   >>> # keys and values are iterated over in the same '
+                 'order\n'
+                 '   >>> list(keys)\n'
+                 "   ['eggs', 'bacon', 'sausage', 'spam']\n"
+                 '   >>> list(values)\n'
+                 '   [2, 1, 1, 500]\n'
+                 '\n'
+                 '   >>> # view objects are dynamic and reflect dict changes\n'
+                 "   >>> del dishes['eggs']\n"
+                 "   >>> del dishes['sausage']\n"
+                 '   >>> list(keys)\n'
+                 "   ['spam', 'bacon']\n"
+                 '\n'
+                 '   >>> # set operations\n'
+                 "   >>> keys & {'eggs', 'bacon', 'salad'}\n"
+                 "   {'bacon'}\n"
+                 "   >>> keys ^ {'sausage', 'juice'}\n"
+                 "   {'juice', 'sausage', 'bacon', 'spam'}\n",
+ 'typesmethods': '\n'
+                 'Methods\n'
+                 '*******\n'
+                 '\n'
+                 'Methods are functions that are called using the attribute '
+                 'notation.\n'
+                 'There are two flavors: built-in methods (such as "append()" '
+                 'on lists)\n'
+                 'and class instance methods.  Built-in methods are described '
+                 'with the\n'
+                 'types that support them.\n'
+                 '\n'
+                 'If you access a method (a function defined in a class '
+                 'namespace)\n'
+                 'through an instance, you get a special object: a *bound '
+                 'method* (also\n'
+                 'called *instance method*) object. When called, it will add '
+                 'the "self"\n'
+                 'argument to the argument list.  Bound methods have two '
+                 'special read-\n'
+                 'only attributes: "m.__self__" is the object on which the '
+                 'method\n'
+                 'operates, and "m.__func__" is the function implementing the '
+                 'method.\n'
+                 'Calling "m(arg-1, arg-2, ..., arg-n)" is completely '
+                 'equivalent to\n'
+                 'calling "m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)".\n'
+                 '\n'
+                 'Like function objects, bound method objects support getting '
+                 'arbitrary\n'
+                 'attributes.  However, since method attributes are actually '
+                 'stored on\n'
+                 'the underlying function object ("meth.__func__"), setting '
+                 'method\n'
+                 'attributes on bound methods is disallowed.  Attempting to '
+                 'set an\n'
+                 'attribute on a method results in an "AttributeError" being '
+                 'raised.  In\n'
+                 'order to set a method attribute, you need to explicitly set '
+                 'it on the\n'
+                 'underlying function object:\n'
+                 '\n'
+                 '   >>> class C:\n'
+                 '   ...     def method(self):\n'
+                 '   ...         pass\n'
+                 '   ...\n'
+                 '   >>> c = C()\n'
+                 "   >>> c.method.whoami = 'my name is method'  # can't set on "
+                 'the method\n'
+                 '   Traceback (most recent call last):\n'
+                 '     File "<stdin>", line 1, in <module>\n'
+                 "   AttributeError: 'method' object has no attribute "
+                 "'whoami'\n"
+                 "   >>> c.method.__func__.whoami = 'my name is method'\n"
+                 '   >>> c.method.whoami\n'
+                 "   'my name is method'\n"
+                 '\n'
+                 'See *The standard type hierarchy* for more information.\n',
+ 'typesmodules': '\n'
+                 'Modules\n'
+                 '*******\n'
+                 '\n'
+                 'The only special operation on a module is attribute access: '
+                 '"m.name",\n'
+                 'where *m* is a module and *name* accesses a name defined in '
+                 "*m*'s\n"
+                 'symbol table. Module attributes can be assigned to.  (Note '
+                 'that the\n'
+                 '"import" statement is not, strictly speaking, an operation '
+                 'on a module\n'
+                 'object; "import foo" does not require a module object named '
+                 '*foo* to\n'
+                 'exist, rather it requires an (external) *definition* for a '
+                 'module\n'
+                 'named *foo* somewhere.)\n'
+                 '\n'
+                 'A special attribute of every module is "__dict__". This is '
+                 'the\n'
+                 "dictionary containing the module's symbol table. Modifying "
+                 'this\n'
+                 "dictionary will actually change the module's symbol table, "
+                 'but direct\n'
+                 'assignment to the "__dict__" attribute is not possible (you '
+                 'can write\n'
+                 '"m.__dict__[\'a\'] = 1", which defines "m.a" to be "1", but '
+                 "you can't\n"
+                 'write "m.__dict__ = {}").  Modifying "__dict__" directly is '
+                 'not\n'
+                 'recommended.\n'
+                 '\n'
+                 'Modules built into the interpreter are written like this: '
+                 '"<module\n'
+                 '\'sys\' (built-in)>".  If loaded from a file, they are '
+                 'written as\n'
+                 '"<module \'os\' from '
+                 '\'/usr/local/lib/pythonX.Y/os.pyc\'>".\n',
+ 'typesseq': '\n'
+             'Sequence Types --- "list", "tuple", "range"\n'
+             '*******************************************\n'
+             '\n'
+             'There are three basic sequence types: lists, tuples, and range\n'
+             'objects. Additional sequence types tailored for processing of '
+             '*binary\n'
+             'data* and *text strings* are described in dedicated sections.\n'
+             '\n'
+             '\n'
+             'Common Sequence Operations\n'
+             '==========================\n'
+             '\n'
+             'The operations in the following table are supported by most '
+             'sequence\n'
+             'types, both mutable and immutable. The '
+             '"collections.abc.Sequence" ABC\n'
+             'is provided to make it easier to correctly implement these '
+             'operations\n'
+             'on custom sequence types.\n'
+             '\n'
+             'This table lists the sequence operations sorted in ascending '
+             'priority.\n'
+             'In the table, *s* and *t* are sequences of the same type, *n*, '
+             '*i*,\n'
+             '*j* and *k* are integers and *x* is an arbitrary object that '
+             'meets any\n'
+             'type and value restrictions imposed by *s*.\n'
+             '\n'
+             'The "in" and "not in" operations have the same priorities as '
+             'the\n'
+             'comparison operations. The "+" (concatenation) and "*" '
+             '(repetition)\n'
+             'operations have the same priority as the corresponding numeric\n'
+             'operations. [3]\n'
+             '\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '| Operation                  | Result                           '
+             '| Notes      |\n'
+             '+============================+==================================+============+\n'
+             '| "x in s"                   | "True" if an item of *s* is      '
+             '| (1)        |\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '| "x not in s"               | "False" if an item of *s* is     '
+             '| (1)        |\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '| "s + t"                    | the concatenation of *s* and *t* '
+             '| (6)(7)     |\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '| "s * n" or "n * s"         | equivalent to adding *s* to      '
+             '| (2)(7)     |\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '| "s[i]"                     | *i*th item of *s*, origin 0      '
+             '| (3)        |\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '| "s[i:j]"                   | slice of *s* from *i* to *j*     '
+             '| (3)(4)     |\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '| "s[i:j:k]"                 | slice of *s* from *i* to *j*     '
+             '| (3)(5)     |\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '| "s.index(x[, i[, j]])"     | index of the first occurrence of '
+             '| (8)        |\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '+----------------------------+----------------------------------+------------+\n'
+             '\n'
+             'Sequences of the same type also support comparisons.  In '
+             'particular,\n'
+             'tuples and lists are compared lexicographically by comparing\n'
+             'corresponding elements. This means that to compare equal, every\n'
+             'element must compare equal and the two sequences must be of the '
+             'same\n'
+             'type and have the same length.  (For full details see '
+             '*Comparisons* in\n'
+             'the language reference.)\n'
+             '\n'
+             'Notes:\n'
+             '\n'
+             '1. While the "in" and "not in" operations are used only for '
+             'simple\n'
+             '   containment testing in the general case, some specialised '
+             'sequences\n'
+             '   (such as "str", "bytes" and "bytearray") also use them for\n'
+             '   subsequence testing:\n'
+             '\n'
+             '      >>> "gg" in "eggs"\n'
+             '      True\n'
+             '\n'
+             '2. Values of *n* less than "0" are treated as "0" (which yields '
+             'an\n'
+             '   empty sequence of the same type as *s*).  Note that items in '
+             'the\n'
+             '   sequence *s* are not copied; they are referenced multiple '
+             'times.\n'
+             '   This often haunts new Python programmers; consider:\n'
+             '\n'
+             '      >>> lists = [[]] * 3\n'
+             '      >>> lists\n'
+             '      [[], [], []]\n'
+             '      >>> lists[0].append(3)\n'
+             '      >>> lists\n'
+             '      [[3], [3], [3]]\n'
+             '\n'
+             '   What has happened is that "[[]]" is a one-element list '
+             'containing\n'
+             '   an empty list, so all three elements of "[[]] * 3" are '
+             'references\n'
+             '   to this single empty list.  Modifying any of the elements of\n'
+             '   "lists" modifies this single list. You can create a list of\n'
+             '   different lists this way:\n'
+             '\n'
+             '      >>> lists = [[] for i in range(3)]\n'
+             '      >>> lists[0].append(3)\n'
+             '      >>> lists[1].append(5)\n'
+             '      >>> lists[2].append(7)\n'
+             '      >>> lists\n'
+             '      [[3], [5], [7]]\n'
+             '\n'
+             '   Further explanation is available in the FAQ entry *How do I '
+             'create\n'
+             '   a multidimensional list?*.\n'
+             '\n'
+             '3. If *i* or *j* is negative, the index is relative to the end '
+             'of\n'
+             '   sequence *s*: "len(s) + i" or "len(s) + j" is substituted.  '
+             'But\n'
+             '   note that "-0" is still "0".\n'
+             '\n'
+             '4. The slice of *s* from *i* to *j* is defined as the sequence '
+             'of\n'
+             '   items with index *k* such that "i <= k < j".  If *i* or *j* '
+             'is\n'
+             '   greater than "len(s)", use "len(s)".  If *i* is omitted or '
+             '"None",\n'
+             '   use "0".  If *j* is omitted or "None", use "len(s)".  If *i* '
+             'is\n'
+             '   greater than or equal to *j*, the slice is empty.\n'
+             '\n'
+             '5. The slice of *s* from *i* to *j* with step *k* is defined as '
+             'the\n'
+             '   sequence of items with index  "x = i + n*k" such that "0 <= n '
+             '<\n'
+             '   (j-i)/k".  In other words, the indices are "i", "i+k", '
+             '"i+2*k",\n'
+             '   "i+3*k" and so on, stopping when *j* is reached (but never\n'
+             '   including *j*).  When *k* is positive, *i* and *j* are '
+             'reduced to\n'
+             '   "len(s)" if they are greater. When *k* is negative, *i* and '
+             '*j* are\n'
+             '   reduced to "len(s) - 1" if they are greater.  If *i* or *j* '
+             'are\n'
+             '   omitted or "None", they become "end" values (which end '
+             'depends on\n'
+             '   the sign of *k*).  Note, *k* cannot be zero. If *k* is '
+             '"None", it\n'
+             '   is treated like "1".\n'
+             '\n'
+             '6. Concatenating immutable sequences always results in a new\n'
+             '   object. This means that building up a sequence by repeated\n'
+             '   concatenation will have a quadratic runtime cost in the '
+             'total\n'
+             '   sequence length. To get a linear runtime cost, you must '
+             'switch to\n'
+             '   one of the alternatives below:\n'
+             '\n'
+             '   * if concatenating "str" objects, you can build a list and '
+             'use\n'
+             '     "str.join()" at the end or else write to an "io.StringIO"\n'
+             '     instance and retrieve its value when complete\n'
+             '\n'
+             '   * if concatenating "bytes" objects, you can similarly use\n'
+             '     "bytes.join()" or "io.BytesIO", or you can do in-place\n'
+             '     concatenation with a "bytearray" object.  "bytearray" '
+             'objects are\n'
+             '     mutable and have an efficient overallocation mechanism\n'
+             '\n'
+             '   * if concatenating "tuple" objects, extend a "list" instead\n'
+             '\n'
+             '   * for other types, investigate the relevant class '
+             'documentation\n'
+             '\n'
+             '7. Some sequence types (such as "range") only support item\n'
+             "   sequences that follow specific patterns, and hence don't "
+             'support\n'
+             '   sequence concatenation or repetition.\n'
+             '\n'
+             '8. "index" raises "ValueError" when *x* is not found in *s*. '
+             'When\n'
+             '   supported, the additional arguments to the index method '
+             'allow\n'
+             '   efficient searching of subsections of the sequence. Passing '
+             'the\n'
+             '   extra arguments is roughly equivalent to using '
+             '"s[i:j].index(x)",\n'
+             '   only without copying any data and with the returned index '
+             'being\n'
+             '   relative to the start of the sequence rather than the start '
+             'of the\n'
+             '   slice.\n'
+             '\n'
+             '\n'
+             'Immutable Sequence Types\n'
+             '========================\n'
+             '\n'
+             'The only operation that immutable sequence types generally '
+             'implement\n'
+             'that is not also implemented by mutable sequence types is '
+             'support for\n'
+             'the "hash()" built-in.\n'
+             '\n'
+             'This support allows immutable sequences, such as "tuple" '
+             'instances, to\n'
+             'be used as "dict" keys and stored in "set" and "frozenset" '
+             'instances.\n'
+             '\n'
+             'Attempting to hash an immutable sequence that contains '
+             'unhashable\n'
+             'values will result in "TypeError".\n'
+             '\n'
+             '\n'
+             'Mutable Sequence Types\n'
+             '======================\n'
+             '\n'
+             'The operations in the following table are defined on mutable '
+             'sequence\n'
+             'types. The "collections.abc.MutableSequence" ABC is provided to '
+             'make\n'
+             'it easier to correctly implement these operations on custom '
+             'sequence\n'
+             'types.\n'
+             '\n'
+             'In the table *s* is an instance of a mutable sequence type, *t* '
+             'is any\n'
+             'iterable object and *x* is an arbitrary object that meets any '
+             'type and\n'
+             'value restrictions imposed by *s* (for example, "bytearray" '
+             'only\n'
+             'accepts integers that meet the value restriction "0 <= x <= '
+             '255").\n'
+             '\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '| Operation                      | '
+             'Result                           | Notes                 |\n'
+             '+================================+==================================+=======================+\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '| "s[i:j:k] = t"                 | the elements of "s[i:j:k]" '
+             'are   | (1)                   |\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '| "s.clear()"                    | removes all items from "s" '
+             '(same | (5)                   |\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '| "s.copy()"                     | creates a shallow copy of '
+             '"s"    | (5)                   |\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '| "s *= n"                       | updates *s* with its '
+             'contents    | (6)                   |\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '| "s.pop([i])"                   | retrieves the item at *i* '
+             'and    | (2)                   |\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '| "s.remove(x)"                  | remove the first item from '
+             '*s*   | (3)                   |\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '| "s.reverse()"                  | reverses the items of *s* '
+             'in     | (4)                   |\n'
+             '+--------------------------------+----------------------------------+-----------------------+\n'
+             '\n'
+             'Notes:\n'
+             '\n'
+             '1. *t* must have the same length as the slice it is replacing.\n'
+             '\n'
+             '2. The optional argument *i* defaults to "-1", so that by '
+             'default\n'
+             '   the last item is removed and returned.\n'
+             '\n'
+             '3. "remove" raises "ValueError" when *x* is not found in *s*.\n'
+             '\n'
+             '4. The "reverse()" method modifies the sequence in place for\n'
+             '   economy of space when reversing a large sequence.  To remind '
+             'users\n'
+             '   that it operates by side effect, it does not return the '
+             'reversed\n'
+             '   sequence.\n'
+             '\n'
+             '5. "clear()" and "copy()" are included for consistency with the\n'
+             "   interfaces of mutable containers that don't support slicing\n"
+             '   operations (such as "dict" and "set")\n'
+             '\n'
+             '   New in version 3.3: "clear()" and "copy()" methods.\n'
+             '\n'
+             '6. The value *n* is an integer, or an object implementing\n'
+             '   "__index__()".  Zero and negative values of *n* clear the '
+             'sequence.\n'
+             '   Items in the sequence are not copied; they are referenced '
+             'multiple\n'
+             '   times, as explained for "s * n" under *Common Sequence '
+             'Operations*.\n'
+             '\n'
+             '\n'
+             'Lists\n'
+             '=====\n'
+             '\n'
+             'Lists are mutable sequences, typically used to store collections '
+             'of\n'
+             'homogeneous items (where the precise degree of similarity will '
+             'vary by\n'
+             'application).\n'
+             '\n'
+             'class class list([iterable])\n'
+             '\n'
+             '   Lists may be constructed in several ways:\n'
+             '\n'
+             '   * Using a pair of square brackets to denote the empty list: '
+             '"[]"\n'
+             '\n'
+             '   * Using square brackets, separating items with commas: '
+             '"[a]",\n'
+             '     "[a, b, c]"\n'
+             '\n'
+             '   * Using a list comprehension: "[x for x in iterable]"\n'
+             '\n'
+             '   * Using the type constructor: "list()" or "list(iterable)"\n'
+             '\n'
+             '   The constructor builds a list whose items are the same and in '
+             'the\n'
+             "   same order as *iterable*'s items.  *iterable* may be either "
+             'a\n'
+             '   sequence, a container that supports iteration, or an '
+             'iterator\n'
+             '   object.  If *iterable* is already a list, a copy is made and\n'
+             '   returned, similar to "iterable[:]". For example, '
+             '"list(\'abc\')"\n'
+             '   returns "[\'a\', \'b\', \'c\']" and "list( (1, 2, 3) )" '
+             'returns "[1, 2,\n'
+             '   3]". If no argument is given, the constructor creates a new '
+             'empty\n'
+             '   list, "[]".\n'
+             '\n'
+             '   Many other operations also produce lists, including the '
+             '"sorted()"\n'
+             '   built-in.\n'
+             '\n'
+             '   Lists implement all of the *common* and *mutable* sequence\n'
+             '   operations. Lists also provide the following additional '
+             'method:\n'
+             '\n'
+             '   sort(*, key=None, reverse=None)\n'
+             '\n'
+             '      This method sorts the list in place, using only "<" '
+             'comparisons\n'
+             '      between items. Exceptions are not suppressed - if any '
+             'comparison\n'
+             '      operations fail, the entire sort operation will fail (and '
+             'the\n'
+             '      list will likely be left in a partially modified state).\n'
+             '\n'
+             '      "sort()" accepts two arguments that can only be passed by\n'
+             '      keyword (*keyword-only arguments*):\n'
+             '\n'
+             '      *key* specifies a function of one argument that is used '
+             'to\n'
+             '      extract a comparison key from each list element (for '
+             'example,\n'
+             '      "key=str.lower"). The key corresponding to each item in '
+             'the list\n'
+             '      is calculated once and then used for the entire sorting '
+             'process.\n'
+             '      The default value of "None" means that list items are '
+             'sorted\n'
+             '      directly without calculating a separate key value.\n'
+             '\n'
+             '      The "functools.cmp_to_key()" utility is available to '
+             'convert a\n'
+             '      2.x style *cmp* function to a *key* function.\n'
+             '\n'
+             '      *reverse* is a boolean value.  If set to "True", then the '
+             'list\n'
+             '      elements are sorted as if each comparison were reversed.\n'
+             '\n'
+             '      This method modifies the sequence in place for economy of '
+             'space\n'
+             '      when sorting a large sequence.  To remind users that it '
+             'operates\n'
+             '      by side effect, it does not return the sorted sequence '
+             '(use\n'
+             '      "sorted()" to explicitly request a new sorted list '
+             'instance).\n'
+             '\n'
+             '      The "sort()" method is guaranteed to be stable.  A sort '
+             'is\n'
+             '      stable if it guarantees not to change the relative order '
+             'of\n'
+             '      elements that compare equal --- this is helpful for '
+             'sorting in\n'
+             '      multiple passes (for example, sort by department, then by '
+             'salary\n'
+             '      grade).\n'
+             '\n'
+             '      **CPython implementation detail:** While a list is being '
+             'sorted,\n'
+             '      the effect of attempting to mutate, or even inspect, the '
+             'list is\n'
+             '      undefined.  The C implementation of Python makes the list '
+             'appear\n'
+             '      empty for the duration, and raises "ValueError" if it can '
+             'detect\n'
+             '      that the list has been mutated during a sort.\n'
+             '\n'
+             '\n'
+             'Tuples\n'
+             '======\n'
+             '\n'
+             'Tuples are immutable sequences, typically used to store '
+             'collections of\n'
+             'heterogeneous data (such as the 2-tuples produced by the '
+             '"enumerate()"\n'
+             'built-in). Tuples are also used for cases where an immutable '
+             'sequence\n'
+             'of homogeneous data is needed (such as allowing storage in a '
+             '"set" or\n'
+             '"dict" instance).\n'
+             '\n'
+             'class class tuple([iterable])\n'
+             '\n'
+             '   Tuples may be constructed in a number of ways:\n'
+             '\n'
+             '   * Using a pair of parentheses to denote the empty tuple: '
+             '"()"\n'
+             '\n'
+             '   * Using a trailing comma for a singleton tuple: "a," or '
+             '"(a,)"\n'
+             '\n'
+             '   * Separating items with commas: "a, b, c" or "(a, b, c)"\n'
+             '\n'
+             '   * Using the "tuple()" built-in: "tuple()" or '
+             '"tuple(iterable)"\n'
+             '\n'
+             '   The constructor builds a tuple whose items are the same and '
+             'in the\n'
+             "   same order as *iterable*'s items.  *iterable* may be either "
+             'a\n'
+             '   sequence, a container that supports iteration, or an '
+             'iterator\n'
+             '   object.  If *iterable* is already a tuple, it is returned\n'
+             '   unchanged. For example, "tuple(\'abc\')" returns "(\'a\', '
+             '\'b\', \'c\')"\n'
+             '   and "tuple( [1, 2, 3] )" returns "(1, 2, 3)". If no argument '
+             'is\n'
+             '   given, the constructor creates a new empty tuple, "()".\n'
+             '\n'
+             '   Note that it is actually the comma which makes a tuple, not '
+             'the\n'
+             '   parentheses. The parentheses are optional, except in the '
+             'empty\n'
+             '   tuple case, or when they are needed to avoid syntactic '
+             'ambiguity.\n'
+             '   For example, "f(a, b, c)" is a function call with three '
+             'arguments,\n'
+             '   while "f((a, b, c))" is a function call with a 3-tuple as the '
+             'sole\n'
+             '   argument.\n'
+             '\n'
+             '   Tuples implement all of the *common* sequence operations.\n'
+             '\n'
+             'For heterogeneous collections of data where access by name is '
+             'clearer\n'
+             'than access by index, "collections.namedtuple()" may be a more\n'
+             'appropriate choice than a simple tuple object.\n'
+             '\n'
+             '\n'
+             'Ranges\n'
+             '======\n'
+             '\n'
+             'The "range" type represents an immutable sequence of numbers and '
+             'is\n'
+             'commonly used for looping a specific number of times in "for" '
+             'loops.\n'
+             '\n'
+             'class class range(stop)\n'
+             'class class range(start, stop[, step])\n'
+             '\n'
+             '   The arguments to the range constructor must be integers '
+             '(either\n'
+             '   built-in "int" or any object that implements the "__index__"\n'
+             '   special method).  If the *step* argument is omitted, it '
+             'defaults to\n'
+             '   "1". If the *start* argument is omitted, it defaults to "0". '
+             'If\n'
+             '   *step* is zero, "ValueError" is raised.\n'
+             '\n'
+             '   For a positive *step*, the contents of a range "r" are '
+             'determined\n'
+             '   by the formula "r[i] = start + step*i" where "i >= 0" and '
+             '"r[i] <\n'
+             '   stop".\n'
+             '\n'
+             '   For a negative *step*, the contents of the range are still\n'
+             '   determined by the formula "r[i] = start + step*i", but the\n'
+             '   constraints are "i >= 0" and "r[i] > stop".\n'
+             '\n'
+             '   A range object will be empty if "r[0]" does not meet the '
+             'value\n'
+             '   constraint. Ranges do support negative indices, but these '
+             'are\n'
+             '   interpreted as indexing from the end of the sequence '
+             'determined by\n'
+             '   the positive indices.\n'
+             '\n'
+             '   Ranges containing absolute values larger than "sys.maxsize" '
+             'are\n'
+             '   permitted but some features (such as "len()") may raise\n'
+             '   "OverflowError".\n'
+             '\n'
+             '   Range examples:\n'
+             '\n'
+             '      >>> list(range(10))\n'
+             '      [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n'
+             '      >>> list(range(1, 11))\n'
+             '      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n'
+             '      >>> list(range(0, 30, 5))\n'
+             '      [0, 5, 10, 15, 20, 25]\n'
+             '      >>> list(range(0, 10, 3))\n'
+             '      [0, 3, 6, 9]\n'
+             '      >>> list(range(0, -10, -1))\n'
+             '      [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]\n'
+             '      >>> list(range(0))\n'
+             '      []\n'
+             '      >>> list(range(1, 0))\n'
+             '      []\n'
+             '\n'
+             '   Ranges implement all of the *common* sequence operations '
+             'except\n'
+             '   concatenation and repetition (due to the fact that range '
+             'objects\n'
+             '   can only represent sequences that follow a strict pattern '
+             'and\n'
+             '   repetition and concatenation will usually violate that '
+             'pattern).\n'
+             '\n'
+             '   start\n'
+             '\n'
+             '      The value of the *start* parameter (or "0" if the '
+             'parameter was\n'
+             '      not supplied)\n'
+             '\n'
+             '   stop\n'
+             '\n'
+             '      The value of the *stop* parameter\n'
+             '\n'
+             '   step\n'
+             '\n'
+             '      The value of the *step* parameter (or "1" if the parameter '
+             'was\n'
+             '      not supplied)\n'
+             '\n'
+             'The advantage of the "range" type over a regular "list" or '
+             '"tuple" is\n'
+             'that a "range" object will always take the same (small) amount '
+             'of\n'
+             'memory, no matter the size of the range it represents (as it '
+             'only\n'
+             'stores the "start", "stop" and "step" values, calculating '
+             'individual\n'
+             'items and subranges as needed).\n'
+             '\n'
+             'Range objects implement the "collections.abc.Sequence" ABC, and\n'
+             'provide features such as containment tests, element index '
+             'lookup,\n'
+             'slicing and support for negative indices (see *Sequence Types '
+             '---\n'
+             'list, tuple, range*):\n'
+             '\n'
+             '>>> r = range(0, 20, 2)\n'
+             '>>> r\n'
+             'range(0, 20, 2)\n'
+             '>>> 11 in r\n'
+             'False\n'
+             '>>> 10 in r\n'
+             'True\n'
+             '>>> r.index(10)\n'
+             '5\n'
+             '>>> r[5]\n'
+             '10\n'
+             '>>> r[:5]\n'
+             'range(0, 10, 2)\n'
+             '>>> r[-1]\n'
+             '18\n'
+             '\n'
+             'Testing range objects for equality with "==" and "!=" compares '
+             'them as\n'
+             'sequences.  That is, two range objects are considered equal if '
+             'they\n'
+             'represent the same sequence of values.  (Note that two range '
+             'objects\n'
+             'that compare equal might have different "start", "stop" and '
+             '"step"\n'
+             'attributes, for example "range(0) == range(2, 1, 3)" or '
+             '"range(0, 3,\n'
+             '2) == range(0, 4, 2)".)\n'
+             '\n'
+             'Changed in version 3.2: Implement the Sequence ABC. Support '
+             'slicing\n'
+             'and negative indices. Test "int" objects for membership in '
+             'constant\n'
+             'time instead of iterating through all items.\n'
+             '\n'
+             "Changed in version 3.3: Define '==' and '!=' to compare range "
+             'objects\n'
+             'based on the sequence of values they define (instead of '
+             'comparing\n'
+             'based on object identity).\n'
+             '\n'
+             'New in version 3.3: The "start", "stop" and "step" attributes.\n',
+ 'typesseq-mutable': '\n'
+                     'Mutable Sequence Types\n'
+                     '**********************\n'
+                     '\n'
+                     'The operations in the following table are defined on '
+                     'mutable sequence\n'
+                     'types. The "collections.abc.MutableSequence" ABC is '
+                     'provided to make\n'
+                     'it easier to correctly implement these operations on '
+                     'custom sequence\n'
+                     'types.\n'
+                     '\n'
+                     'In the table *s* is an instance of a mutable sequence '
+                     'type, *t* is any\n'
+                     'iterable object and *x* is an arbitrary object that '
+                     'meets any type and\n'
+                     'value restrictions imposed by *s* (for example, '
+                     '"bytearray" only\n'
+                     'accepts integers that meet the value restriction "0 <= x '
+                     '<= 255").\n'
+                     '\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '| Operation                      | '
+                     'Result                           | Notes                 '
+                     '|\n'
+                     '+================================+==================================+=======================+\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '| "s[i:j:k] = t"                 | the elements of '
+                     '"s[i:j:k]" are   | (1)                   |\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '| "s.clear()"                    | removes all items '
+                     'from "s" (same | (5)                   |\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '| "s.copy()"                     | creates a shallow '
+                     'copy of "s"    | (5)                   |\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '| "s *= n"                       | updates *s* with its '
+                     'contents    | (6)                   |\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '| "s.pop([i])"                   | retrieves the item at '
+                     '*i* and    | (2)                   |\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '| "s.remove(x)"                  | remove the first item '
+                     'from *s*   | (3)                   |\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '| "s.reverse()"                  | reverses the items of '
+                     '*s* in     | (4)                   |\n'
+                     '+--------------------------------+----------------------------------+-----------------------+\n'
+                     '\n'
+                     'Notes:\n'
+                     '\n'
+                     '1. *t* must have the same length as the slice it is '
+                     'replacing.\n'
+                     '\n'
+                     '2. The optional argument *i* defaults to "-1", so that '
+                     'by default\n'
+                     '   the last item is removed and returned.\n'
+                     '\n'
+                     '3. "remove" raises "ValueError" when *x* is not found in '
+                     '*s*.\n'
+                     '\n'
+                     '4. The "reverse()" method modifies the sequence in place '
+                     'for\n'
+                     '   economy of space when reversing a large sequence.  To '
+                     'remind users\n'
+                     '   that it operates by side effect, it does not return '
+                     'the reversed\n'
+                     '   sequence.\n'
+                     '\n'
+                     '5. "clear()" and "copy()" are included for consistency '
+                     'with the\n'
+                     "   interfaces of mutable containers that don't support "
+                     'slicing\n'
+                     '   operations (such as "dict" and "set")\n'
+                     '\n'
+                     '   New in version 3.3: "clear()" and "copy()" methods.\n'
+                     '\n'
+                     '6. The value *n* is an integer, or an object '
+                     'implementing\n'
+                     '   "__index__()".  Zero and negative values of *n* clear '
+                     'the sequence.\n'
+                     '   Items in the sequence are not copied; they are '
+                     'referenced multiple\n'
+                     '   times, as explained for "s * n" under *Common '
+                     'Sequence Operations*.\n',
+ 'unary': '\n'
+          'Unary arithmetic and bitwise operations\n'
+          '***************************************\n'
+          '\n'
+          'All unary arithmetic and bitwise operations have the same '
+          'priority:\n'
+          '\n'
+          '   u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n'
+          '\n'
+          'The unary "-" (minus) operator yields the negation of its numeric\n'
+          'argument.\n'
+          '\n'
+          'The unary "+" (plus) operator yields its numeric argument '
+          'unchanged.\n'
+          '\n'
+          'The unary "~" (invert) operator yields the bitwise inversion of '
+          'its\n'
+          'integer argument.  The bitwise inversion of "x" is defined as\n'
+          '"-(x+1)".  It only applies to integral numbers.\n'
+          '\n'
+          'In all three cases, if the argument does not have the proper type, '
+          'a\n'
+          '"TypeError" exception is raised.\n',
+ 'while': '\n'
+          'The "while" statement\n'
+          '*********************\n'
+          '\n'
+          'The "while" statement is used for repeated execution as long as an\n'
+          'expression is true:\n'
+          '\n'
+          '   while_stmt ::= "while" expression ":" suite\n'
+          '                  ["else" ":" suite]\n'
+          '\n'
+          'This repeatedly tests the expression and, if it is true, executes '
+          'the\n'
+          'first suite; if the expression is false (which may be the first '
+          'time\n'
+          'it is tested) the suite of the "else" clause, if present, is '
+          'executed\n'
+          'and the loop terminates.\n'
+          '\n'
+          'A "break" statement executed in the first suite terminates the '
+          'loop\n'
+          'without executing the "else" clause\'s suite.  A "continue" '
+          'statement\n'
+          'executed in the first suite skips the rest of the suite and goes '
+          'back\n'
+          'to testing the expression.\n',
+ 'with': '\n'
+         'The "with" statement\n'
+         '********************\n'
+         '\n'
+         'The "with" statement is used to wrap the execution of a block with\n'
+         'methods defined by a context manager (see section *With Statement\n'
+         'Context Managers*). This allows common "try"..."except"..."finally"\n'
+         'usage patterns to be encapsulated for convenient reuse.\n'
+         '\n'
+         '   with_stmt ::= "with" with_item ("," with_item)* ":" suite\n'
+         '   with_item ::= expression ["as" target]\n'
+         '\n'
+         'The execution of the "with" statement with one "item" proceeds as\n'
+         'follows:\n'
+         '\n'
+         '1. The context expression (the expression given in the "with_item")\n'
+         '   is evaluated to obtain a context manager.\n'
+         '\n'
+         '2. The context manager\'s "__exit__()" is loaded for later use.\n'
+         '\n'
+         '3. The context manager\'s "__enter__()" method is invoked.\n'
+         '\n'
+         '4. If a target was included in the "with" statement, the return\n'
+         '   value from "__enter__()" is assigned to it.\n'
+         '\n'
+         '   Note: The "with" statement guarantees that if the "__enter__()"\n'
+         '     method returns without an error, then "__exit__()" will always '
+         'be\n'
+         '     called. Thus, if an error occurs during the assignment to the\n'
+         '     target list, it will be treated the same as an error occurring\n'
+         '     within the suite would be. See step 6 below.\n'
+         '\n'
+         '5. The suite is executed.\n'
+         '\n'
+         '6. The context manager\'s "__exit__()" method is invoked.  If an\n'
+         '   exception caused the suite to be exited, its type, value, and\n'
+         '   traceback are passed as arguments to "__exit__()". Otherwise, '
+         'three\n'
+         '   "None" arguments are supplied.\n'
+         '\n'
+         '   If the suite was exited due to an exception, and the return '
+         'value\n'
+         '   from the "__exit__()" method was false, the exception is '
+         'reraised.\n'
+         '   If the return value was true, the exception is suppressed, and\n'
+         '   execution continues with the statement following the "with"\n'
+         '   statement.\n'
+         '\n'
+         '   If the suite was exited for any reason other than an exception, '
+         'the\n'
+         '   return value from "__exit__()" is ignored, and execution '
+         'proceeds\n'
+         '   at the normal location for the kind of exit that was taken.\n'
+         '\n'
+         'With more than one item, the context managers are processed as if\n'
+         'multiple "with" statements were nested:\n'
+         '\n'
+         '   with A() as a, B() as b:\n'
+         '       suite\n'
+         '\n'
+         'is equivalent to\n'
+         '\n'
+         '   with A() as a:\n'
+         '       with B() as b:\n'
+         '           suite\n'
+         '\n'
+         'Changed in version 3.1: Support for multiple context expressions.\n'
+         '\n'
+         'See also: **PEP 343** - The "with" statement\n'
+         '\n'
+         '     The specification, background, and examples for the Python '
+         '"with"\n'
+         '     statement.\n',
+ 'yield': '\n'
+          'The "yield" statement\n'
+          '*********************\n'
+          '\n'
+          '   yield_stmt ::= yield_expression\n'
+          '\n'
+          'A "yield" statement is semantically equivalent to a *yield\n'
+          'expression*. The yield statement can be used to omit the '
+          'parentheses\n'
+          'that would otherwise be required in the equivalent yield '
+          'expression\n'
+          'statement. For example, the yield statements\n'
+          '\n'
+          '   yield <expr>\n'
+          '   yield from <expr>\n'
+          '\n'
+          'are equivalent to the yield expression statements\n'
+          '\n'
+          '   (yield <expr>)\n'
+          '   (yield from <expr>)\n'
+          '\n'
+          'Yield expressions and statements are only used when defining a\n'
+          '*generator* function, and are only used in the body of the '
+          'generator\n'
+          'function.  Using yield in a function definition is sufficient to '
+          'cause\n'
+          'that definition to create a generator function instead of a normal\n'
+          'function.\n'
+          '\n'
+          'For full details of "yield" semantics, refer to the *Yield\n'
+          'expressions* section.\n'}
diff --git a/Misc/NEWS.d/3.5.8rc1.rst b/Misc/NEWS.d/3.5.8rc1.rst
new file mode 100644
index 000000000000..e19b18e120b1
--- /dev/null
+++ b/Misc/NEWS.d/3.5.8rc1.rst
@@ -0,0 +1,80 @@
+.. bpo: 37461
+.. date: 2019-07-16-08-11-00
+.. nonce: 1Ahz7O
+.. release date: 2019-09-09
+.. section: Security
+
+Fix an infinite loop when parsing specially crafted email headers. Patch by
+Abhilash Raj.
+
+..
+
+.. bpo: 35907
+.. date: 2019-05-21-23-20-18
+.. nonce: NC_zNK
+.. section: Security
+
+CVE-2019-9948: Avoid file reading by disallowing ``local-file://`` and
+``local_file://`` URL schemes in ``URLopener().open()`` and
+``URLopener().retrieve()`` of :mod:`urllib.request`.
+
+..
+
+.. bpo: 34155
+.. date: 2019-05-04-13-33-37
+.. nonce: MJll68
+.. section: Security
+
+Fix parsing of invalid email addresses with more than one ``@`` (e.g.
+a at b@c.com.) to not return the part before 2nd ``@`` as valid email address.
+Patch by maxking & jpic.
+
+..
+
+.. bpo: 36742
+.. date: 2019-04-29-15-34-59
+.. nonce: QCUY0i
+.. section: Security
+
+Fixes mishandling of pre-normalization characters in urlsplit().
+
+..
+
+.. bpo: 30458
+.. date: 2019-04-10-08-53-30
+.. nonce: 51E-DA
+.. section: Security
+
+Address CVE-2019-9740 by disallowing URL paths with embedded whitespace or
+control characters through into the underlying http client request.  Such
+potentially malicious header injection URLs now cause an
+http.client.InvalidURL exception to be raised.
+
+..
+
+.. bpo: 36816
+.. date: 2019-05-08-15-55-46
+.. nonce: WBKRGZ
+.. section: Tests
+
+Update Lib/test/selfsigned_pythontestdotnet.pem to match
+self-signed.pythontest.net's new TLS certificate.
+
+..
+
+.. bpo: 36576
+.. date: 2019-04-05-10-34-29
+.. nonce: 7Cp2kK
+.. section: Tests
+
+Skip test_ssl and test_asyncio tests failing with OpenSSL 1.1.1.
+
+..
+
+.. bpo: 36478
+.. date: 2019-03-29-14-29-06
+.. nonce: hzyneF
+.. section: Build
+
+Fix compatibility with ISO C89 needed by "gnu89" standard of GCC 4.8: use
+C89 for loops in backported pickle patch. Patch by Anthony Sottile.
diff --git a/Misc/NEWS.d/next/Build/2019-03-29-14-29-06.bpo-36478.hzyneF.rst b/Misc/NEWS.d/next/Build/2019-03-29-14-29-06.bpo-36478.hzyneF.rst
deleted file mode 100644
index 942ad9a31656..000000000000
--- a/Misc/NEWS.d/next/Build/2019-03-29-14-29-06.bpo-36478.hzyneF.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix compatibility with ISO C89 needed by "gnu89" standard of GCC 4.8: use C89 for loops in backported pickle patch. Patch by Anthony Sottile.
diff --git a/Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst b/Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst
deleted file mode 100644
index ed8027fb4d64..000000000000
--- a/Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst
+++ /dev/null
@@ -1 +0,0 @@
-Address CVE-2019-9740 by disallowing URL paths with embedded whitespace or control characters through into the underlying http client request.  Such potentially malicious header injection URLs now cause an http.client.InvalidURL exception to be raised.
diff --git a/Misc/NEWS.d/next/Security/2019-04-29-15-34-59.bpo-36742.QCUY0i.rst b/Misc/NEWS.d/next/Security/2019-04-29-15-34-59.bpo-36742.QCUY0i.rst
deleted file mode 100644
index d729ed2f3cd5..000000000000
--- a/Misc/NEWS.d/next/Security/2019-04-29-15-34-59.bpo-36742.QCUY0i.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fixes mishandling of pre-normalization characters in urlsplit().
diff --git a/Misc/NEWS.d/next/Security/2019-05-04-13-33-37.bpo-34155.MJll68.rst b/Misc/NEWS.d/next/Security/2019-05-04-13-33-37.bpo-34155.MJll68.rst
deleted file mode 100644
index 50292e29ed1d..000000000000
--- a/Misc/NEWS.d/next/Security/2019-05-04-13-33-37.bpo-34155.MJll68.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix parsing of invalid email addresses with more than one ``@`` (e.g. a at b@c.com.) to not return the part before 2nd ``@`` as valid email address. Patch by maxking & jpic.
diff --git a/Misc/NEWS.d/next/Security/2019-05-21-23-20-18.bpo-35907.NC_zNK.rst b/Misc/NEWS.d/next/Security/2019-05-21-23-20-18.bpo-35907.NC_zNK.rst
deleted file mode 100644
index 37b567a5b6f9..000000000000
--- a/Misc/NEWS.d/next/Security/2019-05-21-23-20-18.bpo-35907.NC_zNK.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-CVE-2019-9948: Avoid file reading by disallowing ``local-file://`` and
-``local_file://`` URL schemes in ``URLopener().open()`` and
-``URLopener().retrieve()`` of :mod:`urllib.request`.
diff --git a/Misc/NEWS.d/next/Security/2019-07-16-08-11-00.bpo-37461.1Ahz7O.rst b/Misc/NEWS.d/next/Security/2019-07-16-08-11-00.bpo-37461.1Ahz7O.rst
deleted file mode 100644
index 9d47578c6277..000000000000
--- a/Misc/NEWS.d/next/Security/2019-07-16-08-11-00.bpo-37461.1Ahz7O.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix an infinite loop when parsing specially crafted email headers. Patch by
-Abhilash Raj.
diff --git a/Misc/NEWS.d/next/Tests/2019-04-05-10-34-29.bpo-36576.7Cp2kK.rst b/Misc/NEWS.d/next/Tests/2019-04-05-10-34-29.bpo-36576.7Cp2kK.rst
deleted file mode 100644
index 4d15bdf42796..000000000000
--- a/Misc/NEWS.d/next/Tests/2019-04-05-10-34-29.bpo-36576.7Cp2kK.rst
+++ /dev/null
@@ -1 +0,0 @@
-Skip test_ssl and test_asyncio tests failing with OpenSSL 1.1.1.
diff --git a/Misc/NEWS.d/next/Tests/2019-05-08-15-55-46.bpo-36816.WBKRGZ.rst b/Misc/NEWS.d/next/Tests/2019-05-08-15-55-46.bpo-36816.WBKRGZ.rst
deleted file mode 100644
index 420dfe832366..000000000000
--- a/Misc/NEWS.d/next/Tests/2019-05-08-15-55-46.bpo-36816.WBKRGZ.rst
+++ /dev/null
@@ -1 +0,0 @@
-Update Lib/test/selfsigned_pythontestdotnet.pem to match self-signed.pythontest.net's new TLS certificate.
\ No newline at end of file


From webhook-mailer at python.org  Mon Sep  9 10:50:40 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 14:50:40 -0000
Subject: [Python-checkins] bpo-37876: Tests for ROT-13 codec (GH-15314)
Message-ID: <mailman.349.1568040641.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b3b48c81f09d1472010937f1331c5a208a2a2d48
commit: b3b48c81f09d1472010937f1331c5a208a2a2d48
branch: master
author: Zeth <theology at gmail.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-09T07:50:36-07:00
summary:

bpo-37876: Tests for ROT-13 codec (GH-15314)

The Rot-13 codec is for educational use but does not have unit tests,
dragging down test coverage. This adds a few very simple tests.

files:
A Misc/NEWS.d/next/Tests/2019-08-16-16-15-14.bpo-37876.m3k1w3.rst
M Lib/test/test_codecs.py

diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index ba7f4847468a..b37525bf6604 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -3343,5 +3343,42 @@ def test_decode_unsupported_error_handler(self):
         self.assertEqual(str(cm.exception), 'unsupported error handler')
 
 
+class Rot13Test(unittest.TestCase):
+    """Test the educational ROT-13 codec."""
+    def test_encode(self):
+        ciphertext = codecs.encode("Caesar liked ciphers", 'rot-13')
+        self.assertEqual(ciphertext, 'Pnrfne yvxrq pvcuref')
+
+    def test_decode(self):
+        plaintext = codecs.decode('Rg gh, Oehgr?', 'rot-13')
+        self.assertEqual(plaintext, 'Et tu, Brute?')
+
+    def test_incremental_encode(self):
+        encoder = codecs.getincrementalencoder('rot-13')()
+        ciphertext = encoder.encode('ABBA nag Cheryl Baker')
+        self.assertEqual(ciphertext, 'NOON ant Purely Onxre')
+
+    def test_incremental_decode(self):
+        decoder = codecs.getincrementaldecoder('rot-13')()
+        plaintext = decoder.decode('terra Ares envy tha')
+        self.assertEqual(plaintext, 'green Nerf rail gun')
+
+
+class Rot13UtilTest(unittest.TestCase):
+    """Test the ROT-13 codec via rot13 function,
+    i.e. the user has done something like:
+    $ echo "Hello World" | python -m encodings.rot_13
+    """
+    def test_rot13_func(self):
+        infile = io.StringIO('Gb or, be abg gb or, gung vf gur dhrfgvba')
+        outfile = io.StringIO()
+        encodings.rot_13.rot13(infile, outfile)
+        outfile.seek(0)
+        plain_text = outfile.read()
+        self.assertEqual(
+            plain_text,
+            'To be, or not to be, that is the question')
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Tests/2019-08-16-16-15-14.bpo-37876.m3k1w3.rst b/Misc/NEWS.d/next/Tests/2019-08-16-16-15-14.bpo-37876.m3k1w3.rst
new file mode 100644
index 000000000000..45702fc63510
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-08-16-16-15-14.bpo-37876.m3k1w3.rst
@@ -0,0 +1 @@
+Add tests for ROT-13 codec.
\ No newline at end of file


From webhook-mailer at python.org  Mon Sep  9 10:56:04 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 09 Sep 2019 14:56:04 -0000
Subject: [Python-checkins] bpo-38006: Avoid closure in
 weakref.WeakValueDictionary (GH-15641)
Message-ID: <mailman.350.1568040964.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a2af05a0d3f0da06b8d432f52efa3ecf29038532
commit: a2af05a0d3f0da06b8d432f52efa3ecf29038532
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T16:55:58+02:00
summary:

bpo-38006: Avoid closure in weakref.WeakValueDictionary (GH-15641)

weakref.WeakValueDictionary defines a local remove() function used as
callback for weak references. This function was created with a
closure.  Modify the implementation to avoid the closure.

files:
A Misc/NEWS.d/next/Library/2019-09-02-13-37-27.bpo-38006.Y7vA0Q.rst
M Lib/test/test_weakref.py
M Lib/weakref.py

diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py
index 41f78e7e8383..e14df9ea0894 100644
--- a/Lib/test/test_weakref.py
+++ b/Lib/test/test_weakref.py
@@ -1792,6 +1792,11 @@ def test_threaded_weak_value_dict_deepcopy(self):
         # copying should not result in a crash.
         self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, True)
 
+    @support.cpython_only
+    def test_remove_closure(self):
+        d = weakref.WeakValueDictionary()
+        self.assertIsNone(d._remove.__closure__)
+
 
 from test import mapping_tests
 
diff --git a/Lib/weakref.py b/Lib/weakref.py
index 560deee96c92..d17b3ed38eac 100644
--- a/Lib/weakref.py
+++ b/Lib/weakref.py
@@ -108,12 +108,12 @@ def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref):
                 else:
                     # Atomic removal is necessary since this function
                     # can be called asynchronously by the GC
-                    _atomic_removal(d, wr.key)
+                    _atomic_removal(self.data, wr.key)
         self._remove = remove
         # A list of keys to be removed
         self._pending_removals = []
         self._iterating = set()
-        self.data = d = {}
+        self.data = {}
         self.update(other, **kw)
 
     def _commit_removals(self):
diff --git a/Misc/NEWS.d/next/Library/2019-09-02-13-37-27.bpo-38006.Y7vA0Q.rst b/Misc/NEWS.d/next/Library/2019-09-02-13-37-27.bpo-38006.Y7vA0Q.rst
new file mode 100644
index 000000000000..ff064ad3f1a4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-02-13-37-27.bpo-38006.Y7vA0Q.rst
@@ -0,0 +1,3 @@
+weakref.WeakValueDictionary defines a local remove() function used as
+callback for weak references. This function was created with a closure.
+Modify the implementation to avoid the closure.


From webhook-mailer at python.org  Mon Sep  9 11:01:18 2019
From: webhook-mailer at python.org (Michael Foord)
Date: Mon, 09 Sep 2019 15:01:18 -0000
Subject: [Python-checkins] bpo-34596: Fallback to a default reason when
 @unittest.skip is uncalled (GH-9082) (#15781)
Message-ID: <mailman.351.1568041280.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3bd4bed78a0b068e28bcf2242d33aed227c2532c
commit: 3bd4bed78a0b068e28bcf2242d33aed227c2532c
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Michael Foord <voidspace at users.noreply.github.com>
date: 2019-09-09T17:01:14+02:00
summary:

bpo-34596: Fallback to a default reason when @unittest.skip is uncalled (GH-9082) (#15781)

* bpo-34596: Fallback to a default reason when @unittest.skip is uncalled

* Change default reason to empty string

* Fix rst formatting of NEWS entry
(cherry picked from commit d5fd75c53fad7049fc640c9a6162d35f0c5bea03)

Co-authored-by: Naitree Zhu <Naitreey at gmail.com>

files:
A Misc/NEWS.d/next/Tests/2018-09-07-01-18-27.bpo-34596.r2-EGd.rst
M Lib/unittest/case.py
M Lib/unittest/test/test_skipping.py

diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py
index b363c6351007..b639c64d02a7 100644
--- a/Lib/unittest/case.py
+++ b/Lib/unittest/case.py
@@ -10,6 +10,7 @@
 import collections
 import contextlib
 import traceback
+import types
 
 from . import result
 from .util import (strclass, safe_repr, _count_diff_all_purpose,
@@ -122,6 +123,10 @@ def skip_wrapper(*args, **kwargs):
         test_item.__unittest_skip__ = True
         test_item.__unittest_skip_why__ = reason
         return test_item
+    if isinstance(reason, types.FunctionType):
+        test_item = reason
+        reason = ''
+        return decorator(test_item)
     return decorator
 
 def skipIf(condition, reason):
diff --git a/Lib/unittest/test/test_skipping.py b/Lib/unittest/test/test_skipping.py
index 71f7b70e479d..1c178a95f750 100644
--- a/Lib/unittest/test/test_skipping.py
+++ b/Lib/unittest/test/test_skipping.py
@@ -255,6 +255,17 @@ def test_1(self):
         suite.run(result)
         self.assertEqual(result.skipped, [(test, "testing")])
 
+    def test_skip_without_reason(self):
+        class Foo(unittest.TestCase):
+            @unittest.skip
+            def test_1(self):
+                pass
+
+        result = unittest.TestResult()
+        test = Foo("test_1")
+        suite = unittest.TestSuite([test])
+        suite.run(result)
+        self.assertEqual(result.skipped, [(test, "")])
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Tests/2018-09-07-01-18-27.bpo-34596.r2-EGd.rst b/Misc/NEWS.d/next/Tests/2018-09-07-01-18-27.bpo-34596.r2-EGd.rst
new file mode 100644
index 000000000000..156e8aa8945d
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2018-09-07-01-18-27.bpo-34596.r2-EGd.rst
@@ -0,0 +1,2 @@
+Fallback to a default reason when :func:`unittest.skip` is uncalled. Patch by
+Naitree Zhu.


From webhook-mailer at python.org  Mon Sep  9 11:17:55 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 15:17:55 -0000
Subject: [Python-checkins] bpo-20490: Improve circular import error message
 (GH-15308)
Message-ID: <mailman.352.1568042277.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/65366bc8bdc4716ebc361e622590b45a6e5aef07
commit: 65366bc8bdc4716ebc361e622590b45a6e5aef07
branch: master
author: Anthony Sottile <asottile at umich.edu>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-09T08:17:50-07:00
summary:

bpo-20490: Improve circular import error message (GH-15308)

files:
A Lib/test/test_import/data/circular_imports/from_cycle1.py
A Lib/test/test_import/data/circular_imports/from_cycle2.py
A Misc/NEWS.d/next/Core and Builtins/2019-08-15-12-48-36.bpo-20490.-hXeEn.rst
M Lib/test/test_import/__init__.py
M Python/ceval.c

diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py
index 88746b4315f9..f167c84f1b28 100644
--- a/Lib/test/test_import/__init__.py
+++ b/Lib/test/test_import/__init__.py
@@ -1324,6 +1324,16 @@ def test_crossreference2(self):
         self.assertIn('partially initialized module', errmsg)
         self.assertIn('circular import', errmsg)
 
+    def test_circular_from_import(self):
+        with self.assertRaises(ImportError) as cm:
+            import test.test_import.data.circular_imports.from_cycle1
+        self.assertIn(
+            "cannot import name 'b' from partially initialized module "
+            "'test.test_import.data.circular_imports.from_cycle1' "
+            "(most likely due to a circular import)",
+            str(cm.exception),
+        )
+
 
 if __name__ == '__main__':
     # Test needs to be a package, so we can do relative imports.
diff --git a/Lib/test/test_import/data/circular_imports/from_cycle1.py b/Lib/test/test_import/data/circular_imports/from_cycle1.py
new file mode 100644
index 000000000000..aacfd5f46fca
--- /dev/null
+++ b/Lib/test/test_import/data/circular_imports/from_cycle1.py
@@ -0,0 +1,2 @@
+from .from_cycle2 import a
+b = 1
diff --git a/Lib/test/test_import/data/circular_imports/from_cycle2.py b/Lib/test/test_import/data/circular_imports/from_cycle2.py
new file mode 100644
index 000000000000..62a66e1cfd02
--- /dev/null
+++ b/Lib/test/test_import/data/circular_imports/from_cycle2.py
@@ -0,0 +1,2 @@
+from .from_cycle1 import b
+a = 1
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-15-12-48-36.bpo-20490.-hXeEn.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-15-12-48-36.bpo-20490.-hXeEn.rst
new file mode 100644
index 000000000000..dfee480e1aa7
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-15-12-48-36.bpo-20490.-hXeEn.rst	
@@ -0,0 +1,2 @@
+Improve import error message for partially initialized module on circular
+``from`` imports - by Anthony Sottile.
diff --git a/Python/ceval.c b/Python/ceval.c
index f9e03b3097f0..d280d7977607 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -5233,10 +5233,17 @@ import_from(PyThreadState *tstate, PyObject *v, PyObject *name)
         PyErr_SetImportError(errmsg, pkgname, NULL);
     }
     else {
-        errmsg = PyUnicode_FromFormat(
-            "cannot import name %R from %R (%S)",
-            name, pkgname_or_unknown, pkgpath
-        );
+        _Py_IDENTIFIER(__spec__);
+        PyObject *spec = _PyObject_GetAttrId(v, &PyId___spec__);
+        Py_XINCREF(spec);
+        const char *fmt =
+            _PyModuleSpec_IsInitializing(spec) ?
+            "cannot import name %R from partially initialized module %R "
+            "(most likely due to a circular import) (%S)" :
+            "cannot import name %R from %R (%S)";
+        Py_XDECREF(spec);
+
+        errmsg = PyUnicode_FromFormat(fmt, name, pkgname_or_unknown, pkgpath);
         /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */
         PyErr_SetImportError(errmsg, pkgname, pkgpath);
     }


From webhook-mailer at python.org  Mon Sep  9 11:19:17 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 15:19:17 -0000
Subject: [Python-checkins] bpo-38059: Using sys.exit() over exit() in
 inspect.py (GH-15666)
Message-ID: <mailman.353.1568042357.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b7bf632d4ebacfeffc8770a7848c690fa2a90a62
commit: b7bf632d4ebacfeffc8770a7848c690fa2a90a62
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T08:19:13-07:00
summary:

bpo-38059: Using sys.exit() over exit() in inspect.py (GH-15666)


Constants added by the site module like exit() "should not be used in programs"
(cherry picked from commit e3c59a75279b0df4e7553d6f0031e202de434e43)

Co-authored-by: Alan Yee <alanyee at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Library/2019-09-08-11-36-50.bpo-38059.8SA6co.rst
M Lib/inspect.py

diff --git a/Lib/inspect.py b/Lib/inspect.py
index 4d4f33dcc57b..7f0517207d15 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -3108,7 +3108,7 @@ def _main():
                                                     type(exc).__name__,
                                                     exc)
         print(msg, file=sys.stderr)
-        exit(2)
+        sys.exit(2)
 
     if has_attrs:
         parts = attrs.split(".")
@@ -3118,7 +3118,7 @@ def _main():
 
     if module.__name__ in sys.builtin_module_names:
         print("Can't get info for builtin modules.", file=sys.stderr)
-        exit(1)
+        sys.exit(1)
 
     if args.details:
         print('Target: {}'.format(target))
diff --git a/Misc/NEWS.d/next/Library/2019-09-08-11-36-50.bpo-38059.8SA6co.rst b/Misc/NEWS.d/next/Library/2019-09-08-11-36-50.bpo-38059.8SA6co.rst
new file mode 100644
index 000000000000..001952ae1261
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-08-11-36-50.bpo-38059.8SA6co.rst
@@ -0,0 +1 @@
+inspect.py now uses sys.exit() instead of exit()


From webhook-mailer at python.org  Mon Sep  9 11:20:40 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 15:20:40 -0000
Subject: [Python-checkins] bpo-38059: Using sys.exit() over exit() in
 inspect.py (GH-15666)
Message-ID: <mailman.354.1568042442.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0d4396c04cba5ac2b66fdaa23c01db84b1b54227
commit: 0d4396c04cba5ac2b66fdaa23c01db84b1b54227
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T08:20:36-07:00
summary:

bpo-38059: Using sys.exit() over exit() in inspect.py (GH-15666)


Constants added by the site module like exit() "should not be used in programs"
(cherry picked from commit e3c59a75279b0df4e7553d6f0031e202de434e43)

Co-authored-by: Alan Yee <alanyee at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Library/2019-09-08-11-36-50.bpo-38059.8SA6co.rst
M Lib/inspect.py

diff --git a/Lib/inspect.py b/Lib/inspect.py
index a616f2d49b7d..0a57749ccdd4 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -3118,7 +3118,7 @@ def _main():
                                                     type(exc).__name__,
                                                     exc)
         print(msg, file=sys.stderr)
-        exit(2)
+        sys.exit(2)
 
     if has_attrs:
         parts = attrs.split(".")
@@ -3128,7 +3128,7 @@ def _main():
 
     if module.__name__ in sys.builtin_module_names:
         print("Can't get info for builtin modules.", file=sys.stderr)
-        exit(1)
+        sys.exit(1)
 
     if args.details:
         print('Target: {}'.format(target))
diff --git a/Misc/NEWS.d/next/Library/2019-09-08-11-36-50.bpo-38059.8SA6co.rst b/Misc/NEWS.d/next/Library/2019-09-08-11-36-50.bpo-38059.8SA6co.rst
new file mode 100644
index 000000000000..001952ae1261
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-08-11-36-50.bpo-38059.8SA6co.rst
@@ -0,0 +1 @@
+inspect.py now uses sys.exit() instead of exit()


From webhook-mailer at python.org  Mon Sep  9 11:20:44 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Mon, 09 Sep 2019 15:20:44 -0000
Subject: [Python-checkins] bpo-37758: Cut always-constant conditionals on
 sys.maxunicode. (GH-15302)
Message-ID: <mailman.355.1568042446.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3cbc23aa229bc5ec04845053df78eae5f54e0497
commit: 3cbc23aa229bc5ec04845053df78eae5f54e0497
branch: master
author: Greg Price <gnprice at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-09T08:20:40-07:00
summary:

bpo-37758: Cut always-constant conditionals on sys.maxunicode. (GH-15302)

Since PEP 393 in Python 3.3, this value is always 0x10ffff, the
maximum codepoint in Unicode; there's no longer such a thing as a
UCS-2 build of Python, which couldn't properly represent some
characters.

There are a couple of spots left where we still condition on the value
of this constant.  Take them out.

files:
M Lib/test/test_bigaddrspace.py
M Tools/unicode/mkstringprep.py

diff --git a/Lib/test/test_bigaddrspace.py b/Lib/test/test_bigaddrspace.py
index b639f68c2347..aa1f8ca75a98 100644
--- a/Lib/test/test_bigaddrspace.py
+++ b/Lib/test/test_bigaddrspace.py
@@ -55,7 +55,7 @@ def test_repeat(self):
 
 class StrTest(unittest.TestCase):
 
-    unicodesize = 2 if sys.maxunicode < 65536 else 4
+    unicodesize = 4
 
     @bigaddrspacetest
     def test_concat(self):
diff --git a/Tools/unicode/mkstringprep.py b/Tools/unicode/mkstringprep.py
index ead020c3a7a4..427188389a3b 100644
--- a/Tools/unicode/mkstringprep.py
+++ b/Tools/unicode/mkstringprep.py
@@ -1,9 +1,6 @@
-import re, sys
+import re
 from unicodedata import ucd_3_2_0 as unicodedata
 
-if sys.maxunicode == 65535:
-    raise RuntimeError("need UCS-4 Python")
-
 def gen_category(cats):
     for i in range(0, 0x110000):
         if unicodedata.category(chr(i)) in cats:


From webhook-mailer at python.org  Mon Sep  9 11:45:04 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 09 Sep 2019 15:45:04 -0000
Subject: [Python-checkins] bpo-38070: visit_decref() calls
 _PyObject_IsFreed() (GH-15782)
Message-ID: <mailman.356.1568043906.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d91d4de31745fc1ed4c7e6c208917827c9c472b6
commit: d91d4de31745fc1ed4c7e6c208917827c9c472b6
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T17:44:59+02:00
summary:

bpo-38070: visit_decref() calls _PyObject_IsFreed() (GH-15782)

In debug mode, visit_decref() now calls _PyObject_IsFreed() to ensure
that the object is not freed. If it's freed, the program fails with
an assertion error and Python dumps informations about the freed
object.

files:
M Modules/gcmodule.c

diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
index 134f6d168c96..f6e841d52ff9 100644
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -376,6 +376,8 @@ static int
 visit_decref(PyObject *op, void *data)
 {
     assert(op != NULL);
+    _PyObject_ASSERT(op, !_PyObject_IsFreed(op));
+
     if (PyObject_IS_GC(op)) {
         PyGC_Head *gc = AS_GC(op);
         /* We're only interested in gc_refs for objects in the


From webhook-mailer at python.org  Mon Sep  9 11:48:36 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Mon, 09 Sep 2019 15:48:36 -0000
Subject: [Python-checkins] bpo-36279: Ensure os.wait3() rusage is
 initialized (GH-15111)
Message-ID: <mailman.357.1568044116.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/682107cf458578ee6bd92b7cc6862113034a4fad
commit: 682107cf458578ee6bd92b7cc6862113034a4fad
branch: master
author: Zackery Spytz <zspytz at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-09T08:48:32-07:00
summary:

bpo-36279: Ensure os.wait3() rusage is initialized (GH-15111)

Co-Authored-By: David Wilson <dw at botanicus.net>

files:
A Misc/NEWS.d/next/Core and Builtins/2019-08-04-12-24-18.bpo-36279.8Zy7jZ.rst
M Lib/test/test_wait3.py
M Modules/posixmodule.c

diff --git a/Lib/test/test_wait3.py b/Lib/test/test_wait3.py
index eb51b2c03b23..3825451b058c 100644
--- a/Lib/test/test_wait3.py
+++ b/Lib/test/test_wait3.py
@@ -2,6 +2,8 @@
 """
 
 import os
+import subprocess
+import sys
 import time
 import unittest
 from test.fork_wait import ForkWait
@@ -31,6 +33,22 @@ def wait_impl(self, cpid):
         self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8))
         self.assertTrue(rusage)
 
+    def test_wait3_rusage_initialized(self):
+        # Ensure a successful wait3() call where no child was ready to report
+        # its exit status does not return uninitialized memory in the rusage
+        # structure. See bpo-36279.
+        args = [sys.executable, '-c', 'import sys; sys.stdin.read()']
+        proc = subprocess.Popen(args, stdin=subprocess.PIPE)
+        try:
+            pid, status, rusage = os.wait3(os.WNOHANG)
+            self.assertEqual(0, pid)
+            self.assertEqual(0, status)
+            self.assertEqual(0, sum(rusage))
+        finally:
+            proc.stdin.close()
+            proc.wait()
+
+
 def tearDownModule():
     reap_children()
 
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-04-12-24-18.bpo-36279.8Zy7jZ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-04-12-24-18.bpo-36279.8Zy7jZ.rst
new file mode 100644
index 000000000000..0d045c5f40e0
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-04-12-24-18.bpo-36279.8Zy7jZ.rst	
@@ -0,0 +1 @@
+Fix potential use of uninitialized memory in :func:`os.wait3`.
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 81704eec321c..a0a2a305119e 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -7489,6 +7489,12 @@ wait_helper(pid_t pid, int status, struct rusage *ru)
     if (pid == -1)
         return posix_error();
 
+    // If wait succeeded but no child was ready to report status, ru will not
+    // have been populated.
+    if (pid == 0) {
+        memset(ru, 0, sizeof(*ru));
+    }
+
     if (struct_rusage == NULL) {
         PyObject *m = PyImport_ImportModuleNoBlock("resource");
         if (m == NULL)


From webhook-mailer at python.org  Mon Sep  9 11:49:50 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Mon, 09 Sep 2019 15:49:50 -0000
Subject: [Python-checkins] Fix docs bz.open default mode (GH-15100)
Message-ID: <mailman.358.1568044192.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bb668f798a78af0fd99675a4c7e0d949591f4f2b
commit: bb668f798a78af0fd99675a4c7e0d949591f4f2b
branch: master
author: Richard Sanger <rsangerarj at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-09T08:49:47-07:00
summary:

Fix docs bz.open default mode (GH-15100)

bz2.open()'s default mode is rb, not r

files:
M Doc/library/bz2.rst

diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst
index 277de601cb7b..aa836af2b257 100644
--- a/Doc/library/bz2.rst
+++ b/Doc/library/bz2.rst
@@ -31,7 +31,7 @@ All of the classes in this module may safely be accessed from multiple threads.
 (De)compression of files
 ------------------------
 
-.. function:: open(filename, mode='r', compresslevel=9, encoding=None, errors=None, newline=None)
+.. function:: open(filename, mode='rb', compresslevel=9, encoding=None, errors=None, newline=None)
 
    Open a bzip2-compressed file in binary or text mode, returning a :term:`file
    object`.


From webhook-mailer at python.org  Mon Sep  9 11:52:00 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 15:52:00 -0000
Subject: [Python-checkins] Clarify that shutil's copy functions can accept
 path-like values (GH-15141)
Message-ID: <mailman.359.1568044321.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9488a5289de2ceecdfd2098cd70d215f96c4e745
commit: 9488a5289de2ceecdfd2098cd70d215f96c4e745
branch: master
author: Boris Verhovsky <boris.verk at gmail.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-09T08:51:56-07:00
summary:

Clarify that shutil's copy functions can accept path-like values (GH-15141)

files:
M Doc/library/shutil.rst
M Lib/shutil.py

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index a51e068ebd59..88c5f62dd4c9 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -52,7 +52,7 @@ Directory and files operations
 
    Copy the contents (no metadata) of the file named *src* to a file named
    *dst* and return *dst* in the most efficient way possible.
-   *src* and *dst* are path names given as strings.
+   *src* and *dst* are path-like objects or path names given as strings.
 
    *dst* must be the complete target file name; look at :func:`~shutil.copy`
    for a copy that accepts a target directory path.  If *src* and *dst*
@@ -92,7 +92,8 @@ Directory and files operations
 .. function:: copymode(src, dst, *, follow_symlinks=True)
 
    Copy the permission bits from *src* to *dst*.  The file contents, owner, and
-   group are unaffected.  *src* and *dst* are path names given as strings.
+   group are unaffected.  *src* and *dst* are path-like objects or path names
+   given as strings.
    If *follow_symlinks* is false, and both *src* and *dst* are symbolic links,
    :func:`copymode` will attempt to modify the mode of *dst* itself (rather
    than the file it points to).  This functionality is not available on every
@@ -108,7 +109,8 @@ Directory and files operations
    Copy the permission bits, last access time, last modification time, and
    flags from *src* to *dst*.  On Linux, :func:`copystat` also copies the
    "extended attributes" where possible.  The file contents, owner, and
-   group are unaffected.  *src* and *dst* are path names given as strings.
+   group are unaffected.  *src* and *dst* are path-like objects or path
+   names given as strings.
 
    If *follow_symlinks* is false, and *src* and *dst* both
    refer to symbolic links, :func:`copystat` will operate on
diff --git a/Lib/shutil.py b/Lib/shutil.py
index 561ad71508cc..5c1255a6713d 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -330,7 +330,8 @@ def copystat(src, dst, *, follow_symlinks=True):
     Copy the permission bits, last access time, last modification time, and
     flags from `src` to `dst`. On Linux, copystat() also copies the "extended
     attributes" where possible. The file contents, owner, and group are
-    unaffected. `src` and `dst` are path names given as strings.
+    unaffected. `src` and `dst` are path-like objects or path names given as
+    strings.
 
     If the optional flag `follow_symlinks` is not set, symlinks aren't
     followed if and only if both `src` and `dst` are symlinks.


From webhook-mailer at python.org  Mon Sep  9 11:54:38 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Mon, 09 Sep 2019 15:54:38 -0000
Subject: [Python-checkins] bpo-35803: Document and test dir=PathLike for
 tempfile (GH-11644)
Message-ID: <mailman.360.1568044479.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/370138ba9c29595df773cc057d17ea26fb6f21bd
commit: 370138ba9c29595df773cc057d17ea26fb6f21bd
branch: master
author: Anthony Sottile <asottile at umich.edu>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-09T10:54:34-05:00
summary:

bpo-35803: Document and test dir=PathLike for tempfile (GH-11644)

Co-Authored-By: Ammar Askar <ammar_askar at hotmail.com>

files:
A Misc/NEWS.d/next/Documentation/2019-01-21-14-30-59.bpo-35803.yae6Lq.rst
M Doc/library/tempfile.rst
M Lib/test/test_tempfile.py

diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst
index 0793e43df014..fff7a7a03eb8 100644
--- a/Doc/library/tempfile.rst
+++ b/Doc/library/tempfile.rst
@@ -191,6 +191,9 @@ The module defines the following user-callable items:
       *suffix* and *prefix* now accept and default to ``None`` to cause
       an appropriate default value to be used.
 
+   .. versionchanged:: 3.6
+      The *dir* parameter now accepts a :term:`path-like object`.
+
 
 .. function:: mkdtemp(suffix=None, prefix=None, dir=None)
 
@@ -214,6 +217,9 @@ The module defines the following user-callable items:
       *suffix* and *prefix* now accept and default to ``None`` to cause
       an appropriate default value to be used.
 
+   .. versionchanged:: 3.6
+      The *dir* parameter now accepts a :term:`path-like object`.
+
 
 .. function:: gettempdir()
 
diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py
index bd4db839331b..f995f6c9bfaf 100644
--- a/Lib/test/test_tempfile.py
+++ b/Lib/test/test_tempfile.py
@@ -3,6 +3,7 @@
 import errno
 import io
 import os
+import pathlib
 import signal
 import sys
 import re
@@ -56,6 +57,9 @@ def test_infer_return_type_multiples_and_none(self):
         with self.assertRaises(TypeError):
             tempfile._infer_return_type(b'', None, '')
 
+    def test_infer_return_type_pathlib(self):
+        self.assertIs(str, tempfile._infer_return_type(pathlib.Path('/')))
+
 
 # Common functionality.
 
@@ -79,8 +83,13 @@ def nameCheck(self, name, dir, pre, suf):
         nsuf  = nbase[len(nbase)-len(suf):]
 
         if dir is not None:
-            self.assertIs(type(name), str if type(dir) is str else bytes,
-                          "unexpected return type")
+            self.assertIs(
+                type(name),
+                str
+                if type(dir) is str or isinstance(dir, os.PathLike) else
+                bytes,
+                "unexpected return type",
+            )
         if pre is not None:
             self.assertIs(type(name), str if type(pre) is str else bytes,
                           "unexpected return type")
@@ -425,6 +434,7 @@ def test_choose_directory(self):
         dir = tempfile.mkdtemp()
         try:
             self.do_create(dir=dir).write(b"blat")
+            self.do_create(dir=pathlib.Path(dir)).write(b"blat")
         finally:
             os.rmdir(dir)
 
@@ -659,6 +669,7 @@ def test_choose_directory(self):
         dir = tempfile.mkdtemp()
         try:
             self.do_create(dir=dir)
+            self.do_create(dir=pathlib.Path(dir))
         finally:
             os.rmdir(dir)
 
@@ -728,6 +739,7 @@ def test_choose_directory(self):
         dir = tempfile.mkdtemp()
         try:
             os.rmdir(self.do_create(dir=dir))
+            os.rmdir(self.do_create(dir=pathlib.Path(dir)))
         finally:
             os.rmdir(dir)
 
diff --git a/Misc/NEWS.d/next/Documentation/2019-01-21-14-30-59.bpo-35803.yae6Lq.rst b/Misc/NEWS.d/next/Documentation/2019-01-21-14-30-59.bpo-35803.yae6Lq.rst
new file mode 100644
index 000000000000..b8394560e548
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-01-21-14-30-59.bpo-35803.yae6Lq.rst
@@ -0,0 +1,2 @@
+Document and test that ``tempfile`` functions may accept a
+:term:`path-like object` for the ``dir`` argument.  Patch by Anthony Sottile.


From webhook-mailer at python.org  Mon Sep  9 11:58:25 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 15:58:25 -0000
Subject: [Python-checkins] Fix punctuation in `os.execvpe` docstring.
 (GH-15051)
Message-ID: <mailman.361.1568044706.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/fb6807b043ab586428225920373e551b0432bc40
commit: fb6807b043ab586428225920373e551b0432bc40
branch: master
author: Hasan Ramezani <hasan.r67 at gmail.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-09T08:58:21-07:00
summary:

Fix punctuation in `os.execvpe` docstring. (GH-15051)

files:
M Lib/os.py

diff --git a/Lib/os.py b/Lib/os.py
index 79ff7a22d92e..200902528cbf 100644
--- a/Lib/os.py
+++ b/Lib/os.py
@@ -568,7 +568,7 @@ def execvpe(file, args, env):
     """execvpe(file, args, env)
 
     Execute the executable file (which is searched for along $PATH)
-    with argument list args and environment env , replacing the
+    with argument list args and environment env, replacing the
     current process.
     args may be a list or tuple of strings. """
     _execvpe(file, args, env)


From webhook-mailer at python.org  Mon Sep  9 12:05:25 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 16:05:25 -0000
Subject: [Python-checkins] bpo-37649: Fix exec_prefix check (GH-14897)
Message-ID: <mailman.362.1568045126.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/09090d04ef8d2f4c94157b852d3d530a51e13d22
commit: 09090d04ef8d2f4c94157b852d3d530a51e13d22
branch: master
author: Orivej Desh <orivej at gmx.fr>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-09T09:05:21-07:00
summary:

bpo-37649: Fix exec_prefix check (GH-14897)

files:
M Modules/getpath.c

diff --git a/Modules/getpath.c b/Modules/getpath.c
index 2372172a7b9f..36f9860ea131 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -1142,7 +1142,7 @@ calculate_init(PyCalculatePath *calculate,
         return DECODE_LOCALE_ERR("PREFIX define", len);
     }
     calculate->exec_prefix = Py_DecodeLocale(EXEC_PREFIX, &len);
-    if (!calculate->prefix) {
+    if (!calculate->exec_prefix) {
         return DECODE_LOCALE_ERR("EXEC_PREFIX define", len);
     }
     calculate->lib_python = Py_DecodeLocale("lib/python" VERSION, &len);


From webhook-mailer at python.org  Mon Sep  9 12:06:59 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 16:06:59 -0000
Subject: [Python-checkins] bpo-35941: Fix performance regression in new code
 (GH-12610)
Message-ID: <mailman.363.1568045220.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/915cd3f0696cb8a7206754a8fc34d4cd865a1b4a
commit: 915cd3f0696cb8a7206754a8fc34d4cd865a1b4a
branch: master
author: Christian Heimes <christian at python.org>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-09T09:06:55-07:00
summary:

bpo-35941: Fix performance regression in new code (GH-12610)

Accumulate certificates in a set instead of doing a costly list contain
operation. A Windows cert store can easily contain over hundred
certificates. The old code would result in way over 5,000 comparison
operations

Signed-off-by: Christian Heimes <christian at python.org>

files:
M Lib/test/test_ssl.py
M Modules/_ssl.c

diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 6ad0b61d8a57..b71928368b04 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -835,8 +835,8 @@ def test_enum_certificates(self):
                 cert, enc, trust = element
                 self.assertIsInstance(cert, bytes)
                 self.assertIn(enc, {"x509_asn", "pkcs_7_asn"})
-                self.assertIsInstance(trust, (set, bool))
-                if isinstance(trust, set):
+                self.assertIsInstance(trust, (frozenset, set, bool))
+                if isinstance(trust, (frozenset, set)):
                     trust_oids.update(trust)
 
         serverAuth = "1.3.6.1.5.5.7.3.1"
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index d94efa98e951..64060c64c4df 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -5517,7 +5517,7 @@ parseKeyUsage(PCCERT_CONTEXT pCertCtx, DWORD flags)
         }
         return PyErr_SetFromWindowsErr(error);
     }
-    retval = PySet_New(NULL);
+    retval = PyFrozenSet_New(NULL);
     if (retval == NULL) {
         goto error;
     }
@@ -5592,19 +5592,6 @@ ssl_collect_certificates(const char *store_name)
     return hCollectionStore;
 }
 
-/* code from Objects/listobject.c */
-
-static int
-list_contains(PyListObject *a, PyObject *el)
-{
-    Py_ssize_t i;
-    int cmp;
-
-    for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(a); ++i)
-        cmp = PyObject_RichCompareBool(PyList_GET_ITEM(a, i), el, Py_EQ);
-    return cmp;
-}
-
 /*[clinic input]
 _ssl.enum_certificates
     store_name: str
@@ -5627,7 +5614,7 @@ _ssl_enum_certificates_impl(PyObject *module, const char *store_name)
     PyObject *keyusage = NULL, *cert = NULL, *enc = NULL, *tup = NULL;
     PyObject *result = NULL;
 
-    result = PyList_New(0);
+    result = PySet_New(NULL);
     if (result == NULL) {
         return NULL;
     }
@@ -5667,11 +5654,10 @@ _ssl_enum_certificates_impl(PyObject *module, const char *store_name)
         enc = NULL;
         PyTuple_SET_ITEM(tup, 2, keyusage);
         keyusage = NULL;
-        if (!list_contains((PyListObject*)result, tup)) {
-            if (PyList_Append(result, tup) < 0) {
-                Py_CLEAR(result);
-                break;
-            }
+        if (PySet_Add(result, tup) == -1) {
+            Py_CLEAR(result);
+            Py_CLEAR(tup);
+            break;
         }
         Py_CLEAR(tup);
     }
@@ -5695,7 +5681,14 @@ _ssl_enum_certificates_impl(PyObject *module, const char *store_name)
         return PyErr_SetFromWindowsErr(GetLastError());
     }
 
-    return result;
+    /* convert set to list */
+    if (result == NULL) {
+        return NULL;
+    } else {
+        PyObject *lst = PySequence_List(result);
+        Py_DECREF(result);
+        return lst;
+    }
 }
 
 /*[clinic input]
@@ -5719,7 +5712,7 @@ _ssl_enum_crls_impl(PyObject *module, const char *store_name)
     PyObject *crl = NULL, *enc = NULL, *tup = NULL;
     PyObject *result = NULL;
 
-    result = PyList_New(0);
+    result = PySet_New(NULL);
     if (result == NULL) {
         return NULL;
     }
@@ -5749,11 +5742,10 @@ _ssl_enum_crls_impl(PyObject *module, const char *store_name)
         PyTuple_SET_ITEM(tup, 1, enc);
         enc = NULL;
 
-        if (!list_contains((PyListObject*)result, tup)) {
-            if (PyList_Append(result, tup) < 0) {
-                Py_CLEAR(result);
-                break;
-            }
+        if (PySet_Add(result, tup) == -1) {
+            Py_CLEAR(result);
+            Py_CLEAR(tup);
+            break;
         }
         Py_CLEAR(tup);
     }
@@ -5775,7 +5767,14 @@ _ssl_enum_crls_impl(PyObject *module, const char *store_name)
         Py_XDECREF(result);
         return PyErr_SetFromWindowsErr(GetLastError());
     }
-    return result;
+    /* convert set to list */
+    if (result == NULL) {
+        return NULL;
+    } else {
+        PyObject *lst = PySequence_List(result);
+        Py_DECREF(result);
+        return lst;
+    }
 }
 
 #endif /* _MSC_VER */


From webhook-mailer at python.org  Mon Sep  9 12:12:05 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 16:12:05 -0000
Subject: [Python-checkins] bpo-37876: Tests for ROT-13 codec (GH-15314)
Message-ID: <mailman.364.1568045526.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b6ef8f2beb90678a4a54218d6169040afbbf9fe1
commit: b6ef8f2beb90678a4a54218d6169040afbbf9fe1
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T09:12:01-07:00
summary:

bpo-37876: Tests for ROT-13 codec (GH-15314)


The Rot-13 codec is for educational use but does not have unit tests,
dragging down test coverage. This adds a few very simple tests.
(cherry picked from commit b3b48c81f09d1472010937f1331c5a208a2a2d48)

Co-authored-by: Zeth <theology at gmail.com>

files:
A Misc/NEWS.d/next/Tests/2019-08-16-16-15-14.bpo-37876.m3k1w3.rst
M Lib/test/test_codecs.py

diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index ba7f4847468a..b37525bf6604 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -3343,5 +3343,42 @@ def test_decode_unsupported_error_handler(self):
         self.assertEqual(str(cm.exception), 'unsupported error handler')
 
 
+class Rot13Test(unittest.TestCase):
+    """Test the educational ROT-13 codec."""
+    def test_encode(self):
+        ciphertext = codecs.encode("Caesar liked ciphers", 'rot-13')
+        self.assertEqual(ciphertext, 'Pnrfne yvxrq pvcuref')
+
+    def test_decode(self):
+        plaintext = codecs.decode('Rg gh, Oehgr?', 'rot-13')
+        self.assertEqual(plaintext, 'Et tu, Brute?')
+
+    def test_incremental_encode(self):
+        encoder = codecs.getincrementalencoder('rot-13')()
+        ciphertext = encoder.encode('ABBA nag Cheryl Baker')
+        self.assertEqual(ciphertext, 'NOON ant Purely Onxre')
+
+    def test_incremental_decode(self):
+        decoder = codecs.getincrementaldecoder('rot-13')()
+        plaintext = decoder.decode('terra Ares envy tha')
+        self.assertEqual(plaintext, 'green Nerf rail gun')
+
+
+class Rot13UtilTest(unittest.TestCase):
+    """Test the ROT-13 codec via rot13 function,
+    i.e. the user has done something like:
+    $ echo "Hello World" | python -m encodings.rot_13
+    """
+    def test_rot13_func(self):
+        infile = io.StringIO('Gb or, be abg gb or, gung vf gur dhrfgvba')
+        outfile = io.StringIO()
+        encodings.rot_13.rot13(infile, outfile)
+        outfile.seek(0)
+        plain_text = outfile.read()
+        self.assertEqual(
+            plain_text,
+            'To be, or not to be, that is the question')
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Tests/2019-08-16-16-15-14.bpo-37876.m3k1w3.rst b/Misc/NEWS.d/next/Tests/2019-08-16-16-15-14.bpo-37876.m3k1w3.rst
new file mode 100644
index 000000000000..45702fc63510
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-08-16-16-15-14.bpo-37876.m3k1w3.rst
@@ -0,0 +1 @@
+Add tests for ROT-13 codec.
\ No newline at end of file


From webhook-mailer at python.org  Mon Sep  9 12:24:20 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 16:24:20 -0000
Subject: [Python-checkins] bpo-38006: Avoid closure in
 weakref.WeakValueDictionary (GH-15641)
Message-ID: <mailman.365.1568046260.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/78d15faf6c522619098e94be3e7f6d88a9e61123
commit: 78d15faf6c522619098e94be3e7f6d88a9e61123
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T09:24:16-07:00
summary:

bpo-38006: Avoid closure in weakref.WeakValueDictionary (GH-15641)


weakref.WeakValueDictionary defines a local remove() function used as
callback for weak references. This function was created with a
closure.  Modify the implementation to avoid the closure.
(cherry picked from commit a2af05a0d3f0da06b8d432f52efa3ecf29038532)

Co-authored-by: Victor Stinner <vstinner at redhat.com>

files:
A Misc/NEWS.d/next/Library/2019-09-02-13-37-27.bpo-38006.Y7vA0Q.rst
M Lib/test/test_weakref.py
M Lib/weakref.py

diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py
index 6f15c03ac529..d9e1b201f656 100644
--- a/Lib/test/test_weakref.py
+++ b/Lib/test/test_weakref.py
@@ -1785,6 +1785,11 @@ def test_threaded_weak_value_dict_deepcopy(self):
         # copying should not result in a crash.
         self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, True)
 
+    @support.cpython_only
+    def test_remove_closure(self):
+        d = weakref.WeakValueDictionary()
+        self.assertIsNone(d._remove.__closure__)
+
 
 from test import mapping_tests
 
diff --git a/Lib/weakref.py b/Lib/weakref.py
index 8d71af653b7e..9d7008947f06 100644
--- a/Lib/weakref.py
+++ b/Lib/weakref.py
@@ -108,12 +108,12 @@ def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref):
                 else:
                     # Atomic removal is necessary since this function
                     # can be called asynchronously by the GC
-                    _atomic_removal(d, wr.key)
+                    _atomic_removal(self.data, wr.key)
         self._remove = remove
         # A list of keys to be removed
         self._pending_removals = []
         self._iterating = set()
-        self.data = d = {}
+        self.data = {}
         self.update(other, **kw)
 
     def _commit_removals(self):
diff --git a/Misc/NEWS.d/next/Library/2019-09-02-13-37-27.bpo-38006.Y7vA0Q.rst b/Misc/NEWS.d/next/Library/2019-09-02-13-37-27.bpo-38006.Y7vA0Q.rst
new file mode 100644
index 000000000000..ff064ad3f1a4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-02-13-37-27.bpo-38006.Y7vA0Q.rst
@@ -0,0 +1,3 @@
+weakref.WeakValueDictionary defines a local remove() function used as
+callback for weak references. This function was created with a closure.
+Modify the implementation to avoid the closure.


From webhook-mailer at python.org  Mon Sep  9 12:28:38 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Mon, 09 Sep 2019 16:28:38 -0000
Subject: [Python-checkins] bpo-37840: Fix handling of negative indices in
 bytearray_getitem() (GH-15250)
Message-ID: <mailman.366.1568046519.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/92709a263e9cec0bc646ccc1ea051fc528800d8d
commit: 92709a263e9cec0bc646ccc1ea051fc528800d8d
branch: master
author: Sergey Fedoseev <fedoseev.sergey at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-09T09:28:34-07:00
summary:

bpo-37840: Fix handling of negative indices in bytearray_getitem() (GH-15250)

files:
A Misc/NEWS.d/next/Core and Builtins/2019-08-13-18-05-20.bpo-37840.elLCci.rst
M Lib/test/test_bytes.py
M Modules/_testcapimodule.c
M Objects/bytearrayobject.c

diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py
index b5eeb2b4fc29..ddcf367f38fc 100644
--- a/Lib/test/test_bytes.py
+++ b/Lib/test/test_bytes.py
@@ -962,6 +962,15 @@ def test_translate(self):
         c = b.translate(None, delete=b'e')
         self.assertEqual(c, b'hllo')
 
+    def test_sq_item(self):
+        _testcapi = test.support.import_module('_testcapi')
+        obj = self.type2test((42,))
+        with self.assertRaises(IndexError):
+            _testcapi.sequence_getitem(obj, -2)
+        with self.assertRaises(IndexError):
+            _testcapi.sequence_getitem(obj, 1)
+        self.assertEqual(_testcapi.sequence_getitem(obj, 0), 42)
+
 
 class BytesTest(BaseBytesTest, unittest.TestCase):
     type2test = bytes
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-13-18-05-20.bpo-37840.elLCci.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-13-18-05-20.bpo-37840.elLCci.rst
new file mode 100644
index 000000000000..df689dade9a4
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-13-18-05-20.bpo-37840.elLCci.rst	
@@ -0,0 +1,2 @@
+Fix handling of negative indices in :c:member:`~PySequenceMethods.sq_item`
+of :class:`bytearray`. Patch by Sergey Fedoseev.
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 640ec5914577..147008b0a3dc 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -5164,6 +5164,18 @@ test_write_unraisable_exc(PyObject *self, PyObject *args)
 }
 
 
+static PyObject *
+sequence_getitem(PyObject *self, PyObject *args)
+{
+    PyObject *seq;
+    Py_ssize_t i;
+    if (!PyArg_ParseTuple(args, "On", &seq, &i)) {
+        return NULL;
+    }
+    return PySequence_GetItem(seq, i);
+}
+
+
 static PyMethodDef TestMethods[] = {
     {"raise_exception",         raise_exception,                 METH_VARARGS},
     {"raise_memoryerror",       raise_memoryerror,               METH_NOARGS},
@@ -5413,6 +5425,7 @@ static PyMethodDef TestMethods[] = {
     {"negative_refcount", negative_refcount, METH_NOARGS},
 #endif
     {"write_unraisable_exc", test_write_unraisable_exc, METH_VARARGS},
+    {"sequence_getitem", sequence_getitem, METH_VARARGS},
     {NULL, NULL} /* sentinel */
 };
 
diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c
index 8c16cc630d71..c9bf11bba1dd 100644
--- a/Objects/bytearrayobject.c
+++ b/Objects/bytearrayobject.c
@@ -381,8 +381,6 @@ bytearray_irepeat(PyByteArrayObject *self, Py_ssize_t count)
 static PyObject *
 bytearray_getitem(PyByteArrayObject *self, Py_ssize_t i)
 {
-    if (i < 0)
-        i += Py_SIZE(self);
     if (i < 0 || i >= Py_SIZE(self)) {
         PyErr_SetString(PyExc_IndexError, "bytearray index out of range");
         return NULL;


From webhook-mailer at python.org  Mon Sep  9 12:33:16 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Mon, 09 Sep 2019 16:33:16 -0000
Subject: [Python-checkins] bpo-37995: Add an option to ast.dump() to produce
 a multiline output. (GH-15631)
Message-ID: <mailman.367.1568046797.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/850573b836d5b82d1a1ebe75a635aaa0a3dff997
commit: 850573b836d5b82d1a1ebe75a635aaa0a3dff997
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T19:33:13+03:00
summary:

bpo-37995: Add an option to ast.dump() to produce a multiline output. (GH-15631)

files:
A Misc/NEWS.d/next/Library/2019-08-31-13-36-09.bpo-37995.rS8HzT.rst
M Doc/library/ast.rst
M Doc/whatsnew/3.9.rst
M Lib/ast.py
M Lib/test/test_ast.py

diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst
index 92bf8912eb53..cb8e7ec829be 100644
--- a/Doc/library/ast.rst
+++ b/Doc/library/ast.rst
@@ -319,7 +319,7 @@ and classes for traversing abstract syntax trees:
       node = YourTransformer().visit(node)
 
 
-.. function:: dump(node, annotate_fields=True, include_attributes=False)
+.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None)
 
    Return a formatted dump of the tree in *node*.  This is mainly useful for
    debugging purposes.  If *annotate_fields* is true (by default),
@@ -329,6 +329,17 @@ and classes for traversing abstract syntax trees:
    numbers and column offsets are not dumped by default.  If this is wanted,
    *include_attributes* can be set to true.
 
+   If *indent* is a non-negative integer or string, then the tree will be
+   pretty-printed with that indent level.  An indent level
+   of 0, negative, or ``""`` will only insert newlines.  ``None`` (the default)
+   selects the single line representation. Using a positive integer indent
+   indents that many spaces per level.  If *indent* is a string (such as ``"\t"``),
+   that string is used to indent each level.
+
+   .. versionchanged:: 3.9
+      Added the *indent* option.
+
+
 .. seealso::
 
     `Green Tree Snakes <https://greentreesnakes.readthedocs.io/>`_, an external documentation resource, has good
diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
index 5670cb59f481..deaefc753916 100644
--- a/Doc/whatsnew/3.9.rst
+++ b/Doc/whatsnew/3.9.rst
@@ -109,6 +109,14 @@ New Modules
 Improved Modules
 ================
 
+ast
+---
+
+Added the *indent* option to :func:`~ast.dump` which allows it to produce a
+multiline indented output.
+(Contributed by Serhiy Storchaka in :issue:`37995`.)
+
+
 threading
 ---------
 
diff --git a/Lib/ast.py b/Lib/ast.py
index 5ab023f6c3c0..498484f19855 100644
--- a/Lib/ast.py
+++ b/Lib/ast.py
@@ -96,7 +96,7 @@ def _convert(node):
     return _convert(node_or_string)
 
 
-def dump(node, annotate_fields=True, include_attributes=False):
+def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
     """
     Return a formatted dump of the tree in node.  This is mainly useful for
     debugging purposes.  If annotate_fields is true (by default),
@@ -104,11 +104,21 @@ def dump(node, annotate_fields=True, include_attributes=False):
     If annotate_fields is false, the result string will be more compact by
     omitting unambiguous field names.  Attributes such as line
     numbers and column offsets are not dumped by default.  If this is wanted,
-    include_attributes can be set to true.
+    include_attributes can be set to true.  If indent is a non-negative
+    integer or string, then the tree will be pretty-printed with that indent
+    level. None (the default) selects the single line representation.
     """
-    def _format(node):
+    def _format(node, level=0):
+        if indent is not None:
+            level += 1
+            prefix = '\n' + indent * level
+            sep = ',\n' + indent * level
+        else:
+            prefix = ''
+            sep = ', '
         if isinstance(node, AST):
             args = []
+            allsimple = True
             keywords = annotate_fields
             for field in node._fields:
                 try:
@@ -116,23 +126,36 @@ def _format(node):
                 except AttributeError:
                     keywords = True
                 else:
+                    value, simple = _format(value, level)
+                    allsimple = allsimple and simple
                     if keywords:
-                        args.append('%s=%s' % (field, _format(value)))
+                        args.append('%s=%s' % (field, value))
                     else:
-                        args.append(_format(value))
+                        args.append(value)
             if include_attributes and node._attributes:
-                for a in node._attributes:
+                for attr in node._attributes:
                     try:
-                        args.append('%s=%s' % (a, _format(getattr(node, a))))
+                        value = getattr(node, attr)
                     except AttributeError:
                         pass
-            return '%s(%s)' % (node.__class__.__name__, ', '.join(args))
+                    else:
+                        value, simple = _format(value, level)
+                        allsimple = allsimple and simple
+                        args.append('%s=%s' % (attr, value))
+            if allsimple and len(args) <= 3:
+                return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args
+            return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False
         elif isinstance(node, list):
-            return '[%s]' % ', '.join(_format(x) for x in node)
-        return repr(node)
+            if not node:
+                return '[]', True
+            return '[%s%s]' % (prefix, sep.join(_format(x, level)[0] for x in node)), False
+        return repr(node), True
+
     if not isinstance(node, AST):
         raise TypeError('expected AST, got %r' % node.__class__.__name__)
-    return _format(node)
+    if indent is not None and not isinstance(indent, str):
+        indent = ' ' * indent
+    return _format(node)[0]
 
 
 def copy_location(new_node, old_node):
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 07bbb4cc7723..47e259eb2656 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -645,6 +645,68 @@ def test_dump(self):
             "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)], type_ignores=[])"
         )
 
+    def test_dump_indent(self):
+        node = ast.parse('spam(eggs, "and cheese")')
+        self.assertEqual(ast.dump(node, indent=3), """\
+Module(
+   body=[
+      Expr(
+         value=Call(
+            func=Name(id='spam', ctx=Load()),
+            args=[
+               Name(id='eggs', ctx=Load()),
+               Constant(value='and cheese', kind=None)],
+            keywords=[]))],
+   type_ignores=[])""")
+        self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\
+Module(
+\t[
+\t\tExpr(
+\t\t\tCall(
+\t\t\t\tName('spam', Load()),
+\t\t\t\t[
+\t\t\t\t\tName('eggs', Load()),
+\t\t\t\t\tConstant('and cheese', None)],
+\t\t\t\t[]))],
+\t[])""")
+        self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\
+Module(
+   body=[
+      Expr(
+         value=Call(
+            func=Name(
+               id='spam',
+               ctx=Load(),
+               lineno=1,
+               col_offset=0,
+               end_lineno=1,
+               end_col_offset=4),
+            args=[
+               Name(
+                  id='eggs',
+                  ctx=Load(),
+                  lineno=1,
+                  col_offset=5,
+                  end_lineno=1,
+                  end_col_offset=9),
+               Constant(
+                  value='and cheese',
+                  kind=None,
+                  lineno=1,
+                  col_offset=11,
+                  end_lineno=1,
+                  end_col_offset=23)],
+            keywords=[],
+            lineno=1,
+            col_offset=0,
+            end_lineno=1,
+            end_col_offset=24),
+         lineno=1,
+         col_offset=0,
+         end_lineno=1,
+         end_col_offset=24)],
+   type_ignores=[])""")
+
     def test_dump_incomplete(self):
         node = ast.Raise(lineno=3, col_offset=4)
         self.assertEqual(ast.dump(node),
diff --git a/Misc/NEWS.d/next/Library/2019-08-31-13-36-09.bpo-37995.rS8HzT.rst b/Misc/NEWS.d/next/Library/2019-08-31-13-36-09.bpo-37995.rS8HzT.rst
new file mode 100644
index 000000000000..19482b644e14
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-31-13-36-09.bpo-37995.rS8HzT.rst
@@ -0,0 +1,2 @@
+Added the *indent* option to :func:`ast.dump` which allows it to produce a
+multiline indented output.


From webhook-mailer at python.org  Mon Sep  9 12:37:17 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Mon, 09 Sep 2019 16:37:17 -0000
Subject: [Python-checkins] bpo-36502: Update link to UAX #44,
 the Unicode doc on the UCD. (GH-15301)
Message-ID: <mailman.368.1568047038.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/64c6ac74e254d31f93fcc74bf02b3daa7d3e3f25
commit: 64c6ac74e254d31f93fcc74bf02b3daa7d3e3f25
branch: master
author: Greg Price <gnprice at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-09T09:37:13-07:00
summary:

bpo-36502: Update link to UAX #44, the Unicode doc on the UCD. (GH-15301)

The link we have points to the version from Unicode 6.0.0, dated 2010.
There have been numerous updates to it since then:
  https://www.unicode.org/reports/tr44/#Modifications

Change the link to one that points to the current version. Also, use HTTPS.

files:
M Doc/library/unicodedata.rst

diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst
index ee790c0cd00b..225384cf391e 100644
--- a/Doc/library/unicodedata.rst
+++ b/Doc/library/unicodedata.rst
@@ -22,7 +22,7 @@ this database is compiled from the `UCD version 12.1.0
 
 The module uses the same names and symbols as defined by Unicode
 Standard Annex #44, `"Unicode Character Database"
-<http://www.unicode.org/reports/tr44/tr44-6.html>`_.  It defines the
+<https://www.unicode.org/reports/tr44/>`_.  It defines the
 following functions:
 
 


From webhook-mailer at python.org  Mon Sep  9 12:45:21 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 09 Sep 2019 16:45:21 -0000
Subject: [Python-checkins] bpo-20490: Improve circular import error message
 (GH-15308)
Message-ID: <mailman.369.1568047523.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2d5594fac21a81a06f82c3605318dfa96e72398f
commit: 2d5594fac21a81a06f82c3605318dfa96e72398f
branch: 3.8
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-09T09:45:18-07:00
summary:

bpo-20490: Improve circular import error message (GH-15308)

files:
A Lib/test/test_import/data/circular_imports/from_cycle1.py
A Lib/test/test_import/data/circular_imports/from_cycle2.py
A Misc/NEWS.d/next/Core and Builtins/2019-08-15-12-48-36.bpo-20490.-hXeEn.rst
M Lib/test/test_import/__init__.py
M Python/ceval.c

diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py
index 50406d9aa1d9..7c24f0e50232 100644
--- a/Lib/test/test_import/__init__.py
+++ b/Lib/test/test_import/__init__.py
@@ -1326,6 +1326,16 @@ def test_crossreference2(self):
         self.assertIn('partially initialized module', errmsg)
         self.assertIn('circular import', errmsg)
 
+    def test_circular_from_import(self):
+        with self.assertRaises(ImportError) as cm:
+            import test.test_import.data.circular_imports.from_cycle1
+        self.assertIn(
+            "cannot import name 'b' from partially initialized module "
+            "'test.test_import.data.circular_imports.from_cycle1' "
+            "(most likely due to a circular import)",
+            str(cm.exception),
+        )
+
 
 if __name__ == '__main__':
     # Test needs to be a package, so we can do relative imports.
diff --git a/Lib/test/test_import/data/circular_imports/from_cycle1.py b/Lib/test/test_import/data/circular_imports/from_cycle1.py
new file mode 100644
index 000000000000..aacfd5f46fca
--- /dev/null
+++ b/Lib/test/test_import/data/circular_imports/from_cycle1.py
@@ -0,0 +1,2 @@
+from .from_cycle2 import a
+b = 1
diff --git a/Lib/test/test_import/data/circular_imports/from_cycle2.py b/Lib/test/test_import/data/circular_imports/from_cycle2.py
new file mode 100644
index 000000000000..62a66e1cfd02
--- /dev/null
+++ b/Lib/test/test_import/data/circular_imports/from_cycle2.py
@@ -0,0 +1,2 @@
+from .from_cycle1 import b
+a = 1
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-15-12-48-36.bpo-20490.-hXeEn.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-15-12-48-36.bpo-20490.-hXeEn.rst
new file mode 100644
index 000000000000..dfee480e1aa7
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-15-12-48-36.bpo-20490.-hXeEn.rst	
@@ -0,0 +1,2 @@
+Improve import error message for partially initialized module on circular
+``from`` imports - by Anthony Sottile.
diff --git a/Python/ceval.c b/Python/ceval.c
index 546a4264d8ad..07ec3293adf1 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -5236,10 +5236,17 @@ import_from(PyThreadState *tstate, PyObject *v, PyObject *name)
         PyErr_SetImportError(errmsg, pkgname, NULL);
     }
     else {
-        errmsg = PyUnicode_FromFormat(
-            "cannot import name %R from %R (%S)",
-            name, pkgname_or_unknown, pkgpath
-        );
+        _Py_IDENTIFIER(__spec__);
+        PyObject *spec = _PyObject_GetAttrId(v, &PyId___spec__);
+        Py_XINCREF(spec);
+        const char *fmt =
+            _PyModuleSpec_IsInitializing(spec) ?
+            "cannot import name %R from partially initialized module %R "
+            "(most likely due to a circular import) (%S)" :
+            "cannot import name %R from %R (%S)";
+        Py_XDECREF(spec);
+
+        errmsg = PyUnicode_FromFormat(fmt, name, pkgname_or_unknown, pkgpath);
         /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */
         PyErr_SetImportError(errmsg, pkgname, pkgpath);
     }


From webhook-mailer at python.org  Mon Sep  9 12:48:59 2019
From: webhook-mailer at python.org (Davin Potts)
Date: Mon, 09 Sep 2019 16:48:59 -0000
Subject: [Python-checkins] bpo-38018: Increase code coverage for
 multiprocessing.shared_memory (GH-15662)
Message-ID: <mailman.370.1568047739.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d14e39c8d9a9b525c7dcd83b2a260e2707fa85c1
commit: d14e39c8d9a9b525c7dcd83b2a260e2707fa85c1
branch: master
author: Vinay Sharma <vinay04sharma at icloud.com>
committer: Davin Potts <applio at users.noreply.github.com>
date: 2019-09-09T11:48:55-05:00
summary:

bpo-38018: Increase code coverage for multiprocessing.shared_memory (GH-15662)

files:
A Misc/NEWS.d/next/Tests/2019-09-03-19-33-10.bpo-38018.zTrMu7.rst
M Lib/test/_test_multiprocessing.py

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 2fe0def2bcd2..8269b879eb9d 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3719,6 +3719,30 @@ def test_shared_memory_basics(self):
         self.assertLess(same_sms.size, 20*sms.size)  # Size was ignored.
         same_sms.close()
 
+        # Creating Shared Memory Segment with -ve size
+        with self.assertRaises(ValueError):
+            shared_memory.SharedMemory(create=True, size=-2)
+
+        # Attaching Shared Memory Segment without a name
+        with self.assertRaises(ValueError):
+            shared_memory.SharedMemory(create=False)
+
+        # Test if shared memory segment is created properly,
+        # when _make_filename returns an existing shared memory segment name
+        with unittest.mock.patch(
+            'multiprocessing.shared_memory._make_filename') as mock_make_filename:
+
+            names = ['test01_fn', 'test02_fn']
+            mock_make_filename.side_effect = names
+            shm1 = shared_memory.SharedMemory(create=True, size=1)
+            self.addCleanup(shm1.unlink)
+            self.assertEqual(shm1.name, names[0])
+
+            mock_make_filename.side_effect = names
+            shm2 = shared_memory.SharedMemory(create=True, size=1)
+            self.addCleanup(shm2.unlink)
+            self.assertEqual(shm2.name, names[1])
+
         if shared_memory._USE_POSIX:
             # Posix Shared Memory can only be unlinked once.  Here we
             # test an implementation detail that is not observed across
diff --git a/Misc/NEWS.d/next/Tests/2019-09-03-19-33-10.bpo-38018.zTrMu7.rst b/Misc/NEWS.d/next/Tests/2019-09-03-19-33-10.bpo-38018.zTrMu7.rst
new file mode 100644
index 000000000000..3646404ae282
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-03-19-33-10.bpo-38018.zTrMu7.rst
@@ -0,0 +1 @@
+Increase code coverage for multiprocessing.shared_memory.


From webhook-mailer at python.org  Mon Sep  9 12:54:17 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Mon, 09 Sep 2019 16:54:17 -0000
Subject: [Python-checkins] bpo-37383: Updates docs to reflect AsyncMock
 call_count after await. (#15761)
Message-ID: <mailman.371.1568048058.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b9f65f01fd761da7799f36d29b54518399d3458e
commit: b9f65f01fd761da7799f36d29b54518399d3458e
branch: master
author: Lisa Roach <lisaroach14 at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T17:54:13+01:00
summary:

bpo-37383: Updates docs to reflect AsyncMock call_count after await. (#15761)

* bpo-351428: Updates documentation to reflect AsyncMock call_count after await.

* Adds skip and fixes warning.

* Removes extra >>>.

* Adds ... in front of await mock().

files:
M Doc/library/unittest.mock.rst

diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index 9bb342773a97..04ff8a61da3c 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -514,6 +514,20 @@ the *new_callable* argument to :func:`patch`.
             >>> mock.call_count
             2
 
+        For :class:`AsyncMock` the :attr:`call_count` is only iterated if the function
+        has been awaited:
+
+            >>> mock = AsyncMock()
+            >>> mock()  # doctest: +SKIP
+            <coroutine object AsyncMockMixin._mock_call at ...>
+            >>> mock.call_count
+            0
+            >>> async def main():
+            ...     await mock()
+            ...
+            >>> asyncio.run(main())
+            >>> mock.call_count
+            1
 
     .. attribute:: return_value
 


From webhook-mailer at python.org  Mon Sep  9 13:10:11 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Mon, 09 Sep 2019 17:10:11 -0000
Subject: [Python-checkins] [2.7] bpo-36502: Update link to UAX GH-44,
 the Unicode doc on the UCD. (GH-15808)
Message-ID: <mailman.372.1568049012.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/58d61efd4cdece3b026868a66d829001198d29b1
commit: 58d61efd4cdece3b026868a66d829001198d29b1
branch: 2.7
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-09T10:09:58-07:00
summary:

[2.7] bpo-36502: Update link to UAX GH-44, the Unicode doc on the UCD. (GH-15808)

The link we have points to the version from Unicode 6.0.0, dated 2010.
There have been numerous updates to it since then:
  https://www.unicode.org/reports/tr44/GH-Modifications

Change the link to one that points to the current version. Also, use HTTPS..
(cherry picked from commit 64c6ac74e254d31f93fcc74bf02b3daa7d3e3f25)

Co-authored-by: Greg Price <gnprice at gmail.com>

files:
M Doc/library/unicodedata.rst

diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst
index d7c48c4ab312..bed35effd1fb 100644
--- a/Doc/library/unicodedata.rst
+++ b/Doc/library/unicodedata.rst
@@ -20,7 +20,7 @@ based on the :file:`UnicodeData.txt` file version 5.2.0 which is publicly
 available from ftp://ftp.unicode.org/.
 
 The module uses the same names and symbols as defined by the UnicodeData File
-Format 5.2.0 (see http://www.unicode.org/reports/tr44/tr44-4.html).
+Format 5.2.0 (see https://www.unicode.org/reports/tr44/).
 It defines the following functions:
 
 


From webhook-mailer at python.org  Mon Sep  9 13:18:13 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 17:18:13 -0000
Subject: [Python-checkins] bpo-38070: visit_decref() calls
 _PyObject_IsFreed() (GH-15782)
Message-ID: <mailman.373.1568049494.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5731172bb1e958b1d80b18eaf88d3f2f93cfccdd
commit: 5731172bb1e958b1d80b18eaf88d3f2f93cfccdd
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T10:18:09-07:00
summary:

bpo-38070: visit_decref() calls _PyObject_IsFreed() (GH-15782)


In debug mode, visit_decref() now calls _PyObject_IsFreed() to ensure
that the object is not freed. If it's freed, the program fails with
an assertion error and Python dumps informations about the freed
object.
(cherry picked from commit d91d4de31745fc1ed4c7e6c208917827c9c472b6)

Co-authored-by: Victor Stinner <vstinner at redhat.com>

files:
M Modules/gcmodule.c

diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
index 2741edc0a021..b2ee56623cd4 100644
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -376,6 +376,8 @@ static int
 visit_decref(PyObject *op, void *data)
 {
     assert(op != NULL);
+    _PyObject_ASSERT(op, !_PyObject_IsFreed(op));
+
     if (PyObject_IS_GC(op)) {
         PyGC_Head *gc = AS_GC(op);
         /* We're only interested in gc_refs for objects in the


From webhook-mailer at python.org  Mon Sep  9 13:18:55 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 17:18:55 -0000
Subject: [Python-checkins] Clarify that shutil's copy functions can accept
 path-like values (GH-15141)
Message-ID: <mailman.374.1568049536.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0468a85cc4a3bd870f7e119ef8e6a2af0177ac56
commit: 0468a85cc4a3bd870f7e119ef8e6a2af0177ac56
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T10:18:51-07:00
summary:

Clarify that shutil's copy functions can accept path-like values (GH-15141)

(cherry picked from commit 9488a5289de2ceecdfd2098cd70d215f96c4e745)

Co-authored-by: Boris Verhovsky <boris.verk at gmail.com>

files:
M Doc/library/shutil.rst
M Lib/shutil.py

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index a51e068ebd59..88c5f62dd4c9 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -52,7 +52,7 @@ Directory and files operations
 
    Copy the contents (no metadata) of the file named *src* to a file named
    *dst* and return *dst* in the most efficient way possible.
-   *src* and *dst* are path names given as strings.
+   *src* and *dst* are path-like objects or path names given as strings.
 
    *dst* must be the complete target file name; look at :func:`~shutil.copy`
    for a copy that accepts a target directory path.  If *src* and *dst*
@@ -92,7 +92,8 @@ Directory and files operations
 .. function:: copymode(src, dst, *, follow_symlinks=True)
 
    Copy the permission bits from *src* to *dst*.  The file contents, owner, and
-   group are unaffected.  *src* and *dst* are path names given as strings.
+   group are unaffected.  *src* and *dst* are path-like objects or path names
+   given as strings.
    If *follow_symlinks* is false, and both *src* and *dst* are symbolic links,
    :func:`copymode` will attempt to modify the mode of *dst* itself (rather
    than the file it points to).  This functionality is not available on every
@@ -108,7 +109,8 @@ Directory and files operations
    Copy the permission bits, last access time, last modification time, and
    flags from *src* to *dst*.  On Linux, :func:`copystat` also copies the
    "extended attributes" where possible.  The file contents, owner, and
-   group are unaffected.  *src* and *dst* are path names given as strings.
+   group are unaffected.  *src* and *dst* are path-like objects or path
+   names given as strings.
 
    If *follow_symlinks* is false, and *src* and *dst* both
    refer to symbolic links, :func:`copystat` will operate on
diff --git a/Lib/shutil.py b/Lib/shutil.py
index 561ad71508cc..5c1255a6713d 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -330,7 +330,8 @@ def copystat(src, dst, *, follow_symlinks=True):
     Copy the permission bits, last access time, last modification time, and
     flags from `src` to `dst`. On Linux, copystat() also copies the "extended
     attributes" where possible. The file contents, owner, and group are
-    unaffected. `src` and `dst` are path names given as strings.
+    unaffected. `src` and `dst` are path-like objects or path names given as
+    strings.
 
     If the optional flag `follow_symlinks` is not set, symlinks aren't
     followed if and only if both `src` and `dst` are symlinks.


From webhook-mailer at python.org  Mon Sep  9 13:31:36 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 17:31:36 -0000
Subject: [Python-checkins] bpo-35803: Document and test dir=PathLike for
 tempfile (GH-11644)
Message-ID: <mailman.375.1568050297.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/eadf6b8787e979920c4fb6845797c33d270d2729
commit: eadf6b8787e979920c4fb6845797c33d270d2729
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T10:31:31-07:00
summary:

bpo-35803: Document and test dir=PathLike for tempfile (GH-11644)


Co-Authored-By: Ammar Askar <ammar_askar at hotmail.com>
(cherry picked from commit 370138ba9c29595df773cc057d17ea26fb6f21bd)

Co-authored-by: Anthony Sottile <asottile at umich.edu>

files:
A Misc/NEWS.d/next/Documentation/2019-01-21-14-30-59.bpo-35803.yae6Lq.rst
M Doc/library/tempfile.rst
M Lib/test/test_tempfile.py

diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst
index 0793e43df014..fff7a7a03eb8 100644
--- a/Doc/library/tempfile.rst
+++ b/Doc/library/tempfile.rst
@@ -191,6 +191,9 @@ The module defines the following user-callable items:
       *suffix* and *prefix* now accept and default to ``None`` to cause
       an appropriate default value to be used.
 
+   .. versionchanged:: 3.6
+      The *dir* parameter now accepts a :term:`path-like object`.
+
 
 .. function:: mkdtemp(suffix=None, prefix=None, dir=None)
 
@@ -214,6 +217,9 @@ The module defines the following user-callable items:
       *suffix* and *prefix* now accept and default to ``None`` to cause
       an appropriate default value to be used.
 
+   .. versionchanged:: 3.6
+      The *dir* parameter now accepts a :term:`path-like object`.
+
 
 .. function:: gettempdir()
 
diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py
index bd4db839331b..f995f6c9bfaf 100644
--- a/Lib/test/test_tempfile.py
+++ b/Lib/test/test_tempfile.py
@@ -3,6 +3,7 @@
 import errno
 import io
 import os
+import pathlib
 import signal
 import sys
 import re
@@ -56,6 +57,9 @@ def test_infer_return_type_multiples_and_none(self):
         with self.assertRaises(TypeError):
             tempfile._infer_return_type(b'', None, '')
 
+    def test_infer_return_type_pathlib(self):
+        self.assertIs(str, tempfile._infer_return_type(pathlib.Path('/')))
+
 
 # Common functionality.
 
@@ -79,8 +83,13 @@ def nameCheck(self, name, dir, pre, suf):
         nsuf  = nbase[len(nbase)-len(suf):]
 
         if dir is not None:
-            self.assertIs(type(name), str if type(dir) is str else bytes,
-                          "unexpected return type")
+            self.assertIs(
+                type(name),
+                str
+                if type(dir) is str or isinstance(dir, os.PathLike) else
+                bytes,
+                "unexpected return type",
+            )
         if pre is not None:
             self.assertIs(type(name), str if type(pre) is str else bytes,
                           "unexpected return type")
@@ -425,6 +434,7 @@ def test_choose_directory(self):
         dir = tempfile.mkdtemp()
         try:
             self.do_create(dir=dir).write(b"blat")
+            self.do_create(dir=pathlib.Path(dir)).write(b"blat")
         finally:
             os.rmdir(dir)
 
@@ -659,6 +669,7 @@ def test_choose_directory(self):
         dir = tempfile.mkdtemp()
         try:
             self.do_create(dir=dir)
+            self.do_create(dir=pathlib.Path(dir))
         finally:
             os.rmdir(dir)
 
@@ -728,6 +739,7 @@ def test_choose_directory(self):
         dir = tempfile.mkdtemp()
         try:
             os.rmdir(self.do_create(dir=dir))
+            os.rmdir(self.do_create(dir=pathlib.Path(dir)))
         finally:
             os.rmdir(dir)
 
diff --git a/Misc/NEWS.d/next/Documentation/2019-01-21-14-30-59.bpo-35803.yae6Lq.rst b/Misc/NEWS.d/next/Documentation/2019-01-21-14-30-59.bpo-35803.yae6Lq.rst
new file mode 100644
index 000000000000..b8394560e548
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-01-21-14-30-59.bpo-35803.yae6Lq.rst
@@ -0,0 +1,2 @@
+Document and test that ``tempfile`` functions may accept a
+:term:`path-like object` for the ``dir`` argument.  Patch by Anthony Sottile.


From webhook-mailer at python.org  Mon Sep  9 13:33:22 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 17:33:22 -0000
Subject: [Python-checkins] bpo-35803: Document and test dir=PathLike for
 tempfile (GH-11644)
Message-ID: <mailman.376.1568050403.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b4591ad33a727873c0a07d084211295bf4f5b892
commit: b4591ad33a727873c0a07d084211295bf4f5b892
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T10:33:18-07:00
summary:

bpo-35803: Document and test dir=PathLike for tempfile (GH-11644)


Co-Authored-By: Ammar Askar <ammar_askar at hotmail.com>
(cherry picked from commit 370138ba9c29595df773cc057d17ea26fb6f21bd)

Co-authored-by: Anthony Sottile <asottile at umich.edu>

files:
A Misc/NEWS.d/next/Documentation/2019-01-21-14-30-59.bpo-35803.yae6Lq.rst
M Doc/library/tempfile.rst
M Lib/test/test_tempfile.py

diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst
index 746adb1eee48..dd24a1c6f4ff 100644
--- a/Doc/library/tempfile.rst
+++ b/Doc/library/tempfile.rst
@@ -174,6 +174,9 @@ The module defines the following user-callable items:
       *suffix* and *prefix* now accept and default to ``None`` to cause
       an appropriate default value to be used.
 
+   .. versionchanged:: 3.6
+      The *dir* parameter now accepts a :term:`path-like object`.
+
 
 .. function:: mkdtemp(suffix=None, prefix=None, dir=None)
 
@@ -195,6 +198,9 @@ The module defines the following user-callable items:
       *suffix* and *prefix* now accept and default to ``None`` to cause
       an appropriate default value to be used.
 
+   .. versionchanged:: 3.6
+      The *dir* parameter now accepts a :term:`path-like object`.
+
 
 .. function:: gettempdir()
 
diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py
index 710756bde64c..931312831616 100644
--- a/Lib/test/test_tempfile.py
+++ b/Lib/test/test_tempfile.py
@@ -3,6 +3,7 @@
 import errno
 import io
 import os
+import pathlib
 import signal
 import sys
 import re
@@ -61,6 +62,9 @@ def test_infer_return_type_multiples_and_none(self):
         with self.assertRaises(TypeError):
             tempfile._infer_return_type(b'', None, '')
 
+    def test_infer_return_type_pathlib(self):
+        self.assertIs(str, tempfile._infer_return_type(pathlib.Path('/')))
+
 
 # Common functionality.
 
@@ -84,8 +88,13 @@ def nameCheck(self, name, dir, pre, suf):
         nsuf  = nbase[len(nbase)-len(suf):]
 
         if dir is not None:
-            self.assertIs(type(name), str if type(dir) is str else bytes,
-                          "unexpected return type")
+            self.assertIs(
+                type(name),
+                str
+                if type(dir) is str or isinstance(dir, os.PathLike) else
+                bytes,
+                "unexpected return type",
+            )
         if pre is not None:
             self.assertIs(type(name), str if type(pre) is str else bytes,
                           "unexpected return type")
@@ -430,6 +439,7 @@ def test_choose_directory(self):
         dir = tempfile.mkdtemp()
         try:
             self.do_create(dir=dir).write(b"blat")
+            self.do_create(dir=pathlib.Path(dir)).write(b"blat")
         finally:
             os.rmdir(dir)
 
@@ -666,6 +676,7 @@ def test_choose_directory(self):
         dir = tempfile.mkdtemp()
         try:
             self.do_create(dir=dir)
+            self.do_create(dir=pathlib.Path(dir))
         finally:
             os.rmdir(dir)
 
@@ -735,6 +746,7 @@ def test_choose_directory(self):
         dir = tempfile.mkdtemp()
         try:
             os.rmdir(self.do_create(dir=dir))
+            os.rmdir(self.do_create(dir=pathlib.Path(dir)))
         finally:
             os.rmdir(dir)
 
diff --git a/Misc/NEWS.d/next/Documentation/2019-01-21-14-30-59.bpo-35803.yae6Lq.rst b/Misc/NEWS.d/next/Documentation/2019-01-21-14-30-59.bpo-35803.yae6Lq.rst
new file mode 100644
index 000000000000..b8394560e548
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-01-21-14-30-59.bpo-35803.yae6Lq.rst
@@ -0,0 +1,2 @@
+Document and test that ``tempfile`` functions may accept a
+:term:`path-like object` for the ``dir`` argument.  Patch by Anthony Sottile.


From webhook-mailer at python.org  Mon Sep  9 13:44:50 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 17:44:50 -0000
Subject: [Python-checkins] Fix punctuation in `os.execvpe` docstring.
 (GH-15051)
Message-ID: <mailman.377.1568051091.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/421ef39968a5e59cf09fe1983f76be1c6866c637
commit: 421ef39968a5e59cf09fe1983f76be1c6866c637
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T10:44:46-07:00
summary:

Fix punctuation in `os.execvpe` docstring. (GH-15051)

(cherry picked from commit fb6807b043ab586428225920373e551b0432bc40)

Co-authored-by: Hasan Ramezani <hasan.r67 at gmail.com>

files:
M Lib/os.py

diff --git a/Lib/os.py b/Lib/os.py
index 79ff7a22d92e..200902528cbf 100644
--- a/Lib/os.py
+++ b/Lib/os.py
@@ -568,7 +568,7 @@ def execvpe(file, args, env):
     """execvpe(file, args, env)
 
     Execute the executable file (which is searched for along $PATH)
-    with argument list args and environment env , replacing the
+    with argument list args and environment env, replacing the
     current process.
     args may be a list or tuple of strings. """
     _execvpe(file, args, env)


From webhook-mailer at python.org  Mon Sep  9 13:46:17 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 17:46:17 -0000
Subject: [Python-checkins] Fix punctuation in `os.execvpe` docstring.
 (GH-15051)
Message-ID: <mailman.378.1568051177.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/15d263e95cdbce910ebbb2fc13064330b6000b4b
commit: 15d263e95cdbce910ebbb2fc13064330b6000b4b
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T10:46:13-07:00
summary:

Fix punctuation in `os.execvpe` docstring. (GH-15051)

(cherry picked from commit fb6807b043ab586428225920373e551b0432bc40)

Co-authored-by: Hasan Ramezani <hasan.r67 at gmail.com>

files:
M Lib/os.py

diff --git a/Lib/os.py b/Lib/os.py
index 499e6285618f..41896cd11bfd 100644
--- a/Lib/os.py
+++ b/Lib/os.py
@@ -574,7 +574,7 @@ def execvpe(file, args, env):
     """execvpe(file, args, env)
 
     Execute the executable file (which is searched for along $PATH)
-    with argument list args and environment env , replacing the
+    with argument list args and environment env, replacing the
     current process.
     args may be a list or tuple of strings. """
     _execvpe(file, args, env)


From webhook-mailer at python.org  Mon Sep  9 13:59:21 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 17:59:21 -0000
Subject: [Python-checkins] bpo-37649: Fix exec_prefix check (GH-14897)
Message-ID: <mailman.379.1568051962.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e83296314583ba9c7952d6539c63aae803a197ae
commit: e83296314583ba9c7952d6539c63aae803a197ae
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T10:59:18-07:00
summary:

bpo-37649: Fix exec_prefix check (GH-14897)

(cherry picked from commit 09090d04ef8d2f4c94157b852d3d530a51e13d22)

Co-authored-by: Orivej Desh <orivej at gmx.fr>

files:
M Modules/getpath.c

diff --git a/Modules/getpath.c b/Modules/getpath.c
index 5f8073818802..dc0574851c3b 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -1142,7 +1142,7 @@ calculate_init(PyCalculatePath *calculate,
         return DECODE_LOCALE_ERR("PREFIX define", len);
     }
     calculate->exec_prefix = Py_DecodeLocale(EXEC_PREFIX, &len);
-    if (!calculate->prefix) {
+    if (!calculate->exec_prefix) {
         return DECODE_LOCALE_ERR("EXEC_PREFIX define", len);
     }
     calculate->lib_python = Py_DecodeLocale("lib/python" VERSION, &len);


From webhook-mailer at python.org  Mon Sep  9 14:12:04 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 18:12:04 -0000
Subject: [Python-checkins] bpo-37649: Fix exec_prefix check (GH-14897)
Message-ID: <mailman.380.1568052725.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/328cfde754c6b30c1249c5b9c12ab6b8faab551e
commit: 328cfde754c6b30c1249c5b9c12ab6b8faab551e
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T11:12:00-07:00
summary:

bpo-37649: Fix exec_prefix check (GH-14897)

(cherry picked from commit 09090d04ef8d2f4c94157b852d3d530a51e13d22)

Co-authored-by: Orivej Desh <orivej at gmx.fr>

files:
M Modules/getpath.c

diff --git a/Modules/getpath.c b/Modules/getpath.c
index bfb33316d969..ba8d74b4a042 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -909,7 +909,7 @@ calculate_init(PyCalculatePath *calculate,
         return DECODE_LOCALE_ERR("PREFIX define", len);
     }
     calculate->exec_prefix = Py_DecodeLocale(EXEC_PREFIX, &len);
-    if (!calculate->prefix) {
+    if (!calculate->exec_prefix) {
         return DECODE_LOCALE_ERR("EXEC_PREFIX define", len);
     }
     calculate->lib_python = Py_DecodeLocale("lib/python" VERSION, &len);


From webhook-mailer at python.org  Mon Sep  9 14:40:09 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 18:40:09 -0000
Subject: [Python-checkins] bpo-36502: Update link to UAX GH-44,
 the Unicode doc on the UCD. (GH-15301)
Message-ID: <mailman.381.1568054410.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0a86da87da82c4a28d7ec91eb54c0b9ca40bbea7
commit: 0a86da87da82c4a28d7ec91eb54c0b9ca40bbea7
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T11:40:03-07:00
summary:

bpo-36502: Update link to UAX GH-44, the Unicode doc on the UCD. (GH-15301)


The link we have points to the version from Unicode 6.0.0, dated 2010.
There have been numerous updates to it since then:
  https://www.unicode.org/reports/tr44/GH-Modifications

Change the link to one that points to the current version. Also, use HTTPS.
(cherry picked from commit 64c6ac74e254d31f93fcc74bf02b3daa7d3e3f25)

Co-authored-by: Greg Price <gnprice at gmail.com>

files:
M Doc/library/unicodedata.rst

diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst
index 59548f3e8b4a..d0dae72f8070 100644
--- a/Doc/library/unicodedata.rst
+++ b/Doc/library/unicodedata.rst
@@ -22,7 +22,7 @@ this database is compiled from the `UCD version 11.0.0
 
 The module uses the same names and symbols as defined by Unicode
 Standard Annex #44, `"Unicode Character Database"
-<http://www.unicode.org/reports/tr44/tr44-6.html>`_.  It defines the
+<https://www.unicode.org/reports/tr44/>`_.  It defines the
 following functions:
 
 


From webhook-mailer at python.org  Mon Sep  9 14:41:17 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 09 Sep 2019 18:41:17 -0000
Subject: [Python-checkins] bpo-36502: Update link to UAX GH-44,
 the Unicode doc on the UCD. (GH-15301)
Message-ID: <mailman.382.1568054478.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c1c04cbc24c11cd7a47579af3faffee05a16acd7
commit: c1c04cbc24c11cd7a47579af3faffee05a16acd7
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T11:41:13-07:00
summary:

bpo-36502: Update link to UAX GH-44, the Unicode doc on the UCD. (GH-15301)


The link we have points to the version from Unicode 6.0.0, dated 2010.
There have been numerous updates to it since then:
  https://www.unicode.org/reports/tr44/GH-Modifications

Change the link to one that points to the current version. Also, use HTTPS.
(cherry picked from commit 64c6ac74e254d31f93fcc74bf02b3daa7d3e3f25)

Co-authored-by: Greg Price <gnprice at gmail.com>

files:
M Doc/library/unicodedata.rst

diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst
index ee790c0cd00b..225384cf391e 100644
--- a/Doc/library/unicodedata.rst
+++ b/Doc/library/unicodedata.rst
@@ -22,7 +22,7 @@ this database is compiled from the `UCD version 12.1.0
 
 The module uses the same names and symbols as defined by Unicode
 Standard Annex #44, `"Unicode Character Database"
-<http://www.unicode.org/reports/tr44/tr44-6.html>`_.  It defines the
+<https://www.unicode.org/reports/tr44/>`_.  It defines the
 following functions:
 
 


From webhook-mailer at python.org  Mon Sep  9 16:36:17 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Mon, 09 Sep 2019 20:36:17 -0000
Subject: [Python-checkins] bpo-38049: Add command-line interface for the ast
 module. (GH-15724)
Message-ID: <mailman.383.1568061378.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/832e8640086ac4fa547c055a72929879cc5a963a
commit: 832e8640086ac4fa547c055a72929879cc5a963a
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T23:36:13+03:00
summary:

bpo-38049: Add command-line interface for the ast module. (GH-15724)

files:
A Misc/NEWS.d/next/Library/2019-09-07-12-32-50.bpo-38049.xKP4tf.rst
M Doc/library/ast.rst
M Lib/ast.py

diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst
index cb8e7ec829be..b468f4235df3 100644
--- a/Doc/library/ast.rst
+++ b/Doc/library/ast.rst
@@ -340,6 +340,42 @@ and classes for traversing abstract syntax trees:
       Added the *indent* option.
 
 
+.. _ast-cli:
+
+Command-Line Usage
+------------------
+
+.. versionadded:: 3.9
+
+The :mod:`ast` module can be executed as a script from the command line.
+It is as simple as:
+
+.. code-block:: sh
+
+   python -m ast [-m <mode>] [-a] [infile]
+
+The following options are accepted:
+
+.. program:: ast
+
+.. cmdoption:: -h, --help
+
+   Show the help message and exit.
+
+.. cmdoption:: -m <mode>
+               --mode <mode>
+
+   Specify what kind of code must be compiled, like the *mode* argument
+   in :func:`parse`.
+
+.. cmdoption:: -a, --include-attributes
+
+   Include attributes such as line numbers and column offsets.
+
+If :file:`infile` is specified its contents are parsed to AST and dumped
+to stdout.  Otherwise, the content is read from stdin.
+
+
 .. seealso::
 
     `Green Tree Snakes <https://greentreesnakes.readthedocs.io/>`_, an external documentation resource, has good
diff --git a/Lib/ast.py b/Lib/ast.py
index 498484f19855..720dd48a761b 100644
--- a/Lib/ast.py
+++ b/Lib/ast.py
@@ -550,3 +550,27 @@ def __new__(cls, *args, **kwargs):
     bytes: 'Bytes',
     type(...): 'Ellipsis',
 }
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(prog='python -m ast')
+    parser.add_argument('infile', type=argparse.FileType(mode='rb'), nargs='?',
+                        default='-',
+                        help='the file to parse; defaults to stdin')
+    parser.add_argument('-m', '--mode', default='exec',
+                        choices=('exec', 'single', 'eval', 'func_type'),
+                        help='specify what kind of code must be parsed')
+    parser.add_argument('-a', '--include-attributes', action='store_true',
+                        help='include attributes such as line numbers and '
+                             'column offsets')
+    args = parser.parse_args()
+
+    with args.infile as infile:
+        source = infile.read()
+    tree = parse(source, args.infile.name, args.mode, type_comments=True)
+    print(dump(tree, include_attributes=args.include_attributes, indent=3))
+
+if __name__ == '__main__':
+    main()
diff --git a/Misc/NEWS.d/next/Library/2019-09-07-12-32-50.bpo-38049.xKP4tf.rst b/Misc/NEWS.d/next/Library/2019-09-07-12-32-50.bpo-38049.xKP4tf.rst
new file mode 100644
index 000000000000..9f17683032f1
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-07-12-32-50.bpo-38049.xKP4tf.rst
@@ -0,0 +1 @@
+Added command-line interface for the :mod:`ast` module.


From webhook-mailer at python.org  Mon Sep  9 16:41:02 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Mon, 09 Sep 2019 20:41:02 -0000
Subject: [Python-checkins] Correct overflow check in PyTuple_New() (GH-14838)
Message-ID: <mailman.384.1568061663.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/755d4ef8243050e5ff973524633caebd0ce03af9
commit: 755d4ef8243050e5ff973524633caebd0ce03af9
branch: master
author: Sergey Fedoseev <fedoseev.sergey at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-09T13:40:58-07:00
summary:

Correct overflow check in PyTuple_New() (GH-14838)

files:
M Objects/tupleobject.c

diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c
index a72257f95b08..08f7022fda25 100644
--- a/Objects/tupleobject.c
+++ b/Objects/tupleobject.c
@@ -120,8 +120,8 @@ tuple_alloc(Py_ssize_t size)
 #endif
     {
         /* Check for overflow */
-        if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - sizeof(PyTupleObject) -
-                    sizeof(PyObject *)) / sizeof(PyObject *)) {
+        if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) -
+                    sizeof(PyObject *))) / sizeof(PyObject *)) {
             return (PyTupleObject *)PyErr_NoMemory();
         }
         op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);


From webhook-mailer at python.org  Mon Sep  9 17:11:21 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Mon, 09 Sep 2019 21:11:21 -0000
Subject: [Python-checkins] bpo-36511: clean up python process before deploy
 on ARM Windows buildbots (GH-14431)
Message-ID: <mailman.385.1568063483.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/55d12ce8b8d397dd4e376bb0d1c27b3cced0ef50
commit: 55d12ce8b8d397dd4e376bb0d1c27b3cced0ef50
branch: master
author: Paul Monson <paulmon at users.noreply.github.com>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-09T22:11:17+01:00
summary:

bpo-36511: clean up python process before deploy on ARM Windows buildbots (GH-14431)

files:
M Tools/buildbot/remoteDeploy.bat

diff --git a/Tools/buildbot/remoteDeploy.bat b/Tools/buildbot/remoteDeploy.bat
index 6b86e1e59b07..31f151921b3f 100644
--- a/Tools/buildbot/remoteDeploy.bat
+++ b/Tools/buildbot/remoteDeploy.bat
@@ -24,11 +24,18 @@ if NOT "%REMOTE_PYTHON_DIR:~-1,1%"=="\" (set REMOTE_PYTHON_DIR=%REMOTE_PYTHON_DI
 echo PYTHON_SOURCE = %PYTHON_SOURCE%
 echo REMOTE_PYTHON_DIR = %REMOTE_PYTHON_DIR%
 
+REM stop Python processes and remove existing files if found
+ssh %SSH_SERVER% "kill python.exe"
+ssh %SSH_SERVER% "kill python_d.exe"
 ssh %SSH_SERVER% "if EXIST %REMOTE_PYTHON_DIR% (rd %REMOTE_PYTHON_DIR% /s/q)"
+
+REM Create Python directories
 ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%PCBuild\arm32"
 ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%temp"
 ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%Modules"
 ssh %SSH_SERVER% "md %REMOTE_PYTHON_DIR%PC"
+
+REM Copy Python files
 for /f "USEBACKQ" %%i in (`dir PCbuild\*.bat /b`) do @scp PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild"
 for /f "USEBACKQ" %%i in (`dir PCbuild\*.py /b`) do @scp PCBuild\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild"
 for /f "USEBACKQ" %%i in (`dir PCbuild\arm32\*.exe /b`) do @scp PCBuild\arm32\%%i "%SSH_SERVER%:%REMOTE_PYTHON_DIR%PCBuild\arm32"


From webhook-mailer at python.org  Mon Sep  9 17:52:07 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Mon, 09 Sep 2019 21:52:07 -0000
Subject: [Python-checkins] bpo-34293: Fix PDF documentation paper size
 (GH-8585)
Message-ID: <mailman.386.1568065928.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b5381f669718aa19690f42f3b8bd88f03045b9d2
commit: b5381f669718aa19690f42f3b8bd88f03045b9d2
branch: master
author: Jean-Fran?ois B <jfbu at free.fr>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-09T22:52:03+01:00
summary:

bpo-34293: Fix PDF documentation paper size (GH-8585)

The "A4" pdfs were previously the wrong size due to a change in the options in Sphinx 1.5.

See also sphinx-doc/sphinx#5235

files:
A Misc/NEWS.d/next/Documentation/2018-07-31-15-38-26.bpo-34293.yHupAL.rst
M Doc/Makefile

diff --git a/Doc/Makefile b/Doc/Makefile
index 3bcd9f23752b..f06689f83d87 100644
--- a/Doc/Makefile
+++ b/Doc/Makefile
@@ -13,7 +13,11 @@ SOURCES      =
 DISTVERSION  = $(shell $(PYTHON) tools/extensions/patchlevel.py)
 SPHINXERRORHANDLING = -W
 
-ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PAPER) \
+# Internal variables.
+PAPEROPT_a4     = -D latex_elements.papersize=a4paper
+PAPEROPT_letter = -D latex_elements.papersize=letterpaper
+
+ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) \
                 $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES)
 
 .PHONY: help build html htmlhelp latex text changes linkcheck \
diff --git a/Misc/NEWS.d/next/Documentation/2018-07-31-15-38-26.bpo-34293.yHupAL.rst b/Misc/NEWS.d/next/Documentation/2018-07-31-15-38-26.bpo-34293.yHupAL.rst
new file mode 100644
index 000000000000..912a3ad48d40
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2018-07-31-15-38-26.bpo-34293.yHupAL.rst
@@ -0,0 +1 @@
+Fix the Doc/Makefile regarding PAPER environment variable and PDF builds


From webhook-mailer at python.org  Mon Sep  9 18:11:27 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Mon, 09 Sep 2019 22:11:27 -0000
Subject: [Python-checkins] [3.8] bpo-34293: Fix PDF documentation paper size
 (GH-8585) (GH-15816)
Message-ID: <mailman.387.1568067088.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/99df5e837334b62c29c979bb0806f525778a4f3e
commit: 99df5e837334b62c29c979bb0806f525778a4f3e
branch: 3.8
author: Zachary Ware <zachary.ware at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T23:11:23+01:00
summary:

[3.8] bpo-34293: Fix PDF documentation paper size (GH-8585) (GH-15816)

The "A4" pdfs were previously the wrong size due to a change in the options in Sphinx 1.5.

See also sphinx-doc/sphinxGH-5235
(cherry picked from commit b5381f669718aa19690f42f3b8bd88f03045b9d2)

Authored-by: Jean-Fran?ois B <jfbu at free.fr>

files:
A Misc/NEWS.d/next/Documentation/2018-07-31-15-38-26.bpo-34293.yHupAL.rst
M Doc/Makefile

diff --git a/Doc/Makefile b/Doc/Makefile
index 6f86728ea834..05eeab9124df 100644
--- a/Doc/Makefile
+++ b/Doc/Makefile
@@ -13,7 +13,11 @@ SOURCES      =
 DISTVERSION  = $(shell $(PYTHON) tools/extensions/patchlevel.py)
 SPHINXERRORHANDLING = -W
 
-ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PAPER) \
+# Internal variables.
+PAPEROPT_a4     = -D latex_elements.papersize=a4paper
+PAPEROPT_letter = -D latex_elements.papersize=letterpaper
+
+ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) \
                 $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES)
 
 .PHONY: help build html htmlhelp latex text changes linkcheck \
diff --git a/Misc/NEWS.d/next/Documentation/2018-07-31-15-38-26.bpo-34293.yHupAL.rst b/Misc/NEWS.d/next/Documentation/2018-07-31-15-38-26.bpo-34293.yHupAL.rst
new file mode 100644
index 000000000000..912a3ad48d40
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2018-07-31-15-38-26.bpo-34293.yHupAL.rst
@@ -0,0 +1 @@
+Fix the Doc/Makefile regarding PAPER environment variable and PDF builds


From webhook-mailer at python.org  Mon Sep  9 18:11:38 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Mon, 09 Sep 2019 22:11:38 -0000
Subject: [Python-checkins] [3.7] bpo-34293: Fix PDF documentation paper size
 (GH-8585) (GH-15817)
Message-ID: <mailman.388.1568067099.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/12228ce41de1b8fcfb3f1ba0a86d98a232815e85
commit: 12228ce41de1b8fcfb3f1ba0a86d98a232815e85
branch: 3.7
author: Zachary Ware <zachary.ware at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T23:11:34+01:00
summary:

[3.7] bpo-34293: Fix PDF documentation paper size (GH-8585) (GH-15817)

The "A4" pdfs were previously the wrong size due to a change in the options in Sphinx 1.5.

See also sphinx-doc/sphinxGH-5235.
(cherry picked from commit b5381f669718aa19690f42f3b8bd88f03045b9d2)

Authored-by: Jean-Fran?ois B <jfbu at free.fr>

files:
A Misc/NEWS.d/next/Documentation/2018-07-31-15-38-26.bpo-34293.yHupAL.rst
M Doc/Makefile

diff --git a/Doc/Makefile b/Doc/Makefile
index 307d1e0e7de1..08013c0ef48c 100644
--- a/Doc/Makefile
+++ b/Doc/Makefile
@@ -12,7 +12,11 @@ PAPER        =
 SOURCES      =
 DISTVERSION  = $(shell $(PYTHON) tools/extensions/patchlevel.py)
 
-ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PAPER) \
+# Internal variables.
+PAPEROPT_a4     = -D latex_elements.papersize=a4paper
+PAPEROPT_letter = -D latex_elements.papersize=letterpaper
+
+ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) \
                 $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES)
 
 .PHONY: help build html htmlhelp latex text changes linkcheck \
diff --git a/Misc/NEWS.d/next/Documentation/2018-07-31-15-38-26.bpo-34293.yHupAL.rst b/Misc/NEWS.d/next/Documentation/2018-07-31-15-38-26.bpo-34293.yHupAL.rst
new file mode 100644
index 000000000000..912a3ad48d40
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2018-07-31-15-38-26.bpo-34293.yHupAL.rst
@@ -0,0 +1 @@
+Fix the Doc/Makefile regarding PAPER environment variable and PDF builds


From webhook-mailer at python.org  Mon Sep  9 18:31:25 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 09 Sep 2019 22:31:25 -0000
Subject: [Python-checkins] bpo-38006: Avoid closure in
 weakref.WeakValueDictionary (GH-15641) (GH-15789)
Message-ID: <mailman.389.1568068286.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/23669330b7d0d5ad1a9aac40315ba4c2e765f9dd
commit: 23669330b7d0d5ad1a9aac40315ba4c2e765f9dd
branch: 3.7
author: Victor Stinner <vstinner at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-10T00:31:20+02:00
summary:

bpo-38006: Avoid closure in weakref.WeakValueDictionary (GH-15641) (GH-15789)

weakref.WeakValueDictionary defines a local remove() function used as
callback for weak references. This function was created with a
closure.  Modify the implementation to avoid the closure.

(cherry picked from commit a2af05a0d3f0da06b8d432f52efa3ecf29038532)

files:
A Misc/NEWS.d/next/Library/2019-09-02-13-37-27.bpo-38006.Y7vA0Q.rst
M Lib/test/test_weakref.py
M Lib/weakref.py

diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py
index ad7a6acfcc7d..d3396fc9cf90 100644
--- a/Lib/test/test_weakref.py
+++ b/Lib/test/test_weakref.py
@@ -1770,6 +1770,11 @@ def test_threaded_weak_value_dict_deepcopy(self):
         # copying should not result in a crash.
         self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, True)
 
+    @support.cpython_only
+    def test_remove_closure(self):
+        d = weakref.WeakValueDictionary()
+        self.assertIsNone(d._remove.__closure__)
+
 
 from test import mapping_tests
 
diff --git a/Lib/weakref.py b/Lib/weakref.py
index 59b3aa5621a3..461c997a1293 100644
--- a/Lib/weakref.py
+++ b/Lib/weakref.py
@@ -114,12 +114,12 @@ def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref):
                 else:
                     # Atomic removal is necessary since this function
                     # can be called asynchronously by the GC
-                    _atomic_removal(d, wr.key)
+                    _atomic_removal(self.data, wr.key)
         self._remove = remove
         # A list of keys to be removed
         self._pending_removals = []
         self._iterating = set()
-        self.data = d = {}
+        self.data = {}
         self.update(*args, **kw)
 
     def _commit_removals(self):
diff --git a/Misc/NEWS.d/next/Library/2019-09-02-13-37-27.bpo-38006.Y7vA0Q.rst b/Misc/NEWS.d/next/Library/2019-09-02-13-37-27.bpo-38006.Y7vA0Q.rst
new file mode 100644
index 000000000000..ff064ad3f1a4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-02-13-37-27.bpo-38006.Y7vA0Q.rst
@@ -0,0 +1,3 @@
+weakref.WeakValueDictionary defines a local remove() function used as
+callback for weak references. This function was created with a closure.
+Modify the implementation to avoid the closure.


From webhook-mailer at python.org  Mon Sep  9 23:10:53 2019
From: webhook-mailer at python.org (Terry Jan Reedy)
Date: Tue, 10 Sep 2019 03:10:53 -0000
Subject: [Python-checkins] bpo-38077: IDLE no longer adds 'argv' to the user
 namespace (GH-15818)
Message-ID: <mailman.390.1568085054.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c59295a1ca304f37ca136dd7efca9e560db27d28
commit: c59295a1ca304f37ca136dd7efca9e560db27d28
branch: master
author: Terry Jan Reedy <tjreedy at udel.edu>
committer: GitHub <noreply at github.com>
date: 2019-09-09T23:10:44-04:00
summary:

bpo-38077: IDLE no longer adds 'argv' to the user namespace (GH-15818)

This only happened when initializing the subprocess to run a module.
This recent bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4.

files:
A Misc/NEWS.d/next/IDLE/2019-09-09-22-08-36.bpo-38077.Mzpfe2.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/runscript.py

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 559ffd0cf4f3..3ccc5c07fb00 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,9 @@ Released on 2019-10-20?
 ======================================
 
 
+bpo-38077: IDLE no longer adds 'argv' to the user namespace when
+initializing it.  This bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4.
+
 bpo-38401: Shell restart lines now fill the window width, always start
 with '=', and avoid wrapping unnecessarily. The line will still wrap
 if the included file name is long relative to the width.
diff --git a/Lib/idlelib/runscript.py b/Lib/idlelib/runscript.py
index f97cf528cce6..de73bf845815 100644
--- a/Lib/idlelib/runscript.py
+++ b/Lib/idlelib/runscript.py
@@ -164,7 +164,7 @@ def _run_module_event(self, event, *, customize=False):
                 _sys.argv = argv
             import os as _os
             _os.chdir({dirname!r})
-            del _sys, _basename, _os
+            del _sys, argv, _basename, _os
             \n""")
         interp.prepend_syspath(filename)
         # XXX KBK 03Jul04 When run w/o subprocess, runtime warnings still
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-09-22-08-36.bpo-38077.Mzpfe2.rst b/Misc/NEWS.d/next/IDLE/2019-09-09-22-08-36.bpo-38077.Mzpfe2.rst
new file mode 100644
index 000000000000..ba1fd55defaa
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-09-22-08-36.bpo-38077.Mzpfe2.rst
@@ -0,0 +1,2 @@
+IDLE no longer adds 'argv' to the user namespace when initializing it.  This
+bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4.


From webhook-mailer at python.org  Mon Sep  9 23:29:05 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 03:29:05 -0000
Subject: [Python-checkins] bpo-38077: IDLE no longer adds 'argv' to the user
 namespace (GH-15818)
Message-ID: <mailman.391.1568086147.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/29bde48ade5dbd5d88cfe309653014c84bebb89c
commit: 29bde48ade5dbd5d88cfe309653014c84bebb89c
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T20:29:01-07:00
summary:

bpo-38077: IDLE no longer adds 'argv' to the user namespace (GH-15818)


This only happened when initializing the subprocess to run a module.
This recent bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4.
(cherry picked from commit c59295a1ca304f37ca136dd7efca9e560db27d28)

Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu>

files:
A Misc/NEWS.d/next/IDLE/2019-09-09-22-08-36.bpo-38077.Mzpfe2.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/runscript.py

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 559ffd0cf4f3..3ccc5c07fb00 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,9 @@ Released on 2019-10-20?
 ======================================
 
 
+bpo-38077: IDLE no longer adds 'argv' to the user namespace when
+initializing it.  This bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4.
+
 bpo-38401: Shell restart lines now fill the window width, always start
 with '=', and avoid wrapping unnecessarily. The line will still wrap
 if the included file name is long relative to the width.
diff --git a/Lib/idlelib/runscript.py b/Lib/idlelib/runscript.py
index f97cf528cce6..de73bf845815 100644
--- a/Lib/idlelib/runscript.py
+++ b/Lib/idlelib/runscript.py
@@ -164,7 +164,7 @@ def _run_module_event(self, event, *, customize=False):
                 _sys.argv = argv
             import os as _os
             _os.chdir({dirname!r})
-            del _sys, _basename, _os
+            del _sys, argv, _basename, _os
             \n""")
         interp.prepend_syspath(filename)
         # XXX KBK 03Jul04 When run w/o subprocess, runtime warnings still
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-09-22-08-36.bpo-38077.Mzpfe2.rst b/Misc/NEWS.d/next/IDLE/2019-09-09-22-08-36.bpo-38077.Mzpfe2.rst
new file mode 100644
index 000000000000..ba1fd55defaa
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-09-22-08-36.bpo-38077.Mzpfe2.rst
@@ -0,0 +1,2 @@
+IDLE no longer adds 'argv' to the user namespace when initializing it.  This
+bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4.


From webhook-mailer at python.org  Mon Sep  9 23:33:28 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 03:33:28 -0000
Subject: [Python-checkins] bpo-38077: IDLE no longer adds 'argv' to the user
 namespace (GH-15818)
Message-ID: <mailman.392.1568086409.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/64947dc81a94692fa8ed21c2199a19a0188150ad
commit: 64947dc81a94692fa8ed21c2199a19a0188150ad
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-09T20:33:24-07:00
summary:

bpo-38077: IDLE no longer adds 'argv' to the user namespace (GH-15818)


This only happened when initializing the subprocess to run a module.
This recent bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4.
(cherry picked from commit c59295a1ca304f37ca136dd7efca9e560db27d28)

Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu>

files:
A Misc/NEWS.d/next/IDLE/2019-09-09-22-08-36.bpo-38077.Mzpfe2.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/runscript.py

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 34cdfbd099cb..da2512913f0e 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,9 @@ Released on 2019-09-30?
 ======================================
 
 
+bpo-38077: IDLE no longer adds 'argv' to the user namespace when
+initializing it.  This bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4.
+
 bpo-38401: Shell restart lines now fill the window width, always start
 with '=', and avoid wrapping unnecessarily. The line will still wrap
 if the included file name is long relative to the width.
diff --git a/Lib/idlelib/runscript.py b/Lib/idlelib/runscript.py
index f97cf528cce6..de73bf845815 100644
--- a/Lib/idlelib/runscript.py
+++ b/Lib/idlelib/runscript.py
@@ -164,7 +164,7 @@ def _run_module_event(self, event, *, customize=False):
                 _sys.argv = argv
             import os as _os
             _os.chdir({dirname!r})
-            del _sys, _basename, _os
+            del _sys, argv, _basename, _os
             \n""")
         interp.prepend_syspath(filename)
         # XXX KBK 03Jul04 When run w/o subprocess, runtime warnings still
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-09-22-08-36.bpo-38077.Mzpfe2.rst b/Misc/NEWS.d/next/IDLE/2019-09-09-22-08-36.bpo-38077.Mzpfe2.rst
new file mode 100644
index 000000000000..ba1fd55defaa
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-09-22-08-36.bpo-38077.Mzpfe2.rst
@@ -0,0 +1,2 @@
+IDLE no longer adds 'argv' to the user namespace when initializing it.  This
+bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4.


From webhook-mailer at python.org  Tue Sep 10 02:48:31 2019
From: webhook-mailer at python.org (Pablo Galindo)
Date: Tue, 10 Sep 2019 06:48:31 -0000
Subject: [Python-checkins] bpo-38018: Fix test for
 multiprocessing.shared_memory in BSD systems (GH-15821)
Message-ID: <mailman.393.1568098113.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2fc1160a80733f4c5c88796319154b3f59e98e4b
commit: 2fc1160a80733f4c5c88796319154b3f59e98e4b
branch: master
author: Vinay Sharma <vinay04sharma at icloud.com>
committer: Pablo Galindo <Pablogsal at gmail.com>
date: 2019-09-10T07:48:24+01:00
summary:

bpo-38018: Fix test for multiprocessing.shared_memory in BSD systems (GH-15821)

files:
M Lib/test/_test_multiprocessing.py

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 8269b879eb9d..c717d0aad287 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3732,16 +3732,21 @@ def test_shared_memory_basics(self):
         with unittest.mock.patch(
             'multiprocessing.shared_memory._make_filename') as mock_make_filename:
 
+            NAME_PREFIX = shared_memory._SHM_NAME_PREFIX
             names = ['test01_fn', 'test02_fn']
+            # Prepend NAME_PREFIX which can be '/psm_' or 'wnsm_', necessary
+            # because some POSIX compliant systems require name to start with /
+            names = [NAME_PREFIX + name for name in names]
+
             mock_make_filename.side_effect = names
             shm1 = shared_memory.SharedMemory(create=True, size=1)
             self.addCleanup(shm1.unlink)
-            self.assertEqual(shm1.name, names[0])
+            self.assertEqual(shm1._name, names[0])
 
             mock_make_filename.side_effect = names
             shm2 = shared_memory.SharedMemory(create=True, size=1)
             self.addCleanup(shm2.unlink)
-            self.assertEqual(shm2.name, names[1])
+            self.assertEqual(shm2._name, names[1])
 
         if shared_memory._USE_POSIX:
             # Posix Shared Memory can only be unlinked once.  Here we


From webhook-mailer at python.org  Tue Sep 10 03:31:38 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 07:31:38 -0000
Subject: [Python-checkins] bpo-37383: Updates docs to reflect AsyncMock
 call_count after await. (GH-15761)
Message-ID: <mailman.394.1568100698.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d4391aa5eb4767e19b7b380a836413e7c47f1fb4
commit: d4391aa5eb4767e19b7b380a836413e7c47f1fb4
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T00:31:34-07:00
summary:

bpo-37383: Updates docs to reflect AsyncMock call_count after await. (GH-15761)


* bpo-351428: Updates documentation to reflect AsyncMock call_count after await.

* Adds skip and fixes warning.

* Removes extra >>>.

* Adds ... in front of await mock().
(cherry picked from commit b9f65f01fd761da7799f36d29b54518399d3458e)

Co-authored-by: Lisa Roach <lisaroach14 at gmail.com>

files:
M Doc/library/unittest.mock.rst

diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index 304ba532274d..b2547546f3d6 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -514,6 +514,20 @@ the *new_callable* argument to :func:`patch`.
             >>> mock.call_count
             2
 
+        For :class:`AsyncMock` the :attr:`call_count` is only iterated if the function
+        has been awaited:
+
+            >>> mock = AsyncMock()
+            >>> mock()  # doctest: +SKIP
+            <coroutine object AsyncMockMixin._mock_call at ...>
+            >>> mock.call_count
+            0
+            >>> async def main():
+            ...     await mock()
+            ...
+            >>> asyncio.run(main())
+            >>> mock.call_count
+            1
 
     .. attribute:: return_value
 


From webhook-mailer at python.org  Tue Sep 10 03:32:00 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 07:32:00 -0000
Subject: [Python-checkins] bpo-37662: Documented
 venv.EnvBuilder.upgrade_dependencies(). (GH-15768)
Message-ID: <mailman.395.1568100721.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/74b7413d3a9be5e06b16eb2b9a6bdc1f3fe44bbb
commit: 74b7413d3a9be5e06b16eb2b9a6bdc1f3fe44bbb
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T00:31:56-07:00
summary:

bpo-37662: Documented venv.EnvBuilder.upgrade_dependencies(). (GH-15768)

(cherry picked from commit 264e034f990240e2aa379d8484b15b9e70c1fad5)

Co-authored-by: Vinay Sajip <vinay_sajip at yahoo.co.uk>

files:
M Doc/library/venv.rst

diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst
index 1e825c3c21e8..a834ac75eb1b 100644
--- a/Doc/library/venv.rst
+++ b/Doc/library/venv.rst
@@ -186,6 +186,14 @@ creation according to their needs, the :class:`EnvBuilder` class.
         Installs activation scripts appropriate to the platform into the virtual
         environment.
 
+    .. method:: upgrade_dependencies(context)
+
+       Upgrades the core venv dependency packages (currently ``pip`` and
+       ``setuptools``) in the environment. This is done by shelling out to the
+       ``pip`` executable in the environment.
+
+       .. versionadded:: 3.8
+
     .. method:: post_setup(context)
 
         A placeholder method which can be overridden in third party


From webhook-mailer at python.org  Tue Sep 10 04:47:00 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 08:47:00 -0000
Subject: [Python-checkins] bpo-35941: Fix performance regression in SSL
 certificate code (GH-12610)
Message-ID: <mailman.396.1568105221.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b4fb2c29f34c322855ab6be72b491930cf508f64
commit: b4fb2c29f34c322855ab6be72b491930cf508f64
branch: 3.7
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-10T01:46:40-07:00
summary:

bpo-35941: Fix performance regression in SSL certificate code (GH-12610)

Accumulate certificates in a set instead of doing a costly list contain
operation. A Windows cert store can easily contain over hundred
certificates. The old code would result in way over 5,000 comparison
operations

Signed-off-by: Christian Heimes <christian at python.org>

files:
M Lib/test/test_ssl.py
M Modules/_ssl.c

diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 4a61711f0ee6..04ede28ab7c0 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -849,8 +849,8 @@ def test_enum_certificates(self):
                 cert, enc, trust = element
                 self.assertIsInstance(cert, bytes)
                 self.assertIn(enc, {"x509_asn", "pkcs_7_asn"})
-                self.assertIsInstance(trust, (set, bool))
-                if isinstance(trust, set):
+                self.assertIsInstance(trust, (frozenset, set, bool))
+                if isinstance(trust, (frozenset, set)):
                     trust_oids.update(trust)
 
         serverAuth = "1.3.6.1.5.5.7.3.1"
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index 91820fe9529d..a94dbbaec849 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -5384,7 +5384,7 @@ parseKeyUsage(PCCERT_CONTEXT pCertCtx, DWORD flags)
         }
         return PyErr_SetFromWindowsErr(error);
     }
-    retval = PySet_New(NULL);
+    retval = PyFrozenSet_New(NULL);
     if (retval == NULL) {
         goto error;
     }
@@ -5459,20 +5459,6 @@ ssl_collect_certificates(const char *store_name)
     return hCollectionStore;
 }
 
-/* code from Objects/listobject.c */
-
-static int
-list_contains(PyListObject *a, PyObject *el)
-{
-    Py_ssize_t i;
-    int cmp;
-
-    for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(a); ++i)
-        cmp = PyObject_RichCompareBool(el, PyList_GET_ITEM(a, i),
-                                           Py_EQ);
-    return cmp;
-}
-
 /*[clinic input]
 _ssl.enum_certificates
     store_name: str
@@ -5495,7 +5481,7 @@ _ssl_enum_certificates_impl(PyObject *module, const char *store_name)
     PyObject *keyusage = NULL, *cert = NULL, *enc = NULL, *tup = NULL;
     PyObject *result = NULL;
 
-    result = PyList_New(0);
+    result = PySet_New(NULL);
     if (result == NULL) {
         return NULL;
     }
@@ -5535,11 +5521,10 @@ _ssl_enum_certificates_impl(PyObject *module, const char *store_name)
         enc = NULL;
         PyTuple_SET_ITEM(tup, 2, keyusage);
         keyusage = NULL;
-        if (!list_contains((PyListObject*)result, tup)) {
-            if (PyList_Append(result, tup) < 0) {
-                Py_CLEAR(result);
-                break;
-            }
+        if (PySet_Add(result, tup) == -1) {
+            Py_CLEAR(result);
+            Py_CLEAR(tup);
+            break;
         }
         Py_CLEAR(tup);
     }
@@ -5563,7 +5548,14 @@ _ssl_enum_certificates_impl(PyObject *module, const char *store_name)
         return PyErr_SetFromWindowsErr(GetLastError());
     }
 
-    return result;
+    /* convert set to list */
+    if (result == NULL) {
+        return NULL;
+    } else {
+        PyObject *lst = PySequence_List(result);
+        Py_DECREF(result);
+        return lst;
+    }
 }
 
 /*[clinic input]
@@ -5587,7 +5579,7 @@ _ssl_enum_crls_impl(PyObject *module, const char *store_name)
     PyObject *crl = NULL, *enc = NULL, *tup = NULL;
     PyObject *result = NULL;
 
-    result = PyList_New(0);
+    result = PySet_New(NULL);
     if (result == NULL) {
         return NULL;
     }
@@ -5617,11 +5609,10 @@ _ssl_enum_crls_impl(PyObject *module, const char *store_name)
         PyTuple_SET_ITEM(tup, 1, enc);
         enc = NULL;
 
-        if (!list_contains((PyListObject*)result, tup)) {
-            if (PyList_Append(result, tup) < 0) {
-                Py_CLEAR(result);
-                break;
-            }
+        if (PySet_Add(result, tup) == -1) {
+            Py_CLEAR(result);
+            Py_CLEAR(tup);
+            break;
         }
         Py_CLEAR(tup);
     }
@@ -5643,7 +5634,14 @@ _ssl_enum_crls_impl(PyObject *module, const char *store_name)
         Py_XDECREF(result);
         return PyErr_SetFromWindowsErr(GetLastError());
     }
-    return result;
+    /* convert set to list */
+    if (result == NULL) {
+        return NULL;
+    } else {
+        PyObject *lst = PySequence_List(result);
+        Py_DECREF(result);
+        return lst;
+    }
 }
 
 #endif /* _MSC_VER */


From webhook-mailer at python.org  Tue Sep 10 04:51:10 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Tue, 10 Sep 2019 08:51:10 -0000
Subject: [Python-checkins] Cut tricky `goto` that isn't needed,
 in _PyBytes_DecodeEscape. (GH-15825)
Message-ID: <mailman.397.1568105471.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0711642eec58d00f31031279df799fbddaf6f2c1
commit: 0711642eec58d00f31031279df799fbddaf6f2c1
branch: master
author: Greg Price <gnprice at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-10T09:51:04+01:00
summary:

Cut tricky `goto` that isn't needed, in _PyBytes_DecodeEscape. (GH-15825)

This is the sort of `goto` that requires the reader to stare hard at
the code to unpick what it's doing.

On doing so, the answer is... not very much!

* It jumps from the bottom of the loop to almost the top; the effect
  is to bypass the loop condition `s < end` and also the
  `if`-condition `*s != '\\'`, acting as if both are true.

* We've just decremented `s`, after incrementing it in the `switch`
  condition.  So it has the same value as when `s == end` failed.
  Before that was another increment... and before that we had
  `s < end`.  So `s < end` true, then increment, then `s == end`
  false... that means `s < end` is still true.

* Also this means `s` points to the same character as it did for the
  `switch` condition.  And there was a `case '\\'`, which we didn't
  hit -- so `*s != '\\'` is also true.

* That means this has no effect on the behavior!  The most it might do
  is an optimization -- we get to skip those two checks, because (as
  just proven above) we know they're true.

* But gosh, this is the *invalid escape sequence* path.  This does not
  seem like the kind of code path that calls for extreme optimization
  tricks.

So, take the `goto` and the label out.

Perhaps the compiler will notice the exact same facts we showed above,
and generate identical code.  Or perhaps it won't!  That'll be OK.

But then, crucially, if some future edit to this loop causes the
reasoning above to *stop* holding true... the compiler will adjust
this jump accordingly.  One of us fallible humans might not.

files:
M Objects/bytesobject.c

diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
index 6d553304a403..72f9cd271be0 100644
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -1142,7 +1142,6 @@ PyObject *_PyBytes_DecodeEscape(const char *s,
     end = s + len;
     while (s < end) {
         if (*s != '\\') {
-          non_esc:
             if (!(recode_encoding && (*s & 0x80))) {
                 *p++ = *s++;
             }
@@ -1229,8 +1228,6 @@ PyObject *_PyBytes_DecodeEscape(const char *s,
             }
             *p++ = '\\';
             s--;
-            goto non_esc; /* an arbitrary number of unescaped
-                             UTF-8 bytes may follow. */
         }
     }
 


From webhook-mailer at python.org  Tue Sep 10 04:55:38 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Tue, 10 Sep 2019 08:55:38 -0000
Subject: [Python-checkins] bpo-25237: Documentation for tkinter modules
 (GH-1870)
Message-ID: <mailman.398.1568105738.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/80428ed4e19b31071433806b4d89465c88e084c6
commit: 80428ed4e19b31071433806b4d89465c88e084c6
branch: master
author: Nikhil <12703092+patel-nikhil at users.noreply.github.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-10T10:55:34+02:00
summary:

bpo-25237: Documentation for tkinter modules (GH-1870)

files:
A Doc/library/dialog.rst
A Doc/library/tk_msg.png
A Doc/library/tkinter.colorchooser.rst
A Doc/library/tkinter.dnd.rst
A Doc/library/tkinter.font.rst
A Doc/library/tkinter.messagebox.rst
A Misc/NEWS.d/next/Documentation/2018-06-02-12-55-23.bpo-25237.m8-JMu.rst
M Doc/library/tk.rst
M Doc/library/tkinter.rst
M Doc/library/tkinter.scrolledtext.rst

diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst
new file mode 100644
index 000000000000..836f66296337
--- /dev/null
+++ b/Doc/library/dialog.rst
@@ -0,0 +1,230 @@
+Tkinter Dialogs
+===============
+
+:mod:`tkinter.simpledialog` --- Standard Tkinter input dialogs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. module:: tkinter.simpledialog
+   :platform: Tk
+   :synopsis: Simple dialog windows
+
+**Source code:** :source:`Lib/tkinter/simpledialog.py`
+
+--------------
+
+The :mod:`tkinter.simpledialog` module contains convenience classes and
+functions for creating simple modal dialogs to get a value from the user.
+
+
+.. function:: askfloat(title, prompt, **kw)
+              askinteger(title, prompt, **kw)
+              askstring(title, prompt, **kw)
+
+   The above three functions provide dialogs that prompt the user to enter a value
+   of the desired type.
+
+.. class:: Dialog(parent, title=None)
+
+   The base class for custom dialogs.
+
+    .. method:: body(master)
+
+       Override to construct the dialog's interface and return the widget that
+       should have initial focus.
+
+    .. method:: buttonbox()
+
+       Default behaviour adds OK and Cancel buttons. Override for custom button
+       layouts.
+
+
+
+:mod:`tkinter.filedialog` --- File selection dialogs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. module:: tkinter.filedialog
+   :platform: Tk
+   :synopsis: Dialog classes for file selection
+
+**Source code:** :source:`Lib/tkinter/filedialog.py`
+
+--------------
+
+The :mod:`tkinter.filedialog` module provides classes and factory functions for
+creating file/directory selection windows.
+
+Native Load/Save Dialogs
+------------------------
+
+The following classes and functions provide file dialog windows that combine a
+native look-and-feel with configuration options to customize behaviour.
+The following keyword arguments are applicable to the classes and functions
+listed below:
+
+ | *parent* - the window to place the dialog on top of
+
+ | *title* - the title of the window
+
+ | *initialdir* - the directory that the dialog starts in
+
+ | *initialfile* - the file selected upon opening of the dialog
+
+ | *filetypes* - a sequence of (label, pattern) tuples, '*' wildcard is allowed
+
+ | *defaultextension* - default extension to append to file (save dialogs)
+
+ | *multiple* - when True, selection of multiple items is allowed
+
+
+**Static factory functions**
+
+The below functions when called create a modal, native look-and-feel dialog,
+wait for the user's selection, then return the selected value(s) or ``None`` to the
+caller.
+
+.. function:: askopenfile(mode="r", **options)
+              askopenfiles(mode="r", **options)
+
+   The above two functions create an :class:`Open` dialog and return the opened
+   file object(s) in read-only mode.
+
+.. function:: asksaveasfile(mode="w", **options)
+
+   Create a :class:`SaveAs` dialog and return a file object opened in write-only mode.
+
+.. function:: askopenfilename(**options)
+              askopenfilenames(**options)
+
+   The above two functions create an :class:`Open` dialog and return the
+   selected filename(s) that correspond to existing file(s).
+
+.. function:: asksaveasfilename(**options)
+
+   Create a :class:`SaveAs` dialog and return the selected filename.
+
+.. function:: askdirectory(**options)
+
+ | Prompt user to select a directory.
+ | Additional keyword option:
+ |  *mustexist* - determines if selection must be an existing directory.
+
+.. class:: Open(master=None, **options)
+           SaveAs(master=None, **options)
+
+   The above two classes provide native dialog windows for saving and loading
+   files.
+
+**Convenience classes**
+
+The below classes are used for creating file/directory windows from scratch.
+These do not emulate the native look-and-feel of the platform.
+
+.. class:: Directory(master=None, **options)
+
+   Create a dialog prompting the user to select a directory.
+
+.. note::  The *FileDialog* class should be subclassed for custom event
+   handling and behaviour.
+
+.. class:: FileDialog(master, title=None)
+
+   Create a basic file selection dialog.
+
+   .. method:: cancel_command(event=None)
+
+      Trigger the termination of the dialog window.
+
+   .. method:: dirs_double_event(event)
+
+      Event handler for double-click event on directory.
+
+   .. method:: dirs_select_event(event)
+
+      Event handler for click event on directory.
+
+   .. method:: files_double_event(event)
+
+      Event handler for double-click event on file.
+
+   .. method:: files_select_event(event)
+
+      Event handler for single-click event on file.
+
+   .. method:: filter_command(event=None)
+
+      Filter the files by directory.
+
+   .. method:: get_filter()
+
+      Retrieve the file filter currently in use.
+
+   .. method:: get_selection()
+
+      Retrieve the currently selected item.
+
+   .. method:: go(dir_or_file=os.curdir, pattern="*", default="", key=None)
+
+      Render dialog and start event loop.
+
+   .. method:: ok_event(event)
+
+      Exit dialog returning current selection.
+
+   .. method:: quit(how=None)
+
+      Exit dialog returning filename, if any.
+
+   .. method:: set_filter(dir, pat)
+
+      Set the file filter.
+
+   .. method:: set_selection(file)
+
+      Update the current file selection to *file*.
+
+
+.. class:: LoadFileDialog(master, title=None)
+
+   A subclass of FileDialog that creates a dialog window for selecting an
+   existing file.
+
+   .. method:: ok_command()
+
+      Test that a file is provided and that the selection indicates an
+      already existing file.
+
+.. class:: SaveFileDialog(master, title=None)
+
+   A subclass of FileDialog that creates a dialog window for selecting a
+   destination file.
+
+    .. method:: ok_command()
+
+      Test whether or not the selection points to a valid file that is not a
+      directory. Confirmation is required if an already existing file is
+      selected.
+
+:mod:`tkinter.commondialog` --- Dialog window templates
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. module:: tkinter.commondialog
+   :platform: Tk
+   :synopsis: Tkinter base class for dialogs
+
+**Source code:** :source:`Lib/tkinter/commondialog.py`
+
+--------------
+
+The :mod:`tkinter.commondialog` module provides the :class:`Dialog` class that
+is the base class for dialogs defined in other supporting modules.
+
+.. class:: Dialog(master=None, **options)
+
+   .. method:: show(color=None, **options)
+
+      Render the Dialog window.
+
+
+.. seealso::
+
+   Modules :mod:`tkinter.messagebox`, :ref:`tut-files`
\ No newline at end of file
diff --git a/Doc/library/tk.rst b/Doc/library/tk.rst
index 95cd1c7712e7..c6c73f057cab 100644
--- a/Doc/library/tk.rst
+++ b/Doc/library/tk.rst
@@ -33,14 +33,17 @@ alternatives, see the :ref:`other-gui-packages` section.
 .. toctree::
 
    tkinter.rst
+   tkinter.colorchooser.rst
+   tkinter.font.rst
+   dialog.rst
+   tkinter.messagebox.rst
+   tkinter.scrolledtext.rst
+   tkinter.dnd.rst
    tkinter.ttk.rst
    tkinter.tix.rst
-   tkinter.scrolledtext.rst
    idle.rst
    othergui.rst
 
 .. Other sections I have in mind are
    Tkinter internals
-   Freezing Tkinter applications
-
-
+   Freezing Tkinter applications
\ No newline at end of file
diff --git a/Doc/library/tk_msg.png b/Doc/library/tk_msg.png
new file mode 100644
index 000000000000..c122d8f8ae5b
Binary files /dev/null and b/Doc/library/tk_msg.png differ
diff --git a/Doc/library/tkinter.colorchooser.rst b/Doc/library/tkinter.colorchooser.rst
new file mode 100644
index 000000000000..60f4d707270d
--- /dev/null
+++ b/Doc/library/tkinter.colorchooser.rst
@@ -0,0 +1,29 @@
+:mod:`tkinter.colorchooser` --- Color choosing dialog
+=====================================================
+
+.. module:: tkinter.colorchooser
+   :platform: Tk
+   :synopsis: Color choosing dialog
+
+**Source code:** :source:`Lib/tkinter/colorchooser.py`
+
+--------------
+
+The :mod:`tkinter.colorchooser` module provides the :class:`Chooser` class
+as an interface to the native color picker dialog. ``Chooser`` implements
+a modal color choosing dialog window. The ``Chooser`` class inherits from
+the :class:`~tkinter.commondialog.Dialog` class.
+
+.. class:: Chooser(master=None, **options)
+
+.. function:: askcolor(color=None, **options)
+
+   Create a color choosing dialog. A call to this method will show the window,
+   wait for the user to make a selection, and return the selected color (or
+   ``None``) to the caller.
+
+
+.. seealso::
+
+   Module :mod:`tkinter.commondialog`
+      Tkinter standard dialog module
\ No newline at end of file
diff --git a/Doc/library/tkinter.dnd.rst b/Doc/library/tkinter.dnd.rst
new file mode 100644
index 000000000000..6c11c739e1fa
--- /dev/null
+++ b/Doc/library/tkinter.dnd.rst
@@ -0,0 +1,64 @@
+:mod:`tkinter.dnd` --- Drag and drop support
+============================================
+
+.. module:: tkinter.dnd
+   :platform: Tk
+   :synopsis: Tkinter drag-and-drop interface
+
+**Source code:** :source:`Lib/tkinter/dnd.py`
+
+--------------
+
+.. note:: This is experimental and due to be deprecated when it is replaced
+   with the Tk DND.
+
+The :mod:`tkinter.dnd` module provides drag-and-drop support for objects within
+a single application, within the same window or between windows. To enable an
+object to be dragged, you must create an event binding for it that starts the
+drag-and-drop process. Typically, you bind a ButtonPress event to a callback
+function that you write (see :ref:`Bindings-and-Events`). The function should
+call :func:`dnd_start`, where 'source' is the object to be dragged, and 'event'
+is the event that invoked the call (the argument to your callback function).
+
+Selection of a target object occurs as follows:
+
+#. Top-down search of area under mouse for target widget
+
+ * Target widget should have a callable *dnd_accept* attribute
+ * If *dnd_accept* is not present or returns None, search moves to parent widget
+ * If no target widget is found, then the target object is None
+
+2. Call to *<old_target>.dnd_leave(source, event)*
+#. Call to *<new_target>.dnd_enter(source, event)*
+#. Call to *<target>.dnd_commit(source, event)* to notify of drop
+#. Call to *<source>.dnd_end(target, event)* to signal end of drag-and-drop
+
+
+.. class:: DndHandler(source, event)
+
+   The *DndHandler* class handles drag-and-drop events tracking Motion and
+   ButtonRelease events on the root of the event widget.
+
+   .. method:: cancel(event=None)
+
+      Cancel the drag-and-drop process.
+
+   .. method:: finish(event, commit=0)
+
+      Execute end of drag-and-drop functions.
+
+   .. method:: on_motion(event)
+
+      Inspect area below mouse for target objects while drag is performed.
+
+   .. method:: on_release(event)
+
+      Signal end of drag when the release pattern is triggered.
+
+.. function:: dnd_start(source, event)
+
+   Factory function for drag-and-drop process.
+
+.. seealso::
+
+   :ref:`Bindings-and-Events`
\ No newline at end of file
diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst
new file mode 100644
index 000000000000..30c1e7b5f9eb
--- /dev/null
+++ b/Doc/library/tkinter.font.rst
@@ -0,0 +1,96 @@
+:mod:`tkinter.font` --- Tkinter font wrapper
+============================================
+
+.. module:: tkinter.font
+   :platform: Tk
+   :synopsis: Tkinter font-wrapping class
+
+**Source code:** :source:`Lib/tkinter/font.py`
+
+--------------
+
+The :mod:`tkinter.font` module provides the :class:`Font` class for creating
+and using named fonts.
+
+The different font weights and slants are:
+
+.. data:: NORMAL
+          BOLD
+          ITALIC
+          ROMAN
+
+.. class:: Font(root=None, font=None, name=None, exists=False, **options)
+
+   The :class:`Font` class represents a named font. *Font* instances are given
+   unique names and can be specified by their family, size, and style
+   configuration. Named fonts are Tk's method of creating and identifying
+   fonts as a single object, rather than specifying a font by its attributes
+   with each occurrence.
+
+    arguments:
+
+       | *font* - font specifier tuple (family, size, options)
+       | *name* - unique font name
+       | *exists* - self points to existing named font if true
+
+    additional keyword options (ignored if *font* is specified):
+
+       | *family* - font family i.e. Courier, Times
+       | *size* - font size
+       |     If *size* is positive it is interpreted as size in points.
+       |     If *size* is a negative number its absolute value is treated as
+             as size in pixels.
+       | *weight* - font emphasis (NORMAL, BOLD)
+       | *slant* - ROMAN, ITALIC
+       | *underline* - font underlining (0 - none, 1 - underline)
+       | *overstrike* - font strikeout (0 - none, 1 - strikeout)
+
+   .. method:: actual(option=None, displayof=None)
+
+      Return the attributes of the font.
+
+   .. method:: cget(option)
+
+      Retrieve an attribute of the font.
+
+   .. method:: config(**options)
+
+      Modify attributes of the font.
+
+   .. method:: copy()
+
+      Return new instance of the current font.
+
+   .. method:: measure(text, displayof=None)
+
+      Return amount of space the text would occupy on the specified display
+      when formatted in the current font. If no display is specified then the
+      main application window is assumed.
+
+   .. method:: metrics(*options, **kw)
+
+      Return font-specific data.
+      Options include:
+
+      *ascent* - distance between baseline and highest point that a
+         character of the font can occupy
+
+      *descent* - distance between baseline and lowest point that a
+         character of the font can occupy
+
+      *linespace* - minimum vertical separation necessary between any two
+         characters of the font that ensures no vertical overlap between lines.
+
+      *fixed* - 1 if font is fixed-width else 0
+
+.. function:: families(root=None, displayof=None)
+
+   Return the different font families.
+
+.. function:: names(root=None)
+
+   Return the names of defined fonts.
+
+.. function:: nametofont(name)
+
+   Return a :class:`Font` representation of a tk named font.
\ No newline at end of file
diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst
new file mode 100644
index 000000000000..872e72f7a7e2
--- /dev/null
+++ b/Doc/library/tkinter.messagebox.rst
@@ -0,0 +1,39 @@
+:mod:`tkinter.messagebox` --- Tkinter message prompts
+=====================================================
+
+.. module:: tkinter.messagebox
+   :platform: Tk
+   :synopsis: Various types of alert dialogs
+
+**Source code:** :source:`Lib/tkinter/messagebox.py`
+
+--------------
+
+The :mod:`tkinter.messagebox` module provides a template base class as well as
+a variety of convenience methods for commonly used configurations. The message
+boxes are modal and will return a subset of (True, False, OK, None, Yes, No) based on
+the user's selection. Common message box styles and layouts include but are not
+limited to:
+
+.. figure:: tk_msg.png
+
+.. class:: Message(master=None, **options)
+
+   Create a default information message box.
+
+**Information message box**
+
+.. method:: showinfo(title=None, message=None, **options)
+
+**Warning message boxes**
+
+.. method:: showwarning(title=None, message=None, **options)
+            showerror(title=None, message=None, **options)
+
+**Question message boxes**
+
+.. method:: askquestion(title=None, message=None, **options)
+            askokcancel(title=None, message=None, **options)
+            askretrycancel(title=None, message=None, **options)
+            askyesno(title=None, message=None, **options)
+            askyesnocancel(title=None, message=None, **options)
\ No newline at end of file
diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst
index e1fc051d9597..2dc44ad36a7f 100644
--- a/Doc/library/tkinter.rst
+++ b/Doc/library/tkinter.rst
@@ -109,9 +109,6 @@ Or, more often::
 
 Other modules that provide Tk support include:
 
-:mod:`tkinter.scrolledtext`
-   Text widget with a vertical scroll bar built in.
-
 :mod:`tkinter.colorchooser`
    Dialog to let the user choose a color.
 
@@ -127,6 +124,9 @@ Other modules that provide Tk support include:
 :mod:`tkinter.messagebox`
    Access to standard Tk dialog boxes.
 
+:mod:`tkinter.scrolledtext`
+   Text widget with a vertical scroll bar built in.
+
 :mod:`tkinter.simpledialog`
    Basic dialogs and convenience functions.
 
@@ -680,9 +680,10 @@ scrollcommand
    This is almost always the :meth:`!set` method of some scrollbar widget, but can
    be any widget method that takes a single argument.
 
-wrap:
+wrap
    Must be one of: ``"none"``, ``"char"``, or ``"word"``.
 
+.. _Bindings-and-Events:
 
 Bindings and Events
 ^^^^^^^^^^^^^^^^^^^
@@ -860,4 +861,4 @@ use raw reads or ``os.read(file.fileno(), maxbytecount)``.
           WRITABLE
           EXCEPTION
 
-   Constants used in the *mask* arguments.
+   Constants used in the *mask* arguments.
\ No newline at end of file
diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst
index 138720e4785f..d20365baa386 100644
--- a/Doc/library/tkinter.scrolledtext.rst
+++ b/Doc/library/tkinter.scrolledtext.rst
@@ -14,8 +14,7 @@
 The :mod:`tkinter.scrolledtext` module provides a class of the same name which
 implements a basic text widget which has a vertical scroll bar configured to do
 the "right thing."  Using the :class:`ScrolledText` class is a lot easier than
-setting up a text widget and scroll bar directly.  The constructor is the same
-as that of the :class:`tkinter.Text` class.
+setting up a text widget and scroll bar directly.
 
 The text widget and scrollbar are packed together in a :class:`Frame`, and the
 methods of the :class:`Grid` and :class:`Pack` geometry managers are acquired
@@ -25,12 +24,14 @@ be used directly to achieve most normal geometry management behavior.
 Should more specific control be necessary, the following attributes are
 available:
 
+.. class:: ScrolledText(master=None, **kw)
 
-.. attribute:: ScrolledText.frame
 
-   The frame which surrounds the text and scroll bar widgets.
+   .. attribute:: frame
 
+      The frame which surrounds the text and scroll bar widgets.
 
-.. attribute:: ScrolledText.vbar
 
-   The scroll bar widget.
+   .. attribute:: vbar
+
+      The scroll bar widget.
diff --git a/Misc/NEWS.d/next/Documentation/2018-06-02-12-55-23.bpo-25237.m8-JMu.rst b/Misc/NEWS.d/next/Documentation/2018-06-02-12-55-23.bpo-25237.m8-JMu.rst
new file mode 100644
index 000000000000..5778f377aeaf
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2018-06-02-12-55-23.bpo-25237.m8-JMu.rst
@@ -0,0 +1 @@
+Add documentation for tkinter modules


From webhook-mailer at python.org  Tue Sep 10 05:02:09 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 09:02:09 -0000
Subject: [Python-checkins] bpo-35941: Fix performance regression in SSL
 certificate code (GH-12610)
Message-ID: <mailman.399.1568106131.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/fdd17abc51e363ab19d248375d717512b8b26966
commit: fdd17abc51e363ab19d248375d717512b8b26966
branch: 3.8
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-10T02:02:04-07:00
summary:

bpo-35941: Fix performance regression in SSL certificate code (GH-12610)

Accumulate certificates in a set instead of doing a costly list contain
operation. A Windows cert store can easily contain over hundred
certificates. The old code would result in way over 5,000 comparison
operations

Signed-off-by: Christian Heimes <christian at python.org>

files:
M Lib/test/test_ssl.py
M Modules/_ssl.c

diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 879d944f327e..48c4fecabd14 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -832,8 +832,8 @@ def test_enum_certificates(self):
                 cert, enc, trust = element
                 self.assertIsInstance(cert, bytes)
                 self.assertIn(enc, {"x509_asn", "pkcs_7_asn"})
-                self.assertIsInstance(trust, (set, bool))
-                if isinstance(trust, set):
+                self.assertIsInstance(trust, (frozenset, set, bool))
+                if isinstance(trust, (frozenset, set)):
                     trust_oids.update(trust)
 
         serverAuth = "1.3.6.1.5.5.7.3.1"
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index 6f91b4882528..64060c64c4df 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -5517,7 +5517,7 @@ parseKeyUsage(PCCERT_CONTEXT pCertCtx, DWORD flags)
         }
         return PyErr_SetFromWindowsErr(error);
     }
-    retval = PySet_New(NULL);
+    retval = PyFrozenSet_New(NULL);
     if (retval == NULL) {
         goto error;
     }
@@ -5592,20 +5592,6 @@ ssl_collect_certificates(const char *store_name)
     return hCollectionStore;
 }
 
-/* code from Objects/listobject.c */
-
-static int
-list_contains(PyListObject *a, PyObject *el)
-{
-    Py_ssize_t i;
-    int cmp;
-
-    for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(a); ++i)
-        cmp = PyObject_RichCompareBool(el, PyList_GET_ITEM(a, i),
-                                           Py_EQ);
-    return cmp;
-}
-
 /*[clinic input]
 _ssl.enum_certificates
     store_name: str
@@ -5628,7 +5614,7 @@ _ssl_enum_certificates_impl(PyObject *module, const char *store_name)
     PyObject *keyusage = NULL, *cert = NULL, *enc = NULL, *tup = NULL;
     PyObject *result = NULL;
 
-    result = PyList_New(0);
+    result = PySet_New(NULL);
     if (result == NULL) {
         return NULL;
     }
@@ -5668,11 +5654,10 @@ _ssl_enum_certificates_impl(PyObject *module, const char *store_name)
         enc = NULL;
         PyTuple_SET_ITEM(tup, 2, keyusage);
         keyusage = NULL;
-        if (!list_contains((PyListObject*)result, tup)) {
-            if (PyList_Append(result, tup) < 0) {
-                Py_CLEAR(result);
-                break;
-            }
+        if (PySet_Add(result, tup) == -1) {
+            Py_CLEAR(result);
+            Py_CLEAR(tup);
+            break;
         }
         Py_CLEAR(tup);
     }
@@ -5696,7 +5681,14 @@ _ssl_enum_certificates_impl(PyObject *module, const char *store_name)
         return PyErr_SetFromWindowsErr(GetLastError());
     }
 
-    return result;
+    /* convert set to list */
+    if (result == NULL) {
+        return NULL;
+    } else {
+        PyObject *lst = PySequence_List(result);
+        Py_DECREF(result);
+        return lst;
+    }
 }
 
 /*[clinic input]
@@ -5720,7 +5712,7 @@ _ssl_enum_crls_impl(PyObject *module, const char *store_name)
     PyObject *crl = NULL, *enc = NULL, *tup = NULL;
     PyObject *result = NULL;
 
-    result = PyList_New(0);
+    result = PySet_New(NULL);
     if (result == NULL) {
         return NULL;
     }
@@ -5750,11 +5742,10 @@ _ssl_enum_crls_impl(PyObject *module, const char *store_name)
         PyTuple_SET_ITEM(tup, 1, enc);
         enc = NULL;
 
-        if (!list_contains((PyListObject*)result, tup)) {
-            if (PyList_Append(result, tup) < 0) {
-                Py_CLEAR(result);
-                break;
-            }
+        if (PySet_Add(result, tup) == -1) {
+            Py_CLEAR(result);
+            Py_CLEAR(tup);
+            break;
         }
         Py_CLEAR(tup);
     }
@@ -5776,7 +5767,14 @@ _ssl_enum_crls_impl(PyObject *module, const char *store_name)
         Py_XDECREF(result);
         return PyErr_SetFromWindowsErr(GetLastError());
     }
-    return result;
+    /* convert set to list */
+    if (result == NULL) {
+        return NULL;
+    } else {
+        PyObject *lst = PySequence_List(result);
+        Py_DECREF(result);
+        return lst;
+    }
 }
 
 #endif /* _MSC_VER */


From webhook-mailer at python.org  Tue Sep 10 05:22:17 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Tue, 10 Sep 2019 09:22:17 -0000
Subject: [Python-checkins] Expand comment explaining update_one_slot
 (GH-14810)
Message-ID: <mailman.400.1568107337.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5b00dd8fa8e28acbf1ad7ce6da52464f33dbf991
commit: 5b00dd8fa8e28acbf1ad7ce6da52464f33dbf991
branch: master
author: Jeroen Demeyer <J.Demeyer at UGent.be>
committer: T. Wouters <thomas at python.org>
date: 2019-09-10T02:22:05-07:00
summary:

Expand comment explaining update_one_slot (GH-14810)

files:
M Objects/typeobject.c

diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 1d8216d29887..71d7f6f0e5c5 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -7197,8 +7197,9 @@ resolve_slotdups(PyTypeObject *type, PyObject *name)
         *pp = NULL;
     }
 
-    /* Look in all matching slots of the type; if exactly one of these has
-       a filled-in slot, return its value.      Otherwise return NULL. */
+    /* Look in all slots of the type matching the name. If exactly one of these
+       has a filled-in slot, return a pointer to that slot.
+       Otherwise, return NULL. */
     res = NULL;
     for (pp = ptrs; *pp; pp++) {
         ptr = slotptr(type, (*pp)->offset);
@@ -7211,12 +7212,61 @@ resolve_slotdups(PyTypeObject *type, PyObject *name)
     return res;
 }
 
-/* Common code for update_slots_callback() and fixup_slot_dispatchers().  This
-   does some incredibly complex thinking and then sticks something into the
-   slot.  (It sees if the adjacent slotdefs for the same slot have conflicting
-   interests, and then stores a generic wrapper or a specific function into
-   the slot.)  Return a pointer to the next slotdef with a different offset,
-   because that's convenient  for fixup_slot_dispatchers(). */
+
+/* Common code for update_slots_callback() and fixup_slot_dispatchers().
+ *
+ * This is meant to set a "slot" like type->tp_repr or
+ * type->tp_as_sequence->sq_concat by looking up special methods like
+ * __repr__ or __add__. The opposite (adding special methods from slots) is
+ * done by add_operators(), called from PyType_Ready(). Since update_one_slot()
+ * calls PyType_Ready() if needed, the special methods are already in place.
+ *
+ * The special methods corresponding to each slot are defined in the "slotdef"
+ * array. Note that one slot may correspond to multiple special methods and vice
+ * versa. For example, tp_richcompare uses 6 methods __lt__, ..., __ge__ and
+ * tp_as_number->nb_add uses __add__ and __radd__. In the other direction,
+ * __add__ is used by the number and sequence protocols and __getitem__ by the
+ * sequence and mapping protocols. This causes a lot of complications.
+ *
+ * In detail, update_one_slot() does the following:
+ *
+ * First of all, if the slot in question does not exist, return immediately.
+ * This can happen for example if it's tp_as_number->nb_add but tp_as_number
+ * is NULL.
+ *
+ * For the given slot, we loop over all the special methods with a name
+ * corresponding to that slot (for example, for tp_descr_set, this would be
+ * __set__ and __delete__) and we look up these names in the MRO of the type.
+ * If we don't find any special method, the slot is set to NULL (regardless of
+ * what was in the slot before).
+ *
+ * Suppose that we find exactly one special method. If it's a wrapper_descriptor
+ * (i.e. a special method calling a slot, for example str.__repr__ which calls
+ * the tp_repr for the 'str' class) with the correct name ("__repr__" for
+ * tp_repr), for the right class, calling the right wrapper C function (like
+ * wrap_unaryfunc for tp_repr), then the slot is set to the slot that the
+ * wrapper_descriptor originally wrapped. For example, a class inheriting
+ * from 'str' and not redefining __repr__ will have tp_repr set to the tp_repr
+ * of 'str'.
+ * In all other cases where the special method exists, the slot is set to a
+ * wrapper calling the special method. There is one exception: if the special
+ * method is a wrapper_descriptor with the correct name but the type has
+ * precisely one slot set for that name and that slot is not the one that we
+ * are updating, then NULL is put in the slot (this exception is the only place
+ * in update_one_slot() where the *existing* slots matter).
+ *
+ * When there are multiple special methods for the same slot, the above is
+ * applied for each special method. As long as the results agree, the common
+ * resulting slot is applied. If the results disagree, then a wrapper for
+ * the special methods is installed. This is always safe, but less efficient
+ * because it uses method lookup instead of direct C calls.
+ *
+ * There are some further special cases for specific slots, like supporting
+ * __hash__ = None for tp_hash and special code for tp_new.
+ *
+ * When done, return a pointer to the next slotdef with a different offset,
+ * because that's convenient for fixup_slot_dispatchers(). This function never
+ * sets an exception: if an internal error happens (unlikely), it's ignored. */
 static slotdef *
 update_one_slot(PyTypeObject *type, slotdef *p)
 {
@@ -7241,7 +7291,7 @@ update_one_slot(PyTypeObject *type, slotdef *p)
         descr = find_name_in_mro(type, p->name_strobj, &error);
         if (descr == NULL) {
             if (error == -1) {
-                /* It is unlikely by not impossible that there has been an exception
+                /* It is unlikely but not impossible that there has been an exception
                    during lookup. Since this function originally expected no errors,
                    we ignore them here in order to keep up the interface. */
                 PyErr_Clear();


From webhook-mailer at python.org  Tue Sep 10 05:29:30 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Tue, 10 Sep 2019 09:29:30 -0000
Subject: [Python-checkins] bpo-38043: Move unicodedata.normalize tests into
 test_unicodedata. (GH-15712)
Message-ID: <mailman.401.1568107771.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1ad0c776cb640be9f19c8019bbf34bb4aba312ad
commit: 1ad0c776cb640be9f19c8019bbf34bb4aba312ad
branch: master
author: Greg Price <gnprice at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-10T10:29:26+01:00
summary:

bpo-38043: Move unicodedata.normalize tests into test_unicodedata. (GH-15712)

Having these in a separate file from the one that's named after the
module in the usual way makes it very easy to miss them when looking
for tests for these two functions.

(In fact when working recently on is_normalized, I'd been surprised to
see no tests for it here and concluded the function had evaded being
tested at all.  I'd gone as far as to write up some tests myself
before I spotted this other file.)

Mostly this just means moving all the one file's code into the other,
and moving code from the module toplevel to inside the test class to
keep it tidily separate from the rest of the file's code.

There's one substantive change, which reduces by a bit the amount of
code to be moved: we drop the `x > sys.maxunicode` conditional and all
the `RangeError` logic behind it.  Now if that condition ever occurs
it will cause an error at `chr(x)`, and a test failure.  That's the
right result because, since PEP 393 in Python 3.3, there is no longer
such a thing as an "unsupported character".

files:
D Lib/test/test_normalization.py
M Lib/test/test_ucn.py
M Lib/test/test_unicodedata.py
M PCbuild/lib.pyproj

diff --git a/Lib/test/test_normalization.py b/Lib/test/test_normalization.py
deleted file mode 100644
index ba877e73f7d9..000000000000
--- a/Lib/test/test_normalization.py
+++ /dev/null
@@ -1,117 +0,0 @@
-from test.support import open_urlresource
-import unittest
-
-from http.client import HTTPException
-import sys
-from unicodedata import normalize, is_normalized, unidata_version
-
-TESTDATAFILE = "NormalizationTest.txt"
-TESTDATAURL = "http://www.pythontest.net/unicode/" + unidata_version + "/" + TESTDATAFILE
-
-def check_version(testfile):
-    hdr = testfile.readline()
-    return unidata_version in hdr
-
-class RangeError(Exception):
-    pass
-
-def NFC(str):
-    return normalize("NFC", str)
-
-def NFKC(str):
-    return normalize("NFKC", str)
-
-def NFD(str):
-    return normalize("NFD", str)
-
-def NFKD(str):
-    return normalize("NFKD", str)
-
-def unistr(data):
-    data = [int(x, 16) for x in data.split(" ")]
-    for x in data:
-        if x > sys.maxunicode:
-            raise RangeError
-    return "".join([chr(x) for x in data])
-
-class NormalizationTest(unittest.TestCase):
-    def test_main(self):
-        # Hit the exception early
-        try:
-            testdata = open_urlresource(TESTDATAURL, encoding="utf-8",
-                                        check=check_version)
-        except PermissionError:
-            self.skipTest(f"Permission error when downloading {TESTDATAURL} "
-                          f"into the test data directory")
-        except (OSError, HTTPException):
-            self.fail(f"Could not retrieve {TESTDATAURL}")
-
-        with testdata:
-            self.run_normalization_tests(testdata)
-
-    def run_normalization_tests(self, testdata):
-        part = None
-        part1_data = {}
-
-        for line in testdata:
-            if '#' in line:
-                line = line.split('#')[0]
-            line = line.strip()
-            if not line:
-                continue
-            if line.startswith("@Part"):
-                part = line.split()[0]
-                continue
-            try:
-                c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]]
-            except RangeError:
-                # Skip unsupported characters;
-                # try at least adding c1 if we are in part1
-                if part == "@Part1":
-                    try:
-                        c1 = unistr(line.split(';')[0])
-                    except RangeError:
-                        pass
-                    else:
-                        part1_data[c1] = 1
-                continue
-
-            # Perform tests
-            self.assertTrue(c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3), line)
-            self.assertTrue(c4 ==  NFC(c4) ==  NFC(c5), line)
-            self.assertTrue(c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3), line)
-            self.assertTrue(c5 ==  NFD(c4) ==  NFD(c5), line)
-            self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \
-                            NFKC(c3) == NFKC(c4) == NFKC(c5),
-                            line)
-            self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \
-                            NFKD(c3) == NFKD(c4) == NFKD(c5),
-                            line)
-
-            self.assertTrue(is_normalized("NFC", c2))
-            self.assertTrue(is_normalized("NFC", c4))
-
-            self.assertTrue(is_normalized("NFD", c3))
-            self.assertTrue(is_normalized("NFD", c5))
-
-            self.assertTrue(is_normalized("NFKC", c4))
-            self.assertTrue(is_normalized("NFKD", c5))
-
-            # Record part 1 data
-            if part == "@Part1":
-                part1_data[c1] = 1
-
-        # Perform tests for all other data
-        for c in range(sys.maxunicode+1):
-            X = chr(c)
-            if X in part1_data:
-                continue
-            self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c)
-
-    def test_bug_834676(self):
-        # Check for bug 834676
-        normalize('NFC', '\ud55c\uae00')
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/Lib/test/test_ucn.py b/Lib/test/test_ucn.py
index 8febf0af8627..503df6a51d5c 100644
--- a/Lib/test/test_ucn.py
+++ b/Lib/test/test_ucn.py
@@ -12,7 +12,6 @@
 
 from test import support
 from http.client import HTTPException
-from test.test_normalization import check_version
 
 try:
     from _testcapi import INT_MAX, PY_SSIZE_T_MAX, UINT_MAX
@@ -172,6 +171,9 @@ def test_named_sequences_sample(self):
 
     def test_named_sequences_full(self):
         # Check all the named sequences
+        def check_version(testfile):
+            hdr = testfile.readline()
+            return unicodedata.unidata_version in hdr
         url = ("http://www.pythontest.net/unicode/%s/NamedSequences.txt" %
                unicodedata.unidata_version)
         try:
diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py
index 9ec2f1149726..40c38c1c4262 100644
--- a/Lib/test/test_unicodedata.py
+++ b/Lib/test/test_unicodedata.py
@@ -7,10 +7,11 @@
 """
 
 import hashlib
+from http.client import HTTPException
 import sys
 import unicodedata
 import unittest
-from test.support import script_helper
+from test.support import open_urlresource, script_helper
 
 
 class UnicodeMethodsTest(unittest.TestCase):
@@ -171,13 +172,6 @@ def test_combining(self):
         self.assertRaises(TypeError, self.db.combining)
         self.assertRaises(TypeError, self.db.combining, 'xx')
 
-    def test_normalize(self):
-        self.assertRaises(TypeError, self.db.normalize)
-        self.assertRaises(ValueError, self.db.normalize, 'unknown', 'xx')
-        self.assertEqual(self.db.normalize('NFKC', ''), '')
-        # The rest can be found in test_normalization.py
-        # which requires an external file.
-
     def test_pr29(self):
         # http://www.unicode.org/review/pr-29.html
         # See issues #1054943 and #10254.
@@ -208,9 +202,6 @@ def test_issue29456(self):
         self.assertEqual(self.db.normalize('NFC', u11a7_str_a), u11a7_str_b)
         self.assertEqual(self.db.normalize('NFC', u11c3_str_a), u11c3_str_b)
 
-    # For tests of unicodedata.is_normalized / self.db.is_normalized ,
-    # see test_normalization.py .
-
     def test_east_asian_width(self):
         eaw = self.db.east_asian_width
         self.assertRaises(TypeError, eaw, b'a')
@@ -315,5 +306,102 @@ def test_linebreak_7643(self):
                 self.assertEqual(len(lines), 1,
                                  r"\u%.4x should not be a linebreak" % i)
 
+class NormalizationTest(unittest.TestCase):
+    @staticmethod
+    def check_version(testfile):
+        hdr = testfile.readline()
+        return unicodedata.unidata_version in hdr
+
+    @staticmethod
+    def unistr(data):
+        data = [int(x, 16) for x in data.split(" ")]
+        return "".join([chr(x) for x in data])
+
+    def test_normalization(self):
+        TESTDATAFILE = "NormalizationTest.txt"
+        TESTDATAURL = f"http://www.pythontest.net/unicode/{unicodedata.unidata_version}/{TESTDATAFILE}"
+
+        # Hit the exception early
+        try:
+            testdata = open_urlresource(TESTDATAURL, encoding="utf-8",
+                                        check=self.check_version)
+        except PermissionError:
+            self.skipTest(f"Permission error when downloading {TESTDATAURL} "
+                          f"into the test data directory")
+        except (OSError, HTTPException):
+            self.fail(f"Could not retrieve {TESTDATAURL}")
+
+        with testdata:
+            self.run_normalization_tests(testdata)
+
+    def run_normalization_tests(self, testdata):
+        part = None
+        part1_data = {}
+
+        def NFC(str):
+            return unicodedata.normalize("NFC", str)
+
+        def NFKC(str):
+            return unicodedata.normalize("NFKC", str)
+
+        def NFD(str):
+            return unicodedata.normalize("NFD", str)
+
+        def NFKD(str):
+            return unicodedata.normalize("NFKD", str)
+
+        for line in testdata:
+            if '#' in line:
+                line = line.split('#')[0]
+            line = line.strip()
+            if not line:
+                continue
+            if line.startswith("@Part"):
+                part = line.split()[0]
+                continue
+            c1,c2,c3,c4,c5 = [self.unistr(x) for x in line.split(';')[:-1]]
+
+            # Perform tests
+            self.assertTrue(c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3), line)
+            self.assertTrue(c4 ==  NFC(c4) ==  NFC(c5), line)
+            self.assertTrue(c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3), line)
+            self.assertTrue(c5 ==  NFD(c4) ==  NFD(c5), line)
+            self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \
+                            NFKC(c3) == NFKC(c4) == NFKC(c5),
+                            line)
+            self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \
+                            NFKD(c3) == NFKD(c4) == NFKD(c5),
+                            line)
+
+            self.assertTrue(unicodedata.is_normalized("NFC", c2))
+            self.assertTrue(unicodedata.is_normalized("NFC", c4))
+
+            self.assertTrue(unicodedata.is_normalized("NFD", c3))
+            self.assertTrue(unicodedata.is_normalized("NFD", c5))
+
+            self.assertTrue(unicodedata.is_normalized("NFKC", c4))
+            self.assertTrue(unicodedata.is_normalized("NFKD", c5))
+
+            # Record part 1 data
+            if part == "@Part1":
+                part1_data[c1] = 1
+
+        # Perform tests for all other data
+        for c in range(sys.maxunicode+1):
+            X = chr(c)
+            if X in part1_data:
+                continue
+            self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c)
+
+    def test_edge_cases(self):
+        self.assertRaises(TypeError, unicodedata.normalize)
+        self.assertRaises(ValueError, unicodedata.normalize, 'unknown', 'xx')
+        self.assertEqual(unicodedata.normalize('NFKC', ''), '')
+
+    def test_bug_834676(self):
+        # Check for bug 834676
+        unicodedata.normalize('NFC', '\ud55c\uae00')
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/PCbuild/lib.pyproj b/PCbuild/lib.pyproj
index 0ddeef3eaa3b..401e207ae57e 100644
--- a/PCbuild/lib.pyproj
+++ b/PCbuild/lib.pyproj
@@ -1191,7 +1191,6 @@
     <Compile Include="test\test_netrc.py" />
     <Compile Include="test\test_nis.py" />
     <Compile Include="test\test_nntplib.py" />
-    <Compile Include="test\test_normalization.py" />
     <Compile Include="test\test_ntpath.py" />
     <Compile Include="test\test_numeric_tower.py" />
     <Compile Include="test\test_opcodes.py" />


From webhook-mailer at python.org  Tue Sep 10 05:44:24 2019
From: webhook-mailer at python.org (Dino Viehland)
Date: Tue, 10 Sep 2019 09:44:24 -0000
Subject: [Python-checkins] bpo-37725: have "make clean" remove PGO task data
 (#15033)
Message-ID: <mailman.402.1568108665.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c6bbcd258302b4b9b3d4f3c39bb5f7ff0120ec67
commit: c6bbcd258302b4b9b3d4f3c39bb5f7ff0120ec67
branch: master
author: Neil Schemenauer <nas-github at arctrix.com>
committer: Dino Viehland <dinoviehland at gmail.com>
date: 2019-09-10T10:44:20+01:00
summary:

bpo-37725: have "make clean" remove PGO task data (#15033)

Change "clean" makefile target to also clean the program guided
optimization (PGO) data.  Previously you would have to use "make
clean" and "make profile-removal", or "make clobber".

files:
A Misc/NEWS.d/next/Build/2019-07-30-16-26-11.bpo-37725.MkG1TT.rst
M Makefile.pre.in

diff --git a/Makefile.pre.in b/Makefile.pre.in
index dbf95fdcd181..4f4f096e7283 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -461,7 +461,7 @@ check-clean-src:
 
 # Profile generation build must start from a clean tree.
 profile-clean-stamp:
-	$(MAKE) clean profile-removal
+	$(MAKE) clean
 	touch $@
 
 # Compile with profile generation enabled.
@@ -485,7 +485,7 @@ profile-run-stamp:
 	$(MAKE) run_profile_task
 	$(MAKE) build_all_merge_profile
 	# Remove profile generation binary since we are done with it.
-	$(MAKE) clean
+	$(MAKE) clean-retain-profile
 	# This is an expensive target to build and it does not have proper
 	# makefile dependency information.  So, we create a "stamp" file
 	# to record its completion and avoid re-running it.
@@ -512,7 +512,7 @@ profile-opt: profile-run-stamp
 .PHONY=coverage coverage-lcov coverage-report
 coverage:
 	@echo "Building with support for coverage checking:"
-	$(MAKE) clean profile-removal
+	$(MAKE) clean
 	$(MAKE) @DEF_MAKE_RULE@ CFLAGS="$(CFLAGS) -O0 -pg -fprofile-arcs -ftest-coverage" LIBS="$(LIBS) -lgcov"
 
 coverage-lcov:
@@ -1752,7 +1752,9 @@ docclean:
 	-rm -rf Doc/build
 	-rm -rf Doc/tools/sphinx Doc/tools/pygments Doc/tools/docutils
 
-clean: pycremoval
+# like the 'clean' target but retain the profile guided optimization (PGO)
+# data.  The PGO data is only valid if source code remains unchanged.
+clean-retain-profile: pycremoval
 	find . -name '*.[oa]' -exec rm -f {} ';'
 	find . -name '*.s[ol]' -exec rm -f {} ';'
 	find . -name '*.so.[0-9]*.[0-9]*' -exec rm -f {} ';'
@@ -1774,14 +1776,19 @@ profile-removal:
 	rm -rf $(COVERAGE_REPORT)
 	rm -f profile-run-stamp
 
-clobber: clean profile-removal
+clean: clean-retain-profile
+	@if test @DEF_MAKE_ALL_RULE@ = profile-opt; then \
+		rm -f profile-gen-stamp profile-clean-stamp; \
+		$(MAKE) profile-removal; \
+	fi
+
+clobber: clean
 	-rm -f $(BUILDPYTHON) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \
 		tags TAGS \
 		config.cache config.log pyconfig.h Modules/config.c
 	-rm -rf build platform
 	-rm -rf $(PYTHONFRAMEWORKDIR)
 	-rm -f python-config.py python-config
-	-rm -f profile-gen-stamp profile-clean-stamp
 
 # Make things extra clean, before making a distribution:
 # remove all generated files, even Makefile[.pre]
@@ -1855,6 +1862,8 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h
 .PHONY: frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools
 .PHONY: frameworkaltinstallunixtools recheck clean clobber distclean
 .PHONY: smelly funny patchcheck touch altmaninstall commoninstall
+.PHONY: clean-retain-profile profile-removal run_profile_task
+.PHONY: build_all_generate_profile build_all_merge_profile
 .PHONY: gdbhooks
 
 # IF YOU PUT ANYTHING HERE IT WILL GO AWAY
diff --git a/Misc/NEWS.d/next/Build/2019-07-30-16-26-11.bpo-37725.MkG1TT.rst b/Misc/NEWS.d/next/Build/2019-07-30-16-26-11.bpo-37725.MkG1TT.rst
new file mode 100644
index 000000000000..1687cf223c41
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-07-30-16-26-11.bpo-37725.MkG1TT.rst
@@ -0,0 +1,3 @@
+Change "clean" makefile target to also clean the program guided optimization
+(PGO) data.  Previously you would have to use "make clean" and "make
+profile-removal", or "make clobber".


From webhook-mailer at python.org  Tue Sep 10 05:45:11 2019
From: webhook-mailer at python.org (Dino Viehland)
Date: Tue, 10 Sep 2019 09:45:11 -0000
Subject: [Python-checkins] Fix typo in dict object comment (#15814)
Message-ID: <mailman.403.1568108712.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/359143c68659d165f52320d368667e0eff279dc5
commit: 359143c68659d165f52320d368667e0eff279dc5
branch: master
author: dalgarno <32097481+dalgarno at users.noreply.github.com>
committer: Dino Viehland <dinoviehland at gmail.com>
date: 2019-09-10T10:45:07+01:00
summary:

Fix typo in dict object comment (#15814)

files:
M Objects/dictobject.c

diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 73956df4ab2a..5c0640d1dfc8 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -39,7 +39,7 @@ Size of indices is dk_size.  Type of each index in indices is vary on dk_size:
 * int32 for 2**16 <= dk_size <= 2**31
 * int64 for 2**32 <= dk_size
 
-dk_entries is array of PyDictKeyEntry.  It's size is USABLE_FRACTION(dk_size).
+dk_entries is array of PyDictKeyEntry.  Its size is USABLE_FRACTION(dk_size).
 DK_ENTRIES(dk) can be used to get pointer to entries.
 
 NOTE: Since negative value is used for DKIX_EMPTY and DKIX_DUMMY, type of


From webhook-mailer at python.org  Tue Sep 10 05:57:35 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Tue, 10 Sep 2019 09:57:35 -0000
Subject: [Python-checkins] Revert "bpo-33418: Add tp_clear for function
 object (GH-8058)" (GH-15826)
Message-ID: <mailman.404.1568109456.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ccaea525885e41c5f1e566bb68698847faaa82ca
commit: ccaea525885e41c5f1e566bb68698847faaa82ca
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-10T02:57:31-07:00
summary:

Revert "bpo-33418: Add tp_clear for function object (GH-8058)" (GH-15826)

This reverts commit 3c452404ae178b742967589a0bb4a5ec768d76e0.

files:
M Objects/funcobject.c

diff --git a/Objects/funcobject.c b/Objects/funcobject.c
index df5cc2d3f570..53aebf12f550 100644
--- a/Objects/funcobject.c
+++ b/Objects/funcobject.c
@@ -570,31 +570,23 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals,
     return (PyObject *)newfunc;
 }
 
-static int
-func_clear(PyFunctionObject *op)
-{
-    Py_CLEAR(op->func_code);
-    Py_CLEAR(op->func_globals);
-    Py_CLEAR(op->func_module);
-    Py_CLEAR(op->func_name);
-    Py_CLEAR(op->func_defaults);
-    Py_CLEAR(op->func_kwdefaults);
-    Py_CLEAR(op->func_doc);
-    Py_CLEAR(op->func_dict);
-    Py_CLEAR(op->func_closure);
-    Py_CLEAR(op->func_annotations);
-    Py_CLEAR(op->func_qualname);
-    return 0;
-}
-
 static void
 func_dealloc(PyFunctionObject *op)
 {
     _PyObject_GC_UNTRACK(op);
-    if (op->func_weakreflist != NULL) {
+    if (op->func_weakreflist != NULL)
         PyObject_ClearWeakRefs((PyObject *) op);
-    }
-    (void)func_clear(op);
+    Py_DECREF(op->func_code);
+    Py_DECREF(op->func_globals);
+    Py_XDECREF(op->func_module);
+    Py_DECREF(op->func_name);
+    Py_XDECREF(op->func_defaults);
+    Py_XDECREF(op->func_kwdefaults);
+    Py_XDECREF(op->func_doc);
+    Py_XDECREF(op->func_dict);
+    Py_XDECREF(op->func_closure);
+    Py_XDECREF(op->func_annotations);
+    Py_XDECREF(op->func_qualname);
     PyObject_GC_Del(op);
 }
 
@@ -669,7 +661,7 @@ PyTypeObject PyFunction_Type = {
     Py_TPFLAGS_METHOD_DESCRIPTOR,               /* tp_flags */
     func_new__doc__,                            /* tp_doc */
     (traverseproc)func_traverse,                /* tp_traverse */
-    (inquiry)func_clear,                        /* tp_clear */
+    0,                                          /* tp_clear */
     0,                                          /* tp_richcompare */
     offsetof(PyFunctionObject, func_weakreflist), /* tp_weaklistoffset */
     0,                                          /* tp_iter */


From webhook-mailer at python.org  Tue Sep 10 06:02:34 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Tue, 10 Sep 2019 10:02:34 -0000
Subject: [Python-checkins] bpo-38083: Minor improvements in asdl_c.py and
 Python-ast.c. (GH-15824)
Message-ID: <mailman.405.1568109755.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/43c9731334470573209464b7f67914d386457efc
commit: 43c9731334470573209464b7f67914d386457efc
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-10T03:02:30-07:00
summary:

bpo-38083: Minor improvements in asdl_c.py and Python-ast.c. (GH-15824)

* Use the const qualifier for constant C strings.
* Intern field and attribute names.
* Temporary incref a borrowed reference to a list item.

files:
M Parser/asdl_c.py
M Python/Python-ast.c

diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index f4fa271b6595..9b18bb4e1bf8 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -537,8 +537,11 @@ def visitField(self, field, name, sum=None, prod=None, depth=0):
             self.emit("if (%s == NULL) goto failed;" % field.name, depth+1)
             self.emit("for (i = 0; i < len; i++) {", depth+1)
             self.emit("%s val;" % ctype, depth+2)
-            self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &val, arena);" %
+            self.emit("PyObject *tmp2 = PyList_GET_ITEM(tmp, i);", depth+2)
+            self.emit("Py_INCREF(tmp2);", depth+2)
+            self.emit("res = obj2ast_%s(tmp2, &val, arena);" %
                       field.type, depth+2, reflow=False)
+            self.emit("Py_DECREF(tmp2);", depth+2)
             self.emit("if (res != 0) goto failed;", depth+2)
             self.emit("if (len != PyList_GET_SIZE(tmp)) {", depth+2)
             self.emit("PyErr_SetString(PyExc_RuntimeError, \"%s field \\\"%s\\\" "
@@ -576,14 +579,14 @@ def visitProduct(self, prod, name):
         if prod.attributes:
             for a in prod.attributes:
                 self.emit_identifier(a.name)
-            self.emit("static char *%s_attributes[] = {" % name, 0)
+            self.emit("static const char * const %s_attributes[] = {" % name, 0)
             for a in prod.attributes:
                 self.emit('"%s",' % a.name, 1)
             self.emit("};", 0)
         if prod.fields:
             for f in prod.fields:
                 self.emit_identifier(f.name)
-            self.emit("static char *%s_fields[]={" % name,0)
+            self.emit("static const char * const %s_fields[]={" % name,0)
             for f in prod.fields:
                 self.emit('"%s",' % f.name, 1)
             self.emit("};", 0)
@@ -593,7 +596,7 @@ def visitSum(self, sum, name):
         if sum.attributes:
             for a in sum.attributes:
                 self.emit_identifier(a.name)
-            self.emit("static char *%s_attributes[] = {" % name, 0)
+            self.emit("static const char * const %s_attributes[] = {" % name, 0)
             for a in sum.attributes:
                 self.emit('"%s",' % a.name, 1)
             self.emit("};", 0)
@@ -614,7 +617,7 @@ def visitConstructor(self, cons, name):
         if cons.fields:
             for t in cons.fields:
                 self.emit_identifier(t.name)
-            self.emit("static char *%s_fields[]={" % cons.name, 0)
+            self.emit("static const char * const %s_fields[]={" % cons.name, 0)
             for t in cons.fields:
                 self.emit('"%s",' % t.name, 1)
             self.emit("};",0)
@@ -771,7 +774,8 @@ def visitModule(self, mod):
 };
 
 
-static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields)
+static PyTypeObject *
+make_type(const char *type, PyTypeObject *base, const char * const *fields, int num_fields)
 {
     _Py_IDENTIFIER(__module__);
     _Py_IDENTIFIER(_ast);
@@ -780,7 +784,7 @@ def visitModule(self, mod):
     fnames = PyTuple_New(num_fields);
     if (!fnames) return NULL;
     for (i = 0; i < num_fields; i++) {
-        PyObject *field = PyUnicode_FromString(fields[i]);
+        PyObject *field = PyUnicode_InternFromString(fields[i]);
         if (!field) {
             Py_DECREF(fnames);
             return NULL;
@@ -796,14 +800,15 @@ def visitModule(self, mod):
     return (PyTypeObject*)result;
 }
 
-static int add_attributes(PyTypeObject* type, char**attrs, int num_fields)
+static int
+add_attributes(PyTypeObject *type, const char * const *attrs, int num_fields)
 {
     int i, result;
     PyObject *s, *l = PyTuple_New(num_fields);
     if (!l)
         return 0;
     for (i = 0; i < num_fields; i++) {
-        s = PyUnicode_FromString(attrs[i]);
+        s = PyUnicode_InternFromString(attrs[i]);
         if (!s) {
             Py_DECREF(l);
             return 0;
@@ -1193,7 +1198,7 @@ class PartingShots(StaticVisitor):
 {
     mod_ty res;
     PyObject *req_type[3];
-    char *req_name[] = {"Module", "Expression", "Interactive"};
+    const char * const req_name[] = {"Module", "Expression", "Interactive"};
     int isinstance;
 
     if (PySys_Audit("compile", "OO", ast, Py_None) < 0) {
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index fd96964a1e77..44ad53275728 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -11,27 +11,27 @@ static PyObject* ast2obj_mod(void*);
 static PyTypeObject *Module_type;
 _Py_IDENTIFIER(body);
 _Py_IDENTIFIER(type_ignores);
-static char *Module_fields[]={
+static const char * const Module_fields[]={
     "body",
     "type_ignores",
 };
 static PyTypeObject *Interactive_type;
-static char *Interactive_fields[]={
+static const char * const Interactive_fields[]={
     "body",
 };
 static PyTypeObject *Expression_type;
-static char *Expression_fields[]={
+static const char * const Expression_fields[]={
     "body",
 };
 static PyTypeObject *FunctionType_type;
 _Py_IDENTIFIER(argtypes);
 _Py_IDENTIFIER(returns);
-static char *FunctionType_fields[]={
+static const char * const FunctionType_fields[]={
     "argtypes",
     "returns",
 };
 static PyTypeObject *Suite_type;
-static char *Suite_fields[]={
+static const char * const Suite_fields[]={
     "body",
 };
 static PyTypeObject *stmt_type;
@@ -39,7 +39,7 @@ _Py_IDENTIFIER(lineno);
 _Py_IDENTIFIER(col_offset);
 _Py_IDENTIFIER(end_lineno);
 _Py_IDENTIFIER(end_col_offset);
-static char *stmt_attributes[] = {
+static const char * const stmt_attributes[] = {
     "lineno",
     "col_offset",
     "end_lineno",
@@ -51,7 +51,7 @@ _Py_IDENTIFIER(name);
 _Py_IDENTIFIER(args);
 _Py_IDENTIFIER(decorator_list);
 _Py_IDENTIFIER(type_comment);
-static char *FunctionDef_fields[]={
+static const char * const FunctionDef_fields[]={
     "name",
     "args",
     "body",
@@ -60,7 +60,7 @@ static char *FunctionDef_fields[]={
     "type_comment",
 };
 static PyTypeObject *AsyncFunctionDef_type;
-static char *AsyncFunctionDef_fields[]={
+static const char * const AsyncFunctionDef_fields[]={
     "name",
     "args",
     "body",
@@ -71,7 +71,7 @@ static char *AsyncFunctionDef_fields[]={
 static PyTypeObject *ClassDef_type;
 _Py_IDENTIFIER(bases);
 _Py_IDENTIFIER(keywords);
-static char *ClassDef_fields[]={
+static const char * const ClassDef_fields[]={
     "name",
     "bases",
     "keywords",
@@ -80,16 +80,16 @@ static char *ClassDef_fields[]={
 };
 static PyTypeObject *Return_type;
 _Py_IDENTIFIER(value);
-static char *Return_fields[]={
+static const char * const Return_fields[]={
     "value",
 };
 static PyTypeObject *Delete_type;
 _Py_IDENTIFIER(targets);
-static char *Delete_fields[]={
+static const char * const Delete_fields[]={
     "targets",
 };
 static PyTypeObject *Assign_type;
-static char *Assign_fields[]={
+static const char * const Assign_fields[]={
     "targets",
     "value",
     "type_comment",
@@ -97,7 +97,7 @@ static char *Assign_fields[]={
 static PyTypeObject *AugAssign_type;
 _Py_IDENTIFIER(target);
 _Py_IDENTIFIER(op);
-static char *AugAssign_fields[]={
+static const char * const AugAssign_fields[]={
     "target",
     "op",
     "value",
@@ -105,7 +105,7 @@ static char *AugAssign_fields[]={
 static PyTypeObject *AnnAssign_type;
 _Py_IDENTIFIER(annotation);
 _Py_IDENTIFIER(simple);
-static char *AnnAssign_fields[]={
+static const char * const AnnAssign_fields[]={
     "target",
     "annotation",
     "value",
@@ -114,7 +114,7 @@ static char *AnnAssign_fields[]={
 static PyTypeObject *For_type;
 _Py_IDENTIFIER(iter);
 _Py_IDENTIFIER(orelse);
-static char *For_fields[]={
+static const char * const For_fields[]={
     "target",
     "iter",
     "body",
@@ -122,7 +122,7 @@ static char *For_fields[]={
     "type_comment",
 };
 static PyTypeObject *AsyncFor_type;
-static char *AsyncFor_fields[]={
+static const char * const AsyncFor_fields[]={
     "target",
     "iter",
     "body",
@@ -131,26 +131,26 @@ static char *AsyncFor_fields[]={
 };
 static PyTypeObject *While_type;
 _Py_IDENTIFIER(test);
-static char *While_fields[]={
+static const char * const While_fields[]={
     "test",
     "body",
     "orelse",
 };
 static PyTypeObject *If_type;
-static char *If_fields[]={
+static const char * const If_fields[]={
     "test",
     "body",
     "orelse",
 };
 static PyTypeObject *With_type;
 _Py_IDENTIFIER(items);
-static char *With_fields[]={
+static const char * const With_fields[]={
     "items",
     "body",
     "type_comment",
 };
 static PyTypeObject *AsyncWith_type;
-static char *AsyncWith_fields[]={
+static const char * const AsyncWith_fields[]={
     "items",
     "body",
     "type_comment",
@@ -158,14 +158,14 @@ static char *AsyncWith_fields[]={
 static PyTypeObject *Raise_type;
 _Py_IDENTIFIER(exc);
 _Py_IDENTIFIER(cause);
-static char *Raise_fields[]={
+static const char * const Raise_fields[]={
     "exc",
     "cause",
 };
 static PyTypeObject *Try_type;
 _Py_IDENTIFIER(handlers);
 _Py_IDENTIFIER(finalbody);
-static char *Try_fields[]={
+static const char * const Try_fields[]={
     "body",
     "handlers",
     "orelse",
@@ -173,40 +173,40 @@ static char *Try_fields[]={
 };
 static PyTypeObject *Assert_type;
 _Py_IDENTIFIER(msg);
-static char *Assert_fields[]={
+static const char * const Assert_fields[]={
     "test",
     "msg",
 };
 static PyTypeObject *Import_type;
 _Py_IDENTIFIER(names);
-static char *Import_fields[]={
+static const char * const Import_fields[]={
     "names",
 };
 static PyTypeObject *ImportFrom_type;
 _Py_IDENTIFIER(module);
 _Py_IDENTIFIER(level);
-static char *ImportFrom_fields[]={
+static const char * const ImportFrom_fields[]={
     "module",
     "names",
     "level",
 };
 static PyTypeObject *Global_type;
-static char *Global_fields[]={
+static const char * const Global_fields[]={
     "names",
 };
 static PyTypeObject *Nonlocal_type;
-static char *Nonlocal_fields[]={
+static const char * const Nonlocal_fields[]={
     "names",
 };
 static PyTypeObject *Expr_type;
-static char *Expr_fields[]={
+static const char * const Expr_fields[]={
     "value",
 };
 static PyTypeObject *Pass_type;
 static PyTypeObject *Break_type;
 static PyTypeObject *Continue_type;
 static PyTypeObject *expr_type;
-static char *expr_attributes[] = {
+static const char * const expr_attributes[] = {
     "lineno",
     "col_offset",
     "end_lineno",
@@ -215,98 +215,98 @@ static char *expr_attributes[] = {
 static PyObject* ast2obj_expr(void*);
 static PyTypeObject *BoolOp_type;
 _Py_IDENTIFIER(values);
-static char *BoolOp_fields[]={
+static const char * const BoolOp_fields[]={
     "op",
     "values",
 };
 static PyTypeObject *NamedExpr_type;
-static char *NamedExpr_fields[]={
+static const char * const NamedExpr_fields[]={
     "target",
     "value",
 };
 static PyTypeObject *BinOp_type;
 _Py_IDENTIFIER(left);
 _Py_IDENTIFIER(right);
-static char *BinOp_fields[]={
+static const char * const BinOp_fields[]={
     "left",
     "op",
     "right",
 };
 static PyTypeObject *UnaryOp_type;
 _Py_IDENTIFIER(operand);
-static char *UnaryOp_fields[]={
+static const char * const UnaryOp_fields[]={
     "op",
     "operand",
 };
 static PyTypeObject *Lambda_type;
-static char *Lambda_fields[]={
+static const char * const Lambda_fields[]={
     "args",
     "body",
 };
 static PyTypeObject *IfExp_type;
-static char *IfExp_fields[]={
+static const char * const IfExp_fields[]={
     "test",
     "body",
     "orelse",
 };
 static PyTypeObject *Dict_type;
 _Py_IDENTIFIER(keys);
-static char *Dict_fields[]={
+static const char * const Dict_fields[]={
     "keys",
     "values",
 };
 static PyTypeObject *Set_type;
 _Py_IDENTIFIER(elts);
-static char *Set_fields[]={
+static const char * const Set_fields[]={
     "elts",
 };
 static PyTypeObject *ListComp_type;
 _Py_IDENTIFIER(elt);
 _Py_IDENTIFIER(generators);
-static char *ListComp_fields[]={
+static const char * const ListComp_fields[]={
     "elt",
     "generators",
 };
 static PyTypeObject *SetComp_type;
-static char *SetComp_fields[]={
+static const char * const SetComp_fields[]={
     "elt",
     "generators",
 };
 static PyTypeObject *DictComp_type;
 _Py_IDENTIFIER(key);
-static char *DictComp_fields[]={
+static const char * const DictComp_fields[]={
     "key",
     "value",
     "generators",
 };
 static PyTypeObject *GeneratorExp_type;
-static char *GeneratorExp_fields[]={
+static const char * const GeneratorExp_fields[]={
     "elt",
     "generators",
 };
 static PyTypeObject *Await_type;
-static char *Await_fields[]={
+static const char * const Await_fields[]={
     "value",
 };
 static PyTypeObject *Yield_type;
-static char *Yield_fields[]={
+static const char * const Yield_fields[]={
     "value",
 };
 static PyTypeObject *YieldFrom_type;
-static char *YieldFrom_fields[]={
+static const char * const YieldFrom_fields[]={
     "value",
 };
 static PyTypeObject *Compare_type;
 _Py_IDENTIFIER(ops);
 _Py_IDENTIFIER(comparators);
-static char *Compare_fields[]={
+static const char * const Compare_fields[]={
     "left",
     "ops",
     "comparators",
 };
 static PyTypeObject *Call_type;
 _Py_IDENTIFIER(func);
-static char *Call_fields[]={
+static const char * const Call_fields[]={
     "func",
     "args",
     "keywords",
@@ -314,54 +314,54 @@ static char *Call_fields[]={
 static PyTypeObject *FormattedValue_type;
 _Py_IDENTIFIER(conversion);
 _Py_IDENTIFIER(format_spec);
-static char *FormattedValue_fields[]={
+static const char * const FormattedValue_fields[]={
     "value",
     "conversion",
     "format_spec",
 };
 static PyTypeObject *JoinedStr_type;
-static char *JoinedStr_fields[]={
+static const char * const JoinedStr_fields[]={
     "values",
 };
 static PyTypeObject *Constant_type;
 _Py_IDENTIFIER(kind);
-static char *Constant_fields[]={
+static const char * const Constant_fields[]={
     "value",
     "kind",
 };
 static PyTypeObject *Attribute_type;
 _Py_IDENTIFIER(attr);
 _Py_IDENTIFIER(ctx);
-static char *Attribute_fields[]={
+static const char * const Attribute_fields[]={
     "value",
     "attr",
     "ctx",
 };
 static PyTypeObject *Subscript_type;
 _Py_IDENTIFIER(slice);
-static char *Subscript_fields[]={
+static const char * const Subscript_fields[]={
     "value",
     "slice",
     "ctx",
 };
 static PyTypeObject *Starred_type;
-static char *Starred_fields[]={
+static const char * const Starred_fields[]={
     "value",
     "ctx",
 };
 static PyTypeObject *Name_type;
 _Py_IDENTIFIER(id);
-static char *Name_fields[]={
+static const char * const Name_fields[]={
     "id",
     "ctx",
 };
 static PyTypeObject *List_type;
-static char *List_fields[]={
+static const char * const List_fields[]={
     "elts",
     "ctx",
 };
 static PyTypeObject *Tuple_type;
-static char *Tuple_fields[]={
+static const char * const Tuple_fields[]={
     "elts",
     "ctx",
 };
@@ -381,18 +381,18 @@ static PyTypeObject *Slice_type;
 _Py_IDENTIFIER(lower);
 _Py_IDENTIFIER(upper);
 _Py_IDENTIFIER(step);
-static char *Slice_fields[]={
+static const char * const Slice_fields[]={
     "lower",
     "upper",
     "step",
 };
 static PyTypeObject *ExtSlice_type;
 _Py_IDENTIFIER(dims);
-static char *ExtSlice_fields[]={
+static const char * const ExtSlice_fields[]={
     "dims",
 };
 static PyTypeObject *Index_type;
-static char *Index_fields[]={
+static const char * const Index_fields[]={
     "value",
 };
 static PyTypeObject *boolop_type;
@@ -446,14 +446,14 @@ static PyTypeObject *comprehension_type;
 static PyObject* ast2obj_comprehension(void*);
 _Py_IDENTIFIER(ifs);
 _Py_IDENTIFIER(is_async);
-static char *comprehension_fields[]={
+static const char * const comprehension_fields[]={
     "target",
     "iter",
     "ifs",
     "is_async",
 };
 static PyTypeObject *excepthandler_type;
-static char *excepthandler_attributes[] = {
+static const char * const excepthandler_attributes[] = {
     "lineno",
     "col_offset",
     "end_lineno",
@@ -462,7 +462,7 @@ static char *excepthandler_attributes[] = {
 static PyObject* ast2obj_excepthandler(void*);
 static PyTypeObject *ExceptHandler_type;
 _Py_IDENTIFIER(type);
-static char *ExceptHandler_fields[]={
+static const char * const ExceptHandler_fields[]={
     "type",
     "name",
     "body",
@@ -475,7 +475,7 @@ _Py_IDENTIFIER(kwonlyargs);
 _Py_IDENTIFIER(kw_defaults);
 _Py_IDENTIFIER(kwarg);
 _Py_IDENTIFIER(defaults);
-static char *arguments_fields[]={
+static const char * const arguments_fields[]={
     "posonlyargs",
     "args",
     "vararg",
@@ -486,28 +486,28 @@ static char *arguments_fields[]={
 };
 static PyTypeObject *arg_type;
 static PyObject* ast2obj_arg(void*);
-static char *arg_attributes[] = {
+static const char * const arg_attributes[] = {
     "lineno",
     "col_offset",
     "end_lineno",
     "end_col_offset",
 };
 _Py_IDENTIFIER(arg);
-static char *arg_fields[]={
+static const char * const arg_fields[]={
     "arg",
     "annotation",
     "type_comment",
 };
 static PyTypeObject *keyword_type;
 static PyObject* ast2obj_keyword(void*);
-static char *keyword_fields[]={
+static const char * const keyword_fields[]={
     "arg",
     "value",
 };
 static PyTypeObject *alias_type;
 static PyObject* ast2obj_alias(void*);
 _Py_IDENTIFIER(asname);
-static char *alias_fields[]={
+static const char * const alias_fields[]={
     "name",
     "asname",
 };
@@ -515,7 +515,7 @@ static PyTypeObject *withitem_type;
 static PyObject* ast2obj_withitem(void*);
 _Py_IDENTIFIER(context_expr);
 _Py_IDENTIFIER(optional_vars);
-static char *withitem_fields[]={
+static const char * const withitem_fields[]={
     "context_expr",
     "optional_vars",
 };
@@ -523,7 +523,7 @@ static PyTypeObject *type_ignore_type;
 static PyObject* ast2obj_type_ignore(void*);
 static PyTypeObject *TypeIgnore_type;
 _Py_IDENTIFIER(tag);
-static char *TypeIgnore_fields[]={
+static const char * const TypeIgnore_fields[]={
     "lineno",
     "tag",
 };
@@ -677,7 +677,8 @@ static PyTypeObject AST_type = {
 };
 
 
-static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields)
+static PyTypeObject *
+make_type(const char *type, PyTypeObject *base, const char * const *fields, int num_fields)
 {
     _Py_IDENTIFIER(__module__);
     _Py_IDENTIFIER(_ast);
@@ -686,7 +687,7 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int
     fnames = PyTuple_New(num_fields);
     if (!fnames) return NULL;
     for (i = 0; i < num_fields; i++) {
-        PyObject *field = PyUnicode_FromString(fields[i]);
+        PyObject *field = PyUnicode_InternFromString(fields[i]);
         if (!field) {
             Py_DECREF(fnames);
             return NULL;
@@ -702,14 +703,15 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int
     return (PyTypeObject*)result;
 }
 
-static int add_attributes(PyTypeObject* type, char**attrs, int num_fields)
+static int
+add_attributes(PyTypeObject *type, const char * const *attrs, int num_fields)
 {
     int i, result;
     PyObject *s, *l = PyTuple_New(num_fields);
     if (!l)
         return 0;
     for (i = 0; i < num_fields; i++) {
-        s = PyUnicode_FromString(attrs[i]);
+        s = PyUnicode_InternFromString(attrs[i]);
         if (!s) {
             Py_DECREF(l);
             return 0;
@@ -4212,7 +4214,10 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Module field \"body\" changed size during iteration");
@@ -4242,7 +4247,10 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             if (type_ignores == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 type_ignore_ty val;
-                res = obj2ast_type_ignore(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_type_ignore(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Module field \"type_ignores\" changed size during iteration");
@@ -4283,7 +4291,10 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Interactive field \"body\" changed size during iteration");
@@ -4349,7 +4360,10 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             if (argtypes == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "FunctionType field \"argtypes\" changed size during iteration");
@@ -4403,7 +4417,10 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Suite field \"body\" changed size during iteration");
@@ -4549,7 +4566,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "FunctionDef field \"body\" changed size during iteration");
@@ -4579,7 +4599,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (decorator_list == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "FunctionDef field \"decorator_list\" changed size during iteration");
@@ -4679,7 +4702,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "AsyncFunctionDef field \"body\" changed size during iteration");
@@ -4709,7 +4735,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (decorator_list == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "AsyncFunctionDef field \"decorator_list\" changed size during iteration");
@@ -4795,7 +4824,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (bases == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "ClassDef field \"bases\" changed size during iteration");
@@ -4825,7 +4857,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (keywords == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 keyword_ty val;
-                res = obj2ast_keyword(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_keyword(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "ClassDef field \"keywords\" changed size during iteration");
@@ -4855,7 +4890,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "ClassDef field \"body\" changed size during iteration");
@@ -4885,7 +4923,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (decorator_list == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "ClassDef field \"decorator_list\" changed size during iteration");
@@ -4952,7 +4993,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (targets == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Delete field \"targets\" changed size during iteration");
@@ -4996,7 +5040,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (targets == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Assign field \"targets\" changed size during iteration");
@@ -5214,7 +5261,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "For field \"body\" changed size during iteration");
@@ -5244,7 +5294,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (orelse == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "For field \"orelse\" changed size during iteration");
@@ -5329,7 +5382,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "AsyncFor field \"body\" changed size during iteration");
@@ -5359,7 +5415,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (orelse == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "AsyncFor field \"orelse\" changed size during iteration");
@@ -5429,7 +5488,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "While field \"body\" changed size during iteration");
@@ -5459,7 +5521,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (orelse == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "While field \"orelse\" changed size during iteration");
@@ -5516,7 +5581,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "If field \"body\" changed size during iteration");
@@ -5546,7 +5614,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (orelse == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "If field \"orelse\" changed size during iteration");
@@ -5590,7 +5661,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (items == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 withitem_ty val;
-                res = obj2ast_withitem(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_withitem(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "With field \"items\" changed size during iteration");
@@ -5620,7 +5694,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "With field \"body\" changed size during iteration");
@@ -5677,7 +5754,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (items == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 withitem_ty val;
-                res = obj2ast_withitem(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_withitem(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "AsyncWith field \"items\" changed size during iteration");
@@ -5707,7 +5787,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "AsyncWith field \"body\" changed size during iteration");
@@ -5804,7 +5887,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Try field \"body\" changed size during iteration");
@@ -5834,7 +5920,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (handlers == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 excepthandler_ty val;
-                res = obj2ast_excepthandler(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_excepthandler(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Try field \"handlers\" changed size during iteration");
@@ -5864,7 +5953,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (orelse == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Try field \"orelse\" changed size during iteration");
@@ -5894,7 +5986,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (finalbody == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Try field \"finalbody\" changed size during iteration");
@@ -5975,7 +6070,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (names == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 alias_ty val;
-                res = obj2ast_alias(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_alias(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Import field \"names\" changed size during iteration");
@@ -6032,7 +6130,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (names == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 alias_ty val;
-                res = obj2ast_alias(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_alias(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "ImportFrom field \"names\" changed size during iteration");
@@ -6087,7 +6188,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (names == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 identifier val;
-                res = obj2ast_identifier(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_identifier(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Global field \"names\" changed size during iteration");
@@ -6129,7 +6233,10 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (names == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 identifier val;
-                res = obj2ast_identifier(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_identifier(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Nonlocal field \"names\" changed size during iteration");
@@ -6314,7 +6421,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (values == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "BoolOp field \"values\" changed size during iteration");
@@ -6580,7 +6690,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (keys == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Dict field \"keys\" changed size during iteration");
@@ -6610,7 +6723,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (values == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Dict field \"values\" changed size during iteration");
@@ -6652,7 +6768,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (elts == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Set field \"elts\" changed size during iteration");
@@ -6707,7 +6826,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (generators == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 comprehension_ty val;
-                res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_comprehension(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "ListComp field \"generators\" changed size during iteration");
@@ -6763,7 +6885,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (generators == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 comprehension_ty val;
-                res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_comprehension(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "SetComp field \"generators\" changed size during iteration");
@@ -6833,7 +6958,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (generators == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 comprehension_ty val;
-                res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_comprehension(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "DictComp field \"generators\" changed size during iteration");
@@ -6889,7 +7017,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (generators == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 comprehension_ty val;
-                res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_comprehension(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "GeneratorExp field \"generators\" changed size during iteration");
@@ -7021,7 +7152,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (ops == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 cmpop_ty val;
-                res = obj2ast_cmpop(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_cmpop(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Compare field \"ops\" changed size during iteration");
@@ -7051,7 +7185,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (comparators == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Compare field \"comparators\" changed size during iteration");
@@ -7108,7 +7245,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (args == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Call field \"args\" changed size during iteration");
@@ -7138,7 +7278,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (keywords == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 keyword_ty val;
-                res = obj2ast_keyword(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_keyword(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Call field \"keywords\" changed size during iteration");
@@ -7233,7 +7376,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (values == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "JoinedStr field \"values\" changed size during iteration");
@@ -7499,7 +7645,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (elts == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "List field \"elts\" changed size during iteration");
@@ -7555,7 +7704,10 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (elts == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 expr_ty val;
-                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_expr(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "Tuple field \"elts\" changed size during iteration");
@@ -7738,7 +7890,10 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena)
             if (dims == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 slice_ty val;
-                res = obj2ast_slice(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_slice(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "ExtSlice field \"dims\" changed size during iteration");
@@ -8110,7 +8265,10 @@ obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena)
         if (ifs == NULL) goto failed;
         for (i = 0; i < len; i++) {
             expr_ty val;
-            res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+            PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+            Py_INCREF(tmp2);
+            res = obj2ast_expr(tmp2, &val, arena);
+            Py_DECREF(tmp2);
             if (res != 0) goto failed;
             if (len != PyList_GET_SIZE(tmp)) {
                 PyErr_SetString(PyExc_RuntimeError, "comprehension field \"ifs\" changed size during iteration");
@@ -8262,7 +8420,10 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
             if (body == NULL) goto failed;
             for (i = 0; i < len; i++) {
                 stmt_ty val;
-                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
+                PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+                Py_INCREF(tmp2);
+                res = obj2ast_stmt(tmp2, &val, arena);
+                Py_DECREF(tmp2);
                 if (res != 0) goto failed;
                 if (len != PyList_GET_SIZE(tmp)) {
                     PyErr_SetString(PyExc_RuntimeError, "ExceptHandler field \"body\" changed size during iteration");
@@ -8316,7 +8477,10 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         if (posonlyargs == NULL) goto failed;
         for (i = 0; i < len; i++) {
             arg_ty val;
-            res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &val, arena);
+            PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+            Py_INCREF(tmp2);
+            res = obj2ast_arg(tmp2, &val, arena);
+            Py_DECREF(tmp2);
             if (res != 0) goto failed;
             if (len != PyList_GET_SIZE(tmp)) {
                 PyErr_SetString(PyExc_RuntimeError, "arguments field \"posonlyargs\" changed size during iteration");
@@ -8346,7 +8510,10 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         if (args == NULL) goto failed;
         for (i = 0; i < len; i++) {
             arg_ty val;
-            res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &val, arena);
+            PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+            Py_INCREF(tmp2);
+            res = obj2ast_arg(tmp2, &val, arena);
+            Py_DECREF(tmp2);
             if (res != 0) goto failed;
             if (len != PyList_GET_SIZE(tmp)) {
                 PyErr_SetString(PyExc_RuntimeError, "arguments field \"args\" changed size during iteration");
@@ -8389,7 +8556,10 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         if (kwonlyargs == NULL) goto failed;
         for (i = 0; i < len; i++) {
             arg_ty val;
-            res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &val, arena);
+            PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+            Py_INCREF(tmp2);
+            res = obj2ast_arg(tmp2, &val, arena);
+            Py_DECREF(tmp2);
             if (res != 0) goto failed;
             if (len != PyList_GET_SIZE(tmp)) {
                 PyErr_SetString(PyExc_RuntimeError, "arguments field \"kwonlyargs\" changed size during iteration");
@@ -8419,7 +8589,10 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         if (kw_defaults == NULL) goto failed;
         for (i = 0; i < len; i++) {
             expr_ty val;
-            res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+            PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+            Py_INCREF(tmp2);
+            res = obj2ast_expr(tmp2, &val, arena);
+            Py_DECREF(tmp2);
             if (res != 0) goto failed;
             if (len != PyList_GET_SIZE(tmp)) {
                 PyErr_SetString(PyExc_RuntimeError, "arguments field \"kw_defaults\" changed size during iteration");
@@ -8462,7 +8635,10 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         if (defaults == NULL) goto failed;
         for (i = 0; i < len; i++) {
             expr_ty val;
-            res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
+            PyObject *tmp2 = PyList_GET_ITEM(tmp, i);
+            Py_INCREF(tmp2);
+            res = obj2ast_expr(tmp2, &val, arena);
+            Py_DECREF(tmp2);
             if (res != 0) goto failed;
             if (len != PyList_GET_SIZE(tmp)) {
                 PyErr_SetString(PyExc_RuntimeError, "arguments field \"defaults\" changed size during iteration");
@@ -8993,7 +9169,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
 {
     mod_ty res;
     PyObject *req_type[3];
-    char *req_name[] = {"Module", "Expression", "Interactive"};
+    const char * const req_name[] = {"Module", "Expression", "Interactive"};
     int isinstance;
 
     if (PySys_Audit("compile", "OO", ast, Py_None) < 0) {


From webhook-mailer at python.org  Tue Sep 10 06:13:41 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 10:13:41 -0000
Subject: [Python-checkins] Fix typo in dict object comment (GH-15814)
Message-ID: <mailman.406.1568110421.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/18fa272601a1fa2c5a978b27c0c8dfd7c3ca90af
commit: 18fa272601a1fa2c5a978b27c0c8dfd7c3ca90af
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T03:13:36-07:00
summary:

Fix typo in dict object comment (GH-15814)

(cherry picked from commit 359143c68659d165f52320d368667e0eff279dc5)

Co-authored-by: dalgarno <32097481+dalgarno at users.noreply.github.com>

files:
M Objects/dictobject.c

diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index e417cd2119c6..31f962d16b25 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -39,7 +39,7 @@ Size of indices is dk_size.  Type of each index in indices is vary on dk_size:
 * int32 for 2**16 <= dk_size <= 2**31
 * int64 for 2**32 <= dk_size
 
-dk_entries is array of PyDictKeyEntry.  It's size is USABLE_FRACTION(dk_size).
+dk_entries is array of PyDictKeyEntry.  Its size is USABLE_FRACTION(dk_size).
 DK_ENTRIES(dk) can be used to get pointer to entries.
 
 NOTE: Since negative value is used for DKIX_EMPTY and DKIX_DUMMY, type of


From webhook-mailer at python.org  Tue Sep 10 06:15:46 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 10:15:46 -0000
Subject: [Python-checkins] Fix typo in dict object comment (GH-15814)
Message-ID: <mailman.407.1568110548.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9cbf43adfe268fc65e68d9185b80d3305b2c4f19
commit: 9cbf43adfe268fc65e68d9185b80d3305b2c4f19
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T03:15:43-07:00
summary:

Fix typo in dict object comment (GH-15814)

(cherry picked from commit 359143c68659d165f52320d368667e0eff279dc5)

Co-authored-by: dalgarno <32097481+dalgarno at users.noreply.github.com>

files:
M Objects/dictobject.c

diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index eb635263b04c..943744884900 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -39,7 +39,7 @@ Size of indices is dk_size.  Type of each index in indices is vary on dk_size:
 * int32 for 2**16 <= dk_size <= 2**31
 * int64 for 2**32 <= dk_size
 
-dk_entries is array of PyDictKeyEntry.  It's size is USABLE_FRACTION(dk_size).
+dk_entries is array of PyDictKeyEntry.  Its size is USABLE_FRACTION(dk_size).
 DK_ENTRIES(dk) can be used to get pointer to entries.
 
 NOTE: Since negative value is used for DKIX_EMPTY and DKIX_DUMMY, type of


From webhook-mailer at python.org  Tue Sep 10 06:18:41 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Tue, 10 Sep 2019 10:18:41 -0000
Subject: [Python-checkins] bpo-38076: Make struct module PEP-384 compatible
 (#15805)
Message-ID: <mailman.408.1568110722.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4f384af067d05b16a554bfd976934fca9f87a1cf
commit: 4f384af067d05b16a554bfd976934fca9f87a1cf
branch: master
author: Dino Viehland <dinoviehland at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-10T03:18:37-07:00
summary:

bpo-38076: Make struct module PEP-384 compatible (#15805)

* PEP-384 _struct

* More PEP-384 fixes for _struct

Summary: Add a couple of more fixes for `_struct` that were previously missed such as removing `tp_*` accessors and using `PyBytesWriter` instead of calling `PyBytes_FromStringAndSize` with `NULL`. Also added a test to confirm that `iter_unpack` type is still uninstantiable.

* ?? Added by blurb_it.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-09-16-36-37.bpo-38076.C5dVBl.rst
M Lib/test/test_struct.py
M Modules/_struct.c

diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py
index 454082e66d3f..157efa1347a9 100644
--- a/Lib/test/test_struct.py
+++ b/Lib/test/test_struct.py
@@ -679,6 +679,10 @@ def _check_iterator(it):
         with self.assertRaises(struct.error):
             s.iter_unpack(b"12")
 
+    def test_uninstantiable(self):
+        iter_unpack_type = type(struct.Struct(">ibcp").iter_unpack(b""))
+        self.assertRaises(TypeError, iter_unpack_type)
+
     def test_iterate(self):
         s = struct.Struct('>IB')
         b = bytes(range(1, 16))
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-09-16-36-37.bpo-38076.C5dVBl.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-16-36-37.bpo-38076.C5dVBl.rst
new file mode 100644
index 000000000000..6f18c03403d2
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-16-36-37.bpo-38076.C5dVBl.rst	
@@ -0,0 +1 @@
+The struct module is now PEP-384 compatible
\ No newline at end of file
diff --git a/Modules/_struct.c b/Modules/_struct.c
index ba8f9cd8e966..5fdfdf9da63b 100644
--- a/Modules/_struct.c
+++ b/Modules/_struct.c
@@ -14,7 +14,17 @@ class Struct "PyStructObject *" "&PyStructType"
 [clinic start generated code]*/
 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=9b032058a83ed7c3]*/
 
-static PyTypeObject PyStructType;
+typedef struct {
+    PyObject *PyStructType;
+    PyObject *unpackiter_type;
+    PyObject *StructError;
+} _structmodulestate;
+
+#define _structmodulestate(o) ((_structmodulestate *)PyModule_GetState(o))
+
+static struct PyModuleDef _structmodule;
+
+#define _structmodulestate_global _structmodulestate(PyState_FindModule(&_structmodule))
 
 /* The translation function for each format character is table driven */
 typedef struct _formatdef {
@@ -46,13 +56,8 @@ typedef struct {
 } PyStructObject;
 
 
-#define PyStruct_Check(op) PyObject_TypeCheck(op, &PyStructType)
-#define PyStruct_CheckExact(op) (Py_TYPE(op) == &PyStructType)
-
-
-/* Exception */
-
-static PyObject *StructError;
+#define PyStruct_Check(op) PyObject_TypeCheck(op, (PyTypeObject *)_structmodulestate_global->PyStructType)
+#define PyStruct_CheckExact(op) (Py_TYPE(op) == (PyTypeObject *)_structmodulestate_global->PyStructType)
 
 
 /* Define various structs to figure out the alignments of types */
@@ -115,7 +120,7 @@ get_pylong(PyObject *v)
                 return NULL;
         }
         else {
-            PyErr_SetString(StructError,
+            PyErr_SetString(_structmodulestate_global->StructError,
                             "required argument is not an integer");
             return NULL;
         }
@@ -143,7 +148,7 @@ get_long(PyObject *v, long *p)
     Py_DECREF(v);
     if (x == (long)-1 && PyErr_Occurred()) {
         if (PyErr_ExceptionMatches(PyExc_OverflowError))
-            PyErr_SetString(StructError,
+            PyErr_SetString(_structmodulestate_global->StructError,
                             "argument out of range");
         return -1;
     }
@@ -167,7 +172,7 @@ get_ulong(PyObject *v, unsigned long *p)
     Py_DECREF(v);
     if (x == (unsigned long)-1 && PyErr_Occurred()) {
         if (PyErr_ExceptionMatches(PyExc_OverflowError))
-            PyErr_SetString(StructError,
+            PyErr_SetString(_structmodulestate_global->StructError,
                             "argument out of range");
         return -1;
     }
@@ -190,7 +195,7 @@ get_longlong(PyObject *v, long long *p)
     Py_DECREF(v);
     if (x == (long long)-1 && PyErr_Occurred()) {
         if (PyErr_ExceptionMatches(PyExc_OverflowError))
-            PyErr_SetString(StructError,
+            PyErr_SetString(_structmodulestate_global->StructError,
                             "argument out of range");
         return -1;
     }
@@ -213,7 +218,7 @@ get_ulonglong(PyObject *v, unsigned long long *p)
     Py_DECREF(v);
     if (x == (unsigned long long)-1 && PyErr_Occurred()) {
         if (PyErr_ExceptionMatches(PyExc_OverflowError))
-            PyErr_SetString(StructError,
+            PyErr_SetString(_structmodulestate_global->StructError,
                             "argument out of range");
         return -1;
     }
@@ -236,7 +241,7 @@ get_ssize_t(PyObject *v, Py_ssize_t *p)
     Py_DECREF(v);
     if (x == (Py_ssize_t)-1 && PyErr_Occurred()) {
         if (PyErr_ExceptionMatches(PyExc_OverflowError))
-            PyErr_SetString(StructError,
+            PyErr_SetString(_structmodulestate_global->StructError,
                             "argument out of range");
         return -1;
     }
@@ -259,7 +264,7 @@ get_size_t(PyObject *v, size_t *p)
     Py_DECREF(v);
     if (x == (size_t)-1 && PyErr_Occurred()) {
         if (PyErr_ExceptionMatches(PyExc_OverflowError))
-            PyErr_SetString(StructError,
+            PyErr_SetString(_structmodulestate_global->StructError,
                             "argument out of range");
         return -1;
     }
@@ -293,7 +298,7 @@ pack_halffloat(char *p,      /* start of 2-byte string */
 {
     double x = PyFloat_AsDouble(v);
     if (x == -1.0 && PyErr_Occurred()) {
-        PyErr_SetString(StructError,
+        PyErr_SetString(_structmodulestate_global->StructError,
                         "required argument is not a float");
         return -1;
     }
@@ -339,13 +344,13 @@ _range_error(const formatdef *f, int is_unsigned)
     const size_t ulargest = (size_t)-1 >> ((SIZEOF_SIZE_T - f->size)*8);
     assert(f->size >= 1 && f->size <= SIZEOF_SIZE_T);
     if (is_unsigned)
-        PyErr_Format(StructError,
+        PyErr_Format(_structmodulestate_global->StructError,
             "'%c' format requires 0 <= number <= %zu",
             f->format,
             ulargest);
     else {
         const Py_ssize_t largest = (Py_ssize_t)(ulargest >> 1);
-        PyErr_Format(StructError,
+        PyErr_Format(_structmodulestate_global->StructError,
             "'%c' format requires %zd <= number <= %zd",
             f->format,
             ~ largest,
@@ -524,7 +529,7 @@ np_byte(char *p, PyObject *v, const formatdef *f)
     if (get_long(v, &x) < 0)
         return -1;
     if (x < -128 || x > 127) {
-        PyErr_SetString(StructError,
+        PyErr_SetString(_structmodulestate_global->StructError,
                         "byte format requires -128 <= number <= 127");
         return -1;
     }
@@ -539,7 +544,7 @@ np_ubyte(char *p, PyObject *v, const formatdef *f)
     if (get_long(v, &x) < 0)
         return -1;
     if (x < 0 || x > 255) {
-        PyErr_SetString(StructError,
+        PyErr_SetString(_structmodulestate_global->StructError,
                         "ubyte format requires 0 <= number <= 255");
         return -1;
     }
@@ -550,8 +555,8 @@ np_ubyte(char *p, PyObject *v, const formatdef *f)
 static int
 np_char(char *p, PyObject *v, const formatdef *f)
 {
-    if (!PyBytes_Check(v) || PyBytes_GET_SIZE(v) != 1) {
-        PyErr_SetString(StructError,
+    if (!PyBytes_Check(v) || PyBytes_Size(v) != 1) {
+        PyErr_SetString(_structmodulestate_global->StructError,
                         "char format requires a bytes object of length 1");
         return -1;
     }
@@ -567,7 +572,7 @@ np_short(char *p, PyObject *v, const formatdef *f)
     if (get_long(v, &x) < 0)
         return -1;
     if (x < SHRT_MIN || x > SHRT_MAX) {
-        PyErr_SetString(StructError,
+        PyErr_SetString(_structmodulestate_global->StructError,
                         "short format requires " Py_STRINGIFY(SHRT_MIN)
                         " <= number <= " Py_STRINGIFY(SHRT_MAX));
         return -1;
@@ -585,7 +590,7 @@ np_ushort(char *p, PyObject *v, const formatdef *f)
     if (get_long(v, &x) < 0)
         return -1;
     if (x < 0 || x > USHRT_MAX) {
-        PyErr_SetString(StructError,
+        PyErr_SetString(_structmodulestate_global->StructError,
                         "ushort format requires 0 <= number <= "
                         Py_STRINGIFY(USHRT_MAX));
         return -1;
@@ -716,7 +721,7 @@ np_float(char *p, PyObject *v, const formatdef *f)
 {
     float x = (float)PyFloat_AsDouble(v);
     if (x == -1 && PyErr_Occurred()) {
-        PyErr_SetString(StructError,
+        PyErr_SetString(_structmodulestate_global->StructError,
                         "required argument is not a float");
         return -1;
     }
@@ -729,7 +734,7 @@ np_double(char *p, PyObject *v, const formatdef *f)
 {
     double x = PyFloat_AsDouble(v);
     if (x == -1 && PyErr_Occurred()) {
-        PyErr_SetString(StructError,
+        PyErr_SetString(_structmodulestate_global->StructError,
                         "required argument is not a float");
         return -1;
     }
@@ -948,7 +953,7 @@ bp_float(char *p, PyObject *v, const formatdef *f)
 {
     double x = PyFloat_AsDouble(v);
     if (x == -1 && PyErr_Occurred()) {
-        PyErr_SetString(StructError,
+        PyErr_SetString(_structmodulestate_global->StructError,
                         "required argument is not a float");
         return -1;
     }
@@ -960,7 +965,7 @@ bp_double(char *p, PyObject *v, const formatdef *f)
 {
     double x = PyFloat_AsDouble(v);
     if (x == -1 && PyErr_Occurred()) {
-        PyErr_SetString(StructError,
+        PyErr_SetString(_structmodulestate_global->StructError,
                         "required argument is not a float");
         return -1;
     }
@@ -1163,7 +1168,7 @@ lp_float(char *p, PyObject *v, const formatdef *f)
 {
     double x = PyFloat_AsDouble(v);
     if (x == -1 && PyErr_Occurred()) {
-        PyErr_SetString(StructError,
+        PyErr_SetString(_structmodulestate_global->StructError,
                         "required argument is not a float");
         return -1;
     }
@@ -1175,7 +1180,7 @@ lp_double(char *p, PyObject *v, const formatdef *f)
 {
     double x = PyFloat_AsDouble(v);
     if (x == -1 && PyErr_Occurred()) {
-        PyErr_SetString(StructError,
+        PyErr_SetString(_structmodulestate_global->StructError,
                         "required argument is not a float");
         return -1;
     }
@@ -1242,7 +1247,7 @@ getentry(int c, const formatdef *f)
             return f;
         }
     }
-    PyErr_SetString(StructError, "bad char in struct format");
+    PyErr_SetString(_structmodulestate_global->StructError, "bad char in struct format");
     return NULL;
 }
 
@@ -1307,7 +1312,7 @@ prepare_s(PyStructObject *self)
                 num = num*10 + (c - '0');
             }
             if (c == '\0') {
-                PyErr_SetString(StructError,
+                PyErr_SetString(_structmodulestate_global->StructError,
                                 "repeat count given without format specifier");
                 return -1;
             }
@@ -1397,7 +1402,7 @@ prepare_s(PyStructObject *self)
     return 0;
 
   overflow:
-    PyErr_SetString(StructError,
+    PyErr_SetString(_structmodulestate_global->StructError,
                     "total struct size too long");
     return -1;
 }
@@ -1407,9 +1412,11 @@ s_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
     PyObject *self;
 
-    assert(type != NULL && type->tp_alloc != NULL);
+    assert(type != NULL);
+    allocfunc alloc_func = PyType_GetSlot(type, Py_tp_alloc);
+    assert(alloc_func != NULL);
 
-    self = type->tp_alloc(type, 0);
+    self = alloc_func(type, 0);
     if (self != NULL) {
         PyStructObject *s = (PyStructObject*)self;
         Py_INCREF(Py_None);
@@ -1455,7 +1462,7 @@ Struct___init___impl(PyStructObject *self, PyObject *format)
         PyErr_Format(PyExc_TypeError,
                      "Struct() argument 1 must be a str or bytes object, "
                      "not %.200s",
-                     Py_TYPE(format)->tp_name);
+                     _PyType_Name(Py_TYPE(format)));
         return -1;
     }
 
@@ -1468,13 +1475,16 @@ Struct___init___impl(PyStructObject *self, PyObject *format)
 static void
 s_dealloc(PyStructObject *s)
 {
+    PyTypeObject *tp = Py_TYPE(s);
     if (s->weakreflist != NULL)
         PyObject_ClearWeakRefs((PyObject *)s);
     if (s->s_codes != NULL) {
         PyMem_FREE(s->s_codes);
     }
-    Py_DECREF(s->s_format);
-    Py_TYPE(s)->tp_free((PyObject *)s);
+    Py_XDECREF(s->s_format);
+    freefunc free_func = PyType_GetSlot(Py_TYPE(s), Py_tp_free);
+    free_func(s);
+    Py_DECREF(tp);
 }
 
 static PyObject *
@@ -1535,7 +1545,7 @@ Struct_unpack_impl(PyStructObject *self, Py_buffer *buffer)
 {
     assert(self->s_codes != NULL);
     if (buffer->len != self->s_size) {
-        PyErr_Format(StructError,
+        PyErr_Format(_structmodulestate_global->StructError,
                      "unpack requires a buffer of %zd bytes",
                      self->s_size);
         return NULL;
@@ -1568,7 +1578,7 @@ Struct_unpack_from_impl(PyStructObject *self, Py_buffer *buffer,
 
     if (offset < 0) {
         if (offset + self->s_size > 0) {
-            PyErr_Format(StructError,
+            PyErr_Format(_structmodulestate_global->StructError,
                          "not enough data to unpack %zd bytes at offset %zd",
                          self->s_size,
                          offset);
@@ -1576,7 +1586,7 @@ Struct_unpack_from_impl(PyStructObject *self, Py_buffer *buffer,
         }
 
         if (offset + buffer->len < 0) {
-            PyErr_Format(StructError,
+            PyErr_Format(_structmodulestate_global->StructError,
                          "offset %zd out of range for %zd-byte buffer",
                          offset,
                          buffer->len);
@@ -1586,7 +1596,7 @@ Struct_unpack_from_impl(PyStructObject *self, Py_buffer *buffer,
     }
 
     if ((buffer->len - offset) < self->s_size) {
-        PyErr_Format(StructError,
+        PyErr_Format(_structmodulestate_global->StructError,
                      "unpack_from requires a buffer of at least %zu bytes for "
                      "unpacking %zd bytes at offset %zd "
                      "(actual buffer size is %zd)",
@@ -1614,10 +1624,12 @@ static void
 unpackiter_dealloc(unpackiterobject *self)
 {
     /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyTypeObject *tp = Py_TYPE(self);
     PyObject_GC_UnTrack(self);
     Py_XDECREF(self->so);
     PyBuffer_Release(&self->buf);
     PyObject_GC_Del(self);
+    Py_DECREF(tp);
 }
 
 static int
@@ -1663,35 +1675,28 @@ unpackiter_iternext(unpackiterobject *self)
     return result;
 }
 
-static PyTypeObject unpackiter_type = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    "unpack_iterator",                          /* tp_name */
-    sizeof(unpackiterobject),                   /* tp_basicsize */
-    0,                                          /* tp_itemsize */
-    (destructor)unpackiter_dealloc,             /* tp_dealloc */
-    0,                                          /* tp_vectorcall_offset */
-    0,                                          /* tp_getattr */
-    0,                                          /* tp_setattr */
-    0,                                          /* tp_as_async */
-    0,                                          /* tp_repr */
-    0,                                          /* tp_as_number */
-    0,                                          /* tp_as_sequence */
-    0,                                          /* tp_as_mapping */
-    0,                                          /* tp_hash */
-    0,                                          /* tp_call */
-    0,                                          /* tp_str */
-    PyObject_GenericGetAttr,                    /* tp_getattro */
-    0,                                          /* tp_setattro */
-    0,                                          /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,    /* tp_flags */
-    0,                                          /* tp_doc */
-    (traverseproc)unpackiter_traverse,          /* tp_traverse */
-    0,                                          /* tp_clear */
-    0,                                          /* tp_richcompare */
-    0,                                          /* tp_weaklistoffset */
-    PyObject_SelfIter,                          /* tp_iter */
-    (iternextfunc)unpackiter_iternext,          /* tp_iternext */
-    unpackiter_methods                          /* tp_methods */
+PyObject *unpackiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) {
+    PyErr_Format(PyExc_TypeError, "Cannot create '%.200s objects", _PyType_Name(type));
+    return NULL;
+}
+
+static PyType_Slot unpackiter_type_slots[] = {
+    {Py_tp_dealloc, unpackiter_dealloc},
+    {Py_tp_getattro, PyObject_GenericGetAttr},
+    {Py_tp_traverse, unpackiter_traverse},
+    {Py_tp_iter, PyObject_SelfIter},
+    {Py_tp_iternext, unpackiter_iternext},
+    {Py_tp_methods, unpackiter_methods},
+    {Py_tp_new, unpackiter_new},
+    {0, 0},
+};
+
+static PyType_Spec unpackiter_type_spec = {
+    "_struct.unpack_iterator",
+    sizeof(unpackiterobject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,
+    unpackiter_type_slots
 };
 
 /*[clinic input]
@@ -1717,12 +1722,12 @@ Struct_iter_unpack(PyStructObject *self, PyObject *buffer)
     assert(self->s_codes != NULL);
 
     if (self->s_size == 0) {
-        PyErr_Format(StructError,
+        PyErr_Format(_structmodulestate_global->StructError,
                      "cannot iteratively unpack with a struct of length 0");
         return NULL;
     }
 
-    iter = (unpackiterobject *) PyType_GenericAlloc(&unpackiter_type, 0);
+    iter = (unpackiterobject *) PyType_GenericAlloc((PyTypeObject *)_structmodulestate_global->unpackiter_type, 0);
     if (iter == NULL)
         return NULL;
 
@@ -1731,7 +1736,7 @@ Struct_iter_unpack(PyStructObject *self, PyObject *buffer)
         return NULL;
     }
     if (iter->buf.len % self->s_size != 0) {
-        PyErr_Format(StructError,
+        PyErr_Format(_structmodulestate_global->StructError,
                      "iterative unpacking requires a buffer of "
                      "a multiple of %zd bytes",
                      self->s_size);
@@ -1777,7 +1782,7 @@ s_pack_internal(PyStructObject *soself, PyObject *const *args, int offset, char*
                 void *p;
                 isstring = PyBytes_Check(v);
                 if (!isstring && !PyByteArray_Check(v)) {
-                    PyErr_SetString(StructError,
+                    PyErr_SetString(_structmodulestate_global->StructError,
                                     "argument for 's' must be a bytes object");
                     return -1;
                 }
@@ -1799,7 +1804,7 @@ s_pack_internal(PyStructObject *soself, PyObject *const *args, int offset, char*
                 void *p;
                 isstring = PyBytes_Check(v);
                 if (!isstring && !PyByteArray_Check(v)) {
-                    PyErr_SetString(StructError,
+                    PyErr_SetString(_structmodulestate_global->StructError,
                                     "argument for 'p' must be a bytes object");
                     return -1;
                 }
@@ -1821,7 +1826,7 @@ s_pack_internal(PyStructObject *soself, PyObject *const *args, int offset, char*
             } else {
                 if (e->pack(res, v, e) < 0) {
                     if (PyLong_Check(v) && PyErr_ExceptionMatches(PyExc_OverflowError))
-                        PyErr_SetString(StructError,
+                        PyErr_SetString(_structmodulestate_global->StructError,
                                         "int too large to convert");
                     return -1;
                 }
@@ -1845,8 +1850,8 @@ strings.");
 static PyObject *
 s_pack(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 {
+    char *buf;
     PyStructObject *soself;
-    PyObject *result;
 
     /* Validate arguments. */
     soself = (PyStructObject *)self;
@@ -1854,23 +1859,27 @@ s_pack(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
     assert(soself->s_codes != NULL);
     if (nargs != soself->s_len)
     {
-        PyErr_Format(StructError,
+        PyErr_Format(_structmodulestate_global->StructError,
             "pack expected %zd items for packing (got %zd)", soself->s_len, nargs);
         return NULL;
     }
 
-    /* Allocate a new buffer */
-    result = PyBytes_FromStringAndSize((char *)NULL, soself->s_size);
-    if (result == NULL)
+    /* Allocate a new string */
+    _PyBytesWriter writer;
+    _PyBytesWriter_Init(&writer);
+    buf = _PyBytesWriter_Alloc(&writer, soself->s_size);
+    if (buf == NULL) {
+        _PyBytesWriter_Dealloc(&writer);
         return NULL;
+    }
 
     /* Call the guts */
-    if ( s_pack_internal(soself, args, 0, PyBytes_AS_STRING(result)) != 0 ) {
-        Py_DECREF(result);
+    if ( s_pack_internal(soself, args, 0, buf) != 0 ) {
+        _PyBytesWriter_Dealloc(&writer);
         return NULL;
     }
 
-    return result;
+    return _PyBytesWriter_Finish(&writer, buf + soself->s_size);
 }
 
 PyDoc_STRVAR(s_pack_into__doc__,
@@ -1895,15 +1904,15 @@ s_pack_into(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
     if (nargs != (soself->s_len + 2))
     {
         if (nargs == 0) {
-            PyErr_Format(StructError,
+            PyErr_Format(_structmodulestate_global->StructError,
                         "pack_into expected buffer argument");
         }
         else if (nargs == 1) {
-            PyErr_Format(StructError,
+            PyErr_Format(_structmodulestate_global->StructError,
                         "pack_into expected offset argument");
         }
         else {
-            PyErr_Format(StructError,
+            PyErr_Format(_structmodulestate_global->StructError,
                         "pack_into expected %zd items for packing (got %zd)",
                         soself->s_len, (nargs - 2));
         }
@@ -1926,7 +1935,7 @@ s_pack_into(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
     if (offset < 0) {
          /* Check that negative offset is low enough to fit data */
         if (offset + soself->s_size > 0) {
-            PyErr_Format(StructError,
+            PyErr_Format(_structmodulestate_global->StructError,
                          "no space to pack %zd bytes at offset %zd",
                          soself->s_size,
                          offset);
@@ -1936,7 +1945,7 @@ s_pack_into(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 
         /* Check that negative offset is not crossing buffer boundary */
         if (offset + buffer.len < 0) {
-            PyErr_Format(StructError,
+            PyErr_Format(_structmodulestate_global->StructError,
                          "offset %zd out of range for %zd-byte buffer",
                          offset,
                          buffer.len);
@@ -1952,7 +1961,7 @@ s_pack_into(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
         assert(offset >= 0);
         assert(soself->s_size >= 0);
 
-        PyErr_Format(StructError,
+        PyErr_Format(_structmodulestate_global->StructError,
                      "pack_into requires a buffer of at least %zu bytes for "
                      "packing %zd bytes at offset %zd "
                      "(actual buffer size is %zd)",
@@ -2014,6 +2023,11 @@ static struct PyMethodDef s_methods[] = {
     {NULL,       NULL}          /* sentinel */
 };
 
+static PyMemberDef s_members[] = {
+    {"__weaklistoffset__", T_NONE, offsetof(PyStructObject, weakreflist), READONLY},
+    {NULL}  /* sentinel */
+};
+
 #define OFF(x) offsetof(PyStructObject, x)
 
 static PyGetSetDef s_getsetlist[] = {
@@ -2022,47 +2036,32 @@ static PyGetSetDef s_getsetlist[] = {
     {NULL} /* sentinel */
 };
 
-static
-PyTypeObject PyStructType = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    "Struct",
+PyDoc_STRVAR(s__doc__,
+"Struct(fmt) --> compiled struct object\n"
+"\n"
+);
+
+static PyType_Slot PyStructType_slots[] = {
+    {Py_tp_dealloc, s_dealloc},
+    {Py_tp_getattro, PyObject_GenericGetAttr},
+    {Py_tp_setattro, PyObject_GenericSetAttr},
+    {Py_tp_doc, (void*)s__doc__},
+    {Py_tp_methods, s_methods},
+    {Py_tp_members, s_members},
+    {Py_tp_getset, s_getsetlist},
+    {Py_tp_init, Struct___init__},
+    {Py_tp_alloc, PyType_GenericAlloc},
+    {Py_tp_new, s_new},
+    {Py_tp_free, PyObject_Del},
+    {0, 0},
+};
+
+static PyType_Spec PyStructType_spec = {
+    "_struct.Struct",
     sizeof(PyStructObject),
     0,
-    (destructor)s_dealloc,      /* tp_dealloc */
-    0,                                          /* tp_vectorcall_offset */
-    0,                                          /* tp_getattr */
-    0,                                          /* tp_setattr */
-    0,                                          /* tp_as_async */
-    0,                                          /* tp_repr */
-    0,                                          /* tp_as_number */
-    0,                                          /* tp_as_sequence */
-    0,                                          /* tp_as_mapping */
-    0,                                          /* tp_hash */
-    0,                                          /* tp_call */
-    0,                                          /* tp_str */
-    PyObject_GenericGetAttr,                    /* tp_getattro */
-    PyObject_GenericSetAttr,                    /* tp_setattro */
-    0,                                          /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,   /* tp_flags */
-    Struct___init____doc__,                     /* tp_doc */
-    0,                                          /* tp_traverse */
-    0,                                          /* tp_clear */
-    0,                                          /* tp_richcompare */
-    offsetof(PyStructObject, weakreflist),      /* tp_weaklistoffset */
-    0,                                          /* tp_iter */
-    0,                                          /* tp_iternext */
-    s_methods,                                  /* tp_methods */
-    NULL,                                       /* tp_members */
-    s_getsetlist,                               /* tp_getset */
-    0,                                          /* tp_base */
-    0,                                          /* tp_dict */
-    0,                                          /* tp_descr_get */
-    0,                                          /* tp_descr_set */
-    0,                                          /* tp_dictoffset */
-    Struct___init__,                            /* tp_init */
-    PyType_GenericAlloc,                        /* tp_alloc */
-    s_new,                                      /* tp_new */
-    PyObject_Del,                               /* tp_free */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    PyStructType_slots
 };
 
 
@@ -2098,7 +2097,7 @@ cache_struct_converter(PyObject *fmt, PyStructObject **ptr)
         return 0;
     }
 
-    s_object = _PyObject_CallOneArg((PyObject *)(&PyStructType), fmt);
+    s_object = _PyObject_CallOneArg(_structmodulestate_global->PyStructType, fmt);
     if (s_object != NULL) {
         if (PyDict_GET_SIZE(cache) >= MAXCACHE)
             PyDict_Clear(cache);
@@ -2308,16 +2307,40 @@ Whitespace between formats is ignored.\n\
 The variable struct.error is an exception raised on errors.\n");
 
 
+static int
+_structmodule_traverse(PyObject *module, visitproc visit, void *arg)
+{
+    Py_VISIT(_structmodulestate(module)->PyStructType);
+    Py_VISIT(_structmodulestate(module)->unpackiter_type);
+    Py_VISIT(_structmodulestate(module)->StructError);
+    return 0;
+}
+
+static int
+_structmodule_clear(PyObject *module)
+{
+    Py_CLEAR(_structmodulestate(module)->PyStructType);
+    Py_CLEAR(_structmodulestate(module)->unpackiter_type);
+    Py_CLEAR(_structmodulestate(module)->StructError);
+    return 0;
+}
+
+static void
+_structmodule_free(void *module)
+{
+    _structmodule_clear((PyObject *)module);
+}
+
 static struct PyModuleDef _structmodule = {
     PyModuleDef_HEAD_INIT,
     "_struct",
     module_doc,
-    -1,
+    sizeof(_structmodulestate),
     module_functions,
     NULL,
-    NULL,
-    NULL,
-    NULL
+    _structmodule_traverse,
+    _structmodule_clear,
+    _structmodule_free,
 };
 
 PyMODINIT_FUNC
@@ -2329,12 +2352,19 @@ PyInit__struct(void)
     if (m == NULL)
         return NULL;
 
-    Py_TYPE(&PyStructType) = &PyType_Type;
-    if (PyType_Ready(&PyStructType) < 0)
+    PyObject *PyStructType = PyType_FromSpec(&PyStructType_spec);
+    if (PyStructType == NULL) {
         return NULL;
+    }
+    Py_INCREF(PyStructType);
+    PyModule_AddObject(m, "Struct", PyStructType);
+    _structmodulestate(m)->PyStructType = PyStructType;
 
-    if (PyType_Ready(&unpackiter_type) < 0)
+    PyObject *unpackiter_type = PyType_FromSpec(&unpackiter_type_spec);
+    if (unpackiter_type == NULL) {
         return NULL;
+    }
+    _structmodulestate(m)->unpackiter_type = unpackiter_type;
 
     /* Check endian and swap in faster functions */
     {
@@ -2376,17 +2406,12 @@ PyInit__struct(void)
     }
 
     /* Add some symbolic constants to the module */
-    if (StructError == NULL) {
-        StructError = PyErr_NewException("struct.error", NULL, NULL);
-        if (StructError == NULL)
-            return NULL;
-    }
-
+    PyObject *StructError = PyErr_NewException("struct.error", NULL, NULL);
+    if (StructError == NULL)
+        return NULL;
     Py_INCREF(StructError);
     PyModule_AddObject(m, "error", StructError);
-
-    Py_INCREF((PyObject*)&PyStructType);
-    PyModule_AddObject(m, "Struct", (PyObject*)&PyStructType);
+    _structmodulestate(m)->StructError = StructError;
 
     return m;
 }


From webhook-mailer at python.org  Tue Sep 10 06:27:06 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Tue, 10 Sep 2019 10:27:06 -0000
Subject: [Python-checkins] bpo-38074: Make zlib extension module PEP-384
 compatible (GH-15792)
Message-ID: <mailman.409.1568111228.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a1ffad07195b8b976f8c371a92028240946d4e76
commit: a1ffad07195b8b976f8c371a92028240946d4e76
branch: master
author: Dino Viehland <dinoviehland at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-10T03:27:03-07:00
summary:

bpo-38074: Make zlib extension module PEP-384 compatible (GH-15792)

Updated zlibmodule.c to be PEP 384 compliant.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-40-57.bpo-38074.MsVbeI.rst
M Modules/zlibmodule.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-40-57.bpo-38074.MsVbeI.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-40-57.bpo-38074.MsVbeI.rst
new file mode 100644
index 000000000000..a20b40e750b0
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-40-57.bpo-38074.MsVbeI.rst	
@@ -0,0 +1 @@
+zlib module made PEP-384 compatible
\ No newline at end of file
diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c
index a3d9ed6646de..3e77080a31cb 100644
--- a/Modules/zlibmodule.c
+++ b/Modules/zlibmodule.c
@@ -32,10 +32,16 @@
 /* Initial buffer size. */
 #define DEF_BUF_SIZE (16*1024)
 
-static PyTypeObject Comptype;
-static PyTypeObject Decomptype;
+static PyModuleDef zlibmodule;
 
-static PyObject *ZlibError;
+typedef struct {
+    PyTypeObject *Comptype;
+    PyTypeObject *Decomptype;
+    PyObject *ZlibError;
+} _zlibstate;
+
+#define _zlibstate(o) ((_zlibstate *)PyModule_GetState(o))
+#define _zlibstate_global ((_zlibstate *)PyModule_GetState(PyState_FindModule(&zlibmodule)))
 
 typedef struct
 {
@@ -73,9 +79,9 @@ zlib_error(z_stream zst, int err, const char *msg)
         }
     }
     if (zmsg == Z_NULL)
-        PyErr_Format(ZlibError, "Error %d %s", err, msg);
+        PyErr_Format(_zlibstate_global->ZlibError, "Error %d %s", err, msg);
     else
-        PyErr_Format(ZlibError, "Error %d %s: %.200s", err, msg, zmsg);
+        PyErr_Format(_zlibstate_global->ZlibError, "Error %d %s: %.200s", err, msg, zmsg);
 }
 
 /*[clinic input]
@@ -226,7 +232,7 @@ zlib_compress_impl(PyObject *module, Py_buffer *data, int level)
                         "Out of memory while compressing data");
         goto error;
     case Z_STREAM_ERROR:
-        PyErr_SetString(ZlibError, "Bad compression level");
+        PyErr_SetString(_zlibstate_global->ZlibError, "Bad compression level");
         goto error;
     default:
         deflateEnd(&zst);
@@ -462,7 +468,7 @@ zlib_compressobj_impl(PyObject *module, int level, int method, int wbits,
         goto error;
     }
 
-    self = newcompobject(&Comptype);
+    self = newcompobject(_zlibstate_global->Comptype);
     if (self == NULL)
         goto error;
     self->zst.opaque = NULL;
@@ -558,7 +564,7 @@ zlib_decompressobj_impl(PyObject *module, int wbits, PyObject *zdict)
         return NULL;
     }
 
-    self = newcompobject(&Decomptype);
+    self = newcompobject(_zlibstate_global->Decomptype);
     if (self == NULL)
         return NULL;
     self->zst.opaque = NULL;
@@ -581,7 +587,7 @@ zlib_decompressobj_impl(PyObject *module, int wbits, PyObject *zdict)
                 return NULL;
             }
 #else
-            PyErr_Format(ZlibError,
+            PyErr_Format(_zlibstate_global->ZlibError,
                          "zlib version %s does not allow raw inflate with dictionary",
                          ZLIB_VERSION);
             Py_DECREF(self);
@@ -608,11 +614,13 @@ zlib_decompressobj_impl(PyObject *module, int wbits, PyObject *zdict)
 static void
 Dealloc(compobject *self)
 {
+    PyObject *type = (PyObject *)Py_TYPE(self);
     PyThread_free_lock(self->lock);
     Py_XDECREF(self->unused_data);
     Py_XDECREF(self->unconsumed_tail);
     Py_XDECREF(self->zdict);
     PyObject_Del(self);
+    Py_DECREF(type);
 }
 
 static void
@@ -944,7 +952,7 @@ zlib_Compress_copy_impl(compobject *self)
     compobject *retval = NULL;
     int err;
 
-    retval = newcompobject(&Comptype);
+    retval = newcompobject(_zlibstate_global->Comptype);
     if (!retval) return NULL;
 
     /* Copy the zstream state
@@ -1025,7 +1033,7 @@ zlib_Decompress_copy_impl(compobject *self)
     compobject *retval = NULL;
     int err;
 
-    retval = newcompobject(&Decomptype);
+    retval = newcompobject(_zlibstate_global->Decomptype);
     if (!retval) return NULL;
 
     /* Copy the zstream state
@@ -1309,67 +1317,33 @@ static PyMethodDef zlib_methods[] =
     {NULL, NULL}
 };
 
-static PyTypeObject Comptype = {
-    PyVarObject_HEAD_INIT(0, 0)
+static PyType_Slot Comptype_slots[] = {
+    {Py_tp_dealloc, Comp_dealloc},
+    {Py_tp_methods, comp_methods},
+    {0, 0},
+};
+
+static PyType_Spec Comptype_spec = {
     "zlib.Compress",
     sizeof(compobject),
     0,
-    (destructor)Comp_dealloc,       /*tp_dealloc*/
-    0,                              /*tp_vectorcall_offset*/
-    0,                              /*tp_getattr*/
-    0,                              /*tp_setattr*/
-    0,                              /*tp_as_async*/
-    0,                              /*tp_repr*/
-    0,                              /*tp_as_number*/
-    0,                              /*tp_as_sequence*/
-    0,                              /*tp_as_mapping*/
-    0,                              /*tp_hash*/
-    0,                              /*tp_call*/
-    0,                              /*tp_str*/
-    0,                              /*tp_getattro*/
-    0,                              /*tp_setattro*/
-    0,                              /*tp_as_buffer*/
-    Py_TPFLAGS_DEFAULT,             /*tp_flags*/
-    0,                              /*tp_doc*/
-    0,                              /*tp_traverse*/
-    0,                              /*tp_clear*/
-    0,                              /*tp_richcompare*/
-    0,                              /*tp_weaklistoffset*/
-    0,                              /*tp_iter*/
-    0,                              /*tp_iternext*/
-    comp_methods,                   /*tp_methods*/
+    Py_TPFLAGS_DEFAULT,
+    Comptype_slots
 };
 
-static PyTypeObject Decomptype = {
-    PyVarObject_HEAD_INIT(0, 0)
+static PyType_Slot Decomptype_slots[] = {
+    {Py_tp_dealloc, Decomp_dealloc},
+    {Py_tp_methods, Decomp_methods},
+    {Py_tp_members, Decomp_members},
+    {0, 0},
+};
+
+static PyType_Spec Decomptype_spec = {
     "zlib.Decompress",
     sizeof(compobject),
     0,
-    (destructor)Decomp_dealloc,     /*tp_dealloc*/
-    0,                              /*tp_vectorcall_offset*/
-    0,                              /*tp_getattr*/
-    0,                              /*tp_setattr*/
-    0,                              /*tp_as_async*/
-    0,                              /*tp_repr*/
-    0,                              /*tp_as_number*/
-    0,                              /*tp_as_sequence*/
-    0,                              /*tp_as_mapping*/
-    0,                              /*tp_hash*/
-    0,                              /*tp_call*/
-    0,                              /*tp_str*/
-    0,                              /*tp_getattro*/
-    0,                              /*tp_setattro*/
-    0,                              /*tp_as_buffer*/
-    Py_TPFLAGS_DEFAULT,             /*tp_flags*/
-    0,                              /*tp_doc*/
-    0,                              /*tp_traverse*/
-    0,                              /*tp_clear*/
-    0,                              /*tp_richcompare*/
-    0,                              /*tp_weaklistoffset*/
-    0,                              /*tp_iter*/
-    0,                              /*tp_iternext*/
-    Decomp_methods,                 /*tp_methods*/
-    Decomp_members,                 /*tp_members*/
+    Py_TPFLAGS_DEFAULT,
+    Decomptype_slots
 };
 
 PyDoc_STRVAR(zlib_module_documentation,
@@ -1387,34 +1361,72 @@ PyDoc_STRVAR(zlib_module_documentation,
 "Compressor objects support compress() and flush() methods; decompressor\n"
 "objects support decompress() and flush().");
 
+static int
+zlib_clear(PyObject *m)
+{
+    _zlibstate *state = _zlibstate(m);
+    Py_CLEAR(state->Comptype);
+    Py_CLEAR(state->Decomptype);
+    Py_CLEAR(state->ZlibError);
+    return 0;
+}
+
+static int
+zlib_traverse(PyObject *m, visitproc visit, void *arg)
+{
+    _zlibstate *state = _zlibstate(m);
+    Py_VISIT(state->Comptype);
+    Py_VISIT(state->Decomptype);
+    Py_VISIT(state->ZlibError);
+    return 0;
+}
+
+static void
+zlib_free(void *m)
+{
+    zlib_clear((PyObject *)m);
+}
+
 static struct PyModuleDef zlibmodule = {
         PyModuleDef_HEAD_INIT,
         "zlib",
         zlib_module_documentation,
-        -1,
+        sizeof(_zlibstate),
         zlib_methods,
         NULL,
-        NULL,
-        NULL,
-        NULL
+        zlib_traverse,
+        zlib_clear,
+        zlib_free,
 };
 
 PyMODINIT_FUNC
 PyInit_zlib(void)
 {
     PyObject *m, *ver;
-    if (PyType_Ready(&Comptype) < 0)
-            return NULL;
-    if (PyType_Ready(&Decomptype) < 0)
-            return NULL;
+    m = PyState_FindModule(&zlibmodule);
+    if (m != NULL) {
+        Py_INCREF(m);
+        return m;
+    }
     m = PyModule_Create(&zlibmodule);
     if (m == NULL)
         return NULL;
 
-    ZlibError = PyErr_NewException("zlib.error", NULL, NULL);
+    PyTypeObject *Comptype = (PyTypeObject *)PyType_FromSpec(&Comptype_spec);
+    if (Comptype == NULL)
+        return NULL;
+    _zlibstate(m)->Comptype = Comptype;
+
+    PyTypeObject *Decomptype = (PyTypeObject *)PyType_FromSpec(&Decomptype_spec);
+    if (Decomptype == NULL)
+        return NULL;
+    _zlibstate(m)->Decomptype = Decomptype;
+
+    PyObject *ZlibError = PyErr_NewException("zlib.error", NULL, NULL);
     if (ZlibError != NULL) {
         Py_INCREF(ZlibError);
         PyModule_AddObject(m, "error", ZlibError);
+        _zlibstate(m)->ZlibError = ZlibError;
     }
     PyModule_AddIntMacro(m, MAX_WBITS);
     PyModule_AddIntMacro(m, DEFLATED);
@@ -1457,5 +1469,6 @@ PyInit_zlib(void)
 
     PyModule_AddStringConstant(m, "__version__", "1.0");
 
+    PyState_AddModule(m, &zlibmodule);
     return m;
 }


From webhook-mailer at python.org  Tue Sep 10 06:28:15 2019
From: webhook-mailer at python.org (Dino Viehland)
Date: Tue, 10 Sep 2019 10:28:15 -0000
Subject: [Python-checkins] Skip zoneinfo tests on VxWorks (#13535)
Message-ID: <mailman.410.1568111296.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8e7ff6aba372d1faa3afdd9d62ad7e999dbc58d2
commit: 8e7ff6aba372d1faa3afdd9d62ad7e999dbc58d2
branch: master
author: hliu0 <hongchang.liu at windriver.com>
committer: Dino Viehland <dinoviehland at gmail.com>
date: 2019-09-10T11:28:11+01:00
summary:

Skip zoneinfo tests on VxWorks (#13535)

files:
A Misc/NEWS.d/next/Library/2019-05-17-16-50-02.bpo-31904.4a5ggm.rst
M Lib/test/datetimetester.py

diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index d5de5e91a982..a1ed614cf0cd 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -5780,6 +5780,8 @@ class ZoneInfoTest(unittest.TestCase):
     zonename = 'America/New_York'
 
     def setUp(self):
+        if sys.platform == "vxworks":
+            self.skipTest("Skipping zoneinfo tests on VxWorks")
         if sys.platform == "win32":
             self.skipTest("Skipping zoneinfo tests on Windows")
         try:
diff --git a/Misc/NEWS.d/next/Library/2019-05-17-16-50-02.bpo-31904.4a5ggm.rst b/Misc/NEWS.d/next/Library/2019-05-17-16-50-02.bpo-31904.4a5ggm.rst
new file mode 100644
index 000000000000..eb7163b6a7d7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-17-16-50-02.bpo-31904.4a5ggm.rst
@@ -0,0 +1 @@
+Port test_datetime to VxWorks: skip zoneinfo tests on VxWorks


From webhook-mailer at python.org  Tue Sep 10 06:28:33 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Tue, 10 Sep 2019 10:28:33 -0000
Subject: [Python-checkins] Remove macOS tests from Travis. (GH-15809)
Message-ID: <mailman.411.1568111314.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e45b217ae0e83cfb367e9769cae59b9a5505f9b4
commit: e45b217ae0e83cfb367e9769cae59b9a5505f9b4
branch: master
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-10T11:28:29+01:00
summary:

Remove macOS tests from Travis. (GH-15809)

Azure runs macOS, so we don't need Travis to do it.

files:
M .travis.yml

diff --git a/.travis.yml b/.travis.yml
index ffc4d82c39d9..98b33393f52c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -73,16 +73,6 @@ matrix:
         - make -C Doc/ PYTHON=../python venv
       script:
         xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" doctest
-    - name: "Mac OS X tests"
-      os: osx
-      language: c
-      compiler: clang
-      # Testing under macOS is optional until testing stability has been demonstrated.
-      env: OPTIONAL=true
-      before_install:
-        # Python 3 is needed for Argument Clinic and multissl
-        - HOMEBREW_NO_AUTO_UPDATE=1 brew install xz python3
-        - export PATH=$(brew --prefix)/bin:$(brew --prefix)/sbin:$PATH
     - name: "Test code coverage (Python)"
       os: linux
       language: c


From webhook-mailer at python.org  Tue Sep 10 06:30:00 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Tue, 10 Sep 2019 10:30:00 -0000
Subject: [Python-checkins] Fix calling order of PyEval_InitThreads. (GH-4602)
Message-ID: <mailman.412.1568111400.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9e61066355b4b55c873d56f5f106a22463b56862
commit: 9e61066355b4b55c873d56f5f106a22463b56862
branch: master
author: Kenta Murata <mrkn at users.noreply.github.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-10T11:29:56+01:00
summary:

Fix calling order of PyEval_InitThreads. (GH-4602)

As described in Doc/c-api/init.rst, PyEval_InitThreads() cannot be called
before Py_Initialize() function.

files:
M Modules/_ctypes/callbacks.c

diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c
index 97463b599bc0..d2d9a6587d79 100644
--- a/Modules/_ctypes/callbacks.c
+++ b/Modules/_ctypes/callbacks.c
@@ -422,8 +422,8 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable,
 static void LoadPython(void)
 {
     if (!Py_IsInitialized()) {
-        PyEval_InitThreads();
         Py_Initialize();
+        PyEval_InitThreads();
     }
 }
 


From webhook-mailer at python.org  Tue Sep 10 06:30:40 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Tue, 10 Sep 2019 10:30:40 -0000
Subject: [Python-checkins] bpo-38072: PEP-384 grpmodule (GH-15788)
Message-ID: <mailman.413.1568111441.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/40a5313edfc18173d136bb5e19495880934b7d83
commit: 40a5313edfc18173d136bb5e19495880934b7d83
branch: master
author: Dino Viehland <dinoviehland at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-10T03:30:36-07:00
summary:

bpo-38072: PEP-384 grpmodule (GH-15788)

Make the grp module PEP-384 compliant.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-00-42.bpo-38072.Y1xpDO.rst
M Modules/grpmodule.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-00-42.bpo-38072.Y1xpDO.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-00-42.bpo-38072.Y1xpDO.rst
new file mode 100644
index 000000000000..2860e69ea496
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-00-42.bpo-38072.Y1xpDO.rst	
@@ -0,0 +1 @@
+grp module made PEP-384 compatible
\ No newline at end of file
diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c
index ab766b985093..81f969c51dd0 100644
--- a/Modules/grpmodule.c
+++ b/Modules/grpmodule.c
@@ -34,8 +34,13 @@ static PyStructSequence_Desc struct_group_type_desc = {
 };
 
 
-static int initialized;
-static PyTypeObject StructGrpType;
+typedef struct {
+  PyTypeObject *StructGrpType;
+} grpmodulestate;
+#define modulestate(o) ((grpmodulestate *)PyModule_GetState(o))
+#define modulestate_global modulestate(PyState_FindModule(&grpmodule))
+
+static struct PyModuleDef grpmodule;
 
 #define DEFAULT_BUFFER_SIZE 1024
 
@@ -43,10 +48,10 @@ static PyObject *
 mkgrent(struct group *p)
 {
     int setIndex = 0;
-    PyObject *v = PyStructSequence_New(&StructGrpType), *w;
+    PyObject *v, *w;
     char **member;
 
-    if (v == NULL)
+    if ((v = PyStructSequence_New(modulestate_global->StructGrpType)) == NULL)
         return NULL;
 
     if ((w = PyList_New(0)) == NULL) {
@@ -314,36 +319,52 @@ users are not explicitly listed as members of the groups they are in\n\
 according to the password database.  Check both databases to get\n\
 complete membership information.)");
 
+static int grpmodule_traverse(PyObject *m, visitproc visit, void *arg) {
+    Py_VISIT(modulestate(m)->StructGrpType);
+    return 0;
+}
 
+static int grpmodule_clear(PyObject *m) {
+    Py_CLEAR(modulestate(m)->StructGrpType);
+    return 0;
+}
+
+static void grpmodule_free(void *m) {
+    grpmodule_clear((PyObject *)m);
+}
 
 static struct PyModuleDef grpmodule = {
         PyModuleDef_HEAD_INIT,
         "grp",
         grp__doc__,
-        -1,
+        sizeof(grpmodulestate),
         grp_methods,
         NULL,
-        NULL,
-        NULL,
-        NULL
+        grpmodule_traverse,
+        grpmodule_clear,
+        grpmodule_free,
 };
 
 PyMODINIT_FUNC
 PyInit_grp(void)
 {
-    PyObject *m, *d;
-    m = PyModule_Create(&grpmodule);
-    if (m == NULL)
+    PyObject *m;
+    if ((m = PyState_FindModule(&grpmodule)) != NULL) {
+        Py_INCREF(m);
+        return m;
+    }
+
+    if ((m = PyModule_Create(&grpmodule)) == NULL) {
         return NULL;
-    d = PyModule_GetDict(m);
-    if (!initialized) {
-        if (PyStructSequence_InitType2(&StructGrpType,
-                                       &struct_group_type_desc) < 0)
-            return NULL;
     }
-    if (PyDict_SetItemString(d, "struct_group",
-                             (PyObject *)&StructGrpType) < 0)
+
+    grpmodulestate *state = PyModule_GetState(m);
+    state->StructGrpType = PyStructSequence_NewType(&struct_group_type_desc);
+    if (state->StructGrpType == NULL) {
         return NULL;
-    initialized = 1;
+    }
+
+    Py_INCREF(state->StructGrpType);
+    PyModule_AddObject(m, "struct_group", (PyObject *) state->StructGrpType);
     return m;
 }


From webhook-mailer at python.org  Tue Sep 10 06:33:34 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Tue, 10 Sep 2019 10:33:34 -0000
Subject: [Python-checkins] bpo-38071: Make termios PEP-384 compatible
 (GH-15785)
Message-ID: <mailman.414.1568111616.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bd0c7a12d9e28ce74bfc16244d7694aca906838c
commit: bd0c7a12d9e28ce74bfc16244d7694aca906838c
branch: master
author: Dino Viehland <dinoviehland at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-10T03:33:30-07:00
summary:

bpo-38071: Make termios PEP-384 compatible (GH-15785)

Make the termios module PEP-384 compatible.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-09-14-44-17.bpo-38071.bLwkBJ.rst
M Modules/termios.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-09-14-44-17.bpo-38071.bLwkBJ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-14-44-17.bpo-38071.bLwkBJ.rst
new file mode 100644
index 000000000000..327bf73bfd45
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-14-44-17.bpo-38071.bLwkBJ.rst	
@@ -0,0 +1 @@
+Make termios extension module PEP-384 compatible
\ No newline at end of file
diff --git a/Modules/termios.c b/Modules/termios.c
index aee7f12c57ca..b3b8c72bf6b8 100644
--- a/Modules/termios.c
+++ b/Modules/termios.c
@@ -39,7 +39,11 @@ All functions in this module take a file descriptor fd as their first\n\
 argument. This can be an integer file descriptor, such as returned by\n\
 sys.stdin.fileno(), or a file object, such as sys.stdin itself.");
 
-static PyObject *TermiosError;
+typedef struct {
+  PyObject *TermiosError;
+} termiosmodulestate;
+#define modulestate(o) ((termiosmodulestate *)PyModule_GetState(o))
+#define modulestate_global modulestate(PyState_FindModule(&termiosmodule))
 
 static int fdconv(PyObject* obj, void* p)
 {
@@ -53,6 +57,8 @@ static int fdconv(PyObject* obj, void* p)
     return 0;
 }
 
+static struct PyModuleDef termiosmodule;
+
 PyDoc_STRVAR(termios_tcgetattr__doc__,
 "tcgetattr(fd) -> list_of_attrs\n\
 \n\
@@ -80,7 +86,7 @@ termios_tcgetattr(PyObject *self, PyObject *args)
         return NULL;
 
     if (tcgetattr(fd, &mode) == -1)
-        return PyErr_SetFromErrno(TermiosError);
+        return PyErr_SetFromErrno(modulestate_global->TermiosError);
 
     ispeed = cfgetispeed(&mode);
     ospeed = cfgetospeed(&mode);
@@ -160,8 +166,9 @@ termios_tcsetattr(PyObject *self, PyObject *args)
     }
 
     /* Get the old mode, in case there are any hidden fields... */
+    termiosmodulestate *state = modulestate_global;
     if (tcgetattr(fd, &mode) == -1)
-        return PyErr_SetFromErrno(TermiosError);
+        return PyErr_SetFromErrno(state->TermiosError);
     mode.c_iflag = (tcflag_t) PyLong_AsLong(PyList_GetItem(term, 0));
     mode.c_oflag = (tcflag_t) PyLong_AsLong(PyList_GetItem(term, 1));
     mode.c_cflag = (tcflag_t) PyLong_AsLong(PyList_GetItem(term, 2));
@@ -194,11 +201,11 @@ termios_tcsetattr(PyObject *self, PyObject *args)
     }
 
     if (cfsetispeed(&mode, (speed_t) ispeed) == -1)
-        return PyErr_SetFromErrno(TermiosError);
+        return PyErr_SetFromErrno(state->TermiosError);
     if (cfsetospeed(&mode, (speed_t) ospeed) == -1)
-        return PyErr_SetFromErrno(TermiosError);
+        return PyErr_SetFromErrno(state->TermiosError);
     if (tcsetattr(fd, when, &mode) == -1)
-        return PyErr_SetFromErrno(TermiosError);
+        return PyErr_SetFromErrno(state->TermiosError);
 
     Py_RETURN_NONE;
 }
@@ -219,7 +226,7 @@ termios_tcsendbreak(PyObject *self, PyObject *args)
                           fdconv, &fd, &duration))
         return NULL;
     if (tcsendbreak(fd, duration) == -1)
-        return PyErr_SetFromErrno(TermiosError);
+        return PyErr_SetFromErrno(modulestate_global->TermiosError);
 
     Py_RETURN_NONE;
 }
@@ -238,7 +245,7 @@ termios_tcdrain(PyObject *self, PyObject *args)
                           fdconv, &fd))
         return NULL;
     if (tcdrain(fd) == -1)
-        return PyErr_SetFromErrno(TermiosError);
+        return PyErr_SetFromErrno(modulestate_global->TermiosError);
 
     Py_RETURN_NONE;
 }
@@ -260,7 +267,7 @@ termios_tcflush(PyObject *self, PyObject *args)
                           fdconv, &fd, &queue))
         return NULL;
     if (tcflush(fd, queue) == -1)
-        return PyErr_SetFromErrno(TermiosError);
+        return PyErr_SetFromErrno(modulestate_global->TermiosError);
 
     Py_RETURN_NONE;
 }
@@ -282,7 +289,7 @@ termios_tcflow(PyObject *self, PyObject *args)
                           fdconv, &fd, &action))
         return NULL;
     if (tcflow(fd, action) == -1)
-        return PyErr_SetFromErrno(TermiosError);
+        return PyErr_SetFromErrno(modulestate_global->TermiosError);
 
     Py_RETURN_NONE;
 }
@@ -935,17 +942,30 @@ static struct constant {
     {NULL, 0}
 };
 
+static int termiosmodule_traverse(PyObject *m, visitproc visit, void *arg) {
+    Py_VISIT(modulestate(m)->TermiosError);
+    return 0;
+}
+
+static int termiosmodule_clear(PyObject *m) {
+    Py_CLEAR(modulestate(m)->TermiosError);
+    return 0;
+}
+
+static void termiosmodule_free(void *m) {
+    termiosmodule_clear((PyObject *)m);
+}
 
 static struct PyModuleDef termiosmodule = {
     PyModuleDef_HEAD_INIT,
     "termios",
     termios__doc__,
-    -1,
+    sizeof(termiosmodulestate),
     termios_methods,
     NULL,
-    NULL,
-    NULL,
-    NULL
+    termiosmodule_traverse,
+    termiosmodule_clear,
+    termiosmodule_free,
 };
 
 PyMODINIT_FUNC
@@ -954,15 +974,22 @@ PyInit_termios(void)
     PyObject *m;
     struct constant *constant = termios_constants;
 
-    m = PyModule_Create(&termiosmodule);
-    if (m == NULL)
+    if ((m = PyState_FindModule(&termiosmodule)) != NULL) {
+        Py_INCREF(m);
+        return m;
+    }
+
+    if ((m = PyModule_Create(&termiosmodule)) == NULL) {
         return NULL;
+    }
 
-    if (TermiosError == NULL) {
-        TermiosError = PyErr_NewException("termios.error", NULL, NULL);
+    termiosmodulestate *state = PyModule_GetState(m);
+    state->TermiosError = PyErr_NewException("termios.error", NULL, NULL);
+    if (state->TermiosError == NULL) {
+        return NULL;
     }
-    Py_INCREF(TermiosError);
-    PyModule_AddObject(m, "error", TermiosError);
+    Py_INCREF(state->TermiosError);
+    PyModule_AddObject(m, "error", state->TermiosError);
 
     while (constant->name != NULL) {
         PyModule_AddIntConstant(m, constant->name, constant->value);


From webhook-mailer at python.org  Tue Sep 10 06:37:21 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 10:37:21 -0000
Subject: [Python-checkins] bpo-37052: Add examples for mocking async
 iterators and context managers (GH-14660)
Message-ID: <mailman.415.1568111842.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c8dfa7333d6317d7cd8c5c7366023f5a668e3f91
commit: c8dfa7333d6317d7cd8c5c7366023f5a668e3f91
branch: master
author: Xtreak <tir.karthi at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-10T03:37:17-07:00
summary:

bpo-37052: Add examples for mocking async iterators and context managers (GH-14660)



Add examples for mocking asynchronous iterators and asynchronous context managers.


https://bugs.python.org/issue37052

files:
M Doc/library/unittest.mock-examples.rst

diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst
index 811f0fb1ce93..cf6b671b5bee 100644
--- a/Doc/library/unittest.mock-examples.rst
+++ b/Doc/library/unittest.mock-examples.rst
@@ -12,6 +12,7 @@
 
 .. testsetup::
 
+    import asyncio
     import unittest
     from unittest.mock import Mock, MagicMock, patch, call, sentinel
 
@@ -276,6 +277,44 @@ function returns is what the call returns:
     2
 
 
+Mocking asynchronous iterators
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since Python 3.8, ``MagicMock`` has support to mock :ref:`async-iterators`
+through ``__aiter__``. The :attr:`~Mock.return_value` attribute of ``__aiter__``
+can be used to set the return values to be used for iteration.
+
+    >>> mock = MagicMock()
+    >>> mock.__aiter__.return_value = [1, 2, 3]
+    >>> async def main():
+    ...     return [i async for i in mock]
+    >>> asyncio.run(main())
+    [1, 2, 3]
+
+
+Mocking asynchronous context manager
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since Python 3.8, ``MagicMock`` has support to mock
+:ref:`async-context-managers` through ``__aenter__`` and ``__aexit__``. The
+return value of ``__aenter__`` is an :class:`AsyncMock`.
+
+    >>> class AsyncContextManager:
+    ...
+    ...     async def __aenter__(self):
+    ...         return self
+    ...
+    ...     async def __aexit__(self):
+    ...         pass
+    >>> mock_instance = MagicMock(AsyncContextManager())
+    >>> async def main():
+    ...     async with mock_instance as result:
+    ...         pass
+    >>> asyncio.run(main())
+    >>> mock_instance.__aenter__.assert_called_once()
+    >>> mock_instance.__aexit__.assert_called_once()
+
+
 Creating a Mock from an Existing Object
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 


From webhook-mailer at python.org  Tue Sep 10 06:38:02 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Tue, 10 Sep 2019 10:38:02 -0000
Subject: [Python-checkins] bpo-38068: Clean up gettimeofday configure logic.
 (GH-15775)
Message-ID: <mailman.416.1568111884.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f1c19031fd5f4cf6faad539e30796b42954527db
commit: f1c19031fd5f4cf6faad539e30796b42954527db
branch: master
author: Benjamin Peterson <benjamin at python.org>
committer: T. Wouters <thomas at python.org>
date: 2019-09-10T03:37:59-07:00
summary:

bpo-38068: Clean up gettimeofday configure logic. (GH-15775)

Assume gettimeofday exists and takes two arguments.

files:
M Python/pytime.c
M Python/thread_pthread.h
M configure
M configure.ac
M pyconfig.h.in

diff --git a/Python/pytime.c b/Python/pytime.c
index 9ff300699f04..54ddfc952b81 100644
--- a/Python/pytime.c
+++ b/Python/pytime.c
@@ -718,11 +718,7 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise)
 #else   /* HAVE_CLOCK_GETTIME */
 
      /* test gettimeofday() */
-#ifdef GETTIMEOFDAY_NO_TZ
-    err = gettimeofday(&tv);
-#else
     err = gettimeofday(&tv, (struct timezone *)NULL);
-#endif
     if (err) {
         if (raise) {
             PyErr_SetFromErrno(PyExc_OSError);
diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h
index 5678b05ced36..b9a340530cfe 100644
--- a/Python/thread_pthread.h
+++ b/Python/thread_pthread.h
@@ -97,17 +97,10 @@
 #endif
 
 
-/* We assume all modern POSIX systems have gettimeofday() */
-#ifdef GETTIMEOFDAY_NO_TZ
-#define GETTIMEOFDAY(ptv) gettimeofday(ptv)
-#else
-#define GETTIMEOFDAY(ptv) gettimeofday(ptv, (struct timezone *)NULL)
-#endif
-
 #define MICROSECONDS_TO_TIMESPEC(microseconds, ts) \
 do { \
     struct timeval tv; \
-    GETTIMEOFDAY(&tv); \
+    gettimeofday(&tv, NULL); \
     tv.tv_usec += microseconds % 1000000; \
     tv.tv_sec += microseconds / 1000000; \
     tv.tv_sec += tv.tv_usec / 1000000; \
diff --git a/configure b/configure
index d4d59be8602a..cb60ce256d43 100755
--- a/configure
+++ b/configure
@@ -12735,37 +12735,6 @@ if ac_fn_c_try_compile "$LINENO"; then :
 
 $as_echo "#define SETPGRP_HAVE_ARG 1" >>confdefs.h
 
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-
-fi
-done
-
-for ac_func in gettimeofday
-do :
-  ac_fn_c_check_func "$LINENO" "gettimeofday" "ac_cv_func_gettimeofday"
-if test "x$ac_cv_func_gettimeofday" = xyes; then :
-  cat >>confdefs.h <<_ACEOF
-#define HAVE_GETTIMEOFDAY 1
-_ACEOF
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-#include <sys/time.h>
-int
-main ()
-{
-gettimeofday((struct timeval*)0,(struct timezone*)0);
-  ;
-  return 0;
-}
-_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
-
-else
-
-$as_echo "#define GETTIMEOFDAY_NO_TZ 1" >>confdefs.h
-
-
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
 
diff --git a/configure.ac b/configure.ac
index 43514129dd5c..9f5cea5ca0c7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3865,15 +3865,6 @@ AC_CHECK_FUNCS(setpgrp,
     [AC_DEFINE(SETPGRP_HAVE_ARG, 1, [Define if setpgrp() must be called as setpgrp(0, 0).])],
     [])
 )
-AC_CHECK_FUNCS(gettimeofday,
-  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/time.h>]],
-  				     [[gettimeofday((struct timeval*)0,(struct timezone*)0);]])],
-    [],
-    [AC_DEFINE(GETTIMEOFDAY_NO_TZ, 1,
-      [Define if gettimeofday() does not have second (timezone) argument
-       This is the case on Motorola V4 (R40V4.2)])
-    ])
-)
 
 # We search for both crypt and crypt_r as one or the other may be defined
 # This gets us our -lcrypt in LIBS when required on the target platform.
diff --git a/pyconfig.h.in b/pyconfig.h.in
index 840fbcdb2a6f..5ff5cc2282f0 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -44,10 +44,6 @@
 /* Define if getpgrp() must be called as getpgrp(0). */
 #undef GETPGRP_HAVE_ARG
 
-/* Define if gettimeofday() does not have second (timezone) argument This is
-   the case on Motorola V4 (R40V4.2) */
-#undef GETTIMEOFDAY_NO_TZ
-
 /* Define to 1 if you have the `accept4' function. */
 #undef HAVE_ACCEPT4
 
@@ -510,9 +506,6 @@
 /* Define to 1 if you have the `getspnam' function. */
 #undef HAVE_GETSPNAM
 
-/* Define to 1 if you have the `gettimeofday' function. */
-#undef HAVE_GETTIMEOFDAY
-
 /* Define to 1 if you have the `getwd' function. */
 #undef HAVE_GETWD
 


From webhook-mailer at python.org  Tue Sep 10 06:42:03 2019
From: webhook-mailer at python.org (Dino Viehland)
Date: Tue, 10 Sep 2019 10:42:03 -0000
Subject: [Python-checkins] bpo-36971: add subsections in C API "Common
 Object Structures" page (#13446)
Message-ID: <mailman.417.1568112123.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9669931e5e76cf4b6ae6d3d66e699b5fd6ffe931
commit: 9669931e5e76cf4b6ae6d3d66e699b5fd6ffe931
branch: master
author: Jeroen Demeyer <J.Demeyer at UGent.be>
committer: Dino Viehland <dinoviehland at gmail.com>
date: 2019-09-10T11:41:59+01:00
summary:

bpo-36971: add subsections in C API "Common Object Structures" page (#13446)

files:
M Doc/c-api/structures.rst

diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst
index d4e65afef14d..af797778ce63 100644
--- a/Doc/c-api/structures.rst
+++ b/Doc/c-api/structures.rst
@@ -9,6 +9,10 @@ There are a large number of structures which are used in the definition of
 object types for Python.  This section describes these structures and how they
 are used.
 
+
+Base object types and macros
+----------------------------
+
 All Python objects ultimately share a small number of fields at the beginning
 of the object's representation in memory.  These are represented by the
 :c:type:`PyObject` and :c:type:`PyVarObject` types, which are defined, in turn,
@@ -102,6 +106,9 @@ the definition of all other Python objects.
       1, type, size,
 
 
+Implementing functions and methods
+----------------------------------
+
 .. c:type:: PyCFunction
 
    Type of the functions used to implement most Python callables in C.
@@ -271,6 +278,9 @@ definition with the same method name.
    than wrapper object calls.
 
 
+Accessing attributes of extension types
+---------------------------------------
+
 .. c:type:: PyMemberDef
 
    Structure which describes an attribute of a type which corresponds to a C


From webhook-mailer at python.org  Tue Sep 10 06:54:52 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 10:54:52 -0000
Subject: [Python-checkins] Remove macOS tests from Travis. (GH-15809)
Message-ID: <mailman.418.1568112893.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7d41e400d6acc799f49e754998f1963630252e4f
commit: 7d41e400d6acc799f49e754998f1963630252e4f
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T03:54:48-07:00
summary:

Remove macOS tests from Travis. (GH-15809)


Azure runs macOS, so we don't need Travis to do it.
(cherry picked from commit e45b217ae0e83cfb367e9769cae59b9a5505f9b4)

Co-authored-by: Benjamin Peterson <benjamin at python.org>

files:
M .travis.yml

diff --git a/.travis.yml b/.travis.yml
index 1d470728a536..519dd9c65279 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -73,16 +73,6 @@ matrix:
         - make -C Doc/ PYTHON=../python venv
       script:
         xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" doctest
-    - name: "Mac OS X tests"
-      os: osx
-      language: c
-      compiler: clang
-      # Testing under macOS is optional until testing stability has been demonstrated.
-      env: OPTIONAL=true
-      before_install:
-        # Python 3 is needed for Argument Clinic and multissl
-        - HOMEBREW_NO_AUTO_UPDATE=1 brew install xz python3
-        - export PATH=$(brew --prefix)/bin:$(brew --prefix)/sbin:$PATH
     - name: "Test code coverage (Python)"
       os: linux
       language: c


From webhook-mailer at python.org  Tue Sep 10 06:55:11 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 10:55:11 -0000
Subject: [Python-checkins] bpo-36373: Deprecate explicit loop parameter in
 all public asyncio APIs [locks] (GH-13920)
Message-ID: <mailman.419.1568112912.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/537877d85d1c27d2c2f5189e39da64a7a0c413d3
commit: 537877d85d1c27d2c2f5189e39da64a7a0c413d3
branch: master
author: Emmanuel Arias <emmanuelarias30 at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-10T03:55:07-07:00
summary:

 bpo-36373: Deprecate explicit loop parameter in all public asyncio APIs [locks] (GH-13920)



This PR deprecate explicit loop parameters in all public asyncio APIs

This issues is split to be easier to review.

Third step: locks.py





https://bugs.python.org/issue36373

files:
M Doc/library/asyncio-sync.rst
M Lib/asyncio/locks.py
M Lib/test/test_asyncio/test_events.py
M Lib/test/test_asyncio/test_locks.py
M Lib/test/test_asyncio/test_pep492.py
M Lib/test/test_asyncio/test_queues.py
M Lib/test/test_asyncio/test_tasks.py

diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst
index 79f6b02d85e2..cc8c29dc3b6b 100644
--- a/Doc/library/asyncio-sync.rst
+++ b/Doc/library/asyncio-sync.rst
@@ -59,6 +59,9 @@ Lock
        finally:
            lock.release()
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    .. coroutinemethod:: acquire()
 
       Acquire the lock.
@@ -101,6 +104,10 @@ Event
    :meth:`clear` method.  The :meth:`wait` method blocks until the
    flag is set to *true*.  The flag is set to *false* initially.
 
+
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    .. _asyncio_example_sync_event:
 
    Example::
@@ -173,6 +180,10 @@ Condition
    ``None``.  In the latter case a new Lock object is created
    automatically.
 
+
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    The preferred way to use a Condition is an :keyword:`async with`
    statement::
 
@@ -269,6 +280,10 @@ Semaphore
    internal counter (``1`` by default). If the given value is
    less than ``0`` a :exc:`ValueError` is raised.
 
+
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    The preferred way to use a Semaphore is an :keyword:`async with`
    statement::
 
@@ -322,6 +337,9 @@ BoundedSemaphore
    increases the internal counter above the initial *value*.
 
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
 ---------
 
 
diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py
index 1324eefb5ff4..f63d4cedbbb7 100644
--- a/Lib/asyncio/locks.py
+++ b/Lib/asyncio/locks.py
@@ -160,10 +160,13 @@ class Lock(_ContextManagerMixin):
     def __init__(self, *, loop=None):
         self._waiters = None
         self._locked = False
-        if loop is not None:
-            self._loop = loop
-        else:
+        if loop is None:
             self._loop = events.get_event_loop()
+        else:
+            self._loop = loop
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
 
     def __repr__(self):
         res = super().__repr__()
@@ -253,10 +256,13 @@ class Event:
     def __init__(self, *, loop=None):
         self._waiters = collections.deque()
         self._value = False
-        if loop is not None:
-            self._loop = loop
-        else:
+        if loop is None:
             self._loop = events.get_event_loop()
+        else:
+            self._loop = loop
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
 
     def __repr__(self):
         res = super().__repr__()
@@ -317,10 +323,13 @@ class Condition(_ContextManagerMixin):
     """
 
     def __init__(self, lock=None, *, loop=None):
-        if loop is not None:
-            self._loop = loop
-        else:
+        if loop is None:
             self._loop = events.get_event_loop()
+        else:
+            self._loop = loop
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
 
         if lock is None:
             lock = Lock(loop=self._loop)
@@ -445,10 +454,13 @@ def __init__(self, value=1, *, loop=None):
             raise ValueError("Semaphore initial value must be >= 0")
         self._value = value
         self._waiters = collections.deque()
-        if loop is not None:
-            self._loop = loop
-        else:
+        if loop is None:
             self._loop = events.get_event_loop()
+        else:
+            self._loop = loop
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
 
     def __repr__(self):
         res = super().__repr__()
@@ -508,6 +520,11 @@ class BoundedSemaphore(Semaphore):
     """
 
     def __init__(self, value=1, *, loop=None):
+        if loop:
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
+
         self._bound_value = value
         super().__init__(value, loop=loop)
 
diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py
index 5bc1bc2a621b..bdc861374650 100644
--- a/Lib/test/test_asyncio/test_events.py
+++ b/Lib/test/test_asyncio/test_events.py
@@ -1719,19 +1719,20 @@ def test_subprocess_exec(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
-        self.assertEqual('CONNECTED', proto.state)
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
+            self.assertEqual('CONNECTED', proto.state)
 
-        stdin = transp.get_pipe_transport(0)
-        stdin.write(b'Python The Winner')
-        self.loop.run_until_complete(proto.got_data[1].wait())
-        with test_utils.disable_logger():
-            transp.close()
-        self.loop.run_until_complete(proto.completed)
-        self.check_killed(proto.returncode)
-        self.assertEqual(b'Python The Winner', proto.data[1])
+            stdin = transp.get_pipe_transport(0)
+            stdin.write(b'Python The Winner')
+            self.loop.run_until_complete(proto.got_data[1].wait())
+            with test_utils.disable_logger():
+                transp.close()
+            self.loop.run_until_complete(proto.completed)
+            self.check_killed(proto.returncode)
+            self.assertEqual(b'Python The Winner', proto.data[1])
 
     def test_subprocess_interactive(self):
         prog = os.path.join(os.path.dirname(__file__), 'echo.py')
@@ -1739,47 +1740,52 @@ def test_subprocess_interactive(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
-        self.assertEqual('CONNECTED', proto.state)
 
-        stdin = transp.get_pipe_transport(0)
-        stdin.write(b'Python ')
-        self.loop.run_until_complete(proto.got_data[1].wait())
-        proto.got_data[1].clear()
-        self.assertEqual(b'Python ', proto.data[1])
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
+            self.assertEqual('CONNECTED', proto.state)
 
-        stdin.write(b'The Winner')
-        self.loop.run_until_complete(proto.got_data[1].wait())
-        self.assertEqual(b'Python The Winner', proto.data[1])
+            stdin = transp.get_pipe_transport(0)
+            stdin.write(b'Python ')
+            self.loop.run_until_complete(proto.got_data[1].wait())
+            proto.got_data[1].clear()
+            self.assertEqual(b'Python ', proto.data[1])
 
-        with test_utils.disable_logger():
-            transp.close()
-        self.loop.run_until_complete(proto.completed)
-        self.check_killed(proto.returncode)
+            stdin.write(b'The Winner')
+            self.loop.run_until_complete(proto.got_data[1].wait())
+            self.assertEqual(b'Python The Winner', proto.data[1])
+
+            with test_utils.disable_logger():
+                transp.close()
+            self.loop.run_until_complete(proto.completed)
+            self.check_killed(proto.returncode)
 
     def test_subprocess_shell(self):
-        connect = self.loop.subprocess_shell(
-                        functools.partial(MySubprocessProtocol, self.loop),
-                        'echo Python')
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
+        with self.assertWarns(DeprecationWarning):
+            connect = self.loop.subprocess_shell(
+                            functools.partial(MySubprocessProtocol, self.loop),
+                            'echo Python')
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
 
-        transp.get_pipe_transport(0).close()
-        self.loop.run_until_complete(proto.completed)
-        self.assertEqual(0, proto.returncode)
-        self.assertTrue(all(f.done() for f in proto.disconnects.values()))
-        self.assertEqual(proto.data[1].rstrip(b'\r\n'), b'Python')
-        self.assertEqual(proto.data[2], b'')
-        transp.close()
+            transp.get_pipe_transport(0).close()
+            self.loop.run_until_complete(proto.completed)
+            self.assertEqual(0, proto.returncode)
+            self.assertTrue(all(f.done() for f in proto.disconnects.values()))
+            self.assertEqual(proto.data[1].rstrip(b'\r\n'), b'Python')
+            self.assertEqual(proto.data[2], b'')
+            transp.close()
 
     def test_subprocess_exitcode(self):
         connect = self.loop.subprocess_shell(
                         functools.partial(MySubprocessProtocol, self.loop),
                         'exit 7', stdin=None, stdout=None, stderr=None)
-        transp, proto = self.loop.run_until_complete(connect)
+
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
         self.assertIsInstance(proto, MySubprocessProtocol)
         self.loop.run_until_complete(proto.completed)
         self.assertEqual(7, proto.returncode)
@@ -1789,7 +1795,8 @@ def test_subprocess_close_after_finish(self):
         connect = self.loop.subprocess_shell(
                         functools.partial(MySubprocessProtocol, self.loop),
                         'exit 7', stdin=None, stdout=None, stderr=None)
-        transp, proto = self.loop.run_until_complete(connect)
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
         self.assertIsInstance(proto, MySubprocessProtocol)
         self.assertIsNone(transp.get_pipe_transport(0))
         self.assertIsNone(transp.get_pipe_transport(1))
@@ -1804,14 +1811,16 @@ def test_subprocess_kill(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
 
-        transp.kill()
-        self.loop.run_until_complete(proto.completed)
-        self.check_killed(proto.returncode)
-        transp.close()
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
+
+            transp.kill()
+            self.loop.run_until_complete(proto.completed)
+            self.check_killed(proto.returncode)
+            transp.close()
 
     def test_subprocess_terminate(self):
         prog = os.path.join(os.path.dirname(__file__), 'echo.py')
@@ -1819,14 +1828,16 @@ def test_subprocess_terminate(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
 
-        transp.terminate()
-        self.loop.run_until_complete(proto.completed)
-        self.check_terminated(proto.returncode)
-        transp.close()
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
+
+            transp.terminate()
+            self.loop.run_until_complete(proto.completed)
+            self.check_terminated(proto.returncode)
+            transp.close()
 
     @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP")
     def test_subprocess_send_signal(self):
@@ -1840,14 +1851,16 @@ def test_subprocess_send_signal(self):
             connect = self.loop.subprocess_exec(
                             functools.partial(MySubprocessProtocol, self.loop),
                             sys.executable, prog)
-            transp, proto = self.loop.run_until_complete(connect)
-            self.assertIsInstance(proto, MySubprocessProtocol)
-            self.loop.run_until_complete(proto.connected)
 
-            transp.send_signal(signal.SIGHUP)
-            self.loop.run_until_complete(proto.completed)
-            self.assertEqual(-signal.SIGHUP, proto.returncode)
-            transp.close()
+            with self.assertWarns(DeprecationWarning):
+                transp, proto = self.loop.run_until_complete(connect)
+                self.assertIsInstance(proto, MySubprocessProtocol)
+                self.loop.run_until_complete(proto.connected)
+
+                transp.send_signal(signal.SIGHUP)
+                self.loop.run_until_complete(proto.completed)
+                self.assertEqual(-signal.SIGHUP, proto.returncode)
+                transp.close()
         finally:
             signal.signal(signal.SIGHUP, old_handler)
 
@@ -1857,19 +1870,21 @@ def test_subprocess_stderr(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
 
-        stdin = transp.get_pipe_transport(0)
-        stdin.write(b'test')
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
 
-        self.loop.run_until_complete(proto.completed)
+            stdin = transp.get_pipe_transport(0)
+            stdin.write(b'test')
 
-        transp.close()
-        self.assertEqual(b'OUT:test', proto.data[1])
-        self.assertTrue(proto.data[2].startswith(b'ERR:test'), proto.data[2])
-        self.assertEqual(0, proto.returncode)
+            self.loop.run_until_complete(proto.completed)
+
+            transp.close()
+            self.assertEqual(b'OUT:test', proto.data[1])
+            self.assertTrue(proto.data[2].startswith(b'ERR:test'), proto.data[2])
+            self.assertEqual(0, proto.returncode)
 
     def test_subprocess_stderr_redirect_to_stdout(self):
         prog = os.path.join(os.path.dirname(__file__), 'echo2.py')
@@ -1877,22 +1892,24 @@ def test_subprocess_stderr_redirect_to_stdout(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog, stderr=subprocess.STDOUT)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
 
-        stdin = transp.get_pipe_transport(0)
-        self.assertIsNotNone(transp.get_pipe_transport(1))
-        self.assertIsNone(transp.get_pipe_transport(2))
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
 
-        stdin.write(b'test')
-        self.loop.run_until_complete(proto.completed)
-        self.assertTrue(proto.data[1].startswith(b'OUT:testERR:test'),
-                        proto.data[1])
-        self.assertEqual(b'', proto.data[2])
+            stdin = transp.get_pipe_transport(0)
+            self.assertIsNotNone(transp.get_pipe_transport(1))
+            self.assertIsNone(transp.get_pipe_transport(2))
 
-        transp.close()
-        self.assertEqual(0, proto.returncode)
+            stdin.write(b'test')
+            self.loop.run_until_complete(proto.completed)
+            self.assertTrue(proto.data[1].startswith(b'OUT:testERR:test'),
+                            proto.data[1])
+            self.assertEqual(b'', proto.data[2])
+
+            transp.close()
+            self.assertEqual(0, proto.returncode)
 
     def test_subprocess_close_client_stream(self):
         prog = os.path.join(os.path.dirname(__file__), 'echo3.py')
@@ -1900,32 +1917,33 @@ def test_subprocess_close_client_stream(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
 
-        stdin = transp.get_pipe_transport(0)
-        stdout = transp.get_pipe_transport(1)
-        stdin.write(b'test')
-        self.loop.run_until_complete(proto.got_data[1].wait())
-        self.assertEqual(b'OUT:test', proto.data[1])
+            stdin = transp.get_pipe_transport(0)
+            stdout = transp.get_pipe_transport(1)
+            stdin.write(b'test')
+            self.loop.run_until_complete(proto.got_data[1].wait())
+            self.assertEqual(b'OUT:test', proto.data[1])
 
-        stdout.close()
-        self.loop.run_until_complete(proto.disconnects[1])
-        stdin.write(b'xxx')
-        self.loop.run_until_complete(proto.got_data[2].wait())
-        if sys.platform != 'win32':
-            self.assertEqual(b'ERR:BrokenPipeError', proto.data[2])
-        else:
-            # After closing the read-end of a pipe, writing to the
-            # write-end using os.write() fails with errno==EINVAL and
-            # GetLastError()==ERROR_INVALID_NAME on Windows!?!  (Using
-            # WriteFile() we get ERROR_BROKEN_PIPE as expected.)
-            self.assertEqual(b'ERR:OSError', proto.data[2])
-        with test_utils.disable_logger():
-            transp.close()
-        self.loop.run_until_complete(proto.completed)
-        self.check_killed(proto.returncode)
+            stdout.close()
+            self.loop.run_until_complete(proto.disconnects[1])
+            stdin.write(b'xxx')
+            self.loop.run_until_complete(proto.got_data[2].wait())
+            if sys.platform != 'win32':
+                self.assertEqual(b'ERR:BrokenPipeError', proto.data[2])
+            else:
+                # After closing the read-end of a pipe, writing to the
+                # write-end using os.write() fails with errno==EINVAL and
+                # GetLastError()==ERROR_INVALID_NAME on Windows!?!  (Using
+                # WriteFile() we get ERROR_BROKEN_PIPE as expected.)
+                self.assertEqual(b'ERR:OSError', proto.data[2])
+            with test_utils.disable_logger():
+                transp.close()
+            self.loop.run_until_complete(proto.completed)
+            self.check_killed(proto.returncode)
 
     def test_subprocess_wait_no_same_group(self):
         # start the new process in a new session
@@ -1939,7 +1957,6 @@ def test_subprocess_wait_no_same_group(self):
         self.assertEqual(7, proto.returncode)
 
     def test_subprocess_exec_invalid_args(self):
-
         async def connect(**kwds):
             await self.loop.subprocess_exec(
                 asyncio.SubprocessProtocol,
diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py
index 5063a1da448e..d69b56dcda22 100644
--- a/Lib/test/test_asyncio/test_locks.py
+++ b/Lib/test/test_asyncio/test_locks.py
@@ -28,10 +28,12 @@ def setUp(self):
 
     def test_ctor_loop(self):
         loop = mock.Mock()
-        lock = asyncio.Lock(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=loop)
         self.assertIs(lock._loop, loop)
 
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
         self.assertIs(lock._loop, self.loop)
 
     def test_ctor_noloop(self):
@@ -40,7 +42,8 @@ def test_ctor_noloop(self):
         self.assertIs(lock._loop, self.loop)
 
     def test_repr(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
         self.assertTrue(repr(lock).endswith('[unlocked]>'))
         self.assertTrue(RGX_REPR.match(repr(lock)))
 
@@ -55,9 +58,10 @@ def acquire_lock():
         self.assertTrue(RGX_REPR.match(repr(lock)))
 
     def test_lock(self):
-        lock = asyncio.Lock(loop=self.loop)
-
         with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
+
+
             @asyncio.coroutine
             def acquire_lock():
                 with self.assertWarns(DeprecationWarning):
@@ -74,14 +78,14 @@ def acquire_lock():
     def test_lock_by_with_statement(self):
         loop = asyncio.new_event_loop()  # don't use TestLoop quirks
         self.set_event_loop(loop)
-        primitives = [
-            asyncio.Lock(loop=loop),
-            asyncio.Condition(loop=loop),
-            asyncio.Semaphore(loop=loop),
-            asyncio.BoundedSemaphore(loop=loop),
-        ]
-
         with self.assertWarns(DeprecationWarning):
+            primitives = [
+                asyncio.Lock(loop=loop),
+                asyncio.Condition(loop=loop),
+                asyncio.Semaphore(loop=loop),
+                asyncio.BoundedSemaphore(loop=loop),
+            ]
+
             @asyncio.coroutine
             def test(lock):
                 yield from asyncio.sleep(0.01)
@@ -99,7 +103,8 @@ def test(lock):
             self.assertFalse(primitive.locked())
 
     def test_acquire(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
         result = []
 
         self.assertTrue(self.loop.run_until_complete(lock.acquire()))
@@ -150,7 +155,8 @@ def test_acquire(self):
         self.assertTrue(t3.result())
 
     def test_acquire_cancel(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
         self.assertTrue(self.loop.run_until_complete(lock.acquire()))
 
         task = asyncio.Task(lock.acquire(), loop=self.loop)
@@ -175,7 +181,8 @@ def test_cancel_race(self):
         # B's waiter; instead, it should move on to C's waiter.
 
         # Setup: A has the lock, b and c are waiting.
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
 
         async def lockit(name, blocker):
             await lock.acquire()
@@ -211,7 +218,8 @@ def test_cancel_release_race(self):
         # Issue 32734
         # Acquire 4 locks, cancel second, release first
         # and 2 locks are taken at once.
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
         lock_count = 0
         call_count = 0
 
@@ -256,7 +264,8 @@ def trigger():
         self.assertTrue(t3.cancelled())
 
     def test_finished_waiter_cancelled(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
 
         ta = asyncio.Task(lock.acquire(), loop=self.loop)
         test_utils.run_briefly(self.loop)
@@ -278,12 +287,14 @@ def test_finished_waiter_cancelled(self):
         self.assertTrue(tb.cancelled())
 
     def test_release_not_acquired(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
 
         self.assertRaises(RuntimeError, lock.release)
 
     def test_release_no_waiters(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
         self.loop.run_until_complete(lock.acquire())
         self.assertTrue(lock.locked())
 
@@ -291,9 +302,9 @@ def test_release_no_waiters(self):
         self.assertFalse(lock.locked())
 
     def test_context_manager(self):
-        lock = asyncio.Lock(loop=self.loop)
-
         with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
+
             @asyncio.coroutine
             def acquire_lock():
                 with self.assertWarns(DeprecationWarning):
@@ -305,9 +316,9 @@ def acquire_lock():
         self.assertFalse(lock.locked())
 
     def test_context_manager_cant_reuse(self):
-        lock = asyncio.Lock(loop=self.loop)
-
         with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
+
             @asyncio.coroutine
             def acquire_lock():
                 with self.assertWarns(DeprecationWarning):
@@ -325,7 +336,8 @@ def acquire_lock():
                 pass
 
     def test_context_manager_no_yield(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
 
         try:
             with lock:
@@ -346,10 +358,12 @@ def setUp(self):
 
     def test_ctor_loop(self):
         loop = mock.Mock()
-        ev = asyncio.Event(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=loop)
         self.assertIs(ev._loop, loop)
 
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
         self.assertIs(ev._loop, self.loop)
 
     def test_ctor_noloop(self):
@@ -358,7 +372,8 @@ def test_ctor_noloop(self):
         self.assertIs(ev._loop, self.loop)
 
     def test_repr(self):
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
         self.assertTrue(repr(ev).endswith('[unset]>'))
         match = RGX_REPR.match(repr(ev))
         self.assertEqual(match.group('extras'), 'unset')
@@ -372,7 +387,8 @@ def test_repr(self):
         self.assertTrue(RGX_REPR.match(repr(ev)))
 
     def test_wait(self):
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
         self.assertFalse(ev.is_set())
 
         result = []
@@ -409,14 +425,16 @@ def test_wait(self):
         self.assertIsNone(t3.result())
 
     def test_wait_on_set(self):
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
         ev.set()
 
         res = self.loop.run_until_complete(ev.wait())
         self.assertTrue(res)
 
     def test_wait_cancel(self):
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
 
         wait = asyncio.Task(ev.wait(), loop=self.loop)
         self.loop.call_soon(wait.cancel)
@@ -426,7 +444,8 @@ def test_wait_cancel(self):
         self.assertFalse(ev._waiters)
 
     def test_clear(self):
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
         self.assertFalse(ev.is_set())
 
         ev.set()
@@ -436,7 +455,8 @@ def test_clear(self):
         self.assertFalse(ev.is_set())
 
     def test_clear_with_waiters(self):
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
         result = []
 
         async def c1(result):
@@ -472,19 +492,22 @@ def setUp(self):
 
     def test_ctor_loop(self):
         loop = mock.Mock()
-        cond = asyncio.Condition(loop=loop)
-        self.assertIs(cond._loop, loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=loop)
+            self.assertIs(cond._loop, loop)
 
-        cond = asyncio.Condition(loop=self.loop)
-        self.assertIs(cond._loop, self.loop)
+            cond = asyncio.Condition(loop=self.loop)
+            self.assertIs(cond._loop, self.loop)
 
     def test_ctor_noloop(self):
-        asyncio.set_event_loop(self.loop)
-        cond = asyncio.Condition()
-        self.assertIs(cond._loop, self.loop)
+        with self.assertWarns(DeprecationWarning):
+            asyncio.set_event_loop(self.loop)
+            cond = asyncio.Condition()
+            self.assertIs(cond._loop, self.loop)
 
     def test_wait(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         result = []
 
         async def c1(result):
@@ -547,7 +570,8 @@ def test_wait(self):
         self.assertTrue(t3.result())
 
     def test_wait_cancel(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         self.loop.run_until_complete(cond.acquire())
 
         wait = asyncio.Task(cond.wait(), loop=self.loop)
@@ -559,7 +583,8 @@ def test_wait_cancel(self):
         self.assertTrue(cond.locked())
 
     def test_wait_cancel_contested(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
 
         self.loop.run_until_complete(cond.acquire())
         self.assertTrue(cond.locked())
@@ -585,7 +610,8 @@ def test_wait_cancel_contested(self):
 
     def test_wait_cancel_after_notify(self):
         # See bpo-32841
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         waited = False
 
         async def wait_on_cond():
@@ -609,13 +635,15 @@ def test_wait_cancel_after_notify(self):
         self.assertTrue(waited)
 
     def test_wait_unacquired(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         self.assertRaises(
             RuntimeError,
             self.loop.run_until_complete, cond.wait())
 
     def test_wait_for(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         presult = False
 
         def predicate():
@@ -652,7 +680,8 @@ def predicate():
         self.assertTrue(t.result())
 
     def test_wait_for_unacquired(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
 
         # predicate can return true immediately
         res = self.loop.run_until_complete(cond.wait_for(lambda: [1, 2, 3]))
@@ -664,7 +693,8 @@ def test_wait_for_unacquired(self):
             cond.wait_for(lambda: False))
 
     def test_notify(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         result = []
 
         async def c1(result):
@@ -716,7 +746,8 @@ def test_notify(self):
         self.assertTrue(t3.result())
 
     def test_notify_all(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
 
         result = []
 
@@ -752,15 +783,18 @@ def test_notify_all(self):
         self.assertTrue(t2.result())
 
     def test_notify_unacquired(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         self.assertRaises(RuntimeError, cond.notify)
 
     def test_notify_all_unacquired(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         self.assertRaises(RuntimeError, cond.notify_all)
 
     def test_repr(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         self.assertTrue('unlocked' in repr(cond))
         self.assertTrue(RGX_REPR.match(repr(cond)))
 
@@ -776,7 +810,8 @@ def test_repr(self):
         self.assertTrue(RGX_REPR.match(repr(cond)))
 
     def test_context_manager(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
 
         with self.assertWarns(DeprecationWarning):
             @asyncio.coroutine
@@ -790,7 +825,8 @@ def acquire_cond():
         self.assertFalse(cond.locked())
 
     def test_context_manager_no_yield(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
 
         try:
             with cond:
@@ -803,8 +839,9 @@ def test_context_manager_no_yield(self):
         self.assertFalse(cond.locked())
 
     def test_explicit_lock(self):
-        lock = asyncio.Lock(loop=self.loop)
-        cond = asyncio.Condition(lock, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
+            cond = asyncio.Condition(lock, loop=self.loop)
 
         self.assertIs(cond._lock, lock)
         self.assertIs(cond._loop, lock._loop)
@@ -812,10 +849,10 @@ def test_explicit_lock(self):
     def test_ambiguous_loops(self):
         loop = self.new_test_loop()
         self.addCleanup(loop.close)
-
-        lock = asyncio.Lock(loop=self.loop)
-        with self.assertRaises(ValueError):
-            asyncio.Condition(lock, loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
+            with self.assertRaises(ValueError):
+                asyncio.Condition(lock, loop=loop)
 
     def test_timeout_in_block(self):
         loop = asyncio.new_event_loop()
@@ -827,7 +864,8 @@ def test_timeout_in_block(self):
                 with self.assertRaises(asyncio.TimeoutError):
                     await asyncio.wait_for(condition.wait(), timeout=0.5)
 
-        loop.run_until_complete(task_timeout())
+        with self.assertWarns(DeprecationWarning):
+            loop.run_until_complete(task_timeout())
 
 
 class SemaphoreTests(test_utils.TestCase):
@@ -838,10 +876,12 @@ def setUp(self):
 
     def test_ctor_loop(self):
         loop = mock.Mock()
-        sem = asyncio.Semaphore(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(loop=loop)
         self.assertIs(sem._loop, loop)
 
-        sem = asyncio.Semaphore(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(loop=self.loop)
         self.assertIs(sem._loop, self.loop)
 
     def test_ctor_noloop(self):
@@ -850,11 +890,13 @@ def test_ctor_noloop(self):
         self.assertIs(sem._loop, self.loop)
 
     def test_initial_value_zero(self):
-        sem = asyncio.Semaphore(0, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(0, loop=self.loop)
         self.assertTrue(sem.locked())
 
     def test_repr(self):
-        sem = asyncio.Semaphore(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(loop=self.loop)
         self.assertTrue(repr(sem).endswith('[unlocked, value:1]>'))
         self.assertTrue(RGX_REPR.match(repr(sem)))
 
@@ -872,7 +914,8 @@ def test_repr(self):
         self.assertTrue(RGX_REPR.match(repr(sem)))
 
     def test_semaphore(self):
-        sem = asyncio.Semaphore(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(loop=self.loop)
         self.assertEqual(1, sem._value)
 
         with self.assertWarns(DeprecationWarning):
@@ -895,7 +938,8 @@ def test_semaphore_value(self):
         self.assertRaises(ValueError, asyncio.Semaphore, -1)
 
     def test_acquire(self):
-        sem = asyncio.Semaphore(3, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(3, loop=self.loop)
         result = []
 
         self.assertTrue(self.loop.run_until_complete(sem.acquire()))
@@ -956,7 +1000,8 @@ def test_acquire(self):
         self.loop.run_until_complete(asyncio.gather(*race_tasks))
 
     def test_acquire_cancel(self):
-        sem = asyncio.Semaphore(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(loop=self.loop)
         self.loop.run_until_complete(sem.acquire())
 
         acquire = asyncio.Task(sem.acquire(), loop=self.loop)
@@ -968,7 +1013,8 @@ def test_acquire_cancel(self):
                         all(waiter.done() for waiter in sem._waiters))
 
     def test_acquire_cancel_before_awoken(self):
-        sem = asyncio.Semaphore(value=0, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(value=0, loop=self.loop)
 
         t1 = asyncio.Task(sem.acquire(), loop=self.loop)
         t2 = asyncio.Task(sem.acquire(), loop=self.loop)
@@ -990,7 +1036,8 @@ def test_acquire_cancel_before_awoken(self):
         test_utils.run_briefly(self.loop)
 
     def test_acquire_hang(self):
-        sem = asyncio.Semaphore(value=0, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(value=0, loop=self.loop)
 
         t1 = asyncio.Task(sem.acquire(), loop=self.loop)
         t2 = asyncio.Task(sem.acquire(), loop=self.loop)
@@ -1004,12 +1051,14 @@ def test_acquire_hang(self):
         self.assertTrue(sem.locked())
 
     def test_release_not_acquired(self):
-        sem = asyncio.BoundedSemaphore(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.BoundedSemaphore(loop=self.loop)
 
         self.assertRaises(ValueError, sem.release)
 
     def test_release_no_waiters(self):
-        sem = asyncio.Semaphore(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(loop=self.loop)
         self.loop.run_until_complete(sem.acquire())
         self.assertTrue(sem.locked())
 
@@ -1017,9 +1066,9 @@ def test_release_no_waiters(self):
         self.assertFalse(sem.locked())
 
     def test_context_manager(self):
-        sem = asyncio.Semaphore(2, loop=self.loop)
-
         with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(2, loop=self.loop)
+
             @asyncio.coroutine
             def acquire_lock():
                 with self.assertWarns(DeprecationWarning):
@@ -1035,7 +1084,8 @@ def acquire_lock():
         self.assertEqual(2, sem._value)
 
     def test_context_manager_no_yield(self):
-        sem = asyncio.Semaphore(2, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(2, loop=self.loop)
 
         try:
             with sem:
diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py
index a5cf37ded7c9..f327bbd1d66d 100644
--- a/Lib/test/test_asyncio/test_pep492.py
+++ b/Lib/test/test_asyncio/test_pep492.py
@@ -43,12 +43,13 @@ def setUp(self):
 class LockTests(BaseTest):
 
     def test_context_manager_async_with(self):
-        primitives = [
-            asyncio.Lock(loop=self.loop),
-            asyncio.Condition(loop=self.loop),
-            asyncio.Semaphore(loop=self.loop),
-            asyncio.BoundedSemaphore(loop=self.loop),
-        ]
+        with self.assertWarns(DeprecationWarning):
+            primitives = [
+                asyncio.Lock(loop=self.loop),
+                asyncio.Condition(loop=self.loop),
+                asyncio.Semaphore(loop=self.loop),
+                asyncio.BoundedSemaphore(loop=self.loop),
+            ]
 
         async def test(lock):
             await asyncio.sleep(0.01)
@@ -65,12 +66,13 @@ def test_context_manager_async_with(self):
             self.assertFalse(primitive.locked())
 
     def test_context_manager_with_await(self):
-        primitives = [
-            asyncio.Lock(loop=self.loop),
-            asyncio.Condition(loop=self.loop),
-            asyncio.Semaphore(loop=self.loop),
-            asyncio.BoundedSemaphore(loop=self.loop),
-        ]
+        with self.assertWarns(DeprecationWarning):
+            primitives = [
+                asyncio.Lock(loop=self.loop),
+                asyncio.Condition(loop=self.loop),
+                asyncio.Semaphore(loop=self.loop),
+                asyncio.BoundedSemaphore(loop=self.loop),
+            ]
 
         async def test(lock):
             await asyncio.sleep(0.01)
diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py
index b0f0f9c8fdb7..02e8e43ccee6 100644
--- a/Lib/test/test_asyncio/test_queues.py
+++ b/Lib/test/test_asyncio/test_queues.py
@@ -35,7 +35,8 @@ def gen():
 
         loop = self.new_test_loop(gen)
 
-        q = asyncio.Queue(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=loop)
         self.assertTrue(fn(q).startswith('<Queue'), fn(q))
         id_is_present = hex(id(q)) in fn(q)
         self.assertEqual(expect_id, id_is_present)
@@ -50,7 +51,8 @@ def gen():
             # resume q.get coroutine to finish generator
             q.put_nowait(0)
 
-        loop.run_until_complete(add_getter())
+        with self.assertWarns(DeprecationWarning):
+            loop.run_until_complete(add_getter())
 
         async def add_putter():
             q = asyncio.Queue(maxsize=1, loop=loop)
@@ -63,23 +65,26 @@ def gen():
             # resume q.put coroutine to finish generator
             q.get_nowait()
 
-        loop.run_until_complete(add_putter())
-
-        q = asyncio.Queue(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            loop.run_until_complete(add_putter())
+            q = asyncio.Queue(loop=loop)
         q.put_nowait(1)
         self.assertTrue('_queue=[1]' in fn(q))
 
     def test_ctor_loop(self):
         loop = mock.Mock()
-        q = asyncio.Queue(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=loop)
         self.assertIs(q._loop, loop)
 
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         self.assertIs(q._loop, self.loop)
 
     def test_ctor_noloop(self):
         asyncio.set_event_loop(self.loop)
-        q = asyncio.Queue()
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue()
         self.assertIs(q._loop, self.loop)
 
     def test_repr(self):
@@ -89,7 +94,8 @@ def test_str(self):
         self._test_repr_or_str(str, False)
 
     def test_empty(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         self.assertTrue(q.empty())
         q.put_nowait(1)
         self.assertFalse(q.empty())
@@ -97,15 +103,18 @@ def test_empty(self):
         self.assertTrue(q.empty())
 
     def test_full(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         self.assertFalse(q.full())
 
-        q = asyncio.Queue(maxsize=1, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(maxsize=1, loop=self.loop)
         q.put_nowait(1)
         self.assertTrue(q.full())
 
     def test_order(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         for i in [1, 3, 2]:
             q.put_nowait(i)
 
@@ -123,7 +132,8 @@ def gen():
 
         loop = self.new_test_loop(gen)
 
-        q = asyncio.Queue(maxsize=2, loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(maxsize=2, loop=loop)
         self.assertEqual(2, q.maxsize)
         have_been_put = []
 
@@ -157,7 +167,8 @@ def gen():
 class QueueGetTests(_QueueTestBase):
 
     def test_blocking_get(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         q.put_nowait(1)
 
         async def queue_get():
@@ -167,7 +178,8 @@ def test_blocking_get(self):
         self.assertEqual(1, res)
 
     def test_get_with_putters(self):
-        q = asyncio.Queue(1, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(1, loop=self.loop)
         q.put_nowait(1)
 
         waiter = asyncio.Future(loop=self.loop)
@@ -187,8 +199,9 @@ def gen():
 
         loop = self.new_test_loop(gen)
 
-        q = asyncio.Queue(loop=loop)
-        started = asyncio.Event(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=loop)
+            started = asyncio.Event(loop=loop)
         finished = False
 
         async def queue_get():
@@ -212,12 +225,14 @@ def gen():
         self.assertAlmostEqual(0.01, loop.time())
 
     def test_nonblocking_get(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         q.put_nowait(1)
         self.assertEqual(1, q.get_nowait())
 
     def test_nonblocking_get_exception(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         self.assertRaises(asyncio.QueueEmpty, q.get_nowait)
 
     def test_get_cancelled(self):
@@ -231,7 +246,8 @@ def gen():
 
         loop = self.new_test_loop(gen)
 
-        q = asyncio.Queue(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=loop)
 
         async def queue_get():
             return await asyncio.wait_for(q.get(), 0.051)
@@ -246,7 +262,8 @@ def gen():
         self.assertAlmostEqual(0.06, loop.time())
 
     def test_get_cancelled_race(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
 
         t1 = asyncio.Task(q.get(), loop=self.loop)
         t2 = asyncio.Task(q.get(), loop=self.loop)
@@ -260,7 +277,8 @@ def test_get_cancelled_race(self):
         self.assertEqual(t2.result(), 'a')
 
     def test_get_with_waiting_putters(self):
-        q = asyncio.Queue(loop=self.loop, maxsize=1)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop, maxsize=1)
         asyncio.Task(q.put('a'), loop=self.loop)
         asyncio.Task(q.put('b'), loop=self.loop)
         test_utils.run_briefly(self.loop)
@@ -280,7 +298,9 @@ def test_why_are_getters_waiting(self):
 
         queue_size = 1
         producer_num_items = 5
-        q = asyncio.Queue(queue_size, loop=self.loop)
+
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(queue_size, loop=self.loop)
 
         self.loop.run_until_complete(
             asyncio.gather(producer(q, producer_num_items),
@@ -301,7 +321,8 @@ def a_generator():
             except asyncio.TimeoutError:
                 pass
 
-        queue = asyncio.Queue(loop=self.loop, maxsize=5)
+        with self.assertWarns(DeprecationWarning):
+            queue = asyncio.Queue(loop=self.loop, maxsize=5)
         self.loop.run_until_complete(self.loop.create_task(consumer(queue)))
         self.assertEqual(len(queue._getters), 0)
 
@@ -309,7 +330,8 @@ def a_generator():
 class QueuePutTests(_QueueTestBase):
 
     def test_blocking_put(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
 
         async def queue_put():
             # No maxsize, won't block.
@@ -326,8 +348,9 @@ def gen():
 
         loop = self.new_test_loop(gen)
 
-        q = asyncio.Queue(maxsize=1, loop=loop)
-        started = asyncio.Event(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(maxsize=1, loop=loop)
+            started = asyncio.Event(loop=loop)
         finished = False
 
         async def queue_put():
@@ -349,7 +372,8 @@ def gen():
         self.assertAlmostEqual(0.01, loop.time())
 
     def test_nonblocking_put(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         q.put_nowait(1)
         self.assertEqual(1, q.get_nowait())
 
@@ -360,7 +384,8 @@ def gen():
 
         loop = self.new_test_loop(gen)
 
-        q = asyncio.Queue(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=loop)
 
         reader = loop.create_task(q.get())
 
@@ -389,7 +414,8 @@ def gen():
         loop = self.new_test_loop(gen)
         loop.set_debug(True)
 
-        q = asyncio.Queue(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=loop)
 
         reader1 = loop.create_task(q.get())
         reader2 = loop.create_task(q.get())
@@ -418,7 +444,9 @@ def gen():
             yield 0.1
 
         loop = self.new_test_loop(gen)
-        q = asyncio.Queue(1, loop=loop)
+
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(1, loop=loop)
 
         q.put_nowait(1)
 
@@ -442,18 +470,21 @@ def gen():
         self.assertEqual(q.qsize(), 0)
 
     def test_nonblocking_put_exception(self):
-        q = asyncio.Queue(maxsize=1, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(maxsize=1, loop=self.loop)
         q.put_nowait(1)
         self.assertRaises(asyncio.QueueFull, q.put_nowait, 2)
 
     def test_float_maxsize(self):
-        q = asyncio.Queue(maxsize=1.3, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(maxsize=1.3, loop=self.loop)
         q.put_nowait(1)
         q.put_nowait(2)
         self.assertTrue(q.full())
         self.assertRaises(asyncio.QueueFull, q.put_nowait, 3)
 
-        q = asyncio.Queue(maxsize=1.3, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(maxsize=1.3, loop=self.loop)
 
         async def queue_put():
             await q.put(1)
@@ -462,7 +493,8 @@ def test_float_maxsize(self):
         self.loop.run_until_complete(queue_put())
 
     def test_put_cancelled(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
 
         async def queue_put():
             await q.put(1)
@@ -477,7 +509,8 @@ def test_put_cancelled(self):
         self.assertTrue(t.result())
 
     def test_put_cancelled_race(self):
-        q = asyncio.Queue(loop=self.loop, maxsize=1)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop, maxsize=1)
 
         put_a = asyncio.Task(q.put('a'), loop=self.loop)
         put_b = asyncio.Task(q.put('b'), loop=self.loop)
@@ -497,7 +530,8 @@ def test_put_cancelled_race(self):
         self.loop.run_until_complete(put_b)
 
     def test_put_with_waiting_getters(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         t = asyncio.Task(q.get(), loop=self.loop)
         test_utils.run_briefly(self.loop)
         self.loop.run_until_complete(q.put('a'))
@@ -506,7 +540,8 @@ def test_put_with_waiting_getters(self):
     def test_why_are_putters_waiting(self):
         # From issue #265.
 
-        queue = asyncio.Queue(2, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            queue = asyncio.Queue(2, loop=self.loop)
 
         async def putter(item):
             await queue.put(item)
@@ -532,7 +567,8 @@ def a_generator():
         loop = self.new_test_loop(a_generator)
 
         # Full queue.
-        queue = asyncio.Queue(loop=loop, maxsize=1)
+        with self.assertWarns(DeprecationWarning):
+            queue = asyncio.Queue(loop=loop, maxsize=1)
         queue.put_nowait(1)
 
         # Task waiting for space to put an item in the queue.
@@ -555,7 +591,8 @@ def gen():
         loop = self.new_test_loop(gen)
 
         # Full Queue.
-        queue = asyncio.Queue(1, loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            queue = asyncio.Queue(1, loop=loop)
         queue.put_nowait(1)
 
         # Task waiting for space to put a item in the queue.
@@ -578,7 +615,8 @@ def gen():
 class LifoQueueTests(_QueueTestBase):
 
     def test_order(self):
-        q = asyncio.LifoQueue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.LifoQueue(loop=self.loop)
         for i in [1, 3, 2]:
             q.put_nowait(i)
 
@@ -589,7 +627,8 @@ def test_order(self):
 class PriorityQueueTests(_QueueTestBase):
 
     def test_order(self):
-        q = asyncio.PriorityQueue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.PriorityQueue(loop=self.loop)
         for i in [1, 3, 2]:
             q.put_nowait(i)
 
@@ -602,11 +641,13 @@ class _QueueJoinTestMixin:
     q_class = None
 
     def test_task_done_underflow(self):
-        q = self.q_class(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = self.q_class(loop=self.loop)
         self.assertRaises(ValueError, q.task_done)
 
     def test_task_done(self):
-        q = self.q_class(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = self.q_class(loop=self.loop)
         for i in range(100):
             q.put_nowait(i)
 
@@ -641,7 +682,8 @@ def test_task_done(self):
         self.loop.run_until_complete(asyncio.wait(tasks))
 
     def test_join_empty_queue(self):
-        q = self.q_class(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = self.q_class(loop=self.loop)
 
         # Test that a queue join()s successfully, and before anything else
         # (done twice for insurance).
@@ -653,7 +695,8 @@ def test_join_empty_queue(self):
         self.loop.run_until_complete(join())
 
     def test_format(self):
-        q = self.q_class(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = self.q_class(loop=self.loop)
         self.assertEqual(q._format(), 'maxsize=0')
 
         q._unfinished_tasks = 2
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
index d5f44b8091de..323c0907a23a 100644
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -1231,15 +1231,16 @@ def sleeper(dt, x):
             for f in asyncio.as_completed([b, c, a], loop=loop):
                 values.append(await f)
             return values
-
-        res = loop.run_until_complete(self.new_task(loop, foo()))
+        with self.assertWarns(DeprecationWarning):
+            res = loop.run_until_complete(self.new_task(loop, foo()))
         self.assertAlmostEqual(0.15, loop.time())
         self.assertTrue('a' in res[:2])
         self.assertTrue('b' in res[:2])
         self.assertEqual(res[2], 'c')
 
         # Doing it again should take no time and exercise a different path.
-        res = loop.run_until_complete(self.new_task(loop, foo()))
+        with self.assertWarns(DeprecationWarning):
+            res = loop.run_until_complete(self.new_task(loop, foo()))
         self.assertAlmostEqual(0.15, loop.time())
 
     def test_as_completed_with_timeout(self):
@@ -1267,7 +1268,8 @@ def gen():
                     values.append((2, exc))
             return values
 
-        res = loop.run_until_complete(self.new_task(loop, foo()))
+        with self.assertWarns(DeprecationWarning):
+            res = loop.run_until_complete(self.new_task(loop, foo()))
         self.assertEqual(len(res), 2, res)
         self.assertEqual(res[0], (1, 'a'))
         self.assertEqual(res[1][0], 2)
@@ -1294,7 +1296,8 @@ def gen():
                 v = await f
                 self.assertEqual(v, 'a')
 
-        loop.run_until_complete(self.new_task(loop, foo()))
+        with self.assertWarns(DeprecationWarning):
+            loop.run_until_complete(self.new_task(loop, foo()))
 
     def test_as_completed_reverse_wait(self):
 
@@ -1308,7 +1311,9 @@ def gen():
         a = asyncio.sleep(0.05, 'a')
         b = asyncio.sleep(0.10, 'b')
         fs = {a, b}
-        futs = list(asyncio.as_completed(fs, loop=loop))
+
+        with self.assertWarns(DeprecationWarning):
+            futs = list(asyncio.as_completed(fs, loop=loop))
         self.assertEqual(len(futs), 2)
 
         x = loop.run_until_complete(futs[1])
@@ -1333,7 +1338,8 @@ def gen():
         a = asyncio.sleep(0.05, 'a')
         b = asyncio.sleep(0.05, 'b')
         fs = {a, b}
-        futs = list(asyncio.as_completed(fs, loop=loop))
+        with self.assertWarns(DeprecationWarning):
+            futs = list(asyncio.as_completed(fs, loop=loop))
         self.assertEqual(len(futs), 2)
         waiter = asyncio.wait(futs)
         done, pending = loop.run_until_complete(waiter)
@@ -1356,8 +1362,9 @@ def runner():
                     result.append((yield from f))
                 return result
 
-        fut = self.new_task(self.loop, runner())
-        self.loop.run_until_complete(fut)
+        with self.assertWarns(DeprecationWarning):
+            fut = self.new_task(self.loop, runner())
+            self.loop.run_until_complete(fut)
         result = fut.result()
         self.assertEqual(set(result), {'ham', 'spam'})
         self.assertEqual(len(result), 2)


From webhook-mailer at python.org  Tue Sep 10 07:01:24 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 11:01:24 -0000
Subject: [Python-checkins] bpo-38069: Convert _posixsubprocess to PEP-384
 (GH-15780)
Message-ID: <mailman.420.1568113284.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5a7d2e11aaea2dd32878dc5c6b1aae8caf56cb44
commit: 5a7d2e11aaea2dd32878dc5c6b1aae8caf56cb44
branch: master
author: Dino Viehland <dinoviehland at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-10T04:01:20-07:00
summary:

bpo-38069: Convert _posixsubprocess to PEP-384 (GH-15780)



Summary:
Eliminate uses of `_Py_IDENTIFIER` from `_posixsubprocess`, replacing them with interned strings.

Also tries to find an existing version of the module, which will allow subinterpreters.



https://bugs.python.org/issue38069

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-09-14-46-05.bpo-38069.cn8XLv.rst
M Modules/_posixsubprocess.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-09-14-46-05.bpo-38069.cn8XLv.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-14-46-05.bpo-38069.cn8XLv.rst
new file mode 100644
index 000000000000..6bfe7a744f6e
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-14-46-05.bpo-38069.cn8XLv.rst	
@@ -0,0 +1 @@
+Make _posixsubprocess PEP-384 compatible
\ No newline at end of file
diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c
index f68d362acc04..cbdeecfda1c2 100644
--- a/Modules/_posixsubprocess.c
+++ b/Modules/_posixsubprocess.c
@@ -49,18 +49,28 @@
 
 #define POSIX_CALL(call)   do { if ((call) == -1) goto error; } while (0)
 
+typedef struct {
+    PyObject* disable;
+    PyObject* enable;
+    PyObject* isenabled;
+} _posixsubprocessstate;
+
+static struct PyModuleDef _posixsubprocessmodule;
+
+#define _posixsubprocessstate(o) ((_posixsubprocessstate *)PyModule_GetState(o))
+#define _posixsubprocessstate_global _posixsubprocessstate(PyState_FindModule(&_posixsubprocessmodule))
 
 /* If gc was disabled, call gc.enable().  Return 0 on success. */
 static int
 _enable_gc(int need_to_reenable_gc, PyObject *gc_module)
 {
     PyObject *result;
-    _Py_IDENTIFIER(enable);
     PyObject *exctype, *val, *tb;
 
     if (need_to_reenable_gc) {
         PyErr_Fetch(&exctype, &val, &tb);
-        result = _PyObject_CallMethodIdNoArgs(gc_module, &PyId_enable);
+        result = _PyObject_CallMethodNoArgs(
+            gc_module, _posixsubprocessstate_global->enable);
         if (exctype != NULL) {
             PyErr_Restore(exctype, val, tb);
         }
@@ -602,13 +612,12 @@ subprocess_fork_exec(PyObject* self, PyObject *args)
     /* We need to call gc.disable() when we'll be calling preexec_fn */
     if (preexec_fn != Py_None) {
         PyObject *result;
-        _Py_IDENTIFIER(isenabled);
-        _Py_IDENTIFIER(disable);
 
         gc_module = PyImport_ImportModule("gc");
         if (gc_module == NULL)
             return NULL;
-        result = _PyObject_CallMethodIdNoArgs(gc_module, &PyId_isenabled);
+        result = _PyObject_CallMethodNoArgs(
+            gc_module, _posixsubprocessstate_global->isenabled);
         if (result == NULL) {
             Py_DECREF(gc_module);
             return NULL;
@@ -619,7 +628,8 @@ subprocess_fork_exec(PyObject* self, PyObject *args)
             Py_DECREF(gc_module);
             return NULL;
         }
-        result = _PyObject_CallMethodIdNoArgs(gc_module, &PyId_disable);
+        result = _PyObject_CallMethodNoArgs(
+            gc_module, _posixsubprocessstate_global->disable);
         if (result == NULL) {
             Py_DECREF(gc_module);
             return NULL;
@@ -798,16 +808,56 @@ static PyMethodDef module_methods[] = {
 };
 
 
+static int _posixsubprocess_traverse(PyObject *m, visitproc visit, void *arg) {
+    Py_VISIT(_posixsubprocessstate(m)->disable);
+    Py_VISIT(_posixsubprocessstate(m)->enable);
+    Py_VISIT(_posixsubprocessstate(m)->isenabled);
+    return 0;
+}
+
+static int _posixsubprocess_clear(PyObject *m) {
+    Py_CLEAR(_posixsubprocessstate(m)->disable);
+    Py_CLEAR(_posixsubprocessstate(m)->enable);
+    Py_CLEAR(_posixsubprocessstate(m)->isenabled);
+    return 0;
+}
+
+static void _posixsubprocess_free(void *m) {
+    _posixsubprocess_clear((PyObject *)m);
+}
+
 static struct PyModuleDef _posixsubprocessmodule = {
         PyModuleDef_HEAD_INIT,
         "_posixsubprocess",
         module_doc,
-        -1,  /* No memory is needed. */
+        sizeof(_posixsubprocessstate),
         module_methods,
+        NULL,
+        _posixsubprocess_traverse,
+        _posixsubprocess_clear,
+        _posixsubprocess_free,
 };
 
 PyMODINIT_FUNC
 PyInit__posixsubprocess(void)
 {
-    return PyModule_Create(&_posixsubprocessmodule);
+    PyObject* m;
+
+    m = PyState_FindModule(&_posixsubprocessmodule);
+    if (m != NULL) {
+      Py_INCREF(m);
+      return m;
+    }
+
+    m = PyModule_Create(&_posixsubprocessmodule);
+    if (m == NULL) {
+      return NULL;
+    }
+
+    _posixsubprocessstate(m)->disable = PyUnicode_InternFromString("disable");
+    _posixsubprocessstate(m)->enable = PyUnicode_InternFromString("enable");
+    _posixsubprocessstate(m)->isenabled = PyUnicode_InternFromString("isenabled");
+
+    PyState_AddModule(m, &_posixsubprocessmodule);
+    return m;
 }


From webhook-mailer at python.org  Tue Sep 10 07:08:59 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 11:08:59 -0000
Subject: [Python-checkins] bpo-37052: Add examples for mocking async
 iterators and context managers (GH-14660)
Message-ID: <mailman.421.1568113741.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ab74e52f768be5048faf2a11e78822533afebcb7
commit: ab74e52f768be5048faf2a11e78822533afebcb7
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T04:08:52-07:00
summary:

bpo-37052: Add examples for mocking async iterators and context managers (GH-14660)


Add examples for mocking asynchronous iterators and asynchronous context managers.

https://bugs.python.org/issue37052
(cherry picked from commit c8dfa7333d6317d7cd8c5c7366023f5a668e3f91)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
M Doc/library/unittest.mock-examples.rst

diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst
index 811f0fb1ce93..cf6b671b5bee 100644
--- a/Doc/library/unittest.mock-examples.rst
+++ b/Doc/library/unittest.mock-examples.rst
@@ -12,6 +12,7 @@
 
 .. testsetup::
 
+    import asyncio
     import unittest
     from unittest.mock import Mock, MagicMock, patch, call, sentinel
 
@@ -276,6 +277,44 @@ function returns is what the call returns:
     2
 
 
+Mocking asynchronous iterators
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since Python 3.8, ``MagicMock`` has support to mock :ref:`async-iterators`
+through ``__aiter__``. The :attr:`~Mock.return_value` attribute of ``__aiter__``
+can be used to set the return values to be used for iteration.
+
+    >>> mock = MagicMock()
+    >>> mock.__aiter__.return_value = [1, 2, 3]
+    >>> async def main():
+    ...     return [i async for i in mock]
+    >>> asyncio.run(main())
+    [1, 2, 3]
+
+
+Mocking asynchronous context manager
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since Python 3.8, ``MagicMock`` has support to mock
+:ref:`async-context-managers` through ``__aenter__`` and ``__aexit__``. The
+return value of ``__aenter__`` is an :class:`AsyncMock`.
+
+    >>> class AsyncContextManager:
+    ...
+    ...     async def __aenter__(self):
+    ...         return self
+    ...
+    ...     async def __aexit__(self):
+    ...         pass
+    >>> mock_instance = MagicMock(AsyncContextManager())
+    >>> async def main():
+    ...     async with mock_instance as result:
+    ...         pass
+    >>> asyncio.run(main())
+    >>> mock_instance.__aenter__.assert_called_once()
+    >>> mock_instance.__aexit__.assert_called_once()
+
+
 Creating a Mock from an Existing Object
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 


From webhook-mailer at python.org  Tue Sep 10 07:18:48 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Tue, 10 Sep 2019 11:18:48 -0000
Subject: [Python-checkins] bpo-37251: Removes __code__ check from
 _is_async_obj. (GH-15830)
Message-ID: <mailman.422.1568114329.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f1a297acb60b88917712450ebd3cfa707e6efd6b
commit: f1a297acb60b88917712450ebd3cfa707e6efd6b
branch: master
author: Lisa Roach <lisaroach14 at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T12:18:40+01:00
summary:

bpo-37251: Removes __code__ check from _is_async_obj. (GH-15830)

files:
A Misc/NEWS.d/next/Library/2019-09-10-10-59-50.bpo-37251.8zn2o3.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index c26b367a4dc2..50e495965519 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -46,10 +46,9 @@
 _safe_super = super
 
 def _is_async_obj(obj):
-    if getattr(obj, '__code__', None):
-        return asyncio.iscoroutinefunction(obj) or inspect.isawaitable(obj)
-    else:
+    if _is_instance_mock(obj) and not isinstance(obj, AsyncMock):
         return False
+    return asyncio.iscoroutinefunction(obj) or inspect.isawaitable(obj)
 
 
 def _is_async_func(func):
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index 865cfdc00264..550234223cc3 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -18,6 +18,10 @@ def __init__(self):
     def normal_method(self):
         pass
 
+class AwaitableClass:
+    def __await__(self):
+        yield
+
 async def async_func():
     pass
 
@@ -160,6 +164,10 @@ def test_create_autospec_instance(self):
         with self.assertRaises(RuntimeError):
             create_autospec(async_func, instance=True)
 
+    def test_create_autospec_awaitable_class(self):
+        awaitable_mock = create_autospec(spec=AwaitableClass())
+        self.assertIsInstance(create_autospec(awaitable_mock), AsyncMock)
+
     def test_create_autospec(self):
         spec = create_autospec(async_func_args)
         awaitable = spec(1, 2, c=3)
@@ -321,6 +329,13 @@ def test_is_child_AsyncMock(self):
         self.assertIsInstance(mock.normal_method, MagicMock)
         self.assertIsInstance(mock, MagicMock)
 
+    def test_magicmock_lambda_spec(self):
+        mock_obj = MagicMock()
+        mock_obj.mock_func = MagicMock(spec=lambda x: x)
+
+        with patch.object(mock_obj, "mock_func") as cm:
+            self.assertIsInstance(cm, MagicMock)
+
 
 class AsyncArguments(unittest.TestCase):
     def test_add_return_value(self):
diff --git a/Misc/NEWS.d/next/Library/2019-09-10-10-59-50.bpo-37251.8zn2o3.rst b/Misc/NEWS.d/next/Library/2019-09-10-10-59-50.bpo-37251.8zn2o3.rst
new file mode 100644
index 000000000000..27fd1e46963c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-10-10-59-50.bpo-37251.8zn2o3.rst
@@ -0,0 +1,3 @@
+Remove `__code__` check in AsyncMock that incorrectly
+evaluated function specs as async objects but failed to evaluate classes
+with `__await__` but no `__code__` attribute defined as async objects.


From webhook-mailer at python.org  Tue Sep 10 07:21:13 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 11:21:13 -0000
Subject: [Python-checkins] bpo-37499: Test various C calling conventions
 (GH-15776)
Message-ID: <mailman.423.1568114474.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f958377b67c36a98d4df67b94c01eb29e3104f61
commit: f958377b67c36a98d4df67b94c01eb29e3104f61
branch: master
author: Petr Viktorin <encukou at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-10T04:21:09-07:00
summary:

bpo-37499: Test various C calling conventions (GH-15776)



Add functions with various calling conventions to `_testcapi`, expose them as module-level functions, bound methods, class methods, and static methods, and test calling them and introspecting them through GDB.


https://bugs.python.org/issue37499


Co-authored-by: Jeroen Demeyer <J.Demeyer at UGent.be>
Automerge-Triggered-By: @pganssle

files:
M Lib/test/test_call.py
M Lib/test/test_gdb.py
M Modules/_testcapimodule.c

diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py
index 0bff7ded4670..c233ba1351fe 100644
--- a/Lib/test/test_call.py
+++ b/Lib/test/test_call.py
@@ -27,125 +27,6 @@ def fn(**kw):
         self.assertEqual(list(res.items()), expected)
 
 
-# The test cases here cover several paths through the function calling
-# code.  They depend on the METH_XXX flag that is used to define a C
-# function, which can't be verified from Python.  If the METH_XXX decl
-# for a C function changes, these tests may not cover the right paths.
-
-class CFunctionCalls(unittest.TestCase):
-
-    def test_varargs0(self):
-        self.assertRaises(TypeError, {}.__contains__)
-
-    def test_varargs1(self):
-        {}.__contains__(0)
-
-    def test_varargs2(self):
-        self.assertRaises(TypeError, {}.__contains__, 0, 1)
-
-    def test_varargs0_ext(self):
-        try:
-            {}.__contains__(*())
-        except TypeError:
-            pass
-
-    def test_varargs1_ext(self):
-        {}.__contains__(*(0,))
-
-    def test_varargs2_ext(self):
-        try:
-            {}.__contains__(*(1, 2))
-        except TypeError:
-            pass
-        else:
-            raise RuntimeError
-
-    def test_varargs1_kw(self):
-        self.assertRaises(TypeError, {}.__contains__, x=2)
-
-    def test_varargs2_kw(self):
-        self.assertRaises(TypeError, {}.__contains__, x=2, y=2)
-
-    def test_oldargs0_0(self):
-        {}.keys()
-
-    def test_oldargs0_1(self):
-        self.assertRaises(TypeError, {}.keys, 0)
-
-    def test_oldargs0_2(self):
-        self.assertRaises(TypeError, {}.keys, 0, 1)
-
-    def test_oldargs0_0_ext(self):
-        {}.keys(*())
-
-    def test_oldargs0_1_ext(self):
-        try:
-            {}.keys(*(0,))
-        except TypeError:
-            pass
-        else:
-            raise RuntimeError
-
-    def test_oldargs0_2_ext(self):
-        try:
-            {}.keys(*(1, 2))
-        except TypeError:
-            pass
-        else:
-            raise RuntimeError
-
-    def test_oldargs0_0_kw(self):
-        try:
-            {}.keys(x=2)
-        except TypeError:
-            pass
-        else:
-            raise RuntimeError
-
-    def test_oldargs0_1_kw(self):
-        self.assertRaises(TypeError, {}.keys, x=2)
-
-    def test_oldargs0_2_kw(self):
-        self.assertRaises(TypeError, {}.keys, x=2, y=2)
-
-    def test_oldargs1_0(self):
-        self.assertRaises(TypeError, [].count)
-
-    def test_oldargs1_1(self):
-        [].count(1)
-
-    def test_oldargs1_2(self):
-        self.assertRaises(TypeError, [].count, 1, 2)
-
-    def test_oldargs1_0_ext(self):
-        try:
-            [].count(*())
-        except TypeError:
-            pass
-        else:
-            raise RuntimeError
-
-    def test_oldargs1_1_ext(self):
-        [].count(*(1,))
-
-    def test_oldargs1_2_ext(self):
-        try:
-            [].count(*(1, 2))
-        except TypeError:
-            pass
-        else:
-            raise RuntimeError
-
-    def test_oldargs1_0_kw(self):
-        self.assertRaises(TypeError, [].count, x=2)
-
-    def test_oldargs1_1_kw(self):
-        self.assertRaises(TypeError, [].count, {}, x=2)
-
-    def test_oldargs1_2_kw(self):
-        self.assertRaises(TypeError, [].count, x=2, y=2)
-
-
 @cpython_only
 class CFunctionCallsErrorMessages(unittest.TestCase):
 
@@ -289,6 +170,176 @@ def test_oldargs1_2_kw(self):
         self.assertRaisesRegex(TypeError, msg, [].count, x=2, y=2)
 
 
+
+class TestCallingConventions(unittest.TestCase):
+    """Test calling using various C calling conventions (METH_*) from Python
+
+    Subclasses test several kinds of functions (module-level, methods,
+    class methods static methods) using these attributes:
+      obj: the object that contains tested functions (as attributes)
+      expected_self: expected "self" argument to the C function
+
+    The base class tests module-level functions.
+    """
+
+    def setUp(self):
+        self.obj = self.expected_self = _testcapi
+
+    def test_varargs(self):
+        self.assertEqual(
+            self.obj.meth_varargs(1, 2, 3),
+            (self.expected_self, (1, 2, 3)),
+        )
+
+    def test_varargs_ext(self):
+        self.assertEqual(
+            self.obj.meth_varargs(*(1, 2, 3)),
+            (self.expected_self, (1, 2, 3)),
+        )
+
+    def test_varargs_error_kw(self):
+        msg = r"meth_varargs\(\) takes no keyword arguments"
+        self.assertRaisesRegex(
+            TypeError, msg, lambda: self.obj.meth_varargs(k=1),
+        )
+
+    def test_varargs_keywords(self):
+        self.assertEqual(
+            self.obj.meth_varargs_keywords(1, 2, a=3, b=4),
+            (self.expected_self, (1, 2), {'a': 3, 'b': 4})
+        )
+
+    def test_varargs_keywords_ext(self):
+        self.assertEqual(
+            self.obj.meth_varargs_keywords(*[1, 2], **{'a': 3, 'b': 4}),
+            (self.expected_self, (1, 2), {'a': 3, 'b': 4})
+        )
+
+    def test_o(self):
+        self.assertEqual(self.obj.meth_o(1), (self.expected_self, 1))
+
+    def test_o_ext(self):
+        self.assertEqual(self.obj.meth_o(*[1]), (self.expected_self, 1))
+
+    def test_o_error_no_arg(self):
+        msg = r"meth_o\(\) takes exactly one argument \(0 given\)"
+        self.assertRaisesRegex(TypeError, msg, self.obj.meth_o)
+
+    def test_o_error_two_args(self):
+        msg = r"meth_o\(\) takes exactly one argument \(2 given\)"
+        self.assertRaisesRegex(
+            TypeError, msg, lambda: self.obj.meth_o(1, 2),
+        )
+
+    def test_o_error_ext(self):
+        msg = r"meth_o\(\) takes exactly one argument \(3 given\)"
+        self.assertRaisesRegex(
+            TypeError, msg, lambda: self.obj.meth_o(*(1, 2, 3)),
+        )
+
+    def test_o_error_kw(self):
+        msg = r"meth_o\(\) takes no keyword arguments"
+        self.assertRaisesRegex(
+            TypeError, msg, lambda: self.obj.meth_o(k=1),
+        )
+
+    def test_o_error_arg_kw(self):
+        msg = r"meth_o\(\) takes no keyword arguments"
+        self.assertRaisesRegex(
+            TypeError, msg, lambda: self.obj.meth_o(k=1),
+        )
+
+    def test_noargs(self):
+        self.assertEqual(self.obj.meth_noargs(), self.expected_self)
+
+    def test_noargs_ext(self):
+        self.assertEqual(self.obj.meth_noargs(*[]), self.expected_self)
+
+    def test_noargs_error_arg(self):
+        msg = r"meth_noargs\(\) takes no arguments \(1 given\)"
+        self.assertRaisesRegex(
+            TypeError, msg, lambda: self.obj.meth_noargs(1),
+        )
+
+    def test_noargs_error_arg2(self):
+        msg = r"meth_noargs\(\) takes no arguments \(2 given\)"
+        self.assertRaisesRegex(
+            TypeError, msg, lambda: self.obj.meth_noargs(1, 2),
+        )
+
+    def test_noargs_error_ext(self):
+        msg = r"meth_noargs\(\) takes no arguments \(3 given\)"
+        self.assertRaisesRegex(
+            TypeError, msg, lambda: self.obj.meth_noargs(*(1, 2, 3)),
+        )
+
+    def test_noargs_error_kw(self):
+        msg = r"meth_noargs\(\) takes no keyword arguments"
+        self.assertRaisesRegex(
+            TypeError, msg, lambda: self.obj.meth_noargs(k=1),
+        )
+
+    def test_fastcall(self):
+        self.assertEqual(
+            self.obj.meth_fastcall(1, 2, 3),
+            (self.expected_self, (1, 2, 3)),
+        )
+
+    def test_fastcall_ext(self):
+        self.assertEqual(
+            self.obj.meth_fastcall(*(1, 2, 3)),
+            (self.expected_self, (1, 2, 3)),
+        )
+
+    def test_fastcall_error_kw(self):
+        msg = r"meth_fastcall\(\) takes no keyword arguments"
+        self.assertRaisesRegex(
+            TypeError, msg, lambda: self.obj.meth_fastcall(k=1),
+        )
+
+    def test_fastcall_keywords(self):
+        self.assertEqual(
+            self.obj.meth_fastcall_keywords(1, 2, a=3, b=4),
+            (self.expected_self, (1, 2), {'a': 3, 'b': 4})
+        )
+
+    def test_fastcall_keywords_ext(self):
+        self.assertEqual(
+            self.obj.meth_fastcall_keywords(*(1, 2), **{'a': 3, 'b': 4}),
+            (self.expected_self, (1, 2), {'a': 3, 'b': 4})
+        )
+
+
+class TestCallingConventionsInstance(TestCallingConventions):
+    """Test calling instance methods using various calling conventions"""
+
+    def setUp(self):
+        self.obj = self.expected_self = _testcapi.MethInstance()
+
+
+class TestCallingConventionsClass(TestCallingConventions):
+    """Test calling class methods using various calling conventions"""
+
+    def setUp(self):
+        self.obj = self.expected_self = _testcapi.MethClass
+
+
+class TestCallingConventionsClassInstance(TestCallingConventions):
+    """Test calling class methods on instance"""
+
+    def setUp(self):
+        self.obj = _testcapi.MethClass()
+        self.expected_self = _testcapi.MethClass
+
+
+class TestCallingConventionsStatic(TestCallingConventions):
+    """Test calling static methods using various calling conventions"""
+
+    def setUp(self):
+        self.obj = _testcapi.MethStatic()
+        self.expected_self = None
+
+
 def pyfunc(arg1, arg2):
     return [arg1, arg2]
 
@@ -315,14 +366,14 @@ def static_method():
 
 PYTHON_INSTANCE = PythonClass()
 
+NULL_OR_EMPTY = object()
 
-IGNORE_RESULT = object()
-
-
- at cpython_only
 class FastCallTests(unittest.TestCase):
+    """Test calling using various callables from C
+    """
+
     # Test calls with positional arguments
-    CALLS_POSARGS = (
+    CALLS_POSARGS = [
         # (func, args: tuple, result)
 
         # Python function with 2 arguments
@@ -341,31 +392,11 @@ class FastCallTests(unittest.TestCase):
         (PYTHON_INSTANCE.class_method, (), "classmethod"),
         (PYTHON_INSTANCE.static_method, (), "staticmethod"),
 
-        # C function: METH_NOARGS
-        (globals, (), IGNORE_RESULT),
-
-        # C function: METH_O
-        (id, ("hello",), IGNORE_RESULT),
-
-        # C function: METH_VARARGS
-        (dir, (1,), IGNORE_RESULT),
-
-        # C function: METH_VARARGS | METH_KEYWORDS
-        (min, (5, 9), 5),
-
-        # C function: METH_FASTCALL
-        (divmod, (1000, 33), (30, 10)),
-
-        # C type static method: METH_FASTCALL | METH_CLASS
-        (int.from_bytes, (b'\x01\x00', 'little'), 1),
-
-        # bpo-30524: Test that calling a C type static method with no argument
-        # doesn't crash (ignore the result): METH_FASTCALL | METH_CLASS
-        (datetime.datetime.now, (), IGNORE_RESULT),
-    )
+        # C callables are added later
+    ]
 
     # Test calls with positional and keyword arguments
-    CALLS_KWARGS = (
+    CALLS_KWARGS = [
         # (func, args: tuple, kwargs: dict, result)
 
         # Python function with 2 arguments
@@ -376,17 +407,51 @@ class FastCallTests(unittest.TestCase):
         (PYTHON_INSTANCE.method, (1,), {'arg2': 2}, [1, 2]),
         (PYTHON_INSTANCE.method, (), {'arg1': 1, 'arg2': 2}, [1, 2]),
 
-        # C function: METH_VARARGS | METH_KEYWORDS
-        (max, ([],), {'default': 9}, 9),
-
-        # C type static method: METH_FASTCALL | METH_CLASS
-        (int.from_bytes, (b'\x01\x00',), {'byteorder': 'little'}, 1),
-        (int.from_bytes, (), {'bytes': b'\x01\x00', 'byteorder': 'little'}, 1),
-    )
+        # C callables are added later
+    ]
+
+    # Add all the calling conventions and variants of C callables
+    _instance = _testcapi.MethInstance()
+    for obj, expected_self in (
+        (_testcapi, _testcapi),  # module-level function
+        (_instance, _instance),  # bound method
+        (_testcapi.MethClass, _testcapi.MethClass),  # class method on class
+        (_testcapi.MethClass(), _testcapi.MethClass),  # class method on inst.
+        (_testcapi.MethStatic, None),  # static method
+    ):
+        CALLS_POSARGS.extend([
+            (obj.meth_varargs, (1, 2), (expected_self, (1, 2))),
+            (obj.meth_varargs_keywords,
+                (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)),
+            (obj.meth_fastcall, (1, 2), (expected_self, (1, 2))),
+            (obj.meth_fastcall, (), (expected_self, ())),
+            (obj.meth_fastcall_keywords,
+                (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)),
+            (obj.meth_fastcall_keywords,
+                (), (expected_self, (), NULL_OR_EMPTY)),
+            (obj.meth_noargs, (), expected_self),
+            (obj.meth_o, (123, ), (expected_self, 123)),
+        ])
+
+        CALLS_KWARGS.extend([
+            (obj.meth_varargs_keywords,
+                (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})),
+            (obj.meth_varargs_keywords,
+                (), {'x': 'y'}, (expected_self, (), {'x': 'y'})),
+            (obj.meth_varargs_keywords,
+                (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)),
+            (obj.meth_fastcall_keywords,
+                (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})),
+            (obj.meth_fastcall_keywords,
+                (), {'x': 'y'}, (expected_self, (), {'x': 'y'})),
+            (obj.meth_fastcall_keywords,
+                (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)),
+        ])
 
     def check_result(self, result, expected):
-        if expected is IGNORE_RESULT:
-            return
+        if isinstance(expected, tuple) and expected[-1] is NULL_OR_EMPTY:
+            if result[-1] in ({}, None):
+                expected = (*expected[:-1], result[-1])
         self.assertEqual(result, expected)
 
     def test_fastcall(self):
@@ -411,19 +476,11 @@ def test_vectorcall_dict(self):
                 result = _testcapi.pyobject_fastcalldict(func, args, None)
                 self.check_result(result, expected)
 
-                # kwargs={}
-                result = _testcapi.pyobject_fastcalldict(func, args, {})
-                self.check_result(result, expected)
-
                 if not args:
                     # args=NULL, nargs=0, kwargs=NULL
                     result = _testcapi.pyobject_fastcalldict(func, None, None)
                     self.check_result(result, expected)
 
-                    # args=NULL, nargs=0, kwargs={}
-                    result = _testcapi.pyobject_fastcalldict(func, None, {})
-                    self.check_result(result, expected)
-
         for func, args, kwargs, expected in self.CALLS_KWARGS:
             with self.subTest(func=func, args=args, kwargs=kwargs):
                 result = _testcapi.pyobject_fastcalldict(func, args, kwargs)
diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py
index a2aa16c62b7d..e515d0d3fb51 100644
--- a/Lib/test/test_gdb.py
+++ b/Lib/test/test_gdb.py
@@ -838,47 +838,66 @@ def test_gc(self):
                                           )
         self.assertIn('Garbage-collecting', gdb_output)
 
+
     @unittest.skipIf(python_is_optimized(),
                      "Python was compiled with optimizations")
     # Some older versions of gdb will fail with
     #  "Cannot find new threads: generic error"
     # unless we add LD_PRELOAD=PATH-TO-libpthread.so.1 as a workaround
+    #
+    # gdb will also generate many erroneous errors such as:
+    #     Function "meth_varargs" not defined.
+    # This is because we are calling functions from an "external" module
+    # (_testcapimodule) rather than compiled-in functions. It seems difficult
+    # to suppress these. See also the comment in DebuggerTests.get_stack_trace
     def test_pycfunction(self):
         'Verify that "py-bt" displays invocations of PyCFunction instances'
         # Various optimizations multiply the code paths by which these are
         # called, so test a variety of calling conventions.
-        for py_name, py_args, c_name, expected_frame_number in (
-            ('gmtime', '', 'time_gmtime', 1),  # METH_VARARGS
-            ('len', '[]', 'builtin_len', 1),  # METH_O
-            ('locals', '', 'builtin_locals', 1),  # METH_NOARGS
-            ('iter', '[]', 'builtin_iter', 1),  # METH_FASTCALL
-            ('sorted', '[]', 'builtin_sorted', 1),  # METH_FASTCALL|METH_KEYWORDS
+        for func_name, args, expected_frame in (
+            ('meth_varargs', '', 1),
+            ('meth_varargs_keywords', '', 1),
+            ('meth_o', '[]', 1),
+            ('meth_noargs', '', 1),
+            ('meth_fastcall', '', 1),
+            ('meth_fastcall_keywords', '', 1),
         ):
-            with self.subTest(c_name):
-                cmd = ('from time import gmtime\n'  # (not always needed)
-                    'def foo():\n'
-                    f'    {py_name}({py_args})\n'
-                    'def bar():\n'
-                    '    foo()\n'
-                    'bar()\n')
-                # Verify with "py-bt":
-                gdb_output = self.get_stack_trace(
-                    cmd,
-                    breakpoint=c_name,
-                    cmds_after_breakpoint=['bt', 'py-bt'],
-                )
-                self.assertIn(f'<built-in method {py_name}', gdb_output)
-
-                # Verify with "py-bt-full":
-                gdb_output = self.get_stack_trace(
-                    cmd,
-                    breakpoint=c_name,
-                    cmds_after_breakpoint=['py-bt-full'],
-                )
-                self.assertIn(
-                    f'#{expected_frame_number} <built-in method {py_name}',
-                    gdb_output,
-                )
+            for obj in (
+                '_testcapi',
+                '_testcapi.MethClass',
+                '_testcapi.MethClass()',
+                '_testcapi.MethStatic()',
+
+                # XXX: bound methods don't yet give nice tracebacks
+                # '_testcapi.MethInstance()',
+            ):
+                with self.subTest(f'{obj}.{func_name}'):
+                    cmd = textwrap.dedent(f'''
+                        import _testcapi
+                        def foo():
+                            {obj}.{func_name}({args})
+                        def bar():
+                            foo()
+                        bar()
+                    ''')
+                    # Verify with "py-bt":
+                    gdb_output = self.get_stack_trace(
+                        cmd,
+                        breakpoint=func_name,
+                        cmds_after_breakpoint=['bt', 'py-bt'],
+                    )
+                    self.assertIn(f'<built-in method {func_name}', gdb_output)
+
+                    # Verify with "py-bt-full":
+                    gdb_output = self.get_stack_trace(
+                        cmd,
+                        breakpoint=func_name,
+                        cmds_after_breakpoint=['py-bt-full'],
+                    )
+                    self.assertIn(
+                        f'#{expected_frame} <built-in method {func_name}',
+                        gdb_output,
+                    )
 
     @unittest.skipIf(python_is_optimized(),
                      "Python was compiled with optimizations")
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 147008b0a3dc..2b261b6c86dc 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -5176,6 +5176,83 @@ sequence_getitem(PyObject *self, PyObject *args)
 }
 
 
+/* Functions for testing C calling conventions (METH_*) are named meth_*,
+ * e.g. "meth_varargs" for METH_VARARGS.
+ *
+ * They all return a tuple of their C-level arguments, with None instead
+ * of NULL and Python tuples instead of C arrays.
+ */
+
+
+static PyObject*
+_null_to_none(PyObject* obj)
+{
+    if (obj == NULL) {
+        Py_RETURN_NONE;
+    }
+    Py_INCREF(obj);
+    return obj;
+}
+
+static PyObject*
+meth_varargs(PyObject* self, PyObject* args)
+{
+    return Py_BuildValue("NO", _null_to_none(self), args);
+}
+
+static PyObject*
+meth_varargs_keywords(PyObject* self, PyObject* args, PyObject* kwargs)
+{
+    return Py_BuildValue("NON", _null_to_none(self), args, _null_to_none(kwargs));
+}
+
+static PyObject*
+meth_o(PyObject* self, PyObject* obj)
+{
+    return Py_BuildValue("NO", _null_to_none(self), obj);
+}
+
+static PyObject*
+meth_noargs(PyObject* self, PyObject* ignored)
+{
+    return _null_to_none(self);
+}
+
+static PyObject*
+_fastcall_to_tuple(PyObject* const* args, Py_ssize_t nargs)
+{
+    PyObject *tuple = PyTuple_New(nargs);
+    if (tuple == NULL) {
+        return NULL;
+    }
+    for (Py_ssize_t i=0; i < nargs; i++) {
+        Py_INCREF(args[i]);
+        PyTuple_SET_ITEM(tuple, i, args[i]);
+    }
+    return tuple;
+}
+
+static PyObject*
+meth_fastcall(PyObject* self, PyObject* const* args, Py_ssize_t nargs)
+{
+    return Py_BuildValue(
+        "NN", _null_to_none(self), _fastcall_to_tuple(args, nargs)
+    );
+}
+
+static PyObject*
+meth_fastcall_keywords(PyObject* self, PyObject* const* args,
+                       Py_ssize_t nargs, PyObject* kwargs)
+{
+    PyObject *pyargs = _fastcall_to_tuple(args, nargs);
+    if (pyargs == NULL) {
+        return NULL;
+    }
+    PyObject *pykwargs = _PyObject_Vectorcall((PyObject*)&PyDict_Type,
+                                              args + nargs, 0, kwargs);
+    return Py_BuildValue("NNN", _null_to_none(self), pyargs, pykwargs);
+}
+
 static PyMethodDef TestMethods[] = {
     {"raise_exception",         raise_exception,                 METH_VARARGS},
     {"raise_memoryerror",       raise_memoryerror,               METH_NOARGS},
@@ -5426,6 +5503,12 @@ static PyMethodDef TestMethods[] = {
 #endif
     {"write_unraisable_exc", test_write_unraisable_exc, METH_VARARGS},
     {"sequence_getitem", sequence_getitem, METH_VARARGS},
+    {"meth_varargs", meth_varargs, METH_VARARGS},
+    {"meth_varargs_keywords", (PyCFunction)(void(*)(void))meth_varargs_keywords, METH_VARARGS|METH_KEYWORDS},
+    {"meth_o", meth_o, METH_O},
+    {"meth_noargs", meth_noargs, METH_NOARGS},
+    {"meth_fastcall", (PyCFunction)(void(*)(void))meth_fastcall, METH_FASTCALL},
+    {"meth_fastcall_keywords", (PyCFunction)(void(*)(void))meth_fastcall_keywords, METH_FASTCALL|METH_KEYWORDS},
     {NULL, NULL} /* sentinel */
 };
 
@@ -6086,6 +6169,72 @@ static PyTypeObject MethodDescriptor2_Type = {
     .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | _Py_TPFLAGS_HAVE_VECTORCALL,
 };
 
+static PyMethodDef meth_instance_methods[] = {
+    {"meth_varargs", meth_varargs, METH_VARARGS},
+    {"meth_varargs_keywords", (PyCFunction)(void(*)(void))meth_varargs_keywords, METH_VARARGS|METH_KEYWORDS},
+    {"meth_o", meth_o, METH_O},
+    {"meth_noargs", meth_noargs, METH_NOARGS},
+    {"meth_fastcall", (PyCFunction)(void(*)(void))meth_fastcall, METH_FASTCALL},
+    {"meth_fastcall_keywords", (PyCFunction)(void(*)(void))meth_fastcall_keywords, METH_FASTCALL|METH_KEYWORDS},
+    {NULL, NULL} /* sentinel */
+};
+
+
+static PyTypeObject MethInstance_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "MethInstance",
+    sizeof(PyObject),
+    .tp_new = PyType_GenericNew,
+    .tp_flags = Py_TPFLAGS_DEFAULT,
+    .tp_methods = meth_instance_methods,
+    .tp_doc = PyDoc_STR(
+        "Class with normal (instance) methods to test calling conventions"),
+};
+
+static PyMethodDef meth_class_methods[] = {
+    {"meth_varargs", meth_varargs, METH_VARARGS|METH_CLASS},
+    {"meth_varargs_keywords", (PyCFunction)(void(*)(void))meth_varargs_keywords, METH_VARARGS|METH_KEYWORDS|METH_CLASS},
+    {"meth_o", meth_o, METH_O|METH_CLASS},
+    {"meth_noargs", meth_noargs, METH_NOARGS|METH_CLASS},
+    {"meth_fastcall", (PyCFunction)(void(*)(void))meth_fastcall, METH_FASTCALL|METH_CLASS},
+    {"meth_fastcall_keywords", (PyCFunction)(void(*)(void))meth_fastcall_keywords, METH_FASTCALL|METH_KEYWORDS|METH_CLASS},
+    {NULL, NULL} /* sentinel */
+};
+
+
+static PyTypeObject MethClass_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "MethClass",
+    sizeof(PyObject),
+    .tp_new = PyType_GenericNew,
+    .tp_flags = Py_TPFLAGS_DEFAULT,
+    .tp_methods = meth_class_methods,
+    .tp_doc = PyDoc_STR(
+        "Class with class methods to test calling conventions"),
+};
+
+static PyMethodDef meth_static_methods[] = {
+    {"meth_varargs", meth_varargs, METH_VARARGS|METH_STATIC},
+    {"meth_varargs_keywords", (PyCFunction)(void(*)(void))meth_varargs_keywords, METH_VARARGS|METH_KEYWORDS|METH_STATIC},
+    {"meth_o", meth_o, METH_O|METH_STATIC},
+    {"meth_noargs", meth_noargs, METH_NOARGS|METH_STATIC},
+    {"meth_fastcall", (PyCFunction)(void(*)(void))meth_fastcall, METH_FASTCALL|METH_STATIC},
+    {"meth_fastcall_keywords", (PyCFunction)(void(*)(void))meth_fastcall_keywords, METH_FASTCALL|METH_KEYWORDS|METH_STATIC},
+    {NULL, NULL} /* sentinel */
+};
+
+
+static PyTypeObject MethStatic_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "MethStatic",
+    sizeof(PyObject),
+    .tp_new = PyType_GenericNew,
+    .tp_flags = Py_TPFLAGS_DEFAULT,
+    .tp_methods = meth_static_methods,
+    .tp_doc = PyDoc_STR(
+        "Class with static methods to test calling conventions"),
+};
+
 
 static struct PyModuleDef _testcapimodule = {
     PyModuleDef_HEAD_INIT,
@@ -6172,6 +6321,21 @@ PyInit__testcapi(void)
     Py_INCREF(&Generic_Type);
     PyModule_AddObject(m, "Generic", (PyObject *)&Generic_Type);
 
+    if (PyType_Ready(&MethInstance_Type) < 0)
+        return NULL;
+    Py_INCREF(&MethInstance_Type);
+    PyModule_AddObject(m, "MethInstance", (PyObject *)&MethInstance_Type);
+
+    if (PyType_Ready(&MethClass_Type) < 0)
+        return NULL;
+    Py_INCREF(&MethClass_Type);
+    PyModule_AddObject(m, "MethClass", (PyObject *)&MethClass_Type);
+
+    if (PyType_Ready(&MethStatic_Type) < 0)
+        return NULL;
+    Py_INCREF(&MethStatic_Type);
+    PyModule_AddObject(m, "MethStatic", (PyObject *)&MethStatic_Type);
+
     PyRecursingInfinitelyError_Type.tp_base = (PyTypeObject *)PyExc_Exception;
     if (PyType_Ready(&PyRecursingInfinitelyError_Type) < 0) {
         return NULL;


From webhook-mailer at python.org  Tue Sep 10 07:22:00 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Tue, 10 Sep 2019 11:22:00 -0000
Subject: [Python-checkins] bpo-37619: update_one_slot() should not ignore
 wrapper descriptors for wrong type (GH-14836)
Message-ID: <mailman.424.1568114521.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/57ea33560662e0f20a3b0334bb20065771edf4da
commit: 57ea33560662e0f20a3b0334bb20065771edf4da
branch: master
author: Jeroen Demeyer <J.Demeyer at UGent.be>
committer: T. Wouters <thomas at python.org>
date: 2019-09-10T04:21:57-07:00
summary:

bpo-37619: update_one_slot() should not ignore wrapper descriptors for wrong type (GH-14836)

files:
A Misc/NEWS.d/next/Core and Builtins/2019-07-18-11-50-49.bpo-37619.X6Lulo.rst
M Lib/test/test_descr.py
M Objects/typeobject.c

diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py
index 4368bb585f31..796e60a77047 100644
--- a/Lib/test/test_descr.py
+++ b/Lib/test/test_descr.py
@@ -4649,6 +4649,18 @@ class X(dict):
         self.assertEqual(x["y"], 42)
         self.assertEqual(x, -x)
 
+    def test_wrong_class_slot_wrapper(self):
+        # Check bpo-37619: a wrapper descriptor taken from the wrong class
+        # should raise an exception instead of silently being ignored
+        class A(int):
+            __eq__ = str.__eq__
+            __add__ = str.__add__
+        a = A()
+        with self.assertRaises(TypeError):
+            a == a
+        with self.assertRaises(TypeError):
+            a + a
+
     def test_slot_shadows_class_variable(self):
         with self.assertRaises(ValueError) as cm:
             class X:
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-07-18-11-50-49.bpo-37619.X6Lulo.rst b/Misc/NEWS.d/next/Core and Builtins/2019-07-18-11-50-49.bpo-37619.X6Lulo.rst
new file mode 100644
index 000000000000..8723d3d9e9da
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-07-18-11-50-49.bpo-37619.X6Lulo.rst	
@@ -0,0 +1,3 @@
+When adding a wrapper descriptor from one class to a different class
+(for example, setting ``__add__ = str.__add__`` on an ``int`` subclass),
+an exception is correctly raised when the operator is called.
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 71d7f6f0e5c5..883bc22bbde1 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -7307,14 +7307,21 @@ update_one_slot(PyTypeObject *type, slotdef *p)
             if (tptr == NULL || tptr == ptr)
                 generic = p->function;
             d = (PyWrapperDescrObject *)descr;
-            if (d->d_base->wrapper == p->wrapper &&
+            if ((specific == NULL || specific == d->d_wrapped) &&
+                d->d_base->wrapper == p->wrapper &&
                 PyType_IsSubtype(type, PyDescr_TYPE(d)))
             {
-                if (specific == NULL ||
-                    specific == d->d_wrapped)
-                    specific = d->d_wrapped;
-                else
-                    use_generic = 1;
+                specific = d->d_wrapped;
+            }
+            else {
+                /* We cannot use the specific slot function because either
+                   - it is not unique: there are multiple methods for this
+                     slot and they conflict
+                   - the signature is wrong (as checked by the ->wrapper
+                     comparison above)
+                   - it's wrapping the wrong class
+                 */
+                use_generic = 1;
             }
         }
         else if (Py_TYPE(descr) == &PyCFunction_Type &&


From webhook-mailer at python.org  Tue Sep 10 07:26:58 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 11:26:58 -0000
Subject: [Python-checkins] bpo-36373: Deprecate explicit loop parameter in
 all public asyncio APIs [locks] (GH-13920)
Message-ID: <mailman.425.1568114819.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bb8fc8bd309419c159b632068dff73c3c76d478c
commit: bb8fc8bd309419c159b632068dff73c3c76d478c
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T04:26:54-07:00
summary:

bpo-36373: Deprecate explicit loop parameter in all public asyncio APIs [locks] (GH-13920)


This PR deprecate explicit loop parameters in all public asyncio APIs

This issues is split to be easier to review.

Third step: locks.py

https://bugs.python.org/issue36373
(cherry picked from commit 537877d85d1c27d2c2f5189e39da64a7a0c413d3)

Co-authored-by: Emmanuel Arias <emmanuelarias30 at gmail.com>

files:
M Doc/library/asyncio-sync.rst
M Lib/asyncio/locks.py
M Lib/test/test_asyncio/test_events.py
M Lib/test/test_asyncio/test_locks.py
M Lib/test/test_asyncio/test_pep492.py
M Lib/test/test_asyncio/test_queues.py
M Lib/test/test_asyncio/test_tasks.py

diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst
index 79f6b02d85e2..cc8c29dc3b6b 100644
--- a/Doc/library/asyncio-sync.rst
+++ b/Doc/library/asyncio-sync.rst
@@ -59,6 +59,9 @@ Lock
        finally:
            lock.release()
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    .. coroutinemethod:: acquire()
 
       Acquire the lock.
@@ -101,6 +104,10 @@ Event
    :meth:`clear` method.  The :meth:`wait` method blocks until the
    flag is set to *true*.  The flag is set to *false* initially.
 
+
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    .. _asyncio_example_sync_event:
 
    Example::
@@ -173,6 +180,10 @@ Condition
    ``None``.  In the latter case a new Lock object is created
    automatically.
 
+
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    The preferred way to use a Condition is an :keyword:`async with`
    statement::
 
@@ -269,6 +280,10 @@ Semaphore
    internal counter (``1`` by default). If the given value is
    less than ``0`` a :exc:`ValueError` is raised.
 
+
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    The preferred way to use a Semaphore is an :keyword:`async with`
    statement::
 
@@ -322,6 +337,9 @@ BoundedSemaphore
    increases the internal counter above the initial *value*.
 
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
 ---------
 
 
diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py
index 1324eefb5ff4..f63d4cedbbb7 100644
--- a/Lib/asyncio/locks.py
+++ b/Lib/asyncio/locks.py
@@ -160,10 +160,13 @@ class Lock(_ContextManagerMixin):
     def __init__(self, *, loop=None):
         self._waiters = None
         self._locked = False
-        if loop is not None:
-            self._loop = loop
-        else:
+        if loop is None:
             self._loop = events.get_event_loop()
+        else:
+            self._loop = loop
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
 
     def __repr__(self):
         res = super().__repr__()
@@ -253,10 +256,13 @@ class Event:
     def __init__(self, *, loop=None):
         self._waiters = collections.deque()
         self._value = False
-        if loop is not None:
-            self._loop = loop
-        else:
+        if loop is None:
             self._loop = events.get_event_loop()
+        else:
+            self._loop = loop
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
 
     def __repr__(self):
         res = super().__repr__()
@@ -317,10 +323,13 @@ class Condition(_ContextManagerMixin):
     """
 
     def __init__(self, lock=None, *, loop=None):
-        if loop is not None:
-            self._loop = loop
-        else:
+        if loop is None:
             self._loop = events.get_event_loop()
+        else:
+            self._loop = loop
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
 
         if lock is None:
             lock = Lock(loop=self._loop)
@@ -445,10 +454,13 @@ def __init__(self, value=1, *, loop=None):
             raise ValueError("Semaphore initial value must be >= 0")
         self._value = value
         self._waiters = collections.deque()
-        if loop is not None:
-            self._loop = loop
-        else:
+        if loop is None:
             self._loop = events.get_event_loop()
+        else:
+            self._loop = loop
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
 
     def __repr__(self):
         res = super().__repr__()
@@ -508,6 +520,11 @@ class BoundedSemaphore(Semaphore):
     """
 
     def __init__(self, value=1, *, loop=None):
+        if loop:
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
+
         self._bound_value = value
         super().__init__(value, loop=loop)
 
diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py
index e5ad72fe5ba8..95dac72b6d7e 100644
--- a/Lib/test/test_asyncio/test_events.py
+++ b/Lib/test/test_asyncio/test_events.py
@@ -1718,19 +1718,20 @@ def test_subprocess_exec(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
-        self.assertEqual('CONNECTED', proto.state)
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
+            self.assertEqual('CONNECTED', proto.state)
 
-        stdin = transp.get_pipe_transport(0)
-        stdin.write(b'Python The Winner')
-        self.loop.run_until_complete(proto.got_data[1].wait())
-        with test_utils.disable_logger():
-            transp.close()
-        self.loop.run_until_complete(proto.completed)
-        self.check_killed(proto.returncode)
-        self.assertEqual(b'Python The Winner', proto.data[1])
+            stdin = transp.get_pipe_transport(0)
+            stdin.write(b'Python The Winner')
+            self.loop.run_until_complete(proto.got_data[1].wait())
+            with test_utils.disable_logger():
+                transp.close()
+            self.loop.run_until_complete(proto.completed)
+            self.check_killed(proto.returncode)
+            self.assertEqual(b'Python The Winner', proto.data[1])
 
     def test_subprocess_interactive(self):
         prog = os.path.join(os.path.dirname(__file__), 'echo.py')
@@ -1738,47 +1739,52 @@ def test_subprocess_interactive(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
-        self.assertEqual('CONNECTED', proto.state)
 
-        stdin = transp.get_pipe_transport(0)
-        stdin.write(b'Python ')
-        self.loop.run_until_complete(proto.got_data[1].wait())
-        proto.got_data[1].clear()
-        self.assertEqual(b'Python ', proto.data[1])
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
+            self.assertEqual('CONNECTED', proto.state)
 
-        stdin.write(b'The Winner')
-        self.loop.run_until_complete(proto.got_data[1].wait())
-        self.assertEqual(b'Python The Winner', proto.data[1])
+            stdin = transp.get_pipe_transport(0)
+            stdin.write(b'Python ')
+            self.loop.run_until_complete(proto.got_data[1].wait())
+            proto.got_data[1].clear()
+            self.assertEqual(b'Python ', proto.data[1])
 
-        with test_utils.disable_logger():
-            transp.close()
-        self.loop.run_until_complete(proto.completed)
-        self.check_killed(proto.returncode)
+            stdin.write(b'The Winner')
+            self.loop.run_until_complete(proto.got_data[1].wait())
+            self.assertEqual(b'Python The Winner', proto.data[1])
+
+            with test_utils.disable_logger():
+                transp.close()
+            self.loop.run_until_complete(proto.completed)
+            self.check_killed(proto.returncode)
 
     def test_subprocess_shell(self):
-        connect = self.loop.subprocess_shell(
-                        functools.partial(MySubprocessProtocol, self.loop),
-                        'echo Python')
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
+        with self.assertWarns(DeprecationWarning):
+            connect = self.loop.subprocess_shell(
+                            functools.partial(MySubprocessProtocol, self.loop),
+                            'echo Python')
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
 
-        transp.get_pipe_transport(0).close()
-        self.loop.run_until_complete(proto.completed)
-        self.assertEqual(0, proto.returncode)
-        self.assertTrue(all(f.done() for f in proto.disconnects.values()))
-        self.assertEqual(proto.data[1].rstrip(b'\r\n'), b'Python')
-        self.assertEqual(proto.data[2], b'')
-        transp.close()
+            transp.get_pipe_transport(0).close()
+            self.loop.run_until_complete(proto.completed)
+            self.assertEqual(0, proto.returncode)
+            self.assertTrue(all(f.done() for f in proto.disconnects.values()))
+            self.assertEqual(proto.data[1].rstrip(b'\r\n'), b'Python')
+            self.assertEqual(proto.data[2], b'')
+            transp.close()
 
     def test_subprocess_exitcode(self):
         connect = self.loop.subprocess_shell(
                         functools.partial(MySubprocessProtocol, self.loop),
                         'exit 7', stdin=None, stdout=None, stderr=None)
-        transp, proto = self.loop.run_until_complete(connect)
+
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
         self.assertIsInstance(proto, MySubprocessProtocol)
         self.loop.run_until_complete(proto.completed)
         self.assertEqual(7, proto.returncode)
@@ -1788,7 +1794,8 @@ def test_subprocess_close_after_finish(self):
         connect = self.loop.subprocess_shell(
                         functools.partial(MySubprocessProtocol, self.loop),
                         'exit 7', stdin=None, stdout=None, stderr=None)
-        transp, proto = self.loop.run_until_complete(connect)
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
         self.assertIsInstance(proto, MySubprocessProtocol)
         self.assertIsNone(transp.get_pipe_transport(0))
         self.assertIsNone(transp.get_pipe_transport(1))
@@ -1803,14 +1810,16 @@ def test_subprocess_kill(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
 
-        transp.kill()
-        self.loop.run_until_complete(proto.completed)
-        self.check_killed(proto.returncode)
-        transp.close()
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
+
+            transp.kill()
+            self.loop.run_until_complete(proto.completed)
+            self.check_killed(proto.returncode)
+            transp.close()
 
     def test_subprocess_terminate(self):
         prog = os.path.join(os.path.dirname(__file__), 'echo.py')
@@ -1818,14 +1827,16 @@ def test_subprocess_terminate(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
 
-        transp.terminate()
-        self.loop.run_until_complete(proto.completed)
-        self.check_terminated(proto.returncode)
-        transp.close()
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
+
+            transp.terminate()
+            self.loop.run_until_complete(proto.completed)
+            self.check_terminated(proto.returncode)
+            transp.close()
 
     @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP")
     def test_subprocess_send_signal(self):
@@ -1839,14 +1850,16 @@ def test_subprocess_send_signal(self):
             connect = self.loop.subprocess_exec(
                             functools.partial(MySubprocessProtocol, self.loop),
                             sys.executable, prog)
-            transp, proto = self.loop.run_until_complete(connect)
-            self.assertIsInstance(proto, MySubprocessProtocol)
-            self.loop.run_until_complete(proto.connected)
 
-            transp.send_signal(signal.SIGHUP)
-            self.loop.run_until_complete(proto.completed)
-            self.assertEqual(-signal.SIGHUP, proto.returncode)
-            transp.close()
+            with self.assertWarns(DeprecationWarning):
+                transp, proto = self.loop.run_until_complete(connect)
+                self.assertIsInstance(proto, MySubprocessProtocol)
+                self.loop.run_until_complete(proto.connected)
+
+                transp.send_signal(signal.SIGHUP)
+                self.loop.run_until_complete(proto.completed)
+                self.assertEqual(-signal.SIGHUP, proto.returncode)
+                transp.close()
         finally:
             signal.signal(signal.SIGHUP, old_handler)
 
@@ -1856,19 +1869,21 @@ def test_subprocess_stderr(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
 
-        stdin = transp.get_pipe_transport(0)
-        stdin.write(b'test')
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
 
-        self.loop.run_until_complete(proto.completed)
+            stdin = transp.get_pipe_transport(0)
+            stdin.write(b'test')
 
-        transp.close()
-        self.assertEqual(b'OUT:test', proto.data[1])
-        self.assertTrue(proto.data[2].startswith(b'ERR:test'), proto.data[2])
-        self.assertEqual(0, proto.returncode)
+            self.loop.run_until_complete(proto.completed)
+
+            transp.close()
+            self.assertEqual(b'OUT:test', proto.data[1])
+            self.assertTrue(proto.data[2].startswith(b'ERR:test'), proto.data[2])
+            self.assertEqual(0, proto.returncode)
 
     def test_subprocess_stderr_redirect_to_stdout(self):
         prog = os.path.join(os.path.dirname(__file__), 'echo2.py')
@@ -1876,22 +1891,24 @@ def test_subprocess_stderr_redirect_to_stdout(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog, stderr=subprocess.STDOUT)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
 
-        stdin = transp.get_pipe_transport(0)
-        self.assertIsNotNone(transp.get_pipe_transport(1))
-        self.assertIsNone(transp.get_pipe_transport(2))
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
 
-        stdin.write(b'test')
-        self.loop.run_until_complete(proto.completed)
-        self.assertTrue(proto.data[1].startswith(b'OUT:testERR:test'),
-                        proto.data[1])
-        self.assertEqual(b'', proto.data[2])
+            stdin = transp.get_pipe_transport(0)
+            self.assertIsNotNone(transp.get_pipe_transport(1))
+            self.assertIsNone(transp.get_pipe_transport(2))
 
-        transp.close()
-        self.assertEqual(0, proto.returncode)
+            stdin.write(b'test')
+            self.loop.run_until_complete(proto.completed)
+            self.assertTrue(proto.data[1].startswith(b'OUT:testERR:test'),
+                            proto.data[1])
+            self.assertEqual(b'', proto.data[2])
+
+            transp.close()
+            self.assertEqual(0, proto.returncode)
 
     def test_subprocess_close_client_stream(self):
         prog = os.path.join(os.path.dirname(__file__), 'echo3.py')
@@ -1899,32 +1916,33 @@ def test_subprocess_close_client_stream(self):
         connect = self.loop.subprocess_exec(
                         functools.partial(MySubprocessProtocol, self.loop),
                         sys.executable, prog)
-        transp, proto = self.loop.run_until_complete(connect)
-        self.assertIsInstance(proto, MySubprocessProtocol)
-        self.loop.run_until_complete(proto.connected)
+        with self.assertWarns(DeprecationWarning):
+            transp, proto = self.loop.run_until_complete(connect)
+            self.assertIsInstance(proto, MySubprocessProtocol)
+            self.loop.run_until_complete(proto.connected)
 
-        stdin = transp.get_pipe_transport(0)
-        stdout = transp.get_pipe_transport(1)
-        stdin.write(b'test')
-        self.loop.run_until_complete(proto.got_data[1].wait())
-        self.assertEqual(b'OUT:test', proto.data[1])
+            stdin = transp.get_pipe_transport(0)
+            stdout = transp.get_pipe_transport(1)
+            stdin.write(b'test')
+            self.loop.run_until_complete(proto.got_data[1].wait())
+            self.assertEqual(b'OUT:test', proto.data[1])
 
-        stdout.close()
-        self.loop.run_until_complete(proto.disconnects[1])
-        stdin.write(b'xxx')
-        self.loop.run_until_complete(proto.got_data[2].wait())
-        if sys.platform != 'win32':
-            self.assertEqual(b'ERR:BrokenPipeError', proto.data[2])
-        else:
-            # After closing the read-end of a pipe, writing to the
-            # write-end using os.write() fails with errno==EINVAL and
-            # GetLastError()==ERROR_INVALID_NAME on Windows!?!  (Using
-            # WriteFile() we get ERROR_BROKEN_PIPE as expected.)
-            self.assertEqual(b'ERR:OSError', proto.data[2])
-        with test_utils.disable_logger():
-            transp.close()
-        self.loop.run_until_complete(proto.completed)
-        self.check_killed(proto.returncode)
+            stdout.close()
+            self.loop.run_until_complete(proto.disconnects[1])
+            stdin.write(b'xxx')
+            self.loop.run_until_complete(proto.got_data[2].wait())
+            if sys.platform != 'win32':
+                self.assertEqual(b'ERR:BrokenPipeError', proto.data[2])
+            else:
+                # After closing the read-end of a pipe, writing to the
+                # write-end using os.write() fails with errno==EINVAL and
+                # GetLastError()==ERROR_INVALID_NAME on Windows!?!  (Using
+                # WriteFile() we get ERROR_BROKEN_PIPE as expected.)
+                self.assertEqual(b'ERR:OSError', proto.data[2])
+            with test_utils.disable_logger():
+                transp.close()
+            self.loop.run_until_complete(proto.completed)
+            self.check_killed(proto.returncode)
 
     def test_subprocess_wait_no_same_group(self):
         # start the new process in a new session
@@ -1938,7 +1956,6 @@ def test_subprocess_wait_no_same_group(self):
         self.assertEqual(7, proto.returncode)
 
     def test_subprocess_exec_invalid_args(self):
-
         async def connect(**kwds):
             await self.loop.subprocess_exec(
                 asyncio.SubprocessProtocol,
diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py
index 5063a1da448e..d69b56dcda22 100644
--- a/Lib/test/test_asyncio/test_locks.py
+++ b/Lib/test/test_asyncio/test_locks.py
@@ -28,10 +28,12 @@ def setUp(self):
 
     def test_ctor_loop(self):
         loop = mock.Mock()
-        lock = asyncio.Lock(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=loop)
         self.assertIs(lock._loop, loop)
 
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
         self.assertIs(lock._loop, self.loop)
 
     def test_ctor_noloop(self):
@@ -40,7 +42,8 @@ def test_ctor_noloop(self):
         self.assertIs(lock._loop, self.loop)
 
     def test_repr(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
         self.assertTrue(repr(lock).endswith('[unlocked]>'))
         self.assertTrue(RGX_REPR.match(repr(lock)))
 
@@ -55,9 +58,10 @@ def acquire_lock():
         self.assertTrue(RGX_REPR.match(repr(lock)))
 
     def test_lock(self):
-        lock = asyncio.Lock(loop=self.loop)
-
         with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
+
+
             @asyncio.coroutine
             def acquire_lock():
                 with self.assertWarns(DeprecationWarning):
@@ -74,14 +78,14 @@ def acquire_lock():
     def test_lock_by_with_statement(self):
         loop = asyncio.new_event_loop()  # don't use TestLoop quirks
         self.set_event_loop(loop)
-        primitives = [
-            asyncio.Lock(loop=loop),
-            asyncio.Condition(loop=loop),
-            asyncio.Semaphore(loop=loop),
-            asyncio.BoundedSemaphore(loop=loop),
-        ]
-
         with self.assertWarns(DeprecationWarning):
+            primitives = [
+                asyncio.Lock(loop=loop),
+                asyncio.Condition(loop=loop),
+                asyncio.Semaphore(loop=loop),
+                asyncio.BoundedSemaphore(loop=loop),
+            ]
+
             @asyncio.coroutine
             def test(lock):
                 yield from asyncio.sleep(0.01)
@@ -99,7 +103,8 @@ def test(lock):
             self.assertFalse(primitive.locked())
 
     def test_acquire(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
         result = []
 
         self.assertTrue(self.loop.run_until_complete(lock.acquire()))
@@ -150,7 +155,8 @@ def test_acquire(self):
         self.assertTrue(t3.result())
 
     def test_acquire_cancel(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
         self.assertTrue(self.loop.run_until_complete(lock.acquire()))
 
         task = asyncio.Task(lock.acquire(), loop=self.loop)
@@ -175,7 +181,8 @@ def test_cancel_race(self):
         # B's waiter; instead, it should move on to C's waiter.
 
         # Setup: A has the lock, b and c are waiting.
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
 
         async def lockit(name, blocker):
             await lock.acquire()
@@ -211,7 +218,8 @@ def test_cancel_release_race(self):
         # Issue 32734
         # Acquire 4 locks, cancel second, release first
         # and 2 locks are taken at once.
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
         lock_count = 0
         call_count = 0
 
@@ -256,7 +264,8 @@ def trigger():
         self.assertTrue(t3.cancelled())
 
     def test_finished_waiter_cancelled(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
 
         ta = asyncio.Task(lock.acquire(), loop=self.loop)
         test_utils.run_briefly(self.loop)
@@ -278,12 +287,14 @@ def test_finished_waiter_cancelled(self):
         self.assertTrue(tb.cancelled())
 
     def test_release_not_acquired(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
 
         self.assertRaises(RuntimeError, lock.release)
 
     def test_release_no_waiters(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
         self.loop.run_until_complete(lock.acquire())
         self.assertTrue(lock.locked())
 
@@ -291,9 +302,9 @@ def test_release_no_waiters(self):
         self.assertFalse(lock.locked())
 
     def test_context_manager(self):
-        lock = asyncio.Lock(loop=self.loop)
-
         with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
+
             @asyncio.coroutine
             def acquire_lock():
                 with self.assertWarns(DeprecationWarning):
@@ -305,9 +316,9 @@ def acquire_lock():
         self.assertFalse(lock.locked())
 
     def test_context_manager_cant_reuse(self):
-        lock = asyncio.Lock(loop=self.loop)
-
         with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
+
             @asyncio.coroutine
             def acquire_lock():
                 with self.assertWarns(DeprecationWarning):
@@ -325,7 +336,8 @@ def acquire_lock():
                 pass
 
     def test_context_manager_no_yield(self):
-        lock = asyncio.Lock(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
 
         try:
             with lock:
@@ -346,10 +358,12 @@ def setUp(self):
 
     def test_ctor_loop(self):
         loop = mock.Mock()
-        ev = asyncio.Event(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=loop)
         self.assertIs(ev._loop, loop)
 
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
         self.assertIs(ev._loop, self.loop)
 
     def test_ctor_noloop(self):
@@ -358,7 +372,8 @@ def test_ctor_noloop(self):
         self.assertIs(ev._loop, self.loop)
 
     def test_repr(self):
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
         self.assertTrue(repr(ev).endswith('[unset]>'))
         match = RGX_REPR.match(repr(ev))
         self.assertEqual(match.group('extras'), 'unset')
@@ -372,7 +387,8 @@ def test_repr(self):
         self.assertTrue(RGX_REPR.match(repr(ev)))
 
     def test_wait(self):
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
         self.assertFalse(ev.is_set())
 
         result = []
@@ -409,14 +425,16 @@ def test_wait(self):
         self.assertIsNone(t3.result())
 
     def test_wait_on_set(self):
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
         ev.set()
 
         res = self.loop.run_until_complete(ev.wait())
         self.assertTrue(res)
 
     def test_wait_cancel(self):
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
 
         wait = asyncio.Task(ev.wait(), loop=self.loop)
         self.loop.call_soon(wait.cancel)
@@ -426,7 +444,8 @@ def test_wait_cancel(self):
         self.assertFalse(ev._waiters)
 
     def test_clear(self):
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
         self.assertFalse(ev.is_set())
 
         ev.set()
@@ -436,7 +455,8 @@ def test_clear(self):
         self.assertFalse(ev.is_set())
 
     def test_clear_with_waiters(self):
-        ev = asyncio.Event(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            ev = asyncio.Event(loop=self.loop)
         result = []
 
         async def c1(result):
@@ -472,19 +492,22 @@ def setUp(self):
 
     def test_ctor_loop(self):
         loop = mock.Mock()
-        cond = asyncio.Condition(loop=loop)
-        self.assertIs(cond._loop, loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=loop)
+            self.assertIs(cond._loop, loop)
 
-        cond = asyncio.Condition(loop=self.loop)
-        self.assertIs(cond._loop, self.loop)
+            cond = asyncio.Condition(loop=self.loop)
+            self.assertIs(cond._loop, self.loop)
 
     def test_ctor_noloop(self):
-        asyncio.set_event_loop(self.loop)
-        cond = asyncio.Condition()
-        self.assertIs(cond._loop, self.loop)
+        with self.assertWarns(DeprecationWarning):
+            asyncio.set_event_loop(self.loop)
+            cond = asyncio.Condition()
+            self.assertIs(cond._loop, self.loop)
 
     def test_wait(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         result = []
 
         async def c1(result):
@@ -547,7 +570,8 @@ def test_wait(self):
         self.assertTrue(t3.result())
 
     def test_wait_cancel(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         self.loop.run_until_complete(cond.acquire())
 
         wait = asyncio.Task(cond.wait(), loop=self.loop)
@@ -559,7 +583,8 @@ def test_wait_cancel(self):
         self.assertTrue(cond.locked())
 
     def test_wait_cancel_contested(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
 
         self.loop.run_until_complete(cond.acquire())
         self.assertTrue(cond.locked())
@@ -585,7 +610,8 @@ def test_wait_cancel_contested(self):
 
     def test_wait_cancel_after_notify(self):
         # See bpo-32841
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         waited = False
 
         async def wait_on_cond():
@@ -609,13 +635,15 @@ def test_wait_cancel_after_notify(self):
         self.assertTrue(waited)
 
     def test_wait_unacquired(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         self.assertRaises(
             RuntimeError,
             self.loop.run_until_complete, cond.wait())
 
     def test_wait_for(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         presult = False
 
         def predicate():
@@ -652,7 +680,8 @@ def predicate():
         self.assertTrue(t.result())
 
     def test_wait_for_unacquired(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
 
         # predicate can return true immediately
         res = self.loop.run_until_complete(cond.wait_for(lambda: [1, 2, 3]))
@@ -664,7 +693,8 @@ def test_wait_for_unacquired(self):
             cond.wait_for(lambda: False))
 
     def test_notify(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         result = []
 
         async def c1(result):
@@ -716,7 +746,8 @@ def test_notify(self):
         self.assertTrue(t3.result())
 
     def test_notify_all(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
 
         result = []
 
@@ -752,15 +783,18 @@ def test_notify_all(self):
         self.assertTrue(t2.result())
 
     def test_notify_unacquired(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         self.assertRaises(RuntimeError, cond.notify)
 
     def test_notify_all_unacquired(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         self.assertRaises(RuntimeError, cond.notify_all)
 
     def test_repr(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
         self.assertTrue('unlocked' in repr(cond))
         self.assertTrue(RGX_REPR.match(repr(cond)))
 
@@ -776,7 +810,8 @@ def test_repr(self):
         self.assertTrue(RGX_REPR.match(repr(cond)))
 
     def test_context_manager(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
 
         with self.assertWarns(DeprecationWarning):
             @asyncio.coroutine
@@ -790,7 +825,8 @@ def acquire_cond():
         self.assertFalse(cond.locked())
 
     def test_context_manager_no_yield(self):
-        cond = asyncio.Condition(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            cond = asyncio.Condition(loop=self.loop)
 
         try:
             with cond:
@@ -803,8 +839,9 @@ def test_context_manager_no_yield(self):
         self.assertFalse(cond.locked())
 
     def test_explicit_lock(self):
-        lock = asyncio.Lock(loop=self.loop)
-        cond = asyncio.Condition(lock, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
+            cond = asyncio.Condition(lock, loop=self.loop)
 
         self.assertIs(cond._lock, lock)
         self.assertIs(cond._loop, lock._loop)
@@ -812,10 +849,10 @@ def test_explicit_lock(self):
     def test_ambiguous_loops(self):
         loop = self.new_test_loop()
         self.addCleanup(loop.close)
-
-        lock = asyncio.Lock(loop=self.loop)
-        with self.assertRaises(ValueError):
-            asyncio.Condition(lock, loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            lock = asyncio.Lock(loop=self.loop)
+            with self.assertRaises(ValueError):
+                asyncio.Condition(lock, loop=loop)
 
     def test_timeout_in_block(self):
         loop = asyncio.new_event_loop()
@@ -827,7 +864,8 @@ def test_timeout_in_block(self):
                 with self.assertRaises(asyncio.TimeoutError):
                     await asyncio.wait_for(condition.wait(), timeout=0.5)
 
-        loop.run_until_complete(task_timeout())
+        with self.assertWarns(DeprecationWarning):
+            loop.run_until_complete(task_timeout())
 
 
 class SemaphoreTests(test_utils.TestCase):
@@ -838,10 +876,12 @@ def setUp(self):
 
     def test_ctor_loop(self):
         loop = mock.Mock()
-        sem = asyncio.Semaphore(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(loop=loop)
         self.assertIs(sem._loop, loop)
 
-        sem = asyncio.Semaphore(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(loop=self.loop)
         self.assertIs(sem._loop, self.loop)
 
     def test_ctor_noloop(self):
@@ -850,11 +890,13 @@ def test_ctor_noloop(self):
         self.assertIs(sem._loop, self.loop)
 
     def test_initial_value_zero(self):
-        sem = asyncio.Semaphore(0, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(0, loop=self.loop)
         self.assertTrue(sem.locked())
 
     def test_repr(self):
-        sem = asyncio.Semaphore(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(loop=self.loop)
         self.assertTrue(repr(sem).endswith('[unlocked, value:1]>'))
         self.assertTrue(RGX_REPR.match(repr(sem)))
 
@@ -872,7 +914,8 @@ def test_repr(self):
         self.assertTrue(RGX_REPR.match(repr(sem)))
 
     def test_semaphore(self):
-        sem = asyncio.Semaphore(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(loop=self.loop)
         self.assertEqual(1, sem._value)
 
         with self.assertWarns(DeprecationWarning):
@@ -895,7 +938,8 @@ def test_semaphore_value(self):
         self.assertRaises(ValueError, asyncio.Semaphore, -1)
 
     def test_acquire(self):
-        sem = asyncio.Semaphore(3, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(3, loop=self.loop)
         result = []
 
         self.assertTrue(self.loop.run_until_complete(sem.acquire()))
@@ -956,7 +1000,8 @@ def test_acquire(self):
         self.loop.run_until_complete(asyncio.gather(*race_tasks))
 
     def test_acquire_cancel(self):
-        sem = asyncio.Semaphore(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(loop=self.loop)
         self.loop.run_until_complete(sem.acquire())
 
         acquire = asyncio.Task(sem.acquire(), loop=self.loop)
@@ -968,7 +1013,8 @@ def test_acquire_cancel(self):
                         all(waiter.done() for waiter in sem._waiters))
 
     def test_acquire_cancel_before_awoken(self):
-        sem = asyncio.Semaphore(value=0, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(value=0, loop=self.loop)
 
         t1 = asyncio.Task(sem.acquire(), loop=self.loop)
         t2 = asyncio.Task(sem.acquire(), loop=self.loop)
@@ -990,7 +1036,8 @@ def test_acquire_cancel_before_awoken(self):
         test_utils.run_briefly(self.loop)
 
     def test_acquire_hang(self):
-        sem = asyncio.Semaphore(value=0, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(value=0, loop=self.loop)
 
         t1 = asyncio.Task(sem.acquire(), loop=self.loop)
         t2 = asyncio.Task(sem.acquire(), loop=self.loop)
@@ -1004,12 +1051,14 @@ def test_acquire_hang(self):
         self.assertTrue(sem.locked())
 
     def test_release_not_acquired(self):
-        sem = asyncio.BoundedSemaphore(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.BoundedSemaphore(loop=self.loop)
 
         self.assertRaises(ValueError, sem.release)
 
     def test_release_no_waiters(self):
-        sem = asyncio.Semaphore(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(loop=self.loop)
         self.loop.run_until_complete(sem.acquire())
         self.assertTrue(sem.locked())
 
@@ -1017,9 +1066,9 @@ def test_release_no_waiters(self):
         self.assertFalse(sem.locked())
 
     def test_context_manager(self):
-        sem = asyncio.Semaphore(2, loop=self.loop)
-
         with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(2, loop=self.loop)
+
             @asyncio.coroutine
             def acquire_lock():
                 with self.assertWarns(DeprecationWarning):
@@ -1035,7 +1084,8 @@ def acquire_lock():
         self.assertEqual(2, sem._value)
 
     def test_context_manager_no_yield(self):
-        sem = asyncio.Semaphore(2, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            sem = asyncio.Semaphore(2, loop=self.loop)
 
         try:
             with sem:
diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py
index a5cf37ded7c9..f327bbd1d66d 100644
--- a/Lib/test/test_asyncio/test_pep492.py
+++ b/Lib/test/test_asyncio/test_pep492.py
@@ -43,12 +43,13 @@ def setUp(self):
 class LockTests(BaseTest):
 
     def test_context_manager_async_with(self):
-        primitives = [
-            asyncio.Lock(loop=self.loop),
-            asyncio.Condition(loop=self.loop),
-            asyncio.Semaphore(loop=self.loop),
-            asyncio.BoundedSemaphore(loop=self.loop),
-        ]
+        with self.assertWarns(DeprecationWarning):
+            primitives = [
+                asyncio.Lock(loop=self.loop),
+                asyncio.Condition(loop=self.loop),
+                asyncio.Semaphore(loop=self.loop),
+                asyncio.BoundedSemaphore(loop=self.loop),
+            ]
 
         async def test(lock):
             await asyncio.sleep(0.01)
@@ -65,12 +66,13 @@ def test_context_manager_async_with(self):
             self.assertFalse(primitive.locked())
 
     def test_context_manager_with_await(self):
-        primitives = [
-            asyncio.Lock(loop=self.loop),
-            asyncio.Condition(loop=self.loop),
-            asyncio.Semaphore(loop=self.loop),
-            asyncio.BoundedSemaphore(loop=self.loop),
-        ]
+        with self.assertWarns(DeprecationWarning):
+            primitives = [
+                asyncio.Lock(loop=self.loop),
+                asyncio.Condition(loop=self.loop),
+                asyncio.Semaphore(loop=self.loop),
+                asyncio.BoundedSemaphore(loop=self.loop),
+            ]
 
         async def test(lock):
             await asyncio.sleep(0.01)
diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py
index b0f0f9c8fdb7..02e8e43ccee6 100644
--- a/Lib/test/test_asyncio/test_queues.py
+++ b/Lib/test/test_asyncio/test_queues.py
@@ -35,7 +35,8 @@ def gen():
 
         loop = self.new_test_loop(gen)
 
-        q = asyncio.Queue(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=loop)
         self.assertTrue(fn(q).startswith('<Queue'), fn(q))
         id_is_present = hex(id(q)) in fn(q)
         self.assertEqual(expect_id, id_is_present)
@@ -50,7 +51,8 @@ def gen():
             # resume q.get coroutine to finish generator
             q.put_nowait(0)
 
-        loop.run_until_complete(add_getter())
+        with self.assertWarns(DeprecationWarning):
+            loop.run_until_complete(add_getter())
 
         async def add_putter():
             q = asyncio.Queue(maxsize=1, loop=loop)
@@ -63,23 +65,26 @@ def gen():
             # resume q.put coroutine to finish generator
             q.get_nowait()
 
-        loop.run_until_complete(add_putter())
-
-        q = asyncio.Queue(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            loop.run_until_complete(add_putter())
+            q = asyncio.Queue(loop=loop)
         q.put_nowait(1)
         self.assertTrue('_queue=[1]' in fn(q))
 
     def test_ctor_loop(self):
         loop = mock.Mock()
-        q = asyncio.Queue(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=loop)
         self.assertIs(q._loop, loop)
 
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         self.assertIs(q._loop, self.loop)
 
     def test_ctor_noloop(self):
         asyncio.set_event_loop(self.loop)
-        q = asyncio.Queue()
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue()
         self.assertIs(q._loop, self.loop)
 
     def test_repr(self):
@@ -89,7 +94,8 @@ def test_str(self):
         self._test_repr_or_str(str, False)
 
     def test_empty(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         self.assertTrue(q.empty())
         q.put_nowait(1)
         self.assertFalse(q.empty())
@@ -97,15 +103,18 @@ def test_empty(self):
         self.assertTrue(q.empty())
 
     def test_full(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         self.assertFalse(q.full())
 
-        q = asyncio.Queue(maxsize=1, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(maxsize=1, loop=self.loop)
         q.put_nowait(1)
         self.assertTrue(q.full())
 
     def test_order(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         for i in [1, 3, 2]:
             q.put_nowait(i)
 
@@ -123,7 +132,8 @@ def gen():
 
         loop = self.new_test_loop(gen)
 
-        q = asyncio.Queue(maxsize=2, loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(maxsize=2, loop=loop)
         self.assertEqual(2, q.maxsize)
         have_been_put = []
 
@@ -157,7 +167,8 @@ def gen():
 class QueueGetTests(_QueueTestBase):
 
     def test_blocking_get(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         q.put_nowait(1)
 
         async def queue_get():
@@ -167,7 +178,8 @@ def test_blocking_get(self):
         self.assertEqual(1, res)
 
     def test_get_with_putters(self):
-        q = asyncio.Queue(1, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(1, loop=self.loop)
         q.put_nowait(1)
 
         waiter = asyncio.Future(loop=self.loop)
@@ -187,8 +199,9 @@ def gen():
 
         loop = self.new_test_loop(gen)
 
-        q = asyncio.Queue(loop=loop)
-        started = asyncio.Event(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=loop)
+            started = asyncio.Event(loop=loop)
         finished = False
 
         async def queue_get():
@@ -212,12 +225,14 @@ def gen():
         self.assertAlmostEqual(0.01, loop.time())
 
     def test_nonblocking_get(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         q.put_nowait(1)
         self.assertEqual(1, q.get_nowait())
 
     def test_nonblocking_get_exception(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         self.assertRaises(asyncio.QueueEmpty, q.get_nowait)
 
     def test_get_cancelled(self):
@@ -231,7 +246,8 @@ def gen():
 
         loop = self.new_test_loop(gen)
 
-        q = asyncio.Queue(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=loop)
 
         async def queue_get():
             return await asyncio.wait_for(q.get(), 0.051)
@@ -246,7 +262,8 @@ def gen():
         self.assertAlmostEqual(0.06, loop.time())
 
     def test_get_cancelled_race(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
 
         t1 = asyncio.Task(q.get(), loop=self.loop)
         t2 = asyncio.Task(q.get(), loop=self.loop)
@@ -260,7 +277,8 @@ def test_get_cancelled_race(self):
         self.assertEqual(t2.result(), 'a')
 
     def test_get_with_waiting_putters(self):
-        q = asyncio.Queue(loop=self.loop, maxsize=1)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop, maxsize=1)
         asyncio.Task(q.put('a'), loop=self.loop)
         asyncio.Task(q.put('b'), loop=self.loop)
         test_utils.run_briefly(self.loop)
@@ -280,7 +298,9 @@ def test_why_are_getters_waiting(self):
 
         queue_size = 1
         producer_num_items = 5
-        q = asyncio.Queue(queue_size, loop=self.loop)
+
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(queue_size, loop=self.loop)
 
         self.loop.run_until_complete(
             asyncio.gather(producer(q, producer_num_items),
@@ -301,7 +321,8 @@ def a_generator():
             except asyncio.TimeoutError:
                 pass
 
-        queue = asyncio.Queue(loop=self.loop, maxsize=5)
+        with self.assertWarns(DeprecationWarning):
+            queue = asyncio.Queue(loop=self.loop, maxsize=5)
         self.loop.run_until_complete(self.loop.create_task(consumer(queue)))
         self.assertEqual(len(queue._getters), 0)
 
@@ -309,7 +330,8 @@ def a_generator():
 class QueuePutTests(_QueueTestBase):
 
     def test_blocking_put(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
 
         async def queue_put():
             # No maxsize, won't block.
@@ -326,8 +348,9 @@ def gen():
 
         loop = self.new_test_loop(gen)
 
-        q = asyncio.Queue(maxsize=1, loop=loop)
-        started = asyncio.Event(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(maxsize=1, loop=loop)
+            started = asyncio.Event(loop=loop)
         finished = False
 
         async def queue_put():
@@ -349,7 +372,8 @@ def gen():
         self.assertAlmostEqual(0.01, loop.time())
 
     def test_nonblocking_put(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         q.put_nowait(1)
         self.assertEqual(1, q.get_nowait())
 
@@ -360,7 +384,8 @@ def gen():
 
         loop = self.new_test_loop(gen)
 
-        q = asyncio.Queue(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=loop)
 
         reader = loop.create_task(q.get())
 
@@ -389,7 +414,8 @@ def gen():
         loop = self.new_test_loop(gen)
         loop.set_debug(True)
 
-        q = asyncio.Queue(loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=loop)
 
         reader1 = loop.create_task(q.get())
         reader2 = loop.create_task(q.get())
@@ -418,7 +444,9 @@ def gen():
             yield 0.1
 
         loop = self.new_test_loop(gen)
-        q = asyncio.Queue(1, loop=loop)
+
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(1, loop=loop)
 
         q.put_nowait(1)
 
@@ -442,18 +470,21 @@ def gen():
         self.assertEqual(q.qsize(), 0)
 
     def test_nonblocking_put_exception(self):
-        q = asyncio.Queue(maxsize=1, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(maxsize=1, loop=self.loop)
         q.put_nowait(1)
         self.assertRaises(asyncio.QueueFull, q.put_nowait, 2)
 
     def test_float_maxsize(self):
-        q = asyncio.Queue(maxsize=1.3, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(maxsize=1.3, loop=self.loop)
         q.put_nowait(1)
         q.put_nowait(2)
         self.assertTrue(q.full())
         self.assertRaises(asyncio.QueueFull, q.put_nowait, 3)
 
-        q = asyncio.Queue(maxsize=1.3, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(maxsize=1.3, loop=self.loop)
 
         async def queue_put():
             await q.put(1)
@@ -462,7 +493,8 @@ def test_float_maxsize(self):
         self.loop.run_until_complete(queue_put())
 
     def test_put_cancelled(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
 
         async def queue_put():
             await q.put(1)
@@ -477,7 +509,8 @@ def test_put_cancelled(self):
         self.assertTrue(t.result())
 
     def test_put_cancelled_race(self):
-        q = asyncio.Queue(loop=self.loop, maxsize=1)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop, maxsize=1)
 
         put_a = asyncio.Task(q.put('a'), loop=self.loop)
         put_b = asyncio.Task(q.put('b'), loop=self.loop)
@@ -497,7 +530,8 @@ def test_put_cancelled_race(self):
         self.loop.run_until_complete(put_b)
 
     def test_put_with_waiting_getters(self):
-        q = asyncio.Queue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.Queue(loop=self.loop)
         t = asyncio.Task(q.get(), loop=self.loop)
         test_utils.run_briefly(self.loop)
         self.loop.run_until_complete(q.put('a'))
@@ -506,7 +540,8 @@ def test_put_with_waiting_getters(self):
     def test_why_are_putters_waiting(self):
         # From issue #265.
 
-        queue = asyncio.Queue(2, loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            queue = asyncio.Queue(2, loop=self.loop)
 
         async def putter(item):
             await queue.put(item)
@@ -532,7 +567,8 @@ def a_generator():
         loop = self.new_test_loop(a_generator)
 
         # Full queue.
-        queue = asyncio.Queue(loop=loop, maxsize=1)
+        with self.assertWarns(DeprecationWarning):
+            queue = asyncio.Queue(loop=loop, maxsize=1)
         queue.put_nowait(1)
 
         # Task waiting for space to put an item in the queue.
@@ -555,7 +591,8 @@ def gen():
         loop = self.new_test_loop(gen)
 
         # Full Queue.
-        queue = asyncio.Queue(1, loop=loop)
+        with self.assertWarns(DeprecationWarning):
+            queue = asyncio.Queue(1, loop=loop)
         queue.put_nowait(1)
 
         # Task waiting for space to put a item in the queue.
@@ -578,7 +615,8 @@ def gen():
 class LifoQueueTests(_QueueTestBase):
 
     def test_order(self):
-        q = asyncio.LifoQueue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.LifoQueue(loop=self.loop)
         for i in [1, 3, 2]:
             q.put_nowait(i)
 
@@ -589,7 +627,8 @@ def test_order(self):
 class PriorityQueueTests(_QueueTestBase):
 
     def test_order(self):
-        q = asyncio.PriorityQueue(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = asyncio.PriorityQueue(loop=self.loop)
         for i in [1, 3, 2]:
             q.put_nowait(i)
 
@@ -602,11 +641,13 @@ class _QueueJoinTestMixin:
     q_class = None
 
     def test_task_done_underflow(self):
-        q = self.q_class(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = self.q_class(loop=self.loop)
         self.assertRaises(ValueError, q.task_done)
 
     def test_task_done(self):
-        q = self.q_class(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = self.q_class(loop=self.loop)
         for i in range(100):
             q.put_nowait(i)
 
@@ -641,7 +682,8 @@ def test_task_done(self):
         self.loop.run_until_complete(asyncio.wait(tasks))
 
     def test_join_empty_queue(self):
-        q = self.q_class(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = self.q_class(loop=self.loop)
 
         # Test that a queue join()s successfully, and before anything else
         # (done twice for insurance).
@@ -653,7 +695,8 @@ def test_join_empty_queue(self):
         self.loop.run_until_complete(join())
 
     def test_format(self):
-        q = self.q_class(loop=self.loop)
+        with self.assertWarns(DeprecationWarning):
+            q = self.q_class(loop=self.loop)
         self.assertEqual(q._format(), 'maxsize=0')
 
         q._unfinished_tasks = 2
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
index d5f44b8091de..323c0907a23a 100644
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -1231,15 +1231,16 @@ def sleeper(dt, x):
             for f in asyncio.as_completed([b, c, a], loop=loop):
                 values.append(await f)
             return values
-
-        res = loop.run_until_complete(self.new_task(loop, foo()))
+        with self.assertWarns(DeprecationWarning):
+            res = loop.run_until_complete(self.new_task(loop, foo()))
         self.assertAlmostEqual(0.15, loop.time())
         self.assertTrue('a' in res[:2])
         self.assertTrue('b' in res[:2])
         self.assertEqual(res[2], 'c')
 
         # Doing it again should take no time and exercise a different path.
-        res = loop.run_until_complete(self.new_task(loop, foo()))
+        with self.assertWarns(DeprecationWarning):
+            res = loop.run_until_complete(self.new_task(loop, foo()))
         self.assertAlmostEqual(0.15, loop.time())
 
     def test_as_completed_with_timeout(self):
@@ -1267,7 +1268,8 @@ def gen():
                     values.append((2, exc))
             return values
 
-        res = loop.run_until_complete(self.new_task(loop, foo()))
+        with self.assertWarns(DeprecationWarning):
+            res = loop.run_until_complete(self.new_task(loop, foo()))
         self.assertEqual(len(res), 2, res)
         self.assertEqual(res[0], (1, 'a'))
         self.assertEqual(res[1][0], 2)
@@ -1294,7 +1296,8 @@ def gen():
                 v = await f
                 self.assertEqual(v, 'a')
 
-        loop.run_until_complete(self.new_task(loop, foo()))
+        with self.assertWarns(DeprecationWarning):
+            loop.run_until_complete(self.new_task(loop, foo()))
 
     def test_as_completed_reverse_wait(self):
 
@@ -1308,7 +1311,9 @@ def gen():
         a = asyncio.sleep(0.05, 'a')
         b = asyncio.sleep(0.10, 'b')
         fs = {a, b}
-        futs = list(asyncio.as_completed(fs, loop=loop))
+
+        with self.assertWarns(DeprecationWarning):
+            futs = list(asyncio.as_completed(fs, loop=loop))
         self.assertEqual(len(futs), 2)
 
         x = loop.run_until_complete(futs[1])
@@ -1333,7 +1338,8 @@ def gen():
         a = asyncio.sleep(0.05, 'a')
         b = asyncio.sleep(0.05, 'b')
         fs = {a, b}
-        futs = list(asyncio.as_completed(fs, loop=loop))
+        with self.assertWarns(DeprecationWarning):
+            futs = list(asyncio.as_completed(fs, loop=loop))
         self.assertEqual(len(futs), 2)
         waiter = asyncio.wait(futs)
         done, pending = loop.run_until_complete(waiter)
@@ -1356,8 +1362,9 @@ def runner():
                     result.append((yield from f))
                 return result
 
-        fut = self.new_task(self.loop, runner())
-        self.loop.run_until_complete(fut)
+        with self.assertWarns(DeprecationWarning):
+            fut = self.new_task(self.loop, runner())
+            self.loop.run_until_complete(fut)
         result = fut.result()
         self.assertEqual(set(result), {'ham', 'spam'})
         self.assertEqual(len(result), 2)


From webhook-mailer at python.org  Tue Sep 10 07:46:16 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 11:46:16 -0000
Subject: [Python-checkins] bpo-36373: Deprecate explicit loop parameter in
 all public asyncio APIs [queue] (GH-13950)
Message-ID: <mailman.426.1568115977.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9008be303a89bfab8c3314c6a42330b5523adc8b
commit: 9008be303a89bfab8c3314c6a42330b5523adc8b
branch: master
author: Emmanuel Arias <emmanuelarias30 at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-10T04:46:12-07:00
summary:

bpo-36373: Deprecate explicit loop parameter in all public asyncio APIs [queue] (GH-13950)



This PR deprecate explicit loop parameters in all public asyncio APIs

This issues is split to be easier to review.

fourth step: queue.py





https://bugs.python.org/issue36373

files:
M Doc/library/asyncio-queue.rst
M Lib/asyncio/queues.py

diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst
index 7be1023c80cc..e7775da54f58 100644
--- a/Doc/library/asyncio-queue.rst
+++ b/Doc/library/asyncio-queue.rst
@@ -32,6 +32,10 @@ Queue
    the queue is always known and can be returned by calling the
    :meth:`qsize` method.
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
+
    This class is :ref:`not thread safe <asyncio-multithreading>`.
 
    .. attribute:: maxsize
diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py
index 2a8d3e760441..c96b4a0c1ba3 100644
--- a/Lib/asyncio/queues.py
+++ b/Lib/asyncio/queues.py
@@ -2,6 +2,7 @@
 
 import collections
 import heapq
+import warnings
 
 from . import events
 from . import locks
@@ -34,6 +35,9 @@ def __init__(self, maxsize=0, *, loop=None):
             self._loop = events.get_event_loop()
         else:
             self._loop = loop
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
         self._maxsize = maxsize
 
         # Futures.


From webhook-mailer at python.org  Tue Sep 10 08:44:02 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Tue, 10 Sep 2019 12:44:02 -0000
Subject: [Python-checkins] closes bpo-25461: Update os.walk() docstring to
 match the online docs. (GH-11836)
Message-ID: <mailman.427.1568119443.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/734f1202a50641eb2c4bfbcd5b75247c1dc99a8f
commit: 734f1202a50641eb2c4bfbcd5b75247c1dc99a8f
branch: master
author: Bernt R?skar Brenna <bernt.brenna at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-10T13:43:58+01:00
summary:

closes bpo-25461: Update os.walk() docstring to match the online docs. (GH-11836)

files:
M Lib/os.py

diff --git a/Lib/os.py b/Lib/os.py
index 200902528cbf..52d3f1d74158 100644
--- a/Lib/os.py
+++ b/Lib/os.py
@@ -300,10 +300,11 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
     (e.g., via del or slice assignment), and walk will only recurse into the
     subdirectories whose names remain in dirnames; this can be used to prune the
     search, or to impose a specific order of visiting.  Modifying dirnames when
-    topdown is false is ineffective, since the directories in dirnames have
-    already been generated by the time dirnames itself is generated. No matter
-    the value of topdown, the list of subdirectories is retrieved before the
-    tuples for the directory and its subdirectories are generated.
+    topdown is false has no effect on the behavior of os.walk(), since the
+    directories in dirnames have already been generated by the time dirnames
+    itself is generated. No matter the value of topdown, the list of
+    subdirectories is retrieved before the tuples for the directory and its
+    subdirectories are generated.
 
     By default errors from the os.scandir() call are ignored.  If
     optional arg 'onerror' is specified, it should be a function; it


From webhook-mailer at python.org  Tue Sep 10 08:50:18 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 12:50:18 -0000
Subject: [Python-checkins] bpo-36373: Deprecate explicit loop parameter in
 all public asyncio APIs [queue] (GH-13950)
Message-ID: <mailman.428.1568119819.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/55daf1a56163e0673654712bb92ce47a9b6f7be3
commit: 55daf1a56163e0673654712bb92ce47a9b6f7be3
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T05:50:14-07:00
summary:

bpo-36373: Deprecate explicit loop parameter in all public asyncio APIs [queue] (GH-13950)


This PR deprecate explicit loop parameters in all public asyncio APIs

This issues is split to be easier to review.

fourth step: queue.py

https://bugs.python.org/issue36373
(cherry picked from commit 9008be303a89bfab8c3314c6a42330b5523adc8b)

Co-authored-by: Emmanuel Arias <emmanuelarias30 at gmail.com>

files:
M Doc/library/asyncio-queue.rst
M Lib/asyncio/queues.py

diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst
index 7be1023c80cc..e7775da54f58 100644
--- a/Doc/library/asyncio-queue.rst
+++ b/Doc/library/asyncio-queue.rst
@@ -32,6 +32,10 @@ Queue
    the queue is always known and can be returned by calling the
    :meth:`qsize` method.
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
+
    This class is :ref:`not thread safe <asyncio-multithreading>`.
 
    .. attribute:: maxsize
diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py
index 2a8d3e760441..c96b4a0c1ba3 100644
--- a/Lib/asyncio/queues.py
+++ b/Lib/asyncio/queues.py
@@ -2,6 +2,7 @@
 
 import collections
 import heapq
+import warnings
 
 from . import events
 from . import locks
@@ -34,6 +35,9 @@ def __init__(self, maxsize=0, *, loop=None):
             self._loop = events.get_event_loop()
         else:
             self._loop = loop
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
         self._maxsize = maxsize
 
         # Futures.


From webhook-mailer at python.org  Tue Sep 10 08:51:13 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Tue, 10 Sep 2019 12:51:13 -0000
Subject: [Python-checkins] bpo-21018: added missing documentation about
 escaping characters for configparser (GH-6137)
Message-ID: <mailman.429.1568119874.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9a94093189417adddd6b59d6c80cc5544630c8aa
commit: 9a94093189417adddd6b59d6c80cc5544630c8aa
branch: master
author: Arun Persaud <arun at nubati.net>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-10T14:51:09+02:00
summary:

bpo-21018: added missing documentation about escaping characters for configparser (GH-6137)

Document how $ and % can be escaped in configparser.

files:
M Doc/library/configparser.rst

diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst
index 04b52dc7b215..739477f55fdd 100644
--- a/Doc/library/configparser.rst
+++ b/Doc/library/configparser.rst
@@ -313,6 +313,8 @@ from ``get()`` calls.
       my_dir: %(home_dir)s/lumberjack
       my_pictures: %(my_dir)s/Pictures
 
+      [Escape]
+      gain: 80%%  # use a %% to escape the % sign (% is the only character that needs to be escaped)
 
    In the example above, :class:`ConfigParser` with *interpolation* set to
    ``BasicInterpolation()`` would resolve ``%(home_dir)s`` to the value of
@@ -346,6 +348,9 @@ from ``get()`` calls.
       my_dir: ${home_dir}/lumberjack
       my_pictures: ${my_dir}/Pictures
 
+      [Escape]
+      cost: $$80  # use a $$ to escape the $ sign ($ is the only character that needs to be escaped)
+
    Values from other sections can be fetched as well:
 
    .. code-block:: ini


From webhook-mailer at python.org  Tue Sep 10 08:55:15 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Tue, 10 Sep 2019 12:55:15 -0000
Subject: [Python-checkins] bpo-37619: update_one_slot() should not ignore
 wrapper descriptors for wrong type (GH-15838)
Message-ID: <mailman.430.1568120116.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/eb1bc48c74f4f8af88b5276729f9652201e46324
commit: eb1bc48c74f4f8af88b5276729f9652201e46324
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-10T05:55:12-07:00
summary:

bpo-37619: update_one_slot() should not ignore wrapper descriptors for wrong type (GH-15838)

(cherry picked from commit 57ea33560662e0f20a3b0334bb20065771edf4da)

Co-authored-by: Jeroen Demeyer <J.Demeyer at UGent.be>

files:
A Misc/NEWS.d/next/Core and Builtins/2019-07-18-11-50-49.bpo-37619.X6Lulo.rst
M Lib/test/test_descr.py
M Objects/typeobject.c

diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py
index 741cb6dce285..c4d900a8c1bc 100644
--- a/Lib/test/test_descr.py
+++ b/Lib/test/test_descr.py
@@ -4649,6 +4649,18 @@ class X(dict):
         self.assertEqual(x["y"], 42)
         self.assertEqual(x, -x)
 
+    def test_wrong_class_slot_wrapper(self):
+        # Check bpo-37619: a wrapper descriptor taken from the wrong class
+        # should raise an exception instead of silently being ignored
+        class A(int):
+            __eq__ = str.__eq__
+            __add__ = str.__add__
+        a = A()
+        with self.assertRaises(TypeError):
+            a == a
+        with self.assertRaises(TypeError):
+            a + a
+
     def test_slot_shadows_class_variable(self):
         with self.assertRaises(ValueError) as cm:
             class X:
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-07-18-11-50-49.bpo-37619.X6Lulo.rst b/Misc/NEWS.d/next/Core and Builtins/2019-07-18-11-50-49.bpo-37619.X6Lulo.rst
new file mode 100644
index 000000000000..8723d3d9e9da
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-07-18-11-50-49.bpo-37619.X6Lulo.rst	
@@ -0,0 +1,3 @@
+When adding a wrapper descriptor from one class to a different class
+(for example, setting ``__add__ = str.__add__`` on an ``int`` subclass),
+an exception is correctly raised when the operator is called.
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 8ece21827519..f42caedf6cc2 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -7243,14 +7243,21 @@ update_one_slot(PyTypeObject *type, slotdef *p)
             if (tptr == NULL || tptr == ptr)
                 generic = p->function;
             d = (PyWrapperDescrObject *)descr;
-            if (d->d_base->wrapper == p->wrapper &&
+            if ((specific == NULL || specific == d->d_wrapped) &&
+                d->d_base->wrapper == p->wrapper &&
                 PyType_IsSubtype(type, PyDescr_TYPE(d)))
             {
-                if (specific == NULL ||
-                    specific == d->d_wrapped)
-                    specific = d->d_wrapped;
-                else
-                    use_generic = 1;
+                specific = d->d_wrapped;
+            }
+            else {
+                /* We cannot use the specific slot function because either
+                   - it is not unique: there are multiple methods for this
+                     slot and they conflict
+                   - the signature is wrong (as checked by the ->wrapper
+                     comparison above)
+                   - it's wrapping the wrong class
+                 */
+                use_generic = 1;
             }
         }
         else if (Py_TYPE(descr) == &PyCFunction_Type &&


From webhook-mailer at python.org  Tue Sep 10 08:56:18 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 12:56:18 -0000
Subject: [Python-checkins] bpo-38066: Hide internal Stream methods (GH-15762)
Message-ID: <mailman.431.1568120179.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/12c122ae958a55c9874ed4c7d7805ceb084411d7
commit: 12c122ae958a55c9874ed4c7d7805ceb084411d7
branch: master
author: Andrew Svetlov <andrew.svetlov at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-10T05:56:14-07:00
summary:

bpo-38066: Hide internal Stream methods (GH-15762)



feed_eof(), feed_data(), set_exception(), and set_transport() are prefixed with underscore now.


https://bugs.python.org/issue38066

files:
A Misc/NEWS.d/next/Library/2019-09-09-14-39-47.bpo-38066.l9mWv-.rst
M Lib/asyncio/streams.py
M Lib/asyncio/subprocess.py
M Lib/test/test_asyncio/test_pep492.py
M Lib/test/test_asyncio/test_streams.py

diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py
index 33b2fa55673e..4943e8e83e1e 100644
--- a/Lib/asyncio/streams.py
+++ b/Lib/asyncio/streams.py
@@ -1133,9 +1133,9 @@ def connection_lost(self, exc):
         stream = self._stream
         if stream is not None:
             if exc is None:
-                stream.feed_eof()
+                stream._feed_eof()
             else:
-                stream.set_exception(exc)
+                stream._set_exception(exc)
         if not self._closed.done():
             if exc is None:
                 self._closed.set_result(None)
@@ -1147,12 +1147,12 @@ def connection_lost(self, exc):
     def data_received(self, data):
         stream = self._stream
         if stream is not None:
-            stream.feed_data(data)
+            stream._feed_data(data)
 
     def eof_received(self):
         stream = self._stream
         if stream is not None:
-            stream.feed_eof()
+            stream._feed_eof()
         if self._over_ssl:
             # Prevent a warning in SSLProtocol.eof_received:
             # "returning true from eof_received()
@@ -1219,7 +1219,7 @@ def connection_made(self, transport):
         stream = self._stream
         if stream is None:
             return
-        stream.set_transport(transport)
+        stream._set_transport(transport)
         stream._protocol = self
 
     def connection_lost(self, exc):
@@ -1351,6 +1351,11 @@ def is_server_side(self):
 
     @property
     def transport(self):
+        warnings.warn("Stream.transport attribute is deprecated "
+                      "since Python 3.8 and is scheduled for removal in 3.10; "
+                      "it is an internal API",
+                      DeprecationWarning,
+                      stacklevel=2)
         return self._transport
 
     def write(self, data):
@@ -1366,7 +1371,7 @@ def writelines(self, data):
     def _fast_drain(self):
         # The helper tries to use fast-path to return already existing
         # complete future object if underlying transport is not paused
-        #and actual waiting for writing resume is not needed
+        # and actual waiting for writing resume is not needed
         exc = self.exception()
         if exc is not None:
             fut = self._loop.create_future()
@@ -1450,6 +1455,14 @@ def exception(self):
         return self._exception
 
     def set_exception(self, exc):
+        warnings.warn("Stream.set_exception() is deprecated "
+                      "since Python 3.8 and is scheduled for removal in 3.10; "
+                      "it is an internal API",
+                      DeprecationWarning,
+                      stacklevel=2)
+        self._set_exception(exc)
+
+    def _set_exception(self, exc):
         self._exception = exc
 
         waiter = self._waiter
@@ -1467,6 +1480,14 @@ def _wakeup_waiter(self):
                 waiter.set_result(None)
 
     def set_transport(self, transport):
+        warnings.warn("Stream.set_transport() is deprecated "
+                      "since Python 3.8 and is scheduled for removal in 3.10; "
+                      "it is an internal API",
+                      DeprecationWarning,
+                      stacklevel=2)
+        self._set_transport(transport)
+
+    def _set_transport(self, transport):
         if transport is self._transport:
             return
         assert self._transport is None, 'Transport already set'
@@ -1478,6 +1499,14 @@ def _maybe_resume_transport(self):
             self._transport.resume_reading()
 
     def feed_eof(self):
+        warnings.warn("Stream.feed_eof() is deprecated "
+                      "since Python 3.8 and is scheduled for removal in 3.10; "
+                      "it is an internal API",
+                      DeprecationWarning,
+                      stacklevel=2)
+        self._feed_eof()
+
+    def _feed_eof(self):
         self._eof = True
         self._wakeup_waiter()
 
@@ -1486,6 +1515,14 @@ def at_eof(self):
         return self._eof and not self._buffer
 
     def feed_data(self, data):
+        warnings.warn("Stream.feed_data() is deprecated "
+                      "since Python 3.8 and is scheduled for removal in 3.10; "
+                      "it is an internal API",
+                      DeprecationWarning,
+                      stacklevel=2)
+        self._feed_data(data)
+
+    def _feed_data(self, data):
         _ensure_can_read(self._mode)
         assert not self._eof, 'feed_data after feed_eof'
 
diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py
index 2a68c9e4cb06..e4f9e5297a19 100644
--- a/Lib/asyncio/subprocess.py
+++ b/Lib/asyncio/subprocess.py
@@ -50,7 +50,7 @@ def connection_made(self, transport):
                                          limit=self._limit,
                                          loop=self._loop,
                                          _asyncio_internal=True)
-            self.stdout.set_transport(stdout_transport)
+            self.stdout._set_transport(stdout_transport)
             self._pipe_fds.append(1)
 
         stderr_transport = transport.get_pipe_transport(2)
@@ -61,7 +61,7 @@ def connection_made(self, transport):
                                          limit=self._limit,
                                          loop=self._loop,
                                          _asyncio_internal=True)
-            self.stderr.set_transport(stderr_transport)
+            self.stderr._set_transport(stderr_transport)
             self._pipe_fds.append(2)
 
         stdin_transport = transport.get_pipe_transport(0)
@@ -80,7 +80,7 @@ def pipe_data_received(self, fd, data):
         else:
             reader = None
         if reader is not None:
-            reader.feed_data(data)
+            reader._feed_data(data)
 
     def pipe_connection_lost(self, fd, exc):
         if fd == 0:
@@ -101,9 +101,9 @@ def pipe_connection_lost(self, fd, exc):
             reader = None
         if reader is not None:
             if exc is None:
-                reader.feed_eof()
+                reader._feed_eof()
             else:
-                reader.set_exception(exc)
+                reader._set_exception(exc)
 
         if fd in self._pipe_fds:
             self._pipe_fds.remove(fd)
diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py
index f327bbd1d66d..dfb68b8e85d4 100644
--- a/Lib/test/test_asyncio/test_pep492.py
+++ b/Lib/test/test_asyncio/test_pep492.py
@@ -98,8 +98,8 @@ def test_readline(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(DATA)
-        stream.feed_eof()
+        stream._feed_data(DATA)
+        stream._feed_eof()
 
         async def reader():
             data = []
diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py
index eab71e80308f..428c863d9fa6 100644
--- a/Lib/test/test_asyncio/test_streams.py
+++ b/Lib/test/test_asyncio/test_streams.py
@@ -165,7 +165,7 @@ def test_feed_empty_data(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
 
-        stream.feed_data(b'')
+        stream._feed_data(b'')
         self.assertEqual(b'', stream._buffer)
 
     def test_feed_nonempty_data(self):
@@ -173,7 +173,7 @@ def test_feed_nonempty_data(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
 
-        stream.feed_data(self.DATA)
+        stream._feed_data(self.DATA)
         self.assertEqual(self.DATA, stream._buffer)
 
     def test_read_zero(self):
@@ -181,7 +181,7 @@ def test_read_zero(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(self.DATA)
+        stream._feed_data(self.DATA)
 
         data = self.loop.run_until_complete(stream.read(0))
         self.assertEqual(b'', data)
@@ -195,7 +195,7 @@ def test_read(self):
         read_task = asyncio.Task(stream.read(30), loop=self.loop)
 
         def cb():
-            stream.feed_data(self.DATA)
+            stream._feed_data(self.DATA)
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -207,8 +207,8 @@ def test_read_line_breaks(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'line1')
-        stream.feed_data(b'line2')
+        stream._feed_data(b'line1')
+        stream._feed_data(b'line2')
 
         data = self.loop.run_until_complete(stream.read(5))
 
@@ -223,7 +223,7 @@ def test_read_eof(self):
         read_task = asyncio.Task(stream.read(1024), loop=self.loop)
 
         def cb():
-            stream.feed_eof()
+            stream._feed_eof()
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -238,9 +238,9 @@ def test_read_until_eof(self):
         read_task = asyncio.Task(stream.read(-1), loop=self.loop)
 
         def cb():
-            stream.feed_data(b'chunk1\n')
-            stream.feed_data(b'chunk2')
-            stream.feed_eof()
+            stream._feed_data(b'chunk1\n')
+            stream._feed_data(b'chunk2')
+            stream._feed_eof()
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -252,12 +252,12 @@ def test_read_exception(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'line\n')
+        stream._feed_data(b'line\n')
 
         data = self.loop.run_until_complete(stream.read(2))
         self.assertEqual(b'li', data)
 
-        stream.set_exception(ValueError())
+        stream._set_exception(ValueError())
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.read(2))
 
@@ -276,7 +276,7 @@ def test_read_limit(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 limit=3, loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'chunk')
+        stream._feed_data(b'chunk')
         data = self.loop.run_until_complete(stream.read(5))
         self.assertEqual(b'chunk', data)
         self.assertEqual(b'', stream._buffer)
@@ -287,13 +287,13 @@ def test_readline(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'chunk1 ')
+        stream._feed_data(b'chunk1 ')
         read_task = asyncio.Task(stream.readline(), loop=self.loop)
 
         def cb():
-            stream.feed_data(b'chunk2 ')
-            stream.feed_data(b'chunk3 ')
-            stream.feed_data(b'\n chunk4')
+            stream._feed_data(b'chunk2 ')
+            stream._feed_data(b'chunk3 ')
+            stream._feed_data(b'\n chunk4')
         self.loop.call_soon(cb)
 
         line = self.loop.run_until_complete(read_task)
@@ -307,8 +307,8 @@ def test_readline_limit_with_existing_data(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 limit=3, loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'li')
-        stream.feed_data(b'ne1\nline2\n')
+        stream._feed_data(b'li')
+        stream._feed_data(b'ne1\nline2\n')
 
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readline())
@@ -318,9 +318,9 @@ def test_readline_limit_with_existing_data(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 limit=3, loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'li')
-        stream.feed_data(b'ne1')
-        stream.feed_data(b'li')
+        stream._feed_data(b'li')
+        stream._feed_data(b'ne1')
+        stream._feed_data(b'li')
 
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readline())
@@ -337,14 +337,14 @@ def test_at_eof(self):
                                 _asyncio_internal=True)
         self.assertFalse(stream.at_eof())
 
-        stream.feed_data(b'some data\n')
+        stream._feed_data(b'some data\n')
         self.assertFalse(stream.at_eof())
 
         self.loop.run_until_complete(stream.readline())
         self.assertFalse(stream.at_eof())
 
-        stream.feed_data(b'some data\n')
-        stream.feed_eof()
+        stream._feed_data(b'some data\n')
+        stream._feed_eof()
         self.loop.run_until_complete(stream.readline())
         self.assertTrue(stream.at_eof())
 
@@ -356,10 +356,10 @@ def test_readline_limit(self):
                                 limit=7, loop=self.loop,
                                 _asyncio_internal=True)
         def cb():
-            stream.feed_data(b'chunk1')
-            stream.feed_data(b'chunk2')
-            stream.feed_data(b'chunk3\n')
-            stream.feed_eof()
+            stream._feed_data(b'chunk1')
+            stream._feed_data(b'chunk2')
+            stream._feed_data(b'chunk3\n')
+            stream._feed_eof()
         self.loop.call_soon(cb)
 
         self.assertRaises(
@@ -372,10 +372,10 @@ def cb():
                                 limit=7, loop=self.loop,
                                 _asyncio_internal=True)
         def cb():
-            stream.feed_data(b'chunk1')
-            stream.feed_data(b'chunk2\n')
-            stream.feed_data(b'chunk3\n')
-            stream.feed_eof()
+            stream._feed_data(b'chunk1')
+            stream._feed_data(b'chunk2\n')
+            stream._feed_data(b'chunk3\n')
+            stream._feed_eof()
         self.loop.call_soon(cb)
 
         self.assertRaises(
@@ -386,17 +386,17 @@ def cb():
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 limit=7, loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'1234567\n')
+        stream._feed_data(b'1234567\n')
         line = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'1234567\n', line)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'12345678\n')
+        stream._feed_data(b'12345678\n')
         with self.assertRaises(ValueError) as cm:
             self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'12345678')
+        stream._feed_data(b'12345678')
         with self.assertRaises(ValueError) as cm:
             self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'', stream._buffer)
@@ -407,8 +407,8 @@ def test_readline_nolimit_nowait(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(self.DATA[:6])
-        stream.feed_data(self.DATA[6:])
+        stream._feed_data(self.DATA[:6])
+        stream._feed_data(self.DATA[6:])
 
         line = self.loop.run_until_complete(stream.readline())
 
@@ -419,8 +419,8 @@ def test_readline_eof(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'some data')
-        stream.feed_eof()
+        stream._feed_data(b'some data')
+        stream._feed_eof()
 
         line = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'some data', line)
@@ -429,7 +429,7 @@ def test_readline_empty_eof(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_eof()
+        stream._feed_eof()
 
         line = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'', line)
@@ -438,7 +438,7 @@ def test_readline_read_byte_count(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(self.DATA)
+        stream._feed_data(self.DATA)
 
         self.loop.run_until_complete(stream.readline())
 
@@ -451,12 +451,12 @@ def test_readline_exception(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'line\n')
+        stream._feed_data(b'line\n')
 
         data = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'line\n', data)
 
-        stream.set_exception(ValueError())
+        stream._set_exception(ValueError())
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readline())
         self.assertEqual(b'', stream._buffer)
@@ -473,17 +473,17 @@ def test_readuntil_multi_chunks(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
 
-        stream.feed_data(b'lineAAA')
+        stream._feed_data(b'lineAAA')
         data = self.loop.run_until_complete(stream.readuntil(separator=b'AAA'))
         self.assertEqual(b'lineAAA', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'lineAAA')
+        stream._feed_data(b'lineAAA')
         data = self.loop.run_until_complete(stream.readuntil(b'AAA'))
         self.assertEqual(b'lineAAA', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'lineAAAxxx')
+        stream._feed_data(b'lineAAAxxx')
         data = self.loop.run_until_complete(stream.readuntil(b'AAA'))
         self.assertEqual(b'lineAAA', data)
         self.assertEqual(b'xxx', stream._buffer)
@@ -493,34 +493,34 @@ def test_readuntil_multi_chunks_1(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
 
-        stream.feed_data(b'QWEaa')
-        stream.feed_data(b'XYaa')
-        stream.feed_data(b'a')
+        stream._feed_data(b'QWEaa')
+        stream._feed_data(b'XYaa')
+        stream._feed_data(b'a')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'QWEaaXYaaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'QWEaa')
-        stream.feed_data(b'XYa')
-        stream.feed_data(b'aa')
+        stream._feed_data(b'QWEaa')
+        stream._feed_data(b'XYa')
+        stream._feed_data(b'aa')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'QWEaaXYaaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'aaa')
+        stream._feed_data(b'aaa')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'aaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'Xaaa')
+        stream._feed_data(b'Xaaa')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'Xaaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'XXX')
-        stream.feed_data(b'a')
-        stream.feed_data(b'a')
-        stream.feed_data(b'a')
+        stream._feed_data(b'XXX')
+        stream._feed_data(b'a')
+        stream._feed_data(b'a')
+        stream._feed_data(b'a')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'XXXaaa', data)
         self.assertEqual(b'', stream._buffer)
@@ -529,8 +529,8 @@ def test_readuntil_eof(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'some dataAA')
-        stream.feed_eof()
+        stream._feed_data(b'some dataAA')
+        stream._feed_eof()
 
         with self.assertRaises(asyncio.IncompleteReadError) as cm:
             self.loop.run_until_complete(stream.readuntil(b'AAA'))
@@ -542,7 +542,7 @@ def test_readuntil_limit_found_sep(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop, limit=3,
                                 _asyncio_internal=True)
-        stream.feed_data(b'some dataAA')
+        stream._feed_data(b'some dataAA')
 
         with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                     'not found') as cm:
@@ -550,7 +550,7 @@ def test_readuntil_limit_found_sep(self):
 
         self.assertEqual(b'some dataAA', stream._buffer)
 
-        stream.feed_data(b'A')
+        stream._feed_data(b'A')
         with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                     'is found') as cm:
             self.loop.run_until_complete(stream.readuntil(b'AAA'))
@@ -562,7 +562,7 @@ def test_readexactly_zero_or_less(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(self.DATA)
+        stream._feed_data(self.DATA)
 
         data = self.loop.run_until_complete(stream.readexactly(0))
         self.assertEqual(b'', data)
@@ -582,9 +582,9 @@ def test_readexactly(self):
         read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
 
         def cb():
-            stream.feed_data(self.DATA)
-            stream.feed_data(self.DATA)
-            stream.feed_data(self.DATA)
+            stream._feed_data(self.DATA)
+            stream._feed_data(self.DATA)
+            stream._feed_data(self.DATA)
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -595,7 +595,7 @@ def test_readexactly_limit(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 limit=3, loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'chunk')
+        stream._feed_data(b'chunk')
         data = self.loop.run_until_complete(stream.readexactly(5))
         self.assertEqual(b'chunk', data)
         self.assertEqual(b'', stream._buffer)
@@ -609,8 +609,8 @@ def test_readexactly_eof(self):
         read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
 
         def cb():
-            stream.feed_data(self.DATA)
-            stream.feed_eof()
+            stream._feed_data(self.DATA)
+            stream._feed_eof()
         self.loop.call_soon(cb)
 
         with self.assertRaises(asyncio.IncompleteReadError) as cm:
@@ -625,12 +625,12 @@ def test_readexactly_exception(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'line\n')
+        stream._feed_data(b'line\n')
 
         data = self.loop.run_until_complete(stream.readexactly(2))
         self.assertEqual(b'li', data)
 
-        stream.set_exception(ValueError())
+        stream._set_exception(ValueError())
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readexactly(2))
 
@@ -641,7 +641,7 @@ def test_exception(self):
         self.assertIsNone(stream.exception())
 
         exc = ValueError()
-        stream.set_exception(exc)
+        stream._set_exception(exc)
         self.assertIs(stream.exception(), exc)
 
     def test_exception_waiter(self):
@@ -650,7 +650,7 @@ def test_exception_waiter(self):
                                 _asyncio_internal=True)
 
         async def set_err():
-            stream.set_exception(ValueError())
+            stream._set_exception(ValueError())
 
         t1 = asyncio.Task(stream.readline(), loop=self.loop)
         t2 = asyncio.Task(set_err(), loop=self.loop)
@@ -669,7 +669,7 @@ def test_exception_cancel(self):
         t.cancel()
         test_utils.run_briefly(self.loop)
         # The following line fails if set_exception() isn't careful.
-        stream.set_exception(RuntimeError('message'))
+        stream._set_exception(RuntimeError('message'))
         test_utils.run_briefly(self.loop)
         self.assertIs(stream._waiter, None)
 
@@ -993,14 +993,14 @@ def test___repr__eof(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_eof()
+        stream._feed_eof()
         self.assertEqual("<Stream mode=StreamMode.READ eof>", repr(stream))
 
     def test___repr__data(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'data')
+        stream._feed_data(b'data')
         self.assertEqual("<Stream mode=StreamMode.READ 4 bytes>", repr(stream))
 
     def test___repr__exception(self):
@@ -1008,7 +1008,7 @@ def test___repr__exception(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
         exc = RuntimeError()
-        stream.set_exception(exc)
+        stream._set_exception(exc)
         self.assertEqual("<Stream mode=StreamMode.READ exception=RuntimeError()>",
                          repr(stream))
 
@@ -1260,7 +1260,7 @@ def test_stream_writer_forbidden_ops(self):
             stream = asyncio.Stream(mode=asyncio.StreamMode.WRITE,
                                     _asyncio_internal=True)
             with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                stream.feed_data(b'data')
+                stream._feed_data(b'data')
             with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
                 await stream.readline()
             with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
@@ -1785,6 +1785,30 @@ def test_stream_ctor_forbidden(self):
                                     "by asyncio internals only"):
             asyncio.Stream(asyncio.StreamMode.READWRITE)
 
+    def test_deprecated_methods(self):
+        async def f():
+            return asyncio.Stream(mode=asyncio.StreamMode.READWRITE,
+                                  _asyncio_internal=True)
+
+        stream = self.loop.run_until_complete(f())
+
+        tr = mock.Mock()
+
+        with self.assertWarns(DeprecationWarning):
+            stream.set_transport(tr)
+
+        with self.assertWarns(DeprecationWarning):
+            stream.transport is tr
+
+        with self.assertWarns(DeprecationWarning):
+            stream.feed_data(b'data')
+
+        with self.assertWarns(DeprecationWarning):
+            stream.feed_eof()
+
+        with self.assertWarns(DeprecationWarning):
+            stream.set_exception(ConnectionResetError("test"))
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Misc/NEWS.d/next/Library/2019-09-09-14-39-47.bpo-38066.l9mWv-.rst b/Misc/NEWS.d/next/Library/2019-09-09-14-39-47.bpo-38066.l9mWv-.rst
new file mode 100644
index 000000000000..a69924fe2aa4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-09-14-39-47.bpo-38066.l9mWv-.rst
@@ -0,0 +1,2 @@
+Hide internal asyncio.Stream methods: feed_eof(), feed_data(),
+set_exception() and set_transport().


From webhook-mailer at python.org  Tue Sep 10 08:59:55 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 12:59:55 -0000
Subject: [Python-checkins] bpo-38073: Make pwd module PEP-384 compatible
 (GH-15790)
Message-ID: <mailman.432.1568120396.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b7f8e52433b656a6a524229fd7a65e4682a43729
commit: b7f8e52433b656a6a524229fd7a65e4682a43729
branch: master
author: Dino Viehland <dinoviehland at fb.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-10T05:59:43-07:00
summary:

bpo-38073: Make pwd module PEP-384 compatible (GH-15790)



Makes the pwd module PEP-384 compatible


https://bugs.python.org/issue38073



Automerge-Triggered-By: @tiran

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-17-58.bpo-38073.ZoKYOU.rst
M Modules/pwdmodule.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-17-58.bpo-38073.ZoKYOU.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-17-58.bpo-38073.ZoKYOU.rst
new file mode 100644
index 000000000000..d59630f88808
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-17-58.bpo-38073.ZoKYOU.rst	
@@ -0,0 +1 @@
+Make pwd extension module PEP-384 compatible
diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c
index e0232b8d589b..b5ef2557db77 100644
--- a/Modules/pwdmodule.c
+++ b/Modules/pwdmodule.c
@@ -47,8 +47,13 @@ The uid and gid items are integers, all others are strings. An\n\
 exception is raised if the entry asked for cannot be found.");
 
 
-static int initialized;
-static PyTypeObject StructPwdType;
+typedef struct {
+    PyTypeObject *StructPwdType;
+} pwdmodulestate;
+#define modulestate(o) ((pwdmodulestate *)PyModule_GetState(o))
+#define modulestate_global modulestate(PyState_FindModule(&pwdmodule))
+
+static struct PyModuleDef pwdmodule;
 
 #define DEFAULT_BUFFER_SIZE 1024
 
@@ -69,7 +74,7 @@ static PyObject *
 mkpwent(struct passwd *p)
 {
     int setIndex = 0;
-    PyObject *v = PyStructSequence_New(&StructPwdType);
+    PyObject *v = PyStructSequence_New(modulestate_global->StructPwdType);
     if (v == NULL)
         return NULL;
 
@@ -310,16 +315,28 @@ static PyMethodDef pwd_methods[] = {
     {NULL,              NULL}           /* sentinel */
 };
 
+static int pwdmodule_traverse(PyObject *m, visitproc visit, void *arg) {
+    Py_VISIT(modulestate(m)->StructPwdType);
+    return 0;
+}
+static int pwdmodule_clear(PyObject *m) {
+    Py_CLEAR(modulestate(m)->StructPwdType);
+    return 0;
+}
+static void pwdmodule_free(void *m) {
+    pwdmodule_clear((PyObject *)m);
+}
+
 static struct PyModuleDef pwdmodule = {
     PyModuleDef_HEAD_INIT,
     "pwd",
     pwd__doc__,
-    -1,
+    sizeof(pwdmodulestate),
     pwd_methods,
     NULL,
-    NULL,
-    NULL,
-    NULL
+    pwdmodule_traverse,
+    pwdmodule_clear,
+    pwdmodule_free,
 };
 
 
@@ -327,17 +344,19 @@ PyMODINIT_FUNC
 PyInit_pwd(void)
 {
     PyObject *m;
-    m = PyModule_Create(&pwdmodule);
-    if (m == NULL)
+    if ((m = PyState_FindModule(&pwdmodule)) != NULL) {
+        Py_INCREF(m);
+        return m;
+    }
+    if ((m = PyModule_Create(&pwdmodule)) == NULL)
         return NULL;
 
-    if (!initialized) {
-        if (PyStructSequence_InitType2(&StructPwdType,
-                                       &struct_pwd_type_desc) < 0)
-            return NULL;
-        initialized = 1;
+    pwdmodulestate *state = PyModule_GetState(m);
+    state->StructPwdType = PyStructSequence_NewType(&struct_pwd_type_desc);
+    if (state->StructPwdType == NULL) {
+        return NULL;
     }
-    Py_INCREF((PyObject *) &StructPwdType);
-    PyModule_AddObject(m, "struct_passwd", (PyObject *) &StructPwdType);
+    Py_INCREF(state->StructPwdType);
+    PyModule_AddObject(m, "struct_passwd", (PyObject *) state->StructPwdType);
     return m;
 }


From webhook-mailer at python.org  Tue Sep 10 09:04:26 2019
From: webhook-mailer at python.org (Dino Viehland)
Date: Tue, 10 Sep 2019 13:04:26 -0000
Subject: [Python-checkins] Note regarding + mode truncation applies to both
 text and binary mode (#11314)
Message-ID: <mailman.433.1568120667.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c1d8c1cb8e90a54a3daaa7fcdb8d6ca7f08d6a73
commit: c1d8c1cb8e90a54a3daaa7fcdb8d6ca7f08d6a73
branch: master
author: Andre Delfino <adelfino at gmail.com>
committer: Dino Viehland <dinoviehland at gmail.com>
date: 2019-09-10T14:04:22+01:00
summary:

 Note regarding + mode truncation applies to both text and binary mode (#11314)

* Improve doc on open's mode +

* Improve wording

* Address comment from R?mi

files:
M Doc/library/functions.rst

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index ce026ebb4b0f..98a55c61eff2 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -1070,12 +1070,12 @@ are always available.  They are listed here in alphabetical order.
    ``'a'``   open for writing, appending to the end of the file if it exists
    ``'b'``   binary mode
    ``'t'``   text mode (default)
-   ``'+'``   open a disk file for updating (reading and writing)
+   ``'+'``   open for updating (reading and writing)
    ========= ===============================================================
 
    The default mode is ``'r'`` (open for reading text, synonym of ``'rt'``).
-   For binary read-write access, the mode ``'w+b'`` opens and truncates the file
-   to 0 bytes.  ``'r+b'`` opens the file without truncation.
+   Modes ``'w+'`` and ``'w+b'`` opens and truncates the file.  Modes ``'r+'``
+   and ``'r+b'`` opens the file with no truncation.
 
    As mentioned in the :ref:`io-overview`, Python distinguishes between binary
    and text I/O.  Files opened in binary mode (including ``'b'`` in the *mode*


From webhook-mailer at python.org  Tue Sep 10 09:15:23 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 13:15:23 -0000
Subject: [Python-checkins] docs: Add references to AsyncMock in
 unittest.mock.patch (GH-13681)
Message-ID: <mailman.434.1568121324.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/eaa1b09412cc82ba830fd731016278af26b9099b
commit: eaa1b09412cc82ba830fd731016278af26b9099b
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T06:15:19-07:00
summary:

docs: Add references to AsyncMock in unittest.mock.patch (GH-13681)


Update the docs as patch can now return an AsyncMock if the patched
object is an async function.
(cherry picked from commit f5e7f39d2916ed150e80381faed125f405a11e11)

Co-authored-by: Mario Corchero <mcorcherojim at bloomberg.net>

files:
M Doc/library/unittest.mock.rst
M Lib/unittest/mock.py

diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index b2547546f3d6..04ff8a61da3c 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -1321,8 +1321,10 @@ patch
     is patched with a *new* object. When the function/with statement exits
     the patch is undone.
 
-    If *new* is omitted, then the target is replaced with a
-    :class:`MagicMock`. If :func:`patch` is used as a decorator and *new* is
+    If *new* is omitted, then the target is replaced with an
+    :class:`AsyncMock` if the patched object is an async function or
+    a :class:`MagicMock` otherwise.
+    If :func:`patch` is used as a decorator and *new* is
     omitted, the created mock is passed in as an extra argument to the
     decorated function. If :func:`patch` is used as a context manager the created
     mock is returned by the context manager.
@@ -1340,8 +1342,8 @@ patch
     patch to pass in the object being mocked as the spec/spec_set object.
 
     *new_callable* allows you to specify a different class, or callable object,
-    that will be called to create the *new* object. By default :class:`MagicMock` is
-    used.
+    that will be called to create the *new* object. By default :class:`AsyncMock`
+    is used for async functions and :class:`MagicMock` for the rest.
 
     A more powerful form of *spec* is *autospec*. If you set ``autospec=True``
     then the mock will be created with a spec from the object being replaced.
@@ -1505,6 +1507,10 @@ work as expected::
     ...
     >>> test()
 
+.. versionchanged:: 3.8
+
+    :func:`patch` now returns an :class:`AsyncMock` if the target is an async function.
+
 
 patch.object
 ~~~~~~~~~~~~
@@ -2289,6 +2295,12 @@ See :ref:`auto-speccing` for examples of how to use auto-speccing with
 :func:`create_autospec` and the *autospec* argument to :func:`patch`.
 
 
+.. versionchanged:: 3.8
+
+    :func:`create_autospec` now returns an :class:`AsyncMock` if the target is
+    an async function.
+
+
 ANY
 ~~~
 
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 7a4fcf4e3aa9..915883db8802 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -1631,8 +1631,9 @@ def patch(
     is patched with a `new` object. When the function/with statement exits
     the patch is undone.
 
-    If `new` is omitted, then the target is replaced with a
-    `MagicMock`. If `patch` is used as a decorator and `new` is
+    If `new` is omitted, then the target is replaced with an
+    `AsyncMock if the patched object is an async function or a
+    `MagicMock` otherwise. If `patch` is used as a decorator and `new` is
     omitted, the created mock is passed in as an extra argument to the
     decorated function. If `patch` is used as a context manager the created
     mock is returned by the context manager.
@@ -1650,8 +1651,8 @@ def patch(
     patch to pass in the object being mocked as the spec/spec_set object.
 
     `new_callable` allows you to specify a different class, or callable object,
-    that will be called to create the `new` object. By default `MagicMock` is
-    used.
+    that will be called to create the `new` object. By default `AsyncMock` is
+    used for async functions and `MagicMock` for the rest.
 
     A more powerful form of `spec` is `autospec`. If you set `autospec=True`
     then the mock will be created with a spec from the object being replaced.


From webhook-mailer at python.org  Tue Sep 10 09:16:04 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 13:16:04 -0000
Subject: [Python-checkins] bpo-37251: Removes __code__ check from
 _is_async_obj. (GH-15830)
Message-ID: <mailman.435.1568121364.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c3008dd480d645678409273eecbfd24bbc9669d7
commit: c3008dd480d645678409273eecbfd24bbc9669d7
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T06:16:00-07:00
summary:

bpo-37251: Removes __code__ check from _is_async_obj. (GH-15830)

(cherry picked from commit f1a297acb60b88917712450ebd3cfa707e6efd6b)

Co-authored-by: Lisa Roach <lisaroach14 at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-10-10-59-50.bpo-37251.8zn2o3.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 915883db8802..7bfbfc92eef9 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -46,10 +46,9 @@
 _safe_super = super
 
 def _is_async_obj(obj):
-    if getattr(obj, '__code__', None):
-        return asyncio.iscoroutinefunction(obj) or inspect.isawaitable(obj)
-    else:
+    if _is_instance_mock(obj) and not isinstance(obj, AsyncMock):
         return False
+    return asyncio.iscoroutinefunction(obj) or inspect.isawaitable(obj)
 
 
 def _is_async_func(func):
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index fa906e4f7152..4660bea3e123 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -18,6 +18,10 @@ def __init__(self):
     def normal_method(self):
         pass
 
+class AwaitableClass:
+    def __await__(self):
+        yield
+
 async def async_func():
     pass
 
@@ -160,6 +164,10 @@ def test_create_autospec_instance(self):
         with self.assertRaises(RuntimeError):
             create_autospec(async_func, instance=True)
 
+    def test_create_autospec_awaitable_class(self):
+        awaitable_mock = create_autospec(spec=AwaitableClass())
+        self.assertIsInstance(create_autospec(awaitable_mock), AsyncMock)
+
     def test_create_autospec(self):
         spec = create_autospec(async_func_args)
         awaitable = spec(1, 2, c=3)
@@ -321,6 +329,13 @@ def test_is_child_AsyncMock(self):
         self.assertIsInstance(mock.normal_method, MagicMock)
         self.assertIsInstance(mock, MagicMock)
 
+    def test_magicmock_lambda_spec(self):
+        mock_obj = MagicMock()
+        mock_obj.mock_func = MagicMock(spec=lambda x: x)
+
+        with patch.object(mock_obj, "mock_func") as cm:
+            self.assertIsInstance(cm, MagicMock)
+
 
 class AsyncArguments(unittest.TestCase):
     def test_add_return_value(self):
diff --git a/Misc/NEWS.d/next/Library/2019-09-10-10-59-50.bpo-37251.8zn2o3.rst b/Misc/NEWS.d/next/Library/2019-09-10-10-59-50.bpo-37251.8zn2o3.rst
new file mode 100644
index 000000000000..27fd1e46963c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-10-10-59-50.bpo-37251.8zn2o3.rst
@@ -0,0 +1,3 @@
+Remove `__code__` check in AsyncMock that incorrectly
+evaluated function specs as async objects but failed to evaluate classes
+with `__await__` but no `__code__` attribute defined as async objects.


From webhook-mailer at python.org  Tue Sep 10 09:22:01 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 13:22:01 -0000
Subject: [Python-checkins] closes bpo-25461: Update os.walk() docstring to
 match the online docs. (GH-11836)
Message-ID: <mailman.436.1568121722.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c43f26eca35f22707a52fb8f3fbfc9340639b58d
commit: c43f26eca35f22707a52fb8f3fbfc9340639b58d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T06:21:57-07:00
summary:

closes bpo-25461: Update os.walk() docstring to match the online docs. (GH-11836)

(cherry picked from commit 734f1202a50641eb2c4bfbcd5b75247c1dc99a8f)

Co-authored-by: Bernt R?skar Brenna <bernt.brenna at gmail.com>

files:
M Lib/os.py

diff --git a/Lib/os.py b/Lib/os.py
index 200902528cbf..52d3f1d74158 100644
--- a/Lib/os.py
+++ b/Lib/os.py
@@ -300,10 +300,11 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
     (e.g., via del or slice assignment), and walk will only recurse into the
     subdirectories whose names remain in dirnames; this can be used to prune the
     search, or to impose a specific order of visiting.  Modifying dirnames when
-    topdown is false is ineffective, since the directories in dirnames have
-    already been generated by the time dirnames itself is generated. No matter
-    the value of topdown, the list of subdirectories is retrieved before the
-    tuples for the directory and its subdirectories are generated.
+    topdown is false has no effect on the behavior of os.walk(), since the
+    directories in dirnames have already been generated by the time dirnames
+    itself is generated. No matter the value of topdown, the list of
+    subdirectories is retrieved before the tuples for the directory and its
+    subdirectories are generated.
 
     By default errors from the os.scandir() call are ignored.  If
     optional arg 'onerror' is specified, it should be a function; it


From webhook-mailer at python.org  Tue Sep 10 09:30:22 2019
From: webhook-mailer at python.org (Ned Deily)
Date: Tue, 10 Sep 2019 13:30:22 -0000
Subject: [Python-checkins] bpo-34720: Fix test_importlib.test_bad_traverse
 for AIX (GH-14238)
Message-ID: <mailman.437.1568122222.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/61f34f9f63aab7503420603627ad5bd8902d904b
commit: 61f34f9f63aab7503420603627ad5bd8902d904b
branch: 3.7
author: Michael Felt <aixtools at users.noreply.github.com>
committer: Ned Deily <nad at python.org>
date: 2019-09-10T14:30:18+01:00
summary:

bpo-34720: Fix test_importlib.test_bad_traverse for AIX  (GH-14238)

files:
A Misc/NEWS.d/next/Tests/2018-12-26-12-31-16.bpo-34720.T268vz.rst
M Modules/_testmultiphase.c

diff --git a/Misc/NEWS.d/next/Tests/2018-12-26-12-31-16.bpo-34720.T268vz.rst b/Misc/NEWS.d/next/Tests/2018-12-26-12-31-16.bpo-34720.T268vz.rst
new file mode 100644
index 000000000000..fc490285db3c
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2018-12-26-12-31-16.bpo-34720.T268vz.rst
@@ -0,0 +1,2 @@
+Assert m_state != NULL to mimic GC traversal functions that do not correctly
+handle module creation when the module state has not been created.
diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c
index 5776df7d765d..c6dfc2f01dc0 100644
--- a/Modules/_testmultiphase.c
+++ b/Modules/_testmultiphase.c
@@ -624,6 +624,14 @@ bad_traverse(PyObject *self, visitproc visit, void *arg) {
     testmultiphase_state *m_state;
 
     m_state = PyModule_GetState(self);
+
+    /* The following assertion mimics any traversal function that doesn't correctly handle
+     * the case during module creation where the module state hasn't been created yet.
+     *
+     * The check that it is used to test only runs in debug mode, so it is OK that the
+     * assert() will get compiled out in fully optimised release builds.
+     */
+    assert(m_state != NULL);
     Py_VISIT(m_state->integer);
     return 0;
 }


From webhook-mailer at python.org  Tue Sep 10 09:31:05 2019
From: webhook-mailer at python.org (Dino Viehland)
Date: Tue, 10 Sep 2019 13:31:05 -0000
Subject: [Python-checkins] bpo-36781: Optimize sum() for bools. (#13074)
Message-ID: <mailman.438.1568122266.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/88bdb9280b251d753f1b1c8d9183de0fff003622
commit: 88bdb9280b251d753f1b1c8d9183de0fff003622
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: Dino Viehland <dinoviehland at gmail.com>
date: 2019-09-10T14:31:01+01:00
summary:

bpo-36781: Optimize sum() for bools. (#13074)

* Optimize sum() for bools.

* Fix sum([], False).

* Add a NEWS entry.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-05-06-09-59-31.bpo-36781.uocWt6.rst
M Lib/test/test_builtin.py
M Python/bltinmodule.c

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index db2f6cd49b2c..7b3c054d5eb9 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -1373,6 +1373,24 @@ def test_sum(self):
 
         self.assertEqual(sum(range(10), 1000), 1045)
         self.assertEqual(sum(range(10), start=1000), 1045)
+        self.assertEqual(sum(range(10), 2**31-5), 2**31+40)
+        self.assertEqual(sum(range(10), 2**63-5), 2**63+40)
+
+        self.assertEqual(sum(i % 2 != 0 for i in range(10)), 5)
+        self.assertEqual(sum((i % 2 != 0 for i in range(10)), 2**31-3),
+                         2**31+2)
+        self.assertEqual(sum((i % 2 != 0 for i in range(10)), 2**63-3),
+                         2**63+2)
+        self.assertIs(sum([], False), False)
+
+        self.assertEqual(sum(i / 2 for i in range(10)), 22.5)
+        self.assertEqual(sum((i / 2 for i in range(10)), 1000), 1022.5)
+        self.assertEqual(sum((i / 2 for i in range(10)), 1000.25), 1022.75)
+        self.assertEqual(sum([0.5, 1]), 1.5)
+        self.assertEqual(sum([1, 0.5]), 1.5)
+        self.assertEqual(repr(sum([-0.0])), '0.0')
+        self.assertEqual(repr(sum([-0.0], -0.0)), '-0.0')
+        self.assertEqual(repr(sum([], -0.0)), '-0.0')
 
         self.assertRaises(TypeError, sum)
         self.assertRaises(TypeError, sum, 42)
@@ -1384,6 +1402,9 @@ def test_sum(self):
         self.assertRaises(TypeError, sum, [[1], [2], [3]])
         self.assertRaises(TypeError, sum, [{2:3}])
         self.assertRaises(TypeError, sum, [{2:3}]*2, {2:3})
+        self.assertRaises(TypeError, sum, [], '')
+        self.assertRaises(TypeError, sum, [], b'')
+        self.assertRaises(TypeError, sum, [], bytearray())
 
         class BadSeq:
             def __getitem__(self, index):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-06-09-59-31.bpo-36781.uocWt6.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-06-09-59-31.bpo-36781.uocWt6.rst
new file mode 100644
index 000000000000..2e9dcdcb9034
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-06-09-59-31.bpo-36781.uocWt6.rst	
@@ -0,0 +1 @@
+:func:`sum` has been optimized for boolean values.
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 3b0d64ff14ab..d069f2fd2610 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -2342,7 +2342,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start)
                     return NULL;
                 return PyLong_FromLong(i_result);
             }
-            if (PyLong_CheckExact(item)) {
+            if (PyLong_CheckExact(item) || PyBool_Check(item)) {
                 long b = PyLong_AsLongAndOverflow(item, &overflow);
                 if (overflow == 0 &&
                     (i_result >= 0 ? (b <= LONG_MAX - i_result)
@@ -2390,7 +2390,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start)
                 Py_DECREF(item);
                 continue;
             }
-            if (PyLong_CheckExact(item)) {
+            if (PyLong_Check(item)) {
                 long value;
                 int overflow;
                 value = PyLong_AsLongAndOverflow(item, &overflow);


From webhook-mailer at python.org  Tue Sep 10 09:32:47 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Tue, 10 Sep 2019 13:32:47 -0000
Subject: [Python-checkins] bpo-21018: added missing documentation about
 escaping characters for configparser (GH-6137) (GH-15845)
Message-ID: <mailman.439.1568122368.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/122bbf7f8ac2cef4368e3601af593a027dc62ccf
commit: 122bbf7f8ac2cef4368e3601af593a027dc62ccf
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-10T15:32:42+02:00
summary:

bpo-21018: added missing documentation about escaping characters for configparser (GH-6137) (GH-15845)

Document how $ and % can be escaped in configparser.
(cherry picked from commit 9a94093189417adddd6b59d6c80cc5544630c8aa)

Co-authored-by: Arun Persaud <arun at nubati.net>

files:
M Doc/library/configparser.rst

diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst
index 68b663ff7f65..6ab858578ba2 100644
--- a/Doc/library/configparser.rst
+++ b/Doc/library/configparser.rst
@@ -313,6 +313,8 @@ from ``get()`` calls.
       my_dir: %(home_dir)s/lumberjack
       my_pictures: %(my_dir)s/Pictures
 
+      [Escape]
+      gain: 80%%  # use a %% to escape the % sign (% is the only character that needs to be escaped)
 
    In the example above, :class:`ConfigParser` with *interpolation* set to
    ``BasicInterpolation()`` would resolve ``%(home_dir)s`` to the value of
@@ -346,6 +348,9 @@ from ``get()`` calls.
       my_dir: ${home_dir}/lumberjack
       my_pictures: ${my_dir}/Pictures
 
+      [Escape]
+      cost: $$80  # use a $$ to escape the $ sign ($ is the only character that needs to be escaped)
+
    Values from other sections can be fetched as well:
 
    .. code-block:: ini


From webhook-mailer at python.org  Tue Sep 10 09:32:59 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 13:32:59 -0000
Subject: [Python-checkins] bpo-38090: Fix reference leak in ceval.c
 (GH-15848)
Message-ID: <mailman.440.1568122381.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a511c7a4961a684db1f8d0ed438822d87d7d3dcf
commit: a511c7a4961a684db1f8d0ed438822d87d7d3dcf
branch: master
author: Pablo Galindo <Pablogsal at gmail.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-10T14:32:54+01:00
summary:

bpo-38090: Fix reference leak in ceval.c (GH-15848)

files:
M Python/ceval.c

diff --git a/Python/ceval.c b/Python/ceval.c
index d280d7977607..01e2a1b9666f 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -5235,7 +5235,6 @@ import_from(PyThreadState *tstate, PyObject *v, PyObject *name)
     else {
         _Py_IDENTIFIER(__spec__);
         PyObject *spec = _PyObject_GetAttrId(v, &PyId___spec__);
-        Py_XINCREF(spec);
         const char *fmt =
             _PyModuleSpec_IsInitializing(spec) ?
             "cannot import name %R from partially initialized module %R "


From webhook-mailer at python.org  Tue Sep 10 09:33:00 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Tue, 10 Sep 2019 13:33:00 -0000
Subject: [Python-checkins] bpo-21018: added missing documentation about
 escaping characters for configparser (GH-6137) (GH-15846)
Message-ID: <mailman.441.1568122381.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5cf8155bbd2c65572295b26e96c221763b3322f0
commit: 5cf8155bbd2c65572295b26e96c221763b3322f0
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-10T15:32:56+02:00
summary:

bpo-21018: added missing documentation about escaping characters for configparser (GH-6137) (GH-15846)

Document how $ and % can be escaped in configparser.
(cherry picked from commit 9a94093189417adddd6b59d6c80cc5544630c8aa)

Co-authored-by: Arun Persaud <arun at nubati.net>

files:
M Doc/library/configparser.rst

diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst
index 04b52dc7b215..739477f55fdd 100644
--- a/Doc/library/configparser.rst
+++ b/Doc/library/configparser.rst
@@ -313,6 +313,8 @@ from ``get()`` calls.
       my_dir: %(home_dir)s/lumberjack
       my_pictures: %(my_dir)s/Pictures
 
+      [Escape]
+      gain: 80%%  # use a %% to escape the % sign (% is the only character that needs to be escaped)
 
    In the example above, :class:`ConfigParser` with *interpolation* set to
    ``BasicInterpolation()`` would resolve ``%(home_dir)s`` to the value of
@@ -346,6 +348,9 @@ from ``get()`` calls.
       my_dir: ${home_dir}/lumberjack
       my_pictures: ${my_dir}/Pictures
 
+      [Escape]
+      cost: $$80  # use a $$ to escape the $ sign ($ is the only character that needs to be escaped)
+
    Values from other sections can be fetched as well:
 
    .. code-block:: ini


From webhook-mailer at python.org  Tue Sep 10 09:34:07 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 13:34:07 -0000
Subject: [Python-checkins] closes bpo-25461: Update os.walk() docstring to
 match the online docs. (GH-11836)
Message-ID: <mailman.442.1568122448.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d94b762ce824e97c441f9231f0e69ef8f59beeab
commit: d94b762ce824e97c441f9231f0e69ef8f59beeab
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T06:34:03-07:00
summary:

closes bpo-25461: Update os.walk() docstring to match the online docs. (GH-11836)

(cherry picked from commit 734f1202a50641eb2c4bfbcd5b75247c1dc99a8f)

Co-authored-by: Bernt R?skar Brenna <bernt.brenna at gmail.com>

files:
M Lib/os.py

diff --git a/Lib/os.py b/Lib/os.py
index 41896cd11bfd..b93f95d98ede 100644
--- a/Lib/os.py
+++ b/Lib/os.py
@@ -300,10 +300,11 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
     (e.g., via del or slice assignment), and walk will only recurse into the
     subdirectories whose names remain in dirnames; this can be used to prune the
     search, or to impose a specific order of visiting.  Modifying dirnames when
-    topdown is false is ineffective, since the directories in dirnames have
-    already been generated by the time dirnames itself is generated. No matter
-    the value of topdown, the list of subdirectories is retrieved before the
-    tuples for the directory and its subdirectories are generated.
+    topdown is false has no effect on the behavior of os.walk(), since the
+    directories in dirnames have already been generated by the time dirnames
+    itself is generated. No matter the value of topdown, the list of
+    subdirectories is retrieved before the tuples for the directory and its
+    subdirectories are generated.
 
     By default errors from the os.scandir() call are ignored.  If
     optional arg 'onerror' is specified, it should be a function; it


From webhook-mailer at python.org  Tue Sep 10 09:40:18 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 13:40:18 -0000
Subject: [Python-checkins] Fix typo in ssl.RAND_bytes documentation
 (GH-14791)
Message-ID: <mailman.443.1568122818.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c2f056be82a433ff1fe756c1a463c91633c230ab
commit: c2f056be82a433ff1fe756c1a463c91633c230ab
branch: master
author: Zach Thompson <zthompson47 at users.noreply.github.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-10T06:40:14-07:00
summary:

Fix typo in ssl.RAND_bytes documentation (GH-14791)



It looks like "cryptographically strong" is the preferred phrase from the surrounding documentation.

Automerge-Triggered-By: @tiran

files:
M Doc/library/ssl.rst

diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index 279af5728913..2db48ae841da 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -293,7 +293,7 @@ Random generation
    Read the Wikipedia article, `Cryptographically secure pseudorandom number
    generator (CSPRNG)
    <https://en.wikipedia.org/wiki/Cryptographically_secure_pseudorandom_number_generator>`_,
-   to get the requirements of a cryptographically generator.
+   to get the requirements of a cryptographically strong generator.
 
    .. versionadded:: 3.3
 


From webhook-mailer at python.org  Tue Sep 10 09:44:39 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 13:44:39 -0000
Subject: [Python-checkins] bpo-38066: Hide internal Stream methods (GH-15762)
Message-ID: <mailman.444.1568123079.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f12ff05bc07ac087743a4615c4af5f66b73f8d2c
commit: f12ff05bc07ac087743a4615c4af5f66b73f8d2c
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T06:44:32-07:00
summary:

bpo-38066: Hide internal Stream methods (GH-15762)


feed_eof(), feed_data(), set_exception(), and set_transport() are prefixed with underscore now.

https://bugs.python.org/issue38066
(cherry picked from commit 12c122ae958a55c9874ed4c7d7805ceb084411d7)

Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-09-14-39-47.bpo-38066.l9mWv-.rst
M Lib/asyncio/streams.py
M Lib/asyncio/subprocess.py
M Lib/test/test_asyncio/test_pep492.py
M Lib/test/test_asyncio/test_streams.py

diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py
index 1d3e487b1d93..01aa6ffd49e4 100644
--- a/Lib/asyncio/streams.py
+++ b/Lib/asyncio/streams.py
@@ -1133,9 +1133,9 @@ def connection_lost(self, exc):
         stream = self._stream
         if stream is not None:
             if exc is None:
-                stream.feed_eof()
+                stream._feed_eof()
             else:
-                stream.set_exception(exc)
+                stream._set_exception(exc)
         if not self._closed.done():
             if exc is None:
                 self._closed.set_result(None)
@@ -1147,12 +1147,12 @@ def connection_lost(self, exc):
     def data_received(self, data):
         stream = self._stream
         if stream is not None:
-            stream.feed_data(data)
+            stream._feed_data(data)
 
     def eof_received(self):
         stream = self._stream
         if stream is not None:
-            stream.feed_eof()
+            stream._feed_eof()
         if self._over_ssl:
             # Prevent a warning in SSLProtocol.eof_received:
             # "returning true from eof_received()
@@ -1219,7 +1219,7 @@ def connection_made(self, transport):
         stream = self._stream
         if stream is None:
             return
-        stream.set_transport(transport)
+        stream._set_transport(transport)
         stream._protocol = self
 
     def connection_lost(self, exc):
@@ -1351,6 +1351,11 @@ def is_server_side(self):
 
     @property
     def transport(self):
+        warnings.warn("Stream.transport attribute is deprecated "
+                      "since Python 3.8 and is scheduled for removal in 3.10; "
+                      "it is an internal API",
+                      DeprecationWarning,
+                      stacklevel=2)
         return self._transport
 
     def write(self, data):
@@ -1366,7 +1371,7 @@ def writelines(self, data):
     def _fast_drain(self):
         # The helper tries to use fast-path to return already existing
         # complete future object if underlying transport is not paused
-        #and actual waiting for writing resume is not needed
+        # and actual waiting for writing resume is not needed
         exc = self.exception()
         if exc is not None:
             fut = self._loop.create_future()
@@ -1450,6 +1455,14 @@ def exception(self):
         return self._exception
 
     def set_exception(self, exc):
+        warnings.warn("Stream.set_exception() is deprecated "
+                      "since Python 3.8 and is scheduled for removal in 3.10; "
+                      "it is an internal API",
+                      DeprecationWarning,
+                      stacklevel=2)
+        self._set_exception(exc)
+
+    def _set_exception(self, exc):
         self._exception = exc
 
         waiter = self._waiter
@@ -1467,6 +1480,14 @@ def _wakeup_waiter(self):
                 waiter.set_result(None)
 
     def set_transport(self, transport):
+        warnings.warn("Stream.set_transport() is deprecated "
+                      "since Python 3.8 and is scheduled for removal in 3.10; "
+                      "it is an internal API",
+                      DeprecationWarning,
+                      stacklevel=2)
+        self._set_transport(transport)
+
+    def _set_transport(self, transport):
         if transport is self._transport:
             return
         assert self._transport is None, 'Transport already set'
@@ -1478,6 +1499,14 @@ def _maybe_resume_transport(self):
             self._transport.resume_reading()
 
     def feed_eof(self):
+        warnings.warn("Stream.feed_eof() is deprecated "
+                      "since Python 3.8 and is scheduled for removal in 3.10; "
+                      "it is an internal API",
+                      DeprecationWarning,
+                      stacklevel=2)
+        self._feed_eof()
+
+    def _feed_eof(self):
         self._eof = True
         self._wakeup_waiter()
 
@@ -1486,6 +1515,14 @@ def at_eof(self):
         return self._eof and not self._buffer
 
     def feed_data(self, data):
+        warnings.warn("Stream.feed_data() is deprecated "
+                      "since Python 3.8 and is scheduled for removal in 3.10; "
+                      "it is an internal API",
+                      DeprecationWarning,
+                      stacklevel=2)
+        self._feed_data(data)
+
+    def _feed_data(self, data):
         _ensure_can_read(self._mode)
         assert not self._eof, 'feed_data after feed_eof'
 
diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py
index e6bec71d6c7d..e6c1c8b61cc3 100644
--- a/Lib/asyncio/subprocess.py
+++ b/Lib/asyncio/subprocess.py
@@ -50,7 +50,7 @@ def connection_made(self, transport):
                                          limit=self._limit,
                                          loop=self._loop,
                                          _asyncio_internal=True)
-            self.stdout.set_transport(stdout_transport)
+            self.stdout._set_transport(stdout_transport)
             self._pipe_fds.append(1)
 
         stderr_transport = transport.get_pipe_transport(2)
@@ -61,7 +61,7 @@ def connection_made(self, transport):
                                          limit=self._limit,
                                          loop=self._loop,
                                          _asyncio_internal=True)
-            self.stderr.set_transport(stderr_transport)
+            self.stderr._set_transport(stderr_transport)
             self._pipe_fds.append(2)
 
         stdin_transport = transport.get_pipe_transport(0)
@@ -80,7 +80,7 @@ def pipe_data_received(self, fd, data):
         else:
             reader = None
         if reader is not None:
-            reader.feed_data(data)
+            reader._feed_data(data)
 
     def pipe_connection_lost(self, fd, exc):
         if fd == 0:
@@ -101,9 +101,9 @@ def pipe_connection_lost(self, fd, exc):
             reader = None
         if reader is not None:
             if exc is None:
-                reader.feed_eof()
+                reader._feed_eof()
             else:
-                reader.set_exception(exc)
+                reader._set_exception(exc)
 
         if fd in self._pipe_fds:
             self._pipe_fds.remove(fd)
diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py
index f327bbd1d66d..dfb68b8e85d4 100644
--- a/Lib/test/test_asyncio/test_pep492.py
+++ b/Lib/test/test_asyncio/test_pep492.py
@@ -98,8 +98,8 @@ def test_readline(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(DATA)
-        stream.feed_eof()
+        stream._feed_data(DATA)
+        stream._feed_eof()
 
         async def reader():
             data = []
diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py
index eab71e80308f..428c863d9fa6 100644
--- a/Lib/test/test_asyncio/test_streams.py
+++ b/Lib/test/test_asyncio/test_streams.py
@@ -165,7 +165,7 @@ def test_feed_empty_data(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
 
-        stream.feed_data(b'')
+        stream._feed_data(b'')
         self.assertEqual(b'', stream._buffer)
 
     def test_feed_nonempty_data(self):
@@ -173,7 +173,7 @@ def test_feed_nonempty_data(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
 
-        stream.feed_data(self.DATA)
+        stream._feed_data(self.DATA)
         self.assertEqual(self.DATA, stream._buffer)
 
     def test_read_zero(self):
@@ -181,7 +181,7 @@ def test_read_zero(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(self.DATA)
+        stream._feed_data(self.DATA)
 
         data = self.loop.run_until_complete(stream.read(0))
         self.assertEqual(b'', data)
@@ -195,7 +195,7 @@ def test_read(self):
         read_task = asyncio.Task(stream.read(30), loop=self.loop)
 
         def cb():
-            stream.feed_data(self.DATA)
+            stream._feed_data(self.DATA)
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -207,8 +207,8 @@ def test_read_line_breaks(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'line1')
-        stream.feed_data(b'line2')
+        stream._feed_data(b'line1')
+        stream._feed_data(b'line2')
 
         data = self.loop.run_until_complete(stream.read(5))
 
@@ -223,7 +223,7 @@ def test_read_eof(self):
         read_task = asyncio.Task(stream.read(1024), loop=self.loop)
 
         def cb():
-            stream.feed_eof()
+            stream._feed_eof()
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -238,9 +238,9 @@ def test_read_until_eof(self):
         read_task = asyncio.Task(stream.read(-1), loop=self.loop)
 
         def cb():
-            stream.feed_data(b'chunk1\n')
-            stream.feed_data(b'chunk2')
-            stream.feed_eof()
+            stream._feed_data(b'chunk1\n')
+            stream._feed_data(b'chunk2')
+            stream._feed_eof()
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -252,12 +252,12 @@ def test_read_exception(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'line\n')
+        stream._feed_data(b'line\n')
 
         data = self.loop.run_until_complete(stream.read(2))
         self.assertEqual(b'li', data)
 
-        stream.set_exception(ValueError())
+        stream._set_exception(ValueError())
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.read(2))
 
@@ -276,7 +276,7 @@ def test_read_limit(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 limit=3, loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'chunk')
+        stream._feed_data(b'chunk')
         data = self.loop.run_until_complete(stream.read(5))
         self.assertEqual(b'chunk', data)
         self.assertEqual(b'', stream._buffer)
@@ -287,13 +287,13 @@ def test_readline(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'chunk1 ')
+        stream._feed_data(b'chunk1 ')
         read_task = asyncio.Task(stream.readline(), loop=self.loop)
 
         def cb():
-            stream.feed_data(b'chunk2 ')
-            stream.feed_data(b'chunk3 ')
-            stream.feed_data(b'\n chunk4')
+            stream._feed_data(b'chunk2 ')
+            stream._feed_data(b'chunk3 ')
+            stream._feed_data(b'\n chunk4')
         self.loop.call_soon(cb)
 
         line = self.loop.run_until_complete(read_task)
@@ -307,8 +307,8 @@ def test_readline_limit_with_existing_data(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 limit=3, loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'li')
-        stream.feed_data(b'ne1\nline2\n')
+        stream._feed_data(b'li')
+        stream._feed_data(b'ne1\nline2\n')
 
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readline())
@@ -318,9 +318,9 @@ def test_readline_limit_with_existing_data(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 limit=3, loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'li')
-        stream.feed_data(b'ne1')
-        stream.feed_data(b'li')
+        stream._feed_data(b'li')
+        stream._feed_data(b'ne1')
+        stream._feed_data(b'li')
 
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readline())
@@ -337,14 +337,14 @@ def test_at_eof(self):
                                 _asyncio_internal=True)
         self.assertFalse(stream.at_eof())
 
-        stream.feed_data(b'some data\n')
+        stream._feed_data(b'some data\n')
         self.assertFalse(stream.at_eof())
 
         self.loop.run_until_complete(stream.readline())
         self.assertFalse(stream.at_eof())
 
-        stream.feed_data(b'some data\n')
-        stream.feed_eof()
+        stream._feed_data(b'some data\n')
+        stream._feed_eof()
         self.loop.run_until_complete(stream.readline())
         self.assertTrue(stream.at_eof())
 
@@ -356,10 +356,10 @@ def test_readline_limit(self):
                                 limit=7, loop=self.loop,
                                 _asyncio_internal=True)
         def cb():
-            stream.feed_data(b'chunk1')
-            stream.feed_data(b'chunk2')
-            stream.feed_data(b'chunk3\n')
-            stream.feed_eof()
+            stream._feed_data(b'chunk1')
+            stream._feed_data(b'chunk2')
+            stream._feed_data(b'chunk3\n')
+            stream._feed_eof()
         self.loop.call_soon(cb)
 
         self.assertRaises(
@@ -372,10 +372,10 @@ def cb():
                                 limit=7, loop=self.loop,
                                 _asyncio_internal=True)
         def cb():
-            stream.feed_data(b'chunk1')
-            stream.feed_data(b'chunk2\n')
-            stream.feed_data(b'chunk3\n')
-            stream.feed_eof()
+            stream._feed_data(b'chunk1')
+            stream._feed_data(b'chunk2\n')
+            stream._feed_data(b'chunk3\n')
+            stream._feed_eof()
         self.loop.call_soon(cb)
 
         self.assertRaises(
@@ -386,17 +386,17 @@ def cb():
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 limit=7, loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'1234567\n')
+        stream._feed_data(b'1234567\n')
         line = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'1234567\n', line)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'12345678\n')
+        stream._feed_data(b'12345678\n')
         with self.assertRaises(ValueError) as cm:
             self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'12345678')
+        stream._feed_data(b'12345678')
         with self.assertRaises(ValueError) as cm:
             self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'', stream._buffer)
@@ -407,8 +407,8 @@ def test_readline_nolimit_nowait(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(self.DATA[:6])
-        stream.feed_data(self.DATA[6:])
+        stream._feed_data(self.DATA[:6])
+        stream._feed_data(self.DATA[6:])
 
         line = self.loop.run_until_complete(stream.readline())
 
@@ -419,8 +419,8 @@ def test_readline_eof(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'some data')
-        stream.feed_eof()
+        stream._feed_data(b'some data')
+        stream._feed_eof()
 
         line = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'some data', line)
@@ -429,7 +429,7 @@ def test_readline_empty_eof(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_eof()
+        stream._feed_eof()
 
         line = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'', line)
@@ -438,7 +438,7 @@ def test_readline_read_byte_count(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(self.DATA)
+        stream._feed_data(self.DATA)
 
         self.loop.run_until_complete(stream.readline())
 
@@ -451,12 +451,12 @@ def test_readline_exception(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'line\n')
+        stream._feed_data(b'line\n')
 
         data = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'line\n', data)
 
-        stream.set_exception(ValueError())
+        stream._set_exception(ValueError())
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readline())
         self.assertEqual(b'', stream._buffer)
@@ -473,17 +473,17 @@ def test_readuntil_multi_chunks(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
 
-        stream.feed_data(b'lineAAA')
+        stream._feed_data(b'lineAAA')
         data = self.loop.run_until_complete(stream.readuntil(separator=b'AAA'))
         self.assertEqual(b'lineAAA', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'lineAAA')
+        stream._feed_data(b'lineAAA')
         data = self.loop.run_until_complete(stream.readuntil(b'AAA'))
         self.assertEqual(b'lineAAA', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'lineAAAxxx')
+        stream._feed_data(b'lineAAAxxx')
         data = self.loop.run_until_complete(stream.readuntil(b'AAA'))
         self.assertEqual(b'lineAAA', data)
         self.assertEqual(b'xxx', stream._buffer)
@@ -493,34 +493,34 @@ def test_readuntil_multi_chunks_1(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
 
-        stream.feed_data(b'QWEaa')
-        stream.feed_data(b'XYaa')
-        stream.feed_data(b'a')
+        stream._feed_data(b'QWEaa')
+        stream._feed_data(b'XYaa')
+        stream._feed_data(b'a')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'QWEaaXYaaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'QWEaa')
-        stream.feed_data(b'XYa')
-        stream.feed_data(b'aa')
+        stream._feed_data(b'QWEaa')
+        stream._feed_data(b'XYa')
+        stream._feed_data(b'aa')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'QWEaaXYaaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'aaa')
+        stream._feed_data(b'aaa')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'aaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'Xaaa')
+        stream._feed_data(b'Xaaa')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'Xaaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream.feed_data(b'XXX')
-        stream.feed_data(b'a')
-        stream.feed_data(b'a')
-        stream.feed_data(b'a')
+        stream._feed_data(b'XXX')
+        stream._feed_data(b'a')
+        stream._feed_data(b'a')
+        stream._feed_data(b'a')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'XXXaaa', data)
         self.assertEqual(b'', stream._buffer)
@@ -529,8 +529,8 @@ def test_readuntil_eof(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'some dataAA')
-        stream.feed_eof()
+        stream._feed_data(b'some dataAA')
+        stream._feed_eof()
 
         with self.assertRaises(asyncio.IncompleteReadError) as cm:
             self.loop.run_until_complete(stream.readuntil(b'AAA'))
@@ -542,7 +542,7 @@ def test_readuntil_limit_found_sep(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop, limit=3,
                                 _asyncio_internal=True)
-        stream.feed_data(b'some dataAA')
+        stream._feed_data(b'some dataAA')
 
         with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                     'not found') as cm:
@@ -550,7 +550,7 @@ def test_readuntil_limit_found_sep(self):
 
         self.assertEqual(b'some dataAA', stream._buffer)
 
-        stream.feed_data(b'A')
+        stream._feed_data(b'A')
         with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                     'is found') as cm:
             self.loop.run_until_complete(stream.readuntil(b'AAA'))
@@ -562,7 +562,7 @@ def test_readexactly_zero_or_less(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(self.DATA)
+        stream._feed_data(self.DATA)
 
         data = self.loop.run_until_complete(stream.readexactly(0))
         self.assertEqual(b'', data)
@@ -582,9 +582,9 @@ def test_readexactly(self):
         read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
 
         def cb():
-            stream.feed_data(self.DATA)
-            stream.feed_data(self.DATA)
-            stream.feed_data(self.DATA)
+            stream._feed_data(self.DATA)
+            stream._feed_data(self.DATA)
+            stream._feed_data(self.DATA)
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -595,7 +595,7 @@ def test_readexactly_limit(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 limit=3, loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'chunk')
+        stream._feed_data(b'chunk')
         data = self.loop.run_until_complete(stream.readexactly(5))
         self.assertEqual(b'chunk', data)
         self.assertEqual(b'', stream._buffer)
@@ -609,8 +609,8 @@ def test_readexactly_eof(self):
         read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
 
         def cb():
-            stream.feed_data(self.DATA)
-            stream.feed_eof()
+            stream._feed_data(self.DATA)
+            stream._feed_eof()
         self.loop.call_soon(cb)
 
         with self.assertRaises(asyncio.IncompleteReadError) as cm:
@@ -625,12 +625,12 @@ def test_readexactly_exception(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'line\n')
+        stream._feed_data(b'line\n')
 
         data = self.loop.run_until_complete(stream.readexactly(2))
         self.assertEqual(b'li', data)
 
-        stream.set_exception(ValueError())
+        stream._set_exception(ValueError())
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readexactly(2))
 
@@ -641,7 +641,7 @@ def test_exception(self):
         self.assertIsNone(stream.exception())
 
         exc = ValueError()
-        stream.set_exception(exc)
+        stream._set_exception(exc)
         self.assertIs(stream.exception(), exc)
 
     def test_exception_waiter(self):
@@ -650,7 +650,7 @@ def test_exception_waiter(self):
                                 _asyncio_internal=True)
 
         async def set_err():
-            stream.set_exception(ValueError())
+            stream._set_exception(ValueError())
 
         t1 = asyncio.Task(stream.readline(), loop=self.loop)
         t2 = asyncio.Task(set_err(), loop=self.loop)
@@ -669,7 +669,7 @@ def test_exception_cancel(self):
         t.cancel()
         test_utils.run_briefly(self.loop)
         # The following line fails if set_exception() isn't careful.
-        stream.set_exception(RuntimeError('message'))
+        stream._set_exception(RuntimeError('message'))
         test_utils.run_briefly(self.loop)
         self.assertIs(stream._waiter, None)
 
@@ -993,14 +993,14 @@ def test___repr__eof(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_eof()
+        stream._feed_eof()
         self.assertEqual("<Stream mode=StreamMode.READ eof>", repr(stream))
 
     def test___repr__data(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream.feed_data(b'data')
+        stream._feed_data(b'data')
         self.assertEqual("<Stream mode=StreamMode.READ 4 bytes>", repr(stream))
 
     def test___repr__exception(self):
@@ -1008,7 +1008,7 @@ def test___repr__exception(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
         exc = RuntimeError()
-        stream.set_exception(exc)
+        stream._set_exception(exc)
         self.assertEqual("<Stream mode=StreamMode.READ exception=RuntimeError()>",
                          repr(stream))
 
@@ -1260,7 +1260,7 @@ def test_stream_writer_forbidden_ops(self):
             stream = asyncio.Stream(mode=asyncio.StreamMode.WRITE,
                                     _asyncio_internal=True)
             with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                stream.feed_data(b'data')
+                stream._feed_data(b'data')
             with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
                 await stream.readline()
             with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
@@ -1785,6 +1785,30 @@ def test_stream_ctor_forbidden(self):
                                     "by asyncio internals only"):
             asyncio.Stream(asyncio.StreamMode.READWRITE)
 
+    def test_deprecated_methods(self):
+        async def f():
+            return asyncio.Stream(mode=asyncio.StreamMode.READWRITE,
+                                  _asyncio_internal=True)
+
+        stream = self.loop.run_until_complete(f())
+
+        tr = mock.Mock()
+
+        with self.assertWarns(DeprecationWarning):
+            stream.set_transport(tr)
+
+        with self.assertWarns(DeprecationWarning):
+            stream.transport is tr
+
+        with self.assertWarns(DeprecationWarning):
+            stream.feed_data(b'data')
+
+        with self.assertWarns(DeprecationWarning):
+            stream.feed_eof()
+
+        with self.assertWarns(DeprecationWarning):
+            stream.set_exception(ConnectionResetError("test"))
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Misc/NEWS.d/next/Library/2019-09-09-14-39-47.bpo-38066.l9mWv-.rst b/Misc/NEWS.d/next/Library/2019-09-09-14-39-47.bpo-38066.l9mWv-.rst
new file mode 100644
index 000000000000..a69924fe2aa4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-09-14-39-47.bpo-38066.l9mWv-.rst
@@ -0,0 +1,2 @@
+Hide internal asyncio.Stream methods: feed_eof(), feed_data(),
+set_exception() and set_transport().


From webhook-mailer at python.org  Tue Sep 10 09:51:14 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Tue, 10 Sep 2019 13:51:14 -0000
Subject: [Python-checkins] Fix subprocess docstring typo (GH-15812)
Message-ID: <mailman.445.1568123474.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/182e1d1f849757439f2031504f142fa4e1251611
commit: 182e1d1f849757439f2031504f142fa4e1251611
branch: master
author: Matthias <xmatthias at outlook.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-10T14:51:09+01:00
summary:

Fix subprocess docstring typo (GH-15812)

files:
M Lib/subprocess.py

diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 5bbeba47a374..aed7292541e0 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -395,7 +395,7 @@ def check_output(*popenargs, timeout=None, **kwargs):
     b'when in the course of barman events\n'
 
     By default, all communication is in bytes, and therefore any "input"
-    should be bytes, and the return value wil be bytes.  If in text mode,
+    should be bytes, and the return value will be bytes.  If in text mode,
     any "input" should be a string, and the return value will be a string
     decoded according to locale encoding, or by "encoding" if set. Text mode
     is triggered by setting any of text, encoding, errors or universal_newlines.


From webhook-mailer at python.org  Tue Sep 10 09:52:28 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Tue, 10 Sep 2019 13:52:28 -0000
Subject: [Python-checkins] bpo-38088: Fixes distutils not finding
 vcruntime140.dll with only v142 toolset installed (GH-15849)
Message-ID: <mailman.446.1568123549.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/cd8221152dd235ec5d06e3d9d0d8787645bbac8e
commit: cd8221152dd235ec5d06e3d9d0d8787645bbac8e
branch: master
author: Steve Dower <steve.dower at python.org>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-10T14:52:23+01:00
summary:

bpo-38088: Fixes distutils not finding vcruntime140.dll with only v142 toolset installed (GH-15849)

files:
A Misc/NEWS.d/next/Windows/2019-09-10-14-17-25.bpo-38088.FOvWSM.rst
M Lib/distutils/_msvccompiler.py

diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py
index 6e14f330d7f4..e8e4b717b973 100644
--- a/Lib/distutils/_msvccompiler.py
+++ b/Lib/distutils/_msvccompiler.py
@@ -107,7 +107,7 @@ def _find_vcvarsall(plat_spec):
 
     if best_dir:
         vcredist = os.path.join(best_dir, "..", "..", "redist", "MSVC", "**",
-            vcruntime_plat, "Microsoft.VC141.CRT", "vcruntime140.dll")
+            vcruntime_plat, "Microsoft.VC14*.CRT", "vcruntime140.dll")
         try:
             import glob
             vcruntime = glob.glob(vcredist, recursive=True)[-1]
diff --git a/Misc/NEWS.d/next/Windows/2019-09-10-14-17-25.bpo-38088.FOvWSM.rst b/Misc/NEWS.d/next/Windows/2019-09-10-14-17-25.bpo-38088.FOvWSM.rst
new file mode 100644
index 000000000000..37bdeeadd894
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-10-14-17-25.bpo-38088.FOvWSM.rst
@@ -0,0 +1,2 @@
+Fixes distutils not finding vcruntime140.dll with only the v142 toolset
+installed.


From webhook-mailer at python.org  Tue Sep 10 09:52:53 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Tue, 10 Sep 2019 13:52:53 -0000
Subject: [Python-checkins] bpo-38087: Fix case sensitivity in test_pathlib
 and test_ntpath (GH-15850)
Message-ID: <mailman.447.1568123574.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/97d7906e30eeee1261b20a45a22242a8accb1cfb
commit: 97d7906e30eeee1261b20a45a22242a8accb1cfb
branch: master
author: Steve Dower <steve.dower at python.org>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-10T14:52:48+01:00
summary:

bpo-38087: Fix case sensitivity in test_pathlib and test_ntpath (GH-15850)

files:
A Misc/NEWS.d/next/Windows/2019-09-10-14-21-40.bpo-38087.--eIib.rst
M Lib/test/test_ntpath.py
M Lib/test/test_pathlib.py

diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
index 285fb69dc1e8..c5c96e32d747 100644
--- a/Lib/test/test_ntpath.py
+++ b/Lib/test/test_ntpath.py
@@ -23,10 +23,18 @@
     HAVE_GETFINALPATHNAME = True
 
 
+def _norm(path):
+    if isinstance(path, (bytes, str, os.PathLike)):
+        return ntpath.normcase(os.fsdecode(path))
+    elif hasattr(path, "__iter__"):
+        return tuple(ntpath.normcase(os.fsdecode(p)) for p in path)
+    return path
+
+
 def tester(fn, wantResult):
     fn = fn.replace("\\", "\\\\")
     gotResult = eval(fn)
-    if wantResult != gotResult:
+    if wantResult != gotResult and _norm(wantResult) != _norm(gotResult):
         raise TestFailed("%s should return: %s but returned: %s" \
               %(str(fn), str(wantResult), str(gotResult)))
 
@@ -42,16 +50,22 @@ def tester(fn, wantResult):
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", DeprecationWarning)
         gotResult = eval(fn)
-    if isinstance(wantResult, str):
-        wantResult = os.fsencode(wantResult)
-    elif isinstance(wantResult, tuple):
-        wantResult = tuple(os.fsencode(r) for r in wantResult)
-    if wantResult != gotResult:
+    if _norm(wantResult) != _norm(gotResult):
         raise TestFailed("%s should return: %s but returned: %s" \
               %(str(fn), str(wantResult), repr(gotResult)))
 
 
-class TestNtpath(unittest.TestCase):
+class NtpathTestCase(unittest.TestCase):
+    def assertPathEqual(self, path1, path2):
+        if path1 == path2 or _norm(path1) == _norm(path2):
+            return
+        self.assertEqual(path1, path2)
+
+    def assertPathIn(self, path, pathset):
+        self.assertIn(_norm(path), _norm(pathset))
+
+
+class TestNtpath(NtpathTestCase):
     def test_splitext(self):
         tester('ntpath.splitext("foo.ext")', ('foo', '.ext'))
         tester('ntpath.splitext("/foo/foo.ext")', ('/foo/foo', '.ext'))
@@ -232,8 +246,8 @@ def test_realpath_basic(self):
         self.addCleanup(support.unlink, ABSTFN + "1")
 
         os.symlink(ABSTFN, ABSTFN + "1")
-        self.assertEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN)
-        self.assertEqual(ntpath.realpath(os.fsencode(ABSTFN + "1")),
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN)
+        self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1")),
                          os.fsencode(ABSTFN))
 
     @support.skip_unless_symlink
@@ -245,7 +259,7 @@ def test_realpath_relative(self):
         self.addCleanup(support.unlink, ABSTFN + "1")
 
         os.symlink(ABSTFN, ntpath.relpath(ABSTFN + "1"))
-        self.assertEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN)
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN)
 
     @support.skip_unless_symlink
     @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
@@ -268,39 +282,39 @@ def test_realpath_broken_symlinks(self):
             os.symlink(ABSTFN + r"\broken", "broken4")
             os.symlink(r"recursive\..\broken", "broken5")
 
-            self.assertEqual(ntpath.realpath("broken"),
-                             ABSTFN + r"\missing")
-            self.assertEqual(ntpath.realpath(r"broken\foo"),
-                             ABSTFN + r"\missing\foo")
-            self.assertEqual(ntpath.realpath(r"broken1"),
-                             ABSTFN + r"\missing\bar")
-            self.assertEqual(ntpath.realpath(r"broken1\baz"),
-                             ABSTFN + r"\missing\bar\baz")
-            self.assertEqual(ntpath.realpath("broken2"),
-                             ABSTFN + r"\missing")
-            self.assertEqual(ntpath.realpath("broken3"),
-                             ABSTFN + r"\missing")
-            self.assertEqual(ntpath.realpath("broken4"),
-                             ABSTFN + r"\missing")
-            self.assertEqual(ntpath.realpath("broken5"),
-                             ABSTFN + r"\missing")
-
-            self.assertEqual(ntpath.realpath(b"broken"),
-                             os.fsencode(ABSTFN + r"\missing"))
-            self.assertEqual(ntpath.realpath(rb"broken\foo"),
-                             os.fsencode(ABSTFN + r"\missing\foo"))
-            self.assertEqual(ntpath.realpath(rb"broken1"),
-                             os.fsencode(ABSTFN + r"\missing\bar"))
-            self.assertEqual(ntpath.realpath(rb"broken1\baz"),
-                             os.fsencode(ABSTFN + r"\missing\bar\baz"))
-            self.assertEqual(ntpath.realpath(b"broken2"),
-                             os.fsencode(ABSTFN + r"\missing"))
-            self.assertEqual(ntpath.realpath(rb"broken3"),
-                             os.fsencode(ABSTFN + r"\missing"))
-            self.assertEqual(ntpath.realpath(b"broken4"),
-                             os.fsencode(ABSTFN + r"\missing"))
-            self.assertEqual(ntpath.realpath(b"broken5"),
-                             os.fsencode(ABSTFN + r"\missing"))
+            self.assertPathEqual(ntpath.realpath("broken"),
+                                 ABSTFN + r"\missing")
+            self.assertPathEqual(ntpath.realpath(r"broken\foo"),
+                                 ABSTFN + r"\missing\foo")
+            self.assertPathEqual(ntpath.realpath(r"broken1"),
+                                 ABSTFN + r"\missing\bar")
+            self.assertPathEqual(ntpath.realpath(r"broken1\baz"),
+                                 ABSTFN + r"\missing\bar\baz")
+            self.assertPathEqual(ntpath.realpath("broken2"),
+                                 ABSTFN + r"\missing")
+            self.assertPathEqual(ntpath.realpath("broken3"),
+                                 ABSTFN + r"\missing")
+            self.assertPathEqual(ntpath.realpath("broken4"),
+                                 ABSTFN + r"\missing")
+            self.assertPathEqual(ntpath.realpath("broken5"),
+                                 ABSTFN + r"\missing")
+
+            self.assertPathEqual(ntpath.realpath(b"broken"),
+                                 os.fsencode(ABSTFN + r"\missing"))
+            self.assertPathEqual(ntpath.realpath(rb"broken\foo"),
+                                 os.fsencode(ABSTFN + r"\missing\foo"))
+            self.assertPathEqual(ntpath.realpath(rb"broken1"),
+                                 os.fsencode(ABSTFN + r"\missing\bar"))
+            self.assertPathEqual(ntpath.realpath(rb"broken1\baz"),
+                                 os.fsencode(ABSTFN + r"\missing\bar\baz"))
+            self.assertPathEqual(ntpath.realpath(b"broken2"),
+                                 os.fsencode(ABSTFN + r"\missing"))
+            self.assertPathEqual(ntpath.realpath(rb"broken3"),
+                                 os.fsencode(ABSTFN + r"\missing"))
+            self.assertPathEqual(ntpath.realpath(b"broken4"),
+                                 os.fsencode(ABSTFN + r"\missing"))
+            self.assertPathEqual(ntpath.realpath(b"broken5"),
+                                 os.fsencode(ABSTFN + r"\missing"))
 
     @support.skip_unless_symlink
     @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
@@ -318,39 +332,39 @@ def test_realpath_symlink_loops(self):
         P = "\\\\?\\"
 
         os.symlink(ABSTFN, ABSTFN)
-        self.assertEqual(ntpath.realpath(ABSTFN), P + ABSTFN)
+        self.assertPathEqual(ntpath.realpath(ABSTFN), P + ABSTFN)
 
         # cycles are non-deterministic as to which path is returned, but
         # it will always be the fully resolved path of one member of the cycle
         os.symlink(ABSTFN + "1", ABSTFN + "2")
         os.symlink(ABSTFN + "2", ABSTFN + "1")
         expected = (P + ABSTFN + "1", P + ABSTFN + "2")
-        self.assertIn(ntpath.realpath(ABSTFN + "1"), expected)
-        self.assertIn(ntpath.realpath(ABSTFN + "2"), expected)
-
-        self.assertIn(ntpath.realpath(ABSTFN + "1\\x"),
-                      (ntpath.join(r, "x") for r in expected))
-        self.assertEqual(ntpath.realpath(ABSTFN + "1\\.."),
-                         ntpath.dirname(ABSTFN))
-        self.assertEqual(ntpath.realpath(ABSTFN + "1\\..\\x"),
-                         ntpath.dirname(ABSTFN) + "\\x")
+        self.assertPathIn(ntpath.realpath(ABSTFN + "1"), expected)
+        self.assertPathIn(ntpath.realpath(ABSTFN + "2"), expected)
+
+        self.assertPathIn(ntpath.realpath(ABSTFN + "1\\x"),
+                          (ntpath.join(r, "x") for r in expected))
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\.."),
+                             ntpath.dirname(ABSTFN))
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\x"),
+                             ntpath.dirname(ABSTFN) + "\\x")
         os.symlink(ABSTFN + "x", ABSTFN + "y")
-        self.assertEqual(ntpath.realpath(ABSTFN + "1\\..\\"
-                                         + ntpath.basename(ABSTFN) + "y"),
-                         ABSTFN + "x")
-        self.assertIn(ntpath.realpath(ABSTFN + "1\\..\\"
-                                      + ntpath.basename(ABSTFN) + "1"),
-                      expected)
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\"
+                                             + ntpath.basename(ABSTFN) + "y"),
+                             ABSTFN + "x")
+        self.assertPathIn(ntpath.realpath(ABSTFN + "1\\..\\"
+                                          + ntpath.basename(ABSTFN) + "1"),
+                          expected)
 
         os.symlink(ntpath.basename(ABSTFN) + "a\\b", ABSTFN + "a")
-        self.assertEqual(ntpath.realpath(ABSTFN + "a"), P + ABSTFN + "a")
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "a"), P + ABSTFN + "a")
 
         os.symlink("..\\" + ntpath.basename(ntpath.dirname(ABSTFN))
                    + "\\" + ntpath.basename(ABSTFN) + "c", ABSTFN + "c")
-        self.assertEqual(ntpath.realpath(ABSTFN + "c"), P + ABSTFN + "c")
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "c"), P + ABSTFN + "c")
 
         # Test using relative path as well.
-        self.assertEqual(ntpath.realpath(ntpath.basename(ABSTFN)), P + ABSTFN)
+        self.assertPathEqual(ntpath.realpath(ntpath.basename(ABSTFN)), P + ABSTFN)
 
     @support.skip_unless_symlink
     @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
@@ -369,10 +383,10 @@ def test_realpath_symlink_prefix(self):
             f.write(b'1')
         os.symlink("\\\\?\\" + ABSTFN + "3.", ABSTFN + "3.link")
 
-        self.assertEqual(ntpath.realpath(ABSTFN + "3link"),
-                         ABSTFN + "3")
-        self.assertEqual(ntpath.realpath(ABSTFN + "3.link"),
-                         "\\\\?\\" + ABSTFN + "3.")
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "3link"),
+                             ABSTFN + "3")
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link"),
+                             "\\\\?\\" + ABSTFN + "3.")
 
         # Resolved paths should be usable to open target files
         with open(ntpath.realpath(ABSTFN + "3link"), "rb") as f:
@@ -381,10 +395,10 @@ def test_realpath_symlink_prefix(self):
             self.assertEqual(f.read(), b'1')
 
         # When the prefix is included, it is not stripped
-        self.assertEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link"),
-                         "\\\\?\\" + ABSTFN + "3")
-        self.assertEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link"),
-                         "\\\\?\\" + ABSTFN + "3.")
+        self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link"),
+                             "\\\\?\\" + ABSTFN + "3")
+        self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link"),
+                             "\\\\?\\" + ABSTFN + "3.")
 
     def test_expandvars(self):
         with support.EnvironmentVarGuard() as env:
@@ -658,7 +672,7 @@ class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase):
     attributes = ['relpath']
 
 
-class PathLikeTests(unittest.TestCase):
+class PathLikeTests(NtpathTestCase):
 
     path = ntpath
 
@@ -669,67 +683,67 @@ def setUp(self):
         with open(self.file_name, 'xb', 0) as file:
             file.write(b"test_ntpath.PathLikeTests")
 
-    def assertPathEqual(self, func):
-        self.assertEqual(func(self.file_path), func(self.file_name))
+    def _check_function(self, func):
+        self.assertPathEqual(func(self.file_path), func(self.file_name))
 
     def test_path_normcase(self):
-        self.assertPathEqual(self.path.normcase)
+        self._check_function(self.path.normcase)
 
     def test_path_isabs(self):
-        self.assertPathEqual(self.path.isabs)
+        self._check_function(self.path.isabs)
 
     def test_path_join(self):
         self.assertEqual(self.path.join('a', FakePath('b'), 'c'),
                          self.path.join('a', 'b', 'c'))
 
     def test_path_split(self):
-        self.assertPathEqual(self.path.split)
+        self._check_function(self.path.split)
 
     def test_path_splitext(self):
-        self.assertPathEqual(self.path.splitext)
+        self._check_function(self.path.splitext)
 
     def test_path_splitdrive(self):
-        self.assertPathEqual(self.path.splitdrive)
+        self._check_function(self.path.splitdrive)
 
     def test_path_basename(self):
-        self.assertPathEqual(self.path.basename)
+        self._check_function(self.path.basename)
 
     def test_path_dirname(self):
-        self.assertPathEqual(self.path.dirname)
+        self._check_function(self.path.dirname)
 
     def test_path_islink(self):
-        self.assertPathEqual(self.path.islink)
+        self._check_function(self.path.islink)
 
     def test_path_lexists(self):
-        self.assertPathEqual(self.path.lexists)
+        self._check_function(self.path.lexists)
 
     def test_path_ismount(self):
-        self.assertPathEqual(self.path.ismount)
+        self._check_function(self.path.ismount)
 
     def test_path_expanduser(self):
-        self.assertPathEqual(self.path.expanduser)
+        self._check_function(self.path.expanduser)
 
     def test_path_expandvars(self):
-        self.assertPathEqual(self.path.expandvars)
+        self._check_function(self.path.expandvars)
 
     def test_path_normpath(self):
-        self.assertPathEqual(self.path.normpath)
+        self._check_function(self.path.normpath)
 
     def test_path_abspath(self):
-        self.assertPathEqual(self.path.abspath)
+        self._check_function(self.path.abspath)
 
     def test_path_realpath(self):
-        self.assertPathEqual(self.path.realpath)
+        self._check_function(self.path.realpath)
 
     def test_path_relpath(self):
-        self.assertPathEqual(self.path.relpath)
+        self._check_function(self.path.relpath)
 
     def test_path_commonpath(self):
         common_path = self.path.commonpath([self.file_path, self.file_name])
-        self.assertEqual(common_path, self.file_name)
+        self.assertPathEqual(common_path, self.file_name)
 
     def test_path_isdir(self):
-        self.assertPathEqual(self.path.isdir)
+        self._check_function(self.path.isdir)
 
 
 if __name__ == "__main__":
diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py
index cfda0a218826..34c66f3137ff 100644
--- a/Lib/test/test_pathlib.py
+++ b/Lib/test/test_pathlib.py
@@ -1361,10 +1361,13 @@ def assertFileNotFound(self, func, *args, **kwargs):
             func(*args, **kwargs)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
 
+    def assertEqualNormCase(self, path_a, path_b):
+        self.assertEqual(os.path.normcase(path_a), os.path.normcase(path_b))
+
     def _test_cwd(self, p):
         q = self.cls(os.getcwd())
         self.assertEqual(p, q)
-        self.assertEqual(str(p), str(q))
+        self.assertEqualNormCase(str(p), str(q))
         self.assertIs(type(p), type(q))
         self.assertTrue(p.is_absolute())
 
@@ -1375,7 +1378,7 @@ def test_cwd(self):
     def _test_home(self, p):
         q = self.cls(os.path.expanduser('~'))
         self.assertEqual(p, q)
-        self.assertEqual(str(p), str(q))
+        self.assertEqualNormCase(str(p), str(q))
         self.assertIs(type(p), type(q))
         self.assertTrue(p.is_absolute())
 
@@ -1583,14 +1586,14 @@ def test_resolve_common(self):
             p.resolve(strict=True)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         # Non-strict
-        self.assertEqual(str(p.resolve(strict=False)),
-                         os.path.join(BASE, 'foo'))
+        self.assertEqualNormCase(str(p.resolve(strict=False)),
+                                 os.path.join(BASE, 'foo'))
         p = P(BASE, 'foo', 'in', 'spam')
-        self.assertEqual(str(p.resolve(strict=False)),
-                         os.path.join(BASE, 'foo', 'in', 'spam'))
+        self.assertEqualNormCase(str(p.resolve(strict=False)),
+                                 os.path.join(BASE, 'foo', 'in', 'spam'))
         p = P(BASE, '..', 'foo', 'in', 'spam')
-        self.assertEqual(str(p.resolve(strict=False)),
-                         os.path.abspath(os.path.join('foo', 'in', 'spam')))
+        self.assertEqualNormCase(str(p.resolve(strict=False)),
+                                 os.path.abspath(os.path.join('foo', 'in', 'spam')))
         # These are all relative symlinks.
         p = P(BASE, 'dirB', 'fileB')
         self._check_resolve_relative(p, p)
@@ -2137,16 +2140,16 @@ def _check_complex_symlinks(self, link0_target):
         # Resolve absolute paths.
         p = (P / 'link0').resolve()
         self.assertEqual(p, P)
-        self.assertEqual(str(p), BASE)
+        self.assertEqualNormCase(str(p), BASE)
         p = (P / 'link1').resolve()
         self.assertEqual(p, P)
-        self.assertEqual(str(p), BASE)
+        self.assertEqualNormCase(str(p), BASE)
         p = (P / 'link2').resolve()
         self.assertEqual(p, P)
-        self.assertEqual(str(p), BASE)
+        self.assertEqualNormCase(str(p), BASE)
         p = (P / 'link3').resolve()
         self.assertEqual(p, P)
-        self.assertEqual(str(p), BASE)
+        self.assertEqualNormCase(str(p), BASE)
 
         # Resolve relative paths.
         old_path = os.getcwd()
@@ -2154,16 +2157,16 @@ def _check_complex_symlinks(self, link0_target):
         try:
             p = self.cls('link0').resolve()
             self.assertEqual(p, P)
-            self.assertEqual(str(p), BASE)
+            self.assertEqualNormCase(str(p), BASE)
             p = self.cls('link1').resolve()
             self.assertEqual(p, P)
-            self.assertEqual(str(p), BASE)
+            self.assertEqualNormCase(str(p), BASE)
             p = self.cls('link2').resolve()
             self.assertEqual(p, P)
-            self.assertEqual(str(p), BASE)
+            self.assertEqualNormCase(str(p), BASE)
             p = self.cls('link3').resolve()
             self.assertEqual(p, P)
-            self.assertEqual(str(p), BASE)
+            self.assertEqualNormCase(str(p), BASE)
         finally:
             os.chdir(old_path)
 
diff --git a/Misc/NEWS.d/next/Windows/2019-09-10-14-21-40.bpo-38087.--eIib.rst b/Misc/NEWS.d/next/Windows/2019-09-10-14-21-40.bpo-38087.--eIib.rst
new file mode 100644
index 000000000000..ca625cc775a2
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-10-14-21-40.bpo-38087.--eIib.rst
@@ -0,0 +1 @@
+Fix case sensitivity in test_pathlib and test_ntpath.


From webhook-mailer at python.org  Tue Sep 10 09:53:36 2019
From: webhook-mailer at python.org (Brett Cannon)
Date: Tue, 10 Sep 2019 13:53:36 -0000
Subject: [Python-checkins] bpo-38086: Sync importlib.metadata with
 importlib_metadata 0.21. (GH-15840)
Message-ID: <mailman.448.1568123618.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/17499d82702432955d8e442a1871ff276ca64bc5
commit: 17499d82702432955d8e442a1871ff276ca64bc5
branch: master
author: Jason R. Coombs <jaraco at jaraco.com>
committer: Brett Cannon <54418+brettcannon at users.noreply.github.com>
date: 2019-09-10T14:53:31+01:00
summary:

bpo-38086: Sync importlib.metadata with importlib_metadata 0.21. (GH-15840)

https://gitlab.com/python-devs/importlib_metadata/-/tags/0.21

files:
A Misc/NEWS.d/next/Library/2019-09-10-11-42-59.bpo-38086.w5TlG-.rst
M Doc/library/importlib.metadata.rst
M Doc/tools/susp-ignored.csv
M Lib/importlib/_bootstrap_external.py
M Lib/importlib/metadata.py
M Lib/test/test_importlib/test_main.py
M Lib/test/test_importlib/test_metadata_api.py
M Python/importlib_external.h

diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst
index 23c8cdeb21c2..4a4a8f7dfa0f 100644
--- a/Doc/library/importlib.metadata.rst
+++ b/Doc/library/importlib.metadata.rst
@@ -172,10 +172,10 @@ Distribution requirements
 -------------------------
 
 To get the full set of requirements for a distribution, use the ``requires()``
-function.  Note that this returns an iterator::
+function::
 
-    >>> list(requires('wheel'))  # doctest: +SKIP
-    ["pytest (>=3.0.0) ; extra == 'test'"]
+    >>> requires('wheel')  # doctest: +SKIP
+    ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"]
 
 
 Distributions
@@ -224,23 +224,25 @@ The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the
 interface expected of finders by Python's import system.
 ``importlib.metadata`` extends this protocol by looking for an optional
 ``find_distributions`` callable on the finders from
-``sys.meta_path``.  If the finder has this method, it must return
-an iterator over instances of the ``Distribution`` abstract class. This
-method must have the signature::
+``sys.meta_path`` and presents this extended interface as the
+``DistributionFinder`` abstract base class, which defines this abstract
+method::
 
-    def find_distributions(name=None, path=None):
+    @abc.abstractmethod
+    def find_distributions(context=DistributionFinder.Context()):
         """Return an iterable of all Distribution instances capable of
-        loading the metadata for packages matching the name
-        (or all names if not supplied) along the paths in the list
-        of directories ``path`` (defaults to sys.path).
+        loading the metadata for packages for the indicated ``context``.
         """
 
+The ``DistributionFinder.Context`` object provides ``.path`` and ``.name``
+properties indicating the path to search and names to match and may
+supply other relevant context.
+
 What this means in practice is that to support finding distribution package
 metadata in locations other than the file system, you should derive from
-``Distribution`` and implement the ``load_metadata()`` method.  This takes a
-single argument which is the name of the package whose metadata is being
-found.  This instance of the ``Distribution`` base abstract class is what your
-finder's ``find_distributions()`` method should return.
+``Distribution`` and implement the ``load_metadata()`` method. Then from
+your finder, return instances of this derived ``Distribution`` in the
+``find_distributions()`` method.
 
 
 .. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points
diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv
index 43d8d6e28f2f..bc0dcb60b65b 100644
--- a/Doc/tools/susp-ignored.csv
+++ b/Doc/tools/susp-ignored.csv
@@ -352,4 +352,4 @@ whatsnew/changelog,,::,error::BytesWarning
 whatsnew/changelog,,::,default::BytesWarning
 whatsnew/changelog,,::,default::DeprecationWarning
 library/importlib.metadata,,:main,"EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts')"
-library/importlib.metadata,,`,of directories ``path`` (defaults to sys.path).
+library/importlib.metadata,,`,loading the metadata for packages for the indicated ``context``.
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index 671b043c73d5..ec4bbec41e5a 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -1370,21 +1370,19 @@ def find_module(cls, fullname, path=None):
         return spec.loader
 
     @classmethod
-    def find_distributions(cls, name=None, path=None):
+    def find_distributions(self, context=None):
         """
         Find distributions.
 
         Return an iterable of all Distribution instances capable of
-        loading the metadata for packages matching the ``name``
-        (or all names if not supplied) along the paths in the list
-        of directories ``path`` (defaults to sys.path).
+        loading the metadata for packages matching ``context.name``
+        (or all names if ``None`` indicated) along the paths in the list
+        of directories ``context.path``.
         """
-        import re
-        from importlib.metadata import PathDistribution
-        if path is None:
-            path = sys.path
-        pattern = '.*' if name is None else re.escape(name)
-        found = cls._search_paths(pattern, path)
+        from importlib.metadata import PathDistribution, DistributionFinder
+        if context is None:
+            context = DistributionFinder.Context()
+        found = self._search_paths(context.pattern, context.path)
         return map(PathDistribution, found)
 
     @classmethod
diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py
index 3b46142231ec..e23076654fe4 100644
--- a/Lib/importlib/metadata.py
+++ b/Lib/importlib/metadata.py
@@ -19,6 +19,7 @@
 
 __all__ = [
     'Distribution',
+    'DistributionFinder',
     'PackageNotFoundError',
     'distribution',
     'distributions',
@@ -158,7 +159,7 @@ def from_name(cls, name):
             metadata cannot be found.
         """
         for resolver in cls._discover_resolvers():
-            dists = resolver(name)
+            dists = resolver(DistributionFinder.Context(name=name))
             dist = next(dists, None)
             if dist is not None:
                 return dist
@@ -166,16 +167,33 @@ def from_name(cls, name):
             raise PackageNotFoundError(name)
 
     @classmethod
-    def discover(cls):
+    def discover(cls, **kwargs):
         """Return an iterable of Distribution objects for all packages.
 
+        Pass a ``context`` or pass keyword arguments for constructing
+        a context.
+
+        :context: A ``DistributionFinder.Context`` object.
         :return: Iterable of Distribution objects for all packages.
         """
+        context = kwargs.pop('context', None)
+        if context and kwargs:
+            raise ValueError("cannot accept context and kwargs")
+        context = context or DistributionFinder.Context(**kwargs)
         return itertools.chain.from_iterable(
-            resolver()
+            resolver(context)
             for resolver in cls._discover_resolvers()
             )
 
+    @staticmethod
+    def at(path):
+        """Return a Distribution for the indicated metadata path
+
+        :param path: a string or path-like object
+        :return: a concrete Distribution instance for the path
+        """
+        return PathDistribution(pathlib.Path(path))
+
     @staticmethod
     def _discover_resolvers():
         """Search the meta_path for resolvers."""
@@ -215,7 +233,7 @@ def entry_points(self):
     def files(self):
         """Files in this distribution.
 
-        :return: Iterable of PackagePath for this distribution or None
+        :return: List of PackagePath for this distribution or None
 
         Result is `None` if the metadata file that enumerates files
         (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is
@@ -231,7 +249,7 @@ def make_file(name, hash=None, size_str=None):
             result.dist = self
             return result
 
-        return file_lines and starmap(make_file, csv.reader(file_lines))
+        return file_lines and list(starmap(make_file, csv.reader(file_lines)))
 
     def _read_files_distinfo(self):
         """
@@ -251,7 +269,8 @@ def _read_files_egginfo(self):
     @property
     def requires(self):
         """Generated requirements specified for this Distribution"""
-        return self._read_dist_info_reqs() or self._read_egg_info_reqs()
+        reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs()
+        return reqs and list(reqs)
 
     def _read_dist_info_reqs(self):
         return self.metadata.get_all('Requires-Dist')
@@ -312,15 +331,35 @@ class DistributionFinder(MetaPathFinder):
     A MetaPathFinder capable of discovering installed distributions.
     """
 
+    class Context:
+
+        name = None
+        """
+        Specific name for which a distribution finder should match.
+        """
+
+        def __init__(self, **kwargs):
+            vars(self).update(kwargs)
+
+        @property
+        def path(self):
+            """
+            The path that a distribution finder should search.
+            """
+            return vars(self).get('path', sys.path)
+
+        @property
+        def pattern(self):
+            return '.*' if self.name is None else re.escape(self.name)
+
     @abc.abstractmethod
-    def find_distributions(self, name=None, path=None):
+    def find_distributions(self, context=Context()):
         """
         Find distributions.
 
         Return an iterable of all Distribution instances capable of
-        loading the metadata for packages matching the ``name``
-        (or all names if not supplied) along the paths in the list
-        of directories ``path`` (defaults to sys.path).
+        loading the metadata for packages matching the ``context``,
+        a DistributionFinder.Context instance.
         """
 
 
@@ -352,12 +391,12 @@ def distribution(package):
     return Distribution.from_name(package)
 
 
-def distributions():
+def distributions(**kwargs):
     """Get all ``Distribution`` instances in the current environment.
 
     :return: An iterable of ``Distribution`` instances.
     """
-    return Distribution.discover()
+    return Distribution.discover(**kwargs)
 
 
 def metadata(package):
diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py
index 3d7da819b343..4d5b1273d9d1 100644
--- a/Lib/test/test_importlib/test_main.py
+++ b/Lib/test/test_importlib/test_main.py
@@ -162,6 +162,10 @@ def test_package_discovery(self):
             for dist in dists
             )
 
+    def test_invalid_usage(self):
+        with self.assertRaises(ValueError):
+            list(distributions(context='something', name='else'))
+
 
 class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
     def test_egg_info(self):
diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py
index af3bab3558ef..1d7b29ae05fd 100644
--- a/Lib/test/test_importlib/test_metadata_api.py
+++ b/Lib/test/test_importlib/test_metadata_api.py
@@ -1,7 +1,6 @@
 import re
 import textwrap
 import unittest
-import itertools
 
 from collections.abc import Iterator
 
@@ -61,9 +60,7 @@ def test_metadata_for_this_package(self):
         assert 'Topic :: Software Development :: Libraries' in classifiers
 
     @staticmethod
-    def _test_files(files_iter):
-        assert isinstance(files_iter, Iterator), files_iter
-        files = list(files_iter)
+    def _test_files(files):
         root = files[0].root
         for file in files:
             assert file.root == root
@@ -99,16 +96,18 @@ def test_requires_egg_info_file(self):
         requirements = requires('egginfo-file')
         self.assertIsNone(requirements)
 
-    def test_requires(self):
+    def test_requires_egg_info(self):
         deps = requires('egginfo-pkg')
+        assert len(deps) == 2
         assert any(
             dep == 'wheel >= 1.0; python_version >= "2.7"'
             for dep in deps
             )
 
     def test_requires_dist_info(self):
-        deps = list(requires('distinfo-pkg'))
-        assert deps and all(deps)
+        deps = requires('distinfo-pkg')
+        assert len(deps) == 2
+        assert all(deps)
         assert 'wheel >= 1.0' in deps
         assert "pytest; extra == 'test'" in deps
 
@@ -143,11 +142,20 @@ def test_more_complex_deps_requires_text(self):
 
 class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase):
     def test_find_distributions_specified_path(self):
-        dists = itertools.chain.from_iterable(
-            resolver(path=[str(self.site_dir)])
-            for resolver in Distribution._discover_resolvers()
-            )
+        dists = Distribution.discover(path=[str(self.site_dir)])
         assert any(
             dist.metadata['Name'] == 'distinfo-pkg'
             for dist in dists
             )
+
+    def test_distribution_at_pathlib(self):
+        """Demonstrate how to load metadata direct from a directory.
+        """
+        dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info'
+        dist = Distribution.at(dist_info_path)
+        assert dist.version == '1.0.0'
+
+    def test_distribution_at_str(self):
+        dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info'
+        dist = Distribution.at(str(dist_info_path))
+        assert dist.version == '1.0.0'
diff --git a/Misc/NEWS.d/next/Library/2019-09-10-11-42-59.bpo-38086.w5TlG-.rst b/Misc/NEWS.d/next/Library/2019-09-10-11-42-59.bpo-38086.w5TlG-.rst
new file mode 100644
index 000000000000..40db53c97da4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-10-11-42-59.bpo-38086.w5TlG-.rst
@@ -0,0 +1 @@
+Update importlib.metadata with changes from `importlib_metadata 0.21 <https://gitlab.com/python-devs/importlib_metadata/blob/0.21/importlib_metadata/docs/changelog.rst>`_.
\ No newline at end of file
diff --git a/Python/importlib_external.h b/Python/importlib_external.h
index c69819676af8..b9d14d64b402 100644
--- a/Python/importlib_external.h
+++ b/Python/importlib_external.h
@@ -2235,619 +2235,618 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,205,
     0,0,0,79,5,0,0,115,8,0,0,0,0,8,12,1,
     8,1,4,1,122,22,80,97,116,104,70,105,110,100,101,114,
-    46,102,105,110,100,95,109,111,100,117,108,101,99,3,0,0,
-    0,0,0,0,0,0,0,0,0,7,0,0,0,4,0,0,
-    0,67,0,0,0,115,78,0,0,0,100,1,100,2,108,0,
-    125,3,100,1,100,3,108,1,109,2,125,4,1,0,124,2,
-    100,2,107,8,114,34,116,3,106,4,125,2,124,1,100,2,
-    107,8,114,46,100,4,110,8,124,3,160,5,124,1,161,1,
-    125,5,124,0,160,6,124,5,124,2,161,2,125,6,116,7,
-    124,4,124,6,131,2,83,0,41,5,97,37,1,0,0,10,
-    32,32,32,32,32,32,32,32,70,105,110,100,32,100,105,115,
-    116,114,105,98,117,116,105,111,110,115,46,10,10,32,32,32,
-    32,32,32,32,32,82,101,116,117,114,110,32,97,110,32,105,
-    116,101,114,97,98,108,101,32,111,102,32,97,108,108,32,68,
-    105,115,116,114,105,98,117,116,105,111,110,32,105,110,115,116,
-    97,110,99,101,115,32,99,97,112,97,98,108,101,32,111,102,
-    10,32,32,32,32,32,32,32,32,108,111,97,100,105,110,103,
-    32,116,104,101,32,109,101,116,97,100,97,116,97,32,102,111,
-    114,32,112,97,99,107,97,103,101,115,32,109,97,116,99,104,
-    105,110,103,32,116,104,101,32,96,96,110,97,109,101,96,96,
+    46,102,105,110,100,95,109,111,100,117,108,101,99,2,0,0,
+    0,0,0,0,0,0,0,0,0,5,0,0,0,4,0,0,
+    0,67,0,0,0,115,58,0,0,0,100,1,100,2,108,0,
+    109,1,125,2,109,2,125,3,1,0,124,1,100,3,107,8,
+    114,32,124,3,160,3,161,0,125,1,124,0,160,4,124,1,
+    106,5,124,1,106,6,161,2,125,4,116,7,124,2,124,4,
+    131,2,83,0,41,4,97,32,1,0,0,10,32,32,32,32,
+    32,32,32,32,70,105,110,100,32,100,105,115,116,114,105,98,
+    117,116,105,111,110,115,46,10,10,32,32,32,32,32,32,32,
+    32,82,101,116,117,114,110,32,97,110,32,105,116,101,114,97,
+    98,108,101,32,111,102,32,97,108,108,32,68,105,115,116,114,
+    105,98,117,116,105,111,110,32,105,110,115,116,97,110,99,101,
+    115,32,99,97,112,97,98,108,101,32,111,102,10,32,32,32,
+    32,32,32,32,32,108,111,97,100,105,110,103,32,116,104,101,
+    32,109,101,116,97,100,97,116,97,32,102,111,114,32,112,97,
+    99,107,97,103,101,115,32,109,97,116,99,104,105,110,103,32,
+    96,96,99,111,110,116,101,120,116,46,110,97,109,101,96,96,
     10,32,32,32,32,32,32,32,32,40,111,114,32,97,108,108,
-    32,110,97,109,101,115,32,105,102,32,110,111,116,32,115,117,
-    112,112,108,105,101,100,41,32,97,108,111,110,103,32,116,104,
-    101,32,112,97,116,104,115,32,105,110,32,116,104,101,32,108,
-    105,115,116,10,32,32,32,32,32,32,32,32,111,102,32,100,
-    105,114,101,99,116,111,114,105,101,115,32,96,96,112,97,116,
-    104,96,96,32,40,100,101,102,97,117,108,116,115,32,116,111,
-    32,115,121,115,46,112,97,116,104,41,46,10,32,32,32,32,
-    32,32,32,32,114,72,0,0,0,78,41,1,218,16,80,97,
-    116,104,68,105,115,116,114,105,98,117,116,105,111,110,122,2,
-    46,42,41,8,218,2,114,101,90,18,105,109,112,111,114,116,
-    108,105,98,46,109,101,116,97,100,97,116,97,114,58,1,0,
-    0,114,8,0,0,0,114,43,0,0,0,90,6,101,115,99,
-    97,112,101,218,13,95,115,101,97,114,99,104,95,112,97,116,
-    104,115,218,3,109,97,112,41,7,114,192,0,0,0,114,116,
-    0,0,0,114,43,0,0,0,114,59,1,0,0,114,58,1,
-    0,0,218,7,112,97,116,116,101,114,110,90,5,102,111,117,
-    110,100,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
-    0,218,18,102,105,110,100,95,100,105,115,116,114,105,98,117,
-    116,105,111,110,115,92,5,0,0,115,14,0,0,0,0,10,
-    8,1,12,1,8,1,6,1,22,1,12,1,122,29,80,97,
-    116,104,70,105,110,100,101,114,46,102,105,110,100,95,100,105,
-    115,116,114,105,98,117,116,105,111,110,115,99,3,0,0,0,
-    0,0,0,0,0,0,0,0,4,0,0,0,6,0,0,0,
-    3,0,0,0,115,44,0,0,0,100,1,100,2,108,0,125,
-    3,124,3,106,1,160,2,135,0,135,1,102,2,100,3,100,
-    4,132,8,116,3,136,0,106,4,124,2,131,2,68,0,131,
-    1,161,1,83,0,41,5,122,49,70,105,110,100,32,109,101,
-    116,97,100,97,116,97,32,100,105,114,101,99,116,111,114,105,
-    101,115,32,105,110,32,112,97,116,104,115,32,104,101,117,114,
-    105,115,116,105,99,97,108,108,121,46,114,72,0,0,0,78,
-    99,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,
-    0,5,0,0,0,51,0,0,0,115,26,0,0,0,124,0,
-    93,18,125,1,136,0,160,0,124,1,136,1,161,2,86,0,
-    1,0,113,2,100,0,83,0,114,109,0,0,0,41,1,218,
-    12,95,115,101,97,114,99,104,95,112,97,116,104,41,2,114,
-    31,0,0,0,114,43,0,0,0,169,2,114,192,0,0,0,
-    114,62,1,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    18,1,0,0,114,5,0,0,115,4,0,0,0,4,2,2,
-    255,122,43,80,97,116,104,70,105,110,100,101,114,46,95,115,
-    101,97,114,99,104,95,112,97,116,104,115,46,60,108,111,99,
-    97,108,115,62,46,60,103,101,110,101,120,112,114,62,41,5,
-    218,9,105,116,101,114,116,111,111,108,115,90,5,99,104,97,
-    105,110,90,13,102,114,111,109,95,105,116,101,114,97,98,108,
-    101,114,61,1,0,0,218,12,95,115,119,105,116,99,104,95,
-    112,97,116,104,41,4,114,192,0,0,0,114,62,1,0,0,
-    90,5,112,97,116,104,115,114,66,1,0,0,114,3,0,0,
-    0,114,65,1,0,0,114,6,0,0,0,114,60,1,0,0,
-    110,5,0,0,115,8,0,0,0,0,3,8,1,18,2,10,
-    254,122,24,80,97,116,104,70,105,110,100,101,114,46,95,115,
-    101,97,114,99,104,95,112,97,116,104,115,99,1,0,0,0,
-    0,0,0,0,0,0,0,0,5,0,0,0,10,0,0,0,
-    67,0,0,0,115,96,0,0,0,100,1,100,2,108,0,109,
-    1,125,1,1,0,100,1,100,0,108,2,125,2,100,1,100,
-    0,108,3,125,3,100,3,125,4,124,4,114,48,116,4,106,
-    5,160,6,124,0,161,1,114,86,124,1,116,7,131,1,143,
-    24,1,0,124,2,160,8,124,0,161,1,87,0,2,0,53,
-    0,81,0,82,0,163,0,83,0,81,0,82,0,88,0,124,
-    3,160,8,124,0,161,1,83,0,41,4,78,114,72,0,0,
-    0,41,1,218,8,115,117,112,112,114,101,115,115,70,41,9,
-    90,10,99,111,110,116,101,120,116,108,105,98,114,68,1,0,
-    0,218,7,122,105,112,102,105,108,101,218,7,112,97,116,104,
-    108,105,98,90,2,111,115,114,43,0,0,0,90,6,105,115,
-    102,105,108,101,218,9,69,120,99,101,112,116,105,111,110,90,
-    4,80,97,116,104,41,5,114,43,0,0,0,114,68,1,0,
-    0,114,69,1,0,0,114,70,1,0,0,90,13,80,89,80,
-    89,95,79,80,69,78,95,66,85,71,114,3,0,0,0,114,
-    3,0,0,0,114,6,0,0,0,114,67,1,0,0,119,5,
-    0,0,115,16,0,0,0,0,2,12,1,8,1,8,1,4,
-    1,16,1,10,1,28,1,122,23,80,97,116,104,70,105,110,
-    100,101,114,46,95,115,119,105,116,99,104,95,112,97,116,104,
-    99,3,0,0,0,0,0,0,0,0,0,0,0,6,0,0,
-    0,5,0,0,0,67,0,0,0,115,44,0,0,0,100,1,
-    100,0,108,0,125,3,100,2,125,4,124,4,106,1,124,1,
-    100,3,141,1,125,5,124,3,106,2,124,5,124,2,106,3,
+    32,110,97,109,101,115,32,105,102,32,96,96,78,111,110,101,
+    96,96,32,105,110,100,105,99,97,116,101,100,41,32,97,108,
+    111,110,103,32,116,104,101,32,112,97,116,104,115,32,105,110,
+    32,116,104,101,32,108,105,115,116,10,32,32,32,32,32,32,
+    32,32,111,102,32,100,105,114,101,99,116,111,114,105,101,115,
+    32,96,96,99,111,110,116,101,120,116,46,112,97,116,104,96,
+    96,46,10,32,32,32,32,32,32,32,32,114,72,0,0,0,
+    41,2,218,16,80,97,116,104,68,105,115,116,114,105,98,117,
+    116,105,111,110,218,18,68,105,115,116,114,105,98,117,116,105,
+    111,110,70,105,110,100,101,114,78,41,8,90,18,105,109,112,
+    111,114,116,108,105,98,46,109,101,116,97,100,97,116,97,114,
+    58,1,0,0,114,59,1,0,0,90,7,67,111,110,116,101,
+    120,116,218,13,95,115,101,97,114,99,104,95,112,97,116,104,
+    115,218,7,112,97,116,116,101,114,110,114,43,0,0,0,218,
+    3,109,97,112,41,5,114,118,0,0,0,90,7,99,111,110,
+    116,101,120,116,114,58,1,0,0,114,59,1,0,0,90,5,
+    102,111,117,110,100,114,3,0,0,0,114,3,0,0,0,114,
+    6,0,0,0,218,18,102,105,110,100,95,100,105,115,116,114,
+    105,98,117,116,105,111,110,115,92,5,0,0,115,10,0,0,
+    0,0,10,16,1,8,1,8,1,16,1,122,29,80,97,116,
+    104,70,105,110,100,101,114,46,102,105,110,100,95,100,105,115,
+    116,114,105,98,117,116,105,111,110,115,99,3,0,0,0,0,
+    0,0,0,0,0,0,0,4,0,0,0,6,0,0,0,3,
+    0,0,0,115,44,0,0,0,100,1,100,2,108,0,125,3,
+    124,3,106,1,160,2,135,0,135,1,102,2,100,3,100,4,
+    132,8,116,3,136,0,106,4,124,2,131,2,68,0,131,1,
+    161,1,83,0,41,5,122,49,70,105,110,100,32,109,101,116,
+    97,100,97,116,97,32,100,105,114,101,99,116,111,114,105,101,
+    115,32,105,110,32,112,97,116,104,115,32,104,101,117,114,105,
+    115,116,105,99,97,108,108,121,46,114,72,0,0,0,78,99,
+    1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,
+    5,0,0,0,51,0,0,0,115,26,0,0,0,124,0,93,
+    18,125,1,136,0,160,0,124,1,136,1,161,2,86,0,1,
+    0,113,2,100,0,83,0,114,109,0,0,0,41,1,218,12,
+    95,115,101,97,114,99,104,95,112,97,116,104,41,2,114,31,
+    0,0,0,114,43,0,0,0,169,2,114,192,0,0,0,114,
+    61,1,0,0,114,3,0,0,0,114,6,0,0,0,114,18,
+    1,0,0,112,5,0,0,115,4,0,0,0,4,2,2,255,
+    122,43,80,97,116,104,70,105,110,100,101,114,46,95,115,101,
+    97,114,99,104,95,112,97,116,104,115,46,60,108,111,99,97,
+    108,115,62,46,60,103,101,110,101,120,112,114,62,41,5,218,
+    9,105,116,101,114,116,111,111,108,115,90,5,99,104,97,105,
+    110,90,13,102,114,111,109,95,105,116,101,114,97,98,108,101,
+    114,62,1,0,0,218,12,95,115,119,105,116,99,104,95,112,
+    97,116,104,41,4,114,192,0,0,0,114,61,1,0,0,90,
+    5,112,97,116,104,115,114,66,1,0,0,114,3,0,0,0,
+    114,65,1,0,0,114,6,0,0,0,114,60,1,0,0,108,
+    5,0,0,115,8,0,0,0,0,3,8,1,18,2,10,254,
+    122,24,80,97,116,104,70,105,110,100,101,114,46,95,115,101,
+    97,114,99,104,95,112,97,116,104,115,99,1,0,0,0,0,
+    0,0,0,0,0,0,0,5,0,0,0,10,0,0,0,67,
+    0,0,0,115,96,0,0,0,100,1,100,2,108,0,109,1,
+    125,1,1,0,100,1,100,0,108,2,125,2,100,1,100,0,
+    108,3,125,3,100,3,125,4,124,4,114,48,116,4,106,5,
+    160,6,124,0,161,1,114,86,124,1,116,7,131,1,143,24,
+    1,0,124,2,160,8,124,0,161,1,87,0,2,0,53,0,
+    81,0,82,0,163,0,83,0,81,0,82,0,88,0,124,3,
+    160,8,124,0,161,1,83,0,41,4,78,114,72,0,0,0,
+    41,1,218,8,115,117,112,112,114,101,115,115,70,41,9,90,
+    10,99,111,110,116,101,120,116,108,105,98,114,68,1,0,0,
+    218,7,122,105,112,102,105,108,101,218,7,112,97,116,104,108,
+    105,98,90,2,111,115,114,43,0,0,0,90,6,105,115,102,
+    105,108,101,218,9,69,120,99,101,112,116,105,111,110,90,4,
+    80,97,116,104,41,5,114,43,0,0,0,114,68,1,0,0,
+    114,69,1,0,0,114,70,1,0,0,90,13,80,89,80,89,
+    95,79,80,69,78,95,66,85,71,114,3,0,0,0,114,3,
+    0,0,0,114,6,0,0,0,114,67,1,0,0,117,5,0,
+    0,115,16,0,0,0,0,2,12,1,8,1,8,1,4,1,
+    16,1,10,1,28,1,122,23,80,97,116,104,70,105,110,100,
+    101,114,46,95,115,119,105,116,99,104,95,112,97,116,104,99,
+    3,0,0,0,0,0,0,0,0,0,0,0,6,0,0,0,
+    5,0,0,0,67,0,0,0,115,44,0,0,0,100,1,100,
+    0,108,0,125,3,100,2,125,4,124,4,106,1,124,1,100,
+    3,141,1,125,5,124,3,106,2,124,5,124,2,106,3,124,
+    3,106,4,100,4,141,3,83,0,41,5,78,114,72,0,0,
+    0,122,32,123,112,97,116,116,101,114,110,125,40,45,46,42,
+    41,63,92,46,40,100,105,115,116,124,101,103,103,41,45,105,
+    110,102,111,169,1,114,61,1,0,0,169,1,114,82,0,0,
+    0,41,5,218,2,114,101,114,61,0,0,0,90,5,109,97,
+    116,99,104,114,116,0,0,0,218,10,73,71,78,79,82,69,
+    67,65,83,69,169,6,114,192,0,0,0,218,10,110,111,114,
+    109,97,108,105,122,101,100,114,40,1,0,0,114,74,1,0,
+    0,90,8,116,101,109,112,108,97,116,101,90,8,109,97,110,
+    105,102,101,115,116,114,3,0,0,0,114,3,0,0,0,114,
+    6,0,0,0,218,13,95,109,97,116,99,104,101,115,95,105,
+    110,102,111,128,5,0,0,115,8,0,0,0,0,2,8,1,
+    4,1,12,1,122,24,80,97,116,104,70,105,110,100,101,114,
+    46,95,109,97,116,99,104,101,115,95,105,110,102,111,99,3,
+    0,0,0,0,0,0,0,0,0,0,0,6,0,0,0,5,
+    0,0,0,67,0,0,0,115,46,0,0,0,100,1,100,0,
+    108,0,125,3,100,2,125,4,124,4,106,1,124,1,100,3,
+    141,1,125,5,124,3,106,2,124,5,116,3,124,2,131,1,
     124,3,106,4,100,4,141,3,83,0,41,5,78,114,72,0,
-    0,0,122,32,123,112,97,116,116,101,114,110,125,40,45,46,
-    42,41,63,92,46,40,100,105,115,116,124,101,103,103,41,45,
-    105,110,102,111,169,1,114,62,1,0,0,169,1,114,82,0,
-    0,0,41,5,114,59,1,0,0,114,61,0,0,0,90,5,
-    109,97,116,99,104,114,116,0,0,0,218,10,73,71,78,79,
-    82,69,67,65,83,69,169,6,114,192,0,0,0,218,10,110,
-    111,114,109,97,108,105,122,101,100,114,40,1,0,0,114,59,
-    1,0,0,90,8,116,101,109,112,108,97,116,101,90,8,109,
-    97,110,105,102,101,115,116,114,3,0,0,0,114,3,0,0,
-    0,114,6,0,0,0,218,13,95,109,97,116,99,104,101,115,
-    95,105,110,102,111,130,5,0,0,115,8,0,0,0,0,2,
-    8,1,4,1,12,1,122,24,80,97,116,104,70,105,110,100,
-    101,114,46,95,109,97,116,99,104,101,115,95,105,110,102,111,
-    99,3,0,0,0,0,0,0,0,0,0,0,0,6,0,0,
-    0,5,0,0,0,67,0,0,0,115,46,0,0,0,100,1,
-    100,0,108,0,125,3,100,2,125,4,124,4,106,1,124,1,
-    100,3,141,1,125,5,124,3,106,2,124,5,116,3,124,2,
-    131,1,124,3,106,4,100,4,141,3,83,0,41,5,78,114,
-    72,0,0,0,122,30,123,112,97,116,116,101,114,110,125,45,
-    46,42,92,46,101,103,103,91,92,92,47,93,69,71,71,45,
-    73,78,70,79,114,72,1,0,0,114,73,1,0,0,41,5,
-    114,59,1,0,0,114,61,0,0,0,90,6,115,101,97,114,
-    99,104,114,84,0,0,0,114,74,1,0,0,114,75,1,0,
-    0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
-    218,15,95,109,97,116,99,104,101,115,95,108,101,103,97,99,
-    121,137,5,0,0,115,8,0,0,0,0,2,8,1,4,1,
-    12,1,122,26,80,97,116,104,70,105,110,100,101,114,46,95,
-    109,97,116,99,104,101,115,95,108,101,103,97,99,121,99,3,
-    0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,4,
-    0,0,0,3,0,0,0,115,48,0,0,0,124,1,160,0,
-    161,0,115,12,100,1,83,0,124,2,160,1,100,2,100,3,
-    161,2,137,1,135,0,135,1,102,2,100,4,100,5,132,8,
-    124,1,160,2,161,0,68,0,131,1,83,0,41,6,78,114,
-    3,0,0,0,250,1,45,114,44,0,0,0,99,1,0,0,
-    0,0,0,0,0,0,0,0,0,2,0,0,0,5,0,0,
-    0,51,0,0,0,115,42,0,0,0,124,0,93,34,125,1,
-    136,0,160,0,136,1,124,1,161,2,115,30,136,0,160,1,
-    136,1,124,1,161,2,114,2,124,1,86,0,1,0,113,2,
-    100,0,83,0,114,109,0,0,0,41,2,114,77,1,0,0,
-    114,78,1,0,0,41,2,114,31,0,0,0,114,40,1,0,
-    0,169,2,114,192,0,0,0,114,76,1,0,0,114,3,0,
-    0,0,114,6,0,0,0,114,18,1,0,0,149,5,0,0,
-    115,8,0,0,0,4,0,2,1,12,1,12,254,122,42,80,
-    97,116,104,70,105,110,100,101,114,46,95,115,101,97,114,99,
-    104,95,112,97,116,104,46,60,108,111,99,97,108,115,62,46,
-    60,103,101,110,101,120,112,114,62,41,3,90,6,105,115,95,
-    100,105,114,114,66,0,0,0,90,7,105,116,101,114,100,105,
-    114,41,3,114,192,0,0,0,90,4,114,111,111,116,114,62,
-    1,0,0,114,3,0,0,0,114,80,1,0,0,114,6,0,
-    0,0,114,64,1,0,0,144,5,0,0,115,8,0,0,0,
-    0,2,8,1,4,1,12,1,122,23,80,97,116,104,70,105,
-    110,100,101,114,46,95,115,101,97,114,99,104,95,112,97,116,
-    104,41,1,78,41,2,78,78,41,1,78,41,2,78,78,41,
-    19,114,124,0,0,0,114,123,0,0,0,114,125,0,0,0,
-    114,126,0,0,0,114,206,0,0,0,114,45,1,0,0,114,
-    51,1,0,0,114,53,1,0,0,114,54,1,0,0,114,57,
-    1,0,0,114,202,0,0,0,114,205,0,0,0,114,63,1,
-    0,0,114,60,1,0,0,218,12,115,116,97,116,105,99,109,
-    101,116,104,111,100,114,67,1,0,0,114,77,1,0,0,114,
-    78,1,0,0,114,64,1,0,0,114,3,0,0,0,114,3,
-    0,0,0,114,3,0,0,0,114,6,0,0,0,114,44,1,
-    0,0,215,4,0,0,115,54,0,0,0,8,2,4,2,2,
-    1,10,9,2,1,10,12,2,1,10,21,2,1,10,14,2,
-    1,12,31,2,1,12,23,2,1,12,12,2,1,12,17,2,
-    1,10,8,2,1,10,10,2,1,10,6,2,1,10,6,2,
-    1,114,44,1,0,0,99,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,3,0,0,0,64,0,0,0,115,
-    90,0,0,0,101,0,90,1,100,0,90,2,100,1,90,3,
-    100,2,100,3,132,0,90,4,100,4,100,5,132,0,90,5,
-    101,6,90,7,100,6,100,7,132,0,90,8,100,8,100,9,
-    132,0,90,9,100,19,100,11,100,12,132,1,90,10,100,13,
-    100,14,132,0,90,11,101,12,100,15,100,16,132,0,131,1,
-    90,13,100,17,100,18,132,0,90,14,100,10,83,0,41,20,
-    218,10,70,105,108,101,70,105,110,100,101,114,122,172,70,105,
-    108,101,45,98,97,115,101,100,32,102,105,110,100,101,114,46,
-    10,10,32,32,32,32,73,110,116,101,114,97,99,116,105,111,
-    110,115,32,119,105,116,104,32,116,104,101,32,102,105,108,101,
-    32,115,121,115,116,101,109,32,97,114,101,32,99,97,99,104,
-    101,100,32,102,111,114,32,112,101,114,102,111,114,109,97,110,
-    99,101,44,32,98,101,105,110,103,10,32,32,32,32,114,101,
-    102,114,101,115,104,101,100,32,119,104,101,110,32,116,104,101,
-    32,100,105,114,101,99,116,111,114,121,32,116,104,101,32,102,
-    105,110,100,101,114,32,105,115,32,104,97,110,100,108,105,110,
-    103,32,104,97,115,32,98,101,101,110,32,109,111,100,105,102,
-    105,101,100,46,10,10,32,32,32,32,99,2,0,0,0,0,
-    0,0,0,0,0,0,0,5,0,0,0,6,0,0,0,7,
-    0,0,0,115,84,0,0,0,103,0,125,3,124,2,68,0,
-    93,32,92,2,137,0,125,4,124,3,160,0,135,0,102,1,
-    100,1,100,2,132,8,124,4,68,0,131,1,161,1,1,0,
-    113,8,124,3,124,0,95,1,124,1,112,54,100,3,124,0,
-    95,2,100,4,124,0,95,3,116,4,131,0,124,0,95,5,
-    116,4,131,0,124,0,95,6,100,5,83,0,41,6,122,154,
-    73,110,105,116,105,97,108,105,122,101,32,119,105,116,104,32,
-    116,104,101,32,112,97,116,104,32,116,111,32,115,101,97,114,
-    99,104,32,111,110,32,97,110,100,32,97,32,118,97,114,105,
-    97,98,108,101,32,110,117,109,98,101,114,32,111,102,10,32,
-    32,32,32,32,32,32,32,50,45,116,117,112,108,101,115,32,
-    99,111,110,116,97,105,110,105,110,103,32,116,104,101,32,108,
-    111,97,100,101,114,32,97,110,100,32,116,104,101,32,102,105,
-    108,101,32,115,117,102,102,105,120,101,115,32,116,104,101,32,
-    108,111,97,100,101,114,10,32,32,32,32,32,32,32,32,114,
-    101,99,111,103,110,105,122,101,115,46,99,1,0,0,0,0,
-    0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,51,
-    0,0,0,115,22,0,0,0,124,0,93,14,125,1,124,1,
-    136,0,102,2,86,0,1,0,113,2,100,0,83,0,114,109,
-    0,0,0,114,3,0,0,0,114,15,1,0,0,169,1,114,
-    139,0,0,0,114,3,0,0,0,114,6,0,0,0,114,18,
-    1,0,0,169,5,0,0,115,4,0,0,0,4,0,2,0,
-    122,38,70,105,108,101,70,105,110,100,101,114,46,95,95,105,
-    110,105,116,95,95,46,60,108,111,99,97,108,115,62,46,60,
-    103,101,110,101,120,112,114,62,114,70,0,0,0,114,104,0,
-    0,0,78,41,7,114,166,0,0,0,218,8,95,108,111,97,
-    100,101,114,115,114,43,0,0,0,218,11,95,112,97,116,104,
-    95,109,116,105,109,101,218,3,115,101,116,218,11,95,112,97,
-    116,104,95,99,97,99,104,101,218,19,95,114,101,108,97,120,
-    101,100,95,112,97,116,104,95,99,97,99,104,101,41,5,114,
-    118,0,0,0,114,43,0,0,0,218,14,108,111,97,100,101,
-    114,95,100,101,116,97,105,108,115,90,7,108,111,97,100,101,
-    114,115,114,188,0,0,0,114,3,0,0,0,114,83,1,0,
-    0,114,6,0,0,0,114,208,0,0,0,163,5,0,0,115,
-    16,0,0,0,0,4,4,1,12,1,26,1,6,2,10,1,
-    6,1,8,1,122,19,70,105,108,101,70,105,110,100,101,114,
-    46,95,95,105,110,105,116,95,95,99,1,0,0,0,0,0,
-    0,0,0,0,0,0,1,0,0,0,2,0,0,0,67,0,
-    0,0,115,10,0,0,0,100,1,124,0,95,0,100,2,83,
-    0,41,3,122,31,73,110,118,97,108,105,100,97,116,101,32,
-    116,104,101,32,100,105,114,101,99,116,111,114,121,32,109,116,
-    105,109,101,46,114,104,0,0,0,78,41,1,114,85,1,0,
-    0,114,245,0,0,0,114,3,0,0,0,114,3,0,0,0,
-    114,6,0,0,0,114,45,1,0,0,177,5,0,0,115,2,
-    0,0,0,0,2,122,28,70,105,108,101,70,105,110,100,101,
-    114,46,105,110,118,97,108,105,100,97,116,101,95,99,97,99,
-    104,101,115,99,2,0,0,0,0,0,0,0,0,0,0,0,
-    3,0,0,0,3,0,0,0,67,0,0,0,115,42,0,0,
-    0,124,0,160,0,124,1,161,1,125,2,124,2,100,1,107,
-    8,114,26,100,1,103,0,102,2,83,0,124,2,106,1,124,
-    2,106,2,112,38,103,0,102,2,83,0,41,2,122,197,84,
-    114,121,32,116,111,32,102,105,110,100,32,97,32,108,111,97,
-    100,101,114,32,102,111,114,32,116,104,101,32,115,112,101,99,
-    105,102,105,101,100,32,109,111,100,117,108,101,44,32,111,114,
-    32,116,104,101,32,110,97,109,101,115,112,97,99,101,10,32,
-    32,32,32,32,32,32,32,112,97,99,107,97,103,101,32,112,
-    111,114,116,105,111,110,115,46,32,82,101,116,117,114,110,115,
-    32,40,108,111,97,100,101,114,44,32,108,105,115,116,45,111,
-    102,45,112,111,114,116,105,111,110,115,41,46,10,10,32,32,
-    32,32,32,32,32,32,84,104,105,115,32,109,101,116,104,111,
-    100,32,105,115,32,100,101,112,114,101,99,97,116,101,100,46,
-    32,32,85,115,101,32,102,105,110,100,95,115,112,101,99,40,
-    41,32,105,110,115,116,101,97,100,46,10,10,32,32,32,32,
-    32,32,32,32,78,41,3,114,202,0,0,0,114,139,0,0,
-    0,114,177,0,0,0,41,3,114,118,0,0,0,114,138,0,
-    0,0,114,186,0,0,0,114,3,0,0,0,114,3,0,0,
-    0,114,6,0,0,0,114,136,0,0,0,183,5,0,0,115,
-    8,0,0,0,0,7,10,1,8,1,8,1,122,22,70,105,
-    108,101,70,105,110,100,101,114,46,102,105,110,100,95,108,111,
-    97,100,101,114,99,6,0,0,0,0,0,0,0,0,0,0,
-    0,7,0,0,0,6,0,0,0,67,0,0,0,115,26,0,
-    0,0,124,1,124,2,124,3,131,2,125,6,116,0,124,2,
-    124,3,124,6,124,4,100,1,141,4,83,0,41,2,78,114,
-    176,0,0,0,41,1,114,189,0,0,0,41,7,114,118,0,
-    0,0,114,187,0,0,0,114,138,0,0,0,114,43,0,0,
-    0,90,4,115,109,115,108,114,201,0,0,0,114,139,0,0,
-    0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
-    114,57,1,0,0,195,5,0,0,115,8,0,0,0,0,1,
-    10,1,8,1,2,255,122,20,70,105,108,101,70,105,110,100,
-    101,114,46,95,103,101,116,95,115,112,101,99,78,99,3,0,
-    0,0,0,0,0,0,0,0,0,0,14,0,0,0,8,0,
-    0,0,67,0,0,0,115,98,1,0,0,100,1,125,3,124,
-    1,160,0,100,2,161,1,100,3,25,0,125,4,122,24,116,
-    1,124,0,106,2,112,34,116,3,160,4,161,0,131,1,106,
-    5,125,5,87,0,110,24,4,0,116,6,107,10,114,66,1,
-    0,1,0,1,0,100,4,125,5,89,0,110,2,88,0,124,
-    5,124,0,106,7,107,3,114,92,124,0,160,8,161,0,1,
-    0,124,5,124,0,95,7,116,9,131,0,114,114,124,0,106,
-    10,125,6,124,4,160,11,161,0,125,7,110,10,124,0,106,
-    12,125,6,124,4,125,7,124,7,124,6,107,6,114,218,116,
-    13,124,0,106,2,124,4,131,2,125,8,124,0,106,14,68,
-    0,93,58,92,2,125,9,125,10,100,5,124,9,23,0,125,
-    11,116,13,124,8,124,11,131,2,125,12,116,15,124,12,131,
-    1,114,150,124,0,160,16,124,10,124,1,124,12,124,8,103,
-    1,124,2,161,5,2,0,1,0,83,0,113,150,116,17,124,
-    8,131,1,125,3,124,0,106,14,68,0,93,82,92,2,125,
-    9,125,10,116,13,124,0,106,2,124,4,124,9,23,0,131,
-    2,125,12,116,18,106,19,100,6,124,12,100,3,100,7,141,
-    3,1,0,124,7,124,9,23,0,124,6,107,6,114,224,116,
-    15,124,12,131,1,114,224,124,0,160,16,124,10,124,1,124,
-    12,100,8,124,2,161,5,2,0,1,0,83,0,113,224,124,
-    3,144,1,114,94,116,18,160,19,100,9,124,8,161,2,1,
-    0,116,18,160,20,124,1,100,8,161,2,125,13,124,8,103,
-    1,124,13,95,21,124,13,83,0,100,8,83,0,41,10,122,
-    111,84,114,121,32,116,111,32,102,105,110,100,32,97,32,115,
-    112,101,99,32,102,111,114,32,116,104,101,32,115,112,101,99,
-    105,102,105,101,100,32,109,111,100,117,108,101,46,10,10,32,
-    32,32,32,32,32,32,32,82,101,116,117,114,110,115,32,116,
-    104,101,32,109,97,116,99,104,105,110,103,32,115,112,101,99,
-    44,32,111,114,32,78,111,110,101,32,105,102,32,110,111,116,
-    32,102,111,117,110,100,46,10,32,32,32,32,32,32,32,32,
-    70,114,70,0,0,0,114,27,0,0,0,114,104,0,0,0,
-    114,208,0,0,0,122,9,116,114,121,105,110,103,32,123,125,
-    41,1,90,9,118,101,114,98,111,115,105,116,121,78,122,25,
-    112,111,115,115,105,98,108,101,32,110,97,109,101,115,112,97,
-    99,101,32,102,111,114,32,123,125,41,22,114,40,0,0,0,
-    114,48,0,0,0,114,43,0,0,0,114,2,0,0,0,114,
-    54,0,0,0,114,9,1,0,0,114,49,0,0,0,114,85,
-    1,0,0,218,11,95,102,105,108,108,95,99,97,99,104,101,
-    114,7,0,0,0,114,88,1,0,0,114,105,0,0,0,114,
-    87,1,0,0,114,37,0,0,0,114,84,1,0,0,114,53,
-    0,0,0,114,57,1,0,0,114,55,0,0,0,114,133,0,
-    0,0,114,148,0,0,0,114,182,0,0,0,114,177,0,0,
-    0,41,14,114,118,0,0,0,114,138,0,0,0,114,201,0,
-    0,0,90,12,105,115,95,110,97,109,101,115,112,97,99,101,
-    90,11,116,97,105,108,95,109,111,100,117,108,101,114,168,0,
-    0,0,90,5,99,97,99,104,101,90,12,99,97,99,104,101,
-    95,109,111,100,117,108,101,90,9,98,97,115,101,95,112,97,
-    116,104,114,16,1,0,0,114,187,0,0,0,90,13,105,110,
-    105,116,95,102,105,108,101,110,97,109,101,90,9,102,117,108,
-    108,95,112,97,116,104,114,186,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,202,0,0,0,200,
-    5,0,0,115,74,0,0,0,0,5,4,1,14,1,2,1,
-    24,1,14,1,10,1,10,1,8,1,6,2,6,1,6,1,
-    10,2,6,1,4,2,8,1,12,1,14,1,8,1,10,1,
-    8,1,26,4,8,2,14,1,16,1,16,1,12,1,8,1,
-    10,1,2,0,2,255,10,2,6,1,12,1,12,1,8,1,
-    4,1,122,20,70,105,108,101,70,105,110,100,101,114,46,102,
-    105,110,100,95,115,112,101,99,99,1,0,0,0,0,0,0,
-    0,0,0,0,0,9,0,0,0,10,0,0,0,67,0,0,
-    0,115,190,0,0,0,124,0,106,0,125,1,122,22,116,1,
-    160,2,124,1,112,22,116,1,160,3,161,0,161,1,125,2,
-    87,0,110,30,4,0,116,4,116,5,116,6,102,3,107,10,
-    114,58,1,0,1,0,1,0,103,0,125,2,89,0,110,2,
-    88,0,116,7,106,8,160,9,100,1,161,1,115,84,116,10,
-    124,2,131,1,124,0,95,11,110,74,116,10,131,0,125,3,
-    124,2,68,0,93,56,125,4,124,4,160,12,100,2,161,1,
-    92,3,125,5,125,6,125,7,124,6,114,136,100,3,160,13,
-    124,5,124,7,160,14,161,0,161,2,125,8,110,4,124,5,
-    125,8,124,3,160,15,124,8,161,1,1,0,113,94,124,3,
-    124,0,95,11,116,7,106,8,160,9,116,16,161,1,114,186,
-    100,4,100,5,132,0,124,2,68,0,131,1,124,0,95,17,
-    100,6,83,0,41,7,122,68,70,105,108,108,32,116,104,101,
-    32,99,97,99,104,101,32,111,102,32,112,111,116,101,110,116,
-    105,97,108,32,109,111,100,117,108,101,115,32,97,110,100,32,
-    112,97,99,107,97,103,101,115,32,102,111,114,32,116,104,105,
-    115,32,100,105,114,101,99,116,111,114,121,46,114,0,0,0,
-    0,114,70,0,0,0,114,60,0,0,0,99,1,0,0,0,
-    0,0,0,0,0,0,0,0,2,0,0,0,4,0,0,0,
-    83,0,0,0,115,20,0,0,0,104,0,124,0,93,12,125,
-    1,124,1,160,0,161,0,146,2,113,4,83,0,114,3,0,
-    0,0,41,1,114,105,0,0,0,41,2,114,31,0,0,0,
-    90,2,102,110,114,3,0,0,0,114,3,0,0,0,114,6,
-    0,0,0,218,9,60,115,101,116,99,111,109,112,62,21,6,
-    0,0,115,4,0,0,0,6,0,2,0,122,41,70,105,108,
-    101,70,105,110,100,101,114,46,95,102,105,108,108,95,99,97,
-    99,104,101,46,60,108,111,99,97,108,115,62,46,60,115,101,
-    116,99,111,109,112,62,78,41,18,114,43,0,0,0,114,2,
-    0,0,0,114,6,1,0,0,114,54,0,0,0,114,2,1,
-    0,0,218,15,80,101,114,109,105,115,115,105,111,110,69,114,
-    114,111,114,218,18,78,111,116,65,68,105,114,101,99,116,111,
-    114,121,69,114,114,111,114,114,8,0,0,0,114,9,0,0,
-    0,114,10,0,0,0,114,86,1,0,0,114,87,1,0,0,
-    114,100,0,0,0,114,61,0,0,0,114,105,0,0,0,218,
-    3,97,100,100,114,11,0,0,0,114,88,1,0,0,41,9,
-    114,118,0,0,0,114,43,0,0,0,114,7,1,0,0,90,
-    21,108,111,119,101,114,95,115,117,102,102,105,120,95,99,111,
-    110,116,101,110,116,115,114,40,1,0,0,114,116,0,0,0,
-    114,28,1,0,0,114,16,1,0,0,90,8,110,101,119,95,
-    110,97,109,101,114,3,0,0,0,114,3,0,0,0,114,6,
-    0,0,0,114,90,1,0,0,248,5,0,0,115,34,0,0,
-    0,0,2,6,1,2,1,22,1,20,3,10,3,12,1,12,
-    7,6,1,8,1,16,1,4,1,18,2,4,1,12,1,6,
-    1,12,1,122,22,70,105,108,101,70,105,110,100,101,114,46,
-    95,102,105,108,108,95,99,97,99,104,101,99,1,0,0,0,
-    0,0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,
-    7,0,0,0,115,18,0,0,0,135,0,135,1,102,2,100,
-    1,100,2,132,8,125,2,124,2,83,0,41,3,97,20,1,
-    0,0,65,32,99,108,97,115,115,32,109,101,116,104,111,100,
-    32,119,104,105,99,104,32,114,101,116,117,114,110,115,32,97,
-    32,99,108,111,115,117,114,101,32,116,111,32,117,115,101,32,
-    111,110,32,115,121,115,46,112,97,116,104,95,104,111,111,107,
-    10,32,32,32,32,32,32,32,32,119,104,105,99,104,32,119,
-    105,108,108,32,114,101,116,117,114,110,32,97,110,32,105,110,
-    115,116,97,110,99,101,32,117,115,105,110,103,32,116,104,101,
-    32,115,112,101,99,105,102,105,101,100,32,108,111,97,100,101,
-    114,115,32,97,110,100,32,116,104,101,32,112,97,116,104,10,
-    32,32,32,32,32,32,32,32,99,97,108,108,101,100,32,111,
-    110,32,116,104,101,32,99,108,111,115,117,114,101,46,10,10,
-    32,32,32,32,32,32,32,32,73,102,32,116,104,101,32,112,
-    97,116,104,32,99,97,108,108,101,100,32,111,110,32,116,104,
-    101,32,99,108,111,115,117,114,101,32,105,115,32,110,111,116,
-    32,97,32,100,105,114,101,99,116,111,114,121,44,32,73,109,
-    112,111,114,116,69,114,114,111,114,32,105,115,10,32,32,32,
-    32,32,32,32,32,114,97,105,115,101,100,46,10,10,32,32,
-    32,32,32,32,32,32,99,1,0,0,0,0,0,0,0,0,
-    0,0,0,1,0,0,0,4,0,0,0,19,0,0,0,115,
-    34,0,0,0,116,0,124,0,131,1,115,20,116,1,100,1,
-    124,0,100,2,141,2,130,1,136,0,124,0,102,1,136,1,
-    158,2,142,0,83,0,41,3,122,45,80,97,116,104,32,104,
-    111,111,107,32,102,111,114,32,105,109,112,111,114,116,108,105,
-    98,46,109,97,99,104,105,110,101,114,121,46,70,105,108,101,
-    70,105,110,100,101,114,46,122,30,111,110,108,121,32,100,105,
-    114,101,99,116,111,114,105,101,115,32,97,114,101,32,115,117,
-    112,112,111,114,116,101,100,114,47,0,0,0,41,2,114,55,
-    0,0,0,114,117,0,0,0,114,47,0,0,0,169,2,114,
-    192,0,0,0,114,89,1,0,0,114,3,0,0,0,114,6,
-    0,0,0,218,24,112,97,116,104,95,104,111,111,107,95,102,
-    111,114,95,70,105,108,101,70,105,110,100,101,114,33,6,0,
-    0,115,6,0,0,0,0,2,8,1,12,1,122,54,70,105,
-    108,101,70,105,110,100,101,114,46,112,97,116,104,95,104,111,
-    111,107,46,60,108,111,99,97,108,115,62,46,112,97,116,104,
-    95,104,111,111,107,95,102,111,114,95,70,105,108,101,70,105,
-    110,100,101,114,114,3,0,0,0,41,3,114,192,0,0,0,
-    114,89,1,0,0,114,96,1,0,0,114,3,0,0,0,114,
-    95,1,0,0,114,6,0,0,0,218,9,112,97,116,104,95,
-    104,111,111,107,23,6,0,0,115,4,0,0,0,0,10,14,
-    6,122,20,70,105,108,101,70,105,110,100,101,114,46,112,97,
-    116,104,95,104,111,111,107,99,1,0,0,0,0,0,0,0,
-    0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0,
-    115,12,0,0,0,100,1,160,0,124,0,106,1,161,1,83,
-    0,41,2,78,122,16,70,105,108,101,70,105,110,100,101,114,
-    40,123,33,114,125,41,41,2,114,61,0,0,0,114,43,0,
-    0,0,114,245,0,0,0,114,3,0,0,0,114,3,0,0,
-    0,114,6,0,0,0,114,38,1,0,0,41,6,0,0,115,
-    2,0,0,0,0,1,122,19,70,105,108,101,70,105,110,100,
-    101,114,46,95,95,114,101,112,114,95,95,41,1,78,41,15,
-    114,124,0,0,0,114,123,0,0,0,114,125,0,0,0,114,
-    126,0,0,0,114,208,0,0,0,114,45,1,0,0,114,142,
-    0,0,0,114,205,0,0,0,114,136,0,0,0,114,57,1,
-    0,0,114,202,0,0,0,114,90,1,0,0,114,206,0,0,
-    0,114,97,1,0,0,114,38,1,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    82,1,0,0,154,5,0,0,115,22,0,0,0,8,2,4,
-    7,8,14,8,4,4,2,8,12,8,5,10,48,8,31,2,
-    1,10,17,114,82,1,0,0,99,4,0,0,0,0,0,0,
-    0,0,0,0,0,6,0,0,0,8,0,0,0,67,0,0,
-    0,115,146,0,0,0,124,0,160,0,100,1,161,1,125,4,
-    124,0,160,0,100,2,161,1,125,5,124,4,115,66,124,5,
-    114,36,124,5,106,1,125,4,110,30,124,2,124,3,107,2,
-    114,56,116,2,124,1,124,2,131,2,125,4,110,10,116,3,
-    124,1,124,2,131,2,125,4,124,5,115,84,116,4,124,1,
-    124,2,124,4,100,3,141,3,125,5,122,36,124,5,124,0,
-    100,2,60,0,124,4,124,0,100,1,60,0,124,2,124,0,
-    100,4,60,0,124,3,124,0,100,5,60,0,87,0,110,20,
-    4,0,116,5,107,10,114,140,1,0,1,0,1,0,89,0,
-    110,2,88,0,100,0,83,0,41,6,78,218,10,95,95,108,
-    111,97,100,101,114,95,95,218,8,95,95,115,112,101,99,95,
-    95,114,83,1,0,0,90,8,95,95,102,105,108,101,95,95,
-    90,10,95,95,99,97,99,104,101,100,95,95,41,6,218,3,
-    103,101,116,114,139,0,0,0,114,14,1,0,0,114,8,1,
-    0,0,114,189,0,0,0,114,71,1,0,0,41,6,90,2,
-    110,115,114,116,0,0,0,90,8,112,97,116,104,110,97,109,
-    101,90,9,99,112,97,116,104,110,97,109,101,114,139,0,0,
-    0,114,186,0,0,0,114,3,0,0,0,114,3,0,0,0,
-    114,6,0,0,0,218,14,95,102,105,120,95,117,112,95,109,
-    111,100,117,108,101,47,6,0,0,115,34,0,0,0,0,2,
-    10,1,10,1,4,1,4,1,8,1,8,1,12,2,10,1,
-    4,1,14,1,2,1,8,1,8,1,8,1,12,1,14,2,
-    114,101,1,0,0,99,0,0,0,0,0,0,0,0,0,0,
-    0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,38,
-    0,0,0,116,0,116,1,160,2,161,0,102,2,125,0,116,
-    3,116,4,102,2,125,1,116,5,116,6,102,2,125,2,124,
-    0,124,1,124,2,103,3,83,0,41,1,122,95,82,101,116,
-    117,114,110,115,32,97,32,108,105,115,116,32,111,102,32,102,
-    105,108,101,45,98,97,115,101,100,32,109,111,100,117,108,101,
-    32,108,111,97,100,101,114,115,46,10,10,32,32,32,32,69,
-    97,99,104,32,105,116,101,109,32,105,115,32,97,32,116,117,
-    112,108,101,32,40,108,111,97,100,101,114,44,32,115,117,102,
-    102,105,120,101,115,41,46,10,32,32,32,32,41,7,114,251,
-    0,0,0,114,162,0,0,0,218,18,101,120,116,101,110,115,
-    105,111,110,95,115,117,102,102,105,120,101,115,114,8,1,0,
-    0,114,101,0,0,0,114,14,1,0,0,114,88,0,0,0,
-    41,3,90,10,101,120,116,101,110,115,105,111,110,115,90,6,
-    115,111,117,114,99,101,90,8,98,121,116,101,99,111,100,101,
-    114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    183,0,0,0,70,6,0,0,115,8,0,0,0,0,5,12,
-    1,8,1,8,1,114,183,0,0,0,99,1,0,0,0,0,
-    0,0,0,0,0,0,0,12,0,0,0,9,0,0,0,67,
-    0,0,0,115,178,1,0,0,124,0,97,0,116,0,106,1,
-    97,1,116,0,106,2,97,2,116,1,106,3,116,4,25,0,
-    125,1,100,1,68,0,93,48,125,2,124,2,116,1,106,3,
-    107,7,114,56,116,0,160,5,124,2,161,1,125,3,110,10,
-    116,1,106,3,124,2,25,0,125,3,116,6,124,1,124,2,
-    124,3,131,3,1,0,113,30,100,2,100,3,103,1,102,2,
-    100,4,100,5,100,3,103,2,102,2,102,2,125,4,124,4,
-    68,0,93,110,92,2,125,5,125,6,116,7,100,6,100,7,
-    132,0,124,6,68,0,131,1,131,1,115,136,74,0,130,1,
-    124,6,100,8,25,0,125,7,124,5,116,1,106,3,107,6,
-    114,170,116,1,106,3,124,5,25,0,125,8,1,0,113,226,
-    113,106,122,20,116,0,160,5,124,5,161,1,125,8,87,0,
-    1,0,113,226,87,0,113,106,4,0,116,8,107,10,114,214,
-    1,0,1,0,1,0,89,0,113,106,89,0,113,106,88,0,
-    113,106,116,8,100,9,131,1,130,1,116,6,124,1,100,10,
-    124,8,131,3,1,0,116,6,124,1,100,11,124,7,131,3,
-    1,0,116,6,124,1,100,12,100,13,160,9,124,6,161,1,
-    131,3,1,0,116,6,124,1,100,14,100,15,100,16,132,0,
-    124,6,68,0,131,1,131,3,1,0,116,0,160,5,100,17,
-    161,1,125,9,116,6,124,1,100,17,124,9,131,3,1,0,
-    116,0,160,5,100,18,161,1,125,10,116,6,124,1,100,18,
-    124,10,131,3,1,0,124,5,100,4,107,2,144,1,114,110,
-    116,0,160,5,100,19,161,1,125,11,116,6,124,1,100,20,
-    124,11,131,3,1,0,116,6,124,1,100,21,116,10,131,0,
-    131,3,1,0,116,11,160,12,116,2,160,13,161,0,161,1,
-    1,0,124,5,100,4,107,2,144,1,114,174,116,14,160,15,
-    100,22,161,1,1,0,100,23,116,11,107,6,144,1,114,174,
-    100,24,116,16,95,17,100,25,83,0,41,26,122,205,83,101,
-    116,117,112,32,116,104,101,32,112,97,116,104,45,98,97,115,
-    101,100,32,105,109,112,111,114,116,101,114,115,32,102,111,114,
-    32,105,109,112,111,114,116,108,105,98,32,98,121,32,105,109,
-    112,111,114,116,105,110,103,32,110,101,101,100,101,100,10,32,
-    32,32,32,98,117,105,108,116,45,105,110,32,109,111,100,117,
-    108,101,115,32,97,110,100,32,105,110,106,101,99,116,105,110,
-    103,32,116,104,101,109,32,105,110,116,111,32,116,104,101,32,
-    103,108,111,98,97,108,32,110,97,109,101,115,112,97,99,101,
-    46,10,10,32,32,32,32,79,116,104,101,114,32,99,111,109,
-    112,111,110,101,110,116,115,32,97,114,101,32,101,120,116,114,
-    97,99,116,101,100,32,102,114,111,109,32,116,104,101,32,99,
-    111,114,101,32,98,111,111,116,115,116,114,97,112,32,109,111,
-    100,117,108,101,46,10,10,32,32,32,32,41,4,114,63,0,
-    0,0,114,74,0,0,0,218,8,98,117,105,108,116,105,110,
-    115,114,159,0,0,0,90,5,112,111,115,105,120,250,1,47,
-    90,2,110,116,250,1,92,99,1,0,0,0,0,0,0,0,
-    0,0,0,0,2,0,0,0,3,0,0,0,115,0,0,0,
-    115,26,0,0,0,124,0,93,18,125,1,116,0,124,1,131,
-    1,100,0,107,2,86,0,1,0,113,2,100,1,83,0,41,
-    2,114,38,0,0,0,78,41,1,114,22,0,0,0,41,2,
-    114,31,0,0,0,114,94,0,0,0,114,3,0,0,0,114,
-    3,0,0,0,114,6,0,0,0,114,18,1,0,0,106,6,
-    0,0,115,4,0,0,0,4,0,2,0,122,25,95,115,101,
-    116,117,112,46,60,108,111,99,97,108,115,62,46,60,103,101,
-    110,101,120,112,114,62,114,72,0,0,0,122,30,105,109,112,
-    111,114,116,108,105,98,32,114,101,113,117,105,114,101,115,32,
-    112,111,115,105,120,32,111,114,32,110,116,114,2,0,0,0,
-    114,34,0,0,0,114,30,0,0,0,114,39,0,0,0,114,
-    57,0,0,0,99,1,0,0,0,0,0,0,0,0,0,0,
-    0,2,0,0,0,4,0,0,0,83,0,0,0,115,22,0,
-    0,0,104,0,124,0,93,14,125,1,100,0,124,1,155,0,
-    157,2,146,2,113,4,83,0,41,1,114,73,0,0,0,114,
-    3,0,0,0,41,2,114,31,0,0,0,218,1,115,114,3,
-    0,0,0,114,3,0,0,0,114,6,0,0,0,114,91,1,
-    0,0,122,6,0,0,115,4,0,0,0,6,0,2,0,122,
-    25,95,115,101,116,117,112,46,60,108,111,99,97,108,115,62,
-    46,60,115,101,116,99,111,109,112,62,90,7,95,116,104,114,
-    101,97,100,90,8,95,119,101,97,107,114,101,102,90,6,119,
-    105,110,114,101,103,114,191,0,0,0,114,7,0,0,0,122,
-    4,46,112,121,119,122,6,95,100,46,112,121,100,84,78,41,
-    18,114,133,0,0,0,114,8,0,0,0,114,162,0,0,0,
-    114,30,1,0,0,114,124,0,0,0,90,18,95,98,117,105,
-    108,116,105,110,95,102,114,111,109,95,110,97,109,101,114,128,
-    0,0,0,218,3,97,108,108,114,117,0,0,0,114,35,0,
-    0,0,114,13,0,0,0,114,20,1,0,0,114,166,0,0,
-    0,114,102,1,0,0,114,101,0,0,0,114,185,0,0,0,
-    114,190,0,0,0,114,194,0,0,0,41,12,218,17,95,98,
-    111,111,116,115,116,114,97,112,95,109,111,100,117,108,101,90,
-    11,115,101,108,102,95,109,111,100,117,108,101,90,12,98,117,
-    105,108,116,105,110,95,110,97,109,101,90,14,98,117,105,108,
-    116,105,110,95,109,111,100,117,108,101,90,10,111,115,95,100,
-    101,116,97,105,108,115,90,10,98,117,105,108,116,105,110,95,
-    111,115,114,30,0,0,0,114,34,0,0,0,90,9,111,115,
-    95,109,111,100,117,108,101,90,13,116,104,114,101,97,100,95,
-    109,111,100,117,108,101,90,14,119,101,97,107,114,101,102,95,
-    109,111,100,117,108,101,90,13,119,105,110,114,101,103,95,109,
-    111,100,117,108,101,114,3,0,0,0,114,3,0,0,0,114,
-    6,0,0,0,218,6,95,115,101,116,117,112,81,6,0,0,
-    115,78,0,0,0,0,8,4,1,6,1,6,3,10,1,8,
-    1,10,1,12,2,10,1,14,3,22,1,12,2,22,1,8,
-    1,10,1,10,1,6,2,2,1,10,1,10,1,14,1,12,
-    2,8,1,12,1,12,1,18,1,22,3,10,1,12,3,10,
-    1,12,3,10,1,10,1,12,3,14,1,14,1,10,1,10,
-    1,10,1,114,109,1,0,0,99,1,0,0,0,0,0,0,
-    0,0,0,0,0,2,0,0,0,4,0,0,0,67,0,0,
-    0,115,50,0,0,0,116,0,124,0,131,1,1,0,116,1,
-    131,0,125,1,116,2,106,3,160,4,116,5,106,6,124,1,
-    142,0,103,1,161,1,1,0,116,2,106,7,160,8,116,9,
-    161,1,1,0,100,1,83,0,41,2,122,41,73,110,115,116,
-    97,108,108,32,116,104,101,32,112,97,116,104,45,98,97,115,
-    101,100,32,105,109,112,111,114,116,32,99,111,109,112,111,110,
-    101,110,116,115,46,78,41,10,114,109,1,0,0,114,183,0,
-    0,0,114,8,0,0,0,114,50,1,0,0,114,166,0,0,
-    0,114,82,1,0,0,114,97,1,0,0,218,9,109,101,116,
-    97,95,112,97,116,104,114,185,0,0,0,114,44,1,0,0,
-    41,2,114,108,1,0,0,90,17,115,117,112,112,111,114,116,
-    101,100,95,108,111,97,100,101,114,115,114,3,0,0,0,114,
-    3,0,0,0,114,6,0,0,0,218,8,95,105,110,115,116,
-    97,108,108,146,6,0,0,115,8,0,0,0,0,2,8,1,
-    6,1,20,1,114,111,1,0,0,41,1,114,59,0,0,0,
-    41,1,78,41,3,78,78,78,41,2,114,72,0,0,0,114,
-    72,0,0,0,41,1,84,41,1,78,41,1,78,41,63,114,
-    126,0,0,0,114,12,0,0,0,90,37,95,67,65,83,69,
-    95,73,78,83,69,78,83,73,84,73,86,69,95,80,76,65,
-    84,70,79,82,77,83,95,66,89,84,69,83,95,75,69,89,
-    114,11,0,0,0,114,13,0,0,0,114,20,0,0,0,114,
-    26,0,0,0,114,28,0,0,0,114,37,0,0,0,114,46,
-    0,0,0,114,48,0,0,0,114,52,0,0,0,114,53,0,
-    0,0,114,55,0,0,0,114,58,0,0,0,114,68,0,0,
-    0,218,4,116,121,112,101,218,8,95,95,99,111,100,101,95,
-    95,114,161,0,0,0,114,18,0,0,0,114,147,0,0,0,
-    114,17,0,0,0,114,23,0,0,0,114,235,0,0,0,114,
-    91,0,0,0,114,87,0,0,0,114,101,0,0,0,114,88,
-    0,0,0,90,23,68,69,66,85,71,95,66,89,84,69,67,
-    79,68,69,95,83,85,70,70,73,88,69,83,90,27,79,80,
-    84,73,77,73,90,69,68,95,66,89,84,69,67,79,68,69,
-    95,83,85,70,70,73,88,69,83,114,97,0,0,0,114,102,
-    0,0,0,114,108,0,0,0,114,112,0,0,0,114,114,0,
-    0,0,114,135,0,0,0,114,142,0,0,0,114,151,0,0,
-    0,114,155,0,0,0,114,157,0,0,0,114,164,0,0,0,
-    114,169,0,0,0,114,170,0,0,0,114,175,0,0,0,218,
-    6,111,98,106,101,99,116,114,184,0,0,0,114,189,0,0,
-    0,114,190,0,0,0,114,207,0,0,0,114,220,0,0,0,
-    114,238,0,0,0,114,8,1,0,0,114,14,1,0,0,114,
-    20,1,0,0,114,251,0,0,0,114,21,1,0,0,114,42,
-    1,0,0,114,44,1,0,0,114,82,1,0,0,114,101,1,
-    0,0,114,183,0,0,0,114,109,1,0,0,114,111,1,0,
-    0,114,3,0,0,0,114,3,0,0,0,114,3,0,0,0,
-    114,6,0,0,0,218,8,60,109,111,100,117,108,101,62,1,
-    0,0,0,115,126,0,0,0,4,22,4,1,4,1,2,1,
-    2,255,4,4,8,17,8,5,8,5,8,6,8,6,8,12,
-    8,10,8,9,8,5,8,7,8,9,10,22,10,127,0,14,
-    16,1,12,2,4,1,4,2,6,2,6,2,8,2,16,71,
-    8,40,8,19,8,12,8,12,8,28,8,17,8,33,8,28,
-    8,24,10,13,10,10,10,11,8,14,6,3,4,1,2,255,
-    12,68,14,64,14,29,16,127,0,17,14,72,18,45,18,26,
-    4,3,18,53,14,63,14,42,14,127,0,68,14,127,0,22,
-    10,23,8,11,8,65,
+    0,0,122,30,123,112,97,116,116,101,114,110,125,45,46,42,
+    92,46,101,103,103,91,92,92,47,93,69,71,71,45,73,78,
+    70,79,114,72,1,0,0,114,73,1,0,0,41,5,114,74,
+    1,0,0,114,61,0,0,0,90,6,115,101,97,114,99,104,
+    114,84,0,0,0,114,75,1,0,0,114,76,1,0,0,114,
+    3,0,0,0,114,3,0,0,0,114,6,0,0,0,218,15,
+    95,109,97,116,99,104,101,115,95,108,101,103,97,99,121,135,
+    5,0,0,115,8,0,0,0,0,2,8,1,4,1,12,1,
+    122,26,80,97,116,104,70,105,110,100,101,114,46,95,109,97,
+    116,99,104,101,115,95,108,101,103,97,99,121,99,3,0,0,
+    0,0,0,0,0,0,0,0,0,3,0,0,0,4,0,0,
+    0,3,0,0,0,115,48,0,0,0,124,1,160,0,161,0,
+    115,12,100,1,83,0,124,2,160,1,100,2,100,3,161,2,
+    137,1,135,0,135,1,102,2,100,4,100,5,132,8,124,1,
+    160,2,161,0,68,0,131,1,83,0,41,6,78,114,3,0,
+    0,0,250,1,45,114,44,0,0,0,99,1,0,0,0,0,
+    0,0,0,0,0,0,0,2,0,0,0,5,0,0,0,51,
+    0,0,0,115,42,0,0,0,124,0,93,34,125,1,136,0,
+    160,0,136,1,124,1,161,2,115,30,136,0,160,1,136,1,
+    124,1,161,2,114,2,124,1,86,0,1,0,113,2,100,0,
+    83,0,114,109,0,0,0,41,2,114,78,1,0,0,114,79,
+    1,0,0,41,2,114,31,0,0,0,114,40,1,0,0,169,
+    2,114,192,0,0,0,114,77,1,0,0,114,3,0,0,0,
+    114,6,0,0,0,114,18,1,0,0,147,5,0,0,115,8,
+    0,0,0,4,0,2,1,12,1,12,254,122,42,80,97,116,
+    104,70,105,110,100,101,114,46,95,115,101,97,114,99,104,95,
+    112,97,116,104,46,60,108,111,99,97,108,115,62,46,60,103,
+    101,110,101,120,112,114,62,41,3,90,6,105,115,95,100,105,
+    114,114,66,0,0,0,90,7,105,116,101,114,100,105,114,41,
+    3,114,192,0,0,0,90,4,114,111,111,116,114,61,1,0,
+    0,114,3,0,0,0,114,81,1,0,0,114,6,0,0,0,
+    114,64,1,0,0,142,5,0,0,115,8,0,0,0,0,2,
+    8,1,4,1,12,1,122,23,80,97,116,104,70,105,110,100,
+    101,114,46,95,115,101,97,114,99,104,95,112,97,116,104,41,
+    1,78,41,2,78,78,41,1,78,41,1,78,41,19,114,124,
+    0,0,0,114,123,0,0,0,114,125,0,0,0,114,126,0,
+    0,0,114,206,0,0,0,114,45,1,0,0,114,51,1,0,
+    0,114,53,1,0,0,114,54,1,0,0,114,57,1,0,0,
+    114,202,0,0,0,114,205,0,0,0,114,63,1,0,0,114,
+    60,1,0,0,218,12,115,116,97,116,105,99,109,101,116,104,
+    111,100,114,67,1,0,0,114,78,1,0,0,114,79,1,0,
+    0,114,64,1,0,0,114,3,0,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,44,1,0,0,215,
+    4,0,0,115,54,0,0,0,8,2,4,2,2,1,10,9,
+    2,1,10,12,2,1,10,21,2,1,10,14,2,1,12,31,
+    2,1,12,23,2,1,12,12,2,1,12,15,2,1,10,8,
+    2,1,10,10,2,1,10,6,2,1,10,6,2,1,114,44,
+    1,0,0,99,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,3,0,0,0,64,0,0,0,115,90,0,0,
+    0,101,0,90,1,100,0,90,2,100,1,90,3,100,2,100,
+    3,132,0,90,4,100,4,100,5,132,0,90,5,101,6,90,
+    7,100,6,100,7,132,0,90,8,100,8,100,9,132,0,90,
+    9,100,19,100,11,100,12,132,1,90,10,100,13,100,14,132,
+    0,90,11,101,12,100,15,100,16,132,0,131,1,90,13,100,
+    17,100,18,132,0,90,14,100,10,83,0,41,20,218,10,70,
+    105,108,101,70,105,110,100,101,114,122,172,70,105,108,101,45,
+    98,97,115,101,100,32,102,105,110,100,101,114,46,10,10,32,
+    32,32,32,73,110,116,101,114,97,99,116,105,111,110,115,32,
+    119,105,116,104,32,116,104,101,32,102,105,108,101,32,115,121,
+    115,116,101,109,32,97,114,101,32,99,97,99,104,101,100,32,
+    102,111,114,32,112,101,114,102,111,114,109,97,110,99,101,44,
+    32,98,101,105,110,103,10,32,32,32,32,114,101,102,114,101,
+    115,104,101,100,32,119,104,101,110,32,116,104,101,32,100,105,
+    114,101,99,116,111,114,121,32,116,104,101,32,102,105,110,100,
+    101,114,32,105,115,32,104,97,110,100,108,105,110,103,32,104,
+    97,115,32,98,101,101,110,32,109,111,100,105,102,105,101,100,
+    46,10,10,32,32,32,32,99,2,0,0,0,0,0,0,0,
+    0,0,0,0,5,0,0,0,6,0,0,0,7,0,0,0,
+    115,84,0,0,0,103,0,125,3,124,2,68,0,93,32,92,
+    2,137,0,125,4,124,3,160,0,135,0,102,1,100,1,100,
+    2,132,8,124,4,68,0,131,1,161,1,1,0,113,8,124,
+    3,124,0,95,1,124,1,112,54,100,3,124,0,95,2,100,
+    4,124,0,95,3,116,4,131,0,124,0,95,5,116,4,131,
+    0,124,0,95,6,100,5,83,0,41,6,122,154,73,110,105,
+    116,105,97,108,105,122,101,32,119,105,116,104,32,116,104,101,
+    32,112,97,116,104,32,116,111,32,115,101,97,114,99,104,32,
+    111,110,32,97,110,100,32,97,32,118,97,114,105,97,98,108,
+    101,32,110,117,109,98,101,114,32,111,102,10,32,32,32,32,
+    32,32,32,32,50,45,116,117,112,108,101,115,32,99,111,110,
+    116,97,105,110,105,110,103,32,116,104,101,32,108,111,97,100,
+    101,114,32,97,110,100,32,116,104,101,32,102,105,108,101,32,
+    115,117,102,102,105,120,101,115,32,116,104,101,32,108,111,97,
+    100,101,114,10,32,32,32,32,32,32,32,32,114,101,99,111,
+    103,110,105,122,101,115,46,99,1,0,0,0,0,0,0,0,
+    0,0,0,0,2,0,0,0,3,0,0,0,51,0,0,0,
+    115,22,0,0,0,124,0,93,14,125,1,124,1,136,0,102,
+    2,86,0,1,0,113,2,100,0,83,0,114,109,0,0,0,
+    114,3,0,0,0,114,15,1,0,0,169,1,114,139,0,0,
+    0,114,3,0,0,0,114,6,0,0,0,114,18,1,0,0,
+    167,5,0,0,115,4,0,0,0,4,0,2,0,122,38,70,
+    105,108,101,70,105,110,100,101,114,46,95,95,105,110,105,116,
+    95,95,46,60,108,111,99,97,108,115,62,46,60,103,101,110,
+    101,120,112,114,62,114,70,0,0,0,114,104,0,0,0,78,
+    41,7,114,166,0,0,0,218,8,95,108,111,97,100,101,114,
+    115,114,43,0,0,0,218,11,95,112,97,116,104,95,109,116,
+    105,109,101,218,3,115,101,116,218,11,95,112,97,116,104,95,
+    99,97,99,104,101,218,19,95,114,101,108,97,120,101,100,95,
+    112,97,116,104,95,99,97,99,104,101,41,5,114,118,0,0,
+    0,114,43,0,0,0,218,14,108,111,97,100,101,114,95,100,
+    101,116,97,105,108,115,90,7,108,111,97,100,101,114,115,114,
+    188,0,0,0,114,3,0,0,0,114,84,1,0,0,114,6,
+    0,0,0,114,208,0,0,0,161,5,0,0,115,16,0,0,
+    0,0,4,4,1,12,1,26,1,6,2,10,1,6,1,8,
+    1,122,19,70,105,108,101,70,105,110,100,101,114,46,95,95,
+    105,110,105,116,95,95,99,1,0,0,0,0,0,0,0,0,
+    0,0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,
+    10,0,0,0,100,1,124,0,95,0,100,2,83,0,41,3,
+    122,31,73,110,118,97,108,105,100,97,116,101,32,116,104,101,
+    32,100,105,114,101,99,116,111,114,121,32,109,116,105,109,101,
+    46,114,104,0,0,0,78,41,1,114,86,1,0,0,114,245,
+    0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
+    0,0,114,45,1,0,0,175,5,0,0,115,2,0,0,0,
+    0,2,122,28,70,105,108,101,70,105,110,100,101,114,46,105,
+    110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115,
+    99,2,0,0,0,0,0,0,0,0,0,0,0,3,0,0,
+    0,3,0,0,0,67,0,0,0,115,42,0,0,0,124,0,
+    160,0,124,1,161,1,125,2,124,2,100,1,107,8,114,26,
+    100,1,103,0,102,2,83,0,124,2,106,1,124,2,106,2,
+    112,38,103,0,102,2,83,0,41,2,122,197,84,114,121,32,
+    116,111,32,102,105,110,100,32,97,32,108,111,97,100,101,114,
+    32,102,111,114,32,116,104,101,32,115,112,101,99,105,102,105,
+    101,100,32,109,111,100,117,108,101,44,32,111,114,32,116,104,
+    101,32,110,97,109,101,115,112,97,99,101,10,32,32,32,32,
+    32,32,32,32,112,97,99,107,97,103,101,32,112,111,114,116,
+    105,111,110,115,46,32,82,101,116,117,114,110,115,32,40,108,
+    111,97,100,101,114,44,32,108,105,115,116,45,111,102,45,112,
+    111,114,116,105,111,110,115,41,46,10,10,32,32,32,32,32,
+    32,32,32,84,104,105,115,32,109,101,116,104,111,100,32,105,
+    115,32,100,101,112,114,101,99,97,116,101,100,46,32,32,85,
+    115,101,32,102,105,110,100,95,115,112,101,99,40,41,32,105,
+    110,115,116,101,97,100,46,10,10,32,32,32,32,32,32,32,
+    32,78,41,3,114,202,0,0,0,114,139,0,0,0,114,177,
+    0,0,0,41,3,114,118,0,0,0,114,138,0,0,0,114,
+    186,0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
+    0,0,0,114,136,0,0,0,181,5,0,0,115,8,0,0,
+    0,0,7,10,1,8,1,8,1,122,22,70,105,108,101,70,
+    105,110,100,101,114,46,102,105,110,100,95,108,111,97,100,101,
+    114,99,6,0,0,0,0,0,0,0,0,0,0,0,7,0,
+    0,0,6,0,0,0,67,0,0,0,115,26,0,0,0,124,
+    1,124,2,124,3,131,2,125,6,116,0,124,2,124,3,124,
+    6,124,4,100,1,141,4,83,0,41,2,78,114,176,0,0,
+    0,41,1,114,189,0,0,0,41,7,114,118,0,0,0,114,
+    187,0,0,0,114,138,0,0,0,114,43,0,0,0,90,4,
+    115,109,115,108,114,201,0,0,0,114,139,0,0,0,114,3,
+    0,0,0,114,3,0,0,0,114,6,0,0,0,114,57,1,
+    0,0,193,5,0,0,115,8,0,0,0,0,1,10,1,8,
+    1,2,255,122,20,70,105,108,101,70,105,110,100,101,114,46,
+    95,103,101,116,95,115,112,101,99,78,99,3,0,0,0,0,
+    0,0,0,0,0,0,0,14,0,0,0,8,0,0,0,67,
+    0,0,0,115,98,1,0,0,100,1,125,3,124,1,160,0,
+    100,2,161,1,100,3,25,0,125,4,122,24,116,1,124,0,
+    106,2,112,34,116,3,160,4,161,0,131,1,106,5,125,5,
+    87,0,110,24,4,0,116,6,107,10,114,66,1,0,1,0,
+    1,0,100,4,125,5,89,0,110,2,88,0,124,5,124,0,
+    106,7,107,3,114,92,124,0,160,8,161,0,1,0,124,5,
+    124,0,95,7,116,9,131,0,114,114,124,0,106,10,125,6,
+    124,4,160,11,161,0,125,7,110,10,124,0,106,12,125,6,
+    124,4,125,7,124,7,124,6,107,6,114,218,116,13,124,0,
+    106,2,124,4,131,2,125,8,124,0,106,14,68,0,93,58,
+    92,2,125,9,125,10,100,5,124,9,23,0,125,11,116,13,
+    124,8,124,11,131,2,125,12,116,15,124,12,131,1,114,150,
+    124,0,160,16,124,10,124,1,124,12,124,8,103,1,124,2,
+    161,5,2,0,1,0,83,0,113,150,116,17,124,8,131,1,
+    125,3,124,0,106,14,68,0,93,82,92,2,125,9,125,10,
+    116,13,124,0,106,2,124,4,124,9,23,0,131,2,125,12,
+    116,18,106,19,100,6,124,12,100,3,100,7,141,3,1,0,
+    124,7,124,9,23,0,124,6,107,6,114,224,116,15,124,12,
+    131,1,114,224,124,0,160,16,124,10,124,1,124,12,100,8,
+    124,2,161,5,2,0,1,0,83,0,113,224,124,3,144,1,
+    114,94,116,18,160,19,100,9,124,8,161,2,1,0,116,18,
+    160,20,124,1,100,8,161,2,125,13,124,8,103,1,124,13,
+    95,21,124,13,83,0,100,8,83,0,41,10,122,111,84,114,
+    121,32,116,111,32,102,105,110,100,32,97,32,115,112,101,99,
+    32,102,111,114,32,116,104,101,32,115,112,101,99,105,102,105,
+    101,100,32,109,111,100,117,108,101,46,10,10,32,32,32,32,
+    32,32,32,32,82,101,116,117,114,110,115,32,116,104,101,32,
+    109,97,116,99,104,105,110,103,32,115,112,101,99,44,32,111,
+    114,32,78,111,110,101,32,105,102,32,110,111,116,32,102,111,
+    117,110,100,46,10,32,32,32,32,32,32,32,32,70,114,70,
+    0,0,0,114,27,0,0,0,114,104,0,0,0,114,208,0,
+    0,0,122,9,116,114,121,105,110,103,32,123,125,41,1,90,
+    9,118,101,114,98,111,115,105,116,121,78,122,25,112,111,115,
+    115,105,98,108,101,32,110,97,109,101,115,112,97,99,101,32,
+    102,111,114,32,123,125,41,22,114,40,0,0,0,114,48,0,
+    0,0,114,43,0,0,0,114,2,0,0,0,114,54,0,0,
+    0,114,9,1,0,0,114,49,0,0,0,114,86,1,0,0,
+    218,11,95,102,105,108,108,95,99,97,99,104,101,114,7,0,
+    0,0,114,89,1,0,0,114,105,0,0,0,114,88,1,0,
+    0,114,37,0,0,0,114,85,1,0,0,114,53,0,0,0,
+    114,57,1,0,0,114,55,0,0,0,114,133,0,0,0,114,
+    148,0,0,0,114,182,0,0,0,114,177,0,0,0,41,14,
+    114,118,0,0,0,114,138,0,0,0,114,201,0,0,0,90,
+    12,105,115,95,110,97,109,101,115,112,97,99,101,90,11,116,
+    97,105,108,95,109,111,100,117,108,101,114,168,0,0,0,90,
+    5,99,97,99,104,101,90,12,99,97,99,104,101,95,109,111,
+    100,117,108,101,90,9,98,97,115,101,95,112,97,116,104,114,
+    16,1,0,0,114,187,0,0,0,90,13,105,110,105,116,95,
+    102,105,108,101,110,97,109,101,90,9,102,117,108,108,95,112,
+    97,116,104,114,186,0,0,0,114,3,0,0,0,114,3,0,
+    0,0,114,6,0,0,0,114,202,0,0,0,198,5,0,0,
+    115,74,0,0,0,0,5,4,1,14,1,2,1,24,1,14,
+    1,10,1,10,1,8,1,6,2,6,1,6,1,10,2,6,
+    1,4,2,8,1,12,1,14,1,8,1,10,1,8,1,26,
+    4,8,2,14,1,16,1,16,1,12,1,8,1,10,1,2,
+    0,2,255,10,2,6,1,12,1,12,1,8,1,4,1,122,
+    20,70,105,108,101,70,105,110,100,101,114,46,102,105,110,100,
+    95,115,112,101,99,99,1,0,0,0,0,0,0,0,0,0,
+    0,0,9,0,0,0,10,0,0,0,67,0,0,0,115,190,
+    0,0,0,124,0,106,0,125,1,122,22,116,1,160,2,124,
+    1,112,22,116,1,160,3,161,0,161,1,125,2,87,0,110,
+    30,4,0,116,4,116,5,116,6,102,3,107,10,114,58,1,
+    0,1,0,1,0,103,0,125,2,89,0,110,2,88,0,116,
+    7,106,8,160,9,100,1,161,1,115,84,116,10,124,2,131,
+    1,124,0,95,11,110,74,116,10,131,0,125,3,124,2,68,
+    0,93,56,125,4,124,4,160,12,100,2,161,1,92,3,125,
+    5,125,6,125,7,124,6,114,136,100,3,160,13,124,5,124,
+    7,160,14,161,0,161,2,125,8,110,4,124,5,125,8,124,
+    3,160,15,124,8,161,1,1,0,113,94,124,3,124,0,95,
+    11,116,7,106,8,160,9,116,16,161,1,114,186,100,4,100,
+    5,132,0,124,2,68,0,131,1,124,0,95,17,100,6,83,
+    0,41,7,122,68,70,105,108,108,32,116,104,101,32,99,97,
+    99,104,101,32,111,102,32,112,111,116,101,110,116,105,97,108,
+    32,109,111,100,117,108,101,115,32,97,110,100,32,112,97,99,
+    107,97,103,101,115,32,102,111,114,32,116,104,105,115,32,100,
+    105,114,101,99,116,111,114,121,46,114,0,0,0,0,114,70,
+    0,0,0,114,60,0,0,0,99,1,0,0,0,0,0,0,
+    0,0,0,0,0,2,0,0,0,4,0,0,0,83,0,0,
+    0,115,20,0,0,0,104,0,124,0,93,12,125,1,124,1,
+    160,0,161,0,146,2,113,4,83,0,114,3,0,0,0,41,
+    1,114,105,0,0,0,41,2,114,31,0,0,0,90,2,102,
+    110,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
+    218,9,60,115,101,116,99,111,109,112,62,19,6,0,0,115,
+    4,0,0,0,6,0,2,0,122,41,70,105,108,101,70,105,
+    110,100,101,114,46,95,102,105,108,108,95,99,97,99,104,101,
+    46,60,108,111,99,97,108,115,62,46,60,115,101,116,99,111,
+    109,112,62,78,41,18,114,43,0,0,0,114,2,0,0,0,
+    114,6,1,0,0,114,54,0,0,0,114,2,1,0,0,218,
+    15,80,101,114,109,105,115,115,105,111,110,69,114,114,111,114,
+    218,18,78,111,116,65,68,105,114,101,99,116,111,114,121,69,
+    114,114,111,114,114,8,0,0,0,114,9,0,0,0,114,10,
+    0,0,0,114,87,1,0,0,114,88,1,0,0,114,100,0,
+    0,0,114,61,0,0,0,114,105,0,0,0,218,3,97,100,
+    100,114,11,0,0,0,114,89,1,0,0,41,9,114,118,0,
+    0,0,114,43,0,0,0,114,7,1,0,0,90,21,108,111,
+    119,101,114,95,115,117,102,102,105,120,95,99,111,110,116,101,
+    110,116,115,114,40,1,0,0,114,116,0,0,0,114,28,1,
+    0,0,114,16,1,0,0,90,8,110,101,119,95,110,97,109,
+    101,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
+    114,91,1,0,0,246,5,0,0,115,34,0,0,0,0,2,
+    6,1,2,1,22,1,20,3,10,3,12,1,12,7,6,1,
+    8,1,16,1,4,1,18,2,4,1,12,1,6,1,12,1,
+    122,22,70,105,108,101,70,105,110,100,101,114,46,95,102,105,
+    108,108,95,99,97,99,104,101,99,1,0,0,0,0,0,0,
+    0,0,0,0,0,3,0,0,0,3,0,0,0,7,0,0,
+    0,115,18,0,0,0,135,0,135,1,102,2,100,1,100,2,
+    132,8,125,2,124,2,83,0,41,3,97,20,1,0,0,65,
+    32,99,108,97,115,115,32,109,101,116,104,111,100,32,119,104,
+    105,99,104,32,114,101,116,117,114,110,115,32,97,32,99,108,
+    111,115,117,114,101,32,116,111,32,117,115,101,32,111,110,32,
+    115,121,115,46,112,97,116,104,95,104,111,111,107,10,32,32,
+    32,32,32,32,32,32,119,104,105,99,104,32,119,105,108,108,
+    32,114,101,116,117,114,110,32,97,110,32,105,110,115,116,97,
+    110,99,101,32,117,115,105,110,103,32,116,104,101,32,115,112,
+    101,99,105,102,105,101,100,32,108,111,97,100,101,114,115,32,
+    97,110,100,32,116,104,101,32,112,97,116,104,10,32,32,32,
+    32,32,32,32,32,99,97,108,108,101,100,32,111,110,32,116,
+    104,101,32,99,108,111,115,117,114,101,46,10,10,32,32,32,
+    32,32,32,32,32,73,102,32,116,104,101,32,112,97,116,104,
+    32,99,97,108,108,101,100,32,111,110,32,116,104,101,32,99,
+    108,111,115,117,114,101,32,105,115,32,110,111,116,32,97,32,
+    100,105,114,101,99,116,111,114,121,44,32,73,109,112,111,114,
+    116,69,114,114,111,114,32,105,115,10,32,32,32,32,32,32,
+    32,32,114,97,105,115,101,100,46,10,10,32,32,32,32,32,
+    32,32,32,99,1,0,0,0,0,0,0,0,0,0,0,0,
+    1,0,0,0,4,0,0,0,19,0,0,0,115,34,0,0,
+    0,116,0,124,0,131,1,115,20,116,1,100,1,124,0,100,
+    2,141,2,130,1,136,0,124,0,102,1,136,1,158,2,142,
+    0,83,0,41,3,122,45,80,97,116,104,32,104,111,111,107,
+    32,102,111,114,32,105,109,112,111,114,116,108,105,98,46,109,
+    97,99,104,105,110,101,114,121,46,70,105,108,101,70,105,110,
+    100,101,114,46,122,30,111,110,108,121,32,100,105,114,101,99,
+    116,111,114,105,101,115,32,97,114,101,32,115,117,112,112,111,
+    114,116,101,100,114,47,0,0,0,41,2,114,55,0,0,0,
+    114,117,0,0,0,114,47,0,0,0,169,2,114,192,0,0,
+    0,114,90,1,0,0,114,3,0,0,0,114,6,0,0,0,
+    218,24,112,97,116,104,95,104,111,111,107,95,102,111,114,95,
+    70,105,108,101,70,105,110,100,101,114,31,6,0,0,115,6,
+    0,0,0,0,2,8,1,12,1,122,54,70,105,108,101,70,
+    105,110,100,101,114,46,112,97,116,104,95,104,111,111,107,46,
+    60,108,111,99,97,108,115,62,46,112,97,116,104,95,104,111,
+    111,107,95,102,111,114,95,70,105,108,101,70,105,110,100,101,
+    114,114,3,0,0,0,41,3,114,192,0,0,0,114,90,1,
+    0,0,114,97,1,0,0,114,3,0,0,0,114,96,1,0,
+    0,114,6,0,0,0,218,9,112,97,116,104,95,104,111,111,
+    107,21,6,0,0,115,4,0,0,0,0,10,14,6,122,20,
+    70,105,108,101,70,105,110,100,101,114,46,112,97,116,104,95,
+    104,111,111,107,99,1,0,0,0,0,0,0,0,0,0,0,
+    0,1,0,0,0,3,0,0,0,67,0,0,0,115,12,0,
+    0,0,100,1,160,0,124,0,106,1,161,1,83,0,41,2,
+    78,122,16,70,105,108,101,70,105,110,100,101,114,40,123,33,
+    114,125,41,41,2,114,61,0,0,0,114,43,0,0,0,114,
+    245,0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
+    0,0,0,114,38,1,0,0,39,6,0,0,115,2,0,0,
+    0,0,1,122,19,70,105,108,101,70,105,110,100,101,114,46,
+    95,95,114,101,112,114,95,95,41,1,78,41,15,114,124,0,
+    0,0,114,123,0,0,0,114,125,0,0,0,114,126,0,0,
+    0,114,208,0,0,0,114,45,1,0,0,114,142,0,0,0,
+    114,205,0,0,0,114,136,0,0,0,114,57,1,0,0,114,
+    202,0,0,0,114,91,1,0,0,114,206,0,0,0,114,98,
+    1,0,0,114,38,1,0,0,114,3,0,0,0,114,3,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,114,83,1,0,
+    0,152,5,0,0,115,22,0,0,0,8,2,4,7,8,14,
+    8,4,4,2,8,12,8,5,10,48,8,31,2,1,10,17,
+    114,83,1,0,0,99,4,0,0,0,0,0,0,0,0,0,
+    0,0,6,0,0,0,8,0,0,0,67,0,0,0,115,146,
+    0,0,0,124,0,160,0,100,1,161,1,125,4,124,0,160,
+    0,100,2,161,1,125,5,124,4,115,66,124,5,114,36,124,
+    5,106,1,125,4,110,30,124,2,124,3,107,2,114,56,116,
+    2,124,1,124,2,131,2,125,4,110,10,116,3,124,1,124,
+    2,131,2,125,4,124,5,115,84,116,4,124,1,124,2,124,
+    4,100,3,141,3,125,5,122,36,124,5,124,0,100,2,60,
+    0,124,4,124,0,100,1,60,0,124,2,124,0,100,4,60,
+    0,124,3,124,0,100,5,60,0,87,0,110,20,4,0,116,
+    5,107,10,114,140,1,0,1,0,1,0,89,0,110,2,88,
+    0,100,0,83,0,41,6,78,218,10,95,95,108,111,97,100,
+    101,114,95,95,218,8,95,95,115,112,101,99,95,95,114,84,
+    1,0,0,90,8,95,95,102,105,108,101,95,95,90,10,95,
+    95,99,97,99,104,101,100,95,95,41,6,218,3,103,101,116,
+    114,139,0,0,0,114,14,1,0,0,114,8,1,0,0,114,
+    189,0,0,0,114,71,1,0,0,41,6,90,2,110,115,114,
+    116,0,0,0,90,8,112,97,116,104,110,97,109,101,90,9,
+    99,112,97,116,104,110,97,109,101,114,139,0,0,0,114,186,
+    0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
+    0,0,218,14,95,102,105,120,95,117,112,95,109,111,100,117,
+    108,101,45,6,0,0,115,34,0,0,0,0,2,10,1,10,
+    1,4,1,4,1,8,1,8,1,12,2,10,1,4,1,14,
+    1,2,1,8,1,8,1,8,1,12,1,14,2,114,102,1,
+    0,0,99,0,0,0,0,0,0,0,0,0,0,0,0,3,
+    0,0,0,3,0,0,0,67,0,0,0,115,38,0,0,0,
+    116,0,116,1,160,2,161,0,102,2,125,0,116,3,116,4,
+    102,2,125,1,116,5,116,6,102,2,125,2,124,0,124,1,
+    124,2,103,3,83,0,41,1,122,95,82,101,116,117,114,110,
+    115,32,97,32,108,105,115,116,32,111,102,32,102,105,108,101,
+    45,98,97,115,101,100,32,109,111,100,117,108,101,32,108,111,
+    97,100,101,114,115,46,10,10,32,32,32,32,69,97,99,104,
+    32,105,116,101,109,32,105,115,32,97,32,116,117,112,108,101,
+    32,40,108,111,97,100,101,114,44,32,115,117,102,102,105,120,
+    101,115,41,46,10,32,32,32,32,41,7,114,251,0,0,0,
+    114,162,0,0,0,218,18,101,120,116,101,110,115,105,111,110,
+    95,115,117,102,102,105,120,101,115,114,8,1,0,0,114,101,
+    0,0,0,114,14,1,0,0,114,88,0,0,0,41,3,90,
+    10,101,120,116,101,110,115,105,111,110,115,90,6,115,111,117,
+    114,99,101,90,8,98,121,116,101,99,111,100,101,114,3,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,114,183,0,0,
+    0,68,6,0,0,115,8,0,0,0,0,5,12,1,8,1,
+    8,1,114,183,0,0,0,99,1,0,0,0,0,0,0,0,
+    0,0,0,0,12,0,0,0,9,0,0,0,67,0,0,0,
+    115,178,1,0,0,124,0,97,0,116,0,106,1,97,1,116,
+    0,106,2,97,2,116,1,106,3,116,4,25,0,125,1,100,
+    1,68,0,93,48,125,2,124,2,116,1,106,3,107,7,114,
+    56,116,0,160,5,124,2,161,1,125,3,110,10,116,1,106,
+    3,124,2,25,0,125,3,116,6,124,1,124,2,124,3,131,
+    3,1,0,113,30,100,2,100,3,103,1,102,2,100,4,100,
+    5,100,3,103,2,102,2,102,2,125,4,124,4,68,0,93,
+    110,92,2,125,5,125,6,116,7,100,6,100,7,132,0,124,
+    6,68,0,131,1,131,1,115,136,74,0,130,1,124,6,100,
+    8,25,0,125,7,124,5,116,1,106,3,107,6,114,170,116,
+    1,106,3,124,5,25,0,125,8,1,0,113,226,113,106,122,
+    20,116,0,160,5,124,5,161,1,125,8,87,0,1,0,113,
+    226,87,0,113,106,4,0,116,8,107,10,114,214,1,0,1,
+    0,1,0,89,0,113,106,89,0,113,106,88,0,113,106,116,
+    8,100,9,131,1,130,1,116,6,124,1,100,10,124,8,131,
+    3,1,0,116,6,124,1,100,11,124,7,131,3,1,0,116,
+    6,124,1,100,12,100,13,160,9,124,6,161,1,131,3,1,
+    0,116,6,124,1,100,14,100,15,100,16,132,0,124,6,68,
+    0,131,1,131,3,1,0,116,0,160,5,100,17,161,1,125,
+    9,116,6,124,1,100,17,124,9,131,3,1,0,116,0,160,
+    5,100,18,161,1,125,10,116,6,124,1,100,18,124,10,131,
+    3,1,0,124,5,100,4,107,2,144,1,114,110,116,0,160,
+    5,100,19,161,1,125,11,116,6,124,1,100,20,124,11,131,
+    3,1,0,116,6,124,1,100,21,116,10,131,0,131,3,1,
+    0,116,11,160,12,116,2,160,13,161,0,161,1,1,0,124,
+    5,100,4,107,2,144,1,114,174,116,14,160,15,100,22,161,
+    1,1,0,100,23,116,11,107,6,144,1,114,174,100,24,116,
+    16,95,17,100,25,83,0,41,26,122,205,83,101,116,117,112,
+    32,116,104,101,32,112,97,116,104,45,98,97,115,101,100,32,
+    105,109,112,111,114,116,101,114,115,32,102,111,114,32,105,109,
+    112,111,114,116,108,105,98,32,98,121,32,105,109,112,111,114,
+    116,105,110,103,32,110,101,101,100,101,100,10,32,32,32,32,
+    98,117,105,108,116,45,105,110,32,109,111,100,117,108,101,115,
+    32,97,110,100,32,105,110,106,101,99,116,105,110,103,32,116,
+    104,101,109,32,105,110,116,111,32,116,104,101,32,103,108,111,
+    98,97,108,32,110,97,109,101,115,112,97,99,101,46,10,10,
+    32,32,32,32,79,116,104,101,114,32,99,111,109,112,111,110,
+    101,110,116,115,32,97,114,101,32,101,120,116,114,97,99,116,
+    101,100,32,102,114,111,109,32,116,104,101,32,99,111,114,101,
+    32,98,111,111,116,115,116,114,97,112,32,109,111,100,117,108,
+    101,46,10,10,32,32,32,32,41,4,114,63,0,0,0,114,
+    74,0,0,0,218,8,98,117,105,108,116,105,110,115,114,159,
+    0,0,0,90,5,112,111,115,105,120,250,1,47,90,2,110,
+    116,250,1,92,99,1,0,0,0,0,0,0,0,0,0,0,
+    0,2,0,0,0,3,0,0,0,115,0,0,0,115,26,0,
+    0,0,124,0,93,18,125,1,116,0,124,1,131,1,100,0,
+    107,2,86,0,1,0,113,2,100,1,83,0,41,2,114,38,
+    0,0,0,78,41,1,114,22,0,0,0,41,2,114,31,0,
+    0,0,114,94,0,0,0,114,3,0,0,0,114,3,0,0,
+    0,114,6,0,0,0,114,18,1,0,0,104,6,0,0,115,
+    4,0,0,0,4,0,2,0,122,25,95,115,101,116,117,112,
+    46,60,108,111,99,97,108,115,62,46,60,103,101,110,101,120,
+    112,114,62,114,72,0,0,0,122,30,105,109,112,111,114,116,
+    108,105,98,32,114,101,113,117,105,114,101,115,32,112,111,115,
+    105,120,32,111,114,32,110,116,114,2,0,0,0,114,34,0,
+    0,0,114,30,0,0,0,114,39,0,0,0,114,57,0,0,
+    0,99,1,0,0,0,0,0,0,0,0,0,0,0,2,0,
+    0,0,4,0,0,0,83,0,0,0,115,22,0,0,0,104,
+    0,124,0,93,14,125,1,100,0,124,1,155,0,157,2,146,
+    2,113,4,83,0,41,1,114,73,0,0,0,114,3,0,0,
+    0,41,2,114,31,0,0,0,218,1,115,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,92,1,0,0,120,
+    6,0,0,115,4,0,0,0,6,0,2,0,122,25,95,115,
+    101,116,117,112,46,60,108,111,99,97,108,115,62,46,60,115,
+    101,116,99,111,109,112,62,90,7,95,116,104,114,101,97,100,
+    90,8,95,119,101,97,107,114,101,102,90,6,119,105,110,114,
+    101,103,114,191,0,0,0,114,7,0,0,0,122,4,46,112,
+    121,119,122,6,95,100,46,112,121,100,84,78,41,18,114,133,
+    0,0,0,114,8,0,0,0,114,162,0,0,0,114,30,1,
+    0,0,114,124,0,0,0,90,18,95,98,117,105,108,116,105,
+    110,95,102,114,111,109,95,110,97,109,101,114,128,0,0,0,
+    218,3,97,108,108,114,117,0,0,0,114,35,0,0,0,114,
+    13,0,0,0,114,20,1,0,0,114,166,0,0,0,114,103,
+    1,0,0,114,101,0,0,0,114,185,0,0,0,114,190,0,
+    0,0,114,194,0,0,0,41,12,218,17,95,98,111,111,116,
+    115,116,114,97,112,95,109,111,100,117,108,101,90,11,115,101,
+    108,102,95,109,111,100,117,108,101,90,12,98,117,105,108,116,
+    105,110,95,110,97,109,101,90,14,98,117,105,108,116,105,110,
+    95,109,111,100,117,108,101,90,10,111,115,95,100,101,116,97,
+    105,108,115,90,10,98,117,105,108,116,105,110,95,111,115,114,
+    30,0,0,0,114,34,0,0,0,90,9,111,115,95,109,111,
+    100,117,108,101,90,13,116,104,114,101,97,100,95,109,111,100,
+    117,108,101,90,14,119,101,97,107,114,101,102,95,109,111,100,
+    117,108,101,90,13,119,105,110,114,101,103,95,109,111,100,117,
+    108,101,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
+    0,218,6,95,115,101,116,117,112,79,6,0,0,115,78,0,
+    0,0,0,8,4,1,6,1,6,3,10,1,8,1,10,1,
+    12,2,10,1,14,3,22,1,12,2,22,1,8,1,10,1,
+    10,1,6,2,2,1,10,1,10,1,14,1,12,2,8,1,
+    12,1,12,1,18,1,22,3,10,1,12,3,10,1,12,3,
+    10,1,10,1,12,3,14,1,14,1,10,1,10,1,10,1,
+    114,110,1,0,0,99,1,0,0,0,0,0,0,0,0,0,
+    0,0,2,0,0,0,4,0,0,0,67,0,0,0,115,50,
+    0,0,0,116,0,124,0,131,1,1,0,116,1,131,0,125,
+    1,116,2,106,3,160,4,116,5,106,6,124,1,142,0,103,
+    1,161,1,1,0,116,2,106,7,160,8,116,9,161,1,1,
+    0,100,1,83,0,41,2,122,41,73,110,115,116,97,108,108,
+    32,116,104,101,32,112,97,116,104,45,98,97,115,101,100,32,
+    105,109,112,111,114,116,32,99,111,109,112,111,110,101,110,116,
+    115,46,78,41,10,114,110,1,0,0,114,183,0,0,0,114,
+    8,0,0,0,114,50,1,0,0,114,166,0,0,0,114,83,
+    1,0,0,114,98,1,0,0,218,9,109,101,116,97,95,112,
+    97,116,104,114,185,0,0,0,114,44,1,0,0,41,2,114,
+    109,1,0,0,90,17,115,117,112,112,111,114,116,101,100,95,
+    108,111,97,100,101,114,115,114,3,0,0,0,114,3,0,0,
+    0,114,6,0,0,0,218,8,95,105,110,115,116,97,108,108,
+    144,6,0,0,115,8,0,0,0,0,2,8,1,6,1,20,
+    1,114,112,1,0,0,41,1,114,59,0,0,0,41,1,78,
+    41,3,78,78,78,41,2,114,72,0,0,0,114,72,0,0,
+    0,41,1,84,41,1,78,41,1,78,41,63,114,126,0,0,
+    0,114,12,0,0,0,90,37,95,67,65,83,69,95,73,78,
+    83,69,78,83,73,84,73,86,69,95,80,76,65,84,70,79,
+    82,77,83,95,66,89,84,69,83,95,75,69,89,114,11,0,
+    0,0,114,13,0,0,0,114,20,0,0,0,114,26,0,0,
+    0,114,28,0,0,0,114,37,0,0,0,114,46,0,0,0,
+    114,48,0,0,0,114,52,0,0,0,114,53,0,0,0,114,
+    55,0,0,0,114,58,0,0,0,114,68,0,0,0,218,4,
+    116,121,112,101,218,8,95,95,99,111,100,101,95,95,114,161,
+    0,0,0,114,18,0,0,0,114,147,0,0,0,114,17,0,
+    0,0,114,23,0,0,0,114,235,0,0,0,114,91,0,0,
+    0,114,87,0,0,0,114,101,0,0,0,114,88,0,0,0,
+    90,23,68,69,66,85,71,95,66,89,84,69,67,79,68,69,
+    95,83,85,70,70,73,88,69,83,90,27,79,80,84,73,77,
+    73,90,69,68,95,66,89,84,69,67,79,68,69,95,83,85,
+    70,70,73,88,69,83,114,97,0,0,0,114,102,0,0,0,
+    114,108,0,0,0,114,112,0,0,0,114,114,0,0,0,114,
+    135,0,0,0,114,142,0,0,0,114,151,0,0,0,114,155,
+    0,0,0,114,157,0,0,0,114,164,0,0,0,114,169,0,
+    0,0,114,170,0,0,0,114,175,0,0,0,218,6,111,98,
+    106,101,99,116,114,184,0,0,0,114,189,0,0,0,114,190,
+    0,0,0,114,207,0,0,0,114,220,0,0,0,114,238,0,
+    0,0,114,8,1,0,0,114,14,1,0,0,114,20,1,0,
+    0,114,251,0,0,0,114,21,1,0,0,114,42,1,0,0,
+    114,44,1,0,0,114,83,1,0,0,114,102,1,0,0,114,
+    183,0,0,0,114,110,1,0,0,114,112,1,0,0,114,3,
+    0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
+    0,0,218,8,60,109,111,100,117,108,101,62,1,0,0,0,
+    115,126,0,0,0,4,22,4,1,4,1,2,1,2,255,4,
+    4,8,17,8,5,8,5,8,6,8,6,8,12,8,10,8,
+    9,8,5,8,7,8,9,10,22,10,127,0,14,16,1,12,
+    2,4,1,4,2,6,2,6,2,8,2,16,71,8,40,8,
+    19,8,12,8,12,8,28,8,17,8,33,8,28,8,24,10,
+    13,10,10,10,11,8,14,6,3,4,1,2,255,12,68,14,
+    64,14,29,16,127,0,17,14,72,18,45,18,26,4,3,18,
+    53,14,63,14,42,14,127,0,66,14,127,0,22,10,23,8,
+    11,8,65,
 };


From webhook-mailer at python.org  Tue Sep 10 10:01:17 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 14:01:17 -0000
Subject: [Python-checkins] bpo-37913: document that __length_hint__ can
 return NotImplemented (GH-15383)
Message-ID: <mailman.449.1568124078.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/009ef2955d91e04f62a7159a1c7ddaddbfedc3b6
commit: 009ef2955d91e04f62a7159a1c7ddaddbfedc3b6
branch: master
author: Jeroen Demeyer <J.Demeyer at UGent.be>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-10T15:01:13+01:00
summary:

bpo-37913: document that __length_hint__ can return NotImplemented (GH-15383)

files:
M Doc/reference/datamodel.rst

diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index 8813f57587f0..6b9325da2f39 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -2155,8 +2155,10 @@ through the container; for mappings, :meth:`__iter__` should be the same as
 
    Called to implement :func:`operator.length_hint`. Should return an estimated
    length for the object (which may be greater or less than the actual length).
-   The length must be an integer ``>=`` 0. This method is purely an
-   optimization and is never required for correctness.
+   The length must be an integer ``>=`` 0. The return value may also be
+   *NotImplemented*, which is treated the same as if the ``__length_hint__``
+   method didn't exist at all. This method is purely an optimization and is
+   never required for correctness.
 
    .. versionadded:: 3.4
 


From webhook-mailer at python.org  Tue Sep 10 10:17:46 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 14:17:46 -0000
Subject: [Python-checkins] bpo-38087: Fix case sensitivity in test_pathlib
 and test_ntpath (GH-15850)
Message-ID: <mailman.450.1568125067.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0d7e6a6d2d9e564d670efb6e3705598846de188d
commit: 0d7e6a6d2d9e564d670efb6e3705598846de188d
branch: 3.7
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-10T15:17:42+01:00
summary:

bpo-38087: Fix case sensitivity in test_pathlib and test_ntpath (GH-15850)

files:
A Misc/NEWS.d/next/Windows/2019-09-10-14-21-40.bpo-38087.--eIib.rst
M Lib/test/test_ntpath.py
M Lib/test/test_pathlib.py

diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
index 52ddc18c7e46..89b0d5303db5 100644
--- a/Lib/test/test_ntpath.py
+++ b/Lib/test/test_ntpath.py
@@ -14,10 +14,19 @@
     # but for those that require it we import here.
     nt = None
 
+
+def _norm(path):
+    if isinstance(path, (bytes, str, os.PathLike)):
+        return ntpath.normcase(os.fsdecode(path))
+    elif hasattr(path, "__iter__"):
+        return tuple(ntpath.normcase(os.fsdecode(p)) for p in path)
+    return path
+
+
 def tester(fn, wantResult):
     fn = fn.replace("\\", "\\\\")
     gotResult = eval(fn)
-    if wantResult != gotResult:
+    if wantResult != gotResult and _norm(wantResult) != _norm(gotResult):
         raise TestFailed("%s should return: %s but returned: %s" \
               %(str(fn), str(wantResult), str(gotResult)))
 
@@ -33,16 +42,22 @@ def tester(fn, wantResult):
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", DeprecationWarning)
         gotResult = eval(fn)
-    if isinstance(wantResult, str):
-        wantResult = os.fsencode(wantResult)
-    elif isinstance(wantResult, tuple):
-        wantResult = tuple(os.fsencode(r) for r in wantResult)
-    if wantResult != gotResult:
+    if _norm(wantResult) != _norm(gotResult):
         raise TestFailed("%s should return: %s but returned: %s" \
               %(str(fn), str(wantResult), repr(gotResult)))
 
 
-class TestNtpath(unittest.TestCase):
+class NtpathTestCase(unittest.TestCase):
+    def assertPathEqual(self, path1, path2):
+        if path1 == path2 or _norm(path1) == _norm(path2):
+            return
+        self.assertEqual(path1, path2)
+
+    def assertPathIn(self, path, pathset):
+        self.assertIn(_norm(path), _norm(pathset))
+
+
+class TestNtpath(NtpathTestCase):
     def test_splitext(self):
         tester('ntpath.splitext("foo.ext")', ('foo', '.ext'))
         tester('ntpath.splitext("/foo/foo.ext")', ('/foo/foo', '.ext'))
@@ -459,7 +474,7 @@ class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase):
     attributes = ['relpath']
 
 
-class PathLikeTests(unittest.TestCase):
+class PathLikeTests(NtpathTestCase):
 
     path = ntpath
 
@@ -470,67 +485,67 @@ def setUp(self):
         with open(self.file_name, 'xb', 0) as file:
             file.write(b"test_ntpath.PathLikeTests")
 
-    def assertPathEqual(self, func):
-        self.assertEqual(func(self.file_path), func(self.file_name))
+    def _check_function(self, func):
+        self.assertPathEqual(func(self.file_path), func(self.file_name))
 
     def test_path_normcase(self):
-        self.assertPathEqual(self.path.normcase)
+        self._check_function(self.path.normcase)
 
     def test_path_isabs(self):
-        self.assertPathEqual(self.path.isabs)
+        self._check_function(self.path.isabs)
 
     def test_path_join(self):
         self.assertEqual(self.path.join('a', FakePath('b'), 'c'),
                          self.path.join('a', 'b', 'c'))
 
     def test_path_split(self):
-        self.assertPathEqual(self.path.split)
+        self._check_function(self.path.split)
 
     def test_path_splitext(self):
-        self.assertPathEqual(self.path.splitext)
+        self._check_function(self.path.splitext)
 
     def test_path_splitdrive(self):
-        self.assertPathEqual(self.path.splitdrive)
+        self._check_function(self.path.splitdrive)
 
     def test_path_basename(self):
-        self.assertPathEqual(self.path.basename)
+        self._check_function(self.path.basename)
 
     def test_path_dirname(self):
-        self.assertPathEqual(self.path.dirname)
+        self._check_function(self.path.dirname)
 
     def test_path_islink(self):
-        self.assertPathEqual(self.path.islink)
+        self._check_function(self.path.islink)
 
     def test_path_lexists(self):
-        self.assertPathEqual(self.path.lexists)
+        self._check_function(self.path.lexists)
 
     def test_path_ismount(self):
-        self.assertPathEqual(self.path.ismount)
+        self._check_function(self.path.ismount)
 
     def test_path_expanduser(self):
-        self.assertPathEqual(self.path.expanduser)
+        self._check_function(self.path.expanduser)
 
     def test_path_expandvars(self):
-        self.assertPathEqual(self.path.expandvars)
+        self._check_function(self.path.expandvars)
 
     def test_path_normpath(self):
-        self.assertPathEqual(self.path.normpath)
+        self._check_function(self.path.normpath)
 
     def test_path_abspath(self):
-        self.assertPathEqual(self.path.abspath)
+        self._check_function(self.path.abspath)
 
     def test_path_realpath(self):
-        self.assertPathEqual(self.path.realpath)
+        self._check_function(self.path.realpath)
 
     def test_path_relpath(self):
-        self.assertPathEqual(self.path.relpath)
+        self._check_function(self.path.relpath)
 
     def test_path_commonpath(self):
         common_path = self.path.commonpath([self.file_path, self.file_name])
-        self.assertEqual(common_path, self.file_name)
+        self.assertPathEqual(common_path, self.file_name)
 
     def test_path_isdir(self):
-        self.assertPathEqual(self.path.isdir)
+        self._check_function(self.path.isdir)
 
 
 if __name__ == "__main__":
diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py
index f7ed1d1e48fb..9c82c6451479 100644
--- a/Lib/test/test_pathlib.py
+++ b/Lib/test/test_pathlib.py
@@ -1271,10 +1271,13 @@ def assertFileNotFound(self, func, *args, **kwargs):
             func(*args, **kwargs)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
 
+    def assertEqualNormCase(self, path_a, path_b):
+        self.assertEqual(os.path.normcase(path_a), os.path.normcase(path_b))
+
     def _test_cwd(self, p):
         q = self.cls(os.getcwd())
         self.assertEqual(p, q)
-        self.assertEqual(str(p), str(q))
+        self.assertEqualNormCase(str(p), str(q))
         self.assertIs(type(p), type(q))
         self.assertTrue(p.is_absolute())
 
@@ -1285,7 +1288,7 @@ def test_cwd(self):
     def _test_home(self, p):
         q = self.cls(os.path.expanduser('~'))
         self.assertEqual(p, q)
-        self.assertEqual(str(p), str(q))
+        self.assertEqualNormCase(str(p), str(q))
         self.assertIs(type(p), type(q))
         self.assertTrue(p.is_absolute())
 
@@ -1491,15 +1494,15 @@ def test_resolve_common(self):
             p.resolve(strict=True)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         # Non-strict
-        self.assertEqual(str(p.resolve(strict=False)),
-                         os.path.join(BASE, 'foo'))
+        self.assertEqualNormCase(str(p.resolve(strict=False)),
+                                 os.path.join(BASE, 'foo'))
         p = P(BASE, 'foo', 'in', 'spam')
-        self.assertEqual(str(p.resolve(strict=False)),
-                         os.path.join(BASE, 'foo', 'in', 'spam'))
+        self.assertEqualNormCase(str(p.resolve(strict=False)),
+                                 os.path.join(BASE, 'foo', 'in', 'spam'))
         p = P(BASE, '..', 'foo', 'in', 'spam')
-        self.assertEqual(str(p.resolve(strict=False)),
-                         os.path.abspath(os.path.join('foo', 'in', 'spam')))
-        # These are all relative symlinks
+        self.assertEqualNormCase(str(p.resolve(strict=False)),
+                                 os.path.abspath(os.path.join('foo', 'in', 'spam')))
+        # These are all relative symlinks.
         p = P(BASE, 'dirB', 'fileB')
         self._check_resolve_relative(p, p)
         p = P(BASE, 'linkA')
@@ -1996,16 +1999,16 @@ def _check_complex_symlinks(self, link0_target):
         # Resolve absolute paths
         p = (P / 'link0').resolve()
         self.assertEqual(p, P)
-        self.assertEqual(str(p), BASE)
+        self.assertEqualNormCase(str(p), BASE)
         p = (P / 'link1').resolve()
         self.assertEqual(p, P)
-        self.assertEqual(str(p), BASE)
+        self.assertEqualNormCase(str(p), BASE)
         p = (P / 'link2').resolve()
         self.assertEqual(p, P)
-        self.assertEqual(str(p), BASE)
+        self.assertEqualNormCase(str(p), BASE)
         p = (P / 'link3').resolve()
         self.assertEqual(p, P)
-        self.assertEqual(str(p), BASE)
+        self.assertEqualNormCase(str(p), BASE)
 
         # Resolve relative paths
         old_path = os.getcwd()
@@ -2013,16 +2016,16 @@ def _check_complex_symlinks(self, link0_target):
         try:
             p = self.cls('link0').resolve()
             self.assertEqual(p, P)
-            self.assertEqual(str(p), BASE)
+            self.assertEqualNormCase(str(p), BASE)
             p = self.cls('link1').resolve()
             self.assertEqual(p, P)
-            self.assertEqual(str(p), BASE)
+            self.assertEqualNormCase(str(p), BASE)
             p = self.cls('link2').resolve()
             self.assertEqual(p, P)
-            self.assertEqual(str(p), BASE)
+            self.assertEqualNormCase(str(p), BASE)
             p = self.cls('link3').resolve()
             self.assertEqual(p, P)
-            self.assertEqual(str(p), BASE)
+            self.assertEqualNormCase(str(p), BASE)
         finally:
             os.chdir(old_path)
 
diff --git a/Misc/NEWS.d/next/Windows/2019-09-10-14-21-40.bpo-38087.--eIib.rst b/Misc/NEWS.d/next/Windows/2019-09-10-14-21-40.bpo-38087.--eIib.rst
new file mode 100644
index 000000000000..ca625cc775a2
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-10-14-21-40.bpo-38087.--eIib.rst
@@ -0,0 +1 @@
+Fix case sensitivity in test_pathlib and test_ntpath.


From webhook-mailer at python.org  Tue Sep 10 10:18:06 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 14:18:06 -0000
Subject: [Python-checkins] bpo-38088: Fixes distutils not finding
 vcruntime140.dll with only v142 toolset installed (GH-15849)
Message-ID: <mailman.451.1568125086.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0fccd4ab5b2238b6340901dd0023a1a157852792
commit: 0fccd4ab5b2238b6340901dd0023a1a157852792
branch: 3.7
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-10T15:18:02+01:00
summary:

bpo-38088: Fixes distutils not finding vcruntime140.dll with only v142 toolset installed (GH-15849)

files:
A Misc/NEWS.d/next/Windows/2019-09-10-14-17-25.bpo-38088.FOvWSM.rst
M Lib/distutils/_msvccompiler.py

diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py
index 58b20a210247..26112b481af3 100644
--- a/Lib/distutils/_msvccompiler.py
+++ b/Lib/distutils/_msvccompiler.py
@@ -95,7 +95,7 @@ def _find_vcvarsall(plat_spec):
     vcruntime_plat = 'x64' if 'amd64' in plat_spec else 'x86'
     if best_dir:
         vcredist = os.path.join(best_dir, "..", "..", "redist", "MSVC", "**",
-            "Microsoft.VC141.CRT", "vcruntime140.dll")
+            vcruntime_plat, "Microsoft.VC14*.CRT", "vcruntime140.dll")
         try:
             import glob
             vcruntime = glob.glob(vcredist, recursive=True)[-1]
diff --git a/Misc/NEWS.d/next/Windows/2019-09-10-14-17-25.bpo-38088.FOvWSM.rst b/Misc/NEWS.d/next/Windows/2019-09-10-14-17-25.bpo-38088.FOvWSM.rst
new file mode 100644
index 000000000000..37bdeeadd894
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-10-14-17-25.bpo-38088.FOvWSM.rst
@@ -0,0 +1,2 @@
+Fixes distutils not finding vcruntime140.dll with only the v142 toolset
+installed.


From webhook-mailer at python.org  Tue Sep 10 10:25:16 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 14:25:16 -0000
Subject: [Python-checkins] bpo-37913: Link to NotImplemented from new docs
 (GH-15860)
Message-ID: <mailman.452.1568125518.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a39a4c7439ed09069e619c10d30a6d0ec9332d1a
commit: a39a4c7439ed09069e619c10d30a6d0ec9332d1a
branch: master
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-10T15:25:12+01:00
summary:

bpo-37913: Link to NotImplemented from new docs (GH-15860)

files:
M Doc/reference/datamodel.rst

diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index 6b9325da2f39..c6c6e4075039 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -2156,9 +2156,9 @@ through the container; for mappings, :meth:`__iter__` should be the same as
    Called to implement :func:`operator.length_hint`. Should return an estimated
    length for the object (which may be greater or less than the actual length).
    The length must be an integer ``>=`` 0. The return value may also be
-   *NotImplemented*, which is treated the same as if the ``__length_hint__``
-   method didn't exist at all. This method is purely an optimization and is
-   never required for correctness.
+   :const:`NotImplemented`, which is treated the same as if the
+   ``__length_hint__`` method didn't exist at all. This method is purely an
+   optimization and is never required for correctness.
 
    .. versionadded:: 3.4
 


From webhook-mailer at python.org  Tue Sep 10 10:29:34 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 14:29:34 -0000
Subject: [Python-checkins] bpo-38087: Fix case sensitivity in test_pathlib
 and test_ntpath (GH-15850)
Message-ID: <mailman.453.1568125777.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/206e4c3d3547b024935ea9655f960061dffbb80f
commit: 206e4c3d3547b024935ea9655f960061dffbb80f
branch: 3.8
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-10T15:29:28+01:00
summary:

bpo-38087: Fix case sensitivity in test_pathlib and test_ntpath (GH-15850)

files:
A Misc/NEWS.d/next/Windows/2019-09-10-14-21-40.bpo-38087.--eIib.rst
M Lib/test/test_ntpath.py
M Lib/test/test_pathlib.py

diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
index 285fb69dc1e8..c5c96e32d747 100644
--- a/Lib/test/test_ntpath.py
+++ b/Lib/test/test_ntpath.py
@@ -23,10 +23,18 @@
     HAVE_GETFINALPATHNAME = True
 
 
+def _norm(path):
+    if isinstance(path, (bytes, str, os.PathLike)):
+        return ntpath.normcase(os.fsdecode(path))
+    elif hasattr(path, "__iter__"):
+        return tuple(ntpath.normcase(os.fsdecode(p)) for p in path)
+    return path
+
+
 def tester(fn, wantResult):
     fn = fn.replace("\\", "\\\\")
     gotResult = eval(fn)
-    if wantResult != gotResult:
+    if wantResult != gotResult and _norm(wantResult) != _norm(gotResult):
         raise TestFailed("%s should return: %s but returned: %s" \
               %(str(fn), str(wantResult), str(gotResult)))
 
@@ -42,16 +50,22 @@ def tester(fn, wantResult):
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", DeprecationWarning)
         gotResult = eval(fn)
-    if isinstance(wantResult, str):
-        wantResult = os.fsencode(wantResult)
-    elif isinstance(wantResult, tuple):
-        wantResult = tuple(os.fsencode(r) for r in wantResult)
-    if wantResult != gotResult:
+    if _norm(wantResult) != _norm(gotResult):
         raise TestFailed("%s should return: %s but returned: %s" \
               %(str(fn), str(wantResult), repr(gotResult)))
 
 
-class TestNtpath(unittest.TestCase):
+class NtpathTestCase(unittest.TestCase):
+    def assertPathEqual(self, path1, path2):
+        if path1 == path2 or _norm(path1) == _norm(path2):
+            return
+        self.assertEqual(path1, path2)
+
+    def assertPathIn(self, path, pathset):
+        self.assertIn(_norm(path), _norm(pathset))
+
+
+class TestNtpath(NtpathTestCase):
     def test_splitext(self):
         tester('ntpath.splitext("foo.ext")', ('foo', '.ext'))
         tester('ntpath.splitext("/foo/foo.ext")', ('/foo/foo', '.ext'))
@@ -232,8 +246,8 @@ def test_realpath_basic(self):
         self.addCleanup(support.unlink, ABSTFN + "1")
 
         os.symlink(ABSTFN, ABSTFN + "1")
-        self.assertEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN)
-        self.assertEqual(ntpath.realpath(os.fsencode(ABSTFN + "1")),
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN)
+        self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1")),
                          os.fsencode(ABSTFN))
 
     @support.skip_unless_symlink
@@ -245,7 +259,7 @@ def test_realpath_relative(self):
         self.addCleanup(support.unlink, ABSTFN + "1")
 
         os.symlink(ABSTFN, ntpath.relpath(ABSTFN + "1"))
-        self.assertEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN)
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN)
 
     @support.skip_unless_symlink
     @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
@@ -268,39 +282,39 @@ def test_realpath_broken_symlinks(self):
             os.symlink(ABSTFN + r"\broken", "broken4")
             os.symlink(r"recursive\..\broken", "broken5")
 
-            self.assertEqual(ntpath.realpath("broken"),
-                             ABSTFN + r"\missing")
-            self.assertEqual(ntpath.realpath(r"broken\foo"),
-                             ABSTFN + r"\missing\foo")
-            self.assertEqual(ntpath.realpath(r"broken1"),
-                             ABSTFN + r"\missing\bar")
-            self.assertEqual(ntpath.realpath(r"broken1\baz"),
-                             ABSTFN + r"\missing\bar\baz")
-            self.assertEqual(ntpath.realpath("broken2"),
-                             ABSTFN + r"\missing")
-            self.assertEqual(ntpath.realpath("broken3"),
-                             ABSTFN + r"\missing")
-            self.assertEqual(ntpath.realpath("broken4"),
-                             ABSTFN + r"\missing")
-            self.assertEqual(ntpath.realpath("broken5"),
-                             ABSTFN + r"\missing")
-
-            self.assertEqual(ntpath.realpath(b"broken"),
-                             os.fsencode(ABSTFN + r"\missing"))
-            self.assertEqual(ntpath.realpath(rb"broken\foo"),
-                             os.fsencode(ABSTFN + r"\missing\foo"))
-            self.assertEqual(ntpath.realpath(rb"broken1"),
-                             os.fsencode(ABSTFN + r"\missing\bar"))
-            self.assertEqual(ntpath.realpath(rb"broken1\baz"),
-                             os.fsencode(ABSTFN + r"\missing\bar\baz"))
-            self.assertEqual(ntpath.realpath(b"broken2"),
-                             os.fsencode(ABSTFN + r"\missing"))
-            self.assertEqual(ntpath.realpath(rb"broken3"),
-                             os.fsencode(ABSTFN + r"\missing"))
-            self.assertEqual(ntpath.realpath(b"broken4"),
-                             os.fsencode(ABSTFN + r"\missing"))
-            self.assertEqual(ntpath.realpath(b"broken5"),
-                             os.fsencode(ABSTFN + r"\missing"))
+            self.assertPathEqual(ntpath.realpath("broken"),
+                                 ABSTFN + r"\missing")
+            self.assertPathEqual(ntpath.realpath(r"broken\foo"),
+                                 ABSTFN + r"\missing\foo")
+            self.assertPathEqual(ntpath.realpath(r"broken1"),
+                                 ABSTFN + r"\missing\bar")
+            self.assertPathEqual(ntpath.realpath(r"broken1\baz"),
+                                 ABSTFN + r"\missing\bar\baz")
+            self.assertPathEqual(ntpath.realpath("broken2"),
+                                 ABSTFN + r"\missing")
+            self.assertPathEqual(ntpath.realpath("broken3"),
+                                 ABSTFN + r"\missing")
+            self.assertPathEqual(ntpath.realpath("broken4"),
+                                 ABSTFN + r"\missing")
+            self.assertPathEqual(ntpath.realpath("broken5"),
+                                 ABSTFN + r"\missing")
+
+            self.assertPathEqual(ntpath.realpath(b"broken"),
+                                 os.fsencode(ABSTFN + r"\missing"))
+            self.assertPathEqual(ntpath.realpath(rb"broken\foo"),
+                                 os.fsencode(ABSTFN + r"\missing\foo"))
+            self.assertPathEqual(ntpath.realpath(rb"broken1"),
+                                 os.fsencode(ABSTFN + r"\missing\bar"))
+            self.assertPathEqual(ntpath.realpath(rb"broken1\baz"),
+                                 os.fsencode(ABSTFN + r"\missing\bar\baz"))
+            self.assertPathEqual(ntpath.realpath(b"broken2"),
+                                 os.fsencode(ABSTFN + r"\missing"))
+            self.assertPathEqual(ntpath.realpath(rb"broken3"),
+                                 os.fsencode(ABSTFN + r"\missing"))
+            self.assertPathEqual(ntpath.realpath(b"broken4"),
+                                 os.fsencode(ABSTFN + r"\missing"))
+            self.assertPathEqual(ntpath.realpath(b"broken5"),
+                                 os.fsencode(ABSTFN + r"\missing"))
 
     @support.skip_unless_symlink
     @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
@@ -318,39 +332,39 @@ def test_realpath_symlink_loops(self):
         P = "\\\\?\\"
 
         os.symlink(ABSTFN, ABSTFN)
-        self.assertEqual(ntpath.realpath(ABSTFN), P + ABSTFN)
+        self.assertPathEqual(ntpath.realpath(ABSTFN), P + ABSTFN)
 
         # cycles are non-deterministic as to which path is returned, but
         # it will always be the fully resolved path of one member of the cycle
         os.symlink(ABSTFN + "1", ABSTFN + "2")
         os.symlink(ABSTFN + "2", ABSTFN + "1")
         expected = (P + ABSTFN + "1", P + ABSTFN + "2")
-        self.assertIn(ntpath.realpath(ABSTFN + "1"), expected)
-        self.assertIn(ntpath.realpath(ABSTFN + "2"), expected)
-
-        self.assertIn(ntpath.realpath(ABSTFN + "1\\x"),
-                      (ntpath.join(r, "x") for r in expected))
-        self.assertEqual(ntpath.realpath(ABSTFN + "1\\.."),
-                         ntpath.dirname(ABSTFN))
-        self.assertEqual(ntpath.realpath(ABSTFN + "1\\..\\x"),
-                         ntpath.dirname(ABSTFN) + "\\x")
+        self.assertPathIn(ntpath.realpath(ABSTFN + "1"), expected)
+        self.assertPathIn(ntpath.realpath(ABSTFN + "2"), expected)
+
+        self.assertPathIn(ntpath.realpath(ABSTFN + "1\\x"),
+                          (ntpath.join(r, "x") for r in expected))
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\.."),
+                             ntpath.dirname(ABSTFN))
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\x"),
+                             ntpath.dirname(ABSTFN) + "\\x")
         os.symlink(ABSTFN + "x", ABSTFN + "y")
-        self.assertEqual(ntpath.realpath(ABSTFN + "1\\..\\"
-                                         + ntpath.basename(ABSTFN) + "y"),
-                         ABSTFN + "x")
-        self.assertIn(ntpath.realpath(ABSTFN + "1\\..\\"
-                                      + ntpath.basename(ABSTFN) + "1"),
-                      expected)
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\"
+                                             + ntpath.basename(ABSTFN) + "y"),
+                             ABSTFN + "x")
+        self.assertPathIn(ntpath.realpath(ABSTFN + "1\\..\\"
+                                          + ntpath.basename(ABSTFN) + "1"),
+                          expected)
 
         os.symlink(ntpath.basename(ABSTFN) + "a\\b", ABSTFN + "a")
-        self.assertEqual(ntpath.realpath(ABSTFN + "a"), P + ABSTFN + "a")
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "a"), P + ABSTFN + "a")
 
         os.symlink("..\\" + ntpath.basename(ntpath.dirname(ABSTFN))
                    + "\\" + ntpath.basename(ABSTFN) + "c", ABSTFN + "c")
-        self.assertEqual(ntpath.realpath(ABSTFN + "c"), P + ABSTFN + "c")
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "c"), P + ABSTFN + "c")
 
         # Test using relative path as well.
-        self.assertEqual(ntpath.realpath(ntpath.basename(ABSTFN)), P + ABSTFN)
+        self.assertPathEqual(ntpath.realpath(ntpath.basename(ABSTFN)), P + ABSTFN)
 
     @support.skip_unless_symlink
     @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
@@ -369,10 +383,10 @@ def test_realpath_symlink_prefix(self):
             f.write(b'1')
         os.symlink("\\\\?\\" + ABSTFN + "3.", ABSTFN + "3.link")
 
-        self.assertEqual(ntpath.realpath(ABSTFN + "3link"),
-                         ABSTFN + "3")
-        self.assertEqual(ntpath.realpath(ABSTFN + "3.link"),
-                         "\\\\?\\" + ABSTFN + "3.")
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "3link"),
+                             ABSTFN + "3")
+        self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link"),
+                             "\\\\?\\" + ABSTFN + "3.")
 
         # Resolved paths should be usable to open target files
         with open(ntpath.realpath(ABSTFN + "3link"), "rb") as f:
@@ -381,10 +395,10 @@ def test_realpath_symlink_prefix(self):
             self.assertEqual(f.read(), b'1')
 
         # When the prefix is included, it is not stripped
-        self.assertEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link"),
-                         "\\\\?\\" + ABSTFN + "3")
-        self.assertEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link"),
-                         "\\\\?\\" + ABSTFN + "3.")
+        self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link"),
+                             "\\\\?\\" + ABSTFN + "3")
+        self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link"),
+                             "\\\\?\\" + ABSTFN + "3.")
 
     def test_expandvars(self):
         with support.EnvironmentVarGuard() as env:
@@ -658,7 +672,7 @@ class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase):
     attributes = ['relpath']
 
 
-class PathLikeTests(unittest.TestCase):
+class PathLikeTests(NtpathTestCase):
 
     path = ntpath
 
@@ -669,67 +683,67 @@ def setUp(self):
         with open(self.file_name, 'xb', 0) as file:
             file.write(b"test_ntpath.PathLikeTests")
 
-    def assertPathEqual(self, func):
-        self.assertEqual(func(self.file_path), func(self.file_name))
+    def _check_function(self, func):
+        self.assertPathEqual(func(self.file_path), func(self.file_name))
 
     def test_path_normcase(self):
-        self.assertPathEqual(self.path.normcase)
+        self._check_function(self.path.normcase)
 
     def test_path_isabs(self):
-        self.assertPathEqual(self.path.isabs)
+        self._check_function(self.path.isabs)
 
     def test_path_join(self):
         self.assertEqual(self.path.join('a', FakePath('b'), 'c'),
                          self.path.join('a', 'b', 'c'))
 
     def test_path_split(self):
-        self.assertPathEqual(self.path.split)
+        self._check_function(self.path.split)
 
     def test_path_splitext(self):
-        self.assertPathEqual(self.path.splitext)
+        self._check_function(self.path.splitext)
 
     def test_path_splitdrive(self):
-        self.assertPathEqual(self.path.splitdrive)
+        self._check_function(self.path.splitdrive)
 
     def test_path_basename(self):
-        self.assertPathEqual(self.path.basename)
+        self._check_function(self.path.basename)
 
     def test_path_dirname(self):
-        self.assertPathEqual(self.path.dirname)
+        self._check_function(self.path.dirname)
 
     def test_path_islink(self):
-        self.assertPathEqual(self.path.islink)
+        self._check_function(self.path.islink)
 
     def test_path_lexists(self):
-        self.assertPathEqual(self.path.lexists)
+        self._check_function(self.path.lexists)
 
     def test_path_ismount(self):
-        self.assertPathEqual(self.path.ismount)
+        self._check_function(self.path.ismount)
 
     def test_path_expanduser(self):
-        self.assertPathEqual(self.path.expanduser)
+        self._check_function(self.path.expanduser)
 
     def test_path_expandvars(self):
-        self.assertPathEqual(self.path.expandvars)
+        self._check_function(self.path.expandvars)
 
     def test_path_normpath(self):
-        self.assertPathEqual(self.path.normpath)
+        self._check_function(self.path.normpath)
 
     def test_path_abspath(self):
-        self.assertPathEqual(self.path.abspath)
+        self._check_function(self.path.abspath)
 
     def test_path_realpath(self):
-        self.assertPathEqual(self.path.realpath)
+        self._check_function(self.path.realpath)
 
     def test_path_relpath(self):
-        self.assertPathEqual(self.path.relpath)
+        self._check_function(self.path.relpath)
 
     def test_path_commonpath(self):
         common_path = self.path.commonpath([self.file_path, self.file_name])
-        self.assertEqual(common_path, self.file_name)
+        self.assertPathEqual(common_path, self.file_name)
 
     def test_path_isdir(self):
-        self.assertPathEqual(self.path.isdir)
+        self._check_function(self.path.isdir)
 
 
 if __name__ == "__main__":
diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py
index f74524d992a1..027331f95911 100644
--- a/Lib/test/test_pathlib.py
+++ b/Lib/test/test_pathlib.py
@@ -1274,10 +1274,13 @@ def assertFileNotFound(self, func, *args, **kwargs):
             func(*args, **kwargs)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
 
+    def assertEqualNormCase(self, path_a, path_b):
+        self.assertEqual(os.path.normcase(path_a), os.path.normcase(path_b))
+
     def _test_cwd(self, p):
         q = self.cls(os.getcwd())
         self.assertEqual(p, q)
-        self.assertEqual(str(p), str(q))
+        self.assertEqualNormCase(str(p), str(q))
         self.assertIs(type(p), type(q))
         self.assertTrue(p.is_absolute())
 
@@ -1288,7 +1291,7 @@ def test_cwd(self):
     def _test_home(self, p):
         q = self.cls(os.path.expanduser('~'))
         self.assertEqual(p, q)
-        self.assertEqual(str(p), str(q))
+        self.assertEqualNormCase(str(p), str(q))
         self.assertIs(type(p), type(q))
         self.assertTrue(p.is_absolute())
 
@@ -1496,14 +1499,14 @@ def test_resolve_common(self):
             p.resolve(strict=True)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         # Non-strict
-        self.assertEqual(str(p.resolve(strict=False)),
-                         os.path.join(BASE, 'foo'))
+        self.assertEqualNormCase(str(p.resolve(strict=False)),
+                                 os.path.join(BASE, 'foo'))
         p = P(BASE, 'foo', 'in', 'spam')
-        self.assertEqual(str(p.resolve(strict=False)),
-                         os.path.join(BASE, 'foo', 'in', 'spam'))
+        self.assertEqualNormCase(str(p.resolve(strict=False)),
+                                 os.path.join(BASE, 'foo', 'in', 'spam'))
         p = P(BASE, '..', 'foo', 'in', 'spam')
-        self.assertEqual(str(p.resolve(strict=False)),
-                         os.path.abspath(os.path.join('foo', 'in', 'spam')))
+        self.assertEqualNormCase(str(p.resolve(strict=False)),
+                                 os.path.abspath(os.path.join('foo', 'in', 'spam')))
         # These are all relative symlinks.
         p = P(BASE, 'dirB', 'fileB')
         self._check_resolve_relative(p, p)
@@ -2050,16 +2053,16 @@ def _check_complex_symlinks(self, link0_target):
         # Resolve absolute paths.
         p = (P / 'link0').resolve()
         self.assertEqual(p, P)
-        self.assertEqual(str(p), BASE)
+        self.assertEqualNormCase(str(p), BASE)
         p = (P / 'link1').resolve()
         self.assertEqual(p, P)
-        self.assertEqual(str(p), BASE)
+        self.assertEqualNormCase(str(p), BASE)
         p = (P / 'link2').resolve()
         self.assertEqual(p, P)
-        self.assertEqual(str(p), BASE)
+        self.assertEqualNormCase(str(p), BASE)
         p = (P / 'link3').resolve()
         self.assertEqual(p, P)
-        self.assertEqual(str(p), BASE)
+        self.assertEqualNormCase(str(p), BASE)
 
         # Resolve relative paths.
         old_path = os.getcwd()
@@ -2067,16 +2070,16 @@ def _check_complex_symlinks(self, link0_target):
         try:
             p = self.cls('link0').resolve()
             self.assertEqual(p, P)
-            self.assertEqual(str(p), BASE)
+            self.assertEqualNormCase(str(p), BASE)
             p = self.cls('link1').resolve()
             self.assertEqual(p, P)
-            self.assertEqual(str(p), BASE)
+            self.assertEqualNormCase(str(p), BASE)
             p = self.cls('link2').resolve()
             self.assertEqual(p, P)
-            self.assertEqual(str(p), BASE)
+            self.assertEqualNormCase(str(p), BASE)
             p = self.cls('link3').resolve()
             self.assertEqual(p, P)
-            self.assertEqual(str(p), BASE)
+            self.assertEqualNormCase(str(p), BASE)
         finally:
             os.chdir(old_path)
 
diff --git a/Misc/NEWS.d/next/Windows/2019-09-10-14-21-40.bpo-38087.--eIib.rst b/Misc/NEWS.d/next/Windows/2019-09-10-14-21-40.bpo-38087.--eIib.rst
new file mode 100644
index 000000000000..ca625cc775a2
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-10-14-21-40.bpo-38087.--eIib.rst
@@ -0,0 +1 @@
+Fix case sensitivity in test_pathlib and test_ntpath.


From webhook-mailer at python.org  Tue Sep 10 10:31:11 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 14:31:11 -0000
Subject: [Python-checkins] bpo-38090: Fix reference leak in ceval.c
 (GH-15848)
Message-ID: <mailman.454.1568125873.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/45bc3928e232603a97451dea3106d824b0f7a392
commit: 45bc3928e232603a97451dea3106d824b0f7a392
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-10T15:31:07+01:00
summary:

bpo-38090: Fix reference leak in ceval.c (GH-15848)

(cherry picked from commit a511c7a4961a684db1f8d0ed438822d87d7d3dcf)

Co-authored-by: Pablo Galindo <Pablogsal at gmail.com>

files:
M Python/ceval.c

diff --git a/Python/ceval.c b/Python/ceval.c
index 07ec3293adf1..3306fb9728e8 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -5238,7 +5238,6 @@ import_from(PyThreadState *tstate, PyObject *v, PyObject *name)
     else {
         _Py_IDENTIFIER(__spec__);
         PyObject *spec = _PyObject_GetAttrId(v, &PyId___spec__);
-        Py_XINCREF(spec);
         const char *fmt =
             _PyModuleSpec_IsInitializing(spec) ?
             "cannot import name %R from partially initialized module %R "


From webhook-mailer at python.org  Tue Sep 10 10:31:30 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 14:31:30 -0000
Subject: [Python-checkins] bpo-37913: document that __length_hint__ can
 return NotImplemented (GH-15383)
Message-ID: <mailman.455.1568125891.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ed99bb9ca68b37cfaec3629afa67d70289f3ffc7
commit: ed99bb9ca68b37cfaec3629afa67d70289f3ffc7
branch: 3.8
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-10T15:31:26+01:00
summary:

bpo-37913: document that __length_hint__ can return NotImplemented (GH-15383)

files:
M Doc/reference/datamodel.rst

diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index 8813f57587f0..c6c6e4075039 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -2155,7 +2155,9 @@ through the container; for mappings, :meth:`__iter__` should be the same as
 
    Called to implement :func:`operator.length_hint`. Should return an estimated
    length for the object (which may be greater or less than the actual length).
-   The length must be an integer ``>=`` 0. This method is purely an
+   The length must be an integer ``>=`` 0. The return value may also be
+   :const:`NotImplemented`, which is treated the same as if the
+   ``__length_hint__`` method didn't exist at all. This method is purely an
    optimization and is never required for correctness.
 
    .. versionadded:: 3.4


From webhook-mailer at python.org  Tue Sep 10 10:37:38 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Tue, 10 Sep 2019 14:37:38 -0000
Subject: [Python-checkins] Fix calling order of PyEval_InitThreads. (GH-5833)
Message-ID: <mailman.456.1568126259.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4fbe0ee5487055216c71596d694e323dbac02582
commit: 4fbe0ee5487055216c71596d694e323dbac02582
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-10T15:37:34+01:00
summary:

Fix calling order of PyEval_InitThreads. (GH-5833)

As described in Doc/c-api/init.rst, PyEval_InitThreads() cannot be called
before Py_Initialize() function.
(cherry picked from commit 9e61066355b4b55c873d56f5f106a22463b56862)

Co-authored-by: Kenta Murata <mrkn at users.noreply.github.com>

files:
M Modules/_ctypes/callbacks.c

diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c
index 5a8303fba290..a3d1c386b724 100644
--- a/Modules/_ctypes/callbacks.c
+++ b/Modules/_ctypes/callbacks.c
@@ -417,8 +417,8 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable,
 static void LoadPython(void)
 {
     if (!Py_IsInitialized()) {
-        PyEval_InitThreads();
         Py_Initialize();
+        PyEval_InitThreads();
     }
 }
 


From webhook-mailer at python.org  Tue Sep 10 10:37:59 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Tue, 10 Sep 2019 14:37:59 -0000
Subject: [Python-checkins] Fix calling order of PyEval_InitThreads.
 (GH-15836)
Message-ID: <mailman.457.1568126281.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/21dacea6ad1c49e1cafbe8bac803ddc22a534274
commit: 21dacea6ad1c49e1cafbe8bac803ddc22a534274
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-10T15:37:56+01:00
summary:

Fix calling order of PyEval_InitThreads. (GH-15836)

As described in Doc/c-api/init.rst, PyEval_InitThreads() cannot be called
before Py_Initialize() function.
(cherry picked from commit 9e61066355b4b55c873d56f5f106a22463b56862)

Co-authored-by: Kenta Murata <mrkn at users.noreply.github.com>

files:
M Modules/_ctypes/callbacks.c

diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c
index 97463b599bc0..d2d9a6587d79 100644
--- a/Modules/_ctypes/callbacks.c
+++ b/Modules/_ctypes/callbacks.c
@@ -422,8 +422,8 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable,
 static void LoadPython(void)
 {
     if (!Py_IsInitialized()) {
-        PyEval_InitThreads();
         Py_Initialize();
+        PyEval_InitThreads();
     }
 }
 


From webhook-mailer at python.org  Tue Sep 10 10:40:54 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 14:40:54 -0000
Subject: [Python-checkins] bpo-37504: Fix documentation build with texinfo
 builder (GH-14606)
Message-ID: <mailman.458.1568126455.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c3d679fd398f42a2e489fbe3dab17fac1fb2439c
commit: c3d679fd398f42a2e489fbe3dab17fac1fb2439c
branch: master
author: Dmitry Shachnev <mitya57 at users.noreply.github.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-10T15:40:50+01:00
summary:

bpo-37504: Fix documentation build with texinfo builder (GH-14606)

In the table model used by docutils, the `cols` attribute of `tgroup`
nodes is mandatory, see [1]. It is used in texinfo builder in [2].

[1]: https://www.oasis-open.org/specs/tm9901.htm#AEN348
[2]: https://github.com/sphinx-doc/sphinx/blob/v2.1.2/sphinx/writers/texinfo.py#L1129

* Doc: Add texinfo support to the Makefile

files:
M Doc/Makefile
M Doc/tools/extensions/pyspecific.py

diff --git a/Doc/Makefile b/Doc/Makefile
index f06689f83d87..86207fe57a0f 100644
--- a/Doc/Makefile
+++ b/Doc/Makefile
@@ -20,7 +20,7 @@ PAPEROPT_letter = -D latex_elements.papersize=letterpaper
 ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) \
                 $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES)
 
-.PHONY: help build html htmlhelp latex text changes linkcheck \
+.PHONY: help build html htmlhelp latex text texinfo changes linkcheck \
 	suspicious coverage doctest pydoc-topics htmlview clean dist check serve \
 	autobuild-dev autobuild-stable venv
 
@@ -33,6 +33,7 @@ help:
 	@echo "  htmlhelp   to make HTML files and a HTML help project"
 	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
 	@echo "  text       to make plain text files"
+	@echo "  texinfo    to make Texinfo file"
 	@echo "  epub       to make EPUB files"
 	@echo "  changes    to make an overview over all changed/added/deprecated items"
 	@echo "  linkcheck  to check all external links for integrity"
@@ -89,6 +90,11 @@ text: BUILDER = text
 text: build
 	@echo "Build finished; the text files are in build/text."
 
+texinfo: BUILDER = texinfo
+texinfo: build
+	@echo "Build finished; the python.texi file is in build/texinfo."
+	@echo "Run \`make info' in that directory to run it through makeinfo."
+
 epub: BUILDER = epub
 epub: build
 	@echo "Build finished; the epub files are in build/epub."
@@ -183,6 +189,17 @@ dist:
 	make epub
 	cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub
 
+	# archive the texinfo build
+	rm -rf build/texinfo
+	make texinfo
+	make info --directory=build/texinfo
+	cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo
+	tar -C dist -cf dist/python-$(DISTVERSION)-docs-texinfo.tar python-$(DISTVERSION)-docs-texinfo
+	bzip2 -9 -k dist/python-$(DISTVERSION)-docs-texinfo.tar
+	(cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-texinfo.zip python-$(DISTVERSION)-docs-texinfo)
+	rm -r dist/python-$(DISTVERSION)-docs-texinfo
+	rm dist/python-$(DISTVERSION)-docs-texinfo.tar
+
 check:
 	$(PYTHON) tools/rstlint.py -i tools -i $(VENVDIR) -i README.rst
 
diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index 975c87d12383..4e2ad7fc2a1d 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -540,6 +540,7 @@ def process_audit_events(app, doctree, fromdocname):
         nodes.colspec(colwidth=30),
         nodes.colspec(colwidth=55),
         nodes.colspec(colwidth=15),
+        cols=3,
     )
     head = nodes.thead()
     body = nodes.tbody()


From webhook-mailer at python.org  Tue Sep 10 10:42:02 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 14:42:02 -0000
Subject: [Python-checkins] bpo-37913: document that __length_hint__ can
 return NotImplemented (GH-15383)
Message-ID: <mailman.459.1568126522.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/aa3b629a09bfcdf26cdfbbf7d6445ae333c0d44d
commit: aa3b629a09bfcdf26cdfbbf7d6445ae333c0d44d
branch: 3.7
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-10T15:41:58+01:00
summary:

bpo-37913: document that __length_hint__ can return NotImplemented (GH-15383)

files:
M Doc/reference/datamodel.rst

diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index 962ec7e6c719..f5e23d85b9d8 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -2142,7 +2142,9 @@ through the container; for mappings, :meth:`__iter__` should be the same as
 
    Called to implement :func:`operator.length_hint`. Should return an estimated
    length for the object (which may be greater or less than the actual length).
-   The length must be an integer ``>=`` 0. This method is purely an
+   The length must be an integer ``>=`` 0. The return value may also be
+   :const:`NotImplemented`, which is treated the same as if the
+   ``__length_hint__`` method didn't exist at all. This method is purely an
    optimization and is never required for correctness.
 
    .. versionadded:: 3.4


From webhook-mailer at python.org  Tue Sep 10 10:42:30 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 14:42:30 -0000
Subject: [Python-checkins] bpo-38088: Fixes distutils not finding
 vcruntime140.dll with only v142 toolset installed (GH-15849)
Message-ID: <mailman.460.1568126551.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2ed0ac6bf17d599bc246c513c32710c154e50a64
commit: 2ed0ac6bf17d599bc246c513c32710c154e50a64
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-10T15:42:26+01:00
summary:

bpo-38088: Fixes distutils not finding vcruntime140.dll with only v142 toolset installed (GH-15849)

(cherry picked from commit cd8221152dd235ec5d06e3d9d0d8787645bbac8e)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-10-14-17-25.bpo-38088.FOvWSM.rst
M Lib/distutils/_msvccompiler.py

diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py
index 6e14f330d7f4..e8e4b717b973 100644
--- a/Lib/distutils/_msvccompiler.py
+++ b/Lib/distutils/_msvccompiler.py
@@ -107,7 +107,7 @@ def _find_vcvarsall(plat_spec):
 
     if best_dir:
         vcredist = os.path.join(best_dir, "..", "..", "redist", "MSVC", "**",
-            vcruntime_plat, "Microsoft.VC141.CRT", "vcruntime140.dll")
+            vcruntime_plat, "Microsoft.VC14*.CRT", "vcruntime140.dll")
         try:
             import glob
             vcruntime = glob.glob(vcredist, recursive=True)[-1]
diff --git a/Misc/NEWS.d/next/Windows/2019-09-10-14-17-25.bpo-38088.FOvWSM.rst b/Misc/NEWS.d/next/Windows/2019-09-10-14-17-25.bpo-38088.FOvWSM.rst
new file mode 100644
index 000000000000..37bdeeadd894
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-10-14-17-25.bpo-38088.FOvWSM.rst
@@ -0,0 +1,2 @@
+Fixes distutils not finding vcruntime140.dll with only the v142 toolset
+installed.


From webhook-mailer at python.org  Tue Sep 10 10:48:09 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Tue, 10 Sep 2019 14:48:09 -0000
Subject: [Python-checkins] Correct minor grammatical mistake in open docs
 (GH-15865)
Message-ID: <mailman.461.1568126890.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/05184515f924c3880067aad87940dd466f751585
commit: 05184515f924c3880067aad87940dd466f751585
branch: master
author: Andre Delfino <adelfino at gmail.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-10T15:48:05+01:00
summary:

Correct minor grammatical mistake in open docs (GH-15865)

files:
M Doc/library/functions.rst

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 98a55c61eff2..2e385a2a37d5 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -1074,8 +1074,8 @@ are always available.  They are listed here in alphabetical order.
    ========= ===============================================================
 
    The default mode is ``'r'`` (open for reading text, synonym of ``'rt'``).
-   Modes ``'w+'`` and ``'w+b'`` opens and truncates the file.  Modes ``'r+'``
-   and ``'r+b'`` opens the file with no truncation.
+   Modes ``'w+'`` and ``'w+b'`` open and truncate the file.  Modes ``'r+'``
+   and ``'r+b'`` open the file with no truncation.
 
    As mentioned in the :ref:`io-overview`, Python distinguishes between binary
    and text I/O.  Files opened in binary mode (including ``'b'`` in the *mode*


From webhook-mailer at python.org  Tue Sep 10 10:48:25 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 14:48:25 -0000
Subject: [Python-checkins] Fix subprocess docstring typo (GH-15812)
Message-ID: <mailman.462.1568126906.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9f54feabee26133ac64c6d0e58969005b30614d3
commit: 9f54feabee26133ac64c6d0e58969005b30614d3
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T07:48:22-07:00
summary:

Fix subprocess docstring typo (GH-15812)

(cherry picked from commit 182e1d1f849757439f2031504f142fa4e1251611)

Co-authored-by: Matthias <xmatthias at outlook.com>

files:
M Lib/subprocess.py

diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 2c30e344e61f..395e5e7ba75c 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -394,7 +394,7 @@ def check_output(*popenargs, timeout=None, **kwargs):
     b'when in the course of barman events\n'
 
     By default, all communication is in bytes, and therefore any "input"
-    should be bytes, and the return value wil be bytes.  If in text mode,
+    should be bytes, and the return value will be bytes.  If in text mode,
     any "input" should be a string, and the return value will be a string
     decoded according to locale encoding, or by "encoding" if set. Text mode
     is triggered by setting any of text, encoding, errors or universal_newlines.


From webhook-mailer at python.org  Tue Sep 10 10:50:30 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Tue, 10 Sep 2019 14:50:30 -0000
Subject: [Python-checkins] Correct info about "f.read(size)". (GH13852)
Message-ID: <mailman.463.1568127030.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/faff81c05f838b0b7a64bbc8c53c02a9b04bb79d
commit: faff81c05f838b0b7a64bbc8c53c02a9b04bb79d
branch: master
author: William Andrea <william.j.andrea at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-10T15:50:26+01:00
summary:

Correct info about "f.read(size)". (GH13852)

In text mode, the "size" parameter indicates the number of characters, not bytes.

files:
M Doc/tutorial/inputoutput.rst

diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst
index fc2bd5578c4c..9fe1276ec9d1 100644
--- a/Doc/tutorial/inputoutput.rst
+++ b/Doc/tutorial/inputoutput.rst
@@ -358,8 +358,8 @@ To read a file's contents, call ``f.read(size)``, which reads some quantity of
 data and returns it as a string (in text mode) or bytes object (in binary mode).
 *size* is an optional numeric argument.  When *size* is omitted or negative, the
 entire contents of the file will be read and returned; it's your problem if the
-file is twice as large as your machine's memory. Otherwise, at most *size* bytes
-are read and returned.
+file is twice as large as your machine's memory. Otherwise, at most *size*
+characters (in text mode) or *size* bytes (in binary mode) are read and returned.
 If the end of the file has been reached, ``f.read()`` will return an empty
 string (``''``).  ::
 


From webhook-mailer at python.org  Tue Sep 10 10:51:33 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 14:51:33 -0000
Subject: [Python-checkins] Fix subprocess docstring typo (GH-15812)
Message-ID: <mailman.464.1568127093.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/41c965f93007b53710b9f63918f6710dde321226
commit: 41c965f93007b53710b9f63918f6710dde321226
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T07:51:29-07:00
summary:

Fix subprocess docstring typo (GH-15812)

(cherry picked from commit 182e1d1f849757439f2031504f142fa4e1251611)

Co-authored-by: Matthias <xmatthias at outlook.com>

files:
M Lib/subprocess.py

diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 5bbeba47a374..aed7292541e0 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -395,7 +395,7 @@ def check_output(*popenargs, timeout=None, **kwargs):
     b'when in the course of barman events\n'
 
     By default, all communication is in bytes, and therefore any "input"
-    should be bytes, and the return value wil be bytes.  If in text mode,
+    should be bytes, and the return value will be bytes.  If in text mode,
     any "input" should be a string, and the return value will be a string
     decoded according to locale encoding, or by "encoding" if set. Text mode
     is triggered by setting any of text, encoding, errors or universal_newlines.


From webhook-mailer at python.org  Tue Sep 10 10:51:48 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 14:51:48 -0000
Subject: [Python-checkins] bpo-37504: Fix documentation build with texinfo
 builder (GH-14606)
Message-ID: <mailman.465.1568127109.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/313f80192a7396ea3e5ab107175afe4c5a017ab8
commit: 313f80192a7396ea3e5ab107175afe4c5a017ab8
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-10T15:51:43+01:00
summary:

bpo-37504: Fix documentation build with texinfo builder (GH-14606)

In the table model used by docutils, the `cols` attribute of `tgroup`
nodes is mandatory, see [1]. It is used in texinfo builder in [2].

[1]: https://www.oasis-open.org/specs/tm9901.htmGH-AEN348
[2]: https://github.com/sphinx-doc/sphinx/blob/v2.1.2/sphinx/writers/texinfo.pyGH-L1129

* Doc: Add texinfo support to the Makefile
(cherry picked from commit c3d679fd398f42a2e489fbe3dab17fac1fb2439c)

Co-authored-by: Dmitry Shachnev <mitya57 at users.noreply.github.com>

files:
M Doc/Makefile
M Doc/tools/extensions/pyspecific.py

diff --git a/Doc/Makefile b/Doc/Makefile
index 05eeab9124df..f589b6e75f61 100644
--- a/Doc/Makefile
+++ b/Doc/Makefile
@@ -20,7 +20,7 @@ PAPEROPT_letter = -D latex_elements.papersize=letterpaper
 ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) \
                 $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES)
 
-.PHONY: help build html htmlhelp latex text changes linkcheck \
+.PHONY: help build html htmlhelp latex text texinfo changes linkcheck \
 	suspicious coverage doctest pydoc-topics htmlview clean dist check serve \
 	autobuild-dev autobuild-stable venv
 
@@ -33,6 +33,7 @@ help:
 	@echo "  htmlhelp   to make HTML files and a HTML help project"
 	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
 	@echo "  text       to make plain text files"
+	@echo "  texinfo    to make Texinfo file"
 	@echo "  epub       to make EPUB files"
 	@echo "  changes    to make an overview over all changed/added/deprecated items"
 	@echo "  linkcheck  to check all external links for integrity"
@@ -89,6 +90,11 @@ text: BUILDER = text
 text: build
 	@echo "Build finished; the text files are in build/text."
 
+texinfo: BUILDER = texinfo
+texinfo: build
+	@echo "Build finished; the python.texi file is in build/texinfo."
+	@echo "Run \`make info' in that directory to run it through makeinfo."
+
 epub: BUILDER = epub
 epub: build
 	@echo "Build finished; the epub files are in build/epub."
@@ -183,6 +189,17 @@ dist:
 	make epub
 	cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub
 
+	# archive the texinfo build
+	rm -rf build/texinfo
+	make texinfo
+	make info --directory=build/texinfo
+	cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo
+	tar -C dist -cf dist/python-$(DISTVERSION)-docs-texinfo.tar python-$(DISTVERSION)-docs-texinfo
+	bzip2 -9 -k dist/python-$(DISTVERSION)-docs-texinfo.tar
+	(cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-texinfo.zip python-$(DISTVERSION)-docs-texinfo)
+	rm -r dist/python-$(DISTVERSION)-docs-texinfo
+	rm dist/python-$(DISTVERSION)-docs-texinfo.tar
+
 check:
 	$(PYTHON) tools/rstlint.py -i tools -i $(VENVDIR) -i README.rst
 
diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index f41077b07615..82dfc6bb263b 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -540,6 +540,7 @@ def process_audit_events(app, doctree, fromdocname):
         nodes.colspec(colwidth=30),
         nodes.colspec(colwidth=55),
         nodes.colspec(colwidth=15),
+        cols=3,
     )
     head = nodes.thead()
     body = nodes.tbody()


From webhook-mailer at python.org  Tue Sep 10 10:57:20 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 14:57:20 -0000
Subject: [Python-checkins] bpo-37913: document that __length_hint__ can
 return NotImplemented (GH-15383)
Message-ID: <mailman.466.1568127441.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/756eb849d7d968f0a0924fa33b68c62831e8884e
commit: 756eb849d7d968f0a0924fa33b68c62831e8884e
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-10T15:57:16+01:00
summary:

bpo-37913: document that __length_hint__ can return NotImplemented (GH-15383)

(cherry picked from commit ed99bb9ca68b37cfaec3629afa67d70289f3ffc7)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:




From webhook-mailer at python.org  Tue Sep 10 10:58:33 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Tue, 10 Sep 2019 14:58:33 -0000
Subject: [Python-checkins] [3.8] bpo-38086: Sync importlib.metadata with
 importlib_metadata 0.21. (GH-15840) (#15861)
Message-ID: <mailman.467.1568127516.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/97c2f68a4c0bf4e69a1e769928d7928fab73a666
commit: 97c2f68a4c0bf4e69a1e769928d7928fab73a666
branch: 3.8
author: Jason R. Coombs <jaraco at jaraco.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T15:58:29+01:00
summary:

[3.8] bpo-38086: Sync importlib.metadata with importlib_metadata 0.21. (GH-15840) (#15861)

https://gitlab.com/python-devs/importlib_metadata/-/tags/0.21.
(cherry picked from commit 17499d82702432955d8e442a1871ff276ca64bc5)

Co-authored-by: Jason R. Coombs <jaraco at jaraco.com>

files:
A Misc/NEWS.d/next/Library/2019-09-10-11-42-59.bpo-38086.w5TlG-.rst
M Doc/library/importlib.metadata.rst
M Doc/tools/susp-ignored.csv
M Lib/importlib/_bootstrap_external.py
M Lib/importlib/metadata.py
M Lib/test/test_importlib/test_main.py
M Lib/test/test_importlib/test_metadata_api.py
M Python/importlib_external.h

diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst
index 23c8cdeb21c2..4a4a8f7dfa0f 100644
--- a/Doc/library/importlib.metadata.rst
+++ b/Doc/library/importlib.metadata.rst
@@ -172,10 +172,10 @@ Distribution requirements
 -------------------------
 
 To get the full set of requirements for a distribution, use the ``requires()``
-function.  Note that this returns an iterator::
+function::
 
-    >>> list(requires('wheel'))  # doctest: +SKIP
-    ["pytest (>=3.0.0) ; extra == 'test'"]
+    >>> requires('wheel')  # doctest: +SKIP
+    ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"]
 
 
 Distributions
@@ -224,23 +224,25 @@ The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the
 interface expected of finders by Python's import system.
 ``importlib.metadata`` extends this protocol by looking for an optional
 ``find_distributions`` callable on the finders from
-``sys.meta_path``.  If the finder has this method, it must return
-an iterator over instances of the ``Distribution`` abstract class. This
-method must have the signature::
+``sys.meta_path`` and presents this extended interface as the
+``DistributionFinder`` abstract base class, which defines this abstract
+method::
 
-    def find_distributions(name=None, path=None):
+    @abc.abstractmethod
+    def find_distributions(context=DistributionFinder.Context()):
         """Return an iterable of all Distribution instances capable of
-        loading the metadata for packages matching the name
-        (or all names if not supplied) along the paths in the list
-        of directories ``path`` (defaults to sys.path).
+        loading the metadata for packages for the indicated ``context``.
         """
 
+The ``DistributionFinder.Context`` object provides ``.path`` and ``.name``
+properties indicating the path to search and names to match and may
+supply other relevant context.
+
 What this means in practice is that to support finding distribution package
 metadata in locations other than the file system, you should derive from
-``Distribution`` and implement the ``load_metadata()`` method.  This takes a
-single argument which is the name of the package whose metadata is being
-found.  This instance of the ``Distribution`` base abstract class is what your
-finder's ``find_distributions()`` method should return.
+``Distribution`` and implement the ``load_metadata()`` method. Then from
+your finder, return instances of this derived ``Distribution`` in the
+``find_distributions()`` method.
 
 
 .. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points
diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv
index d6252ed2ffa2..c97c859fe891 100644
--- a/Doc/tools/susp-ignored.csv
+++ b/Doc/tools/susp-ignored.csv
@@ -350,4 +350,4 @@ whatsnew/changelog,,::,error::BytesWarning
 whatsnew/changelog,,::,default::BytesWarning
 whatsnew/changelog,,::,default::DeprecationWarning
 library/importlib.metadata,,:main,"EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts')"
-library/importlib.metadata,,`,of directories ``path`` (defaults to sys.path).
+library/importlib.metadata,,`,loading the metadata for packages for the indicated ``context``.
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index 67bd1d3cf99e..ba78c57eebc3 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -1369,21 +1369,19 @@ def find_module(cls, fullname, path=None):
         return spec.loader
 
     @classmethod
-    def find_distributions(cls, name=None, path=None):
+    def find_distributions(self, context=None):
         """
         Find distributions.
 
         Return an iterable of all Distribution instances capable of
-        loading the metadata for packages matching the ``name``
-        (or all names if not supplied) along the paths in the list
-        of directories ``path`` (defaults to sys.path).
+        loading the metadata for packages matching ``context.name``
+        (or all names if ``None`` indicated) along the paths in the list
+        of directories ``context.path``.
         """
-        import re
-        from importlib.metadata import PathDistribution
-        if path is None:
-            path = sys.path
-        pattern = '.*' if name is None else re.escape(name)
-        found = cls._search_paths(pattern, path)
+        from importlib.metadata import PathDistribution, DistributionFinder
+        if context is None:
+            context = DistributionFinder.Context()
+        found = self._search_paths(context.pattern, context.path)
         return map(PathDistribution, found)
 
     @classmethod
diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py
index 3b46142231ec..e23076654fe4 100644
--- a/Lib/importlib/metadata.py
+++ b/Lib/importlib/metadata.py
@@ -19,6 +19,7 @@
 
 __all__ = [
     'Distribution',
+    'DistributionFinder',
     'PackageNotFoundError',
     'distribution',
     'distributions',
@@ -158,7 +159,7 @@ def from_name(cls, name):
             metadata cannot be found.
         """
         for resolver in cls._discover_resolvers():
-            dists = resolver(name)
+            dists = resolver(DistributionFinder.Context(name=name))
             dist = next(dists, None)
             if dist is not None:
                 return dist
@@ -166,16 +167,33 @@ def from_name(cls, name):
             raise PackageNotFoundError(name)
 
     @classmethod
-    def discover(cls):
+    def discover(cls, **kwargs):
         """Return an iterable of Distribution objects for all packages.
 
+        Pass a ``context`` or pass keyword arguments for constructing
+        a context.
+
+        :context: A ``DistributionFinder.Context`` object.
         :return: Iterable of Distribution objects for all packages.
         """
+        context = kwargs.pop('context', None)
+        if context and kwargs:
+            raise ValueError("cannot accept context and kwargs")
+        context = context or DistributionFinder.Context(**kwargs)
         return itertools.chain.from_iterable(
-            resolver()
+            resolver(context)
             for resolver in cls._discover_resolvers()
             )
 
+    @staticmethod
+    def at(path):
+        """Return a Distribution for the indicated metadata path
+
+        :param path: a string or path-like object
+        :return: a concrete Distribution instance for the path
+        """
+        return PathDistribution(pathlib.Path(path))
+
     @staticmethod
     def _discover_resolvers():
         """Search the meta_path for resolvers."""
@@ -215,7 +233,7 @@ def entry_points(self):
     def files(self):
         """Files in this distribution.
 
-        :return: Iterable of PackagePath for this distribution or None
+        :return: List of PackagePath for this distribution or None
 
         Result is `None` if the metadata file that enumerates files
         (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is
@@ -231,7 +249,7 @@ def make_file(name, hash=None, size_str=None):
             result.dist = self
             return result
 
-        return file_lines and starmap(make_file, csv.reader(file_lines))
+        return file_lines and list(starmap(make_file, csv.reader(file_lines)))
 
     def _read_files_distinfo(self):
         """
@@ -251,7 +269,8 @@ def _read_files_egginfo(self):
     @property
     def requires(self):
         """Generated requirements specified for this Distribution"""
-        return self._read_dist_info_reqs() or self._read_egg_info_reqs()
+        reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs()
+        return reqs and list(reqs)
 
     def _read_dist_info_reqs(self):
         return self.metadata.get_all('Requires-Dist')
@@ -312,15 +331,35 @@ class DistributionFinder(MetaPathFinder):
     A MetaPathFinder capable of discovering installed distributions.
     """
 
+    class Context:
+
+        name = None
+        """
+        Specific name for which a distribution finder should match.
+        """
+
+        def __init__(self, **kwargs):
+            vars(self).update(kwargs)
+
+        @property
+        def path(self):
+            """
+            The path that a distribution finder should search.
+            """
+            return vars(self).get('path', sys.path)
+
+        @property
+        def pattern(self):
+            return '.*' if self.name is None else re.escape(self.name)
+
     @abc.abstractmethod
-    def find_distributions(self, name=None, path=None):
+    def find_distributions(self, context=Context()):
         """
         Find distributions.
 
         Return an iterable of all Distribution instances capable of
-        loading the metadata for packages matching the ``name``
-        (or all names if not supplied) along the paths in the list
-        of directories ``path`` (defaults to sys.path).
+        loading the metadata for packages matching the ``context``,
+        a DistributionFinder.Context instance.
         """
 
 
@@ -352,12 +391,12 @@ def distribution(package):
     return Distribution.from_name(package)
 
 
-def distributions():
+def distributions(**kwargs):
     """Get all ``Distribution`` instances in the current environment.
 
     :return: An iterable of ``Distribution`` instances.
     """
-    return Distribution.discover()
+    return Distribution.discover(**kwargs)
 
 
 def metadata(package):
diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py
index 3d7da819b343..4d5b1273d9d1 100644
--- a/Lib/test/test_importlib/test_main.py
+++ b/Lib/test/test_importlib/test_main.py
@@ -162,6 +162,10 @@ def test_package_discovery(self):
             for dist in dists
             )
 
+    def test_invalid_usage(self):
+        with self.assertRaises(ValueError):
+            list(distributions(context='something', name='else'))
+
 
 class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
     def test_egg_info(self):
diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py
index af3bab3558ef..1d7b29ae05fd 100644
--- a/Lib/test/test_importlib/test_metadata_api.py
+++ b/Lib/test/test_importlib/test_metadata_api.py
@@ -1,7 +1,6 @@
 import re
 import textwrap
 import unittest
-import itertools
 
 from collections.abc import Iterator
 
@@ -61,9 +60,7 @@ def test_metadata_for_this_package(self):
         assert 'Topic :: Software Development :: Libraries' in classifiers
 
     @staticmethod
-    def _test_files(files_iter):
-        assert isinstance(files_iter, Iterator), files_iter
-        files = list(files_iter)
+    def _test_files(files):
         root = files[0].root
         for file in files:
             assert file.root == root
@@ -99,16 +96,18 @@ def test_requires_egg_info_file(self):
         requirements = requires('egginfo-file')
         self.assertIsNone(requirements)
 
-    def test_requires(self):
+    def test_requires_egg_info(self):
         deps = requires('egginfo-pkg')
+        assert len(deps) == 2
         assert any(
             dep == 'wheel >= 1.0; python_version >= "2.7"'
             for dep in deps
             )
 
     def test_requires_dist_info(self):
-        deps = list(requires('distinfo-pkg'))
-        assert deps and all(deps)
+        deps = requires('distinfo-pkg')
+        assert len(deps) == 2
+        assert all(deps)
         assert 'wheel >= 1.0' in deps
         assert "pytest; extra == 'test'" in deps
 
@@ -143,11 +142,20 @@ def test_more_complex_deps_requires_text(self):
 
 class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase):
     def test_find_distributions_specified_path(self):
-        dists = itertools.chain.from_iterable(
-            resolver(path=[str(self.site_dir)])
-            for resolver in Distribution._discover_resolvers()
-            )
+        dists = Distribution.discover(path=[str(self.site_dir)])
         assert any(
             dist.metadata['Name'] == 'distinfo-pkg'
             for dist in dists
             )
+
+    def test_distribution_at_pathlib(self):
+        """Demonstrate how to load metadata direct from a directory.
+        """
+        dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info'
+        dist = Distribution.at(dist_info_path)
+        assert dist.version == '1.0.0'
+
+    def test_distribution_at_str(self):
+        dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info'
+        dist = Distribution.at(str(dist_info_path))
+        assert dist.version == '1.0.0'
diff --git a/Misc/NEWS.d/next/Library/2019-09-10-11-42-59.bpo-38086.w5TlG-.rst b/Misc/NEWS.d/next/Library/2019-09-10-11-42-59.bpo-38086.w5TlG-.rst
new file mode 100644
index 000000000000..40db53c97da4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-10-11-42-59.bpo-38086.w5TlG-.rst
@@ -0,0 +1 @@
+Update importlib.metadata with changes from `importlib_metadata 0.21 <https://gitlab.com/python-devs/importlib_metadata/blob/0.21/importlib_metadata/docs/changelog.rst>`_.
\ No newline at end of file
diff --git a/Python/importlib_external.h b/Python/importlib_external.h
index 9bcf05057d1f..1e3415e2efbd 100644
--- a/Python/importlib_external.h
+++ b/Python/importlib_external.h
@@ -2237,618 +2237,617 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,114,206,0,0,0,78,5,0,0,115,8,0,0,
     0,0,8,12,1,8,1,4,1,122,22,80,97,116,104,70,
     105,110,100,101,114,46,102,105,110,100,95,109,111,100,117,108,
-    101,99,3,0,0,0,0,0,0,0,0,0,0,0,7,0,
-    0,0,4,0,0,0,67,0,0,0,115,78,0,0,0,100,
-    1,100,2,108,0,125,3,100,1,100,3,108,1,109,2,125,
-    4,1,0,124,2,100,2,107,8,114,34,116,3,106,4,125,
-    2,124,1,100,2,107,8,114,46,100,4,110,8,124,3,160,
-    5,124,1,161,1,125,5,124,0,160,6,124,5,124,2,161,
-    2,125,6,116,7,124,4,124,6,131,2,83,0,41,5,97,
-    37,1,0,0,10,32,32,32,32,32,32,32,32,70,105,110,
-    100,32,100,105,115,116,114,105,98,117,116,105,111,110,115,46,
-    10,10,32,32,32,32,32,32,32,32,82,101,116,117,114,110,
-    32,97,110,32,105,116,101,114,97,98,108,101,32,111,102,32,
-    97,108,108,32,68,105,115,116,114,105,98,117,116,105,111,110,
-    32,105,110,115,116,97,110,99,101,115,32,99,97,112,97,98,
-    108,101,32,111,102,10,32,32,32,32,32,32,32,32,108,111,
-    97,100,105,110,103,32,116,104,101,32,109,101,116,97,100,97,
-    116,97,32,102,111,114,32,112,97,99,107,97,103,101,115,32,
-    109,97,116,99,104,105,110,103,32,116,104,101,32,96,96,110,
+    101,99,2,0,0,0,0,0,0,0,0,0,0,0,5,0,
+    0,0,4,0,0,0,67,0,0,0,115,58,0,0,0,100,
+    1,100,2,108,0,109,1,125,2,109,2,125,3,1,0,124,
+    1,100,3,107,8,114,32,124,3,160,3,161,0,125,1,124,
+    0,160,4,124,1,106,5,124,1,106,6,161,2,125,4,116,
+    7,124,2,124,4,131,2,83,0,41,4,97,32,1,0,0,
+    10,32,32,32,32,32,32,32,32,70,105,110,100,32,100,105,
+    115,116,114,105,98,117,116,105,111,110,115,46,10,10,32,32,
+    32,32,32,32,32,32,82,101,116,117,114,110,32,97,110,32,
+    105,116,101,114,97,98,108,101,32,111,102,32,97,108,108,32,
+    68,105,115,116,114,105,98,117,116,105,111,110,32,105,110,115,
+    116,97,110,99,101,115,32,99,97,112,97,98,108,101,32,111,
+    102,10,32,32,32,32,32,32,32,32,108,111,97,100,105,110,
+    103,32,116,104,101,32,109,101,116,97,100,97,116,97,32,102,
+    111,114,32,112,97,99,107,97,103,101,115,32,109,97,116,99,
+    104,105,110,103,32,96,96,99,111,110,116,101,120,116,46,110,
     97,109,101,96,96,10,32,32,32,32,32,32,32,32,40,111,
-    114,32,97,108,108,32,110,97,109,101,115,32,105,102,32,110,
-    111,116,32,115,117,112,112,108,105,101,100,41,32,97,108,111,
-    110,103,32,116,104,101,32,112,97,116,104,115,32,105,110,32,
-    116,104,101,32,108,105,115,116,10,32,32,32,32,32,32,32,
-    32,111,102,32,100,105,114,101,99,116,111,114,105,101,115,32,
-    96,96,112,97,116,104,96,96,32,40,100,101,102,97,117,108,
-    116,115,32,116,111,32,115,121,115,46,112,97,116,104,41,46,
-    10,32,32,32,32,32,32,32,32,114,73,0,0,0,78,41,
-    1,218,16,80,97,116,104,68,105,115,116,114,105,98,117,116,
-    105,111,110,122,2,46,42,41,8,218,2,114,101,90,18,105,
-    109,112,111,114,116,108,105,98,46,109,101,116,97,100,97,116,
-    97,114,59,1,0,0,114,8,0,0,0,114,44,0,0,0,
-    90,6,101,115,99,97,112,101,218,13,95,115,101,97,114,99,
-    104,95,112,97,116,104,115,218,3,109,97,112,41,7,114,193,
-    0,0,0,114,117,0,0,0,114,44,0,0,0,114,60,1,
-    0,0,114,59,1,0,0,218,7,112,97,116,116,101,114,110,
-    90,5,102,111,117,110,100,114,3,0,0,0,114,3,0,0,
-    0,114,6,0,0,0,218,18,102,105,110,100,95,100,105,115,
-    116,114,105,98,117,116,105,111,110,115,91,5,0,0,115,14,
-    0,0,0,0,10,8,1,12,1,8,1,6,1,22,1,12,
-    1,122,29,80,97,116,104,70,105,110,100,101,114,46,102,105,
-    110,100,95,100,105,115,116,114,105,98,117,116,105,111,110,115,
-    99,3,0,0,0,0,0,0,0,0,0,0,0,4,0,0,
-    0,6,0,0,0,3,0,0,0,115,44,0,0,0,100,1,
-    100,2,108,0,125,3,124,3,106,1,160,2,135,0,135,1,
-    102,2,100,3,100,4,132,8,116,3,136,0,106,4,124,2,
-    131,2,68,0,131,1,161,1,83,0,41,5,122,49,70,105,
-    110,100,32,109,101,116,97,100,97,116,97,32,100,105,114,101,
-    99,116,111,114,105,101,115,32,105,110,32,112,97,116,104,115,
-    32,104,101,117,114,105,115,116,105,99,97,108,108,121,46,114,
-    73,0,0,0,78,99,1,0,0,0,0,0,0,0,0,0,
-    0,0,2,0,0,0,5,0,0,0,51,0,0,0,115,26,
-    0,0,0,124,0,93,18,125,1,136,0,160,0,124,1,136,
-    1,161,2,86,0,1,0,113,2,100,0,83,0,114,110,0,
-    0,0,41,1,218,12,95,115,101,97,114,99,104,95,112,97,
-    116,104,41,2,114,32,0,0,0,114,44,0,0,0,169,2,
-    114,193,0,0,0,114,63,1,0,0,114,3,0,0,0,114,
-    6,0,0,0,114,19,1,0,0,113,5,0,0,115,4,0,
-    0,0,4,2,2,255,122,43,80,97,116,104,70,105,110,100,
-    101,114,46,95,115,101,97,114,99,104,95,112,97,116,104,115,
-    46,60,108,111,99,97,108,115,62,46,60,103,101,110,101,120,
-    112,114,62,41,5,218,9,105,116,101,114,116,111,111,108,115,
-    90,5,99,104,97,105,110,90,13,102,114,111,109,95,105,116,
-    101,114,97,98,108,101,114,62,1,0,0,218,12,95,115,119,
-    105,116,99,104,95,112,97,116,104,41,4,114,193,0,0,0,
-    114,63,1,0,0,90,5,112,97,116,104,115,114,67,1,0,
-    0,114,3,0,0,0,114,66,1,0,0,114,6,0,0,0,
-    114,61,1,0,0,109,5,0,0,115,8,0,0,0,0,3,
-    8,1,18,2,10,254,122,24,80,97,116,104,70,105,110,100,
-    101,114,46,95,115,101,97,114,99,104,95,112,97,116,104,115,
-    99,1,0,0,0,0,0,0,0,0,0,0,0,5,0,0,
-    0,10,0,0,0,67,0,0,0,115,96,0,0,0,100,1,
-    100,2,108,0,109,1,125,1,1,0,100,1,100,0,108,2,
-    125,2,100,1,100,0,108,3,125,3,100,3,125,4,124,4,
-    114,48,116,4,106,5,160,6,124,0,161,1,114,86,124,1,
-    116,7,131,1,143,24,1,0,124,2,160,8,124,0,161,1,
-    87,0,2,0,53,0,81,0,82,0,163,0,83,0,81,0,
-    82,0,88,0,124,3,160,8,124,0,161,1,83,0,41,4,
-    78,114,73,0,0,0,41,1,218,8,115,117,112,112,114,101,
-    115,115,70,41,9,90,10,99,111,110,116,101,120,116,108,105,
-    98,114,69,1,0,0,218,7,122,105,112,102,105,108,101,218,
-    7,112,97,116,104,108,105,98,90,2,111,115,114,44,0,0,
-    0,90,6,105,115,102,105,108,101,218,9,69,120,99,101,112,
-    116,105,111,110,90,4,80,97,116,104,41,5,114,44,0,0,
-    0,114,69,1,0,0,114,70,1,0,0,114,71,1,0,0,
-    90,13,80,89,80,89,95,79,80,69,78,95,66,85,71,114,
-    3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,68,
-    1,0,0,118,5,0,0,115,16,0,0,0,0,2,12,1,
-    8,1,8,1,4,1,16,1,10,1,28,1,122,23,80,97,
-    116,104,70,105,110,100,101,114,46,95,115,119,105,116,99,104,
-    95,112,97,116,104,99,3,0,0,0,0,0,0,0,0,0,
-    0,0,6,0,0,0,5,0,0,0,67,0,0,0,115,44,
-    0,0,0,100,1,100,0,108,0,125,3,100,2,125,4,124,
-    4,106,1,124,1,100,3,141,1,125,5,124,3,106,2,124,
-    5,124,2,106,3,124,3,106,4,100,4,141,3,83,0,41,
-    5,78,114,73,0,0,0,122,32,123,112,97,116,116,101,114,
-    110,125,40,45,46,42,41,63,92,46,40,100,105,115,116,124,
-    101,103,103,41,45,105,110,102,111,169,1,114,63,1,0,0,
-    169,1,114,83,0,0,0,41,5,114,60,1,0,0,114,62,
-    0,0,0,90,5,109,97,116,99,104,114,117,0,0,0,218,
-    10,73,71,78,79,82,69,67,65,83,69,169,6,114,193,0,
-    0,0,218,10,110,111,114,109,97,108,105,122,101,100,114,41,
-    1,0,0,114,60,1,0,0,90,8,116,101,109,112,108,97,
-    116,101,90,8,109,97,110,105,102,101,115,116,114,3,0,0,
-    0,114,3,0,0,0,114,6,0,0,0,218,13,95,109,97,
-    116,99,104,101,115,95,105,110,102,111,129,5,0,0,115,8,
-    0,0,0,0,2,8,1,4,1,12,1,122,24,80,97,116,
-    104,70,105,110,100,101,114,46,95,109,97,116,99,104,101,115,
-    95,105,110,102,111,99,3,0,0,0,0,0,0,0,0,0,
-    0,0,6,0,0,0,5,0,0,0,67,0,0,0,115,46,
-    0,0,0,100,1,100,0,108,0,125,3,100,2,125,4,124,
-    4,106,1,124,1,100,3,141,1,125,5,124,3,106,2,124,
-    5,116,3,124,2,131,1,124,3,106,4,100,4,141,3,83,
-    0,41,5,78,114,73,0,0,0,122,30,123,112,97,116,116,
-    101,114,110,125,45,46,42,92,46,101,103,103,91,92,92,47,
-    93,69,71,71,45,73,78,70,79,114,73,1,0,0,114,74,
-    1,0,0,41,5,114,60,1,0,0,114,62,0,0,0,90,
-    6,115,101,97,114,99,104,114,85,0,0,0,114,75,1,0,
-    0,114,76,1,0,0,114,3,0,0,0,114,3,0,0,0,
-    114,6,0,0,0,218,15,95,109,97,116,99,104,101,115,95,
-    108,101,103,97,99,121,136,5,0,0,115,8,0,0,0,0,
-    2,8,1,4,1,12,1,122,26,80,97,116,104,70,105,110,
-    100,101,114,46,95,109,97,116,99,104,101,115,95,108,101,103,
-    97,99,121,99,3,0,0,0,0,0,0,0,0,0,0,0,
-    3,0,0,0,4,0,0,0,3,0,0,0,115,48,0,0,
-    0,124,1,160,0,161,0,115,12,100,1,83,0,124,2,160,
-    1,100,2,100,3,161,2,137,1,135,0,135,1,102,2,100,
-    4,100,5,132,8,124,1,160,2,161,0,68,0,131,1,83,
-    0,41,6,78,114,3,0,0,0,250,1,45,114,45,0,0,
-    0,99,1,0,0,0,0,0,0,0,0,0,0,0,2,0,
-    0,0,5,0,0,0,51,0,0,0,115,42,0,0,0,124,
-    0,93,34,125,1,136,0,160,0,136,1,124,1,161,2,115,
-    30,136,0,160,1,136,1,124,1,161,2,114,2,124,1,86,
-    0,1,0,113,2,100,0,83,0,114,110,0,0,0,41,2,
-    114,78,1,0,0,114,79,1,0,0,41,2,114,32,0,0,
-    0,114,41,1,0,0,169,2,114,193,0,0,0,114,77,1,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,19,1,0,
-    0,148,5,0,0,115,8,0,0,0,4,0,2,1,12,1,
-    12,254,122,42,80,97,116,104,70,105,110,100,101,114,46,95,
-    115,101,97,114,99,104,95,112,97,116,104,46,60,108,111,99,
-    97,108,115,62,46,60,103,101,110,101,120,112,114,62,41,3,
-    90,6,105,115,95,100,105,114,114,67,0,0,0,90,7,105,
-    116,101,114,100,105,114,41,3,114,193,0,0,0,90,4,114,
-    111,111,116,114,63,1,0,0,114,3,0,0,0,114,81,1,
-    0,0,114,6,0,0,0,114,65,1,0,0,143,5,0,0,
-    115,8,0,0,0,0,2,8,1,4,1,12,1,122,23,80,
-    97,116,104,70,105,110,100,101,114,46,95,115,101,97,114,99,
-    104,95,112,97,116,104,41,1,78,41,2,78,78,41,1,78,
-    41,2,78,78,41,19,114,125,0,0,0,114,124,0,0,0,
-    114,126,0,0,0,114,127,0,0,0,114,207,0,0,0,114,
-    46,1,0,0,114,52,1,0,0,114,54,1,0,0,114,55,
-    1,0,0,114,58,1,0,0,114,203,0,0,0,114,206,0,
-    0,0,114,64,1,0,0,114,61,1,0,0,218,12,115,116,
-    97,116,105,99,109,101,116,104,111,100,114,68,1,0,0,114,
-    78,1,0,0,114,79,1,0,0,114,65,1,0,0,114,3,
-    0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,114,45,1,0,0,214,4,0,0,115,54,0,0,0,
-    8,2,4,2,2,1,10,9,2,1,10,12,2,1,10,21,
-    2,1,10,14,2,1,12,31,2,1,12,23,2,1,12,12,
-    2,1,12,17,2,1,10,8,2,1,10,10,2,1,10,6,
-    2,1,10,6,2,1,114,45,1,0,0,99,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,
-    64,0,0,0,115,90,0,0,0,101,0,90,1,100,0,90,
-    2,100,1,90,3,100,2,100,3,132,0,90,4,100,4,100,
-    5,132,0,90,5,101,6,90,7,100,6,100,7,132,0,90,
-    8,100,8,100,9,132,0,90,9,100,19,100,11,100,12,132,
-    1,90,10,100,13,100,14,132,0,90,11,101,12,100,15,100,
-    16,132,0,131,1,90,13,100,17,100,18,132,0,90,14,100,
-    10,83,0,41,20,218,10,70,105,108,101,70,105,110,100,101,
-    114,122,172,70,105,108,101,45,98,97,115,101,100,32,102,105,
-    110,100,101,114,46,10,10,32,32,32,32,73,110,116,101,114,
-    97,99,116,105,111,110,115,32,119,105,116,104,32,116,104,101,
-    32,102,105,108,101,32,115,121,115,116,101,109,32,97,114,101,
-    32,99,97,99,104,101,100,32,102,111,114,32,112,101,114,102,
-    111,114,109,97,110,99,101,44,32,98,101,105,110,103,10,32,
-    32,32,32,114,101,102,114,101,115,104,101,100,32,119,104,101,
-    110,32,116,104,101,32,100,105,114,101,99,116,111,114,121,32,
-    116,104,101,32,102,105,110,100,101,114,32,105,115,32,104,97,
-    110,100,108,105,110,103,32,104,97,115,32,98,101,101,110,32,
-    109,111,100,105,102,105,101,100,46,10,10,32,32,32,32,99,
-    2,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,
-    6,0,0,0,7,0,0,0,115,84,0,0,0,103,0,125,
-    3,124,2,68,0,93,32,92,2,137,0,125,4,124,3,160,
-    0,135,0,102,1,100,1,100,2,132,8,124,4,68,0,131,
-    1,161,1,1,0,113,8,124,3,124,0,95,1,124,1,112,
-    54,100,3,124,0,95,2,100,4,124,0,95,3,116,4,131,
-    0,124,0,95,5,116,4,131,0,124,0,95,6,100,5,83,
-    0,41,6,122,154,73,110,105,116,105,97,108,105,122,101,32,
-    119,105,116,104,32,116,104,101,32,112,97,116,104,32,116,111,
-    32,115,101,97,114,99,104,32,111,110,32,97,110,100,32,97,
-    32,118,97,114,105,97,98,108,101,32,110,117,109,98,101,114,
-    32,111,102,10,32,32,32,32,32,32,32,32,50,45,116,117,
-    112,108,101,115,32,99,111,110,116,97,105,110,105,110,103,32,
-    116,104,101,32,108,111,97,100,101,114,32,97,110,100,32,116,
-    104,101,32,102,105,108,101,32,115,117,102,102,105,120,101,115,
-    32,116,104,101,32,108,111,97,100,101,114,10,32,32,32,32,
-    32,32,32,32,114,101,99,111,103,110,105,122,101,115,46,99,
+    114,32,97,108,108,32,110,97,109,101,115,32,105,102,32,96,
+    96,78,111,110,101,96,96,32,105,110,100,105,99,97,116,101,
+    100,41,32,97,108,111,110,103,32,116,104,101,32,112,97,116,
+    104,115,32,105,110,32,116,104,101,32,108,105,115,116,10,32,
+    32,32,32,32,32,32,32,111,102,32,100,105,114,101,99,116,
+    111,114,105,101,115,32,96,96,99,111,110,116,101,120,116,46,
+    112,97,116,104,96,96,46,10,32,32,32,32,32,32,32,32,
+    114,73,0,0,0,41,2,218,16,80,97,116,104,68,105,115,
+    116,114,105,98,117,116,105,111,110,218,18,68,105,115,116,114,
+    105,98,117,116,105,111,110,70,105,110,100,101,114,78,41,8,
+    90,18,105,109,112,111,114,116,108,105,98,46,109,101,116,97,
+    100,97,116,97,114,59,1,0,0,114,60,1,0,0,90,7,
+    67,111,110,116,101,120,116,218,13,95,115,101,97,114,99,104,
+    95,112,97,116,104,115,218,7,112,97,116,116,101,114,110,114,
+    44,0,0,0,218,3,109,97,112,41,5,114,119,0,0,0,
+    90,7,99,111,110,116,101,120,116,114,59,1,0,0,114,60,
+    1,0,0,90,5,102,111,117,110,100,114,3,0,0,0,114,
+    3,0,0,0,114,6,0,0,0,218,18,102,105,110,100,95,
+    100,105,115,116,114,105,98,117,116,105,111,110,115,91,5,0,
+    0,115,10,0,0,0,0,10,16,1,8,1,8,1,16,1,
+    122,29,80,97,116,104,70,105,110,100,101,114,46,102,105,110,
+    100,95,100,105,115,116,114,105,98,117,116,105,111,110,115,99,
+    3,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,
+    6,0,0,0,3,0,0,0,115,44,0,0,0,100,1,100,
+    2,108,0,125,3,124,3,106,1,160,2,135,0,135,1,102,
+    2,100,3,100,4,132,8,116,3,136,0,106,4,124,2,131,
+    2,68,0,131,1,161,1,83,0,41,5,122,49,70,105,110,
+    100,32,109,101,116,97,100,97,116,97,32,100,105,114,101,99,
+    116,111,114,105,101,115,32,105,110,32,112,97,116,104,115,32,
+    104,101,117,114,105,115,116,105,99,97,108,108,121,46,114,73,
+    0,0,0,78,99,1,0,0,0,0,0,0,0,0,0,0,
+    0,2,0,0,0,5,0,0,0,51,0,0,0,115,26,0,
+    0,0,124,0,93,18,125,1,136,0,160,0,124,1,136,1,
+    161,2,86,0,1,0,113,2,100,0,83,0,114,110,0,0,
+    0,41,1,218,12,95,115,101,97,114,99,104,95,112,97,116,
+    104,41,2,114,32,0,0,0,114,44,0,0,0,169,2,114,
+    193,0,0,0,114,62,1,0,0,114,3,0,0,0,114,6,
+    0,0,0,114,19,1,0,0,111,5,0,0,115,4,0,0,
+    0,4,2,2,255,122,43,80,97,116,104,70,105,110,100,101,
+    114,46,95,115,101,97,114,99,104,95,112,97,116,104,115,46,
+    60,108,111,99,97,108,115,62,46,60,103,101,110,101,120,112,
+    114,62,41,5,218,9,105,116,101,114,116,111,111,108,115,90,
+    5,99,104,97,105,110,90,13,102,114,111,109,95,105,116,101,
+    114,97,98,108,101,114,63,1,0,0,218,12,95,115,119,105,
+    116,99,104,95,112,97,116,104,41,4,114,193,0,0,0,114,
+    62,1,0,0,90,5,112,97,116,104,115,114,67,1,0,0,
+    114,3,0,0,0,114,66,1,0,0,114,6,0,0,0,114,
+    61,1,0,0,107,5,0,0,115,8,0,0,0,0,3,8,
+    1,18,2,10,254,122,24,80,97,116,104,70,105,110,100,101,
+    114,46,95,115,101,97,114,99,104,95,112,97,116,104,115,99,
+    1,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,
+    10,0,0,0,67,0,0,0,115,96,0,0,0,100,1,100,
+    2,108,0,109,1,125,1,1,0,100,1,100,0,108,2,125,
+    2,100,1,100,0,108,3,125,3,100,3,125,4,124,4,114,
+    48,116,4,106,5,160,6,124,0,161,1,114,86,124,1,116,
+    7,131,1,143,24,1,0,124,2,160,8,124,0,161,1,87,
+    0,2,0,53,0,81,0,82,0,163,0,83,0,81,0,82,
+    0,88,0,124,3,160,8,124,0,161,1,83,0,41,4,78,
+    114,73,0,0,0,41,1,218,8,115,117,112,112,114,101,115,
+    115,70,41,9,90,10,99,111,110,116,101,120,116,108,105,98,
+    114,69,1,0,0,218,7,122,105,112,102,105,108,101,218,7,
+    112,97,116,104,108,105,98,90,2,111,115,114,44,0,0,0,
+    90,6,105,115,102,105,108,101,218,9,69,120,99,101,112,116,
+    105,111,110,90,4,80,97,116,104,41,5,114,44,0,0,0,
+    114,69,1,0,0,114,70,1,0,0,114,71,1,0,0,90,
+    13,80,89,80,89,95,79,80,69,78,95,66,85,71,114,3,
+    0,0,0,114,3,0,0,0,114,6,0,0,0,114,68,1,
+    0,0,116,5,0,0,115,16,0,0,0,0,2,12,1,8,
+    1,8,1,4,1,16,1,10,1,28,1,122,23,80,97,116,
+    104,70,105,110,100,101,114,46,95,115,119,105,116,99,104,95,
+    112,97,116,104,99,3,0,0,0,0,0,0,0,0,0,0,
+    0,6,0,0,0,5,0,0,0,67,0,0,0,115,44,0,
+    0,0,100,1,100,0,108,0,125,3,100,2,125,4,124,4,
+    106,1,124,1,100,3,141,1,125,5,124,3,106,2,124,5,
+    124,2,106,3,124,3,106,4,100,4,141,3,83,0,41,5,
+    78,114,73,0,0,0,122,32,123,112,97,116,116,101,114,110,
+    125,40,45,46,42,41,63,92,46,40,100,105,115,116,124,101,
+    103,103,41,45,105,110,102,111,169,1,114,62,1,0,0,169,
+    1,114,83,0,0,0,41,5,218,2,114,101,114,62,0,0,
+    0,90,5,109,97,116,99,104,114,117,0,0,0,218,10,73,
+    71,78,79,82,69,67,65,83,69,169,6,114,193,0,0,0,
+    218,10,110,111,114,109,97,108,105,122,101,100,114,41,1,0,
+    0,114,75,1,0,0,90,8,116,101,109,112,108,97,116,101,
+    90,8,109,97,110,105,102,101,115,116,114,3,0,0,0,114,
+    3,0,0,0,114,6,0,0,0,218,13,95,109,97,116,99,
+    104,101,115,95,105,110,102,111,127,5,0,0,115,8,0,0,
+    0,0,2,8,1,4,1,12,1,122,24,80,97,116,104,70,
+    105,110,100,101,114,46,95,109,97,116,99,104,101,115,95,105,
+    110,102,111,99,3,0,0,0,0,0,0,0,0,0,0,0,
+    6,0,0,0,5,0,0,0,67,0,0,0,115,46,0,0,
+    0,100,1,100,0,108,0,125,3,100,2,125,4,124,4,106,
+    1,124,1,100,3,141,1,125,5,124,3,106,2,124,5,116,
+    3,124,2,131,1,124,3,106,4,100,4,141,3,83,0,41,
+    5,78,114,73,0,0,0,122,30,123,112,97,116,116,101,114,
+    110,125,45,46,42,92,46,101,103,103,91,92,92,47,93,69,
+    71,71,45,73,78,70,79,114,73,1,0,0,114,74,1,0,
+    0,41,5,114,75,1,0,0,114,62,0,0,0,90,6,115,
+    101,97,114,99,104,114,85,0,0,0,114,76,1,0,0,114,
+    77,1,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
+    0,0,0,218,15,95,109,97,116,99,104,101,115,95,108,101,
+    103,97,99,121,134,5,0,0,115,8,0,0,0,0,2,8,
+    1,4,1,12,1,122,26,80,97,116,104,70,105,110,100,101,
+    114,46,95,109,97,116,99,104,101,115,95,108,101,103,97,99,
+    121,99,3,0,0,0,0,0,0,0,0,0,0,0,3,0,
+    0,0,4,0,0,0,3,0,0,0,115,48,0,0,0,124,
+    1,160,0,161,0,115,12,100,1,83,0,124,2,160,1,100,
+    2,100,3,161,2,137,1,135,0,135,1,102,2,100,4,100,
+    5,132,8,124,1,160,2,161,0,68,0,131,1,83,0,41,
+    6,78,114,3,0,0,0,250,1,45,114,45,0,0,0,99,
     1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,
-    3,0,0,0,51,0,0,0,115,22,0,0,0,124,0,93,
-    14,125,1,124,1,136,0,102,2,86,0,1,0,113,2,100,
-    0,83,0,114,110,0,0,0,114,3,0,0,0,114,16,1,
-    0,0,169,1,114,140,0,0,0,114,3,0,0,0,114,6,
-    0,0,0,114,19,1,0,0,168,5,0,0,115,4,0,0,
-    0,4,0,2,0,122,38,70,105,108,101,70,105,110,100,101,
-    114,46,95,95,105,110,105,116,95,95,46,60,108,111,99,97,
-    108,115,62,46,60,103,101,110,101,120,112,114,62,114,71,0,
-    0,0,114,105,0,0,0,78,41,7,114,167,0,0,0,218,
-    8,95,108,111,97,100,101,114,115,114,44,0,0,0,218,11,
-    95,112,97,116,104,95,109,116,105,109,101,218,3,115,101,116,
-    218,11,95,112,97,116,104,95,99,97,99,104,101,218,19,95,
-    114,101,108,97,120,101,100,95,112,97,116,104,95,99,97,99,
-    104,101,41,5,114,119,0,0,0,114,44,0,0,0,218,14,
-    108,111,97,100,101,114,95,100,101,116,97,105,108,115,90,7,
-    108,111,97,100,101,114,115,114,189,0,0,0,114,3,0,0,
-    0,114,84,1,0,0,114,6,0,0,0,114,209,0,0,0,
-    162,5,0,0,115,16,0,0,0,0,4,4,1,12,1,26,
-    1,6,2,10,1,6,1,8,1,122,19,70,105,108,101,70,
-    105,110,100,101,114,46,95,95,105,110,105,116,95,95,99,1,
-    0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,2,
-    0,0,0,67,0,0,0,115,10,0,0,0,100,1,124,0,
-    95,0,100,2,83,0,41,3,122,31,73,110,118,97,108,105,
-    100,97,116,101,32,116,104,101,32,100,105,114,101,99,116,111,
-    114,121,32,109,116,105,109,101,46,114,105,0,0,0,78,41,
-    1,114,86,1,0,0,114,246,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,46,1,0,0,176,
-    5,0,0,115,2,0,0,0,0,2,122,28,70,105,108,101,
-    70,105,110,100,101,114,46,105,110,118,97,108,105,100,97,116,
-    101,95,99,97,99,104,101,115,99,2,0,0,0,0,0,0,
-    0,0,0,0,0,3,0,0,0,3,0,0,0,67,0,0,
-    0,115,42,0,0,0,124,0,160,0,124,1,161,1,125,2,
-    124,2,100,1,107,8,114,26,100,1,103,0,102,2,83,0,
-    124,2,106,1,124,2,106,2,112,38,103,0,102,2,83,0,
-    41,2,122,197,84,114,121,32,116,111,32,102,105,110,100,32,
-    97,32,108,111,97,100,101,114,32,102,111,114,32,116,104,101,
-    32,115,112,101,99,105,102,105,101,100,32,109,111,100,117,108,
-    101,44,32,111,114,32,116,104,101,32,110,97,109,101,115,112,
-    97,99,101,10,32,32,32,32,32,32,32,32,112,97,99,107,
-    97,103,101,32,112,111,114,116,105,111,110,115,46,32,82,101,
-    116,117,114,110,115,32,40,108,111,97,100,101,114,44,32,108,
-    105,115,116,45,111,102,45,112,111,114,116,105,111,110,115,41,
-    46,10,10,32,32,32,32,32,32,32,32,84,104,105,115,32,
-    109,101,116,104,111,100,32,105,115,32,100,101,112,114,101,99,
-    97,116,101,100,46,32,32,85,115,101,32,102,105,110,100,95,
-    115,112,101,99,40,41,32,105,110,115,116,101,97,100,46,10,
-    10,32,32,32,32,32,32,32,32,78,41,3,114,203,0,0,
-    0,114,140,0,0,0,114,178,0,0,0,41,3,114,119,0,
-    0,0,114,139,0,0,0,114,187,0,0,0,114,3,0,0,
-    0,114,3,0,0,0,114,6,0,0,0,114,137,0,0,0,
-    182,5,0,0,115,8,0,0,0,0,7,10,1,8,1,8,
-    1,122,22,70,105,108,101,70,105,110,100,101,114,46,102,105,
-    110,100,95,108,111,97,100,101,114,99,6,0,0,0,0,0,
-    0,0,0,0,0,0,7,0,0,0,6,0,0,0,67,0,
-    0,0,115,26,0,0,0,124,1,124,2,124,3,131,2,125,
-    6,116,0,124,2,124,3,124,6,124,4,100,1,141,4,83,
-    0,41,2,78,114,177,0,0,0,41,1,114,190,0,0,0,
-    41,7,114,119,0,0,0,114,188,0,0,0,114,139,0,0,
-    0,114,44,0,0,0,90,4,115,109,115,108,114,202,0,0,
-    0,114,140,0,0,0,114,3,0,0,0,114,3,0,0,0,
-    114,6,0,0,0,114,58,1,0,0,194,5,0,0,115,8,
-    0,0,0,0,1,10,1,8,1,2,255,122,20,70,105,108,
-    101,70,105,110,100,101,114,46,95,103,101,116,95,115,112,101,
-    99,78,99,3,0,0,0,0,0,0,0,0,0,0,0,14,
-    0,0,0,8,0,0,0,67,0,0,0,115,98,1,0,0,
-    100,1,125,3,124,1,160,0,100,2,161,1,100,3,25,0,
-    125,4,122,24,116,1,124,0,106,2,112,34,116,3,160,4,
-    161,0,131,1,106,5,125,5,87,0,110,24,4,0,116,6,
-    107,10,114,66,1,0,1,0,1,0,100,4,125,5,89,0,
-    110,2,88,0,124,5,124,0,106,7,107,3,114,92,124,0,
-    160,8,161,0,1,0,124,5,124,0,95,7,116,9,131,0,
-    114,114,124,0,106,10,125,6,124,4,160,11,161,0,125,7,
-    110,10,124,0,106,12,125,6,124,4,125,7,124,7,124,6,
-    107,6,114,218,116,13,124,0,106,2,124,4,131,2,125,8,
-    124,0,106,14,68,0,93,58,92,2,125,9,125,10,100,5,
-    124,9,23,0,125,11,116,13,124,8,124,11,131,2,125,12,
-    116,15,124,12,131,1,114,150,124,0,160,16,124,10,124,1,
-    124,12,124,8,103,1,124,2,161,5,2,0,1,0,83,0,
-    113,150,116,17,124,8,131,1,125,3,124,0,106,14,68,0,
-    93,82,92,2,125,9,125,10,116,13,124,0,106,2,124,4,
-    124,9,23,0,131,2,125,12,116,18,106,19,100,6,124,12,
-    100,3,100,7,141,3,1,0,124,7,124,9,23,0,124,6,
-    107,6,114,224,116,15,124,12,131,1,114,224,124,0,160,16,
-    124,10,124,1,124,12,100,8,124,2,161,5,2,0,1,0,
-    83,0,113,224,124,3,144,1,114,94,116,18,160,19,100,9,
-    124,8,161,2,1,0,116,18,160,20,124,1,100,8,161,2,
-    125,13,124,8,103,1,124,13,95,21,124,13,83,0,100,8,
-    83,0,41,10,122,111,84,114,121,32,116,111,32,102,105,110,
-    100,32,97,32,115,112,101,99,32,102,111,114,32,116,104,101,
-    32,115,112,101,99,105,102,105,101,100,32,109,111,100,117,108,
-    101,46,10,10,32,32,32,32,32,32,32,32,82,101,116,117,
-    114,110,115,32,116,104,101,32,109,97,116,99,104,105,110,103,
-    32,115,112,101,99,44,32,111,114,32,78,111,110,101,32,105,
-    102,32,110,111,116,32,102,111,117,110,100,46,10,32,32,32,
-    32,32,32,32,32,70,114,71,0,0,0,114,28,0,0,0,
-    114,105,0,0,0,114,209,0,0,0,122,9,116,114,121,105,
-    110,103,32,123,125,41,1,90,9,118,101,114,98,111,115,105,
-    116,121,78,122,25,112,111,115,115,105,98,108,101,32,110,97,
-    109,101,115,112,97,99,101,32,102,111,114,32,123,125,41,22,
-    114,41,0,0,0,114,49,0,0,0,114,44,0,0,0,114,
-    2,0,0,0,114,55,0,0,0,114,10,1,0,0,114,50,
-    0,0,0,114,86,1,0,0,218,11,95,102,105,108,108,95,
-    99,97,99,104,101,114,7,0,0,0,114,89,1,0,0,114,
-    106,0,0,0,114,88,1,0,0,114,38,0,0,0,114,85,
-    1,0,0,114,54,0,0,0,114,58,1,0,0,114,56,0,
-    0,0,114,134,0,0,0,114,149,0,0,0,114,183,0,0,
-    0,114,178,0,0,0,41,14,114,119,0,0,0,114,139,0,
-    0,0,114,202,0,0,0,90,12,105,115,95,110,97,109,101,
-    115,112,97,99,101,90,11,116,97,105,108,95,109,111,100,117,
-    108,101,114,169,0,0,0,90,5,99,97,99,104,101,90,12,
-    99,97,99,104,101,95,109,111,100,117,108,101,90,9,98,97,
-    115,101,95,112,97,116,104,114,17,1,0,0,114,188,0,0,
-    0,90,13,105,110,105,116,95,102,105,108,101,110,97,109,101,
-    90,9,102,117,108,108,95,112,97,116,104,114,187,0,0,0,
+    5,0,0,0,51,0,0,0,115,42,0,0,0,124,0,93,
+    34,125,1,136,0,160,0,136,1,124,1,161,2,115,30,136,
+    0,160,1,136,1,124,1,161,2,114,2,124,1,86,0,1,
+    0,113,2,100,0,83,0,114,110,0,0,0,41,2,114,79,
+    1,0,0,114,80,1,0,0,41,2,114,32,0,0,0,114,
+    41,1,0,0,169,2,114,193,0,0,0,114,78,1,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,19,1,0,0,146,
+    5,0,0,115,8,0,0,0,4,0,2,1,12,1,12,254,
+    122,42,80,97,116,104,70,105,110,100,101,114,46,95,115,101,
+    97,114,99,104,95,112,97,116,104,46,60,108,111,99,97,108,
+    115,62,46,60,103,101,110,101,120,112,114,62,41,3,90,6,
+    105,115,95,100,105,114,114,67,0,0,0,90,7,105,116,101,
+    114,100,105,114,41,3,114,193,0,0,0,90,4,114,111,111,
+    116,114,62,1,0,0,114,3,0,0,0,114,82,1,0,0,
+    114,6,0,0,0,114,65,1,0,0,141,5,0,0,115,8,
+    0,0,0,0,2,8,1,4,1,12,1,122,23,80,97,116,
+    104,70,105,110,100,101,114,46,95,115,101,97,114,99,104,95,
+    112,97,116,104,41,1,78,41,2,78,78,41,1,78,41,1,
+    78,41,19,114,125,0,0,0,114,124,0,0,0,114,126,0,
+    0,0,114,127,0,0,0,114,207,0,0,0,114,46,1,0,
+    0,114,52,1,0,0,114,54,1,0,0,114,55,1,0,0,
+    114,58,1,0,0,114,203,0,0,0,114,206,0,0,0,114,
+    64,1,0,0,114,61,1,0,0,218,12,115,116,97,116,105,
+    99,109,101,116,104,111,100,114,68,1,0,0,114,79,1,0,
+    0,114,80,1,0,0,114,65,1,0,0,114,3,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    203,0,0,0,199,5,0,0,115,74,0,0,0,0,5,4,
-    1,14,1,2,1,24,1,14,1,10,1,10,1,8,1,6,
-    2,6,1,6,1,10,2,6,1,4,2,8,1,12,1,14,
-    1,8,1,10,1,8,1,26,4,8,2,14,1,16,1,16,
-    1,12,1,8,1,10,1,2,0,2,255,10,2,6,1,12,
-    1,12,1,8,1,4,1,122,20,70,105,108,101,70,105,110,
-    100,101,114,46,102,105,110,100,95,115,112,101,99,99,1,0,
-    0,0,0,0,0,0,0,0,0,0,9,0,0,0,10,0,
-    0,0,67,0,0,0,115,190,0,0,0,124,0,106,0,125,
-    1,122,22,116,1,160,2,124,1,112,22,116,1,160,3,161,
-    0,161,1,125,2,87,0,110,30,4,0,116,4,116,5,116,
-    6,102,3,107,10,114,58,1,0,1,0,1,0,103,0,125,
-    2,89,0,110,2,88,0,116,7,106,8,160,9,100,1,161,
-    1,115,84,116,10,124,2,131,1,124,0,95,11,110,74,116,
-    10,131,0,125,3,124,2,68,0,93,56,125,4,124,4,160,
-    12,100,2,161,1,92,3,125,5,125,6,125,7,124,6,114,
-    136,100,3,160,13,124,5,124,7,160,14,161,0,161,2,125,
-    8,110,4,124,5,125,8,124,3,160,15,124,8,161,1,1,
-    0,113,94,124,3,124,0,95,11,116,7,106,8,160,9,116,
-    16,161,1,114,186,100,4,100,5,132,0,124,2,68,0,131,
-    1,124,0,95,17,100,6,83,0,41,7,122,68,70,105,108,
-    108,32,116,104,101,32,99,97,99,104,101,32,111,102,32,112,
-    111,116,101,110,116,105,97,108,32,109,111,100,117,108,101,115,
-    32,97,110,100,32,112,97,99,107,97,103,101,115,32,102,111,
-    114,32,116,104,105,115,32,100,105,114,101,99,116,111,114,121,
-    46,114,0,0,0,0,114,71,0,0,0,114,61,0,0,0,
-    99,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,
-    0,4,0,0,0,83,0,0,0,115,20,0,0,0,104,0,
-    124,0,93,12,125,1,124,1,160,0,161,0,146,2,113,4,
-    83,0,114,3,0,0,0,41,1,114,106,0,0,0,41,2,
-    114,32,0,0,0,90,2,102,110,114,3,0,0,0,114,3,
-    0,0,0,114,6,0,0,0,218,9,60,115,101,116,99,111,
-    109,112,62,20,6,0,0,115,4,0,0,0,6,0,2,0,
-    122,41,70,105,108,101,70,105,110,100,101,114,46,95,102,105,
-    108,108,95,99,97,99,104,101,46,60,108,111,99,97,108,115,
-    62,46,60,115,101,116,99,111,109,112,62,78,41,18,114,44,
-    0,0,0,114,2,0,0,0,114,7,1,0,0,114,55,0,
-    0,0,114,3,1,0,0,218,15,80,101,114,109,105,115,115,
-    105,111,110,69,114,114,111,114,218,18,78,111,116,65,68,105,
-    114,101,99,116,111,114,121,69,114,114,111,114,114,8,0,0,
-    0,114,9,0,0,0,114,10,0,0,0,114,87,1,0,0,
-    114,88,1,0,0,114,101,0,0,0,114,62,0,0,0,114,
-    106,0,0,0,218,3,97,100,100,114,11,0,0,0,114,89,
-    1,0,0,41,9,114,119,0,0,0,114,44,0,0,0,114,
-    8,1,0,0,90,21,108,111,119,101,114,95,115,117,102,102,
-    105,120,95,99,111,110,116,101,110,116,115,114,41,1,0,0,
-    114,117,0,0,0,114,29,1,0,0,114,17,1,0,0,90,
-    8,110,101,119,95,110,97,109,101,114,3,0,0,0,114,3,
-    0,0,0,114,6,0,0,0,114,91,1,0,0,247,5,0,
-    0,115,34,0,0,0,0,2,6,1,2,1,22,1,20,3,
-    10,3,12,1,12,7,6,1,8,1,16,1,4,1,18,2,
-    4,1,12,1,6,1,12,1,122,22,70,105,108,101,70,105,
-    110,100,101,114,46,95,102,105,108,108,95,99,97,99,104,101,
-    99,1,0,0,0,0,0,0,0,0,0,0,0,3,0,0,
-    0,3,0,0,0,7,0,0,0,115,18,0,0,0,135,0,
-    135,1,102,2,100,1,100,2,132,8,125,2,124,2,83,0,
-    41,3,97,20,1,0,0,65,32,99,108,97,115,115,32,109,
-    101,116,104,111,100,32,119,104,105,99,104,32,114,101,116,117,
-    114,110,115,32,97,32,99,108,111,115,117,114,101,32,116,111,
-    32,117,115,101,32,111,110,32,115,121,115,46,112,97,116,104,
-    95,104,111,111,107,10,32,32,32,32,32,32,32,32,119,104,
-    105,99,104,32,119,105,108,108,32,114,101,116,117,114,110,32,
-    97,110,32,105,110,115,116,97,110,99,101,32,117,115,105,110,
-    103,32,116,104,101,32,115,112,101,99,105,102,105,101,100,32,
-    108,111,97,100,101,114,115,32,97,110,100,32,116,104,101,32,
-    112,97,116,104,10,32,32,32,32,32,32,32,32,99,97,108,
-    108,101,100,32,111,110,32,116,104,101,32,99,108,111,115,117,
-    114,101,46,10,10,32,32,32,32,32,32,32,32,73,102,32,
-    116,104,101,32,112,97,116,104,32,99,97,108,108,101,100,32,
-    111,110,32,116,104,101,32,99,108,111,115,117,114,101,32,105,
-    115,32,110,111,116,32,97,32,100,105,114,101,99,116,111,114,
-    121,44,32,73,109,112,111,114,116,69,114,114,111,114,32,105,
-    115,10,32,32,32,32,32,32,32,32,114,97,105,115,101,100,
-    46,10,10,32,32,32,32,32,32,32,32,99,1,0,0,0,
-    0,0,0,0,0,0,0,0,1,0,0,0,4,0,0,0,
-    19,0,0,0,115,34,0,0,0,116,0,124,0,131,1,115,
-    20,116,1,100,1,124,0,100,2,141,2,130,1,136,0,124,
-    0,102,1,136,1,158,2,142,0,83,0,41,3,122,45,80,
-    97,116,104,32,104,111,111,107,32,102,111,114,32,105,109,112,
-    111,114,116,108,105,98,46,109,97,99,104,105,110,101,114,121,
-    46,70,105,108,101,70,105,110,100,101,114,46,122,30,111,110,
-    108,121,32,100,105,114,101,99,116,111,114,105,101,115,32,97,
-    114,101,32,115,117,112,112,111,114,116,101,100,114,48,0,0,
-    0,41,2,114,56,0,0,0,114,118,0,0,0,114,48,0,
-    0,0,169,2,114,193,0,0,0,114,90,1,0,0,114,3,
-    0,0,0,114,6,0,0,0,218,24,112,97,116,104,95,104,
-    111,111,107,95,102,111,114,95,70,105,108,101,70,105,110,100,
-    101,114,32,6,0,0,115,6,0,0,0,0,2,8,1,12,
-    1,122,54,70,105,108,101,70,105,110,100,101,114,46,112,97,
-    116,104,95,104,111,111,107,46,60,108,111,99,97,108,115,62,
-    46,112,97,116,104,95,104,111,111,107,95,102,111,114,95,70,
-    105,108,101,70,105,110,100,101,114,114,3,0,0,0,41,3,
-    114,193,0,0,0,114,90,1,0,0,114,97,1,0,0,114,
-    3,0,0,0,114,96,1,0,0,114,6,0,0,0,218,9,
-    112,97,116,104,95,104,111,111,107,22,6,0,0,115,4,0,
-    0,0,0,10,14,6,122,20,70,105,108,101,70,105,110,100,
-    101,114,46,112,97,116,104,95,104,111,111,107,99,1,0,0,
-    0,0,0,0,0,0,0,0,0,1,0,0,0,3,0,0,
-    0,67,0,0,0,115,12,0,0,0,100,1,160,0,124,0,
-    106,1,161,1,83,0,41,2,78,122,16,70,105,108,101,70,
-    105,110,100,101,114,40,123,33,114,125,41,41,2,114,62,0,
-    0,0,114,44,0,0,0,114,246,0,0,0,114,3,0,0,
-    0,114,3,0,0,0,114,6,0,0,0,114,39,1,0,0,
-    40,6,0,0,115,2,0,0,0,0,1,122,19,70,105,108,
-    101,70,105,110,100,101,114,46,95,95,114,101,112,114,95,95,
-    41,1,78,41,15,114,125,0,0,0,114,124,0,0,0,114,
-    126,0,0,0,114,127,0,0,0,114,209,0,0,0,114,46,
-    1,0,0,114,143,0,0,0,114,206,0,0,0,114,137,0,
-    0,0,114,58,1,0,0,114,203,0,0,0,114,91,1,0,
-    0,114,207,0,0,0,114,98,1,0,0,114,39,1,0,0,
-    114,3,0,0,0,114,3,0,0,0,114,3,0,0,0,114,
-    6,0,0,0,114,83,1,0,0,153,5,0,0,115,22,0,
-    0,0,8,2,4,7,8,14,8,4,4,2,8,12,8,5,
-    10,48,8,31,2,1,10,17,114,83,1,0,0,99,4,0,
-    0,0,0,0,0,0,0,0,0,0,6,0,0,0,8,0,
-    0,0,67,0,0,0,115,146,0,0,0,124,0,160,0,100,
-    1,161,1,125,4,124,0,160,0,100,2,161,1,125,5,124,
-    4,115,66,124,5,114,36,124,5,106,1,125,4,110,30,124,
-    2,124,3,107,2,114,56,116,2,124,1,124,2,131,2,125,
-    4,110,10,116,3,124,1,124,2,131,2,125,4,124,5,115,
-    84,116,4,124,1,124,2,124,4,100,3,141,3,125,5,122,
-    36,124,5,124,0,100,2,60,0,124,4,124,0,100,1,60,
-    0,124,2,124,0,100,4,60,0,124,3,124,0,100,5,60,
-    0,87,0,110,20,4,0,116,5,107,10,114,140,1,0,1,
-    0,1,0,89,0,110,2,88,0,100,0,83,0,41,6,78,
-    218,10,95,95,108,111,97,100,101,114,95,95,218,8,95,95,
-    115,112,101,99,95,95,114,84,1,0,0,90,8,95,95,102,
-    105,108,101,95,95,90,10,95,95,99,97,99,104,101,100,95,
-    95,41,6,218,3,103,101,116,114,140,0,0,0,114,15,1,
-    0,0,114,9,1,0,0,114,190,0,0,0,114,72,1,0,
-    0,41,6,90,2,110,115,114,117,0,0,0,90,8,112,97,
-    116,104,110,97,109,101,90,9,99,112,97,116,104,110,97,109,
-    101,114,140,0,0,0,114,187,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,218,14,95,102,105,120,
-    95,117,112,95,109,111,100,117,108,101,46,6,0,0,115,34,
-    0,0,0,0,2,10,1,10,1,4,1,4,1,8,1,8,
-    1,12,2,10,1,4,1,14,1,2,1,8,1,8,1,8,
-    1,12,1,14,2,114,102,1,0,0,99,0,0,0,0,0,
-    0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,67,
-    0,0,0,115,38,0,0,0,116,0,116,1,160,2,161,0,
-    102,2,125,0,116,3,116,4,102,2,125,1,116,5,116,6,
-    102,2,125,2,124,0,124,1,124,2,103,3,83,0,41,1,
-    122,95,82,101,116,117,114,110,115,32,97,32,108,105,115,116,
-    32,111,102,32,102,105,108,101,45,98,97,115,101,100,32,109,
-    111,100,117,108,101,32,108,111,97,100,101,114,115,46,10,10,
-    32,32,32,32,69,97,99,104,32,105,116,101,109,32,105,115,
-    32,97,32,116,117,112,108,101,32,40,108,111,97,100,101,114,
-    44,32,115,117,102,102,105,120,101,115,41,46,10,32,32,32,
-    32,41,7,114,252,0,0,0,114,163,0,0,0,218,18,101,
-    120,116,101,110,115,105,111,110,95,115,117,102,102,105,120,101,
-    115,114,9,1,0,0,114,102,0,0,0,114,15,1,0,0,
-    114,89,0,0,0,41,3,90,10,101,120,116,101,110,115,105,
-    111,110,115,90,6,115,111,117,114,99,101,90,8,98,121,116,
-    101,99,111,100,101,114,3,0,0,0,114,3,0,0,0,114,
-    6,0,0,0,114,184,0,0,0,69,6,0,0,115,8,0,
-    0,0,0,5,12,1,8,1,8,1,114,184,0,0,0,99,
-    1,0,0,0,0,0,0,0,0,0,0,0,12,0,0,0,
-    9,0,0,0,67,0,0,0,115,178,1,0,0,124,0,97,
-    0,116,0,106,1,97,1,116,0,106,2,97,2,116,1,106,
-    3,116,4,25,0,125,1,100,1,68,0,93,48,125,2,124,
-    2,116,1,106,3,107,7,114,56,116,0,160,5,124,2,161,
-    1,125,3,110,10,116,1,106,3,124,2,25,0,125,3,116,
-    6,124,1,124,2,124,3,131,3,1,0,113,30,100,2,100,
-    3,103,1,102,2,100,4,100,5,100,3,103,2,102,2,102,
-    2,125,4,124,4,68,0,93,110,92,2,125,5,125,6,116,
-    7,100,6,100,7,132,0,124,6,68,0,131,1,131,1,115,
-    136,116,8,130,1,124,6,100,8,25,0,125,7,124,5,116,
-    1,106,3,107,6,114,170,116,1,106,3,124,5,25,0,125,
-    8,1,0,113,226,113,106,122,20,116,0,160,5,124,5,161,
-    1,125,8,87,0,1,0,113,226,87,0,113,106,4,0,116,
-    9,107,10,114,214,1,0,1,0,1,0,89,0,113,106,89,
-    0,113,106,88,0,113,106,116,9,100,9,131,1,130,1,116,
-    6,124,1,100,10,124,8,131,3,1,0,116,6,124,1,100,
-    11,124,7,131,3,1,0,116,6,124,1,100,12,100,13,160,
-    10,124,6,161,1,131,3,1,0,116,6,124,1,100,14,100,
-    15,100,16,132,0,124,6,68,0,131,1,131,3,1,0,116,
-    0,160,5,100,17,161,1,125,9,116,6,124,1,100,17,124,
-    9,131,3,1,0,116,0,160,5,100,18,161,1,125,10,116,
-    6,124,1,100,18,124,10,131,3,1,0,124,5,100,4,107,
-    2,144,1,114,110,116,0,160,5,100,19,161,1,125,11,116,
-    6,124,1,100,20,124,11,131,3,1,0,116,6,124,1,100,
-    21,116,11,131,0,131,3,1,0,116,12,160,13,116,2,160,
-    14,161,0,161,1,1,0,124,5,100,4,107,2,144,1,114,
-    174,116,15,160,16,100,22,161,1,1,0,100,23,116,12,107,
-    6,144,1,114,174,100,24,116,17,95,18,100,25,83,0,41,
-    26,122,205,83,101,116,117,112,32,116,104,101,32,112,97,116,
-    104,45,98,97,115,101,100,32,105,109,112,111,114,116,101,114,
-    115,32,102,111,114,32,105,109,112,111,114,116,108,105,98,32,
-    98,121,32,105,109,112,111,114,116,105,110,103,32,110,101,101,
-    100,101,100,10,32,32,32,32,98,117,105,108,116,45,105,110,
-    32,109,111,100,117,108,101,115,32,97,110,100,32,105,110,106,
-    101,99,116,105,110,103,32,116,104,101,109,32,105,110,116,111,
-    32,116,104,101,32,103,108,111,98,97,108,32,110,97,109,101,
-    115,112,97,99,101,46,10,10,32,32,32,32,79,116,104,101,
-    114,32,99,111,109,112,111,110,101,110,116,115,32,97,114,101,
-    32,101,120,116,114,97,99,116,101,100,32,102,114,111,109,32,
-    116,104,101,32,99,111,114,101,32,98,111,111,116,115,116,114,
-    97,112,32,109,111,100,117,108,101,46,10,10,32,32,32,32,
-    41,4,114,64,0,0,0,114,75,0,0,0,218,8,98,117,
-    105,108,116,105,110,115,114,160,0,0,0,90,5,112,111,115,
-    105,120,250,1,47,90,2,110,116,250,1,92,99,1,0,0,
+    45,1,0,0,214,4,0,0,115,54,0,0,0,8,2,4,
+    2,2,1,10,9,2,1,10,12,2,1,10,21,2,1,10,
+    14,2,1,12,31,2,1,12,23,2,1,12,12,2,1,12,
+    15,2,1,10,8,2,1,10,10,2,1,10,6,2,1,10,
+    6,2,1,114,45,1,0,0,99,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,3,0,0,0,64,0,0,
+    0,115,90,0,0,0,101,0,90,1,100,0,90,2,100,1,
+    90,3,100,2,100,3,132,0,90,4,100,4,100,5,132,0,
+    90,5,101,6,90,7,100,6,100,7,132,0,90,8,100,8,
+    100,9,132,0,90,9,100,19,100,11,100,12,132,1,90,10,
+    100,13,100,14,132,0,90,11,101,12,100,15,100,16,132,0,
+    131,1,90,13,100,17,100,18,132,0,90,14,100,10,83,0,
+    41,20,218,10,70,105,108,101,70,105,110,100,101,114,122,172,
+    70,105,108,101,45,98,97,115,101,100,32,102,105,110,100,101,
+    114,46,10,10,32,32,32,32,73,110,116,101,114,97,99,116,
+    105,111,110,115,32,119,105,116,104,32,116,104,101,32,102,105,
+    108,101,32,115,121,115,116,101,109,32,97,114,101,32,99,97,
+    99,104,101,100,32,102,111,114,32,112,101,114,102,111,114,109,
+    97,110,99,101,44,32,98,101,105,110,103,10,32,32,32,32,
+    114,101,102,114,101,115,104,101,100,32,119,104,101,110,32,116,
+    104,101,32,100,105,114,101,99,116,111,114,121,32,116,104,101,
+    32,102,105,110,100,101,114,32,105,115,32,104,97,110,100,108,
+    105,110,103,32,104,97,115,32,98,101,101,110,32,109,111,100,
+    105,102,105,101,100,46,10,10,32,32,32,32,99,2,0,0,
+    0,0,0,0,0,0,0,0,0,5,0,0,0,6,0,0,
+    0,7,0,0,0,115,84,0,0,0,103,0,125,3,124,2,
+    68,0,93,32,92,2,137,0,125,4,124,3,160,0,135,0,
+    102,1,100,1,100,2,132,8,124,4,68,0,131,1,161,1,
+    1,0,113,8,124,3,124,0,95,1,124,1,112,54,100,3,
+    124,0,95,2,100,4,124,0,95,3,116,4,131,0,124,0,
+    95,5,116,4,131,0,124,0,95,6,100,5,83,0,41,6,
+    122,154,73,110,105,116,105,97,108,105,122,101,32,119,105,116,
+    104,32,116,104,101,32,112,97,116,104,32,116,111,32,115,101,
+    97,114,99,104,32,111,110,32,97,110,100,32,97,32,118,97,
+    114,105,97,98,108,101,32,110,117,109,98,101,114,32,111,102,
+    10,32,32,32,32,32,32,32,32,50,45,116,117,112,108,101,
+    115,32,99,111,110,116,97,105,110,105,110,103,32,116,104,101,
+    32,108,111,97,100,101,114,32,97,110,100,32,116,104,101,32,
+    102,105,108,101,32,115,117,102,102,105,120,101,115,32,116,104,
+    101,32,108,111,97,100,101,114,10,32,32,32,32,32,32,32,
+    32,114,101,99,111,103,110,105,122,101,115,46,99,1,0,0,
     0,0,0,0,0,0,0,0,0,2,0,0,0,3,0,0,
-    0,115,0,0,0,115,26,0,0,0,124,0,93,18,125,1,
-    116,0,124,1,131,1,100,0,107,2,86,0,1,0,113,2,
-    100,1,83,0,41,2,114,39,0,0,0,78,41,1,114,22,
-    0,0,0,41,2,114,32,0,0,0,114,95,0,0,0,114,
-    3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,19,
-    1,0,0,105,6,0,0,115,4,0,0,0,4,0,2,0,
-    122,25,95,115,101,116,117,112,46,60,108,111,99,97,108,115,
-    62,46,60,103,101,110,101,120,112,114,62,114,73,0,0,0,
-    122,30,105,109,112,111,114,116,108,105,98,32,114,101,113,117,
-    105,114,101,115,32,112,111,115,105,120,32,111,114,32,110,116,
-    114,2,0,0,0,114,35,0,0,0,114,31,0,0,0,114,
-    40,0,0,0,114,58,0,0,0,99,1,0,0,0,0,0,
-    0,0,0,0,0,0,2,0,0,0,4,0,0,0,83,0,
-    0,0,115,22,0,0,0,104,0,124,0,93,14,125,1,100,
-    0,124,1,155,0,157,2,146,2,113,4,83,0,41,1,114,
-    74,0,0,0,114,3,0,0,0,41,2,114,32,0,0,0,
-    218,1,115,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,114,92,1,0,0,121,6,0,0,115,4,0,0,0,
-    6,0,2,0,122,25,95,115,101,116,117,112,46,60,108,111,
-    99,97,108,115,62,46,60,115,101,116,99,111,109,112,62,90,
-    7,95,116,104,114,101,97,100,90,8,95,119,101,97,107,114,
-    101,102,90,6,119,105,110,114,101,103,114,192,0,0,0,114,
-    7,0,0,0,122,4,46,112,121,119,122,6,95,100,46,112,
-    121,100,84,78,41,19,114,134,0,0,0,114,8,0,0,0,
-    114,163,0,0,0,114,31,1,0,0,114,125,0,0,0,90,
-    18,95,98,117,105,108,116,105,110,95,102,114,111,109,95,110,
-    97,109,101,114,129,0,0,0,218,3,97,108,108,114,23,0,
-    0,0,114,118,0,0,0,114,36,0,0,0,114,13,0,0,
-    0,114,21,1,0,0,114,167,0,0,0,114,103,1,0,0,
-    114,102,0,0,0,114,186,0,0,0,114,191,0,0,0,114,
-    195,0,0,0,41,12,218,17,95,98,111,111,116,115,116,114,
-    97,112,95,109,111,100,117,108,101,90,11,115,101,108,102,95,
-    109,111,100,117,108,101,90,12,98,117,105,108,116,105,110,95,
-    110,97,109,101,90,14,98,117,105,108,116,105,110,95,109,111,
-    100,117,108,101,90,10,111,115,95,100,101,116,97,105,108,115,
-    90,10,98,117,105,108,116,105,110,95,111,115,114,31,0,0,
-    0,114,35,0,0,0,90,9,111,115,95,109,111,100,117,108,
-    101,90,13,116,104,114,101,97,100,95,109,111,100,117,108,101,
-    90,14,119,101,97,107,114,101,102,95,109,111,100,117,108,101,
-    90,13,119,105,110,114,101,103,95,109,111,100,117,108,101,114,
-    3,0,0,0,114,3,0,0,0,114,6,0,0,0,218,6,
-    95,115,101,116,117,112,80,6,0,0,115,78,0,0,0,0,
-    8,4,1,6,1,6,3,10,1,8,1,10,1,12,2,10,
-    1,14,3,22,1,12,2,22,1,8,1,10,1,10,1,6,
-    2,2,1,10,1,10,1,14,1,12,2,8,1,12,1,12,
-    1,18,1,22,3,10,1,12,3,10,1,12,3,10,1,10,
-    1,12,3,14,1,14,1,10,1,10,1,10,1,114,110,1,
-    0,0,99,1,0,0,0,0,0,0,0,0,0,0,0,2,
-    0,0,0,4,0,0,0,67,0,0,0,115,50,0,0,0,
-    116,0,124,0,131,1,1,0,116,1,131,0,125,1,116,2,
-    106,3,160,4,116,5,106,6,124,1,142,0,103,1,161,1,
-    1,0,116,2,106,7,160,8,116,9,161,1,1,0,100,1,
-    83,0,41,2,122,41,73,110,115,116,97,108,108,32,116,104,
-    101,32,112,97,116,104,45,98,97,115,101,100,32,105,109,112,
-    111,114,116,32,99,111,109,112,111,110,101,110,116,115,46,78,
-    41,10,114,110,1,0,0,114,184,0,0,0,114,8,0,0,
-    0,114,51,1,0,0,114,167,0,0,0,114,83,1,0,0,
-    114,98,1,0,0,218,9,109,101,116,97,95,112,97,116,104,
-    114,186,0,0,0,114,45,1,0,0,41,2,114,109,1,0,
-    0,90,17,115,117,112,112,111,114,116,101,100,95,108,111,97,
-    100,101,114,115,114,3,0,0,0,114,3,0,0,0,114,6,
-    0,0,0,218,8,95,105,110,115,116,97,108,108,145,6,0,
-    0,115,8,0,0,0,0,2,8,1,6,1,20,1,114,112,
-    1,0,0,41,1,114,60,0,0,0,41,1,78,41,3,78,
-    78,78,41,2,114,73,0,0,0,114,73,0,0,0,41,1,
-    84,41,1,78,41,1,78,41,63,114,127,0,0,0,114,12,
-    0,0,0,90,37,95,67,65,83,69,95,73,78,83,69,78,
-    83,73,84,73,86,69,95,80,76,65,84,70,79,82,77,83,
-    95,66,89,84,69,83,95,75,69,89,114,11,0,0,0,114,
-    13,0,0,0,114,20,0,0,0,114,27,0,0,0,114,29,
-    0,0,0,114,38,0,0,0,114,47,0,0,0,114,49,0,
-    0,0,114,53,0,0,0,114,54,0,0,0,114,56,0,0,
-    0,114,59,0,0,0,114,69,0,0,0,218,4,116,121,112,
-    101,218,8,95,95,99,111,100,101,95,95,114,162,0,0,0,
-    114,18,0,0,0,114,148,0,0,0,114,17,0,0,0,114,
-    24,0,0,0,114,236,0,0,0,114,92,0,0,0,114,88,
-    0,0,0,114,102,0,0,0,114,89,0,0,0,90,23,68,
-    69,66,85,71,95,66,89,84,69,67,79,68,69,95,83,85,
-    70,70,73,88,69,83,90,27,79,80,84,73,77,73,90,69,
-    68,95,66,89,84,69,67,79,68,69,95,83,85,70,70,73,
-    88,69,83,114,98,0,0,0,114,103,0,0,0,114,109,0,
-    0,0,114,113,0,0,0,114,115,0,0,0,114,136,0,0,
-    0,114,143,0,0,0,114,152,0,0,0,114,156,0,0,0,
-    114,158,0,0,0,114,165,0,0,0,114,170,0,0,0,114,
-    171,0,0,0,114,176,0,0,0,218,6,111,98,106,101,99,
-    116,114,185,0,0,0,114,190,0,0,0,114,191,0,0,0,
-    114,208,0,0,0,114,221,0,0,0,114,239,0,0,0,114,
-    9,1,0,0,114,15,1,0,0,114,21,1,0,0,114,252,
-    0,0,0,114,22,1,0,0,114,43,1,0,0,114,45,1,
-    0,0,114,83,1,0,0,114,102,1,0,0,114,184,0,0,
-    0,114,110,1,0,0,114,112,1,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,218,
-    8,60,109,111,100,117,108,101,62,1,0,0,0,115,126,0,
-    0,0,4,22,4,1,4,1,2,1,2,255,4,4,8,17,
-    8,5,8,5,8,6,8,6,8,12,8,10,8,9,8,5,
-    8,7,8,9,10,22,10,127,0,13,16,1,12,2,4,1,
-    4,2,6,2,6,2,8,2,16,71,8,40,8,19,8,12,
-    8,12,8,28,8,17,8,33,8,28,8,24,10,13,10,10,
-    10,11,8,14,6,3,4,1,2,255,12,68,14,64,14,29,
-    16,127,0,17,14,72,18,45,18,26,4,3,18,53,14,63,
-    14,42,14,127,0,68,14,127,0,22,10,23,8,11,8,65,
+    0,51,0,0,0,115,22,0,0,0,124,0,93,14,125,1,
+    124,1,136,0,102,2,86,0,1,0,113,2,100,0,83,0,
+    114,110,0,0,0,114,3,0,0,0,114,16,1,0,0,169,
+    1,114,140,0,0,0,114,3,0,0,0,114,6,0,0,0,
+    114,19,1,0,0,166,5,0,0,115,4,0,0,0,4,0,
+    2,0,122,38,70,105,108,101,70,105,110,100,101,114,46,95,
+    95,105,110,105,116,95,95,46,60,108,111,99,97,108,115,62,
+    46,60,103,101,110,101,120,112,114,62,114,71,0,0,0,114,
+    105,0,0,0,78,41,7,114,167,0,0,0,218,8,95,108,
+    111,97,100,101,114,115,114,44,0,0,0,218,11,95,112,97,
+    116,104,95,109,116,105,109,101,218,3,115,101,116,218,11,95,
+    112,97,116,104,95,99,97,99,104,101,218,19,95,114,101,108,
+    97,120,101,100,95,112,97,116,104,95,99,97,99,104,101,41,
+    5,114,119,0,0,0,114,44,0,0,0,218,14,108,111,97,
+    100,101,114,95,100,101,116,97,105,108,115,90,7,108,111,97,
+    100,101,114,115,114,189,0,0,0,114,3,0,0,0,114,85,
+    1,0,0,114,6,0,0,0,114,209,0,0,0,160,5,0,
+    0,115,16,0,0,0,0,4,4,1,12,1,26,1,6,2,
+    10,1,6,1,8,1,122,19,70,105,108,101,70,105,110,100,
+    101,114,46,95,95,105,110,105,116,95,95,99,1,0,0,0,
+    0,0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,
+    67,0,0,0,115,10,0,0,0,100,1,124,0,95,0,100,
+    2,83,0,41,3,122,31,73,110,118,97,108,105,100,97,116,
+    101,32,116,104,101,32,100,105,114,101,99,116,111,114,121,32,
+    109,116,105,109,101,46,114,105,0,0,0,78,41,1,114,87,
+    1,0,0,114,246,0,0,0,114,3,0,0,0,114,3,0,
+    0,0,114,6,0,0,0,114,46,1,0,0,174,5,0,0,
+    115,2,0,0,0,0,2,122,28,70,105,108,101,70,105,110,
+    100,101,114,46,105,110,118,97,108,105,100,97,116,101,95,99,
+    97,99,104,101,115,99,2,0,0,0,0,0,0,0,0,0,
+    0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,42,
+    0,0,0,124,0,160,0,124,1,161,1,125,2,124,2,100,
+    1,107,8,114,26,100,1,103,0,102,2,83,0,124,2,106,
+    1,124,2,106,2,112,38,103,0,102,2,83,0,41,2,122,
+    197,84,114,121,32,116,111,32,102,105,110,100,32,97,32,108,
+    111,97,100,101,114,32,102,111,114,32,116,104,101,32,115,112,
+    101,99,105,102,105,101,100,32,109,111,100,117,108,101,44,32,
+    111,114,32,116,104,101,32,110,97,109,101,115,112,97,99,101,
+    10,32,32,32,32,32,32,32,32,112,97,99,107,97,103,101,
+    32,112,111,114,116,105,111,110,115,46,32,82,101,116,117,114,
+    110,115,32,40,108,111,97,100,101,114,44,32,108,105,115,116,
+    45,111,102,45,112,111,114,116,105,111,110,115,41,46,10,10,
+    32,32,32,32,32,32,32,32,84,104,105,115,32,109,101,116,
+    104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101,
+    100,46,32,32,85,115,101,32,102,105,110,100,95,115,112,101,
+    99,40,41,32,105,110,115,116,101,97,100,46,10,10,32,32,
+    32,32,32,32,32,32,78,41,3,114,203,0,0,0,114,140,
+    0,0,0,114,178,0,0,0,41,3,114,119,0,0,0,114,
+    139,0,0,0,114,187,0,0,0,114,3,0,0,0,114,3,
+    0,0,0,114,6,0,0,0,114,137,0,0,0,180,5,0,
+    0,115,8,0,0,0,0,7,10,1,8,1,8,1,122,22,
+    70,105,108,101,70,105,110,100,101,114,46,102,105,110,100,95,
+    108,111,97,100,101,114,99,6,0,0,0,0,0,0,0,0,
+    0,0,0,7,0,0,0,6,0,0,0,67,0,0,0,115,
+    26,0,0,0,124,1,124,2,124,3,131,2,125,6,116,0,
+    124,2,124,3,124,6,124,4,100,1,141,4,83,0,41,2,
+    78,114,177,0,0,0,41,1,114,190,0,0,0,41,7,114,
+    119,0,0,0,114,188,0,0,0,114,139,0,0,0,114,44,
+    0,0,0,90,4,115,109,115,108,114,202,0,0,0,114,140,
+    0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
+    0,0,114,58,1,0,0,192,5,0,0,115,8,0,0,0,
+    0,1,10,1,8,1,2,255,122,20,70,105,108,101,70,105,
+    110,100,101,114,46,95,103,101,116,95,115,112,101,99,78,99,
+    3,0,0,0,0,0,0,0,0,0,0,0,14,0,0,0,
+    8,0,0,0,67,0,0,0,115,98,1,0,0,100,1,125,
+    3,124,1,160,0,100,2,161,1,100,3,25,0,125,4,122,
+    24,116,1,124,0,106,2,112,34,116,3,160,4,161,0,131,
+    1,106,5,125,5,87,0,110,24,4,0,116,6,107,10,114,
+    66,1,0,1,0,1,0,100,4,125,5,89,0,110,2,88,
+    0,124,5,124,0,106,7,107,3,114,92,124,0,160,8,161,
+    0,1,0,124,5,124,0,95,7,116,9,131,0,114,114,124,
+    0,106,10,125,6,124,4,160,11,161,0,125,7,110,10,124,
+    0,106,12,125,6,124,4,125,7,124,7,124,6,107,6,114,
+    218,116,13,124,0,106,2,124,4,131,2,125,8,124,0,106,
+    14,68,0,93,58,92,2,125,9,125,10,100,5,124,9,23,
+    0,125,11,116,13,124,8,124,11,131,2,125,12,116,15,124,
+    12,131,1,114,150,124,0,160,16,124,10,124,1,124,12,124,
+    8,103,1,124,2,161,5,2,0,1,0,83,0,113,150,116,
+    17,124,8,131,1,125,3,124,0,106,14,68,0,93,82,92,
+    2,125,9,125,10,116,13,124,0,106,2,124,4,124,9,23,
+    0,131,2,125,12,116,18,106,19,100,6,124,12,100,3,100,
+    7,141,3,1,0,124,7,124,9,23,0,124,6,107,6,114,
+    224,116,15,124,12,131,1,114,224,124,0,160,16,124,10,124,
+    1,124,12,100,8,124,2,161,5,2,0,1,0,83,0,113,
+    224,124,3,144,1,114,94,116,18,160,19,100,9,124,8,161,
+    2,1,0,116,18,160,20,124,1,100,8,161,2,125,13,124,
+    8,103,1,124,13,95,21,124,13,83,0,100,8,83,0,41,
+    10,122,111,84,114,121,32,116,111,32,102,105,110,100,32,97,
+    32,115,112,101,99,32,102,111,114,32,116,104,101,32,115,112,
+    101,99,105,102,105,101,100,32,109,111,100,117,108,101,46,10,
+    10,32,32,32,32,32,32,32,32,82,101,116,117,114,110,115,
+    32,116,104,101,32,109,97,116,99,104,105,110,103,32,115,112,
+    101,99,44,32,111,114,32,78,111,110,101,32,105,102,32,110,
+    111,116,32,102,111,117,110,100,46,10,32,32,32,32,32,32,
+    32,32,70,114,71,0,0,0,114,28,0,0,0,114,105,0,
+    0,0,114,209,0,0,0,122,9,116,114,121,105,110,103,32,
+    123,125,41,1,90,9,118,101,114,98,111,115,105,116,121,78,
+    122,25,112,111,115,115,105,98,108,101,32,110,97,109,101,115,
+    112,97,99,101,32,102,111,114,32,123,125,41,22,114,41,0,
+    0,0,114,49,0,0,0,114,44,0,0,0,114,2,0,0,
+    0,114,55,0,0,0,114,10,1,0,0,114,50,0,0,0,
+    114,87,1,0,0,218,11,95,102,105,108,108,95,99,97,99,
+    104,101,114,7,0,0,0,114,90,1,0,0,114,106,0,0,
+    0,114,89,1,0,0,114,38,0,0,0,114,86,1,0,0,
+    114,54,0,0,0,114,58,1,0,0,114,56,0,0,0,114,
+    134,0,0,0,114,149,0,0,0,114,183,0,0,0,114,178,
+    0,0,0,41,14,114,119,0,0,0,114,139,0,0,0,114,
+    202,0,0,0,90,12,105,115,95,110,97,109,101,115,112,97,
+    99,101,90,11,116,97,105,108,95,109,111,100,117,108,101,114,
+    169,0,0,0,90,5,99,97,99,104,101,90,12,99,97,99,
+    104,101,95,109,111,100,117,108,101,90,9,98,97,115,101,95,
+    112,97,116,104,114,17,1,0,0,114,188,0,0,0,90,13,
+    105,110,105,116,95,102,105,108,101,110,97,109,101,90,9,102,
+    117,108,108,95,112,97,116,104,114,187,0,0,0,114,3,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,114,203,0,0,
+    0,197,5,0,0,115,74,0,0,0,0,5,4,1,14,1,
+    2,1,24,1,14,1,10,1,10,1,8,1,6,2,6,1,
+    6,1,10,2,6,1,4,2,8,1,12,1,14,1,8,1,
+    10,1,8,1,26,4,8,2,14,1,16,1,16,1,12,1,
+    8,1,10,1,2,0,2,255,10,2,6,1,12,1,12,1,
+    8,1,4,1,122,20,70,105,108,101,70,105,110,100,101,114,
+    46,102,105,110,100,95,115,112,101,99,99,1,0,0,0,0,
+    0,0,0,0,0,0,0,9,0,0,0,10,0,0,0,67,
+    0,0,0,115,190,0,0,0,124,0,106,0,125,1,122,22,
+    116,1,160,2,124,1,112,22,116,1,160,3,161,0,161,1,
+    125,2,87,0,110,30,4,0,116,4,116,5,116,6,102,3,
+    107,10,114,58,1,0,1,0,1,0,103,0,125,2,89,0,
+    110,2,88,0,116,7,106,8,160,9,100,1,161,1,115,84,
+    116,10,124,2,131,1,124,0,95,11,110,74,116,10,131,0,
+    125,3,124,2,68,0,93,56,125,4,124,4,160,12,100,2,
+    161,1,92,3,125,5,125,6,125,7,124,6,114,136,100,3,
+    160,13,124,5,124,7,160,14,161,0,161,2,125,8,110,4,
+    124,5,125,8,124,3,160,15,124,8,161,1,1,0,113,94,
+    124,3,124,0,95,11,116,7,106,8,160,9,116,16,161,1,
+    114,186,100,4,100,5,132,0,124,2,68,0,131,1,124,0,
+    95,17,100,6,83,0,41,7,122,68,70,105,108,108,32,116,
+    104,101,32,99,97,99,104,101,32,111,102,32,112,111,116,101,
+    110,116,105,97,108,32,109,111,100,117,108,101,115,32,97,110,
+    100,32,112,97,99,107,97,103,101,115,32,102,111,114,32,116,
+    104,105,115,32,100,105,114,101,99,116,111,114,121,46,114,0,
+    0,0,0,114,71,0,0,0,114,61,0,0,0,99,1,0,
+    0,0,0,0,0,0,0,0,0,0,2,0,0,0,4,0,
+    0,0,83,0,0,0,115,20,0,0,0,104,0,124,0,93,
+    12,125,1,124,1,160,0,161,0,146,2,113,4,83,0,114,
+    3,0,0,0,41,1,114,106,0,0,0,41,2,114,32,0,
+    0,0,90,2,102,110,114,3,0,0,0,114,3,0,0,0,
+    114,6,0,0,0,218,9,60,115,101,116,99,111,109,112,62,
+    18,6,0,0,115,4,0,0,0,6,0,2,0,122,41,70,
+    105,108,101,70,105,110,100,101,114,46,95,102,105,108,108,95,
+    99,97,99,104,101,46,60,108,111,99,97,108,115,62,46,60,
+    115,101,116,99,111,109,112,62,78,41,18,114,44,0,0,0,
+    114,2,0,0,0,114,7,1,0,0,114,55,0,0,0,114,
+    3,1,0,0,218,15,80,101,114,109,105,115,115,105,111,110,
+    69,114,114,111,114,218,18,78,111,116,65,68,105,114,101,99,
+    116,111,114,121,69,114,114,111,114,114,8,0,0,0,114,9,
+    0,0,0,114,10,0,0,0,114,88,1,0,0,114,89,1,
+    0,0,114,101,0,0,0,114,62,0,0,0,114,106,0,0,
+    0,218,3,97,100,100,114,11,0,0,0,114,90,1,0,0,
+    41,9,114,119,0,0,0,114,44,0,0,0,114,8,1,0,
+    0,90,21,108,111,119,101,114,95,115,117,102,102,105,120,95,
+    99,111,110,116,101,110,116,115,114,41,1,0,0,114,117,0,
+    0,0,114,29,1,0,0,114,17,1,0,0,90,8,110,101,
+    119,95,110,97,109,101,114,3,0,0,0,114,3,0,0,0,
+    114,6,0,0,0,114,92,1,0,0,245,5,0,0,115,34,
+    0,0,0,0,2,6,1,2,1,22,1,20,3,10,3,12,
+    1,12,7,6,1,8,1,16,1,4,1,18,2,4,1,12,
+    1,6,1,12,1,122,22,70,105,108,101,70,105,110,100,101,
+    114,46,95,102,105,108,108,95,99,97,99,104,101,99,1,0,
+    0,0,0,0,0,0,0,0,0,0,3,0,0,0,3,0,
+    0,0,7,0,0,0,115,18,0,0,0,135,0,135,1,102,
+    2,100,1,100,2,132,8,125,2,124,2,83,0,41,3,97,
+    20,1,0,0,65,32,99,108,97,115,115,32,109,101,116,104,
+    111,100,32,119,104,105,99,104,32,114,101,116,117,114,110,115,
+    32,97,32,99,108,111,115,117,114,101,32,116,111,32,117,115,
+    101,32,111,110,32,115,121,115,46,112,97,116,104,95,104,111,
+    111,107,10,32,32,32,32,32,32,32,32,119,104,105,99,104,
+    32,119,105,108,108,32,114,101,116,117,114,110,32,97,110,32,
+    105,110,115,116,97,110,99,101,32,117,115,105,110,103,32,116,
+    104,101,32,115,112,101,99,105,102,105,101,100,32,108,111,97,
+    100,101,114,115,32,97,110,100,32,116,104,101,32,112,97,116,
+    104,10,32,32,32,32,32,32,32,32,99,97,108,108,101,100,
+    32,111,110,32,116,104,101,32,99,108,111,115,117,114,101,46,
+    10,10,32,32,32,32,32,32,32,32,73,102,32,116,104,101,
+    32,112,97,116,104,32,99,97,108,108,101,100,32,111,110,32,
+    116,104,101,32,99,108,111,115,117,114,101,32,105,115,32,110,
+    111,116,32,97,32,100,105,114,101,99,116,111,114,121,44,32,
+    73,109,112,111,114,116,69,114,114,111,114,32,105,115,10,32,
+    32,32,32,32,32,32,32,114,97,105,115,101,100,46,10,10,
+    32,32,32,32,32,32,32,32,99,1,0,0,0,0,0,0,
+    0,0,0,0,0,1,0,0,0,4,0,0,0,19,0,0,
+    0,115,34,0,0,0,116,0,124,0,131,1,115,20,116,1,
+    100,1,124,0,100,2,141,2,130,1,136,0,124,0,102,1,
+    136,1,158,2,142,0,83,0,41,3,122,45,80,97,116,104,
+    32,104,111,111,107,32,102,111,114,32,105,109,112,111,114,116,
+    108,105,98,46,109,97,99,104,105,110,101,114,121,46,70,105,
+    108,101,70,105,110,100,101,114,46,122,30,111,110,108,121,32,
+    100,105,114,101,99,116,111,114,105,101,115,32,97,114,101,32,
+    115,117,112,112,111,114,116,101,100,114,48,0,0,0,41,2,
+    114,56,0,0,0,114,118,0,0,0,114,48,0,0,0,169,
+    2,114,193,0,0,0,114,91,1,0,0,114,3,0,0,0,
+    114,6,0,0,0,218,24,112,97,116,104,95,104,111,111,107,
+    95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,30,
+    6,0,0,115,6,0,0,0,0,2,8,1,12,1,122,54,
+    70,105,108,101,70,105,110,100,101,114,46,112,97,116,104,95,
+    104,111,111,107,46,60,108,111,99,97,108,115,62,46,112,97,
+    116,104,95,104,111,111,107,95,102,111,114,95,70,105,108,101,
+    70,105,110,100,101,114,114,3,0,0,0,41,3,114,193,0,
+    0,0,114,91,1,0,0,114,98,1,0,0,114,3,0,0,
+    0,114,97,1,0,0,114,6,0,0,0,218,9,112,97,116,
+    104,95,104,111,111,107,20,6,0,0,115,4,0,0,0,0,
+    10,14,6,122,20,70,105,108,101,70,105,110,100,101,114,46,
+    112,97,116,104,95,104,111,111,107,99,1,0,0,0,0,0,
+    0,0,0,0,0,0,1,0,0,0,3,0,0,0,67,0,
+    0,0,115,12,0,0,0,100,1,160,0,124,0,106,1,161,
+    1,83,0,41,2,78,122,16,70,105,108,101,70,105,110,100,
+    101,114,40,123,33,114,125,41,41,2,114,62,0,0,0,114,
+    44,0,0,0,114,246,0,0,0,114,3,0,0,0,114,3,
+    0,0,0,114,6,0,0,0,114,39,1,0,0,38,6,0,
+    0,115,2,0,0,0,0,1,122,19,70,105,108,101,70,105,
+    110,100,101,114,46,95,95,114,101,112,114,95,95,41,1,78,
+    41,15,114,125,0,0,0,114,124,0,0,0,114,126,0,0,
+    0,114,127,0,0,0,114,209,0,0,0,114,46,1,0,0,
+    114,143,0,0,0,114,206,0,0,0,114,137,0,0,0,114,
+    58,1,0,0,114,203,0,0,0,114,92,1,0,0,114,207,
+    0,0,0,114,99,1,0,0,114,39,1,0,0,114,3,0,
+    0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
+    0,114,84,1,0,0,151,5,0,0,115,22,0,0,0,8,
+    2,4,7,8,14,8,4,4,2,8,12,8,5,10,48,8,
+    31,2,1,10,17,114,84,1,0,0,99,4,0,0,0,0,
+    0,0,0,0,0,0,0,6,0,0,0,8,0,0,0,67,
+    0,0,0,115,146,0,0,0,124,0,160,0,100,1,161,1,
+    125,4,124,0,160,0,100,2,161,1,125,5,124,4,115,66,
+    124,5,114,36,124,5,106,1,125,4,110,30,124,2,124,3,
+    107,2,114,56,116,2,124,1,124,2,131,2,125,4,110,10,
+    116,3,124,1,124,2,131,2,125,4,124,5,115,84,116,4,
+    124,1,124,2,124,4,100,3,141,3,125,5,122,36,124,5,
+    124,0,100,2,60,0,124,4,124,0,100,1,60,0,124,2,
+    124,0,100,4,60,0,124,3,124,0,100,5,60,0,87,0,
+    110,20,4,0,116,5,107,10,114,140,1,0,1,0,1,0,
+    89,0,110,2,88,0,100,0,83,0,41,6,78,218,10,95,
+    95,108,111,97,100,101,114,95,95,218,8,95,95,115,112,101,
+    99,95,95,114,85,1,0,0,90,8,95,95,102,105,108,101,
+    95,95,90,10,95,95,99,97,99,104,101,100,95,95,41,6,
+    218,3,103,101,116,114,140,0,0,0,114,15,1,0,0,114,
+    9,1,0,0,114,190,0,0,0,114,72,1,0,0,41,6,
+    90,2,110,115,114,117,0,0,0,90,8,112,97,116,104,110,
+    97,109,101,90,9,99,112,97,116,104,110,97,109,101,114,140,
+    0,0,0,114,187,0,0,0,114,3,0,0,0,114,3,0,
+    0,0,114,6,0,0,0,218,14,95,102,105,120,95,117,112,
+    95,109,111,100,117,108,101,44,6,0,0,115,34,0,0,0,
+    0,2,10,1,10,1,4,1,4,1,8,1,8,1,12,2,
+    10,1,4,1,14,1,2,1,8,1,8,1,8,1,12,1,
+    14,2,114,103,1,0,0,99,0,0,0,0,0,0,0,0,
+    0,0,0,0,3,0,0,0,3,0,0,0,67,0,0,0,
+    115,38,0,0,0,116,0,116,1,160,2,161,0,102,2,125,
+    0,116,3,116,4,102,2,125,1,116,5,116,6,102,2,125,
+    2,124,0,124,1,124,2,103,3,83,0,41,1,122,95,82,
+    101,116,117,114,110,115,32,97,32,108,105,115,116,32,111,102,
+    32,102,105,108,101,45,98,97,115,101,100,32,109,111,100,117,
+    108,101,32,108,111,97,100,101,114,115,46,10,10,32,32,32,
+    32,69,97,99,104,32,105,116,101,109,32,105,115,32,97,32,
+    116,117,112,108,101,32,40,108,111,97,100,101,114,44,32,115,
+    117,102,102,105,120,101,115,41,46,10,32,32,32,32,41,7,
+    114,252,0,0,0,114,163,0,0,0,218,18,101,120,116,101,
+    110,115,105,111,110,95,115,117,102,102,105,120,101,115,114,9,
+    1,0,0,114,102,0,0,0,114,15,1,0,0,114,89,0,
+    0,0,41,3,90,10,101,120,116,101,110,115,105,111,110,115,
+    90,6,115,111,117,114,99,101,90,8,98,121,116,101,99,111,
+    100,101,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
+    0,114,184,0,0,0,67,6,0,0,115,8,0,0,0,0,
+    5,12,1,8,1,8,1,114,184,0,0,0,99,1,0,0,
+    0,0,0,0,0,0,0,0,0,12,0,0,0,9,0,0,
+    0,67,0,0,0,115,178,1,0,0,124,0,97,0,116,0,
+    106,1,97,1,116,0,106,2,97,2,116,1,106,3,116,4,
+    25,0,125,1,100,1,68,0,93,48,125,2,124,2,116,1,
+    106,3,107,7,114,56,116,0,160,5,124,2,161,1,125,3,
+    110,10,116,1,106,3,124,2,25,0,125,3,116,6,124,1,
+    124,2,124,3,131,3,1,0,113,30,100,2,100,3,103,1,
+    102,2,100,4,100,5,100,3,103,2,102,2,102,2,125,4,
+    124,4,68,0,93,110,92,2,125,5,125,6,116,7,100,6,
+    100,7,132,0,124,6,68,0,131,1,131,1,115,136,116,8,
+    130,1,124,6,100,8,25,0,125,7,124,5,116,1,106,3,
+    107,6,114,170,116,1,106,3,124,5,25,0,125,8,1,0,
+    113,226,113,106,122,20,116,0,160,5,124,5,161,1,125,8,
+    87,0,1,0,113,226,87,0,113,106,4,0,116,9,107,10,
+    114,214,1,0,1,0,1,0,89,0,113,106,89,0,113,106,
+    88,0,113,106,116,9,100,9,131,1,130,1,116,6,124,1,
+    100,10,124,8,131,3,1,0,116,6,124,1,100,11,124,7,
+    131,3,1,0,116,6,124,1,100,12,100,13,160,10,124,6,
+    161,1,131,3,1,0,116,6,124,1,100,14,100,15,100,16,
+    132,0,124,6,68,0,131,1,131,3,1,0,116,0,160,5,
+    100,17,161,1,125,9,116,6,124,1,100,17,124,9,131,3,
+    1,0,116,0,160,5,100,18,161,1,125,10,116,6,124,1,
+    100,18,124,10,131,3,1,0,124,5,100,4,107,2,144,1,
+    114,110,116,0,160,5,100,19,161,1,125,11,116,6,124,1,
+    100,20,124,11,131,3,1,0,116,6,124,1,100,21,116,11,
+    131,0,131,3,1,0,116,12,160,13,116,2,160,14,161,0,
+    161,1,1,0,124,5,100,4,107,2,144,1,114,174,116,15,
+    160,16,100,22,161,1,1,0,100,23,116,12,107,6,144,1,
+    114,174,100,24,116,17,95,18,100,25,83,0,41,26,122,205,
+    83,101,116,117,112,32,116,104,101,32,112,97,116,104,45,98,
+    97,115,101,100,32,105,109,112,111,114,116,101,114,115,32,102,
+    111,114,32,105,109,112,111,114,116,108,105,98,32,98,121,32,
+    105,109,112,111,114,116,105,110,103,32,110,101,101,100,101,100,
+    10,32,32,32,32,98,117,105,108,116,45,105,110,32,109,111,
+    100,117,108,101,115,32,97,110,100,32,105,110,106,101,99,116,
+    105,110,103,32,116,104,101,109,32,105,110,116,111,32,116,104,
+    101,32,103,108,111,98,97,108,32,110,97,109,101,115,112,97,
+    99,101,46,10,10,32,32,32,32,79,116,104,101,114,32,99,
+    111,109,112,111,110,101,110,116,115,32,97,114,101,32,101,120,
+    116,114,97,99,116,101,100,32,102,114,111,109,32,116,104,101,
+    32,99,111,114,101,32,98,111,111,116,115,116,114,97,112,32,
+    109,111,100,117,108,101,46,10,10,32,32,32,32,41,4,114,
+    64,0,0,0,114,75,0,0,0,218,8,98,117,105,108,116,
+    105,110,115,114,160,0,0,0,90,5,112,111,115,105,120,250,
+    1,47,90,2,110,116,250,1,92,99,1,0,0,0,0,0,
+    0,0,0,0,0,0,2,0,0,0,3,0,0,0,115,0,
+    0,0,115,26,0,0,0,124,0,93,18,125,1,116,0,124,
+    1,131,1,100,0,107,2,86,0,1,0,113,2,100,1,83,
+    0,41,2,114,39,0,0,0,78,41,1,114,22,0,0,0,
+    41,2,114,32,0,0,0,114,95,0,0,0,114,3,0,0,
+    0,114,3,0,0,0,114,6,0,0,0,114,19,1,0,0,
+    103,6,0,0,115,4,0,0,0,4,0,2,0,122,25,95,
+    115,101,116,117,112,46,60,108,111,99,97,108,115,62,46,60,
+    103,101,110,101,120,112,114,62,114,73,0,0,0,122,30,105,
+    109,112,111,114,116,108,105,98,32,114,101,113,117,105,114,101,
+    115,32,112,111,115,105,120,32,111,114,32,110,116,114,2,0,
+    0,0,114,35,0,0,0,114,31,0,0,0,114,40,0,0,
+    0,114,58,0,0,0,99,1,0,0,0,0,0,0,0,0,
+    0,0,0,2,0,0,0,4,0,0,0,83,0,0,0,115,
+    22,0,0,0,104,0,124,0,93,14,125,1,100,0,124,1,
+    155,0,157,2,146,2,113,4,83,0,41,1,114,74,0,0,
+    0,114,3,0,0,0,41,2,114,32,0,0,0,218,1,115,
+    114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
+    93,1,0,0,119,6,0,0,115,4,0,0,0,6,0,2,
+    0,122,25,95,115,101,116,117,112,46,60,108,111,99,97,108,
+    115,62,46,60,115,101,116,99,111,109,112,62,90,7,95,116,
+    104,114,101,97,100,90,8,95,119,101,97,107,114,101,102,90,
+    6,119,105,110,114,101,103,114,192,0,0,0,114,7,0,0,
+    0,122,4,46,112,121,119,122,6,95,100,46,112,121,100,84,
+    78,41,19,114,134,0,0,0,114,8,0,0,0,114,163,0,
+    0,0,114,31,1,0,0,114,125,0,0,0,90,18,95,98,
+    117,105,108,116,105,110,95,102,114,111,109,95,110,97,109,101,
+    114,129,0,0,0,218,3,97,108,108,114,23,0,0,0,114,
+    118,0,0,0,114,36,0,0,0,114,13,0,0,0,114,21,
+    1,0,0,114,167,0,0,0,114,104,1,0,0,114,102,0,
+    0,0,114,186,0,0,0,114,191,0,0,0,114,195,0,0,
+    0,41,12,218,17,95,98,111,111,116,115,116,114,97,112,95,
+    109,111,100,117,108,101,90,11,115,101,108,102,95,109,111,100,
+    117,108,101,90,12,98,117,105,108,116,105,110,95,110,97,109,
+    101,90,14,98,117,105,108,116,105,110,95,109,111,100,117,108,
+    101,90,10,111,115,95,100,101,116,97,105,108,115,90,10,98,
+    117,105,108,116,105,110,95,111,115,114,31,0,0,0,114,35,
+    0,0,0,90,9,111,115,95,109,111,100,117,108,101,90,13,
+    116,104,114,101,97,100,95,109,111,100,117,108,101,90,14,119,
+    101,97,107,114,101,102,95,109,111,100,117,108,101,90,13,119,
+    105,110,114,101,103,95,109,111,100,117,108,101,114,3,0,0,
+    0,114,3,0,0,0,114,6,0,0,0,218,6,95,115,101,
+    116,117,112,78,6,0,0,115,78,0,0,0,0,8,4,1,
+    6,1,6,3,10,1,8,1,10,1,12,2,10,1,14,3,
+    22,1,12,2,22,1,8,1,10,1,10,1,6,2,2,1,
+    10,1,10,1,14,1,12,2,8,1,12,1,12,1,18,1,
+    22,3,10,1,12,3,10,1,12,3,10,1,10,1,12,3,
+    14,1,14,1,10,1,10,1,10,1,114,111,1,0,0,99,
+    1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,
+    4,0,0,0,67,0,0,0,115,50,0,0,0,116,0,124,
+    0,131,1,1,0,116,1,131,0,125,1,116,2,106,3,160,
+    4,116,5,106,6,124,1,142,0,103,1,161,1,1,0,116,
+    2,106,7,160,8,116,9,161,1,1,0,100,1,83,0,41,
+    2,122,41,73,110,115,116,97,108,108,32,116,104,101,32,112,
+    97,116,104,45,98,97,115,101,100,32,105,109,112,111,114,116,
+    32,99,111,109,112,111,110,101,110,116,115,46,78,41,10,114,
+    111,1,0,0,114,184,0,0,0,114,8,0,0,0,114,51,
+    1,0,0,114,167,0,0,0,114,84,1,0,0,114,99,1,
+    0,0,218,9,109,101,116,97,95,112,97,116,104,114,186,0,
+    0,0,114,45,1,0,0,41,2,114,110,1,0,0,90,17,
+    115,117,112,112,111,114,116,101,100,95,108,111,97,100,101,114,
+    115,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
+    218,8,95,105,110,115,116,97,108,108,143,6,0,0,115,8,
+    0,0,0,0,2,8,1,6,1,20,1,114,113,1,0,0,
+    41,1,114,60,0,0,0,41,1,78,41,3,78,78,78,41,
+    2,114,73,0,0,0,114,73,0,0,0,41,1,84,41,1,
+    78,41,1,78,41,63,114,127,0,0,0,114,12,0,0,0,
+    90,37,95,67,65,83,69,95,73,78,83,69,78,83,73,84,
+    73,86,69,95,80,76,65,84,70,79,82,77,83,95,66,89,
+    84,69,83,95,75,69,89,114,11,0,0,0,114,13,0,0,
+    0,114,20,0,0,0,114,27,0,0,0,114,29,0,0,0,
+    114,38,0,0,0,114,47,0,0,0,114,49,0,0,0,114,
+    53,0,0,0,114,54,0,0,0,114,56,0,0,0,114,59,
+    0,0,0,114,69,0,0,0,218,4,116,121,112,101,218,8,
+    95,95,99,111,100,101,95,95,114,162,0,0,0,114,18,0,
+    0,0,114,148,0,0,0,114,17,0,0,0,114,24,0,0,
+    0,114,236,0,0,0,114,92,0,0,0,114,88,0,0,0,
+    114,102,0,0,0,114,89,0,0,0,90,23,68,69,66,85,
+    71,95,66,89,84,69,67,79,68,69,95,83,85,70,70,73,
+    88,69,83,90,27,79,80,84,73,77,73,90,69,68,95,66,
+    89,84,69,67,79,68,69,95,83,85,70,70,73,88,69,83,
+    114,98,0,0,0,114,103,0,0,0,114,109,0,0,0,114,
+    113,0,0,0,114,115,0,0,0,114,136,0,0,0,114,143,
+    0,0,0,114,152,0,0,0,114,156,0,0,0,114,158,0,
+    0,0,114,165,0,0,0,114,170,0,0,0,114,171,0,0,
+    0,114,176,0,0,0,218,6,111,98,106,101,99,116,114,185,
+    0,0,0,114,190,0,0,0,114,191,0,0,0,114,208,0,
+    0,0,114,221,0,0,0,114,239,0,0,0,114,9,1,0,
+    0,114,15,1,0,0,114,21,1,0,0,114,252,0,0,0,
+    114,22,1,0,0,114,43,1,0,0,114,45,1,0,0,114,
+    84,1,0,0,114,103,1,0,0,114,184,0,0,0,114,111,
+    1,0,0,114,113,1,0,0,114,3,0,0,0,114,3,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,218,8,60,109,
+    111,100,117,108,101,62,1,0,0,0,115,126,0,0,0,4,
+    22,4,1,4,1,2,1,2,255,4,4,8,17,8,5,8,
+    5,8,6,8,6,8,12,8,10,8,9,8,5,8,7,8,
+    9,10,22,10,127,0,13,16,1,12,2,4,1,4,2,6,
+    2,6,2,8,2,16,71,8,40,8,19,8,12,8,12,8,
+    28,8,17,8,33,8,28,8,24,10,13,10,10,10,11,8,
+    14,6,3,4,1,2,255,12,68,14,64,14,29,16,127,0,
+    17,14,72,18,45,18,26,4,3,18,53,14,63,14,42,14,
+    127,0,66,14,127,0,22,10,23,8,11,8,65,
 };


From webhook-mailer at python.org  Tue Sep 10 11:05:00 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Tue, 10 Sep 2019 15:05:00 -0000
Subject: [Python-checkins] Note regarding + mode truncation applies to both
 text and binary mode (GH-11314) (GH-15869)
Message-ID: <mailman.468.1568127901.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/80722308820c112f0e473f2353f550d5aeb90352
commit: 80722308820c112f0e473f2353f550d5aeb90352
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-10T16:04:56+01:00
summary:

Note regarding + mode truncation applies to both text and binary mode (GH-11314) (GH-15869)

* Improve doc on open's mode +

* Improve wording

* Address comment from R?mi
(cherry picked from commit c1d8c1cb8e90a54a3daaa7fcdb8d6ca7f08d6a73)

Co-authored-by: Andre Delfino <adelfino at gmail.com>

files:
M Doc/library/functions.rst

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index c748b086d8ac..a4097b0b30ec 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -1068,12 +1068,12 @@ are always available.  They are listed here in alphabetical order.
    ``'a'``   open for writing, appending to the end of the file if it exists
    ``'b'``   binary mode
    ``'t'``   text mode (default)
-   ``'+'``   open a disk file for updating (reading and writing)
+   ``'+'``   open for updating (reading and writing)
    ========= ===============================================================
 
    The default mode is ``'r'`` (open for reading text, synonym of ``'rt'``).
-   For binary read-write access, the mode ``'w+b'`` opens and truncates the file
-   to 0 bytes.  ``'r+b'`` opens the file without truncation.
+   Modes ``'w+'`` and ``'w+b'`` opens and truncates the file.  Modes ``'r+'``
+   and ``'r+b'`` opens the file with no truncation.
 
    As mentioned in the :ref:`io-overview`, Python distinguishes between binary
    and text I/O.  Files opened in binary mode (including ``'b'`` in the *mode*


From webhook-mailer at python.org  Tue Sep 10 11:09:38 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Tue, 10 Sep 2019 15:09:38 -0000
Subject: [Python-checkins] Docs: Small tweaks to c-api/intro#Include_Files
 (GH-14698)
Message-ID: <mailman.469.1568128179.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b6dafe51399f5c6313a00529118a6052b466942f
commit: b6dafe51399f5c6313a00529118a6052b466942f
branch: master
author: Kyle Stanley <aeros167 at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-10T16:09:34+01:00
summary:

Docs: Small tweaks to c-api/intro#Include_Files (GH-14698)

files:
M Doc/c-api/intro.rst

diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst
index a1c8d34a7ea0..80eebd89ad3f 100644
--- a/Doc/c-api/intro.rst
+++ b/Doc/c-api/intro.rst
@@ -69,10 +69,12 @@ standard headers) have one of the prefixes ``Py`` or ``_Py``.  Names beginning
 with ``_Py`` are for internal use by the Python implementation and should not be
 used by extension writers. Structure member names do not have a reserved prefix.
 
-**Important:** user code should never define names that begin with ``Py`` or
-``_Py``.  This confuses the reader, and jeopardizes the portability of the user
-code to future Python versions, which may define additional names beginning with
-one of these prefixes.
+.. note::
+
+   User code should never define names that begin with ``Py`` or ``_Py``. This
+   confuses the reader, and jeopardizes the portability of the user code to
+   future Python versions, which may define additional names beginning with one
+   of these prefixes.
 
 The header files are typically installed with Python.  On Unix, these  are
 located in the directories :file:`{prefix}/include/pythonversion/` and
@@ -90,9 +92,9 @@ multi-platform builds since the platform independent headers under
 :envvar:`prefix` include the platform specific headers from
 :envvar:`exec_prefix`.
 
-C++ users should note that though the API is defined entirely using C, the
-header files do properly declare the entry points to be ``extern "C"``, so there
-is no need to do anything special to use the API from C++.
+C++ users should note that although the API is defined entirely using C, the
+header files properly declare the entry points to be ``extern "C"``. As a result,
+there is no need to do anything special to use the API from C++.
 
 
 Useful macros


From webhook-mailer at python.org  Tue Sep 10 11:11:20 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Tue, 10 Sep 2019 15:11:20 -0000
Subject: [Python-checkins] bpo-33602: Doc: Remove set and queue references
 from Data Types (GH-7055)
Message-ID: <mailman.470.1568128281.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/912108891db52c2067889be1f4ce5713839807cd
commit: 912108891db52c2067889be1f4ce5713839807cd
branch: master
author: Andre Delfino <adelfino at gmail.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-10T17:11:16+02:00
summary:

bpo-33602: Doc: Remove set and queue references from Data Types (GH-7055)

files:
M Doc/library/datatypes.rst

diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst
index 48af0823aa49..94010c0e391b 100644
--- a/Doc/library/datatypes.rst
+++ b/Doc/library/datatypes.rst
@@ -5,13 +5,14 @@ Data Types
 **********
 
 The modules described in this chapter provide a variety of specialized data
-types such as dates and times, fixed-type arrays, heap queues, synchronized
-queues, and sets.
+types such as dates and times, fixed-type arrays, heap queues, double-ended
+queues, and enumerations.
 
 Python also provides some built-in data types, in particular,
 :class:`dict`, :class:`list`, :class:`set` and :class:`frozenset`, and
 :class:`tuple`.  The :class:`str` class is used to hold
-Unicode strings, and the :class:`bytes` class is used to hold binary data.
+Unicode strings, and the :class:`bytes` and :class:`bytearray` classes are used
+to hold binary data.
 
 The following modules are documented in this chapter:
 


From webhook-mailer at python.org  Tue Sep 10 11:20:19 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Tue, 10 Sep 2019 15:20:19 -0000
Subject: [Python-checkins] bpo-36253: Remove use after free reference in
 ctypes test suite (GH-12257)
Message-ID: <mailman.471.1568128819.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a9b6033179b64b985394ad351501089a6a94fc9d
commit: a9b6033179b64b985394ad351501089a6a94fc9d
branch: master
author: Ben Harper <btharper1221 at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-10T08:20:15-07:00
summary:

bpo-36253: Remove use after free reference in ctypes test suite (GH-12257)

files:
M Lib/ctypes/test/test_stringptr.py

diff --git a/Lib/ctypes/test/test_stringptr.py b/Lib/ctypes/test/test_stringptr.py
index 95cd1614c676..c20951f4ce00 100644
--- a/Lib/ctypes/test/test_stringptr.py
+++ b/Lib/ctypes/test/test_stringptr.py
@@ -70,8 +70,8 @@ def test_functions(self):
         x = r[0], r[1], r[2], r[3], r[4]
         self.assertEqual(x, (b"c", b"d", b"e", b"f", b"\000"))
         del buf
-        # x1 will NOT be the same as x, usually:
-        x1 = r[0], r[1], r[2], r[3], r[4]
+        # Because r is a pointer to memory that is freed after deleting buf,
+        # the pointer is hanging and using it would reference freed memory.
 
 if __name__ == '__main__':
     unittest.main()


From webhook-mailer at python.org  Tue Sep 10 11:22:06 2019
From: webhook-mailer at python.org (Stefan Behnel)
Date: Tue, 10 Sep 2019 15:22:06 -0000
Subject: [Python-checkins] bpo-32424: Deprecate
 xml.etree.ElementTree.Element.copy() in favor of copy.copy() (GH-12995)
Message-ID: <mailman.472.1568128926.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7d952ded6813c896ea3f4234bb8db5247dcb5484
commit: 7d952ded6813c896ea3f4234bb8db5247dcb5484
branch: master
author: Gordon P. Hemsley <me at gphemsley.org>
committer: Stefan Behnel <stefan_ml at behnel.de>
date: 2019-09-10T16:22:01+01:00
summary:

bpo-32424: Deprecate xml.etree.ElementTree.Element.copy() in favor of copy.copy() (GH-12995)

files:
A Misc/NEWS.d/next/Library/2019-04-28-10-34-19.bpo-32424.eqNPhM.rst
M Lib/test/test_xml_etree.py
M Lib/xml/etree/ElementTree.py

diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
index 87a6fa146462..9fbb4ac1813a 100644
--- a/Lib/test/test_xml_etree.py
+++ b/Lib/test/test_xml_etree.py
@@ -2211,6 +2211,35 @@ def test___init__(self):
         self.assertIsNot(element_foo.attrib, attrib)
         self.assertNotEqual(element_foo.attrib, attrib)
 
+    def test_copy(self):
+        # Only run this test if Element.copy() is defined.
+        if "copy" not in dir(ET.Element):
+            raise unittest.SkipTest("Element.copy() not present")
+
+        element_foo = ET.Element("foo", { "zix": "wyp" })
+        element_foo.append(ET.Element("bar", { "baz": "qix" }))
+
+        with self.assertWarns(DeprecationWarning):
+            element_foo2 = element_foo.copy()
+
+        # elements are not the same
+        self.assertIsNot(element_foo2, element_foo)
+
+        # string attributes are equal
+        self.assertEqual(element_foo2.tag, element_foo.tag)
+        self.assertEqual(element_foo2.text, element_foo.text)
+        self.assertEqual(element_foo2.tail, element_foo.tail)
+
+        # number of children is the same
+        self.assertEqual(len(element_foo2), len(element_foo))
+
+        # children are the same
+        for (child1, child2) in itertools.zip_longest(element_foo, element_foo2):
+            self.assertIs(child1, child2)
+
+        # attrib is a copy
+        self.assertEqual(element_foo2.attrib, element_foo.attrib)
+
     def test___copy__(self):
         element_foo = ET.Element("foo", { "zix": "wyp" })
         element_foo.append(ET.Element("bar", { "baz": "qix" }))
diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py
index e75200a1b8a3..c8d898f32816 100644
--- a/Lib/xml/etree/ElementTree.py
+++ b/Lib/xml/etree/ElementTree.py
@@ -195,6 +195,13 @@ def copy(self):
         original tree.
 
         """
+        warnings.warn(
+            "elem.copy() is deprecated. Use copy.copy(elem) instead.",
+            DeprecationWarning
+            )
+        return self.__copy__()
+
+    def __copy__(self):
         elem = self.makeelement(self.tag, self.attrib)
         elem.text = self.text
         elem.tail = self.tail
diff --git a/Misc/NEWS.d/next/Library/2019-04-28-10-34-19.bpo-32424.eqNPhM.rst b/Misc/NEWS.d/next/Library/2019-04-28-10-34-19.bpo-32424.eqNPhM.rst
new file mode 100644
index 000000000000..a60d68bf2de3
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-04-28-10-34-19.bpo-32424.eqNPhM.rst
@@ -0,0 +1,3 @@
+Deprecate xml.etree.ElementTree.Element.copy() in favor of copy.copy().
+
+Patch by Gordon P. Hemsley


From webhook-mailer at python.org  Tue Sep 10 11:43:00 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Tue, 10 Sep 2019 15:43:00 -0000
Subject: [Python-checkins] bpo-33602: Doc: Remove set and queue references
 from Data Types (GH-7055) (GH-15875)
Message-ID: <mailman.473.1568130181.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/58ef7d341c79f649da275bb1d5c11f668d7bac9e
commit: 58ef7d341c79f649da275bb1d5c11f668d7bac9e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-10T17:42:56+02:00
summary:

bpo-33602: Doc: Remove set and queue references from Data Types (GH-7055) (GH-15875)

(cherry picked from commit 912108891db52c2067889be1f4ce5713839807cd)

Co-authored-by: Andre Delfino <adelfino at gmail.com>

files:
M Doc/library/datatypes.rst

diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst
index 48af0823aa49..94010c0e391b 100644
--- a/Doc/library/datatypes.rst
+++ b/Doc/library/datatypes.rst
@@ -5,13 +5,14 @@ Data Types
 **********
 
 The modules described in this chapter provide a variety of specialized data
-types such as dates and times, fixed-type arrays, heap queues, synchronized
-queues, and sets.
+types such as dates and times, fixed-type arrays, heap queues, double-ended
+queues, and enumerations.
 
 Python also provides some built-in data types, in particular,
 :class:`dict`, :class:`list`, :class:`set` and :class:`frozenset`, and
 :class:`tuple`.  The :class:`str` class is used to hold
-Unicode strings, and the :class:`bytes` class is used to hold binary data.
+Unicode strings, and the :class:`bytes` and :class:`bytearray` classes are used
+to hold binary data.
 
 The following modules are documented in this chapter:
 


From webhook-mailer at python.org  Tue Sep 10 11:43:15 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Tue, 10 Sep 2019 15:43:15 -0000
Subject: [Python-checkins] bpo-33602: Doc: Remove set and queue references
 from Data Types (GH-7055) (GH-15876)
Message-ID: <mailman.474.1568130196.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/816825e192ed430438c613d52a58fb9e1a8d90f4
commit: 816825e192ed430438c613d52a58fb9e1a8d90f4
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-10T17:43:11+02:00
summary:

bpo-33602: Doc: Remove set and queue references from Data Types (GH-7055) (GH-15876)

(cherry picked from commit 912108891db52c2067889be1f4ce5713839807cd)

Co-authored-by: Andre Delfino <adelfino at gmail.com>

files:
M Doc/library/datatypes.rst

diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst
index 48af0823aa49..94010c0e391b 100644
--- a/Doc/library/datatypes.rst
+++ b/Doc/library/datatypes.rst
@@ -5,13 +5,14 @@ Data Types
 **********
 
 The modules described in this chapter provide a variety of specialized data
-types such as dates and times, fixed-type arrays, heap queues, synchronized
-queues, and sets.
+types such as dates and times, fixed-type arrays, heap queues, double-ended
+queues, and enumerations.
 
 Python also provides some built-in data types, in particular,
 :class:`dict`, :class:`list`, :class:`set` and :class:`frozenset`, and
 :class:`tuple`.  The :class:`str` class is used to hold
-Unicode strings, and the :class:`bytes` class is used to hold binary data.
+Unicode strings, and the :class:`bytes` and :class:`bytearray` classes are used
+to hold binary data.
 
 The following modules are documented in this chapter:
 


From webhook-mailer at python.org  Tue Sep 10 11:53:07 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Tue, 10 Sep 2019 15:53:07 -0000
Subject: [Python-checkins] bpo-38089: Move Azure Pipelines to latest VM
 versions and make macOS tests optional (GH-15851)
Message-ID: <mailman.475.1568130788.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/801f925998cc393260f36f5ac77369fef2373ad1
commit: 801f925998cc393260f36f5ac77369fef2373ad1
branch: master
author: Steve Dower <steve.dower at python.org>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-10T16:53:03+01:00
summary:

bpo-38089: Move Azure Pipelines to latest VM versions and make macOS tests optional (GH-15851)

files:
A Misc/NEWS.d/next/macOS/2019-09-10-14-24-35.bpo-38089.eedgyD.rst
M .azure-pipelines/ci.yml
M .azure-pipelines/macos-steps.yml
M .azure-pipelines/pr.yml

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index fcfac85ed9c4..12ff7d4c55c9 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -42,7 +42,7 @@ jobs:
     testRunPlatform: macos
 
   pool:
-    vmImage: xcode9-macos10.13
+    vmImage: macos-10.14
 
   steps:
   - template: ./macos-steps.yml
@@ -131,7 +131,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: vs2017-win2016
+    vmImage: windows-2019
 
   strategy:
     matrix:
diff --git a/.azure-pipelines/macos-steps.yml b/.azure-pipelines/macos-steps.yml
index 647081689454..d2ca580a93d7 100644
--- a/.azure-pipelines/macos-steps.yml
+++ b/.azure-pipelines/macos-steps.yml
@@ -14,6 +14,8 @@ steps:
 
 - script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
   displayName: 'Tests'
+  continueOnError: true
+  timeoutInMinutes: 30
 
 - task: PublishTestResults at 2
   displayName: 'Publish Test Results'
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
index 2486f88a63fb..e3e24ae2a4b0 100644
--- a/.azure-pipelines/pr.yml
+++ b/.azure-pipelines/pr.yml
@@ -40,7 +40,7 @@ jobs:
     testRunPlatform: macos
 
   pool:
-    vmImage: xcode9-macos10.13
+    vmImage: macos-10.14
 
   steps:
   - template: ./macos-steps.yml
@@ -131,7 +131,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: vs2017-win2016
+    vmImage: windows-2019
 
   strategy:
     matrix:
diff --git a/Misc/NEWS.d/next/macOS/2019-09-10-14-24-35.bpo-38089.eedgyD.rst b/Misc/NEWS.d/next/macOS/2019-09-10-14-24-35.bpo-38089.eedgyD.rst
new file mode 100644
index 000000000000..41349848621e
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2019-09-10-14-24-35.bpo-38089.eedgyD.rst
@@ -0,0 +1 @@
+Move Azure Pipelines to latest VM versions and make macOS tests optional


From webhook-mailer at python.org  Tue Sep 10 11:54:44 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 10 Sep 2019 15:54:44 -0000
Subject: [Python-checkins] [3.7] bpo-37531: Enhance regrtest multiprocess
 timeout (GH-15345) (GH-15874)
Message-ID: <mailman.476.1568130885.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/494b61aeec8e583755c8ff3a147a58657353bbc3
commit: 494b61aeec8e583755c8ff3a147a58657353bbc3
branch: 3.7
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T17:54:40+02:00
summary:

[3.7] bpo-37531: Enhance regrtest multiprocess timeout (GH-15345) (GH-15874)

bpo-37531: Enhance regrtest multiprocess timeout (GH-15345)

* Write a message when killing a worker process
* Put a timeout on the second popen.communicate() call
  (after killing the process)
* Put a timeout on popen.wait() call
* Catch popen.kill() and popen.wait() exceptions

(cherry picked from commit de2d9eed8bc628533e1628b843cc4c7a5010f6e5)

files:
A Misc/NEWS.d/next/Tests/2019-08-20-19-24-19.bpo-37531.wRoXfU.rst
M Lib/test/libregrtest/main.py
M Lib/test/libregrtest/runtest_mp.py

diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index ef1ecef25350..7d6a62863599 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -589,7 +589,6 @@ def create_temp_dir(self):
 
     def cleanup(self):
         import glob
-        import shutil
 
         path = os.path.join(self.tmp_dir, 'test_python_*')
         print("Cleanup %s directory" % self.tmp_dir)
diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py
index eed643291162..c22479b7976f 100644
--- a/Lib/test/libregrtest/runtest_mp.py
+++ b/Lib/test/libregrtest/runtest_mp.py
@@ -126,6 +126,38 @@ def __repr__(self):
             info.append(f'pid={popen.pid}')
         return '<%s>' % ' '.join(info)
 
+    def _kill(self):
+        dt = time.monotonic() - self.start_time
+
+        popen = self._popen
+        pid = popen.pid
+        print("Kill worker process %s running for %.1f sec" % (pid, dt),
+              file=sys.stderr, flush=True)
+
+        try:
+            popen.kill()
+            return True
+        except OSError as exc:
+            print("WARNING: Failed to kill worker process %s: %r" % (pid, exc),
+                  file=sys.stderr, flush=True)
+            return False
+
+    def _close_wait(self):
+        popen = self._popen
+
+        # stdout and stderr must be closed to ensure that communicate()
+        # does not hang
+        popen.stdout.close()
+        popen.stderr.close()
+
+        try:
+            popen.wait(JOIN_TIMEOUT)
+        except (subprocess.TimeoutExpired, OSError) as exc:
+            print("WARNING: Failed to wait for worker process %s "
+                  "completion (timeout=%.1f sec): %r"
+                  % (popen.pid, JOIN_TIMEOUT, exc),
+                  file=sys.stderr, flush=True)
+
     def kill(self):
         """
         Kill the current process (if any).
@@ -135,15 +167,13 @@ def kill(self):
         """
         self._killed = True
 
-        popen = self._popen
-        if popen is None:
+        if self._popen is None:
             return
-        popen.kill()
-        # stdout and stderr must be closed to ensure that communicate()
-        # does not hang
-        popen.stdout.close()
-        popen.stderr.close()
-        popen.wait()
+
+        if not self._kill():
+            return
+
+        self._close_wait()
 
     def mp_result_error(self, test_name, error_type, stdout='', stderr='',
                         err_msg=None):
@@ -151,6 +181,23 @@ def mp_result_error(self, test_name, error_type, stdout='', stderr='',
         result = TestResult(test_name, error_type, test_time, None)
         return MultiprocessResult(result, stdout, stderr, err_msg)
 
+    def _timedout(self, test_name):
+        self._kill()
+
+        stdout = sterr = ''
+        popen = self._popen
+        try:
+            stdout, stderr = popen.communicate(timeout=JOIN_TIMEOUT)
+        except (subprocess.TimeoutExpired, OSError) as exc:
+            print("WARNING: Failed to read worker process %s output "
+                  "(timeout=%.1f sec): %r"
+                  % (popen.pid, exc, timeout),
+                  file=sys.stderr, flush=True)
+
+        self._close_wait()
+
+        return self.mp_result_error(test_name, TIMEOUT, stdout, stderr)
+
     def _runtest(self, test_name):
         try:
             self.start_time = time.monotonic()
@@ -158,7 +205,7 @@ def _runtest(self, test_name):
 
             self._popen = run_test_in_subprocess(test_name, self.ns)
             popen = self._popen
-            with popen:
+            try:
                 try:
                     if self._killed:
                         # If kill() has been called before self._popen is set,
@@ -175,12 +222,7 @@ def _runtest(self, test_name):
                             # on reading closed stdout/stderr
                             raise ExitThread
 
-                        popen.kill()
-                        stdout, stderr = popen.communicate()
-                        self.kill()
-
-                        return self.mp_result_error(test_name, TIMEOUT,
-                                                    stdout, stderr)
+                        return self._timedout(test_name)
                     except OSError:
                         if self._killed:
                             # kill() has been called: communicate() fails
@@ -190,8 +232,10 @@ def _runtest(self, test_name):
                 except:
                     self.kill()
                     raise
+            finally:
+                self._close_wait()
 
-            retcode = popen.wait()
+            retcode = popen.returncode
         finally:
             self.current_test_name = None
             self._popen = None
@@ -286,10 +330,11 @@ def wait_workers(self):
                 if not worker.is_alive():
                     break
                 dt = time.monotonic() - start_time
-                print("Wait for regrtest worker %r for %.1f sec" % (worker, dt))
+                print("Wait for regrtest worker %r for %.1f sec" % (worker, dt),
+                      flush=True)
                 if dt > JOIN_TIMEOUT:
                     print("Warning -- failed to join a regrtest worker %s"
-                          % worker)
+                          % worker, flush=True)
                     break
 
     def _get_result(self):
diff --git a/Misc/NEWS.d/next/Tests/2019-08-20-19-24-19.bpo-37531.wRoXfU.rst b/Misc/NEWS.d/next/Tests/2019-08-20-19-24-19.bpo-37531.wRoXfU.rst
new file mode 100644
index 000000000000..59500ce67a01
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-08-20-19-24-19.bpo-37531.wRoXfU.rst
@@ -0,0 +1,3 @@
+Enhance regrtest multiprocess timeout: write a message when killing a worker
+process, catch popen.kill() and popen.wait() exceptions, put a timeout on the
+second call to popen.communicate().


From webhook-mailer at python.org  Tue Sep 10 11:54:55 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 10 Sep 2019 15:54:55 -0000
Subject: [Python-checkins] bpo-37531: Enhance regrtest multiprocess timeout
 (GH-15345) (GH-15871)
Message-ID: <mailman.477.1568130897.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d42a4fdc630c54352701a466a9e512bee68b5c48
commit: d42a4fdc630c54352701a466a9e512bee68b5c48
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T17:54:51+02:00
summary:

bpo-37531: Enhance regrtest multiprocess timeout (GH-15345) (GH-15871)

* Write a message when killing a worker process
* Put a timeout on the second popen.communicate() call
  (after killing the process)
* Put a timeout on popen.wait() call
* Catch popen.kill() and popen.wait() exceptions

(cherry picked from commit de2d9eed8bc628533e1628b843cc4c7a5010f6e5)

files:
A Misc/NEWS.d/next/Tests/2019-08-20-19-24-19.bpo-37531.wRoXfU.rst
M Lib/test/libregrtest/runtest_mp.py

diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py
index eed643291162..c22479b7976f 100644
--- a/Lib/test/libregrtest/runtest_mp.py
+++ b/Lib/test/libregrtest/runtest_mp.py
@@ -126,6 +126,38 @@ def __repr__(self):
             info.append(f'pid={popen.pid}')
         return '<%s>' % ' '.join(info)
 
+    def _kill(self):
+        dt = time.monotonic() - self.start_time
+
+        popen = self._popen
+        pid = popen.pid
+        print("Kill worker process %s running for %.1f sec" % (pid, dt),
+              file=sys.stderr, flush=True)
+
+        try:
+            popen.kill()
+            return True
+        except OSError as exc:
+            print("WARNING: Failed to kill worker process %s: %r" % (pid, exc),
+                  file=sys.stderr, flush=True)
+            return False
+
+    def _close_wait(self):
+        popen = self._popen
+
+        # stdout and stderr must be closed to ensure that communicate()
+        # does not hang
+        popen.stdout.close()
+        popen.stderr.close()
+
+        try:
+            popen.wait(JOIN_TIMEOUT)
+        except (subprocess.TimeoutExpired, OSError) as exc:
+            print("WARNING: Failed to wait for worker process %s "
+                  "completion (timeout=%.1f sec): %r"
+                  % (popen.pid, JOIN_TIMEOUT, exc),
+                  file=sys.stderr, flush=True)
+
     def kill(self):
         """
         Kill the current process (if any).
@@ -135,15 +167,13 @@ def kill(self):
         """
         self._killed = True
 
-        popen = self._popen
-        if popen is None:
+        if self._popen is None:
             return
-        popen.kill()
-        # stdout and stderr must be closed to ensure that communicate()
-        # does not hang
-        popen.stdout.close()
-        popen.stderr.close()
-        popen.wait()
+
+        if not self._kill():
+            return
+
+        self._close_wait()
 
     def mp_result_error(self, test_name, error_type, stdout='', stderr='',
                         err_msg=None):
@@ -151,6 +181,23 @@ def mp_result_error(self, test_name, error_type, stdout='', stderr='',
         result = TestResult(test_name, error_type, test_time, None)
         return MultiprocessResult(result, stdout, stderr, err_msg)
 
+    def _timedout(self, test_name):
+        self._kill()
+
+        stdout = sterr = ''
+        popen = self._popen
+        try:
+            stdout, stderr = popen.communicate(timeout=JOIN_TIMEOUT)
+        except (subprocess.TimeoutExpired, OSError) as exc:
+            print("WARNING: Failed to read worker process %s output "
+                  "(timeout=%.1f sec): %r"
+                  % (popen.pid, exc, timeout),
+                  file=sys.stderr, flush=True)
+
+        self._close_wait()
+
+        return self.mp_result_error(test_name, TIMEOUT, stdout, stderr)
+
     def _runtest(self, test_name):
         try:
             self.start_time = time.monotonic()
@@ -158,7 +205,7 @@ def _runtest(self, test_name):
 
             self._popen = run_test_in_subprocess(test_name, self.ns)
             popen = self._popen
-            with popen:
+            try:
                 try:
                     if self._killed:
                         # If kill() has been called before self._popen is set,
@@ -175,12 +222,7 @@ def _runtest(self, test_name):
                             # on reading closed stdout/stderr
                             raise ExitThread
 
-                        popen.kill()
-                        stdout, stderr = popen.communicate()
-                        self.kill()
-
-                        return self.mp_result_error(test_name, TIMEOUT,
-                                                    stdout, stderr)
+                        return self._timedout(test_name)
                     except OSError:
                         if self._killed:
                             # kill() has been called: communicate() fails
@@ -190,8 +232,10 @@ def _runtest(self, test_name):
                 except:
                     self.kill()
                     raise
+            finally:
+                self._close_wait()
 
-            retcode = popen.wait()
+            retcode = popen.returncode
         finally:
             self.current_test_name = None
             self._popen = None
@@ -286,10 +330,11 @@ def wait_workers(self):
                 if not worker.is_alive():
                     break
                 dt = time.monotonic() - start_time
-                print("Wait for regrtest worker %r for %.1f sec" % (worker, dt))
+                print("Wait for regrtest worker %r for %.1f sec" % (worker, dt),
+                      flush=True)
                 if dt > JOIN_TIMEOUT:
                     print("Warning -- failed to join a regrtest worker %s"
-                          % worker)
+                          % worker, flush=True)
                     break
 
     def _get_result(self):
diff --git a/Misc/NEWS.d/next/Tests/2019-08-20-19-24-19.bpo-37531.wRoXfU.rst b/Misc/NEWS.d/next/Tests/2019-08-20-19-24-19.bpo-37531.wRoXfU.rst
new file mode 100644
index 000000000000..59500ce67a01
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-08-20-19-24-19.bpo-37531.wRoXfU.rst
@@ -0,0 +1,3 @@
+Enhance regrtest multiprocess timeout: write a message when killing a worker
+process, catch popen.kill() and popen.wait() exceptions, put a timeout on the
+second call to popen.communicate().


From webhook-mailer at python.org  Tue Sep 10 12:00:12 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 16:00:12 -0000
Subject: [Python-checkins] Correct info about "f.read(size)". (GH13852)
Message-ID: <mailman.478.1568131214.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6c588a00ed88dc621023771d5c50167423d0eea3
commit: 6c588a00ed88dc621023771d5c50167423d0eea3
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T09:00:09-07:00
summary:

Correct info about "f.read(size)". (GH13852)


In text mode, the "size" parameter indicates the number of characters, not bytes.
(cherry picked from commit faff81c05f838b0b7a64bbc8c53c02a9b04bb79d)

Co-authored-by: William Andrea <william.j.andrea at gmail.com>

files:
M Doc/tutorial/inputoutput.rst

diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst
index fc2bd5578c4c..9fe1276ec9d1 100644
--- a/Doc/tutorial/inputoutput.rst
+++ b/Doc/tutorial/inputoutput.rst
@@ -358,8 +358,8 @@ To read a file's contents, call ``f.read(size)``, which reads some quantity of
 data and returns it as a string (in text mode) or bytes object (in binary mode).
 *size* is an optional numeric argument.  When *size* is omitted or negative, the
 entire contents of the file will be read and returned; it's your problem if the
-file is twice as large as your machine's memory. Otherwise, at most *size* bytes
-are read and returned.
+file is twice as large as your machine's memory. Otherwise, at most *size*
+characters (in text mode) or *size* bytes (in binary mode) are read and returned.
 If the end of the file has been reached, ``f.read()`` will return an empty
 string (``''``).  ::
 


From webhook-mailer at python.org  Tue Sep 10 12:02:44 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 16:02:44 -0000
Subject: [Python-checkins] Correct info about "f.read(size)". (GH13852)
Message-ID: <mailman.479.1568131365.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/075a441fa0c900802e331167f617058c7d3a9607
commit: 075a441fa0c900802e331167f617058c7d3a9607
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T09:02:40-07:00
summary:

Correct info about "f.read(size)". (GH13852)


In text mode, the "size" parameter indicates the number of characters, not bytes.
(cherry picked from commit faff81c05f838b0b7a64bbc8c53c02a9b04bb79d)

Co-authored-by: William Andrea <william.j.andrea at gmail.com>

files:
M Doc/tutorial/inputoutput.rst

diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst
index 79427860f518..1dc01e82102d 100644
--- a/Doc/tutorial/inputoutput.rst
+++ b/Doc/tutorial/inputoutput.rst
@@ -356,8 +356,8 @@ To read a file's contents, call ``f.read(size)``, which reads some quantity of
 data and returns it as a string (in text mode) or bytes object (in binary mode).
 *size* is an optional numeric argument.  When *size* is omitted or negative, the
 entire contents of the file will be read and returned; it's your problem if the
-file is twice as large as your machine's memory. Otherwise, at most *size* bytes
-are read and returned.
+file is twice as large as your machine's memory. Otherwise, at most *size*
+characters (in text mode) or *size* bytes (in binary mode) are read and returned.
 If the end of the file has been reached, ``f.read()`` will return an empty
 string (``''``).  ::
 


From webhook-mailer at python.org  Tue Sep 10 12:04:12 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 16:04:12 -0000
Subject: [Python-checkins] bpo-37752: Delete redundant Py_CHARMASK in
 normalizestring() (GH-15095)
Message-ID: <mailman.480.1568131453.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2ec70102066fe5534f1a62e8f496d2005e1697db
commit: 2ec70102066fe5534f1a62e8f496d2005e1697db
branch: master
author: Jordon Xu <46997731+qigangxu at users.noreply.github.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-10T17:04:08+01:00
summary:

bpo-37752: Delete redundant Py_CHARMASK in normalizestring() (GH-15095)

files:
M Modules/_struct.c
M Modules/_tkinter.c
M Modules/unicodedata.c
M Objects/longobject.c
M Python/dynload_aix.c
M Python/getargs.c
M Python/mystrtoul.c

diff --git a/Modules/_struct.c b/Modules/_struct.c
index 5fdfdf9da63b..c387ae2ffc19 100644
--- a/Modules/_struct.c
+++ b/Modules/_struct.c
@@ -1298,7 +1298,7 @@ prepare_s(PyStructObject *self)
     len = 0;
     ncodes = 0;
     while ((c = *s++) != '\0') {
-        if (Py_ISSPACE(Py_CHARMASK(c)))
+        if (Py_ISSPACE(c))
             continue;
         if ('0' <= c && c <= '9') {
             num = c - '0';
@@ -1363,7 +1363,7 @@ prepare_s(PyStructObject *self)
     s = fmt;
     size = 0;
     while ((c = *s++) != '\0') {
-        if (Py_ISSPACE(Py_CHARMASK(c)))
+        if (Py_ISSPACE(c))
             continue;
         if ('0' <= c && c <= '9') {
             num = c - '0';
diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c
index a832099c0950..036f9b68bb23 100644
--- a/Modules/_tkinter.c
+++ b/Modules/_tkinter.c
@@ -671,8 +671,8 @@ Tkapp_New(const char *screenName, const char *className,
     }
 
     strcpy(argv0, className);
-    if (Py_ISUPPER(Py_CHARMASK(argv0[0])))
-        argv0[0] = Py_TOLOWER(Py_CHARMASK(argv0[0]));
+    if (Py_ISUPPER(argv0[0]))
+        argv0[0] = Py_TOLOWER(argv0[0]);
     Tcl_SetVar(v->interp, "argv0", argv0, TCL_GLOBAL_ONLY);
     PyMem_Free(argv0);
 
diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c
index bb09b7d21f36..e99d914b797b 100644
--- a/Modules/unicodedata.c
+++ b/Modules/unicodedata.c
@@ -987,7 +987,7 @@ _gethash(const char *s, int len, int scale)
     unsigned long h = 0;
     unsigned long ix;
     for (i = 0; i < len; i++) {
-        h = (h * scale) + (unsigned char) Py_TOUPPER(Py_CHARMASK(s[i]));
+        h = (h * scale) + (unsigned char) Py_TOUPPER(s[i]);
         ix = h & 0xff000000;
         if (ix)
             h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff;
@@ -1157,7 +1157,7 @@ _cmpname(PyObject *self, int code, const char* name, int namelen)
     if (!_getucname(self, code, buffer, NAME_MAXLEN, 1))
         return 0;
     for (i = 0; i < namelen; i++) {
-        if (Py_TOUPPER(Py_CHARMASK(name[i])) != buffer[i])
+        if (Py_TOUPPER(name[i]) != buffer[i])
             return 0;
     }
     return buffer[namelen] == '\0';
diff --git a/Objects/longobject.c b/Objects/longobject.c
index 4cf2b0726e81..ed25e0bee329 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -2300,7 +2300,7 @@ PyLong_FromString(const char *str, char **pend, int base)
                         "int() arg 2 must be >= 2 and <= 36");
         return NULL;
     }
-    while (*str != '\0' && Py_ISSPACE(Py_CHARMASK(*str))) {
+    while (*str != '\0' && Py_ISSPACE(*str)) {
         str++;
     }
     if (*str == '+') {
@@ -2609,7 +2609,7 @@ digit beyond the first.
     if (sign < 0) {
         Py_SIZE(z) = -(Py_SIZE(z));
     }
-    while (*str && Py_ISSPACE(Py_CHARMASK(*str))) {
+    while (*str && Py_ISSPACE(*str)) {
         str++;
     }
     if (*str != '\0') {
diff --git a/Python/dynload_aix.c b/Python/dynload_aix.c
index b3ff8e288c1a..684f10a8b919 100644
--- a/Python/dynload_aix.c
+++ b/Python/dynload_aix.c
@@ -140,7 +140,7 @@ aix_loaderror(const char *pathname)
             if (nerr == load_errtab[j].errNo && load_errtab[j].errstr)
             ERRBUF_APPEND(load_errtab[j].errstr);
         }
-        while (Py_ISDIGIT(Py_CHARMASK(*message[i]))) message[i]++ ;
+        while (Py_ISDIGIT(*message[i])) message[i]++ ;
         ERRBUF_APPEND(message[i]);
         ERRBUF_APPEND("\n");
     }
diff --git a/Python/getargs.c b/Python/getargs.c
index 02a0366ece46..7723ae353710 100644
--- a/Python/getargs.c
+++ b/Python/getargs.c
@@ -312,7 +312,7 @@ vgetargs1_impl(PyObject *compat_args, PyObject *const *stack, Py_ssize_t nargs,
             break;
         default:
             if (level == 0) {
-                if (Py_ISALPHA(Py_CHARMASK(c)))
+                if (Py_ISALPHA(c))
                     if (c != 'e') /* skip encoded */
                         max++;
             }
@@ -397,7 +397,7 @@ vgetargs1_impl(PyObject *compat_args, PyObject *const *stack, Py_ssize_t nargs,
         }
     }
 
-    if (*format != '\0' && !Py_ISALPHA(Py_CHARMASK(*format)) &&
+    if (*format != '\0' && !Py_ISALPHA(*format) &&
         *format != '(' &&
         *format != '|' && *format != ':' && *format != ';') {
         PyErr_Format(PyExc_SystemError,
@@ -521,7 +521,7 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
         }
         else if (c == ':' || c == ';' || c == '\0')
             break;
-        else if (level == 0 && Py_ISALPHA(Py_CHARMASK(c)))
+        else if (level == 0 && Py_ISALPHA(c))
             n++;
     }
 
diff --git a/Python/mystrtoul.c b/Python/mystrtoul.c
index 7ab5814272d5..19fa57aa144e 100644
--- a/Python/mystrtoul.c
+++ b/Python/mystrtoul.c
@@ -99,7 +99,7 @@ PyOS_strtoul(const char *str, char **ptr, int base)
     int ovlimit;       /* required digits to overflow */
 
     /* skip leading white space */
-    while (*str && Py_ISSPACE(Py_CHARMASK(*str)))
+    while (*str && Py_ISSPACE(*str))
         ++str;
 
     /* check for leading 0b, 0o or 0x for auto-base or base 16 */
@@ -138,7 +138,7 @@ PyOS_strtoul(const char *str, char **ptr, int base)
                 /* skip all zeroes... */
                 while (*str == '0')
                     ++str;
-                while (Py_ISSPACE(Py_CHARMASK(*str)))
+                while (Py_ISSPACE(*str))
                     ++str;
                 if (ptr)
                     *ptr = (char *)str;
@@ -266,7 +266,7 @@ PyOS_strtol(const char *str, char **ptr, int base)
     unsigned long uresult;
     char sign;
 
-    while (*str && Py_ISSPACE(Py_CHARMASK(*str)))
+    while (*str && Py_ISSPACE(*str))
         str++;
 
     sign = *str;


From webhook-mailer at python.org  Tue Sep 10 12:06:29 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 16:06:29 -0000
Subject: [Python-checkins] bpo-37574: Mention helper functions for find_spec
 documentation (GH-14739)
Message-ID: <mailman.481.1568131590.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9cbb97b29eac4b23e916a3233f26b60ac69e335b
commit: 9cbb97b29eac4b23e916a3233f26b60ac69e335b
branch: master
author: jdkandersson <51036209+jdkandersson at users.noreply.github.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-10T17:06:22+01:00
summary:

bpo-37574: Mention helper functions for find_spec documentation (GH-14739)

files:
M Doc/library/importlib.rst

diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
index 0976360c6965..54ca8b8b50a4 100644
--- a/Doc/library/importlib.rst
+++ b/Doc/library/importlib.rst
@@ -274,6 +274,8 @@ ABC hierarchy::
       parent package. If a spec cannot be found, ``None`` is returned.
       When passed in, ``target`` is a module object that the finder may
       use to make a more educated guess about what spec to return.
+      :func:`importlib.util.spec_from_loader` may be useful for implementing
+      concrete ``MetaPathFinders``.
 
       .. versionadded:: 3.4
 
@@ -323,7 +325,8 @@ ABC hierarchy::
       within the :term:`path entry` to which it is assigned.  If a spec
       cannot be found, ``None`` is returned.  When passed in, ``target``
       is a module object that the finder may use to make a more educated
-      guess about what spec to return.
+      guess about what spec to return. :func:`importlib.util.spec_from_loader`
+      may be useful for implementing concrete ``PathEntryFinders``.
 
       .. versionadded:: 3.4
 


From webhook-mailer at python.org  Tue Sep 10 12:11:14 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Tue, 10 Sep 2019 16:11:14 -0000
Subject: [Python-checkins] Remove unneeded assignment in PyBytes_Concat()
 (GH-15274)
Message-ID: <mailman.482.1568131876.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/afdeb189e97033b54cef44a7490d89d2013cb4e5
commit: afdeb189e97033b54cef44a7490d89d2013cb4e5
branch: master
author: Sergey Fedoseev <fedoseev.sergey at gmail.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-10T17:11:10+01:00
summary:

Remove unneeded assignment in PyBytes_Concat() (GH-15274)

The `wb.len = -1` assignment is unneeded since its introduction in 161d695fb0455ce52530d4f43a9eac4c738f64bb as `PyObject_GetBuffer` always fills it in.

files:
M Objects/bytesobject.c

diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
index 72f9cd271be0..e1f5ee2f62f3 100644
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -2935,7 +2935,6 @@ PyBytes_Concat(PyObject **pv, PyObject *w)
         Py_ssize_t oldsize;
         Py_buffer wb;
 
-        wb.len = -1;
         if (PyObject_GetBuffer(w, &wb, PyBUF_SIMPLE) != 0) {
             PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s",
                          Py_TYPE(w)->tp_name, Py_TYPE(*pv)->tp_name);


From webhook-mailer at python.org  Tue Sep 10 12:14:15 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Tue, 10 Sep 2019 16:14:15 -0000
Subject: [Python-checkins] bpo-28494: Test existing zipfile working
 behavior. (GH-15853)
Message-ID: <mailman.483.1568132056.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3f4db4a0bab073b768fae958e93288bd5d24eadd
commit: 3f4db4a0bab073b768fae958e93288bd5d24eadd
branch: master
author: Gregory P. Smith <greg at krypto.org>
committer: T. Wouters <thomas at python.org>
date: 2019-09-10T09:14:11-07:00
summary:

bpo-28494: Test existing zipfile working behavior. (GH-15853)

Add unittests for executables with a zipfile appended to test_zipfile, as zipfile.is_zipfile and zipfile.ZipFile work properly on these today.

files:
A Lib/test/ziptestdata/README.md
A Lib/test/ziptestdata/exe_with_z64
A Lib/test/ziptestdata/exe_with_zip
A Lib/test/ziptestdata/header.sh
A Lib/test/ziptestdata/testdata_module_inside_zip.py
M Lib/test/test_zipfile.py

diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
index f9ee740eb6fa..99d599eac79a 100644
--- a/Lib/test/test_zipfile.py
+++ b/Lib/test/test_zipfile.py
@@ -5,6 +5,8 @@
 import pathlib
 import posixpath
 import struct
+import subprocess
+import sys
 import time
 import unittest
 import zipfile
@@ -2470,6 +2472,44 @@ def build_alpharep_fixture():
     return zf
 
 
+class TestExecutablePrependedZip(unittest.TestCase):
+    """Test our ability to open zip files with an executable prepended."""
+
+    def setUp(self):
+        self.exe_zip = findfile('exe_with_zip', subdir='ziptestdata')
+        self.exe_zip64 = findfile('exe_with_z64', subdir='ziptestdata')
+
+    def _test_zip_works(self, name):
+        # bpo-28494 sanity check: ensure is_zipfile works on these.
+        self.assertTrue(zipfile.is_zipfile(name),
+                        f'is_zipfile failed on {name}')
+        # Ensure we can operate on these via ZipFile.
+        with zipfile.ZipFile(name) as zipfp:
+            for n in zipfp.namelist():
+                data = zipfp.read(n)
+                self.assertIn(b'FAVORITE_NUMBER', data)
+
+    def test_read_zip_with_exe_prepended(self):
+        self._test_zip_works(self.exe_zip)
+
+    def test_read_zip64_with_exe_prepended(self):
+        self._test_zip_works(self.exe_zip64)
+
+    @unittest.skipUnless(sys.executable, 'sys.executable required.')
+    @unittest.skipUnless(os.access('/bin/bash', os.X_OK),
+                         'Test relies on #!/bin/bash working.')
+    def test_execute_zip2(self):
+        output = subprocess.check_output([self.exe_zip, sys.executable])
+        self.assertIn(b'number in executable: 5', output)
+
+    @unittest.skipUnless(sys.executable, 'sys.executable required.')
+    @unittest.skipUnless(os.access('/bin/bash', os.X_OK),
+                         'Test relies on #!/bin/bash working.')
+    def test_execute_zip64(self):
+        output = subprocess.check_output([self.exe_zip64, sys.executable])
+        self.assertIn(b'number in executable: 5', output)
+
+
 class TestPath(unittest.TestCase):
     def setUp(self):
         self.fixtures = contextlib.ExitStack()
diff --git a/Lib/test/ziptestdata/README.md b/Lib/test/ziptestdata/README.md
new file mode 100644
index 000000000000..6b9147db76e1
--- /dev/null
+++ b/Lib/test/ziptestdata/README.md
@@ -0,0 +1,35 @@
+# Test data for `test_zipfile`
+
+The test executables in this directory are created manually from header.sh and
+the `testdata_module_inside_zip.py` file.  You must have infozip's zip utility
+installed (`apt install zip` on Debian).
+
+## Purpose
+
+These are used to test executable files with an appended zipfile, in a scenario
+where the executable is _not_ a Python interpreter itself so our automatic
+zipimport machinery (that'd look for `__main__.py`) is not being used.
+
+## Updating the test executables
+
+If you update header.sh or the testdata_module_inside_zip.py file, rerun the
+commands below.  These are expected to be rarely changed, if ever.
+
+### Standard old format (2.0) zip file
+
+```
+zip -0 zip2.zip testdata_module_inside_zip.py
+cat header.sh zip2.zip >exe_with_zip
+rm zip2.zip
+```
+
+### Modern format (4.5) zip64 file
+
+Redirecting from stdin forces infozip's zip tool to create a zip64.
+
+```
+zip -0 <testdata_module_inside_zip.py >zip64.zip
+cat header.sh zip64.zip >exe_with_z64
+rm zip64.zip
+```
+
diff --git a/Lib/test/ziptestdata/exe_with_z64 b/Lib/test/ziptestdata/exe_with_z64
new file mode 100755
index 000000000000..82b03cf39d91
Binary files /dev/null and b/Lib/test/ziptestdata/exe_with_z64 differ
diff --git a/Lib/test/ziptestdata/exe_with_zip b/Lib/test/ziptestdata/exe_with_zip
new file mode 100755
index 000000000000..c833cdf9f934
Binary files /dev/null and b/Lib/test/ziptestdata/exe_with_zip differ
diff --git a/Lib/test/ziptestdata/header.sh b/Lib/test/ziptestdata/header.sh
new file mode 100755
index 000000000000..52dc91acf740
--- /dev/null
+++ b/Lib/test/ziptestdata/header.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+INTERPRETER_UNDER_TEST="$1"
+if [[ ! -x "${INTERPRETER_UNDER_TEST}" ]]; then
+    echo "Interpreter must be the command line argument."
+    exit 4
+fi
+EXECUTABLE="$0" exec "${INTERPRETER_UNDER_TEST}" -E - <<END_OF_PYTHON
+import os
+import zipfile
+
+namespace = {}
+
+filename = os.environ['EXECUTABLE']
+print(f'Opening {filename} as a zipfile.')
+with zipfile.ZipFile(filename, mode='r') as exe_zip:
+  for file_info in exe_zip.infolist():
+    data = exe_zip.read(file_info)
+    exec(data, namespace, namespace)
+    break  # Only use the first file in the archive.
+
+print('Favorite number in executable:', namespace["FAVORITE_NUMBER"])
+
+### Archive contents will be appended after this file. ###
+END_OF_PYTHON
diff --git a/Lib/test/ziptestdata/testdata_module_inside_zip.py b/Lib/test/ziptestdata/testdata_module_inside_zip.py
new file mode 100644
index 000000000000..6f8dcd6c0e84
--- /dev/null
+++ b/Lib/test/ziptestdata/testdata_module_inside_zip.py
@@ -0,0 +1,2 @@
+# Test data file to be stored within a zip file.
+FAVORITE_NUMBER = 5


From webhook-mailer at python.org  Tue Sep 10 12:20:20 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 16:20:20 -0000
Subject: [Python-checkins] bpo-38089: Move Azure Pipelines to latest VM
 versions and make macOS tests optional (GH-15851)
Message-ID: <mailman.484.1568132420.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/cdce233f61349d029640b2dd2c0403e502050e39
commit: cdce233f61349d029640b2dd2c0403e502050e39
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-10T17:20:16+01:00
summary:

bpo-38089: Move Azure Pipelines to latest VM versions and make macOS tests optional (GH-15851)

(cherry picked from commit 801f925998cc393260f36f5ac77369fef2373ad1)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/macOS/2019-09-10-14-24-35.bpo-38089.eedgyD.rst
M .azure-pipelines/ci.yml
M .azure-pipelines/macos-steps.yml
M .azure-pipelines/pr.yml

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index fcfac85ed9c4..12ff7d4c55c9 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -42,7 +42,7 @@ jobs:
     testRunPlatform: macos
 
   pool:
-    vmImage: xcode9-macos10.13
+    vmImage: macos-10.14
 
   steps:
   - template: ./macos-steps.yml
@@ -131,7 +131,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: vs2017-win2016
+    vmImage: windows-2019
 
   strategy:
     matrix:
diff --git a/.azure-pipelines/macos-steps.yml b/.azure-pipelines/macos-steps.yml
index 647081689454..d2ca580a93d7 100644
--- a/.azure-pipelines/macos-steps.yml
+++ b/.azure-pipelines/macos-steps.yml
@@ -14,6 +14,8 @@ steps:
 
 - script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
   displayName: 'Tests'
+  continueOnError: true
+  timeoutInMinutes: 30
 
 - task: PublishTestResults at 2
   displayName: 'Publish Test Results'
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
index 2486f88a63fb..e3e24ae2a4b0 100644
--- a/.azure-pipelines/pr.yml
+++ b/.azure-pipelines/pr.yml
@@ -40,7 +40,7 @@ jobs:
     testRunPlatform: macos
 
   pool:
-    vmImage: xcode9-macos10.13
+    vmImage: macos-10.14
 
   steps:
   - template: ./macos-steps.yml
@@ -131,7 +131,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: vs2017-win2016
+    vmImage: windows-2019
 
   strategy:
     matrix:
diff --git a/Misc/NEWS.d/next/macOS/2019-09-10-14-24-35.bpo-38089.eedgyD.rst b/Misc/NEWS.d/next/macOS/2019-09-10-14-24-35.bpo-38089.eedgyD.rst
new file mode 100644
index 000000000000..41349848621e
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2019-09-10-14-24-35.bpo-38089.eedgyD.rst
@@ -0,0 +1 @@
+Move Azure Pipelines to latest VM versions and make macOS tests optional


From webhook-mailer at python.org  Tue Sep 10 12:24:10 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 16:24:10 -0000
Subject: [Python-checkins] Correct minor grammatical mistake in open docs
 (GH-15865)
Message-ID: <mailman.485.1568132651.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/30a8fd7e73802e2df94829b004d45ff18b9abc91
commit: 30a8fd7e73802e2df94829b004d45ff18b9abc91
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T09:24:06-07:00
summary:

Correct minor grammatical mistake in open docs (GH-15865)

(cherry picked from commit 05184515f924c3880067aad87940dd466f751585)

Co-authored-by: Andre Delfino <adelfino at gmail.com>

files:
M Doc/library/functions.rst

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index a4097b0b30ec..9ea7b7b55a97 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -1072,8 +1072,8 @@ are always available.  They are listed here in alphabetical order.
    ========= ===============================================================
 
    The default mode is ``'r'`` (open for reading text, synonym of ``'rt'``).
-   Modes ``'w+'`` and ``'w+b'`` opens and truncates the file.  Modes ``'r+'``
-   and ``'r+b'`` opens the file with no truncation.
+   Modes ``'w+'`` and ``'w+b'`` open and truncate the file.  Modes ``'r+'``
+   and ``'r+b'`` open the file with no truncation.
 
    As mentioned in the :ref:`io-overview`, Python distinguishes between binary
    and text I/O.  Files opened in binary mode (including ``'b'`` in the *mode*


From webhook-mailer at python.org  Tue Sep 10 12:38:11 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 16:38:11 -0000
Subject: [Python-checkins] Docs: Small tweaks to c-api/introGH-Include_Files
 (GH-14698)
Message-ID: <mailman.486.1568133492.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/78c3949407580593f92968ea268630c36ebe6634
commit: 78c3949407580593f92968ea268630c36ebe6634
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T09:38:07-07:00
summary:

Docs: Small tweaks to c-api/introGH-Include_Files (GH-14698)

(cherry picked from commit b6dafe51399f5c6313a00529118a6052b466942f)

Co-authored-by: Kyle Stanley <aeros167 at gmail.com>

files:
M Doc/c-api/intro.rst

diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst
index a1c8d34a7ea0..80eebd89ad3f 100644
--- a/Doc/c-api/intro.rst
+++ b/Doc/c-api/intro.rst
@@ -69,10 +69,12 @@ standard headers) have one of the prefixes ``Py`` or ``_Py``.  Names beginning
 with ``_Py`` are for internal use by the Python implementation and should not be
 used by extension writers. Structure member names do not have a reserved prefix.
 
-**Important:** user code should never define names that begin with ``Py`` or
-``_Py``.  This confuses the reader, and jeopardizes the portability of the user
-code to future Python versions, which may define additional names beginning with
-one of these prefixes.
+.. note::
+
+   User code should never define names that begin with ``Py`` or ``_Py``. This
+   confuses the reader, and jeopardizes the portability of the user code to
+   future Python versions, which may define additional names beginning with one
+   of these prefixes.
 
 The header files are typically installed with Python.  On Unix, these  are
 located in the directories :file:`{prefix}/include/pythonversion/` and
@@ -90,9 +92,9 @@ multi-platform builds since the platform independent headers under
 :envvar:`prefix` include the platform specific headers from
 :envvar:`exec_prefix`.
 
-C++ users should note that though the API is defined entirely using C, the
-header files do properly declare the entry points to be ``extern "C"``, so there
-is no need to do anything special to use the API from C++.
+C++ users should note that although the API is defined entirely using C, the
+header files properly declare the entry points to be ``extern "C"``. As a result,
+there is no need to do anything special to use the API from C++.
 
 
 Useful macros


From webhook-mailer at python.org  Tue Sep 10 12:40:36 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 16:40:36 -0000
Subject: [Python-checkins] Docs: Small tweaks to c-api/introGH-Include_Files
 (GH-14698)
Message-ID: <mailman.487.1568133637.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/14514b6e71b6f2c26dc95f45f24e748e72052292
commit: 14514b6e71b6f2c26dc95f45f24e748e72052292
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T09:40:32-07:00
summary:

Docs: Small tweaks to c-api/introGH-Include_Files (GH-14698)

(cherry picked from commit b6dafe51399f5c6313a00529118a6052b466942f)

Co-authored-by: Kyle Stanley <aeros167 at gmail.com>

files:
M Doc/c-api/intro.rst

diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst
index 330871bc2ae3..964d348c2df1 100644
--- a/Doc/c-api/intro.rst
+++ b/Doc/c-api/intro.rst
@@ -69,10 +69,12 @@ standard headers) have one of the prefixes ``Py`` or ``_Py``.  Names beginning
 with ``_Py`` are for internal use by the Python implementation and should not be
 used by extension writers. Structure member names do not have a reserved prefix.
 
-**Important:** user code should never define names that begin with ``Py`` or
-``_Py``.  This confuses the reader, and jeopardizes the portability of the user
-code to future Python versions, which may define additional names beginning with
-one of these prefixes.
+.. note::
+
+   User code should never define names that begin with ``Py`` or ``_Py``. This
+   confuses the reader, and jeopardizes the portability of the user code to
+   future Python versions, which may define additional names beginning with one
+   of these prefixes.
 
 The header files are typically installed with Python.  On Unix, these  are
 located in the directories :file:`{prefix}/include/pythonversion/` and
@@ -90,9 +92,9 @@ multi-platform builds since the platform independent headers under
 :envvar:`prefix` include the platform specific headers from
 :envvar:`exec_prefix`.
 
-C++ users should note that though the API is defined entirely using C, the
-header files do properly declare the entry points to be ``extern "C"``, so there
-is no need to do anything special to use the API from C++.
+C++ users should note that although the API is defined entirely using C, the
+header files properly declare the entry points to be ``extern "C"``. As a result,
+there is no need to do anything special to use the API from C++.
 
 
 Useful macros


From webhook-mailer at python.org  Tue Sep 10 13:06:42 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Tue, 10 Sep 2019 17:06:42 -0000
Subject: [Python-checkins] bpo-38089: Move Azure Pipelines to latest VM
 versions and make macOS tests optional (GH-15851)
Message-ID: <mailman.488.1568135203.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6a494e77fc20543dcc33c1299b46ee6433d80019
commit: 6a494e77fc20543dcc33c1299b46ee6433d80019
branch: 3.7
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-10T18:06:38+01:00
summary:

bpo-38089: Move Azure Pipelines to latest VM versions and make macOS tests optional (GH-15851)

files:
A Misc/NEWS.d/next/macOS/2019-09-10-14-24-35.bpo-38089.eedgyD.rst
M .azure-pipelines/ci.yml
M .azure-pipelines/macos-steps.yml
M .azure-pipelines/pr.yml

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index fcfac85ed9c4..9d8da910acda 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -42,7 +42,7 @@ jobs:
     testRunPlatform: macos
 
   pool:
-    vmImage: xcode9-macos10.13
+    vmImage: macos-10.14
 
   steps:
   - template: ./macos-steps.yml
diff --git a/.azure-pipelines/macos-steps.yml b/.azure-pipelines/macos-steps.yml
index 647081689454..d2ca580a93d7 100644
--- a/.azure-pipelines/macos-steps.yml
+++ b/.azure-pipelines/macos-steps.yml
@@ -14,6 +14,8 @@ steps:
 
 - script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
   displayName: 'Tests'
+  continueOnError: true
+  timeoutInMinutes: 30
 
 - task: PublishTestResults at 2
   displayName: 'Publish Test Results'
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
index 2486f88a63fb..bafed733a75d 100644
--- a/.azure-pipelines/pr.yml
+++ b/.azure-pipelines/pr.yml
@@ -40,7 +40,7 @@ jobs:
     testRunPlatform: macos
 
   pool:
-    vmImage: xcode9-macos10.13
+    vmImage: macos-10.14
 
   steps:
   - template: ./macos-steps.yml
diff --git a/Misc/NEWS.d/next/macOS/2019-09-10-14-24-35.bpo-38089.eedgyD.rst b/Misc/NEWS.d/next/macOS/2019-09-10-14-24-35.bpo-38089.eedgyD.rst
new file mode 100644
index 000000000000..41349848621e
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2019-09-10-14-24-35.bpo-38089.eedgyD.rst
@@ -0,0 +1 @@
+Move Azure Pipelines to latest VM versions and make macOS tests optional


From webhook-mailer at python.org  Tue Sep 10 13:13:59 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 17:13:59 -0000
Subject: [Python-checkins] bpo-37574: Mention helper functions for find_spec
 documentation (GH-14739)
Message-ID: <mailman.489.1568135640.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/63909cdc2ff4eec58cc2b56426f2c725ccd4ba2b
commit: 63909cdc2ff4eec58cc2b56426f2c725ccd4ba2b
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T10:13:55-07:00
summary:

bpo-37574: Mention helper functions for find_spec documentation (GH-14739)

(cherry picked from commit 9cbb97b29eac4b23e916a3233f26b60ac69e335b)

Co-authored-by: jdkandersson <51036209+jdkandersson at users.noreply.github.com>

files:
M Doc/library/importlib.rst

diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
index cb8360caca5e..47e3dd03a687 100644
--- a/Doc/library/importlib.rst
+++ b/Doc/library/importlib.rst
@@ -274,6 +274,8 @@ ABC hierarchy::
       parent package. If a spec cannot be found, ``None`` is returned.
       When passed in, ``target`` is a module object that the finder may
       use to make a more educated guess about what spec to return.
+      :func:`importlib.util.spec_from_loader` may be useful for implementing
+      concrete ``MetaPathFinders``.
 
       .. versionadded:: 3.4
 
@@ -323,7 +325,8 @@ ABC hierarchy::
       within the :term:`path entry` to which it is assigned.  If a spec
       cannot be found, ``None`` is returned.  When passed in, ``target``
       is a module object that the finder may use to make a more educated
-      guess about what spec to return.
+      guess about what spec to return. :func:`importlib.util.spec_from_loader`
+      may be useful for implementing concrete ``PathEntryFinders``.
 
       .. versionadded:: 3.4
 


From webhook-mailer at python.org  Tue Sep 10 13:16:07 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 17:16:07 -0000
Subject: [Python-checkins] bpo-37574: Mention helper functions for find_spec
 documentation (GH-14739)
Message-ID: <mailman.490.1568135768.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e0a1561f93545f38034d6c48f21e79acfd4db04b
commit: e0a1561f93545f38034d6c48f21e79acfd4db04b
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T10:16:03-07:00
summary:

bpo-37574: Mention helper functions for find_spec documentation (GH-14739)

(cherry picked from commit 9cbb97b29eac4b23e916a3233f26b60ac69e335b)

Co-authored-by: jdkandersson <51036209+jdkandersson at users.noreply.github.com>

files:
M Doc/library/importlib.rst

diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
index df184b33d0e7..974dbb461a15 100644
--- a/Doc/library/importlib.rst
+++ b/Doc/library/importlib.rst
@@ -274,6 +274,8 @@ ABC hierarchy::
       parent package. If a spec cannot be found, ``None`` is returned.
       When passed in, ``target`` is a module object that the finder may
       use to make a more educated guess about what spec to return.
+      :func:`importlib.util.spec_from_loader` may be useful for implementing
+      concrete ``MetaPathFinders``.
 
       .. versionadded:: 3.4
 
@@ -323,7 +325,8 @@ ABC hierarchy::
       within the :term:`path entry` to which it is assigned.  If a spec
       cannot be found, ``None`` is returned.  When passed in, ``target``
       is a module object that the finder may use to make a more educated
-      guess about what spec to return.
+      guess about what spec to return. :func:`importlib.util.spec_from_loader`
+      may be useful for implementing concrete ``PathEntryFinders``.
 
       .. versionadded:: 3.4
 


From webhook-mailer at python.org  Tue Sep 10 13:46:58 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 17:46:58 -0000
Subject: [Python-checkins] Remove unneeded assignment in PyBytes_Concat()
 (GH-15274)
Message-ID: <mailman.491.1568137619.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/98224d24d1d69383ccc6336e87e362f986c99169
commit: 98224d24d1d69383ccc6336e87e362f986c99169
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T10:46:52-07:00
summary:

Remove unneeded assignment in PyBytes_Concat() (GH-15274)


The `wb.len = -1` assignment is unneeded since its introduction in 161d695fb0455ce52530d4f43a9eac4c738f64bb as `PyObject_GetBuffer` always fills it in.
(cherry picked from commit afdeb189e97033b54cef44a7490d89d2013cb4e5)

Co-authored-by: Sergey Fedoseev <fedoseev.sergey at gmail.com>

files:
M Objects/bytesobject.c

diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
index 06c87b0c62d3..9358e5effd56 100644
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -2939,7 +2939,6 @@ PyBytes_Concat(PyObject **pv, PyObject *w)
         Py_ssize_t oldsize;
         Py_buffer wb;
 
-        wb.len = -1;
         if (PyObject_GetBuffer(w, &wb, PyBUF_SIMPLE) != 0) {
             PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s",
                          Py_TYPE(w)->tp_name, Py_TYPE(*pv)->tp_name);


From webhook-mailer at python.org  Tue Sep 10 18:58:00 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 10 Sep 2019 22:58:00 -0000
Subject: [Python-checkins] bpo-28494: Test existing zipfile working
 behavior. (GH-15853)
Message-ID: <mailman.492.1568156281.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/74b0291b03db60dd244d31e9c97407cccb8d30dd
commit: 74b0291b03db60dd244d31e9c97407cccb8d30dd
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-10T15:57:54-07:00
summary:

bpo-28494: Test existing zipfile working behavior. (GH-15853)


Add unittests for executables with a zipfile appended to test_zipfile, as zipfile.is_zipfile and zipfile.ZipFile work properly on these today.
(cherry picked from commit 3f4db4a0bab073b768fae958e93288bd5d24eadd)

Co-authored-by: Gregory P. Smith <greg at krypto.org>

files:
A Lib/test/ziptestdata/README.md
A Lib/test/ziptestdata/exe_with_z64
A Lib/test/ziptestdata/exe_with_zip
A Lib/test/ziptestdata/header.sh
A Lib/test/ziptestdata/testdata_module_inside_zip.py
M Lib/test/test_zipfile.py

diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
index 8e437e5cb2b9..955e540f96e6 100644
--- a/Lib/test/test_zipfile.py
+++ b/Lib/test/test_zipfile.py
@@ -5,6 +5,8 @@
 import pathlib
 import posixpath
 import struct
+import subprocess
+import sys
 import time
 import unittest
 import zipfile
@@ -2443,6 +2445,44 @@ def build_alpharep_fixture():
     return zf
 
 
+class TestExecutablePrependedZip(unittest.TestCase):
+    """Test our ability to open zip files with an executable prepended."""
+
+    def setUp(self):
+        self.exe_zip = findfile('exe_with_zip', subdir='ziptestdata')
+        self.exe_zip64 = findfile('exe_with_z64', subdir='ziptestdata')
+
+    def _test_zip_works(self, name):
+        # bpo-28494 sanity check: ensure is_zipfile works on these.
+        self.assertTrue(zipfile.is_zipfile(name),
+                        f'is_zipfile failed on {name}')
+        # Ensure we can operate on these via ZipFile.
+        with zipfile.ZipFile(name) as zipfp:
+            for n in zipfp.namelist():
+                data = zipfp.read(n)
+                self.assertIn(b'FAVORITE_NUMBER', data)
+
+    def test_read_zip_with_exe_prepended(self):
+        self._test_zip_works(self.exe_zip)
+
+    def test_read_zip64_with_exe_prepended(self):
+        self._test_zip_works(self.exe_zip64)
+
+    @unittest.skipUnless(sys.executable, 'sys.executable required.')
+    @unittest.skipUnless(os.access('/bin/bash', os.X_OK),
+                         'Test relies on #!/bin/bash working.')
+    def test_execute_zip2(self):
+        output = subprocess.check_output([self.exe_zip, sys.executable])
+        self.assertIn(b'number in executable: 5', output)
+
+    @unittest.skipUnless(sys.executable, 'sys.executable required.')
+    @unittest.skipUnless(os.access('/bin/bash', os.X_OK),
+                         'Test relies on #!/bin/bash working.')
+    def test_execute_zip64(self):
+        output = subprocess.check_output([self.exe_zip64, sys.executable])
+        self.assertIn(b'number in executable: 5', output)
+
+
 class TestPath(unittest.TestCase):
     def setUp(self):
         self.fixtures = contextlib.ExitStack()
diff --git a/Lib/test/ziptestdata/README.md b/Lib/test/ziptestdata/README.md
new file mode 100644
index 000000000000..6b9147db76e1
--- /dev/null
+++ b/Lib/test/ziptestdata/README.md
@@ -0,0 +1,35 @@
+# Test data for `test_zipfile`
+
+The test executables in this directory are created manually from header.sh and
+the `testdata_module_inside_zip.py` file.  You must have infozip's zip utility
+installed (`apt install zip` on Debian).
+
+## Purpose
+
+These are used to test executable files with an appended zipfile, in a scenario
+where the executable is _not_ a Python interpreter itself so our automatic
+zipimport machinery (that'd look for `__main__.py`) is not being used.
+
+## Updating the test executables
+
+If you update header.sh or the testdata_module_inside_zip.py file, rerun the
+commands below.  These are expected to be rarely changed, if ever.
+
+### Standard old format (2.0) zip file
+
+```
+zip -0 zip2.zip testdata_module_inside_zip.py
+cat header.sh zip2.zip >exe_with_zip
+rm zip2.zip
+```
+
+### Modern format (4.5) zip64 file
+
+Redirecting from stdin forces infozip's zip tool to create a zip64.
+
+```
+zip -0 <testdata_module_inside_zip.py >zip64.zip
+cat header.sh zip64.zip >exe_with_z64
+rm zip64.zip
+```
+
diff --git a/Lib/test/ziptestdata/exe_with_z64 b/Lib/test/ziptestdata/exe_with_z64
new file mode 100755
index 000000000000..82b03cf39d91
Binary files /dev/null and b/Lib/test/ziptestdata/exe_with_z64 differ
diff --git a/Lib/test/ziptestdata/exe_with_zip b/Lib/test/ziptestdata/exe_with_zip
new file mode 100755
index 000000000000..c833cdf9f934
Binary files /dev/null and b/Lib/test/ziptestdata/exe_with_zip differ
diff --git a/Lib/test/ziptestdata/header.sh b/Lib/test/ziptestdata/header.sh
new file mode 100755
index 000000000000..52dc91acf740
--- /dev/null
+++ b/Lib/test/ziptestdata/header.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+INTERPRETER_UNDER_TEST="$1"
+if [[ ! -x "${INTERPRETER_UNDER_TEST}" ]]; then
+    echo "Interpreter must be the command line argument."
+    exit 4
+fi
+EXECUTABLE="$0" exec "${INTERPRETER_UNDER_TEST}" -E - <<END_OF_PYTHON
+import os
+import zipfile
+
+namespace = {}
+
+filename = os.environ['EXECUTABLE']
+print(f'Opening {filename} as a zipfile.')
+with zipfile.ZipFile(filename, mode='r') as exe_zip:
+  for file_info in exe_zip.infolist():
+    data = exe_zip.read(file_info)
+    exec(data, namespace, namespace)
+    break  # Only use the first file in the archive.
+
+print('Favorite number in executable:', namespace["FAVORITE_NUMBER"])
+
+### Archive contents will be appended after this file. ###
+END_OF_PYTHON
diff --git a/Lib/test/ziptestdata/testdata_module_inside_zip.py b/Lib/test/ziptestdata/testdata_module_inside_zip.py
new file mode 100644
index 000000000000..6f8dcd6c0e84
--- /dev/null
+++ b/Lib/test/ziptestdata/testdata_module_inside_zip.py
@@ -0,0 +1,2 @@
+# Test data file to be stored within a zip file.
+FAVORITE_NUMBER = 5


From webhook-mailer at python.org  Wed Sep 11 02:44:43 2019
From: webhook-mailer at python.org (Carol Willing)
Date: Wed, 11 Sep 2019 06:44:43 -0000
Subject: [Python-checkins] bpo-38034: Fix typo in logging.handlers.rst
 (GH-15708)
Message-ID: <mailman.493.1568184284.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/efd5741ae953e50a6654e04cf731da86a1307296
commit: efd5741ae953e50a6654e04cf731da86a1307296
branch: master
author: wwuck <301402+wwuck at users.noreply.github.com>
committer: Carol Willing <carolcode at willingconsulting.com>
date: 2019-09-11T07:44:37+01:00
summary:

bpo-38034: Fix typo in logging.handlers.rst (GH-15708)

files:
M Doc/library/logging.handlers.rst

diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst
index b7445a135b74..eccd0ae41da7 100644
--- a/Doc/library/logging.handlers.rst
+++ b/Doc/library/logging.handlers.rst
@@ -1067,7 +1067,7 @@ possible, while any potentially slow operations (such as sending an email via
    versions - to always pass each message to each handler.
 
    .. versionchanged:: 3.5
-      The ``respect_handler_levels`` argument was added.
+      The ``respect_handler_level`` argument was added.
 
    .. method:: dequeue(block)
 


From webhook-mailer at python.org  Wed Sep 11 02:56:14 2019
From: webhook-mailer at python.org (Carol Willing)
Date: Wed, 11 Sep 2019 06:56:14 -0000
Subject: [Python-checkins] bpo-38034: Fix typo in logging.handlers.rst
 (GH-15708) (GH-15893)
Message-ID: <mailman.494.1568184975.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f357cd022ec82a4099ac7a09047d2b556a1c91ec
commit: f357cd022ec82a4099ac7a09047d2b556a1c91ec
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Carol Willing <carolcode at willingconsulting.com>
date: 2019-09-11T07:56:10+01:00
summary:

bpo-38034: Fix typo in logging.handlers.rst (GH-15708) (GH-15893)

(cherry picked from commit efd5741ae953e50a6654e04cf731da86a1307296)

Co-authored-by: wwuck <301402+wwuck at users.noreply.github.com>

files:
M Doc/library/logging.handlers.rst

diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst
index 32919c1a2e29..0e9870408ff4 100644
--- a/Doc/library/logging.handlers.rst
+++ b/Doc/library/logging.handlers.rst
@@ -1046,7 +1046,7 @@ possible, while any potentially slow operations (such as sending an email via
    versions - to always pass each message to each handler.
 
    .. versionchanged:: 3.5
-      The ``respect_handler_levels`` argument was added.
+      The ``respect_handler_level`` argument was added.
 
    .. method:: dequeue(block)
 


From webhook-mailer at python.org  Wed Sep 11 02:56:53 2019
From: webhook-mailer at python.org (Carol Willing)
Date: Wed, 11 Sep 2019 06:56:53 -0000
Subject: [Python-checkins] bpo-38034: Fix typo in logging.handlers.rst
 (GH-15708) (GH-15894)
Message-ID: <mailman.495.1568185015.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2a8560ae99213954158789b9dd7275b8ffa2086a
commit: 2a8560ae99213954158789b9dd7275b8ffa2086a
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Carol Willing <carolcode at willingconsulting.com>
date: 2019-09-11T07:56:50+01:00
summary:

bpo-38034: Fix typo in logging.handlers.rst (GH-15708) (GH-15894)

(cherry picked from commit efd5741ae953e50a6654e04cf731da86a1307296)

Co-authored-by: wwuck <301402+wwuck at users.noreply.github.com>

files:
M Doc/library/logging.handlers.rst

diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst
index 32919c1a2e29..0e9870408ff4 100644
--- a/Doc/library/logging.handlers.rst
+++ b/Doc/library/logging.handlers.rst
@@ -1046,7 +1046,7 @@ possible, while any potentially slow operations (such as sending an email via
    versions - to always pass each message to each handler.
 
    .. versionchanged:: 3.5
-      The ``respect_handler_levels`` argument was added.
+      The ``respect_handler_level`` argument was added.
 
    .. method:: dequeue(block)
 


From webhook-mailer at python.org  Wed Sep 11 04:20:28 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 08:20:28 -0000
Subject: [Python-checkins] bpo-36373: Fix deprecation warnings (GH-15889)
Message-ID: <mailman.496.1568190030.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7264e92b718d307cc499b2f10eab7644b00f0499
commit: 7264e92b718d307cc499b2f10eab7644b00f0499
branch: master
author: Andrew Svetlov <andrew.svetlov at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T01:20:24-07:00
summary:

bpo-36373: Fix deprecation warnings (GH-15889)



https://bugs.python.org/issue36373

files:
M Lib/asyncio/locks.py
M Lib/asyncio/queues.py
M Lib/test/test_asyncio/test_locks.py
M Lib/test/test_asyncio/test_queues.py
M Lib/unittest/async_case.py

diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py
index f63d4cedbbb7..d94daeb5a173 100644
--- a/Lib/asyncio/locks.py
+++ b/Lib/asyncio/locks.py
@@ -332,7 +332,7 @@ def __init__(self, lock=None, *, loop=None):
                           DeprecationWarning, stacklevel=2)
 
         if lock is None:
-            lock = Lock(loop=self._loop)
+            lock = Lock(loop=loop)
         elif lock._loop is not self._loop:
             raise ValueError("loop argument must agree with lock")
 
diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py
index c96b4a0c1ba3..390ae9a6821c 100644
--- a/Lib/asyncio/queues.py
+++ b/Lib/asyncio/queues.py
@@ -45,7 +45,7 @@ def __init__(self, maxsize=0, *, loop=None):
         # Futures.
         self._putters = collections.deque()
         self._unfinished_tasks = 0
-        self._finished = locks.Event(loop=self._loop)
+        self._finished = locks.Event(loop=loop)
         self._finished.set()
         self._init(maxsize)
 
diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py
index d69b56dcda22..a9953b4b2a2d 100644
--- a/Lib/test/test_asyncio/test_locks.py
+++ b/Lib/test/test_asyncio/test_locks.py
@@ -500,10 +500,9 @@ def test_ctor_loop(self):
             self.assertIs(cond._loop, self.loop)
 
     def test_ctor_noloop(self):
-        with self.assertWarns(DeprecationWarning):
-            asyncio.set_event_loop(self.loop)
-            cond = asyncio.Condition()
-            self.assertIs(cond._loop, self.loop)
+        asyncio.set_event_loop(self.loop)
+        cond = asyncio.Condition()
+        self.assertIs(cond._loop, self.loop)
 
     def test_wait(self):
         with self.assertWarns(DeprecationWarning):
diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py
index 02e8e43ccee6..171176c9fc53 100644
--- a/Lib/test/test_asyncio/test_queues.py
+++ b/Lib/test/test_asyncio/test_queues.py
@@ -83,8 +83,7 @@ def test_ctor_loop(self):
 
     def test_ctor_noloop(self):
         asyncio.set_event_loop(self.loop)
-        with self.assertWarns(DeprecationWarning):
-            q = asyncio.Queue()
+        q = asyncio.Queue()
         self.assertIs(q._loop, self.loop)
 
     def test_repr(self):
diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py
index a3c8bfb9eca7..1bc1312c8c2e 100644
--- a/Lib/unittest/async_case.py
+++ b/Lib/unittest/async_case.py
@@ -89,8 +89,9 @@ def _callMaybeAsync(self, func, /, *args, **kwargs):
         else:
             return ret
 
-    async def _asyncioLoopRunner(self):
-        queue = self._asyncioCallsQueue
+    async def _asyncioLoopRunner(self, fut):
+        self._asyncioCallsQueue = queue = asyncio.Queue()
+        fut.set_result(None)
         while True:
             query = await queue.get()
             queue.task_done()
@@ -113,8 +114,9 @@ def _setupAsyncioLoop(self):
         asyncio.set_event_loop(loop)
         loop.set_debug(True)
         self._asyncioTestLoop = loop
-        self._asyncioCallsQueue = asyncio.Queue(loop=loop)
-        self._asyncioCallsTask = loop.create_task(self._asyncioLoopRunner())
+        fut = loop.create_future()
+        self._asyncioCallsTask = loop.create_task(self._asyncioLoopRunner(fut))
+        loop.run_until_complete(fut)
 
     def _tearDownAsyncioLoop(self):
         assert self._asyncioTestLoop is not None


From webhook-mailer at python.org  Wed Sep 11 05:17:18 2019
From: webhook-mailer at python.org (Petr Viktorin)
Date: Wed, 11 Sep 2019 09:17:18 -0000
Subject: [Python-checkins] bpo-37879: Suppress subtype_dealloc decref when
 base type is a C heap type (GH-15323)
Message-ID: <mailman.497.1568193438.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ff023ed36ea260ab64be5895f1f1f087c798987a
commit: ff023ed36ea260ab64be5895f1f1f087c798987a
branch: master
author: Eddie Elizondo <eduardo.elizondorueda at gmail.com>
committer: Petr Viktorin <encukou at gmail.com>
date: 2019-09-11T10:17:13+01:00
summary:

bpo-37879: Suppress subtype_dealloc decref when base type is a C heap type (GH-15323)

The instance destructor for a type is responsible for preparing
an instance for deallocation by decrementing the reference counts
of its referents.

If an instance belongs to a heap type, the type object of an instance
has its reference count decremented while for static types, which
are permanently allocated, the type object is unaffected by the
instance destructor.

Previously, the default instance destructor searched the class
hierarchy for an inherited instance destructor and, if present,
would invoke it.

Then, if the instance type is a heap type, it would decrement the
reference count of that heap type.  However, this could result in the
premature destruction of a type because the inherited instance
destructor should have already decremented the reference count
of the type object.

This change avoids the premature destruction of the type object
by suppressing the decrement of its reference count when an
inherited, non-default instance destructor has been invoked.

Finally, an assertion on the Py_SIZE of a type was deleted.  Heap
types have a non zero size, making this into an incorrect assertion.

https://github.com/python/cpython/pull/15323

files:
A Misc/NEWS.d/next/C API/2019-08-17-13-50-21.bpo-37879.CZeUem.rst
M Lib/test/test_capi.py
M Modules/_testcapimodule.c
M Objects/typeobject.c

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 7b35ba60b53a..4d6e2f21551a 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -383,6 +383,92 @@ def __del__(self):
             del L
             self.assertEqual(PyList.num, 0)
 
+    def test_subclass_of_heap_gc_ctype_with_tpdealloc_decrefs_once(self):
+        class HeapGcCTypeSubclass(_testcapi.HeapGcCType):
+            def __init__(self):
+                self.value2 = 20
+                super().__init__()
+
+        subclass_instance = HeapGcCTypeSubclass()
+        type_refcnt = sys.getrefcount(HeapGcCTypeSubclass)
+
+        # Test that subclass instance was fully created
+        self.assertEqual(subclass_instance.value, 10)
+        self.assertEqual(subclass_instance.value2, 20)
+
+        # Test that the type reference count is only decremented once
+        del subclass_instance
+        self.assertEqual(type_refcnt - 1, sys.getrefcount(HeapGcCTypeSubclass))
+
+    def test_subclass_of_heap_gc_ctype_with_del_modifying_dunder_class_only_decrefs_once(self):
+        class A(_testcapi.HeapGcCType):
+            def __init__(self):
+                self.value2 = 20
+                super().__init__()
+
+        class B(A):
+            def __init__(self):
+                super().__init__()
+
+            def __del__(self):
+                self.__class__ = A
+                A.refcnt_in_del = sys.getrefcount(A)
+                B.refcnt_in_del = sys.getrefcount(B)
+
+        subclass_instance = B()
+        type_refcnt = sys.getrefcount(B)
+        new_type_refcnt = sys.getrefcount(A)
+
+        # Test that subclass instance was fully created
+        self.assertEqual(subclass_instance.value, 10)
+        self.assertEqual(subclass_instance.value2, 20)
+
+        del subclass_instance
+
+        # Test that setting __class__ modified the reference counts of the types
+        self.assertEqual(type_refcnt - 1, B.refcnt_in_del)
+        self.assertEqual(new_type_refcnt + 1, A.refcnt_in_del)
+
+        # Test that the original type already has decreased its refcnt
+        self.assertEqual(type_refcnt - 1, sys.getrefcount(B))
+
+        # Test that subtype_dealloc decref the newly assigned __class__ only once
+        self.assertEqual(new_type_refcnt, sys.getrefcount(A))
+
+    def test_c_subclass_of_heap_ctype_with_tpdealloc_decrefs_once(self):
+        subclass_instance = _testcapi.HeapCTypeSubclass()
+        type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass)
+
+        # Test that subclass instance was fully created
+        self.assertEqual(subclass_instance.value, 10)
+        self.assertEqual(subclass_instance.value2, 20)
+
+        # Test that the type reference count is only decremented once
+        del subclass_instance
+        self.assertEqual(type_refcnt - 1, sys.getrefcount(_testcapi.HeapCTypeSubclass))
+
+    def test_c_subclass_of_heap_ctype_with_del_modifying_dunder_class_only_decrefs_once(self):
+        subclass_instance = _testcapi.HeapCTypeSubclassWithFinalizer()
+        type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclassWithFinalizer)
+        new_type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass)
+
+        # Test that subclass instance was fully created
+        self.assertEqual(subclass_instance.value, 10)
+        self.assertEqual(subclass_instance.value2, 20)
+
+        # The tp_finalize slot will set __class__ to HeapCTypeSubclass
+        del subclass_instance
+
+        # Test that setting __class__ modified the reference counts of the types
+        self.assertEqual(type_refcnt - 1, _testcapi.HeapCTypeSubclassWithFinalizer.refcnt_in_del)
+        self.assertEqual(new_type_refcnt + 1, _testcapi.HeapCTypeSubclass.refcnt_in_del)
+
+        # Test that the original type already has decreased its refcnt
+        self.assertEqual(type_refcnt - 1, sys.getrefcount(_testcapi.HeapCTypeSubclassWithFinalizer))
+
+        # Test that subtype_dealloc decref the newly assigned __class__ only once
+        self.assertEqual(new_type_refcnt, sys.getrefcount(_testcapi.HeapCTypeSubclass))
+
 
 class TestPendingCalls(unittest.TestCase):
 
diff --git a/Misc/NEWS.d/next/C API/2019-08-17-13-50-21.bpo-37879.CZeUem.rst b/Misc/NEWS.d/next/C API/2019-08-17-13-50-21.bpo-37879.CZeUem.rst
new file mode 100644
index 000000000000..87322fbf5f23
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-08-17-13-50-21.bpo-37879.CZeUem.rst	
@@ -0,0 +1,2 @@
+Fix subtype_dealloc to suppress the type decref when the base type is a C
+heap type
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 2b261b6c86dc..21061a2171a0 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -32,6 +32,8 @@
 #  error "_testcapi must test the public Python C API, not CPython internal C API"
 #endif
 
+static struct PyModuleDef _testcapimodule;
+
 static PyObject *TestError;     /* set to exception object in init */
 
 /* Raise TestError with test_name + ": " + msg, and return NULL. */
@@ -6169,6 +6171,189 @@ static PyTypeObject MethodDescriptor2_Type = {
     .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | _Py_TPFLAGS_HAVE_VECTORCALL,
 };
 
+PyDoc_STRVAR(heapgctype__doc__,
+"A heap type with GC, and with overridden dealloc.\n\n"
+"The 'value' attribute is set to 10 in __init__.");
+
+typedef struct {
+    PyObject_HEAD
+    int value;
+} HeapCTypeObject;
+
+static struct PyMemberDef heapctype_members[] = {
+    {"value", T_INT, offsetof(HeapCTypeObject, value)},
+    {NULL} /* Sentinel */
+};
+
+static int
+heapctype_init(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+    ((HeapCTypeObject *)self)->value = 10;
+    return 0;
+}
+
+static void
+heapgcctype_dealloc(HeapCTypeObject *self)
+{
+    PyTypeObject *tp = Py_TYPE(self);
+    PyObject_GC_UnTrack(self);
+    PyObject_GC_Del(self);
+    Py_DECREF(tp);
+}
+
+static PyType_Slot HeapGcCType_slots[] = {
+    {Py_tp_init, heapctype_init},
+    {Py_tp_members, heapctype_members},
+    {Py_tp_dealloc, heapgcctype_dealloc},
+    {Py_tp_doc, heapgctype__doc__},
+    {0, 0},
+};
+
+static PyType_Spec HeapGcCType_spec = {
+    "_testcapi.HeapGcCType",
+    sizeof(HeapCTypeObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+    HeapGcCType_slots
+};
+
+PyDoc_STRVAR(heapctype__doc__,
+"A heap type without GC, but with overridden dealloc.\n\n"
+"The 'value' attribute is set to 10 in __init__.");
+
+static void
+heapctype_dealloc(HeapCTypeObject *self)
+{
+    PyTypeObject *tp = Py_TYPE(self);
+    PyObject_Del(self);
+    Py_DECREF(tp);
+}
+
+static PyType_Slot HeapCType_slots[] = {
+    {Py_tp_init, heapctype_init},
+    {Py_tp_members, heapctype_members},
+    {Py_tp_dealloc, heapctype_dealloc},
+    {Py_tp_doc, heapctype__doc__},
+    {0, 0},
+};
+
+static PyType_Spec HeapCType_spec = {
+    "_testcapi.HeapCType",
+    sizeof(HeapCTypeObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    HeapCType_slots
+};
+
+PyDoc_STRVAR(heapctypesubclass__doc__,
+"Subclass of HeapCType, without GC.\n\n"
+"__init__ sets the 'value' attribute to 10 and 'value2' to 20.");
+
+typedef struct {
+    HeapCTypeObject base;
+    int value2;
+} HeapCTypeSubclassObject;
+
+static int
+heapctypesubclass_init(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+    /* Call __init__ of the superclass */
+    if (heapctype_init(self, args, kwargs) < 0) {
+        return -1;
+    }
+    /* Initialize additional element */
+    ((HeapCTypeSubclassObject *)self)->value2 = 20;
+    return 0;
+}
+
+static struct PyMemberDef heapctypesubclass_members[] = {
+    {"value2", T_INT, offsetof(HeapCTypeSubclassObject, value2)},
+    {NULL} /* Sentinel */
+};
+
+static PyType_Slot HeapCTypeSubclass_slots[] = {
+    {Py_tp_init, heapctypesubclass_init},
+    {Py_tp_members, heapctypesubclass_members},
+    {Py_tp_doc, heapctypesubclass__doc__},
+    {0, 0},
+};
+
+static PyType_Spec HeapCTypeSubclass_spec = {
+    "_testcapi.HeapCTypeSubclass",
+    sizeof(HeapCTypeSubclassObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    HeapCTypeSubclass_slots
+};
+
+PyDoc_STRVAR(heapctypesubclasswithfinalizer__doc__,
+"Subclass of HeapCType with a finalizer that reassigns __class__.\n\n"
+"__class__ is set to plain HeapCTypeSubclass during finalization.\n"
+"__init__ sets the 'value' attribute to 10 and 'value2' to 20.");
+
+static int
+heapctypesubclasswithfinalizer_init(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+    PyTypeObject *base = (PyTypeObject *)PyType_GetSlot(Py_TYPE(self), Py_tp_base);
+    initproc base_init = PyType_GetSlot(base, Py_tp_init);
+    base_init(self, args, kwargs);
+    return 0;
+}
+
+static void
+heapctypesubclasswithfinalizer_finalize(PyObject *self)
+{
+    PyObject *error_type, *error_value, *error_traceback, *m, *oldtype, *newtype;
+
+    /* Save the current exception, if any. */
+    PyErr_Fetch(&error_type, &error_value, &error_traceback);
+
+    m = PyState_FindModule(&_testcapimodule);
+    if (m == NULL) {
+        goto cleanup_finalize;
+    }
+    oldtype = PyObject_GetAttrString(m, "HeapCTypeSubclassWithFinalizer");
+    newtype = PyObject_GetAttrString(m, "HeapCTypeSubclass");
+    if (oldtype == NULL || newtype == NULL) {
+        goto cleanup_finalize;
+    }
+
+    if (PyObject_SetAttrString(self, "__class__", newtype) < 0) {
+        goto cleanup_finalize;
+    }
+    if (PyObject_SetAttrString(
+        oldtype, "refcnt_in_del", PyLong_FromSsize_t(Py_REFCNT(oldtype))) < 0) {
+        goto cleanup_finalize;
+    }
+    if (PyObject_SetAttrString(
+        newtype, "refcnt_in_del", PyLong_FromSsize_t(Py_REFCNT(newtype))) < 0) {
+        goto cleanup_finalize;
+    }
+
+cleanup_finalize:
+    Py_XDECREF(oldtype);
+    Py_XDECREF(newtype);
+
+    /* Restore the saved exception. */
+    PyErr_Restore(error_type, error_value, error_traceback);
+}
+
+static PyType_Slot HeapCTypeSubclassWithFinalizer_slots[] = {
+    {Py_tp_init, heapctypesubclasswithfinalizer_init},
+    {Py_tp_members, heapctypesubclass_members},
+    {Py_tp_finalize, heapctypesubclasswithfinalizer_finalize},
+    {Py_tp_doc, heapctypesubclasswithfinalizer__doc__},
+    {0, 0},
+};
+
+static PyType_Spec HeapCTypeSubclassWithFinalizer_spec = {
+    "_testcapi.HeapCTypeSubclassWithFinalizer",
+    sizeof(HeapCTypeSubclassObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_FINALIZE,
+    HeapCTypeSubclassWithFinalizer_slots
+};
+
 static PyMethodDef meth_instance_methods[] = {
     {"meth_varargs", meth_varargs, METH_VARARGS},
     {"meth_varargs_keywords", (PyCFunction)(void(*)(void))meth_varargs_keywords, METH_VARARGS|METH_KEYWORDS},
@@ -6380,5 +6565,40 @@ PyInit__testcapi(void)
     TestError = PyErr_NewException("_testcapi.error", NULL, NULL);
     Py_INCREF(TestError);
     PyModule_AddObject(m, "error", TestError);
+
+    PyObject *HeapGcCType = PyType_FromSpec(&HeapGcCType_spec);
+    if (HeapGcCType == NULL) {
+        return NULL;
+    }
+    PyModule_AddObject(m, "HeapGcCType", HeapGcCType);
+
+    PyObject *HeapCType = PyType_FromSpec(&HeapCType_spec);
+    if (HeapCType == NULL) {
+        return NULL;
+    }
+    PyObject *subclass_bases = PyTuple_Pack(1, HeapCType);
+    if (subclass_bases == NULL) {
+        return NULL;
+    }
+    PyObject *HeapCTypeSubclass = PyType_FromSpecWithBases(&HeapCTypeSubclass_spec, subclass_bases);
+    if (HeapCTypeSubclass == NULL) {
+        return NULL;
+    }
+    Py_DECREF(subclass_bases);
+    PyModule_AddObject(m, "HeapCTypeSubclass", HeapCTypeSubclass);
+
+    PyObject *subclass_with_finalizer_bases = PyTuple_Pack(1, HeapCTypeSubclass);
+    if (subclass_with_finalizer_bases == NULL) {
+        return NULL;
+    }
+    PyObject *HeapCTypeSubclassWithFinalizer = PyType_FromSpecWithBases(
+        &HeapCTypeSubclassWithFinalizer_spec, subclass_with_finalizer_bases);
+    if (HeapCTypeSubclassWithFinalizer == NULL) {
+        return NULL;
+    }
+    Py_DECREF(subclass_with_finalizer_bases);
+    PyModule_AddObject(m, "HeapCTypeSubclassWithFinalizer", HeapCTypeSubclassWithFinalizer);
+
+    PyState_AddModule(m, &_testcapimodule);
     return m;
 }
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 883bc22bbde1..7575e5580bf9 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1157,11 +1157,9 @@ subtype_dealloc(PyObject *self)
     /* Test whether the type has GC exactly once */
 
     if (!PyType_IS_GC(type)) {
-        /* It's really rare to find a dynamic type that doesn't have
-           GC; it can only happen when deriving from 'object' and not
-           adding any slots or instance variables.  This allows
-           certain simplifications: there's no need to call
-           clear_slots(), or DECREF the dict, or clear weakrefs. */
+        /* A non GC dynamic type allows certain simplifications:
+           there's no need to call clear_slots(), or DECREF the dict,
+           or clear weakrefs. */
 
         /* Maybe call finalizer; exit early if resurrected */
         if (type->tp_finalize) {
@@ -1177,7 +1175,6 @@ subtype_dealloc(PyObject *self)
         /* Find the nearest base with a different tp_dealloc */
         base = type;
         while ((basedealloc = base->tp_dealloc) == subtype_dealloc) {
-            assert(Py_SIZE(base) == 0);
             base = base->tp_base;
             assert(base);
         }
@@ -1189,8 +1186,10 @@ subtype_dealloc(PyObject *self)
         assert(basedealloc);
         basedealloc(self);
 
-        /* Can't reference self beyond this point */
-        Py_DECREF(type);
+       /* Only decref if the base type is not already a heap allocated type.
+          Otherwise, basedealloc should have decref'd it already */
+        if (type->tp_flags & Py_TPFLAGS_HEAPTYPE && !(base->tp_flags & Py_TPFLAGS_HEAPTYPE))
+            Py_DECREF(type);
 
         /* Done */
         return;
@@ -1289,8 +1288,9 @@ subtype_dealloc(PyObject *self)
 
     /* Can't reference self beyond this point. It's possible tp_del switched
        our type from a HEAPTYPE to a non-HEAPTYPE, so be careful about
-       reference counting. */
-    if (type->tp_flags & Py_TPFLAGS_HEAPTYPE)
+       reference counting. Only decref if the base type is not already a heap
+       allocated type. Otherwise, basedealloc should have decref'd it already */
+    if (type->tp_flags & Py_TPFLAGS_HEAPTYPE && !(base->tp_flags & Py_TPFLAGS_HEAPTYPE))
       Py_DECREF(type);
 
   endlabel:


From webhook-mailer at python.org  Wed Sep 11 05:19:09 2019
From: webhook-mailer at python.org (Paul Ganssle)
Date: Wed, 11 Sep 2019 09:19:09 -0000
Subject: [Python-checkins] Overhaul datetime documentation (GH-13410)
Message-ID: <mailman.498.1568193550.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3fb1363fe87a24cdb2ee1dd9746f1c49046af958
commit: 3fb1363fe87a24cdb2ee1dd9746f1c49046af958
branch: master
author: Brad <brad.solomon.1124 at gmail.com>
committer: Paul Ganssle <paul at ganssle.io>
date: 2019-09-11T10:19:05+01:00
summary:

Overhaul datetime documentation (GH-13410)

This is a restructuring of the datetime documentation to hopefully make
them more user-friendly and approachable to new users without losing any
of the detail.

Changes include:
 - Creating dedicated subsections for some concepts such as:
    - "Constants"
    - "Naive vs Aware"
    - "Determining if an Object is Aware"
 - Give 'naive vs aware' its own subsection
 - Give 'constants' their own subsection
 - Overhauling the strftime-strptime section by:
    - Breaking it into logical, linkable, and digestable parts
    - Adding a high-level comparison table
    - Moving the technical detail to bottom: readers come to this
      section primarily to remind themselves to things:
      - How do I write the format code for X?
      - strptime/strftime: which one is which again?
 - Touching up fromisoformat + isoformat sections by:
    - Revising fromisoformat + isoformat for date, time, and
      datetime
    - Adding basic examples
    - Enforcing consistency about putting formats (i.e. ``HH:MM``)
      in double backticks.  This was previously done in some places
      but not all
    - Putting long 'supported formats', on their own line to improve
      readability
 - Moving the 'seealso' section to the top and add a link to dateutil
    Rationale: This doesn't really belong nested under the
    'constants' section.  Let readers know right away that
    datetime is one of several related tools.
 - Moving common features of several types into one place:
    Previously, each type went out of its way to note separately
    that it was hashable and picklable.  These can be brought
    into one single place that is more prominent.
 - Reducing some verbose explanations to improve readability
 - Breaking up long paragraphs into digestable chunks
 - Displaying longer "equivalent to" examples, as short code blocks
 - Using the dot notation for datetime/time classes:
    Use :class:`.time` and :class:`.datetime` rather than :class:`time` and
    :class:`datetime`; otherwise, the generated links will route to the
    respective modules, not classes.
 - Rewording the tzinfo class description
    The top paragraph should get straight to the point of telling the reader
    what subclasses of tzinfo _do_.  Previously, that was hidden in a later
    paragraph.
 - Adding a note on .today() versus .now()
 - Rearranging and expanding example blocks, including:
    - Moved long, multiline inline examples to standalone examples
    - Simplified the example block for timedelta arithmetic:
        - Broke the example into two logical sections:
          1. normalization/parameter 'merging'
          2. timedelta arithmetic
        - Reduced the complexity of the some of the examples.  Show
          reasonable, real-world uses cases that are easy to follow
          along with and progres in difficult slightly.
    - Broke up the example sections for date and datetime sections by putting
      the easy examples first, progressing to more esoteric situations and
      breaking it up into logical sections based on what the methods are
      doing at a high level.
    - Simplified the KabulTz example:
        - Put the class definition itself into a non-REPL block since there is
          no interactive output involved there
        - Briefly explained what's happening before launching into the code
        - Broke the example section into visually separate chunks
 - Various whitespace, formatting, style and grammar fixes including:
    - Consistently using backctics for 'date_string' formats
    - Consistently using one space after periods.
    - Consistently using bold for vocab terms
    - Consistently using italics when referring to params:
      See https://devguide.python.org/documenting/#id4
    - Using '::' to lead into code blocks
        Per https://devguide.python.org/documenting/#source-code, this will
        let the reader use the 'expand/collapse' top-right button for REPL
        blocks to hide or show the prompt.
    - Using consistent captialization schemes
    - Removing use of the default role
    - Put 'example' blocks in Markdown subsections

This is a combination of 66 commits.

See bpo-36960: https://bugs.python.org/issue36960

files:
A Misc/NEWS.d/next/Documentation/2019-05-18-16-25-44.bpo-36960.xEKHXj.rst
M Doc/library/datetime.rst

diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index 50f1b9e74949..882d4e16e1f6 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -14,40 +14,59 @@
 
 .. XXX what order should the types be discussed in?
 
-The :mod:`datetime` module supplies classes for manipulating dates and times in
-both simple and complex ways.  While date and time arithmetic is supported, the
-focus of the implementation is on efficient attribute extraction for output
-formatting and manipulation. For related functionality, see also the
-:mod:`time` and :mod:`calendar` modules.
-
-There are two kinds of date and time objects: "naive" and "aware".
-
-An aware object has sufficient knowledge of applicable algorithmic and
-political time adjustments, such as time zone and daylight saving time
-information, to locate itself relative to other aware objects.  An aware object
-is used to represent a specific moment in time that is not open to
-interpretation [#]_.
-
-A naive object does not contain enough information to unambiguously locate
-itself relative to other date/time objects.  Whether a naive object represents
+The :mod:`datetime` module supplies classes for manipulating dates and times.
+
+While date and time arithmetic is supported, the focus of the implementation is
+on efficient attribute extraction for output formatting and manipulation.
+
+.. seealso::
+
+   Module :mod:`calendar`
+      General calendar related functions.
+
+   Module :mod:`time`
+      Time access and conversions.
+
+   Package `dateutil <https://dateutil.readthedocs.io/en/stable/>`_
+      Third-party library with expanded time zone and parsing support.
+
+.. _datetime-naive-aware:
+
+Aware and Naive Objects
+-----------------------
+
+Date and time objects may be categorized as "aware" or "naive."
+
+With sufficient knowledge of applicable algorithmic and political time
+adjustments, such as time zone and daylight saving time information,
+an **aware** object can locate itself relative to other aware objects.
+An aware object represents a specific moment in time that is not open to
+interpretation. [#]_
+
+A **naive** object does not contain enough information to unambiguously locate
+itself relative to other date/time objects. Whether a naive object represents
 Coordinated Universal Time (UTC), local time, or time in some other timezone is
 purely up to the program, just like it is up to the program whether a
-particular number represents metres, miles, or mass.  Naive objects are easy to
+particular number represents metres, miles, or mass. Naive objects are easy to
 understand and to work with, at the cost of ignoring some aspects of reality.
 
 For applications requiring aware objects, :class:`.datetime` and :class:`.time`
 objects have an optional time zone information attribute, :attr:`!tzinfo`, that
 can be set to an instance of a subclass of the abstract :class:`tzinfo` class.
 These :class:`tzinfo` objects capture information about the offset from UTC
-time, the time zone name, and whether Daylight Saving Time is in effect.  Note
-that only one concrete :class:`tzinfo` class, the :class:`timezone` class, is
-supplied by the :mod:`datetime` module.  The :class:`timezone` class can
-represent simple timezones with fixed offset from UTC, such as UTC itself or
-North American EST and EDT timezones.  Supporting timezones at deeper levels of
-detail is up to the application.  The rules for time adjustment across the
+time, the time zone name, and whether daylight saving time is in effect.
+
+Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is
+supplied by the :mod:`datetime` module. The :class:`timezone` class can
+represent simple timezones with fixed offsets from UTC, such as UTC itself or
+North American EST and EDT timezones. Supporting timezones at deeper levels of
+detail is up to the application. The rules for time adjustment across the
 world are more political than rational, change frequently, and there is no
 standard suitable for every application aside from UTC.
 
+Constants
+---------
+
 The :mod:`datetime` module exports the following constants:
 
 .. data:: MINYEAR
@@ -61,16 +80,6 @@ The :mod:`datetime` module exports the following constants:
    The largest year number allowed in a :class:`date` or :class:`.datetime` object.
    :const:`MAXYEAR` is ``9999``.
 
-
-.. seealso::
-
-   Module :mod:`calendar`
-      General calendar related functions.
-
-   Module :mod:`time`
-      Time access and conversions.
-
-
 Available Types
 ---------------
 
@@ -86,7 +95,7 @@ Available Types
    :noindex:
 
    An idealized time, independent of any particular day, assuming that every day
-   has exactly 24\*60\*60 seconds (there is no notion of "leap seconds" here).
+   has exactly 24\*60\*60 seconds.  (There is no notion of "leap seconds" here.)
    Attributes: :attr:`hour`, :attr:`minute`, :attr:`second`, :attr:`microsecond`,
    and :attr:`.tzinfo`.
 
@@ -109,7 +118,7 @@ Available Types
 .. class:: tzinfo
    :noindex:
 
-   An abstract base class for time zone information objects.  These are used by the
+   An abstract base class for time zone information objects. These are used by the
    :class:`.datetime` and :class:`.time` classes to provide a customizable notion of
    time adjustment (for example, to account for time zone and/or daylight saving
    time).
@@ -122,22 +131,8 @@ Available Types
 
    .. versionadded:: 3.2
 
-
 Objects of these types are immutable.
 
-Objects of the :class:`date` type are always naive.
-
-An object of type :class:`.time` or :class:`.datetime` may be naive or aware.
-A :class:`.datetime` object *d* is aware if ``d.tzinfo`` is not ``None`` and
-``d.tzinfo.utcoffset(d)`` does not return ``None``.  If ``d.tzinfo`` is
-``None``, or if ``d.tzinfo`` is not ``None`` but ``d.tzinfo.utcoffset(d)``
-returns ``None``, *d* is naive.  A :class:`.time` object *t* is aware
-if ``t.tzinfo`` is not ``None`` and ``t.tzinfo.utcoffset(None)`` does not return
-``None``.  Otherwise, *t* is naive.
-
-The distinction between naive and aware doesn't apply to :class:`timedelta`
-objects.
-
 Subclass relationships::
 
    object
@@ -148,6 +143,40 @@ Subclass relationships::
        date
            datetime
 
+Common Properties
+^^^^^^^^^^^^^^^^^
+
+The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` types
+share these common features:
+
+- Objects of these types are immutable.
+- Objects of these types are hashable, meaning that they can be used as
+  dictionary keys.
+- Objects of these types support efficient pickling via the :mod:`pickle` module.
+
+Determining if an Object is Aware or Naive
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Objects of the :class:`date` type are always naive.
+
+An object of type :class:`.time` or :class:`.datetime` may be aware or naive.
+
+A :class:`.datetime` object *d* is aware if both of the following hold:
+
+1. ``d.tzinfo`` is not ``None``
+2. ``d.tzinfo.utcoffset(d)`` does not return ``None``
+
+Otherwise, *d* is naive.
+
+A :class:`.time` object *t* is aware if both of the following hold:
+
+1. ``t.tzinfo`` is not ``None``
+2. ``t.tzinfo.utcoffset(None)`` does not return ``None``.
+
+Otherwise, *t* is naive.
+
+The distinction between aware and naive doesn't apply to :class:`timedelta`
+objects.
 
 .. _datetime-timedelta:
 
@@ -159,11 +188,11 @@ dates or times.
 
 .. class:: timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0)
 
-   All arguments are optional and default to ``0``.  Arguments may be integers
+   All arguments are optional and default to ``0``. Arguments may be integers
    or floats, and may be positive or negative.
 
-   Only *days*, *seconds* and *microseconds* are stored internally.  Arguments are
-   converted to those units:
+   Only *days*, *seconds* and *microseconds* are stored internally.
+   Arguments are converted to those units:
 
    * A millisecond is converted to 1000 microseconds.
    * A minute is converted to 60 seconds.
@@ -177,10 +206,28 @@ dates or times.
    * ``0 <= seconds < 3600*24`` (the number of seconds in one day)
    * ``-999999999 <= days <= 999999999``
 
+   The following example illustrates how any arguments besides
+   *days*, *seconds* and *microseconds* are "merged" and normalized into those
+   three resulting attributes::
+
+       >>> from datetime import timedelta
+       >>> delta = timedelta(
+       ...     days=50,
+       ...     seconds=27,
+       ...     microseconds=10,
+       ...     milliseconds=29000,
+       ...     minutes=5,
+       ...     hours=8,
+       ...     weeks=2
+       ... )
+       >>> # Only days, seconds, and microseconds remain
+       >>> delta
+       datetime.timedelta(days=64, seconds=29156, microseconds=10)
+
    If any argument is a float and there are fractional microseconds,
    the fractional microseconds left over from all arguments are
    combined and their sum is rounded to the nearest microsecond using
-   round-half-to-even tiebreaker.  If no argument is a float, the
+   round-half-to-even tiebreaker. If no argument is a float, the
    conversion and normalization processes are exact (no information is
    lost).
 
@@ -188,7 +235,7 @@ dates or times.
    :exc:`OverflowError` is raised.
 
    Note that normalization of negative values may be surprising at first. For
-   example,
+   example::
 
       >>> from datetime import timedelta
       >>> d = timedelta(microseconds=-1)
@@ -196,7 +243,7 @@ dates or times.
       (-1, 86399, 999999)
 
 
-Class attributes are:
+Class attributes:
 
 .. attribute:: timedelta.min
 
@@ -263,7 +310,7 @@ Supported operations:
 |                                | timedelta.resolution using round-half-to-even.|
 +--------------------------------+-----------------------------------------------+
 | ``t1 = t2 // i`` or            | The floor is computed and the remainder (if   |
-| ``t1 = t2 // t3``              | any) is thrown away.  In the second case, an  |
+| ``t1 = t2 // t3``              | any) is thrown away. In the second case, an   |
 |                                | integer is returned. (3)                      |
 +--------------------------------+-----------------------------------------------+
 | ``t1 = t2 % t3``               | The remainder is computed as a                |
@@ -282,8 +329,8 @@ Supported operations:
 |                                | -*t1.seconds*, -*t1.microseconds*),           |
 |                                | and to *t1*\* -1. (1)(4)                      |
 +--------------------------------+-----------------------------------------------+
-| ``abs(t)``                     | equivalent to +\ *t* when ``t.days >= 0``, and|
-|                                | to -*t* when ``t.days < 0``. (2)              |
+| ``abs(t)``                     | equivalent to +\ *t* when ``t.days >= 0``,    |
+|                                | and to -*t* when ``t.days < 0``. (2)          |
 +--------------------------------+-----------------------------------------------+
 | ``str(t)``                     | Returns a string in the form                  |
 |                                | ``[D day[s], ][H]H:MM:SS[.UUUUUU]``, where D  |
@@ -298,10 +345,10 @@ Supported operations:
 Notes:
 
 (1)
-   This is exact, but may overflow.
+   This is exact but may overflow.
 
 (2)
-   This is exact, and cannot overflow.
+   This is exact and cannot overflow.
 
 (3)
    Division by 0 raises :exc:`ZeroDivisionError`.
@@ -310,41 +357,56 @@ Notes:
    -*timedelta.max* is not representable as a :class:`timedelta` object.
 
 (5)
-  String representations of :class:`timedelta` objects are normalized
-  similarly to their internal representation.  This leads to somewhat
-  unusual results for negative timedeltas.  For example:
+   String representations of :class:`timedelta` objects are normalized
+   similarly to their internal representation. This leads to somewhat
+   unusual results for negative timedeltas. For example::
 
-  >>> timedelta(hours=-5)
-  datetime.timedelta(days=-1, seconds=68400)
-  >>> print(_)
-  -1 day, 19:00:00
+      >>> timedelta(hours=-5)
+      datetime.timedelta(days=-1, seconds=68400)
+      >>> print(_)
+      -1 day, 19:00:00
 
 (6)
    The expression ``t2 - t3`` will always be equal to the expression ``t2 + (-t3)`` except
    when t3 is equal to ``timedelta.max``; in that case the former will produce a result
    while the latter will overflow.
 
-In addition to the operations listed above :class:`timedelta` objects support
+In addition to the operations listed above, :class:`timedelta` objects support
 certain additions and subtractions with :class:`date` and :class:`.datetime`
 objects (see below).
 
 .. versionchanged:: 3.2
    Floor division and true division of a :class:`timedelta` object by another
    :class:`timedelta` object are now supported, as are remainder operations and
-   the :func:`divmod` function.  True division and multiplication of a
+   the :func:`divmod` function. True division and multiplication of a
    :class:`timedelta` object by a :class:`float` object are now supported.
 
 
-Comparisons of :class:`timedelta` objects are supported with the
-:class:`timedelta` object representing the smaller duration considered to be the
-smaller timedelta. In order to stop mixed-type comparisons from falling back to
-the default comparison by object address, when a :class:`timedelta` object is
-compared to an object of a different type, :exc:`TypeError` is raised unless the
-comparison is ``==`` or ``!=``.  The latter cases return :const:`False` or
-:const:`True`, respectively.
+Comparisons of :class:`timedelta` objects are supported, with some caveats.
+
+The comparisons ``==`` or ``!=`` *always* return a :class:`bool`, no matter
+the type of the compared object::
 
-:class:`timedelta` objects are :term:`hashable` (usable as dictionary keys), support
-efficient pickling, and in Boolean contexts, a :class:`timedelta` object is
+    >>> from datetime import timedelta
+    >>> delta1 = timedelta(seconds=57)
+    >>> delta2 = timedelta(hours=25, seconds=2)
+    >>> delta2 != delta1
+    True
+    >>> delta2 == 5
+    False
+
+For all other comparisons (such as ``<`` and ``>``), when a :class:`timedelta`
+object is compared to an object of a different type, :exc:`TypeError`
+is raised::
+
+    >>> delta2 > delta1
+    True
+    >>> delta2 > 5
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+    TypeError: '>' not supported between instances of 'datetime.timedelta' and 'int'
+
+In Boolean contexts, a :class:`timedelta` object is
 considered to be true if and only if it isn't equal to ``timedelta(0)``.
 
 Instance methods:
@@ -360,29 +422,36 @@ Instance methods:
 
    .. versionadded:: 3.2
 
+Examples of usage: :class:`timedelta`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Example usage:
+An additional example of normalization::
 
+    >>> # Components of another_year add up to exactly 365 days
     >>> from datetime import timedelta
     >>> year = timedelta(days=365)
     >>> another_year = timedelta(weeks=40, days=84, hours=23,
-    ...                          minutes=50, seconds=600)  # adds up to 365 days
-    >>> year.total_seconds()
-    31536000.0
+    ...                          minutes=50, seconds=600)
     >>> year == another_year
     True
+    >>> year.total_seconds()
+    31536000.0
+
+Examples of :class:`timedelta` arithmetic::
+
+    >>> from datetime import timedelta
+    >>> year = timedelta(days=365)
     >>> ten_years = 10 * year
-    >>> ten_years, ten_years.days // 365
-    (datetime.timedelta(days=3650), 10)
+    >>> ten_years
+    datetime.timedelta(days=3650)
+    >>> ten_years.days // 365
+    10
     >>> nine_years = ten_years - year
-    >>> nine_years, nine_years.days // 365
-    (datetime.timedelta(days=3285), 9)
+    >>> nine_years
+    datetime.timedelta(days=3285)
     >>> three_years = nine_years // 3
     >>> three_years, three_years.days // 365
     (datetime.timedelta(days=1095), 3)
-    >>> abs(three_years - ten_years) == 2 * three_years + year
-    True
-
 
 .. _datetime-date:
 
@@ -391,17 +460,14 @@ Example usage:
 
 A :class:`date` object represents a date (year, month and day) in an idealized
 calendar, the current Gregorian calendar indefinitely extended in both
-directions.  January 1 of year 1 is called day number 1, January 2 of year 1 is
-called day number 2, and so on.  This matches the definition of the "proleptic
-Gregorian" calendar in Dershowitz and Reingold's book Calendrical Calculations,
-where it's the base calendar for all computations.  See the book for algorithms
-for converting between proleptic Gregorian ordinals and many other calendar
-systems.
+directions.
 
+January 1 of year 1 is called day number 1, January 2 of year 1 is
+called day number 2, and so on. [#]_
 
 .. class:: date(year, month, day)
 
-   All arguments are required.  Arguments must be integers in the following
+   All arguments are required. Arguments must be integers, in the following
    ranges:
 
    * ``MINYEAR <= year <= MAXYEAR``
@@ -415,17 +481,19 @@ Other constructors, all class methods:
 
 .. classmethod:: date.today()
 
-   Return the current local date.  This is equivalent to
-   ``date.fromtimestamp(time.time())``.
+   Return the current local date.
 
+   This is equivalent to ``date.fromtimestamp(time.time())``.
 
 .. classmethod:: date.fromtimestamp(timestamp)
 
-   Return the local date corresponding to the POSIX timestamp, such as is returned
-   by :func:`time.time`.  This may raise :exc:`OverflowError`, if the timestamp is out
-   of the range of values supported by the platform C :c:func:`localtime` function,
-   and :exc:`OSError` on :c:func:`localtime` failure.
-   It's common for this to be restricted to years from 1970 through 2038.  Note
+   Return the local date corresponding to the POSIX timestamp, such as is
+   returned by :func:`time.time`.
+
+   This may raise :exc:`OverflowError`, if the timestamp is out
+   of the range of values supported by the platform C :c:func:`localtime`
+   function, and :exc:`OSError` on :c:func:`localtime` failure.
+   It's common for this to be restricted to years from 1970 through 2038. Note
    that on non-POSIX systems that include leap seconds in their notion of a
    timestamp, leap seconds are ignored by :meth:`fromtimestamp`.
 
@@ -438,24 +506,27 @@ Other constructors, all class methods:
 
 .. classmethod:: date.fromordinal(ordinal)
 
-   Return the date corresponding to the proleptic Gregorian ordinal, where January
-   1 of year 1 has ordinal 1.  :exc:`ValueError` is raised unless ``1 <= ordinal <=
-   date.max.toordinal()``. For any date *d*, ``date.fromordinal(d.toordinal()) ==
-   d``.
+   Return the date corresponding to the proleptic Gregorian ordinal, where
+   January 1 of year 1 has ordinal 1.
+
+   :exc:`ValueError` is raised unless ``1 <= ordinal <=
+   date.max.toordinal()``. For any date *d*,
+   ``date.fromordinal(d.toordinal()) == d``.
 
 
 .. classmethod:: date.fromisoformat(date_string)
 
-  Return a :class:`date` corresponding to a *date_string* in the format emitted
-  by :meth:`date.isoformat`. Specifically, this function supports strings in
-  the format(s) ``YYYY-MM-DD``.
+   Return a :class:`date` corresponding to a *date_string* given in the format
+   ``YYYY-MM-DD``::
 
-  .. caution::
+      >>> from datetime import date
+      >>> date.fromisoformat('2019-12-04')
+      datetime.date(2019, 12, 4)
 
-    This does not support parsing arbitrary ISO 8601 strings - it is only intended
-    as the inverse operation of :meth:`date.isoformat`.
+   This is the inverse of :meth:`date.isoformat`. It only supports the format
+   ``YYYY-MM-DD``.
 
-  .. versionadded:: 3.7
+   .. versionadded:: 3.7
 
 
 .. classmethod:: date.fromisocalendar(year, week, day)
@@ -507,7 +578,7 @@ Supported operations:
 | Operation                     | Result                                       |
 +===============================+==============================================+
 | ``date2 = date1 + timedelta`` | *date2* is ``timedelta.days`` days removed   |
-|                               | from *date1*.  (1)                           |
+|                               | from *date1*. (1)                            |
 +-------------------------------+----------------------------------------------+
 | ``date2 = date1 - timedelta`` | Computes *date2* such that ``date2 +         |
 |                               | timedelta == date1``. (2)                    |
@@ -522,7 +593,7 @@ Notes:
 
 (1)
    *date2* is moved forward in time if ``timedelta.days > 0``, or backward if
-   ``timedelta.days < 0``.  Afterward ``date2 - date1 == timedelta.days``.
+   ``timedelta.days < 0``. Afterward ``date2 - date1 == timedelta.days``.
    ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored.
    :exc:`OverflowError` is raised if ``date2.year`` would be smaller than
    :const:`MINYEAR` or larger than :const:`MAXYEAR`.
@@ -531,7 +602,7 @@ Notes:
    ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored.
 
 (3)
-   This is exact, and cannot overflow.  timedelta.seconds and
+   This is exact, and cannot overflow. timedelta.seconds and
    timedelta.microseconds are 0, and date2 + timedelta == date1 after.
 
 (4)
@@ -539,38 +610,47 @@ Notes:
    date2.toordinal()``. Date comparison raises :exc:`TypeError` if
    the other comparand isn't also a :class:`date` object. However,
    ``NotImplemented`` is returned instead if the other comparand has a
-   :meth:`timetuple` attribute.  This hook gives other kinds of date objects a
+   :meth:`timetuple` attribute. This hook gives other kinds of date objects a
    chance at implementing mixed-type comparison. If not, when a :class:`date`
    object is compared to an object of a different type, :exc:`TypeError` is raised
-   unless the comparison is ``==`` or ``!=``.  The latter cases return
+   unless the comparison is ``==`` or ``!=``. The latter cases return
    :const:`False` or :const:`True`, respectively.
 
-Dates can be used as dictionary keys. In Boolean contexts, all :class:`date`
-objects are considered to be true.
+In Boolean contexts, all :class:`date` objects are considered to be true.
 
 Instance methods:
 
 .. method:: date.replace(year=self.year, month=self.month, day=self.day)
 
    Return a date with the same value, except for those parameters given new
-   values by whichever keyword arguments are specified.  For example, if ``d ==
-   date(2002, 12, 31)``, then ``d.replace(day=26) == date(2002, 12, 26)``.
+   values by whichever keyword arguments are specified.
+
+   Example::
+
+       >>> from datetime import date
+       >>> d = date(2002, 12, 31)
+       >>> d.replace(day=26)
+       datetime.date(2002, 12, 26)
 
 
 .. method:: date.timetuple()
 
    Return a :class:`time.struct_time` such as returned by :func:`time.localtime`.
-   The hours, minutes and seconds are 0, and the DST flag is -1. ``d.timetuple()``
-   is equivalent to ``time.struct_time((d.year, d.month, d.day, 0, 0, 0,
-   d.weekday(), yday, -1))``, where ``yday = d.toordinal() - date(d.year, 1,
-   1).toordinal() + 1`` is the day number within the current year starting with
-   ``1`` for January 1st.
+
+   The hours, minutes and seconds are 0, and the DST flag is -1.
+
+   ``d.timetuple()`` is equivalent to::
+
+     time.struct_time((d.year, d.month, d.day, 0, 0, 0, d.weekday(), yday, -1))
+
+   where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1``
+   is the day number within the current year starting with ``1`` for January 1st.
 
 
 .. method:: date.toordinal()
 
    Return the proleptic Gregorian ordinal of the date, where January 1 of year 1
-   has ordinal 1.  For any :class:`date` object *d*,
+   has ordinal 1. For any :class:`date` object *d*,
    ``date.fromordinal(d.toordinal()) == d``.
 
 
@@ -592,26 +672,31 @@ Instance methods:
 
    Return a 3-tuple, (ISO year, ISO week number, ISO weekday).
 
-   The ISO calendar is a widely used variant of the Gregorian calendar. See
-   https://www.staff.science.uu.nl/~gent0113/calendar/isocalendar.htm for a good
-   explanation.
+   The ISO calendar is a widely used variant of the Gregorian calendar. [#]_
 
    The ISO year consists of 52 or 53 full weeks, and where a week starts on a
-   Monday and ends on a Sunday.  The first week of an ISO year is the first
+   Monday and ends on a Sunday. The first week of an ISO year is the first
    (Gregorian) calendar week of a year containing a Thursday. This is called week
    number 1, and the ISO year of that Thursday is the same as its Gregorian year.
 
    For example, 2004 begins on a Thursday, so the first week of ISO year 2004
-   begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004, so that
-   ``date(2003, 12, 29).isocalendar() == (2004, 1, 1)`` and ``date(2004, 1,
-   4).isocalendar() == (2004, 1, 7)``.
+   begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004::
 
+       >>> from datetime import date
+       >>> date(2003, 12, 29).isocalendar()
+       (2004, 1, 1)
+       >>> date(2004, 1, 4).isocalendar()
+       (2004, 1, 7)
 
 .. method:: date.isoformat()
 
-   Return a string representing the date in ISO 8601 format, 'YYYY-MM-DD'.  For
-   example, ``date(2002, 12, 4).isoformat() == '2002-12-04'``.
+   Return a string representing the date in ISO 8601 format, ``YYYY-MM-DD``::
 
+       >>> from datetime import date
+       >>> date(2002, 12, 4).isoformat()
+       '2002-12-04'
+
+   This is the inverse of :meth:`date.fromisoformat`.
 
 .. method:: date.__str__()
 
@@ -620,9 +705,17 @@ Instance methods:
 
 .. method:: date.ctime()
 
-   Return a string representing the date, for example ``date(2002, 12,
-   4).ctime() == 'Wed Dec 4 00:00:00 2002'``. ``d.ctime()`` is equivalent to
-   ``time.ctime(time.mktime(d.timetuple()))`` on platforms where the native C
+   Return a string representing the date::
+
+       >>> from datetime import date
+       >>> date(2002, 12, 4).ctime()
+       'Wed Dec  4 00:00:00 2002'
+
+   ``d.ctime()`` is equivalent to::
+
+     time.ctime(time.mktime(d.timetuple()))
+
+   on platforms where the native C
    :c:func:`ctime` function (which :func:`time.ctime` invokes, but which
    :meth:`date.ctime` does not invoke) conforms to the C standard.
 
@@ -643,6 +736,8 @@ Instance methods:
    complete list of formatting directives, see
    :ref:`strftime-strptime-behavior`.
 
+Examples of Usage: :class:`date`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Example of counting days to an event::
 
@@ -662,7 +757,7 @@ Example of counting days to an event::
     >>> time_to_birthday.days
     202
 
-Example of working with :class:`date`:
+More examples of working with :class:`date`:
 
 .. doctest::
 
@@ -670,6 +765,20 @@ Example of working with :class:`date`:
     >>> d = date.fromordinal(730920) # 730920th day after 1. 1. 0001
     >>> d
     datetime.date(2002, 3, 11)
+
+    >>> # Methods related to formatting string output
+    >>> d.isoformat()
+    '2002-03-11'
+    >>> d.strftime("%d/%m/%y")
+    '11/03/02'
+    >>> d.strftime("%A %d. %B %Y")
+    'Monday 11. March 2002'
+    >>> d.ctime()
+    'Mon Mar 11 00:00:00 2002'
+    >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month")
+    'The day is 11, the month is March.'
+
+    >>> # Methods for to extracting 'components' under different calendars
     >>> t = d.timetuple()
     >>> for i in t:     # doctest: +SKIP
     ...     print(i)
@@ -688,14 +797,10 @@ Example of working with :class:`date`:
     2002                # ISO year
     11                  # ISO week number
     1                   # ISO day number ( 1 = Monday )
-    >>> d.isoformat()
-    '2002-03-11'
-    >>> d.strftime("%d/%m/%y")
-    '11/03/02'
-    >>> d.strftime("%A %d. %B %Y")
-    'Monday 11. March 2002'
-    >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month")
-    'The day is 11, the month is March.'
+
+    >>> # A date object is immutable; all operations produce a new object
+    >>> d.replace(year=2005)
+    datetime.date(2005, 3, 11)
 
 
 .. _datetime-datetime:
@@ -704,17 +809,18 @@ Example of working with :class:`date`:
 --------------------------
 
 A :class:`.datetime` object is a single object containing all the information
-from a :class:`date` object and a :class:`.time` object.  Like a :class:`date`
-object, :class:`.datetime` assumes the current Gregorian calendar extended in
-both directions; like a time object, :class:`.datetime` assumes there are exactly
-3600\*24 seconds in every day.
+from a :class:`date` object and a :class:`.time` object.
+
+Like a :class:`date` object, :class:`.datetime` assumes the current Gregorian
+calendar extended in both directions; like a :class:`.time` object,
+:class:`.datetime` assumes there are exactly 3600\*24 seconds in every day.
 
 Constructor:
 
 .. class:: datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0)
 
-   The year, month and day arguments are required.  *tzinfo* may be ``None``, or an
-   instance of a :class:`tzinfo` subclass.  The remaining arguments must be integers
+   The *year*, *month* and *day* arguments are required. *tzinfo* may be ``None``, or an
+   instance of a :class:`tzinfo` subclass. The remaining arguments must be integers
    in the following ranges:
 
    * ``MINYEAR <= year <= MAXYEAR``,
@@ -735,31 +841,43 @@ Other constructors, all class methods:
 
 .. classmethod:: datetime.today()
 
-   Return the current local datetime, with :attr:`.tzinfo` ``None``. This is
-   equivalent to ``datetime.fromtimestamp(time.time())``. See also :meth:`now`,
-   :meth:`fromtimestamp`.
+   Return the current local datetime, with :attr:`.tzinfo` ``None``.
+
+   Equivalent to::
+
+     datetime.fromtimestamp(time.time())
+
+   See also :meth:`now`, :meth:`fromtimestamp`.
 
+   This method is functionally equivalent to :meth:`now`, but without a
+   ``tz`` parameter.
 
 .. classmethod:: datetime.now(tz=None)
 
-   Return the current local date and time.  If optional argument *tz* is ``None``
+   Return the current local date and time.
+
+   If optional argument *tz* is ``None``
    or not specified, this is like :meth:`today`, but, if possible, supplies more
    precision than can be gotten from going through a :func:`time.time` timestamp
    (for example, this may be possible on platforms supplying the C
    :c:func:`gettimeofday` function).
 
    If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the
-   current date and time are converted to *tz*?s time zone.  In this case the
-   result is equivalent to ``tz.fromutc(datetime.utcnow().replace(tzinfo=tz))``.
+   current date and time are converted to *tz*?s time zone. In this case the
+   result is equivalent to::
+
+     tz.fromutc(datetime.utcnow().replace(tzinfo=tz))
+
    See also :meth:`today`, :meth:`utcnow`.
 
 
 .. classmethod:: datetime.utcnow()
 
-   Return the current UTC date and time, with :attr:`.tzinfo` ``None``. This is like
-   :meth:`now`, but returns the current UTC date and time, as a naive
-   :class:`.datetime` object.  An aware current UTC datetime can be obtained by
-   calling ``datetime.now(timezone.utc)``.  See also :meth:`now`.
+   Return the current UTC date and time, with :attr:`.tzinfo` ``None``.
+
+   This is like :meth:`now`, but returns the current UTC date and time, as a naive
+   :class:`.datetime` object. An aware current UTC datetime can be obtained by
+   calling ``datetime.now(timezone.utc)``. See also :meth:`now`.
 
 .. classmethod:: datetime.fromtimestamp(timestamp, tz=None)
 
@@ -769,9 +887,11 @@ Other constructors, all class methods:
    the returned :class:`.datetime` object is naive.
 
    If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the
-   timestamp is converted to *tz*?s time zone.  In this case the result is
-   equivalent to
-   ``tz.fromutc(datetime.utcfromtimestamp(timestamp).replace(tzinfo=tz))``.
+   timestamp is converted to *tz*?s time zone.
+
+   In this case the result is equivalent to::
+
+     tz.fromutc(datetime.utcfromtimestamp(timestamp).replace(tzinfo=tz))
 
    :meth:`fromtimestamp` may raise :exc:`OverflowError`, if the timestamp is out of
    the range of values supported by the platform C :c:func:`localtime` or
@@ -796,7 +916,9 @@ Other constructors, all class methods:
 .. classmethod:: datetime.utcfromtimestamp(timestamp)
 
    Return the UTC :class:`.datetime` corresponding to the POSIX timestamp, with
-   :attr:`.tzinfo` ``None``. This may raise :exc:`OverflowError`, if the timestamp is
+   :attr:`.tzinfo` ``None``.  (The resulting object is naive.)
+
+   This may raise :exc:`OverflowError`, if the timestamp is
    out of the range of values supported by the platform C :c:func:`gmtime` function,
    and :exc:`OSError` on :c:func:`gmtime` failure.
    It's common for this to be restricted to years in 1970 through 2038.
@@ -824,7 +946,7 @@ Other constructors, all class methods:
 
    Return the :class:`.datetime` corresponding to the proleptic Gregorian ordinal,
    where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1
-   <= ordinal <= datetime.max.toordinal()``.  The hour, minute, second and
+   <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and
    microsecond of the result are all 0, and :attr:`.tzinfo` is ``None``.
 
 
@@ -832,13 +954,13 @@ Other constructors, all class methods:
 
    Return a new :class:`.datetime` object whose date components are equal to the
    given :class:`date` object's, and whose time components
-   are equal to the given :class:`.time` object's.  If the *tzinfo*
+   are equal to the given :class:`.time` object's. If the *tzinfo*
    argument is provided, its value is used to set the :attr:`.tzinfo` attribute
    of the result, otherwise the :attr:`~.time.tzinfo` attribute of the *time* argument
    is used.
 
    For any :class:`.datetime` object *d*,
-   ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``.  If date is a
+   ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``. If date is a
    :class:`.datetime` object, its time components and :attr:`.tzinfo` attributes
    are ignored.
 
@@ -848,25 +970,46 @@ Other constructors, all class methods:
 
 .. classmethod:: datetime.fromisoformat(date_string)
 
-  Return a :class:`datetime` corresponding to a *date_string* in one of the
-  formats emitted by :meth:`date.isoformat` and :meth:`datetime.isoformat`.
-  Specifically, this function supports strings in the format(s)
-  ``YYYY-MM-DD[*HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]]``,
-  where ``*`` can match any single character.
+   Return a :class:`.datetime` corresponding to a *date_string* in one of the
+   formats emitted by :meth:`date.isoformat` and :meth:`datetime.isoformat`.
+
+   Specifically, this function supports strings in the format:
+
+   .. code-block:: none
+
+      YYYY-MM-DD[*HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]]
 
-  .. caution::
+   where ``*`` can match any single character.
 
-    This does not support parsing arbitrary ISO 8601 strings - it is only intended
-    as the inverse operation of :meth:`datetime.isoformat`. A more full-featured
-    ISO 8601 parser, ``dateutil.parser.isoparse`` is available in the third-party package
-    `dateutil <https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.isoparse>`_.
+   .. caution::
 
-  .. versionadded:: 3.7
+     This does *not* support parsing arbitrary ISO 8601 strings - it is only intended
+     as the inverse operation of :meth:`datetime.isoformat`. A more full-featured
+     ISO 8601 parser, ``dateutil.parser.isoparse`` is available in the third-party package
+     `dateutil <https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.isoparse>`__.
+     This does not support parsing arbitrary ISO 8601 strings - it is only intended
+     as the inverse operation of :meth:`datetime.isoformat`.
 
+   Examples::
+
+       >>> from datetime import datetime
+       >>> datetime.fromisoformat('2011-11-04')
+       datetime.datetime(2011, 11, 4, 0, 0)
+       >>> datetime.fromisoformat('2011-11-04T00:05:23')
+       datetime.datetime(2011, 11, 4, 0, 5, 23)
+       >>> datetime.fromisoformat('2011-11-04 00:05:23.283')
+       datetime.datetime(2011, 11, 4, 0, 5, 23, 283000)
+       >>> datetime.fromisoformat('2011-11-04 00:05:23.283+00:00')
+       datetime.datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=datetime.timezone.utc)
+       >>> datetime.fromisoformat('2011-11-04T00:05:23+04:00')   # doctest: +NORMALIZE_WHITESPACE
+       datetime.datetime(2011, 11, 4, 0, 5, 23,
+           tzinfo=datetime.timezone(datetime.timedelta(seconds=14400)))
+
+   .. versionadded:: 3.7
 
 .. classmethod:: datetime.fromisocalendar(year, week, day)
 
-   Return a :class:`datetime` corresponding to the ISO calendar date specified
+   Return a :class:`.datetime` corresponding to the ISO calendar date specified
    by year, week and day. The non-date components of the datetime are populated
    with their normal default values. This is the inverse of the function
    :meth:`datetime.isocalendar`.
@@ -876,8 +1019,13 @@ Other constructors, all class methods:
 .. classmethod:: datetime.strptime(date_string, format)
 
    Return a :class:`.datetime` corresponding to *date_string*, parsed according to
-   *format*.  This is equivalent to ``datetime(*(time.strptime(date_string,
-   format)[0:6]))``. :exc:`ValueError` is raised if the date_string and format
+   *format*.
+
+   This is equivalent to::
+
+     datetime(*(time.strptime(date_string, format)[0:6]))
+
+   :exc:`ValueError` is raised if the date_string and format
    can't be parsed by :func:`time.strptime` or if it returns a value which isn't a
    time tuple. For a complete list of formatting directives, see
    :ref:`strftime-strptime-behavior`.
@@ -949,7 +1097,7 @@ Instance attributes (read-only):
 
 .. attribute:: datetime.fold
 
-   In ``[0, 1]``.  Used to disambiguate wall times during a repeated interval.  (A
+   In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A
    repeated interval occurs when clocks are rolled back at the end of daylight saving
    time or when the UTC offset for the current zone is decreased for political reasons.)
    The value 0 (1) represents the earlier (later) of the two moments with the same wall
@@ -974,7 +1122,7 @@ Supported operations:
 
 (1)
    datetime2 is a duration of timedelta removed from datetime1, moving forward in
-   time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0.  The
+   time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0. The
    result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and
    datetime2 - datetime1 == timedelta after. :exc:`OverflowError` is raised if
    datetime2.year would be smaller than :const:`MINYEAR` or larger than
@@ -988,16 +1136,16 @@ Supported operations:
 
 (3)
    Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if
-   both operands are naive, or if both are aware.  If one is aware and the other is
+   both operands are naive, or if both are aware. If one is aware and the other is
    naive, :exc:`TypeError` is raised.
 
    If both are naive, or both are aware and have the same :attr:`~.datetime.tzinfo` attribute,
    the :attr:`~.datetime.tzinfo` attributes are ignored, and the result is a :class:`timedelta`
-   object *t* such that ``datetime2 + t == datetime1``.  No time zone adjustments
+   object *t* such that ``datetime2 + t == datetime1``. No time zone adjustments
    are done in this case.
 
    If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts
-   as if *a* and *b* were first converted to naive UTC datetimes first.  The
+   as if *a* and *b* were first converted to naive UTC datetimes first. The
    result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None)
    - b.utcoffset())`` except that the implementation never overflows.
 
@@ -1006,34 +1154,31 @@ Supported operations:
    *datetime2* in time.
 
    If one comparand is naive and the other is aware, :exc:`TypeError`
-   is raised if an order comparison is attempted.  For equality
+   is raised if an order comparison is attempted. For equality
    comparisons, naive instances are never equal to aware instances.
 
    If both comparands are aware, and have the same :attr:`~.datetime.tzinfo` attribute, the
    common :attr:`~.datetime.tzinfo` attribute is ignored and the base datetimes are
-   compared.  If both comparands are aware and have different :attr:`~.datetime.tzinfo`
+   compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo`
    attributes, the comparands are first adjusted by subtracting their UTC
    offsets (obtained from ``self.utcoffset()``).
 
    .. versionchanged:: 3.3
-      Equality comparisons between naive and aware :class:`.datetime`
+      Equality comparisons between aware and naive :class:`.datetime`
       instances don't raise :exc:`TypeError`.
 
    .. note::
 
       In order to stop comparison from falling back to the default scheme of comparing
       object addresses, datetime comparison normally raises :exc:`TypeError` if the
-      other comparand isn't also a :class:`.datetime` object.  However,
+      other comparand isn't also a :class:`.datetime` object. However,
       ``NotImplemented`` is returned instead if the other comparand has a
-      :meth:`timetuple` attribute.  This hook gives other kinds of date objects a
-      chance at implementing mixed-type comparison.  If not, when a :class:`.datetime`
+      :meth:`timetuple` attribute. This hook gives other kinds of date objects a
+      chance at implementing mixed-type comparison. If not, when a :class:`.datetime`
       object is compared to an object of a different type, :exc:`TypeError` is raised
-      unless the comparison is ``==`` or ``!=``.  The latter cases return
+      unless the comparison is ``==`` or ``!=``. The latter cases return
       :const:`False` or :const:`True`, respectively.
 
-:class:`.datetime` objects can be used as dictionary keys. In Boolean contexts,
-all :class:`.datetime` objects are considered to be true.
-
 Instance methods:
 
 .. method:: datetime.date()
@@ -1044,7 +1189,7 @@ Instance methods:
 .. method:: datetime.time()
 
    Return :class:`.time` object with same hour, minute, second, microsecond and fold.
-   :attr:`.tzinfo` is ``None``.  See also method :meth:`timetz`.
+   :attr:`.tzinfo` is ``None``. See also method :meth:`timetz`.
 
    .. versionchanged:: 3.6
       The fold value is copied to the returned :class:`.time` object.
@@ -1053,7 +1198,7 @@ Instance methods:
 .. method:: datetime.timetz()
 
    Return :class:`.time` object with same hour, minute, second, microsecond, fold, and
-   tzinfo attributes.  See also method :meth:`time`.
+   tzinfo attributes. See also method :meth:`time`.
 
    .. versionchanged:: 3.6
       The fold value is copied to the returned :class:`.time` object.
@@ -1064,7 +1209,7 @@ Instance methods:
    tzinfo=self.tzinfo, * fold=0)
 
    Return a datetime with the same attributes, except for those attributes given
-   new values by whichever keyword arguments are specified.  Note that
+   new values by whichever keyword arguments are specified. Note that
    ``tzinfo=None`` can be specified to create a naive datetime from an aware
    datetime with no conversion of date and time data.
 
@@ -1079,11 +1224,11 @@ Instance methods:
    *self*, but in *tz*'s local time.
 
    If provided, *tz* must be an instance of a :class:`tzinfo` subclass, and its
-   :meth:`utcoffset` and :meth:`dst` methods must not return ``None``.  If *self*
+   :meth:`utcoffset` and :meth:`dst` methods must not return ``None``. If *self*
    is naive, it is presumed to represent time in the system timezone.
 
    If called without arguments (or with ``tz=None``) the system local
-   timezone is assumed for the target timezone.  The ``.tzinfo`` attribute of the converted
+   timezone is assumed for the target timezone. The ``.tzinfo`` attribute of the converted
    datetime instance will be set to an instance of :class:`timezone`
    with the zone name and offset obtained from the OS.
 
@@ -1094,7 +1239,7 @@ Instance methods:
    the same date and time data as ``dt - dt.utcoffset()``.
 
    If you merely want to attach a time zone object *tz* to a datetime *dt* without
-   adjustment of date and time data, use ``dt.replace(tzinfo=tz)``.  If you
+   adjustment of date and time data, use ``dt.replace(tzinfo=tz)``. If you
    merely want to remove the time zone object from an aware datetime *dt* without
    conversion of date and time data, use ``dt.replace(tzinfo=None)``.
 
@@ -1148,25 +1293,31 @@ Instance methods:
 .. method:: datetime.timetuple()
 
    Return a :class:`time.struct_time` such as returned by :func:`time.localtime`.
-   ``d.timetuple()`` is equivalent to ``time.struct_time((d.year, d.month, d.day,
-   d.hour, d.minute, d.second, d.weekday(), yday, dst))``, where ``yday =
-   d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` is the day number within
-   the current year starting with ``1`` for January 1st. The :attr:`tm_isdst` flag
-   of the result is set according to the :meth:`dst` method: :attr:`.tzinfo` is
-   ``None`` or :meth:`dst` returns ``None``, :attr:`tm_isdst` is set to ``-1``;
-   else if :meth:`dst` returns a non-zero value, :attr:`tm_isdst` is set to ``1``;
-   else :attr:`tm_isdst` is set to ``0``.
+
+   ``d.timetuple()`` is equivalent to::
+
+     time.struct_time((d.year, d.month, d.day,
+                       d.hour, d.minute, d.second,
+                       d.weekday(), yday, dst))
+
+   where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1``
+   is the day number within the current year starting with ``1`` for January
+   1st. The :attr:`tm_isdst` flag of the result is set according to the
+   :meth:`dst` method: :attr:`.tzinfo` is ``None`` or :meth:`dst` returns
+   ``None``, :attr:`tm_isdst` is set to ``-1``; else if :meth:`dst` returns a
+   non-zero value, :attr:`tm_isdst` is set to ``1``; else :attr:`tm_isdst` is
+   set to ``0``.
 
 
 .. method:: datetime.utctimetuple()
 
    If :class:`.datetime` instance *d* is naive, this is the same as
    ``d.timetuple()`` except that :attr:`tm_isdst` is forced to 0 regardless of what
-   ``d.dst()`` returns.  DST is never in effect for a UTC time.
+   ``d.dst()`` returns. DST is never in effect for a UTC time.
 
    If *d* is aware, *d* is normalized to UTC time, by subtracting
    ``d.utcoffset()``, and a :class:`time.struct_time` for the
-   normalized time is returned.  :attr:`tm_isdst` is forced to 0. Note
+   normalized time is returned. :attr:`tm_isdst` is forced to 0. Note
    that an :exc:`OverflowError` may be raised if *d*.year was
    ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year
    boundary.
@@ -1174,18 +1325,18 @@ Instance methods:
 
 .. method:: datetime.toordinal()
 
-   Return the proleptic Gregorian ordinal of the date.  The same as
+   Return the proleptic Gregorian ordinal of the date. The same as
    ``self.date().toordinal()``.
 
 .. method:: datetime.timestamp()
 
    Return POSIX timestamp corresponding to the :class:`.datetime`
-   instance.  The return value is a :class:`float` similar to that
+   instance. The return value is a :class:`float` similar to that
    returned by :func:`time.time`.
 
    Naive :class:`.datetime` instances are assumed to represent local
    time and this method relies on the platform C :c:func:`mktime`
-   function to perform the conversion.  Since :class:`.datetime`
+   function to perform the conversion. Since :class:`.datetime`
    supports wider range of values than :c:func:`mktime` on many
    platforms, this method may raise :exc:`OverflowError` for times far
    in the past or far in the future.
@@ -1204,7 +1355,7 @@ Instance methods:
    .. note::
 
       There is no method to obtain the POSIX timestamp directly from a
-      naive :class:`.datetime` instance representing UTC time.  If your
+      naive :class:`.datetime` instance representing UTC time. If your
       application uses this convention and your system timezone is not
       set to UTC, you can obtain the POSIX timestamp by supplying
       ``tzinfo=timezone.utc``::
@@ -1230,30 +1381,45 @@ Instance methods:
 
 .. method:: datetime.isocalendar()
 
-   Return a 3-tuple, (ISO year, ISO week number, ISO weekday).  The same as
+   Return a 3-tuple, (ISO year, ISO week number, ISO weekday). The same as
    ``self.date().isocalendar()``.
 
 
 .. method:: datetime.isoformat(sep='T', timespec='auto')
 
-   Return a string representing the date and time in ISO 8601 format,
-   YYYY-MM-DDTHH:MM:SS.ffffff or, if :attr:`microsecond` is 0,
-   YYYY-MM-DDTHH:MM:SS
+   Return a string representing the date and time in ISO 8601 format:
+
+   - ``YYYY-MM-DDTHH:MM:SS.ffffff``, if :attr:`microsecond` is not 0
+   - ``YYYY-MM-DDTHH:MM:SS``, if :attr:`microsecond` is 0
 
    If :meth:`utcoffset` does not return ``None``, a string is
    appended, giving the UTC offset:
-   YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]] or, if :attr:`microsecond`
-   is 0 YYYY-MM-DDTHH:MM:SS+HH:MM[:SS[.ffffff]].
+
+   - ``YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond`
+     is not 0
+   - ``YYYY-MM-DDTHH:MM:SS+HH:MM[:SS[.ffffff]]``,  if :attr:`microsecond` is 0
+
+   Examples::
+
+       >>> from datetime import datetime, timezone
+       >>> datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat()
+       '2019-05-18T15:17:08.132263'
+       >>> datetime(2019, 5, 18, 15, 17, tzinfo=timezone.utc).isoformat()
+       '2019-05-18T15:17:00+00:00'
 
    The optional argument *sep* (default ``'T'``) is a one-character separator,
-   placed between the date and time portions of the result.  For example,
+   placed between the date and time portions of the result. For example::
 
       >>> from datetime import tzinfo, timedelta, datetime
       >>> class TZ(tzinfo):
-      ...     def utcoffset(self, dt): return timedelta(minutes=-399)
+      ...     """A time zone with an arbitrary, constant -06:39 offset."""
+      ...     def utcoffset(self, dt):
+      ...         return timedelta(hours=-6, minutes=-39)
       ...
       >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ')
       '2002-12-25 00:00:00-06:39'
+      >>> datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat()
+      '2009-11-27T00:00:00.000100-06:39'
 
    The optional argument *timespec* specifies the number of additional
    components of the time to include (the default is ``'auto'``).
@@ -1261,19 +1427,19 @@ Instance methods:
 
    - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0,
      same as ``'microseconds'`` otherwise.
-   - ``'hours'``: Include the :attr:`hour` in the two-digit HH format.
-   - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format.
+   - ``'hours'``: Include the :attr:`hour` in the two-digit ``HH`` format.
+   - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in ``HH:MM`` format.
    - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second`
-     in HH:MM:SS format.
+     in ``HH:MM:SS`` format.
    - ``'milliseconds'``: Include full time, but truncate fractional second
-     part to milliseconds. HH:MM:SS.sss format.
-   - ``'microseconds'``: Include full time in HH:MM:SS.ffffff format.
+     part to milliseconds. ``HH:MM:SS.sss`` format.
+   - ``'microseconds'``: Include full time in ``HH:MM:SS.ffffff`` format.
 
    .. note::
 
       Excluded time components are truncated, not rounded.
 
-   :exc:`ValueError` will be raised on an invalid *timespec* argument.
+   :exc:`ValueError` will be raised on an invalid *timespec* argument::
 
 
       >>> from datetime import datetime
@@ -1295,48 +1461,64 @@ Instance methods:
 
 .. method:: datetime.ctime()
 
-   Return a string representing the date and time, for example ``datetime(2002, 12,
-   4, 20, 30, 40).ctime() == 'Wed Dec  4 20:30:40 2002'``. ``d.ctime()`` is
-   equivalent to ``time.ctime(time.mktime(d.timetuple()))`` on platforms where the
-   native C :c:func:`ctime` function (which :func:`time.ctime` invokes, but which
-   :meth:`datetime.ctime` does not invoke) conforms to the C standard.
+   Return a string representing the date and time::
+
+       >>> from datetime import datetime
+       >>> datetime(2002, 12, 4, 20, 30, 40).ctime()
+       'Wed Dec  4 20:30:40 2002'
+
+   The output string will *not* include time zone information, regardless
+   of whether the input is aware or naive.
 
+   ``d.ctime()`` is equivalent to::
+
+     time.ctime(time.mktime(d.timetuple()))
+
+   on platforms where the native C :c:func:`ctime` function
+   (which :func:`time.ctime` invokes, but which
+   :meth:`datetime.ctime` does not invoke) conforms to the C standard.
 
 .. method:: datetime.strftime(format)
 
    Return a string representing the date and time, controlled by an explicit format
-   string.  For a complete list of formatting directives, see
+   string. For a complete list of formatting directives, see
    :ref:`strftime-strptime-behavior`.
 
 
 .. method:: datetime.__format__(format)
 
-   Same as :meth:`.datetime.strftime`.  This makes it possible to specify a format
+   Same as :meth:`.datetime.strftime`. This makes it possible to specify a format
    string for a :class:`.datetime` object in :ref:`formatted string
-   literals <f-strings>` and when using :meth:`str.format`.  For a
+   literals <f-strings>` and when using :meth:`str.format`. For a
    complete list of formatting directives, see
    :ref:`strftime-strptime-behavior`.
 
+Examples of Usage: :class:`.datetime`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Examples of working with datetime objects:
+Examples of working with :class:`~datetime.datetime` objects:
 
 .. doctest::
 
     >>> from datetime import datetime, date, time
+
     >>> # Using datetime.combine()
     >>> d = date(2005, 7, 14)
     >>> t = time(12, 30)
     >>> datetime.combine(d, t)
     datetime.datetime(2005, 7, 14, 12, 30)
+
     >>> # Using datetime.now() or datetime.utcnow()
     >>> datetime.now()   # doctest: +SKIP
     datetime.datetime(2007, 12, 6, 16, 29, 43, 79043)   # GMT +1
     >>> datetime.utcnow()   # doctest: +SKIP
     datetime.datetime(2007, 12, 6, 15, 29, 43, 79060)
+
     >>> # Using datetime.strptime()
     >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M")
     >>> dt
     datetime.datetime(2006, 11, 21, 16, 30)
+
     >>> # Using datetime.timetuple() to get tuple of all attributes
     >>> tt = dt.timetuple()
     >>> for it in tt:   # doctest: +SKIP
@@ -1351,6 +1533,7 @@ Examples of working with datetime objects:
     1       # weekday (0 = Monday)
     325     # number of days since 1st January
     -1      # dst - method tzinfo.dst() returned None
+
     >>> # Date in ISO format
     >>> ic = dt.isocalendar()
     >>> for it in ic:   # doctest: +SKIP
@@ -1359,89 +1542,95 @@ Examples of working with datetime objects:
     2006    # ISO year
     47      # ISO week
     2       # ISO weekday
-    >>> # Formatting datetime
+
+    >>> # Formatting a datetime
     >>> dt.strftime("%A, %d. %B %Y %I:%M%p")
     'Tuesday, 21. November 2006 04:30PM'
     >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time")
     'The day is 21, the month is November, the time is 04:30PM.'
 
-Using datetime with tzinfo:
-
-    >>> from datetime import timedelta, datetime, tzinfo, timezone
-    >>> class KabulTz(tzinfo):
-    ...     # Kabul used +4 until 1945, when they moved to +4:30
-    ...     UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc)
-    ...     def utcoffset(self, dt):
-    ...         if dt.year < 1945:
-    ...             return timedelta(hours=4)
-    ...         elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30):
-    ...             # If dt falls in the imaginary range, use fold to decide how
-    ...             # to resolve. See PEP495
-    ...             return timedelta(hours=4, minutes=(30 if dt.fold else 0))
-    ...         else:
-    ...             return timedelta(hours=4, minutes=30)
-    ...
-    ...     def fromutc(self, dt):
-    ...         # A custom implementation is required for fromutc as
-    ...         # the input to this function is a datetime with utc values
-    ...         # but with a tzinfo set to self
-    ...         # See datetime.astimezone or fromtimestamp
-    ...
-    ...         # Follow same validations as in datetime.tzinfo
-    ...         if not isinstance(dt, datetime):
-    ...             raise TypeError("fromutc() requires a datetime argument")
-    ...         if dt.tzinfo is not self:
-    ...             raise ValueError("dt.tzinfo is not self")
-    ...
-    ...         if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE:
-    ...             return dt + timedelta(hours=4, minutes=30)
-    ...         else:
-    ...             return dt + timedelta(hours=4)
-    ...
-    ...     def dst(self, dt):
-    ...         return timedelta(0)
-    ...
-    ...     def tzname(self, dt):
-    ...         if dt >= self.UTC_MOVE_DATE:
-    ...             return "+04:30"
-    ...         else:
-    ...             return "+04"
-    ...
-    ...     def  __repr__(self):
-    ...         return f"{self.__class__.__name__}()"
-    ...
-    >>> tz1 = KabulTz()
-    >>> # Datetime before the change
-    >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1)
-    >>> print(dt1.utcoffset())
-    4:00:00
-    >>> # Datetime after the change
-    >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1)
-    >>> print(dt2.utcoffset())
-    4:30:00
-    >>> # Convert datetime to another time zone
-    >>> dt3 = dt2.astimezone(timezone.utc)
-    >>> dt3
-    datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc)
-    >>> dt2
-    datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz())
-    >>> dt2.utctimetuple() == dt3.utctimetuple()
-    True
-
-
+The example below defines a :class:`tzinfo` subclass capturing time zone
+information for Kabul, Afghanistan, which used +4 UTC until 1945
+and then +4:30 UTC thereafter::
+
+   from datetime import timedelta, datetime, tzinfo, timezone
+
+   class KabulTz(tzinfo):
+       # Kabul used +4 until 1945, when they moved to +4:30
+       UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc)
+
+       def utcoffset(self, dt):
+           if dt.year < 1945:
+               return timedelta(hours=4)
+           elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30):
+               # An ambiguous ("imaginary") half-hour range representing
+               # a 'fold' in time due to the shift from +4 to +4:30.
+               # If dt falls in the imaginary range, use fold to decide how
+               # to resolve. See PEP495.
+               return timedelta(hours=4, minutes=(30 if dt.fold else 0))
+           else:
+               return timedelta(hours=4, minutes=30)
+
+       def fromutc(self, dt):
+           # Follow same validations as in datetime.tzinfo
+           if not isinstance(dt, datetime):
+               raise TypeError("fromutc() requires a datetime argument")
+           if dt.tzinfo is not self:
+               raise ValueError("dt.tzinfo is not self")
+
+           # A custom implementation is required for fromutc as
+           # the input to this function is a datetime with utc values
+           # but with a tzinfo set to self.
+           # See datetime.astimezone or fromtimestamp.
+           if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE:
+               return dt + timedelta(hours=4, minutes=30)
+           else:
+               return dt + timedelta(hours=4)
+
+       def dst(self, dt):
+           # Kabul does not observe daylight saving time.
+           return timedelta(0)
+
+       def tzname(self, dt):
+           if dt >= self.UTC_MOVE_DATE:
+               return "+04:30"
+           return "+04"
+
+Usage of ``KabulTz`` from above::
+
+   >>> tz1 = KabulTz()
+
+   >>> # Datetime before the change
+   >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1)
+   >>> print(dt1.utcoffset())
+   4:00:00
+
+   >>> # Datetime after the change
+   >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1)
+   >>> print(dt2.utcoffset())
+   4:30:00
+
+   >>> # Convert datetime to another time zone
+   >>> dt3 = dt2.astimezone(timezone.utc)
+   >>> dt3
+   datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc)
+   >>> dt2
+   datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz())
+   >>> dt2.utctimetuple() == dt3.utctimetuple()
+   True
 
 .. _datetime-time:
 
 :class:`.time` Objects
 ----------------------
 
-A time object represents a (local) time of day, independent of any particular
+A :class:`time` object represents a (local) time of day, independent of any particular
 day, and subject to adjustment via a :class:`tzinfo` object.
 
 .. class:: time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0)
 
-   All arguments are optional.  *tzinfo* may be ``None``, or an instance of a
-   :class:`tzinfo` subclass.  The remaining arguments must be integers in the
+   All arguments are optional. *tzinfo* may be ``None``, or an instance of a
+   :class:`tzinfo` subclass. The remaining arguments must be integers in the
    following ranges:
 
    * ``0 <= hour < 24``,
@@ -1450,7 +1639,7 @@ day, and subject to adjustment via a :class:`tzinfo` object.
    * ``0 <= microsecond < 1000000``,
    * ``fold in [0, 1]``.
 
-   If an argument outside those ranges is given, :exc:`ValueError` is raised.  All
+   If an argument outside those ranges is given, :exc:`ValueError` is raised. All
    default to ``0`` except *tzinfo*, which defaults to :const:`None`.
 
 Class attributes:
@@ -1503,7 +1692,7 @@ Instance attributes (read-only):
 
 .. attribute:: time.fold
 
-   In ``[0, 1]``.  Used to disambiguate wall times during a repeated interval.  (A
+   In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A
    repeated interval occurs when clocks are rolled back at the end of daylight saving
    time or when the UTC offset for the current zone is decreased for political reasons.)
    The value 0 (1) represents the earlier (later) of the two moments with the same wall
@@ -1511,38 +1700,32 @@ Instance attributes (read-only):
 
    .. versionadded:: 3.6
 
+:class:`.time` objects support comparison of :class:`.time` to :class:`.time`,
+where *a* is considered less
+than *b* when *a* precedes *b* in time. If one comparand is naive and the other
+is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality
+comparisons, naive instances are never equal to aware instances.
 
-Supported operations:
-
-* comparison of :class:`.time` to :class:`.time`, where *a* is considered less
-  than *b* when *a* precedes *b* in time.  If one comparand is naive and the other
-  is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality
-  comparisons, naive instances are never equal to aware instances.
+If both comparands are aware, and have
+the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is
+ignored and the base times are compared. If both comparands are aware and
+have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by
+subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order
+to stop mixed-type comparisons from falling back to the default comparison by
+object address, when a :class:`.time` object is compared to an object of a
+different type, :exc:`TypeError` is raised unless the comparison is ``==`` or
+``!=``. The latter cases return :const:`False` or :const:`True`, respectively.
 
-  If both comparands are aware, and have
-  the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is
-  ignored and the base times are compared.  If both comparands are aware and
-  have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by
-  subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order
-  to stop mixed-type comparisons from falling back to the default comparison by
-  object address, when a :class:`.time` object is compared to an object of a
-  different type, :exc:`TypeError` is raised unless the comparison is ``==`` or
-  ``!=``.  The latter cases return :const:`False` or :const:`True`, respectively.
+.. versionchanged:: 3.3
+  Equality comparisons between aware and naive :class:`~datetime.time` instances
+  don't raise :exc:`TypeError`.
 
-  .. versionchanged:: 3.3
-     Equality comparisons between naive and aware :class:`~datetime.time` instances
-     don't raise :exc:`TypeError`.
-
-* hash, use as dict key
-
-* efficient pickling
-
-In boolean contexts, a :class:`.time` object is always considered to be true.
+In Boolean contexts, a :class:`.time` object is always considered to be true.
 
 .. versionchanged:: 3.5
    Before Python 3.5, a :class:`.time` object was considered to be false if it
-   represented midnight in UTC.  This behavior was considered obscure and
-   error-prone and has been removed in Python 3.5.  See :issue:`13936` for full
+   represented midnight in UTC. This behavior was considered obscure and
+   error-prone and has been removed in Python 3.5. See :issue:`13936` for full
    details.
 
 
@@ -1550,16 +1733,30 @@ Other constructor:
 
 .. classmethod:: time.fromisoformat(time_string)
 
-  Return a :class:`time` corresponding to a *time_string* in one of the
-  formats emitted by :meth:`time.isoformat`. Specifically, this function supports
-  strings in the format(s) ``HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]``.
+   Return a :class:`.time` corresponding to a *time_string* in one of the
+   formats emitted by :meth:`time.isoformat`. Specifically, this function supports
+   strings in the format:
+
+   .. code-block:: none
+
+      HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]
+
+   .. caution::
 
-  .. caution::
+     This does *not* support parsing arbitrary ISO 8601 strings. It is only
+     intended as the inverse operation of :meth:`time.isoformat`.
 
-    This does not support parsing arbitrary ISO 8601 strings - it is only intended
-    as the inverse operation of :meth:`time.isoformat`.
+   Examples::
 
-  .. versionadded:: 3.7
+       >>> from datetime import time
+       >>> time.fromisoformat('04:23:01')
+       datetime.time(4, 23, 1)
+       >>> time.fromisoformat('04:23:01.000384')
+       datetime.time(4, 23, 1, 384)
+       >>> time.fromisoformat('04:23:01+04:00')
+       datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400)))
+
+   .. versionadded:: 3.7
 
 
 Instance methods:
@@ -1568,7 +1765,7 @@ Instance methods:
    microsecond=self.microsecond, tzinfo=self.tzinfo, * fold=0)
 
    Return a :class:`.time` with the same value, except for those attributes given
-   new values by whichever keyword arguments are specified.  Note that
+   new values by whichever keyword arguments are specified. Note that
    ``tzinfo=None`` can be specified to create a naive :class:`.time` from an
    aware :class:`.time`, without conversion of the time data.
 
@@ -1578,10 +1775,12 @@ Instance methods:
 
 .. method:: time.isoformat(timespec='auto')
 
-   Return a string representing the time in ISO 8601 format, HH:MM:SS.ffffff or, if
-   :attr:`microsecond` is 0, HH:MM:SS If :meth:`utcoffset` does not return ``None``, a
-   string is appended, giving the UTC offset: HH:MM:SS.ffffff+HH:MM[:SS[.ffffff]]
-   or, if self.microsecond is 0, HH:MM:SS+HH:MM[:SS[.ffffff]].
+   Return a string representing the time in ISO 8601 format, one of:
+
+   - ``HH:MM:SS.ffffff``, if :attr:`microsecond` is not 0
+   - ``HH:MM:SS``, if :attr:`microsecond` is 0
+   - ``HH:MM:SS.ffffff+HH:MM[:SS[.ffffff]]``, if :meth:`utcoffset` does not return ``None``
+   - ``HH:MM:SS+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond` is 0 and :meth:`utcoffset` does not return ``None``
 
    The optional argument *timespec* specifies the number of additional
    components of the time to include (the default is ``'auto'``).
@@ -1589,13 +1788,13 @@ Instance methods:
 
    - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0,
      same as ``'microseconds'`` otherwise.
-   - ``'hours'``: Include the :attr:`hour` in the two-digit HH format.
-   - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format.
+   - ``'hours'``: Include the :attr:`hour` in the two-digit ``HH`` format.
+   - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in ``HH:MM`` format.
    - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second`
-     in HH:MM:SS format.
+     in ``HH:MM:SS`` format.
    - ``'milliseconds'``: Include full time, but truncate fractional second
-     part to milliseconds. HH:MM:SS.sss format.
-   - ``'microseconds'``: Include full time in HH:MM:SS.ffffff format.
+     part to milliseconds. ``HH:MM:SS.sss`` format.
+   - ``'microseconds'``: Include full time in ``HH:MM:SS.ffffff`` format.
 
    .. note::
 
@@ -1603,6 +1802,7 @@ Instance methods:
 
    :exc:`ValueError` will be raised on an invalid *timespec* argument.
 
+   Example::
 
       >>> from datetime import time
       >>> time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes')
@@ -1625,7 +1825,7 @@ Instance methods:
 .. method:: time.strftime(format)
 
    Return a string representing the time, controlled by an explicit format
-   string.  For a complete list of formatting directives, see
+   string. For a complete list of formatting directives, see
    :ref:`strftime-strptime-behavior`.
 
 
@@ -1633,7 +1833,7 @@ Instance methods:
 
    Same as :meth:`.time.strftime`. This makes it possible to specify a format string
    for a :class:`.time` object in :ref:`formatted string
-   literals <f-strings>` and when using :meth:`str.format`.  For a
+   literals <f-strings>` and when using :meth:`str.format`. For a
    complete list of formatting directives, see
    :ref:`strftime-strptime-behavior`.
 
@@ -1663,7 +1863,10 @@ Instance methods:
    ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't
    return ``None`` or a string object.
 
-Example:
+Examples of Usage: :class:`.time`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Examples of working with a :class:`.time` object::
 
     >>> from datetime import time, tzinfo, timedelta
     >>> class TZ1(tzinfo):
@@ -1699,12 +1902,8 @@ Example:
 .. class:: tzinfo()
 
    This is an abstract base class, meaning that this class should not be
-   instantiated directly.  You need to derive a concrete subclass, and (at least)
-   supply implementations of the standard :class:`tzinfo` methods needed by the
-   :class:`.datetime` methods you use.  The :mod:`datetime` module supplies
-   a simple concrete subclass of :class:`tzinfo`, :class:`timezone`, which can represent
-   timezones with fixed offset from UTC such as UTC itself or North American EST and
-   EDT.
+   instantiated directly.  Define a subclass of :class:`tzinfo` to capture
+   information about a particular time zone.
 
    An instance of (a concrete subclass of) :class:`tzinfo` can be passed to the
    constructors for :class:`.datetime` and :class:`.time` objects. The latter objects
@@ -1712,28 +1911,35 @@ Example:
    supports methods revealing offset of local time from UTC, the name of the time
    zone, and DST offset, all relative to a date or time object passed to them.
 
+   You need to derive a concrete subclass, and (at least)
+   supply implementations of the standard :class:`tzinfo` methods needed by the
+   :class:`.datetime` methods you use. The :mod:`datetime` module provides
+   :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can
+   represent timezones with fixed offset from UTC such as UTC itself or North
+   American EST and EDT.
+
    Special requirement for pickling:  A :class:`tzinfo` subclass must have an
-   :meth:`__init__` method that can be called with no arguments, else it can be
-   pickled but possibly not unpickled again.  This is a technical requirement that
+   :meth:`__init__` method that can be called with no arguments, otherwise it can be
+   pickled but possibly not unpickled again. This is a technical requirement that
    may be relaxed in the future.
 
    A concrete subclass of :class:`tzinfo` may need to implement the following
-   methods.  Exactly which methods are needed depends on the uses made of aware
-   :mod:`datetime` objects.  If in doubt, simply implement all of them.
+   methods. Exactly which methods are needed depends on the uses made of aware
+   :mod:`datetime` objects. If in doubt, simply implement all of them.
 
 
 .. method:: tzinfo.utcoffset(dt)
 
    Return offset of local time from UTC, as a :class:`timedelta` object that is
-   positive east of UTC.  If local time is
-   west of UTC, this should be negative.  Note that this is intended to be the
-   total offset from UTC; for example, if a :class:`tzinfo` object represents both
-   time zone and DST adjustments, :meth:`utcoffset` should return their sum.  If
-   the UTC offset isn't known, return ``None``.  Else the value returned must be a
-   :class:`timedelta` object strictly between ``-timedelta(hours=24)`` and
-   ``timedelta(hours=24)`` (the magnitude of the offset must be less
-   than one day).  Most implementations of :meth:`utcoffset` will probably look
-   like one of these two::
+   positive east of UTC. If local time is west of UTC, this should be negative.
+
+   This represents the *total* offset from UTC; for example, if a
+   :class:`tzinfo` object represents both time zone and DST adjustments,
+   :meth:`utcoffset` should return their sum. If the UTC offset isn't known,
+   return ``None``. Else the value returned must be a :class:`timedelta` object
+   strictly between ``-timedelta(hours=24)`` and ``timedelta(hours=24)``
+   (the magnitude of the offset must be less than one day). Most implementations
+   of :meth:`utcoffset` will probably look like one of these two::
 
       return CONSTANT                 # fixed-offset class
       return CONSTANT + self.dst(dt)  # daylight-aware class
@@ -1752,12 +1958,14 @@ Example:
 
    Return the daylight saving time (DST) adjustment, as a :class:`timedelta`
    object or
-   ``None`` if DST information isn't known.  Return ``timedelta(0)`` if DST is not
-   in effect. If DST is in effect, return the offset as a :class:`timedelta` object
+   ``None`` if DST information isn't known.
+
+   Return ``timedelta(0)`` if DST is not in effect.
+   If DST is in effect, return the offset as a :class:`timedelta` object
    (see :meth:`utcoffset` for details). Note that DST offset, if applicable, has
    already been added to the UTC offset returned by :meth:`utcoffset`, so there's
    no need to consult :meth:`dst` unless you're interested in obtaining DST info
-   separately.  For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo`
+   separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo`
    attribute's :meth:`dst` method to determine how the :attr:`tm_isdst` flag
    should be set, and :meth:`tzinfo.fromutc` calls :meth:`dst` to account for
    DST changes when crossing time zones.
@@ -1770,9 +1978,9 @@ Example:
    must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo ==
    tz``  For sane :class:`tzinfo` subclasses, this expression yields the time
    zone's "standard offset", which should not depend on the date or the time, but
-   only on geographic location.  The implementation of :meth:`datetime.astimezone`
+   only on geographic location. The implementation of :meth:`datetime.astimezone`
    relies on this, but cannot detect violations; it's the programmer's
-   responsibility to ensure it.  If a :class:`tzinfo` subclass cannot guarantee
+   responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee
    this, it may be able to override the default implementation of
    :meth:`tzinfo.fromutc` to work correctly with :meth:`astimezone` regardless.
 
@@ -1782,12 +1990,12 @@ Example:
           # a fixed-offset class:  doesn't account for DST
           return timedelta(0)
 
-   or ::
+   or::
 
       def dst(self, dt):
           # Code to set dston and dstoff to the time zone's DST
           # transition times based on the input dt.year, and expressed
-          # in standard local time.  Then
+          # in standard local time.
 
           if dston <= dt.replace(tzinfo=None) < dstoff:
               return timedelta(hours=1)
@@ -1804,9 +2012,9 @@ Example:
 
    Return the time zone name corresponding to the :class:`.datetime` object *dt*, as
    a string. Nothing about string names is defined by the :mod:`datetime` module,
-   and there's no requirement that it mean anything in particular.  For example,
+   and there's no requirement that it mean anything in particular. For example,
    "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all
-   valid replies.  Return ``None`` if a string name isn't known.  Note that this is
+   valid replies. Return ``None`` if a string name isn't known. Note that this is
    a method rather than a fixed string primarily because some :class:`tzinfo`
    subclasses will wish to return different names depending on the specific value
    of *dt* passed, especially if the :class:`tzinfo` class is accounting for
@@ -1816,20 +2024,20 @@ Example:
 
 
 These methods are called by a :class:`.datetime` or :class:`.time` object, in
-response to their methods of the same names.  A :class:`.datetime` object passes
+response to their methods of the same names. A :class:`.datetime` object passes
 itself as the argument, and a :class:`.time` object passes ``None`` as the
-argument.  A :class:`tzinfo` subclass's methods should therefore be prepared to
+argument. A :class:`tzinfo` subclass's methods should therefore be prepared to
 accept a *dt* argument of ``None``, or of class :class:`.datetime`.
 
 When ``None`` is passed, it's up to the class designer to decide the best
-response.  For example, returning ``None`` is appropriate if the class wishes to
-say that time objects don't participate in the :class:`tzinfo` protocols.  It
+response. For example, returning ``None`` is appropriate if the class wishes to
+say that time objects don't participate in the :class:`tzinfo` protocols. It
 may be more useful for ``utcoffset(None)`` to return the standard UTC offset, as
 there is no other convention for discovering the standard offset.
 
 When a :class:`.datetime` object is passed in response to a :class:`.datetime`
-method, ``dt.tzinfo`` is the same object as *self*.  :class:`tzinfo` methods can
-rely on this, unless user code calls :class:`tzinfo` methods directly.  The
+method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can
+rely on this, unless user code calls :class:`tzinfo` methods directly. The
 intent is that the :class:`tzinfo` methods interpret *dt* as being in local
 time, and not need worry about objects in other timezones.
 
@@ -1839,16 +2047,16 @@ There is one more :class:`tzinfo` method that a subclass may wish to override:
 .. method:: tzinfo.fromutc(dt)
 
    This is called from the default :class:`datetime.astimezone()`
-   implementation.  When called from that, ``dt.tzinfo`` is *self*, and *dt*'s
-   date and time data are to be viewed as expressing a UTC time.  The purpose
+   implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s
+   date and time data are to be viewed as expressing a UTC time. The purpose
    of :meth:`fromutc` is to adjust the date and time data, returning an
    equivalent datetime in *self*'s local time.
 
    Most :class:`tzinfo` subclasses should be able to inherit the default
-   :meth:`fromutc` implementation without problems.  It's strong enough to handle
+   :meth:`fromutc` implementation without problems. It's strong enough to handle
    fixed-offset time zones, and time zones accounting for both standard and
    daylight time, and the latter even if the DST transition times differ in
-   different years.  An example of a time zone the default :meth:`fromutc`
+   different years. An example of a time zone the default :meth:`fromutc`
    implementation may not handle correctly in all cases is one where the standard
    offset (from UTC) depends on the specific date and time passed, which can happen
    for political reasons. The default implementations of :meth:`astimezone` and
@@ -1881,7 +2089,7 @@ In the following :download:`tzinfo_examples.py
 
 Note that there are unavoidable subtleties twice per year in a :class:`tzinfo`
 subclass accounting for both standard and daylight time, at the DST transition
-points.  For concreteness, consider US Eastern (UTC -0500), where EDT begins the
+points. For concreteness, consider US Eastern (UTC -0500), where EDT begins the
 minute after 1:59 (EST) on the second Sunday in March, and ends the minute after
 1:59 (EDT) on the first Sunday in November::
 
@@ -1894,9 +2102,9 @@ minute after 1:59 (EST) on the second Sunday in March, and ends the minute after
      end  23:MM  0:MM  1:MM  1:MM  2:MM  3:MM
 
 When DST starts (the "start" line), the local wall clock leaps from 1:59 to
-3:00.  A wall time of the form 2:MM doesn't really make sense on that day, so
+3:00. A wall time of the form 2:MM doesn't really make sense on that day, so
 ``astimezone(Eastern)`` won't deliver a result with ``hour == 2`` on the day DST
-begins.  For example, at the Spring forward transition of 2016, we get
+begins. For example, at the Spring forward transition of 2016, we get::
 
     >>> from datetime import datetime, timezone
     >>> from tzinfo_examples import HOUR, Eastern
@@ -1914,14 +2122,14 @@ begins.  For example, at the Spring forward transition of 2016, we get
 
 When DST ends (the "end" line), there's a potentially worse problem: there's an
 hour that can't be spelled unambiguously in local wall time: the last hour of
-daylight time.  In Eastern, that's times of the form 5:MM UTC on the day
-daylight time ends.  The local wall clock leaps from 1:59 (daylight time) back
+daylight time. In Eastern, that's times of the form 5:MM UTC on the day
+daylight time ends. The local wall clock leaps from 1:59 (daylight time) back
 to 1:00 (standard time) again. Local times of the form 1:MM are ambiguous.
 :meth:`astimezone` mimics the local clock's behavior by mapping two adjacent UTC
-hours into the same local hour then.  In the Eastern example, UTC times of the
+hours into the same local hour then. In the Eastern example, UTC times of the
 form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times
 have the :attr:`~datetime.fold` attribute set to 0 and the later times have it set to 1.
-For example, at the Fall back transition of 2016, we get
+For example, at the Fall back transition of 2016, we get::
 
     >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc)
     >>> for i in range(4):
@@ -1934,7 +2142,7 @@ For example, at the Fall back transition of 2016, we get
     06:00:00 UTC = 01:00:00 EST 1
     07:00:00 UTC = 02:00:00 EST 0
 
-Note that the :class:`datetime` instances that differ only by the value of the
+Note that the :class:`.datetime` instances that differ only by the value of the
 :attr:`~datetime.fold` attribute are considered equal in comparisons.
 
 Applications that can't bear wall-time ambiguities should explicitly check the
@@ -1946,15 +2154,17 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)).
 .. seealso::
 
    `dateutil.tz <https://dateutil.readthedocs.io/en/stable/tz.html>`_
-      The standard library has :class:`timezone` class for handling arbitrary
-      fixed offsets from UTC and :attr:`timezone.utc` as UTC timezone instance.
+      The :mod:`datetime` module has a basic :class:`timezone` class (for
+      handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc`
+      attribute (a UTC timezone instance).
 
-      *dateutil.tz* library brings the *IANA timezone database* (also known as the
-      Olson database) to Python and its usage is recommended.
+      *dateutil.tz* library brings the *IANA timezone database*
+      (also known as the Olson database) to Python, and its usage is
+      recommended.
 
    `IANA timezone database <https://www.iana.org/time-zones>`_
-      The Time Zone Database (often called tz, tzdata or zoneinfo) contains code and
-      data that represent the history of local time for many representative
+      The Time Zone Database (often called tz, tzdata or zoneinfo) contains code
+      and data that represent the history of local time for many representative
       locations around the globe. It is updated periodically to reflect changes
       made by political bodies to time zone boundaries, UTC offsets, and
       daylight-saving rules.
@@ -1967,20 +2177,21 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)).
 
 The :class:`timezone` class is a subclass of :class:`tzinfo`, each
 instance of which represents a timezone defined by a fixed offset from
-UTC.  Note that objects of this class cannot be used to represent
-timezone information in the locations where different offsets are used
-in different days of the year or where historical changes have been
-made to civil time.
+UTC.
+
+Objects of this class cannot be used to represent timezone information in the
+locations where different offsets are used in different days of the year or
+where historical changes have been made to civil time.
 
 
 .. class:: timezone(offset, name=None)
 
   The *offset* argument must be specified as a :class:`timedelta`
-  object representing the difference between the local time and UTC.  It must
+  object representing the difference between the local time and UTC. It must
   be strictly between ``-timedelta(hours=24)`` and
   ``timedelta(hours=24)``, otherwise :exc:`ValueError` is raised.
 
-  The *name* argument is optional.  If specified it must be a string that
+  The *name* argument is optional. If specified it must be a string that
   will be used as the value returned by the :meth:`datetime.tzname` method.
 
   .. versionadded:: 3.2
@@ -1992,9 +2203,10 @@ made to civil time.
 .. method:: timezone.utcoffset(dt)
 
   Return the fixed value specified when the :class:`timezone` instance is
-  constructed.  The *dt* argument is ignored.  The return value is a
-  :class:`timedelta` instance equal to the difference between the
-  local time and UTC.
+  constructed.
+
+  The *dt* argument is ignored. The return value is a :class:`timedelta`
+  instance equal to the difference between the local time and UTC.
 
   .. versionchanged:: 3.7
      The UTC offset is not restricted to a whole number of minutes.
@@ -2002,16 +2214,17 @@ made to civil time.
 .. method:: timezone.tzname(dt)
 
   Return the fixed value specified when the :class:`timezone` instance
-  is constructed.  If *name* is not provided in the constructor, the
-  name returned by ``tzname(dt)`` is generated from the value of the
-  ``offset`` as follows.  If *offset* is ``timedelta(0)``, the name
-  is "UTC", otherwise it is a string 'UTC?HH:MM', where ? is the sign
-  of ``offset``, HH and MM are two digits of ``offset.hours`` and
-  ``offset.minutes`` respectively.
+  is constructed.
+
+  If *name* is not provided in the constructor, the name returned by
+  ``tzname(dt)`` is generated from the value of the ``offset`` as follows. If
+  *offset* is ``timedelta(0)``, the name is "UTC", otherwise it is a string in
+  the format ``UTC?HH:MM``, where ? is the sign of ``offset``, HH and MM are
+  two digits of ``offset.hours`` and ``offset.minutes`` respectively.
 
   .. versionchanged:: 3.6
-     Name generated from ``offset=timedelta(0)`` is now plain 'UTC', not
-     'UTC+00:00'.
+     Name generated from ``offset=timedelta(0)`` is now plain `'UTC'`, not
+     ``'UTC+00:00'``.
 
 
 .. method:: timezone.dst(dt)
@@ -2020,7 +2233,7 @@ made to civil time.
 
 .. method:: timezone.fromutc(dt)
 
-  Return ``dt + offset``.  The *dt* argument must be an aware
+  Return ``dt + offset``. The *dt* argument must be an aware
   :class:`.datetime` instance, with ``tzinfo`` set to ``self``.
 
 Class attributes:
@@ -2040,43 +2253,33 @@ Class attributes:
 
 :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a
 ``strftime(format)`` method, to create a string representing the time under the
-control of an explicit format string.  Broadly speaking, ``d.strftime(fmt)``
-acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())``
-although not all objects support a :meth:`timetuple` method.
+control of an explicit format string.
 
 Conversely, the :meth:`datetime.strptime` class method creates a
 :class:`.datetime` object from a string representing a date and time and a
-corresponding format string. ``datetime.strptime(date_string, format)`` is
-equivalent to ``datetime(*(time.strptime(date_string, format)[0:6]))``, except
-when the format includes sub-second components or timezone offset information,
-which are supported in ``datetime.strptime`` but are discarded by ``time.strptime``.
-
-For :class:`.time` objects, the format codes for year, month, and day should not
-be used, as time objects have no such values.  If they're used anyway, ``1900``
-is substituted for the year, and ``1`` for the month and day.
+corresponding format string.
 
-For :class:`date` objects, the format codes for hours, minutes, seconds, and
-microseconds should not be used, as :class:`date` objects have no such
-values.  If they're used anyway, ``0`` is substituted for them.
+The table below provides a high-level comparison of :meth:`strftime`
+versus :meth:`strptime`:
 
-For the :meth:`datetime.strptime` class method, the default value is ``1900-01-01T00:00:00.000``:
-any components not specified in the format string will be pulled from the default value. [#]_
++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+
+|                | ``strftime``                                           | ``strptime``                                                                 |
++================+========================================================+==============================================================================+
+| Usage          | Convert object to a string according to a given format | Parse a string into a :class:`.datetime` object given a corresponding format |
++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+
+| Type of method | Instance method                                        | Class method                                                                 |
++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+
+| Method of      | :class:`date`; :class:`.datetime`; :class:`.time`      | :class:`.datetime`                                                           |
++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+
+| Signature      | ``strftime(format)``                                   | ``strptime(date_string, format)``                                            |
++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+
 
-The full set of format codes supported varies across platforms, because Python
-calls the platform C library's :func:`strftime` function, and platform
-variations are common.  To see the full set of format codes supported on your
-platform, consult the :manpage:`strftime(3)` documentation.
 
-For the same reason, handling of format strings containing Unicode code points
-that can't be represented in the charset of the current locale is also
-platform-dependent. On some platforms such code points are preserved intact in
-the output, while on others ``strftime`` may raise :exc:`UnicodeError` or return
-an empty string instead.
+:meth:`strftime` and :meth:`strptime` Format Codes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The following is a list of all the format codes that the C standard (1989
-version) requires, and these work on all platforms with a standard C
-implementation.  Note that the 1999 version of the C standard added additional
-format codes.
+The following is a list of all the format codes that the 1989 C standard
+requires, and these work on all platforms with a standard C implementation.
 
 +-----------+--------------------------------+------------------------+-------+
 | Directive | Meaning                        | Example                | Notes |
@@ -2137,7 +2340,7 @@ format codes.
 |           | left.                          |                        |       |
 +-----------+--------------------------------+------------------------+-------+
 | ``%z``    | UTC offset in the form         | (empty), +0000,        | \(6)  |
-|           | ?HHMM[SS[.ffffff]] (empty      | -0400, +1030,          |       |
+|           | ``?HHMM[SS[.ffffff]]`` (empty  | -0400, +1030,          |       |
 |           | string if the object is        | +063415,               |       |
 |           | naive).                        | -030712.345216         |       |
 +-----------+--------------------------------+------------------------+-------+
@@ -2179,11 +2382,7 @@ format codes.
 +-----------+--------------------------------+------------------------+-------+
 
 Several additional directives not required by the C89 standard are included for
-convenience. These parameters all correspond to ISO 8601 date values. These
-may not be available on all platforms when used with the :meth:`strftime`
-method. The ISO 8601 year and ISO 8601 week directives are not interchangeable
-with the year and week number directives above. Calling :meth:`strptime` with
-incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`.
+convenience. These parameters all correspond to ISO 8601 date values.
 
 +-----------+--------------------------------+------------------------+-------+
 | Directive | Meaning                        | Example                | Notes |
@@ -2203,9 +2402,52 @@ incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`.
 |           | Jan 4.                         |                        |       |
 +-----------+--------------------------------+------------------------+-------+
 
+These may not be available on all platforms when used with the :meth:`strftime`
+method. The ISO 8601 year and ISO 8601 week directives are not interchangeable
+with the year and week number directives above. Calling :meth:`strptime` with
+incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`.
+
+The full set of format codes supported varies across platforms, because Python
+calls the platform C library's :func:`strftime` function, and platform
+variations are common. To see the full set of format codes supported on your
+platform, consult the :manpage:`strftime(3)` documentation.
+
 .. versionadded:: 3.6
    ``%G``, ``%u`` and ``%V`` were added.
 
+Technical Detail
+^^^^^^^^^^^^^^^^
+
+Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's
+``time.strftime(fmt, d.timetuple())`` although not all objects support a
+:meth:`timetuple` method.
+
+For the :meth:`datetime.strptime` class method, the default value is
+``1900-01-01T00:00:00.000``: any components not specified in the format string
+will be pulled from the default value. [#]_
+
+Using ``datetime.strptime(date_string, format)`` is equivalent to::
+
+  datetime(*(time.strptime(date_string, format)[0:6]))
+
+except when the format includes sub-second components or timezone offset
+information, which are supported in ``datetime.strptime`` but are discarded by
+``time.strptime``.
+
+For :class:`.time` objects, the format codes for year, month, and day should not
+be used, as :class:`time` objects have no such values. If they're used anyway,
+``1900`` is substituted for the year, and ``1`` for the month and day.
+
+For :class:`date` objects, the format codes for hours, minutes, seconds, and
+microseconds should not be used, as :class:`date` objects have no such
+values. If they're used anyway, ``0`` is substituted for them.
+
+For the same reason, handling of format strings containing Unicode code points
+that can't be represented in the charset of the current locale is also
+platform-dependent. On some platforms such code points are preserved intact in
+the output, while on others ``strftime`` may raise :exc:`UnicodeError` or return
+an empty string instead.
+
 Notes:
 
 (1)
@@ -2239,7 +2481,7 @@ Notes:
 
 (5)
    When used with the :meth:`strptime` method, the ``%f`` directive
-   accepts from one to six digits and zero pads on the right.  ``%f`` is
+   accepts from one to six digits and zero pads on the right. ``%f`` is
    an extension to the set of format characters in the C standard (but
    implemented separately in datetime objects, and therefore always
    available).
@@ -2252,13 +2494,13 @@ Notes:
 
    ``%z``
       :meth:`utcoffset` is transformed into a string of the form
-      ?HHMM[SS[.ffffff]], where HH is a 2-digit string giving the number of UTC
-      offset hours, MM is a 2-digit string giving the number of UTC offset
-      minutes, SS is a 2-digit string giving the number of UTC offset
-      seconds and ffffff is a 6-digit string giving the number of UTC
-      offset microseconds.  The ffffff part is omitted when the offset is a
-      whole number of seconds and both the ffffff and the SS part is omitted
-      when the offset is a whole number of minutes.  For example, if
+      ``?HHMM[SS[.ffffff]]``, where ``HH`` is a 2-digit string giving the number
+      of UTC offset hours, ``MM`` is a 2-digit string giving the number of UTC
+      offset minutes, SS is a 2-digit string giving the number of UTC offset
+      seconds and ``ffffff`` is a 6-digit string giving the number of UTC
+      offset microseconds. The ``ffffff`` part is omitted when the offset is a
+      whole number of seconds and both the ``ffffff`` and the ``SS`` part is
+      omitted when the offset is a whole number of minutes. For example, if
       :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is
       replaced with the string ``'-0330'``.
 
@@ -2274,12 +2516,12 @@ Notes:
 
    ``%Z``
       If :meth:`tzname` returns ``None``, ``%Z`` is replaced by an empty
-      string.  Otherwise ``%Z`` is replaced by the returned value, which must
+      string. Otherwise ``%Z`` is replaced by the returned value, which must
       be a string.
 
    .. versionchanged:: 3.2
       When the ``%z`` directive is provided to the :meth:`strptime` method, an
-      aware :class:`.datetime` object will be produced.  The ``tzinfo`` of the
+      aware :class:`.datetime` object will be produced. The ``tzinfo`` of the
       result will be set to a :class:`timezone` instance.
 
 (7)
@@ -2301,4 +2543,15 @@ Notes:
 .. rubric:: Footnotes
 
 .. [#] If, that is, we ignore the effects of Relativity
+
+.. [#] This matches the definition of the "proleptic Gregorian" calendar in
+       Dershowitz and Reingold's book *Calendrical Calculations*,
+       where it's the base calendar for all computations. See the book for
+       algorithms for converting between proleptic Gregorian ordinals and
+       many other calendar systems.
+
+.. [#] See R. H. van Gent's `guide to the mathematics of the ISO 8601 calendar
+       <https://www.staff.science.uu.nl/~gent0113/calendar/isocalendar.htm>`_
+       for a good explanation.
+
 .. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since ``1900`` is not a leap year.
diff --git a/Misc/NEWS.d/next/Documentation/2019-05-18-16-25-44.bpo-36960.xEKHXj.rst b/Misc/NEWS.d/next/Documentation/2019-05-18-16-25-44.bpo-36960.xEKHXj.rst
new file mode 100644
index 000000000000..1d4b213367e7
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-05-18-16-25-44.bpo-36960.xEKHXj.rst
@@ -0,0 +1,2 @@
+Restructured the :mod:`datetime` docs in the interest of making them more
+user-friendly and improving readability. Patch by Brad Solomon.


From webhook-mailer at python.org  Wed Sep 11 05:23:09 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Wed, 11 Sep 2019 09:23:09 -0000
Subject: [Python-checkins] bpo-37424: Avoid a hang in subprocess.run timeout
 output capture (GH-14490)
Message-ID: <mailman.499.1568193790.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/580d2782f70f8e0bed7ec20abb03d740cb83b5da
commit: 580d2782f70f8e0bed7ec20abb03d740cb83b5da
branch: master
author: Gregory P. Smith <greg at krypto.org>
committer: T. Wouters <thomas at python.org>
date: 2019-09-11T02:23:05-07:00
summary:

bpo-37424: Avoid a hang in subprocess.run timeout output capture (GH-14490)

Fixes a possible hang when using a timeout on subprocess.run() while
capturing output. If the child process spawned its own children or otherwise
connected its stdout or stderr handles with another process, we could hang
after the timeout was reached and our child was killed when attempting to read
final output from the pipes.

files:
A Misc/NEWS.d/next/Library/2019-07-04-13-00-20.bpo-37424.0i1MR-.rst
M Lib/subprocess.py
M Lib/test/test_subprocess.py

diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index aed7292541e0..85b9ea078546 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -489,11 +489,20 @@ def run(*popenargs,
     with Popen(*popenargs, **kwargs) as process:
         try:
             stdout, stderr = process.communicate(input, timeout=timeout)
-        except TimeoutExpired:
+        except TimeoutExpired as exc:
             process.kill()
-            stdout, stderr = process.communicate()
-            raise TimeoutExpired(process.args, timeout, output=stdout,
-                                 stderr=stderr)
+            if _mswindows:
+                # Windows accumulates the output in a single blocking
+                # read() call run on child threads, with the timeout
+                # being done in a join() on those threads.  communicate()
+                # _after_ kill() is required to collect that and add it
+                # to the exception.
+                exc.stdout, exc.stderr = process.communicate()
+            else:
+                # POSIX _communicate already populated the output so
+                # far into the TimeoutExpired exception.
+                process.wait()
+            raise
         except:  # Including KeyboardInterrupt, communicate handled that.
             process.kill()
             # We don't call process.wait() as .__exit__ does that for us.
@@ -1050,12 +1059,16 @@ def _remaining_time(self, endtime):
             return endtime - _time()
 
 
-    def _check_timeout(self, endtime, orig_timeout):
+    def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq,
+                       skip_check_and_raise=False):
         """Convenience for checking if a timeout has expired."""
         if endtime is None:
             return
-        if _time() > endtime:
-            raise TimeoutExpired(self.args, orig_timeout)
+        if skip_check_and_raise or _time() > endtime:
+            raise TimeoutExpired(
+                    self.args, orig_timeout,
+                    output=b''.join(stdout_seq) if stdout_seq else None,
+                    stderr=b''.join(stderr_seq) if stderr_seq else None)
 
 
     def wait(self, timeout=None):
@@ -1843,10 +1856,15 @@ def _communicate(self, input, endtime, orig_timeout):
                 while selector.get_map():
                     timeout = self._remaining_time(endtime)
                     if timeout is not None and timeout < 0:
-                        raise TimeoutExpired(self.args, orig_timeout)
+                        self._check_timeout(endtime, orig_timeout,
+                                            stdout, stderr,
+                                            skip_check_and_raise=True)
+                        raise RuntimeError(  # Impossible :)
+                            '_check_timeout(..., skip_check_and_raise=True) '
+                            'failed to raise TimeoutExpired.')
 
                     ready = selector.select(timeout)
-                    self._check_timeout(endtime, orig_timeout)
+                    self._check_timeout(endtime, orig_timeout, stdout, stderr)
 
                     # XXX Rewrite these to use non-blocking I/O on the file
                     # objects; they are no longer using C stdio!
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index 4fe74bf504b4..91f525df4607 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -10,6 +10,7 @@
 import errno
 import tempfile
 import time
+import traceback
 import selectors
 import sysconfig
 import select
@@ -1557,6 +1558,26 @@ def test_stderr_with_capture_output_arg(self):
         self.assertIn('stderr', c.exception.args[0])
         self.assertIn('capture_output', c.exception.args[0])
 
+    # This test _might_ wind up a bit fragile on loaded build+test machines
+    # as it depends on the timing with wide enough margins for normal situations
+    # but does assert that it happened "soon enough" to believe the right thing
+    # happened.
+    @unittest.skipIf(mswindows, "requires posix like 'sleep' shell command")
+    def test_run_with_shell_timeout_and_capture_output(self):
+        """Output capturing after a timeout mustn't hang forever on open filehandles."""
+        before_secs = time.monotonic()
+        try:
+            subprocess.run('sleep 3', shell=True, timeout=0.1,
+                           capture_output=True)  # New session unspecified.
+        except subprocess.TimeoutExpired as exc:
+            after_secs = time.monotonic()
+            stacks = traceback.format_exc()  # assertRaises doesn't give this.
+        else:
+            self.fail("TimeoutExpired not raised.")
+        self.assertLess(after_secs - before_secs, 1.5,
+                        msg="TimeoutExpired was delayed! Bad traceback:\n```\n"
+                        f"{stacks}```")
+
 
 @unittest.skipIf(mswindows, "POSIX specific tests")
 class POSIXProcessTestCase(BaseTestCase):
diff --git a/Misc/NEWS.d/next/Library/2019-07-04-13-00-20.bpo-37424.0i1MR-.rst b/Misc/NEWS.d/next/Library/2019-07-04-13-00-20.bpo-37424.0i1MR-.rst
new file mode 100644
index 000000000000..b98a17e241e0
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-07-04-13-00-20.bpo-37424.0i1MR-.rst
@@ -0,0 +1,5 @@
+Fixes a possible hang when using a timeout on `subprocess.run()` while
+capturing output.  If the child process spawned its own children or
+otherwise connected its stdout or stderr handles with another process, we
+could hang after the timeout was reached and our child was killed when
+attempting to read final output from the pipes.


From webhook-mailer at python.org  Wed Sep 11 05:25:30 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Wed, 11 Sep 2019 09:25:30 -0000
Subject: [Python-checkins] bpo-37936: Systematically distinguish rooted vs.
 unrooted in .gitignore (GH-15823)
Message-ID: <mailman.500.1568193931.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/455122a0094c8cfdf7e062eccc5e5b5885c75e8b
commit: 455122a0094c8cfdf7e062eccc5e5b5885c75e8b
branch: master
author: Greg Price <gnprice at gmail.com>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-11T10:25:26+01:00
summary:

bpo-37936: Systematically distinguish rooted vs. unrooted in .gitignore (GH-15823)

A root cause of bpo-37936 is that it's easy to write a .gitignore
rule that's intended to apply to a specific file (e.g., the
`pyconfig.h` generated by `./configure`) but actually applies to all
similarly-named files in the tree (e.g., `PC/pyconfig.h`.)

Specifically, any rule with no non-trailing slashes is applied in an
"unrooted" way, to files anywhere in the tree.  This means that if we
write the rules in the most obvious-looking way, then

 * for specific files we want to ignore that happen to be in
   subdirectories (like `Modules/config.c`), the rule will work
   as intended, staying "rooted" to the top of the tree; but

 * when a specific file we want to ignore happens to be at the root of
   the repo (like `platform`), then the obvious rule (`platform`) will
   apply much more broadly than intended: if someone tries to add a
   file or directory named `platform` somewhere else in the tree, it
   will unexpectedly get ignored.

That's surprising behavior that can make the .gitignore file's
behavior feel finicky and unpredictable.

To avoid it, we can simply always give a rule "rooted" behavior when
that's what's intended, by systematically using leading slashes.

Further, to help make the pattern obvious when looking at the file and
minimize any need for thinking about the syntax when adding new rules:
separate the rules into one group for each type, with brief comments
identifying them.

For most of these rules it's clear whether they're meant to be rooted
or unrooted, but in a handful of cases I've only guessed.  In that
case the safer default (the choice that won't hide information) is the
narrower, rooted meaning, with a leading slash.  If for some of these
the unrooted meaning is desired after all, it'll be easy to move them
to the unrooted section at the top.

files:
A Misc/NEWS.d/next/Build/2019-09-10-00-54-48.bpo-37936.E7XEwu.rst
M .gitignore
M Lib/test/libregrtest/runtest.py

diff --git a/.gitignore b/.gitignore
index 54c598e109e0..0e3f973d0fc7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,11 +1,14 @@
-# Two-trick pony for OSX and other case insensitive file systems:
-# Ignore ./python binary on Unix but still look into ./Python/ directory.
-/python
-!/Python/
+#####
+# First, rules intended to apply in all subdirectories.
+# These contain no slash, or only a trailing slash.
 
 *.cover
 *.iml
 *.o
+*.a
+*.so*
+*.dylib
+*.dll
 *.orig
 *.pyc
 *.pyd
@@ -18,6 +21,31 @@
 *.profraw
 *.dyn
 .gdb_history
+.purify
+__pycache__
+.hg/
+.svn/
+.idea/
+tags
+TAGS
+.vs/
+.vscode/
+gmon.out
+.coverage
+.mypy_cache/
+
+*.exe
+!Lib/distutils/command/*.exe
+
+# Ignore core dumps... but not Tools/msi/core/ or the like.
+core
+!core/
+
+
+#####
+# Then, rules meant for a specific location relative to the repo root.
+# These must contain a non-trailing slash (and may also have a trailing slash.)
+
 Doc/build/
 Doc/venv/
 Doc/.venv/
@@ -29,7 +57,7 @@ Lib/lib2to3/*.pickle
 Lib/test/data/*
 !Lib/test/data/README
 /Makefile
-Makefile.pre
+/Makefile.pre
 Misc/python.pc
 Misc/python-embed.pc
 Misc/python-config.sh
@@ -38,12 +66,9 @@ Modules/Setup.local
 Modules/config.c
 Modules/ld_so_aix
 Programs/_freeze_importlib
-Programs/_freeze_importlib.exe
 Programs/_testembed
-Programs/_testembed.exe
 PC/python_nt*.h
 PC/pythonnt_rc*.h
-PC/*/*.exe
 PC/*/*.exp
 PC/*/*.lib
 PC/*/*.bsc
@@ -62,53 +87,35 @@ PCbuild/*-pgi
 PCbuild/*-pgo
 PCbuild/*.VC.db
 PCbuild/*.VC.opendb
-PCbuild/.vs/
 PCbuild/amd64/
 PCbuild/arm32/
 PCbuild/arm64/
 PCbuild/obj/
 PCbuild/win32/
 Tools/unicode/data/
-.purify
-__pycache__
-autom4te.cache
-build/
-buildno
-config.cache
-config.log
-config.status
-config.status.lineno
-core
-!Tools/msi/core/
-db_home
-.hg/
-.idea/
-ipch/
-libpython*.a
-libpython*.so*
-libpython*.dylib
-libpython*.dll
-platform
-pybuilddir.txt
+/autom4te.cache
+/build/
+/config.cache
+/config.log
+/config.status
+/config.status.lineno
+/platform
+/pybuilddir.txt
 /pyconfig.h
-python-config
-python-config.py
-python.bat
-python.exe
-python-gdb.py
-python.exe-gdb.py
-reflog.txt
-.svn/
-tags
-TAGS
-.coverage
-coverage/
-externals/
-htmlcov/
+/python-config
+/python-config.py
+/python.bat
+/python-gdb.py
+/python.exe-gdb.py
+/reflog.txt
+/coverage/
+/externals/
+/htmlcov/
 Tools/msi/obj
 Tools/ssl/amd64
 Tools/ssl/win32
-.vs/
-.vscode/
-gmon.out
-.mypy_cache/
+
+# Two-trick pony for OSX and other case insensitive file systems:
+# Ignore ./python binary on Unix but still look into ./Python/ directory.
+/python
+!/Python/
diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py
index e7dce180cb37..eeb108bb4473 100644
--- a/Lib/test/libregrtest/runtest.py
+++ b/Lib/test/libregrtest/runtest.py
@@ -313,9 +313,7 @@ def cleanup_test_droppings(test_name, verbose):
     # since if a test leaves a file open, it cannot be deleted by name (while
     # there's nothing we can do about that here either, we can display the
     # name of the offending test, which is a real help).
-    for name in (support.TESTFN,
-                 "db_home",
-                ):
+    for name in (support.TESTFN,):
         if not os.path.exists(name):
             continue
 
diff --git a/Misc/NEWS.d/next/Build/2019-09-10-00-54-48.bpo-37936.E7XEwu.rst b/Misc/NEWS.d/next/Build/2019-09-10-00-54-48.bpo-37936.E7XEwu.rst
new file mode 100644
index 000000000000..5ded61eafe94
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-09-10-00-54-48.bpo-37936.E7XEwu.rst
@@ -0,0 +1,5 @@
+The :file:`.gitignore` file systematically keeps "rooted", with a
+non-trailing slash, all the rules that are meant to apply to files in a
+specific place in the repo.  Previously, when the intended file to ignore
+happened to be at the root of the repo, we'd most often accidentally also
+ignore files and directories with the same name anywhere in the tree.


From webhook-mailer at python.org  Wed Sep 11 05:39:39 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Wed, 11 Sep 2019 09:39:39 -0000
Subject: [Python-checkins] bpo-37445: Include FORMAT_MESSAGE_IGNORE_INSERTS
 in FormatMessageW() calls (GH-15822)
Message-ID: <mailman.501.1568194779.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/289c5ea7737e44e3b150532b3498e3d3d4c70d99
commit: 289c5ea7737e44e3b150532b3498e3d3d4c70d99
branch: 2.7
author: Zackery Spytz <zspytz at gmail.com>
committer: Steve Dower <steve.dower at python.org>
date: 2019-09-11T10:39:34+01:00
summary:

bpo-37445: Include FORMAT_MESSAGE_IGNORE_INSERTS in FormatMessageW() calls (GH-15822)

If FormatMessageW() is passed the FORMAT_MESSAGE_FROM_SYSTEM flag
without FORMAT_MESSAGE_IGNORE_INSERTS, it will fail if there are
insert sequences in the message definition.
(cherry picked from commit a656365)

files:
A Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
M Modules/_ctypes/callproc.c
M PC/bdist_wininst/install.c

diff --git a/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst b/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
new file mode 100644
index 000000000000..e4805b4e02fb
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
@@ -0,0 +1,2 @@
+Include the ``FORMAT_MESSAGE_IGNORE_INSERTS`` flag in ``FormatMessageW()``
+calls.
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
index defcde1ff3e9..066fefc0cca6 100644
--- a/Modules/_ctypes/callproc.c
+++ b/Modules/_ctypes/callproc.c
@@ -242,7 +242,9 @@ static TCHAR *FormatError(DWORD code)
 {
     TCHAR *lpMsgBuf;
     DWORD n;
-    n = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+    n = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |
+                      FORMAT_MESSAGE_FROM_SYSTEM |
+                      FORMAT_MESSAGE_IGNORE_INSERTS,
                       NULL,
                       code,
                       MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), /* Default language */
diff --git a/PC/bdist_wininst/install.c b/PC/bdist_wininst/install.c
index f1cc7feed7a4..e3b52a80554b 100644
--- a/PC/bdist_wininst/install.c
+++ b/PC/bdist_wininst/install.c
@@ -895,7 +895,8 @@ static BOOL SystemError(int error, char *msg)
         LPVOID lpMsgBuf;
         FormatMessage(
             FORMAT_MESSAGE_ALLOCATE_BUFFER |
-            FORMAT_MESSAGE_FROM_SYSTEM,
+            FORMAT_MESSAGE_FROM_SYSTEM |
+            FORMAT_MESSAGE_IGNORE_INSERTS,
             NULL,
             error,
             MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),


From webhook-mailer at python.org  Wed Sep 11 05:48:40 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Wed, 11 Sep 2019 09:48:40 -0000
Subject: [Python-checkins] bpo-38081: Fixes ntpath.realpath('NUL') (GH-15899)
Message-ID: <mailman.502.1568195321.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/92521fea5d0d4aeb9b6a3c3fdd4654af700ad5c8
commit: 92521fea5d0d4aeb9b6a3c3fdd4654af700ad5c8
branch: master
author: Steve Dower <steve.dower at python.org>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-11T10:48:36+01:00
summary:

bpo-38081: Fixes ntpath.realpath('NUL') (GH-15899)

files:
A Misc/NEWS.d/next/Windows/2019-09-11-10-22-01.bpo-38081.8JhzjD.rst
M Lib/ntpath.py
M Lib/test/test_ntpath.py

diff --git a/Lib/ntpath.py b/Lib/ntpath.py
index 1d22d5f1dc2a..1b5e16f95f16 100644
--- a/Lib/ntpath.py
+++ b/Lib/ntpath.py
@@ -535,9 +535,10 @@ def _readlink_deep(path, seen=None):
             try:
                 path = _nt_readlink(path)
             except OSError as ex:
-                # Stop on file (2) or directory (3) not found, or
-                # paths that are not reparse points (4390)
-                if ex.winerror in (2, 3, 4390):
+                # Stop on incorrect function (1), file (2) or
+                # directory (3) not found, or paths that are
+                # not reparse points (4390)
+                if ex.winerror in (1, 2, 3, 4390):
                     break
                 raise
             except ValueError:
@@ -553,9 +554,9 @@ def _getfinalpathname_nonstrict(path):
         except OSError:
             pass
 
-        # Allow file (2) or directory (3) not found, invalid syntax (123),
-        # and symlinks that cannot be followed (1921)
-        allowed_winerror = 2, 3, 123, 1921
+        # Allow file (2) or directory (3) not found, incorrect parameter (87),
+        # invalid syntax (123), and symlinks that cannot be followed (1921)
+        allowed_winerror = 2, 3, 87, 123, 1921
 
         # Non-strict algorithm is to find as much of the target directory
         # as we can and join the rest.
diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
index c5c96e32d747..2f0faf94726f 100644
--- a/Lib/test/test_ntpath.py
+++ b/Lib/test/test_ntpath.py
@@ -400,6 +400,10 @@ def test_realpath_symlink_prefix(self):
         self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link"),
                              "\\\\?\\" + ABSTFN + "3.")
 
+    @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
+    def test_realpath_nul(self):
+        tester("ntpath.realpath('NUL')", r'\\.\NUL')
+
     def test_expandvars(self):
         with support.EnvironmentVarGuard() as env:
             env.clear()
diff --git a/Misc/NEWS.d/next/Windows/2019-09-11-10-22-01.bpo-38081.8JhzjD.rst b/Misc/NEWS.d/next/Windows/2019-09-11-10-22-01.bpo-38081.8JhzjD.rst
new file mode 100644
index 000000000000..e9d7a30eed9f
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-11-10-22-01.bpo-38081.8JhzjD.rst
@@ -0,0 +1 @@
+Prevent error calling :func:`os.path.realpath` on ``'NUL'``.


From webhook-mailer at python.org  Wed Sep 11 05:58:31 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Wed, 11 Sep 2019 09:58:31 -0000
Subject: [Python-checkins] bpo-16438: Doc: confusing text regarding numeric
 precedence corrected (GH-10521)
Message-ID: <mailman.503.1568195912.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4576b5431bd597df7581fe3c852b315e47e4b230
commit: 4576b5431bd597df7581fe3c852b315e47e4b230
branch: master
author: Anjali <anjali30malik at gmail.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-11T11:58:27+02:00
summary:

bpo-16438: Doc: confusing text regarding numeric precedence corrected (GH-10521)

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 08c5ae876c1b..0a565c3e5b6b 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -265,9 +265,8 @@ which is narrower than complex.  Comparisons between numbers of mixed type use
 the same rule. [2]_ The constructors :func:`int`, :func:`float`, and
 :func:`complex` can be used to produce numbers of a specific type.
 
-All numeric types (except complex) support the following operations, sorted by
-ascending priority (all numeric operations have a higher priority than
-comparison operations):
+All numeric types (except complex) support the following operations (for priorities of
+the operations, see :ref:`operator-summary`):
 
 +---------------------+---------------------------------+---------+--------------------+
 | Operation           | Result                          | Notes   | Full documentation |


From webhook-mailer at python.org  Wed Sep 11 06:01:46 2019
From: webhook-mailer at python.org (Brett Cannon)
Date: Wed, 11 Sep 2019 10:01:46 -0000
Subject: [Python-checkins] bpo-37585: Add clarification regarding comparing
 dict.values() (GH-14954)
Message-ID: <mailman.504.1568196107.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6472ece5a0fe82809d3aa0ffb281796fcd252d76
commit: 6472ece5a0fe82809d3aa0ffb281796fcd252d76
branch: master
author: Kyle Stanley <aeros167 at gmail.com>
committer: Brett Cannon <54418+brettcannon at users.noreply.github.com>
date: 2019-09-11T11:01:41+01:00
summary:

bpo-37585: Add clarification regarding comparing dict.values() (GH-14954)

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 0a565c3e5b6b..c78f4ba8f068 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -4357,6 +4357,14 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098:
       Return a new view of the dictionary's values.  See the
       :ref:`documentation of view objects <dict-views>`.
 
+      An equality comparison between one ``dict.values()`` view and another
+      will always return ``False``. This also applies when comparing
+      ``dict.values()`` to itself::
+
+         >>> d = {'a': 1}
+         >>> d.values() == d.values()
+         False
+
    Dictionaries compare equal if and only if they have the same ``(key,
    value)`` pairs. Order comparisons ('<', '<=', '>=', '>') raise
    :exc:`TypeError`.


From webhook-mailer at python.org  Wed Sep 11 06:05:18 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 10:05:18 -0000
Subject: [Python-checkins] bpo-37424: Avoid a hang in subprocess.run timeout
 output capture (GH-14490)
Message-ID: <mailman.505.1568196319.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/872c85a1796290baef89a20dbde819c4da45830c
commit: 872c85a1796290baef89a20dbde819c4da45830c
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T03:05:14-07:00
summary:

bpo-37424: Avoid a hang in subprocess.run timeout output capture (GH-14490)


Fixes a possible hang when using a timeout on subprocess.run() while
capturing output. If the child process spawned its own children or otherwise
connected its stdout or stderr handles with another process, we could hang
after the timeout was reached and our child was killed when attempting to read
final output from the pipes.
(cherry picked from commit 580d2782f70f8e0bed7ec20abb03d740cb83b5da)

Co-authored-by: Gregory P. Smith <greg at krypto.org>

files:
A Misc/NEWS.d/next/Library/2019-07-04-13-00-20.bpo-37424.0i1MR-.rst
M Lib/subprocess.py
M Lib/test/test_subprocess.py

diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index aed7292541e0..85b9ea078546 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -489,11 +489,20 @@ def run(*popenargs,
     with Popen(*popenargs, **kwargs) as process:
         try:
             stdout, stderr = process.communicate(input, timeout=timeout)
-        except TimeoutExpired:
+        except TimeoutExpired as exc:
             process.kill()
-            stdout, stderr = process.communicate()
-            raise TimeoutExpired(process.args, timeout, output=stdout,
-                                 stderr=stderr)
+            if _mswindows:
+                # Windows accumulates the output in a single blocking
+                # read() call run on child threads, with the timeout
+                # being done in a join() on those threads.  communicate()
+                # _after_ kill() is required to collect that and add it
+                # to the exception.
+                exc.stdout, exc.stderr = process.communicate()
+            else:
+                # POSIX _communicate already populated the output so
+                # far into the TimeoutExpired exception.
+                process.wait()
+            raise
         except:  # Including KeyboardInterrupt, communicate handled that.
             process.kill()
             # We don't call process.wait() as .__exit__ does that for us.
@@ -1050,12 +1059,16 @@ def _remaining_time(self, endtime):
             return endtime - _time()
 
 
-    def _check_timeout(self, endtime, orig_timeout):
+    def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq,
+                       skip_check_and_raise=False):
         """Convenience for checking if a timeout has expired."""
         if endtime is None:
             return
-        if _time() > endtime:
-            raise TimeoutExpired(self.args, orig_timeout)
+        if skip_check_and_raise or _time() > endtime:
+            raise TimeoutExpired(
+                    self.args, orig_timeout,
+                    output=b''.join(stdout_seq) if stdout_seq else None,
+                    stderr=b''.join(stderr_seq) if stderr_seq else None)
 
 
     def wait(self, timeout=None):
@@ -1843,10 +1856,15 @@ def _communicate(self, input, endtime, orig_timeout):
                 while selector.get_map():
                     timeout = self._remaining_time(endtime)
                     if timeout is not None and timeout < 0:
-                        raise TimeoutExpired(self.args, orig_timeout)
+                        self._check_timeout(endtime, orig_timeout,
+                                            stdout, stderr,
+                                            skip_check_and_raise=True)
+                        raise RuntimeError(  # Impossible :)
+                            '_check_timeout(..., skip_check_and_raise=True) '
+                            'failed to raise TimeoutExpired.')
 
                     ready = selector.select(timeout)
-                    self._check_timeout(endtime, orig_timeout)
+                    self._check_timeout(endtime, orig_timeout, stdout, stderr)
 
                     # XXX Rewrite these to use non-blocking I/O on the file
                     # objects; they are no longer using C stdio!
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index e58d0925df3b..69ac584667d3 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -10,6 +10,7 @@
 import errno
 import tempfile
 import time
+import traceback
 import selectors
 import sysconfig
 import select
@@ -1557,6 +1558,26 @@ def test_stderr_with_capture_output_arg(self):
         self.assertIn('stderr', c.exception.args[0])
         self.assertIn('capture_output', c.exception.args[0])
 
+    # This test _might_ wind up a bit fragile on loaded build+test machines
+    # as it depends on the timing with wide enough margins for normal situations
+    # but does assert that it happened "soon enough" to believe the right thing
+    # happened.
+    @unittest.skipIf(mswindows, "requires posix like 'sleep' shell command")
+    def test_run_with_shell_timeout_and_capture_output(self):
+        """Output capturing after a timeout mustn't hang forever on open filehandles."""
+        before_secs = time.monotonic()
+        try:
+            subprocess.run('sleep 3', shell=True, timeout=0.1,
+                           capture_output=True)  # New session unspecified.
+        except subprocess.TimeoutExpired as exc:
+            after_secs = time.monotonic()
+            stacks = traceback.format_exc()  # assertRaises doesn't give this.
+        else:
+            self.fail("TimeoutExpired not raised.")
+        self.assertLess(after_secs - before_secs, 1.5,
+                        msg="TimeoutExpired was delayed! Bad traceback:\n```\n"
+                        f"{stacks}```")
+
 
 @unittest.skipIf(mswindows, "POSIX specific tests")
 class POSIXProcessTestCase(BaseTestCase):
diff --git a/Misc/NEWS.d/next/Library/2019-07-04-13-00-20.bpo-37424.0i1MR-.rst b/Misc/NEWS.d/next/Library/2019-07-04-13-00-20.bpo-37424.0i1MR-.rst
new file mode 100644
index 000000000000..b98a17e241e0
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-07-04-13-00-20.bpo-37424.0i1MR-.rst
@@ -0,0 +1,5 @@
+Fixes a possible hang when using a timeout on `subprocess.run()` while
+capturing output.  If the child process spawned its own children or
+otherwise connected its stdout or stderr handles with another process, we
+could hang after the timeout was reached and our child was killed when
+attempting to read final output from the pipes.


From webhook-mailer at python.org  Wed Sep 11 06:06:01 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 10:06:01 -0000
Subject: [Python-checkins] bpo-37424: Avoid a hang in subprocess.run timeout
 output capture (GH-14490)
Message-ID: <mailman.506.1568196362.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5fe153c21de020f37712910b0dcce93341bba272
commit: 5fe153c21de020f37712910b0dcce93341bba272
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T03:05:58-07:00
summary:

bpo-37424: Avoid a hang in subprocess.run timeout output capture (GH-14490)


Fixes a possible hang when using a timeout on subprocess.run() while
capturing output. If the child process spawned its own children or otherwise
connected its stdout or stderr handles with another process, we could hang
after the timeout was reached and our child was killed when attempting to read
final output from the pipes.
(cherry picked from commit 580d2782f70f8e0bed7ec20abb03d740cb83b5da)

Co-authored-by: Gregory P. Smith <greg at krypto.org>

files:
A Misc/NEWS.d/next/Library/2019-07-04-13-00-20.bpo-37424.0i1MR-.rst
M Lib/subprocess.py
M Lib/test/test_subprocess.py

diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 395e5e7ba75c..d1b6de578578 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -488,11 +488,20 @@ def run(*popenargs,
     with Popen(*popenargs, **kwargs) as process:
         try:
             stdout, stderr = process.communicate(input, timeout=timeout)
-        except TimeoutExpired:
+        except TimeoutExpired as exc:
             process.kill()
-            stdout, stderr = process.communicate()
-            raise TimeoutExpired(process.args, timeout, output=stdout,
-                                 stderr=stderr)
+            if _mswindows:
+                # Windows accumulates the output in a single blocking
+                # read() call run on child threads, with the timeout
+                # being done in a join() on those threads.  communicate()
+                # _after_ kill() is required to collect that and add it
+                # to the exception.
+                exc.stdout, exc.stderr = process.communicate()
+            else:
+                # POSIX _communicate already populated the output so
+                # far into the TimeoutExpired exception.
+                process.wait()
+            raise
         except:  # Including KeyboardInterrupt, communicate handled that.
             process.kill()
             # We don't call process.wait() as .__exit__ does that for us.
@@ -990,12 +999,16 @@ def _remaining_time(self, endtime):
             return endtime - _time()
 
 
-    def _check_timeout(self, endtime, orig_timeout):
+    def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq,
+                       skip_check_and_raise=False):
         """Convenience for checking if a timeout has expired."""
         if endtime is None:
             return
-        if _time() > endtime:
-            raise TimeoutExpired(self.args, orig_timeout)
+        if skip_check_and_raise or _time() > endtime:
+            raise TimeoutExpired(
+                    self.args, orig_timeout,
+                    output=b''.join(stdout_seq) if stdout_seq else None,
+                    stderr=b''.join(stderr_seq) if stderr_seq else None)
 
 
     def wait(self, timeout=None):
@@ -1692,10 +1705,15 @@ def _communicate(self, input, endtime, orig_timeout):
                 while selector.get_map():
                     timeout = self._remaining_time(endtime)
                     if timeout is not None and timeout < 0:
-                        raise TimeoutExpired(self.args, orig_timeout)
+                        self._check_timeout(endtime, orig_timeout,
+                                            stdout, stderr,
+                                            skip_check_and_raise=True)
+                        raise RuntimeError(  # Impossible :)
+                            '_check_timeout(..., skip_check_and_raise=True) '
+                            'failed to raise TimeoutExpired.')
 
                     ready = selector.select(timeout)
-                    self._check_timeout(endtime, orig_timeout)
+                    self._check_timeout(endtime, orig_timeout, stdout, stderr)
 
                     # XXX Rewrite these to use non-blocking I/O on the file
                     # objects; they are no longer using C stdio!
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index 41e6a0901585..9f494dab408c 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -11,6 +11,7 @@
 import errno
 import tempfile
 import time
+import traceback
 import selectors
 import sysconfig
 import select
@@ -1508,6 +1509,26 @@ def test_stderr_with_capture_output_arg(self):
         self.assertIn('stderr', c.exception.args[0])
         self.assertIn('capture_output', c.exception.args[0])
 
+    # This test _might_ wind up a bit fragile on loaded build+test machines
+    # as it depends on the timing with wide enough margins for normal situations
+    # but does assert that it happened "soon enough" to believe the right thing
+    # happened.
+    @unittest.skipIf(mswindows, "requires posix like 'sleep' shell command")
+    def test_run_with_shell_timeout_and_capture_output(self):
+        """Output capturing after a timeout mustn't hang forever on open filehandles."""
+        before_secs = time.monotonic()
+        try:
+            subprocess.run('sleep 3', shell=True, timeout=0.1,
+                           capture_output=True)  # New session unspecified.
+        except subprocess.TimeoutExpired as exc:
+            after_secs = time.monotonic()
+            stacks = traceback.format_exc()  # assertRaises doesn't give this.
+        else:
+            self.fail("TimeoutExpired not raised.")
+        self.assertLess(after_secs - before_secs, 1.5,
+                        msg="TimeoutExpired was delayed! Bad traceback:\n```\n"
+                        f"{stacks}```")
+
 
 @unittest.skipIf(mswindows, "POSIX specific tests")
 class POSIXProcessTestCase(BaseTestCase):
diff --git a/Misc/NEWS.d/next/Library/2019-07-04-13-00-20.bpo-37424.0i1MR-.rst b/Misc/NEWS.d/next/Library/2019-07-04-13-00-20.bpo-37424.0i1MR-.rst
new file mode 100644
index 000000000000..b98a17e241e0
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-07-04-13-00-20.bpo-37424.0i1MR-.rst
@@ -0,0 +1,5 @@
+Fixes a possible hang when using a timeout on `subprocess.run()` while
+capturing output.  If the child process spawned its own children or
+otherwise connected its stdout or stderr handles with another process, we
+could hang after the timeout was reached and our child was killed when
+attempting to read final output from the pipes.


From webhook-mailer at python.org  Wed Sep 11 06:09:53 2019
From: webhook-mailer at python.org (Brett Cannon)
Date: Wed, 11 Sep 2019 10:09:53 -0000
Subject: [Python-checkins] bpo-37409: fix relative import with no parent
 (#14956)
Message-ID: <mailman.507.1568196593.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/92420b3e679959a7d0ce875875601a4cee45231e
commit: 92420b3e679959a7d0ce875875601a4cee45231e
branch: master
author: Ben Lewis <benjimin at users.noreply.github.com>
committer: Brett Cannon <54418+brettcannon at users.noreply.github.com>
date: 2019-09-11T11:09:47+01:00
summary:

bpo-37409: fix relative import with no parent (#14956)

Relative imports use resolve_name to get the absolute target name,
which first seeks the current module's absolute package name from the globals:
If __package__ (and __spec__.parent) are missing then
import uses __name__, truncating the last segment if
the module is a submodule rather than a package __init__.py
(which it guesses from whether __path__ is defined).

The __name__ attempt should fail if there is no parent package (top level modules),
if __name__ is '__main__' (-m entry points), or both (scripts).
That is, if both __name__ has no subcomponents and the module does not seem
to be a package __init__ module then import should fail.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-08-06-23-39-05.bpo-37409.1qwzn2.rst
M Lib/test/test_builtin.py
M Lib/test/test_import/__init__.py
M Misc/ACKS
M Python/import.c

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 7b3c054d5eb9..185d8f794b25 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -160,6 +160,10 @@ def test_import(self):
         self.assertRaises(TypeError, __import__, 1, 2, 3, 4)
         self.assertRaises(ValueError, __import__, '')
         self.assertRaises(TypeError, __import__, 'sys', name='sys')
+        # Relative import outside of a package with no __package__ or __spec__ (bpo-37409).
+        self.assertRaises(ImportError, __import__, '',
+                          {'__package__': None, '__spec__': None, '__name__': '__main__'},
+                          locals={}, fromlist=('foo',), level=1)
         # embedded null character
         self.assertRaises(ModuleNotFoundError, __import__, 'string\x00')
 
diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py
index f167c84f1b28..13d04815da7e 100644
--- a/Lib/test/test_import/__init__.py
+++ b/Lib/test/test_import/__init__.py
@@ -772,6 +772,11 @@ def check_relative():
         ns = dict(__package__=object())
         self.assertRaises(TypeError, check_relative)
 
+    def test_parentless_import_shadowed_by_global(self):
+        # Test as if this were done from the REPL where this error most commonly occurs (bpo-37409).
+        script_helper.assert_python_failure('-W', 'ignore', '-c',
+            "foo = 1; from . import foo")
+
     def test_absolute_import_without_future(self):
         # If explicit relative import syntax is used, then do not try
         # to perform an absolute import in the face of failure.
diff --git a/Misc/ACKS b/Misc/ACKS
index 63189984e64a..d37d2bdcc43c 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -975,6 +975,7 @@ Alain Leufroy
 Mark Levinson
 Mark Levitt
 Ivan Levkivskyi
+Ben Lewis
 William Lewis
 Akira Li
 Robert Li
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-06-23-39-05.bpo-37409.1qwzn2.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-06-23-39-05.bpo-37409.1qwzn2.rst
new file mode 100644
index 000000000000..9cfa71548020
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-06-23-39-05.bpo-37409.1qwzn2.rst	
@@ -0,0 +1,2 @@
+Ensure explicit relative imports from interactive sessions and scripts (having no parent package) always raise ImportError, rather than treating the current module as the package.
+Patch by Ben Lewis.
diff --git a/Python/import.c b/Python/import.c
index 41c2f34f12c6..5be4d196a355 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -1646,23 +1646,20 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level
             if (dot == -2) {
                 goto error;
             }
-
-            if (dot >= 0) {
-                PyObject *substr = PyUnicode_Substring(package, 0, dot);
-                if (substr == NULL) {
-                    goto error;
-                }
-                Py_SETREF(package, substr);
+            else if (dot == -1) {
+                goto no_parent_error;
             }
+            PyObject *substr = PyUnicode_Substring(package, 0, dot);
+            if (substr == NULL) {
+                goto error;
+            }
+            Py_SETREF(package, substr);
         }
     }
 
     last_dot = PyUnicode_GET_LENGTH(package);
     if (last_dot == 0) {
-        _PyErr_SetString(tstate, PyExc_ImportError,
-                         "attempted relative import "
-                         "with no known parent package");
-        goto error;
+        goto no_parent_error;
     }
 
     for (level_up = 1; level_up < level; level_up += 1) {
@@ -1688,6 +1685,11 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level
     Py_DECREF(base);
     return abs_name;
 
+  no_parent_error:
+    _PyErr_SetString(tstate, PyExc_ImportError,
+                     "attempted relative import "
+                     "with no known parent package");
+
   error:
     Py_XDECREF(package);
     return NULL;


From webhook-mailer at python.org  Wed Sep 11 06:30:09 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Wed, 11 Sep 2019 10:30:09 -0000
Subject: [Python-checkins] bpo-25810: Clarify eval() docs,
 it does not keywords (GH-15173)
Message-ID: <mailman.508.1568197810.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7a0023e8d17566eb32c836b65c33663303a2224f
commit: 7a0023e8d17566eb32c836b65c33663303a2224f
branch: master
author: smokephil <smokephil at gmail.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-11T05:30:04-05:00
summary:

bpo-25810: Clarify eval() docs, it does not keywords (GH-15173)

files:
M Doc/library/functions.rst

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 2e385a2a37d5..ea2777f75367 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -456,7 +456,7 @@ are always available.  They are listed here in alphabetical order.
               n += 1
 
 
-.. function:: eval(expression, globals=None, locals=None)
+.. function:: eval(expression[, globals[, locals]])
 
    The arguments are a string and optional globals and locals.  If provided,
    *globals* must be a dictionary.  If provided, *locals* can be any mapping


From webhook-mailer at python.org  Wed Sep 11 06:31:16 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Wed, 11 Sep 2019 10:31:16 -0000
Subject: [Python-checkins] [3.8] bpo-37936: Systematically distinguish
 rooted vs. unrooted in .gitignore (GH-15823) (GH-15900)
Message-ID: <mailman.509.1568197876.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/df935b5f0bcc40522c7aac1e844aa176cd8bbdef
commit: df935b5f0bcc40522c7aac1e844aa176cd8bbdef
branch: 3.8
author: Zachary Ware <zachary.ware at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T11:31:12+01:00
summary:

[3.8] bpo-37936: Systematically distinguish rooted vs. unrooted in .gitignore (GH-15823) (GH-15900)

A root cause of bpo-37936 is that it's easy to write a .gitignore
rule that's intended to apply to a specific file (e.g., the
`pyconfig.h` generated by `./configure`) but actually applies to all
similarly-named files in the tree (e.g., `PC/pyconfig.h`.)

Specifically, any rule with no non-trailing slashes is applied in an
"unrooted" way, to files anywhere in the tree.  This means that if we
write the rules in the most obvious-looking way, then

 * for specific files we want to ignore that happen to be in
   subdirectories (like `Modules/config.c`), the rule will work
   as intended, staying "rooted" to the top of the tree; but

 * when a specific file we want to ignore happens to be at the root of
   the repo (like `platform`), then the obvious rule (`platform`) will
   apply much more broadly than intended: if someone tries to add a
   file or directory named `platform` somewhere else in the tree, it
   will unexpectedly get ignored.

That's surprising behavior that can make the .gitignore file's
behavior feel finicky and unpredictable.

To avoid it, we can simply always give a rule "rooted" behavior when
that's what's intended, by systematically using leading slashes.

Further, to help make the pattern obvious when looking at the file and
minimize any need for thinking about the syntax when adding new rules:
separate the rules into one group for each type, with brief comments
identifying them.

For most of these rules it's clear whether they're meant to be rooted
or unrooted, but in a handful of cases I've only guessed.  In that
case the safer default (the choice that won't hide information) is the
narrower, rooted meaning, with a leading slash.  If for some of these
the unrooted meaning is desired after all, it'll be easy to move them
to the unrooted section at the top.

(cherry picked from commit 455122a0094c8cfdf7e062eccc5e5b5885c75e8b)

Co-authored-by: Greg Price <gnprice at gmail.com>

files:
A Misc/NEWS.d/next/Build/2019-09-10-00-54-48.bpo-37936.E7XEwu.rst
M .gitignore
M Lib/test/libregrtest/runtest.py

diff --git a/.gitignore b/.gitignore
index 648f07e765b1..b662374b1255 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,11 +1,14 @@
-# Two-trick pony for OSX and other case insensitive file systems:
-# Ignore ./python binary on Unix but still look into ./Python/ directory.
-/python
-!/Python/
+#####
+# First, rules intended to apply in all subdirectories.
+# These contain no slash, or only a trailing slash.
 
 *.cover
 *.iml
 *.o
+*.a
+*.so*
+*.dylib
+*.dll
 *.orig
 *.pyc
 *.pyd
@@ -18,6 +21,31 @@
 *.profraw
 *.dyn
 .gdb_history
+.purify
+__pycache__
+.hg/
+.svn/
+.idea/
+tags
+TAGS
+.vs/
+.vscode/
+gmon.out
+.coverage
+.mypy_cache/
+
+*.exe
+!Lib/distutils/command/*.exe
+
+# Ignore core dumps... but not Tools/msi/core/ or the like.
+core
+!core/
+
+
+#####
+# Then, rules meant for a specific location relative to the repo root.
+# These must contain a non-trailing slash (and may also have a trailing slash.)
+
 Doc/build/
 Doc/venv/
 Doc/.venv/
@@ -29,7 +57,7 @@ Lib/lib2to3/*.pickle
 Lib/test/data/*
 !Lib/test/data/README
 /Makefile
-Makefile.pre
+/Makefile.pre
 Misc/python.pc
 Misc/python-embed.pc
 Misc/python-config.sh
@@ -38,12 +66,9 @@ Modules/Setup.local
 Modules/config.c
 Modules/ld_so_aix
 Programs/_freeze_importlib
-Programs/_freeze_importlib.exe
 Programs/_testembed
-Programs/_testembed.exe
 PC/python_nt*.h
 PC/pythonnt_rc*.h
-PC/*/*.exe
 PC/*/*.exp
 PC/*/*.lib
 PC/*/*.bsc
@@ -62,52 +87,34 @@ PCbuild/*-pgi
 PCbuild/*-pgo
 PCbuild/*.VC.db
 PCbuild/*.VC.opendb
-PCbuild/.vs/
 PCbuild/amd64/
 PCbuild/arm32/
 PCbuild/arm64/
 PCbuild/obj/
 PCbuild/win32/
-.purify
-__pycache__
-autom4te.cache
-build/
-buildno
-config.cache
-config.log
-config.status
-config.status.lineno
-core
-!Tools/msi/core/
-db_home
-.hg/
-.idea/
-ipch/
-libpython*.a
-libpython*.so*
-libpython*.dylib
-libpython*.dll
-platform
-pybuilddir.txt
+/autom4te.cache
+/build/
+/config.cache
+/config.log
+/config.status
+/config.status.lineno
+/platform
+/pybuilddir.txt
 /pyconfig.h
-python-config
-python-config.py
-python.bat
-python.exe
-python-gdb.py
-python.exe-gdb.py
-reflog.txt
-.svn/
-tags
-TAGS
-.coverage
-coverage/
-externals/
-htmlcov/
+/python-config
+/python-config.py
+/python.bat
+/python-gdb.py
+/python.exe-gdb.py
+/reflog.txt
+/coverage/
+/externals/
+/htmlcov/
 Tools/msi/obj
 Tools/ssl/amd64
 Tools/ssl/win32
-.vs/
-.vscode/
-gmon.out
-.mypy_cache/
+
+# Two-trick pony for OSX and other case insensitive file systems:
+# Ignore ./python binary on Unix but still look into ./Python/ directory.
+/python
+!/Python/
diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py
index e7dce180cb37..eeb108bb4473 100644
--- a/Lib/test/libregrtest/runtest.py
+++ b/Lib/test/libregrtest/runtest.py
@@ -313,9 +313,7 @@ def cleanup_test_droppings(test_name, verbose):
     # since if a test leaves a file open, it cannot be deleted by name (while
     # there's nothing we can do about that here either, we can display the
     # name of the offending test, which is a real help).
-    for name in (support.TESTFN,
-                 "db_home",
-                ):
+    for name in (support.TESTFN,):
         if not os.path.exists(name):
             continue
 
diff --git a/Misc/NEWS.d/next/Build/2019-09-10-00-54-48.bpo-37936.E7XEwu.rst b/Misc/NEWS.d/next/Build/2019-09-10-00-54-48.bpo-37936.E7XEwu.rst
new file mode 100644
index 000000000000..5ded61eafe94
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-09-10-00-54-48.bpo-37936.E7XEwu.rst
@@ -0,0 +1,5 @@
+The :file:`.gitignore` file systematically keeps "rooted", with a
+non-trailing slash, all the rules that are meant to apply to files in a
+specific place in the repo.  Previously, when the intended file to ignore
+happened to be at the root of the repo, we'd most often accidentally also
+ignore files and directories with the same name anywhere in the tree.


From webhook-mailer at python.org  Wed Sep 11 06:31:40 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Wed, 11 Sep 2019 10:31:40 -0000
Subject: [Python-checkins] bpo-28494: install ziptestdata to fix install bot
 (GH-15902)
Message-ID: <mailman.510.1568197900.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c37447481ec8f6d0e49d0587ec0de3f9e7d56b28
commit: c37447481ec8f6d0e49d0587ec0de3f9e7d56b28
branch: master
author: Gregory P. Smith <greg at krypto.org>
committer: GitHub <noreply at github.com>
date: 2019-09-11T05:31:37-05:00
summary:

bpo-28494: install ziptestdata to fix install bot (GH-15902)

files:
M Makefile.pre.in

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 4f4f096e7283..1c0958ec974b 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1374,6 +1374,7 @@ LIBSUBDIRS=	tkinter tkinter/test tkinter/test/test_tkinter \
 		test/test_importlib/source \
 		test/test_importlib/zipdata01 \
 		test/test_importlib/zipdata02 \
+		test/ziptestdata \
 		asyncio \
 		test/test_asyncio \
 		collections concurrent concurrent/futures encodings \


From webhook-mailer at python.org  Wed Sep 11 06:39:40 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 10:39:40 -0000
Subject: [Python-checkins] bpo-16438: Doc: confusing text regarding numeric
 precedence corrected (GH-10521)
Message-ID: <mailman.511.1568198381.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/80db4b4be54ccdb5b67821506b6db2b27bd7c28a
commit: 80db4b4be54ccdb5b67821506b6db2b27bd7c28a
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T03:39:36-07:00
summary:

bpo-16438: Doc: confusing text regarding numeric precedence corrected (GH-10521)

(cherry picked from commit 4576b5431bd597df7581fe3c852b315e47e4b230)

Co-authored-by: Anjali <anjali30malik at gmail.com>

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 0f7c369ea51f..dade2cd69fee 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -265,9 +265,8 @@ which is narrower than complex.  Comparisons between numbers of mixed type use
 the same rule. [2]_ The constructors :func:`int`, :func:`float`, and
 :func:`complex` can be used to produce numbers of a specific type.
 
-All numeric types (except complex) support the following operations, sorted by
-ascending priority (all numeric operations have a higher priority than
-comparison operations):
+All numeric types (except complex) support the following operations (for priorities of
+the operations, see :ref:`operator-summary`):
 
 +---------------------+---------------------------------+---------+--------------------+
 | Operation           | Result                          | Notes   | Full documentation |


From webhook-mailer at python.org  Wed Sep 11 06:40:40 2019
From: webhook-mailer at python.org (Andrew Svetlov)
Date: Wed, 11 Sep 2019 10:40:40 -0000
Subject: [Python-checkins] [3.8] bpo-36373: Fix deprecation warnings
 (GH-15889) (GH-15901)
Message-ID: <mailman.512.1568198440.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4601f7a49fe8ed00c4b6b70b0eda2b3922568e9b
commit: 4601f7a49fe8ed00c4b6b70b0eda2b3922568e9b
branch: 3.8
author: Andrew Svetlov <andrew.svetlov at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T13:40:36+03:00
summary:

[3.8] bpo-36373: Fix deprecation warnings (GH-15889) (GH-15901)

https://bugs.python.org/issue36373
(cherry picked from commit 7264e92b718d307cc499b2f10eab7644b00f0499)

Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com>

files:
M Lib/asyncio/locks.py
M Lib/asyncio/queues.py
M Lib/test/test_asyncio/test_locks.py
M Lib/test/test_asyncio/test_queues.py
M Lib/unittest/async_case.py

diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py
index f63d4cedbbb7..d94daeb5a173 100644
--- a/Lib/asyncio/locks.py
+++ b/Lib/asyncio/locks.py
@@ -332,7 +332,7 @@ def __init__(self, lock=None, *, loop=None):
                           DeprecationWarning, stacklevel=2)
 
         if lock is None:
-            lock = Lock(loop=self._loop)
+            lock = Lock(loop=loop)
         elif lock._loop is not self._loop:
             raise ValueError("loop argument must agree with lock")
 
diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py
index c96b4a0c1ba3..390ae9a6821c 100644
--- a/Lib/asyncio/queues.py
+++ b/Lib/asyncio/queues.py
@@ -45,7 +45,7 @@ def __init__(self, maxsize=0, *, loop=None):
         # Futures.
         self._putters = collections.deque()
         self._unfinished_tasks = 0
-        self._finished = locks.Event(loop=self._loop)
+        self._finished = locks.Event(loop=loop)
         self._finished.set()
         self._init(maxsize)
 
diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py
index d69b56dcda22..a9953b4b2a2d 100644
--- a/Lib/test/test_asyncio/test_locks.py
+++ b/Lib/test/test_asyncio/test_locks.py
@@ -500,10 +500,9 @@ def test_ctor_loop(self):
             self.assertIs(cond._loop, self.loop)
 
     def test_ctor_noloop(self):
-        with self.assertWarns(DeprecationWarning):
-            asyncio.set_event_loop(self.loop)
-            cond = asyncio.Condition()
-            self.assertIs(cond._loop, self.loop)
+        asyncio.set_event_loop(self.loop)
+        cond = asyncio.Condition()
+        self.assertIs(cond._loop, self.loop)
 
     def test_wait(self):
         with self.assertWarns(DeprecationWarning):
diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py
index 02e8e43ccee6..171176c9fc53 100644
--- a/Lib/test/test_asyncio/test_queues.py
+++ b/Lib/test/test_asyncio/test_queues.py
@@ -83,8 +83,7 @@ def test_ctor_loop(self):
 
     def test_ctor_noloop(self):
         asyncio.set_event_loop(self.loop)
-        with self.assertWarns(DeprecationWarning):
-            q = asyncio.Queue()
+        q = asyncio.Queue()
         self.assertIs(q._loop, self.loop)
 
     def test_repr(self):
diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py
index a3c8bfb9eca7..1bc1312c8c2e 100644
--- a/Lib/unittest/async_case.py
+++ b/Lib/unittest/async_case.py
@@ -89,8 +89,9 @@ def _callMaybeAsync(self, func, /, *args, **kwargs):
         else:
             return ret
 
-    async def _asyncioLoopRunner(self):
-        queue = self._asyncioCallsQueue
+    async def _asyncioLoopRunner(self, fut):
+        self._asyncioCallsQueue = queue = asyncio.Queue()
+        fut.set_result(None)
         while True:
             query = await queue.get()
             queue.task_done()
@@ -113,8 +114,9 @@ def _setupAsyncioLoop(self):
         asyncio.set_event_loop(loop)
         loop.set_debug(True)
         self._asyncioTestLoop = loop
-        self._asyncioCallsQueue = asyncio.Queue(loop=loop)
-        self._asyncioCallsTask = loop.create_task(self._asyncioLoopRunner())
+        fut = loop.create_future()
+        self._asyncioCallsTask = loop.create_task(self._asyncioLoopRunner(fut))
+        loop.run_until_complete(fut)
 
     def _tearDownAsyncioLoop(self):
         assert self._asyncioTestLoop is not None


From webhook-mailer at python.org  Wed Sep 11 06:41:31 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 10:41:31 -0000
Subject: [Python-checkins] bpo-16438: Doc: confusing text regarding numeric
 precedence corrected (GH-10521)
Message-ID: <mailman.513.1568198492.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9210b5826fab5248ac28a15c1b8b39ae949d0ef8
commit: 9210b5826fab5248ac28a15c1b8b39ae949d0ef8
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T03:41:28-07:00
summary:

bpo-16438: Doc: confusing text regarding numeric precedence corrected (GH-10521)

(cherry picked from commit 4576b5431bd597df7581fe3c852b315e47e4b230)

Co-authored-by: Anjali <anjali30malik at gmail.com>

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index b9581ce1c9ae..07282dce00d3 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -267,9 +267,8 @@ which is narrower than complex.  Comparisons between numbers of mixed type use
 the same rule. [2]_ The constructors :func:`int`, :func:`float`, and
 :func:`complex` can be used to produce numbers of a specific type.
 
-All numeric types (except complex) support the following operations, sorted by
-ascending priority (all numeric operations have a higher priority than
-comparison operations):
+All numeric types (except complex) support the following operations (for priorities of
+the operations, see :ref:`operator-summary`):
 
 +---------------------+---------------------------------+---------+--------------------+
 | Operation           | Result                          | Notes   | Full documentation |


From webhook-mailer at python.org  Wed Sep 11 06:43:34 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 10:43:34 -0000
Subject: [Python-checkins] bpo-38081: Fixes ntpath.realpath('NUL') (GH-15899)
Message-ID: <mailman.514.1568198615.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/57491de7c33c5886c4cae2f468b474d9b4e6fed2
commit: 57491de7c33c5886c4cae2f468b474d9b4e6fed2
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T03:43:30-07:00
summary:

bpo-38081: Fixes ntpath.realpath('NUL') (GH-15899)

(cherry picked from commit 92521fea5d0d4aeb9b6a3c3fdd4654af700ad5c8)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-11-10-22-01.bpo-38081.8JhzjD.rst
M Lib/ntpath.py
M Lib/test/test_ntpath.py

diff --git a/Lib/ntpath.py b/Lib/ntpath.py
index 1d22d5f1dc2a..1b5e16f95f16 100644
--- a/Lib/ntpath.py
+++ b/Lib/ntpath.py
@@ -535,9 +535,10 @@ def _readlink_deep(path, seen=None):
             try:
                 path = _nt_readlink(path)
             except OSError as ex:
-                # Stop on file (2) or directory (3) not found, or
-                # paths that are not reparse points (4390)
-                if ex.winerror in (2, 3, 4390):
+                # Stop on incorrect function (1), file (2) or
+                # directory (3) not found, or paths that are
+                # not reparse points (4390)
+                if ex.winerror in (1, 2, 3, 4390):
                     break
                 raise
             except ValueError:
@@ -553,9 +554,9 @@ def _getfinalpathname_nonstrict(path):
         except OSError:
             pass
 
-        # Allow file (2) or directory (3) not found, invalid syntax (123),
-        # and symlinks that cannot be followed (1921)
-        allowed_winerror = 2, 3, 123, 1921
+        # Allow file (2) or directory (3) not found, incorrect parameter (87),
+        # invalid syntax (123), and symlinks that cannot be followed (1921)
+        allowed_winerror = 2, 3, 87, 123, 1921
 
         # Non-strict algorithm is to find as much of the target directory
         # as we can and join the rest.
diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
index c5c96e32d747..2f0faf94726f 100644
--- a/Lib/test/test_ntpath.py
+++ b/Lib/test/test_ntpath.py
@@ -400,6 +400,10 @@ def test_realpath_symlink_prefix(self):
         self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link"),
                              "\\\\?\\" + ABSTFN + "3.")
 
+    @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname')
+    def test_realpath_nul(self):
+        tester("ntpath.realpath('NUL')", r'\\.\NUL')
+
     def test_expandvars(self):
         with support.EnvironmentVarGuard() as env:
             env.clear()
diff --git a/Misc/NEWS.d/next/Windows/2019-09-11-10-22-01.bpo-38081.8JhzjD.rst b/Misc/NEWS.d/next/Windows/2019-09-11-10-22-01.bpo-38081.8JhzjD.rst
new file mode 100644
index 000000000000..e9d7a30eed9f
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-11-10-22-01.bpo-38081.8JhzjD.rst
@@ -0,0 +1 @@
+Prevent error calling :func:`os.path.realpath` on ``'NUL'``.


From webhook-mailer at python.org  Wed Sep 11 06:45:58 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 10:45:58 -0000
Subject: [Python-checkins] bpo-37585: Add clarification regarding comparing
 dict.values() (GH-14954)
Message-ID: <mailman.515.1568198759.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3cd147bf599f116593dc06ed7dfe948b759aabd3
commit: 3cd147bf599f116593dc06ed7dfe948b759aabd3
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T03:45:54-07:00
summary:

bpo-37585: Add clarification regarding comparing dict.values() (GH-14954)

(cherry picked from commit 6472ece5a0fe82809d3aa0ffb281796fcd252d76)

Co-authored-by: Kyle Stanley <aeros167 at gmail.com>

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 07282dce00d3..1903864a55e2 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -4267,6 +4267,14 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098:
       Return a new view of the dictionary's values.  See the
       :ref:`documentation of view objects <dict-views>`.
 
+      An equality comparison between one ``dict.values()`` view and another
+      will always return ``False``. This also applies when comparing
+      ``dict.values()`` to itself::
+
+         >>> d = {'a': 1}
+         >>> d.values() == d.values()
+         False
+
    Dictionaries compare equal if and only if they have the same ``(key,
    value)`` pairs. Order comparisons ('<', '<=', '>=', '>') raise
    :exc:`TypeError`.


From webhook-mailer at python.org  Wed Sep 11 06:46:56 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 10:46:56 -0000
Subject: [Python-checkins] bpo-37585: Add clarification regarding comparing
 dict.values() (GH-14954)
Message-ID: <mailman.516.1568198818.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/690a16d455603500a0c6df0bd87e49c9b37a6950
commit: 690a16d455603500a0c6df0bd87e49c9b37a6950
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T03:46:53-07:00
summary:

bpo-37585: Add clarification regarding comparing dict.values() (GH-14954)

(cherry picked from commit 6472ece5a0fe82809d3aa0ffb281796fcd252d76)

Co-authored-by: Kyle Stanley <aeros167 at gmail.com>

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index dade2cd69fee..32bd7d2b894d 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -4343,6 +4343,14 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098:
       Return a new view of the dictionary's values.  See the
       :ref:`documentation of view objects <dict-views>`.
 
+      An equality comparison between one ``dict.values()`` view and another
+      will always return ``False``. This also applies when comparing
+      ``dict.values()`` to itself::
+
+         >>> d = {'a': 1}
+         >>> d.values() == d.values()
+         False
+
    Dictionaries compare equal if and only if they have the same ``(key,
    value)`` pairs. Order comparisons ('<', '<=', '>=', '>') raise
    :exc:`TypeError`.


From webhook-mailer at python.org  Wed Sep 11 06:50:42 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Wed, 11 Sep 2019 10:50:42 -0000
Subject: [Python-checkins] bpo-35066: Make trailing percent test more
 portable. (GH-15907)
Message-ID: <mailman.517.1568199043.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f2173ae38fa49235c3cdc28ae2ca2e19a375a596
commit: f2173ae38fa49235c3cdc28ae2ca2e19a375a596
branch: master
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-11T11:50:38+01:00
summary:

bpo-35066: Make trailing percent test more portable. (GH-15907)

Different libc implementations have different behavior when presented with trailing % in strftime strings. To make test_strftime_trailing_percent more portable, compare the output of datetime.strftime directly to that of time.strftime rather than hardcoding.

files:
M Lib/test/datetimetester.py

diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index a1ed614cf0cd..32977b12ebae 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -1449,15 +1449,20 @@ def test_strftime(self):
         t.strftime("%f")
 
     def test_strftime_trailing_percent(self):
-        # bpo-35066: make sure trailing '%' doesn't cause
-        # datetime's strftime to complain
+        # bpo-35066: Make sure trailing '%' doesn't cause datetime's strftime to
+        # complain. Different libcs have different handling of trailing
+        # percents, so we simply check datetime's strftime acts the same as
+        # time.strftime.
         t = self.theclass(2005, 3, 2)
         try:
             _time.strftime('%')
         except ValueError:
             self.skipTest('time module does not support trailing %')
-        self.assertEqual(t.strftime('%'), '%')
-        self.assertEqual(t.strftime("m:%m d:%d y:%y %"), "m:03 d:02 y:05 %")
+        self.assertEqual(t.strftime('%'), _time.strftime('%', t.timetuple()))
+        self.assertEqual(
+            t.strftime("m:%m d:%d y:%y %"),
+            _time.strftime("m:03 d:02 y:05 %", t.timetuple()),
+        )
 
     def test_format(self):
         dt = self.theclass(2007, 9, 10)


From webhook-mailer at python.org  Wed Sep 11 06:53:20 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 10:53:20 -0000
Subject: [Python-checkins] bpo-25810: Clarify eval() docs,
 it does not keywords (GH-15173)
Message-ID: <mailman.518.1568199201.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4e914ab29f6d48a9fd045ea6a25dbf9e2fb603f9
commit: 4e914ab29f6d48a9fd045ea6a25dbf9e2fb603f9
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T03:53:16-07:00
summary:

bpo-25810: Clarify eval() docs, it does not keywords (GH-15173)

(cherry picked from commit 7a0023e8d17566eb32c836b65c33663303a2224f)

Co-authored-by: smokephil <smokephil at gmail.com>

files:
M Doc/library/functions.rst

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 9ea7b7b55a97..2f3ef4f7fc76 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -454,7 +454,7 @@ are always available.  They are listed here in alphabetical order.
               n += 1
 
 
-.. function:: eval(expression, globals=None, locals=None)
+.. function:: eval(expression[, globals[, locals]])
 
    The arguments are a string and optional globals and locals.  If provided,
    *globals* must be a dictionary.  If provided, *locals* can be any mapping


From webhook-mailer at python.org  Wed Sep 11 06:55:33 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 10:55:33 -0000
Subject: [Python-checkins] bpo-25810: Clarify eval() docs,
 it does not keywords (GH-15173)
Message-ID: <mailman.519.1568199334.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d378fdb10a5476b86b5a01d74c1bcc4e2f52e003
commit: d378fdb10a5476b86b5a01d74c1bcc4e2f52e003
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T03:55:29-07:00
summary:

bpo-25810: Clarify eval() docs, it does not keywords (GH-15173)

(cherry picked from commit 7a0023e8d17566eb32c836b65c33663303a2224f)

Co-authored-by: smokephil <smokephil at gmail.com>

files:
M Doc/library/functions.rst

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 11147525d7e8..313dbffdcde9 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -426,7 +426,7 @@ are always available.  They are listed here in alphabetical order.
               n += 1
 
 
-.. function:: eval(expression, globals=None, locals=None)
+.. function:: eval(expression[, globals[, locals]])
 
    The arguments are a string and optional globals and locals.  If provided,
    *globals* must be a dictionary.  If provided, *locals* can be any mapping


From webhook-mailer at python.org  Wed Sep 11 06:58:03 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Wed, 11 Sep 2019 10:58:03 -0000
Subject: [Python-checkins] bpo-38103: fix conflicting labels in the docs.
 (GH-15906)
Message-ID: <mailman.520.1568199484.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2d8d597bb8f882a7677db5a2739df0e617098634
commit: 2d8d597bb8f882a7677db5a2739df0e617098634
branch: master
author: Ezio Melotti <ezio.melotti at gmail.com>
committer: Lisa Roach <lisaroach14 at gmail.com>
date: 2019-09-11T11:57:59+01:00
summary:

bpo-38103: fix conflicting labels in the docs. (GH-15906)

files:
M Doc/c-api/typeobj.rst
M Doc/distutils/examples.rst

diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index 638fb0c32d78..f8e30a0380b9 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -20,7 +20,7 @@ functionality.  The fields of the type object are examined in detail in this
 section.  The fields will be described in the order in which they occur in the
 structure.
 
-In addition to the following quick reference, the :ref:`examples`
+In addition to the following quick reference, the :ref:`typedef-examples`
 section provides at-a-glance insight into the meaning and use of
 :c:type:`PyTypeObject`.
 
@@ -2450,7 +2450,7 @@ Slot Type typedefs
 .. c:type:: int (*objobjargproc)(PyObject *, PyObject *, PyObject *)
 
 
-.. _examples:
+.. _typedef-examples:
 
 Examples
 ========
diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst
index 4ac552c7c699..44f70831d0bf 100644
--- a/Doc/distutils/examples.rst
+++ b/Doc/distutils/examples.rst
@@ -1,8 +1,8 @@
 .. _distutils_examples:
 
-********
-Examples
-********
+******************
+Distutils Examples
+******************
 
 .. include:: ./_setuptools_disclaimer.rst
 


From webhook-mailer at python.org  Wed Sep 11 07:01:05 2019
From: webhook-mailer at python.org (Petr Viktorin)
Date: Wed, 11 Sep 2019 11:01:05 -0000
Subject: [Python-checkins] bpo-37151: remove special case for PyCFunction
 from PyObject_Call (GH-14684)
Message-ID: <mailman.521.1568199666.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7a6873cdb1f496447ac5d57ae457eacbb56b7972
commit: 7a6873cdb1f496447ac5d57ae457eacbb56b7972
branch: master
author: Jeroen Demeyer <J.Demeyer at UGent.be>
committer: Petr Viktorin <encukou at gmail.com>
date: 2019-09-11T12:01:01+01:00
summary:

bpo-37151: remove special case for PyCFunction from PyObject_Call (GH-14684)

bpo-37151: remove special case for PyCFunction from PyObject_Call

Alse, make the undocumented function PyCFunction_Call an alias
of PyObject_Call and deprecate it.

files:
A Misc/NEWS.d/next/C API/2019-07-16-11-02-00.bpo-37151.YKfuNA.rst
M Include/methodobject.h
M Objects/call.c
M Objects/methodobject.c
M Python/ceval.c
M Tools/gdb/libpython.py

diff --git a/Include/methodobject.h b/Include/methodobject.h
index 9f5f7c482c87..3bccf5a3f60d 100644
--- a/Include/methodobject.h
+++ b/Include/methodobject.h
@@ -39,7 +39,7 @@ PyAPI_FUNC(int) PyCFunction_GetFlags(PyObject *);
 #define PyCFunction_GET_FLAGS(func) \
         (((PyCFunctionObject *)func) -> m_ml -> ml_flags)
 #endif
-PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *);
+Py_DEPRECATED(3.9) PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *);
 
 struct PyMethodDef {
     const char  *ml_name;   /* The name of the built-in function/method */
diff --git a/Misc/NEWS.d/next/C API/2019-07-16-11-02-00.bpo-37151.YKfuNA.rst b/Misc/NEWS.d/next/C API/2019-07-16-11-02-00.bpo-37151.YKfuNA.rst
new file mode 100644
index 000000000000..0841e570f6c9
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-07-16-11-02-00.bpo-37151.YKfuNA.rst	
@@ -0,0 +1 @@
+``PyCFunction_Call`` is now a deprecated alias of :c:func:`PyObject_Call`.
diff --git a/Objects/call.c b/Objects/call.c
index 8a60b3e58e30..a715bcbee4ad 100644
--- a/Objects/call.c
+++ b/Objects/call.c
@@ -5,9 +5,6 @@
 #include "frameobject.h"
 
 
-static PyObject *
-cfunction_call_varargs(PyObject *func, PyObject *args, PyObject *kwargs);
-
 static PyObject *const *
 _PyStack_UnpackDict(PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs,
                     PyObject **p_kwnames);
@@ -236,11 +233,6 @@ PyObject_Call(PyObject *callable, PyObject *args, PyObject *kwargs)
     if (_PyVectorcall_Function(callable) != NULL) {
         return PyVectorcall_Call(callable, args, kwargs);
     }
-    else if (PyCFunction_Check(callable)) {
-        /* This must be a METH_VARARGS function, otherwise we would be
-         * in the previous case */
-        return cfunction_call_varargs(callable, args, kwargs);
-    }
     else {
         call = callable->ob_type->tp_call;
         if (call == NULL) {
@@ -261,6 +253,13 @@ PyObject_Call(PyObject *callable, PyObject *args, PyObject *kwargs)
 }
 
 
+PyObject *
+PyCFunction_Call(PyObject *callable, PyObject *args, PyObject *kwargs)
+{
+    return PyObject_Call(callable, args, kwargs);
+}
+
+
 /* --- PyFunction call functions ---------------------------------- */
 
 static PyObject* _Py_HOT_FUNCTION
@@ -363,60 +362,6 @@ _PyFunction_Vectorcall(PyObject *func, PyObject* const* stack,
 }
 
 
-/* --- PyCFunction call functions --------------------------------- */
-
-static PyObject *
-cfunction_call_varargs(PyObject *func, PyObject *args, PyObject *kwargs)
-{
-    assert(!PyErr_Occurred());
-    assert(kwargs == NULL || PyDict_Check(kwargs));
-
-    PyCFunction meth = PyCFunction_GET_FUNCTION(func);
-    PyObject *self = PyCFunction_GET_SELF(func);
-    PyObject *result;
-
-    assert(PyCFunction_GET_FLAGS(func) & METH_VARARGS);
-    if (PyCFunction_GET_FLAGS(func) & METH_KEYWORDS) {
-        if (Py_EnterRecursiveCall(" while calling a Python object")) {
-            return NULL;
-        }
-
-        result = (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, args, kwargs);
-
-        Py_LeaveRecursiveCall();
-    }
-    else {
-        if (kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) {
-            PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments",
-                         ((PyCFunctionObject*)func)->m_ml->ml_name);
-            return NULL;
-        }
-
-        if (Py_EnterRecursiveCall(" while calling a Python object")) {
-            return NULL;
-        }
-
-        result = (*meth)(self, args);
-
-        Py_LeaveRecursiveCall();
-    }
-
-    return _Py_CheckFunctionResult(func, result, NULL);
-}
-
-
-PyObject *
-PyCFunction_Call(PyObject *func, PyObject *args, PyObject *kwargs)
-{
-    /* For METH_VARARGS, we cannot use vectorcall as the vectorcall pointer
-     * is NULL. This is intentional, since vectorcall would be slower. */
-    if (PyCFunction_GET_FLAGS(func) & METH_VARARGS) {
-        return cfunction_call_varargs(func, args, kwargs);
-    }
-    return PyVectorcall_Call(func, args, kwargs);
-}
-
-
 /* --- More complex call functions -------------------------------- */
 
 /* External interface to call any callable object.
diff --git a/Objects/methodobject.c b/Objects/methodobject.c
index 7d70cc04a238..a5f0c5d3465d 100644
--- a/Objects/methodobject.c
+++ b/Objects/methodobject.c
@@ -19,6 +19,8 @@ static PyObject * cfunction_vectorcall_NOARGS(
     PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames);
 static PyObject * cfunction_vectorcall_O(
     PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames);
+static PyObject * cfunction_call(
+    PyObject *func, PyObject *args, PyObject *kwargs);
 
 
 PyObject *
@@ -289,7 +291,7 @@ PyTypeObject PyCFunction_Type = {
     0,                                          /* tp_as_sequence */
     0,                                          /* tp_as_mapping */
     (hashfunc)meth_hash,                        /* tp_hash */
-    PyCFunction_Call,                           /* tp_call */
+    cfunction_call,                             /* tp_call */
     0,                                          /* tp_str */
     PyObject_GenericGetAttr,                    /* tp_getattro */
     0,                                          /* tp_setattro */
@@ -441,3 +443,36 @@ cfunction_vectorcall_O(
     Py_LeaveRecursiveCall();
     return result;
 }
+
+
+static PyObject *
+cfunction_call(PyObject *func, PyObject *args, PyObject *kwargs)
+{
+    assert(!PyErr_Occurred());
+    assert(kwargs == NULL || PyDict_Check(kwargs));
+
+    int flags = PyCFunction_GET_FLAGS(func);
+    if (!(flags & METH_VARARGS)) {
+        /* If this is not a METH_VARARGS function, delegate to vectorcall */
+        return PyVectorcall_Call(func, args, kwargs);
+    }
+
+    /* For METH_VARARGS, we cannot use vectorcall as the vectorcall pointer
+     * is NULL. This is intentional, since vectorcall would be slower. */
+    PyCFunction meth = PyCFunction_GET_FUNCTION(func);
+    PyObject *self = PyCFunction_GET_SELF(func);
+
+    PyObject *result;
+    if (flags & METH_KEYWORDS) {
+        result = (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, args, kwargs);
+    }
+    else {
+        if (kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) {
+            PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments",
+                         ((PyCFunctionObject*)func)->m_ml->ml_name);
+            return NULL;
+        }
+        result = meth(self, args);
+    }
+    return _Py_CheckFunctionResult(func, result, NULL);
+}
diff --git a/Python/ceval.c b/Python/ceval.c
index 01e2a1b9666f..964ce1333694 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -5001,7 +5001,7 @@ do_call_core(PyThreadState *tstate, PyObject *func, PyObject *callargs, PyObject
     PyObject *result;
 
     if (PyCFunction_Check(func)) {
-        C_TRACE(result, PyCFunction_Call(func, callargs, kwdict));
+        C_TRACE(result, PyObject_Call(func, callargs, kwdict));
         return result;
     }
     else if (Py_TYPE(func) == &PyMethodDescr_Type) {
diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py
index e40f79b5c3c7..e28513db261b 100755
--- a/Tools/gdb/libpython.py
+++ b/Tools/gdb/libpython.py
@@ -1564,7 +1564,7 @@ def is_other_python_frame(self):
             return False
 
         if (caller.startswith('cfunction_vectorcall_') or
-            caller == 'cfunction_call_varargs'):
+            caller == 'cfunction_call'):
             arg_name = 'func'
             # Within that frame:
             #   "func" is the local containing the PyObject* of the


From webhook-mailer at python.org  Wed Sep 11 07:02:18 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 11:02:18 -0000
Subject: [Python-checkins] bpo-32972: Document IsolatedAsyncioTestCase of
 unittest module (GH-15878)
Message-ID: <mailman.522.1568199740.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6a9fd66f6e4445a418c43c92585b9e06d76df4b1
commit: 6a9fd66f6e4445a418c43c92585b9e06d76df4b1
branch: master
author: Xtreak <tir.karthi at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T04:02:14-07:00
summary:

bpo-32972: Document IsolatedAsyncioTestCase of unittest module (GH-15878)



* Document `unittest.IsolatedAsyncioTestCase` API
* Add a simple example with respect to order of evaluation of setup and teardown calls.


https://bugs.python.org/issue32972



Automerge-Triggered-By: @asvetlov

files:
M Doc/library/unittest.rst
M Doc/whatsnew/3.8.rst
M Misc/NEWS.d/3.8.0b1.rst

diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst
index 320d898fc8d4..8c9affe9db95 100644
--- a/Doc/library/unittest.rst
+++ b/Doc/library/unittest.rst
@@ -1486,8 +1486,84 @@ Test cases
       .. versionadded:: 3.8
 
 
+.. class:: IsolatedAsyncioTestCase(methodName='runTest')
 
+   This class provides an API similar to :class:`TestCase` and also accepts
+   coroutines as test functions.
 
+   .. versionadded:: 3.8
+
+   .. coroutinemethod:: asyncSetUp()
+
+      Method called to prepare the test fixture. This is called after :meth:`setUp`.
+      This is called immediately before calling the test method; other than
+      :exc:`AssertionError` or :exc:`SkipTest`, any exception raised by this method
+      will be considered an error rather than a test failure. The default implementation
+      does nothing.
+
+   .. coroutinemethod:: asyncTearDown()
+
+      Method called immediately after the test method has been called and the
+      result recorded.  This is called before :meth:`tearDown`. This is called even if
+      the test method raised an exception, so the implementation in subclasses may need
+      to be particularly careful about checking internal state.  Any exception, other than
+      :exc:`AssertionError` or :exc:`SkipTest`, raised by this method will be
+      considered an additional error rather than a test failure (thus increasing
+      the total number of reported errors). This method will only be called if
+      the :meth:`asyncSetUp` succeeds, regardless of the outcome of the test method.
+      The default implementation does nothing.
+
+   .. method:: addAsyncCleanup(function, /, *args, **kwargs)
+
+      This method accepts a coroutine that can be used as a cleanup function.
+
+   .. method:: run(result=None)
+
+      Sets up a new event loop to run the test, collecting the result into
+      the :class:`TestResult` object passed as *result*.  If *result* is
+      omitted or ``None``, a temporary result object is created (by calling
+      the :meth:`defaultTestResult` method) and used. The result object is
+      returned to :meth:`run`'s caller. At the end of the test all the tasks
+      in the event loop are cancelled.
+
+
+   An example illustrating the order::
+
+      from unittest import IsolatedAsyncioTestCase
+
+      events = []
+
+
+      class Test(IsolatedAsyncioTestCase):
+
+
+          def setUp(self):
+              events.append("setUp")
+
+          async def asyncSetUp(self):
+              self._async_connection = await AsyncConnection()
+              events.append("asyncSetUp")
+
+          async def test_response(self):
+              events.append("test_response")
+              response = await self._async_connection.get("https://example.com")
+              self.assertEqual(response.status_code, 200)
+              self.addAsyncCleanup(self.on_cleanup)
+
+          def tearDown(self):
+              events.append("tearDown")
+
+          async def asyncTearDown(self):
+              await self._async_connection.close()
+              events.append("asyncTearDown")
+
+          async def on_cleanup(self):
+              events.append("cleanup")
+
+      if __name__ == "__main__":
+          unittest.main()
+
+   After running the test ``events`` would contain ``["setUp", "asyncSetUp", "test_response", "asyncTearDown", "tearDown", "cleanup"]``
 
 
 .. class:: FunctionTestCase(testFunc, setUp=None, tearDown=None, description=None)
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 2b4eb63d61f6..07202ea4ff08 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -1111,6 +1111,32 @@ unittest
 * Several mock assert functions now also print a list of actual calls upon
   failure. (Contributed by Petter Strandmark in :issue:`35047`.)
 
+* :mod:`unittest` module gained support for coroutines to be used as test cases
+  with :class:`unittest.IsolatedAsyncioTestCase`.
+  (Contributed by Andrew Svetlov in :issue:`32972`.)
+
+  Example::
+
+    import unittest
+
+
+    class TestRequest(unittest.IsolatedAsyncioTestCase):
+
+        async def asyncSetUp(self):
+            self.connection = await AsyncConnection()
+
+        async def test_get(self):
+            response = await self.connection.get("https://example.com")
+            self.assertEqual(response.status_code, 200)
+
+        async def asyncTearDown(self):
+            await self.connection.close()
+
+
+    if __name__ == "__main__":
+        unittest.main()
+
+
 venv
 ----
 
diff --git a/Misc/NEWS.d/3.8.0b1.rst b/Misc/NEWS.d/3.8.0b1.rst
index 84b0350134fb..43a88a37c5cb 100644
--- a/Misc/NEWS.d/3.8.0b1.rst
+++ b/Misc/NEWS.d/3.8.0b1.rst
@@ -808,7 +808,7 @@ detect classes that can be passed to `hex()`, `oct()` and `bin()`.
 .. nonce: LoeUNh
 .. section: Library
 
-Implement ``unittest.AsyncTestCase`` to help testing asyncio-based code.
+Implement ``unittest.IsolatedAsyncioTestCase`` to help testing asyncio-based code.
 
 ..
 


From webhook-mailer at python.org  Wed Sep 11 07:04:08 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Wed, 11 Sep 2019 11:04:08 -0000
Subject: [Python-checkins] bpo-35168: Make shlex.punctuation_chars read-only
 (#11631)
Message-ID: <mailman.523.1568199850.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/972cf5c06a5ba16ad243a442dbb9c15307fbed95
commit: 972cf5c06a5ba16ad243a442dbb9c15307fbed95
branch: master
author: Alex <a.v.shkop at gmail.com>
committer: Vinay Sajip <vinay_sajip at yahoo.co.uk>
date: 2019-09-11T12:04:04+01:00
summary:

bpo-35168: Make shlex.punctuation_chars read-only (#11631)

* bpo-35168: Documentation about shlex.punctuation_chars now states that it should be set in __init__.py

* bpo-35168: Convert shlex.punctuation_chars to read-only property

* Add NEWS.d entry

files:
A Misc/NEWS.d/next/Library/2019-01-22-09-23-20.bpo-35168.UGv2yW.rst
M Doc/library/shlex.rst
M Lib/shlex.py
M Lib/test/test_shlex.py

diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst
index a8421fdb7008..adc23da6d491 100644
--- a/Doc/library/shlex.rst
+++ b/Doc/library/shlex.rst
@@ -113,7 +113,9 @@ The :mod:`shlex` module defines the following class:
    characters, those characters will be used as the punctuation characters.  Any
    characters in the :attr:`wordchars` attribute that appear in
    *punctuation_chars* will be removed from :attr:`wordchars`.  See
-   :ref:`improved-shell-compatibility` for more information.
+   :ref:`improved-shell-compatibility` for more information. *punctuation_chars*
+   can be set only upon :class:`~shlex.shlex` instance creation and can't be
+   modified later.
 
    .. versionchanged:: 3.6
       The *punctuation_chars* parameter was added.
@@ -317,8 +319,8 @@ variables which either control lexical analysis or can be used for debugging:
 
 .. attribute:: shlex.punctuation_chars
 
-   Characters that will be considered punctuation. Runs of punctuation
-   characters will be returned as a single token. However, note that no
+   A read-only property. Characters that will be considered punctuation. Runs of
+   punctuation characters will be returned as a single token. However, note that no
    semantic validity checking will be performed: for example, '>>>' could be
    returned as a token, even though it may not be recognised as such by shells.
 
diff --git a/Lib/shlex.py b/Lib/shlex.py
index edea07794860..ae0f5ddec187 100644
--- a/Lib/shlex.py
+++ b/Lib/shlex.py
@@ -55,7 +55,7 @@ def __init__(self, instream=None, infile=None, posix=False,
             punctuation_chars = ''
         elif punctuation_chars is True:
             punctuation_chars = '();<>|&'
-        self.punctuation_chars = punctuation_chars
+        self._punctuation_chars = punctuation_chars
         if punctuation_chars:
             # _pushback_chars is a push back queue used by lookahead logic
             self._pushback_chars = deque()
@@ -65,6 +65,10 @@ def __init__(self, instream=None, infile=None, posix=False,
             t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars))
             self.wordchars = self.wordchars.translate(t)
 
+    @property
+    def punctuation_chars(self):
+        return self._punctuation_chars
+
     def push_token(self, tok):
         "Push a token onto the stack popped by the get_token method"
         if self.debug >= 1:
diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py
index 376c5e88d381..a21ccd2fdf44 100644
--- a/Lib/test/test_shlex.py
+++ b/Lib/test/test_shlex.py
@@ -353,6 +353,13 @@ def testJoinRoundtrip(self):
                 resplit = shlex.split(joined)
                 self.assertEqual(split_command, resplit)
 
+    def testPunctuationCharsReadOnly(self):
+        punctuation_chars = "/|$%^"
+        shlex_instance = shlex.shlex(punctuation_chars=punctuation_chars)
+        self.assertEqual(shlex_instance.punctuation_chars, punctuation_chars)
+        with self.assertRaises(AttributeError):
+            shlex_instance.punctuation_chars = False
+
 
 # Allow this test to be used with old shlex.py
 if not getattr(shlex, "split", None):
diff --git a/Misc/NEWS.d/next/Library/2019-01-22-09-23-20.bpo-35168.UGv2yW.rst b/Misc/NEWS.d/next/Library/2019-01-22-09-23-20.bpo-35168.UGv2yW.rst
new file mode 100644
index 000000000000..10684fdbde2f
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-01-22-09-23-20.bpo-35168.UGv2yW.rst
@@ -0,0 +1 @@
+:attr:`shlex.shlex.punctuation_chars` is now a read-only property.


From webhook-mailer at python.org  Wed Sep 11 07:05:57 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Wed, 11 Sep 2019 11:05:57 -0000
Subject: [Python-checkins] =?utf-8?q?bpo-20504_=3A_in_cgi=2Epy=2C_fix_bug?=
 =?utf-8?q?_when_a_multipart/form-data_request_has=E2=80=A6_=28=2310638=29?=
Message-ID: <mailman.524.1568199958.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2d7cacacc310b65b43e7e2de89e7722291dea6a4
commit: 2d7cacacc310b65b43e7e2de89e7722291dea6a4
branch: master
author: Pierre Quentel <pierre.quentel at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-11T12:05:53+01:00
summary:

bpo-20504 : in cgi.py, fix bug when a multipart/form-data request has? (#10638)

* bpo-20504 : in cgi.py, fix bug when a multipart/form-data request has no content-length header

* Add Misc/NEWS.d/next file.

* Add rst formatting for NEWS.d/next file

* Reaplce assert by self.assertEqual

files:
A Misc/NEWS.d/next/Library/2018-11-21-18-05-50.bpo-20504.kG0ub5.rst
M Lib/cgi.py
M Lib/test/test_cgi.py

diff --git a/Lib/cgi.py b/Lib/cgi.py
index b96bd1f0fe39..c22c71b38785 100755
--- a/Lib/cgi.py
+++ b/Lib/cgi.py
@@ -461,7 +461,7 @@ def __init__(self, fp=None, headers=None, outerboundary=b'',
             if maxlen and clen > maxlen:
                 raise ValueError('Maximum content length exceeded')
         self.length = clen
-        if self.limit is None and clen:
+        if self.limit is None and clen >= 0:
             self.limit = clen
 
         self.list = self.file = None
@@ -642,8 +642,10 @@ def read_multi(self, environ, keep_blank_values, strict_parsing):
             if 'content-length' in headers:
                 del headers['content-length']
 
+            limit = None if self.limit is None \
+                else self.limit - self.bytes_read
             part = klass(self.fp, headers, ib, environ, keep_blank_values,
-                         strict_parsing,self.limit-self.bytes_read,
+                         strict_parsing, limit,
                          self.encoding, self.errors, max_num_fields)
 
             if max_num_fields is not None:
@@ -734,7 +736,7 @@ def read_lines_to_outerboundary(self):
         last_line_lfend = True
         _read = 0
         while 1:
-            if _read >= self.limit:
+            if self.limit is not None and _read >= self.limit:
                 break
             line = self.fp.readline(1<<16) # bytes
             self.bytes_read += len(line)
diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py
index 092255598259..ab8677199f32 100644
--- a/Lib/test/test_cgi.py
+++ b/Lib/test/test_cgi.py
@@ -352,6 +352,23 @@ def test_fieldstorage_part_content_length(self):
         self.assertEqual(fs.list[0].name, 'submit-name')
         self.assertEqual(fs.list[0].value, 'Larry')
 
+    def test_field_storage_multipart_no_content_length(self):
+        fp = BytesIO(b"""--MyBoundary
+Content-Disposition: form-data; name="my-arg"; filename="foo"
+
+Test
+
+--MyBoundary--
+""")
+        env = {
+            "REQUEST_METHOD": "POST",
+            "CONTENT_TYPE": "multipart/form-data; boundary=MyBoundary",
+            "wsgi.input": fp,
+        }
+        fields = cgi.FieldStorage(fp, environ=env)
+
+        self.assertEqual(len(fields["my-arg"].file.read()), 5)
+
     def test_fieldstorage_as_context_manager(self):
         fp = BytesIO(b'x' * 10)
         env = {'REQUEST_METHOD': 'PUT'}
diff --git a/Misc/NEWS.d/next/Library/2018-11-21-18-05-50.bpo-20504.kG0ub5.rst b/Misc/NEWS.d/next/Library/2018-11-21-18-05-50.bpo-20504.kG0ub5.rst
new file mode 100644
index 000000000000..726329ad0d65
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-11-21-18-05-50.bpo-20504.kG0ub5.rst
@@ -0,0 +1,2 @@
+Fixes a bug in :mod:`cgi` module when a multipart/form-data request has no
+`Content-Length` header.


From webhook-mailer at python.org  Wed Sep 11 07:09:57 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 11:09:57 -0000
Subject: [Python-checkins] bpo-28494: install ziptestdata to fix install bot
 (GH-15902)
Message-ID: <mailman.525.1568200199.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7acb22e6e9061f85988c0c6c5ee25ebdf2950841
commit: 7acb22e6e9061f85988c0c6c5ee25ebdf2950841
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T04:09:53-07:00
summary:

bpo-28494: install ziptestdata to fix install bot (GH-15902)

(cherry picked from commit c37447481ec8f6d0e49d0587ec0de3f9e7d56b28)

Co-authored-by: Gregory P. Smith <greg at krypto.org>

files:
M Makefile.pre.in

diff --git a/Makefile.pre.in b/Makefile.pre.in
index b02f78412f16..502317aa0c78 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1372,6 +1372,7 @@ LIBSUBDIRS=	tkinter tkinter/test tkinter/test/test_tkinter \
 		test/test_importlib/source \
 		test/test_importlib/zipdata01 \
 		test/test_importlib/zipdata02 \
+		test/ziptestdata \
 		asyncio \
 		test/test_asyncio \
 		collections concurrent concurrent/futures encodings \


From webhook-mailer at python.org  Wed Sep 11 07:17:19 2019
From: webhook-mailer at python.org (Andrew Svetlov)
Date: Wed, 11 Sep 2019 11:17:19 -0000
Subject: [Python-checkins] bpo-32972: Document IsolatedAsyncioTestCase of
 unittest module (GH-15878) (GH-15918)
Message-ID: <mailman.526.1568200641.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0ba5dbd992d68d7df23396148ad55624200a1dbc
commit: 0ba5dbd992d68d7df23396148ad55624200a1dbc
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Andrew Svetlov <andrew.svetlov at gmail.com>
date: 2019-09-11T14:17:14+03:00
summary:

bpo-32972: Document IsolatedAsyncioTestCase of unittest module (GH-15878) (GH-15918)

* Document `unittest.IsolatedAsyncioTestCase` API
* Add a simple example with respect to order of evaluation of setup and teardown calls.

https://bugs.python.org/issue32972

Automerge-Triggered-By: @asvetlov
(cherry picked from commit 6a9fd66f6e4445a418c43c92585b9e06d76df4b1)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
M Doc/library/unittest.rst
M Doc/whatsnew/3.8.rst
M Misc/NEWS.d/3.8.0b1.rst

diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst
index 5ec4b40856ae..cbee94e52c0a 100644
--- a/Doc/library/unittest.rst
+++ b/Doc/library/unittest.rst
@@ -1486,8 +1486,84 @@ Test cases
       .. versionadded:: 3.8
 
 
+.. class:: IsolatedAsyncioTestCase(methodName='runTest')
 
+   This class provides an API similar to :class:`TestCase` and also accepts
+   coroutines as test functions.
 
+   .. versionadded:: 3.8
+
+   .. coroutinemethod:: asyncSetUp()
+
+      Method called to prepare the test fixture. This is called after :meth:`setUp`.
+      This is called immediately before calling the test method; other than
+      :exc:`AssertionError` or :exc:`SkipTest`, any exception raised by this method
+      will be considered an error rather than a test failure. The default implementation
+      does nothing.
+
+   .. coroutinemethod:: asyncTearDown()
+
+      Method called immediately after the test method has been called and the
+      result recorded.  This is called before :meth:`tearDown`. This is called even if
+      the test method raised an exception, so the implementation in subclasses may need
+      to be particularly careful about checking internal state.  Any exception, other than
+      :exc:`AssertionError` or :exc:`SkipTest`, raised by this method will be
+      considered an additional error rather than a test failure (thus increasing
+      the total number of reported errors). This method will only be called if
+      the :meth:`asyncSetUp` succeeds, regardless of the outcome of the test method.
+      The default implementation does nothing.
+
+   .. method:: addAsyncCleanup(function, /, *args, **kwargs)
+
+      This method accepts a coroutine that can be used as a cleanup function.
+
+   .. method:: run(result=None)
+
+      Sets up a new event loop to run the test, collecting the result into
+      the :class:`TestResult` object passed as *result*.  If *result* is
+      omitted or ``None``, a temporary result object is created (by calling
+      the :meth:`defaultTestResult` method) and used. The result object is
+      returned to :meth:`run`'s caller. At the end of the test all the tasks
+      in the event loop are cancelled.
+
+
+   An example illustrating the order::
+
+      from unittest import IsolatedAsyncioTestCase
+
+      events = []
+
+
+      class Test(IsolatedAsyncioTestCase):
+
+
+          def setUp(self):
+              events.append("setUp")
+
+          async def asyncSetUp(self):
+              self._async_connection = await AsyncConnection()
+              events.append("asyncSetUp")
+
+          async def test_response(self):
+              events.append("test_response")
+              response = await self._async_connection.get("https://example.com")
+              self.assertEqual(response.status_code, 200)
+              self.addAsyncCleanup(self.on_cleanup)
+
+          def tearDown(self):
+              events.append("tearDown")
+
+          async def asyncTearDown(self):
+              await self._async_connection.close()
+              events.append("asyncTearDown")
+
+          async def on_cleanup(self):
+              events.append("cleanup")
+
+      if __name__ == "__main__":
+          unittest.main()
+
+   After running the test ``events`` would contain ``["setUp", "asyncSetUp", "test_response", "asyncTearDown", "tearDown", "cleanup"]``
 
 
 .. class:: FunctionTestCase(testFunc, setUp=None, tearDown=None, description=None)
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 2b4eb63d61f6..07202ea4ff08 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -1111,6 +1111,32 @@ unittest
 * Several mock assert functions now also print a list of actual calls upon
   failure. (Contributed by Petter Strandmark in :issue:`35047`.)
 
+* :mod:`unittest` module gained support for coroutines to be used as test cases
+  with :class:`unittest.IsolatedAsyncioTestCase`.
+  (Contributed by Andrew Svetlov in :issue:`32972`.)
+
+  Example::
+
+    import unittest
+
+
+    class TestRequest(unittest.IsolatedAsyncioTestCase):
+
+        async def asyncSetUp(self):
+            self.connection = await AsyncConnection()
+
+        async def test_get(self):
+            response = await self.connection.get("https://example.com")
+            self.assertEqual(response.status_code, 200)
+
+        async def asyncTearDown(self):
+            await self.connection.close()
+
+
+    if __name__ == "__main__":
+        unittest.main()
+
+
 venv
 ----
 
diff --git a/Misc/NEWS.d/3.8.0b1.rst b/Misc/NEWS.d/3.8.0b1.rst
index 84b0350134fb..43a88a37c5cb 100644
--- a/Misc/NEWS.d/3.8.0b1.rst
+++ b/Misc/NEWS.d/3.8.0b1.rst
@@ -808,7 +808,7 @@ detect classes that can be passed to `hex()`, `oct()` and `bin()`.
 .. nonce: LoeUNh
 .. section: Library
 
-Implement ``unittest.AsyncTestCase`` to help testing asyncio-based code.
+Implement ``unittest.IsolatedAsyncioTestCase`` to help testing asyncio-based code.
 
 ..
 


From webhook-mailer at python.org  Wed Sep 11 07:21:34 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Wed, 11 Sep 2019 11:21:34 -0000
Subject: [Python-checkins] bpo-35603: Add a note on difflib table header
 interpreted as HTML (GH-11439)
Message-ID: <mailman.527.1568200895.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c78dae8d2b890d487e428dce00c7f600612cce7b
commit: c78dae8d2b890d487e428dce00c7f600612cce7b
branch: master
author: Xtreak <tir.karthi at gmail.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-11T13:21:30+02:00
summary:

bpo-35603: Add a note on difflib table header interpreted as HTML (GH-11439)

files:
M Doc/library/difflib.rst

diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst
index e245ab81cfb9..c2a19dc019bb 100644
--- a/Doc/library/difflib.rst
+++ b/Doc/library/difflib.rst
@@ -127,6 +127,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module.
       the next difference highlight at the top of the browser without any leading
       context).
 
+      .. note::
+         *fromdesc* and *todesc* are interpreted as unescaped HTML and should be
+         properly escaped while receiving input from untrusted sources.
+
       .. versionchanged:: 3.5
          *charset* keyword-only argument was added.  The default charset of
          HTML document changed from ``'ISO-8859-1'`` to ``'utf-8'``.


From webhook-mailer at python.org  Wed Sep 11 07:38:25 2019
From: webhook-mailer at python.org (Brett Cannon)
Date: Wed, 11 Sep 2019 11:38:25 -0000
Subject: [Python-checkins] [3.8] bpo-37409: fix relative import with no
 parent (GH-14956) (GH-15913)
Message-ID: <mailman.528.1568201907.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0a6693a469cfb1dd5c8048d8cb4231a7b5883251
commit: 0a6693a469cfb1dd5c8048d8cb4231a7b5883251
branch: 3.8
author: Brett Cannon <54418+brettcannon at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T12:38:22+01:00
summary:

[3.8] bpo-37409: fix relative import with no parent (GH-14956) (GH-15913)

Relative imports use resolve_name to get the absolute target name,
which first seeks the current module's absolute package name from the globals:
If __package__ (and __spec__.parent) are missing then
import uses __name__, truncating the last segment if
the module is a submodule rather than a package __init__.py
(which it guesses from whether __path__ is defined).

The __name__ attempt should fail if there is no parent package (top level modules),
if __name__ is '__main__' (-m entry points), or both (scripts).
That is, if both __name__ has no subcomponents and the module does not seem
to be a package __init__ module then import should fail..
(cherry picked from commit 92420b3e679959a7d0ce875875601a4cee45231e)

Co-authored-by: Ben Lewis <benjimin at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2019-08-06-23-39-05.bpo-37409.1qwzn2.rst
M Lib/test/test_builtin.py
M Lib/test/test_import/__init__.py
M Misc/ACKS
M Python/import.c

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 1100c49e9b88..0bd4772dbb6e 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -160,6 +160,10 @@ def test_import(self):
         self.assertRaises(TypeError, __import__, 1, 2, 3, 4)
         self.assertRaises(ValueError, __import__, '')
         self.assertRaises(TypeError, __import__, 'sys', name='sys')
+        # Relative import outside of a package with no __package__ or __spec__ (bpo-37409).
+        self.assertRaises(ImportError, __import__, '',
+                          {'__package__': None, '__spec__': None, '__name__': '__main__'},
+                          locals={}, fromlist=('foo',), level=1)
         # embedded null character
         self.assertRaises(ModuleNotFoundError, __import__, 'string\x00')
 
diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py
index 7c24f0e50232..f037821dda2c 100644
--- a/Lib/test/test_import/__init__.py
+++ b/Lib/test/test_import/__init__.py
@@ -774,6 +774,11 @@ def check_relative():
         ns = dict(__package__=object())
         self.assertRaises(TypeError, check_relative)
 
+    def test_parentless_import_shadowed_by_global(self):
+        # Test as if this were done from the REPL where this error most commonly occurs (bpo-37409).
+        script_helper.assert_python_failure('-W', 'ignore', '-c',
+            "foo = 1; from . import foo")
+
     def test_absolute_import_without_future(self):
         # If explicit relative import syntax is used, then do not try
         # to perform an absolute import in the face of failure.
diff --git a/Misc/ACKS b/Misc/ACKS
index 55aab6b6b2b3..adc15c94cd31 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -969,6 +969,7 @@ Alain Leufroy
 Mark Levinson
 Mark Levitt
 Ivan Levkivskyi
+Ben Lewis
 William Lewis
 Akira Li
 Robert Li
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-06-23-39-05.bpo-37409.1qwzn2.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-06-23-39-05.bpo-37409.1qwzn2.rst
new file mode 100644
index 000000000000..9cfa71548020
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-06-23-39-05.bpo-37409.1qwzn2.rst	
@@ -0,0 +1,2 @@
+Ensure explicit relative imports from interactive sessions and scripts (having no parent package) always raise ImportError, rather than treating the current module as the package.
+Patch by Ben Lewis.
diff --git a/Python/import.c b/Python/import.c
index ab7db6bc17f6..495012d1c7da 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -1601,22 +1601,20 @@ resolve_name(PyObject *name, PyObject *globals, int level)
             if (dot == -2) {
                 goto error;
             }
-
-            if (dot >= 0) {
-                PyObject *substr = PyUnicode_Substring(package, 0, dot);
-                if (substr == NULL) {
-                    goto error;
-                }
-                Py_SETREF(package, substr);
+            else if (dot == -1) {
+                goto no_parent_error;
             }
+            PyObject *substr = PyUnicode_Substring(package, 0, dot);
+            if (substr == NULL) {
+                goto error;
+            }
+            Py_SETREF(package, substr);
         }
     }
 
     last_dot = PyUnicode_GET_LENGTH(package);
     if (last_dot == 0) {
-        PyErr_SetString(PyExc_ImportError,
-                "attempted relative import with no known parent package");
-        goto error;
+        goto no_parent_error;
     }
 
     for (level_up = 1; level_up < level; level_up += 1) {
@@ -1642,6 +1640,11 @@ resolve_name(PyObject *name, PyObject *globals, int level)
     Py_DECREF(base);
     return abs_name;
 
+  no_parent_error:
+    PyErr_SetString(PyExc_ImportError,
+                     "attempted relative import "
+                     "with no known parent package");
+
   error:
     Py_XDECREF(package);
     return NULL;


From webhook-mailer at python.org  Wed Sep 11 07:47:21 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 11:47:21 -0000
Subject: [Python-checkins] bpo-35066: Make trailing percent test more
 portable. (GH-15907)
Message-ID: <mailman.529.1568202442.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f3e430b07975c84cf34c927851df234d04d5753f
commit: f3e430b07975c84cf34c927851df234d04d5753f
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T04:47:16-07:00
summary:

bpo-35066: Make trailing percent test more portable. (GH-15907)


Different libc implementations have different behavior when presented with trailing % in strftime strings. To make test_strftime_trailing_percent more portable, compare the output of datetime.strftime directly to that of time.strftime rather than hardcoding.
(cherry picked from commit f2173ae38fa49235c3cdc28ae2ca2e19a375a596)

Co-authored-by: Benjamin Peterson <benjamin at python.org>

files:
M Lib/test/datetimetester.py

diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index b440e5ab5fa6..d1a3c2ff9abc 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -1449,15 +1449,20 @@ def test_strftime(self):
         t.strftime("%f")
 
     def test_strftime_trailing_percent(self):
-        # bpo-35066: make sure trailing '%' doesn't cause
-        # datetime's strftime to complain
+        # bpo-35066: Make sure trailing '%' doesn't cause datetime's strftime to
+        # complain. Different libcs have different handling of trailing
+        # percents, so we simply check datetime's strftime acts the same as
+        # time.strftime.
         t = self.theclass(2005, 3, 2)
         try:
             _time.strftime('%')
         except ValueError:
             self.skipTest('time module does not support trailing %')
-        self.assertEqual(t.strftime('%'), '%')
-        self.assertEqual(t.strftime("m:%m d:%d y:%y %"), "m:03 d:02 y:05 %")
+        self.assertEqual(t.strftime('%'), _time.strftime('%', t.timetuple()))
+        self.assertEqual(
+            t.strftime("m:%m d:%d y:%y %"),
+            _time.strftime("m:03 d:02 y:05 %", t.timetuple()),
+        )
 
     def test_format(self):
         dt = self.theclass(2007, 9, 10)


From webhook-mailer at python.org  Wed Sep 11 07:51:05 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 11:51:05 -0000
Subject: [Python-checkins] bpo-35066: Make trailing percent test more
 portable. (GH-15907)
Message-ID: <mailman.530.1568202667.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0553369b9827bb5497bb7a65f64dd259781ae792
commit: 0553369b9827bb5497bb7a65f64dd259781ae792
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T04:51:01-07:00
summary:

bpo-35066: Make trailing percent test more portable. (GH-15907)


Different libc implementations have different behavior when presented with trailing % in strftime strings. To make test_strftime_trailing_percent more portable, compare the output of datetime.strftime directly to that of time.strftime rather than hardcoding.
(cherry picked from commit f2173ae38fa49235c3cdc28ae2ca2e19a375a596)

Co-authored-by: Benjamin Peterson <benjamin at python.org>

files:
M Lib/test/datetimetester.py

diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index 3cce953a5823..ba3722295ddb 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -1411,15 +1411,20 @@ def test_strftime(self):
         t.strftime("%f")
 
     def test_strftime_trailing_percent(self):
-        # bpo-35066: make sure trailing '%' doesn't cause
-        # datetime's strftime to complain
+        # bpo-35066: Make sure trailing '%' doesn't cause datetime's strftime to
+        # complain. Different libcs have different handling of trailing
+        # percents, so we simply check datetime's strftime acts the same as
+        # time.strftime.
         t = self.theclass(2005, 3, 2)
         try:
             _time.strftime('%')
         except ValueError:
             self.skipTest('time module does not support trailing %')
-        self.assertEqual(t.strftime('%'), '%')
-        self.assertEqual(t.strftime("m:%m d:%d y:%y %"), "m:03 d:02 y:05 %")
+        self.assertEqual(t.strftime('%'), _time.strftime('%', t.timetuple()))
+        self.assertEqual(
+            t.strftime("m:%m d:%d y:%y %"),
+            _time.strftime("m:03 d:02 y:05 %", t.timetuple()),
+        )
 
     def test_format(self):
         dt = self.theclass(2007, 9, 10)


From webhook-mailer at python.org  Wed Sep 11 08:09:27 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 12:09:27 -0000
Subject: [Python-checkins] =?utf-8?q?bpo-20504_=3A_in_cgi=2Epy=2C_fix_bug?=
 =?utf-8?q?_when_a_multipart/form-data_request_has=E2=80=A6_=28GH-10638=29?=
Message-ID: <mailman.531.1568203768.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e3bd941e4e6f4465f17a0e5a4a6bdf4ea0afdd0d
commit: e3bd941e4e6f4465f17a0e5a4a6bdf4ea0afdd0d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T05:09:23-07:00
summary:

bpo-20504 : in cgi.py, fix bug when a multipart/form-data request has? (GH-10638)


* bpo-20504 : in cgi.py, fix bug when a multipart/form-data request has no content-length header

* Add Misc/NEWS.d/next file.

* Add rst formatting for NEWS.d/next file

* Reaplce assert by self.assertEqual
(cherry picked from commit 2d7cacacc310b65b43e7e2de89e7722291dea6a4)

Co-authored-by: Pierre Quentel <pierre.quentel at gmail.com>

files:
A Misc/NEWS.d/next/Library/2018-11-21-18-05-50.bpo-20504.kG0ub5.rst
M Lib/cgi.py
M Lib/test/test_cgi.py

diff --git a/Lib/cgi.py b/Lib/cgi.py
index b96bd1f0fe39..c22c71b38785 100755
--- a/Lib/cgi.py
+++ b/Lib/cgi.py
@@ -461,7 +461,7 @@ def __init__(self, fp=None, headers=None, outerboundary=b'',
             if maxlen and clen > maxlen:
                 raise ValueError('Maximum content length exceeded')
         self.length = clen
-        if self.limit is None and clen:
+        if self.limit is None and clen >= 0:
             self.limit = clen
 
         self.list = self.file = None
@@ -642,8 +642,10 @@ def read_multi(self, environ, keep_blank_values, strict_parsing):
             if 'content-length' in headers:
                 del headers['content-length']
 
+            limit = None if self.limit is None \
+                else self.limit - self.bytes_read
             part = klass(self.fp, headers, ib, environ, keep_blank_values,
-                         strict_parsing,self.limit-self.bytes_read,
+                         strict_parsing, limit,
                          self.encoding, self.errors, max_num_fields)
 
             if max_num_fields is not None:
@@ -734,7 +736,7 @@ def read_lines_to_outerboundary(self):
         last_line_lfend = True
         _read = 0
         while 1:
-            if _read >= self.limit:
+            if self.limit is not None and _read >= self.limit:
                 break
             line = self.fp.readline(1<<16) # bytes
             self.bytes_read += len(line)
diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py
index 092255598259..ab8677199f32 100644
--- a/Lib/test/test_cgi.py
+++ b/Lib/test/test_cgi.py
@@ -352,6 +352,23 @@ def test_fieldstorage_part_content_length(self):
         self.assertEqual(fs.list[0].name, 'submit-name')
         self.assertEqual(fs.list[0].value, 'Larry')
 
+    def test_field_storage_multipart_no_content_length(self):
+        fp = BytesIO(b"""--MyBoundary
+Content-Disposition: form-data; name="my-arg"; filename="foo"
+
+Test
+
+--MyBoundary--
+""")
+        env = {
+            "REQUEST_METHOD": "POST",
+            "CONTENT_TYPE": "multipart/form-data; boundary=MyBoundary",
+            "wsgi.input": fp,
+        }
+        fields = cgi.FieldStorage(fp, environ=env)
+
+        self.assertEqual(len(fields["my-arg"].file.read()), 5)
+
     def test_fieldstorage_as_context_manager(self):
         fp = BytesIO(b'x' * 10)
         env = {'REQUEST_METHOD': 'PUT'}
diff --git a/Misc/NEWS.d/next/Library/2018-11-21-18-05-50.bpo-20504.kG0ub5.rst b/Misc/NEWS.d/next/Library/2018-11-21-18-05-50.bpo-20504.kG0ub5.rst
new file mode 100644
index 000000000000..726329ad0d65
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-11-21-18-05-50.bpo-20504.kG0ub5.rst
@@ -0,0 +1,2 @@
+Fixes a bug in :mod:`cgi` module when a multipart/form-data request has no
+`Content-Length` header.


From webhook-mailer at python.org  Wed Sep 11 08:22:42 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 12:22:42 -0000
Subject: [Python-checkins] =?utf-8?q?bpo-20504_=3A_in_cgi=2Epy=2C_fix_bug?=
 =?utf-8?q?_when_a_multipart/form-data_request_has=E2=80=A6_=28GH-10638=29?=
Message-ID: <mailman.532.1568204563.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/99f0e81f43f64b83e18e8cb2a0b66c53a81a74ab
commit: 99f0e81f43f64b83e18e8cb2a0b66c53a81a74ab
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T05:22:38-07:00
summary:

bpo-20504 : in cgi.py, fix bug when a multipart/form-data request has? (GH-10638)


* bpo-20504 : in cgi.py, fix bug when a multipart/form-data request has no content-length header

* Add Misc/NEWS.d/next file.

* Add rst formatting for NEWS.d/next file

* Reaplce assert by self.assertEqual
(cherry picked from commit 2d7cacacc310b65b43e7e2de89e7722291dea6a4)

Co-authored-by: Pierre Quentel <pierre.quentel at gmail.com>

files:
A Misc/NEWS.d/next/Library/2018-11-21-18-05-50.bpo-20504.kG0ub5.rst
M Lib/cgi.py
M Lib/test/test_cgi.py

diff --git a/Lib/cgi.py b/Lib/cgi.py
index 8cf668718ded..df84f1fe69cf 100755
--- a/Lib/cgi.py
+++ b/Lib/cgi.py
@@ -478,7 +478,7 @@ def __init__(self, fp=None, headers=None, outerboundary=b'',
             if maxlen and clen > maxlen:
                 raise ValueError('Maximum content length exceeded')
         self.length = clen
-        if self.limit is None and clen:
+        if self.limit is None and clen >= 0:
             self.limit = clen
 
         self.list = self.file = None
@@ -659,8 +659,10 @@ def read_multi(self, environ, keep_blank_values, strict_parsing):
             if 'content-length' in headers:
                 del headers['content-length']
 
+            limit = None if self.limit is None \
+                else self.limit - self.bytes_read
             part = klass(self.fp, headers, ib, environ, keep_blank_values,
-                         strict_parsing,self.limit-self.bytes_read,
+                         strict_parsing, limit,
                          self.encoding, self.errors, max_num_fields)
 
             if max_num_fields is not None:
@@ -751,7 +753,7 @@ def read_lines_to_outerboundary(self):
         last_line_lfend = True
         _read = 0
         while 1:
-            if _read >= self.limit:
+            if self.limit is not None and _read >= self.limit:
                 break
             line = self.fp.readline(1<<16) # bytes
             self.bytes_read += len(line)
diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py
index f4e00c7a72c5..b46be67f7732 100644
--- a/Lib/test/test_cgi.py
+++ b/Lib/test/test_cgi.py
@@ -363,6 +363,23 @@ def test_fieldstorage_part_content_length(self):
         self.assertEqual(fs.list[0].name, 'submit-name')
         self.assertEqual(fs.list[0].value, 'Larry')
 
+    def test_field_storage_multipart_no_content_length(self):
+        fp = BytesIO(b"""--MyBoundary
+Content-Disposition: form-data; name="my-arg"; filename="foo"
+
+Test
+
+--MyBoundary--
+""")
+        env = {
+            "REQUEST_METHOD": "POST",
+            "CONTENT_TYPE": "multipart/form-data; boundary=MyBoundary",
+            "wsgi.input": fp,
+        }
+        fields = cgi.FieldStorage(fp, environ=env)
+
+        self.assertEqual(len(fields["my-arg"].file.read()), 5)
+
     def test_fieldstorage_as_context_manager(self):
         fp = BytesIO(b'x' * 10)
         env = {'REQUEST_METHOD': 'PUT'}
diff --git a/Misc/NEWS.d/next/Library/2018-11-21-18-05-50.bpo-20504.kG0ub5.rst b/Misc/NEWS.d/next/Library/2018-11-21-18-05-50.bpo-20504.kG0ub5.rst
new file mode 100644
index 000000000000..726329ad0d65
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-11-21-18-05-50.bpo-20504.kG0ub5.rst
@@ -0,0 +1,2 @@
+Fixes a bug in :mod:`cgi` module when a multipart/form-data request has no
+`Content-Length` header.


From webhook-mailer at python.org  Wed Sep 11 08:24:58 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 12:24:58 -0000
Subject: [Python-checkins] bpo-35603: Add a note on difflib table header
 interpreted as HTML (GH-11439)
Message-ID: <mailman.533.1568204699.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/44e36e80456dabaeb59c6e2a93e0c1322bfeb179
commit: 44e36e80456dabaeb59c6e2a93e0c1322bfeb179
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T05:24:55-07:00
summary:

bpo-35603: Add a note on difflib table header interpreted as HTML (GH-11439)

(cherry picked from commit c78dae8d2b890d487e428dce00c7f600612cce7b)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
M Doc/library/difflib.rst

diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst
index e245ab81cfb9..c2a19dc019bb 100644
--- a/Doc/library/difflib.rst
+++ b/Doc/library/difflib.rst
@@ -127,6 +127,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module.
       the next difference highlight at the top of the browser without any leading
       context).
 
+      .. note::
+         *fromdesc* and *todesc* are interpreted as unescaped HTML and should be
+         properly escaped while receiving input from untrusted sources.
+
       .. versionchanged:: 3.5
          *charset* keyword-only argument was added.  The default charset of
          HTML document changed from ``'ISO-8859-1'`` to ``'utf-8'``.


From webhook-mailer at python.org  Wed Sep 11 08:28:10 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 12:28:10 -0000
Subject: [Python-checkins] bpo-38103: fix conflicting labels in the docs.
 (GH-15906)
Message-ID: <mailman.534.1568204891.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e0dd713370f94d0f4a59604cc769e0cc643b7740
commit: e0dd713370f94d0f4a59604cc769e0cc643b7740
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T05:28:06-07:00
summary:

bpo-38103: fix conflicting labels in the docs. (GH-15906)

(cherry picked from commit 2d8d597bb8f882a7677db5a2739df0e617098634)

Co-authored-by: Ezio Melotti <ezio.melotti at gmail.com>

files:
M Doc/c-api/typeobj.rst
M Doc/distutils/examples.rst

diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index 638fb0c32d78..f8e30a0380b9 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -20,7 +20,7 @@ functionality.  The fields of the type object are examined in detail in this
 section.  The fields will be described in the order in which they occur in the
 structure.
 
-In addition to the following quick reference, the :ref:`examples`
+In addition to the following quick reference, the :ref:`typedef-examples`
 section provides at-a-glance insight into the meaning and use of
 :c:type:`PyTypeObject`.
 
@@ -2450,7 +2450,7 @@ Slot Type typedefs
 .. c:type:: int (*objobjargproc)(PyObject *, PyObject *, PyObject *)
 
 
-.. _examples:
+.. _typedef-examples:
 
 Examples
 ========
diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst
index 4ac552c7c699..44f70831d0bf 100644
--- a/Doc/distutils/examples.rst
+++ b/Doc/distutils/examples.rst
@@ -1,8 +1,8 @@
 .. _distutils_examples:
 
-********
-Examples
-********
+******************
+Distutils Examples
+******************
 
 .. include:: ./_setuptools_disclaimer.rst
 


From webhook-mailer at python.org  Wed Sep 11 08:32:20 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 12:32:20 -0000
Subject: [Python-checkins] Minor ReST formatting fixes in subprocess docs
 (GH-14876)
Message-ID: <mailman.535.1568205141.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/01ae0e269899c244c6cf779dc3a7ebdd29248859
commit: 01ae0e269899c244c6cf779dc3a7ebdd29248859
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T05:32:16-07:00
summary:

Minor ReST formatting fixes in subprocess docs (GH-14876)

(cherry picked from commit 1a13efb7e05b545def26f29c954751fdb6b22fa3)

Co-authored-by: Tim Hoffmann <2836374+timhoffm at users.noreply.github.com>

files:
M Doc/library/subprocess.rst

diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
index 167ed9a6ead4..954e0fec1182 100644
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -1048,7 +1048,7 @@ calls these functions.
    Run the command described by *args*.  Wait for command to complete, then
    return the :attr:`~Popen.returncode` attribute.
 
-   Code needing to capture stdout or stderr should use :func:`run` instead:
+   Code needing to capture stdout or stderr should use :func:`run` instead::
 
        run(...).returncode
 
@@ -1076,7 +1076,7 @@ calls these functions.
    :exc:`CalledProcessError` object will have the return code in the
    :attr:`~CalledProcessError.returncode` attribute.
 
-   Code needing to capture stdout or stderr should use :func:`run` instead:
+   Code needing to capture stdout or stderr should use :func:`run` instead::
 
        run(..., check=True)
 
@@ -1198,8 +1198,8 @@ becomes::
    p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
    output = p2.communicate()[0]
 
-The p1.stdout.close() call after starting the p2 is important in order for p1
-to receive a SIGPIPE if p2 exits before p1.
+The ``p1.stdout.close()`` call after starting the p2 is important in order for
+p1 to receive a SIGPIPE if p2 exits before p1.
 
 Alternatively, for trusted input, the shell's own pipeline support may still
 be used directly:


From webhook-mailer at python.org  Wed Sep 11 08:39:12 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Wed, 11 Sep 2019 12:39:12 -0000
Subject: [Python-checkins] [3.7] bpo-35168: Make shlex.punctuation_chars
 read-only (GH-11631) (GH-15926)
Message-ID: <mailman.536.1568205553.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/aca878ecf18b9e915e9b551f095a550b5c6e9bc5
commit: aca878ecf18b9e915e9b551f095a550b5c6e9bc5
branch: 3.7
author: Vinay Sajip <vinay_sajip at yahoo.co.uk>
committer: GitHub <noreply at github.com>
date: 2019-09-11T13:39:08+01:00
summary:

[3.7] bpo-35168: Make shlex.punctuation_chars read-only (GH-11631) (GH-15926)

(cherry picked from commit 972cf5c06a5ba16ad243a442dbb9c15307fbed95)

Co-authored-by: Alex <a.v.shkop at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-01-22-09-23-20.bpo-35168.UGv2yW.rst
M Doc/library/shlex.rst
M Lib/shlex.py
M Lib/test/test_shlex.py

diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst
index fb335c690068..d5b5ec6b14e4 100644
--- a/Doc/library/shlex.rst
+++ b/Doc/library/shlex.rst
@@ -98,7 +98,9 @@ The :mod:`shlex` module defines the following class:
    characters, those characters will be used as the punctuation characters.  Any
    characters in the :attr:`wordchars` attribute that appear in
    *punctuation_chars* will be removed from :attr:`wordchars`.  See
-   :ref:`improved-shell-compatibility` for more information.
+   :ref:`improved-shell-compatibility` for more information. *punctuation_chars*
+   can be set only upon :class:`~shlex.shlex` instance creation and can't be
+   modified later.
 
    .. versionchanged:: 3.6
       The *punctuation_chars* parameter was added.
@@ -299,8 +301,8 @@ variables which either control lexical analysis or can be used for debugging:
 
 .. attribute:: shlex.punctuation_chars
 
-   Characters that will be considered punctuation. Runs of punctuation
-   characters will be returned as a single token. However, note that no
+   A read-only property. Characters that will be considered punctuation. Runs of
+   punctuation characters will be returned as a single token. However, note that no
    semantic validity checking will be performed: for example, '>>>' could be
    returned as a token, even though it may not be recognised as such by shells.
 
diff --git a/Lib/shlex.py b/Lib/shlex.py
index 2c9786c517a3..195dc12bbce9 100644
--- a/Lib/shlex.py
+++ b/Lib/shlex.py
@@ -55,7 +55,7 @@ def __init__(self, instream=None, infile=None, posix=False,
             punctuation_chars = ''
         elif punctuation_chars is True:
             punctuation_chars = '();<>|&'
-        self.punctuation_chars = punctuation_chars
+        self._punctuation_chars = punctuation_chars
         if punctuation_chars:
             # _pushback_chars is a push back queue used by lookahead logic
             self._pushback_chars = deque()
@@ -65,6 +65,10 @@ def __init__(self, instream=None, infile=None, posix=False,
             t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars))
             self.wordchars = self.wordchars.translate(t)
 
+    @property
+    def punctuation_chars(self):
+        return self._punctuation_chars
+
     def push_token(self, tok):
         "Push a token onto the stack popped by the get_token method"
         if self.debug >= 1:
diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py
index fd35788e81b2..6d4627d7f4ea 100644
--- a/Lib/test/test_shlex.py
+++ b/Lib/test/test_shlex.py
@@ -308,6 +308,14 @@ def testQuote(self):
             self.assertEqual(shlex.quote("test%s'name'" % u),
                              "'test%s'\"'\"'name'\"'\"''" % u)
 
+    def testPunctuationCharsReadOnly(self):
+        punctuation_chars = "/|$%^"
+        shlex_instance = shlex.shlex(punctuation_chars=punctuation_chars)
+        self.assertEqual(shlex_instance.punctuation_chars, punctuation_chars)
+        with self.assertRaises(AttributeError):
+            shlex_instance.punctuation_chars = False
+
+
 # Allow this test to be used with old shlex.py
 if not getattr(shlex, "split", None):
     for methname in dir(ShlexTest):
diff --git a/Misc/NEWS.d/next/Library/2019-01-22-09-23-20.bpo-35168.UGv2yW.rst b/Misc/NEWS.d/next/Library/2019-01-22-09-23-20.bpo-35168.UGv2yW.rst
new file mode 100644
index 000000000000..10684fdbde2f
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-01-22-09-23-20.bpo-35168.UGv2yW.rst
@@ -0,0 +1 @@
+:attr:`shlex.shlex.punctuation_chars` is now a read-only property.


From webhook-mailer at python.org  Wed Sep 11 08:39:56 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Wed, 11 Sep 2019 12:39:56 -0000
Subject: [Python-checkins] [3.8] bpo-35168: Make shlex.punctuation_chars
 read-only (GH-11631) (GH-15927)
Message-ID: <mailman.537.1568205597.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3b92ddb7612fd8fe2be95ff3d8022ed18335b13f
commit: 3b92ddb7612fd8fe2be95ff3d8022ed18335b13f
branch: 3.8
author: Vinay Sajip <vinay_sajip at yahoo.co.uk>
committer: GitHub <noreply at github.com>
date: 2019-09-11T13:39:52+01:00
summary:

[3.8] bpo-35168: Make shlex.punctuation_chars read-only (GH-11631) (GH-15927)

(cherry picked from commit 972cf5c06a5ba16ad243a442dbb9c15307fbed95)

Co-authored-by: Alex <a.v.shkop at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-01-22-09-23-20.bpo-35168.UGv2yW.rst
M Doc/library/shlex.rst
M Lib/shlex.py
M Lib/test/test_shlex.py

diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst
index a8421fdb7008..adc23da6d491 100644
--- a/Doc/library/shlex.rst
+++ b/Doc/library/shlex.rst
@@ -113,7 +113,9 @@ The :mod:`shlex` module defines the following class:
    characters, those characters will be used as the punctuation characters.  Any
    characters in the :attr:`wordchars` attribute that appear in
    *punctuation_chars* will be removed from :attr:`wordchars`.  See
-   :ref:`improved-shell-compatibility` for more information.
+   :ref:`improved-shell-compatibility` for more information. *punctuation_chars*
+   can be set only upon :class:`~shlex.shlex` instance creation and can't be
+   modified later.
 
    .. versionchanged:: 3.6
       The *punctuation_chars* parameter was added.
@@ -317,8 +319,8 @@ variables which either control lexical analysis or can be used for debugging:
 
 .. attribute:: shlex.punctuation_chars
 
-   Characters that will be considered punctuation. Runs of punctuation
-   characters will be returned as a single token. However, note that no
+   A read-only property. Characters that will be considered punctuation. Runs of
+   punctuation characters will be returned as a single token. However, note that no
    semantic validity checking will be performed: for example, '>>>' could be
    returned as a token, even though it may not be recognised as such by shells.
 
diff --git a/Lib/shlex.py b/Lib/shlex.py
index edea07794860..ae0f5ddec187 100644
--- a/Lib/shlex.py
+++ b/Lib/shlex.py
@@ -55,7 +55,7 @@ def __init__(self, instream=None, infile=None, posix=False,
             punctuation_chars = ''
         elif punctuation_chars is True:
             punctuation_chars = '();<>|&'
-        self.punctuation_chars = punctuation_chars
+        self._punctuation_chars = punctuation_chars
         if punctuation_chars:
             # _pushback_chars is a push back queue used by lookahead logic
             self._pushback_chars = deque()
@@ -65,6 +65,10 @@ def __init__(self, instream=None, infile=None, posix=False,
             t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars))
             self.wordchars = self.wordchars.translate(t)
 
+    @property
+    def punctuation_chars(self):
+        return self._punctuation_chars
+
     def push_token(self, tok):
         "Push a token onto the stack popped by the get_token method"
         if self.debug >= 1:
diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py
index 376c5e88d381..a21ccd2fdf44 100644
--- a/Lib/test/test_shlex.py
+++ b/Lib/test/test_shlex.py
@@ -353,6 +353,13 @@ def testJoinRoundtrip(self):
                 resplit = shlex.split(joined)
                 self.assertEqual(split_command, resplit)
 
+    def testPunctuationCharsReadOnly(self):
+        punctuation_chars = "/|$%^"
+        shlex_instance = shlex.shlex(punctuation_chars=punctuation_chars)
+        self.assertEqual(shlex_instance.punctuation_chars, punctuation_chars)
+        with self.assertRaises(AttributeError):
+            shlex_instance.punctuation_chars = False
+
 
 # Allow this test to be used with old shlex.py
 if not getattr(shlex, "split", None):
diff --git a/Misc/NEWS.d/next/Library/2019-01-22-09-23-20.bpo-35168.UGv2yW.rst b/Misc/NEWS.d/next/Library/2019-01-22-09-23-20.bpo-35168.UGv2yW.rst
new file mode 100644
index 000000000000..10684fdbde2f
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-01-22-09-23-20.bpo-35168.UGv2yW.rst
@@ -0,0 +1 @@
+:attr:`shlex.shlex.punctuation_chars` is now a read-only property.


From webhook-mailer at python.org  Wed Sep 11 08:41:57 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Wed, 11 Sep 2019 12:41:57 -0000
Subject: [Python-checkins] bpo-35649: update http client example (GH-11441)
Message-ID: <mailman.538.1568205718.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/62cf6981425c6a6b136c5e2abef853364f535e9d
commit: 62cf6981425c6a6b136c5e2abef853364f535e9d
branch: master
author: Ashwin Ramaswami <aramaswamis at gmail.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-11T14:41:54+02:00
summary:

bpo-35649: update http client example (GH-11441)

files:
M Doc/library/http.client.rst

diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst
index 4e761cd39a01..48bc35ca76cc 100644
--- a/Doc/library/http.client.rst
+++ b/Doc/library/http.client.rst
@@ -516,8 +516,11 @@ Here is an example session that uses the ``GET`` method::
    >>> # The following example demonstrates reading data in chunks.
    >>> conn.request("GET", "/")
    >>> r1 = conn.getresponse()
-   >>> while not r1.closed:
-   ...     print(r1.read(200))  # 200 bytes
+   >>> while True:
+   ...     chunk = r1.read(200)  # 200 bytes
+   ...     if not chunk:
+   ...          break
+   ...     print(repr(chunk))
    b'<!doctype html>\n<!--[if"...
    ...
    >>> # Example of an invalid request


From webhook-mailer at python.org  Wed Sep 11 08:43:04 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Wed, 11 Sep 2019 12:43:04 -0000
Subject: [Python-checkins] bpo-37326: Include libffi license when available
 (GH-15921)
Message-ID: <mailman.539.1568205784.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8fcaffb048d2359a4c06ed5d8fbbf46d3ae1a472
commit: 8fcaffb048d2359a4c06ed5d8fbbf46d3ae1a472
branch: master
author: Steve Dower <steve.dower at python.org>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-11T13:43:00+01:00
summary:

bpo-37326: Include libffi license when available (GH-15921)

files:
M PCbuild/python.vcxproj

diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj
index e37bcfb0b6e1..2094420a8df3 100644
--- a/PCbuild/python.vcxproj
+++ b/PCbuild/python.vcxproj
@@ -150,12 +150,13 @@ $(_PGOPath)
       <LicenseFiles Include="$(PySourcePath)LICENSE;
                              $(PySourcePath)PC\crtlicense.txt;
                              $(bz2Dir)LICENSE;
-                             $(opensslOutDir)LICENSE;" />
+                             $(opensslOutDir)LICENSE;
+                             $(libffiDir)LICENSE;" />
       <LicenseFiles Include="$(tcltkDir)tcllicense.terms;
                              $(tcltkDir)tklicense.terms;
                              $(tcltkDir)tixlicense.terms" Condition="$(IncludeTkinter)" />
       <_LicenseFiles Include="@(LicenseFiles)">
-        <Content>$([System.IO.File]::ReadAllText(%(FullPath)))</Content>
+        <Content Condition="Exists(%(FullPath))">$([System.IO.File]::ReadAllText(%(FullPath)))</Content>
       </_LicenseFiles>
     </ItemGroup>
 


From webhook-mailer at python.org  Wed Sep 11 08:47:43 2019
From: webhook-mailer at python.org (Brett Cannon)
Date: Wed, 11 Sep 2019 12:47:43 -0000
Subject: [Python-checkins] bpo-35943: Prevent PyImport_GetModule() from
 returning a partially-initialized module (GH-15057)
Message-ID: <mailman.540.1568206064.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/37c22206981f52ae35c28b39f7530f8438afbfdb
commit: 37c22206981f52ae35c28b39f7530f8438afbfdb
branch: master
author: Joannah Nanjekye <33177550+nanjekyejoannah at users.noreply.github.com>
committer: Brett Cannon <54418+brettcannon at users.noreply.github.com>
date: 2019-09-11T13:47:39+01:00
summary:

bpo-35943: Prevent PyImport_GetModule() from returning a partially-initialized module (GH-15057)

files:
A Misc/NEWS.d/next/Library/2019-07-31-15-52-51.bpo-35943.-KswoB.rst
M Python/import.c

diff --git a/Misc/NEWS.d/next/Library/2019-07-31-15-52-51.bpo-35943.-KswoB.rst b/Misc/NEWS.d/next/Library/2019-07-31-15-52-51.bpo-35943.-KswoB.rst
new file mode 100644
index 000000000000..b1c55608f598
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-07-31-15-52-51.bpo-35943.-KswoB.rst
@@ -0,0 +1,2 @@
+The function :c:func:`PyImport_GetModule` now ensures any module it returns is fully initialized.
+Patch by Joannah Nanjekye.
diff --git a/Python/import.c b/Python/import.c
index 5be4d196a355..6eb079d2a471 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -387,11 +387,33 @@ import_get_module(PyThreadState *tstate, PyObject *name)
 }
 
 
-PyObject *
-PyImport_GetModule(PyObject *name)
+static int
+import_ensure_initialized(PyThreadState *tstate, PyObject *mod, PyObject *name)
 {
-    PyThreadState *tstate = _PyThreadState_GET();
-    return import_get_module(tstate, name);
+    PyInterpreterState *interp = tstate->interp;
+    PyObject *spec;
+
+    _Py_IDENTIFIER(__spec__);
+    _Py_IDENTIFIER(_lock_unlock_module);
+
+    /* Optimization: only call _bootstrap._lock_unlock_module() if
+       __spec__._initializing is true.
+       NOTE: because of this, initializing must be set *before*
+       stuffing the new module in sys.modules.
+    */
+    spec = _PyObject_GetAttrId(mod, &PyId___spec__);
+    int busy = _PyModuleSpec_IsInitializing(spec);
+    Py_XDECREF(spec);
+    if (busy) {
+        /* Wait until module is done importing. */
+        PyObject *value = _PyObject_CallMethodIdOneArg(
+            interp->importlib, &PyId__lock_unlock_module, name);
+        if (value == NULL) {
+            return -1;
+        }
+        Py_DECREF(value);
+    }
+    return 0;
 }
 
 
@@ -1461,6 +1483,7 @@ PyImport_ImportModule(const char *name)
     return result;
 }
 
+
 /* Import a module without blocking
  *
  * At first it tries to fetch the module from sys.modules. If the module was
@@ -1762,6 +1785,23 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name)
     return mod;
 }
 
+PyObject *
+PyImport_GetModule(PyObject *name)
+{
+    PyThreadState *tstate = _PyThreadState_GET();
+    PyObject *mod;
+
+    mod = import_get_module(tstate, name);
+    if (mod != NULL && mod != Py_None) {
+        if (import_ensure_initialized(tstate, mod, name) < 0) {
+            Py_DECREF(mod);
+            remove_importlib_frames(tstate);
+            return NULL;
+        }
+    }
+    return mod;
+}
+
 PyObject *
 PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals,
                                  PyObject *locals, PyObject *fromlist,
@@ -1817,26 +1857,9 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals,
     }
 
     if (mod != NULL && mod != Py_None) {
-        _Py_IDENTIFIER(__spec__);
-        _Py_IDENTIFIER(_lock_unlock_module);
-        PyObject *spec;
-
-        /* Optimization: only call _bootstrap._lock_unlock_module() if
-           __spec__._initializing is true.
-           NOTE: because of this, initializing must be set *before*
-           stuffing the new module in sys.modules.
-         */
-        spec = _PyObject_GetAttrId(mod, &PyId___spec__);
-        if (_PyModuleSpec_IsInitializing(spec)) {
-            PyObject *value = _PyObject_CallMethodIdOneArg(
-                interp->importlib, &PyId__lock_unlock_module, abs_name);
-            if (value == NULL) {
-                Py_DECREF(spec);
-                goto error;
-            }
-            Py_DECREF(value);
+        if (import_ensure_initialized(tstate, mod, name) < 0) {
+            goto error;
         }
-        Py_XDECREF(spec);
     }
     else {
         Py_XDECREF(mod);


From webhook-mailer at python.org  Wed Sep 11 08:50:19 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 12:50:19 -0000
Subject: [Python-checkins] [3.8] bpo-37409: fix relative import with no
 parent (GH-14956) (GH-15913)
Message-ID: <mailman.541.1568206220.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f3480ad08823a9bc7df490bb5b54593d9483be70
commit: f3480ad08823a9bc7df490bb5b54593d9483be70
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T05:50:14-07:00
summary:

[3.8] bpo-37409: fix relative import with no parent (GH-14956) (GH-15913)


Relative imports use resolve_name to get the absolute target name,
which first seeks the current module's absolute package name from the globals:
If __package__ (and __spec__.parent) are missing then
import uses __name__, truncating the last segment if
the module is a submodule rather than a package __init__.py
(which it guesses from whether __path__ is defined).

The __name__ attempt should fail if there is no parent package (top level modules),
if __name__ is '__main__' (-m entry points), or both (scripts).
That is, if both __name__ has no subcomponents and the module does not seem
to be a package __init__ module then import should fail..
(cherry picked from commit 92420b3e679959a7d0ce875875601a4cee45231e)

Co-authored-by: Ben Lewis <benjimin at users.noreply.github.com>
(cherry picked from commit 0a6693a469cfb1dd5c8048d8cb4231a7b5883251)

Co-authored-by: Brett Cannon <54418+brettcannon at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2019-08-06-23-39-05.bpo-37409.1qwzn2.rst
M Lib/test/test_builtin.py
M Lib/test/test_import/__init__.py
M Misc/ACKS
M Python/import.c

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index e0d2a698dcff..5216da459f67 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -154,6 +154,10 @@ def test_import(self):
         self.assertRaises(TypeError, __import__, 1, 2, 3, 4)
         self.assertRaises(ValueError, __import__, '')
         self.assertRaises(TypeError, __import__, 'sys', name='sys')
+        # Relative import outside of a package with no __package__ or __spec__ (bpo-37409).
+        self.assertRaises(ImportError, __import__, '',
+                          {'__package__': None, '__spec__': None, '__name__': '__main__'},
+                          locals={}, fromlist=('foo',), level=1)
         # embedded null character
         self.assertRaises(ModuleNotFoundError, __import__, 'string\x00')
 
diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py
index 1fc4de11e178..4b9907d5329d 100644
--- a/Lib/test/test_import/__init__.py
+++ b/Lib/test/test_import/__init__.py
@@ -711,6 +711,11 @@ def check_relative():
         ns = dict(__package__=object())
         self.assertRaises(TypeError, check_relative)
 
+    def test_parentless_import_shadowed_by_global(self):
+        # Test as if this were done from the REPL where this error most commonly occurs (bpo-37409).
+        script_helper.assert_python_failure('-W', 'ignore', '-c',
+            "foo = 1; from . import foo")
+
     def test_absolute_import_without_future(self):
         # If explicit relative import syntax is used, then do not try
         # to perform an absolute import in the face of failure.
diff --git a/Misc/ACKS b/Misc/ACKS
index c7d6f6c35b61..35fcec2b2088 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -949,6 +949,7 @@ Alain Leufroy
 Mark Levinson
 Mark Levitt
 Ivan Levkivskyi
+Ben Lewis
 William Lewis
 Akira Li
 Robert Li
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-06-23-39-05.bpo-37409.1qwzn2.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-06-23-39-05.bpo-37409.1qwzn2.rst
new file mode 100644
index 000000000000..9cfa71548020
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-06-23-39-05.bpo-37409.1qwzn2.rst	
@@ -0,0 +1,2 @@
+Ensure explicit relative imports from interactive sessions and scripts (having no parent package) always raise ImportError, rather than treating the current module as the package.
+Patch by Ben Lewis.
diff --git a/Python/import.c b/Python/import.c
index 63c99ea5c1ab..edc59249622b 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -1563,22 +1563,20 @@ resolve_name(PyObject *name, PyObject *globals, int level)
             if (dot == -2) {
                 goto error;
             }
-
-            if (dot >= 0) {
-                PyObject *substr = PyUnicode_Substring(package, 0, dot);
-                if (substr == NULL) {
-                    goto error;
-                }
-                Py_SETREF(package, substr);
+            else if (dot == -1) {
+                goto no_parent_error;
             }
+            PyObject *substr = PyUnicode_Substring(package, 0, dot);
+            if (substr == NULL) {
+                goto error;
+            }
+            Py_SETREF(package, substr);
         }
     }
 
     last_dot = PyUnicode_GET_LENGTH(package);
     if (last_dot == 0) {
-        PyErr_SetString(PyExc_ImportError,
-                "attempted relative import with no known parent package");
-        goto error;
+        goto no_parent_error;
     }
 
     for (level_up = 1; level_up < level; level_up += 1) {
@@ -1604,6 +1602,11 @@ resolve_name(PyObject *name, PyObject *globals, int level)
     Py_DECREF(base);
     return abs_name;
 
+  no_parent_error:
+    PyErr_SetString(PyExc_ImportError,
+                     "attempted relative import "
+                     "with no known parent package");
+
   error:
     Py_XDECREF(package);
     return NULL;


From webhook-mailer at python.org  Wed Sep 11 09:02:28 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Wed, 11 Sep 2019 13:02:28 -0000
Subject: [Python-checkins] bpo-35649: update http client example (GH-11441)
 (GH-15930)
Message-ID: <mailman.542.1568206950.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/43fb3bb223338511a7aee9b55d75af4a415134dc
commit: 43fb3bb223338511a7aee9b55d75af4a415134dc
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-11T15:02:25+02:00
summary:

bpo-35649: update http client example (GH-11441) (GH-15930)

(cherry picked from commit 62cf6981425c6a6b136c5e2abef853364f535e9d)

Co-authored-by: Ashwin Ramaswami <aramaswamis at gmail.com>

files:
M Doc/library/http.client.rst

diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst
index 4e761cd39a01..48bc35ca76cc 100644
--- a/Doc/library/http.client.rst
+++ b/Doc/library/http.client.rst
@@ -516,8 +516,11 @@ Here is an example session that uses the ``GET`` method::
    >>> # The following example demonstrates reading data in chunks.
    >>> conn.request("GET", "/")
    >>> r1 = conn.getresponse()
-   >>> while not r1.closed:
-   ...     print(r1.read(200))  # 200 bytes
+   >>> while True:
+   ...     chunk = r1.read(200)  # 200 bytes
+   ...     if not chunk:
+   ...          break
+   ...     print(repr(chunk))
    b'<!doctype html>\n<!--[if"...
    ...
    >>> # Example of an invalid request


From webhook-mailer at python.org  Wed Sep 11 09:03:50 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Wed, 11 Sep 2019 13:03:50 -0000
Subject: [Python-checkins] bpo-35640: Allow passing PathLike arguments to
 SimpleHTTPRequestHandler (GH-11398)
Message-ID: <mailman.543.1568207031.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/781266ebb60e7ac781a3e07030d92275721ff3cf
commit: 781266ebb60e7ac781a3e07030d92275721ff3cf
branch: master
author: G?ry Ogam <gery.ogam at gmail.com>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-11T14:03:46+01:00
summary:

bpo-35640: Allow passing PathLike arguments to SimpleHTTPRequestHandler (GH-11398)

files:
A Misc/NEWS.d/next/Library/2019-09-10-15-52-55.bpo-35640.X0lp5f.rst
M Doc/library/http.server.rst
M Lib/http/server.py
M Lib/test/test_httpservers.py

diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst
index a367e373dc3c..5173decb2b2e 100644
--- a/Doc/library/http.server.rst
+++ b/Doc/library/http.server.rst
@@ -344,6 +344,9 @@ provides three different variants:
 
       If not specified, the directory to serve is the current working directory.
 
+      .. versionchanged:: 3.9
+         Accepts a :term:`path-like object`.
+
    The :class:`SimpleHTTPRequestHandler` class defines the following methods:
 
    .. method:: do_HEAD()
diff --git a/Lib/http/server.py b/Lib/http/server.py
index b247675ec45e..005dd824f9a7 100644
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -642,7 +642,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
     def __init__(self, *args, directory=None, **kwargs):
         if directory is None:
             directory = os.getcwd()
-        self.directory = directory
+        self.directory = os.fspath(directory)
         super().__init__(*args, **kwargs)
 
     def do_GET(self):
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
index 87d4924a34b3..1c980a2fa66e 100644
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -14,6 +14,7 @@
 import re
 import base64
 import ntpath
+import pathlib
 import shutil
 import email.message
 import email.utils
@@ -790,10 +791,10 @@ def test_query_with_continuous_slashes(self):
 
 
 class SocketlessRequestHandler(SimpleHTTPRequestHandler):
-    def __init__(self, *args, **kwargs):
+    def __init__(self, directory=None):
         request = mock.Mock()
         request.makefile.return_value = BytesIO()
-        super().__init__(request, None, None)
+        super().__init__(request, None, None, directory=directory)
 
         self.get_called = False
         self.protocol_version = "HTTP/1.1"
@@ -1068,41 +1069,91 @@ def test_date_time_string(self):
 class SimpleHTTPRequestHandlerTestCase(unittest.TestCase):
     """ Test url parsing """
     def setUp(self):
-        self.translated = os.getcwd()
-        self.translated = os.path.join(self.translated, 'filename')
-        self.handler = SocketlessRequestHandler()
+        self.translated_1 = os.path.join(os.getcwd(), 'filename')
+        self.translated_2 = os.path.join('foo', 'filename')
+        self.translated_3 = os.path.join('bar', 'filename')
+        self.handler_1 = SocketlessRequestHandler()
+        self.handler_2 = SocketlessRequestHandler(directory='foo')
+        self.handler_3 = SocketlessRequestHandler(directory=pathlib.PurePath('bar'))
 
     def test_query_arguments(self):
-        path = self.handler.translate_path('/filename')
-        self.assertEqual(path, self.translated)
-        path = self.handler.translate_path('/filename?foo=bar')
-        self.assertEqual(path, self.translated)
-        path = self.handler.translate_path('/filename?a=b&spam=eggs#zot')
-        self.assertEqual(path, self.translated)
+        path = self.handler_1.translate_path('/filename')
+        self.assertEqual(path, self.translated_1)
+        path = self.handler_2.translate_path('/filename')
+        self.assertEqual(path, self.translated_2)
+        path = self.handler_3.translate_path('/filename')
+        self.assertEqual(path, self.translated_3)
+
+        path = self.handler_1.translate_path('/filename?foo=bar')
+        self.assertEqual(path, self.translated_1)
+        path = self.handler_2.translate_path('/filename?foo=bar')
+        self.assertEqual(path, self.translated_2)
+        path = self.handler_3.translate_path('/filename?foo=bar')
+        self.assertEqual(path, self.translated_3)
+
+        path = self.handler_1.translate_path('/filename?a=b&spam=eggs#zot')
+        self.assertEqual(path, self.translated_1)
+        path = self.handler_2.translate_path('/filename?a=b&spam=eggs#zot')
+        self.assertEqual(path, self.translated_2)
+        path = self.handler_3.translate_path('/filename?a=b&spam=eggs#zot')
+        self.assertEqual(path, self.translated_3)
 
     def test_start_with_double_slash(self):
-        path = self.handler.translate_path('//filename')
-        self.assertEqual(path, self.translated)
-        path = self.handler.translate_path('//filename?foo=bar')
-        self.assertEqual(path, self.translated)
+        path = self.handler_1.translate_path('//filename')
+        self.assertEqual(path, self.translated_1)
+        path = self.handler_2.translate_path('//filename')
+        self.assertEqual(path, self.translated_2)
+        path = self.handler_3.translate_path('//filename')
+        self.assertEqual(path, self.translated_3)
+
+        path = self.handler_1.translate_path('//filename?foo=bar')
+        self.assertEqual(path, self.translated_1)
+        path = self.handler_2.translate_path('//filename?foo=bar')
+        self.assertEqual(path, self.translated_2)
+        path = self.handler_3.translate_path('//filename?foo=bar')
+        self.assertEqual(path, self.translated_3)
 
     def test_windows_colon(self):
         with support.swap_attr(server.os, 'path', ntpath):
-            path = self.handler.translate_path('c:c:c:foo/filename')
+            path = self.handler_1.translate_path('c:c:c:foo/filename')
+            path = path.replace(ntpath.sep, os.sep)
+            self.assertEqual(path, self.translated_1)
+            path = self.handler_2.translate_path('c:c:c:foo/filename')
+            path = path.replace(ntpath.sep, os.sep)
+            self.assertEqual(path, self.translated_2)
+            path = self.handler_3.translate_path('c:c:c:foo/filename')
             path = path.replace(ntpath.sep, os.sep)
-            self.assertEqual(path, self.translated)
+            self.assertEqual(path, self.translated_3)
 
-            path = self.handler.translate_path('\\c:../filename')
+            path = self.handler_1.translate_path('\\c:../filename')
+            path = path.replace(ntpath.sep, os.sep)
+            self.assertEqual(path, self.translated_1)
+            path = self.handler_2.translate_path('\\c:../filename')
+            path = path.replace(ntpath.sep, os.sep)
+            self.assertEqual(path, self.translated_2)
+            path = self.handler_3.translate_path('\\c:../filename')
             path = path.replace(ntpath.sep, os.sep)
-            self.assertEqual(path, self.translated)
+            self.assertEqual(path, self.translated_3)
 
-            path = self.handler.translate_path('c:\\c:..\\foo/filename')
+            path = self.handler_1.translate_path('c:\\c:..\\foo/filename')
             path = path.replace(ntpath.sep, os.sep)
-            self.assertEqual(path, self.translated)
+            self.assertEqual(path, self.translated_1)
+            path = self.handler_2.translate_path('c:\\c:..\\foo/filename')
+            path = path.replace(ntpath.sep, os.sep)
+            self.assertEqual(path, self.translated_2)
+            path = self.handler_3.translate_path('c:\\c:..\\foo/filename')
+            path = path.replace(ntpath.sep, os.sep)
+            self.assertEqual(path, self.translated_3)
 
-            path = self.handler.translate_path('c:c:foo\\c:c:bar/filename')
+            path = self.handler_1.translate_path('c:c:foo\\c:c:bar/filename')
+            path = path.replace(ntpath.sep, os.sep)
+            self.assertEqual(path, self.translated_1)
+            path = self.handler_2.translate_path('c:c:foo\\c:c:bar/filename')
+            path = path.replace(ntpath.sep, os.sep)
+            self.assertEqual(path, self.translated_2)
+            path = self.handler_3.translate_path('c:c:foo\\c:c:bar/filename')
             path = path.replace(ntpath.sep, os.sep)
-            self.assertEqual(path, self.translated)
+            self.assertEqual(path, self.translated_3)
 
 
 class MiscTestCase(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Library/2019-09-10-15-52-55.bpo-35640.X0lp5f.rst b/Misc/NEWS.d/next/Library/2019-09-10-15-52-55.bpo-35640.X0lp5f.rst
new file mode 100644
index 000000000000..abdf87a62388
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-10-15-52-55.bpo-35640.X0lp5f.rst
@@ -0,0 +1,2 @@
+Allow passing a :term:`path-like object` as ``directory`` argument to the
+:class:`http.server.SimpleHTTPRequestHandler` class. Patch by G?ry Ogam.


From webhook-mailer at python.org  Wed Sep 11 09:07:40 2019
From: webhook-mailer at python.org (Andrew Svetlov)
Date: Wed, 11 Sep 2019 13:07:40 -0000
Subject: [Python-checkins] bpo-38107: Replace direct future and task
 contructor calls with factories in asyncio tests (GH-15928)
Message-ID: <mailman.544.1568207262.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9aee90018a5213e3529768e0b873955d23f5e50b
commit: 9aee90018a5213e3529768e0b873955d23f5e50b
branch: master
author: Andrew Svetlov <andrew.svetlov at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T16:07:37+03:00
summary:

bpo-38107: Replace direct future and task contructor calls with factories in asyncio tests (GH-15928)

files:
M Lib/test/test_asyncio/test_base_events.py
M Lib/test/test_asyncio/test_events.py
M Lib/test/test_asyncio/test_locks.py
M Lib/test/test_asyncio/test_pep492.py
M Lib/test/test_asyncio/test_proactor_events.py
M Lib/test/test_asyncio/test_queues.py
M Lib/test/test_asyncio/test_selector_events.py
M Lib/test/test_asyncio/test_sock_lowlevel.py
M Lib/test/test_asyncio/test_sslproto.py
M Lib/test/test_asyncio/test_streams.py
M Lib/test/test_asyncio/test_subprocess.py
M Lib/test/test_asyncio/test_tasks.py
M Lib/test/test_asyncio/test_unix_events.py

diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
index 08d4792fa726..370e6a6dfa79 100644
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -193,7 +193,7 @@ def test_close(self):
         self.loop.close()
 
         # operation blocked when the loop is closed
-        f = asyncio.Future(loop=self.loop)
+        f = self.loop.create_future()
         self.assertRaises(RuntimeError, self.loop.run_forever)
         self.assertRaises(RuntimeError, self.loop.run_until_complete, f)
 
@@ -324,7 +324,7 @@ def check_in_thread(loop, event, debug, create_loop, fut):
 
         def test_thread(loop, debug, create_loop=False):
             event = threading.Event()
-            fut = asyncio.Future(loop=loop)
+            fut = loop.create_future()
             loop.call_soon(event.set)
             args = (loop, event, debug, create_loop, fut)
             thread = threading.Thread(target=check_in_thread, args=args)
@@ -472,7 +472,7 @@ def test_run_until_complete_type_error(self):
             self.loop.run_until_complete, 'blah')
 
     def test_run_until_complete_loop(self):
-        task = asyncio.Future(loop=self.loop)
+        task = self.loop.create_future()
         other_loop = self.new_test_loop()
         self.addCleanup(other_loop.close)
         self.assertRaises(ValueError,
@@ -555,7 +555,7 @@ def zero_error(fut):
 
         # Test call_soon (events.Handle)
         with mock.patch('asyncio.base_events.logger') as log:
-            fut = asyncio.Future(loop=self.loop)
+            fut = self.loop.create_future()
             self.loop.call_soon(zero_error, fut)
             fut.add_done_callback(lambda fut: self.loop.stop())
             self.loop.run_forever()
@@ -565,7 +565,7 @@ def zero_error(fut):
 
         # Test call_later (events.TimerHandle)
         with mock.patch('asyncio.base_events.logger') as log:
-            fut = asyncio.Future(loop=self.loop)
+            fut = self.loop.create_future()
             self.loop.call_later(0.01, zero_error, fut)
             fut.add_done_callback(lambda fut: self.loop.stop())
             self.loop.run_forever()
@@ -996,7 +996,7 @@ def __init__(self, create_future=False):
         self.state = 'INITIAL'
         self.nbytes = 0
         if create_future:
-            self.done = asyncio.Future()
+            self.done = asyncio.get_running_loop().create_future()
 
     def connection_made(self, transport):
         self.transport = transport
@@ -1026,7 +1026,7 @@ def __init__(self, create_future=False, loop=None):
         self.state = 'INITIAL'
         self.nbytes = 0
         if create_future:
-            self.done = asyncio.Future(loop=loop)
+            self.done = loop.create_future()
 
     def connection_made(self, transport):
         self.transport = transport
@@ -1071,7 +1071,7 @@ class MyProto(asyncio.Protocol):
                     (2, 1, 6, '', ('107.6.106.82', 80))]
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         idx = -1
         errors = ['err1', 'err2']
@@ -1098,7 +1098,7 @@ def test_create_connection_timeout(self, m_socket):
         m_socket.socket.return_value = sock
 
         def getaddrinfo(*args, **kw):
-            fut = asyncio.Future(loop=self.loop)
+            fut = self.loop.create_future()
             addr = (socket.AF_INET, socket.SOCK_STREAM, 0, '',
                     ('127.0.0.1', 80))
             fut.set_result([addr])
@@ -1190,7 +1190,7 @@ def test_create_connection_no_getaddrinfo(self):
             return []
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         self.loop.getaddrinfo = getaddrinfo_task
         coro = self.loop.create_connection(MyProto, 'example.com', 80)
@@ -1202,7 +1202,7 @@ def test_create_connection_connect_err(self):
             return [(2, 1, 6, '', ('107.6.106.82', 80))]
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         self.loop.getaddrinfo = getaddrinfo_task
         self.loop.sock_connect = mock.Mock()
@@ -1218,7 +1218,7 @@ def test_create_connection_multiple(self):
                     (2, 1, 6, '', ('0.0.0.2', 80))]
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         self.loop.getaddrinfo = getaddrinfo_task
         self.loop.sock_connect = mock.Mock()
@@ -1245,7 +1245,7 @@ def bind(addr):
                     (2, 1, 6, '', ('0.0.0.2', 80))]
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         self.loop.getaddrinfo = getaddrinfo_task
         self.loop.sock_connect = mock.Mock()
@@ -1377,7 +1377,7 @@ def test_create_connection_no_local_addr(self):
                 return []
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
         self.loop.getaddrinfo = getaddrinfo_task
 
         coro = self.loop.create_connection(
@@ -1405,7 +1405,7 @@ def test_create_connection_ssl_server_hostname_default(self):
         self.loop.getaddrinfo = mock.Mock()
 
         def mock_getaddrinfo(*args, **kwds):
-            f = asyncio.Future(loop=self.loop)
+            f = self.loop.create_future()
             f.set_result([(socket.AF_INET, socket.SOCK_STREAM,
                            socket.SOL_TCP, '', ('1.2.3.4', 80))])
             return f
@@ -1513,7 +1513,7 @@ def test_create_server_empty_host(self):
             return []
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         self.loop.getaddrinfo = getaddrinfo_task
         fut = self.loop.create_server(MyProto, '', 0)
diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py
index bdc861374650..7256758465da 100644
--- a/Lib/test/test_asyncio/test_events.py
+++ b/Lib/test/test_asyncio/test_events.py
@@ -82,8 +82,8 @@ def __init__(self, loop=None):
         self.state = 'INITIAL'
         self.nbytes = 0
         if loop is not None:
-            self.connected = asyncio.Future(loop=loop)
-            self.done = asyncio.Future(loop=loop)
+            self.connected = loop.create_future()
+            self.done = loop.create_future()
 
     def connection_made(self, transport):
         self.transport = transport
@@ -120,7 +120,7 @@ def __init__(self, loop=None):
         self.state = 'INITIAL'
         self.nbytes = 0
         if loop is not None:
-            self.done = asyncio.Future(loop=loop)
+            self.done = loop.create_future()
 
     def connection_made(self, transport):
         self.transport = transport
@@ -149,7 +149,7 @@ def __init__(self, loop=None):
         self.nbytes = 0
         self.transport = None
         if loop is not None:
-            self.done = asyncio.Future(loop=loop)
+            self.done = loop.create_future()
 
     def connection_made(self, transport):
         self.transport = transport
@@ -180,7 +180,7 @@ def __init__(self, loop=None):
         self.state = 'INITIAL'
         self.transport = None
         if loop is not None:
-            self.done = asyncio.Future(loop=loop)
+            self.done = loop.create_future()
 
     def connection_made(self, transport):
         self.transport = transport
@@ -199,9 +199,9 @@ class MySubprocessProtocol(asyncio.SubprocessProtocol):
     def __init__(self, loop):
         self.state = 'INITIAL'
         self.transport = None
-        self.connected = asyncio.Future(loop=loop)
-        self.completed = asyncio.Future(loop=loop)
-        self.disconnects = {fd: asyncio.Future(loop=loop) for fd in range(3)}
+        self.connected = loop.create_future()
+        self.completed = loop.create_future()
+        self.disconnects = {fd: loop.create_future() for fd in range(3)}
         self.data = {1: b'', 2: b''}
         self.returncode = None
         self.got_data = {1: asyncio.Event(loop=loop),
@@ -739,7 +739,7 @@ def create_server_multiple_hosts(self, family, hosts, mock_sock):
                 return [(family, socket.SOCK_STREAM, 6, '', (host, port, 0, 0))]
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         unique_hosts = set(hosts)
 
@@ -1095,7 +1095,7 @@ def test_create_server_ssl_verified(self):
         client, pr = self.loop.run_until_complete(f_c)
 
         # extra info is available
-        self.check_ssl_extra_info(client,peername=(host, port),
+        self.check_ssl_extra_info(client, peername=(host, port),
                                   peercert=test_utils.PEERCERT)
 
         # close connection
@@ -1105,7 +1105,7 @@ def test_create_server_ssl_verified(self):
         self.loop.run_until_complete(proto.done)
 
     def test_create_server_sock(self):
-        proto = asyncio.Future(loop=self.loop)
+        proto = self.loop.create_future()
 
         class TestMyProto(MyProto):
             def connection_made(self, transport):
@@ -1144,7 +1144,7 @@ def test_create_server_addr_in_use(self):
 
     @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 not supported or enabled')
     def test_create_server_dual_stack(self):
-        f_proto = asyncio.Future(loop=self.loop)
+        f_proto = self.loop.create_future()
 
         class TestMyProto(MyProto):
             def connection_made(self, transport):
@@ -1173,7 +1173,7 @@ def connection_made(self, transport):
         proto.transport.close()
         client.close()
 
-        f_proto = asyncio.Future(loop=self.loop)
+        f_proto = self.loop.create_future()
         client = socket.socket(socket.AF_INET6)
         client.connect(('::1', port))
         client.send(b'xxx')
@@ -1588,7 +1588,7 @@ def test_prompt_cancellation(self):
             return res
 
         start = time.monotonic()
-        t = asyncio.Task(main(), loop=self.loop)
+        t = self.loop.create_task(main())
         self.loop.run_forever()
         elapsed = time.monotonic() - start
 
@@ -1674,7 +1674,7 @@ def test_close(self):
         with self.assertRaises(RuntimeError):
             self.loop.run_forever()
         with self.assertRaises(RuntimeError):
-            fut = asyncio.Future(loop=self.loop)
+            fut = self.loop.create_future()
             self.loop.run_until_complete(fut)
         with self.assertRaises(RuntimeError):
             self.loop.call_soon(func)
diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py
index a9953b4b2a2d..9468e740b3c1 100644
--- a/Lib/test/test_asyncio/test_locks.py
+++ b/Lib/test/test_asyncio/test_locks.py
@@ -124,8 +124,8 @@ def test_acquire(self):
                 result.append(3)
             return True
 
-        t1 = asyncio.Task(c1(result), loop=self.loop)
-        t2 = asyncio.Task(c2(result), loop=self.loop)
+        t1 = self.loop.create_task(c1(result))
+        t2 = self.loop.create_task(c2(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
@@ -137,7 +137,7 @@ def test_acquire(self):
         test_utils.run_briefly(self.loop)
         self.assertEqual([1], result)
 
-        t3 = asyncio.Task(c3(result), loop=self.loop)
+        t3 = self.loop.create_task(c3(result))
 
         lock.release()
         test_utils.run_briefly(self.loop)
@@ -159,7 +159,7 @@ def test_acquire_cancel(self):
             lock = asyncio.Lock(loop=self.loop)
         self.assertTrue(self.loop.run_until_complete(lock.acquire()))
 
-        task = asyncio.Task(lock.acquire(), loop=self.loop)
+        task = self.loop.create_task(lock.acquire())
         self.loop.call_soon(task.cancel)
         self.assertRaises(
             asyncio.CancelledError,
@@ -192,14 +192,14 @@ def test_cancel_race(self):
             finally:
                 lock.release()
 
-        fa = asyncio.Future(loop=self.loop)
-        ta = asyncio.Task(lockit('A', fa), loop=self.loop)
+        fa = self.loop.create_future()
+        ta = self.loop.create_task(lockit('A', fa))
         test_utils.run_briefly(self.loop)
         self.assertTrue(lock.locked())
-        tb = asyncio.Task(lockit('B', None), loop=self.loop)
+        tb = self.loop.create_task(lockit('B', None))
         test_utils.run_briefly(self.loop)
         self.assertEqual(len(lock._waiters), 1)
-        tc = asyncio.Task(lockit('C', None), loop=self.loop)
+        tc = self.loop.create_task(lockit('C', None))
         test_utils.run_briefly(self.loop)
         self.assertEqual(len(lock._waiters), 2)
 
@@ -267,17 +267,17 @@ def test_finished_waiter_cancelled(self):
         with self.assertWarns(DeprecationWarning):
             lock = asyncio.Lock(loop=self.loop)
 
-        ta = asyncio.Task(lock.acquire(), loop=self.loop)
+        ta = self.loop.create_task(lock.acquire())
         test_utils.run_briefly(self.loop)
         self.assertTrue(lock.locked())
 
-        tb = asyncio.Task(lock.acquire(), loop=self.loop)
+        tb = self.loop.create_task(lock.acquire())
         test_utils.run_briefly(self.loop)
         self.assertEqual(len(lock._waiters), 1)
 
         # Create a second waiter, wake up the first, and cancel it.
         # Without the fix, the second was not woken up.
-        tc = asyncio.Task(lock.acquire(), loop=self.loop)
+        tc = self.loop.create_task(lock.acquire())
         lock.release()
         tb.cancel()
         test_utils.run_briefly(self.loop)
@@ -405,13 +405,13 @@ def test_wait(self):
             if await ev.wait():
                 result.append(3)
 
-        t1 = asyncio.Task(c1(result), loop=self.loop)
-        t2 = asyncio.Task(c2(result), loop=self.loop)
+        t1 = self.loop.create_task(c1(result))
+        t2 = self.loop.create_task(c2(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
 
-        t3 = asyncio.Task(c3(result), loop=self.loop)
+        t3 = self.loop.create_task(c3(result))
 
         ev.set()
         test_utils.run_briefly(self.loop)
@@ -436,7 +436,7 @@ def test_wait_cancel(self):
         with self.assertWarns(DeprecationWarning):
             ev = asyncio.Event(loop=self.loop)
 
-        wait = asyncio.Task(ev.wait(), loop=self.loop)
+        wait = self.loop.create_task(ev.wait())
         self.loop.call_soon(wait.cancel)
         self.assertRaises(
             asyncio.CancelledError,
@@ -464,7 +464,7 @@ def test_clear_with_waiters(self):
                 result.append(1)
             return True
 
-        t = asyncio.Task(c1(result), loop=self.loop)
+        t = self.loop.create_task(c1(result))
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
 
@@ -527,9 +527,9 @@ def test_wait(self):
                 result.append(3)
             return True
 
-        t1 = asyncio.Task(c1(result), loop=self.loop)
-        t2 = asyncio.Task(c2(result), loop=self.loop)
-        t3 = asyncio.Task(c3(result), loop=self.loop)
+        t1 = self.loop.create_task(c1(result))
+        t2 = self.loop.create_task(c2(result))
+        t3 = self.loop.create_task(c3(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
@@ -573,7 +573,7 @@ def test_wait_cancel(self):
             cond = asyncio.Condition(loop=self.loop)
         self.loop.run_until_complete(cond.acquire())
 
-        wait = asyncio.Task(cond.wait(), loop=self.loop)
+        wait = self.loop.create_task(cond.wait())
         self.loop.call_soon(wait.cancel)
         self.assertRaises(
             asyncio.CancelledError,
@@ -588,7 +588,7 @@ def test_wait_cancel_contested(self):
         self.loop.run_until_complete(cond.acquire())
         self.assertTrue(cond.locked())
 
-        wait_task = asyncio.Task(cond.wait(), loop=self.loop)
+        wait_task = self.loop.create_task(cond.wait())
         test_utils.run_briefly(self.loop)
         self.assertFalse(cond.locked())
 
@@ -657,7 +657,7 @@ def predicate():
                 cond.release()
             return True
 
-        t = asyncio.Task(c1(result), loop=self.loop)
+        t = self.loop.create_task(c1(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
@@ -717,9 +717,9 @@ def test_notify(self):
                 cond.release()
             return True
 
-        t1 = asyncio.Task(c1(result), loop=self.loop)
-        t2 = asyncio.Task(c2(result), loop=self.loop)
-        t3 = asyncio.Task(c3(result), loop=self.loop)
+        t1 = self.loop.create_task(c1(result))
+        t2 = self.loop.create_task(c2(result))
+        t3 = self.loop.create_task(c3(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
@@ -764,8 +764,8 @@ def test_notify_all(self):
                 cond.release()
             return True
 
-        t1 = asyncio.Task(c1(result), loop=self.loop)
-        t2 = asyncio.Task(c2(result), loop=self.loop)
+        t1 = self.loop.create_task(c1(result))
+        t2 = self.loop.create_task(c2(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
@@ -965,9 +965,9 @@ def test_acquire(self):
             result.append(4)
             return True
 
-        t1 = asyncio.Task(c1(result), loop=self.loop)
-        t2 = asyncio.Task(c2(result), loop=self.loop)
-        t3 = asyncio.Task(c3(result), loop=self.loop)
+        t1 = self.loop.create_task(c1(result))
+        t2 = self.loop.create_task(c2(result))
+        t3 = self.loop.create_task(c3(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([1], result)
@@ -975,7 +975,7 @@ def test_acquire(self):
         self.assertEqual(2, len(sem._waiters))
         self.assertEqual(0, sem._value)
 
-        t4 = asyncio.Task(c4(result), loop=self.loop)
+        t4 = self.loop.create_task(c4(result))
 
         sem.release()
         sem.release()
@@ -1003,7 +1003,7 @@ def test_acquire_cancel(self):
             sem = asyncio.Semaphore(loop=self.loop)
         self.loop.run_until_complete(sem.acquire())
 
-        acquire = asyncio.Task(sem.acquire(), loop=self.loop)
+        acquire = self.loop.create_task(sem.acquire())
         self.loop.call_soon(acquire.cancel)
         self.assertRaises(
             asyncio.CancelledError,
@@ -1015,10 +1015,10 @@ def test_acquire_cancel_before_awoken(self):
         with self.assertWarns(DeprecationWarning):
             sem = asyncio.Semaphore(value=0, loop=self.loop)
 
-        t1 = asyncio.Task(sem.acquire(), loop=self.loop)
-        t2 = asyncio.Task(sem.acquire(), loop=self.loop)
-        t3 = asyncio.Task(sem.acquire(), loop=self.loop)
-        t4 = asyncio.Task(sem.acquire(), loop=self.loop)
+        t1 = self.loop.create_task(sem.acquire())
+        t2 = self.loop.create_task(sem.acquire())
+        t3 = self.loop.create_task(sem.acquire())
+        t4 = self.loop.create_task(sem.acquire())
 
         test_utils.run_briefly(self.loop)
 
@@ -1038,8 +1038,8 @@ def test_acquire_hang(self):
         with self.assertWarns(DeprecationWarning):
             sem = asyncio.Semaphore(value=0, loop=self.loop)
 
-        t1 = asyncio.Task(sem.acquire(), loop=self.loop)
-        t2 = asyncio.Task(sem.acquire(), loop=self.loop)
+        t1 = self.loop.create_task(sem.acquire())
+        t2 = self.loop.create_task(sem.acquire())
 
         test_utils.run_briefly(self.loop)
 
diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py
index dfb68b8e85d4..58a609444285 100644
--- a/Lib/test/test_asyncio/test_pep492.py
+++ b/Lib/test/test_asyncio/test_pep492.py
@@ -206,7 +206,7 @@ def test_double_await(self):
 
         async def runner():
             coro = afunc()
-            t = asyncio.Task(coro, loop=self.loop)
+            t = self.loop.create_task(coro)
             try:
                 await asyncio.sleep(0)
                 await coro
diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py
index b2fd60683c57..c838c5ce85ae 100644
--- a/Lib/test/test_asyncio/test_proactor_events.py
+++ b/Lib/test/test_asyncio/test_proactor_events.py
@@ -47,7 +47,7 @@ def socket_transport(self, waiter=None):
         return transport
 
     def test_ctor(self):
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         tr = self.socket_transport(waiter=fut)
         test_utils.run_briefly(self.loop)
         self.assertIsNone(fut.result())
@@ -62,7 +62,7 @@ def test_loop_reading(self):
         self.assertFalse(self.protocol.eof_received.called)
 
     def test_loop_reading_data(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(b'data')
 
         tr = self.socket_transport()
@@ -72,7 +72,7 @@ def test_loop_reading_data(self):
         self.protocol.data_received.assert_called_with(b'data')
 
     def test_loop_reading_no_data(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(b'')
 
         tr = self.socket_transport()
@@ -182,7 +182,7 @@ def test_loop_writing_err(self, m_log):
         m_log.warning.assert_called_with('socket.send() raised exception.')
 
     def test_loop_writing_stop(self):
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         fut.set_result(b'data')
 
         tr = self.socket_transport()
@@ -191,7 +191,7 @@ def test_loop_writing_stop(self):
         self.assertIsNone(tr._write_fut)
 
     def test_loop_writing_closing(self):
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         fut.set_result(1)
 
         tr = self.socket_transport()
@@ -260,7 +260,7 @@ def test_force_close(self):
     def test_loop_writing_force_close(self):
         exc_handler = mock.Mock()
         self.loop.set_exception_handler(exc_handler)
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         fut.set_result(1)
         self.proactor.send.return_value = fut
 
@@ -303,7 +303,7 @@ def test_write_eof(self):
 
     def test_write_eof_buffer(self):
         tr = self.socket_transport()
-        f = asyncio.Future(loop=self.loop)
+        f = self.loop.create_future()
         tr._loop._proactor.send.return_value = f
         tr.write(b'data')
         tr.write_eof()
@@ -327,7 +327,7 @@ def test_write_eof_write_pipe(self):
 
     def test_write_eof_buffer_write_pipe(self):
         tr = _ProactorWritePipeTransport(self.loop, self.sock, self.protocol)
-        f = asyncio.Future(loop=self.loop)
+        f = self.loop.create_future()
         tr._loop._proactor.send.return_value = f
         tr.write(b'data')
         tr.write_eof()
@@ -352,7 +352,7 @@ def test_pause_resume_reading(self):
         tr = self.socket_transport()
         futures = []
         for msg in [b'data1', b'data2', b'data3', b'data4', b'data5', b'']:
-            f = asyncio.Future(loop=self.loop)
+            f = self.loop.create_future()
             f.set_result(msg)
             futures.append(f)
 
@@ -406,7 +406,7 @@ def test_pause_resume_writing(self):
         tr = self.pause_writing_transport(high=4)
 
         # write a large chunk, must pause writing
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         self.loop._proactor.send.return_value = fut
         tr.write(b'large data')
         self.loop._run_once()
@@ -422,7 +422,7 @@ def test_pause_writing_2write(self):
         tr = self.pause_writing_transport(high=4)
 
         # first short write, the buffer is not full (3 <= 4)
-        fut1 = asyncio.Future(loop=self.loop)
+        fut1 = self.loop.create_future()
         self.loop._proactor.send.return_value = fut1
         tr.write(b'123')
         self.loop._run_once()
@@ -439,7 +439,7 @@ def test_pause_writing_3write(self):
         tr = self.pause_writing_transport(high=4)
 
         # first short write, the buffer is not full (1 <= 4)
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         self.loop._proactor.send.return_value = fut
         tr.write(b'1')
         self.loop._run_once()
@@ -463,7 +463,7 @@ def test_dont_pause_writing(self):
 
         # write a large chunk which completes immediately,
         # it should not pause writing
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         fut.set_result(None)
         self.loop._proactor.send.return_value = fut
         tr.write(b'very large data')
@@ -496,7 +496,7 @@ def socket_transport(self, waiter=None):
         return transport
 
     def test_ctor(self):
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         tr = self.socket_transport(waiter=fut)
         test_utils.run_briefly(self.loop)
         self.assertIsNone(fut.result())
@@ -541,7 +541,7 @@ def test_proto_type_switch(self):
         self.protocol = test_utils.make_test_protocol(asyncio.Protocol)
         tr = self.socket_transport()
 
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(b'data')
 
         tr = self.socket_transport()
@@ -558,7 +558,7 @@ def test_proto_type_switch(self):
 
         tr.set_protocol(buf_proto)
         test_utils.run_briefly(self.loop)
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(4)
 
         tr._read_fut = res
@@ -591,7 +591,7 @@ def test_buffer_updated_error(self):
         self.loop.call_exception_handler = mock.Mock()
         self.protocol.buffer_updated.side_effect = LookupError()
 
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(10)
         transport._read_fut = res
         transport._loop_reading(res)
@@ -601,7 +601,7 @@ def test_buffer_updated_error(self):
         self.assertTrue(self.protocol.buffer_updated.called)
 
     def test_loop_eof_received_error(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(0)
 
         self.protocol.eof_received.side_effect = LookupError()
@@ -617,7 +617,7 @@ def test_loop_eof_received_error(self):
         self.assertTrue(tr._fatal_error.called)
 
     def test_loop_reading_data(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(4)
 
         tr = self.socket_transport()
@@ -627,7 +627,7 @@ def test_loop_reading_data(self):
         self.protocol.buffer_updated.assert_called_with(4)
 
     def test_loop_reading_no_data(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(0)
 
         tr = self.socket_transport()
@@ -691,7 +691,7 @@ def test_pause_resume_reading(self):
         tr = self.socket_transport()
         futures = []
         for msg in [10, 20, 30, 40, 0]:
-            f = asyncio.Future(loop=self.loop)
+            f = self.loop.create_future()
             f.set_result(msg)
             futures.append(f)
 
@@ -1044,7 +1044,7 @@ def test_create_server_cancel(self):
         loop = call_soon.call_args[0][0]
 
         # cancelled
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         fut.cancel()
         loop(fut)
         self.assertTrue(self.sock.close.called)
@@ -1094,7 +1094,7 @@ def test_datagram_loop_reading(self):
         close_transport(tr)
 
     def test_datagram_loop_reading_data(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result((b'data', ('127.0.0.1', 12068)))
 
         tr = self.datagram_transport()
@@ -1105,7 +1105,7 @@ def test_datagram_loop_reading_data(self):
         close_transport(tr)
 
     def test_datagram_loop_reading_no_data(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result((b'', ('127.0.0.1', 12068)))
 
         tr = self.datagram_transport()
diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py
index 171176c9fc53..5c9aaa82c311 100644
--- a/Lib/test/test_asyncio/test_queues.py
+++ b/Lib/test/test_asyncio/test_queues.py
@@ -44,7 +44,7 @@ def gen():
         async def add_getter():
             q = asyncio.Queue(loop=loop)
             # Start a task that waits to get.
-            asyncio.Task(q.get(), loop=loop)
+            loop.create_task(q.get())
             # Let it start waiting.
             await asyncio.sleep(0.1)
             self.assertTrue('_getters[1]' in fn(q))
@@ -58,7 +58,7 @@ def gen():
             q = asyncio.Queue(maxsize=1, loop=loop)
             q.put_nowait(1)
             # Start a task that waits to put.
-            asyncio.Task(q.put(2), loop=loop)
+            loop.create_task(q.put(2))
             # Let it start waiting.
             await asyncio.sleep(0.1)
             self.assertTrue('_putters[1]' in fn(q))
@@ -143,7 +143,7 @@ def gen():
             return True
 
         async def test():
-            t = asyncio.Task(putter(), loop=loop)
+            t = loop.create_task(putter())
             await asyncio.sleep(0.01)
 
             # The putter is blocked after putting two items.
@@ -181,7 +181,7 @@ def test_get_with_putters(self):
             q = asyncio.Queue(1, loop=self.loop)
         q.put_nowait(1)
 
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         q._putters.append(waiter)
 
         res = self.loop.run_until_complete(q.get())
@@ -212,7 +212,7 @@ def gen():
 
         async def queue_put():
             loop.call_later(0.01, q.put_nowait, 1)
-            queue_get_task = asyncio.Task(queue_get(), loop=loop)
+            queue_get_task = loop.create_task(queue_get())
             await started.wait()
             self.assertFalse(finished)
             res = await queue_get_task
@@ -252,7 +252,7 @@ def gen():
             return await asyncio.wait_for(q.get(), 0.051)
 
         async def test():
-            get_task = asyncio.Task(queue_get(), loop=loop)
+            get_task = loop.create_task(queue_get())
             await asyncio.sleep(0.01)  # let the task start
             q.put_nowait(1)
             return await get_task
@@ -264,8 +264,8 @@ def test_get_cancelled_race(self):
         with self.assertWarns(DeprecationWarning):
             q = asyncio.Queue(loop=self.loop)
 
-        t1 = asyncio.Task(q.get(), loop=self.loop)
-        t2 = asyncio.Task(q.get(), loop=self.loop)
+        t1 = self.loop.create_task(q.get())
+        t2 = self.loop.create_task(q.get())
 
         test_utils.run_briefly(self.loop)
         t1.cancel()
@@ -278,8 +278,8 @@ def test_get_cancelled_race(self):
     def test_get_with_waiting_putters(self):
         with self.assertWarns(DeprecationWarning):
             q = asyncio.Queue(loop=self.loop, maxsize=1)
-        asyncio.Task(q.put('a'), loop=self.loop)
-        asyncio.Task(q.put('b'), loop=self.loop)
+        self.loop.create_task(q.put('a'))
+        self.loop.create_task(q.put('b'))
         test_utils.run_briefly(self.loop)
         self.assertEqual(self.loop.run_until_complete(q.get()), 'a')
         self.assertEqual(self.loop.run_until_complete(q.get()), 'b')
@@ -361,7 +361,7 @@ def gen():
 
         async def queue_get():
             loop.call_later(0.01, q.get_nowait)
-            queue_put_task = asyncio.Task(queue_put(), loop=loop)
+            queue_put_task = loop.create_task(queue_put())
             await started.wait()
             self.assertFalse(finished)
             await queue_put_task
@@ -502,7 +502,7 @@ def test_put_cancelled(self):
         async def test():
             return await q.get()
 
-        t = asyncio.Task(queue_put(), loop=self.loop)
+        t = self.loop.create_task(queue_put())
         self.assertEqual(1, self.loop.run_until_complete(test()))
         self.assertTrue(t.done())
         self.assertTrue(t.result())
@@ -511,9 +511,9 @@ def test_put_cancelled_race(self):
         with self.assertWarns(DeprecationWarning):
             q = asyncio.Queue(loop=self.loop, maxsize=1)
 
-        put_a = asyncio.Task(q.put('a'), loop=self.loop)
-        put_b = asyncio.Task(q.put('b'), loop=self.loop)
-        put_c = asyncio.Task(q.put('X'), loop=self.loop)
+        put_a = self.loop.create_task(q.put('a'))
+        put_b = self.loop.create_task(q.put('b'))
+        put_c = self.loop.create_task(q.put('X'))
 
         test_utils.run_briefly(self.loop)
         self.assertTrue(put_a.done())
@@ -531,7 +531,7 @@ def test_put_cancelled_race(self):
     def test_put_with_waiting_getters(self):
         with self.assertWarns(DeprecationWarning):
             q = asyncio.Queue(loop=self.loop)
-        t = asyncio.Task(q.get(), loop=self.loop)
+        t = self.loop.create_task(q.get())
         test_utils.run_briefly(self.loop)
         self.loop.run_until_complete(q.put('a'))
         self.assertEqual(self.loop.run_until_complete(t), 'a')
@@ -665,7 +665,7 @@ def test_task_done(self):
                 q.task_done()
 
         async def test():
-            tasks = [asyncio.Task(worker(), loop=self.loop)
+            tasks = [self.loop.create_task(worker())
                      for index in range(2)]
 
             await q.join()
diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py
index 68b7853b2eba..1613c753c26e 100644
--- a/Lib/test/test_asyncio/test_selector_events.py
+++ b/Lib/test/test_asyncio/test_selector_events.py
@@ -78,7 +78,7 @@ def test_make_ssl_transport(self):
         self.loop._add_writer = mock.Mock()
         self.loop._remove_reader = mock.Mock()
         self.loop._remove_writer = mock.Mock()
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         with test_utils.disable_logger():
             transport = self.loop._make_ssl_transport(
                 m, asyncio.Protocol(), m, waiter)
@@ -154,7 +154,7 @@ def _make_self_pipe(self):
         self.loop.close()
 
         # operation blocked when the loop is closed
-        f = asyncio.Future(loop=self.loop)
+        f = self.loop.create_future()
         self.assertRaises(RuntimeError, self.loop.run_forever)
         self.assertRaises(RuntimeError, self.loop.run_until_complete, f)
         fd = 0
@@ -516,7 +516,7 @@ def socket_transport(self, waiter=None):
         return transport
 
     def test_ctor(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         tr = self.socket_transport(waiter=waiter)
         self.loop.run_until_complete(waiter)
 
@@ -525,7 +525,7 @@ def test_ctor(self):
         self.protocol.connection_made.assert_called_with(tr)
 
     def test_ctor_with_waiter(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         self.socket_transport(waiter=waiter)
         self.loop.run_until_complete(waiter)
 
@@ -911,7 +911,7 @@ def socket_transport(self, waiter=None):
         return transport
 
     def test_ctor(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         tr = self.socket_transport(waiter=waiter)
         self.loop.run_until_complete(waiter)
 
diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py
index 7b40a765e4a0..89c2af9f5ec0 100644
--- a/Lib/test/test_asyncio/test_sock_lowlevel.py
+++ b/Lib/test/test_asyncio/test_sock_lowlevel.py
@@ -16,8 +16,8 @@ def __init__(self, loop=None):
         self.state = 'INITIAL'
         self.nbytes = 0
         if loop is not None:
-            self.connected = asyncio.Future(loop=loop)
-            self.done = asyncio.Future(loop=loop)
+            self.connected = loop.create_future()
+            self.done = loop.create_future()
 
     def connection_made(self, transport):
         self.transport = transport
diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py
index 9457bc982b06..c115e46e705b 100644
--- a/Lib/test/test_asyncio/test_sslproto.py
+++ b/Lib/test/test_asyncio/test_sslproto.py
@@ -72,7 +72,7 @@ def test_handshake_timeout_negative(self):
                                  ssl_handshake_timeout=-10)
 
     def test_eof_received_waiter(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         ssl_proto = self.ssl_protocol(waiter=waiter)
         self.connection_made(ssl_proto)
         ssl_proto.eof_received()
@@ -83,7 +83,7 @@ def test_fatal_error_no_name_error(self):
         # From issue #363.
         # _fatal_error() generates a NameError if sslproto.py
         # does not import base_events.
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         ssl_proto = self.ssl_protocol(waiter=waiter)
         # Temporarily turn off error logging so as not to spoil test output.
         log_level = log.logger.getEffectiveLevel()
@@ -97,7 +97,7 @@ def test_fatal_error_no_name_error(self):
     def test_connection_lost(self):
         # From issue #472.
         # yield from waiter hang if lost_connection was called.
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         ssl_proto = self.ssl_protocol(waiter=waiter)
         self.connection_made(ssl_proto)
         ssl_proto.connection_lost(ConnectionAbortedError)
@@ -106,7 +106,7 @@ def test_connection_lost(self):
 
     def test_close_during_handshake(self):
         # bpo-29743 Closing transport during handshake process leaks socket
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         ssl_proto = self.ssl_protocol(waiter=waiter)
 
         transport = self.connection_made(ssl_proto)
@@ -116,7 +116,7 @@ def test_close_during_handshake(self):
         self.assertTrue(transport.abort.called)
 
     def test_get_extra_info_on_closed_connection(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         ssl_proto = self.ssl_protocol(waiter=waiter)
         self.assertIsNone(ssl_proto._get_extra_info('socket'))
         default = object()
@@ -127,7 +127,7 @@ def test_get_extra_info_on_closed_connection(self):
         self.assertIsNone(ssl_proto._get_extra_info('socket'))
 
     def test_set_new_app_protocol(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         ssl_proto = self.ssl_protocol(waiter=waiter)
         new_app_proto = asyncio.Protocol()
         ssl_proto._app_transport.set_protocol(new_app_proto)
diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py
index 428c863d9fa6..60fe52fa4b37 100644
--- a/Lib/test/test_asyncio/test_streams.py
+++ b/Lib/test/test_asyncio/test_streams.py
@@ -192,7 +192,7 @@ def test_read(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        read_task = asyncio.Task(stream.read(30), loop=self.loop)
+        read_task = self.loop.create_task(stream.read(30))
 
         def cb():
             stream._feed_data(self.DATA)
@@ -220,7 +220,7 @@ def test_read_eof(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        read_task = asyncio.Task(stream.read(1024), loop=self.loop)
+        read_task = self.loop.create_task(stream.read(1024))
 
         def cb():
             stream._feed_eof()
@@ -235,7 +235,7 @@ def test_read_until_eof(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        read_task = asyncio.Task(stream.read(-1), loop=self.loop)
+        read_task = self.loop.create_task(stream.read(-1))
 
         def cb():
             stream._feed_data(b'chunk1\n')
@@ -288,7 +288,7 @@ def test_readline(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
         stream._feed_data(b'chunk1 ')
-        read_task = asyncio.Task(stream.readline(), loop=self.loop)
+        read_task = self.loop.create_task(stream.readline())
 
         def cb():
             stream._feed_data(b'chunk2 ')
@@ -579,7 +579,7 @@ def test_readexactly(self):
                                 _asyncio_internal=True)
 
         n = 2 * len(self.DATA)
-        read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
+        read_task = self.loop.create_task(stream.readexactly(n))
 
         def cb():
             stream._feed_data(self.DATA)
@@ -606,7 +606,7 @@ def test_readexactly_eof(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
         n = 2 * len(self.DATA)
-        read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
+        read_task = self.loop.create_task(stream.readexactly(n))
 
         def cb():
             stream._feed_data(self.DATA)
@@ -652,8 +652,8 @@ def test_exception_waiter(self):
         async def set_err():
             stream._set_exception(ValueError())
 
-        t1 = asyncio.Task(stream.readline(), loop=self.loop)
-        t2 = asyncio.Task(set_err(), loop=self.loop)
+        t1 = self.loop.create_task(stream.readline())
+        t2 = self.loop.create_task(set_err())
 
         self.loop.run_until_complete(asyncio.wait([t1, t2]))
 
@@ -664,7 +664,7 @@ def test_exception_cancel(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
 
-        t = asyncio.Task(stream.readline(), loop=self.loop)
+        t = self.loop.create_task(stream.readline())
         test_utils.run_briefly(self.loop)
         t.cancel()
         test_utils.run_briefly(self.loop)
@@ -735,8 +735,7 @@ def stop(self):
         server = MyServer(self.loop)
         with self.assertWarns(DeprecationWarning):
             addr = server.start()
-        msg = self.loop.run_until_complete(asyncio.Task(client(addr),
-                                                        loop=self.loop))
+        msg = self.loop.run_until_complete(self.loop.create_task(client(addr)))
         server.stop()
         self.assertEqual(msg, b"hello world!\n")
 
@@ -744,8 +743,7 @@ def stop(self):
         server = MyServer(self.loop)
         with self.assertWarns(DeprecationWarning):
             addr = server.start_callback()
-        msg = self.loop.run_until_complete(asyncio.Task(client(addr),
-                                                        loop=self.loop))
+        msg = self.loop.run_until_complete(self.loop.create_task(client(addr)))
         server.stop()
         self.assertEqual(msg, b"hello world!\n")
 
@@ -810,8 +808,8 @@ def stop(self):
             server = MyServer(self.loop, path)
             with self.assertWarns(DeprecationWarning):
                 server.start()
-            msg = self.loop.run_until_complete(asyncio.Task(client(path),
-                                                            loop=self.loop))
+            msg = self.loop.run_until_complete(
+                self.loop.create_task(client(path)))
             server.stop()
             self.assertEqual(msg, b"hello world!\n")
 
@@ -820,8 +818,8 @@ def stop(self):
             server = MyServer(self.loop, path)
             with self.assertWarns(DeprecationWarning):
                 server.start_callback()
-            msg = self.loop.run_until_complete(asyncio.Task(client(path),
-                                                            loop=self.loop))
+            msg = self.loop.run_until_complete(
+                self.loop.create_task(client(path)))
             server.stop()
             self.assertEqual(msg, b"hello world!\n")
 
@@ -1016,7 +1014,7 @@ def test___repr__waiter(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream._waiter = asyncio.Future(loop=self.loop)
+        stream._waiter = self.loop.create_future()
         self.assertRegex(
             repr(stream),
             r"<Stream .+ waiter=<Future pending[\S ]*>>")
diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py
index b9578b2866c0..2cfe6576d623 100644
--- a/Lib/test/test_asyncio/test_subprocess.py
+++ b/Lib/test/test_asyncio/test_subprocess.py
@@ -43,7 +43,6 @@ def setUp(self):
         self.loop = self.new_test_loop()
         self.set_event_loop(self.loop)
 
-
     def create_transport(self, waiter=None):
         protocol = mock.Mock()
         protocol.connection_made._is_coroutine = False
@@ -54,7 +53,7 @@ def create_transport(self, waiter=None):
         return (transport, protocol)
 
     def test_proc_exited(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         transport, protocol = self.create_transport(waiter)
         transport._process_exited(6)
         self.loop.run_until_complete(waiter)
@@ -80,7 +79,7 @@ def test_proc_exited(self):
         transport.close()
 
     def test_subprocess_repr(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         transport, protocol = self.create_transport(waiter)
         transport._process_exited(6)
         self.loop.run_until_complete(waiter)
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
index 323c0907a23a..3fbb2a1a1629 100644
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -2660,7 +2660,7 @@ def get_loop(self):
         try:
             fut = Fut(loop=self.loop)
             self.loop.call_later(0.1, fut.set_result, 1)
-            task = asyncio.Task(coro(), loop=self.loop)
+            task = self.loop.create_task(coro())
             res = self.loop.run_until_complete(task)
         finally:
             self.loop.close()
@@ -2878,7 +2878,7 @@ def _run_loop(self, loop):
             test_utils.run_briefly(loop)
 
     def _check_success(self, **kwargs):
-        a, b, c = [asyncio.Future(loop=self.one_loop) for i in range(3)]
+        a, b, c = [self.one_loop.create_future() for i in range(3)]
         fut = asyncio.gather(*self.wrap_futures(a, b, c), **kwargs)
         cb = test_utils.MockCallback()
         fut.add_done_callback(cb)
@@ -2900,7 +2900,7 @@ def test_result_exception_success(self):
         self._check_success(return_exceptions=True)
 
     def test_one_exception(self):
-        a, b, c, d, e = [asyncio.Future(loop=self.one_loop) for i in range(5)]
+        a, b, c, d, e = [self.one_loop.create_future() for i in range(5)]
         fut = asyncio.gather(*self.wrap_futures(a, b, c, d, e))
         cb = test_utils.MockCallback()
         fut.add_done_callback(cb)
@@ -2918,7 +2918,7 @@ def test_one_exception(self):
         e.exception()
 
     def test_return_exceptions(self):
-        a, b, c, d = [asyncio.Future(loop=self.one_loop) for i in range(4)]
+        a, b, c, d = [self.one_loop.create_future() for i in range(4)]
         fut = asyncio.gather(*self.wrap_futures(a, b, c, d),
                              return_exceptions=True)
         cb = test_utils.MockCallback()
@@ -2991,15 +2991,15 @@ def test_constructor_empty_sequence(self):
         self._check_empty_sequence(iter(""))
 
     def test_constructor_heterogenous_futures(self):
-        fut1 = asyncio.Future(loop=self.one_loop)
-        fut2 = asyncio.Future(loop=self.other_loop)
+        fut1 = self.one_loop.create_future()
+        fut2 = self.other_loop.create_future()
         with self.assertRaises(ValueError):
             asyncio.gather(fut1, fut2)
         with self.assertRaises(ValueError):
             asyncio.gather(fut1, loop=self.other_loop)
 
     def test_constructor_homogenous_futures(self):
-        children = [asyncio.Future(loop=self.other_loop) for i in range(3)]
+        children = [self.other_loop.create_future() for i in range(3)]
         fut = asyncio.gather(*children)
         self.assertIs(fut._loop, self.other_loop)
         self._run_loop(self.other_loop)
@@ -3010,7 +3010,7 @@ def test_constructor_homogenous_futures(self):
         self.assertFalse(fut.done())
 
     def test_one_cancellation(self):
-        a, b, c, d, e = [asyncio.Future(loop=self.one_loop) for i in range(5)]
+        a, b, c, d, e = [self.one_loop.create_future() for i in range(5)]
         fut = asyncio.gather(a, b, c, d, e)
         cb = test_utils.MockCallback()
         fut.add_done_callback(cb)
@@ -3028,7 +3028,7 @@ def test_one_cancellation(self):
         e.exception()
 
     def test_result_exception_one_cancellation(self):
-        a, b, c, d, e, f = [asyncio.Future(loop=self.one_loop)
+        a, b, c, d, e, f = [self.one_loop.create_future()
                             for i in range(6)]
         fut = asyncio.gather(a, b, c, d, e, f, return_exceptions=True)
         cb = test_utils.MockCallback()
@@ -3094,7 +3094,7 @@ def coro(s):
     def test_cancellation_broadcast(self):
         # Cancelling outer() cancels all children.
         proof = 0
-        waiter = asyncio.Future(loop=self.one_loop)
+        waiter = self.one_loop.create_future()
 
         async def inner():
             nonlocal proof
@@ -3130,8 +3130,8 @@ def test_exception_marking(self):
             await f
             raise RuntimeError('should not be ignored')
 
-        a = asyncio.Future(loop=self.one_loop)
-        b = asyncio.Future(loop=self.one_loop)
+        a = self.one_loop.create_future()
+        b = self.one_loop.create_future()
 
         async def outer():
             await asyncio.gather(inner(a), inner(b), loop=self.one_loop)
diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py
index 1daa870a7b27..02ce7e89da09 100644
--- a/Lib/test/test_asyncio/test_unix_events.py
+++ b/Lib/test/test_asyncio/test_unix_events.py
@@ -669,7 +669,7 @@ def read_pipe_transport(self, waiter=None):
         return transport
 
     def test_ctor(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         tr = self.read_pipe_transport(waiter=waiter)
         self.loop.run_until_complete(waiter)
 
@@ -819,7 +819,7 @@ def write_pipe_transport(self, waiter=None):
         return transport
 
     def test_ctor(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         tr = self.write_pipe_transport(waiter=waiter)
         self.loop.run_until_complete(waiter)
 


From webhook-mailer at python.org  Wed Sep 11 09:08:45 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Wed, 11 Sep 2019 13:08:45 -0000
Subject: [Python-checkins] bpo-34519: Add additional aliases for HP Roman 8
 (GH-8956)
Message-ID: <mailman.545.1568207326.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a828514cc3715ebbc700fefa41c0ef044cbd6741
commit: a828514cc3715ebbc700fefa41c0ef044cbd6741
branch: master
author: Michael Osipov <michael.osipov at siemens.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-11T14:08:41+01:00
summary:

bpo-34519: Add additional aliases for HP Roman 8 (GH-8956)

* bpo-34519: Add additional aliases for HP Roman 8

HP Roman 8 is known under mode aliases than listed in aliases.py.

Patch by Michael Osipov.

files:
A Misc/NEWS.d/next/Library/2018-08-27-15-44-50.bpo-34519.cPlH1h.rst
M Lib/encodings/aliases.py

diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py
index 8b621add1b1b..d85afd6d5cf7 100644
--- a/Lib/encodings/aliases.py
+++ b/Lib/encodings/aliases.py
@@ -266,6 +266,8 @@
     'roman8'             : 'hp_roman8',
     'r8'                 : 'hp_roman8',
     'csHPRoman8'         : 'hp_roman8',
+    'cp1051'             : 'hp_roman8',
+    'ibm1051'            : 'hp_roman8',
 
     # hz codec
     'hzgb'               : 'hz',
diff --git a/Misc/NEWS.d/next/Library/2018-08-27-15-44-50.bpo-34519.cPlH1h.rst b/Misc/NEWS.d/next/Library/2018-08-27-15-44-50.bpo-34519.cPlH1h.rst
new file mode 100644
index 000000000000..e17e2dc0fdcc
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-08-27-15-44-50.bpo-34519.cPlH1h.rst
@@ -0,0 +1 @@
+Add additional aliases for HP Roman 8. Patch by Michael Osipov.


From webhook-mailer at python.org  Wed Sep 11 09:14:00 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Wed, 11 Sep 2019 13:14:00 -0000
Subject: [Python-checkins] bpo-35649: update http client example (GH-11441)
 (GH-15931)
Message-ID: <mailman.546.1568207641.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0fd8c0560b2099d2c976b17cf01cb596badc1ec6
commit: 0fd8c0560b2099d2c976b17cf01cb596badc1ec6
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-11T15:13:55+02:00
summary:

bpo-35649: update http client example (GH-11441) (GH-15931)

(cherry picked from commit 62cf6981425c6a6b136c5e2abef853364f535e9d)

Co-authored-by: Ashwin Ramaswami <aramaswamis at gmail.com>

files:
M Doc/library/http.client.rst

diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst
index 3ebeb10aee9a..bc73c7a9bdf7 100644
--- a/Doc/library/http.client.rst
+++ b/Doc/library/http.client.rst
@@ -497,8 +497,11 @@ Here is an example session that uses the ``GET`` method::
    >>> # The following example demonstrates reading data in chunks.
    >>> conn.request("GET", "/")
    >>> r1 = conn.getresponse()
-   >>> while not r1.closed:
-   ...     print(r1.read(200))  # 200 bytes
+   >>> while True:
+   ...     chunk = r1.read(200)  # 200 bytes
+   ...     if not chunk:
+   ...          break
+   ...     print(repr(chunk))
    b'<!doctype html>\n<!--[if"...
    ...
    >>> # Example of an invalid request


From webhook-mailer at python.org  Wed Sep 11 09:16:15 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 13:16:15 -0000
Subject: [Python-checkins] bpo-33459: Fix "tuple displays" term in
 Expressions.rst (GH-6760)
Message-ID: <mailman.547.1568207776.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/dc269971091710563a0d730a0d4b084901826c15
commit: dc269971091710563a0d730a0d4b084901826c15
branch: master
author: Andre Delfino <adelfino at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T06:16:11-07:00
summary:

bpo-33459: Fix "tuple displays" term in Expressions.rst (GH-6760)



https://bugs.python.org/issue33459



Automerge-Triggered-By: @matrixise

files:
M Doc/reference/expressions.rst

diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst
index ee13c5f4c6fc..73a2f271ca54 100644
--- a/Doc/reference/expressions.rst
+++ b/Doc/reference/expressions.rst
@@ -148,9 +148,8 @@ immutable, the same rules as for literals apply (i.e., two occurrences of the em
 tuple may or may not yield the same object).
 
 .. index::
-   single: comma; tuple display
-   pair: tuple; display
-   single: , (comma); tuple display
+   single: comma
+   single: , (comma)
 
 Note that tuples are not formed by the parentheses, but rather by use of the
 comma operator.  The exception is the empty tuple, for which parentheses *are*
@@ -1822,7 +1821,8 @@ precedence and have a left-to-right chaining feature as described in the
 | ``x[index]``, ``x[index:index]``,             | Subscription, slicing,              |
 | ``x(arguments...)``, ``x.attribute``          | call, attribute reference           |
 +-----------------------------------------------+-------------------------------------+
-| ``(expressions...)``,                         | Binding or tuple display,           |
+| ``(expressions...)``,                         | Binding or parenthesized            |
+|                                               | expression,                         |
 | ``[expressions...]``,                         | list display,                       |
 | ``{key: value...}``,                          | dictionary display,                 |
 | ``{expressions...}``                          | set display                         |


From webhook-mailer at python.org  Wed Sep 11 09:17:57 2019
From: webhook-mailer at python.org (Brett Cannon)
Date: Wed, 11 Sep 2019 13:17:57 -0000
Subject: [Python-checkins] Turn off AppVeyor CI (GH-15929)
Message-ID: <mailman.548.1568207878.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6f55b0394bce8e64a59c61f465fec8079a00e55c
commit: 6f55b0394bce8e64a59c61f465fec8079a00e55c
branch: master
author: Zachary Ware <zachary.ware at gmail.com>
committer: Brett Cannon <54418+brettcannon at users.noreply.github.com>
date: 2019-09-11T14:17:53+01:00
summary:

Turn off AppVeyor CI (GH-15929)

files:
D .github/appveyor.yml

diff --git a/.github/appveyor.yml b/.github/appveyor.yml
deleted file mode 100644
index 8556cf8437e5..000000000000
--- a/.github/appveyor.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-version: 3.9build{build}
-clone_depth: 5
-branches:
-  only:
-    - master
-    - /\d\.\d/
-    - buildbot-custom
-cache:
-  - externals -> PCbuild
-before_build:
-  - ps: |+
-      if ($env:APPVEYOR_RE_BUILD) {
-        echo 'Doing full build due to re-build request.'
-      } elseif (!$env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT) {
-        echo 'Not a PR, doing full build.'
-      } else {
-        git fetch -q origin +refs/heads/$env:APPVEYOR_REPO_BRANCH
-        $mergebase = git merge-base HEAD FETCH_HEAD
-        $changes = git diff --name-only HEAD $mergebase | grep -vE '(\.rst$)|(^Doc)|(^Misc)'
-        If (!$changes) {
-          echo 'Only docs were updated, stopping build process.'
-          Exit-AppveyorBuild
-        } else {
-          echo 'Doing full build due to non-doc changes in these files:'
-          echo $changes
-        }
-      }
-
-
-build_script:
-  - cmd: PCbuild\build.bat -e -p x64
-  - cmd: PCbuild\amd64\python.exe -m test.pythoninfo
-test_script:
-  - cmd: PCbuild\rt.bat -x64 -q -uall -u-cpu -u-largefile -rwW --slowest --timeout=1200 --fail-env-changed -j0
-environment:
-  HOST_PYTHON: C:\Python36\python.exe
-image:
-  - Visual Studio 2017


From webhook-mailer at python.org  Wed Sep 11 09:21:08 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 13:21:08 -0000
Subject: [Python-checkins] bpo-33944: note about the intended use of code in
 .pth files (GH-10131)
Message-ID: <mailman.549.1568208069.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f9b5840fb4497a9e2ba2c1f01ad0dafba04c8496
commit: f9b5840fb4497a9e2ba2c1f01ad0dafba04c8496
branch: master
author: native-api <ivan_pozdeev at mail.ru>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T06:21:04-07:00
summary:

bpo-33944: note about the intended use of code in .pth files (GH-10131)



https://bugs.python.org/issue33944

files:
A Misc/NEWS.d/next/Documentation/2018-10-26-18-10-29.bpo-33944.V1YeOA.rst
M Doc/library/site.rst

diff --git a/Doc/library/site.rst b/Doc/library/site.rst
index 9e4d402c1f5e..e1ca160c107b 100644
--- a/Doc/library/site.rst
+++ b/Doc/library/site.rst
@@ -61,6 +61,19 @@ directory rather than a file.  No item is added to ``sys.path`` more than
 once.  Blank lines and lines beginning with ``#`` are skipped.  Lines starting
 with ``import`` (followed by space or tab) are executed.
 
+.. note::
+
+   An executable line in a :file:`.pth` file is run at every Python startup,
+   regardless of whether a particular module is actually going to be used.
+   Its impact should thus be kept to a minimum.
+   The primary intended purpose of executable lines is to make the
+   corresponding module(s) importable
+   (load 3rd-party import hooks, adjust :envvar:`PATH` etc).
+   Any other initialization is supposed to be done upon a module's
+   actual import, if and when it happens.
+   Limiting a code chunk to a single line is a deliberate measure
+   to discourage putting anything more complex here.
+
 .. index::
    single: package
    triple: path; configuration; file
diff --git a/Misc/NEWS.d/next/Documentation/2018-10-26-18-10-29.bpo-33944.V1YeOA.rst b/Misc/NEWS.d/next/Documentation/2018-10-26-18-10-29.bpo-33944.V1YeOA.rst
new file mode 100644
index 000000000000..29bec88c9649
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2018-10-26-18-10-29.bpo-33944.V1YeOA.rst
@@ -0,0 +1 @@
+Added a note about the intended use of code in .pth files.


From webhook-mailer at python.org  Wed Sep 11 09:22:12 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Wed, 11 Sep 2019 13:22:12 -0000
Subject: [Python-checkins] closes bpo-36711: Remove duplicate method in
 Lib/email/feedparser.py. (GH-14801)
Message-ID: <mailman.550.1568208132.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b239ab9107cd5bcbfbfa3b46290191298718ced3
commit: b239ab9107cd5bcbfbfa3b46290191298718ced3
branch: 2.7
author: Zackery Spytz <zspytz at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-11T14:22:08+01:00
summary:

closes bpo-36711: Remove duplicate method in Lib/email/feedparser.py. (GH-14801)

files:
M Lib/email/feedparser.py

diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py
index 8031ca666eef..298fe79df8d9 100644
--- a/Lib/email/feedparser.py
+++ b/Lib/email/feedparser.py
@@ -117,26 +117,6 @@ def push(self, data):
             self._partial = [parts.pop()]
         self.pushlines(parts)
 
-    def pushlines(self, lines):
-        # Crack into lines, but preserve the newlines on the end of each
-        parts = NLCRE_crack.split(data)
-        # The *ahem* interesting behaviour of re.split when supplied grouping
-        # parentheses is that the last element of the resulting list is the
-        # data after the final RE.  In the case of a NL/CR terminated string,
-        # this is the empty string.
-        self._partial = parts.pop()
-        #GAN 29Mar09  bugs 1555570, 1721862  Confusion at 8K boundary ending with \r:
-        # is there a \n to follow later?
-        if not self._partial and parts and parts[-1].endswith('\r'):
-            self._partial = parts.pop(-2)+parts.pop()
-        # parts is a list of strings, alternating between the line contents
-        # and the eol character(s).  Gather up a list of lines after
-        # re-attaching the newlines.
-        lines = []
-        for i in range(len(parts) // 2):
-            lines.append(parts[i*2] + parts[i*2+1])
-        self.pushlines(lines)
-
     def pushlines(self, lines):
         # Reverse and insert at the front of the lines.
         self._lines[:0] = lines[::-1]


From webhook-mailer at python.org  Wed Sep 11 09:26:11 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Wed, 11 Sep 2019 13:26:11 -0000
Subject: [Python-checkins] closes bpo-36712: Fix duplicate method in
 Lib/email/test/test_email_renamed.py. (GH-14800)
Message-ID: <mailman.551.1568208373.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f9db011c323556ec68761263c6b91b2f75ca95ba
commit: f9db011c323556ec68761263c6b91b2f75ca95ba
branch: 2.7
author: Zackery Spytz <zspytz at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-11T14:26:07+01:00
summary:

closes bpo-36712: Fix duplicate method in Lib/email/test/test_email_renamed.py. (GH-14800)

files:
M Lib/email/test/test_email_renamed.py

diff --git a/Lib/email/test/test_email_renamed.py b/Lib/email/test/test_email_renamed.py
index 5a41701271c5..206baaf59a6d 100644
--- a/Lib/email/test/test_email_renamed.py
+++ b/Lib/email/test/test_email_renamed.py
@@ -513,11 +513,6 @@ def test_encode_empty_payload(self):
         msg.set_charset('us-ascii')
         eq(msg['content-transfer-encoding'], '7bit')
 
-    def test_default_cte(self):
-        eq = self.assertEqual
-        msg = MIMEText('hello world')
-        eq(msg['content-transfer-encoding'], '7bit')
-
     def test_default_cte(self):
         eq = self.assertEqual
         # With no explicit _charset its us-ascii, and all are 7-bit


From webhook-mailer at python.org  Wed Sep 11 09:26:53 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Wed, 11 Sep 2019 13:26:53 -0000
Subject: [Python-checkins] bpo-31163: Added return values to pathlib.Path
 instance's rename and replace methods. (GH-13582)
Message-ID: <mailman.552.1568208414.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/088a09af4bdeff52b9dedeb7acd1e82069f37d98
commit: 088a09af4bdeff52b9dedeb7acd1e82069f37d98
branch: master
author: hui shang <shangdahao at gmail.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-11T14:26:49+01:00
summary:

bpo-31163: Added return values to pathlib.Path instance's rename and replace methods. (GH-13582)

* bpo-31163: Added return values to pathlib.Path instance's rename and replace methods.

files:
A Misc/NEWS.d/next/Library/2019-05-26-16-34-53.bpo-31163.21A802.rst
M Doc/library/pathlib.rst
M Lib/pathlib.py
M Lib/test/test_pathlib.py

diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst
index 33fac5f69725..d33e2d0c89b1 100644
--- a/Doc/library/pathlib.rst
+++ b/Doc/library/pathlib.rst
@@ -946,23 +946,32 @@ call fails (for example because the path doesn't exist).
 
 .. method:: Path.rename(target)
 
-   Rename this file or directory to the given *target*.  On Unix, if
-   *target* exists and is a file, it will be replaced silently if the user
-   has permission.  *target* can be either a string or another path object::
+   Rename this file or directory to the given *target*, and return a new Path
+   instance pointing to *target*.  On Unix, if *target* exists and is a file,
+   it will be replaced silently if the user has permission.  *target* can be
+   either a string or another path object::
 
       >>> p = Path('foo')
       >>> p.open('w').write('some text')
       9
       >>> target = Path('bar')
       >>> p.rename(target)
+      PosixPath('bar')
       >>> target.open().read()
       'some text'
 
+   .. versionchanged:: 3.8
+      Added return value, return the new Path instance.
+
 
 .. method:: Path.replace(target)
 
-   Rename this file or directory to the given *target*.  If *target* points
-   to an existing file or directory, it will be unconditionally replaced.
+   Rename this file or directory to the given *target*, and return a new Path
+   instance pointing to *target*.  If *target* points to an existing file or
+   directory, it will be unconditionally replaced.
+
+   .. versionchanged:: 3.8
+      Added return value, return the new Path instance.
 
 
 .. method:: Path.resolve(strict=False)
diff --git a/Lib/pathlib.py b/Lib/pathlib.py
index 80923c768268..7bd66c10cbdb 100644
--- a/Lib/pathlib.py
+++ b/Lib/pathlib.py
@@ -1341,20 +1341,24 @@ def link_to(self, target):
 
     def rename(self, target):
         """
-        Rename this path to the given path.
+        Rename this path to the given path,
+        and return a new Path instance pointing to the given path.
         """
         if self._closed:
             self._raise_closed()
         self._accessor.rename(self, target)
+        return self.__class__(target)
 
     def replace(self, target):
         """
         Rename this path to the given path, clobbering the existing
-        destination if it exists.
+        destination if it exists, and return a new Path instance
+        pointing to the given path.
         """
         if self._closed:
             self._raise_closed()
         self._accessor.replace(self, target)
+        return self.__class__(target)
 
     def symlink_to(self, target, target_is_directory=False):
         """
diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py
index 34c66f3137ff..f3b385fbf367 100644
--- a/Lib/test/test_pathlib.py
+++ b/Lib/test/test_pathlib.py
@@ -1767,12 +1767,14 @@ def test_rename(self):
         size = p.stat().st_size
         # Renaming to another path.
         q = P / 'dirA' / 'fileAA'
-        p.rename(q)
+        renamed_p = p.rename(q)
+        self.assertEqual(renamed_p, q)
         self.assertEqual(q.stat().st_size, size)
         self.assertFileNotFound(p.stat)
         # Renaming to a str of a relative path.
         r = rel_join('fileAAA')
-        q.rename(r)
+        renamed_q = q.rename(r)
+        self.assertEqual(renamed_q, self.cls(r))
         self.assertEqual(os.stat(r).st_size, size)
         self.assertFileNotFound(q.stat)
 
@@ -1782,12 +1784,14 @@ def test_replace(self):
         size = p.stat().st_size
         # Replacing a non-existing path.
         q = P / 'dirA' / 'fileAA'
-        p.replace(q)
+        replaced_p = p.replace(q)
+        self.assertEqual(replaced_p, q)
         self.assertEqual(q.stat().st_size, size)
         self.assertFileNotFound(p.stat)
         # Replacing another (existing) path.
         r = rel_join('dirB', 'fileB')
-        q.replace(r)
+        replaced_q = q.replace(r)
+        self.assertEqual(replaced_q, self.cls(r))
         self.assertEqual(os.stat(r).st_size, size)
         self.assertFileNotFound(q.stat)
 
diff --git a/Misc/NEWS.d/next/Library/2019-05-26-16-34-53.bpo-31163.21A802.rst b/Misc/NEWS.d/next/Library/2019-05-26-16-34-53.bpo-31163.21A802.rst
new file mode 100644
index 000000000000..6ec074612141
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-26-16-34-53.bpo-31163.21A802.rst
@@ -0,0 +1,2 @@
+pathlib.Path instance's rename and replace methods now return the new Path
+instance.


From webhook-mailer at python.org  Wed Sep 11 09:31:31 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 13:31:31 -0000
Subject: [Python-checkins] bpo-37326: Include libffi license when available
 (GH-15921)
Message-ID: <mailman.553.1568208692.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/da38ec1d4862ffb58faec41ed367719e81801766
commit: da38ec1d4862ffb58faec41ed367719e81801766
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T06:31:27-07:00
summary:

bpo-37326: Include libffi license when available (GH-15921)

(cherry picked from commit 8fcaffb048d2359a4c06ed5d8fbbf46d3ae1a472)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
M PCbuild/python.vcxproj

diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj
index e37bcfb0b6e1..2094420a8df3 100644
--- a/PCbuild/python.vcxproj
+++ b/PCbuild/python.vcxproj
@@ -150,12 +150,13 @@ $(_PGOPath)
       <LicenseFiles Include="$(PySourcePath)LICENSE;
                              $(PySourcePath)PC\crtlicense.txt;
                              $(bz2Dir)LICENSE;
-                             $(opensslOutDir)LICENSE;" />
+                             $(opensslOutDir)LICENSE;
+                             $(libffiDir)LICENSE;" />
       <LicenseFiles Include="$(tcltkDir)tcllicense.terms;
                              $(tcltkDir)tklicense.terms;
                              $(tcltkDir)tixlicense.terms" Condition="$(IncludeTkinter)" />
       <_LicenseFiles Include="@(LicenseFiles)">
-        <Content>$([System.IO.File]::ReadAllText(%(FullPath)))</Content>
+        <Content Condition="Exists(%(FullPath))">$([System.IO.File]::ReadAllText(%(FullPath)))</Content>
       </_LicenseFiles>
     </ItemGroup>
 


From webhook-mailer at python.org  Wed Sep 11 09:32:08 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Wed, 11 Sep 2019 13:32:08 -0000
Subject: [Python-checkins] bpo-37305: add MIME type for Web App Manifest
 (#14199)
Message-ID: <mailman.554.1568208730.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4b41745b3c65ef4807cd1108dce12d3e0c173f20
commit: 4b41745b3c65ef4807cd1108dce12d3e0c173f20
branch: master
author: Filip ? <filip.stamcar at hotmail.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-11T14:32:04+01:00
summary:

bpo-37305: add MIME type for Web App Manifest (#14199)

* bpo-37305: add MIME type for Web App Manifest

* bpo-37305: add news entry

* Restore indentation and sort by value

files:
A Misc/NEWS.d/next/Library/2019-06-18-13-59-55.bpo-37305.fGzWlP.rst
M Lib/mimetypes.py

diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py
index f38005c9d295..9fdd6ebd6348 100644
--- a/Lib/mimetypes.py
+++ b/Lib/mimetypes.py
@@ -415,6 +415,7 @@ def _default_mime_types():
         '.js'     : 'application/javascript',
         '.mjs'    : 'application/javascript',
         '.json'   : 'application/json',
+        '.webmanifest': 'application/manifest+json',
         '.doc'    : 'application/msword',
         '.dot'    : 'application/msword',
         '.wiz'    : 'application/msword',
diff --git a/Misc/NEWS.d/next/Library/2019-06-18-13-59-55.bpo-37305.fGzWlP.rst b/Misc/NEWS.d/next/Library/2019-06-18-13-59-55.bpo-37305.fGzWlP.rst
new file mode 100644
index 000000000000..5bd2cd4d30af
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-18-13-59-55.bpo-37305.fGzWlP.rst
@@ -0,0 +1,2 @@
+Add .webmanifest -> application/manifest+json to list of recognized file types
+and content type headers


From webhook-mailer at python.org  Wed Sep 11 09:42:09 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Wed, 11 Sep 2019 13:42:09 -0000
Subject: [Python-checkins] closes bpo-37252: Fix devpoll tests. (GH-14017)
Message-ID: <mailman.555.1568209330.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/95da826db9ed4acbc81f32296f14429c06bd2124
commit: 95da826db9ed4acbc81f32296f14429c06bd2124
branch: master
author: Jakub Kul?k <Kulikjak at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-11T14:42:04+01:00
summary:

closes bpo-37252: Fix devpoll tests. (GH-14017)

files:
A Misc/NEWS.d/next/Tests/2019-06-12-14-30-29.bpo-37252.4o-uLs.rst
M Lib/test/test_devpoll.py

diff --git a/Lib/test/test_devpoll.py b/Lib/test/test_devpoll.py
index c133c811b197..110c00686273 100644
--- a/Lib/test/test_devpoll.py
+++ b/Lib/test/test_devpoll.py
@@ -109,7 +109,7 @@ def test_close(self):
         # operations must fail with ValueError("I/O operation on closed ...")
         self.assertRaises(ValueError, devpoll.modify, fd, select.POLLIN)
         self.assertRaises(ValueError, devpoll.poll)
-        self.assertRaises(ValueError, devpoll.register, fd, fd, select.POLLIN)
+        self.assertRaises(ValueError, devpoll.register, fd, select.POLLIN)
         self.assertRaises(ValueError, devpoll.unregister, fd)
 
     def test_fd_non_inheritable(self):
@@ -122,9 +122,9 @@ def test_events_mask_overflow(self):
         w, r = os.pipe()
         pollster.register(w)
         # Issue #17919
-        self.assertRaises(OverflowError, pollster.register, 0, -1)
+        self.assertRaises(ValueError, pollster.register, 0, -1)
         self.assertRaises(OverflowError, pollster.register, 0, 1 << 64)
-        self.assertRaises(OverflowError, pollster.modify, 1, -1)
+        self.assertRaises(ValueError, pollster.modify, 1, -1)
         self.assertRaises(OverflowError, pollster.modify, 1, 1 << 64)
 
     @cpython_only
diff --git a/Misc/NEWS.d/next/Tests/2019-06-12-14-30-29.bpo-37252.4o-uLs.rst b/Misc/NEWS.d/next/Tests/2019-06-12-14-30-29.bpo-37252.4o-uLs.rst
new file mode 100644
index 000000000000..1bd7d2872f13
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-06-12-14-30-29.bpo-37252.4o-uLs.rst
@@ -0,0 +1,2 @@
+Fix assertions in ``test_close`` and ``test_events_mask_overflow`` devpoll
+tests.


From webhook-mailer at python.org  Wed Sep 11 09:57:36 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Wed, 11 Sep 2019 13:57:36 -0000
Subject: [Python-checkins] bpo-36919: make
 test_source_encoding.test_issue2301 implementation-independent (#13639)
Message-ID: <mailman.556.1568210258.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b6643dcfc26859f935e4b3a6a2a203e8ef5320e2
commit: b6643dcfc26859f935e4b3a6a2a203e8ef5320e2
branch: master
author: Pavel Koneski <pavel.koneski at gmail.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-11T14:57:32+01:00
summary:

bpo-36919: make test_source_encoding.test_issue2301 implementation-independent (#13639)

* bpo-36919: make test_issue2301 implementation-independent

files:
A Misc/NEWS.d/next/Tests/2019-05-28-15-41-34.bpo-36919.-vGt_m.rst
M Lib/test/test_source_encoding.py

diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py
index 38734009c008..a0bd741c36ac 100644
--- a/Lib/test/test_source_encoding.py
+++ b/Lib/test/test_source_encoding.py
@@ -31,7 +31,7 @@ def test_issue2301(self):
         try:
             compile(b"# coding: cp932\nprint '\x94\x4e'", "dummy", "exec")
         except SyntaxError as v:
-            self.assertEqual(v.text, "print '\u5e74'\n")
+            self.assertEqual(v.text.rstrip('\n'), "print '\u5e74'")
         else:
             self.fail()
 
diff --git a/Misc/NEWS.d/next/Tests/2019-05-28-15-41-34.bpo-36919.-vGt_m.rst b/Misc/NEWS.d/next/Tests/2019-05-28-15-41-34.bpo-36919.-vGt_m.rst
new file mode 100644
index 000000000000..9400bdda5da1
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-05-28-15-41-34.bpo-36919.-vGt_m.rst
@@ -0,0 +1,2 @@
+Make ``test_source_encoding.test_issue2301`` implementation independent. The
+test will work now for both CPython and IronPython.


From webhook-mailer at python.org  Wed Sep 11 09:58:46 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 13:58:46 -0000
Subject: [Python-checkins] bpo-37488 : Document a warning for
 datetime.utcnow() and utcfromtimestamp() (GH-15773)
Message-ID: <mailman.557.1568210327.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1a53c785e62e00bad87ae19466c3a32ebcebb915
commit: 1a53c785e62e00bad87ae19466c3a32ebcebb915
branch: master
author: Joannah Nanjekye <33177550+nanjekyejoannah at users.noreply.github.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T06:58:42-07:00
summary:

bpo-37488 : Document a warning for datetime.utcnow() and utcfromtimestamp() (GH-15773)



https://bugs.python.org/issue37488



Automerge-Triggered-By: @pganssle

files:
A Misc/NEWS.d/next/Library/2019-09-11-11-44-16.bpo-37488.S8CJUL.rst
M Doc/library/datetime.rst

diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index 882d4e16e1f6..b1e1b25691d8 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -862,13 +862,10 @@ Other constructors, all class methods:
    (for example, this may be possible on platforms supplying the C
    :c:func:`gettimeofday` function).
 
-   If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the
-   current date and time are converted to *tz*?s time zone. In this case the
-   result is equivalent to::
-
-     tz.fromutc(datetime.utcnow().replace(tzinfo=tz))
+   If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass,
+   and the current date and time are converted to *tz*?s time zone.
 
-   See also :meth:`today`, :meth:`utcnow`.
+   This function is preferred over :meth:`today` and :meth:`utcnow`.
 
 
 .. classmethod:: datetime.utcnow()
@@ -879,6 +876,14 @@ Other constructors, all class methods:
    :class:`.datetime` object. An aware current UTC datetime can be obtained by
    calling ``datetime.now(timezone.utc)``. See also :meth:`now`.
 
+   .. warning::
+
+      Because naive ``datetime`` objects are treated by many ``datetime`` methods
+      as local times, it is preferred to use aware datetimes to represent times
+      in UTC. As such, the recommended way to create an object representing the
+      current time in UTC  by calling ``datetime.now(timezone.utc)``.
+
+
 .. classmethod:: datetime.fromtimestamp(timestamp, tz=None)
 
    Return the local date and time corresponding to the POSIX timestamp, such as is
@@ -889,10 +894,6 @@ Other constructors, all class methods:
    If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the
    timestamp is converted to *tz*?s time zone.
 
-   In this case the result is equivalent to::
-
-     tz.fromutc(datetime.utcfromtimestamp(timestamp).replace(tzinfo=tz))
-
    :meth:`fromtimestamp` may raise :exc:`OverflowError`, if the timestamp is out of
    the range of values supported by the platform C :c:func:`localtime` or
    :c:func:`gmtime` functions, and :exc:`OSError` on :c:func:`localtime` or
@@ -901,7 +902,8 @@ Other constructors, all class methods:
    1970 through 2038. Note that on non-POSIX systems that include leap seconds in
    their notion of a timestamp, leap seconds are ignored by :meth:`fromtimestamp`,
    and then it's possible to have two timestamps differing by a second that yield
-   identical :class:`.datetime` objects. See also :meth:`utcfromtimestamp`.
+   identical :class:`.datetime` objects. This method is preferred over
+   :meth:`utcfromtimestamp`.
 
    .. versionchanged:: 3.3
       Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp
@@ -935,6 +937,14 @@ Other constructors, all class methods:
    except the latter formula always supports the full years range: between
    :const:`MINYEAR` and :const:`MAXYEAR` inclusive.
 
+   .. warning::
+
+      Because naive ``datetime`` objects are treated by many ``datetime`` methods
+      as local times, it is preferred to use aware datetimes to represent times
+      in UTC. As such, the recommended way to create an object representing a
+      specific timestamp in UTC  by calling
+      ``datetime.fromtimestamp(timestamp, tz=timezone.utc)``.
+
    .. versionchanged:: 3.3
       Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp
       is out of the range of values supported by the platform C
@@ -1322,6 +1332,14 @@ Instance methods:
    ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year
    boundary.
 
+   .. warning::
+
+      Because naive ``datetime`` objects are treated by many ``datetime`` methods
+      as local times, it is preferred to use aware datetimes to represent times
+      in UTC; as a result, using ``utcfromtimetuple`` may give misleading
+      results. If you have a naive ``datetime`` representing UTC, use
+      ``datetime.replace(tzinfo=timezone.utc)`` to make it aware, at which point
+      you can use :meth:`.datetime.timetuple`.
 
 .. method:: datetime.toordinal()
 
@@ -1500,7 +1518,7 @@ Examples of working with :class:`~datetime.datetime` objects:
 
 .. doctest::
 
-    >>> from datetime import datetime, date, time
+    >>> from datetime import datetime, date, time, timezone
 
     >>> # Using datetime.combine()
     >>> d = date(2005, 7, 14)
@@ -1508,11 +1526,11 @@ Examples of working with :class:`~datetime.datetime` objects:
     >>> datetime.combine(d, t)
     datetime.datetime(2005, 7, 14, 12, 30)
 
-    >>> # Using datetime.now() or datetime.utcnow()
+    >>> # Using datetime.now()
     >>> datetime.now()   # doctest: +SKIP
     datetime.datetime(2007, 12, 6, 16, 29, 43, 79043)   # GMT +1
-    >>> datetime.utcnow()   # doctest: +SKIP
-    datetime.datetime(2007, 12, 6, 15, 29, 43, 79060)
+    >>> datetime.now(timezone.utc)   # doctest: +SKIP
+    datetime.datetime(2007, 12, 6, 15, 29, 43, 79060, tzinfo=datetime.timezone.utc)
 
     >>> # Using datetime.strptime()
     >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M")
@@ -1616,7 +1634,7 @@ Usage of ``KabulTz`` from above::
    datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc)
    >>> dt2
    datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz())
-   >>> dt2.utctimetuple() == dt3.utctimetuple()
+   >>> dt2 == dt3
    True
 
 .. _datetime-time:
diff --git a/Misc/NEWS.d/next/Library/2019-09-11-11-44-16.bpo-37488.S8CJUL.rst b/Misc/NEWS.d/next/Library/2019-09-11-11-44-16.bpo-37488.S8CJUL.rst
new file mode 100644
index 000000000000..4f9041502a91
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-11-11-44-16.bpo-37488.S8CJUL.rst
@@ -0,0 +1 @@
+Add warning to :meth:`datetime.utctimetuple`,  :meth:`datetime.utcnow` and :meth:`datetime.utcfromtimestamp` .
\ No newline at end of file


From webhook-mailer at python.org  Wed Sep 11 09:59:41 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Wed, 11 Sep 2019 13:59:41 -0000
Subject: [Python-checkins] bpo-23460: Fix documentation for decimal string
 :g formatting (GH-11850)
Message-ID: <mailman.558.1568210383.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1660a61a105bcd62e2dfa77885959a8992e9f14e
commit: 1660a61a105bcd62e2dfa77885959a8992e9f14e
branch: master
author: Brennan D Baraban <34765317+bdbaraban at users.noreply.github.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-11T15:59:37+02:00
summary:

bpo-23460: Fix documentation for decimal string :g formatting (GH-11850)

files:
A Misc/NEWS.d/next/Documentation/2019-02-14-07-12-48.bpo-23460.Iqiqtm.rst
M Doc/library/string.rst

diff --git a/Doc/library/string.rst b/Doc/library/string.rst
index af8b9b358cc3..af52af0482c4 100644
--- a/Doc/library/string.rst
+++ b/Doc/library/string.rst
@@ -500,9 +500,10 @@ The available presentation types for floating point and decimal values are:
    |         |                                                          |
    |         | The precise rules are as follows: suppose that the       |
    |         | result formatted with presentation type ``'e'`` and      |
-   |         | precision ``p-1`` would have exponent ``exp``.  Then     |
-   |         | if ``-4 <= exp < p``, the number is formatted            |
-   |         | with presentation type ``'f'`` and precision             |
+   |         | precision ``p-1`` would have exponent ``exp``.  Then,    |
+   |         | if ``m <= exp < p``, where ``m`` is -4 for floats and -6 |
+   |         | for :class:`Decimals <decimal.Decimal>`, the number is   |
+   |         | formatted with presentation type ``'f'`` and precision   |
    |         | ``p-1-exp``.  Otherwise, the number is formatted         |
    |         | with presentation type ``'e'`` and precision ``p-1``.    |
    |         | In both cases insignificant trailing zeros are removed   |
diff --git a/Misc/NEWS.d/next/Documentation/2019-02-14-07-12-48.bpo-23460.Iqiqtm.rst b/Misc/NEWS.d/next/Documentation/2019-02-14-07-12-48.bpo-23460.Iqiqtm.rst
new file mode 100644
index 000000000000..4674d551d379
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-02-14-07-12-48.bpo-23460.Iqiqtm.rst
@@ -0,0 +1 @@
+The documentation for decimal string formatting using the `:g` specifier has been updated to reflect the correct exponential notation cutoff point. Original patch contributed by Tuomas Suutari.
\ No newline at end of file


From webhook-mailer at python.org  Wed Sep 11 10:09:57 2019
From: webhook-mailer at python.org (Stefan Behnel)
Date: Wed, 11 Sep 2019 14:09:57 -0000
Subject: [Python-checkins] bpo-33187: Document ElementInclude (XInclude)
 support in ElementTree (GH-8861)
Message-ID: <mailman.559.1568210998.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/97b817eae34b77be1ced382e15098a112f547848
commit: 97b817eae34b77be1ced382e15098a112f547848
branch: master
author: Anjali Bansal <anjali.mca17.du at gmail.com>
committer: Stefan Behnel <stefan_ml at behnel.de>
date: 2019-09-11T15:09:53+01:00
summary:

bpo-33187: Document ElementInclude (XInclude) support in ElementTree (GH-8861)

files:
M Doc/library/xml.etree.elementtree.rst
M Doc/tools/susp-ignored.csv

diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst
index 1b4e2c9b142e..abc15d44b2f6 100644
--- a/Doc/library/xml.etree.elementtree.rst
+++ b/Doc/library/xml.etree.elementtree.rst
@@ -738,6 +738,95 @@ Functions
    :class:`Element` instance and a dictionary.
 
 
+.. _elementtree-xinclude:
+
+XInclude support
+----------------
+
+This module provides limited support for
+`XInclude directives <https://www.w3.org/TR/xinclude/>`_, via the :mod:`xml.etree.ElementInclude` helper module.  This module can be used to insert subtrees and text strings into element trees, based on information in the tree.
+
+Example
+^^^^^^^
+
+Here's an example that demonstrates use of the XInclude module. To include an XML document in the current document, use the ``{http://www.w3.org/2001/XInclude}include`` element and set the **parse** attribute to ``"xml"``, and use the **href** attribute to specify the document to include.
+
+.. code-block:: xml
+
+    <?xml version="1.0"?>
+    <document xmlns:xi="http://www.w3.org/2001/XInclude">
+      <xi:include href="source.xml" parse="xml" />
+    </document>
+
+By default, the **href** attribute is treated as a file name. You can use custom loaders to override this behaviour. Also note that the standard helper does not support XPointer syntax.
+
+To process this file, load it as usual, and pass the root element to the :mod:`xml.etree.ElementTree` module:
+
+.. code-block:: python
+
+   from xml.etree import ElementTree, ElementInclude
+
+   tree = ElementTree.parse("document.xml")
+   root = tree.getroot()
+
+   ElementInclude.include(root)
+
+The ElementInclude module replaces the ``{http://www.w3.org/2001/XInclude}include`` element with the root element from the **source.xml** document. The result might look something like this:
+
+.. code-block:: xml
+
+    <document xmlns:xi="http://www.w3.org/2001/XInclude">
+      <para>This is a paragraph.</para>
+    </document>
+
+If the **parse** attribute is omitted, it defaults to "xml". The href attribute is required.
+
+To include a text document, use the ``{http://www.w3.org/2001/XInclude}include`` element, and set the **parse** attribute to "text":
+
+.. code-block:: xml
+
+    <?xml version="1.0"?>
+    <document xmlns:xi="http://www.w3.org/2001/XInclude">
+      Copyright (c) <xi:include href="year.txt" parse="text" />.
+    </document>
+
+The result might look something like:
+
+.. code-block:: xml
+
+    <document xmlns:xi="http://www.w3.org/2001/XInclude">
+      Copyright (c) 2003.
+    </document>
+
+Reference
+---------
+
+.. _elementinclude-functions:
+
+Functions
+^^^^^^^^^
+
+.. function:: xml.etree.ElementInclude.default_loader( href, parse, encoding=None)
+
+   Default loader. This default loader reads an included resource from disk.  *href* is a URL.
+   *parse* is for parse mode either "xml" or "text".  *encoding*
+   is an optional text encoding.  If not given, encoding is ``utf-8``.  Returns the
+   expanded resource.  If the parse mode is ``"xml"``, this is an ElementTree
+   instance.  If the parse mode is "text", this is a Unicode string.  If the
+   loader fails, it can return None or raise an exception.
+
+
+.. function:: xml.etree.ElementInclude.include( elem, loader=None)
+
+   This function expands XInclude directives.  *elem* is the root element.  *loader* is
+   an optional resource loader.  If omitted, it defaults to :func:`default_loader`.
+   If given, it should be a callable that implements the same interface as
+   :func:`default_loader`.  Returns the expanded resource.  If the parse mode is
+   ``"xml"``, this is an ElementTree instance.  If the parse mode is "text",
+   this is a Unicode string.  If the loader fails, it can return None or
+   raise an exception.
+
+
 .. _elementtree-element-objects:
 
 Element Objects
diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv
index bc0dcb60b65b..a7408620aeb4 100644
--- a/Doc/tools/susp-ignored.csv
+++ b/Doc/tools/susp-ignored.csv
@@ -333,6 +333,9 @@ library/xml.etree.elementtree,,:character,<fictional:character>Commander Clement
 library/xml.etree.elementtree,,:actor,"for actor in root.findall('real_person:actor', ns):"
 library/xml.etree.elementtree,,:name,"name = actor.find('real_person:name', ns)"
 library/xml.etree.elementtree,,:character,"for char in actor.findall('role:character', ns):"
+library/xml.etree.elementtree,,:xi,<document xmlns:xi="http://www.w3.org/2001/XInclude">
+library/xml.etree.elementtree,,:include,  <xi:include href="source.xml" parse="xml" />
+library/xml.etree.elementtree,,:include,  Copyright (c) <xi:include href="year.txt" parse="text" />.
 library/zipapp,,:main,"$ python -m zipapp myapp -m ""myapp:main"""
 library/zipapp,,:fn,"pkg.mod:fn"
 library/zipapp,,:callable,"pkg.module:callable"


From webhook-mailer at python.org  Wed Sep 11 10:12:55 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 14:12:55 -0000
Subject: [Python-checkins] bpo-37651: Document CancelledError is now a
 subclass of BaseException (GH-15950)
Message-ID: <mailman.560.1568211177.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7b69069e9aa0047a0dbe8af1a67aa2b355dc68d8
commit: 7b69069e9aa0047a0dbe8af1a67aa2b355dc68d8
branch: master
author: Xtreak <tir.karthi at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T07:12:51-07:00
summary:

bpo-37651: Document CancelledError is now a subclass of BaseException (GH-15950)



https://bugs.python.org/issue37651



Automerge-Triggered-By: @1st1

files:
M Doc/library/asyncio-exceptions.rst

diff --git a/Doc/library/asyncio-exceptions.rst b/Doc/library/asyncio-exceptions.rst
index e49577a203e8..b1e6ecfda15e 100644
--- a/Doc/library/asyncio-exceptions.rst
+++ b/Doc/library/asyncio-exceptions.rst
@@ -25,26 +25,9 @@ Exceptions
    when asyncio Tasks are cancelled.  In almost all situations the
    exception must be re-raised.
 
-   .. important::
-
-      This exception is a subclass of :exc:`Exception`, so it can be
-      accidentally suppressed by an overly broad ``try..except`` block::
-
-        try:
-            await operation
-        except Exception:
-            # The cancellation is broken because the *except* block
-            # suppresses the CancelledError exception.
-            log.log('an error has occurred')
-
-      Instead, the following pattern should be used::
+   .. versionchanged:: 3.8
 
-        try:
-            await operation
-        except asyncio.CancelledError:
-            raise
-        except Exception:
-            log.log('an error has occurred')
+      :exc:`CancelledError` is now a subclass of :class:`BaseException`.
 
 
 .. exception:: InvalidStateError


From webhook-mailer at python.org  Wed Sep 11 10:12:58 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Wed, 11 Sep 2019 14:12:58 -0000
Subject: [Python-checkins] bpo-31163: Added return values to pathlib.Path
 instance's rename and replace methods. (GH-13582) (GH-15944)
Message-ID: <mailman.561.1568211180.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/cbd7b2a399a8ff2ed9994c380b07ef598892b6b1
commit: cbd7b2a399a8ff2ed9994c380b07ef598892b6b1
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-11T15:12:54+01:00
summary:

bpo-31163: Added return values to pathlib.Path instance's rename and replace methods. (GH-13582) (GH-15944)

* bpo-31163: Added return values to pathlib.Path instance's rename and replace methods.
(cherry picked from commit 088a09af4bdeff52b9dedeb7acd1e82069f37d98)

Co-authored-by: hui shang <shangdahao at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-05-26-16-34-53.bpo-31163.21A802.rst
M Doc/library/pathlib.rst
M Lib/pathlib.py
M Lib/test/test_pathlib.py

diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst
index 166de8de1f06..ad8f6011ad02 100644
--- a/Doc/library/pathlib.rst
+++ b/Doc/library/pathlib.rst
@@ -933,23 +933,32 @@ call fails (for example because the path doesn't exist).
 
 .. method:: Path.rename(target)
 
-   Rename this file or directory to the given *target*.  On Unix, if
-   *target* exists and is a file, it will be replaced silently if the user
-   has permission.  *target* can be either a string or another path object::
+   Rename this file or directory to the given *target*, and return a new Path
+   instance pointing to *target*.  On Unix, if *target* exists and is a file,
+   it will be replaced silently if the user has permission.  *target* can be
+   either a string or another path object::
 
       >>> p = Path('foo')
       >>> p.open('w').write('some text')
       9
       >>> target = Path('bar')
       >>> p.rename(target)
+      PosixPath('bar')
       >>> target.open().read()
       'some text'
 
+   .. versionchanged:: 3.8
+      Added return value, return the new Path instance.
+
 
 .. method:: Path.replace(target)
 
-   Rename this file or directory to the given *target*.  If *target* points
-   to an existing file or directory, it will be unconditionally replaced.
+   Rename this file or directory to the given *target*, and return a new Path
+   instance pointing to *target*.  If *target* points to an existing file or
+   directory, it will be unconditionally replaced.
+
+   .. versionchanged:: 3.8
+      Added return value, return the new Path instance.
 
 
 .. method:: Path.resolve(strict=False)
diff --git a/Lib/pathlib.py b/Lib/pathlib.py
index 6355ae864108..91ce4a125740 100644
--- a/Lib/pathlib.py
+++ b/Lib/pathlib.py
@@ -1332,20 +1332,24 @@ def link_to(self, target):
 
     def rename(self, target):
         """
-        Rename this path to the given path.
+        Rename this path to the given path,
+        and return a new Path instance pointing to the given path.
         """
         if self._closed:
             self._raise_closed()
         self._accessor.rename(self, target)
+        return self.__class__(target)
 
     def replace(self, target):
         """
         Rename this path to the given path, clobbering the existing
-        destination if it exists.
+        destination if it exists, and return a new Path instance
+        pointing to the given path.
         """
         if self._closed:
             self._raise_closed()
         self._accessor.replace(self, target)
+        return self.__class__(target)
 
     def symlink_to(self, target, target_is_directory=False):
         """
diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py
index 027331f95911..ebc71ec219c8 100644
--- a/Lib/test/test_pathlib.py
+++ b/Lib/test/test_pathlib.py
@@ -1680,12 +1680,14 @@ def test_rename(self):
         size = p.stat().st_size
         # Renaming to another path.
         q = P / 'dirA' / 'fileAA'
-        p.rename(q)
+        renamed_p = p.rename(q)
+        self.assertEqual(renamed_p, q)
         self.assertEqual(q.stat().st_size, size)
         self.assertFileNotFound(p.stat)
         # Renaming to a str of a relative path.
         r = rel_join('fileAAA')
-        q.rename(r)
+        renamed_q = q.rename(r)
+        self.assertEqual(renamed_q, self.cls(r))
         self.assertEqual(os.stat(r).st_size, size)
         self.assertFileNotFound(q.stat)
 
@@ -1695,12 +1697,14 @@ def test_replace(self):
         size = p.stat().st_size
         # Replacing a non-existing path.
         q = P / 'dirA' / 'fileAA'
-        p.replace(q)
+        replaced_p = p.replace(q)
+        self.assertEqual(replaced_p, q)
         self.assertEqual(q.stat().st_size, size)
         self.assertFileNotFound(p.stat)
         # Replacing another (existing) path.
         r = rel_join('dirB', 'fileB')
-        q.replace(r)
+        replaced_q = q.replace(r)
+        self.assertEqual(replaced_q, self.cls(r))
         self.assertEqual(os.stat(r).st_size, size)
         self.assertFileNotFound(q.stat)
 
diff --git a/Misc/NEWS.d/next/Library/2019-05-26-16-34-53.bpo-31163.21A802.rst b/Misc/NEWS.d/next/Library/2019-05-26-16-34-53.bpo-31163.21A802.rst
new file mode 100644
index 000000000000..6ec074612141
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-26-16-34-53.bpo-31163.21A802.rst
@@ -0,0 +1,2 @@
+pathlib.Path instance's rename and replace methods now return the new Path
+instance.


From webhook-mailer at python.org  Wed Sep 11 10:13:35 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Wed, 11 Sep 2019 14:13:35 -0000
Subject: [Python-checkins] bpo-37305: add MIME type for Web App Manifest
 (GH-14199) (#15946)
Message-ID: <mailman.562.1568211216.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/35d0934040223f1360b2b343005fcc8ebbc65c1a
commit: 35d0934040223f1360b2b343005fcc8ebbc65c1a
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-11T15:13:31+01:00
summary:

bpo-37305: add MIME type for Web App Manifest (GH-14199) (#15946)

* bpo-37305: add MIME type for Web App Manifest

Co-authored-by: Filip ? <filip.stamcar at hotmail.com>

files:
A Misc/NEWS.d/next/Library/2019-06-18-13-59-55.bpo-37305.fGzWlP.rst
M Lib/mimetypes.py

diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py
index f38005c9d295..9fdd6ebd6348 100644
--- a/Lib/mimetypes.py
+++ b/Lib/mimetypes.py
@@ -415,6 +415,7 @@ def _default_mime_types():
         '.js'     : 'application/javascript',
         '.mjs'    : 'application/javascript',
         '.json'   : 'application/json',
+        '.webmanifest': 'application/manifest+json',
         '.doc'    : 'application/msword',
         '.dot'    : 'application/msword',
         '.wiz'    : 'application/msword',
diff --git a/Misc/NEWS.d/next/Library/2019-06-18-13-59-55.bpo-37305.fGzWlP.rst b/Misc/NEWS.d/next/Library/2019-06-18-13-59-55.bpo-37305.fGzWlP.rst
new file mode 100644
index 000000000000..5bd2cd4d30af
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-18-13-59-55.bpo-37305.fGzWlP.rst
@@ -0,0 +1,2 @@
+Add .webmanifest -> application/manifest+json to list of recognized file types
+and content type headers


From webhook-mailer at python.org  Wed Sep 11 10:13:58 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Wed, 11 Sep 2019 14:13:58 -0000
Subject: [Python-checkins] bpo-34519: Add additional aliases for HP Roman 8
 (GH-8956) (GH-15945)
Message-ID: <mailman.563.1568211239.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/fef5bdc645bc5d037505e3da2965dfabb73f58eb
commit: fef5bdc645bc5d037505e3da2965dfabb73f58eb
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-11T15:13:54+01:00
summary:

bpo-34519: Add additional aliases for HP Roman 8 (GH-8956) (GH-15945)

* bpo-34519: Add additional aliases for HP Roman 8

HP Roman 8 is known under mode aliases than listed in aliases.py.

Patch by Michael Osipov.
(cherry picked from commit a828514cc3715ebbc700fefa41c0ef044cbd6741)

Co-authored-by: Michael Osipov <michael.osipov at siemens.com>

files:
A Misc/NEWS.d/next/Library/2018-08-27-15-44-50.bpo-34519.cPlH1h.rst
M Lib/encodings/aliases.py

diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py
index 5ef40a3438b9..2444f9f1795e 100644
--- a/Lib/encodings/aliases.py
+++ b/Lib/encodings/aliases.py
@@ -266,6 +266,8 @@
     'roman8'             : 'hp_roman8',
     'r8'                 : 'hp_roman8',
     'csHPRoman8'         : 'hp_roman8',
+    'cp1051'             : 'hp_roman8',
+    'ibm1051'            : 'hp_roman8',
 
     # hz codec
     'hzgb'               : 'hz',
diff --git a/Misc/NEWS.d/next/Library/2018-08-27-15-44-50.bpo-34519.cPlH1h.rst b/Misc/NEWS.d/next/Library/2018-08-27-15-44-50.bpo-34519.cPlH1h.rst
new file mode 100644
index 000000000000..e17e2dc0fdcc
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-08-27-15-44-50.bpo-34519.cPlH1h.rst
@@ -0,0 +1 @@
+Add additional aliases for HP Roman 8. Patch by Michael Osipov.


From webhook-mailer at python.org  Wed Sep 11 10:17:36 2019
From: webhook-mailer at python.org (Paul Ganssle)
Date: Wed, 11 Sep 2019 14:17:36 -0000
Subject: [Python-checkins] bpo-38096: Clean up the "struct sequence" /
 "named tuple" docs (GH-15895)
Message-ID: <mailman.564.1568211458.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7117074410118086938044c7a4ef6846ec1662b2
commit: 7117074410118086938044c7a4ef6846ec1662b2
branch: master
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: Paul Ganssle <paul at ganssle.io>
date: 2019-09-11T15:17:32+01:00
summary:

bpo-38096: Clean up the "struct sequence" / "named tuple" docs (GH-15895)

* bpo-38096: Clean up the "struct sequence" / "named tuple" docs

* Fix remaining occurrences of "struct sequence"

* Repair a user visible docstring

files:
M Doc/glossary.rst
M Doc/library/sys.rst
M Doc/whatsnew/3.3.rst
M Doc/whatsnew/3.4.rst
M Objects/floatobject.c
M Objects/longobject.c
M Python/sysmodule.c
M Python/thread.c

diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 0f2a3a1fdf05..84d0fcab9d75 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -739,17 +739,28 @@ Glossary
       also :term:`immutable`.
 
    named tuple
-      Any tuple-like class whose indexable elements are also accessible using
-      named attributes (for example, :func:`time.localtime` returns a
-      tuple-like object where the *year* is accessible either with an
-      index such as ``t[0]`` or with a named attribute like ``t.tm_year``).
-
-      A named tuple can be a built-in type such as :class:`time.struct_time`,
-      or it can be created with a regular class definition.  A full featured
-      named tuple can also be created with the factory function
-      :func:`collections.namedtuple`.  The latter approach automatically
-      provides extra features such as a self-documenting representation like
-      ``Employee(name='jones', title='programmer')``.
+      The term "named tuple" applies to any type or class that inherits from
+      tuple and whose indexable elements are also accessible using named
+      attributes.  The type or class may have other features as well.
+
+      Several built-in types are named tuples, including the values returned
+      by :func:`time.localtime` and :func:`os.stat`.  Another example is
+      :data:`sys.float_info`::
+
+           >>> sys.float_info[1]                   # indexed access
+           1024
+           >>> sys.float_info.max_exp              # named field access
+           1024
+           >>> isinstance(sys.float_info, tuple)   # kind of tuple
+           True
+
+      Some named tuples are built-in types (such as the above examples).
+      Alternatively, a named tuple can be created from a regular class
+      definition that inherits from :class:`tuple` and that defines named
+      fields.  Such as class can be written by hand or it can be created with
+      the factory function :func:`collections.namedtuple`.  The latter
+      technique also adds some extra methods that may not be found in
+      hand-written or built-in named tuples.
 
    namespace
       The place where a variable is stored.  Namespaces are implemented as
@@ -1032,14 +1043,6 @@ Glossary
       an :term:`expression` or one of several constructs with a keyword, such
       as :keyword:`if`, :keyword:`while` or :keyword:`for`.
 
-   struct sequence
-      A tuple with named elements. Struct sequences expose an interface similar
-      to :term:`named tuple` in that elements can be accessed either by
-      index or as an attribute. However, they do not have any of the named tuple
-      methods like :meth:`~collections.somenamedtuple._make` or
-      :meth:`~collections.somenamedtuple._asdict`. Examples of struct sequences
-      include :data:`sys.float_info` and the return value of :func:`os.stat`.
-
    text encoding
       A codec which encodes Unicode strings to bytes.
 
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 01df026a6e8a..a5528f72e4b7 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -408,7 +408,7 @@ always available.
 
 .. data:: flags
 
-   The :term:`struct sequence` *flags* exposes the status of command line
+   The :term:`named tuple` *flags* exposes the status of command line
    flags. The attributes are read only.
 
    ============================= =============================
@@ -450,7 +450,7 @@ always available.
 
 .. data:: float_info
 
-   A :term:`struct sequence` holding information about the float type. It
+   A :term:`named tuple` holding information about the float type. It
    contains low level information about the precision and internal
    representation.  The values correspond to the various floating-point
    constants defined in the standard header file :file:`float.h` for the 'C'
@@ -782,7 +782,7 @@ always available.
 
 .. data:: hash_info
 
-   A :term:`struct sequence` giving parameters of the numeric hash
+   A :term:`named tuple` giving parameters of the numeric hash
    implementation.  For more details about hashing of numeric types, see
    :ref:`numeric-hash`.
 
@@ -830,7 +830,7 @@ always available.
 
    This is called ``hexversion`` since it only really looks meaningful when viewed
    as the result of passing it to the built-in :func:`hex` function.  The
-   :term:`struct sequence`  :data:`sys.version_info` may be used for a more
+   :term:`named tuple`  :data:`sys.version_info` may be used for a more
    human-friendly encoding of the same information.
 
    More details of ``hexversion`` can be found at :ref:`apiabiversion`.
@@ -882,7 +882,7 @@ always available.
 
 .. data:: int_info
 
-   A :term:`struct sequence` that holds information about Python's internal
+   A :term:`named tuple` that holds information about Python's internal
    representation of integers.  The attributes are read only.
 
    .. tabularcolumns:: |l|L|
@@ -1457,7 +1457,7 @@ always available.
 
 .. data:: thread_info
 
-   A :term:`struct sequence` holding information about the thread
+   A :term:`named tuple` holding information about the thread
    implementation.
 
    .. tabularcolumns:: |l|p{0.7\linewidth}|
diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst
index ea036840d671..f1a033c6dae6 100644
--- a/Doc/whatsnew/3.3.rst
+++ b/Doc/whatsnew/3.3.rst
@@ -2002,8 +2002,8 @@ platform-independent fashion.  (Contributed by Ross Lagerwall in
 sys
 ---
 
-The :mod:`sys` module has a new :data:`~sys.thread_info` :term:`struct
-sequence` holding information about the thread implementation
+The :mod:`sys` module has a new :data:`~sys.thread_info` :term:`named
+tuple` holding information about the thread implementation
 (:issue:`11223`).
 
 
diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst
index 845e327cabc8..822ba81be8a7 100644
--- a/Doc/whatsnew/3.4.rst
+++ b/Doc/whatsnew/3.4.rst
@@ -1849,7 +1849,7 @@ Python's default implementation to a SipHash implementation on platforms that
 have a 64 bit data type.  Any performance differences in comparison with the
 older FNV algorithm are trivial.
 
-The PEP adds additional fields to the :attr:`sys.hash_info` struct sequence to
+The PEP adds additional fields to the :attr:`sys.hash_info` named tuple to
 describe the hash algorithm in use by the currently executing binary.  Otherwise,
 the PEP does not alter any existing CPython APIs.
 
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
index 6643ff340ea5..20b7f5120bd4 100644
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -43,7 +43,7 @@ static PyTypeObject FloatInfoType;
 PyDoc_STRVAR(floatinfo__doc__,
 "sys.float_info\n\
 \n\
-A structseq holding information about the float type. It contains low level\n\
+A named tuple holding information about the float type. It contains low level\n\
 information about the precision and internal representation. Please study\n\
 your system's :file:`float.h` for more information.");
 
diff --git a/Objects/longobject.c b/Objects/longobject.c
index ed25e0bee329..6afec180a56e 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -5780,7 +5780,7 @@ static PyTypeObject Int_InfoType;
 PyDoc_STRVAR(int_info__doc__,
 "sys.int_info\n\
 \n\
-A struct sequence that holds information about Python's\n\
+A named tuple that holds information about Python's\n\
 internal representation of integers.  The attributes are read only.");
 
 static PyStructSequence_Field int_info_fields[] = {
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 8509aaf86693..476e154b1194 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -1160,7 +1160,7 @@ static PyTypeObject AsyncGenHooksType;
 PyDoc_STRVAR(asyncgen_hooks_doc,
 "asyncgen_hooks\n\
 \n\
-A struct sequence providing information about asynchronous\n\
+A named tuple providing information about asynchronous\n\
 generators hooks.  The attributes are read only.");
 
 static PyStructSequence_Field asyncgen_hooks_fields[] = {
@@ -1269,7 +1269,7 @@ static PyTypeObject Hash_InfoType;
 PyDoc_STRVAR(hash_info_doc,
 "hash_info\n\
 \n\
-A struct sequence providing parameters used for computing\n\
+A named tuple providing parameters used for computing\n\
 hashes. The attributes are read only.");
 
 static PyStructSequence_Field hash_info_fields[] = {
@@ -2293,17 +2293,17 @@ builtin_module_names -- tuple of module names built into this interpreter\n\
 copyright -- copyright notice pertaining to this interpreter\n\
 exec_prefix -- prefix used to find the machine-specific Python library\n\
 executable -- absolute path of the executable binary of the Python interpreter\n\
-float_info -- a struct sequence with information about the float implementation.\n\
+float_info -- a named tuple with information about the float implementation.\n\
 float_repr_style -- string indicating the style of repr() output for floats\n\
-hash_info -- a struct sequence with information about the hash algorithm.\n\
+hash_info -- a named tuple with information about the hash algorithm.\n\
 hexversion -- version information encoded as a single integer\n\
 implementation -- Python implementation information.\n\
-int_info -- a struct sequence with information about the int implementation.\n\
+int_info -- a named tuple with information about the int implementation.\n\
 maxsize -- the largest supported length of containers.\n\
 maxunicode -- the value of the largest Unicode code point\n\
 platform -- platform identifier\n\
 prefix -- prefix used to find the Python library\n\
-thread_info -- a struct sequence with information about the thread implementation.\n\
+thread_info -- a named tuple with information about the thread implementation.\n\
 version -- the version of this interpreter as a string\n\
 version_info -- version information as a named tuple\n\
 "
diff --git a/Python/thread.c b/Python/thread.c
index c5364f91948f..c36ce6ff9835 100644
--- a/Python/thread.c
+++ b/Python/thread.c
@@ -147,7 +147,7 @@ PyThread_tss_is_created(Py_tss_t *key)
 PyDoc_STRVAR(threadinfo__doc__,
 "sys.thread_info\n\
 \n\
-A struct sequence holding information about the thread implementation.");
+A named tuple holding information about the thread implementation.");
 
 static PyStructSequence_Field threadinfo_fields[] = {
     {"name",    "name of the thread implementation"},


From webhook-mailer at python.org  Wed Sep 11 10:21:57 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 14:21:57 -0000
Subject: [Python-checkins] bpo-38107: Replace direct future and task
 contructor calls with factories in asyncio tests (GH-15928)
Message-ID: <mailman.565.1568211718.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b18b19809d24182e9837b27b1c0af65458dff3ba
commit: b18b19809d24182e9837b27b1c0af65458dff3ba
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T07:21:53-07:00
summary:

bpo-38107: Replace direct future and task contructor calls with factories in asyncio tests (GH-15928)

(cherry picked from commit 9aee90018a5213e3529768e0b873955d23f5e50b)

Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com>

files:
M Lib/test/test_asyncio/test_base_events.py
M Lib/test/test_asyncio/test_events.py
M Lib/test/test_asyncio/test_locks.py
M Lib/test/test_asyncio/test_pep492.py
M Lib/test/test_asyncio/test_proactor_events.py
M Lib/test/test_asyncio/test_queues.py
M Lib/test/test_asyncio/test_selector_events.py
M Lib/test/test_asyncio/test_sock_lowlevel.py
M Lib/test/test_asyncio/test_sslproto.py
M Lib/test/test_asyncio/test_streams.py
M Lib/test/test_asyncio/test_subprocess.py
M Lib/test/test_asyncio/test_tasks.py
M Lib/test/test_asyncio/test_unix_events.py

diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
index 08d4792fa726..370e6a6dfa79 100644
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -193,7 +193,7 @@ def test_close(self):
         self.loop.close()
 
         # operation blocked when the loop is closed
-        f = asyncio.Future(loop=self.loop)
+        f = self.loop.create_future()
         self.assertRaises(RuntimeError, self.loop.run_forever)
         self.assertRaises(RuntimeError, self.loop.run_until_complete, f)
 
@@ -324,7 +324,7 @@ def check_in_thread(loop, event, debug, create_loop, fut):
 
         def test_thread(loop, debug, create_loop=False):
             event = threading.Event()
-            fut = asyncio.Future(loop=loop)
+            fut = loop.create_future()
             loop.call_soon(event.set)
             args = (loop, event, debug, create_loop, fut)
             thread = threading.Thread(target=check_in_thread, args=args)
@@ -472,7 +472,7 @@ def test_run_until_complete_type_error(self):
             self.loop.run_until_complete, 'blah')
 
     def test_run_until_complete_loop(self):
-        task = asyncio.Future(loop=self.loop)
+        task = self.loop.create_future()
         other_loop = self.new_test_loop()
         self.addCleanup(other_loop.close)
         self.assertRaises(ValueError,
@@ -555,7 +555,7 @@ def zero_error(fut):
 
         # Test call_soon (events.Handle)
         with mock.patch('asyncio.base_events.logger') as log:
-            fut = asyncio.Future(loop=self.loop)
+            fut = self.loop.create_future()
             self.loop.call_soon(zero_error, fut)
             fut.add_done_callback(lambda fut: self.loop.stop())
             self.loop.run_forever()
@@ -565,7 +565,7 @@ def zero_error(fut):
 
         # Test call_later (events.TimerHandle)
         with mock.patch('asyncio.base_events.logger') as log:
-            fut = asyncio.Future(loop=self.loop)
+            fut = self.loop.create_future()
             self.loop.call_later(0.01, zero_error, fut)
             fut.add_done_callback(lambda fut: self.loop.stop())
             self.loop.run_forever()
@@ -996,7 +996,7 @@ def __init__(self, create_future=False):
         self.state = 'INITIAL'
         self.nbytes = 0
         if create_future:
-            self.done = asyncio.Future()
+            self.done = asyncio.get_running_loop().create_future()
 
     def connection_made(self, transport):
         self.transport = transport
@@ -1026,7 +1026,7 @@ def __init__(self, create_future=False, loop=None):
         self.state = 'INITIAL'
         self.nbytes = 0
         if create_future:
-            self.done = asyncio.Future(loop=loop)
+            self.done = loop.create_future()
 
     def connection_made(self, transport):
         self.transport = transport
@@ -1071,7 +1071,7 @@ class MyProto(asyncio.Protocol):
                     (2, 1, 6, '', ('107.6.106.82', 80))]
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         idx = -1
         errors = ['err1', 'err2']
@@ -1098,7 +1098,7 @@ def test_create_connection_timeout(self, m_socket):
         m_socket.socket.return_value = sock
 
         def getaddrinfo(*args, **kw):
-            fut = asyncio.Future(loop=self.loop)
+            fut = self.loop.create_future()
             addr = (socket.AF_INET, socket.SOCK_STREAM, 0, '',
                     ('127.0.0.1', 80))
             fut.set_result([addr])
@@ -1190,7 +1190,7 @@ def test_create_connection_no_getaddrinfo(self):
             return []
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         self.loop.getaddrinfo = getaddrinfo_task
         coro = self.loop.create_connection(MyProto, 'example.com', 80)
@@ -1202,7 +1202,7 @@ def test_create_connection_connect_err(self):
             return [(2, 1, 6, '', ('107.6.106.82', 80))]
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         self.loop.getaddrinfo = getaddrinfo_task
         self.loop.sock_connect = mock.Mock()
@@ -1218,7 +1218,7 @@ def test_create_connection_multiple(self):
                     (2, 1, 6, '', ('0.0.0.2', 80))]
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         self.loop.getaddrinfo = getaddrinfo_task
         self.loop.sock_connect = mock.Mock()
@@ -1245,7 +1245,7 @@ def bind(addr):
                     (2, 1, 6, '', ('0.0.0.2', 80))]
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         self.loop.getaddrinfo = getaddrinfo_task
         self.loop.sock_connect = mock.Mock()
@@ -1377,7 +1377,7 @@ def test_create_connection_no_local_addr(self):
                 return []
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
         self.loop.getaddrinfo = getaddrinfo_task
 
         coro = self.loop.create_connection(
@@ -1405,7 +1405,7 @@ def test_create_connection_ssl_server_hostname_default(self):
         self.loop.getaddrinfo = mock.Mock()
 
         def mock_getaddrinfo(*args, **kwds):
-            f = asyncio.Future(loop=self.loop)
+            f = self.loop.create_future()
             f.set_result([(socket.AF_INET, socket.SOCK_STREAM,
                            socket.SOL_TCP, '', ('1.2.3.4', 80))])
             return f
@@ -1513,7 +1513,7 @@ def test_create_server_empty_host(self):
             return []
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         self.loop.getaddrinfo = getaddrinfo_task
         fut = self.loop.create_server(MyProto, '', 0)
diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py
index 95dac72b6d7e..b0ade1ed3ba1 100644
--- a/Lib/test/test_asyncio/test_events.py
+++ b/Lib/test/test_asyncio/test_events.py
@@ -81,8 +81,8 @@ def __init__(self, loop=None):
         self.state = 'INITIAL'
         self.nbytes = 0
         if loop is not None:
-            self.connected = asyncio.Future(loop=loop)
-            self.done = asyncio.Future(loop=loop)
+            self.connected = loop.create_future()
+            self.done = loop.create_future()
 
     def connection_made(self, transport):
         self.transport = transport
@@ -119,7 +119,7 @@ def __init__(self, loop=None):
         self.state = 'INITIAL'
         self.nbytes = 0
         if loop is not None:
-            self.done = asyncio.Future(loop=loop)
+            self.done = loop.create_future()
 
     def connection_made(self, transport):
         self.transport = transport
@@ -148,7 +148,7 @@ def __init__(self, loop=None):
         self.nbytes = 0
         self.transport = None
         if loop is not None:
-            self.done = asyncio.Future(loop=loop)
+            self.done = loop.create_future()
 
     def connection_made(self, transport):
         self.transport = transport
@@ -179,7 +179,7 @@ def __init__(self, loop=None):
         self.state = 'INITIAL'
         self.transport = None
         if loop is not None:
-            self.done = asyncio.Future(loop=loop)
+            self.done = loop.create_future()
 
     def connection_made(self, transport):
         self.transport = transport
@@ -198,9 +198,9 @@ class MySubprocessProtocol(asyncio.SubprocessProtocol):
     def __init__(self, loop):
         self.state = 'INITIAL'
         self.transport = None
-        self.connected = asyncio.Future(loop=loop)
-        self.completed = asyncio.Future(loop=loop)
-        self.disconnects = {fd: asyncio.Future(loop=loop) for fd in range(3)}
+        self.connected = loop.create_future()
+        self.completed = loop.create_future()
+        self.disconnects = {fd: loop.create_future() for fd in range(3)}
         self.data = {1: b'', 2: b''}
         self.returncode = None
         self.got_data = {1: asyncio.Event(loop=loop),
@@ -738,7 +738,7 @@ def create_server_multiple_hosts(self, family, hosts, mock_sock):
                 return [(family, socket.SOCK_STREAM, 6, '', (host, port, 0, 0))]
 
         def getaddrinfo_task(*args, **kwds):
-            return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+            return self.loop.create_task(getaddrinfo(*args, **kwds))
 
         unique_hosts = set(hosts)
 
@@ -1094,7 +1094,7 @@ def test_create_server_ssl_verified(self):
         client, pr = self.loop.run_until_complete(f_c)
 
         # extra info is available
-        self.check_ssl_extra_info(client,peername=(host, port),
+        self.check_ssl_extra_info(client, peername=(host, port),
                                   peercert=test_utils.PEERCERT)
 
         # close connection
@@ -1104,7 +1104,7 @@ def test_create_server_ssl_verified(self):
         self.loop.run_until_complete(proto.done)
 
     def test_create_server_sock(self):
-        proto = asyncio.Future(loop=self.loop)
+        proto = self.loop.create_future()
 
         class TestMyProto(MyProto):
             def connection_made(self, transport):
@@ -1143,7 +1143,7 @@ def test_create_server_addr_in_use(self):
 
     @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 not supported or enabled')
     def test_create_server_dual_stack(self):
-        f_proto = asyncio.Future(loop=self.loop)
+        f_proto = self.loop.create_future()
 
         class TestMyProto(MyProto):
             def connection_made(self, transport):
@@ -1172,7 +1172,7 @@ def connection_made(self, transport):
         proto.transport.close()
         client.close()
 
-        f_proto = asyncio.Future(loop=self.loop)
+        f_proto = self.loop.create_future()
         client = socket.socket(socket.AF_INET6)
         client.connect(('::1', port))
         client.send(b'xxx')
@@ -1587,7 +1587,7 @@ def test_prompt_cancellation(self):
             return res
 
         start = time.monotonic()
-        t = asyncio.Task(main(), loop=self.loop)
+        t = self.loop.create_task(main())
         self.loop.run_forever()
         elapsed = time.monotonic() - start
 
@@ -1673,7 +1673,7 @@ def test_close(self):
         with self.assertRaises(RuntimeError):
             self.loop.run_forever()
         with self.assertRaises(RuntimeError):
-            fut = asyncio.Future(loop=self.loop)
+            fut = self.loop.create_future()
             self.loop.run_until_complete(fut)
         with self.assertRaises(RuntimeError):
             self.loop.call_soon(func)
diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py
index a9953b4b2a2d..9468e740b3c1 100644
--- a/Lib/test/test_asyncio/test_locks.py
+++ b/Lib/test/test_asyncio/test_locks.py
@@ -124,8 +124,8 @@ def test_acquire(self):
                 result.append(3)
             return True
 
-        t1 = asyncio.Task(c1(result), loop=self.loop)
-        t2 = asyncio.Task(c2(result), loop=self.loop)
+        t1 = self.loop.create_task(c1(result))
+        t2 = self.loop.create_task(c2(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
@@ -137,7 +137,7 @@ def test_acquire(self):
         test_utils.run_briefly(self.loop)
         self.assertEqual([1], result)
 
-        t3 = asyncio.Task(c3(result), loop=self.loop)
+        t3 = self.loop.create_task(c3(result))
 
         lock.release()
         test_utils.run_briefly(self.loop)
@@ -159,7 +159,7 @@ def test_acquire_cancel(self):
             lock = asyncio.Lock(loop=self.loop)
         self.assertTrue(self.loop.run_until_complete(lock.acquire()))
 
-        task = asyncio.Task(lock.acquire(), loop=self.loop)
+        task = self.loop.create_task(lock.acquire())
         self.loop.call_soon(task.cancel)
         self.assertRaises(
             asyncio.CancelledError,
@@ -192,14 +192,14 @@ def test_cancel_race(self):
             finally:
                 lock.release()
 
-        fa = asyncio.Future(loop=self.loop)
-        ta = asyncio.Task(lockit('A', fa), loop=self.loop)
+        fa = self.loop.create_future()
+        ta = self.loop.create_task(lockit('A', fa))
         test_utils.run_briefly(self.loop)
         self.assertTrue(lock.locked())
-        tb = asyncio.Task(lockit('B', None), loop=self.loop)
+        tb = self.loop.create_task(lockit('B', None))
         test_utils.run_briefly(self.loop)
         self.assertEqual(len(lock._waiters), 1)
-        tc = asyncio.Task(lockit('C', None), loop=self.loop)
+        tc = self.loop.create_task(lockit('C', None))
         test_utils.run_briefly(self.loop)
         self.assertEqual(len(lock._waiters), 2)
 
@@ -267,17 +267,17 @@ def test_finished_waiter_cancelled(self):
         with self.assertWarns(DeprecationWarning):
             lock = asyncio.Lock(loop=self.loop)
 
-        ta = asyncio.Task(lock.acquire(), loop=self.loop)
+        ta = self.loop.create_task(lock.acquire())
         test_utils.run_briefly(self.loop)
         self.assertTrue(lock.locked())
 
-        tb = asyncio.Task(lock.acquire(), loop=self.loop)
+        tb = self.loop.create_task(lock.acquire())
         test_utils.run_briefly(self.loop)
         self.assertEqual(len(lock._waiters), 1)
 
         # Create a second waiter, wake up the first, and cancel it.
         # Without the fix, the second was not woken up.
-        tc = asyncio.Task(lock.acquire(), loop=self.loop)
+        tc = self.loop.create_task(lock.acquire())
         lock.release()
         tb.cancel()
         test_utils.run_briefly(self.loop)
@@ -405,13 +405,13 @@ def test_wait(self):
             if await ev.wait():
                 result.append(3)
 
-        t1 = asyncio.Task(c1(result), loop=self.loop)
-        t2 = asyncio.Task(c2(result), loop=self.loop)
+        t1 = self.loop.create_task(c1(result))
+        t2 = self.loop.create_task(c2(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
 
-        t3 = asyncio.Task(c3(result), loop=self.loop)
+        t3 = self.loop.create_task(c3(result))
 
         ev.set()
         test_utils.run_briefly(self.loop)
@@ -436,7 +436,7 @@ def test_wait_cancel(self):
         with self.assertWarns(DeprecationWarning):
             ev = asyncio.Event(loop=self.loop)
 
-        wait = asyncio.Task(ev.wait(), loop=self.loop)
+        wait = self.loop.create_task(ev.wait())
         self.loop.call_soon(wait.cancel)
         self.assertRaises(
             asyncio.CancelledError,
@@ -464,7 +464,7 @@ def test_clear_with_waiters(self):
                 result.append(1)
             return True
 
-        t = asyncio.Task(c1(result), loop=self.loop)
+        t = self.loop.create_task(c1(result))
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
 
@@ -527,9 +527,9 @@ def test_wait(self):
                 result.append(3)
             return True
 
-        t1 = asyncio.Task(c1(result), loop=self.loop)
-        t2 = asyncio.Task(c2(result), loop=self.loop)
-        t3 = asyncio.Task(c3(result), loop=self.loop)
+        t1 = self.loop.create_task(c1(result))
+        t2 = self.loop.create_task(c2(result))
+        t3 = self.loop.create_task(c3(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
@@ -573,7 +573,7 @@ def test_wait_cancel(self):
             cond = asyncio.Condition(loop=self.loop)
         self.loop.run_until_complete(cond.acquire())
 
-        wait = asyncio.Task(cond.wait(), loop=self.loop)
+        wait = self.loop.create_task(cond.wait())
         self.loop.call_soon(wait.cancel)
         self.assertRaises(
             asyncio.CancelledError,
@@ -588,7 +588,7 @@ def test_wait_cancel_contested(self):
         self.loop.run_until_complete(cond.acquire())
         self.assertTrue(cond.locked())
 
-        wait_task = asyncio.Task(cond.wait(), loop=self.loop)
+        wait_task = self.loop.create_task(cond.wait())
         test_utils.run_briefly(self.loop)
         self.assertFalse(cond.locked())
 
@@ -657,7 +657,7 @@ def predicate():
                 cond.release()
             return True
 
-        t = asyncio.Task(c1(result), loop=self.loop)
+        t = self.loop.create_task(c1(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
@@ -717,9 +717,9 @@ def test_notify(self):
                 cond.release()
             return True
 
-        t1 = asyncio.Task(c1(result), loop=self.loop)
-        t2 = asyncio.Task(c2(result), loop=self.loop)
-        t3 = asyncio.Task(c3(result), loop=self.loop)
+        t1 = self.loop.create_task(c1(result))
+        t2 = self.loop.create_task(c2(result))
+        t3 = self.loop.create_task(c3(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
@@ -764,8 +764,8 @@ def test_notify_all(self):
                 cond.release()
             return True
 
-        t1 = asyncio.Task(c1(result), loop=self.loop)
-        t2 = asyncio.Task(c2(result), loop=self.loop)
+        t1 = self.loop.create_task(c1(result))
+        t2 = self.loop.create_task(c2(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([], result)
@@ -965,9 +965,9 @@ def test_acquire(self):
             result.append(4)
             return True
 
-        t1 = asyncio.Task(c1(result), loop=self.loop)
-        t2 = asyncio.Task(c2(result), loop=self.loop)
-        t3 = asyncio.Task(c3(result), loop=self.loop)
+        t1 = self.loop.create_task(c1(result))
+        t2 = self.loop.create_task(c2(result))
+        t3 = self.loop.create_task(c3(result))
 
         test_utils.run_briefly(self.loop)
         self.assertEqual([1], result)
@@ -975,7 +975,7 @@ def test_acquire(self):
         self.assertEqual(2, len(sem._waiters))
         self.assertEqual(0, sem._value)
 
-        t4 = asyncio.Task(c4(result), loop=self.loop)
+        t4 = self.loop.create_task(c4(result))
 
         sem.release()
         sem.release()
@@ -1003,7 +1003,7 @@ def test_acquire_cancel(self):
             sem = asyncio.Semaphore(loop=self.loop)
         self.loop.run_until_complete(sem.acquire())
 
-        acquire = asyncio.Task(sem.acquire(), loop=self.loop)
+        acquire = self.loop.create_task(sem.acquire())
         self.loop.call_soon(acquire.cancel)
         self.assertRaises(
             asyncio.CancelledError,
@@ -1015,10 +1015,10 @@ def test_acquire_cancel_before_awoken(self):
         with self.assertWarns(DeprecationWarning):
             sem = asyncio.Semaphore(value=0, loop=self.loop)
 
-        t1 = asyncio.Task(sem.acquire(), loop=self.loop)
-        t2 = asyncio.Task(sem.acquire(), loop=self.loop)
-        t3 = asyncio.Task(sem.acquire(), loop=self.loop)
-        t4 = asyncio.Task(sem.acquire(), loop=self.loop)
+        t1 = self.loop.create_task(sem.acquire())
+        t2 = self.loop.create_task(sem.acquire())
+        t3 = self.loop.create_task(sem.acquire())
+        t4 = self.loop.create_task(sem.acquire())
 
         test_utils.run_briefly(self.loop)
 
@@ -1038,8 +1038,8 @@ def test_acquire_hang(self):
         with self.assertWarns(DeprecationWarning):
             sem = asyncio.Semaphore(value=0, loop=self.loop)
 
-        t1 = asyncio.Task(sem.acquire(), loop=self.loop)
-        t2 = asyncio.Task(sem.acquire(), loop=self.loop)
+        t1 = self.loop.create_task(sem.acquire())
+        t2 = self.loop.create_task(sem.acquire())
 
         test_utils.run_briefly(self.loop)
 
diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py
index dfb68b8e85d4..58a609444285 100644
--- a/Lib/test/test_asyncio/test_pep492.py
+++ b/Lib/test/test_asyncio/test_pep492.py
@@ -206,7 +206,7 @@ def test_double_await(self):
 
         async def runner():
             coro = afunc()
-            t = asyncio.Task(coro, loop=self.loop)
+            t = self.loop.create_task(coro)
             try:
                 await asyncio.sleep(0)
                 await coro
diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py
index b2fd60683c57..c838c5ce85ae 100644
--- a/Lib/test/test_asyncio/test_proactor_events.py
+++ b/Lib/test/test_asyncio/test_proactor_events.py
@@ -47,7 +47,7 @@ def socket_transport(self, waiter=None):
         return transport
 
     def test_ctor(self):
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         tr = self.socket_transport(waiter=fut)
         test_utils.run_briefly(self.loop)
         self.assertIsNone(fut.result())
@@ -62,7 +62,7 @@ def test_loop_reading(self):
         self.assertFalse(self.protocol.eof_received.called)
 
     def test_loop_reading_data(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(b'data')
 
         tr = self.socket_transport()
@@ -72,7 +72,7 @@ def test_loop_reading_data(self):
         self.protocol.data_received.assert_called_with(b'data')
 
     def test_loop_reading_no_data(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(b'')
 
         tr = self.socket_transport()
@@ -182,7 +182,7 @@ def test_loop_writing_err(self, m_log):
         m_log.warning.assert_called_with('socket.send() raised exception.')
 
     def test_loop_writing_stop(self):
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         fut.set_result(b'data')
 
         tr = self.socket_transport()
@@ -191,7 +191,7 @@ def test_loop_writing_stop(self):
         self.assertIsNone(tr._write_fut)
 
     def test_loop_writing_closing(self):
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         fut.set_result(1)
 
         tr = self.socket_transport()
@@ -260,7 +260,7 @@ def test_force_close(self):
     def test_loop_writing_force_close(self):
         exc_handler = mock.Mock()
         self.loop.set_exception_handler(exc_handler)
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         fut.set_result(1)
         self.proactor.send.return_value = fut
 
@@ -303,7 +303,7 @@ def test_write_eof(self):
 
     def test_write_eof_buffer(self):
         tr = self.socket_transport()
-        f = asyncio.Future(loop=self.loop)
+        f = self.loop.create_future()
         tr._loop._proactor.send.return_value = f
         tr.write(b'data')
         tr.write_eof()
@@ -327,7 +327,7 @@ def test_write_eof_write_pipe(self):
 
     def test_write_eof_buffer_write_pipe(self):
         tr = _ProactorWritePipeTransport(self.loop, self.sock, self.protocol)
-        f = asyncio.Future(loop=self.loop)
+        f = self.loop.create_future()
         tr._loop._proactor.send.return_value = f
         tr.write(b'data')
         tr.write_eof()
@@ -352,7 +352,7 @@ def test_pause_resume_reading(self):
         tr = self.socket_transport()
         futures = []
         for msg in [b'data1', b'data2', b'data3', b'data4', b'data5', b'']:
-            f = asyncio.Future(loop=self.loop)
+            f = self.loop.create_future()
             f.set_result(msg)
             futures.append(f)
 
@@ -406,7 +406,7 @@ def test_pause_resume_writing(self):
         tr = self.pause_writing_transport(high=4)
 
         # write a large chunk, must pause writing
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         self.loop._proactor.send.return_value = fut
         tr.write(b'large data')
         self.loop._run_once()
@@ -422,7 +422,7 @@ def test_pause_writing_2write(self):
         tr = self.pause_writing_transport(high=4)
 
         # first short write, the buffer is not full (3 <= 4)
-        fut1 = asyncio.Future(loop=self.loop)
+        fut1 = self.loop.create_future()
         self.loop._proactor.send.return_value = fut1
         tr.write(b'123')
         self.loop._run_once()
@@ -439,7 +439,7 @@ def test_pause_writing_3write(self):
         tr = self.pause_writing_transport(high=4)
 
         # first short write, the buffer is not full (1 <= 4)
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         self.loop._proactor.send.return_value = fut
         tr.write(b'1')
         self.loop._run_once()
@@ -463,7 +463,7 @@ def test_dont_pause_writing(self):
 
         # write a large chunk which completes immediately,
         # it should not pause writing
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         fut.set_result(None)
         self.loop._proactor.send.return_value = fut
         tr.write(b'very large data')
@@ -496,7 +496,7 @@ def socket_transport(self, waiter=None):
         return transport
 
     def test_ctor(self):
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         tr = self.socket_transport(waiter=fut)
         test_utils.run_briefly(self.loop)
         self.assertIsNone(fut.result())
@@ -541,7 +541,7 @@ def test_proto_type_switch(self):
         self.protocol = test_utils.make_test_protocol(asyncio.Protocol)
         tr = self.socket_transport()
 
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(b'data')
 
         tr = self.socket_transport()
@@ -558,7 +558,7 @@ def test_proto_type_switch(self):
 
         tr.set_protocol(buf_proto)
         test_utils.run_briefly(self.loop)
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(4)
 
         tr._read_fut = res
@@ -591,7 +591,7 @@ def test_buffer_updated_error(self):
         self.loop.call_exception_handler = mock.Mock()
         self.protocol.buffer_updated.side_effect = LookupError()
 
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(10)
         transport._read_fut = res
         transport._loop_reading(res)
@@ -601,7 +601,7 @@ def test_buffer_updated_error(self):
         self.assertTrue(self.protocol.buffer_updated.called)
 
     def test_loop_eof_received_error(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(0)
 
         self.protocol.eof_received.side_effect = LookupError()
@@ -617,7 +617,7 @@ def test_loop_eof_received_error(self):
         self.assertTrue(tr._fatal_error.called)
 
     def test_loop_reading_data(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(4)
 
         tr = self.socket_transport()
@@ -627,7 +627,7 @@ def test_loop_reading_data(self):
         self.protocol.buffer_updated.assert_called_with(4)
 
     def test_loop_reading_no_data(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result(0)
 
         tr = self.socket_transport()
@@ -691,7 +691,7 @@ def test_pause_resume_reading(self):
         tr = self.socket_transport()
         futures = []
         for msg in [10, 20, 30, 40, 0]:
-            f = asyncio.Future(loop=self.loop)
+            f = self.loop.create_future()
             f.set_result(msg)
             futures.append(f)
 
@@ -1044,7 +1044,7 @@ def test_create_server_cancel(self):
         loop = call_soon.call_args[0][0]
 
         # cancelled
-        fut = asyncio.Future(loop=self.loop)
+        fut = self.loop.create_future()
         fut.cancel()
         loop(fut)
         self.assertTrue(self.sock.close.called)
@@ -1094,7 +1094,7 @@ def test_datagram_loop_reading(self):
         close_transport(tr)
 
     def test_datagram_loop_reading_data(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result((b'data', ('127.0.0.1', 12068)))
 
         tr = self.datagram_transport()
@@ -1105,7 +1105,7 @@ def test_datagram_loop_reading_data(self):
         close_transport(tr)
 
     def test_datagram_loop_reading_no_data(self):
-        res = asyncio.Future(loop=self.loop)
+        res = self.loop.create_future()
         res.set_result((b'', ('127.0.0.1', 12068)))
 
         tr = self.datagram_transport()
diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py
index 171176c9fc53..5c9aaa82c311 100644
--- a/Lib/test/test_asyncio/test_queues.py
+++ b/Lib/test/test_asyncio/test_queues.py
@@ -44,7 +44,7 @@ def gen():
         async def add_getter():
             q = asyncio.Queue(loop=loop)
             # Start a task that waits to get.
-            asyncio.Task(q.get(), loop=loop)
+            loop.create_task(q.get())
             # Let it start waiting.
             await asyncio.sleep(0.1)
             self.assertTrue('_getters[1]' in fn(q))
@@ -58,7 +58,7 @@ def gen():
             q = asyncio.Queue(maxsize=1, loop=loop)
             q.put_nowait(1)
             # Start a task that waits to put.
-            asyncio.Task(q.put(2), loop=loop)
+            loop.create_task(q.put(2))
             # Let it start waiting.
             await asyncio.sleep(0.1)
             self.assertTrue('_putters[1]' in fn(q))
@@ -143,7 +143,7 @@ def gen():
             return True
 
         async def test():
-            t = asyncio.Task(putter(), loop=loop)
+            t = loop.create_task(putter())
             await asyncio.sleep(0.01)
 
             # The putter is blocked after putting two items.
@@ -181,7 +181,7 @@ def test_get_with_putters(self):
             q = asyncio.Queue(1, loop=self.loop)
         q.put_nowait(1)
 
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         q._putters.append(waiter)
 
         res = self.loop.run_until_complete(q.get())
@@ -212,7 +212,7 @@ def gen():
 
         async def queue_put():
             loop.call_later(0.01, q.put_nowait, 1)
-            queue_get_task = asyncio.Task(queue_get(), loop=loop)
+            queue_get_task = loop.create_task(queue_get())
             await started.wait()
             self.assertFalse(finished)
             res = await queue_get_task
@@ -252,7 +252,7 @@ def gen():
             return await asyncio.wait_for(q.get(), 0.051)
 
         async def test():
-            get_task = asyncio.Task(queue_get(), loop=loop)
+            get_task = loop.create_task(queue_get())
             await asyncio.sleep(0.01)  # let the task start
             q.put_nowait(1)
             return await get_task
@@ -264,8 +264,8 @@ def test_get_cancelled_race(self):
         with self.assertWarns(DeprecationWarning):
             q = asyncio.Queue(loop=self.loop)
 
-        t1 = asyncio.Task(q.get(), loop=self.loop)
-        t2 = asyncio.Task(q.get(), loop=self.loop)
+        t1 = self.loop.create_task(q.get())
+        t2 = self.loop.create_task(q.get())
 
         test_utils.run_briefly(self.loop)
         t1.cancel()
@@ -278,8 +278,8 @@ def test_get_cancelled_race(self):
     def test_get_with_waiting_putters(self):
         with self.assertWarns(DeprecationWarning):
             q = asyncio.Queue(loop=self.loop, maxsize=1)
-        asyncio.Task(q.put('a'), loop=self.loop)
-        asyncio.Task(q.put('b'), loop=self.loop)
+        self.loop.create_task(q.put('a'))
+        self.loop.create_task(q.put('b'))
         test_utils.run_briefly(self.loop)
         self.assertEqual(self.loop.run_until_complete(q.get()), 'a')
         self.assertEqual(self.loop.run_until_complete(q.get()), 'b')
@@ -361,7 +361,7 @@ def gen():
 
         async def queue_get():
             loop.call_later(0.01, q.get_nowait)
-            queue_put_task = asyncio.Task(queue_put(), loop=loop)
+            queue_put_task = loop.create_task(queue_put())
             await started.wait()
             self.assertFalse(finished)
             await queue_put_task
@@ -502,7 +502,7 @@ def test_put_cancelled(self):
         async def test():
             return await q.get()
 
-        t = asyncio.Task(queue_put(), loop=self.loop)
+        t = self.loop.create_task(queue_put())
         self.assertEqual(1, self.loop.run_until_complete(test()))
         self.assertTrue(t.done())
         self.assertTrue(t.result())
@@ -511,9 +511,9 @@ def test_put_cancelled_race(self):
         with self.assertWarns(DeprecationWarning):
             q = asyncio.Queue(loop=self.loop, maxsize=1)
 
-        put_a = asyncio.Task(q.put('a'), loop=self.loop)
-        put_b = asyncio.Task(q.put('b'), loop=self.loop)
-        put_c = asyncio.Task(q.put('X'), loop=self.loop)
+        put_a = self.loop.create_task(q.put('a'))
+        put_b = self.loop.create_task(q.put('b'))
+        put_c = self.loop.create_task(q.put('X'))
 
         test_utils.run_briefly(self.loop)
         self.assertTrue(put_a.done())
@@ -531,7 +531,7 @@ def test_put_cancelled_race(self):
     def test_put_with_waiting_getters(self):
         with self.assertWarns(DeprecationWarning):
             q = asyncio.Queue(loop=self.loop)
-        t = asyncio.Task(q.get(), loop=self.loop)
+        t = self.loop.create_task(q.get())
         test_utils.run_briefly(self.loop)
         self.loop.run_until_complete(q.put('a'))
         self.assertEqual(self.loop.run_until_complete(t), 'a')
@@ -665,7 +665,7 @@ def test_task_done(self):
                 q.task_done()
 
         async def test():
-            tasks = [asyncio.Task(worker(), loop=self.loop)
+            tasks = [self.loop.create_task(worker())
                      for index in range(2)]
 
             await q.join()
diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py
index 68b7853b2eba..1613c753c26e 100644
--- a/Lib/test/test_asyncio/test_selector_events.py
+++ b/Lib/test/test_asyncio/test_selector_events.py
@@ -78,7 +78,7 @@ def test_make_ssl_transport(self):
         self.loop._add_writer = mock.Mock()
         self.loop._remove_reader = mock.Mock()
         self.loop._remove_writer = mock.Mock()
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         with test_utils.disable_logger():
             transport = self.loop._make_ssl_transport(
                 m, asyncio.Protocol(), m, waiter)
@@ -154,7 +154,7 @@ def _make_self_pipe(self):
         self.loop.close()
 
         # operation blocked when the loop is closed
-        f = asyncio.Future(loop=self.loop)
+        f = self.loop.create_future()
         self.assertRaises(RuntimeError, self.loop.run_forever)
         self.assertRaises(RuntimeError, self.loop.run_until_complete, f)
         fd = 0
@@ -516,7 +516,7 @@ def socket_transport(self, waiter=None):
         return transport
 
     def test_ctor(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         tr = self.socket_transport(waiter=waiter)
         self.loop.run_until_complete(waiter)
 
@@ -525,7 +525,7 @@ def test_ctor(self):
         self.protocol.connection_made.assert_called_with(tr)
 
     def test_ctor_with_waiter(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         self.socket_transport(waiter=waiter)
         self.loop.run_until_complete(waiter)
 
@@ -911,7 +911,7 @@ def socket_transport(self, waiter=None):
         return transport
 
     def test_ctor(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         tr = self.socket_transport(waiter=waiter)
         self.loop.run_until_complete(waiter)
 
diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py
index 7b40a765e4a0..89c2af9f5ec0 100644
--- a/Lib/test/test_asyncio/test_sock_lowlevel.py
+++ b/Lib/test/test_asyncio/test_sock_lowlevel.py
@@ -16,8 +16,8 @@ def __init__(self, loop=None):
         self.state = 'INITIAL'
         self.nbytes = 0
         if loop is not None:
-            self.connected = asyncio.Future(loop=loop)
-            self.done = asyncio.Future(loop=loop)
+            self.connected = loop.create_future()
+            self.done = loop.create_future()
 
     def connection_made(self, transport):
         self.transport = transport
diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py
index 9457bc982b06..c115e46e705b 100644
--- a/Lib/test/test_asyncio/test_sslproto.py
+++ b/Lib/test/test_asyncio/test_sslproto.py
@@ -72,7 +72,7 @@ def test_handshake_timeout_negative(self):
                                  ssl_handshake_timeout=-10)
 
     def test_eof_received_waiter(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         ssl_proto = self.ssl_protocol(waiter=waiter)
         self.connection_made(ssl_proto)
         ssl_proto.eof_received()
@@ -83,7 +83,7 @@ def test_fatal_error_no_name_error(self):
         # From issue #363.
         # _fatal_error() generates a NameError if sslproto.py
         # does not import base_events.
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         ssl_proto = self.ssl_protocol(waiter=waiter)
         # Temporarily turn off error logging so as not to spoil test output.
         log_level = log.logger.getEffectiveLevel()
@@ -97,7 +97,7 @@ def test_fatal_error_no_name_error(self):
     def test_connection_lost(self):
         # From issue #472.
         # yield from waiter hang if lost_connection was called.
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         ssl_proto = self.ssl_protocol(waiter=waiter)
         self.connection_made(ssl_proto)
         ssl_proto.connection_lost(ConnectionAbortedError)
@@ -106,7 +106,7 @@ def test_connection_lost(self):
 
     def test_close_during_handshake(self):
         # bpo-29743 Closing transport during handshake process leaks socket
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         ssl_proto = self.ssl_protocol(waiter=waiter)
 
         transport = self.connection_made(ssl_proto)
@@ -116,7 +116,7 @@ def test_close_during_handshake(self):
         self.assertTrue(transport.abort.called)
 
     def test_get_extra_info_on_closed_connection(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         ssl_proto = self.ssl_protocol(waiter=waiter)
         self.assertIsNone(ssl_proto._get_extra_info('socket'))
         default = object()
@@ -127,7 +127,7 @@ def test_get_extra_info_on_closed_connection(self):
         self.assertIsNone(ssl_proto._get_extra_info('socket'))
 
     def test_set_new_app_protocol(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         ssl_proto = self.ssl_protocol(waiter=waiter)
         new_app_proto = asyncio.Protocol()
         ssl_proto._app_transport.set_protocol(new_app_proto)
diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py
index 428c863d9fa6..60fe52fa4b37 100644
--- a/Lib/test/test_asyncio/test_streams.py
+++ b/Lib/test/test_asyncio/test_streams.py
@@ -192,7 +192,7 @@ def test_read(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        read_task = asyncio.Task(stream.read(30), loop=self.loop)
+        read_task = self.loop.create_task(stream.read(30))
 
         def cb():
             stream._feed_data(self.DATA)
@@ -220,7 +220,7 @@ def test_read_eof(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        read_task = asyncio.Task(stream.read(1024), loop=self.loop)
+        read_task = self.loop.create_task(stream.read(1024))
 
         def cb():
             stream._feed_eof()
@@ -235,7 +235,7 @@ def test_read_until_eof(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        read_task = asyncio.Task(stream.read(-1), loop=self.loop)
+        read_task = self.loop.create_task(stream.read(-1))
 
         def cb():
             stream._feed_data(b'chunk1\n')
@@ -288,7 +288,7 @@ def test_readline(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
         stream._feed_data(b'chunk1 ')
-        read_task = asyncio.Task(stream.readline(), loop=self.loop)
+        read_task = self.loop.create_task(stream.readline())
 
         def cb():
             stream._feed_data(b'chunk2 ')
@@ -579,7 +579,7 @@ def test_readexactly(self):
                                 _asyncio_internal=True)
 
         n = 2 * len(self.DATA)
-        read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
+        read_task = self.loop.create_task(stream.readexactly(n))
 
         def cb():
             stream._feed_data(self.DATA)
@@ -606,7 +606,7 @@ def test_readexactly_eof(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
         n = 2 * len(self.DATA)
-        read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
+        read_task = self.loop.create_task(stream.readexactly(n))
 
         def cb():
             stream._feed_data(self.DATA)
@@ -652,8 +652,8 @@ def test_exception_waiter(self):
         async def set_err():
             stream._set_exception(ValueError())
 
-        t1 = asyncio.Task(stream.readline(), loop=self.loop)
-        t2 = asyncio.Task(set_err(), loop=self.loop)
+        t1 = self.loop.create_task(stream.readline())
+        t2 = self.loop.create_task(set_err())
 
         self.loop.run_until_complete(asyncio.wait([t1, t2]))
 
@@ -664,7 +664,7 @@ def test_exception_cancel(self):
                                 loop=self.loop,
                                 _asyncio_internal=True)
 
-        t = asyncio.Task(stream.readline(), loop=self.loop)
+        t = self.loop.create_task(stream.readline())
         test_utils.run_briefly(self.loop)
         t.cancel()
         test_utils.run_briefly(self.loop)
@@ -735,8 +735,7 @@ def stop(self):
         server = MyServer(self.loop)
         with self.assertWarns(DeprecationWarning):
             addr = server.start()
-        msg = self.loop.run_until_complete(asyncio.Task(client(addr),
-                                                        loop=self.loop))
+        msg = self.loop.run_until_complete(self.loop.create_task(client(addr)))
         server.stop()
         self.assertEqual(msg, b"hello world!\n")
 
@@ -744,8 +743,7 @@ def stop(self):
         server = MyServer(self.loop)
         with self.assertWarns(DeprecationWarning):
             addr = server.start_callback()
-        msg = self.loop.run_until_complete(asyncio.Task(client(addr),
-                                                        loop=self.loop))
+        msg = self.loop.run_until_complete(self.loop.create_task(client(addr)))
         server.stop()
         self.assertEqual(msg, b"hello world!\n")
 
@@ -810,8 +808,8 @@ def stop(self):
             server = MyServer(self.loop, path)
             with self.assertWarns(DeprecationWarning):
                 server.start()
-            msg = self.loop.run_until_complete(asyncio.Task(client(path),
-                                                            loop=self.loop))
+            msg = self.loop.run_until_complete(
+                self.loop.create_task(client(path)))
             server.stop()
             self.assertEqual(msg, b"hello world!\n")
 
@@ -820,8 +818,8 @@ def stop(self):
             server = MyServer(self.loop, path)
             with self.assertWarns(DeprecationWarning):
                 server.start_callback()
-            msg = self.loop.run_until_complete(asyncio.Task(client(path),
-                                                            loop=self.loop))
+            msg = self.loop.run_until_complete(
+                self.loop.create_task(client(path)))
             server.stop()
             self.assertEqual(msg, b"hello world!\n")
 
@@ -1016,7 +1014,7 @@ def test___repr__waiter(self):
         stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
                                 loop=self.loop,
                                 _asyncio_internal=True)
-        stream._waiter = asyncio.Future(loop=self.loop)
+        stream._waiter = self.loop.create_future()
         self.assertRegex(
             repr(stream),
             r"<Stream .+ waiter=<Future pending[\S ]*>>")
diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py
index b9578b2866c0..2cfe6576d623 100644
--- a/Lib/test/test_asyncio/test_subprocess.py
+++ b/Lib/test/test_asyncio/test_subprocess.py
@@ -43,7 +43,6 @@ def setUp(self):
         self.loop = self.new_test_loop()
         self.set_event_loop(self.loop)
 
-
     def create_transport(self, waiter=None):
         protocol = mock.Mock()
         protocol.connection_made._is_coroutine = False
@@ -54,7 +53,7 @@ def create_transport(self, waiter=None):
         return (transport, protocol)
 
     def test_proc_exited(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         transport, protocol = self.create_transport(waiter)
         transport._process_exited(6)
         self.loop.run_until_complete(waiter)
@@ -80,7 +79,7 @@ def test_proc_exited(self):
         transport.close()
 
     def test_subprocess_repr(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         transport, protocol = self.create_transport(waiter)
         transport._process_exited(6)
         self.loop.run_until_complete(waiter)
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
index 323c0907a23a..3fbb2a1a1629 100644
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -2660,7 +2660,7 @@ def get_loop(self):
         try:
             fut = Fut(loop=self.loop)
             self.loop.call_later(0.1, fut.set_result, 1)
-            task = asyncio.Task(coro(), loop=self.loop)
+            task = self.loop.create_task(coro())
             res = self.loop.run_until_complete(task)
         finally:
             self.loop.close()
@@ -2878,7 +2878,7 @@ def _run_loop(self, loop):
             test_utils.run_briefly(loop)
 
     def _check_success(self, **kwargs):
-        a, b, c = [asyncio.Future(loop=self.one_loop) for i in range(3)]
+        a, b, c = [self.one_loop.create_future() for i in range(3)]
         fut = asyncio.gather(*self.wrap_futures(a, b, c), **kwargs)
         cb = test_utils.MockCallback()
         fut.add_done_callback(cb)
@@ -2900,7 +2900,7 @@ def test_result_exception_success(self):
         self._check_success(return_exceptions=True)
 
     def test_one_exception(self):
-        a, b, c, d, e = [asyncio.Future(loop=self.one_loop) for i in range(5)]
+        a, b, c, d, e = [self.one_loop.create_future() for i in range(5)]
         fut = asyncio.gather(*self.wrap_futures(a, b, c, d, e))
         cb = test_utils.MockCallback()
         fut.add_done_callback(cb)
@@ -2918,7 +2918,7 @@ def test_one_exception(self):
         e.exception()
 
     def test_return_exceptions(self):
-        a, b, c, d = [asyncio.Future(loop=self.one_loop) for i in range(4)]
+        a, b, c, d = [self.one_loop.create_future() for i in range(4)]
         fut = asyncio.gather(*self.wrap_futures(a, b, c, d),
                              return_exceptions=True)
         cb = test_utils.MockCallback()
@@ -2991,15 +2991,15 @@ def test_constructor_empty_sequence(self):
         self._check_empty_sequence(iter(""))
 
     def test_constructor_heterogenous_futures(self):
-        fut1 = asyncio.Future(loop=self.one_loop)
-        fut2 = asyncio.Future(loop=self.other_loop)
+        fut1 = self.one_loop.create_future()
+        fut2 = self.other_loop.create_future()
         with self.assertRaises(ValueError):
             asyncio.gather(fut1, fut2)
         with self.assertRaises(ValueError):
             asyncio.gather(fut1, loop=self.other_loop)
 
     def test_constructor_homogenous_futures(self):
-        children = [asyncio.Future(loop=self.other_loop) for i in range(3)]
+        children = [self.other_loop.create_future() for i in range(3)]
         fut = asyncio.gather(*children)
         self.assertIs(fut._loop, self.other_loop)
         self._run_loop(self.other_loop)
@@ -3010,7 +3010,7 @@ def test_constructor_homogenous_futures(self):
         self.assertFalse(fut.done())
 
     def test_one_cancellation(self):
-        a, b, c, d, e = [asyncio.Future(loop=self.one_loop) for i in range(5)]
+        a, b, c, d, e = [self.one_loop.create_future() for i in range(5)]
         fut = asyncio.gather(a, b, c, d, e)
         cb = test_utils.MockCallback()
         fut.add_done_callback(cb)
@@ -3028,7 +3028,7 @@ def test_one_cancellation(self):
         e.exception()
 
     def test_result_exception_one_cancellation(self):
-        a, b, c, d, e, f = [asyncio.Future(loop=self.one_loop)
+        a, b, c, d, e, f = [self.one_loop.create_future()
                             for i in range(6)]
         fut = asyncio.gather(a, b, c, d, e, f, return_exceptions=True)
         cb = test_utils.MockCallback()
@@ -3094,7 +3094,7 @@ def coro(s):
     def test_cancellation_broadcast(self):
         # Cancelling outer() cancels all children.
         proof = 0
-        waiter = asyncio.Future(loop=self.one_loop)
+        waiter = self.one_loop.create_future()
 
         async def inner():
             nonlocal proof
@@ -3130,8 +3130,8 @@ def test_exception_marking(self):
             await f
             raise RuntimeError('should not be ignored')
 
-        a = asyncio.Future(loop=self.one_loop)
-        b = asyncio.Future(loop=self.one_loop)
+        a = self.one_loop.create_future()
+        b = self.one_loop.create_future()
 
         async def outer():
             await asyncio.gather(inner(a), inner(b), loop=self.one_loop)
diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py
index 1daa870a7b27..02ce7e89da09 100644
--- a/Lib/test/test_asyncio/test_unix_events.py
+++ b/Lib/test/test_asyncio/test_unix_events.py
@@ -669,7 +669,7 @@ def read_pipe_transport(self, waiter=None):
         return transport
 
     def test_ctor(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         tr = self.read_pipe_transport(waiter=waiter)
         self.loop.run_until_complete(waiter)
 
@@ -819,7 +819,7 @@ def write_pipe_transport(self, waiter=None):
         return transport
 
     def test_ctor(self):
-        waiter = asyncio.Future(loop=self.loop)
+        waiter = self.loop.create_future()
         tr = self.write_pipe_transport(waiter=waiter)
         self.loop.run_until_complete(waiter)
 


From webhook-mailer at python.org  Wed Sep 11 10:28:10 2019
From: webhook-mailer at python.org (Michael Foord)
Date: Wed, 11 Sep 2019 14:28:10 -0000
Subject: [Python-checkins] bpo-37972: unittest.mock._Call now passes on
 __getitem__ to the __getattr__ chaining so that call() can be subscriptable
 (GH-15565)
Message-ID: <mailman.566.1568212091.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/72c359912d36705a94fca8b63d80451905a14ae4
commit: 72c359912d36705a94fca8b63d80451905a14ae4
branch: master
author: blhsing <github at ydooby.com>
committer: Michael Foord <voidspace at users.noreply.github.com>
date: 2019-09-11T16:28:06+02:00
summary:

bpo-37972: unittest.mock._Call now passes on __getitem__ to the __getattr__ chaining so that call() can be subscriptable (GH-15565)

* bpo-37972: unittest.mock._Call now passes on __getitem__ to the __getattr__ chaining so that call() can be subscriptable

* ?? Added by blurb_it.

* Update 2019-08-28-21-40-12.bpo-37972.kP-n4L.rst

added name of the contributor

* bpo-37972: made all dunder methods chainable for _Call

* bpo-37972: delegate only attributes of tuple instead to __getattr__

files:
A Misc/NEWS.d/next/Library/2019-08-28-21-40-12.bpo-37972.kP-n4L.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testhelpers.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 50e495965519..a98decc2c9f0 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -2462,6 +2462,12 @@ def __getattr__(self, attr):
         return _Call(name=name, parent=self, from_kall=False)
 
 
+    def __getattribute__(self, attr):
+        if attr in tuple.__dict__:
+            raise AttributeError
+        return tuple.__getattribute__(self, attr)
+
+
     def count(self, /, *args, **kwargs):
         return self.__getattr__('count')(*args, **kwargs)
 
diff --git a/Lib/unittest/test/testmock/testhelpers.py b/Lib/unittest/test/testmock/testhelpers.py
index 301bca430c13..f3c7acb98c21 100644
--- a/Lib/unittest/test/testmock/testhelpers.py
+++ b/Lib/unittest/test/testmock/testhelpers.py
@@ -334,6 +334,26 @@ def test_call_with_name(self):
         self.assertEqual(_Call((('bar', 'barz'),),)[0], '')
         self.assertEqual(_Call((('bar', 'barz'), {'hello': 'world'}),)[0], '')
 
+    def test_dunder_call(self):
+        m = MagicMock()
+        m().foo()['bar']()
+        self.assertEqual(
+            m.mock_calls,
+            [call(), call().foo(), call().foo().__getitem__('bar'), call().foo().__getitem__()()]
+        )
+        m = MagicMock()
+        m().foo()['bar'] = 1
+        self.assertEqual(
+            m.mock_calls,
+            [call(), call().foo(), call().foo().__setitem__('bar', 1)]
+        )
+        m = MagicMock()
+        iter(m().foo())
+        self.assertEqual(
+            m.mock_calls,
+            [call(), call().foo(), call().foo().__iter__()]
+        )
+
 
 class SpecSignatureTest(unittest.TestCase):
 
diff --git a/Misc/NEWS.d/next/Library/2019-08-28-21-40-12.bpo-37972.kP-n4L.rst b/Misc/NEWS.d/next/Library/2019-08-28-21-40-12.bpo-37972.kP-n4L.rst
new file mode 100644
index 000000000000..73d9ef77762b
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-28-21-40-12.bpo-37972.kP-n4L.rst
@@ -0,0 +1,5 @@
+Subscripts to the `unittest.mock.call` objects now receive the same chaining mechanism as any other custom attributes, so that the following usage no longer raises a `TypeError`:
+
+    call().foo().__getitem__('bar')
+
+Patch by blhsing


From webhook-mailer at python.org  Wed Sep 11 10:36:39 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Wed, 11 Sep 2019 14:36:39 -0000
Subject: [Python-checkins] bpo-36919: make
 test_source_encoding.test_issue2301 implementation-independent (GH-13639)
 (GH-15953)
Message-ID: <mailman.567.1568212600.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c2119399488a297275db7f0590cff957b6ce8a2e
commit: c2119399488a297275db7f0590cff957b6ce8a2e
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-11T15:36:36+01:00
summary:

bpo-36919: make test_source_encoding.test_issue2301 implementation-independent (GH-13639) (GH-15953)

* bpo-36919: make test_issue2301 implementation-independent
(cherry picked from commit b6643dcfc26859f935e4b3a6a2a203e8ef5320e2)

Co-authored-by: Pavel Koneski <pavel.koneski at gmail.com>

files:
A Misc/NEWS.d/next/Tests/2019-05-28-15-41-34.bpo-36919.-vGt_m.rst
M Lib/test/test_source_encoding.py

diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py
index 38734009c008..a0bd741c36ac 100644
--- a/Lib/test/test_source_encoding.py
+++ b/Lib/test/test_source_encoding.py
@@ -31,7 +31,7 @@ def test_issue2301(self):
         try:
             compile(b"# coding: cp932\nprint '\x94\x4e'", "dummy", "exec")
         except SyntaxError as v:
-            self.assertEqual(v.text, "print '\u5e74'\n")
+            self.assertEqual(v.text.rstrip('\n'), "print '\u5e74'")
         else:
             self.fail()
 
diff --git a/Misc/NEWS.d/next/Tests/2019-05-28-15-41-34.bpo-36919.-vGt_m.rst b/Misc/NEWS.d/next/Tests/2019-05-28-15-41-34.bpo-36919.-vGt_m.rst
new file mode 100644
index 000000000000..9400bdda5da1
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-05-28-15-41-34.bpo-36919.-vGt_m.rst
@@ -0,0 +1,2 @@
+Make ``test_source_encoding.test_issue2301`` implementation independent. The
+test will work now for both CPython and IronPython.


From webhook-mailer at python.org  Wed Sep 11 10:37:16 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 14:37:16 -0000
Subject: [Python-checkins] bpo-35224: Additional documentation for
 Assignment Expressions (GH-15935)
Message-ID: <mailman.568.1568212637.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6357c95716d89ac1f80587fbc4133df8d2e8396c
commit: 6357c95716d89ac1f80587fbc4133df8d2e8396c
branch: master
author: Emily Morehouse <emily at cuttlesoft.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T07:37:12-07:00
summary:

bpo-35224: Additional documentation for Assignment Expressions (GH-15935)



Add or update assignment expression documentation for:
- FAQ - Design
- Reference - Expressions
- Reference - Lexical Analysis


https://bugs.python.org/issue35224



Automerge-Triggered-By: @matrixise

files:
M Doc/faq/design.rst
M Doc/reference/expressions.rst
M Doc/reference/lexical_analysis.rst
M Doc/whatsnew/3.8.rst

diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst
index 47fc4d498e76..75cd20fb71dd 100644
--- a/Doc/faq/design.rst
+++ b/Doc/faq/design.rst
@@ -151,66 +151,15 @@ to tell Python which namespace to use.
 Why can't I use an assignment in an expression?
 -----------------------------------------------
 
-Many people used to C or Perl complain that they want to use this C idiom:
+Starting in Python 3.8, you can!
 
-.. code-block:: c
+Assignment expressions using the walrus operator `:=` assign a variable in an
+expression::
 
-   while (line = readline(f)) {
-       // do something with line
-   }
-
-where in Python you're forced to write this::
-
-   while True:
-       line = f.readline()
-       if not line:
-           break
-       ...  # do something with line
-
-The reason for not allowing assignment in Python expressions is a common,
-hard-to-find bug in those other languages, caused by this construct:
-
-.. code-block:: c
-
-    if (x = 0) {
-        // error handling
-    }
-    else {
-        // code that only works for nonzero x
-    }
-
-The error is a simple typo: ``x = 0``, which assigns 0 to the variable ``x``,
-was written while the comparison ``x == 0`` is certainly what was intended.
-
-Many alternatives have been proposed.  Most are hacks that save some typing but
-use arbitrary or cryptic syntax or keywords, and fail the simple criterion for
-language change proposals: it should intuitively suggest the proper meaning to a
-human reader who has not yet been introduced to the construct.
-
-An interesting phenomenon is that most experienced Python programmers recognize
-the ``while True`` idiom and don't seem to be missing the assignment in
-expression construct much; it's only newcomers who express a strong desire to
-add this to the language.
-
-There's an alternative way of spelling this that seems attractive but is
-generally less robust than the "while True" solution::
-
-   line = f.readline()
-   while line:
-       ...  # do something with line...
-       line = f.readline()
-
-The problem with this is that if you change your mind about exactly how you get
-the next line (e.g. you want to change it into ``sys.stdin.readline()``) you
-have to remember to change two places in your program -- the second occurrence
-is hidden at the bottom of the loop.
-
-The best approach is to use iterators, making it possible to loop through
-objects using the ``for`` statement.  For example, :term:`file objects
-<file object>` support the iterator protocol, so you can write simply::
+   while chunk := fp.read(200):
+      print(chunk)
 
-   for line in f:
-       ...  # do something with line...
+See :pep:`572` for more information.
 
 
 
diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst
index 73a2f271ca54..d9db33a78568 100644
--- a/Doc/reference/expressions.rst
+++ b/Doc/reference/expressions.rst
@@ -1784,6 +1784,8 @@ precedence and have a left-to-right chaining feature as described in the
 +-----------------------------------------------+-------------------------------------+
 | Operator                                      | Description                         |
 +===============================================+=====================================+
+| ``:=``                                        | Assignment expression               |
++-----------------------------------------------+-------------------------------------+
 | :keyword:`lambda`                             | Lambda expression                   |
 +-----------------------------------------------+-------------------------------------+
 | :keyword:`if <if_expr>` -- :keyword:`!else`   | Conditional expression              |
diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst
index 7e1e17edb2d8..c0e13b53698e 100644
--- a/Doc/reference/lexical_analysis.rst
+++ b/Doc/reference/lexical_analysis.rst
@@ -887,7 +887,7 @@ The following tokens are operators:
 
 
    +       -       *       **      /       //      %      @
-   <<      >>      &       |       ^       ~
+   <<      >>      &       |       ^       ~       :=
    <       >       <=      >=      ==      !=
 
 
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index aefe9b53746f..c2455f487b14 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -122,8 +122,6 @@ See :pep:`572` for a full description.
 
 (Contributed by Emily Morehouse in :issue:`35224`.)
 
-.. TODO: Emily will sprint on docs at PyCon US 2019.
-
 
 Positional-only parameters
 --------------------------


From webhook-mailer at python.org  Wed Sep 11 10:37:22 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Wed, 11 Sep 2019 14:37:22 -0000
Subject: [Python-checkins] bpo-36919: make
 test_source_encoding.test_issue2301 implementation-independent (GH-13639)
 (GH-15952)
Message-ID: <mailman.569.1568212643.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/42edfcfd129c7ac4842b9e0cd453bbe3ff006669
commit: 42edfcfd129c7ac4842b9e0cd453bbe3ff006669
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-11T15:37:18+01:00
summary:

bpo-36919: make test_source_encoding.test_issue2301 implementation-independent (GH-13639) (GH-15952)

* bpo-36919: make test_issue2301 implementation-independent
(cherry picked from commit b6643dcfc26859f935e4b3a6a2a203e8ef5320e2)

Co-authored-by: Pavel Koneski <pavel.koneski at gmail.com>

files:
A Misc/NEWS.d/next/Tests/2019-05-28-15-41-34.bpo-36919.-vGt_m.rst
M Lib/test/test_source_encoding.py

diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py
index 38734009c008..a0bd741c36ac 100644
--- a/Lib/test/test_source_encoding.py
+++ b/Lib/test/test_source_encoding.py
@@ -31,7 +31,7 @@ def test_issue2301(self):
         try:
             compile(b"# coding: cp932\nprint '\x94\x4e'", "dummy", "exec")
         except SyntaxError as v:
-            self.assertEqual(v.text, "print '\u5e74'\n")
+            self.assertEqual(v.text.rstrip('\n'), "print '\u5e74'")
         else:
             self.fail()
 
diff --git a/Misc/NEWS.d/next/Tests/2019-05-28-15-41-34.bpo-36919.-vGt_m.rst b/Misc/NEWS.d/next/Tests/2019-05-28-15-41-34.bpo-36919.-vGt_m.rst
new file mode 100644
index 000000000000..9400bdda5da1
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-05-28-15-41-34.bpo-36919.-vGt_m.rst
@@ -0,0 +1,2 @@
+Make ``test_source_encoding.test_issue2301`` implementation independent. The
+test will work now for both CPython and IronPython.


From webhook-mailer at python.org  Wed Sep 11 10:38:29 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Wed, 11 Sep 2019 14:38:29 -0000
Subject: [Python-checkins] bpo-23460: Fix documentation for decimal string
 :g formatting (GH-11850) (GH-15954)
Message-ID: <mailman.570.1568212710.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/629f1f87e9b0bfd9f1dc3d02f966decde5c65201
commit: 629f1f87e9b0bfd9f1dc3d02f966decde5c65201
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-11T16:38:25+02:00
summary:

bpo-23460: Fix documentation for decimal string :g formatting (GH-11850) (GH-15954)

(cherry picked from commit 1660a61a105bcd62e2dfa77885959a8992e9f14e)

Co-authored-by: Brennan D Baraban <34765317+bdbaraban at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Documentation/2019-02-14-07-12-48.bpo-23460.Iqiqtm.rst
M Doc/library/string.rst

diff --git a/Doc/library/string.rst b/Doc/library/string.rst
index af8b9b358cc3..af52af0482c4 100644
--- a/Doc/library/string.rst
+++ b/Doc/library/string.rst
@@ -500,9 +500,10 @@ The available presentation types for floating point and decimal values are:
    |         |                                                          |
    |         | The precise rules are as follows: suppose that the       |
    |         | result formatted with presentation type ``'e'`` and      |
-   |         | precision ``p-1`` would have exponent ``exp``.  Then     |
-   |         | if ``-4 <= exp < p``, the number is formatted            |
-   |         | with presentation type ``'f'`` and precision             |
+   |         | precision ``p-1`` would have exponent ``exp``.  Then,    |
+   |         | if ``m <= exp < p``, where ``m`` is -4 for floats and -6 |
+   |         | for :class:`Decimals <decimal.Decimal>`, the number is   |
+   |         | formatted with presentation type ``'f'`` and precision   |
    |         | ``p-1-exp``.  Otherwise, the number is formatted         |
    |         | with presentation type ``'e'`` and precision ``p-1``.    |
    |         | In both cases insignificant trailing zeros are removed   |
diff --git a/Misc/NEWS.d/next/Documentation/2019-02-14-07-12-48.bpo-23460.Iqiqtm.rst b/Misc/NEWS.d/next/Documentation/2019-02-14-07-12-48.bpo-23460.Iqiqtm.rst
new file mode 100644
index 000000000000..4674d551d379
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-02-14-07-12-48.bpo-23460.Iqiqtm.rst
@@ -0,0 +1 @@
+The documentation for decimal string formatting using the `:g` specifier has been updated to reflect the correct exponential notation cutoff point. Original patch contributed by Tuomas Suutari.
\ No newline at end of file


From webhook-mailer at python.org  Wed Sep 11 10:39:36 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Wed, 11 Sep 2019 14:39:36 -0000
Subject: [Python-checkins] [3.8] Turn off AppVeyor CI (GH-15929) (GH-15941)
Message-ID: <mailman.571.1568212777.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8ee8ad22d7abf48418fd911b7ce8c2c115e540a0
commit: 8ee8ad22d7abf48418fd911b7ce8c2c115e540a0
branch: 3.8
author: Zachary Ware <zachary.ware at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T15:39:32+01:00
summary:

[3.8] Turn off AppVeyor CI (GH-15929) (GH-15941)

(cherry picked from commit 6f55b0394bce8e64a59c61f465fec8079a00e55c)

files:
D .github/appveyor.yml

diff --git a/.github/appveyor.yml b/.github/appveyor.yml
deleted file mode 100644
index e8012f69ee5b..000000000000
--- a/.github/appveyor.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-version: 3.8build{build}
-clone_depth: 5
-branches:
-  only:
-    - master
-    - /\d\.\d/
-    - buildbot-custom
-cache:
-  - externals -> PCbuild
-before_build:
-  - ps: |+
-      if ($env:APPVEYOR_RE_BUILD) {
-        echo 'Doing full build due to re-build request.'
-      } elseif (!$env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT) {
-        echo 'Not a PR, doing full build.'
-      } else {
-        git fetch -q origin +refs/heads/$env:APPVEYOR_REPO_BRANCH
-        $mergebase = git merge-base HEAD FETCH_HEAD
-        $changes = git diff --name-only HEAD $mergebase | grep -vE '(\.rst$)|(^Doc)|(^Misc)'
-        If (!$changes) {
-          echo 'Only docs were updated, stopping build process.'
-          Exit-AppveyorBuild
-        } else {
-          echo 'Doing full build due to non-doc changes in these files:'
-          echo $changes
-        }
-      }
-
-
-build_script:
-  - cmd: PCbuild\build.bat -e -p x64
-  - cmd: PCbuild\amd64\python.exe -m test.pythoninfo
-test_script:
-  - cmd: PCbuild\rt.bat -x64 -q -uall -u-cpu -u-largefile -rwW --slowest --timeout=1200 --fail-env-changed -j0
-environment:
-  HOST_PYTHON: C:\Python36\python.exe
-image:
-  - Visual Studio 2017


From webhook-mailer at python.org  Wed Sep 11 10:39:56 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Wed, 11 Sep 2019 14:39:56 -0000
Subject: [Python-checkins] [3.7] Turn off AppVeyor CI (GH-15929) (GH-15943)
Message-ID: <mailman.572.1568212796.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e58e18dfcd2e638773990fb5f307d2b97b848031
commit: e58e18dfcd2e638773990fb5f307d2b97b848031
branch: 3.7
author: Zachary Ware <zachary.ware at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T15:39:52+01:00
summary:

[3.7] Turn off AppVeyor CI (GH-15929) (GH-15943)

(cherry picked from commit 6f55b0394bce8e64a59c61f465fec8079a00e55c)

files:
D .github/appveyor.yml

diff --git a/.github/appveyor.yml b/.github/appveyor.yml
deleted file mode 100644
index 6662732326a3..000000000000
--- a/.github/appveyor.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-version: 3.7build{build}
-clone_depth: 5
-branches:
-  only:
-    - master
-    - /\d\.\d/
-    - buildbot-custom
-cache:
-  - externals -> PCbuild\*
-before_build:
-  - ps: |+
-      if ($env:APPVEYOR_RE_BUILD) {
-        echo 'Doing full build due to re-build request.'
-      } elseif (!$env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT) {
-        echo 'Not a PR, doing full build.'
-      } else {
-        git fetch -q origin +refs/heads/$env:APPVEYOR_REPO_BRANCH
-        $mergebase = git merge-base HEAD FETCH_HEAD
-        $changes = git diff --name-only HEAD $mergebase | grep -vE '(\.rst$)|(^Doc)|(^Misc)'
-        If (!$changes) {
-          echo 'Only docs were updated, stopping build process.'
-          Exit-AppveyorBuild
-        } else {
-          echo 'Doing full build due to non-doc changes in these files:'
-          echo $changes
-        }
-      }
-
-
-build_script:
-  - cmd: PCbuild\build.bat -e
-  - cmd: PCbuild\win32\python.exe -m test.pythoninfo
-test_script:
-  - cmd: PCbuild\rt.bat -q -uall -u-cpu -u-largefile -rwW --slowest --timeout=1200 --fail-env-changed -j0
-environment:
-  HOST_PYTHON: C:\Python36\python.exe
-image:
-  - Visual Studio 2017


From webhook-mailer at python.org  Wed Sep 11 10:41:25 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Wed, 11 Sep 2019 14:41:25 -0000
Subject: [Python-checkins] bpo-36634: Fixes activate.bat when existing
 values contain double quotes (GH-15924)
Message-ID: <mailman.573.1568212886.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/574b324bdc9a126b5a4488c3613f11ad2555415e
commit: 574b324bdc9a126b5a4488c3613f11ad2555415e
branch: master
author: Steve Dower <steve.dower at python.org>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-11T15:41:21+01:00
summary:

bpo-36634: Fixes activate.bat when existing values contain double quotes (GH-15924)

files:
A Misc/NEWS.d/next/Windows/2019-09-11-12-34-31.bpo-36634.xLaGgb.rst
M Lib/venv/scripts/nt/activate.bat

diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat
index da831bb42c79..8eb5c1ff664f 100644
--- a/Lib/venv/scripts/nt/activate.bat
+++ b/Lib/venv/scripts/nt/activate.bat
@@ -2,44 +2,32 @@
 
 rem This file is UTF-8 encoded, so we need to update the current code page while executing it
 for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do (
-    set "_OLD_CODEPAGE=%%a"
+    set _OLD_CODEPAGE=%%a
 )
 if defined _OLD_CODEPAGE (
     "%SystemRoot%\System32\chcp.com" 65001 > nul
 )
 
-set "VIRTUAL_ENV=__VENV_DIR__"
+set VIRTUAL_ENV=__VENV_DIR__
 
-if not defined PROMPT (
-    set "PROMPT=$P$G"
-)
-
-if defined _OLD_VIRTUAL_PROMPT (
-    set "PROMPT=%_OLD_VIRTUAL_PROMPT%"
-)
+if not defined PROMPT set PROMPT=$P$G
 
-if defined _OLD_VIRTUAL_PYTHONHOME (
-    set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%"
-)
+if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT%
+if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%
 
-set "_OLD_VIRTUAL_PROMPT=%PROMPT%"
-set "PROMPT=__VENV_PROMPT__%PROMPT%"
+set _OLD_VIRTUAL_PROMPT=%PROMPT%
+set PROMPT=__VENV_PROMPT__%PROMPT%
 
-if defined PYTHONHOME (
-    set "_OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%"
-    set PYTHONHOME=
-)
+if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%
+set PYTHONHOME=
 
-if defined _OLD_VIRTUAL_PATH (
-    set "PATH=%_OLD_VIRTUAL_PATH%"
-) else (
-    set "_OLD_VIRTUAL_PATH=%PATH%"
-)
+if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH%
+else set _OLD_VIRTUAL_PATH=%PATH%
 
-set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%"
+set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%
 
 :END
 if defined _OLD_CODEPAGE (
     "%SystemRoot%\System32\chcp.com" %_OLD_CODEPAGE% > nul
-    set "_OLD_CODEPAGE="
+    set _OLD_CODEPAGE=
 )
diff --git a/Misc/NEWS.d/next/Windows/2019-09-11-12-34-31.bpo-36634.xLaGgb.rst b/Misc/NEWS.d/next/Windows/2019-09-11-12-34-31.bpo-36634.xLaGgb.rst
new file mode 100644
index 000000000000..c0bd2beaae2d
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-11-12-34-31.bpo-36634.xLaGgb.rst
@@ -0,0 +1,2 @@
+venv activate.bat now works when the existing variables contain double quote
+characters.


From webhook-mailer at python.org  Wed Sep 11 10:48:38 2019
From: webhook-mailer at python.org (Stefan Behnel)
Date: Wed, 11 Sep 2019 14:48:38 -0000
Subject: [Python-checkins] bpo-33187: Document ElementInclude (XInclude)
 support in ElementTree (GH-8861) (GH-15958)
Message-ID: <mailman.574.1568213319.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6cf0ba8a632b1c0bd3576ed33c971ca3778000de
commit: 6cf0ba8a632b1c0bd3576ed33c971ca3778000de
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Stefan Behnel <stefan_ml at behnel.de>
date: 2019-09-11T15:48:33+01:00
summary:

bpo-33187: Document ElementInclude (XInclude) support in ElementTree (GH-8861) (GH-15958)

(cherry picked from commit 97b817eae34b77be1ced382e15098a112f547848)

Co-authored-by: Anjali Bansal <anjali.mca17.du at gmail.com>

files:
M Doc/library/xml.etree.elementtree.rst
M Doc/tools/susp-ignored.csv

diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst
index 9f46755c2685..5e87d8f0285f 100644
--- a/Doc/library/xml.etree.elementtree.rst
+++ b/Doc/library/xml.etree.elementtree.rst
@@ -726,6 +726,95 @@ Functions
    :class:`Element` instance and a dictionary.
 
 
+.. _elementtree-xinclude:
+
+XInclude support
+----------------
+
+This module provides limited support for
+`XInclude directives <https://www.w3.org/TR/xinclude/>`_, via the :mod:`xml.etree.ElementInclude` helper module.  This module can be used to insert subtrees and text strings into element trees, based on information in the tree.
+
+Example
+^^^^^^^
+
+Here's an example that demonstrates use of the XInclude module. To include an XML document in the current document, use the ``{http://www.w3.org/2001/XInclude}include`` element and set the **parse** attribute to ``"xml"``, and use the **href** attribute to specify the document to include.
+
+.. code-block:: xml
+
+    <?xml version="1.0"?>
+    <document xmlns:xi="http://www.w3.org/2001/XInclude">
+      <xi:include href="source.xml" parse="xml" />
+    </document>
+
+By default, the **href** attribute is treated as a file name. You can use custom loaders to override this behaviour. Also note that the standard helper does not support XPointer syntax.
+
+To process this file, load it as usual, and pass the root element to the :mod:`xml.etree.ElementTree` module:
+
+.. code-block:: python
+
+   from xml.etree import ElementTree, ElementInclude
+
+   tree = ElementTree.parse("document.xml")
+   root = tree.getroot()
+
+   ElementInclude.include(root)
+
+The ElementInclude module replaces the ``{http://www.w3.org/2001/XInclude}include`` element with the root element from the **source.xml** document. The result might look something like this:
+
+.. code-block:: xml
+
+    <document xmlns:xi="http://www.w3.org/2001/XInclude">
+      <para>This is a paragraph.</para>
+    </document>
+
+If the **parse** attribute is omitted, it defaults to "xml". The href attribute is required.
+
+To include a text document, use the ``{http://www.w3.org/2001/XInclude}include`` element, and set the **parse** attribute to "text":
+
+.. code-block:: xml
+
+    <?xml version="1.0"?>
+    <document xmlns:xi="http://www.w3.org/2001/XInclude">
+      Copyright (c) <xi:include href="year.txt" parse="text" />.
+    </document>
+
+The result might look something like:
+
+.. code-block:: xml
+
+    <document xmlns:xi="http://www.w3.org/2001/XInclude">
+      Copyright (c) 2003.
+    </document>
+
+Reference
+---------
+
+.. _elementinclude-functions:
+
+Functions
+^^^^^^^^^
+
+.. function:: xml.etree.ElementInclude.default_loader( href, parse, encoding=None)
+
+   Default loader. This default loader reads an included resource from disk.  *href* is a URL.
+   *parse* is for parse mode either "xml" or "text".  *encoding*
+   is an optional text encoding.  If not given, encoding is ``utf-8``.  Returns the
+   expanded resource.  If the parse mode is ``"xml"``, this is an ElementTree
+   instance.  If the parse mode is "text", this is a Unicode string.  If the
+   loader fails, it can return None or raise an exception.
+
+
+.. function:: xml.etree.ElementInclude.include( elem, loader=None)
+
+   This function expands XInclude directives.  *elem* is the root element.  *loader* is
+   an optional resource loader.  If omitted, it defaults to :func:`default_loader`.
+   If given, it should be a callable that implements the same interface as
+   :func:`default_loader`.  Returns the expanded resource.  If the parse mode is
+   ``"xml"``, this is an ElementTree instance.  If the parse mode is "text",
+   this is a Unicode string.  If the loader fails, it can return None or
+   raise an exception.
+
+
 .. _elementtree-element-objects:
 
 Element Objects
diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv
index c97c859fe891..1f0cf3fd6ed4 100644
--- a/Doc/tools/susp-ignored.csv
+++ b/Doc/tools/susp-ignored.csv
@@ -331,6 +331,9 @@ library/xml.etree.elementtree,,:character,<fictional:character>Commander Clement
 library/xml.etree.elementtree,,:actor,"for actor in root.findall('real_person:actor', ns):"
 library/xml.etree.elementtree,,:name,"name = actor.find('real_person:name', ns)"
 library/xml.etree.elementtree,,:character,"for char in actor.findall('role:character', ns):"
+library/xml.etree.elementtree,,:xi,<document xmlns:xi="http://www.w3.org/2001/XInclude">
+library/xml.etree.elementtree,,:include,  <xi:include href="source.xml" parse="xml" />
+library/xml.etree.elementtree,,:include,  Copyright (c) <xi:include href="year.txt" parse="text" />.
 library/zipapp,,:main,"$ python -m zipapp myapp -m ""myapp:main"""
 library/zipapp,,:fn,"pkg.mod:fn"
 library/zipapp,,:callable,"pkg.module:callable"


From webhook-mailer at python.org  Wed Sep 11 10:54:48 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 14:54:48 -0000
Subject: [Python-checkins] Update ftplib.all_errors documentation to match
 code (GH-15026)
Message-ID: <mailman.575.1568213689.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2f2a867fe334df84963ddc6b0003c5e24b703856
commit: 2f2a867fe334df84963ddc6b0003c5e24b703856
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T07:54:44-07:00
summary:

Update ftplib.all_errors documentation to match code (GH-15026)


The documentation doesn't mention the `EOFError` that https://github.com/python/cpython/blob/master/Lib/ftplib.pyGH-L66 includes
(cherry picked from commit 51adfc6ed2c6c466dda84edc11d9b2b8ae2903e8)

Co-authored-by: Andrew Scheller <github at loowis.durge.org>

files:
M Doc/library/ftplib.rst

diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst
index e006d019b837..7423413d209a 100644
--- a/Doc/library/ftplib.rst
+++ b/Doc/library/ftplib.rst
@@ -144,7 +144,7 @@ The module defines the following items:
    The set of all exceptions (as a tuple) that methods of :class:`FTP`
    instances may raise as a result of problems with the FTP connection (as
    opposed to programming errors made by the caller).  This set includes the
-   four exceptions listed above as well as :exc:`OSError`.
+   four exceptions listed above as well as :exc:`OSError` and :exc:`EOFError`.
 
 
 .. seealso::


From webhook-mailer at python.org  Wed Sep 11 10:55:16 2019
From: webhook-mailer at python.org (Carol Willing)
Date: Wed, 11 Sep 2019 14:55:16 -0000
Subject: [Python-checkins] Improve the io module documentation (GH-15099)
Message-ID: <mailman.576.1568213717.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3b58a70d9cf1c0f963adce9b07060116b2775687
commit: 3b58a70d9cf1c0f963adce9b07060116b2775687
branch: master
author: G?ry Ogam <gery.ogam at gmail.com>
committer: Carol Willing <carolcode at willingconsulting.com>
date: 2019-09-11T15:55:13+01:00
summary:

Improve the io module documentation (GH-15099)

* Update io.rst

* Apply suggestions from code review

Co-Authored-By: Ashwin Ramaswami <aramaswamis at gmail.com>

Co-Authored-By: Carol Willing <carolcode at willingconsulting.com>

files:
M Doc/library/io.rst

diff --git a/Doc/library/io.rst b/Doc/library/io.rst
index 70e01153d419..f0987da9b6a4 100644
--- a/Doc/library/io.rst
+++ b/Doc/library/io.rst
@@ -195,18 +195,18 @@ The :class:`RawIOBase` ABC extends :class:`IOBase`.  It deals with the reading
 and writing of bytes to a stream.  :class:`FileIO` subclasses :class:`RawIOBase`
 to provide an interface to files in the machine's file system.
 
-The :class:`BufferedIOBase` ABC deals with buffering on a raw byte stream
-(:class:`RawIOBase`).  Its subclasses, :class:`BufferedWriter`,
-:class:`BufferedReader`, and :class:`BufferedRWPair` buffer streams that are
-readable, writable, and both readable and writable.  :class:`BufferedRandom`
-provides a buffered interface to random access streams.  Another
-:class:`BufferedIOBase` subclass, :class:`BytesIO`, is a stream of in-memory
-bytes.
-
-The :class:`TextIOBase` ABC, another subclass of :class:`IOBase`, deals with
+The :class:`BufferedIOBase` ABC extends :class:`IOBase`.  It deals with
+buffering on a raw binary stream (:class:`RawIOBase`).  Its subclasses,
+:class:`BufferedWriter`, :class:`BufferedReader`, and :class:`BufferedRWPair`
+buffer raw binary streams that are readable, writable, and both readable and writable,
+respectively. :class:`BufferedRandom` provides a buffered interface to seekable streams.
+Another :class:`BufferedIOBase` subclass, :class:`BytesIO`, is a stream of
+in-memory bytes.
+
+The :class:`TextIOBase` ABC extends :class:`IOBase`.  It deals with
 streams whose bytes represent text, and handles encoding and decoding to and
-from strings. :class:`TextIOWrapper`, which extends it, is a buffered text
-interface to a buffered raw stream (:class:`BufferedIOBase`). Finally,
+from strings.  :class:`TextIOWrapper`, which extends :class:`TextIOBase`, is a buffered text
+interface to a buffered raw stream (:class:`BufferedIOBase`).  Finally,
 :class:`StringIO` is an in-memory stream for text.
 
 Argument names are not part of the specification, and only the arguments of
@@ -391,15 +391,16 @@ I/O Base Classes
 
 .. class:: RawIOBase
 
-   Base class for raw binary I/O.  It inherits :class:`IOBase`.  There is no
+   Base class for raw binary streams.  It inherits :class:`IOBase`.  There is no
    public constructor.
 
-   Raw binary I/O typically provides low-level access to an underlying OS
-   device or API, and does not try to encapsulate it in high-level primitives
-   (this is left to Buffered I/O and Text I/O, described later in this page).
+   Raw binary streams typically provide low-level access to an underlying OS
+   device or API, and do not try to encapsulate it in high-level primitives
+   (this functionality is done at a higher-level in buffered binary streams and text streams, described later
+   in this page).
 
-   In addition to the attributes and methods from :class:`IOBase`,
-   :class:`RawIOBase` provides the following methods:
+   :class:`RawIOBase` provides these methods in addition to those from
+   :class:`IOBase`:
 
    .. method:: read(size=-1)
 
@@ -463,8 +464,8 @@ I/O Base Classes
    :class:`RawIOBase` implementation, but wrap one, like
    :class:`BufferedWriter` and :class:`BufferedReader` do.
 
-   :class:`BufferedIOBase` provides or overrides these methods and attribute in
-   addition to those from :class:`IOBase`:
+   :class:`BufferedIOBase` provides or overrides these data attributes and
+   methods in addition to those from :class:`IOBase`:
 
    .. attribute:: raw
 
@@ -557,9 +558,8 @@ Raw File I/O
 
 .. class:: FileIO(name, mode='r', closefd=True, opener=None)
 
-   :class:`FileIO` represents an OS-level file containing bytes data.
-   It implements the :class:`RawIOBase` interface (and therefore the
-   :class:`IOBase` interface, too).
+   A raw binary stream representing an OS-level file containing bytes data.  It
+   inherits :class:`RawIOBase`.
 
    The *name* can be one of two things:
 
@@ -600,9 +600,8 @@ Raw File I/O
    .. versionchanged:: 3.4
       The file is now non-inheritable.
 
-   In addition to the attributes and methods from :class:`IOBase` and
-   :class:`RawIOBase`, :class:`FileIO` provides the following data
-   attributes:
+   :class:`FileIO` provides these data attributes in addition to those from
+   :class:`RawIOBase` and :class:`IOBase`:
 
    .. attribute:: mode
 
@@ -622,7 +621,7 @@ than raw I/O does.
 
 .. class:: BytesIO([initial_bytes])
 
-   A stream implementation using an in-memory bytes buffer.  It inherits
+   A binary stream using an in-memory bytes buffer.  It inherits
    :class:`BufferedIOBase`.  The buffer is discarded when the
    :meth:`~IOBase.close` method is called.
 
@@ -670,8 +669,10 @@ than raw I/O does.
 
 .. class:: BufferedReader(raw, buffer_size=DEFAULT_BUFFER_SIZE)
 
-   A buffer providing higher-level access to a readable, sequential
-   :class:`RawIOBase` object.  It inherits :class:`BufferedIOBase`.
+   A buffered binary stream providing higher-level access to a readable, non
+   seekable :class:`RawIOBase` raw binary stream.  It inherits
+   :class:`BufferedIOBase`.
+
    When reading data from this object, a larger amount of data may be
    requested from the underlying raw stream, and kept in an internal buffer.
    The buffered data can then be returned directly on subsequent reads.
@@ -706,8 +707,10 @@ than raw I/O does.
 
 .. class:: BufferedWriter(raw, buffer_size=DEFAULT_BUFFER_SIZE)
 
-   A buffer providing higher-level access to a writeable, sequential
-   :class:`RawIOBase` object.  It inherits :class:`BufferedIOBase`.
+   A buffered binary stream providing higher-level access to a writeable, non
+   seekable :class:`RawIOBase` raw binary stream.  It inherits
+   :class:`BufferedIOBase`.
+
    When writing to this object, data is normally placed into an internal
    buffer.  The buffer will be written out to the underlying :class:`RawIOBase`
    object under various conditions, including:
@@ -739,8 +742,9 @@ than raw I/O does.
 
 .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE)
 
-   A buffered interface to random access streams.  It inherits
-   :class:`BufferedReader` and :class:`BufferedWriter`.
+   A buffered binary stream providing higher-level access to a seekable
+   :class:`RawIOBase` raw binary stream.  It inherits :class:`BufferedReader`
+   and :class:`BufferedWriter`.
 
    The constructor creates a reader and writer for a seekable raw stream, given
    in the first argument.  If the *buffer_size* is omitted it defaults to
@@ -753,9 +757,9 @@ than raw I/O does.
 
 .. class:: BufferedRWPair(reader, writer, buffer_size=DEFAULT_BUFFER_SIZE)
 
-   A buffered I/O object combining two unidirectional :class:`RawIOBase`
-   objects -- one readable, the other writeable -- into a single bidirectional
-   endpoint.  It inherits :class:`BufferedIOBase`.
+   A buffered binary stream providing higher-level access to two non seekable
+   :class:`RawIOBase` raw binary streams---one readable, the other writeable.
+   It inherits :class:`BufferedIOBase`.
 
    *reader* and *writer* are :class:`RawIOBase` objects that are readable and
    writeable respectively.  If the *buffer_size* is omitted it defaults to
@@ -778,8 +782,8 @@ Text I/O
 .. class:: TextIOBase
 
    Base class for text streams.  This class provides a character and line based
-   interface to stream I/O.  It inherits :class:`IOBase`.
-   There is no public constructor.
+   interface to stream I/O.  It inherits :class:`IOBase`.  There is no public
+   constructor.
 
    :class:`TextIOBase` provides or overrides these data attributes and
    methods in addition to those from :class:`IOBase`:
@@ -867,8 +871,9 @@ Text I/O
 .. class:: TextIOWrapper(buffer, encoding=None, errors=None, newline=None, \
                          line_buffering=False, write_through=False)
 
-   A buffered text stream over a :class:`BufferedIOBase` binary stream.
-   It inherits :class:`TextIOBase`.
+   A buffered text stream providing higher-level access to a
+   :class:`BufferedIOBase` buffered binary stream.  It inherits
+   :class:`TextIOBase`.
 
    *encoding* gives the name of the encoding that the stream will be decoded or
    encoded with.  It defaults to
@@ -896,11 +901,11 @@ Text I/O
    * When reading input from the stream, if *newline* is ``None``,
      :term:`universal newlines` mode is enabled.  Lines in the input can end in
      ``'\n'``, ``'\r'``, or ``'\r\n'``, and these are translated into ``'\n'``
-     before being returned to the caller.  If it is ``''``, universal newlines
-     mode is enabled, but line endings are returned to the caller untranslated.
-     If it has any of the other legal values, input lines are only terminated
-     by the given string, and the line ending is returned to the caller
-     untranslated.
+     before being returned to the caller.  If *newline* is ``''``, universal
+     newlines mode is enabled, but line endings are returned to the caller
+     untranslated.  If *newline* has any of the other legal values, input lines
+     are only terminated by the given string, and the line ending is returned to
+     the caller untranslated.
 
    * When writing output to the stream, if *newline* is ``None``, any ``'\n'``
      characters written are translated to the system default line separator,
@@ -924,8 +929,8 @@ Text I/O
       locale encoding using :func:`locale.setlocale`, use the current locale
       encoding instead of the user preferred encoding.
 
-   :class:`TextIOWrapper` provides these members in addition to those of
-   :class:`TextIOBase` and its parents:
+   :class:`TextIOWrapper` provides these data attributes and methods in
+   addition to those from :class:`TextIOBase` and :class:`IOBase`:
 
    .. attribute:: line_buffering
 
@@ -960,22 +965,23 @@ Text I/O
 
 .. class:: StringIO(initial_value='', newline='\\n')
 
-   An in-memory stream for text I/O.  The text buffer is discarded when the
-   :meth:`~IOBase.close` method is called.
+   A text stream using an in-memory text buffer.  It inherits
+   :class:`TextIOBase`.
+
+   The text buffer is discarded when the :meth:`~IOBase.close` method is
+   called.
 
    The initial value of the buffer can be set by providing *initial_value*.
    If newline translation is enabled, newlines will be encoded as if by
    :meth:`~TextIOBase.write`.  The stream is positioned at the start of
    the buffer.
 
-   The *newline* argument works like that of :class:`TextIOWrapper`.
-   The default is to consider only ``\n`` characters as ends of lines and
-   to do no newline translation.  If *newline* is set to ``None``,
-   newlines are written as ``\n`` on all platforms, but universal
-   newline decoding is still performed when reading.
+   The *newline* argument works like that of :class:`TextIOWrapper`,
+   except that when writing output to the stream, if *newline* is ``None``,
+   newlines are written as ``\n`` on all platforms.
 
    :class:`StringIO` provides this method in addition to those from
-   :class:`TextIOBase` and its parents:
+   :class:`TextIOBase` and :class:`IOBase`:
 
    .. method:: getvalue()
 
@@ -1066,5 +1072,5 @@ buffered object.
 
 The above implicitly extends to text files, since the :func:`open()` function
 will wrap a buffered object inside a :class:`TextIOWrapper`.  This includes
-standard streams and therefore affects the built-in function :func:`print()` as
+standard streams and therefore affects the built-in :func:`print()` function as
 well.


From webhook-mailer at python.org  Wed Sep 11 10:59:00 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Wed, 11 Sep 2019 14:59:00 -0000
Subject: [Python-checkins] bpo-37885: venv: Don't produce unbound variable
 warning on deactivate (GH-15330)
Message-ID: <mailman.577.1568213940.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5209e586b7cac9a43b2c44349a26b1b0af06ead3
commit: 5209e586b7cac9a43b2c44349a26b1b0af06ead3
branch: master
author: Daniel Abrahamsson <hamsson at gmail.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-11T07:58:56-07:00
summary:

bpo-37885: venv: Don't produce unbound variable warning on deactivate (GH-15330)

Before, running deactivate from a bash shell configured to treat undefined variables as errors (`set -u`) would produce a warning:

``` 
$ python3 -m venv test
$ source test/bin/activate
(test) $ deactivate
-bash: $1: unbound variable
```

files:
A Misc/NEWS.d/next/Library/2019-08-19-10-31-41.bpo-37885.4Nc9sp.rst
M Lib/test/test_venv.py
M Lib/venv/scripts/common/activate

diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
index de93d9539c2c..0103de8828dd 100644
--- a/Lib/test/test_venv.py
+++ b/Lib/test/test_venv.py
@@ -9,6 +9,7 @@
 import os
 import os.path
 import re
+import shutil
 import struct
 import subprocess
 import sys
@@ -360,6 +361,25 @@ def test_multiprocessing(self):
             'pool.terminate()'])
         self.assertEqual(out.strip(), "python".encode())
 
+    @unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
+    def test_deactivate_with_strict_bash_opts(self):
+        bash = shutil.which("bash")
+        if bash is None:
+            self.skipTest("bash required for this test")
+        rmtree(self.env_dir)
+        builder = venv.EnvBuilder(clear=True)
+        builder.create(self.env_dir)
+        activate = os.path.join(self.env_dir, self.bindir, "activate")
+        test_script = os.path.join(self.env_dir, "test_strict.sh")
+        with open(test_script, "w") as f:
+            f.write("set -euo pipefail\n"
+                    f"source {activate}\n"
+                    "deactivate\n")
+        out, err = check_output([bash, test_script])
+        self.assertEqual(out, "".encode())
+        self.assertEqual(err, "".encode())
+
+
 @requireVenvCreate
 class EnsurePipTest(BaseTest):
     """Test venv module installation of pip."""
diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate
index 6a2320a369cb..b1b4625fdddc 100644
--- a/Lib/venv/scripts/common/activate
+++ b/Lib/venv/scripts/common/activate
@@ -28,7 +28,7 @@ deactivate () {
     fi
 
     unset VIRTUAL_ENV
-    if [ ! "$1" = "nondestructive" ] ; then
+    if [ ! "${1:-}" = "nondestructive" ] ; then
     # Self destruct!
         unset -f deactivate
     fi
diff --git a/Misc/NEWS.d/next/Library/2019-08-19-10-31-41.bpo-37885.4Nc9sp.rst b/Misc/NEWS.d/next/Library/2019-08-19-10-31-41.bpo-37885.4Nc9sp.rst
new file mode 100644
index 000000000000..055d02970146
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-19-10-31-41.bpo-37885.4Nc9sp.rst
@@ -0,0 +1 @@
+venv: Don't generate unset variable warning on deactivate.
\ No newline at end of file


From webhook-mailer at python.org  Wed Sep 11 11:04:06 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 15:04:06 -0000
Subject: [Python-checkins] Doc: Indicate locations of parse_qs, parse_qsl,
 escape (GH-14828)
Message-ID: <mailman.578.1568214247.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/aa527174f0b508cb4454956333b1a7c98bcc1ab1
commit: aa527174f0b508cb4454956333b1a7c98bcc1ab1
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T08:04:02-07:00
summary:

Doc: Indicate locations of parse_qs, parse_qsl, escape (GH-14828)


Since they have been removed from cgi it's useful to remind people where they
can be found instead.
(cherry picked from commit 1abf54336fd2cb545c453d22dd5501392b3350b2)

Co-authored-by: Simon Willison <swillison at gmail.com>

files:
M Doc/whatsnew/3.8.rst

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 07202ea4ff08..aefe9b53746f 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -1467,7 +1467,8 @@ The following features and APIs have been removed from Python 3.8:
   script is tied to. (Contributed by Brett Cannon in :issue:`25427`.)
 
 * ``parse_qs``, ``parse_qsl``, and ``escape`` are removed from :mod:`cgi`
-  module.  They are deprecated from Python 3.2 or older.
+  module.  They are deprecated from Python 3.2 or older. They should be imported
+  from the ``urllib.parse`` and ``html`` modules instead.
 
 * ``filemode`` function is removed from :mod:`tarfile` module.
   It is not documented and deprecated since Python 3.3.


From webhook-mailer at python.org  Wed Sep 11 11:04:15 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Wed, 11 Sep 2019 15:04:15 -0000
Subject: [Python-checkins] bpo-36260: Add pitfalls to zipfile module
 documentation (#13378)
Message-ID: <mailman.579.1568214256.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3ba51d587f6897a45301ce9126300c14fcd4eba2
commit: 3ba51d587f6897a45301ce9126300c14fcd4eba2
branch: master
author: JunWei Song <sungboss2004 at gmail.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-11T16:04:11+01:00
summary:

bpo-36260: Add pitfalls to zipfile module documentation (#13378)

* bpo-36260: Add pitfalls to zipfile module documentation

We saw vulnerability warning description (including zip bomb) in Doc/library/xml.rst file.
This gave us the idea of documentation improvement. 

So, we moved a little bit forward :P
And the doc patch can be found (pr).

* fix trailing whitespace

* ?? Added by blurb_it.

* Reformat text for consistency.

files:
A Misc/NEWS.d/next/Documentation/2019-06-04-09-29-00.bpo-36260.WrGuc-.rst
M Doc/library/zipfile.rst

diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst
index 9db9697105d6..18d2e9a60a7b 100644
--- a/Doc/library/zipfile.rst
+++ b/Doc/library/zipfile.rst
@@ -816,5 +816,45 @@ Command-line options
 
    Test whether the zipfile is valid or not.
 
+Decompression pitfalls
+----------------------
+
+The extraction in zipfile module might fail due to some pitfalls listed below.
+
+From file itself
+~~~~~~~~~~~~~~~~
+
+Decompression may fail due to incorrect password / CRC checksum / ZIP format or
+unsupported compression method / decryption.
+
+File System limitations
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Exceeding limitations on different file systems can cause decompression failed.
+Such as allowable characters in the directory entries, length of the file name,
+length of the pathname, size of a single file, and number of files, etc.
+
+Resources limitations
+~~~~~~~~~~~~~~~~~~~~~
+
+The lack of memory or disk volume would lead to decompression
+failed. For example, decompression bombs (aka `ZIP bomb`_)
+apply to zipfile library that can cause disk volume exhaustion.
+
+Interruption
+~~~~~~~~~~~~
+
+Interruption during the decompression, such as pressing control-C or killing the
+decompression process may result in incomplete decompression of the archive.
+
+Default behaviors of extraction
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Not knowing the default extraction behaviors
+can cause unexpected decompression results.
+For example, when extracting the same archive twice,
+it overwrites files without asking.
+
 
+.. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb
 .. _PKZIP Application Note: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
diff --git a/Misc/NEWS.d/next/Documentation/2019-06-04-09-29-00.bpo-36260.WrGuc-.rst b/Misc/NEWS.d/next/Documentation/2019-06-04-09-29-00.bpo-36260.WrGuc-.rst
new file mode 100644
index 000000000000..9276516a8823
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-06-04-09-29-00.bpo-36260.WrGuc-.rst
@@ -0,0 +1 @@
+Add decompression pitfalls to zipfile module documentation.
\ No newline at end of file


From webhook-mailer at python.org  Wed Sep 11 11:08:14 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Wed, 11 Sep 2019 15:08:14 -0000
Subject: [Python-checkins] bpo-36182: Update pathlib.Path.write_text() docs
 (GH-12161)
Message-ID: <mailman.580.1568214495.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/af636f4f91b8289b6dad95cb84123f6e22fd7f4f
commit: af636f4f91b8289b6dad95cb84123f6e22fd7f4f
branch: master
author: Lysandros Nikolaou <lisandrosnik at gmail.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-11T17:08:10+02:00
summary:

bpo-36182: Update pathlib.Path.write_text() docs (GH-12161)

with the case of an existing file

files:
M Doc/library/pathlib.rst

diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst
index d33e2d0c89b1..acbd0e48af24 100644
--- a/Doc/library/pathlib.rst
+++ b/Doc/library/pathlib.rst
@@ -1119,6 +1119,9 @@ call fails (for example because the path doesn't exist).
       >>> p.read_text()
       'Text file contents'
 
+   An existing file of the same name is overwritten. The optional parameters
+   have the same meaning as in :func:`open`.
+
    .. versionadded:: 3.5
 
 Correspondence to tools in the :mod:`os` module


From webhook-mailer at python.org  Wed Sep 11 11:08:51 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 15:08:51 -0000
Subject: [Python-checkins] bpo-37651: Document CancelledError is now a
 subclass of BaseException (GH-15950)
Message-ID: <mailman.581.1568214531.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e784bb7c6b2a52e80d7c03cb85e5faab11a1ccbd
commit: e784bb7c6b2a52e80d7c03cb85e5faab11a1ccbd
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T08:08:47-07:00
summary:

bpo-37651: Document CancelledError is now a subclass of BaseException (GH-15950)


https://bugs.python.org/issue37651

Automerge-Triggered-By: @1st1
(cherry picked from commit 7b69069e9aa0047a0dbe8af1a67aa2b355dc68d8)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
M Doc/library/asyncio-exceptions.rst

diff --git a/Doc/library/asyncio-exceptions.rst b/Doc/library/asyncio-exceptions.rst
index e49577a203e8..b1e6ecfda15e 100644
--- a/Doc/library/asyncio-exceptions.rst
+++ b/Doc/library/asyncio-exceptions.rst
@@ -25,26 +25,9 @@ Exceptions
    when asyncio Tasks are cancelled.  In almost all situations the
    exception must be re-raised.
 
-   .. important::
-
-      This exception is a subclass of :exc:`Exception`, so it can be
-      accidentally suppressed by an overly broad ``try..except`` block::
-
-        try:
-            await operation
-        except Exception:
-            # The cancellation is broken because the *except* block
-            # suppresses the CancelledError exception.
-            log.log('an error has occurred')
-
-      Instead, the following pattern should be used::
+   .. versionchanged:: 3.8
 
-        try:
-            await operation
-        except asyncio.CancelledError:
-            raise
-        except Exception:
-            log.log('an error has occurred')
+      :exc:`CancelledError` is now a subclass of :class:`BaseException`.
 
 
 .. exception:: InvalidStateError


From webhook-mailer at python.org  Wed Sep 11 11:12:02 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Wed, 11 Sep 2019 15:12:02 -0000
Subject: [Python-checkins] bpo-38110: Use fdwalk for os.closerange() when
 available. (GH-15224)
Message-ID: <mailman.582.1568214723.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e20134f889a0cfcc37a46979f31a1c98b800de07
commit: e20134f889a0cfcc37a46979f31a1c98b800de07
branch: master
author: Jakub Kul?k <Kulikjak at gmail.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-11T10:11:57-05:00
summary:

bpo-38110: Use fdwalk for os.closerange() when available. (GH-15224)

Use fdwalk() on platforms that support it to implement os.closerange().

files:
A Misc/NEWS.d/next/Library/2019-09-11-14-49-20.bpo-38110.A19Y-q.rst
M Modules/posixmodule.c
M configure
M configure.ac
M pyconfig.h.in

diff --git a/Misc/NEWS.d/next/Library/2019-09-11-14-49-20.bpo-38110.A19Y-q.rst b/Misc/NEWS.d/next/Library/2019-09-11-14-49-20.bpo-38110.A19Y-q.rst
new file mode 100644
index 000000000000..7b94a2e90a50
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-11-14-49-20.bpo-38110.A19Y-q.rst
@@ -0,0 +1,2 @@
+The os.closewalk() implementation now uses the libc fdwalk() API on
+platforms where it is available.
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index a0a2a305119e..88d3d8323950 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -8422,6 +8422,21 @@ os_close_impl(PyObject *module, int fd)
 }
 
 
+#ifdef HAVE_FDWALK
+static int
+_fdwalk_close_func(void *lohi, int fd)
+{
+    int lo = ((int *)lohi)[0];
+    int hi = ((int *)lohi)[1];
+
+    if (fd >= hi)
+        return 1;
+    else if (fd >= lo)
+        close(fd);
+    return 0;
+}
+#endif /* HAVE_FDWALK */
+
 /*[clinic input]
 os.closerange
 
@@ -8436,11 +8451,21 @@ static PyObject *
 os_closerange_impl(PyObject *module, int fd_low, int fd_high)
 /*[clinic end generated code: output=0ce5c20fcda681c2 input=5855a3d053ebd4ec]*/
 {
+#ifdef HAVE_FDWALK
+    int lohi[2];
+#else
     int i;
+#endif
     Py_BEGIN_ALLOW_THREADS
     _Py_BEGIN_SUPPRESS_IPH
+#ifdef HAVE_FDWALK
+    lohi[0] = Py_MAX(fd_low, 0);
+    lohi[1] = fd_high;
+    fdwalk(_fdwalk_close_func, lohi);
+#else
     for (i = Py_MAX(fd_low, 0); i < fd_high; i++)
         close(i);
+#endif
     _Py_END_SUPPRESS_IPH
     Py_END_ALLOW_THREADS
     Py_RETURN_NONE;
diff --git a/configure b/configure
index cb60ce256d43..25f9de9fbbb8 100755
--- a/configure
+++ b/configure
@@ -11500,7 +11500,7 @@ fi
 for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
  clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \
  faccessat fchmod fchmodat fchown fchownat \
- fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
+ fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
  futimens futimes gai_strerror getentropy \
  getgrgid_r getgrnam_r \
  getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
diff --git a/configure.ac b/configure.ac
index 9f5cea5ca0c7..5ca775cced09 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3538,7 +3538,7 @@ fi
 AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
  clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \
  faccessat fchmod fchmodat fchown fchownat \
- fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
+ fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
  futimens futimes gai_strerror getentropy \
  getgrgid_r getgrnam_r \
  getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
diff --git a/pyconfig.h.in b/pyconfig.h.in
index 5ff5cc2282f0..aca9751d4b75 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -337,6 +337,9 @@
 /* Define to 1 if you have the `fdopendir' function. */
 #undef HAVE_FDOPENDIR
 
+/* Define to 1 if you have the `fdwalk' function. */
+#undef HAVE_FDWALK
+
 /* Define to 1 if you have the `fexecve' function. */
 #undef HAVE_FEXECVE
 


From webhook-mailer at python.org  Wed Sep 11 11:15:43 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Wed, 11 Sep 2019 15:15:43 -0000
Subject: [Python-checkins] bpo-33166: Change os.cpu_count to return active
 (real) processors (GH-15949)
Message-ID: <mailman.583.1568214944.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/aa929273caca2f4e24e3aa9e790272fd4458ad35
commit: aa929273caca2f4e24e3aa9e790272fd4458ad35
branch: master
author: Steve Dower <steve.dower at python.org>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-11T16:15:39+01:00
summary:

bpo-33166: Change os.cpu_count to return active (real) processors (GH-15949)

files:
A Misc/NEWS.d/next/Windows/2019-09-11-14-42-04.bpo-36634.8Un8ih.rst
M Modules/posixmodule.c

diff --git a/Misc/NEWS.d/next/Windows/2019-09-11-14-42-04.bpo-36634.8Un8ih.rst b/Misc/NEWS.d/next/Windows/2019-09-11-14-42-04.bpo-36634.8Un8ih.rst
new file mode 100644
index 000000000000..1affdd81cbfa
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-11-14-42-04.bpo-36634.8Un8ih.rst
@@ -0,0 +1,2 @@
+:func:`os.cpu_count` now returns active processors rather than maximum
+processors.
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 88d3d8323950..089572f4fdab 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -12204,23 +12204,9 @@ os_cpu_count_impl(PyObject *module)
 {
     int ncpu = 0;
 #ifdef MS_WINDOWS
-    /* Vista is supported and the GetMaximumProcessorCount API is Win7+
-       Need to fallback to Vista behavior if this call isn't present */
-    HINSTANCE hKernel32;
-    static DWORD(CALLBACK *_GetMaximumProcessorCount)(WORD) = NULL;
-    Py_BEGIN_ALLOW_THREADS
-    hKernel32 = GetModuleHandleW(L"KERNEL32");
-    *(FARPROC*)&_GetMaximumProcessorCount = GetProcAddress(hKernel32,
-        "GetMaximumProcessorCount");
-    Py_END_ALLOW_THREADS
-    if (_GetMaximumProcessorCount != NULL) {
-        ncpu = _GetMaximumProcessorCount(ALL_PROCESSOR_GROUPS);
-    }
-    else {
-        SYSTEM_INFO sysinfo;
-        GetSystemInfo(&sysinfo);
-        ncpu = sysinfo.dwNumberOfProcessors;
-    }
+    /* Declare prototype here to avoid pulling in all of the Win7 APIs in 3.8 */
+    DWORD WINAPI GetActiveProcessorCount(WORD group);
+    ncpu = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
 #elif defined(__hpux)
     ncpu = mpctl(MPC_GETNUMSPUS, NULL, NULL);
 #elif defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN)


From webhook-mailer at python.org  Wed Sep 11 11:16:31 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Wed, 11 Sep 2019 15:16:31 -0000
Subject: [Python-checkins] bpo-38114: Do not include pip.ini in Nuget
 package (GH-15964)
Message-ID: <mailman.584.1568214992.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/19f6940cd7fb91246b88e1fbdbce97a02e7f3fa1
commit: 19f6940cd7fb91246b88e1fbdbce97a02e7f3fa1
branch: master
author: Steve Dower <steve.dower at python.org>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-11T16:16:27+01:00
summary:

bpo-38114: Do not include pip.ini in Nuget package (GH-15964)

files:
A Misc/NEWS.d/next/Windows/2019-09-11-15-24-04.bpo-38114.cc0E5E.rst
M PC/layout/support/options.py
M PC/layout/support/pip.py

diff --git a/Misc/NEWS.d/next/Windows/2019-09-11-15-24-04.bpo-38114.cc0E5E.rst b/Misc/NEWS.d/next/Windows/2019-09-11-15-24-04.bpo-38114.cc0E5E.rst
new file mode 100644
index 000000000000..5dab8d728e88
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-11-15-24-04.bpo-38114.cc0E5E.rst
@@ -0,0 +1 @@
+The ``pip.ini`` is no longer included in the Nuget package.
diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py
index c8ae4e30a8c4..5e1543b59ccd 100644
--- a/PC/layout/support/options.py
+++ b/PC/layout/support/options.py
@@ -17,6 +17,7 @@ def public(f):
 OPTIONS = {
     "stable": {"help": "stable ABI stub"},
     "pip": {"help": "pip"},
+    "pip-user": {"help": "pip.ini file for default --user"},
     "distutils": {"help": "distutils"},
     "tcltk": {"help": "Tcl, Tk and tkinter"},
     "idle": {"help": "Idle"},
@@ -42,6 +43,7 @@ def public(f):
         "options": [
             "stable",
             "pip",
+            "pip-user",
             "distutils",
             "tcltk",
             "idle",
diff --git a/PC/layout/support/pip.py b/PC/layout/support/pip.py
index 4ad3b1dd5bc0..c54acb250a25 100644
--- a/PC/layout/support/pip.py
+++ b/PC/layout/support/pip.py
@@ -33,11 +33,12 @@ def get_pip_layout(ns):
         pkg_root = "packages/{}" if ns.zip_lib else "Lib/site-packages/{}"
         for dest, src in rglob(pip_dir, "**/*"):
             yield pkg_root.format(dest), src
-        content = "\n".join(
-            "[{}]\nuser=yes".format(n)
-            for n in ["install", "uninstall", "freeze", "list"]
-        )
-        yield "pip.ini", ("pip.ini", content.encode())
+        if ns.include_pip_user:
+            content = "\n".join(
+                "[{}]\nuser=yes".format(n)
+                for n in ["install", "uninstall", "freeze", "list"]
+            )
+            yield "pip.ini", ("pip.ini", content.encode())
 
 
 def extract_pip_files(ns):


From webhook-mailer at python.org  Wed Sep 11 11:22:02 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 15:22:02 -0000
Subject: [Python-checkins] bpo-36634: Fixes activate.bat when existing
 values contain double quotes (GH-15924)
Message-ID: <mailman.585.1568215323.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/20f80bfc83506037e40d918fcac887cee9093c93
commit: 20f80bfc83506037e40d918fcac887cee9093c93
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T08:21:58-07:00
summary:

bpo-36634: Fixes activate.bat when existing values contain double quotes (GH-15924)

(cherry picked from commit 574b324bdc9a126b5a4488c3613f11ad2555415e)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-11-12-34-31.bpo-36634.xLaGgb.rst
M Lib/venv/scripts/nt/activate.bat

diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat
index da831bb42c79..8eb5c1ff664f 100644
--- a/Lib/venv/scripts/nt/activate.bat
+++ b/Lib/venv/scripts/nt/activate.bat
@@ -2,44 +2,32 @@
 
 rem This file is UTF-8 encoded, so we need to update the current code page while executing it
 for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do (
-    set "_OLD_CODEPAGE=%%a"
+    set _OLD_CODEPAGE=%%a
 )
 if defined _OLD_CODEPAGE (
     "%SystemRoot%\System32\chcp.com" 65001 > nul
 )
 
-set "VIRTUAL_ENV=__VENV_DIR__"
+set VIRTUAL_ENV=__VENV_DIR__
 
-if not defined PROMPT (
-    set "PROMPT=$P$G"
-)
-
-if defined _OLD_VIRTUAL_PROMPT (
-    set "PROMPT=%_OLD_VIRTUAL_PROMPT%"
-)
+if not defined PROMPT set PROMPT=$P$G
 
-if defined _OLD_VIRTUAL_PYTHONHOME (
-    set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%"
-)
+if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT%
+if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%
 
-set "_OLD_VIRTUAL_PROMPT=%PROMPT%"
-set "PROMPT=__VENV_PROMPT__%PROMPT%"
+set _OLD_VIRTUAL_PROMPT=%PROMPT%
+set PROMPT=__VENV_PROMPT__%PROMPT%
 
-if defined PYTHONHOME (
-    set "_OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%"
-    set PYTHONHOME=
-)
+if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%
+set PYTHONHOME=
 
-if defined _OLD_VIRTUAL_PATH (
-    set "PATH=%_OLD_VIRTUAL_PATH%"
-) else (
-    set "_OLD_VIRTUAL_PATH=%PATH%"
-)
+if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH%
+else set _OLD_VIRTUAL_PATH=%PATH%
 
-set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%"
+set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%
 
 :END
 if defined _OLD_CODEPAGE (
     "%SystemRoot%\System32\chcp.com" %_OLD_CODEPAGE% > nul
-    set "_OLD_CODEPAGE="
+    set _OLD_CODEPAGE=
 )
diff --git a/Misc/NEWS.d/next/Windows/2019-09-11-12-34-31.bpo-36634.xLaGgb.rst b/Misc/NEWS.d/next/Windows/2019-09-11-12-34-31.bpo-36634.xLaGgb.rst
new file mode 100644
index 000000000000..c0bd2beaae2d
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-11-12-34-31.bpo-36634.xLaGgb.rst
@@ -0,0 +1,2 @@
+venv activate.bat now works when the existing variables contain double quote
+characters.


From webhook-mailer at python.org  Wed Sep 11 11:23:11 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 15:23:11 -0000
Subject: [Python-checkins] bpo-36634: Fixes activate.bat when existing
 values contain double quotes (GH-15924)
Message-ID: <mailman.586.1568215393.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8a19a91e28b86592ee1ad9d1761e59b30d8511b9
commit: 8a19a91e28b86592ee1ad9d1761e59b30d8511b9
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T08:23:07-07:00
summary:

bpo-36634: Fixes activate.bat when existing values contain double quotes (GH-15924)

(cherry picked from commit 574b324bdc9a126b5a4488c3613f11ad2555415e)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-11-12-34-31.bpo-36634.xLaGgb.rst
M Lib/venv/scripts/nt/activate.bat

diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat
index da831bb42c79..8eb5c1ff664f 100644
--- a/Lib/venv/scripts/nt/activate.bat
+++ b/Lib/venv/scripts/nt/activate.bat
@@ -2,44 +2,32 @@
 
 rem This file is UTF-8 encoded, so we need to update the current code page while executing it
 for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do (
-    set "_OLD_CODEPAGE=%%a"
+    set _OLD_CODEPAGE=%%a
 )
 if defined _OLD_CODEPAGE (
     "%SystemRoot%\System32\chcp.com" 65001 > nul
 )
 
-set "VIRTUAL_ENV=__VENV_DIR__"
+set VIRTUAL_ENV=__VENV_DIR__
 
-if not defined PROMPT (
-    set "PROMPT=$P$G"
-)
-
-if defined _OLD_VIRTUAL_PROMPT (
-    set "PROMPT=%_OLD_VIRTUAL_PROMPT%"
-)
+if not defined PROMPT set PROMPT=$P$G
 
-if defined _OLD_VIRTUAL_PYTHONHOME (
-    set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%"
-)
+if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT%
+if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%
 
-set "_OLD_VIRTUAL_PROMPT=%PROMPT%"
-set "PROMPT=__VENV_PROMPT__%PROMPT%"
+set _OLD_VIRTUAL_PROMPT=%PROMPT%
+set PROMPT=__VENV_PROMPT__%PROMPT%
 
-if defined PYTHONHOME (
-    set "_OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%"
-    set PYTHONHOME=
-)
+if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%
+set PYTHONHOME=
 
-if defined _OLD_VIRTUAL_PATH (
-    set "PATH=%_OLD_VIRTUAL_PATH%"
-) else (
-    set "_OLD_VIRTUAL_PATH=%PATH%"
-)
+if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH%
+else set _OLD_VIRTUAL_PATH=%PATH%
 
-set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%"
+set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%
 
 :END
 if defined _OLD_CODEPAGE (
     "%SystemRoot%\System32\chcp.com" %_OLD_CODEPAGE% > nul
-    set "_OLD_CODEPAGE="
+    set _OLD_CODEPAGE=
 )
diff --git a/Misc/NEWS.d/next/Windows/2019-09-11-12-34-31.bpo-36634.xLaGgb.rst b/Misc/NEWS.d/next/Windows/2019-09-11-12-34-31.bpo-36634.xLaGgb.rst
new file mode 100644
index 000000000000..c0bd2beaae2d
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-11-12-34-31.bpo-36634.xLaGgb.rst
@@ -0,0 +1,2 @@
+venv activate.bat now works when the existing variables contain double quote
+characters.


From webhook-mailer at python.org  Wed Sep 11 11:36:45 2019
From: webhook-mailer at python.org (Stefan Behnel)
Date: Wed, 11 Sep 2019 15:36:45 -0000
Subject: [Python-checkins] bpo-33187: Document ElementInclude (XInclude)
 support in ElementTree (GH-8861) (GH-15972)
Message-ID: <mailman.587.1568216206.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/195dc142f84804ad9c8ce91414ab4a0bf9615f09
commit: 195dc142f84804ad9c8ce91414ab4a0bf9615f09
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Stefan Behnel <stefan_ml at behnel.de>
date: 2019-09-11T16:36:41+01:00
summary:

bpo-33187: Document ElementInclude (XInclude) support in ElementTree (GH-8861) (GH-15972)

(cherry picked from commit 97b817eae34b77be1ced382e15098a112f547848)

Co-authored-by: Anjali Bansal <anjali.mca17.du at gmail.com>

files:
M Doc/library/xml.etree.elementtree.rst
M Doc/tools/susp-ignored.csv

diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst
index 5f799aba994d..ca6bdf6095b8 100644
--- a/Doc/library/xml.etree.elementtree.rst
+++ b/Doc/library/xml.etree.elementtree.rst
@@ -640,6 +640,95 @@ Functions
    :class:`Element` instance and a dictionary.
 
 
+.. _elementtree-xinclude:
+
+XInclude support
+----------------
+
+This module provides limited support for
+`XInclude directives <https://www.w3.org/TR/xinclude/>`_, via the :mod:`xml.etree.ElementInclude` helper module.  This module can be used to insert subtrees and text strings into element trees, based on information in the tree.
+
+Example
+^^^^^^^
+
+Here's an example that demonstrates use of the XInclude module. To include an XML document in the current document, use the ``{http://www.w3.org/2001/XInclude}include`` element and set the **parse** attribute to ``"xml"``, and use the **href** attribute to specify the document to include.
+
+.. code-block:: xml
+
+    <?xml version="1.0"?>
+    <document xmlns:xi="http://www.w3.org/2001/XInclude">
+      <xi:include href="source.xml" parse="xml" />
+    </document>
+
+By default, the **href** attribute is treated as a file name. You can use custom loaders to override this behaviour. Also note that the standard helper does not support XPointer syntax.
+
+To process this file, load it as usual, and pass the root element to the :mod:`xml.etree.ElementTree` module:
+
+.. code-block:: python
+
+   from xml.etree import ElementTree, ElementInclude
+
+   tree = ElementTree.parse("document.xml")
+   root = tree.getroot()
+
+   ElementInclude.include(root)
+
+The ElementInclude module replaces the ``{http://www.w3.org/2001/XInclude}include`` element with the root element from the **source.xml** document. The result might look something like this:
+
+.. code-block:: xml
+
+    <document xmlns:xi="http://www.w3.org/2001/XInclude">
+      <para>This is a paragraph.</para>
+    </document>
+
+If the **parse** attribute is omitted, it defaults to "xml". The href attribute is required.
+
+To include a text document, use the ``{http://www.w3.org/2001/XInclude}include`` element, and set the **parse** attribute to "text":
+
+.. code-block:: xml
+
+    <?xml version="1.0"?>
+    <document xmlns:xi="http://www.w3.org/2001/XInclude">
+      Copyright (c) <xi:include href="year.txt" parse="text" />.
+    </document>
+
+The result might look something like:
+
+.. code-block:: xml
+
+    <document xmlns:xi="http://www.w3.org/2001/XInclude">
+      Copyright (c) 2003.
+    </document>
+
+Reference
+---------
+
+.. _elementinclude-functions:
+
+Functions
+^^^^^^^^^
+
+.. function:: xml.etree.ElementInclude.default_loader( href, parse, encoding=None)
+
+   Default loader. This default loader reads an included resource from disk.  *href* is a URL.
+   *parse* is for parse mode either "xml" or "text".  *encoding*
+   is an optional text encoding.  If not given, encoding is ``utf-8``.  Returns the
+   expanded resource.  If the parse mode is ``"xml"``, this is an ElementTree
+   instance.  If the parse mode is "text", this is a Unicode string.  If the
+   loader fails, it can return None or raise an exception.
+
+
+.. function:: xml.etree.ElementInclude.include( elem, loader=None)
+
+   This function expands XInclude directives.  *elem* is the root element.  *loader* is
+   an optional resource loader.  If omitted, it defaults to :func:`default_loader`.
+   If given, it should be a callable that implements the same interface as
+   :func:`default_loader`.  Returns the expanded resource.  If the parse mode is
+   ``"xml"``, this is an ElementTree instance.  If the parse mode is "text",
+   this is a Unicode string.  If the loader fails, it can return None or
+   raise an exception.
+
+
 .. _elementtree-element-objects:
 
 Element Objects
diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv
index 30271bb9ce93..35dfce2c5c80 100644
--- a/Doc/tools/susp-ignored.csv
+++ b/Doc/tools/susp-ignored.csv
@@ -332,6 +332,9 @@ library/xml.etree.elementtree,,:character,<fictional:character>Commander Clement
 library/xml.etree.elementtree,,:actor,"for actor in root.findall('real_person:actor', ns):"
 library/xml.etree.elementtree,,:name,"name = actor.find('real_person:name', ns)"
 library/xml.etree.elementtree,,:character,"for char in actor.findall('role:character', ns):"
+library/xml.etree.elementtree,,:xi,<document xmlns:xi="http://www.w3.org/2001/XInclude">
+library/xml.etree.elementtree,,:include,  <xi:include href="source.xml" parse="xml" />
+library/xml.etree.elementtree,,:include,  Copyright (c) <xi:include href="year.txt" parse="text" />.
 library/zipapp,,:main,"$ python -m zipapp myapp -m ""myapp:main"""
 library/zipapp,,:fn,"pkg.mod:fn"
 library/zipapp,,:callable,"pkg.module:callable"


From webhook-mailer at python.org  Wed Sep 11 11:47:20 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 15:47:20 -0000
Subject: [Python-checkins] bpo-38113: Update Python/ast.c to PEP-384
 (GH-15975)
Message-ID: <mailman.588.1568216841.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5b172c27f760dc5437e37dd8f1540eb76440e9a7
commit: 5b172c27f760dc5437e37dd8f1540eb76440e9a7
branch: master
author: Dino Viehland <dinoviehland at fb.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T08:47:16-07:00
summary:

bpo-38113: Update Python/ast.c to PEP-384 (GH-15975)



Removes statics for better subinterpreter support and moves to _PyType_Name


https://bugs.python.org/issue38113



Automerge-Triggered-By: @tiran

files:
M Python/ast.c

diff --git a/Python/ast.c b/Python/ast.c
index 11524029ae26..46815c271b62 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -294,7 +294,7 @@ validate_expr(expr_ty exp, expr_context_ty ctx)
         if (!validate_constant(exp->v.Constant.value)) {
             PyErr_Format(PyExc_TypeError,
                          "got an invalid type in Constant: %s",
-                         Py_TYPE(exp->v.Constant.value)->tp_name);
+                         _PyType_Name(Py_TYPE(exp->v.Constant.value)));
             return 0;
         }
         return 1;
@@ -618,12 +618,11 @@ new_identifier(const char *n, struct compiling *c)
        identifier; if so, normalize to NFKC. */
     if (!PyUnicode_IS_ASCII(id)) {
         PyObject *id2;
-        _Py_IDENTIFIER(NFKC);
         if (!c->c_normalize && !init_normalization(c)) {
             Py_DECREF(id);
             return NULL;
         }
-        PyObject *form = _PyUnicode_FromId(&PyId_NFKC);
+        PyObject *form = PyUnicode_InternFromString("NFKC");
         if (form == NULL) {
             Py_DECREF(id);
             return NULL;
@@ -637,7 +636,7 @@ new_identifier(const char *n, struct compiling *c)
             PyErr_Format(PyExc_TypeError,
                          "unicodedata.normalize() must return a string, not "
                          "%.200s",
-                         Py_TYPE(id2)->tp_name);
+                         _PyType_Name(Py_TYPE(id2)));
             Py_DECREF(id2);
             return NULL;
         }


From webhook-mailer at python.org  Wed Sep 11 11:55:26 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 15:55:26 -0000
Subject: [Python-checkins] Doc: Fix missing negation. (GH-14640)
Message-ID: <mailman.589.1568217327.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1fae844451b120b93880d9360f288c70e125520c
commit: 1fae844451b120b93880d9360f288c70e125520c
branch: master
author: Julien Palard <julien at palard.fr>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T08:55:22-07:00
summary:

Doc: Fix missing negation. (GH-14640)



Reported by Hug Capella on docs at .



Automerge-Triggered-By: @matrixise

files:
M Doc/library/re.rst

diff --git a/Doc/library/re.rst b/Doc/library/re.rst
index 158248c3d147..df29c4d39090 100644
--- a/Doc/library/re.rst
+++ b/Doc/library/re.rst
@@ -540,8 +540,8 @@ character ``'$'``.
    Matches any character which is not a word character. This is
    the opposite of ``\w``. If the :const:`ASCII` flag is used this
    becomes the equivalent of ``[^a-zA-Z0-9_]``.  If the :const:`LOCALE` flag is
-   used, matches characters considered alphanumeric in the current locale
-   and the underscore.
+   used, matches characters which are neither alphanumeric in the current locale
+   nor the underscore.
 
 .. index:: single: \Z; in regular expressions
 


From webhook-mailer at python.org  Wed Sep 11 11:55:35 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Wed, 11 Sep 2019 15:55:35 -0000
Subject: [Python-checkins] bpo-36182: Update pathlib.Path.write_text() docs
 (GH-12161) (GH-15977)
Message-ID: <mailman.590.1568217336.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/893653357cc83d49049debfeb9074a4ce99cd478
commit: 893653357cc83d49049debfeb9074a4ce99cd478
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-11T17:55:31+02:00
summary:

bpo-36182: Update pathlib.Path.write_text() docs (GH-12161) (GH-15977)

with the case of an existing file
(cherry picked from commit af636f4f91b8289b6dad95cb84123f6e22fd7f4f)

Co-authored-by: Lysandros Nikolaou <lisandrosnik at gmail.com>

files:
M Doc/library/pathlib.rst

diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst
index ad8f6011ad02..326bb2e933cd 100644
--- a/Doc/library/pathlib.rst
+++ b/Doc/library/pathlib.rst
@@ -1106,6 +1106,9 @@ call fails (for example because the path doesn't exist).
       >>> p.read_text()
       'Text file contents'
 
+   An existing file of the same name is overwritten. The optional parameters
+   have the same meaning as in :func:`open`.
+
    .. versionadded:: 3.5
 
 Correspondence to tools in the :mod:`os` module


From webhook-mailer at python.org  Wed Sep 11 11:56:01 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Wed, 11 Sep 2019 15:56:01 -0000
Subject: [Python-checkins] bpo-37885: venv: Don't produce unbound variable
 warning on deactivate (GH-15973)
Message-ID: <mailman.591.1568217363.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/63eefc35674ec12ab4d00af4feaf21de4cb1c91c
commit: 63eefc35674ec12ab4d00af4feaf21de4cb1c91c
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-11T08:55:57-07:00
summary:

bpo-37885: venv: Don't produce unbound variable warning on deactivate (GH-15973)

Before, running deactivate from a bash shell configured to treat undefined variables as errors (`set -u`) would produce a warning:

```
$ python3 -m venv test
$ source test/bin/activate
(test) $ deactivate
-bash: $1: unbound variable
```
(cherry picked from commit 5209e586b7cac9a43b2c44349a26b1b0af06ead3)

Co-authored-by: Daniel Abrahamsson <hamsson at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-08-19-10-31-41.bpo-37885.4Nc9sp.rst
M Lib/test/test_venv.py
M Lib/venv/scripts/common/activate

diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
index 7130dc6f8309..ede65935b650 100644
--- a/Lib/test/test_venv.py
+++ b/Lib/test/test_venv.py
@@ -9,6 +9,7 @@
 import os
 import os.path
 import re
+import shutil
 import struct
 import subprocess
 import sys
@@ -338,6 +339,25 @@ def test_multiprocessing(self):
             'pool.terminate()'])
         self.assertEqual(out.strip(), "python".encode())
 
+    @unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
+    def test_deactivate_with_strict_bash_opts(self):
+        bash = shutil.which("bash")
+        if bash is None:
+            self.skipTest("bash required for this test")
+        rmtree(self.env_dir)
+        builder = venv.EnvBuilder(clear=True)
+        builder.create(self.env_dir)
+        activate = os.path.join(self.env_dir, self.bindir, "activate")
+        test_script = os.path.join(self.env_dir, "test_strict.sh")
+        with open(test_script, "w") as f:
+            f.write("set -euo pipefail\n"
+                    f"source {activate}\n"
+                    "deactivate\n")
+        out, err = check_output([bash, test_script])
+        self.assertEqual(out, "".encode())
+        self.assertEqual(err, "".encode())
+
+
 @requireVenvCreate
 class EnsurePipTest(BaseTest):
     """Test venv module installation of pip."""
diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate
index fff0765af53b..b9d498fb2ef0 100644
--- a/Lib/venv/scripts/common/activate
+++ b/Lib/venv/scripts/common/activate
@@ -28,7 +28,7 @@ deactivate () {
     fi
 
     unset VIRTUAL_ENV
-    if [ ! "$1" = "nondestructive" ] ; then
+    if [ ! "${1:-}" = "nondestructive" ] ; then
     # Self destruct!
         unset -f deactivate
     fi
diff --git a/Misc/NEWS.d/next/Library/2019-08-19-10-31-41.bpo-37885.4Nc9sp.rst b/Misc/NEWS.d/next/Library/2019-08-19-10-31-41.bpo-37885.4Nc9sp.rst
new file mode 100644
index 000000000000..055d02970146
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-19-10-31-41.bpo-37885.4Nc9sp.rst
@@ -0,0 +1 @@
+venv: Don't generate unset variable warning on deactivate.
\ No newline at end of file


From webhook-mailer at python.org  Wed Sep 11 11:56:16 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 15:56:16 -0000
Subject: [Python-checkins] bpo-33166: Change os.cpu_count to return active
 (real) processors (GH-15949)
Message-ID: <mailman.592.1568217378.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/43ee0e2ca33ae28d37a3dddf10a648895cd46bda
commit: 43ee0e2ca33ae28d37a3dddf10a648895cd46bda
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T08:56:13-07:00
summary:

bpo-33166: Change os.cpu_count to return active (real) processors (GH-15949)

(cherry picked from commit aa929273caca2f4e24e3aa9e790272fd4458ad35)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-11-14-42-04.bpo-36634.8Un8ih.rst
M Modules/posixmodule.c

diff --git a/Misc/NEWS.d/next/Windows/2019-09-11-14-42-04.bpo-36634.8Un8ih.rst b/Misc/NEWS.d/next/Windows/2019-09-11-14-42-04.bpo-36634.8Un8ih.rst
new file mode 100644
index 000000000000..1affdd81cbfa
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-11-14-42-04.bpo-36634.8Un8ih.rst
@@ -0,0 +1,2 @@
+:func:`os.cpu_count` now returns active processors rather than maximum
+processors.
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 771c5616152c..cf6c00b7d6ae 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -12176,23 +12176,9 @@ os_cpu_count_impl(PyObject *module)
 {
     int ncpu = 0;
 #ifdef MS_WINDOWS
-    /* Vista is supported and the GetMaximumProcessorCount API is Win7+
-       Need to fallback to Vista behavior if this call isn't present */
-    HINSTANCE hKernel32;
-    static DWORD(CALLBACK *_GetMaximumProcessorCount)(WORD) = NULL;
-    Py_BEGIN_ALLOW_THREADS
-    hKernel32 = GetModuleHandleW(L"KERNEL32");
-    *(FARPROC*)&_GetMaximumProcessorCount = GetProcAddress(hKernel32,
-        "GetMaximumProcessorCount");
-    Py_END_ALLOW_THREADS
-    if (_GetMaximumProcessorCount != NULL) {
-        ncpu = _GetMaximumProcessorCount(ALL_PROCESSOR_GROUPS);
-    }
-    else {
-        SYSTEM_INFO sysinfo;
-        GetSystemInfo(&sysinfo);
-        ncpu = sysinfo.dwNumberOfProcessors;
-    }
+    /* Declare prototype here to avoid pulling in all of the Win7 APIs in 3.8 */
+    DWORD WINAPI GetActiveProcessorCount(WORD group);
+    ncpu = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
 #elif defined(__hpux)
     ncpu = mpctl(MPC_GETNUMSPUS, NULL, NULL);
 #elif defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN)


From webhook-mailer at python.org  Wed Sep 11 11:56:32 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Wed, 11 Sep 2019 15:56:32 -0000
Subject: [Python-checkins] bpo-37885: venv: Don't produce unbound variable
 warning on deactivate (GH-15974)
Message-ID: <mailman.593.1568217393.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d126fbddc960afd93ff070bc34209be256152943
commit: d126fbddc960afd93ff070bc34209be256152943
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-11T08:56:27-07:00
summary:

bpo-37885: venv: Don't produce unbound variable warning on deactivate (GH-15974)

Before, running deactivate from a bash shell configured to treat undefined variables as errors (`set -u`) would produce a warning:

```
$ python3 -m venv test
$ source test/bin/activate
(test) $ deactivate
-bash: $1: unbound variable
```
(cherry picked from commit 5209e586b7cac9a43b2c44349a26b1b0af06ead3)

Co-authored-by: Daniel Abrahamsson <hamsson at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-08-19-10-31-41.bpo-37885.4Nc9sp.rst
M Lib/test/test_venv.py
M Lib/venv/scripts/common/activate

diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
index 921512e03721..9cfd72f8b1fd 100644
--- a/Lib/test/test_venv.py
+++ b/Lib/test/test_venv.py
@@ -9,6 +9,7 @@
 import os
 import os.path
 import re
+import shutil
 import struct
 import subprocess
 import sys
@@ -325,6 +326,25 @@ def test_multiprocessing(self):
             'pool.terminate()'])
         self.assertEqual(out.strip(), "python".encode())
 
+    @unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
+    def test_deactivate_with_strict_bash_opts(self):
+        bash = shutil.which("bash")
+        if bash is None:
+            self.skipTest("bash required for this test")
+        rmtree(self.env_dir)
+        builder = venv.EnvBuilder(clear=True)
+        builder.create(self.env_dir)
+        activate = os.path.join(self.env_dir, self.bindir, "activate")
+        test_script = os.path.join(self.env_dir, "test_strict.sh")
+        with open(test_script, "w") as f:
+            f.write("set -euo pipefail\n"
+                    f"source {activate}\n"
+                    "deactivate\n")
+        out, err = check_output([bash, test_script])
+        self.assertEqual(out, "".encode())
+        self.assertEqual(err, "".encode())
+
+
 @requireVenvCreate
 class EnsurePipTest(BaseTest):
     """Test venv module installation of pip."""
diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate
index fff0765af53b..b9d498fb2ef0 100644
--- a/Lib/venv/scripts/common/activate
+++ b/Lib/venv/scripts/common/activate
@@ -28,7 +28,7 @@ deactivate () {
     fi
 
     unset VIRTUAL_ENV
-    if [ ! "$1" = "nondestructive" ] ; then
+    if [ ! "${1:-}" = "nondestructive" ] ; then
     # Self destruct!
         unset -f deactivate
     fi
diff --git a/Misc/NEWS.d/next/Library/2019-08-19-10-31-41.bpo-37885.4Nc9sp.rst b/Misc/NEWS.d/next/Library/2019-08-19-10-31-41.bpo-37885.4Nc9sp.rst
new file mode 100644
index 000000000000..055d02970146
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-19-10-31-41.bpo-37885.4Nc9sp.rst
@@ -0,0 +1 @@
+venv: Don't generate unset variable warning on deactivate.
\ No newline at end of file


From webhook-mailer at python.org  Wed Sep 11 11:57:59 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 15:57:59 -0000
Subject: [Python-checkins] bpo-38114: Do not include pip.ini in Nuget
 package (GH-15964)
Message-ID: <mailman.594.1568217480.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/436cd2c20b1d7c86be1c53a3ba3bceefcaff2b73
commit: 436cd2c20b1d7c86be1c53a3ba3bceefcaff2b73
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T08:57:54-07:00
summary:

bpo-38114: Do not include pip.ini in Nuget package (GH-15964)

(cherry picked from commit 19f6940cd7fb91246b88e1fbdbce97a02e7f3fa1)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-11-15-24-04.bpo-38114.cc0E5E.rst
M PC/layout/support/options.py
M PC/layout/support/pip.py

diff --git a/Misc/NEWS.d/next/Windows/2019-09-11-15-24-04.bpo-38114.cc0E5E.rst b/Misc/NEWS.d/next/Windows/2019-09-11-15-24-04.bpo-38114.cc0E5E.rst
new file mode 100644
index 000000000000..5dab8d728e88
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-11-15-24-04.bpo-38114.cc0E5E.rst
@@ -0,0 +1 @@
+The ``pip.ini`` is no longer included in the Nuget package.
diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py
index c8ae4e30a8c4..5e1543b59ccd 100644
--- a/PC/layout/support/options.py
+++ b/PC/layout/support/options.py
@@ -17,6 +17,7 @@ def public(f):
 OPTIONS = {
     "stable": {"help": "stable ABI stub"},
     "pip": {"help": "pip"},
+    "pip-user": {"help": "pip.ini file for default --user"},
     "distutils": {"help": "distutils"},
     "tcltk": {"help": "Tcl, Tk and tkinter"},
     "idle": {"help": "Idle"},
@@ -42,6 +43,7 @@ def public(f):
         "options": [
             "stable",
             "pip",
+            "pip-user",
             "distutils",
             "tcltk",
             "idle",
diff --git a/PC/layout/support/pip.py b/PC/layout/support/pip.py
index 4ad3b1dd5bc0..c54acb250a25 100644
--- a/PC/layout/support/pip.py
+++ b/PC/layout/support/pip.py
@@ -33,11 +33,12 @@ def get_pip_layout(ns):
         pkg_root = "packages/{}" if ns.zip_lib else "Lib/site-packages/{}"
         for dest, src in rglob(pip_dir, "**/*"):
             yield pkg_root.format(dest), src
-        content = "\n".join(
-            "[{}]\nuser=yes".format(n)
-            for n in ["install", "uninstall", "freeze", "list"]
-        )
-        yield "pip.ini", ("pip.ini", content.encode())
+        if ns.include_pip_user:
+            content = "\n".join(
+                "[{}]\nuser=yes".format(n)
+                for n in ["install", "uninstall", "freeze", "list"]
+            )
+            yield "pip.ini", ("pip.ini", content.encode())
 
 
 def extract_pip_files(ns):


From webhook-mailer at python.org  Wed Sep 11 12:00:06 2019
From: webhook-mailer at python.org (Brett Cannon)
Date: Wed, 11 Sep 2019 16:00:06 -0000
Subject: [Python-checkins] bpo-35923: Update the BuiltinImporter to use
 loader._ORIGIN instead of a hard-coded value (GH-15651)
Message-ID: <mailman.595.1568217607.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/145cf1f50c8a8e8233e641f345cd5e25ee69190a
commit: 145cf1f50c8a8e8233e641f345cd5e25ee69190a
branch: master
author: Dong-hee Na <donghee.na92 at gmail.com>
committer: Brett Cannon <54418+brettcannon at users.noreply.github.com>
date: 2019-09-11T17:00:02+01:00
summary:

bpo-35923: Update the BuiltinImporter to use loader._ORIGIN instead of a hard-coded value (GH-15651)

files:
A Misc/NEWS.d/next/Library/2019-09-03-01-41-35.bpo-35923.lYpKbY.rst
M Lib/importlib/_bootstrap.py
M Python/importlib.h

diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index e17eeb658538..8de0e9ee79fd 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -713,6 +713,8 @@ class BuiltinImporter:
 
     """
 
+    _ORIGIN = "built-in"
+
     @staticmethod
     def module_repr(module):
         """Return repr for the module.
@@ -720,14 +722,14 @@ def module_repr(module):
         The method is deprecated.  The import machinery does the job itself.
 
         """
-        return '<module {!r} (built-in)>'.format(module.__name__)
+        return f'<module {module.__name__!r} ({BuiltinImporter._ORIGIN})>'
 
     @classmethod
     def find_spec(cls, fullname, path=None, target=None):
         if path is not None:
             return None
         if _imp.is_builtin(fullname):
-            return spec_from_loader(fullname, cls, origin='built-in')
+            return spec_from_loader(fullname, cls, origin=cls._ORIGIN)
         else:
             return None
 
diff --git a/Misc/NEWS.d/next/Library/2019-09-03-01-41-35.bpo-35923.lYpKbY.rst b/Misc/NEWS.d/next/Library/2019-09-03-01-41-35.bpo-35923.lYpKbY.rst
new file mode 100644
index 000000000000..bd2bba75675b
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-03-01-41-35.bpo-35923.lYpKbY.rst
@@ -0,0 +1,2 @@
+Update :class:`importlib.machinery.BuiltinImporter` to use ``loader._ORIGIN``
+instead of a hardcoded value. Patch by Dong-hee Na.
diff --git a/Python/importlib.h b/Python/importlib.h
index 8f25dcef31a6..67c820cd0364 100644
--- a/Python/importlib.h
+++ b/Python/importlib.h
@@ -1023,768 +1023,769 @@ const unsigned char _Py_M__importlib_bootstrap[] = {
     0,0,0,114,10,0,0,0,114,11,0,0,0,114,93,0,
     0,0,180,2,0,0,115,4,0,0,0,0,9,12,1,114,
     93,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,4,0,0,0,64,0,0,0,115,136,0,
-    0,0,101,0,90,1,100,0,90,2,100,1,90,3,101,4,
-    100,2,100,3,132,0,131,1,90,5,101,6,100,19,100,5,
-    100,6,132,1,131,1,90,7,101,6,100,20,100,7,100,8,
-    132,1,131,1,90,8,101,6,100,9,100,10,132,0,131,1,
-    90,9,101,6,100,11,100,12,132,0,131,1,90,10,101,6,
-    101,11,100,13,100,14,132,0,131,1,131,1,90,12,101,6,
-    101,11,100,15,100,16,132,0,131,1,131,1,90,13,101,6,
-    101,11,100,17,100,18,132,0,131,1,131,1,90,14,101,6,
-    101,15,131,1,90,16,100,4,83,0,41,21,218,15,66,117,
-    105,108,116,105,110,73,109,112,111,114,116,101,114,122,144,77,
-    101,116,97,32,112,97,116,104,32,105,109,112,111,114,116,32,
-    102,111,114,32,98,117,105,108,116,45,105,110,32,109,111,100,
+    0,0,0,0,0,4,0,0,0,64,0,0,0,115,140,0,
+    0,0,101,0,90,1,100,0,90,2,100,1,90,3,100,2,
+    90,4,101,5,100,3,100,4,132,0,131,1,90,6,101,7,
+    100,20,100,6,100,7,132,1,131,1,90,8,101,7,100,21,
+    100,8,100,9,132,1,131,1,90,9,101,7,100,10,100,11,
+    132,0,131,1,90,10,101,7,100,12,100,13,132,0,131,1,
+    90,11,101,7,101,12,100,14,100,15,132,0,131,1,131,1,
+    90,13,101,7,101,12,100,16,100,17,132,0,131,1,131,1,
+    90,14,101,7,101,12,100,18,100,19,132,0,131,1,131,1,
+    90,15,101,7,101,16,131,1,90,17,100,5,83,0,41,22,
+    218,15,66,117,105,108,116,105,110,73,109,112,111,114,116,101,
+    114,122,144,77,101,116,97,32,112,97,116,104,32,105,109,112,
+    111,114,116,32,102,111,114,32,98,117,105,108,116,45,105,110,
+    32,109,111,100,117,108,101,115,46,10,10,32,32,32,32,65,
+    108,108,32,109,101,116,104,111,100,115,32,97,114,101,32,101,
+    105,116,104,101,114,32,99,108,97,115,115,32,111,114,32,115,
+    116,97,116,105,99,32,109,101,116,104,111,100,115,32,116,111,
+    32,97,118,111,105,100,32,116,104,101,32,110,101,101,100,32,
+    116,111,10,32,32,32,32,105,110,115,116,97,110,116,105,97,
+    116,101,32,116,104,101,32,99,108,97,115,115,46,10,10,32,
+    32,32,32,122,8,98,117,105,108,116,45,105,110,99,1,0,
+    0,0,0,0,0,0,0,0,0,0,1,0,0,0,5,0,
+    0,0,67,0,0,0,115,22,0,0,0,100,1,124,0,106,
+    0,155,2,100,2,116,1,106,2,155,0,100,3,157,5,83,
+    0,41,4,250,115,82,101,116,117,114,110,32,114,101,112,114,
+    32,102,111,114,32,116,104,101,32,109,111,100,117,108,101,46,
+    10,10,32,32,32,32,32,32,32,32,84,104,101,32,109,101,
+    116,104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,
+    101,100,46,32,32,84,104,101,32,105,109,112,111,114,116,32,
+    109,97,99,104,105,110,101,114,121,32,100,111,101,115,32,116,
+    104,101,32,106,111,98,32,105,116,115,101,108,102,46,10,10,
+    32,32,32,32,32,32,32,32,122,8,60,109,111,100,117,108,
+    101,32,122,2,32,40,122,2,41,62,41,3,114,1,0,0,
+    0,114,159,0,0,0,114,137,0,0,0,41,1,114,95,0,
+    0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,
+    0,114,98,0,0,0,206,2,0,0,115,2,0,0,0,0,
+    7,122,27,66,117,105,108,116,105,110,73,109,112,111,114,116,
+    101,114,46,109,111,100,117,108,101,95,114,101,112,114,78,99,
+    4,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,
+    5,0,0,0,67,0,0,0,115,46,0,0,0,124,2,100,
+    0,107,9,114,12,100,0,83,0,116,0,160,1,124,1,161,
+    1,114,38,116,2,124,1,124,0,124,0,106,3,100,1,141,
+    3,83,0,100,0,83,0,100,0,83,0,169,2,78,114,136,
+    0,0,0,41,4,114,56,0,0,0,90,10,105,115,95,98,
+    117,105,108,116,105,110,114,90,0,0,0,114,137,0,0,0,
+    169,4,218,3,99,108,115,114,80,0,0,0,218,4,112,97,
+    116,104,218,6,116,97,114,103,101,116,114,10,0,0,0,114,
+    10,0,0,0,114,11,0,0,0,218,9,102,105,110,100,95,
+    115,112,101,99,215,2,0,0,115,10,0,0,0,0,2,8,
+    1,4,1,10,1,16,2,122,25,66,117,105,108,116,105,110,
+    73,109,112,111,114,116,101,114,46,102,105,110,100,95,115,112,
+    101,99,99,3,0,0,0,0,0,0,0,0,0,0,0,4,
+    0,0,0,4,0,0,0,67,0,0,0,115,30,0,0,0,
+    124,0,160,0,124,1,124,2,161,2,125,3,124,3,100,1,
+    107,9,114,26,124,3,106,1,83,0,100,1,83,0,41,2,
+    122,175,70,105,110,100,32,116,104,101,32,98,117,105,108,116,
+    45,105,110,32,109,111,100,117,108,101,46,10,10,32,32,32,
+    32,32,32,32,32,73,102,32,39,112,97,116,104,39,32,105,
+    115,32,101,118,101,114,32,115,112,101,99,105,102,105,101,100,
+    32,116,104,101,110,32,116,104,101,32,115,101,97,114,99,104,
+    32,105,115,32,99,111,110,115,105,100,101,114,101,100,32,97,
+    32,102,97,105,108,117,114,101,46,10,10,32,32,32,32,32,
+    32,32,32,84,104,105,115,32,109,101,116,104,111,100,32,105,
+    115,32,100,101,112,114,101,99,97,116,101,100,46,32,32,85,
+    115,101,32,102,105,110,100,95,115,112,101,99,40,41,32,105,
+    110,115,116,101,97,100,46,10,10,32,32,32,32,32,32,32,
+    32,78,41,2,114,166,0,0,0,114,108,0,0,0,41,4,
+    114,163,0,0,0,114,80,0,0,0,114,164,0,0,0,114,
+    94,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,
+    0,0,0,218,11,102,105,110,100,95,109,111,100,117,108,101,
+    224,2,0,0,115,4,0,0,0,0,9,12,1,122,27,66,
+    117,105,108,116,105,110,73,109,112,111,114,116,101,114,46,102,
+    105,110,100,95,109,111,100,117,108,101,99,2,0,0,0,0,
+    0,0,0,0,0,0,0,2,0,0,0,4,0,0,0,67,
+    0,0,0,115,46,0,0,0,124,1,106,0,116,1,106,2,
+    107,7,114,34,116,3,100,1,160,4,124,1,106,0,161,1,
+    124,1,106,0,100,2,141,2,130,1,116,5,116,6,106,7,
+    124,1,131,2,83,0,41,3,122,24,67,114,101,97,116,101,
+    32,97,32,98,117,105,108,116,45,105,110,32,109,111,100,117,
+    108,101,114,76,0,0,0,114,16,0,0,0,41,8,114,17,
+    0,0,0,114,15,0,0,0,114,77,0,0,0,114,78,0,
+    0,0,114,44,0,0,0,114,66,0,0,0,114,56,0,0,
+    0,90,14,99,114,101,97,116,101,95,98,117,105,108,116,105,
+    110,41,2,114,30,0,0,0,114,94,0,0,0,114,10,0,
+    0,0,114,10,0,0,0,114,11,0,0,0,114,148,0,0,
+    0,236,2,0,0,115,10,0,0,0,0,3,12,1,12,1,
+    4,255,6,2,122,29,66,117,105,108,116,105,110,73,109,112,
+    111,114,116,101,114,46,99,114,101,97,116,101,95,109,111,100,
+    117,108,101,99,2,0,0,0,0,0,0,0,0,0,0,0,
+    2,0,0,0,3,0,0,0,67,0,0,0,115,16,0,0,
+    0,116,0,116,1,106,2,124,1,131,2,1,0,100,1,83,
+    0,41,2,122,22,69,120,101,99,32,97,32,98,117,105,108,
+    116,45,105,110,32,109,111,100,117,108,101,78,41,3,114,66,
+    0,0,0,114,56,0,0,0,90,12,101,120,101,99,95,98,
+    117,105,108,116,105,110,41,2,114,30,0,0,0,114,95,0,
+    0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,
+    0,114,149,0,0,0,244,2,0,0,115,2,0,0,0,0,
+    3,122,27,66,117,105,108,116,105,110,73,109,112,111,114,116,
+    101,114,46,101,120,101,99,95,109,111,100,117,108,101,99,2,
+    0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,1,
+    0,0,0,67,0,0,0,115,4,0,0,0,100,1,83,0,
+    41,2,122,57,82,101,116,117,114,110,32,78,111,110,101,32,
+    97,115,32,98,117,105,108,116,45,105,110,32,109,111,100,117,
+    108,101,115,32,100,111,32,110,111,116,32,104,97,118,101,32,
+    99,111,100,101,32,111,98,106,101,99,116,115,46,78,114,10,
+    0,0,0,169,2,114,163,0,0,0,114,80,0,0,0,114,
+    10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,8,
+    103,101,116,95,99,111,100,101,249,2,0,0,115,2,0,0,
+    0,0,4,122,24,66,117,105,108,116,105,110,73,109,112,111,
+    114,116,101,114,46,103,101,116,95,99,111,100,101,99,2,0,
+    0,0,0,0,0,0,0,0,0,0,2,0,0,0,1,0,
+    0,0,67,0,0,0,115,4,0,0,0,100,1,83,0,41,
+    2,122,56,82,101,116,117,114,110,32,78,111,110,101,32,97,
+    115,32,98,117,105,108,116,45,105,110,32,109,111,100,117,108,
+    101,115,32,100,111,32,110,111,116,32,104,97,118,101,32,115,
+    111,117,114,99,101,32,99,111,100,101,46,78,114,10,0,0,
+    0,114,168,0,0,0,114,10,0,0,0,114,10,0,0,0,
+    114,11,0,0,0,218,10,103,101,116,95,115,111,117,114,99,
+    101,255,2,0,0,115,2,0,0,0,0,4,122,26,66,117,
+    105,108,116,105,110,73,109,112,111,114,116,101,114,46,103,101,
+    116,95,115,111,117,114,99,101,99,2,0,0,0,0,0,0,
+    0,0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,
+    0,115,4,0,0,0,100,1,83,0,41,2,122,52,82,101,
+    116,117,114,110,32,70,97,108,115,101,32,97,115,32,98,117,
+    105,108,116,45,105,110,32,109,111,100,117,108,101,115,32,97,
+    114,101,32,110,101,118,101,114,32,112,97,99,107,97,103,101,
+    115,46,70,114,10,0,0,0,114,168,0,0,0,114,10,0,
+    0,0,114,10,0,0,0,114,11,0,0,0,114,114,0,0,
+    0,5,3,0,0,115,2,0,0,0,0,4,122,26,66,117,
+    105,108,116,105,110,73,109,112,111,114,116,101,114,46,105,115,
+    95,112,97,99,107,97,103,101,41,2,78,78,41,1,78,41,
+    18,114,1,0,0,0,114,0,0,0,0,114,2,0,0,0,
+    114,3,0,0,0,114,137,0,0,0,218,12,115,116,97,116,
+    105,99,109,101,116,104,111,100,114,98,0,0,0,218,11,99,
+    108,97,115,115,109,101,116,104,111,100,114,166,0,0,0,114,
+    167,0,0,0,114,148,0,0,0,114,149,0,0,0,114,85,
+    0,0,0,114,169,0,0,0,114,170,0,0,0,114,114,0,
+    0,0,114,96,0,0,0,114,155,0,0,0,114,10,0,0,
+    0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,
+    114,159,0,0,0,195,2,0,0,115,44,0,0,0,8,2,
+    4,7,4,2,2,1,10,8,2,1,12,8,2,1,12,11,
+    2,1,10,7,2,1,10,4,2,1,2,1,12,4,2,1,
+    2,1,12,4,2,1,2,1,12,4,114,159,0,0,0,99,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    4,0,0,0,64,0,0,0,115,144,0,0,0,101,0,90,
+    1,100,0,90,2,100,1,90,3,100,2,90,4,101,5,100,
+    3,100,4,132,0,131,1,90,6,101,7,100,22,100,6,100,
+    7,132,1,131,1,90,8,101,7,100,23,100,8,100,9,132,
+    1,131,1,90,9,101,7,100,10,100,11,132,0,131,1,90,
+    10,101,5,100,12,100,13,132,0,131,1,90,11,101,7,100,
+    14,100,15,132,0,131,1,90,12,101,7,101,13,100,16,100,
+    17,132,0,131,1,131,1,90,14,101,7,101,13,100,18,100,
+    19,132,0,131,1,131,1,90,15,101,7,101,13,100,20,100,
+    21,132,0,131,1,131,1,90,16,100,5,83,0,41,24,218,
+    14,70,114,111,122,101,110,73,109,112,111,114,116,101,114,122,
+    142,77,101,116,97,32,112,97,116,104,32,105,109,112,111,114,
+    116,32,102,111,114,32,102,114,111,122,101,110,32,109,111,100,
     117,108,101,115,46,10,10,32,32,32,32,65,108,108,32,109,
     101,116,104,111,100,115,32,97,114,101,32,101,105,116,104,101,
     114,32,99,108,97,115,115,32,111,114,32,115,116,97,116,105,
     99,32,109,101,116,104,111,100,115,32,116,111,32,97,118,111,
     105,100,32,116,104,101,32,110,101,101,100,32,116,111,10,32,
     32,32,32,105,110,115,116,97,110,116,105,97,116,101,32,116,
-    104,101,32,99,108,97,115,115,46,10,10,32,32,32,32,99,
-    1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,
-    3,0,0,0,67,0,0,0,115,12,0,0,0,100,1,160,
-    0,124,0,106,1,161,1,83,0,41,2,250,115,82,101,116,
-    117,114,110,32,114,101,112,114,32,102,111,114,32,116,104,101,
-    32,109,111,100,117,108,101,46,10,10,32,32,32,32,32,32,
-    32,32,84,104,101,32,109,101,116,104,111,100,32,105,115,32,
-    100,101,112,114,101,99,97,116,101,100,46,32,32,84,104,101,
-    32,105,109,112,111,114,116,32,109,97,99,104,105,110,101,114,
-    121,32,100,111,101,115,32,116,104,101,32,106,111,98,32,105,
-    116,115,101,108,102,46,10,10,32,32,32,32,32,32,32,32,
-    122,24,60,109,111,100,117,108,101,32,123,33,114,125,32,40,
-    98,117,105,108,116,45,105,110,41,62,41,2,114,44,0,0,
-    0,114,1,0,0,0,41,1,114,95,0,0,0,114,10,0,
-    0,0,114,10,0,0,0,114,11,0,0,0,114,98,0,0,
-    0,204,2,0,0,115,2,0,0,0,0,7,122,27,66,117,
-    105,108,116,105,110,73,109,112,111,114,116,101,114,46,109,111,
-    100,117,108,101,95,114,101,112,114,78,99,4,0,0,0,0,
-    0,0,0,0,0,0,0,4,0,0,0,5,0,0,0,67,
-    0,0,0,115,44,0,0,0,124,2,100,0,107,9,114,12,
-    100,0,83,0,116,0,160,1,124,1,161,1,114,36,116,2,
-    124,1,124,0,100,1,100,2,141,3,83,0,100,0,83,0,
-    100,0,83,0,41,3,78,122,8,98,117,105,108,116,45,105,
-    110,114,136,0,0,0,41,3,114,56,0,0,0,90,10,105,
-    115,95,98,117,105,108,116,105,110,114,90,0,0,0,169,4,
-    218,3,99,108,115,114,80,0,0,0,218,4,112,97,116,104,
-    218,6,116,97,114,103,101,116,114,10,0,0,0,114,10,0,
-    0,0,114,11,0,0,0,218,9,102,105,110,100,95,115,112,
-    101,99,213,2,0,0,115,10,0,0,0,0,2,8,1,4,
-    1,10,1,14,2,122,25,66,117,105,108,116,105,110,73,109,
-    112,111,114,116,101,114,46,102,105,110,100,95,115,112,101,99,
-    99,3,0,0,0,0,0,0,0,0,0,0,0,4,0,0,
-    0,4,0,0,0,67,0,0,0,115,30,0,0,0,124,0,
-    160,0,124,1,124,2,161,2,125,3,124,3,100,1,107,9,
-    114,26,124,3,106,1,83,0,100,1,83,0,41,2,122,175,
-    70,105,110,100,32,116,104,101,32,98,117,105,108,116,45,105,
-    110,32,109,111,100,117,108,101,46,10,10,32,32,32,32,32,
-    32,32,32,73,102,32,39,112,97,116,104,39,32,105,115,32,
-    101,118,101,114,32,115,112,101,99,105,102,105,101,100,32,116,
-    104,101,110,32,116,104,101,32,115,101,97,114,99,104,32,105,
-    115,32,99,111,110,115,105,100,101,114,101,100,32,97,32,102,
-    97,105,108,117,114,101,46,10,10,32,32,32,32,32,32,32,
-    32,84,104,105,115,32,109,101,116,104,111,100,32,105,115,32,
-    100,101,112,114,101,99,97,116,101,100,46,32,32,85,115,101,
-    32,102,105,110,100,95,115,112,101,99,40,41,32,105,110,115,
-    116,101,97,100,46,10,10,32,32,32,32,32,32,32,32,78,
-    41,2,114,165,0,0,0,114,108,0,0,0,41,4,114,162,
-    0,0,0,114,80,0,0,0,114,163,0,0,0,114,94,0,
-    0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,
-    0,218,11,102,105,110,100,95,109,111,100,117,108,101,222,2,
-    0,0,115,4,0,0,0,0,9,12,1,122,27,66,117,105,
-    108,116,105,110,73,109,112,111,114,116,101,114,46,102,105,110,
-    100,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0,
-    0,0,0,0,0,2,0,0,0,4,0,0,0,67,0,0,
-    0,115,46,0,0,0,124,1,106,0,116,1,106,2,107,7,
-    114,34,116,3,100,1,160,4,124,1,106,0,161,1,124,1,
-    106,0,100,2,141,2,130,1,116,5,116,6,106,7,124,1,
-    131,2,83,0,41,3,122,24,67,114,101,97,116,101,32,97,
-    32,98,117,105,108,116,45,105,110,32,109,111,100,117,108,101,
-    114,76,0,0,0,114,16,0,0,0,41,8,114,17,0,0,
-    0,114,15,0,0,0,114,77,0,0,0,114,78,0,0,0,
-    114,44,0,0,0,114,66,0,0,0,114,56,0,0,0,90,
-    14,99,114,101,97,116,101,95,98,117,105,108,116,105,110,41,
-    2,114,30,0,0,0,114,94,0,0,0,114,10,0,0,0,
-    114,10,0,0,0,114,11,0,0,0,114,148,0,0,0,234,
-    2,0,0,115,10,0,0,0,0,3,12,1,12,1,4,255,
-    6,2,122,29,66,117,105,108,116,105,110,73,109,112,111,114,
-    116,101,114,46,99,114,101,97,116,101,95,109,111,100,117,108,
+    104,101,32,99,108,97,115,115,46,10,10,32,32,32,32,90,
+    6,102,114,111,122,101,110,99,1,0,0,0,0,0,0,0,
+    0,0,0,0,1,0,0,0,4,0,0,0,67,0,0,0,
+    115,16,0,0,0,100,1,160,0,124,0,106,1,116,2,106,
+    3,161,2,83,0,41,2,114,160,0,0,0,114,152,0,0,
+    0,41,4,114,44,0,0,0,114,1,0,0,0,114,173,0,
+    0,0,114,137,0,0,0,41,1,218,1,109,114,10,0,0,
+    0,114,10,0,0,0,114,11,0,0,0,114,98,0,0,0,
+    25,3,0,0,115,2,0,0,0,0,7,122,26,70,114,111,
+    122,101,110,73,109,112,111,114,116,101,114,46,109,111,100,117,
+    108,101,95,114,101,112,114,78,99,4,0,0,0,0,0,0,
+    0,0,0,0,0,4,0,0,0,5,0,0,0,67,0,0,
+    0,115,34,0,0,0,116,0,160,1,124,1,161,1,114,26,
+    116,2,124,1,124,0,124,0,106,3,100,1,141,3,83,0,
+    100,0,83,0,100,0,83,0,114,161,0,0,0,41,4,114,
+    56,0,0,0,114,87,0,0,0,114,90,0,0,0,114,137,
+    0,0,0,114,162,0,0,0,114,10,0,0,0,114,10,0,
+    0,0,114,11,0,0,0,114,166,0,0,0,34,3,0,0,
+    115,6,0,0,0,0,2,10,1,16,2,122,24,70,114,111,
+    122,101,110,73,109,112,111,114,116,101,114,46,102,105,110,100,
+    95,115,112,101,99,99,3,0,0,0,0,0,0,0,0,0,
+    0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,18,
+    0,0,0,116,0,160,1,124,1,161,1,114,14,124,0,83,
+    0,100,1,83,0,41,2,122,93,70,105,110,100,32,97,32,
+    102,114,111,122,101,110,32,109,111,100,117,108,101,46,10,10,
+    32,32,32,32,32,32,32,32,84,104,105,115,32,109,101,116,
+    104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101,
+    100,46,32,32,85,115,101,32,102,105,110,100,95,115,112,101,
+    99,40,41,32,105,110,115,116,101,97,100,46,10,10,32,32,
+    32,32,32,32,32,32,78,41,2,114,56,0,0,0,114,87,
+    0,0,0,41,3,114,163,0,0,0,114,80,0,0,0,114,
+    164,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,
+    0,0,0,114,167,0,0,0,41,3,0,0,115,2,0,0,
+    0,0,7,122,26,70,114,111,122,101,110,73,109,112,111,114,
+    116,101,114,46,102,105,110,100,95,109,111,100,117,108,101,99,
+    2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,
+    1,0,0,0,67,0,0,0,115,4,0,0,0,100,1,83,
+    0,41,2,122,42,85,115,101,32,100,101,102,97,117,108,116,
+    32,115,101,109,97,110,116,105,99,115,32,102,111,114,32,109,
+    111,100,117,108,101,32,99,114,101,97,116,105,111,110,46,78,
+    114,10,0,0,0,41,2,114,163,0,0,0,114,94,0,0,
+    0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,
+    114,148,0,0,0,50,3,0,0,115,2,0,0,0,0,2,
+    122,28,70,114,111,122,101,110,73,109,112,111,114,116,101,114,
+    46,99,114,101,97,116,101,95,109,111,100,117,108,101,99,1,
+    0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,4,
+    0,0,0,67,0,0,0,115,64,0,0,0,124,0,106,0,
+    106,1,125,1,116,2,160,3,124,1,161,1,115,36,116,4,
+    100,1,160,5,124,1,161,1,124,1,100,2,141,2,130,1,
+    116,6,116,2,106,7,124,1,131,2,125,2,116,8,124,2,
+    124,0,106,9,131,2,1,0,100,0,83,0,114,86,0,0,
+    0,41,10,114,104,0,0,0,114,17,0,0,0,114,56,0,
+    0,0,114,87,0,0,0,114,78,0,0,0,114,44,0,0,
+    0,114,66,0,0,0,218,17,103,101,116,95,102,114,111,122,
+    101,110,95,111,98,106,101,99,116,218,4,101,120,101,99,114,
+    7,0,0,0,41,3,114,95,0,0,0,114,17,0,0,0,
+    218,4,99,111,100,101,114,10,0,0,0,114,10,0,0,0,
+    114,11,0,0,0,114,149,0,0,0,54,3,0,0,115,14,
+    0,0,0,0,2,8,1,10,1,10,1,2,255,6,2,12,
+    1,122,26,70,114,111,122,101,110,73,109,112,111,114,116,101,
+    114,46,101,120,101,99,95,109,111,100,117,108,101,99,2,0,
+    0,0,0,0,0,0,0,0,0,0,2,0,0,0,3,0,
+    0,0,67,0,0,0,115,10,0,0,0,116,0,124,0,124,
+    1,131,2,83,0,41,1,122,95,76,111,97,100,32,97,32,
+    102,114,111,122,101,110,32,109,111,100,117,108,101,46,10,10,
+    32,32,32,32,32,32,32,32,84,104,105,115,32,109,101,116,
+    104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101,
+    100,46,32,32,85,115,101,32,101,120,101,99,95,109,111,100,
+    117,108,101,40,41,32,105,110,115,116,101,97,100,46,10,10,
+    32,32,32,32,32,32,32,32,41,1,114,96,0,0,0,114,
+    168,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,
+    0,0,0,114,155,0,0,0,63,3,0,0,115,2,0,0,
+    0,0,7,122,26,70,114,111,122,101,110,73,109,112,111,114,
+    116,101,114,46,108,111,97,100,95,109,111,100,117,108,101,99,
+    2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,
+    3,0,0,0,67,0,0,0,115,10,0,0,0,116,0,160,
+    1,124,1,161,1,83,0,41,1,122,45,82,101,116,117,114,
+    110,32,116,104,101,32,99,111,100,101,32,111,98,106,101,99,
+    116,32,102,111,114,32,116,104,101,32,102,114,111,122,101,110,
+    32,109,111,100,117,108,101,46,41,2,114,56,0,0,0,114,
+    175,0,0,0,114,168,0,0,0,114,10,0,0,0,114,10,
+    0,0,0,114,11,0,0,0,114,169,0,0,0,72,3,0,
+    0,115,2,0,0,0,0,4,122,23,70,114,111,122,101,110,
+    73,109,112,111,114,116,101,114,46,103,101,116,95,99,111,100,
     101,99,2,0,0,0,0,0,0,0,0,0,0,0,2,0,
-    0,0,3,0,0,0,67,0,0,0,115,16,0,0,0,116,
-    0,116,1,106,2,124,1,131,2,1,0,100,1,83,0,41,
-    2,122,22,69,120,101,99,32,97,32,98,117,105,108,116,45,
-    105,110,32,109,111,100,117,108,101,78,41,3,114,66,0,0,
-    0,114,56,0,0,0,90,12,101,120,101,99,95,98,117,105,
-    108,116,105,110,41,2,114,30,0,0,0,114,95,0,0,0,
-    114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,
-    149,0,0,0,242,2,0,0,115,2,0,0,0,0,3,122,
-    27,66,117,105,108,116,105,110,73,109,112,111,114,116,101,114,
-    46,101,120,101,99,95,109,111,100,117,108,101,99,2,0,0,
-    0,0,0,0,0,0,0,0,0,2,0,0,0,1,0,0,
-    0,67,0,0,0,115,4,0,0,0,100,1,83,0,41,2,
-    122,57,82,101,116,117,114,110,32,78,111,110,101,32,97,115,
-    32,98,117,105,108,116,45,105,110,32,109,111,100,117,108,101,
-    115,32,100,111,32,110,111,116,32,104,97,118,101,32,99,111,
-    100,101,32,111,98,106,101,99,116,115,46,78,114,10,0,0,
-    0,169,2,114,162,0,0,0,114,80,0,0,0,114,10,0,
-    0,0,114,10,0,0,0,114,11,0,0,0,218,8,103,101,
-    116,95,99,111,100,101,247,2,0,0,115,2,0,0,0,0,
-    4,122,24,66,117,105,108,116,105,110,73,109,112,111,114,116,
-    101,114,46,103,101,116,95,99,111,100,101,99,2,0,0,0,
-    0,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,
-    67,0,0,0,115,4,0,0,0,100,1,83,0,41,2,122,
-    56,82,101,116,117,114,110,32,78,111,110,101,32,97,115,32,
-    98,117,105,108,116,45,105,110,32,109,111,100,117,108,101,115,
-    32,100,111,32,110,111,116,32,104,97,118,101,32,115,111,117,
-    114,99,101,32,99,111,100,101,46,78,114,10,0,0,0,114,
-    167,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,
-    0,0,0,218,10,103,101,116,95,115,111,117,114,99,101,253,
-    2,0,0,115,2,0,0,0,0,4,122,26,66,117,105,108,
-    116,105,110,73,109,112,111,114,116,101,114,46,103,101,116,95,
-    115,111,117,114,99,101,99,2,0,0,0,0,0,0,0,0,
-    0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,115,
-    4,0,0,0,100,1,83,0,41,2,122,52,82,101,116,117,
-    114,110,32,70,97,108,115,101,32,97,115,32,98,117,105,108,
-    116,45,105,110,32,109,111,100,117,108,101,115,32,97,114,101,
-    32,110,101,118,101,114,32,112,97,99,107,97,103,101,115,46,
-    70,114,10,0,0,0,114,167,0,0,0,114,10,0,0,0,
-    114,10,0,0,0,114,11,0,0,0,114,114,0,0,0,3,
-    3,0,0,115,2,0,0,0,0,4,122,26,66,117,105,108,
-    116,105,110,73,109,112,111,114,116,101,114,46,105,115,95,112,
-    97,99,107,97,103,101,41,2,78,78,41,1,78,41,17,114,
-    1,0,0,0,114,0,0,0,0,114,2,0,0,0,114,3,
-    0,0,0,218,12,115,116,97,116,105,99,109,101,116,104,111,
-    100,114,98,0,0,0,218,11,99,108,97,115,115,109,101,116,
-    104,111,100,114,165,0,0,0,114,166,0,0,0,114,148,0,
-    0,0,114,149,0,0,0,114,85,0,0,0,114,168,0,0,
-    0,114,169,0,0,0,114,114,0,0,0,114,96,0,0,0,
-    114,155,0,0,0,114,10,0,0,0,114,10,0,0,0,114,
-    10,0,0,0,114,11,0,0,0,114,159,0,0,0,195,2,
-    0,0,115,42,0,0,0,8,2,4,7,2,1,10,8,2,
-    1,12,8,2,1,12,11,2,1,10,7,2,1,10,4,2,
-    1,2,1,12,4,2,1,2,1,12,4,2,1,2,1,12,
-    4,114,159,0,0,0,99,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,4,0,0,0,64,0,0,0,115,
-    144,0,0,0,101,0,90,1,100,0,90,2,100,1,90,3,
-    100,2,90,4,101,5,100,3,100,4,132,0,131,1,90,6,
-    101,7,100,22,100,6,100,7,132,1,131,1,90,8,101,7,
-    100,23,100,8,100,9,132,1,131,1,90,9,101,7,100,10,
-    100,11,132,0,131,1,90,10,101,5,100,12,100,13,132,0,
-    131,1,90,11,101,7,100,14,100,15,132,0,131,1,90,12,
-    101,7,101,13,100,16,100,17,132,0,131,1,131,1,90,14,
-    101,7,101,13,100,18,100,19,132,0,131,1,131,1,90,15,
-    101,7,101,13,100,20,100,21,132,0,131,1,131,1,90,16,
-    100,5,83,0,41,24,218,14,70,114,111,122,101,110,73,109,
-    112,111,114,116,101,114,122,142,77,101,116,97,32,112,97,116,
-    104,32,105,109,112,111,114,116,32,102,111,114,32,102,114,111,
-    122,101,110,32,109,111,100,117,108,101,115,46,10,10,32,32,
-    32,32,65,108,108,32,109,101,116,104,111,100,115,32,97,114,
-    101,32,101,105,116,104,101,114,32,99,108,97,115,115,32,111,
-    114,32,115,116,97,116,105,99,32,109,101,116,104,111,100,115,
-    32,116,111,32,97,118,111,105,100,32,116,104,101,32,110,101,
-    101,100,32,116,111,10,32,32,32,32,105,110,115,116,97,110,
-    116,105,97,116,101,32,116,104,101,32,99,108,97,115,115,46,
-    10,10,32,32,32,32,90,6,102,114,111,122,101,110,99,1,
-    0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,4,
-    0,0,0,67,0,0,0,115,16,0,0,0,100,1,160,0,
-    124,0,106,1,116,2,106,3,161,2,83,0,41,2,114,160,
-    0,0,0,114,152,0,0,0,41,4,114,44,0,0,0,114,
-    1,0,0,0,114,172,0,0,0,114,137,0,0,0,41,1,
-    218,1,109,114,10,0,0,0,114,10,0,0,0,114,11,0,
-    0,0,114,98,0,0,0,23,3,0,0,115,2,0,0,0,
-    0,7,122,26,70,114,111,122,101,110,73,109,112,111,114,116,
-    101,114,46,109,111,100,117,108,101,95,114,101,112,114,78,99,
-    4,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,
-    5,0,0,0,67,0,0,0,115,34,0,0,0,116,0,160,
-    1,124,1,161,1,114,26,116,2,124,1,124,0,124,0,106,
-    3,100,1,141,3,83,0,100,0,83,0,100,0,83,0,41,
-    2,78,114,136,0,0,0,41,4,114,56,0,0,0,114,87,
-    0,0,0,114,90,0,0,0,114,137,0,0,0,114,161,0,
-    0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,
-    0,114,165,0,0,0,32,3,0,0,115,6,0,0,0,0,
-    2,10,1,16,2,122,24,70,114,111,122,101,110,73,109,112,
-    111,114,116,101,114,46,102,105,110,100,95,115,112,101,99,99,
-    3,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,
-    3,0,0,0,67,0,0,0,115,18,0,0,0,116,0,160,
-    1,124,1,161,1,114,14,124,0,83,0,100,1,83,0,41,
-    2,122,93,70,105,110,100,32,97,32,102,114,111,122,101,110,
-    32,109,111,100,117,108,101,46,10,10,32,32,32,32,32,32,
-    32,32,84,104,105,115,32,109,101,116,104,111,100,32,105,115,
-    32,100,101,112,114,101,99,97,116,101,100,46,32,32,85,115,
-    101,32,102,105,110,100,95,115,112,101,99,40,41,32,105,110,
-    115,116,101,97,100,46,10,10,32,32,32,32,32,32,32,32,
-    78,41,2,114,56,0,0,0,114,87,0,0,0,41,3,114,
-    162,0,0,0,114,80,0,0,0,114,163,0,0,0,114,10,
-    0,0,0,114,10,0,0,0,114,11,0,0,0,114,166,0,
-    0,0,39,3,0,0,115,2,0,0,0,0,7,122,26,70,
-    114,111,122,101,110,73,109,112,111,114,116,101,114,46,102,105,
-    110,100,95,109,111,100,117,108,101,99,2,0,0,0,0,0,
-    0,0,0,0,0,0,2,0,0,0,1,0,0,0,67,0,
-    0,0,115,4,0,0,0,100,1,83,0,41,2,122,42,85,
-    115,101,32,100,101,102,97,117,108,116,32,115,101,109,97,110,
-    116,105,99,115,32,102,111,114,32,109,111,100,117,108,101,32,
-    99,114,101,97,116,105,111,110,46,78,114,10,0,0,0,41,
-    2,114,162,0,0,0,114,94,0,0,0,114,10,0,0,0,
-    114,10,0,0,0,114,11,0,0,0,114,148,0,0,0,48,
-    3,0,0,115,2,0,0,0,0,2,122,28,70,114,111,122,
-    101,110,73,109,112,111,114,116,101,114,46,99,114,101,97,116,
-    101,95,109,111,100,117,108,101,99,1,0,0,0,0,0,0,
-    0,0,0,0,0,3,0,0,0,4,0,0,0,67,0,0,
-    0,115,64,0,0,0,124,0,106,0,106,1,125,1,116,2,
-    160,3,124,1,161,1,115,36,116,4,100,1,160,5,124,1,
-    161,1,124,1,100,2,141,2,130,1,116,6,116,2,106,7,
-    124,1,131,2,125,2,116,8,124,2,124,0,106,9,131,2,
-    1,0,100,0,83,0,114,86,0,0,0,41,10,114,104,0,
-    0,0,114,17,0,0,0,114,56,0,0,0,114,87,0,0,
-    0,114,78,0,0,0,114,44,0,0,0,114,66,0,0,0,
-    218,17,103,101,116,95,102,114,111,122,101,110,95,111,98,106,
-    101,99,116,218,4,101,120,101,99,114,7,0,0,0,41,3,
-    114,95,0,0,0,114,17,0,0,0,218,4,99,111,100,101,
-    114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,
-    149,0,0,0,52,3,0,0,115,14,0,0,0,0,2,8,
-    1,10,1,10,1,2,255,6,2,12,1,122,26,70,114,111,
-    122,101,110,73,109,112,111,114,116,101,114,46,101,120,101,99,
-    95,109,111,100,117,108,101,99,2,0,0,0,0,0,0,0,
-    0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,0,
-    115,10,0,0,0,116,0,124,0,124,1,131,2,83,0,41,
-    1,122,95,76,111,97,100,32,97,32,102,114,111,122,101,110,
-    32,109,111,100,117,108,101,46,10,10,32,32,32,32,32,32,
-    32,32,84,104,105,115,32,109,101,116,104,111,100,32,105,115,
-    32,100,101,112,114,101,99,97,116,101,100,46,32,32,85,115,
-    101,32,101,120,101,99,95,109,111,100,117,108,101,40,41,32,
-    105,110,115,116,101,97,100,46,10,10,32,32,32,32,32,32,
-    32,32,41,1,114,96,0,0,0,114,167,0,0,0,114,10,
-    0,0,0,114,10,0,0,0,114,11,0,0,0,114,155,0,
-    0,0,61,3,0,0,115,2,0,0,0,0,7,122,26,70,
-    114,111,122,101,110,73,109,112,111,114,116,101,114,46,108,111,
-    97,100,95,109,111,100,117,108,101,99,2,0,0,0,0,0,
-    0,0,0,0,0,0,2,0,0,0,3,0,0,0,67,0,
-    0,0,115,10,0,0,0,116,0,160,1,124,1,161,1,83,
-    0,41,1,122,45,82,101,116,117,114,110,32,116,104,101,32,
-    99,111,100,101,32,111,98,106,101,99,116,32,102,111,114,32,
-    116,104,101,32,102,114,111,122,101,110,32,109,111,100,117,108,
-    101,46,41,2,114,56,0,0,0,114,174,0,0,0,114,167,
-    0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,
-    0,0,114,168,0,0,0,70,3,0,0,115,2,0,0,0,
-    0,4,122,23,70,114,111,122,101,110,73,109,112,111,114,116,
-    101,114,46,103,101,116,95,99,111,100,101,99,2,0,0,0,
-    0,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,
-    67,0,0,0,115,4,0,0,0,100,1,83,0,41,2,122,
-    54,82,101,116,117,114,110,32,78,111,110,101,32,97,115,32,
-    102,114,111,122,101,110,32,109,111,100,117,108,101,115,32,100,
-    111,32,110,111,116,32,104,97,118,101,32,115,111,117,114,99,
-    101,32,99,111,100,101,46,78,114,10,0,0,0,114,167,0,
-    0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,
-    0,114,169,0,0,0,76,3,0,0,115,2,0,0,0,0,
-    4,122,25,70,114,111,122,101,110,73,109,112,111,114,116,101,
-    114,46,103,101,116,95,115,111,117,114,99,101,99,2,0,0,
-    0,0,0,0,0,0,0,0,0,2,0,0,0,3,0,0,
-    0,67,0,0,0,115,10,0,0,0,116,0,160,1,124,1,
-    161,1,83,0,41,1,122,46,82,101,116,117,114,110,32,84,
-    114,117,101,32,105,102,32,116,104,101,32,102,114,111,122,101,
-    110,32,109,111,100,117,108,101,32,105,115,32,97,32,112,97,
-    99,107,97,103,101,46,41,2,114,56,0,0,0,90,17,105,
-    115,95,102,114,111,122,101,110,95,112,97,99,107,97,103,101,
-    114,167,0,0,0,114,10,0,0,0,114,10,0,0,0,114,
-    11,0,0,0,114,114,0,0,0,82,3,0,0,115,2,0,
-    0,0,0,4,122,25,70,114,111,122,101,110,73,109,112,111,
-    114,116,101,114,46,105,115,95,112,97,99,107,97,103,101,41,
-    2,78,78,41,1,78,41,17,114,1,0,0,0,114,0,0,
-    0,0,114,2,0,0,0,114,3,0,0,0,114,137,0,0,
-    0,114,170,0,0,0,114,98,0,0,0,114,171,0,0,0,
-    114,165,0,0,0,114,166,0,0,0,114,148,0,0,0,114,
-    149,0,0,0,114,155,0,0,0,114,89,0,0,0,114,168,
-    0,0,0,114,169,0,0,0,114,114,0,0,0,114,10,0,
-    0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,
-    0,114,172,0,0,0,12,3,0,0,115,46,0,0,0,8,
-    2,4,7,4,2,2,1,10,8,2,1,12,6,2,1,12,
-    8,2,1,10,3,2,1,10,8,2,1,10,8,2,1,2,
-    1,12,4,2,1,2,1,12,4,2,1,2,1,114,172,0,
-    0,0,99,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,2,0,0,0,64,0,0,0,115,32,0,0,0,
-    101,0,90,1,100,0,90,2,100,1,90,3,100,2,100,3,
-    132,0,90,4,100,4,100,5,132,0,90,5,100,6,83,0,
-    41,7,218,18,95,73,109,112,111,114,116,76,111,99,107,67,
-    111,110,116,101,120,116,122,36,67,111,110,116,101,120,116,32,
-    109,97,110,97,103,101,114,32,102,111,114,32,116,104,101,32,
-    105,109,112,111,114,116,32,108,111,99,107,46,99,1,0,0,
-    0,0,0,0,0,0,0,0,0,1,0,0,0,2,0,0,
-    0,67,0,0,0,115,12,0,0,0,116,0,160,1,161,0,
-    1,0,100,1,83,0,41,2,122,24,65,99,113,117,105,114,
-    101,32,116,104,101,32,105,109,112,111,114,116,32,108,111,99,
-    107,46,78,41,2,114,56,0,0,0,114,57,0,0,0,114,
-    46,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,
-    0,0,0,114,53,0,0,0,95,3,0,0,115,2,0,0,
-    0,0,2,122,28,95,73,109,112,111,114,116,76,111,99,107,
-    67,111,110,116,101,120,116,46,95,95,101,110,116,101,114,95,
-    95,99,4,0,0,0,0,0,0,0,0,0,0,0,4,0,
-    0,0,2,0,0,0,67,0,0,0,115,12,0,0,0,116,
-    0,160,1,161,0,1,0,100,1,83,0,41,2,122,60,82,
-    101,108,101,97,115,101,32,116,104,101,32,105,109,112,111,114,
-    116,32,108,111,99,107,32,114,101,103,97,114,100,108,101,115,
-    115,32,111,102,32,97,110,121,32,114,97,105,115,101,100,32,
-    101,120,99,101,112,116,105,111,110,115,46,78,41,2,114,56,
-    0,0,0,114,58,0,0,0,41,4,114,30,0,0,0,218,
-    8,101,120,99,95,116,121,112,101,218,9,101,120,99,95,118,
-    97,108,117,101,218,13,101,120,99,95,116,114,97,99,101,98,
-    97,99,107,114,10,0,0,0,114,10,0,0,0,114,11,0,
-    0,0,114,55,0,0,0,99,3,0,0,115,2,0,0,0,
-    0,2,122,27,95,73,109,112,111,114,116,76,111,99,107,67,
-    111,110,116,101,120,116,46,95,95,101,120,105,116,95,95,78,
-    41,6,114,1,0,0,0,114,0,0,0,0,114,2,0,0,
-    0,114,3,0,0,0,114,53,0,0,0,114,55,0,0,0,
-    114,10,0,0,0,114,10,0,0,0,114,10,0,0,0,114,
-    11,0,0,0,114,177,0,0,0,91,3,0,0,115,6,0,
-    0,0,8,2,4,2,8,4,114,177,0,0,0,99,3,0,
-    0,0,0,0,0,0,0,0,0,0,5,0,0,0,5,0,
-    0,0,67,0,0,0,115,64,0,0,0,124,1,160,0,100,
-    1,124,2,100,2,24,0,161,2,125,3,116,1,124,3,131,
-    1,124,2,107,0,114,36,116,2,100,3,131,1,130,1,124,
-    3,100,4,25,0,125,4,124,0,114,60,100,5,160,3,124,
-    4,124,0,161,2,83,0,124,4,83,0,41,6,122,50,82,
-    101,115,111,108,118,101,32,97,32,114,101,108,97,116,105,118,
-    101,32,109,111,100,117,108,101,32,110,97,109,101,32,116,111,
-    32,97,110,32,97,98,115,111,108,117,116,101,32,111,110,101,
-    46,114,127,0,0,0,114,37,0,0,0,122,50,97,116,116,
+    0,0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,
+    1,83,0,41,2,122,54,82,101,116,117,114,110,32,78,111,
+    110,101,32,97,115,32,102,114,111,122,101,110,32,109,111,100,
+    117,108,101,115,32,100,111,32,110,111,116,32,104,97,118,101,
+    32,115,111,117,114,99,101,32,99,111,100,101,46,78,114,10,
+    0,0,0,114,168,0,0,0,114,10,0,0,0,114,10,0,
+    0,0,114,11,0,0,0,114,170,0,0,0,78,3,0,0,
+    115,2,0,0,0,0,4,122,25,70,114,111,122,101,110,73,
+    109,112,111,114,116,101,114,46,103,101,116,95,115,111,117,114,
+    99,101,99,2,0,0,0,0,0,0,0,0,0,0,0,2,
+    0,0,0,3,0,0,0,67,0,0,0,115,10,0,0,0,
+    116,0,160,1,124,1,161,1,83,0,41,1,122,46,82,101,
+    116,117,114,110,32,84,114,117,101,32,105,102,32,116,104,101,
+    32,102,114,111,122,101,110,32,109,111,100,117,108,101,32,105,
+    115,32,97,32,112,97,99,107,97,103,101,46,41,2,114,56,
+    0,0,0,90,17,105,115,95,102,114,111,122,101,110,95,112,
+    97,99,107,97,103,101,114,168,0,0,0,114,10,0,0,0,
+    114,10,0,0,0,114,11,0,0,0,114,114,0,0,0,84,
+    3,0,0,115,2,0,0,0,0,4,122,25,70,114,111,122,
+    101,110,73,109,112,111,114,116,101,114,46,105,115,95,112,97,
+    99,107,97,103,101,41,2,78,78,41,1,78,41,17,114,1,
+    0,0,0,114,0,0,0,0,114,2,0,0,0,114,3,0,
+    0,0,114,137,0,0,0,114,171,0,0,0,114,98,0,0,
+    0,114,172,0,0,0,114,166,0,0,0,114,167,0,0,0,
+    114,148,0,0,0,114,149,0,0,0,114,155,0,0,0,114,
+    89,0,0,0,114,169,0,0,0,114,170,0,0,0,114,114,
+    0,0,0,114,10,0,0,0,114,10,0,0,0,114,10,0,
+    0,0,114,11,0,0,0,114,173,0,0,0,14,3,0,0,
+    115,46,0,0,0,8,2,4,7,4,2,2,1,10,8,2,
+    1,12,6,2,1,12,8,2,1,10,3,2,1,10,8,2,
+    1,10,8,2,1,2,1,12,4,2,1,2,1,12,4,2,
+    1,2,1,114,173,0,0,0,99,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,2,0,0,0,64,0,0,
+    0,115,32,0,0,0,101,0,90,1,100,0,90,2,100,1,
+    90,3,100,2,100,3,132,0,90,4,100,4,100,5,132,0,
+    90,5,100,6,83,0,41,7,218,18,95,73,109,112,111,114,
+    116,76,111,99,107,67,111,110,116,101,120,116,122,36,67,111,
+    110,116,101,120,116,32,109,97,110,97,103,101,114,32,102,111,
+    114,32,116,104,101,32,105,109,112,111,114,116,32,108,111,99,
+    107,46,99,1,0,0,0,0,0,0,0,0,0,0,0,1,
+    0,0,0,2,0,0,0,67,0,0,0,115,12,0,0,0,
+    116,0,160,1,161,0,1,0,100,1,83,0,41,2,122,24,
+    65,99,113,117,105,114,101,32,116,104,101,32,105,109,112,111,
+    114,116,32,108,111,99,107,46,78,41,2,114,56,0,0,0,
+    114,57,0,0,0,114,46,0,0,0,114,10,0,0,0,114,
+    10,0,0,0,114,11,0,0,0,114,53,0,0,0,97,3,
+    0,0,115,2,0,0,0,0,2,122,28,95,73,109,112,111,
+    114,116,76,111,99,107,67,111,110,116,101,120,116,46,95,95,
+    101,110,116,101,114,95,95,99,4,0,0,0,0,0,0,0,
+    0,0,0,0,4,0,0,0,2,0,0,0,67,0,0,0,
+    115,12,0,0,0,116,0,160,1,161,0,1,0,100,1,83,
+    0,41,2,122,60,82,101,108,101,97,115,101,32,116,104,101,
+    32,105,109,112,111,114,116,32,108,111,99,107,32,114,101,103,
+    97,114,100,108,101,115,115,32,111,102,32,97,110,121,32,114,
+    97,105,115,101,100,32,101,120,99,101,112,116,105,111,110,115,
+    46,78,41,2,114,56,0,0,0,114,58,0,0,0,41,4,
+    114,30,0,0,0,218,8,101,120,99,95,116,121,112,101,218,
+    9,101,120,99,95,118,97,108,117,101,218,13,101,120,99,95,
+    116,114,97,99,101,98,97,99,107,114,10,0,0,0,114,10,
+    0,0,0,114,11,0,0,0,114,55,0,0,0,101,3,0,
+    0,115,2,0,0,0,0,2,122,27,95,73,109,112,111,114,
+    116,76,111,99,107,67,111,110,116,101,120,116,46,95,95,101,
+    120,105,116,95,95,78,41,6,114,1,0,0,0,114,0,0,
+    0,0,114,2,0,0,0,114,3,0,0,0,114,53,0,0,
+    0,114,55,0,0,0,114,10,0,0,0,114,10,0,0,0,
+    114,10,0,0,0,114,11,0,0,0,114,178,0,0,0,93,
+    3,0,0,115,6,0,0,0,8,2,4,2,8,4,114,178,
+    0,0,0,99,3,0,0,0,0,0,0,0,0,0,0,0,
+    5,0,0,0,5,0,0,0,67,0,0,0,115,64,0,0,
+    0,124,1,160,0,100,1,124,2,100,2,24,0,161,2,125,
+    3,116,1,124,3,131,1,124,2,107,0,114,36,116,2,100,
+    3,131,1,130,1,124,3,100,4,25,0,125,4,124,0,114,
+    60,100,5,160,3,124,4,124,0,161,2,83,0,124,4,83,
+    0,41,6,122,50,82,101,115,111,108,118,101,32,97,32,114,
+    101,108,97,116,105,118,101,32,109,111,100,117,108,101,32,110,
+    97,109,101,32,116,111,32,97,110,32,97,98,115,111,108,117,
+    116,101,32,111,110,101,46,114,127,0,0,0,114,37,0,0,
+    0,122,50,97,116,116,101,109,112,116,101,100,32,114,101,108,
+    97,116,105,118,101,32,105,109,112,111,114,116,32,98,101,121,
+    111,110,100,32,116,111,112,45,108,101,118,101,108,32,112,97,
+    99,107,97,103,101,114,22,0,0,0,250,5,123,125,46,123,
+    125,41,4,218,6,114,115,112,108,105,116,218,3,108,101,110,
+    114,78,0,0,0,114,44,0,0,0,41,5,114,17,0,0,
+    0,218,7,112,97,99,107,97,103,101,218,5,108,101,118,101,
+    108,90,4,98,105,116,115,90,4,98,97,115,101,114,10,0,
+    0,0,114,10,0,0,0,114,11,0,0,0,218,13,95,114,
+    101,115,111,108,118,101,95,110,97,109,101,106,3,0,0,115,
+    10,0,0,0,0,2,16,1,12,1,8,1,8,1,114,187,
+    0,0,0,99,3,0,0,0,0,0,0,0,0,0,0,0,
+    4,0,0,0,4,0,0,0,67,0,0,0,115,34,0,0,
+    0,124,0,160,0,124,1,124,2,161,2,125,3,124,3,100,
+    0,107,8,114,24,100,0,83,0,116,1,124,1,124,3,131,
+    2,83,0,114,13,0,0,0,41,2,114,167,0,0,0,114,
+    90,0,0,0,41,4,218,6,102,105,110,100,101,114,114,17,
+    0,0,0,114,164,0,0,0,114,108,0,0,0,114,10,0,
+    0,0,114,10,0,0,0,114,11,0,0,0,218,17,95,102,
+    105,110,100,95,115,112,101,99,95,108,101,103,97,99,121,115,
+    3,0,0,115,8,0,0,0,0,3,12,1,8,1,4,1,
+    114,189,0,0,0,99,3,0,0,0,0,0,0,0,0,0,
+    0,0,10,0,0,0,10,0,0,0,67,0,0,0,115,12,
+    1,0,0,116,0,106,1,125,3,124,3,100,1,107,8,114,
+    22,116,2,100,2,131,1,130,1,124,3,115,38,116,3,160,
+    4,100,3,116,5,161,2,1,0,124,0,116,0,106,6,107,
+    6,125,4,124,3,68,0,93,210,125,5,116,7,131,0,143,
+    84,1,0,122,10,124,5,106,8,125,6,87,0,110,54,4,
+    0,116,9,107,10,114,128,1,0,1,0,1,0,116,10,124,
+    5,124,0,124,1,131,3,125,7,124,7,100,1,107,8,114,
+    124,89,0,87,0,53,0,81,0,82,0,163,0,113,52,89,
+    0,110,14,88,0,124,6,124,0,124,1,124,2,131,3,125,
+    7,87,0,53,0,81,0,82,0,88,0,124,7,100,1,107,
+    9,114,52,124,4,144,0,115,254,124,0,116,0,106,6,107,
+    6,144,0,114,254,116,0,106,6,124,0,25,0,125,8,122,
+    10,124,8,106,11,125,9,87,0,110,28,4,0,116,9,107,
+    10,114,226,1,0,1,0,1,0,124,7,6,0,89,0,2,
+    0,1,0,83,0,88,0,124,9,100,1,107,8,114,244,124,
+    7,2,0,1,0,83,0,124,9,2,0,1,0,83,0,113,
+    52,124,7,2,0,1,0,83,0,113,52,100,1,83,0,41,
+    4,122,21,70,105,110,100,32,97,32,109,111,100,117,108,101,
+    39,115,32,115,112,101,99,46,78,122,53,115,121,115,46,109,
+    101,116,97,95,112,97,116,104,32,105,115,32,78,111,110,101,
+    44,32,80,121,116,104,111,110,32,105,115,32,108,105,107,101,
+    108,121,32,115,104,117,116,116,105,110,103,32,100,111,119,110,
+    122,22,115,121,115,46,109,101,116,97,95,112,97,116,104,32,
+    105,115,32,101,109,112,116,121,41,12,114,15,0,0,0,218,
+    9,109,101,116,97,95,112,97,116,104,114,78,0,0,0,218,
+    9,95,119,97,114,110,105,110,103,115,218,4,119,97,114,110,
+    218,13,73,109,112,111,114,116,87,97,114,110,105,110,103,114,
+    91,0,0,0,114,178,0,0,0,114,166,0,0,0,114,105,
+    0,0,0,114,189,0,0,0,114,104,0,0,0,41,10,114,
+    17,0,0,0,114,164,0,0,0,114,165,0,0,0,114,190,
+    0,0,0,90,9,105,115,95,114,101,108,111,97,100,114,188,
+    0,0,0,114,166,0,0,0,114,94,0,0,0,114,95,0,
+    0,0,114,104,0,0,0,114,10,0,0,0,114,10,0,0,
+    0,114,11,0,0,0,218,10,95,102,105,110,100,95,115,112,
+    101,99,124,3,0,0,115,54,0,0,0,0,2,6,1,8,
+    2,8,3,4,1,12,5,10,1,8,1,8,1,2,1,10,
+    1,14,1,12,1,8,1,20,2,22,1,8,2,18,1,10,
+    1,2,1,10,1,14,4,14,2,8,1,8,2,10,2,10,
+    2,114,194,0,0,0,99,3,0,0,0,0,0,0,0,0,
+    0,0,0,3,0,0,0,5,0,0,0,67,0,0,0,115,
+    108,0,0,0,116,0,124,0,116,1,131,2,115,28,116,2,
+    100,1,160,3,116,4,124,0,131,1,161,1,131,1,130,1,
+    124,2,100,2,107,0,114,44,116,5,100,3,131,1,130,1,
+    124,2,100,2,107,4,114,84,116,0,124,1,116,1,131,2,
+    115,72,116,2,100,4,131,1,130,1,110,12,124,1,115,84,
+    116,6,100,5,131,1,130,1,124,0,115,104,124,2,100,2,
+    107,2,114,104,116,5,100,6,131,1,130,1,100,7,83,0,
+    41,8,122,28,86,101,114,105,102,121,32,97,114,103,117,109,
+    101,110,116,115,32,97,114,101,32,34,115,97,110,101,34,46,
+    122,31,109,111,100,117,108,101,32,110,97,109,101,32,109,117,
+    115,116,32,98,101,32,115,116,114,44,32,110,111,116,32,123,
+    125,114,22,0,0,0,122,18,108,101,118,101,108,32,109,117,
+    115,116,32,98,101,32,62,61,32,48,122,31,95,95,112,97,
+    99,107,97,103,101,95,95,32,110,111,116,32,115,101,116,32,
+    116,111,32,97,32,115,116,114,105,110,103,122,54,97,116,116,
     101,109,112,116,101,100,32,114,101,108,97,116,105,118,101,32,
-    105,109,112,111,114,116,32,98,101,121,111,110,100,32,116,111,
-    112,45,108,101,118,101,108,32,112,97,99,107,97,103,101,114,
-    22,0,0,0,250,5,123,125,46,123,125,41,4,218,6,114,
-    115,112,108,105,116,218,3,108,101,110,114,78,0,0,0,114,
-    44,0,0,0,41,5,114,17,0,0,0,218,7,112,97,99,
-    107,97,103,101,218,5,108,101,118,101,108,90,4,98,105,116,
-    115,90,4,98,97,115,101,114,10,0,0,0,114,10,0,0,
-    0,114,11,0,0,0,218,13,95,114,101,115,111,108,118,101,
-    95,110,97,109,101,104,3,0,0,115,10,0,0,0,0,2,
-    16,1,12,1,8,1,8,1,114,186,0,0,0,99,3,0,
-    0,0,0,0,0,0,0,0,0,0,4,0,0,0,4,0,
-    0,0,67,0,0,0,115,34,0,0,0,124,0,160,0,124,
-    1,124,2,161,2,125,3,124,3,100,0,107,8,114,24,100,
-    0,83,0,116,1,124,1,124,3,131,2,83,0,114,13,0,
-    0,0,41,2,114,166,0,0,0,114,90,0,0,0,41,4,
-    218,6,102,105,110,100,101,114,114,17,0,0,0,114,163,0,
-    0,0,114,108,0,0,0,114,10,0,0,0,114,10,0,0,
-    0,114,11,0,0,0,218,17,95,102,105,110,100,95,115,112,
-    101,99,95,108,101,103,97,99,121,113,3,0,0,115,8,0,
-    0,0,0,3,12,1,8,1,4,1,114,188,0,0,0,99,
-    3,0,0,0,0,0,0,0,0,0,0,0,10,0,0,0,
-    10,0,0,0,67,0,0,0,115,12,1,0,0,116,0,106,
-    1,125,3,124,3,100,1,107,8,114,22,116,2,100,2,131,
-    1,130,1,124,3,115,38,116,3,160,4,100,3,116,5,161,
-    2,1,0,124,0,116,0,106,6,107,6,125,4,124,3,68,
-    0,93,210,125,5,116,7,131,0,143,84,1,0,122,10,124,
-    5,106,8,125,6,87,0,110,54,4,0,116,9,107,10,114,
-    128,1,0,1,0,1,0,116,10,124,5,124,0,124,1,131,
-    3,125,7,124,7,100,1,107,8,114,124,89,0,87,0,53,
-    0,81,0,82,0,163,0,113,52,89,0,110,14,88,0,124,
-    6,124,0,124,1,124,2,131,3,125,7,87,0,53,0,81,
-    0,82,0,88,0,124,7,100,1,107,9,114,52,124,4,144,
-    0,115,254,124,0,116,0,106,6,107,6,144,0,114,254,116,
-    0,106,6,124,0,25,0,125,8,122,10,124,8,106,11,125,
-    9,87,0,110,28,4,0,116,9,107,10,114,226,1,0,1,
-    0,1,0,124,7,6,0,89,0,2,0,1,0,83,0,88,
-    0,124,9,100,1,107,8,114,244,124,7,2,0,1,0,83,
-    0,124,9,2,0,1,0,83,0,113,52,124,7,2,0,1,
-    0,83,0,113,52,100,1,83,0,41,4,122,21,70,105,110,
-    100,32,97,32,109,111,100,117,108,101,39,115,32,115,112,101,
-    99,46,78,122,53,115,121,115,46,109,101,116,97,95,112,97,
-    116,104,32,105,115,32,78,111,110,101,44,32,80,121,116,104,
-    111,110,32,105,115,32,108,105,107,101,108,121,32,115,104,117,
-    116,116,105,110,103,32,100,111,119,110,122,22,115,121,115,46,
-    109,101,116,97,95,112,97,116,104,32,105,115,32,101,109,112,
-    116,121,41,12,114,15,0,0,0,218,9,109,101,116,97,95,
-    112,97,116,104,114,78,0,0,0,218,9,95,119,97,114,110,
-    105,110,103,115,218,4,119,97,114,110,218,13,73,109,112,111,
-    114,116,87,97,114,110,105,110,103,114,91,0,0,0,114,177,
-    0,0,0,114,165,0,0,0,114,105,0,0,0,114,188,0,
-    0,0,114,104,0,0,0,41,10,114,17,0,0,0,114,163,
-    0,0,0,114,164,0,0,0,114,189,0,0,0,90,9,105,
-    115,95,114,101,108,111,97,100,114,187,0,0,0,114,165,0,
-    0,0,114,94,0,0,0,114,95,0,0,0,114,104,0,0,
+    105,109,112,111,114,116,32,119,105,116,104,32,110,111,32,107,
+    110,111,119,110,32,112,97,114,101,110,116,32,112,97,99,107,
+    97,103,101,122,17,69,109,112,116,121,32,109,111,100,117,108,
+    101,32,110,97,109,101,78,41,7,218,10,105,115,105,110,115,
+    116,97,110,99,101,218,3,115,116,114,218,9,84,121,112,101,
+    69,114,114,111,114,114,44,0,0,0,114,14,0,0,0,218,
+    10,86,97,108,117,101,69,114,114,111,114,114,78,0,0,0,
+    169,3,114,17,0,0,0,114,185,0,0,0,114,186,0,0,
     0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,
-    218,10,95,102,105,110,100,95,115,112,101,99,122,3,0,0,
-    115,54,0,0,0,0,2,6,1,8,2,8,3,4,1,12,
-    5,10,1,8,1,8,1,2,1,10,1,14,1,12,1,8,
-    1,20,2,22,1,8,2,18,1,10,1,2,1,10,1,14,
-    4,14,2,8,1,8,2,10,2,10,2,114,193,0,0,0,
-    99,3,0,0,0,0,0,0,0,0,0,0,0,3,0,0,
-    0,5,0,0,0,67,0,0,0,115,108,0,0,0,116,0,
-    124,0,116,1,131,2,115,28,116,2,100,1,160,3,116,4,
-    124,0,131,1,161,1,131,1,130,1,124,2,100,2,107,0,
-    114,44,116,5,100,3,131,1,130,1,124,2,100,2,107,4,
-    114,84,116,0,124,1,116,1,131,2,115,72,116,2,100,4,
-    131,1,130,1,110,12,124,1,115,84,116,6,100,5,131,1,
-    130,1,124,0,115,104,124,2,100,2,107,2,114,104,116,5,
-    100,6,131,1,130,1,100,7,83,0,41,8,122,28,86,101,
-    114,105,102,121,32,97,114,103,117,109,101,110,116,115,32,97,
-    114,101,32,34,115,97,110,101,34,46,122,31,109,111,100,117,
-    108,101,32,110,97,109,101,32,109,117,115,116,32,98,101,32,
-    115,116,114,44,32,110,111,116,32,123,125,114,22,0,0,0,
-    122,18,108,101,118,101,108,32,109,117,115,116,32,98,101,32,
-    62,61,32,48,122,31,95,95,112,97,99,107,97,103,101,95,
-    95,32,110,111,116,32,115,101,116,32,116,111,32,97,32,115,
-    116,114,105,110,103,122,54,97,116,116,101,109,112,116,101,100,
-    32,114,101,108,97,116,105,118,101,32,105,109,112,111,114,116,
-    32,119,105,116,104,32,110,111,32,107,110,111,119,110,32,112,
-    97,114,101,110,116,32,112,97,99,107,97,103,101,122,17,69,
-    109,112,116,121,32,109,111,100,117,108,101,32,110,97,109,101,
-    78,41,7,218,10,105,115,105,110,115,116,97,110,99,101,218,
-    3,115,116,114,218,9,84,121,112,101,69,114,114,111,114,114,
-    44,0,0,0,114,14,0,0,0,218,10,86,97,108,117,101,
-    69,114,114,111,114,114,78,0,0,0,169,3,114,17,0,0,
-    0,114,184,0,0,0,114,185,0,0,0,114,10,0,0,0,
-    114,10,0,0,0,114,11,0,0,0,218,13,95,115,97,110,
-    105,116,121,95,99,104,101,99,107,169,3,0,0,115,22,0,
-    0,0,0,2,10,1,18,1,8,1,8,1,8,1,10,1,
-    10,1,4,1,8,2,12,1,114,199,0,0,0,122,16,78,
-    111,32,109,111,100,117,108,101,32,110,97,109,101,100,32,122,
-    4,123,33,114,125,99,2,0,0,0,0,0,0,0,0,0,
-    0,0,8,0,0,0,8,0,0,0,67,0,0,0,115,220,
-    0,0,0,100,0,125,2,124,0,160,0,100,1,161,1,100,
-    2,25,0,125,3,124,3,114,134,124,3,116,1,106,2,107,
-    7,114,42,116,3,124,1,124,3,131,2,1,0,124,0,116,
-    1,106,2,107,6,114,62,116,1,106,2,124,0,25,0,83,
-    0,116,1,106,2,124,3,25,0,125,4,122,10,124,4,106,
-    4,125,2,87,0,110,50,4,0,116,5,107,10,114,132,1,
-    0,1,0,1,0,116,6,100,3,23,0,160,7,124,0,124,
-    3,161,2,125,5,116,8,124,5,124,0,100,4,141,2,100,
-    0,130,2,89,0,110,2,88,0,116,9,124,0,124,2,131,
-    2,125,6,124,6,100,0,107,8,114,172,116,8,116,6,160,
-    7,124,0,161,1,124,0,100,4,141,2,130,1,110,8,116,
-    10,124,6,131,1,125,7,124,3,114,216,116,1,106,2,124,
-    3,25,0,125,4,116,11,124,4,124,0,160,0,100,1,161,
-    1,100,5,25,0,124,7,131,3,1,0,124,7,83,0,41,
-    6,78,114,127,0,0,0,114,22,0,0,0,122,23,59,32,
-    123,33,114,125,32,105,115,32,110,111,116,32,97,32,112,97,
-    99,107,97,103,101,114,16,0,0,0,233,2,0,0,0,41,
-    12,114,128,0,0,0,114,15,0,0,0,114,91,0,0,0,
-    114,66,0,0,0,114,140,0,0,0,114,105,0,0,0,218,
-    8,95,69,82,82,95,77,83,71,114,44,0,0,0,218,19,
-    77,111,100,117,108,101,78,111,116,70,111,117,110,100,69,114,
-    114,111,114,114,193,0,0,0,114,158,0,0,0,114,5,0,
-    0,0,41,8,114,17,0,0,0,218,7,105,109,112,111,114,
-    116,95,114,163,0,0,0,114,129,0,0,0,90,13,112,97,
-    114,101,110,116,95,109,111,100,117,108,101,114,156,0,0,0,
-    114,94,0,0,0,114,95,0,0,0,114,10,0,0,0,114,
-    10,0,0,0,114,11,0,0,0,218,23,95,102,105,110,100,
-    95,97,110,100,95,108,111,97,100,95,117,110,108,111,99,107,
-    101,100,188,3,0,0,115,42,0,0,0,0,1,4,1,14,
-    1,4,1,10,1,10,2,10,1,10,1,10,1,2,1,10,
-    1,14,1,16,1,20,1,10,1,8,1,20,2,8,1,4,
-    2,10,1,22,1,114,204,0,0,0,99,2,0,0,0,0,
-    0,0,0,0,0,0,0,4,0,0,0,10,0,0,0,67,
-    0,0,0,115,106,0,0,0,116,0,124,0,131,1,143,50,
-    1,0,116,1,106,2,160,3,124,0,116,4,161,2,125,2,
-    124,2,116,4,107,8,114,54,116,5,124,0,124,1,131,2,
-    87,0,2,0,53,0,81,0,82,0,163,0,83,0,87,0,
-    53,0,81,0,82,0,88,0,124,2,100,1,107,8,114,94,
-    100,2,160,6,124,0,161,1,125,3,116,7,124,3,124,0,
-    100,3,141,2,130,1,116,8,124,0,131,1,1,0,124,2,
-    83,0,41,4,122,25,70,105,110,100,32,97,110,100,32,108,
-    111,97,100,32,116,104,101,32,109,111,100,117,108,101,46,78,
-    122,40,105,109,112,111,114,116,32,111,102,32,123,125,32,104,
-    97,108,116,101,100,59,32,78,111,110,101,32,105,110,32,115,
-    121,115,46,109,111,100,117,108,101,115,114,16,0,0,0,41,
-    9,114,49,0,0,0,114,15,0,0,0,114,91,0,0,0,
-    114,34,0,0,0,218,14,95,78,69,69,68,83,95,76,79,
-    65,68,73,78,71,114,204,0,0,0,114,44,0,0,0,114,
-    202,0,0,0,114,64,0,0,0,41,4,114,17,0,0,0,
-    114,203,0,0,0,114,95,0,0,0,114,74,0,0,0,114,
-    10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,14,
-    95,102,105,110,100,95,97,110,100,95,108,111,97,100,218,3,
-    0,0,115,22,0,0,0,0,2,10,1,14,1,8,1,32,
-    2,8,1,4,1,2,255,4,2,12,2,8,1,114,206,0,
-    0,0,114,22,0,0,0,99,3,0,0,0,0,0,0,0,
-    0,0,0,0,3,0,0,0,4,0,0,0,67,0,0,0,
-    115,42,0,0,0,116,0,124,0,124,1,124,2,131,3,1,
-    0,124,2,100,1,107,4,114,32,116,1,124,0,124,1,124,
-    2,131,3,125,0,116,2,124,0,116,3,131,2,83,0,41,
-    2,97,50,1,0,0,73,109,112,111,114,116,32,97,110,100,
-    32,114,101,116,117,114,110,32,116,104,101,32,109,111,100,117,
-    108,101,32,98,97,115,101,100,32,111,110,32,105,116,115,32,
-    110,97,109,101,44,32,116,104,101,32,112,97,99,107,97,103,
-    101,32,116,104,101,32,99,97,108,108,32,105,115,10,32,32,
-    32,32,98,101,105,110,103,32,109,97,100,101,32,102,114,111,
-    109,44,32,97,110,100,32,116,104,101,32,108,101,118,101,108,
-    32,97,100,106,117,115,116,109,101,110,116,46,10,10,32,32,
-    32,32,84,104,105,115,32,102,117,110,99,116,105,111,110,32,
-    114,101,112,114,101,115,101,110,116,115,32,116,104,101,32,103,
-    114,101,97,116,101,115,116,32,99,111,109,109,111,110,32,100,
-    101,110,111,109,105,110,97,116,111,114,32,111,102,32,102,117,
-    110,99,116,105,111,110,97,108,105,116,121,10,32,32,32,32,
-    98,101,116,119,101,101,110,32,105,109,112,111,114,116,95,109,
-    111,100,117,108,101,32,97,110,100,32,95,95,105,109,112,111,
-    114,116,95,95,46,32,84,104,105,115,32,105,110,99,108,117,
-    100,101,115,32,115,101,116,116,105,110,103,32,95,95,112,97,
-    99,107,97,103,101,95,95,32,105,102,10,32,32,32,32,116,
-    104,101,32,108,111,97,100,101,114,32,100,105,100,32,110,111,
-    116,46,10,10,32,32,32,32,114,22,0,0,0,41,4,114,
-    199,0,0,0,114,186,0,0,0,114,206,0,0,0,218,11,
-    95,103,99,100,95,105,109,112,111,114,116,114,198,0,0,0,
-    114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,
-    207,0,0,0,234,3,0,0,115,8,0,0,0,0,9,12,
-    1,8,1,12,1,114,207,0,0,0,169,1,218,9,114,101,
-    99,117,114,115,105,118,101,99,3,0,0,0,0,0,0,0,
-    1,0,0,0,8,0,0,0,11,0,0,0,67,0,0,0,
-    115,226,0,0,0,124,1,68,0,93,216,125,4,116,0,124,
-    4,116,1,131,2,115,66,124,3,114,34,124,0,106,2,100,
-    1,23,0,125,5,110,4,100,2,125,5,116,3,100,3,124,
-    5,155,0,100,4,116,4,124,4,131,1,106,2,155,0,157,
-    4,131,1,130,1,113,4,124,4,100,5,107,2,114,108,124,
-    3,115,220,116,5,124,0,100,6,131,2,114,220,116,6,124,
-    0,124,0,106,7,124,2,100,7,100,8,141,4,1,0,113,
-    4,116,5,124,0,124,4,131,2,115,4,100,9,160,8,124,
-    0,106,2,124,4,161,2,125,6,122,14,116,9,124,2,124,
-    6,131,2,1,0,87,0,113,4,4,0,116,10,107,10,114,
-    218,1,0,125,7,1,0,122,42,124,7,106,11,124,6,107,
-    2,114,200,116,12,106,13,160,14,124,6,116,15,161,2,100,
-    10,107,9,114,200,87,0,89,0,162,8,113,4,130,0,87,
-    0,53,0,100,10,125,7,126,7,88,0,89,0,113,4,88,
-    0,113,4,124,0,83,0,41,11,122,238,70,105,103,117,114,
-    101,32,111,117,116,32,119,104,97,116,32,95,95,105,109,112,
-    111,114,116,95,95,32,115,104,111,117,108,100,32,114,101,116,
-    117,114,110,46,10,10,32,32,32,32,84,104,101,32,105,109,
-    112,111,114,116,95,32,112,97,114,97,109,101,116,101,114,32,
-    105,115,32,97,32,99,97,108,108,97,98,108,101,32,119,104,
-    105,99,104,32,116,97,107,101,115,32,116,104,101,32,110,97,
-    109,101,32,111,102,32,109,111,100,117,108,101,32,116,111,10,
-    32,32,32,32,105,109,112,111,114,116,46,32,73,116,32,105,
-    115,32,114,101,113,117,105,114,101,100,32,116,111,32,100,101,
-    99,111,117,112,108,101,32,116,104,101,32,102,117,110,99,116,
-    105,111,110,32,102,114,111,109,32,97,115,115,117,109,105,110,
-    103,32,105,109,112,111,114,116,108,105,98,39,115,10,32,32,
-    32,32,105,109,112,111,114,116,32,105,109,112,108,101,109,101,
-    110,116,97,116,105,111,110,32,105,115,32,100,101,115,105,114,
-    101,100,46,10,10,32,32,32,32,122,8,46,95,95,97,108,
-    108,95,95,122,13,96,96,102,114,111,109,32,108,105,115,116,
-    39,39,122,8,73,116,101,109,32,105,110,32,122,18,32,109,
-    117,115,116,32,98,101,32,115,116,114,44,32,110,111,116,32,
-    250,1,42,218,7,95,95,97,108,108,95,95,84,114,208,0,
-    0,0,114,181,0,0,0,78,41,16,114,194,0,0,0,114,
-    195,0,0,0,114,1,0,0,0,114,196,0,0,0,114,14,
-    0,0,0,114,4,0,0,0,218,16,95,104,97,110,100,108,
-    101,95,102,114,111,109,108,105,115,116,114,211,0,0,0,114,
-    44,0,0,0,114,66,0,0,0,114,202,0,0,0,114,17,
-    0,0,0,114,15,0,0,0,114,91,0,0,0,114,34,0,
-    0,0,114,205,0,0,0,41,8,114,95,0,0,0,218,8,
-    102,114,111,109,108,105,115,116,114,203,0,0,0,114,209,0,
-    0,0,218,1,120,90,5,119,104,101,114,101,90,9,102,114,
-    111,109,95,110,97,109,101,90,3,101,120,99,114,10,0,0,
-    0,114,10,0,0,0,114,11,0,0,0,114,212,0,0,0,
-    249,3,0,0,115,44,0,0,0,0,10,8,1,10,1,4,
-    1,12,2,4,1,28,2,8,1,14,1,10,1,2,255,8,
-    2,10,1,14,1,2,1,14,1,16,4,10,1,16,255,2,
-    2,8,1,22,1,114,212,0,0,0,99,1,0,0,0,0,
-    0,0,0,0,0,0,0,3,0,0,0,6,0,0,0,67,
-    0,0,0,115,146,0,0,0,124,0,160,0,100,1,161,1,
-    125,1,124,0,160,0,100,2,161,1,125,2,124,1,100,3,
-    107,9,114,82,124,2,100,3,107,9,114,78,124,1,124,2,
-    106,1,107,3,114,78,116,2,106,3,100,4,124,1,155,2,
-    100,5,124,2,106,1,155,2,100,6,157,5,116,4,100,7,
-    100,8,141,3,1,0,124,1,83,0,124,2,100,3,107,9,
-    114,96,124,2,106,1,83,0,116,2,106,3,100,9,116,4,
-    100,7,100,8,141,3,1,0,124,0,100,10,25,0,125,1,
-    100,11,124,0,107,7,114,142,124,1,160,5,100,12,161,1,
-    100,13,25,0,125,1,124,1,83,0,41,14,122,167,67,97,
-    108,99,117,108,97,116,101,32,119,104,97,116,32,95,95,112,
-    97,99,107,97,103,101,95,95,32,115,104,111,117,108,100,32,
-    98,101,46,10,10,32,32,32,32,95,95,112,97,99,107,97,
-    103,101,95,95,32,105,115,32,110,111,116,32,103,117,97,114,
-    97,110,116,101,101,100,32,116,111,32,98,101,32,100,101,102,
-    105,110,101,100,32,111,114,32,99,111,117,108,100,32,98,101,
-    32,115,101,116,32,116,111,32,78,111,110,101,10,32,32,32,
-    32,116,111,32,114,101,112,114,101,115,101,110,116,32,116,104,
-    97,116,32,105,116,115,32,112,114,111,112,101,114,32,118,97,
-    108,117,101,32,105,115,32,117,110,107,110,111,119,110,46,10,
-    10,32,32,32,32,114,144,0,0,0,114,104,0,0,0,78,
-    122,32,95,95,112,97,99,107,97,103,101,95,95,32,33,61,
-    32,95,95,115,112,101,99,95,95,46,112,97,114,101,110,116,
-    32,40,122,4,32,33,61,32,250,1,41,233,3,0,0,0,
-    41,1,90,10,115,116,97,99,107,108,101,118,101,108,122,89,
-    99,97,110,39,116,32,114,101,115,111,108,118,101,32,112,97,
-    99,107,97,103,101,32,102,114,111,109,32,95,95,115,112,101,
-    99,95,95,32,111,114,32,95,95,112,97,99,107,97,103,101,
-    95,95,44,32,102,97,108,108,105,110,103,32,98,97,99,107,
-    32,111,110,32,95,95,110,97,109,101,95,95,32,97,110,100,
-    32,95,95,112,97,116,104,95,95,114,1,0,0,0,114,140,
-    0,0,0,114,127,0,0,0,114,22,0,0,0,41,6,114,
-    34,0,0,0,114,129,0,0,0,114,190,0,0,0,114,191,
-    0,0,0,114,192,0,0,0,114,128,0,0,0,41,3,218,
-    7,103,108,111,98,97,108,115,114,184,0,0,0,114,94,0,
-    0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,
-    0,218,17,95,99,97,108,99,95,95,95,112,97,99,107,97,
-    103,101,95,95,30,4,0,0,115,38,0,0,0,0,7,10,
-    1,10,1,8,1,18,1,22,2,2,0,2,254,6,3,4,
-    1,8,1,6,2,6,2,2,0,2,254,6,3,8,1,8,
-    1,14,1,114,218,0,0,0,114,10,0,0,0,99,5,0,
-    0,0,0,0,0,0,0,0,0,0,9,0,0,0,5,0,
-    0,0,67,0,0,0,115,180,0,0,0,124,4,100,1,107,
-    2,114,18,116,0,124,0,131,1,125,5,110,36,124,1,100,
-    2,107,9,114,30,124,1,110,2,105,0,125,6,116,1,124,
-    6,131,1,125,7,116,0,124,0,124,7,124,4,131,3,125,
-    5,124,3,115,150,124,4,100,1,107,2,114,84,116,0,124,
-    0,160,2,100,3,161,1,100,1,25,0,131,1,83,0,124,
-    0,115,92,124,5,83,0,116,3,124,0,131,1,116,3,124,
-    0,160,2,100,3,161,1,100,1,25,0,131,1,24,0,125,
-    8,116,4,106,5,124,5,106,6,100,2,116,3,124,5,106,
-    6,131,1,124,8,24,0,133,2,25,0,25,0,83,0,110,
-    26,116,7,124,5,100,4,131,2,114,172,116,8,124,5,124,
-    3,116,0,131,3,83,0,124,5,83,0,100,2,83,0,41,
-    5,97,215,1,0,0,73,109,112,111,114,116,32,97,32,109,
-    111,100,117,108,101,46,10,10,32,32,32,32,84,104,101,32,
-    39,103,108,111,98,97,108,115,39,32,97,114,103,117,109,101,
-    110,116,32,105,115,32,117,115,101,100,32,116,111,32,105,110,
-    102,101,114,32,119,104,101,114,101,32,116,104,101,32,105,109,
-    112,111,114,116,32,105,115,32,111,99,99,117,114,114,105,110,
-    103,32,102,114,111,109,10,32,32,32,32,116,111,32,104,97,
-    110,100,108,101,32,114,101,108,97,116,105,118,101,32,105,109,
-    112,111,114,116,115,46,32,84,104,101,32,39,108,111,99,97,
-    108,115,39,32,97,114,103,117,109,101,110,116,32,105,115,32,
-    105,103,110,111,114,101,100,46,32,84,104,101,10,32,32,32,
-    32,39,102,114,111,109,108,105,115,116,39,32,97,114,103,117,
-    109,101,110,116,32,115,112,101,99,105,102,105,101,115,32,119,
-    104,97,116,32,115,104,111,117,108,100,32,101,120,105,115,116,
-    32,97,115,32,97,116,116,114,105,98,117,116,101,115,32,111,
-    110,32,116,104,101,32,109,111,100,117,108,101,10,32,32,32,
-    32,98,101,105,110,103,32,105,109,112,111,114,116,101,100,32,
-    40,101,46,103,46,32,96,96,102,114,111,109,32,109,111,100,
-    117,108,101,32,105,109,112,111,114,116,32,60,102,114,111,109,
-    108,105,115,116,62,96,96,41,46,32,32,84,104,101,32,39,
-    108,101,118,101,108,39,10,32,32,32,32,97,114,103,117,109,
-    101,110,116,32,114,101,112,114,101,115,101,110,116,115,32,116,
-    104,101,32,112,97,99,107,97,103,101,32,108,111,99,97,116,
-    105,111,110,32,116,111,32,105,109,112,111,114,116,32,102,114,
-    111,109,32,105,110,32,97,32,114,101,108,97,116,105,118,101,
-    10,32,32,32,32,105,109,112,111,114,116,32,40,101,46,103,
-    46,32,96,96,102,114,111,109,32,46,46,112,107,103,32,105,
-    109,112,111,114,116,32,109,111,100,96,96,32,119,111,117,108,
-    100,32,104,97,118,101,32,97,32,39,108,101,118,101,108,39,
-    32,111,102,32,50,41,46,10,10,32,32,32,32,114,22,0,
-    0,0,78,114,127,0,0,0,114,140,0,0,0,41,9,114,
-    207,0,0,0,114,218,0,0,0,218,9,112,97,114,116,105,
-    116,105,111,110,114,183,0,0,0,114,15,0,0,0,114,91,
-    0,0,0,114,1,0,0,0,114,4,0,0,0,114,212,0,
-    0,0,41,9,114,17,0,0,0,114,217,0,0,0,218,6,
-    108,111,99,97,108,115,114,213,0,0,0,114,185,0,0,0,
-    114,95,0,0,0,90,8,103,108,111,98,97,108,115,95,114,
-    184,0,0,0,90,7,99,117,116,95,111,102,102,114,10,0,
-    0,0,114,10,0,0,0,114,11,0,0,0,218,10,95,95,
-    105,109,112,111,114,116,95,95,57,4,0,0,115,30,0,0,
-    0,0,11,8,1,10,2,16,1,8,1,12,1,4,3,8,
-    1,18,1,4,1,4,4,26,3,32,1,10,1,12,2,114,
-    221,0,0,0,99,1,0,0,0,0,0,0,0,0,0,0,
-    0,2,0,0,0,3,0,0,0,67,0,0,0,115,38,0,
-    0,0,116,0,160,1,124,0,161,1,125,1,124,1,100,0,
-    107,8,114,30,116,2,100,1,124,0,23,0,131,1,130,1,
-    116,3,124,1,131,1,83,0,41,2,78,122,25,110,111,32,
-    98,117,105,108,116,45,105,110,32,109,111,100,117,108,101,32,
-    110,97,109,101,100,32,41,4,114,159,0,0,0,114,165,0,
-    0,0,114,78,0,0,0,114,158,0,0,0,41,2,114,17,
+    218,13,95,115,97,110,105,116,121,95,99,104,101,99,107,171,
+    3,0,0,115,22,0,0,0,0,2,10,1,18,1,8,1,
+    8,1,8,1,10,1,10,1,4,1,8,2,12,1,114,200,
+    0,0,0,122,16,78,111,32,109,111,100,117,108,101,32,110,
+    97,109,101,100,32,122,4,123,33,114,125,99,2,0,0,0,
+    0,0,0,0,0,0,0,0,8,0,0,0,8,0,0,0,
+    67,0,0,0,115,220,0,0,0,100,0,125,2,124,0,160,
+    0,100,1,161,1,100,2,25,0,125,3,124,3,114,134,124,
+    3,116,1,106,2,107,7,114,42,116,3,124,1,124,3,131,
+    2,1,0,124,0,116,1,106,2,107,6,114,62,116,1,106,
+    2,124,0,25,0,83,0,116,1,106,2,124,3,25,0,125,
+    4,122,10,124,4,106,4,125,2,87,0,110,50,4,0,116,
+    5,107,10,114,132,1,0,1,0,1,0,116,6,100,3,23,
+    0,160,7,124,0,124,3,161,2,125,5,116,8,124,5,124,
+    0,100,4,141,2,100,0,130,2,89,0,110,2,88,0,116,
+    9,124,0,124,2,131,2,125,6,124,6,100,0,107,8,114,
+    172,116,8,116,6,160,7,124,0,161,1,124,0,100,4,141,
+    2,130,1,110,8,116,10,124,6,131,1,125,7,124,3,114,
+    216,116,1,106,2,124,3,25,0,125,4,116,11,124,4,124,
+    0,160,0,100,1,161,1,100,5,25,0,124,7,131,3,1,
+    0,124,7,83,0,41,6,78,114,127,0,0,0,114,22,0,
+    0,0,122,23,59,32,123,33,114,125,32,105,115,32,110,111,
+    116,32,97,32,112,97,99,107,97,103,101,114,16,0,0,0,
+    233,2,0,0,0,41,12,114,128,0,0,0,114,15,0,0,
+    0,114,91,0,0,0,114,66,0,0,0,114,140,0,0,0,
+    114,105,0,0,0,218,8,95,69,82,82,95,77,83,71,114,
+    44,0,0,0,218,19,77,111,100,117,108,101,78,111,116,70,
+    111,117,110,100,69,114,114,111,114,114,194,0,0,0,114,158,
+    0,0,0,114,5,0,0,0,41,8,114,17,0,0,0,218,
+    7,105,109,112,111,114,116,95,114,164,0,0,0,114,129,0,
+    0,0,90,13,112,97,114,101,110,116,95,109,111,100,117,108,
+    101,114,156,0,0,0,114,94,0,0,0,114,95,0,0,0,
+    114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,
+    23,95,102,105,110,100,95,97,110,100,95,108,111,97,100,95,
+    117,110,108,111,99,107,101,100,190,3,0,0,115,42,0,0,
+    0,0,1,4,1,14,1,4,1,10,1,10,2,10,1,10,
+    1,10,1,2,1,10,1,14,1,16,1,20,1,10,1,8,
+    1,20,2,8,1,4,2,10,1,22,1,114,205,0,0,0,
+    99,2,0,0,0,0,0,0,0,0,0,0,0,4,0,0,
+    0,10,0,0,0,67,0,0,0,115,106,0,0,0,116,0,
+    124,0,131,1,143,50,1,0,116,1,106,2,160,3,124,0,
+    116,4,161,2,125,2,124,2,116,4,107,8,114,54,116,5,
+    124,0,124,1,131,2,87,0,2,0,53,0,81,0,82,0,
+    163,0,83,0,87,0,53,0,81,0,82,0,88,0,124,2,
+    100,1,107,8,114,94,100,2,160,6,124,0,161,1,125,3,
+    116,7,124,3,124,0,100,3,141,2,130,1,116,8,124,0,
+    131,1,1,0,124,2,83,0,41,4,122,25,70,105,110,100,
+    32,97,110,100,32,108,111,97,100,32,116,104,101,32,109,111,
+    100,117,108,101,46,78,122,40,105,109,112,111,114,116,32,111,
+    102,32,123,125,32,104,97,108,116,101,100,59,32,78,111,110,
+    101,32,105,110,32,115,121,115,46,109,111,100,117,108,101,115,
+    114,16,0,0,0,41,9,114,49,0,0,0,114,15,0,0,
+    0,114,91,0,0,0,114,34,0,0,0,218,14,95,78,69,
+    69,68,83,95,76,79,65,68,73,78,71,114,205,0,0,0,
+    114,44,0,0,0,114,203,0,0,0,114,64,0,0,0,41,
+    4,114,17,0,0,0,114,204,0,0,0,114,95,0,0,0,
+    114,74,0,0,0,114,10,0,0,0,114,10,0,0,0,114,
+    11,0,0,0,218,14,95,102,105,110,100,95,97,110,100,95,
+    108,111,97,100,220,3,0,0,115,22,0,0,0,0,2,10,
+    1,14,1,8,1,32,2,8,1,4,1,2,255,4,2,12,
+    2,8,1,114,207,0,0,0,114,22,0,0,0,99,3,0,
+    0,0,0,0,0,0,0,0,0,0,3,0,0,0,4,0,
+    0,0,67,0,0,0,115,42,0,0,0,116,0,124,0,124,
+    1,124,2,131,3,1,0,124,2,100,1,107,4,114,32,116,
+    1,124,0,124,1,124,2,131,3,125,0,116,2,124,0,116,
+    3,131,2,83,0,41,2,97,50,1,0,0,73,109,112,111,
+    114,116,32,97,110,100,32,114,101,116,117,114,110,32,116,104,
+    101,32,109,111,100,117,108,101,32,98,97,115,101,100,32,111,
+    110,32,105,116,115,32,110,97,109,101,44,32,116,104,101,32,
+    112,97,99,107,97,103,101,32,116,104,101,32,99,97,108,108,
+    32,105,115,10,32,32,32,32,98,101,105,110,103,32,109,97,
+    100,101,32,102,114,111,109,44,32,97,110,100,32,116,104,101,
+    32,108,101,118,101,108,32,97,100,106,117,115,116,109,101,110,
+    116,46,10,10,32,32,32,32,84,104,105,115,32,102,117,110,
+    99,116,105,111,110,32,114,101,112,114,101,115,101,110,116,115,
+    32,116,104,101,32,103,114,101,97,116,101,115,116,32,99,111,
+    109,109,111,110,32,100,101,110,111,109,105,110,97,116,111,114,
+    32,111,102,32,102,117,110,99,116,105,111,110,97,108,105,116,
+    121,10,32,32,32,32,98,101,116,119,101,101,110,32,105,109,
+    112,111,114,116,95,109,111,100,117,108,101,32,97,110,100,32,
+    95,95,105,109,112,111,114,116,95,95,46,32,84,104,105,115,
+    32,105,110,99,108,117,100,101,115,32,115,101,116,116,105,110,
+    103,32,95,95,112,97,99,107,97,103,101,95,95,32,105,102,
+    10,32,32,32,32,116,104,101,32,108,111,97,100,101,114,32,
+    100,105,100,32,110,111,116,46,10,10,32,32,32,32,114,22,
+    0,0,0,41,4,114,200,0,0,0,114,187,0,0,0,114,
+    207,0,0,0,218,11,95,103,99,100,95,105,109,112,111,114,
+    116,114,199,0,0,0,114,10,0,0,0,114,10,0,0,0,
+    114,11,0,0,0,114,208,0,0,0,236,3,0,0,115,8,
+    0,0,0,0,9,12,1,8,1,12,1,114,208,0,0,0,
+    169,1,218,9,114,101,99,117,114,115,105,118,101,99,3,0,
+    0,0,0,0,0,0,1,0,0,0,8,0,0,0,11,0,
+    0,0,67,0,0,0,115,226,0,0,0,124,1,68,0,93,
+    216,125,4,116,0,124,4,116,1,131,2,115,66,124,3,114,
+    34,124,0,106,2,100,1,23,0,125,5,110,4,100,2,125,
+    5,116,3,100,3,124,5,155,0,100,4,116,4,124,4,131,
+    1,106,2,155,0,157,4,131,1,130,1,113,4,124,4,100,
+    5,107,2,114,108,124,3,115,220,116,5,124,0,100,6,131,
+    2,114,220,116,6,124,0,124,0,106,7,124,2,100,7,100,
+    8,141,4,1,0,113,4,116,5,124,0,124,4,131,2,115,
+    4,100,9,160,8,124,0,106,2,124,4,161,2,125,6,122,
+    14,116,9,124,2,124,6,131,2,1,0,87,0,113,4,4,
+    0,116,10,107,10,114,218,1,0,125,7,1,0,122,42,124,
+    7,106,11,124,6,107,2,114,200,116,12,106,13,160,14,124,
+    6,116,15,161,2,100,10,107,9,114,200,87,0,89,0,162,
+    8,113,4,130,0,87,0,53,0,100,10,125,7,126,7,88,
+    0,89,0,113,4,88,0,113,4,124,0,83,0,41,11,122,
+    238,70,105,103,117,114,101,32,111,117,116,32,119,104,97,116,
+    32,95,95,105,109,112,111,114,116,95,95,32,115,104,111,117,
+    108,100,32,114,101,116,117,114,110,46,10,10,32,32,32,32,
+    84,104,101,32,105,109,112,111,114,116,95,32,112,97,114,97,
+    109,101,116,101,114,32,105,115,32,97,32,99,97,108,108,97,
+    98,108,101,32,119,104,105,99,104,32,116,97,107,101,115,32,
+    116,104,101,32,110,97,109,101,32,111,102,32,109,111,100,117,
+    108,101,32,116,111,10,32,32,32,32,105,109,112,111,114,116,
+    46,32,73,116,32,105,115,32,114,101,113,117,105,114,101,100,
+    32,116,111,32,100,101,99,111,117,112,108,101,32,116,104,101,
+    32,102,117,110,99,116,105,111,110,32,102,114,111,109,32,97,
+    115,115,117,109,105,110,103,32,105,109,112,111,114,116,108,105,
+    98,39,115,10,32,32,32,32,105,109,112,111,114,116,32,105,
+    109,112,108,101,109,101,110,116,97,116,105,111,110,32,105,115,
+    32,100,101,115,105,114,101,100,46,10,10,32,32,32,32,122,
+    8,46,95,95,97,108,108,95,95,122,13,96,96,102,114,111,
+    109,32,108,105,115,116,39,39,122,8,73,116,101,109,32,105,
+    110,32,122,18,32,109,117,115,116,32,98,101,32,115,116,114,
+    44,32,110,111,116,32,250,1,42,218,7,95,95,97,108,108,
+    95,95,84,114,209,0,0,0,114,182,0,0,0,78,41,16,
+    114,195,0,0,0,114,196,0,0,0,114,1,0,0,0,114,
+    197,0,0,0,114,14,0,0,0,114,4,0,0,0,218,16,
+    95,104,97,110,100,108,101,95,102,114,111,109,108,105,115,116,
+    114,212,0,0,0,114,44,0,0,0,114,66,0,0,0,114,
+    203,0,0,0,114,17,0,0,0,114,15,0,0,0,114,91,
+    0,0,0,114,34,0,0,0,114,206,0,0,0,41,8,114,
+    95,0,0,0,218,8,102,114,111,109,108,105,115,116,114,204,
+    0,0,0,114,210,0,0,0,218,1,120,90,5,119,104,101,
+    114,101,90,9,102,114,111,109,95,110,97,109,101,90,3,101,
+    120,99,114,10,0,0,0,114,10,0,0,0,114,11,0,0,
+    0,114,213,0,0,0,251,3,0,0,115,44,0,0,0,0,
+    10,8,1,10,1,4,1,12,2,4,1,28,2,8,1,14,
+    1,10,1,2,255,8,2,10,1,14,1,2,1,14,1,16,
+    4,10,1,16,255,2,2,8,1,22,1,114,213,0,0,0,
+    99,1,0,0,0,0,0,0,0,0,0,0,0,3,0,0,
+    0,6,0,0,0,67,0,0,0,115,146,0,0,0,124,0,
+    160,0,100,1,161,1,125,1,124,0,160,0,100,2,161,1,
+    125,2,124,1,100,3,107,9,114,82,124,2,100,3,107,9,
+    114,78,124,1,124,2,106,1,107,3,114,78,116,2,106,3,
+    100,4,124,1,155,2,100,5,124,2,106,1,155,2,100,6,
+    157,5,116,4,100,7,100,8,141,3,1,0,124,1,83,0,
+    124,2,100,3,107,9,114,96,124,2,106,1,83,0,116,2,
+    106,3,100,9,116,4,100,7,100,8,141,3,1,0,124,0,
+    100,10,25,0,125,1,100,11,124,0,107,7,114,142,124,1,
+    160,5,100,12,161,1,100,13,25,0,125,1,124,1,83,0,
+    41,14,122,167,67,97,108,99,117,108,97,116,101,32,119,104,
+    97,116,32,95,95,112,97,99,107,97,103,101,95,95,32,115,
+    104,111,117,108,100,32,98,101,46,10,10,32,32,32,32,95,
+    95,112,97,99,107,97,103,101,95,95,32,105,115,32,110,111,
+    116,32,103,117,97,114,97,110,116,101,101,100,32,116,111,32,
+    98,101,32,100,101,102,105,110,101,100,32,111,114,32,99,111,
+    117,108,100,32,98,101,32,115,101,116,32,116,111,32,78,111,
+    110,101,10,32,32,32,32,116,111,32,114,101,112,114,101,115,
+    101,110,116,32,116,104,97,116,32,105,116,115,32,112,114,111,
+    112,101,114,32,118,97,108,117,101,32,105,115,32,117,110,107,
+    110,111,119,110,46,10,10,32,32,32,32,114,144,0,0,0,
+    114,104,0,0,0,78,122,32,95,95,112,97,99,107,97,103,
+    101,95,95,32,33,61,32,95,95,115,112,101,99,95,95,46,
+    112,97,114,101,110,116,32,40,122,4,32,33,61,32,250,1,
+    41,233,3,0,0,0,41,1,90,10,115,116,97,99,107,108,
+    101,118,101,108,122,89,99,97,110,39,116,32,114,101,115,111,
+    108,118,101,32,112,97,99,107,97,103,101,32,102,114,111,109,
+    32,95,95,115,112,101,99,95,95,32,111,114,32,95,95,112,
+    97,99,107,97,103,101,95,95,44,32,102,97,108,108,105,110,
+    103,32,98,97,99,107,32,111,110,32,95,95,110,97,109,101,
+    95,95,32,97,110,100,32,95,95,112,97,116,104,95,95,114,
+    1,0,0,0,114,140,0,0,0,114,127,0,0,0,114,22,
+    0,0,0,41,6,114,34,0,0,0,114,129,0,0,0,114,
+    191,0,0,0,114,192,0,0,0,114,193,0,0,0,114,128,
+    0,0,0,41,3,218,7,103,108,111,98,97,108,115,114,185,
     0,0,0,114,94,0,0,0,114,10,0,0,0,114,10,0,
-    0,0,114,11,0,0,0,218,18,95,98,117,105,108,116,105,
-    110,95,102,114,111,109,95,110,97,109,101,94,4,0,0,115,
-    8,0,0,0,0,1,10,1,8,1,12,1,114,222,0,0,
-    0,99,2,0,0,0,0,0,0,0,0,0,0,0,10,0,
-    0,0,5,0,0,0,67,0,0,0,115,166,0,0,0,124,
-    1,97,0,124,0,97,1,116,2,116,1,131,1,125,2,116,
-    1,106,3,160,4,161,0,68,0,93,72,92,2,125,3,125,
-    4,116,5,124,4,124,2,131,2,114,26,124,3,116,1,106,
-    6,107,6,114,60,116,7,125,5,110,18,116,0,160,8,124,
-    3,161,1,114,26,116,9,125,5,110,2,113,26,116,10,124,
-    4,124,5,131,2,125,6,116,11,124,6,124,4,131,2,1,
-    0,113,26,116,1,106,3,116,12,25,0,125,7,100,1,68,
-    0,93,46,125,8,124,8,116,1,106,3,107,7,114,138,116,
-    13,124,8,131,1,125,9,110,10,116,1,106,3,124,8,25,
-    0,125,9,116,14,124,7,124,8,124,9,131,3,1,0,113,
-    114,100,2,83,0,41,3,122,250,83,101,116,117,112,32,105,
-    109,112,111,114,116,108,105,98,32,98,121,32,105,109,112,111,
-    114,116,105,110,103,32,110,101,101,100,101,100,32,98,117,105,
-    108,116,45,105,110,32,109,111,100,117,108,101,115,32,97,110,
-    100,32,105,110,106,101,99,116,105,110,103,32,116,104,101,109,
-    10,32,32,32,32,105,110,116,111,32,116,104,101,32,103,108,
-    111,98,97,108,32,110,97,109,101,115,112,97,99,101,46,10,
-    10,32,32,32,32,65,115,32,115,121,115,32,105,115,32,110,
-    101,101,100,101,100,32,102,111,114,32,115,121,115,46,109,111,
-    100,117,108,101,115,32,97,99,99,101,115,115,32,97,110,100,
-    32,95,105,109,112,32,105,115,32,110,101,101,100,101,100,32,
-    116,111,32,108,111,97,100,32,98,117,105,108,116,45,105,110,
-    10,32,32,32,32,109,111,100,117,108,101,115,44,32,116,104,
-    111,115,101,32,116,119,111,32,109,111,100,117,108,101,115,32,
-    109,117,115,116,32,98,101,32,101,120,112,108,105,99,105,116,
-    108,121,32,112,97,115,115,101,100,32,105,110,46,10,10,32,
-    32,32,32,41,3,114,23,0,0,0,114,190,0,0,0,114,
-    63,0,0,0,78,41,15,114,56,0,0,0,114,15,0,0,
-    0,114,14,0,0,0,114,91,0,0,0,218,5,105,116,101,
-    109,115,114,194,0,0,0,114,77,0,0,0,114,159,0,0,
-    0,114,87,0,0,0,114,172,0,0,0,114,141,0,0,0,
-    114,147,0,0,0,114,1,0,0,0,114,222,0,0,0,114,
-    5,0,0,0,41,10,218,10,115,121,115,95,109,111,100,117,
-    108,101,218,11,95,105,109,112,95,109,111,100,117,108,101,90,
-    11,109,111,100,117,108,101,95,116,121,112,101,114,17,0,0,
-    0,114,95,0,0,0,114,108,0,0,0,114,94,0,0,0,
-    90,11,115,101,108,102,95,109,111,100,117,108,101,90,12,98,
-    117,105,108,116,105,110,95,110,97,109,101,90,14,98,117,105,
-    108,116,105,110,95,109,111,100,117,108,101,114,10,0,0,0,
-    114,10,0,0,0,114,11,0,0,0,218,6,95,115,101,116,
-    117,112,101,4,0,0,115,36,0,0,0,0,9,4,1,4,
-    3,8,1,18,1,10,1,10,1,6,1,10,1,6,2,2,
-    1,10,1,12,3,10,1,8,1,10,1,10,2,10,1,114,
-    226,0,0,0,99,2,0,0,0,0,0,0,0,0,0,0,
-    0,2,0,0,0,3,0,0,0,67,0,0,0,115,38,0,
-    0,0,116,0,124,0,124,1,131,2,1,0,116,1,106,2,
-    160,3,116,4,161,1,1,0,116,1,106,2,160,3,116,5,
-    161,1,1,0,100,1,83,0,41,2,122,48,73,110,115,116,
-    97,108,108,32,105,109,112,111,114,116,101,114,115,32,102,111,
-    114,32,98,117,105,108,116,105,110,32,97,110,100,32,102,114,
-    111,122,101,110,32,109,111,100,117,108,101,115,78,41,6,114,
-    226,0,0,0,114,15,0,0,0,114,189,0,0,0,114,118,
-    0,0,0,114,159,0,0,0,114,172,0,0,0,41,2,114,
-    224,0,0,0,114,225,0,0,0,114,10,0,0,0,114,10,
-    0,0,0,114,11,0,0,0,218,8,95,105,110,115,116,97,
-    108,108,136,4,0,0,115,6,0,0,0,0,2,10,2,12,
-    1,114,227,0,0,0,99,0,0,0,0,0,0,0,0,0,
-    0,0,0,1,0,0,0,4,0,0,0,67,0,0,0,115,
-    32,0,0,0,100,1,100,2,108,0,125,0,124,0,97,1,
-    124,0,160,2,116,3,106,4,116,5,25,0,161,1,1,0,
-    100,2,83,0,41,3,122,57,73,110,115,116,97,108,108,32,
-    105,109,112,111,114,116,101,114,115,32,116,104,97,116,32,114,
-    101,113,117,105,114,101,32,101,120,116,101,114,110,97,108,32,
-    102,105,108,101,115,121,115,116,101,109,32,97,99,99,101,115,
-    115,114,22,0,0,0,78,41,6,218,26,95,102,114,111,122,
-    101,110,95,105,109,112,111,114,116,108,105,98,95,101,120,116,
-    101,114,110,97,108,114,125,0,0,0,114,227,0,0,0,114,
-    15,0,0,0,114,91,0,0,0,114,1,0,0,0,41,1,
-    114,228,0,0,0,114,10,0,0,0,114,10,0,0,0,114,
-    11,0,0,0,218,27,95,105,110,115,116,97,108,108,95,101,
-    120,116,101,114,110,97,108,95,105,109,112,111,114,116,101,114,
-    115,144,4,0,0,115,6,0,0,0,0,3,8,1,4,1,
-    114,229,0,0,0,41,2,78,78,41,1,78,41,2,78,114,
-    22,0,0,0,41,4,78,78,114,10,0,0,0,114,22,0,
-    0,0,41,50,114,3,0,0,0,114,125,0,0,0,114,12,
-    0,0,0,114,18,0,0,0,114,59,0,0,0,114,33,0,
-    0,0,114,42,0,0,0,114,19,0,0,0,114,20,0,0,
-    0,114,48,0,0,0,114,49,0,0,0,114,52,0,0,0,
-    114,64,0,0,0,114,66,0,0,0,114,75,0,0,0,114,
-    85,0,0,0,114,89,0,0,0,114,96,0,0,0,114,110,
-    0,0,0,114,111,0,0,0,114,90,0,0,0,114,141,0,
-    0,0,114,147,0,0,0,114,151,0,0,0,114,106,0,0,
-    0,114,92,0,0,0,114,157,0,0,0,114,158,0,0,0,
-    114,93,0,0,0,114,159,0,0,0,114,172,0,0,0,114,
-    177,0,0,0,114,186,0,0,0,114,188,0,0,0,114,193,
-    0,0,0,114,199,0,0,0,90,15,95,69,82,82,95,77,
-    83,71,95,80,82,69,70,73,88,114,201,0,0,0,114,204,
-    0,0,0,218,6,111,98,106,101,99,116,114,205,0,0,0,
-    114,206,0,0,0,114,207,0,0,0,114,212,0,0,0,114,
-    218,0,0,0,114,221,0,0,0,114,222,0,0,0,114,226,
-    0,0,0,114,227,0,0,0,114,229,0,0,0,114,10,0,
-    0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,
-    0,218,8,60,109,111,100,117,108,101,62,1,0,0,0,115,
-    94,0,0,0,4,24,4,2,8,8,8,8,4,2,4,3,
-    16,4,14,68,14,21,14,16,8,37,8,17,8,11,14,8,
-    8,11,8,12,8,16,8,36,14,101,16,26,10,45,14,72,
-    8,17,8,17,8,30,8,37,8,42,8,15,14,73,14,79,
-    14,13,8,9,8,9,10,47,8,16,4,1,8,2,8,27,
-    6,3,8,16,10,15,14,37,8,27,10,37,8,7,8,35,
-    8,8,
+    0,0,114,11,0,0,0,218,17,95,99,97,108,99,95,95,
+    95,112,97,99,107,97,103,101,95,95,32,4,0,0,115,38,
+    0,0,0,0,7,10,1,10,1,8,1,18,1,22,2,2,
+    0,2,254,6,3,4,1,8,1,6,2,6,2,2,0,2,
+    254,6,3,8,1,8,1,14,1,114,219,0,0,0,114,10,
+    0,0,0,99,5,0,0,0,0,0,0,0,0,0,0,0,
+    9,0,0,0,5,0,0,0,67,0,0,0,115,180,0,0,
+    0,124,4,100,1,107,2,114,18,116,0,124,0,131,1,125,
+    5,110,36,124,1,100,2,107,9,114,30,124,1,110,2,105,
+    0,125,6,116,1,124,6,131,1,125,7,116,0,124,0,124,
+    7,124,4,131,3,125,5,124,3,115,150,124,4,100,1,107,
+    2,114,84,116,0,124,0,160,2,100,3,161,1,100,1,25,
+    0,131,1,83,0,124,0,115,92,124,5,83,0,116,3,124,
+    0,131,1,116,3,124,0,160,2,100,3,161,1,100,1,25,
+    0,131,1,24,0,125,8,116,4,106,5,124,5,106,6,100,
+    2,116,3,124,5,106,6,131,1,124,8,24,0,133,2,25,
+    0,25,0,83,0,110,26,116,7,124,5,100,4,131,2,114,
+    172,116,8,124,5,124,3,116,0,131,3,83,0,124,5,83,
+    0,100,2,83,0,41,5,97,215,1,0,0,73,109,112,111,
+    114,116,32,97,32,109,111,100,117,108,101,46,10,10,32,32,
+    32,32,84,104,101,32,39,103,108,111,98,97,108,115,39,32,
+    97,114,103,117,109,101,110,116,32,105,115,32,117,115,101,100,
+    32,116,111,32,105,110,102,101,114,32,119,104,101,114,101,32,
+    116,104,101,32,105,109,112,111,114,116,32,105,115,32,111,99,
+    99,117,114,114,105,110,103,32,102,114,111,109,10,32,32,32,
+    32,116,111,32,104,97,110,100,108,101,32,114,101,108,97,116,
+    105,118,101,32,105,109,112,111,114,116,115,46,32,84,104,101,
+    32,39,108,111,99,97,108,115,39,32,97,114,103,117,109,101,
+    110,116,32,105,115,32,105,103,110,111,114,101,100,46,32,84,
+    104,101,10,32,32,32,32,39,102,114,111,109,108,105,115,116,
+    39,32,97,114,103,117,109,101,110,116,32,115,112,101,99,105,
+    102,105,101,115,32,119,104,97,116,32,115,104,111,117,108,100,
+    32,101,120,105,115,116,32,97,115,32,97,116,116,114,105,98,
+    117,116,101,115,32,111,110,32,116,104,101,32,109,111,100,117,
+    108,101,10,32,32,32,32,98,101,105,110,103,32,105,109,112,
+    111,114,116,101,100,32,40,101,46,103,46,32,96,96,102,114,
+    111,109,32,109,111,100,117,108,101,32,105,109,112,111,114,116,
+    32,60,102,114,111,109,108,105,115,116,62,96,96,41,46,32,
+    32,84,104,101,32,39,108,101,118,101,108,39,10,32,32,32,
+    32,97,114,103,117,109,101,110,116,32,114,101,112,114,101,115,
+    101,110,116,115,32,116,104,101,32,112,97,99,107,97,103,101,
+    32,108,111,99,97,116,105,111,110,32,116,111,32,105,109,112,
+    111,114,116,32,102,114,111,109,32,105,110,32,97,32,114,101,
+    108,97,116,105,118,101,10,32,32,32,32,105,109,112,111,114,
+    116,32,40,101,46,103,46,32,96,96,102,114,111,109,32,46,
+    46,112,107,103,32,105,109,112,111,114,116,32,109,111,100,96,
+    96,32,119,111,117,108,100,32,104,97,118,101,32,97,32,39,
+    108,101,118,101,108,39,32,111,102,32,50,41,46,10,10,32,
+    32,32,32,114,22,0,0,0,78,114,127,0,0,0,114,140,
+    0,0,0,41,9,114,208,0,0,0,114,219,0,0,0,218,
+    9,112,97,114,116,105,116,105,111,110,114,184,0,0,0,114,
+    15,0,0,0,114,91,0,0,0,114,1,0,0,0,114,4,
+    0,0,0,114,213,0,0,0,41,9,114,17,0,0,0,114,
+    218,0,0,0,218,6,108,111,99,97,108,115,114,214,0,0,
+    0,114,186,0,0,0,114,95,0,0,0,90,8,103,108,111,
+    98,97,108,115,95,114,185,0,0,0,90,7,99,117,116,95,
+    111,102,102,114,10,0,0,0,114,10,0,0,0,114,11,0,
+    0,0,218,10,95,95,105,109,112,111,114,116,95,95,59,4,
+    0,0,115,30,0,0,0,0,11,8,1,10,2,16,1,8,
+    1,12,1,4,3,8,1,18,1,4,1,4,4,26,3,32,
+    1,10,1,12,2,114,222,0,0,0,99,1,0,0,0,0,
+    0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,67,
+    0,0,0,115,38,0,0,0,116,0,160,1,124,0,161,1,
+    125,1,124,1,100,0,107,8,114,30,116,2,100,1,124,0,
+    23,0,131,1,130,1,116,3,124,1,131,1,83,0,41,2,
+    78,122,25,110,111,32,98,117,105,108,116,45,105,110,32,109,
+    111,100,117,108,101,32,110,97,109,101,100,32,41,4,114,159,
+    0,0,0,114,166,0,0,0,114,78,0,0,0,114,158,0,
+    0,0,41,2,114,17,0,0,0,114,94,0,0,0,114,10,
+    0,0,0,114,10,0,0,0,114,11,0,0,0,218,18,95,
+    98,117,105,108,116,105,110,95,102,114,111,109,95,110,97,109,
+    101,96,4,0,0,115,8,0,0,0,0,1,10,1,8,1,
+    12,1,114,223,0,0,0,99,2,0,0,0,0,0,0,0,
+    0,0,0,0,10,0,0,0,5,0,0,0,67,0,0,0,
+    115,166,0,0,0,124,1,97,0,124,0,97,1,116,2,116,
+    1,131,1,125,2,116,1,106,3,160,4,161,0,68,0,93,
+    72,92,2,125,3,125,4,116,5,124,4,124,2,131,2,114,
+    26,124,3,116,1,106,6,107,6,114,60,116,7,125,5,110,
+    18,116,0,160,8,124,3,161,1,114,26,116,9,125,5,110,
+    2,113,26,116,10,124,4,124,5,131,2,125,6,116,11,124,
+    6,124,4,131,2,1,0,113,26,116,1,106,3,116,12,25,
+    0,125,7,100,1,68,0,93,46,125,8,124,8,116,1,106,
+    3,107,7,114,138,116,13,124,8,131,1,125,9,110,10,116,
+    1,106,3,124,8,25,0,125,9,116,14,124,7,124,8,124,
+    9,131,3,1,0,113,114,100,2,83,0,41,3,122,250,83,
+    101,116,117,112,32,105,109,112,111,114,116,108,105,98,32,98,
+    121,32,105,109,112,111,114,116,105,110,103,32,110,101,101,100,
+    101,100,32,98,117,105,108,116,45,105,110,32,109,111,100,117,
+    108,101,115,32,97,110,100,32,105,110,106,101,99,116,105,110,
+    103,32,116,104,101,109,10,32,32,32,32,105,110,116,111,32,
+    116,104,101,32,103,108,111,98,97,108,32,110,97,109,101,115,
+    112,97,99,101,46,10,10,32,32,32,32,65,115,32,115,121,
+    115,32,105,115,32,110,101,101,100,101,100,32,102,111,114,32,
+    115,121,115,46,109,111,100,117,108,101,115,32,97,99,99,101,
+    115,115,32,97,110,100,32,95,105,109,112,32,105,115,32,110,
+    101,101,100,101,100,32,116,111,32,108,111,97,100,32,98,117,
+    105,108,116,45,105,110,10,32,32,32,32,109,111,100,117,108,
+    101,115,44,32,116,104,111,115,101,32,116,119,111,32,109,111,
+    100,117,108,101,115,32,109,117,115,116,32,98,101,32,101,120,
+    112,108,105,99,105,116,108,121,32,112,97,115,115,101,100,32,
+    105,110,46,10,10,32,32,32,32,41,3,114,23,0,0,0,
+    114,191,0,0,0,114,63,0,0,0,78,41,15,114,56,0,
+    0,0,114,15,0,0,0,114,14,0,0,0,114,91,0,0,
+    0,218,5,105,116,101,109,115,114,195,0,0,0,114,77,0,
+    0,0,114,159,0,0,0,114,87,0,0,0,114,173,0,0,
+    0,114,141,0,0,0,114,147,0,0,0,114,1,0,0,0,
+    114,223,0,0,0,114,5,0,0,0,41,10,218,10,115,121,
+    115,95,109,111,100,117,108,101,218,11,95,105,109,112,95,109,
+    111,100,117,108,101,90,11,109,111,100,117,108,101,95,116,121,
+    112,101,114,17,0,0,0,114,95,0,0,0,114,108,0,0,
+    0,114,94,0,0,0,90,11,115,101,108,102,95,109,111,100,
+    117,108,101,90,12,98,117,105,108,116,105,110,95,110,97,109,
+    101,90,14,98,117,105,108,116,105,110,95,109,111,100,117,108,
+    101,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,
+    218,6,95,115,101,116,117,112,103,4,0,0,115,36,0,0,
+    0,0,9,4,1,4,3,8,1,18,1,10,1,10,1,6,
+    1,10,1,6,2,2,1,10,1,12,3,10,1,8,1,10,
+    1,10,2,10,1,114,227,0,0,0,99,2,0,0,0,0,
+    0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,67,
+    0,0,0,115,38,0,0,0,116,0,124,0,124,1,131,2,
+    1,0,116,1,106,2,160,3,116,4,161,1,1,0,116,1,
+    106,2,160,3,116,5,161,1,1,0,100,1,83,0,41,2,
+    122,48,73,110,115,116,97,108,108,32,105,109,112,111,114,116,
+    101,114,115,32,102,111,114,32,98,117,105,108,116,105,110,32,
+    97,110,100,32,102,114,111,122,101,110,32,109,111,100,117,108,
+    101,115,78,41,6,114,227,0,0,0,114,15,0,0,0,114,
+    190,0,0,0,114,118,0,0,0,114,159,0,0,0,114,173,
+    0,0,0,41,2,114,225,0,0,0,114,226,0,0,0,114,
+    10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,8,
+    95,105,110,115,116,97,108,108,138,4,0,0,115,6,0,0,
+    0,0,2,10,2,12,1,114,228,0,0,0,99,0,0,0,
+    0,0,0,0,0,0,0,0,0,1,0,0,0,4,0,0,
+    0,67,0,0,0,115,32,0,0,0,100,1,100,2,108,0,
+    125,0,124,0,97,1,124,0,160,2,116,3,106,4,116,5,
+    25,0,161,1,1,0,100,2,83,0,41,3,122,57,73,110,
+    115,116,97,108,108,32,105,109,112,111,114,116,101,114,115,32,
+    116,104,97,116,32,114,101,113,117,105,114,101,32,101,120,116,
+    101,114,110,97,108,32,102,105,108,101,115,121,115,116,101,109,
+    32,97,99,99,101,115,115,114,22,0,0,0,78,41,6,218,
+    26,95,102,114,111,122,101,110,95,105,109,112,111,114,116,108,
+    105,98,95,101,120,116,101,114,110,97,108,114,125,0,0,0,
+    114,228,0,0,0,114,15,0,0,0,114,91,0,0,0,114,
+    1,0,0,0,41,1,114,229,0,0,0,114,10,0,0,0,
+    114,10,0,0,0,114,11,0,0,0,218,27,95,105,110,115,
+    116,97,108,108,95,101,120,116,101,114,110,97,108,95,105,109,
+    112,111,114,116,101,114,115,146,4,0,0,115,6,0,0,0,
+    0,3,8,1,4,1,114,230,0,0,0,41,2,78,78,41,
+    1,78,41,2,78,114,22,0,0,0,41,4,78,78,114,10,
+    0,0,0,114,22,0,0,0,41,50,114,3,0,0,0,114,
+    125,0,0,0,114,12,0,0,0,114,18,0,0,0,114,59,
+    0,0,0,114,33,0,0,0,114,42,0,0,0,114,19,0,
+    0,0,114,20,0,0,0,114,48,0,0,0,114,49,0,0,
+    0,114,52,0,0,0,114,64,0,0,0,114,66,0,0,0,
+    114,75,0,0,0,114,85,0,0,0,114,89,0,0,0,114,
+    96,0,0,0,114,110,0,0,0,114,111,0,0,0,114,90,
+    0,0,0,114,141,0,0,0,114,147,0,0,0,114,151,0,
+    0,0,114,106,0,0,0,114,92,0,0,0,114,157,0,0,
+    0,114,158,0,0,0,114,93,0,0,0,114,159,0,0,0,
+    114,173,0,0,0,114,178,0,0,0,114,187,0,0,0,114,
+    189,0,0,0,114,194,0,0,0,114,200,0,0,0,90,15,
+    95,69,82,82,95,77,83,71,95,80,82,69,70,73,88,114,
+    202,0,0,0,114,205,0,0,0,218,6,111,98,106,101,99,
+    116,114,206,0,0,0,114,207,0,0,0,114,208,0,0,0,
+    114,213,0,0,0,114,219,0,0,0,114,222,0,0,0,114,
+    223,0,0,0,114,227,0,0,0,114,228,0,0,0,114,230,
+    0,0,0,114,10,0,0,0,114,10,0,0,0,114,10,0,
+    0,0,114,11,0,0,0,218,8,60,109,111,100,117,108,101,
+    62,1,0,0,0,115,94,0,0,0,4,24,4,2,8,8,
+    8,8,4,2,4,3,16,4,14,68,14,21,14,16,8,37,
+    8,17,8,11,14,8,8,11,8,12,8,16,8,36,14,101,
+    16,26,10,45,14,72,8,17,8,17,8,30,8,37,8,42,
+    8,15,14,75,14,79,14,13,8,9,8,9,10,47,8,16,
+    4,1,8,2,8,27,6,3,8,16,10,15,14,37,8,27,
+    10,37,8,7,8,35,8,8,
 };


From webhook-mailer at python.org  Wed Sep 11 12:02:38 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Wed, 11 Sep 2019 16:02:38 -0000
Subject: [Python-checkins] bpo-36528: Remove duplicate re tests. (GH-2689)
Message-ID: <mailman.596.1568217759.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e6557d3c62800f51ee6530c94a10b93d725504fa
commit: e6557d3c62800f51ee6530c94a10b93d725504fa
branch: master
author: Xtreak <tir.karthi at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-11T17:02:34+01:00
summary:

bpo-36528: Remove duplicate re tests. (GH-2689)

Co-Authored-By: Makdon <makdon at makdon.me>

files:
M Lib/test/re_tests.py

diff --git a/Lib/test/re_tests.py b/Lib/test/re_tests.py
index 5ba9f1652e18..85b026736caa 100755
--- a/Lib/test/re_tests.py
+++ b/Lib/test/re_tests.py
@@ -88,7 +88,6 @@
     # NOTE: not an error under PCRE/PRE:
     (r'\u', '', SYNTAX_ERROR),    # A Perl escape
     # (r'\c\e\g\h\i\j\k\m\o\p\q\y\z', 'ceghijkmopqyz', SUCCEED, 'found', 'ceghijkmopqyz'),
-    (r'\xff', '\377', SUCCEED, 'found', chr(255)),
     # new \x semantics
     (r'\x00ffffffffffffff', '\377', FAIL, 'found', chr(255)),
     (r'\x00f', '\017', FAIL, 'found', chr(15)),
@@ -273,124 +272,32 @@
     # Test octal escapes/memory references
 
     ('\\1', 'a', SYNTAX_ERROR),
-    ('\\09', chr(0) + '9', SUCCEED, 'found', chr(0) + '9'),
-    ('\\141', 'a', SUCCEED, 'found', 'a'),
-    ('(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)\\119', 'abcdefghijklk9', SUCCEED, 'found+"-"+g11', 'abcdefghijklk9-k'),
 
     # All tests from Perl
 
-    ('abc', 'abc', SUCCEED, 'found', 'abc'),
-    ('abc', 'xbc', FAIL),
-    ('abc', 'axc', FAIL),
-    ('abc', 'abx', FAIL),
-    ('abc', 'xabcy', SUCCEED, 'found', 'abc'),
-    ('abc', 'ababc', SUCCEED, 'found', 'abc'),
-    ('ab*c', 'abc', SUCCEED, 'found', 'abc'),
-    ('ab*bc', 'abc', SUCCEED, 'found', 'abc'),
-    ('ab*bc', 'abbc', SUCCEED, 'found', 'abbc'),
-    ('ab*bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'),
     ('ab{0,}bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'),
-    ('ab+bc', 'abbc', SUCCEED, 'found', 'abbc'),
-    ('ab+bc', 'abc', FAIL),
-    ('ab+bc', 'abq', FAIL),
     ('ab{1,}bc', 'abq', FAIL),
-    ('ab+bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'),
     ('ab{1,}bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'),
     ('ab{1,3}bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'),
     ('ab{3,4}bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'),
     ('ab{4,5}bc', 'abbbbc', FAIL),
-    ('ab?bc', 'abbc', SUCCEED, 'found', 'abbc'),
-    ('ab?bc', 'abc', SUCCEED, 'found', 'abc'),
     ('ab{0,1}bc', 'abc', SUCCEED, 'found', 'abc'),
-    ('ab?bc', 'abbbbc', FAIL),
-    ('ab?c', 'abc', SUCCEED, 'found', 'abc'),
     ('ab{0,1}c', 'abc', SUCCEED, 'found', 'abc'),
-    ('^abc$', 'abc', SUCCEED, 'found', 'abc'),
-    ('^abc$', 'abcc', FAIL),
-    ('^abc', 'abcc', SUCCEED, 'found', 'abc'),
-    ('^abc$', 'aabc', FAIL),
-    ('abc$', 'aabc', SUCCEED, 'found', 'abc'),
     ('^', 'abc', SUCCEED, 'found', ''),
     ('$', 'abc', SUCCEED, 'found', ''),
-    ('a.c', 'abc', SUCCEED, 'found', 'abc'),
-    ('a.c', 'axc', SUCCEED, 'found', 'axc'),
-    ('a.*c', 'axyzc', SUCCEED, 'found', 'axyzc'),
-    ('a.*c', 'axyzd', FAIL),
-    ('a[bc]d', 'abc', FAIL),
-    ('a[bc]d', 'abd', SUCCEED, 'found', 'abd'),
-    ('a[b-d]e', 'abd', FAIL),
-    ('a[b-d]e', 'ace', SUCCEED, 'found', 'ace'),
-    ('a[b-d]', 'aac', SUCCEED, 'found', 'ac'),
-    ('a[-b]', 'a-', SUCCEED, 'found', 'a-'),
     ('a[b-]', 'a-', SUCCEED, 'found', 'a-'),
     ('a[b-a]', '-', SYNTAX_ERROR),
-    ('a[]b', '-', SYNTAX_ERROR),
-    ('a[', '-', SYNTAX_ERROR),
-    ('a]', 'a]', SUCCEED, 'found', 'a]'),
-    ('a[]]b', 'a]b', SUCCEED, 'found', 'a]b'),
-    ('a[^bc]d', 'aed', SUCCEED, 'found', 'aed'),
-    ('a[^bc]d', 'abd', FAIL),
-    ('a[^-b]c', 'adc', SUCCEED, 'found', 'adc'),
-    ('a[^-b]c', 'a-c', FAIL),
-    ('a[^]b]c', 'a]c', FAIL),
-    ('a[^]b]c', 'adc', SUCCEED, 'found', 'adc'),
-    ('ab|cd', 'abc', SUCCEED, 'found', 'ab'),
-    ('ab|cd', 'abcd', SUCCEED, 'found', 'ab'),
-    ('()ef', 'def', SUCCEED, 'found+"-"+g1', 'ef-'),
     ('*a', '-', SYNTAX_ERROR),
     ('(*)b', '-', SYNTAX_ERROR),
-    ('$b', 'b', FAIL),
-    ('a\\', '-', SYNTAX_ERROR),
-    ('a\\(b', 'a(b', SUCCEED, 'found+"-"+g1', 'a(b-Error'),
-    ('a\\(*b', 'ab', SUCCEED, 'found', 'ab'),
-    ('a\\(*b', 'a((b', SUCCEED, 'found', 'a((b'),
-    ('a\\\\b', 'a\\b', SUCCEED, 'found', 'a\\b'),
-    ('abc)', '-', SYNTAX_ERROR),
-    ('(abc', '-', SYNTAX_ERROR),
-    ('((a))', 'abc', SUCCEED, 'found+"-"+g1+"-"+g2', 'a-a-a'),
-    ('(a)b(c)', 'abc', SUCCEED, 'found+"-"+g1+"-"+g2', 'abc-a-c'),
-    ('a+b+c', 'aabbabc', SUCCEED, 'found', 'abc'),
     ('a{1,}b{1,}c', 'aabbabc', SUCCEED, 'found', 'abc'),
     ('a**', '-', SYNTAX_ERROR),
     ('a.+?c', 'abcabc', SUCCEED, 'found', 'abc'),
-    ('(a+|b)*', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'),
     ('(a+|b){0,}', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'),
-    ('(a+|b)+', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'),
     ('(a+|b){1,}', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'),
-    ('(a+|b)?', 'ab', SUCCEED, 'found+"-"+g1', 'a-a'),
     ('(a+|b){0,1}', 'ab', SUCCEED, 'found+"-"+g1', 'a-a'),
-    (')(', '-', SYNTAX_ERROR),
-    ('[^ab]*', 'cde', SUCCEED, 'found', 'cde'),
-    ('abc', '', FAIL),
-    ('a*', '', SUCCEED, 'found', ''),
     ('([abc])*d', 'abbbcd', SUCCEED, 'found+"-"+g1', 'abbbcd-c'),
     ('([abc])*bcd', 'abcd', SUCCEED, 'found+"-"+g1', 'abcd-a'),
-    ('a|b|c|d|e', 'e', SUCCEED, 'found', 'e'),
-    ('(a|b|c|d|e)f', 'ef', SUCCEED, 'found+"-"+g1', 'ef-e'),
-    ('abcd*efg', 'abcdefg', SUCCEED, 'found', 'abcdefg'),
-    ('ab*', 'xabyabbbz', SUCCEED, 'found', 'ab'),
-    ('ab*', 'xayabbbz', SUCCEED, 'found', 'a'),
-    ('(ab|cd)e', 'abcde', SUCCEED, 'found+"-"+g1', 'cde-cd'),
-    ('[abhgefdc]ij', 'hij', SUCCEED, 'found', 'hij'),
     ('^(ab|cd)e', 'abcde', FAIL),
-    ('(abc|)ef', 'abcdef', SUCCEED, 'found+"-"+g1', 'ef-'),
-    ('(a|b)c*d', 'abcd', SUCCEED, 'found+"-"+g1', 'bcd-b'),
-    ('(ab|ab*)bc', 'abc', SUCCEED, 'found+"-"+g1', 'abc-a'),
-    ('a([bc]*)c*', 'abc', SUCCEED, 'found+"-"+g1', 'abc-bc'),
-    ('a([bc]*)(c*d)', 'abcd', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcd-bc-d'),
-    ('a([bc]+)(c*d)', 'abcd', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcd-bc-d'),
-    ('a([bc]*)(c+d)', 'abcd', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcd-b-cd'),
-    ('a[bcd]*dcdcde', 'adcdcde', SUCCEED, 'found', 'adcdcde'),
-    ('a[bcd]+dcdcde', 'adcdcde', FAIL),
-    ('(ab|a)b*c', 'abc', SUCCEED, 'found+"-"+g1', 'abc-ab'),
-    ('((a)(b)c)(d)', 'abcd', SUCCEED, 'g1+"-"+g2+"-"+g3+"-"+g4', 'abc-a-b-d'),
-    ('[a-zA-Z_][a-zA-Z0-9_]*', 'alpha', SUCCEED, 'found', 'alpha'),
-    ('^a(bc+|b[eh])g|.h$', 'abh', SUCCEED, 'found+"-"+g1', 'bh-None'),
-    ('(bc+d$|ef*g.|h?i(j|k))', 'effgz', SUCCEED, 'found+"-"+g1+"-"+g2', 'effgz-effgz-None'),
-    ('(bc+d$|ef*g.|h?i(j|k))', 'ij', SUCCEED, 'found+"-"+g1+"-"+g2', 'ij-ij-j'),
-    ('(bc+d$|ef*g.|h?i(j|k))', 'effg', FAIL),
-    ('(bc+d$|ef*g.|h?i(j|k))', 'bcdd', FAIL),
-    ('(bc+d$|ef*g.|h?i(j|k))', 'reffgz', SUCCEED, 'found+"-"+g1+"-"+g2', 'effgz-effgz-None'),
     ('((((((((((a))))))))))', 'a', SUCCEED, 'g10', 'a'),
     ('((((((((((a))))))))))\\10', 'aa', SUCCEED, 'found', 'aa'),
 # Python does not have the same rules for \\41 so this is a syntax error
@@ -398,15 +305,6 @@
 #    ('((((((((((a))))))))))\\41', 'a!', SUCCEED, 'found', 'a!'),
     ('((((((((((a))))))))))\\41', '', SYNTAX_ERROR),
     ('(?i)((((((((((a))))))))))\\41', '', SYNTAX_ERROR),
-    ('(((((((((a)))))))))', 'a', SUCCEED, 'found', 'a'),
-    ('multiple words of text', 'uh-uh', FAIL),
-    ('multiple words', 'multiple words, yeah', SUCCEED, 'found', 'multiple words'),
-    ('(.*)c(.*)', 'abcde', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcde-ab-de'),
-    ('\\((.*), (.*)\\)', '(a, b)', SUCCEED, 'g2+"-"+g1', 'b-a'),
-    ('[k]', 'ab', FAIL),
-    ('a[-]?c', 'ac', SUCCEED, 'found', 'ac'),
-    ('(abc)\\1', 'abcabc', SUCCEED, 'g1', 'abc'),
-    ('([a-c]*)\\1', 'abcabc', SUCCEED, 'g1', 'abc'),
     ('(?i)abc', 'ABC', SUCCEED, 'found', 'ABC'),
     ('(?i)abc', 'XBC', FAIL),
     ('(?i)abc', 'AXC', FAIL),
@@ -546,7 +444,6 @@
     ('a(?:b|c|d)*(.)', 'ace', SUCCEED, 'g1', 'e'),
     ('a(?:b|c|d)+?(.)', 'ace', SUCCEED, 'g1', 'e'),
     ('a(?:b|(c|e){1,2}?|d)+?(.)', 'ace', SUCCEED, 'g1 + g2', 'ce'),
-    ('^(.+)?B', 'AB', SUCCEED, 'g1', 'A'),
 
     # lookbehind: split by : but not if it is escaped by -.
     ('(?<!-):(.*?)(?<!-):', 'a:bc-:de:f', SUCCEED, 'g1', 'bc-:de' ),
@@ -586,10 +483,7 @@
 xyzabc
 123""", SUCCEED, 'found', 'abc'),
 
-    # using the s embedded pattern modifier
 
-    ('a.b', 'a\nb', FAIL),
-    ('(?s)a.b', 'a\nb', SUCCEED, 'found', 'a\nb'),
 
     # test \w, etc. both inside and outside character classes
 


From webhook-mailer at python.org  Wed Sep 11 12:03:22 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Wed, 11 Sep 2019 16:03:22 -0000
Subject: [Python-checkins] bpo-36260: Add pitfalls to zipfile module
 documentation (GH-13378) (GH-15976)
Message-ID: <mailman.597.1568217803.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c5a672315dffbc95acc1ca28584ec84ddb56626f
commit: c5a672315dffbc95acc1ca28584ec84ddb56626f
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Jason R. Coombs <jaraco at jaraco.com>
date: 2019-09-11T17:03:18+01:00
summary:

bpo-36260: Add pitfalls to zipfile module documentation (GH-13378) (GH-15976)

* bpo-36260: Add pitfalls to zipfile module documentation

We saw vulnerability warning description (including zip bomb) in Doc/library/xml.rst file.
This gave us the idea of documentation improvement.

So, we moved a little bit forward :P
And the doc patch can be found (pr).

* fix trailing whitespace

* ?? Added by blurb_it.

* Reformat text for consistency.
(cherry picked from commit 3ba51d587f6897a45301ce9126300c14fcd4eba2)

Co-authored-by: JunWei Song <sungboss2004 at gmail.com>

files:
A Misc/NEWS.d/next/Documentation/2019-06-04-09-29-00.bpo-36260.WrGuc-.rst
M Doc/library/zipfile.rst

diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst
index 9db9697105d6..18d2e9a60a7b 100644
--- a/Doc/library/zipfile.rst
+++ b/Doc/library/zipfile.rst
@@ -816,5 +816,45 @@ Command-line options
 
    Test whether the zipfile is valid or not.
 
+Decompression pitfalls
+----------------------
+
+The extraction in zipfile module might fail due to some pitfalls listed below.
+
+From file itself
+~~~~~~~~~~~~~~~~
+
+Decompression may fail due to incorrect password / CRC checksum / ZIP format or
+unsupported compression method / decryption.
+
+File System limitations
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Exceeding limitations on different file systems can cause decompression failed.
+Such as allowable characters in the directory entries, length of the file name,
+length of the pathname, size of a single file, and number of files, etc.
+
+Resources limitations
+~~~~~~~~~~~~~~~~~~~~~
+
+The lack of memory or disk volume would lead to decompression
+failed. For example, decompression bombs (aka `ZIP bomb`_)
+apply to zipfile library that can cause disk volume exhaustion.
+
+Interruption
+~~~~~~~~~~~~
+
+Interruption during the decompression, such as pressing control-C or killing the
+decompression process may result in incomplete decompression of the archive.
+
+Default behaviors of extraction
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Not knowing the default extraction behaviors
+can cause unexpected decompression results.
+For example, when extracting the same archive twice,
+it overwrites files without asking.
+
 
+.. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb
 .. _PKZIP Application Note: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
diff --git a/Misc/NEWS.d/next/Documentation/2019-06-04-09-29-00.bpo-36260.WrGuc-.rst b/Misc/NEWS.d/next/Documentation/2019-06-04-09-29-00.bpo-36260.WrGuc-.rst
new file mode 100644
index 000000000000..9276516a8823
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-06-04-09-29-00.bpo-36260.WrGuc-.rst
@@ -0,0 +1 @@
+Add decompression pitfalls to zipfile module documentation.
\ No newline at end of file


From webhook-mailer at python.org  Wed Sep 11 12:03:40 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Wed, 11 Sep 2019 16:03:40 -0000
Subject: [Python-checkins] bpo-32592: Set Windows 8 as the minimum required
 version for API support (GH-15951)
Message-ID: <mailman.598.1568217822.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0b72ccff56fb47e14f7b1f6590eafff8d104c229
commit: 0b72ccff56fb47e14f7b1f6590eafff8d104c229
branch: master
author: Steve Dower <steve.dower at python.org>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-11T17:03:37+01:00
summary:

bpo-32592: Set Windows 8 as the minimum required version for API support (GH-15951)

files:
A Misc/NEWS.d/next/Windows/2019-09-11-14-51-56.bpo-32592.jvQMD9.rst
M Modules/posixmodule.c
M PC/pyconfig.h

diff --git a/Misc/NEWS.d/next/Windows/2019-09-11-14-51-56.bpo-32592.jvQMD9.rst b/Misc/NEWS.d/next/Windows/2019-09-11-14-51-56.bpo-32592.jvQMD9.rst
new file mode 100644
index 000000000000..c07a1213308e
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-11-14-51-56.bpo-32592.jvQMD9.rst
@@ -0,0 +1 @@
+Set Windows 8 as the minimum required version for API support
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 089572f4fdab..e71d4b28bd1e 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -12204,8 +12204,6 @@ os_cpu_count_impl(PyObject *module)
 {
     int ncpu = 0;
 #ifdef MS_WINDOWS
-    /* Declare prototype here to avoid pulling in all of the Win7 APIs in 3.8 */
-    DWORD WINAPI GetActiveProcessorCount(WORD group);
     ncpu = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
 #elif defined(__hpux)
     ncpu = mpctl(MPC_GETNUMSPUS, NULL, NULL);
diff --git a/PC/pyconfig.h b/PC/pyconfig.h
index 9304c7257c66..d407e1d4632a 100644
--- a/PC/pyconfig.h
+++ b/PC/pyconfig.h
@@ -135,9 +135,9 @@ WIN32 is still required for the locale module.
 #endif /* MS_WIN64 */
 
 /* set the version macros for the windows headers */
-/* Python 3.5+ requires Windows Vista or greater */
-#define Py_WINVER 0x0600 /* _WIN32_WINNT_VISTA */
-#define Py_NTDDI NTDDI_VISTA
+/* Python 3.9+ requires Windows 8 or greater */
+#define Py_WINVER 0x0602 /* _WIN32_WINNT_WIN8 */
+#define Py_NTDDI NTDDI_WIN8
 
 /* We only set these values when building Python - we don't want to force
    these values on extensions, as that will affect the prototypes and


From webhook-mailer at python.org  Wed Sep 11 12:25:45 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 16:25:45 -0000
Subject: [Python-checkins] Doc: Fix missing negation. (GH-14640)
Message-ID: <mailman.599.1568219145.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/05d692b08b7806126f34e2d48f0700b960c833d3
commit: 05d692b08b7806126f34e2d48f0700b960c833d3
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T09:25:40-07:00
summary:

Doc: Fix missing negation. (GH-14640)


Reported by Hug Capella on docs at .

Automerge-Triggered-By: @matrixise
(cherry picked from commit 1fae844451b120b93880d9360f288c70e125520c)

Co-authored-by: Julien Palard <julien at palard.fr>

files:
M Doc/library/re.rst

diff --git a/Doc/library/re.rst b/Doc/library/re.rst
index 158248c3d147..df29c4d39090 100644
--- a/Doc/library/re.rst
+++ b/Doc/library/re.rst
@@ -540,8 +540,8 @@ character ``'$'``.
    Matches any character which is not a word character. This is
    the opposite of ``\w``. If the :const:`ASCII` flag is used this
    becomes the equivalent of ``[^a-zA-Z0-9_]``.  If the :const:`LOCALE` flag is
-   used, matches characters considered alphanumeric in the current locale
-   and the underscore.
+   used, matches characters which are neither alphanumeric in the current locale
+   nor the underscore.
 
 .. index:: single: \Z; in regular expressions
 


From webhook-mailer at python.org  Wed Sep 11 12:38:50 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 16:38:50 -0000
Subject: [Python-checkins] bpo-37750: Add doc of PyBuffer_FromContiguous
 (GH-15988)
Message-ID: <mailman.600.1568219931.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5a56ce4a0e820fefcd598b94715a7ff7e199858d
commit: 5a56ce4a0e820fefcd598b94715a7ff7e199858d
branch: master
author: Hai Shi <shihai1992 at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T09:38:47-07:00
summary:

bpo-37750: Add doc of PyBuffer_FromContiguous (GH-15988)



https://bugs.python.org/issue37750



Automerge-Triggered-By: @matrixise

files:
M Doc/c-api/buffer.rst

diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index 2536c743b247..fb9d47b16c9f 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -483,6 +483,13 @@ Buffer-related functions
    *indices* must point to an array of ``view->ndim`` indices.
 
 
+.. c:function:: int PyBuffer_FromContiguous(Py_buffer *view, void *buf, Py_ssize_t len, char fort)
+
+   Copy contiguous *len* bytes from *buf* to *view*.
+   *fort* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering).
+   ``0`` is returned on success, ``-1`` on error.
+
+
 .. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order)
 
    Copy *len* bytes from *src* to its contiguous representation in *buf*.


From webhook-mailer at python.org  Wed Sep 11 12:46:32 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 16:46:32 -0000
Subject: [Python-checkins] bpo-36528: Remove duplicate re tests. (GH-2689)
Message-ID: <mailman.601.1568220393.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f90cbcbfe318fc4f0107bfad3cfc653ad1501c58
commit: f90cbcbfe318fc4f0107bfad3cfc653ad1501c58
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T09:46:29-07:00
summary:

bpo-36528: Remove duplicate re tests. (GH-2689)


Co-Authored-By: Makdon <makdon at makdon.me>
(cherry picked from commit e6557d3c62800f51ee6530c94a10b93d725504fa)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
M Lib/test/re_tests.py

diff --git a/Lib/test/re_tests.py b/Lib/test/re_tests.py
index 5ba9f1652e18..85b026736caa 100755
--- a/Lib/test/re_tests.py
+++ b/Lib/test/re_tests.py
@@ -88,7 +88,6 @@
     # NOTE: not an error under PCRE/PRE:
     (r'\u', '', SYNTAX_ERROR),    # A Perl escape
     # (r'\c\e\g\h\i\j\k\m\o\p\q\y\z', 'ceghijkmopqyz', SUCCEED, 'found', 'ceghijkmopqyz'),
-    (r'\xff', '\377', SUCCEED, 'found', chr(255)),
     # new \x semantics
     (r'\x00ffffffffffffff', '\377', FAIL, 'found', chr(255)),
     (r'\x00f', '\017', FAIL, 'found', chr(15)),
@@ -273,124 +272,32 @@
     # Test octal escapes/memory references
 
     ('\\1', 'a', SYNTAX_ERROR),
-    ('\\09', chr(0) + '9', SUCCEED, 'found', chr(0) + '9'),
-    ('\\141', 'a', SUCCEED, 'found', 'a'),
-    ('(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)\\119', 'abcdefghijklk9', SUCCEED, 'found+"-"+g11', 'abcdefghijklk9-k'),
 
     # All tests from Perl
 
-    ('abc', 'abc', SUCCEED, 'found', 'abc'),
-    ('abc', 'xbc', FAIL),
-    ('abc', 'axc', FAIL),
-    ('abc', 'abx', FAIL),
-    ('abc', 'xabcy', SUCCEED, 'found', 'abc'),
-    ('abc', 'ababc', SUCCEED, 'found', 'abc'),
-    ('ab*c', 'abc', SUCCEED, 'found', 'abc'),
-    ('ab*bc', 'abc', SUCCEED, 'found', 'abc'),
-    ('ab*bc', 'abbc', SUCCEED, 'found', 'abbc'),
-    ('ab*bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'),
     ('ab{0,}bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'),
-    ('ab+bc', 'abbc', SUCCEED, 'found', 'abbc'),
-    ('ab+bc', 'abc', FAIL),
-    ('ab+bc', 'abq', FAIL),
     ('ab{1,}bc', 'abq', FAIL),
-    ('ab+bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'),
     ('ab{1,}bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'),
     ('ab{1,3}bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'),
     ('ab{3,4}bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'),
     ('ab{4,5}bc', 'abbbbc', FAIL),
-    ('ab?bc', 'abbc', SUCCEED, 'found', 'abbc'),
-    ('ab?bc', 'abc', SUCCEED, 'found', 'abc'),
     ('ab{0,1}bc', 'abc', SUCCEED, 'found', 'abc'),
-    ('ab?bc', 'abbbbc', FAIL),
-    ('ab?c', 'abc', SUCCEED, 'found', 'abc'),
     ('ab{0,1}c', 'abc', SUCCEED, 'found', 'abc'),
-    ('^abc$', 'abc', SUCCEED, 'found', 'abc'),
-    ('^abc$', 'abcc', FAIL),
-    ('^abc', 'abcc', SUCCEED, 'found', 'abc'),
-    ('^abc$', 'aabc', FAIL),
-    ('abc$', 'aabc', SUCCEED, 'found', 'abc'),
     ('^', 'abc', SUCCEED, 'found', ''),
     ('$', 'abc', SUCCEED, 'found', ''),
-    ('a.c', 'abc', SUCCEED, 'found', 'abc'),
-    ('a.c', 'axc', SUCCEED, 'found', 'axc'),
-    ('a.*c', 'axyzc', SUCCEED, 'found', 'axyzc'),
-    ('a.*c', 'axyzd', FAIL),
-    ('a[bc]d', 'abc', FAIL),
-    ('a[bc]d', 'abd', SUCCEED, 'found', 'abd'),
-    ('a[b-d]e', 'abd', FAIL),
-    ('a[b-d]e', 'ace', SUCCEED, 'found', 'ace'),
-    ('a[b-d]', 'aac', SUCCEED, 'found', 'ac'),
-    ('a[-b]', 'a-', SUCCEED, 'found', 'a-'),
     ('a[b-]', 'a-', SUCCEED, 'found', 'a-'),
     ('a[b-a]', '-', SYNTAX_ERROR),
-    ('a[]b', '-', SYNTAX_ERROR),
-    ('a[', '-', SYNTAX_ERROR),
-    ('a]', 'a]', SUCCEED, 'found', 'a]'),
-    ('a[]]b', 'a]b', SUCCEED, 'found', 'a]b'),
-    ('a[^bc]d', 'aed', SUCCEED, 'found', 'aed'),
-    ('a[^bc]d', 'abd', FAIL),
-    ('a[^-b]c', 'adc', SUCCEED, 'found', 'adc'),
-    ('a[^-b]c', 'a-c', FAIL),
-    ('a[^]b]c', 'a]c', FAIL),
-    ('a[^]b]c', 'adc', SUCCEED, 'found', 'adc'),
-    ('ab|cd', 'abc', SUCCEED, 'found', 'ab'),
-    ('ab|cd', 'abcd', SUCCEED, 'found', 'ab'),
-    ('()ef', 'def', SUCCEED, 'found+"-"+g1', 'ef-'),
     ('*a', '-', SYNTAX_ERROR),
     ('(*)b', '-', SYNTAX_ERROR),
-    ('$b', 'b', FAIL),
-    ('a\\', '-', SYNTAX_ERROR),
-    ('a\\(b', 'a(b', SUCCEED, 'found+"-"+g1', 'a(b-Error'),
-    ('a\\(*b', 'ab', SUCCEED, 'found', 'ab'),
-    ('a\\(*b', 'a((b', SUCCEED, 'found', 'a((b'),
-    ('a\\\\b', 'a\\b', SUCCEED, 'found', 'a\\b'),
-    ('abc)', '-', SYNTAX_ERROR),
-    ('(abc', '-', SYNTAX_ERROR),
-    ('((a))', 'abc', SUCCEED, 'found+"-"+g1+"-"+g2', 'a-a-a'),
-    ('(a)b(c)', 'abc', SUCCEED, 'found+"-"+g1+"-"+g2', 'abc-a-c'),
-    ('a+b+c', 'aabbabc', SUCCEED, 'found', 'abc'),
     ('a{1,}b{1,}c', 'aabbabc', SUCCEED, 'found', 'abc'),
     ('a**', '-', SYNTAX_ERROR),
     ('a.+?c', 'abcabc', SUCCEED, 'found', 'abc'),
-    ('(a+|b)*', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'),
     ('(a+|b){0,}', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'),
-    ('(a+|b)+', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'),
     ('(a+|b){1,}', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'),
-    ('(a+|b)?', 'ab', SUCCEED, 'found+"-"+g1', 'a-a'),
     ('(a+|b){0,1}', 'ab', SUCCEED, 'found+"-"+g1', 'a-a'),
-    (')(', '-', SYNTAX_ERROR),
-    ('[^ab]*', 'cde', SUCCEED, 'found', 'cde'),
-    ('abc', '', FAIL),
-    ('a*', '', SUCCEED, 'found', ''),
     ('([abc])*d', 'abbbcd', SUCCEED, 'found+"-"+g1', 'abbbcd-c'),
     ('([abc])*bcd', 'abcd', SUCCEED, 'found+"-"+g1', 'abcd-a'),
-    ('a|b|c|d|e', 'e', SUCCEED, 'found', 'e'),
-    ('(a|b|c|d|e)f', 'ef', SUCCEED, 'found+"-"+g1', 'ef-e'),
-    ('abcd*efg', 'abcdefg', SUCCEED, 'found', 'abcdefg'),
-    ('ab*', 'xabyabbbz', SUCCEED, 'found', 'ab'),
-    ('ab*', 'xayabbbz', SUCCEED, 'found', 'a'),
-    ('(ab|cd)e', 'abcde', SUCCEED, 'found+"-"+g1', 'cde-cd'),
-    ('[abhgefdc]ij', 'hij', SUCCEED, 'found', 'hij'),
     ('^(ab|cd)e', 'abcde', FAIL),
-    ('(abc|)ef', 'abcdef', SUCCEED, 'found+"-"+g1', 'ef-'),
-    ('(a|b)c*d', 'abcd', SUCCEED, 'found+"-"+g1', 'bcd-b'),
-    ('(ab|ab*)bc', 'abc', SUCCEED, 'found+"-"+g1', 'abc-a'),
-    ('a([bc]*)c*', 'abc', SUCCEED, 'found+"-"+g1', 'abc-bc'),
-    ('a([bc]*)(c*d)', 'abcd', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcd-bc-d'),
-    ('a([bc]+)(c*d)', 'abcd', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcd-bc-d'),
-    ('a([bc]*)(c+d)', 'abcd', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcd-b-cd'),
-    ('a[bcd]*dcdcde', 'adcdcde', SUCCEED, 'found', 'adcdcde'),
-    ('a[bcd]+dcdcde', 'adcdcde', FAIL),
-    ('(ab|a)b*c', 'abc', SUCCEED, 'found+"-"+g1', 'abc-ab'),
-    ('((a)(b)c)(d)', 'abcd', SUCCEED, 'g1+"-"+g2+"-"+g3+"-"+g4', 'abc-a-b-d'),
-    ('[a-zA-Z_][a-zA-Z0-9_]*', 'alpha', SUCCEED, 'found', 'alpha'),
-    ('^a(bc+|b[eh])g|.h$', 'abh', SUCCEED, 'found+"-"+g1', 'bh-None'),
-    ('(bc+d$|ef*g.|h?i(j|k))', 'effgz', SUCCEED, 'found+"-"+g1+"-"+g2', 'effgz-effgz-None'),
-    ('(bc+d$|ef*g.|h?i(j|k))', 'ij', SUCCEED, 'found+"-"+g1+"-"+g2', 'ij-ij-j'),
-    ('(bc+d$|ef*g.|h?i(j|k))', 'effg', FAIL),
-    ('(bc+d$|ef*g.|h?i(j|k))', 'bcdd', FAIL),
-    ('(bc+d$|ef*g.|h?i(j|k))', 'reffgz', SUCCEED, 'found+"-"+g1+"-"+g2', 'effgz-effgz-None'),
     ('((((((((((a))))))))))', 'a', SUCCEED, 'g10', 'a'),
     ('((((((((((a))))))))))\\10', 'aa', SUCCEED, 'found', 'aa'),
 # Python does not have the same rules for \\41 so this is a syntax error
@@ -398,15 +305,6 @@
 #    ('((((((((((a))))))))))\\41', 'a!', SUCCEED, 'found', 'a!'),
     ('((((((((((a))))))))))\\41', '', SYNTAX_ERROR),
     ('(?i)((((((((((a))))))))))\\41', '', SYNTAX_ERROR),
-    ('(((((((((a)))))))))', 'a', SUCCEED, 'found', 'a'),
-    ('multiple words of text', 'uh-uh', FAIL),
-    ('multiple words', 'multiple words, yeah', SUCCEED, 'found', 'multiple words'),
-    ('(.*)c(.*)', 'abcde', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcde-ab-de'),
-    ('\\((.*), (.*)\\)', '(a, b)', SUCCEED, 'g2+"-"+g1', 'b-a'),
-    ('[k]', 'ab', FAIL),
-    ('a[-]?c', 'ac', SUCCEED, 'found', 'ac'),
-    ('(abc)\\1', 'abcabc', SUCCEED, 'g1', 'abc'),
-    ('([a-c]*)\\1', 'abcabc', SUCCEED, 'g1', 'abc'),
     ('(?i)abc', 'ABC', SUCCEED, 'found', 'ABC'),
     ('(?i)abc', 'XBC', FAIL),
     ('(?i)abc', 'AXC', FAIL),
@@ -546,7 +444,6 @@
     ('a(?:b|c|d)*(.)', 'ace', SUCCEED, 'g1', 'e'),
     ('a(?:b|c|d)+?(.)', 'ace', SUCCEED, 'g1', 'e'),
     ('a(?:b|(c|e){1,2}?|d)+?(.)', 'ace', SUCCEED, 'g1 + g2', 'ce'),
-    ('^(.+)?B', 'AB', SUCCEED, 'g1', 'A'),
 
     # lookbehind: split by : but not if it is escaped by -.
     ('(?<!-):(.*?)(?<!-):', 'a:bc-:de:f', SUCCEED, 'g1', 'bc-:de' ),
@@ -586,10 +483,7 @@
 xyzabc
 123""", SUCCEED, 'found', 'abc'),
 
-    # using the s embedded pattern modifier
 
-    ('a.b', 'a\nb', FAIL),
-    ('(?s)a.b', 'a\nb', SUCCEED, 'found', 'a\nb'),
 
     # test \w, etc. both inside and outside character classes
 


From webhook-mailer at python.org  Wed Sep 11 13:09:32 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 17:09:32 -0000
Subject: [Python-checkins] bpo-38117: Test with OpenSSL 1.1.1d (GH-15983)
Message-ID: <mailman.602.1568221773.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0b7f3706e6a4d72f7e9988cebc4826ad2c4c4753
commit: 0b7f3706e6a4d72f7e9988cebc4826ad2c4c4753
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T10:09:28-07:00
summary:

bpo-38117: Test with OpenSSL 1.1.1d (GH-15983)


Signed-off-by: Christian Heimes <christian at python.org>
(cherry picked from commit 58ab13479d854491ac9207bacfae25e8b18b044a)

Co-authored-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Tests/2019-09-11-17-22-32.bpo-38117.X7LgGY.rst
M .azure-pipelines/ci.yml
M .azure-pipelines/pr.yml
M .travis.yml
M Tools/ssl/multissltests.py

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index 12ff7d4c55c9..fe10e45b6792 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -59,7 +59,7 @@ jobs:
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1d
 
   steps:
   - template: ./posix-steps.yml
@@ -116,7 +116,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1d
 
   steps:
   - template: ./posix-steps.yml
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
index e3e24ae2a4b0..09209fc0c954 100644
--- a/.azure-pipelines/pr.yml
+++ b/.azure-pipelines/pr.yml
@@ -59,7 +59,7 @@ jobs:
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1d
 
   steps:
   - template: ./posix-steps.yml
@@ -116,7 +116,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1d
 
   steps:
   - template: ./posix-steps.yml
diff --git a/.travis.yml b/.travis.yml
index 519dd9c65279..8a960c53426f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,7 +11,7 @@ cache:
 
 env:
   global:
-    - OPENSSL=1.1.1c
+    - OPENSSL=1.1.1d
     - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}"
     - PATH="${OPENSSL_DIR}/bin:$PATH"
     - CFLAGS="-I${OPENSSL_DIR}/include"
diff --git a/Misc/NEWS.d/next/Tests/2019-09-11-17-22-32.bpo-38117.X7LgGY.rst b/Misc/NEWS.d/next/Tests/2019-09-11-17-22-32.bpo-38117.X7LgGY.rst
new file mode 100644
index 000000000000..d3280375516a
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-11-17-22-32.bpo-38117.X7LgGY.rst
@@ -0,0 +1 @@
+Test with OpenSSL 1.1.1d
diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py
index 7fda4df55a67..d816810d5707 100755
--- a/Tools/ssl/multissltests.py
+++ b/Tools/ssl/multissltests.py
@@ -45,17 +45,16 @@
 ]
 
 OPENSSL_RECENT_VERSIONS = [
-    "1.0.2s",
-    "1.1.0k",
-    "1.1.1c",
+    "1.0.2t",
+    "1.1.0l",
+    "1.1.1d",
 ]
 
 LIBRESSL_OLD_VERSIONS = [
-    "2.9.2",
 ]
 
 LIBRESSL_RECENT_VERSIONS = [
-    "2.8.3",
+    "2.9.2",
 ]
 
 # store files in ../multissl


From webhook-mailer at python.org  Wed Sep 11 13:12:25 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 11 Sep 2019 17:12:25 -0000
Subject: [Python-checkins] bpo-28724: Add methods send_fds and recv_fds to
 the socket module (GH-12889)
Message-ID: <mailman.603.1568221945.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8d120f75fb8c8731464b5f7531d74cdbb897d924
commit: 8d120f75fb8c8731464b5f7531d74cdbb897d924
branch: master
author: Joannah Nanjekye <33177550+nanjekyejoannah at users.noreply.github.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-11T19:12:21+02:00
summary:

bpo-28724: Add methods send_fds and recv_fds to the socket module (GH-12889)

The socket module now has the socket.send_fds() and socket.recv.fds() functions.
Contributed by Joannah Nanjekye, Shinya Okano (original patch)
and Victor Stinner.

Co-Authored-By: Victor Stinner <vstinner at redhat.com>

files:
A Misc/NEWS.d/next/Library/2019-05-30-15-51-42.bpo-28724.34TrS8.rst
M Doc/library/socket.rst
M Doc/whatsnew/3.8.rst
M Lib/socket.py
M Lib/test/test_socket.py

diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst
old mode 100644
new mode 100755
index eebbe810abe4..d443e972fe98
--- a/Doc/library/socket.rst
+++ b/Doc/library/socket.rst
@@ -1584,6 +1584,29 @@ to sockets.
 
    .. versionadded:: 3.6
 
+.. method:: socket.send_fds(sock, buffers, fds[, flags[, address]])
+
+   Send the list of file descriptors *fds* over an :const:`AF_UNIX` socket.
+   The *fds* parameter is a sequence of file descriptors.
+   Consult :meth:`sendmsg` for the documentation of these parameters.
+
+   .. availability:: Unix supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism.
+
+   .. versionadded:: 3.9
+
+.. method:: socket.recv_fds(sock, bufsize, maxfds[, flags])
+
+   Receive up to *maxfds* file descriptors. Return ``(msg, list(fds), flags, addr)``. Consult
+   :meth:`recvmsg` for the documentation of these parameters.
+
+   .. availability:: Unix supporting :meth:`~socket.recvmsg` and :const:`SCM_RIGHTS` mechanism.
+
+   .. versionadded:: 3.9
+
+   .. note::
+
+      Any truncated integers at the end of the list of file descriptors.
+
 .. method:: socket.sendfile(file, offset=0, count=None)
 
    Send a file until EOF is reached by using high-performance
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
old mode 100644
new mode 100755
diff --git a/Lib/socket.py b/Lib/socket.py
old mode 100644
new mode 100755
index 87b097f2c071..e5989d9dfd40
--- a/Lib/socket.py
+++ b/Lib/socket.py
@@ -12,6 +12,8 @@
 socket() -- create a new socket object
 socketpair() -- create a pair of new socket objects [*]
 fromfd() -- create a socket object from an open file descriptor [*]
+send_fds() -- Send file descriptor to the socket.
+recv_fds() -- Recieve file descriptors from the socket.
 fromshare() -- create a socket object from data received from socket.share() [*]
 gethostname() -- return the current hostname
 gethostbyname() -- map a hostname to its IP number
@@ -542,6 +544,40 @@ def fromfd(fd, family, type, proto=0):
     nfd = dup(fd)
     return socket(family, type, proto, nfd)
 
+if hasattr(_socket.socket, "sendmsg"):
+    import array
+
+    def send_fds(sock, buffers, fds, flags=0, address=None):
+        """ send_fds(sock, buffers, fds[, flags[, address]]) -> integer
+
+        Send the list of file descriptors fds over an AF_UNIX socket.
+        """
+        return sock.sendmsg(buffers, [(_socket.SOL_SOCKET,
+            _socket.SCM_RIGHTS, array.array("i", fds))])
+    __all__.append("send_fds")
+
+if hasattr(_socket.socket, "recvmsg"):
+    import array
+
+    def recv_fds(sock, bufsize, maxfds, flags=0):
+        """ recv_fds(sock, bufsize, maxfds[, flags]) -> (data, list of file
+        descriptors, msg_flags, address)
+
+        Receive up to maxfds file descriptors returning the message
+        data and a list containing the descriptors.
+        """
+        # Array of ints
+        fds = array.array("i")
+        msg, ancdata, flags, addr = sock.recvmsg(bufsize,
+            _socket.CMSG_LEN(maxfds * fds.itemsize))
+        for cmsg_level, cmsg_type, cmsg_data in ancdata:
+            if (cmsg_level == _socket.SOL_SOCKET and cmsg_type == _socket.SCM_RIGHTS):
+                fds.frombytes(cmsg_data[:
+                        len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
+
+        return msg, list(fds), flags, addr
+    __all__.append("recv_fds")
+
 if hasattr(_socket.socket, "share"):
     def fromshare(info):
         """ fromshare(info) -> socket object
diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
old mode 100644
new mode 100755
index 14ff561da94f..b855c523f080
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -6436,11 +6436,53 @@ def test_dual_stack_client_v6(self):
             self.echo_server(sock)
             self.echo_client(("::1", port), socket.AF_INET6)
 
+ at requireAttrs(socket, "send_fds")
+ at requireAttrs(socket, "recv_fds")
+ at requireAttrs(socket, "AF_UNIX")
+class SendRecvFdsTests(unittest.TestCase):
+    def testSendAndRecvFds(self):
+        def close_pipes(pipes):
+            for fd1, fd2 in pipes:
+                os.close(fd1)
+                os.close(fd2)
+
+        def close_fds(fds):
+            for fd in fds:
+                os.close(fd)
+
+        # send 10 file descriptors
+        pipes = [os.pipe() for _ in range(10)]
+        self.addCleanup(close_pipes, pipes)
+        fds = [rfd for rfd, wfd in pipes]
+
+        # use a UNIX socket pair to exchange file descriptors locally
+        sock1, sock2 = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)
+        with sock1, sock2:
+            socket.send_fds(sock1, [MSG], fds)
+            # request more data and file descriptors than expected
+            msg, fds2, flags, addr = socket.recv_fds(sock2, len(MSG) * 2, len(fds) * 2)
+            self.addCleanup(close_fds, fds2)
+
+        self.assertEqual(msg, MSG)
+        self.assertEqual(len(fds2), len(fds))
+        self.assertEqual(flags, 0)
+        # don't test addr
+
+        # test that file descriptors are connected
+        for index, fds in enumerate(pipes):
+            rfd, wfd = fds
+            os.write(wfd, str(index).encode())
+
+        for index, rfd in enumerate(fds2):
+            data = os.read(rfd, 100)
+            self.assertEqual(data,  str(index).encode())
+
 
 def test_main():
     tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest,
              TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest,
-             UDPTimeoutTest, CreateServerTest, CreateServerFunctionalTest]
+             UDPTimeoutTest, CreateServerTest, CreateServerFunctionalTest,
+             SendRecvFdsTests]
 
     tests.extend([
         NonBlockingTCPTests,
@@ -6513,5 +6555,6 @@ def test_main():
     support.run_unittest(*tests)
     support.threading_cleanup(*thread_info)
 
+
 if __name__ == "__main__":
     test_main()
diff --git a/Misc/NEWS.d/next/Library/2019-05-30-15-51-42.bpo-28724.34TrS8.rst b/Misc/NEWS.d/next/Library/2019-05-30-15-51-42.bpo-28724.34TrS8.rst
new file mode 100755
index 000000000000..68c469810a37
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-30-15-51-42.bpo-28724.34TrS8.rst
@@ -0,0 +1,2 @@
+The socket module now has the :func:`socket.send_fds` and :func:`socket.recv.fds` methods.
+Contributed by Joannah Nanjekye, Shinya Okano and Victor Stinner.
\ No newline at end of file


From webhook-mailer at python.org  Wed Sep 11 13:16:39 2019
From: webhook-mailer at python.org (Eric Snow)
Date: Wed, 11 Sep 2019 17:16:39 -0000
Subject: [Python-checkins] bpo-38113: Update the Python-ast.c generator to
 PEP384 (gh-15957)
Message-ID: <mailman.604.1568222200.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ac46eb4ad6662cf6d771b20d8963658b2186c48c
commit: ac46eb4ad6662cf6d771b20d8963658b2186c48c
branch: master
author: Dino Viehland <dinoviehland at fb.com>
committer: Eric Snow <ericsnowcurrently at gmail.com>
date: 2019-09-11T18:16:34+01:00
summary:

bpo-38113: Update the Python-ast.c generator to PEP384 (gh-15957)

Summary: This mostly migrates Python-ast.c to PEP384 and removes all statics from the whole file. This modifies the generator itself that generates the Python-ast.c. It leaves in the usage of _PyObject_LookupAttr even though it's not fully PEP384 compatible (this could always be shimmed in by anyone who needs it).

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-11-14-10-02.bpo-38113.yZXC3P.rst
M Parser/asdl_c.py
M Python/Python-ast.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-11-14-10-02.bpo-38113.yZXC3P.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-11-14-10-02.bpo-38113.yZXC3P.rst
new file mode 100644
index 000000000000..af1210de4242
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-11-14-10-02.bpo-38113.yZXC3P.rst	
@@ -0,0 +1 @@
+ast module updated to PEP-384 and all statics removed
\ No newline at end of file
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 9b18bb4e1bf8..85d96a33f3ff 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -78,14 +78,18 @@ class EmitVisitor(asdl.VisitorBase):
     def __init__(self, file):
         self.file = file
         self.identifiers = set()
+        self.singletons = set()
+        self.types = set()
         super(EmitVisitor, self).__init__()
 
     def emit_identifier(self, name):
-        name = str(name)
-        if name in self.identifiers:
-            return
-        self.emit("_Py_IDENTIFIER(%s);" % name, 0)
-        self.identifiers.add(name)
+        self.identifiers.add(str(name))
+
+    def emit_singleton(self, name):
+        self.singletons.add(str(name))
+
+    def emit_type(self, name):
+        self.types.add(str(name))
 
     def emit(self, s, depth, reflow=True):
         # XXX reflow long lines?
@@ -392,7 +396,7 @@ def simpleSum(self, sum, name):
         self.funcHeader(name)
         for t in sum.types:
             line = ("isinstance = PyObject_IsInstance(obj, "
-                    "(PyObject *)%s_type);")
+                    "astmodulestate_global->%s_type);")
             self.emit(line % (t.name,), 1)
             self.emit("if (isinstance == -1) {", 1)
             self.emit("return 1;", 2)
@@ -409,6 +413,7 @@ def buildArgs(self, fields):
     def complexSum(self, sum, name):
         self.funcHeader(name)
         self.emit("PyObject *tmp = NULL;", 1)
+        self.emit("PyObject *tp;", 1)
         for a in sum.attributes:
             self.visitAttributeDeclaration(a, name, sum=sum)
         self.emit("", 0)
@@ -420,8 +425,8 @@ def complexSum(self, sum, name):
         for a in sum.attributes:
             self.visitField(a, name, sum=sum, depth=1)
         for t in sum.types:
-            line = "isinstance = PyObject_IsInstance(obj, (PyObject*)%s_type);"
-            self.emit(line % (t.name,), 1)
+            self.emit("tp = astmodulestate_global->%s_type;" % (t.name,), 1)
+            self.emit("isinstance = PyObject_IsInstance(obj, tp);", 1)
             self.emit("if (isinstance == -1) {", 1)
             self.emit("return 1;", 2)
             self.emit("}", 1)
@@ -497,7 +502,8 @@ def isSimpleType(self, field):
 
     def visitField(self, field, name, sum=None, prod=None, depth=0):
         ctype = get_c_type(field.type)
-        self.emit("if (_PyObject_LookupAttrId(obj, &PyId_%s, &tmp) < 0) {" % field.name, depth)
+        line = "if (_PyObject_LookupAttr(obj, astmodulestate_global->%s, &tmp) < 0) {"
+        self.emit(line % field.name, depth)
         self.emit("return 1;", depth+1)
         self.emit("}", depth)
         if not field.opt:
@@ -524,7 +530,7 @@ def visitField(self, field, name, sum=None, prod=None, depth=0):
             self.emit("Py_ssize_t i;", depth+1)
             self.emit("if (!PyList_Check(tmp)) {", depth+1)
             self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must "
-                      "be a list, not a %%.200s\", tmp->ob_type->tp_name);" %
+                      "be a list, not a %%.200s\", _PyType_Name(Py_TYPE(tmp)));" %
                       (name, field.name),
                       depth+2, reflow=False)
             self.emit("goto failed;", depth+2)
@@ -574,7 +580,7 @@ def prototype(self, sum, name):
 class PyTypesDeclareVisitor(PickleVisitor):
 
     def visitProduct(self, prod, name):
-        self.emit("static PyTypeObject *%s_type;" % name, 0)
+        self.emit_type("%s_type" % name)
         self.emit("static PyObject* ast2obj_%s(void*);" % name, 0)
         if prod.attributes:
             for a in prod.attributes:
@@ -592,7 +598,7 @@ def visitProduct(self, prod, name):
             self.emit("};", 0)
 
     def visitSum(self, sum, name):
-        self.emit("static PyTypeObject *%s_type;" % name, 0)
+        self.emit_type("%s_type" % name)
         if sum.attributes:
             for a in sum.attributes:
                 self.emit_identifier(a.name)
@@ -603,17 +609,13 @@ def visitSum(self, sum, name):
         ptype = "void*"
         if is_simple(sum):
             ptype = get_c_type(name)
-            tnames = []
             for t in sum.types:
-                tnames.append(str(t.name)+"_singleton")
-            tnames = ", *".join(tnames)
-            self.emit("static PyObject *%s;" % tnames, 0)
+                self.emit_singleton("%s_singleton" % t.name)
         self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0)
         for t in sum.types:
             self.visitConstructor(t, name)
 
     def visitConstructor(self, cons, name):
-        self.emit("static PyTypeObject *%s_type;" % cons.name, 0)
         if cons.fields:
             for t in cons.fields:
                 self.emit_identifier(t.name)
@@ -626,8 +628,6 @@ class PyTypesVisitor(PickleVisitor):
 
     def visitModule(self, mod):
         self.emit("""
-_Py_IDENTIFIER(_fields);
-_Py_IDENTIFIER(_attributes);
 
 typedef struct {
     PyObject_HEAD
@@ -663,7 +663,7 @@ def visitModule(self, mod):
     Py_ssize_t i, numfields = 0;
     int res = -1;
     PyObject *key, *value, *fields;
-    if (_PyObject_LookupAttrId((PyObject*)Py_TYPE(self), &PyId__fields, &fields) < 0) {
+    if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), astmodulestate_global->_fields, &fields) < 0) {
         goto cleanup;
     }
     if (fields) {
@@ -676,7 +676,7 @@ def visitModule(self, mod):
     if (numfields < PyTuple_GET_SIZE(args)) {
         PyErr_Format(PyExc_TypeError, "%.400s constructor takes at most "
                      "%zd positional argument%s",
-                     Py_TYPE(self)->tp_name,
+                     _PyType_Name(Py_TYPE(self)),
                      numfields, numfields == 1 ? "" : "s");
         res = -1;
         goto cleanup;
@@ -710,9 +710,8 @@ def visitModule(self, mod):
 static PyObject *
 ast_type_reduce(PyObject *self, PyObject *unused)
 {
-    _Py_IDENTIFIER(__dict__);
     PyObject *dict;
-    if (_PyObject_LookupAttrId(self, &PyId___dict__, &dict) < 0) {
+    if (_PyObject_LookupAttr(self, astmodulestate_global->__dict__, &dict) < 0) {
         return NULL;
     }
     if (dict) {
@@ -731,54 +730,33 @@ def visitModule(self, mod):
     {NULL}
 };
 
-static PyTypeObject AST_type = {
-    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+static PyType_Slot AST_type_slots[] = {
+    {Py_tp_dealloc, ast_dealloc},
+    {Py_tp_getattro, PyObject_GenericGetAttr},
+    {Py_tp_setattro, PyObject_GenericSetAttr},
+    {Py_tp_traverse, ast_traverse},
+    {Py_tp_clear, ast_clear},
+    {Py_tp_methods, ast_type_methods},
+    {Py_tp_getset, ast_type_getsets},
+    {Py_tp_init, ast_type_init},
+    {Py_tp_alloc, PyType_GenericAlloc},
+    {Py_tp_new, PyType_GenericNew},
+    {Py_tp_free, PyType_GenericNew},
+    {Py_tp_free, PyObject_GC_Del},
+    {0, 0},
+};
+
+static PyType_Spec AST_type_spec = {
     "_ast.AST",
     sizeof(AST_object),
     0,
-    (destructor)ast_dealloc, /* tp_dealloc */
-    0,                       /* tp_vectorcall_offset */
-    0,                       /* tp_getattr */
-    0,                       /* tp_setattr */
-    0,                       /* tp_as_async */
-    0,                       /* tp_repr */
-    0,                       /* tp_as_number */
-    0,                       /* tp_as_sequence */
-    0,                       /* tp_as_mapping */
-    0,                       /* tp_hash */
-    0,                       /* tp_call */
-    0,                       /* tp_str */
-    PyObject_GenericGetAttr, /* tp_getattro */
-    PyObject_GenericSetAttr, /* tp_setattro */
-    0,                       /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */
-    0,                       /* tp_doc */
-    (traverseproc)ast_traverse, /* tp_traverse */
-    (inquiry)ast_clear,      /* tp_clear */
-    0,                       /* tp_richcompare */
-    0,                       /* tp_weaklistoffset */
-    0,                       /* tp_iter */
-    0,                       /* tp_iternext */
-    ast_type_methods,        /* tp_methods */
-    0,                       /* tp_members */
-    ast_type_getsets,        /* tp_getset */
-    0,                       /* tp_base */
-    0,                       /* tp_dict */
-    0,                       /* tp_descr_get */
-    0,                       /* tp_descr_set */
-    offsetof(AST_object, dict),/* tp_dictoffset */
-    (initproc)ast_type_init, /* tp_init */
-    PyType_GenericAlloc,     /* tp_alloc */
-    PyType_GenericNew,       /* tp_new */
-    PyObject_GC_Del,         /* tp_free */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+    AST_type_slots
 };
 
-
-static PyTypeObject *
-make_type(const char *type, PyTypeObject *base, const char * const *fields, int num_fields)
+static PyObject *
+make_type(const char *type, PyObject* base, const char* const* fields, int num_fields)
 {
-    _Py_IDENTIFIER(__module__);
-    _Py_IDENTIFIER(_ast);
     PyObject *fnames, *result;
     int i;
     fnames = PyTuple_New(num_fields);
@@ -793,15 +771,15 @@ def visitModule(self, mod):
     }
     result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){OOOO}",
                     type, base,
-                    _PyUnicode_FromId(&PyId__fields), fnames,
-                    _PyUnicode_FromId(&PyId___module__),
-                    _PyUnicode_FromId(&PyId__ast));
+                    astmodulestate_global->_fields, fnames,
+                    astmodulestate_global->__module__,
+                    astmodulestate_global->_ast);
     Py_DECREF(fnames);
-    return (PyTypeObject*)result;
+    return result;
 }
 
 static int
-add_attributes(PyTypeObject *type, const char * const *attrs, int num_fields)
+add_attributes(PyObject *type, const char * const *attrs, int num_fields)
 {
     int i, result;
     PyObject *s, *l = PyTuple_New(num_fields);
@@ -815,7 +793,7 @@ def visitModule(self, mod):
         }
         PyTuple_SET_ITEM(l, i, s);
     }
-    result = _PyObject_SetAttrId((PyObject*)type, &PyId__attributes, l) >= 0;
+    result = PyObject_SetAttr(type, astmodulestate_global->_attributes, l) >= 0;
     Py_DECREF(l);
     return result;
 }
@@ -921,14 +899,11 @@ def visitModule(self, mod):
 
 static int add_ast_fields(void)
 {
-    PyObject *empty_tuple, *d;
-    if (PyType_Ready(&AST_type) < 0)
-        return -1;
-    d = AST_type.tp_dict;
+    PyObject *empty_tuple;
     empty_tuple = PyTuple_New(0);
     if (!empty_tuple ||
-        _PyDict_SetItemId(d, &PyId__fields, empty_tuple) < 0 ||
-        _PyDict_SetItemId(d, &PyId__attributes, empty_tuple) < 0) {
+        PyObject_SetAttrString(astmodulestate_global->AST_type, "_fields", empty_tuple) < 0 ||
+        PyObject_SetAttrString(astmodulestate_global->AST_type, "_attributes", empty_tuple) < 0) {
         Py_XDECREF(empty_tuple);
         return -1;
     }
@@ -940,12 +915,22 @@ def visitModule(self, mod):
 
         self.emit("static int init_types(void)",0)
         self.emit("{", 0)
-        self.emit("static int initialized;", 1)
-        self.emit("if (initialized) return 1;", 1)
+        self.emit("PyObject *m;", 1)
+        self.emit("if (PyState_FindModule(&_astmodule) == NULL) {", 1)
+        self.emit("m = PyModule_Create(&_astmodule);", 2)
+        self.emit("if (!m) return 0;", 2)
+        self.emit("PyState_AddModule(m, &_astmodule);", 2)
+        self.emit("}", 1)
+        self.emit("astmodulestate *state = astmodulestate_global;", 1)
+        self.emit("if (state->initialized) return 1;", 1)
+        self.emit("if (init_identifiers() < 0) return 0;", 1)
+        self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1)
+        self.emit("if (!state->AST_type) return 0;", 1)
+        self.emit("((PyTypeObject*)state->AST_type)->tp_dictoffset = offsetof(AST_object, dict);", 1)
         self.emit("if (add_ast_fields() < 0) return 0;", 1)
         for dfn in mod.dfns:
             self.visit(dfn)
-        self.emit("initialized = 1;", 1)
+        self.emit("state->initialized = 1;", 1)
         self.emit("return 1;", 1);
         self.emit("}", 0)
 
@@ -954,24 +939,27 @@ def visitProduct(self, prod, name):
             fields = name+"_fields"
         else:
             fields = "NULL"
-        self.emit('%s_type = make_type("%s", &AST_type, %s, %d);' %
+        self.emit('state->%s_type = make_type("%s", state->AST_type, %s, %d);' %
                         (name, name, fields, len(prod.fields)), 1)
-        self.emit("if (!%s_type) return 0;" % name, 1)
+        self.emit("if (!state->%s_type) return 0;" % name, 1)
+        self.emit_type("AST_type")
+        self.emit_type("%s_type" % name)
         if prod.attributes:
-            self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" %
+            self.emit("if (!add_attributes(state->%s_type, %s_attributes, %d)) return 0;" %
                             (name, name, len(prod.attributes)), 1)
         else:
-            self.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name, 1)
+            self.emit("if (!add_attributes(state->%s_type, NULL, 0)) return 0;" % name, 1)
 
     def visitSum(self, sum, name):
-        self.emit('%s_type = make_type("%s", &AST_type, NULL, 0);' %
+        self.emit('state->%s_type = make_type("%s", state->AST_type, NULL, 0);' %
                   (name, name), 1)
-        self.emit("if (!%s_type) return 0;" % name, 1)
+        self.emit_type("%s_type" % name)
+        self.emit("if (!state->%s_type) return 0;" % name, 1)
         if sum.attributes:
-            self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" %
+            self.emit("if (!add_attributes(state->%s_type, %s_attributes, %d)) return 0;" %
                             (name, name, len(sum.attributes)), 1)
         else:
-            self.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name, 1)
+            self.emit("if (!add_attributes(state->%s_type, NULL, 0)) return 0;" % name, 1)
         simple = is_simple(sum)
         for t in sum.types:
             self.visitConstructor(t, name, simple)
@@ -981,30 +969,29 @@ def visitConstructor(self, cons, name, simple):
             fields = cons.name+"_fields"
         else:
             fields = "NULL"
-        self.emit('%s_type = make_type("%s", %s_type, %s, %d);' %
+        self.emit('state->%s_type = make_type("%s", state->%s_type, %s, %d);' %
                             (cons.name, cons.name, name, fields, len(cons.fields)), 1)
-        self.emit("if (!%s_type) return 0;" % cons.name, 1)
+        self.emit("if (!state->%s_type) return 0;" % cons.name, 1)
+        self.emit_type("%s_type" % cons.name)
         if simple:
-            self.emit("%s_singleton = PyType_GenericNew(%s_type, NULL, NULL);" %
+            self.emit("state->%s_singleton = PyType_GenericNew((PyTypeObject *)"
+                      "state->%s_type, NULL, NULL);" %
                              (cons.name, cons.name), 1)
-            self.emit("if (!%s_singleton) return 0;" % cons.name, 1)
+            self.emit("if (!state->%s_singleton) return 0;" % cons.name, 1)
 
 
 class ASTModuleVisitor(PickleVisitor):
 
     def visitModule(self, mod):
-        self.emit("static struct PyModuleDef _astmodule = {", 0)
-        self.emit('  PyModuleDef_HEAD_INIT, "_ast"', 0)
-        self.emit("};", 0)
         self.emit("PyMODINIT_FUNC", 0)
         self.emit("PyInit__ast(void)", 0)
         self.emit("{", 0)
-        self.emit("PyObject *m, *d;", 1)
+        self.emit("PyObject *m;", 1)
         self.emit("if (!init_types()) return NULL;", 1)
-        self.emit('m = PyModule_Create(&_astmodule);', 1)
+        self.emit('m = PyState_FindModule(&_astmodule);', 1)
         self.emit("if (!m) return NULL;", 1)
-        self.emit("d = PyModule_GetDict(m);", 1)
-        self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1)
+        self.emit('Py_INCREF(astmodulestate(m)->AST_type);', 1)
+        self.emit('if (PyModule_AddObject(m, "AST", astmodulestate_global->AST_type) < 0) return NULL;', 1)
         self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0)', 1)
         self.emit("return NULL;", 2)
         self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1)
@@ -1028,7 +1015,9 @@ def visitConstructor(self, cons, name):
         self.addObj(cons.name)
 
     def addObj(self, name):
-        self.emit('if (PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return NULL;' % (name, name), 1)
+        self.emit("if (PyModule_AddObject(m, \"%s\", "
+                  "astmodulestate_global->%s_type) < 0) return NULL;" % (name, name), 1)
+        self.emit("Py_INCREF(astmodulestate(m)->%s_type);" % name, 1)
 
 
 _SPECIALIZED_SEQUENCES = ('stmt', 'expr')
@@ -1066,6 +1055,7 @@ def func_begin(self, name):
         self.emit("{", 0)
         self.emit("%s o = (%s)_o;" % (ctype, ctype), 1)
         self.emit("PyObject *result = NULL, *value = NULL;", 1)
+        self.emit("PyTypeObject *tp;", 1)
         self.emit('if (!o) {', 1)
         self.emit("Py_RETURN_NONE;", 2)
         self.emit("}", 1)
@@ -1093,7 +1083,7 @@ def visitSum(self, sum, name):
         for a in sum.attributes:
             self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1)
             self.emit("if (!value) goto failed;", 1)
-            self.emit('if (_PyObject_SetAttrId(result, &PyId_%s, value) < 0)' % a.name, 1)
+            self.emit('if (PyObject_SetAttr(result, astmodulestate_global->%s, value) < 0)' % a.name, 1)
             self.emit('goto failed;', 2)
             self.emit('Py_DECREF(value);', 1)
         self.func_end()
@@ -1104,8 +1094,8 @@ def simpleSum(self, sum, name):
         self.emit("switch(o) {", 1)
         for t in sum.types:
             self.emit("case %s:" % t.name, 2)
-            self.emit("Py_INCREF(%s_singleton);" % t.name, 3)
-            self.emit("return %s_singleton;" % t.name, 3)
+            self.emit("Py_INCREF(astmodulestate_global->%s_singleton);" % t.name, 3)
+            self.emit("return astmodulestate_global->%s_singleton;" % t.name, 3)
         self.emit("default:", 2)
         self.emit('/* should never happen, but just in case ... */', 3)
         code = "PyErr_Format(PyExc_SystemError, \"unknown %s found\");" % name
@@ -1116,21 +1106,23 @@ def simpleSum(self, sum, name):
 
     def visitProduct(self, prod, name):
         self.func_begin(name)
-        self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % name, 1);
+        self.emit("tp = (PyTypeObject *)astmodulestate_global->%s_type;" % name, 1)
+        self.emit("result = PyType_GenericNew(tp, NULL, NULL);", 1);
         self.emit("if (!result) return NULL;", 1)
         for field in prod.fields:
             self.visitField(field, name, 1, True)
         for a in prod.attributes:
             self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1)
             self.emit("if (!value) goto failed;", 1)
-            self.emit('if (_PyObject_SetAttrId(result, &PyId_%s, value) < 0)' % a.name, 1)
+            self.emit("if (PyObject_SetAttr(result, astmodulestate_global->%s, value) < 0)" % a.name, 1)
             self.emit('goto failed;', 2)
             self.emit('Py_DECREF(value);', 1)
         self.func_end()
 
     def visitConstructor(self, cons, enum, name):
         self.emit("case %s_kind:" % cons.name, 1)
-        self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % cons.name, 2);
+        self.emit("tp = (PyTypeObject *)astmodulestate_global->%s_type;" % cons.name, 2)
+        self.emit("result = PyType_GenericNew(tp, NULL, NULL);", 2);
         self.emit("if (!result) goto failed;", 2)
         for f in cons.fields:
             self.visitField(f, cons.name, 2, False)
@@ -1145,7 +1137,7 @@ def emit(s, d):
             value = "o->v.%s.%s" % (name, field.name)
         self.set(field, value, depth)
         emit("if (!value) goto failed;", 0)
-        emit('if (_PyObject_SetAttrId(result, &PyId_%s, value) == -1)' % field.name, 0)
+        emit("if (PyObject_SetAttr(result, astmodulestate_global->%s, value) == -1)" % field.name, 0)
         emit("goto failed;", 1)
         emit("Py_DECREF(value);", 0)
 
@@ -1205,9 +1197,9 @@ class PartingShots(StaticVisitor):
         return NULL;
     }
 
-    req_type[0] = (PyObject*)Module_type;
-    req_type[1] = (PyObject*)Expression_type;
-    req_type[2] = (PyObject*)Interactive_type;
+    req_type[0] = astmodulestate_global->Module_type;
+    req_type[1] = astmodulestate_global->Expression_type;
+    req_type[2] = astmodulestate_global->Interactive_type;
 
     assert(0 <= mode && mode <= 2);
 
@@ -1219,7 +1211,7 @@ class PartingShots(StaticVisitor):
         return NULL;
     if (!isinstance) {
         PyErr_Format(PyExc_TypeError, "expected %s node, got %.400s",
-                     req_name[mode], Py_TYPE(ast)->tp_name);
+                     req_name[mode], _PyType_Name(Py_TYPE(ast)));
         return NULL;
     }
     if (obj2ast_mod(ast, &res, arena) != 0)
@@ -1232,7 +1224,7 @@ class PartingShots(StaticVisitor):
 {
     if (!init_types())
         return -1;
-    return PyObject_IsInstance(obj, (PyObject*)&AST_type);
+    return PyObject_IsInstance(obj, astmodulestate_global->AST_type);
 }
 """
 
@@ -1245,6 +1237,86 @@ def visit(self, object):
             v.visit(object)
             v.emit("", 0)
 
+
+def generate_module_def(f, mod):
+    # Gather all the data needed for ModuleSpec
+    visitor_list = set()
+    with open(os.devnull, "w") as devnull:
+        visitor = PyTypesDeclareVisitor(devnull)
+        visitor.visit(mod)
+        visitor_list.add(visitor)
+        visitor = PyTypesVisitor(devnull)
+        visitor.visit(mod)
+        visitor_list.add(visitor)
+
+    state_strings = set(["__dict__", "_attributes", "_fields", "__module__", "_ast"])
+    module_state = set(["__dict__", "_attributes", "_fields", "__module__", "_ast"])
+    for visitor in visitor_list:
+        for identifier in visitor.identifiers:
+            module_state.add(identifier)
+            state_strings.add(identifier)
+        for singleton in visitor.singletons:
+            module_state.add(singleton)
+        for tp in visitor.types:
+            module_state.add(tp)
+    state_strings = sorted(state_strings)
+    module_state = sorted(module_state)
+    f.write('typedef struct {\n')
+    f.write('    int initialized;\n')
+    for s in module_state:
+        f.write('    PyObject *' + s + ';\n')
+    f.write('} astmodulestate;\n\n')
+    f.write("""
+#define astmodulestate(o) ((astmodulestate *)PyModule_GetState(o))
+
+static int astmodule_clear(PyObject *module)
+{
+""")
+    for s in module_state:
+        f.write("    Py_CLEAR(astmodulestate(module)->" + s + ');\n')
+    f.write("""
+    return 0;
+}
+
+static int astmodule_traverse(PyObject *module, visitproc visit, void* arg)
+{
+""")
+    for s in module_state:
+        f.write("    Py_VISIT(astmodulestate(module)->" + s + ');\n')
+    f.write("""
+    return 0;
+}
+
+static void astmodule_free(void* module) {
+    astmodule_clear((PyObject*)module);
+}
+
+static struct PyModuleDef _astmodule = {
+    PyModuleDef_HEAD_INIT,
+    "_ast",
+    NULL,
+    sizeof(astmodulestate),
+    NULL,
+    NULL,
+    astmodule_traverse,
+    astmodule_clear,
+    astmodule_free,
+};
+
+#define astmodulestate_global ((astmodulestate *)PyModule_GetState(PyState_FindModule(&_astmodule)))
+
+""")
+    f.write('static int init_identifiers(void)\n')
+    f.write('{\n')
+    f.write('    astmodulestate *state = astmodulestate_global;\n')
+    for identifier in state_strings:
+        f.write('    if ((state->' + identifier)
+        f.write(' = PyUnicode_InternFromString("')
+        f.write(identifier + '")) == NULL) return 0;\n')
+    f.write('    return 1;\n')
+    f.write('};\n\n')
+
+
 common_msg = "/* File automatically generated by %s. */\n\n"
 
 def main(srcfile, dump_module=False):
@@ -1295,7 +1367,9 @@ def main(srcfile, dump_module=False):
             f.write('#include "Python.h"\n')
             f.write('#include "%s-ast.h"\n' % mod.name)
             f.write('\n')
-            f.write("static PyTypeObject AST_type;\n")
+
+            generate_module_def(f, mod)
+
             v = ChainOfVisitors(
                 PyTypesDeclareVisitor(f),
                 PyTypesVisitor(f),
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 44ad53275728..5190a90f13bf 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -5,40 +5,810 @@
 #include "Python.h"
 #include "Python-ast.h"
 
-static PyTypeObject AST_type;
-static PyTypeObject *mod_type;
+typedef struct {
+    int initialized;
+    PyObject *AST_type;
+    PyObject *Add_singleton;
+    PyObject *Add_type;
+    PyObject *And_singleton;
+    PyObject *And_type;
+    PyObject *AnnAssign_type;
+    PyObject *Assert_type;
+    PyObject *Assign_type;
+    PyObject *AsyncFor_type;
+    PyObject *AsyncFunctionDef_type;
+    PyObject *AsyncWith_type;
+    PyObject *Attribute_type;
+    PyObject *AugAssign_type;
+    PyObject *AugLoad_singleton;
+    PyObject *AugLoad_type;
+    PyObject *AugStore_singleton;
+    PyObject *AugStore_type;
+    PyObject *Await_type;
+    PyObject *BinOp_type;
+    PyObject *BitAnd_singleton;
+    PyObject *BitAnd_type;
+    PyObject *BitOr_singleton;
+    PyObject *BitOr_type;
+    PyObject *BitXor_singleton;
+    PyObject *BitXor_type;
+    PyObject *BoolOp_type;
+    PyObject *Break_type;
+    PyObject *Call_type;
+    PyObject *ClassDef_type;
+    PyObject *Compare_type;
+    PyObject *Constant_type;
+    PyObject *Continue_type;
+    PyObject *Del_singleton;
+    PyObject *Del_type;
+    PyObject *Delete_type;
+    PyObject *DictComp_type;
+    PyObject *Dict_type;
+    PyObject *Div_singleton;
+    PyObject *Div_type;
+    PyObject *Eq_singleton;
+    PyObject *Eq_type;
+    PyObject *ExceptHandler_type;
+    PyObject *Expr_type;
+    PyObject *Expression_type;
+    PyObject *ExtSlice_type;
+    PyObject *FloorDiv_singleton;
+    PyObject *FloorDiv_type;
+    PyObject *For_type;
+    PyObject *FormattedValue_type;
+    PyObject *FunctionDef_type;
+    PyObject *FunctionType_type;
+    PyObject *GeneratorExp_type;
+    PyObject *Global_type;
+    PyObject *GtE_singleton;
+    PyObject *GtE_type;
+    PyObject *Gt_singleton;
+    PyObject *Gt_type;
+    PyObject *IfExp_type;
+    PyObject *If_type;
+    PyObject *ImportFrom_type;
+    PyObject *Import_type;
+    PyObject *In_singleton;
+    PyObject *In_type;
+    PyObject *Index_type;
+    PyObject *Interactive_type;
+    PyObject *Invert_singleton;
+    PyObject *Invert_type;
+    PyObject *IsNot_singleton;
+    PyObject *IsNot_type;
+    PyObject *Is_singleton;
+    PyObject *Is_type;
+    PyObject *JoinedStr_type;
+    PyObject *LShift_singleton;
+    PyObject *LShift_type;
+    PyObject *Lambda_type;
+    PyObject *ListComp_type;
+    PyObject *List_type;
+    PyObject *Load_singleton;
+    PyObject *Load_type;
+    PyObject *LtE_singleton;
+    PyObject *LtE_type;
+    PyObject *Lt_singleton;
+    PyObject *Lt_type;
+    PyObject *MatMult_singleton;
+    PyObject *MatMult_type;
+    PyObject *Mod_singleton;
+    PyObject *Mod_type;
+    PyObject *Module_type;
+    PyObject *Mult_singleton;
+    PyObject *Mult_type;
+    PyObject *Name_type;
+    PyObject *NamedExpr_type;
+    PyObject *Nonlocal_type;
+    PyObject *NotEq_singleton;
+    PyObject *NotEq_type;
+    PyObject *NotIn_singleton;
+    PyObject *NotIn_type;
+    PyObject *Not_singleton;
+    PyObject *Not_type;
+    PyObject *Or_singleton;
+    PyObject *Or_type;
+    PyObject *Param_singleton;
+    PyObject *Param_type;
+    PyObject *Pass_type;
+    PyObject *Pow_singleton;
+    PyObject *Pow_type;
+    PyObject *RShift_singleton;
+    PyObject *RShift_type;
+    PyObject *Raise_type;
+    PyObject *Return_type;
+    PyObject *SetComp_type;
+    PyObject *Set_type;
+    PyObject *Slice_type;
+    PyObject *Starred_type;
+    PyObject *Store_singleton;
+    PyObject *Store_type;
+    PyObject *Sub_singleton;
+    PyObject *Sub_type;
+    PyObject *Subscript_type;
+    PyObject *Suite_type;
+    PyObject *Try_type;
+    PyObject *Tuple_type;
+    PyObject *TypeIgnore_type;
+    PyObject *UAdd_singleton;
+    PyObject *UAdd_type;
+    PyObject *USub_singleton;
+    PyObject *USub_type;
+    PyObject *UnaryOp_type;
+    PyObject *While_type;
+    PyObject *With_type;
+    PyObject *YieldFrom_type;
+    PyObject *Yield_type;
+    PyObject *__dict__;
+    PyObject *__module__;
+    PyObject *_ast;
+    PyObject *_attributes;
+    PyObject *_fields;
+    PyObject *alias_type;
+    PyObject *annotation;
+    PyObject *arg;
+    PyObject *arg_type;
+    PyObject *args;
+    PyObject *argtypes;
+    PyObject *arguments_type;
+    PyObject *asname;
+    PyObject *attr;
+    PyObject *bases;
+    PyObject *body;
+    PyObject *boolop_type;
+    PyObject *cause;
+    PyObject *cmpop_type;
+    PyObject *col_offset;
+    PyObject *comparators;
+    PyObject *comprehension_type;
+    PyObject *context_expr;
+    PyObject *conversion;
+    PyObject *ctx;
+    PyObject *decorator_list;
+    PyObject *defaults;
+    PyObject *dims;
+    PyObject *elt;
+    PyObject *elts;
+    PyObject *end_col_offset;
+    PyObject *end_lineno;
+    PyObject *exc;
+    PyObject *excepthandler_type;
+    PyObject *expr_context_type;
+    PyObject *expr_type;
+    PyObject *finalbody;
+    PyObject *format_spec;
+    PyObject *func;
+    PyObject *generators;
+    PyObject *handlers;
+    PyObject *id;
+    PyObject *ifs;
+    PyObject *is_async;
+    PyObject *items;
+    PyObject *iter;
+    PyObject *key;
+    PyObject *keys;
+    PyObject *keyword_type;
+    PyObject *keywords;
+    PyObject *kind;
+    PyObject *kw_defaults;
+    PyObject *kwarg;
+    PyObject *kwonlyargs;
+    PyObject *left;
+    PyObject *level;
+    PyObject *lineno;
+    PyObject *lower;
+    PyObject *mod_type;
+    PyObject *module;
+    PyObject *msg;
+    PyObject *name;
+    PyObject *names;
+    PyObject *op;
+    PyObject *operand;
+    PyObject *operator_type;
+    PyObject *ops;
+    PyObject *optional_vars;
+    PyObject *orelse;
+    PyObject *posonlyargs;
+    PyObject *returns;
+    PyObject *right;
+    PyObject *simple;
+    PyObject *slice;
+    PyObject *slice_type;
+    PyObject *step;
+    PyObject *stmt_type;
+    PyObject *tag;
+    PyObject *target;
+    PyObject *targets;
+    PyObject *test;
+    PyObject *type;
+    PyObject *type_comment;
+    PyObject *type_ignore_type;
+    PyObject *type_ignores;
+    PyObject *unaryop_type;
+    PyObject *upper;
+    PyObject *value;
+    PyObject *values;
+    PyObject *vararg;
+    PyObject *withitem_type;
+} astmodulestate;
+
+
+#define astmodulestate(o) ((astmodulestate *)PyModule_GetState(o))
+
+static int astmodule_clear(PyObject *module)
+{
+    Py_CLEAR(astmodulestate(module)->AST_type);
+    Py_CLEAR(astmodulestate(module)->Add_singleton);
+    Py_CLEAR(astmodulestate(module)->Add_type);
+    Py_CLEAR(astmodulestate(module)->And_singleton);
+    Py_CLEAR(astmodulestate(module)->And_type);
+    Py_CLEAR(astmodulestate(module)->AnnAssign_type);
+    Py_CLEAR(astmodulestate(module)->Assert_type);
+    Py_CLEAR(astmodulestate(module)->Assign_type);
+    Py_CLEAR(astmodulestate(module)->AsyncFor_type);
+    Py_CLEAR(astmodulestate(module)->AsyncFunctionDef_type);
+    Py_CLEAR(astmodulestate(module)->AsyncWith_type);
+    Py_CLEAR(astmodulestate(module)->Attribute_type);
+    Py_CLEAR(astmodulestate(module)->AugAssign_type);
+    Py_CLEAR(astmodulestate(module)->AugLoad_singleton);
+    Py_CLEAR(astmodulestate(module)->AugLoad_type);
+    Py_CLEAR(astmodulestate(module)->AugStore_singleton);
+    Py_CLEAR(astmodulestate(module)->AugStore_type);
+    Py_CLEAR(astmodulestate(module)->Await_type);
+    Py_CLEAR(astmodulestate(module)->BinOp_type);
+    Py_CLEAR(astmodulestate(module)->BitAnd_singleton);
+    Py_CLEAR(astmodulestate(module)->BitAnd_type);
+    Py_CLEAR(astmodulestate(module)->BitOr_singleton);
+    Py_CLEAR(astmodulestate(module)->BitOr_type);
+    Py_CLEAR(astmodulestate(module)->BitXor_singleton);
+    Py_CLEAR(astmodulestate(module)->BitXor_type);
+    Py_CLEAR(astmodulestate(module)->BoolOp_type);
+    Py_CLEAR(astmodulestate(module)->Break_type);
+    Py_CLEAR(astmodulestate(module)->Call_type);
+    Py_CLEAR(astmodulestate(module)->ClassDef_type);
+    Py_CLEAR(astmodulestate(module)->Compare_type);
+    Py_CLEAR(astmodulestate(module)->Constant_type);
+    Py_CLEAR(astmodulestate(module)->Continue_type);
+    Py_CLEAR(astmodulestate(module)->Del_singleton);
+    Py_CLEAR(astmodulestate(module)->Del_type);
+    Py_CLEAR(astmodulestate(module)->Delete_type);
+    Py_CLEAR(astmodulestate(module)->DictComp_type);
+    Py_CLEAR(astmodulestate(module)->Dict_type);
+    Py_CLEAR(astmodulestate(module)->Div_singleton);
+    Py_CLEAR(astmodulestate(module)->Div_type);
+    Py_CLEAR(astmodulestate(module)->Eq_singleton);
+    Py_CLEAR(astmodulestate(module)->Eq_type);
+    Py_CLEAR(astmodulestate(module)->ExceptHandler_type);
+    Py_CLEAR(astmodulestate(module)->Expr_type);
+    Py_CLEAR(astmodulestate(module)->Expression_type);
+    Py_CLEAR(astmodulestate(module)->ExtSlice_type);
+    Py_CLEAR(astmodulestate(module)->FloorDiv_singleton);
+    Py_CLEAR(astmodulestate(module)->FloorDiv_type);
+    Py_CLEAR(astmodulestate(module)->For_type);
+    Py_CLEAR(astmodulestate(module)->FormattedValue_type);
+    Py_CLEAR(astmodulestate(module)->FunctionDef_type);
+    Py_CLEAR(astmodulestate(module)->FunctionType_type);
+    Py_CLEAR(astmodulestate(module)->GeneratorExp_type);
+    Py_CLEAR(astmodulestate(module)->Global_type);
+    Py_CLEAR(astmodulestate(module)->GtE_singleton);
+    Py_CLEAR(astmodulestate(module)->GtE_type);
+    Py_CLEAR(astmodulestate(module)->Gt_singleton);
+    Py_CLEAR(astmodulestate(module)->Gt_type);
+    Py_CLEAR(astmodulestate(module)->IfExp_type);
+    Py_CLEAR(astmodulestate(module)->If_type);
+    Py_CLEAR(astmodulestate(module)->ImportFrom_type);
+    Py_CLEAR(astmodulestate(module)->Import_type);
+    Py_CLEAR(astmodulestate(module)->In_singleton);
+    Py_CLEAR(astmodulestate(module)->In_type);
+    Py_CLEAR(astmodulestate(module)->Index_type);
+    Py_CLEAR(astmodulestate(module)->Interactive_type);
+    Py_CLEAR(astmodulestate(module)->Invert_singleton);
+    Py_CLEAR(astmodulestate(module)->Invert_type);
+    Py_CLEAR(astmodulestate(module)->IsNot_singleton);
+    Py_CLEAR(astmodulestate(module)->IsNot_type);
+    Py_CLEAR(astmodulestate(module)->Is_singleton);
+    Py_CLEAR(astmodulestate(module)->Is_type);
+    Py_CLEAR(astmodulestate(module)->JoinedStr_type);
+    Py_CLEAR(astmodulestate(module)->LShift_singleton);
+    Py_CLEAR(astmodulestate(module)->LShift_type);
+    Py_CLEAR(astmodulestate(module)->Lambda_type);
+    Py_CLEAR(astmodulestate(module)->ListComp_type);
+    Py_CLEAR(astmodulestate(module)->List_type);
+    Py_CLEAR(astmodulestate(module)->Load_singleton);
+    Py_CLEAR(astmodulestate(module)->Load_type);
+    Py_CLEAR(astmodulestate(module)->LtE_singleton);
+    Py_CLEAR(astmodulestate(module)->LtE_type);
+    Py_CLEAR(astmodulestate(module)->Lt_singleton);
+    Py_CLEAR(astmodulestate(module)->Lt_type);
+    Py_CLEAR(astmodulestate(module)->MatMult_singleton);
+    Py_CLEAR(astmodulestate(module)->MatMult_type);
+    Py_CLEAR(astmodulestate(module)->Mod_singleton);
+    Py_CLEAR(astmodulestate(module)->Mod_type);
+    Py_CLEAR(astmodulestate(module)->Module_type);
+    Py_CLEAR(astmodulestate(module)->Mult_singleton);
+    Py_CLEAR(astmodulestate(module)->Mult_type);
+    Py_CLEAR(astmodulestate(module)->Name_type);
+    Py_CLEAR(astmodulestate(module)->NamedExpr_type);
+    Py_CLEAR(astmodulestate(module)->Nonlocal_type);
+    Py_CLEAR(astmodulestate(module)->NotEq_singleton);
+    Py_CLEAR(astmodulestate(module)->NotEq_type);
+    Py_CLEAR(astmodulestate(module)->NotIn_singleton);
+    Py_CLEAR(astmodulestate(module)->NotIn_type);
+    Py_CLEAR(astmodulestate(module)->Not_singleton);
+    Py_CLEAR(astmodulestate(module)->Not_type);
+    Py_CLEAR(astmodulestate(module)->Or_singleton);
+    Py_CLEAR(astmodulestate(module)->Or_type);
+    Py_CLEAR(astmodulestate(module)->Param_singleton);
+    Py_CLEAR(astmodulestate(module)->Param_type);
+    Py_CLEAR(astmodulestate(module)->Pass_type);
+    Py_CLEAR(astmodulestate(module)->Pow_singleton);
+    Py_CLEAR(astmodulestate(module)->Pow_type);
+    Py_CLEAR(astmodulestate(module)->RShift_singleton);
+    Py_CLEAR(astmodulestate(module)->RShift_type);
+    Py_CLEAR(astmodulestate(module)->Raise_type);
+    Py_CLEAR(astmodulestate(module)->Return_type);
+    Py_CLEAR(astmodulestate(module)->SetComp_type);
+    Py_CLEAR(astmodulestate(module)->Set_type);
+    Py_CLEAR(astmodulestate(module)->Slice_type);
+    Py_CLEAR(astmodulestate(module)->Starred_type);
+    Py_CLEAR(astmodulestate(module)->Store_singleton);
+    Py_CLEAR(astmodulestate(module)->Store_type);
+    Py_CLEAR(astmodulestate(module)->Sub_singleton);
+    Py_CLEAR(astmodulestate(module)->Sub_type);
+    Py_CLEAR(astmodulestate(module)->Subscript_type);
+    Py_CLEAR(astmodulestate(module)->Suite_type);
+    Py_CLEAR(astmodulestate(module)->Try_type);
+    Py_CLEAR(astmodulestate(module)->Tuple_type);
+    Py_CLEAR(astmodulestate(module)->TypeIgnore_type);
+    Py_CLEAR(astmodulestate(module)->UAdd_singleton);
+    Py_CLEAR(astmodulestate(module)->UAdd_type);
+    Py_CLEAR(astmodulestate(module)->USub_singleton);
+    Py_CLEAR(astmodulestate(module)->USub_type);
+    Py_CLEAR(astmodulestate(module)->UnaryOp_type);
+    Py_CLEAR(astmodulestate(module)->While_type);
+    Py_CLEAR(astmodulestate(module)->With_type);
+    Py_CLEAR(astmodulestate(module)->YieldFrom_type);
+    Py_CLEAR(astmodulestate(module)->Yield_type);
+    Py_CLEAR(astmodulestate(module)->__dict__);
+    Py_CLEAR(astmodulestate(module)->__module__);
+    Py_CLEAR(astmodulestate(module)->_ast);
+    Py_CLEAR(astmodulestate(module)->_attributes);
+    Py_CLEAR(astmodulestate(module)->_fields);
+    Py_CLEAR(astmodulestate(module)->alias_type);
+    Py_CLEAR(astmodulestate(module)->annotation);
+    Py_CLEAR(astmodulestate(module)->arg);
+    Py_CLEAR(astmodulestate(module)->arg_type);
+    Py_CLEAR(astmodulestate(module)->args);
+    Py_CLEAR(astmodulestate(module)->argtypes);
+    Py_CLEAR(astmodulestate(module)->arguments_type);
+    Py_CLEAR(astmodulestate(module)->asname);
+    Py_CLEAR(astmodulestate(module)->attr);
+    Py_CLEAR(astmodulestate(module)->bases);
+    Py_CLEAR(astmodulestate(module)->body);
+    Py_CLEAR(astmodulestate(module)->boolop_type);
+    Py_CLEAR(astmodulestate(module)->cause);
+    Py_CLEAR(astmodulestate(module)->cmpop_type);
+    Py_CLEAR(astmodulestate(module)->col_offset);
+    Py_CLEAR(astmodulestate(module)->comparators);
+    Py_CLEAR(astmodulestate(module)->comprehension_type);
+    Py_CLEAR(astmodulestate(module)->context_expr);
+    Py_CLEAR(astmodulestate(module)->conversion);
+    Py_CLEAR(astmodulestate(module)->ctx);
+    Py_CLEAR(astmodulestate(module)->decorator_list);
+    Py_CLEAR(astmodulestate(module)->defaults);
+    Py_CLEAR(astmodulestate(module)->dims);
+    Py_CLEAR(astmodulestate(module)->elt);
+    Py_CLEAR(astmodulestate(module)->elts);
+    Py_CLEAR(astmodulestate(module)->end_col_offset);
+    Py_CLEAR(astmodulestate(module)->end_lineno);
+    Py_CLEAR(astmodulestate(module)->exc);
+    Py_CLEAR(astmodulestate(module)->excepthandler_type);
+    Py_CLEAR(astmodulestate(module)->expr_context_type);
+    Py_CLEAR(astmodulestate(module)->expr_type);
+    Py_CLEAR(astmodulestate(module)->finalbody);
+    Py_CLEAR(astmodulestate(module)->format_spec);
+    Py_CLEAR(astmodulestate(module)->func);
+    Py_CLEAR(astmodulestate(module)->generators);
+    Py_CLEAR(astmodulestate(module)->handlers);
+    Py_CLEAR(astmodulestate(module)->id);
+    Py_CLEAR(astmodulestate(module)->ifs);
+    Py_CLEAR(astmodulestate(module)->is_async);
+    Py_CLEAR(astmodulestate(module)->items);
+    Py_CLEAR(astmodulestate(module)->iter);
+    Py_CLEAR(astmodulestate(module)->key);
+    Py_CLEAR(astmodulestate(module)->keys);
+    Py_CLEAR(astmodulestate(module)->keyword_type);
+    Py_CLEAR(astmodulestate(module)->keywords);
+    Py_CLEAR(astmodulestate(module)->kind);
+    Py_CLEAR(astmodulestate(module)->kw_defaults);
+    Py_CLEAR(astmodulestate(module)->kwarg);
+    Py_CLEAR(astmodulestate(module)->kwonlyargs);
+    Py_CLEAR(astmodulestate(module)->left);
+    Py_CLEAR(astmodulestate(module)->level);
+    Py_CLEAR(astmodulestate(module)->lineno);
+    Py_CLEAR(astmodulestate(module)->lower);
+    Py_CLEAR(astmodulestate(module)->mod_type);
+    Py_CLEAR(astmodulestate(module)->module);
+    Py_CLEAR(astmodulestate(module)->msg);
+    Py_CLEAR(astmodulestate(module)->name);
+    Py_CLEAR(astmodulestate(module)->names);
+    Py_CLEAR(astmodulestate(module)->op);
+    Py_CLEAR(astmodulestate(module)->operand);
+    Py_CLEAR(astmodulestate(module)->operator_type);
+    Py_CLEAR(astmodulestate(module)->ops);
+    Py_CLEAR(astmodulestate(module)->optional_vars);
+    Py_CLEAR(astmodulestate(module)->orelse);
+    Py_CLEAR(astmodulestate(module)->posonlyargs);
+    Py_CLEAR(astmodulestate(module)->returns);
+    Py_CLEAR(astmodulestate(module)->right);
+    Py_CLEAR(astmodulestate(module)->simple);
+    Py_CLEAR(astmodulestate(module)->slice);
+    Py_CLEAR(astmodulestate(module)->slice_type);
+    Py_CLEAR(astmodulestate(module)->step);
+    Py_CLEAR(astmodulestate(module)->stmt_type);
+    Py_CLEAR(astmodulestate(module)->tag);
+    Py_CLEAR(astmodulestate(module)->target);
+    Py_CLEAR(astmodulestate(module)->targets);
+    Py_CLEAR(astmodulestate(module)->test);
+    Py_CLEAR(astmodulestate(module)->type);
+    Py_CLEAR(astmodulestate(module)->type_comment);
+    Py_CLEAR(astmodulestate(module)->type_ignore_type);
+    Py_CLEAR(astmodulestate(module)->type_ignores);
+    Py_CLEAR(astmodulestate(module)->unaryop_type);
+    Py_CLEAR(astmodulestate(module)->upper);
+    Py_CLEAR(astmodulestate(module)->value);
+    Py_CLEAR(astmodulestate(module)->values);
+    Py_CLEAR(astmodulestate(module)->vararg);
+    Py_CLEAR(astmodulestate(module)->withitem_type);
+
+    return 0;
+}
+
+static int astmodule_traverse(PyObject *module, visitproc visit, void* arg)
+{
+    Py_VISIT(astmodulestate(module)->AST_type);
+    Py_VISIT(astmodulestate(module)->Add_singleton);
+    Py_VISIT(astmodulestate(module)->Add_type);
+    Py_VISIT(astmodulestate(module)->And_singleton);
+    Py_VISIT(astmodulestate(module)->And_type);
+    Py_VISIT(astmodulestate(module)->AnnAssign_type);
+    Py_VISIT(astmodulestate(module)->Assert_type);
+    Py_VISIT(astmodulestate(module)->Assign_type);
+    Py_VISIT(astmodulestate(module)->AsyncFor_type);
+    Py_VISIT(astmodulestate(module)->AsyncFunctionDef_type);
+    Py_VISIT(astmodulestate(module)->AsyncWith_type);
+    Py_VISIT(astmodulestate(module)->Attribute_type);
+    Py_VISIT(astmodulestate(module)->AugAssign_type);
+    Py_VISIT(astmodulestate(module)->AugLoad_singleton);
+    Py_VISIT(astmodulestate(module)->AugLoad_type);
+    Py_VISIT(astmodulestate(module)->AugStore_singleton);
+    Py_VISIT(astmodulestate(module)->AugStore_type);
+    Py_VISIT(astmodulestate(module)->Await_type);
+    Py_VISIT(astmodulestate(module)->BinOp_type);
+    Py_VISIT(astmodulestate(module)->BitAnd_singleton);
+    Py_VISIT(astmodulestate(module)->BitAnd_type);
+    Py_VISIT(astmodulestate(module)->BitOr_singleton);
+    Py_VISIT(astmodulestate(module)->BitOr_type);
+    Py_VISIT(astmodulestate(module)->BitXor_singleton);
+    Py_VISIT(astmodulestate(module)->BitXor_type);
+    Py_VISIT(astmodulestate(module)->BoolOp_type);
+    Py_VISIT(astmodulestate(module)->Break_type);
+    Py_VISIT(astmodulestate(module)->Call_type);
+    Py_VISIT(astmodulestate(module)->ClassDef_type);
+    Py_VISIT(astmodulestate(module)->Compare_type);
+    Py_VISIT(astmodulestate(module)->Constant_type);
+    Py_VISIT(astmodulestate(module)->Continue_type);
+    Py_VISIT(astmodulestate(module)->Del_singleton);
+    Py_VISIT(astmodulestate(module)->Del_type);
+    Py_VISIT(astmodulestate(module)->Delete_type);
+    Py_VISIT(astmodulestate(module)->DictComp_type);
+    Py_VISIT(astmodulestate(module)->Dict_type);
+    Py_VISIT(astmodulestate(module)->Div_singleton);
+    Py_VISIT(astmodulestate(module)->Div_type);
+    Py_VISIT(astmodulestate(module)->Eq_singleton);
+    Py_VISIT(astmodulestate(module)->Eq_type);
+    Py_VISIT(astmodulestate(module)->ExceptHandler_type);
+    Py_VISIT(astmodulestate(module)->Expr_type);
+    Py_VISIT(astmodulestate(module)->Expression_type);
+    Py_VISIT(astmodulestate(module)->ExtSlice_type);
+    Py_VISIT(astmodulestate(module)->FloorDiv_singleton);
+    Py_VISIT(astmodulestate(module)->FloorDiv_type);
+    Py_VISIT(astmodulestate(module)->For_type);
+    Py_VISIT(astmodulestate(module)->FormattedValue_type);
+    Py_VISIT(astmodulestate(module)->FunctionDef_type);
+    Py_VISIT(astmodulestate(module)->FunctionType_type);
+    Py_VISIT(astmodulestate(module)->GeneratorExp_type);
+    Py_VISIT(astmodulestate(module)->Global_type);
+    Py_VISIT(astmodulestate(module)->GtE_singleton);
+    Py_VISIT(astmodulestate(module)->GtE_type);
+    Py_VISIT(astmodulestate(module)->Gt_singleton);
+    Py_VISIT(astmodulestate(module)->Gt_type);
+    Py_VISIT(astmodulestate(module)->IfExp_type);
+    Py_VISIT(astmodulestate(module)->If_type);
+    Py_VISIT(astmodulestate(module)->ImportFrom_type);
+    Py_VISIT(astmodulestate(module)->Import_type);
+    Py_VISIT(astmodulestate(module)->In_singleton);
+    Py_VISIT(astmodulestate(module)->In_type);
+    Py_VISIT(astmodulestate(module)->Index_type);
+    Py_VISIT(astmodulestate(module)->Interactive_type);
+    Py_VISIT(astmodulestate(module)->Invert_singleton);
+    Py_VISIT(astmodulestate(module)->Invert_type);
+    Py_VISIT(astmodulestate(module)->IsNot_singleton);
+    Py_VISIT(astmodulestate(module)->IsNot_type);
+    Py_VISIT(astmodulestate(module)->Is_singleton);
+    Py_VISIT(astmodulestate(module)->Is_type);
+    Py_VISIT(astmodulestate(module)->JoinedStr_type);
+    Py_VISIT(astmodulestate(module)->LShift_singleton);
+    Py_VISIT(astmodulestate(module)->LShift_type);
+    Py_VISIT(astmodulestate(module)->Lambda_type);
+    Py_VISIT(astmodulestate(module)->ListComp_type);
+    Py_VISIT(astmodulestate(module)->List_type);
+    Py_VISIT(astmodulestate(module)->Load_singleton);
+    Py_VISIT(astmodulestate(module)->Load_type);
+    Py_VISIT(astmodulestate(module)->LtE_singleton);
+    Py_VISIT(astmodulestate(module)->LtE_type);
+    Py_VISIT(astmodulestate(module)->Lt_singleton);
+    Py_VISIT(astmodulestate(module)->Lt_type);
+    Py_VISIT(astmodulestate(module)->MatMult_singleton);
+    Py_VISIT(astmodulestate(module)->MatMult_type);
+    Py_VISIT(astmodulestate(module)->Mod_singleton);
+    Py_VISIT(astmodulestate(module)->Mod_type);
+    Py_VISIT(astmodulestate(module)->Module_type);
+    Py_VISIT(astmodulestate(module)->Mult_singleton);
+    Py_VISIT(astmodulestate(module)->Mult_type);
+    Py_VISIT(astmodulestate(module)->Name_type);
+    Py_VISIT(astmodulestate(module)->NamedExpr_type);
+    Py_VISIT(astmodulestate(module)->Nonlocal_type);
+    Py_VISIT(astmodulestate(module)->NotEq_singleton);
+    Py_VISIT(astmodulestate(module)->NotEq_type);
+    Py_VISIT(astmodulestate(module)->NotIn_singleton);
+    Py_VISIT(astmodulestate(module)->NotIn_type);
+    Py_VISIT(astmodulestate(module)->Not_singleton);
+    Py_VISIT(astmodulestate(module)->Not_type);
+    Py_VISIT(astmodulestate(module)->Or_singleton);
+    Py_VISIT(astmodulestate(module)->Or_type);
+    Py_VISIT(astmodulestate(module)->Param_singleton);
+    Py_VISIT(astmodulestate(module)->Param_type);
+    Py_VISIT(astmodulestate(module)->Pass_type);
+    Py_VISIT(astmodulestate(module)->Pow_singleton);
+    Py_VISIT(astmodulestate(module)->Pow_type);
+    Py_VISIT(astmodulestate(module)->RShift_singleton);
+    Py_VISIT(astmodulestate(module)->RShift_type);
+    Py_VISIT(astmodulestate(module)->Raise_type);
+    Py_VISIT(astmodulestate(module)->Return_type);
+    Py_VISIT(astmodulestate(module)->SetComp_type);
+    Py_VISIT(astmodulestate(module)->Set_type);
+    Py_VISIT(astmodulestate(module)->Slice_type);
+    Py_VISIT(astmodulestate(module)->Starred_type);
+    Py_VISIT(astmodulestate(module)->Store_singleton);
+    Py_VISIT(astmodulestate(module)->Store_type);
+    Py_VISIT(astmodulestate(module)->Sub_singleton);
+    Py_VISIT(astmodulestate(module)->Sub_type);
+    Py_VISIT(astmodulestate(module)->Subscript_type);
+    Py_VISIT(astmodulestate(module)->Suite_type);
+    Py_VISIT(astmodulestate(module)->Try_type);
+    Py_VISIT(astmodulestate(module)->Tuple_type);
+    Py_VISIT(astmodulestate(module)->TypeIgnore_type);
+    Py_VISIT(astmodulestate(module)->UAdd_singleton);
+    Py_VISIT(astmodulestate(module)->UAdd_type);
+    Py_VISIT(astmodulestate(module)->USub_singleton);
+    Py_VISIT(astmodulestate(module)->USub_type);
+    Py_VISIT(astmodulestate(module)->UnaryOp_type);
+    Py_VISIT(astmodulestate(module)->While_type);
+    Py_VISIT(astmodulestate(module)->With_type);
+    Py_VISIT(astmodulestate(module)->YieldFrom_type);
+    Py_VISIT(astmodulestate(module)->Yield_type);
+    Py_VISIT(astmodulestate(module)->__dict__);
+    Py_VISIT(astmodulestate(module)->__module__);
+    Py_VISIT(astmodulestate(module)->_ast);
+    Py_VISIT(astmodulestate(module)->_attributes);
+    Py_VISIT(astmodulestate(module)->_fields);
+    Py_VISIT(astmodulestate(module)->alias_type);
+    Py_VISIT(astmodulestate(module)->annotation);
+    Py_VISIT(astmodulestate(module)->arg);
+    Py_VISIT(astmodulestate(module)->arg_type);
+    Py_VISIT(astmodulestate(module)->args);
+    Py_VISIT(astmodulestate(module)->argtypes);
+    Py_VISIT(astmodulestate(module)->arguments_type);
+    Py_VISIT(astmodulestate(module)->asname);
+    Py_VISIT(astmodulestate(module)->attr);
+    Py_VISIT(astmodulestate(module)->bases);
+    Py_VISIT(astmodulestate(module)->body);
+    Py_VISIT(astmodulestate(module)->boolop_type);
+    Py_VISIT(astmodulestate(module)->cause);
+    Py_VISIT(astmodulestate(module)->cmpop_type);
+    Py_VISIT(astmodulestate(module)->col_offset);
+    Py_VISIT(astmodulestate(module)->comparators);
+    Py_VISIT(astmodulestate(module)->comprehension_type);
+    Py_VISIT(astmodulestate(module)->context_expr);
+    Py_VISIT(astmodulestate(module)->conversion);
+    Py_VISIT(astmodulestate(module)->ctx);
+    Py_VISIT(astmodulestate(module)->decorator_list);
+    Py_VISIT(astmodulestate(module)->defaults);
+    Py_VISIT(astmodulestate(module)->dims);
+    Py_VISIT(astmodulestate(module)->elt);
+    Py_VISIT(astmodulestate(module)->elts);
+    Py_VISIT(astmodulestate(module)->end_col_offset);
+    Py_VISIT(astmodulestate(module)->end_lineno);
+    Py_VISIT(astmodulestate(module)->exc);
+    Py_VISIT(astmodulestate(module)->excepthandler_type);
+    Py_VISIT(astmodulestate(module)->expr_context_type);
+    Py_VISIT(astmodulestate(module)->expr_type);
+    Py_VISIT(astmodulestate(module)->finalbody);
+    Py_VISIT(astmodulestate(module)->format_spec);
+    Py_VISIT(astmodulestate(module)->func);
+    Py_VISIT(astmodulestate(module)->generators);
+    Py_VISIT(astmodulestate(module)->handlers);
+    Py_VISIT(astmodulestate(module)->id);
+    Py_VISIT(astmodulestate(module)->ifs);
+    Py_VISIT(astmodulestate(module)->is_async);
+    Py_VISIT(astmodulestate(module)->items);
+    Py_VISIT(astmodulestate(module)->iter);
+    Py_VISIT(astmodulestate(module)->key);
+    Py_VISIT(astmodulestate(module)->keys);
+    Py_VISIT(astmodulestate(module)->keyword_type);
+    Py_VISIT(astmodulestate(module)->keywords);
+    Py_VISIT(astmodulestate(module)->kind);
+    Py_VISIT(astmodulestate(module)->kw_defaults);
+    Py_VISIT(astmodulestate(module)->kwarg);
+    Py_VISIT(astmodulestate(module)->kwonlyargs);
+    Py_VISIT(astmodulestate(module)->left);
+    Py_VISIT(astmodulestate(module)->level);
+    Py_VISIT(astmodulestate(module)->lineno);
+    Py_VISIT(astmodulestate(module)->lower);
+    Py_VISIT(astmodulestate(module)->mod_type);
+    Py_VISIT(astmodulestate(module)->module);
+    Py_VISIT(astmodulestate(module)->msg);
+    Py_VISIT(astmodulestate(module)->name);
+    Py_VISIT(astmodulestate(module)->names);
+    Py_VISIT(astmodulestate(module)->op);
+    Py_VISIT(astmodulestate(module)->operand);
+    Py_VISIT(astmodulestate(module)->operator_type);
+    Py_VISIT(astmodulestate(module)->ops);
+    Py_VISIT(astmodulestate(module)->optional_vars);
+    Py_VISIT(astmodulestate(module)->orelse);
+    Py_VISIT(astmodulestate(module)->posonlyargs);
+    Py_VISIT(astmodulestate(module)->returns);
+    Py_VISIT(astmodulestate(module)->right);
+    Py_VISIT(astmodulestate(module)->simple);
+    Py_VISIT(astmodulestate(module)->slice);
+    Py_VISIT(astmodulestate(module)->slice_type);
+    Py_VISIT(astmodulestate(module)->step);
+    Py_VISIT(astmodulestate(module)->stmt_type);
+    Py_VISIT(astmodulestate(module)->tag);
+    Py_VISIT(astmodulestate(module)->target);
+    Py_VISIT(astmodulestate(module)->targets);
+    Py_VISIT(astmodulestate(module)->test);
+    Py_VISIT(astmodulestate(module)->type);
+    Py_VISIT(astmodulestate(module)->type_comment);
+    Py_VISIT(astmodulestate(module)->type_ignore_type);
+    Py_VISIT(astmodulestate(module)->type_ignores);
+    Py_VISIT(astmodulestate(module)->unaryop_type);
+    Py_VISIT(astmodulestate(module)->upper);
+    Py_VISIT(astmodulestate(module)->value);
+    Py_VISIT(astmodulestate(module)->values);
+    Py_VISIT(astmodulestate(module)->vararg);
+    Py_VISIT(astmodulestate(module)->withitem_type);
+
+    return 0;
+}
+
+static void astmodule_free(void* module) {
+    astmodule_clear((PyObject*)module);
+}
+
+static struct PyModuleDef _astmodule = {
+    PyModuleDef_HEAD_INIT,
+    "_ast",
+    NULL,
+    sizeof(astmodulestate),
+    NULL,
+    NULL,
+    astmodule_traverse,
+    astmodule_clear,
+    astmodule_free,
+};
+
+#define astmodulestate_global ((astmodulestate *)PyModule_GetState(PyState_FindModule(&_astmodule)))
+
+static int init_identifiers(void)
+{
+    astmodulestate *state = astmodulestate_global;
+    if ((state->__dict__ = PyUnicode_InternFromString("__dict__")) == NULL) return 0;
+    if ((state->__module__ = PyUnicode_InternFromString("__module__")) == NULL) return 0;
+    if ((state->_ast = PyUnicode_InternFromString("_ast")) == NULL) return 0;
+    if ((state->_attributes = PyUnicode_InternFromString("_attributes")) == NULL) return 0;
+    if ((state->_fields = PyUnicode_InternFromString("_fields")) == NULL) return 0;
+    if ((state->annotation = PyUnicode_InternFromString("annotation")) == NULL) return 0;
+    if ((state->arg = PyUnicode_InternFromString("arg")) == NULL) return 0;
+    if ((state->args = PyUnicode_InternFromString("args")) == NULL) return 0;
+    if ((state->argtypes = PyUnicode_InternFromString("argtypes")) == NULL) return 0;
+    if ((state->asname = PyUnicode_InternFromString("asname")) == NULL) return 0;
+    if ((state->attr = PyUnicode_InternFromString("attr")) == NULL) return 0;
+    if ((state->bases = PyUnicode_InternFromString("bases")) == NULL) return 0;
+    if ((state->body = PyUnicode_InternFromString("body")) == NULL) return 0;
+    if ((state->cause = PyUnicode_InternFromString("cause")) == NULL) return 0;
+    if ((state->col_offset = PyUnicode_InternFromString("col_offset")) == NULL) return 0;
+    if ((state->comparators = PyUnicode_InternFromString("comparators")) == NULL) return 0;
+    if ((state->context_expr = PyUnicode_InternFromString("context_expr")) == NULL) return 0;
+    if ((state->conversion = PyUnicode_InternFromString("conversion")) == NULL) return 0;
+    if ((state->ctx = PyUnicode_InternFromString("ctx")) == NULL) return 0;
+    if ((state->decorator_list = PyUnicode_InternFromString("decorator_list")) == NULL) return 0;
+    if ((state->defaults = PyUnicode_InternFromString("defaults")) == NULL) return 0;
+    if ((state->dims = PyUnicode_InternFromString("dims")) == NULL) return 0;
+    if ((state->elt = PyUnicode_InternFromString("elt")) == NULL) return 0;
+    if ((state->elts = PyUnicode_InternFromString("elts")) == NULL) return 0;
+    if ((state->end_col_offset = PyUnicode_InternFromString("end_col_offset")) == NULL) return 0;
+    if ((state->end_lineno = PyUnicode_InternFromString("end_lineno")) == NULL) return 0;
+    if ((state->exc = PyUnicode_InternFromString("exc")) == NULL) return 0;
+    if ((state->finalbody = PyUnicode_InternFromString("finalbody")) == NULL) return 0;
+    if ((state->format_spec = PyUnicode_InternFromString("format_spec")) == NULL) return 0;
+    if ((state->func = PyUnicode_InternFromString("func")) == NULL) return 0;
+    if ((state->generators = PyUnicode_InternFromString("generators")) == NULL) return 0;
+    if ((state->handlers = PyUnicode_InternFromString("handlers")) == NULL) return 0;
+    if ((state->id = PyUnicode_InternFromString("id")) == NULL) return 0;
+    if ((state->ifs = PyUnicode_InternFromString("ifs")) == NULL) return 0;
+    if ((state->is_async = PyUnicode_InternFromString("is_async")) == NULL) return 0;
+    if ((state->items = PyUnicode_InternFromString("items")) == NULL) return 0;
+    if ((state->iter = PyUnicode_InternFromString("iter")) == NULL) return 0;
+    if ((state->key = PyUnicode_InternFromString("key")) == NULL) return 0;
+    if ((state->keys = PyUnicode_InternFromString("keys")) == NULL) return 0;
+    if ((state->keywords = PyUnicode_InternFromString("keywords")) == NULL) return 0;
+    if ((state->kind = PyUnicode_InternFromString("kind")) == NULL) return 0;
+    if ((state->kw_defaults = PyUnicode_InternFromString("kw_defaults")) == NULL) return 0;
+    if ((state->kwarg = PyUnicode_InternFromString("kwarg")) == NULL) return 0;
+    if ((state->kwonlyargs = PyUnicode_InternFromString("kwonlyargs")) == NULL) return 0;
+    if ((state->left = PyUnicode_InternFromString("left")) == NULL) return 0;
+    if ((state->level = PyUnicode_InternFromString("level")) == NULL) return 0;
+    if ((state->lineno = PyUnicode_InternFromString("lineno")) == NULL) return 0;
+    if ((state->lower = PyUnicode_InternFromString("lower")) == NULL) return 0;
+    if ((state->module = PyUnicode_InternFromString("module")) == NULL) return 0;
+    if ((state->msg = PyUnicode_InternFromString("msg")) == NULL) return 0;
+    if ((state->name = PyUnicode_InternFromString("name")) == NULL) return 0;
+    if ((state->names = PyUnicode_InternFromString("names")) == NULL) return 0;
+    if ((state->op = PyUnicode_InternFromString("op")) == NULL) return 0;
+    if ((state->operand = PyUnicode_InternFromString("operand")) == NULL) return 0;
+    if ((state->ops = PyUnicode_InternFromString("ops")) == NULL) return 0;
+    if ((state->optional_vars = PyUnicode_InternFromString("optional_vars")) == NULL) return 0;
+    if ((state->orelse = PyUnicode_InternFromString("orelse")) == NULL) return 0;
+    if ((state->posonlyargs = PyUnicode_InternFromString("posonlyargs")) == NULL) return 0;
+    if ((state->returns = PyUnicode_InternFromString("returns")) == NULL) return 0;
+    if ((state->right = PyUnicode_InternFromString("right")) == NULL) return 0;
+    if ((state->simple = PyUnicode_InternFromString("simple")) == NULL) return 0;
+    if ((state->slice = PyUnicode_InternFromString("slice")) == NULL) return 0;
+    if ((state->step = PyUnicode_InternFromString("step")) == NULL) return 0;
+    if ((state->tag = PyUnicode_InternFromString("tag")) == NULL) return 0;
+    if ((state->target = PyUnicode_InternFromString("target")) == NULL) return 0;
+    if ((state->targets = PyUnicode_InternFromString("targets")) == NULL) return 0;
+    if ((state->test = PyUnicode_InternFromString("test")) == NULL) return 0;
+    if ((state->type = PyUnicode_InternFromString("type")) == NULL) return 0;
+    if ((state->type_comment = PyUnicode_InternFromString("type_comment")) == NULL) return 0;
+    if ((state->type_ignores = PyUnicode_InternFromString("type_ignores")) == NULL) return 0;
+    if ((state->upper = PyUnicode_InternFromString("upper")) == NULL) return 0;
+    if ((state->value = PyUnicode_InternFromString("value")) == NULL) return 0;
+    if ((state->values = PyUnicode_InternFromString("values")) == NULL) return 0;
+    if ((state->vararg = PyUnicode_InternFromString("vararg")) == NULL) return 0;
+    return 1;
+};
+
 static PyObject* ast2obj_mod(void*);
-static PyTypeObject *Module_type;
-_Py_IDENTIFIER(body);
-_Py_IDENTIFIER(type_ignores);
 static const char * const Module_fields[]={
     "body",
     "type_ignores",
 };
-static PyTypeObject *Interactive_type;
 static const char * const Interactive_fields[]={
     "body",
 };
-static PyTypeObject *Expression_type;
 static const char * const Expression_fields[]={
     "body",
 };
-static PyTypeObject *FunctionType_type;
-_Py_IDENTIFIER(argtypes);
-_Py_IDENTIFIER(returns);
 static const char * const FunctionType_fields[]={
     "argtypes",
     "returns",
 };
-static PyTypeObject *Suite_type;
 static const char * const Suite_fields[]={
     "body",
 };
-static PyTypeObject *stmt_type;
-_Py_IDENTIFIER(lineno);
-_Py_IDENTIFIER(col_offset);
-_Py_IDENTIFIER(end_lineno);
-_Py_IDENTIFIER(end_col_offset);
 static const char * const stmt_attributes[] = {
     "lineno",
     "col_offset",
@@ -46,11 +816,6 @@ static const char * const stmt_attributes[] = {
     "end_col_offset",
 };
 static PyObject* ast2obj_stmt(void*);
-static PyTypeObject *FunctionDef_type;
-_Py_IDENTIFIER(name);
-_Py_IDENTIFIER(args);
-_Py_IDENTIFIER(decorator_list);
-_Py_IDENTIFIER(type_comment);
 static const char * const FunctionDef_fields[]={
     "name",
     "args",
@@ -59,7 +824,6 @@ static const char * const FunctionDef_fields[]={
     "returns",
     "type_comment",
 };
-static PyTypeObject *AsyncFunctionDef_type;
 static const char * const AsyncFunctionDef_fields[]={
     "name",
     "args",
@@ -68,9 +832,6 @@ static const char * const AsyncFunctionDef_fields[]={
     "returns",
     "type_comment",
 };
-static PyTypeObject *ClassDef_type;
-_Py_IDENTIFIER(bases);
-_Py_IDENTIFIER(keywords);
 static const char * const ClassDef_fields[]={
     "name",
     "bases",
@@ -78,42 +839,28 @@ static const char * const ClassDef_fields[]={
     "body",
     "decorator_list",
 };
-static PyTypeObject *Return_type;
-_Py_IDENTIFIER(value);
 static const char * const Return_fields[]={
     "value",
 };
-static PyTypeObject *Delete_type;
-_Py_IDENTIFIER(targets);
 static const char * const Delete_fields[]={
     "targets",
 };
-static PyTypeObject *Assign_type;
 static const char * const Assign_fields[]={
     "targets",
     "value",
     "type_comment",
 };
-static PyTypeObject *AugAssign_type;
-_Py_IDENTIFIER(target);
-_Py_IDENTIFIER(op);
 static const char * const AugAssign_fields[]={
     "target",
     "op",
     "value",
 };
-static PyTypeObject *AnnAssign_type;
-_Py_IDENTIFIER(annotation);
-_Py_IDENTIFIER(simple);
 static const char * const AnnAssign_fields[]={
     "target",
     "annotation",
     "value",
     "simple",
 };
-static PyTypeObject *For_type;
-_Py_IDENTIFIER(iter);
-_Py_IDENTIFIER(orelse);
 static const char * const For_fields[]={
     "target",
     "iter",
@@ -121,7 +868,6 @@ static const char * const For_fields[]={
     "orelse",
     "type_comment",
 };
-static PyTypeObject *AsyncFor_type;
 static const char * const AsyncFor_fields[]={
     "target",
     "iter",
@@ -129,83 +875,57 @@ static const char * const AsyncFor_fields[]={
     "orelse",
     "type_comment",
 };
-static PyTypeObject *While_type;
-_Py_IDENTIFIER(test);
 static const char * const While_fields[]={
     "test",
     "body",
     "orelse",
 };
-static PyTypeObject *If_type;
 static const char * const If_fields[]={
     "test",
     "body",
     "orelse",
 };
-static PyTypeObject *With_type;
-_Py_IDENTIFIER(items);
 static const char * const With_fields[]={
     "items",
     "body",
     "type_comment",
 };
-static PyTypeObject *AsyncWith_type;
 static const char * const AsyncWith_fields[]={
     "items",
     "body",
     "type_comment",
 };
-static PyTypeObject *Raise_type;
-_Py_IDENTIFIER(exc);
-_Py_IDENTIFIER(cause);
 static const char * const Raise_fields[]={
     "exc",
     "cause",
 };
-static PyTypeObject *Try_type;
-_Py_IDENTIFIER(handlers);
-_Py_IDENTIFIER(finalbody);
 static const char * const Try_fields[]={
     "body",
     "handlers",
     "orelse",
     "finalbody",
 };
-static PyTypeObject *Assert_type;
-_Py_IDENTIFIER(msg);
 static const char * const Assert_fields[]={
     "test",
     "msg",
 };
-static PyTypeObject *Import_type;
-_Py_IDENTIFIER(names);
 static const char * const Import_fields[]={
     "names",
 };
-static PyTypeObject *ImportFrom_type;
-_Py_IDENTIFIER(module);
-_Py_IDENTIFIER(level);
 static const char * const ImportFrom_fields[]={
     "module",
     "names",
     "level",
 };
-static PyTypeObject *Global_type;
 static const char * const Global_fields[]={
     "names",
 };
-static PyTypeObject *Nonlocal_type;
 static const char * const Nonlocal_fields[]={
     "names",
 };
-static PyTypeObject *Expr_type;
 static const char * const Expr_fields[]={
     "value",
 };
-static PyTypeObject *Pass_type;
-static PyTypeObject *Break_type;
-static PyTypeObject *Continue_type;
-static PyTypeObject *expr_type;
 static const char * const expr_attributes[] = {
     "lineno",
     "col_offset",
@@ -213,246 +933,137 @@ static const char * const expr_attributes[] = {
     "end_col_offset",
 };
 static PyObject* ast2obj_expr(void*);
-static PyTypeObject *BoolOp_type;
-_Py_IDENTIFIER(values);
 static const char * const BoolOp_fields[]={
     "op",
     "values",
 };
-static PyTypeObject *NamedExpr_type;
 static const char * const NamedExpr_fields[]={
     "target",
     "value",
 };
-static PyTypeObject *BinOp_type;
-_Py_IDENTIFIER(left);
-_Py_IDENTIFIER(right);
 static const char * const BinOp_fields[]={
     "left",
     "op",
     "right",
 };
-static PyTypeObject *UnaryOp_type;
-_Py_IDENTIFIER(operand);
 static const char * const UnaryOp_fields[]={
     "op",
     "operand",
 };
-static PyTypeObject *Lambda_type;
 static const char * const Lambda_fields[]={
     "args",
     "body",
 };
-static PyTypeObject *IfExp_type;
 static const char * const IfExp_fields[]={
     "test",
     "body",
     "orelse",
 };
-static PyTypeObject *Dict_type;
-_Py_IDENTIFIER(keys);
 static const char * const Dict_fields[]={
     "keys",
     "values",
 };
-static PyTypeObject *Set_type;
-_Py_IDENTIFIER(elts);
 static const char * const Set_fields[]={
     "elts",
 };
-static PyTypeObject *ListComp_type;
-_Py_IDENTIFIER(elt);
-_Py_IDENTIFIER(generators);
 static const char * const ListComp_fields[]={
     "elt",
     "generators",
 };
-static PyTypeObject *SetComp_type;
 static const char * const SetComp_fields[]={
     "elt",
     "generators",
 };
-static PyTypeObject *DictComp_type;
-_Py_IDENTIFIER(key);
 static const char * const DictComp_fields[]={
     "key",
     "value",
     "generators",
 };
-static PyTypeObject *GeneratorExp_type;
 static const char * const GeneratorExp_fields[]={
     "elt",
     "generators",
 };
-static PyTypeObject *Await_type;
 static const char * const Await_fields[]={
     "value",
 };
-static PyTypeObject *Yield_type;
 static const char * const Yield_fields[]={
     "value",
 };
-static PyTypeObject *YieldFrom_type;
 static const char * const YieldFrom_fields[]={
     "value",
 };
-static PyTypeObject *Compare_type;
-_Py_IDENTIFIER(ops);
-_Py_IDENTIFIER(comparators);
 static const char * const Compare_fields[]={
     "left",
     "ops",
     "comparators",
 };
-static PyTypeObject *Call_type;
-_Py_IDENTIFIER(func);
 static const char * const Call_fields[]={
     "func",
     "args",
     "keywords",
 };
-static PyTypeObject *FormattedValue_type;
-_Py_IDENTIFIER(conversion);
-_Py_IDENTIFIER(format_spec);
 static const char * const FormattedValue_fields[]={
     "value",
     "conversion",
     "format_spec",
 };
-static PyTypeObject *JoinedStr_type;
 static const char * const JoinedStr_fields[]={
     "values",
 };
-static PyTypeObject *Constant_type;
-_Py_IDENTIFIER(kind);
 static const char * const Constant_fields[]={
     "value",
     "kind",
 };
-static PyTypeObject *Attribute_type;
-_Py_IDENTIFIER(attr);
-_Py_IDENTIFIER(ctx);
 static const char * const Attribute_fields[]={
     "value",
     "attr",
     "ctx",
 };
-static PyTypeObject *Subscript_type;
-_Py_IDENTIFIER(slice);
 static const char * const Subscript_fields[]={
     "value",
     "slice",
     "ctx",
 };
-static PyTypeObject *Starred_type;
 static const char * const Starred_fields[]={
     "value",
     "ctx",
 };
-static PyTypeObject *Name_type;
-_Py_IDENTIFIER(id);
 static const char * const Name_fields[]={
     "id",
     "ctx",
 };
-static PyTypeObject *List_type;
 static const char * const List_fields[]={
     "elts",
     "ctx",
 };
-static PyTypeObject *Tuple_type;
 static const char * const Tuple_fields[]={
     "elts",
     "ctx",
 };
-static PyTypeObject *expr_context_type;
-static PyObject *Load_singleton, *Store_singleton, *Del_singleton,
-*AugLoad_singleton, *AugStore_singleton, *Param_singleton;
 static PyObject* ast2obj_expr_context(expr_context_ty);
-static PyTypeObject *Load_type;
-static PyTypeObject *Store_type;
-static PyTypeObject *Del_type;
-static PyTypeObject *AugLoad_type;
-static PyTypeObject *AugStore_type;
-static PyTypeObject *Param_type;
-static PyTypeObject *slice_type;
 static PyObject* ast2obj_slice(void*);
-static PyTypeObject *Slice_type;
-_Py_IDENTIFIER(lower);
-_Py_IDENTIFIER(upper);
-_Py_IDENTIFIER(step);
 static const char * const Slice_fields[]={
     "lower",
     "upper",
     "step",
 };
-static PyTypeObject *ExtSlice_type;
-_Py_IDENTIFIER(dims);
 static const char * const ExtSlice_fields[]={
     "dims",
 };
-static PyTypeObject *Index_type;
 static const char * const Index_fields[]={
     "value",
 };
-static PyTypeObject *boolop_type;
-static PyObject *And_singleton, *Or_singleton;
 static PyObject* ast2obj_boolop(boolop_ty);
-static PyTypeObject *And_type;
-static PyTypeObject *Or_type;
-static PyTypeObject *operator_type;
-static PyObject *Add_singleton, *Sub_singleton, *Mult_singleton,
-*MatMult_singleton, *Div_singleton, *Mod_singleton, *Pow_singleton,
-*LShift_singleton, *RShift_singleton, *BitOr_singleton, *BitXor_singleton,
-*BitAnd_singleton, *FloorDiv_singleton;
 static PyObject* ast2obj_operator(operator_ty);
-static PyTypeObject *Add_type;
-static PyTypeObject *Sub_type;
-static PyTypeObject *Mult_type;
-static PyTypeObject *MatMult_type;
-static PyTypeObject *Div_type;
-static PyTypeObject *Mod_type;
-static PyTypeObject *Pow_type;
-static PyTypeObject *LShift_type;
-static PyTypeObject *RShift_type;
-static PyTypeObject *BitOr_type;
-static PyTypeObject *BitXor_type;
-static PyTypeObject *BitAnd_type;
-static PyTypeObject *FloorDiv_type;
-static PyTypeObject *unaryop_type;
-static PyObject *Invert_singleton, *Not_singleton, *UAdd_singleton,
-*USub_singleton;
 static PyObject* ast2obj_unaryop(unaryop_ty);
-static PyTypeObject *Invert_type;
-static PyTypeObject *Not_type;
-static PyTypeObject *UAdd_type;
-static PyTypeObject *USub_type;
-static PyTypeObject *cmpop_type;
-static PyObject *Eq_singleton, *NotEq_singleton, *Lt_singleton, *LtE_singleton,
-*Gt_singleton, *GtE_singleton, *Is_singleton, *IsNot_singleton, *In_singleton,
-*NotIn_singleton;
 static PyObject* ast2obj_cmpop(cmpop_ty);
-static PyTypeObject *Eq_type;
-static PyTypeObject *NotEq_type;
-static PyTypeObject *Lt_type;
-static PyTypeObject *LtE_type;
-static PyTypeObject *Gt_type;
-static PyTypeObject *GtE_type;
-static PyTypeObject *Is_type;
-static PyTypeObject *IsNot_type;
-static PyTypeObject *In_type;
-static PyTypeObject *NotIn_type;
-static PyTypeObject *comprehension_type;
 static PyObject* ast2obj_comprehension(void*);
-_Py_IDENTIFIER(ifs);
-_Py_IDENTIFIER(is_async);
 static const char * const comprehension_fields[]={
     "target",
     "iter",
     "ifs",
     "is_async",
 };
-static PyTypeObject *excepthandler_type;
 static const char * const excepthandler_attributes[] = {
     "lineno",
     "col_offset",
@@ -460,21 +1071,12 @@ static const char * const excepthandler_attributes[] = {
     "end_col_offset",
 };
 static PyObject* ast2obj_excepthandler(void*);
-static PyTypeObject *ExceptHandler_type;
-_Py_IDENTIFIER(type);
 static const char * const ExceptHandler_fields[]={
     "type",
     "name",
     "body",
 };
-static PyTypeObject *arguments_type;
 static PyObject* ast2obj_arguments(void*);
-_Py_IDENTIFIER(posonlyargs);
-_Py_IDENTIFIER(vararg);
-_Py_IDENTIFIER(kwonlyargs);
-_Py_IDENTIFIER(kw_defaults);
-_Py_IDENTIFIER(kwarg);
-_Py_IDENTIFIER(defaults);
 static const char * const arguments_fields[]={
     "posonlyargs",
     "args",
@@ -484,7 +1086,6 @@ static const char * const arguments_fields[]={
     "kwarg",
     "defaults",
 };
-static PyTypeObject *arg_type;
 static PyObject* ast2obj_arg(void*);
 static const char * const arg_attributes[] = {
     "lineno",
@@ -492,45 +1093,33 @@ static const char * const arg_attributes[] = {
     "end_lineno",
     "end_col_offset",
 };
-_Py_IDENTIFIER(arg);
 static const char * const arg_fields[]={
     "arg",
     "annotation",
     "type_comment",
 };
-static PyTypeObject *keyword_type;
 static PyObject* ast2obj_keyword(void*);
 static const char * const keyword_fields[]={
     "arg",
     "value",
 };
-static PyTypeObject *alias_type;
 static PyObject* ast2obj_alias(void*);
-_Py_IDENTIFIER(asname);
 static const char * const alias_fields[]={
     "name",
     "asname",
 };
-static PyTypeObject *withitem_type;
 static PyObject* ast2obj_withitem(void*);
-_Py_IDENTIFIER(context_expr);
-_Py_IDENTIFIER(optional_vars);
 static const char * const withitem_fields[]={
     "context_expr",
     "optional_vars",
 };
-static PyTypeObject *type_ignore_type;
 static PyObject* ast2obj_type_ignore(void*);
-static PyTypeObject *TypeIgnore_type;
-_Py_IDENTIFIER(tag);
 static const char * const TypeIgnore_fields[]={
     "lineno",
     "tag",
 };
 
 
-_Py_IDENTIFIER(_fields);
-_Py_IDENTIFIER(_attributes);
 
 typedef struct {
     PyObject_HEAD
@@ -566,7 +1155,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
     Py_ssize_t i, numfields = 0;
     int res = -1;
     PyObject *key, *value, *fields;
-    if (_PyObject_LookupAttrId((PyObject*)Py_TYPE(self), &PyId__fields, &fields) < 0) {
+    if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), astmodulestate_global->_fields, &fields) < 0) {
         goto cleanup;
     }
     if (fields) {
@@ -579,7 +1168,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
     if (numfields < PyTuple_GET_SIZE(args)) {
         PyErr_Format(PyExc_TypeError, "%.400s constructor takes at most "
                      "%zd positional argument%s",
-                     Py_TYPE(self)->tp_name,
+                     _PyType_Name(Py_TYPE(self)),
                      numfields, numfields == 1 ? "" : "s");
         res = -1;
         goto cleanup;
@@ -613,9 +1202,8 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
 static PyObject *
 ast_type_reduce(PyObject *self, PyObject *unused)
 {
-    _Py_IDENTIFIER(__dict__);
     PyObject *dict;
-    if (_PyObject_LookupAttrId(self, &PyId___dict__, &dict) < 0) {
+    if (_PyObject_LookupAttr(self, astmodulestate_global->__dict__, &dict) < 0) {
         return NULL;
     }
     if (dict) {
@@ -634,54 +1222,33 @@ static PyGetSetDef ast_type_getsets[] = {
     {NULL}
 };
 
-static PyTypeObject AST_type = {
-    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+static PyType_Slot AST_type_slots[] = {
+    {Py_tp_dealloc, ast_dealloc},
+    {Py_tp_getattro, PyObject_GenericGetAttr},
+    {Py_tp_setattro, PyObject_GenericSetAttr},
+    {Py_tp_traverse, ast_traverse},
+    {Py_tp_clear, ast_clear},
+    {Py_tp_methods, ast_type_methods},
+    {Py_tp_getset, ast_type_getsets},
+    {Py_tp_init, ast_type_init},
+    {Py_tp_alloc, PyType_GenericAlloc},
+    {Py_tp_new, PyType_GenericNew},
+    {Py_tp_free, PyType_GenericNew},
+    {Py_tp_free, PyObject_GC_Del},
+    {0, 0},
+};
+
+static PyType_Spec AST_type_spec = {
     "_ast.AST",
     sizeof(AST_object),
     0,
-    (destructor)ast_dealloc, /* tp_dealloc */
-    0,                       /* tp_vectorcall_offset */
-    0,                       /* tp_getattr */
-    0,                       /* tp_setattr */
-    0,                       /* tp_as_async */
-    0,                       /* tp_repr */
-    0,                       /* tp_as_number */
-    0,                       /* tp_as_sequence */
-    0,                       /* tp_as_mapping */
-    0,                       /* tp_hash */
-    0,                       /* tp_call */
-    0,                       /* tp_str */
-    PyObject_GenericGetAttr, /* tp_getattro */
-    PyObject_GenericSetAttr, /* tp_setattro */
-    0,                       /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */
-    0,                       /* tp_doc */
-    (traverseproc)ast_traverse, /* tp_traverse */
-    (inquiry)ast_clear,      /* tp_clear */
-    0,                       /* tp_richcompare */
-    0,                       /* tp_weaklistoffset */
-    0,                       /* tp_iter */
-    0,                       /* tp_iternext */
-    ast_type_methods,        /* tp_methods */
-    0,                       /* tp_members */
-    ast_type_getsets,        /* tp_getset */
-    0,                       /* tp_base */
-    0,                       /* tp_dict */
-    0,                       /* tp_descr_get */
-    0,                       /* tp_descr_set */
-    offsetof(AST_object, dict),/* tp_dictoffset */
-    (initproc)ast_type_init, /* tp_init */
-    PyType_GenericAlloc,     /* tp_alloc */
-    PyType_GenericNew,       /* tp_new */
-    PyObject_GC_Del,         /* tp_free */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+    AST_type_slots
 };
 
-
-static PyTypeObject *
-make_type(const char *type, PyTypeObject *base, const char * const *fields, int num_fields)
+static PyObject *
+make_type(const char *type, PyObject* base, const char* const* fields, int num_fields)
 {
-    _Py_IDENTIFIER(__module__);
-    _Py_IDENTIFIER(_ast);
     PyObject *fnames, *result;
     int i;
     fnames = PyTuple_New(num_fields);
@@ -696,15 +1263,15 @@ make_type(const char *type, PyTypeObject *base, const char * const *fields, int
     }
     result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){OOOO}",
                     type, base,
-                    _PyUnicode_FromId(&PyId__fields), fnames,
-                    _PyUnicode_FromId(&PyId___module__),
-                    _PyUnicode_FromId(&PyId__ast));
+                    astmodulestate_global->_fields, fnames,
+                    astmodulestate_global->__module__,
+                    astmodulestate_global->_ast);
     Py_DECREF(fnames);
-    return (PyTypeObject*)result;
+    return result;
 }
 
 static int
-add_attributes(PyTypeObject *type, const char * const *attrs, int num_fields)
+add_attributes(PyObject *type, const char * const *attrs, int num_fields)
 {
     int i, result;
     PyObject *s, *l = PyTuple_New(num_fields);
@@ -718,7 +1285,7 @@ add_attributes(PyTypeObject *type, const char * const *attrs, int num_fields)
         }
         PyTuple_SET_ITEM(l, i, s);
     }
-    result = _PyObject_SetAttrId((PyObject*)type, &PyId__attributes, l) >= 0;
+    result = PyObject_SetAttr(type, astmodulestate_global->_attributes, l) >= 0;
     Py_DECREF(l);
     return result;
 }
@@ -824,14 +1391,11 @@ static int obj2ast_int(PyObject* obj, int* out, PyArena* arena)
 
 static int add_ast_fields(void)
 {
-    PyObject *empty_tuple, *d;
-    if (PyType_Ready(&AST_type) < 0)
-        return -1;
-    d = AST_type.tp_dict;
+    PyObject *empty_tuple;
     empty_tuple = PyTuple_New(0);
     if (!empty_tuple ||
-        _PyDict_SetItemId(d, &PyId__fields, empty_tuple) < 0 ||
-        _PyDict_SetItemId(d, &PyId__attributes, empty_tuple) < 0) {
+        PyObject_SetAttrString(astmodulestate_global->AST_type, "_fields", empty_tuple) < 0 ||
+        PyObject_SetAttrString(astmodulestate_global->AST_type, "_attributes", empty_tuple) < 0) {
         Py_XDECREF(empty_tuple);
         return -1;
     }
@@ -842,333 +1406,427 @@ static int add_ast_fields(void)
 
 static int init_types(void)
 {
-    static int initialized;
-    if (initialized) return 1;
+    PyObject *m;
+    if (PyState_FindModule(&_astmodule) == NULL) {
+        m = PyModule_Create(&_astmodule);
+        if (!m) return 0;
+        PyState_AddModule(m, &_astmodule);
+    }
+    astmodulestate *state = astmodulestate_global;
+    if (state->initialized) return 1;
+    if (init_identifiers() < 0) return 0;
+    state->AST_type = PyType_FromSpec(&AST_type_spec);
+    if (!state->AST_type) return 0;
+    ((PyTypeObject*)state->AST_type)->tp_dictoffset = offsetof(AST_object,
+     dict);
     if (add_ast_fields() < 0) return 0;
-    mod_type = make_type("mod", &AST_type, NULL, 0);
-    if (!mod_type) return 0;
-    if (!add_attributes(mod_type, NULL, 0)) return 0;
-    Module_type = make_type("Module", mod_type, Module_fields, 2);
-    if (!Module_type) return 0;
-    Interactive_type = make_type("Interactive", mod_type, Interactive_fields,
-                                 1);
-    if (!Interactive_type) return 0;
-    Expression_type = make_type("Expression", mod_type, Expression_fields, 1);
-    if (!Expression_type) return 0;
-    FunctionType_type = make_type("FunctionType", mod_type,
-                                  FunctionType_fields, 2);
-    if (!FunctionType_type) return 0;
-    Suite_type = make_type("Suite", mod_type, Suite_fields, 1);
-    if (!Suite_type) return 0;
-    stmt_type = make_type("stmt", &AST_type, NULL, 0);
-    if (!stmt_type) return 0;
-    if (!add_attributes(stmt_type, stmt_attributes, 4)) return 0;
-    FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_fields,
-                                 6);
-    if (!FunctionDef_type) return 0;
-    AsyncFunctionDef_type = make_type("AsyncFunctionDef", stmt_type,
-                                      AsyncFunctionDef_fields, 6);
-    if (!AsyncFunctionDef_type) return 0;
-    ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 5);
-    if (!ClassDef_type) return 0;
-    Return_type = make_type("Return", stmt_type, Return_fields, 1);
-    if (!Return_type) return 0;
-    Delete_type = make_type("Delete", stmt_type, Delete_fields, 1);
-    if (!Delete_type) return 0;
-    Assign_type = make_type("Assign", stmt_type, Assign_fields, 3);
-    if (!Assign_type) return 0;
-    AugAssign_type = make_type("AugAssign", stmt_type, AugAssign_fields, 3);
-    if (!AugAssign_type) return 0;
-    AnnAssign_type = make_type("AnnAssign", stmt_type, AnnAssign_fields, 4);
-    if (!AnnAssign_type) return 0;
-    For_type = make_type("For", stmt_type, For_fields, 5);
-    if (!For_type) return 0;
-    AsyncFor_type = make_type("AsyncFor", stmt_type, AsyncFor_fields, 5);
-    if (!AsyncFor_type) return 0;
-    While_type = make_type("While", stmt_type, While_fields, 3);
-    if (!While_type) return 0;
-    If_type = make_type("If", stmt_type, If_fields, 3);
-    if (!If_type) return 0;
-    With_type = make_type("With", stmt_type, With_fields, 3);
-    if (!With_type) return 0;
-    AsyncWith_type = make_type("AsyncWith", stmt_type, AsyncWith_fields, 3);
-    if (!AsyncWith_type) return 0;
-    Raise_type = make_type("Raise", stmt_type, Raise_fields, 2);
-    if (!Raise_type) return 0;
-    Try_type = make_type("Try", stmt_type, Try_fields, 4);
-    if (!Try_type) return 0;
-    Assert_type = make_type("Assert", stmt_type, Assert_fields, 2);
-    if (!Assert_type) return 0;
-    Import_type = make_type("Import", stmt_type, Import_fields, 1);
-    if (!Import_type) return 0;
-    ImportFrom_type = make_type("ImportFrom", stmt_type, ImportFrom_fields, 3);
-    if (!ImportFrom_type) return 0;
-    Global_type = make_type("Global", stmt_type, Global_fields, 1);
-    if (!Global_type) return 0;
-    Nonlocal_type = make_type("Nonlocal", stmt_type, Nonlocal_fields, 1);
-    if (!Nonlocal_type) return 0;
-    Expr_type = make_type("Expr", stmt_type, Expr_fields, 1);
-    if (!Expr_type) return 0;
-    Pass_type = make_type("Pass", stmt_type, NULL, 0);
-    if (!Pass_type) return 0;
-    Break_type = make_type("Break", stmt_type, NULL, 0);
-    if (!Break_type) return 0;
-    Continue_type = make_type("Continue", stmt_type, NULL, 0);
-    if (!Continue_type) return 0;
-    expr_type = make_type("expr", &AST_type, NULL, 0);
-    if (!expr_type) return 0;
-    if (!add_attributes(expr_type, expr_attributes, 4)) return 0;
-    BoolOp_type = make_type("BoolOp", expr_type, BoolOp_fields, 2);
-    if (!BoolOp_type) return 0;
-    NamedExpr_type = make_type("NamedExpr", expr_type, NamedExpr_fields, 2);
-    if (!NamedExpr_type) return 0;
-    BinOp_type = make_type("BinOp", expr_type, BinOp_fields, 3);
-    if (!BinOp_type) return 0;
-    UnaryOp_type = make_type("UnaryOp", expr_type, UnaryOp_fields, 2);
-    if (!UnaryOp_type) return 0;
-    Lambda_type = make_type("Lambda", expr_type, Lambda_fields, 2);
-    if (!Lambda_type) return 0;
-    IfExp_type = make_type("IfExp", expr_type, IfExp_fields, 3);
-    if (!IfExp_type) return 0;
-    Dict_type = make_type("Dict", expr_type, Dict_fields, 2);
-    if (!Dict_type) return 0;
-    Set_type = make_type("Set", expr_type, Set_fields, 1);
-    if (!Set_type) return 0;
-    ListComp_type = make_type("ListComp", expr_type, ListComp_fields, 2);
-    if (!ListComp_type) return 0;
-    SetComp_type = make_type("SetComp", expr_type, SetComp_fields, 2);
-    if (!SetComp_type) return 0;
-    DictComp_type = make_type("DictComp", expr_type, DictComp_fields, 3);
-    if (!DictComp_type) return 0;
-    GeneratorExp_type = make_type("GeneratorExp", expr_type,
-                                  GeneratorExp_fields, 2);
-    if (!GeneratorExp_type) return 0;
-    Await_type = make_type("Await", expr_type, Await_fields, 1);
-    if (!Await_type) return 0;
-    Yield_type = make_type("Yield", expr_type, Yield_fields, 1);
-    if (!Yield_type) return 0;
-    YieldFrom_type = make_type("YieldFrom", expr_type, YieldFrom_fields, 1);
-    if (!YieldFrom_type) return 0;
-    Compare_type = make_type("Compare", expr_type, Compare_fields, 3);
-    if (!Compare_type) return 0;
-    Call_type = make_type("Call", expr_type, Call_fields, 3);
-    if (!Call_type) return 0;
-    FormattedValue_type = make_type("FormattedValue", expr_type,
-                                    FormattedValue_fields, 3);
-    if (!FormattedValue_type) return 0;
-    JoinedStr_type = make_type("JoinedStr", expr_type, JoinedStr_fields, 1);
-    if (!JoinedStr_type) return 0;
-    Constant_type = make_type("Constant", expr_type, Constant_fields, 2);
-    if (!Constant_type) return 0;
-    Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3);
-    if (!Attribute_type) return 0;
-    Subscript_type = make_type("Subscript", expr_type, Subscript_fields, 3);
-    if (!Subscript_type) return 0;
-    Starred_type = make_type("Starred", expr_type, Starred_fields, 2);
-    if (!Starred_type) return 0;
-    Name_type = make_type("Name", expr_type, Name_fields, 2);
-    if (!Name_type) return 0;
-    List_type = make_type("List", expr_type, List_fields, 2);
-    if (!List_type) return 0;
-    Tuple_type = make_type("Tuple", expr_type, Tuple_fields, 2);
-    if (!Tuple_type) return 0;
-    expr_context_type = make_type("expr_context", &AST_type, NULL, 0);
-    if (!expr_context_type) return 0;
-    if (!add_attributes(expr_context_type, NULL, 0)) return 0;
-    Load_type = make_type("Load", expr_context_type, NULL, 0);
-    if (!Load_type) return 0;
-    Load_singleton = PyType_GenericNew(Load_type, NULL, NULL);
-    if (!Load_singleton) return 0;
-    Store_type = make_type("Store", expr_context_type, NULL, 0);
-    if (!Store_type) return 0;
-    Store_singleton = PyType_GenericNew(Store_type, NULL, NULL);
-    if (!Store_singleton) return 0;
-    Del_type = make_type("Del", expr_context_type, NULL, 0);
-    if (!Del_type) return 0;
-    Del_singleton = PyType_GenericNew(Del_type, NULL, NULL);
-    if (!Del_singleton) return 0;
-    AugLoad_type = make_type("AugLoad", expr_context_type, NULL, 0);
-    if (!AugLoad_type) return 0;
-    AugLoad_singleton = PyType_GenericNew(AugLoad_type, NULL, NULL);
-    if (!AugLoad_singleton) return 0;
-    AugStore_type = make_type("AugStore", expr_context_type, NULL, 0);
-    if (!AugStore_type) return 0;
-    AugStore_singleton = PyType_GenericNew(AugStore_type, NULL, NULL);
-    if (!AugStore_singleton) return 0;
-    Param_type = make_type("Param", expr_context_type, NULL, 0);
-    if (!Param_type) return 0;
-    Param_singleton = PyType_GenericNew(Param_type, NULL, NULL);
-    if (!Param_singleton) return 0;
-    slice_type = make_type("slice", &AST_type, NULL, 0);
-    if (!slice_type) return 0;
-    if (!add_attributes(slice_type, NULL, 0)) return 0;
-    Slice_type = make_type("Slice", slice_type, Slice_fields, 3);
-    if (!Slice_type) return 0;
-    ExtSlice_type = make_type("ExtSlice", slice_type, ExtSlice_fields, 1);
-    if (!ExtSlice_type) return 0;
-    Index_type = make_type("Index", slice_type, Index_fields, 1);
-    if (!Index_type) return 0;
-    boolop_type = make_type("boolop", &AST_type, NULL, 0);
-    if (!boolop_type) return 0;
-    if (!add_attributes(boolop_type, NULL, 0)) return 0;
-    And_type = make_type("And", boolop_type, NULL, 0);
-    if (!And_type) return 0;
-    And_singleton = PyType_GenericNew(And_type, NULL, NULL);
-    if (!And_singleton) return 0;
-    Or_type = make_type("Or", boolop_type, NULL, 0);
-    if (!Or_type) return 0;
-    Or_singleton = PyType_GenericNew(Or_type, NULL, NULL);
-    if (!Or_singleton) return 0;
-    operator_type = make_type("operator", &AST_type, NULL, 0);
-    if (!operator_type) return 0;
-    if (!add_attributes(operator_type, NULL, 0)) return 0;
-    Add_type = make_type("Add", operator_type, NULL, 0);
-    if (!Add_type) return 0;
-    Add_singleton = PyType_GenericNew(Add_type, NULL, NULL);
-    if (!Add_singleton) return 0;
-    Sub_type = make_type("Sub", operator_type, NULL, 0);
-    if (!Sub_type) return 0;
-    Sub_singleton = PyType_GenericNew(Sub_type, NULL, NULL);
-    if (!Sub_singleton) return 0;
-    Mult_type = make_type("Mult", operator_type, NULL, 0);
-    if (!Mult_type) return 0;
-    Mult_singleton = PyType_GenericNew(Mult_type, NULL, NULL);
-    if (!Mult_singleton) return 0;
-    MatMult_type = make_type("MatMult", operator_type, NULL, 0);
-    if (!MatMult_type) return 0;
-    MatMult_singleton = PyType_GenericNew(MatMult_type, NULL, NULL);
-    if (!MatMult_singleton) return 0;
-    Div_type = make_type("Div", operator_type, NULL, 0);
-    if (!Div_type) return 0;
-    Div_singleton = PyType_GenericNew(Div_type, NULL, NULL);
-    if (!Div_singleton) return 0;
-    Mod_type = make_type("Mod", operator_type, NULL, 0);
-    if (!Mod_type) return 0;
-    Mod_singleton = PyType_GenericNew(Mod_type, NULL, NULL);
-    if (!Mod_singleton) return 0;
-    Pow_type = make_type("Pow", operator_type, NULL, 0);
-    if (!Pow_type) return 0;
-    Pow_singleton = PyType_GenericNew(Pow_type, NULL, NULL);
-    if (!Pow_singleton) return 0;
-    LShift_type = make_type("LShift", operator_type, NULL, 0);
-    if (!LShift_type) return 0;
-    LShift_singleton = PyType_GenericNew(LShift_type, NULL, NULL);
-    if (!LShift_singleton) return 0;
-    RShift_type = make_type("RShift", operator_type, NULL, 0);
-    if (!RShift_type) return 0;
-    RShift_singleton = PyType_GenericNew(RShift_type, NULL, NULL);
-    if (!RShift_singleton) return 0;
-    BitOr_type = make_type("BitOr", operator_type, NULL, 0);
-    if (!BitOr_type) return 0;
-    BitOr_singleton = PyType_GenericNew(BitOr_type, NULL, NULL);
-    if (!BitOr_singleton) return 0;
-    BitXor_type = make_type("BitXor", operator_type, NULL, 0);
-    if (!BitXor_type) return 0;
-    BitXor_singleton = PyType_GenericNew(BitXor_type, NULL, NULL);
-    if (!BitXor_singleton) return 0;
-    BitAnd_type = make_type("BitAnd", operator_type, NULL, 0);
-    if (!BitAnd_type) return 0;
-    BitAnd_singleton = PyType_GenericNew(BitAnd_type, NULL, NULL);
-    if (!BitAnd_singleton) return 0;
-    FloorDiv_type = make_type("FloorDiv", operator_type, NULL, 0);
-    if (!FloorDiv_type) return 0;
-    FloorDiv_singleton = PyType_GenericNew(FloorDiv_type, NULL, NULL);
-    if (!FloorDiv_singleton) return 0;
-    unaryop_type = make_type("unaryop", &AST_type, NULL, 0);
-    if (!unaryop_type) return 0;
-    if (!add_attributes(unaryop_type, NULL, 0)) return 0;
-    Invert_type = make_type("Invert", unaryop_type, NULL, 0);
-    if (!Invert_type) return 0;
-    Invert_singleton = PyType_GenericNew(Invert_type, NULL, NULL);
-    if (!Invert_singleton) return 0;
-    Not_type = make_type("Not", unaryop_type, NULL, 0);
-    if (!Not_type) return 0;
-    Not_singleton = PyType_GenericNew(Not_type, NULL, NULL);
-    if (!Not_singleton) return 0;
-    UAdd_type = make_type("UAdd", unaryop_type, NULL, 0);
-    if (!UAdd_type) return 0;
-    UAdd_singleton = PyType_GenericNew(UAdd_type, NULL, NULL);
-    if (!UAdd_singleton) return 0;
-    USub_type = make_type("USub", unaryop_type, NULL, 0);
-    if (!USub_type) return 0;
-    USub_singleton = PyType_GenericNew(USub_type, NULL, NULL);
-    if (!USub_singleton) return 0;
-    cmpop_type = make_type("cmpop", &AST_type, NULL, 0);
-    if (!cmpop_type) return 0;
-    if (!add_attributes(cmpop_type, NULL, 0)) return 0;
-    Eq_type = make_type("Eq", cmpop_type, NULL, 0);
-    if (!Eq_type) return 0;
-    Eq_singleton = PyType_GenericNew(Eq_type, NULL, NULL);
-    if (!Eq_singleton) return 0;
-    NotEq_type = make_type("NotEq", cmpop_type, NULL, 0);
-    if (!NotEq_type) return 0;
-    NotEq_singleton = PyType_GenericNew(NotEq_type, NULL, NULL);
-    if (!NotEq_singleton) return 0;
-    Lt_type = make_type("Lt", cmpop_type, NULL, 0);
-    if (!Lt_type) return 0;
-    Lt_singleton = PyType_GenericNew(Lt_type, NULL, NULL);
-    if (!Lt_singleton) return 0;
-    LtE_type = make_type("LtE", cmpop_type, NULL, 0);
-    if (!LtE_type) return 0;
-    LtE_singleton = PyType_GenericNew(LtE_type, NULL, NULL);
-    if (!LtE_singleton) return 0;
-    Gt_type = make_type("Gt", cmpop_type, NULL, 0);
-    if (!Gt_type) return 0;
-    Gt_singleton = PyType_GenericNew(Gt_type, NULL, NULL);
-    if (!Gt_singleton) return 0;
-    GtE_type = make_type("GtE", cmpop_type, NULL, 0);
-    if (!GtE_type) return 0;
-    GtE_singleton = PyType_GenericNew(GtE_type, NULL, NULL);
-    if (!GtE_singleton) return 0;
-    Is_type = make_type("Is", cmpop_type, NULL, 0);
-    if (!Is_type) return 0;
-    Is_singleton = PyType_GenericNew(Is_type, NULL, NULL);
-    if (!Is_singleton) return 0;
-    IsNot_type = make_type("IsNot", cmpop_type, NULL, 0);
-    if (!IsNot_type) return 0;
-    IsNot_singleton = PyType_GenericNew(IsNot_type, NULL, NULL);
-    if (!IsNot_singleton) return 0;
-    In_type = make_type("In", cmpop_type, NULL, 0);
-    if (!In_type) return 0;
-    In_singleton = PyType_GenericNew(In_type, NULL, NULL);
-    if (!In_singleton) return 0;
-    NotIn_type = make_type("NotIn", cmpop_type, NULL, 0);
-    if (!NotIn_type) return 0;
-    NotIn_singleton = PyType_GenericNew(NotIn_type, NULL, NULL);
-    if (!NotIn_singleton) return 0;
-    comprehension_type = make_type("comprehension", &AST_type,
-                                   comprehension_fields, 4);
-    if (!comprehension_type) return 0;
-    if (!add_attributes(comprehension_type, NULL, 0)) return 0;
-    excepthandler_type = make_type("excepthandler", &AST_type, NULL, 0);
-    if (!excepthandler_type) return 0;
-    if (!add_attributes(excepthandler_type, excepthandler_attributes, 4))
-        return 0;
-    ExceptHandler_type = make_type("ExceptHandler", excepthandler_type,
-                                   ExceptHandler_fields, 3);
-    if (!ExceptHandler_type) return 0;
-    arguments_type = make_type("arguments", &AST_type, arguments_fields, 7);
-    if (!arguments_type) return 0;
-    if (!add_attributes(arguments_type, NULL, 0)) return 0;
-    arg_type = make_type("arg", &AST_type, arg_fields, 3);
-    if (!arg_type) return 0;
-    if (!add_attributes(arg_type, arg_attributes, 4)) return 0;
-    keyword_type = make_type("keyword", &AST_type, keyword_fields, 2);
-    if (!keyword_type) return 0;
-    if (!add_attributes(keyword_type, NULL, 0)) return 0;
-    alias_type = make_type("alias", &AST_type, alias_fields, 2);
-    if (!alias_type) return 0;
-    if (!add_attributes(alias_type, NULL, 0)) return 0;
-    withitem_type = make_type("withitem", &AST_type, withitem_fields, 2);
-    if (!withitem_type) return 0;
-    if (!add_attributes(withitem_type, NULL, 0)) return 0;
-    type_ignore_type = make_type("type_ignore", &AST_type, NULL, 0);
-    if (!type_ignore_type) return 0;
-    if (!add_attributes(type_ignore_type, NULL, 0)) return 0;
-    TypeIgnore_type = make_type("TypeIgnore", type_ignore_type,
-                                TypeIgnore_fields, 2);
-    if (!TypeIgnore_type) return 0;
-    initialized = 1;
+    state->mod_type = make_type("mod", state->AST_type, NULL, 0);
+    if (!state->mod_type) return 0;
+    if (!add_attributes(state->mod_type, NULL, 0)) return 0;
+    state->Module_type = make_type("Module", state->mod_type, Module_fields, 2);
+    if (!state->Module_type) return 0;
+    state->Interactive_type = make_type("Interactive", state->mod_type,
+                                        Interactive_fields, 1);
+    if (!state->Interactive_type) return 0;
+    state->Expression_type = make_type("Expression", state->mod_type,
+                                       Expression_fields, 1);
+    if (!state->Expression_type) return 0;
+    state->FunctionType_type = make_type("FunctionType", state->mod_type,
+                                         FunctionType_fields, 2);
+    if (!state->FunctionType_type) return 0;
+    state->Suite_type = make_type("Suite", state->mod_type, Suite_fields, 1);
+    if (!state->Suite_type) return 0;
+    state->stmt_type = make_type("stmt", state->AST_type, NULL, 0);
+    if (!state->stmt_type) return 0;
+    if (!add_attributes(state->stmt_type, stmt_attributes, 4)) return 0;
+    state->FunctionDef_type = make_type("FunctionDef", state->stmt_type,
+                                        FunctionDef_fields, 6);
+    if (!state->FunctionDef_type) return 0;
+    state->AsyncFunctionDef_type = make_type("AsyncFunctionDef",
+                                             state->stmt_type,
+                                             AsyncFunctionDef_fields, 6);
+    if (!state->AsyncFunctionDef_type) return 0;
+    state->ClassDef_type = make_type("ClassDef", state->stmt_type,
+                                     ClassDef_fields, 5);
+    if (!state->ClassDef_type) return 0;
+    state->Return_type = make_type("Return", state->stmt_type, Return_fields,
+                                   1);
+    if (!state->Return_type) return 0;
+    state->Delete_type = make_type("Delete", state->stmt_type, Delete_fields,
+                                   1);
+    if (!state->Delete_type) return 0;
+    state->Assign_type = make_type("Assign", state->stmt_type, Assign_fields,
+                                   3);
+    if (!state->Assign_type) return 0;
+    state->AugAssign_type = make_type("AugAssign", state->stmt_type,
+                                      AugAssign_fields, 3);
+    if (!state->AugAssign_type) return 0;
+    state->AnnAssign_type = make_type("AnnAssign", state->stmt_type,
+                                      AnnAssign_fields, 4);
+    if (!state->AnnAssign_type) return 0;
+    state->For_type = make_type("For", state->stmt_type, For_fields, 5);
+    if (!state->For_type) return 0;
+    state->AsyncFor_type = make_type("AsyncFor", state->stmt_type,
+                                     AsyncFor_fields, 5);
+    if (!state->AsyncFor_type) return 0;
+    state->While_type = make_type("While", state->stmt_type, While_fields, 3);
+    if (!state->While_type) return 0;
+    state->If_type = make_type("If", state->stmt_type, If_fields, 3);
+    if (!state->If_type) return 0;
+    state->With_type = make_type("With", state->stmt_type, With_fields, 3);
+    if (!state->With_type) return 0;
+    state->AsyncWith_type = make_type("AsyncWith", state->stmt_type,
+                                      AsyncWith_fields, 3);
+    if (!state->AsyncWith_type) return 0;
+    state->Raise_type = make_type("Raise", state->stmt_type, Raise_fields, 2);
+    if (!state->Raise_type) return 0;
+    state->Try_type = make_type("Try", state->stmt_type, Try_fields, 4);
+    if (!state->Try_type) return 0;
+    state->Assert_type = make_type("Assert", state->stmt_type, Assert_fields,
+                                   2);
+    if (!state->Assert_type) return 0;
+    state->Import_type = make_type("Import", state->stmt_type, Import_fields,
+                                   1);
+    if (!state->Import_type) return 0;
+    state->ImportFrom_type = make_type("ImportFrom", state->stmt_type,
+                                       ImportFrom_fields, 3);
+    if (!state->ImportFrom_type) return 0;
+    state->Global_type = make_type("Global", state->stmt_type, Global_fields,
+                                   1);
+    if (!state->Global_type) return 0;
+    state->Nonlocal_type = make_type("Nonlocal", state->stmt_type,
+                                     Nonlocal_fields, 1);
+    if (!state->Nonlocal_type) return 0;
+    state->Expr_type = make_type("Expr", state->stmt_type, Expr_fields, 1);
+    if (!state->Expr_type) return 0;
+    state->Pass_type = make_type("Pass", state->stmt_type, NULL, 0);
+    if (!state->Pass_type) return 0;
+    state->Break_type = make_type("Break", state->stmt_type, NULL, 0);
+    if (!state->Break_type) return 0;
+    state->Continue_type = make_type("Continue", state->stmt_type, NULL, 0);
+    if (!state->Continue_type) return 0;
+    state->expr_type = make_type("expr", state->AST_type, NULL, 0);
+    if (!state->expr_type) return 0;
+    if (!add_attributes(state->expr_type, expr_attributes, 4)) return 0;
+    state->BoolOp_type = make_type("BoolOp", state->expr_type, BoolOp_fields,
+                                   2);
+    if (!state->BoolOp_type) return 0;
+    state->NamedExpr_type = make_type("NamedExpr", state->expr_type,
+                                      NamedExpr_fields, 2);
+    if (!state->NamedExpr_type) return 0;
+    state->BinOp_type = make_type("BinOp", state->expr_type, BinOp_fields, 3);
+    if (!state->BinOp_type) return 0;
+    state->UnaryOp_type = make_type("UnaryOp", state->expr_type,
+                                    UnaryOp_fields, 2);
+    if (!state->UnaryOp_type) return 0;
+    state->Lambda_type = make_type("Lambda", state->expr_type, Lambda_fields,
+                                   2);
+    if (!state->Lambda_type) return 0;
+    state->IfExp_type = make_type("IfExp", state->expr_type, IfExp_fields, 3);
+    if (!state->IfExp_type) return 0;
+    state->Dict_type = make_type("Dict", state->expr_type, Dict_fields, 2);
+    if (!state->Dict_type) return 0;
+    state->Set_type = make_type("Set", state->expr_type, Set_fields, 1);
+    if (!state->Set_type) return 0;
+    state->ListComp_type = make_type("ListComp", state->expr_type,
+                                     ListComp_fields, 2);
+    if (!state->ListComp_type) return 0;
+    state->SetComp_type = make_type("SetComp", state->expr_type,
+                                    SetComp_fields, 2);
+    if (!state->SetComp_type) return 0;
+    state->DictComp_type = make_type("DictComp", state->expr_type,
+                                     DictComp_fields, 3);
+    if (!state->DictComp_type) return 0;
+    state->GeneratorExp_type = make_type("GeneratorExp", state->expr_type,
+                                         GeneratorExp_fields, 2);
+    if (!state->GeneratorExp_type) return 0;
+    state->Await_type = make_type("Await", state->expr_type, Await_fields, 1);
+    if (!state->Await_type) return 0;
+    state->Yield_type = make_type("Yield", state->expr_type, Yield_fields, 1);
+    if (!state->Yield_type) return 0;
+    state->YieldFrom_type = make_type("YieldFrom", state->expr_type,
+                                      YieldFrom_fields, 1);
+    if (!state->YieldFrom_type) return 0;
+    state->Compare_type = make_type("Compare", state->expr_type,
+                                    Compare_fields, 3);
+    if (!state->Compare_type) return 0;
+    state->Call_type = make_type("Call", state->expr_type, Call_fields, 3);
+    if (!state->Call_type) return 0;
+    state->FormattedValue_type = make_type("FormattedValue", state->expr_type,
+                                           FormattedValue_fields, 3);
+    if (!state->FormattedValue_type) return 0;
+    state->JoinedStr_type = make_type("JoinedStr", state->expr_type,
+                                      JoinedStr_fields, 1);
+    if (!state->JoinedStr_type) return 0;
+    state->Constant_type = make_type("Constant", state->expr_type,
+                                     Constant_fields, 2);
+    if (!state->Constant_type) return 0;
+    state->Attribute_type = make_type("Attribute", state->expr_type,
+                                      Attribute_fields, 3);
+    if (!state->Attribute_type) return 0;
+    state->Subscript_type = make_type("Subscript", state->expr_type,
+                                      Subscript_fields, 3);
+    if (!state->Subscript_type) return 0;
+    state->Starred_type = make_type("Starred", state->expr_type,
+                                    Starred_fields, 2);
+    if (!state->Starred_type) return 0;
+    state->Name_type = make_type("Name", state->expr_type, Name_fields, 2);
+    if (!state->Name_type) return 0;
+    state->List_type = make_type("List", state->expr_type, List_fields, 2);
+    if (!state->List_type) return 0;
+    state->Tuple_type = make_type("Tuple", state->expr_type, Tuple_fields, 2);
+    if (!state->Tuple_type) return 0;
+    state->expr_context_type = make_type("expr_context", state->AST_type, NULL,
+                                         0);
+    if (!state->expr_context_type) return 0;
+    if (!add_attributes(state->expr_context_type, NULL, 0)) return 0;
+    state->Load_type = make_type("Load", state->expr_context_type, NULL, 0);
+    if (!state->Load_type) return 0;
+    state->Load_singleton = PyType_GenericNew((PyTypeObject *)state->Load_type,
+                                              NULL, NULL);
+    if (!state->Load_singleton) return 0;
+    state->Store_type = make_type("Store", state->expr_context_type, NULL, 0);
+    if (!state->Store_type) return 0;
+    state->Store_singleton = PyType_GenericNew((PyTypeObject
+                                               *)state->Store_type, NULL, NULL);
+    if (!state->Store_singleton) return 0;
+    state->Del_type = make_type("Del", state->expr_context_type, NULL, 0);
+    if (!state->Del_type) return 0;
+    state->Del_singleton = PyType_GenericNew((PyTypeObject *)state->Del_type,
+                                             NULL, NULL);
+    if (!state->Del_singleton) return 0;
+    state->AugLoad_type = make_type("AugLoad", state->expr_context_type, NULL,
+                                    0);
+    if (!state->AugLoad_type) return 0;
+    state->AugLoad_singleton = PyType_GenericNew((PyTypeObject
+                                                 *)state->AugLoad_type, NULL,
+                                                 NULL);
+    if (!state->AugLoad_singleton) return 0;
+    state->AugStore_type = make_type("AugStore", state->expr_context_type,
+                                     NULL, 0);
+    if (!state->AugStore_type) return 0;
+    state->AugStore_singleton = PyType_GenericNew((PyTypeObject
+                                                  *)state->AugStore_type, NULL,
+                                                  NULL);
+    if (!state->AugStore_singleton) return 0;
+    state->Param_type = make_type("Param", state->expr_context_type, NULL, 0);
+    if (!state->Param_type) return 0;
+    state->Param_singleton = PyType_GenericNew((PyTypeObject
+                                               *)state->Param_type, NULL, NULL);
+    if (!state->Param_singleton) return 0;
+    state->slice_type = make_type("slice", state->AST_type, NULL, 0);
+    if (!state->slice_type) return 0;
+    if (!add_attributes(state->slice_type, NULL, 0)) return 0;
+    state->Slice_type = make_type("Slice", state->slice_type, Slice_fields, 3);
+    if (!state->Slice_type) return 0;
+    state->ExtSlice_type = make_type("ExtSlice", state->slice_type,
+                                     ExtSlice_fields, 1);
+    if (!state->ExtSlice_type) return 0;
+    state->Index_type = make_type("Index", state->slice_type, Index_fields, 1);
+    if (!state->Index_type) return 0;
+    state->boolop_type = make_type("boolop", state->AST_type, NULL, 0);
+    if (!state->boolop_type) return 0;
+    if (!add_attributes(state->boolop_type, NULL, 0)) return 0;
+    state->And_type = make_type("And", state->boolop_type, NULL, 0);
+    if (!state->And_type) return 0;
+    state->And_singleton = PyType_GenericNew((PyTypeObject *)state->And_type,
+                                             NULL, NULL);
+    if (!state->And_singleton) return 0;
+    state->Or_type = make_type("Or", state->boolop_type, NULL, 0);
+    if (!state->Or_type) return 0;
+    state->Or_singleton = PyType_GenericNew((PyTypeObject *)state->Or_type,
+                                            NULL, NULL);
+    if (!state->Or_singleton) return 0;
+    state->operator_type = make_type("operator", state->AST_type, NULL, 0);
+    if (!state->operator_type) return 0;
+    if (!add_attributes(state->operator_type, NULL, 0)) return 0;
+    state->Add_type = make_type("Add", state->operator_type, NULL, 0);
+    if (!state->Add_type) return 0;
+    state->Add_singleton = PyType_GenericNew((PyTypeObject *)state->Add_type,
+                                             NULL, NULL);
+    if (!state->Add_singleton) return 0;
+    state->Sub_type = make_type("Sub", state->operator_type, NULL, 0);
+    if (!state->Sub_type) return 0;
+    state->Sub_singleton = PyType_GenericNew((PyTypeObject *)state->Sub_type,
+                                             NULL, NULL);
+    if (!state->Sub_singleton) return 0;
+    state->Mult_type = make_type("Mult", state->operator_type, NULL, 0);
+    if (!state->Mult_type) return 0;
+    state->Mult_singleton = PyType_GenericNew((PyTypeObject *)state->Mult_type,
+                                              NULL, NULL);
+    if (!state->Mult_singleton) return 0;
+    state->MatMult_type = make_type("MatMult", state->operator_type, NULL, 0);
+    if (!state->MatMult_type) return 0;
+    state->MatMult_singleton = PyType_GenericNew((PyTypeObject
+                                                 *)state->MatMult_type, NULL,
+                                                 NULL);
+    if (!state->MatMult_singleton) return 0;
+    state->Div_type = make_type("Div", state->operator_type, NULL, 0);
+    if (!state->Div_type) return 0;
+    state->Div_singleton = PyType_GenericNew((PyTypeObject *)state->Div_type,
+                                             NULL, NULL);
+    if (!state->Div_singleton) return 0;
+    state->Mod_type = make_type("Mod", state->operator_type, NULL, 0);
+    if (!state->Mod_type) return 0;
+    state->Mod_singleton = PyType_GenericNew((PyTypeObject *)state->Mod_type,
+                                             NULL, NULL);
+    if (!state->Mod_singleton) return 0;
+    state->Pow_type = make_type("Pow", state->operator_type, NULL, 0);
+    if (!state->Pow_type) return 0;
+    state->Pow_singleton = PyType_GenericNew((PyTypeObject *)state->Pow_type,
+                                             NULL, NULL);
+    if (!state->Pow_singleton) return 0;
+    state->LShift_type = make_type("LShift", state->operator_type, NULL, 0);
+    if (!state->LShift_type) return 0;
+    state->LShift_singleton = PyType_GenericNew((PyTypeObject
+                                                *)state->LShift_type, NULL,
+                                                NULL);
+    if (!state->LShift_singleton) return 0;
+    state->RShift_type = make_type("RShift", state->operator_type, NULL, 0);
+    if (!state->RShift_type) return 0;
+    state->RShift_singleton = PyType_GenericNew((PyTypeObject
+                                                *)state->RShift_type, NULL,
+                                                NULL);
+    if (!state->RShift_singleton) return 0;
+    state->BitOr_type = make_type("BitOr", state->operator_type, NULL, 0);
+    if (!state->BitOr_type) return 0;
+    state->BitOr_singleton = PyType_GenericNew((PyTypeObject
+                                               *)state->BitOr_type, NULL, NULL);
+    if (!state->BitOr_singleton) return 0;
+    state->BitXor_type = make_type("BitXor", state->operator_type, NULL, 0);
+    if (!state->BitXor_type) return 0;
+    state->BitXor_singleton = PyType_GenericNew((PyTypeObject
+                                                *)state->BitXor_type, NULL,
+                                                NULL);
+    if (!state->BitXor_singleton) return 0;
+    state->BitAnd_type = make_type("BitAnd", state->operator_type, NULL, 0);
+    if (!state->BitAnd_type) return 0;
+    state->BitAnd_singleton = PyType_GenericNew((PyTypeObject
+                                                *)state->BitAnd_type, NULL,
+                                                NULL);
+    if (!state->BitAnd_singleton) return 0;
+    state->FloorDiv_type = make_type("FloorDiv", state->operator_type, NULL, 0);
+    if (!state->FloorDiv_type) return 0;
+    state->FloorDiv_singleton = PyType_GenericNew((PyTypeObject
+                                                  *)state->FloorDiv_type, NULL,
+                                                  NULL);
+    if (!state->FloorDiv_singleton) return 0;
+    state->unaryop_type = make_type("unaryop", state->AST_type, NULL, 0);
+    if (!state->unaryop_type) return 0;
+    if (!add_attributes(state->unaryop_type, NULL, 0)) return 0;
+    state->Invert_type = make_type("Invert", state->unaryop_type, NULL, 0);
+    if (!state->Invert_type) return 0;
+    state->Invert_singleton = PyType_GenericNew((PyTypeObject
+                                                *)state->Invert_type, NULL,
+                                                NULL);
+    if (!state->Invert_singleton) return 0;
+    state->Not_type = make_type("Not", state->unaryop_type, NULL, 0);
+    if (!state->Not_type) return 0;
+    state->Not_singleton = PyType_GenericNew((PyTypeObject *)state->Not_type,
+                                             NULL, NULL);
+    if (!state->Not_singleton) return 0;
+    state->UAdd_type = make_type("UAdd", state->unaryop_type, NULL, 0);
+    if (!state->UAdd_type) return 0;
+    state->UAdd_singleton = PyType_GenericNew((PyTypeObject *)state->UAdd_type,
+                                              NULL, NULL);
+    if (!state->UAdd_singleton) return 0;
+    state->USub_type = make_type("USub", state->unaryop_type, NULL, 0);
+    if (!state->USub_type) return 0;
+    state->USub_singleton = PyType_GenericNew((PyTypeObject *)state->USub_type,
+                                              NULL, NULL);
+    if (!state->USub_singleton) return 0;
+    state->cmpop_type = make_type("cmpop", state->AST_type, NULL, 0);
+    if (!state->cmpop_type) return 0;
+    if (!add_attributes(state->cmpop_type, NULL, 0)) return 0;
+    state->Eq_type = make_type("Eq", state->cmpop_type, NULL, 0);
+    if (!state->Eq_type) return 0;
+    state->Eq_singleton = PyType_GenericNew((PyTypeObject *)state->Eq_type,
+                                            NULL, NULL);
+    if (!state->Eq_singleton) return 0;
+    state->NotEq_type = make_type("NotEq", state->cmpop_type, NULL, 0);
+    if (!state->NotEq_type) return 0;
+    state->NotEq_singleton = PyType_GenericNew((PyTypeObject
+                                               *)state->NotEq_type, NULL, NULL);
+    if (!state->NotEq_singleton) return 0;
+    state->Lt_type = make_type("Lt", state->cmpop_type, NULL, 0);
+    if (!state->Lt_type) return 0;
+    state->Lt_singleton = PyType_GenericNew((PyTypeObject *)state->Lt_type,
+                                            NULL, NULL);
+    if (!state->Lt_singleton) return 0;
+    state->LtE_type = make_type("LtE", state->cmpop_type, NULL, 0);
+    if (!state->LtE_type) return 0;
+    state->LtE_singleton = PyType_GenericNew((PyTypeObject *)state->LtE_type,
+                                             NULL, NULL);
+    if (!state->LtE_singleton) return 0;
+    state->Gt_type = make_type("Gt", state->cmpop_type, NULL, 0);
+    if (!state->Gt_type) return 0;
+    state->Gt_singleton = PyType_GenericNew((PyTypeObject *)state->Gt_type,
+                                            NULL, NULL);
+    if (!state->Gt_singleton) return 0;
+    state->GtE_type = make_type("GtE", state->cmpop_type, NULL, 0);
+    if (!state->GtE_type) return 0;
+    state->GtE_singleton = PyType_GenericNew((PyTypeObject *)state->GtE_type,
+                                             NULL, NULL);
+    if (!state->GtE_singleton) return 0;
+    state->Is_type = make_type("Is", state->cmpop_type, NULL, 0);
+    if (!state->Is_type) return 0;
+    state->Is_singleton = PyType_GenericNew((PyTypeObject *)state->Is_type,
+                                            NULL, NULL);
+    if (!state->Is_singleton) return 0;
+    state->IsNot_type = make_type("IsNot", state->cmpop_type, NULL, 0);
+    if (!state->IsNot_type) return 0;
+    state->IsNot_singleton = PyType_GenericNew((PyTypeObject
+                                               *)state->IsNot_type, NULL, NULL);
+    if (!state->IsNot_singleton) return 0;
+    state->In_type = make_type("In", state->cmpop_type, NULL, 0);
+    if (!state->In_type) return 0;
+    state->In_singleton = PyType_GenericNew((PyTypeObject *)state->In_type,
+                                            NULL, NULL);
+    if (!state->In_singleton) return 0;
+    state->NotIn_type = make_type("NotIn", state->cmpop_type, NULL, 0);
+    if (!state->NotIn_type) return 0;
+    state->NotIn_singleton = PyType_GenericNew((PyTypeObject
+                                               *)state->NotIn_type, NULL, NULL);
+    if (!state->NotIn_singleton) return 0;
+    state->comprehension_type = make_type("comprehension", state->AST_type,
+                                          comprehension_fields, 4);
+    if (!state->comprehension_type) return 0;
+    if (!add_attributes(state->comprehension_type, NULL, 0)) return 0;
+    state->excepthandler_type = make_type("excepthandler", state->AST_type,
+                                          NULL, 0);
+    if (!state->excepthandler_type) return 0;
+    if (!add_attributes(state->excepthandler_type, excepthandler_attributes,
+        4)) return 0;
+    state->ExceptHandler_type = make_type("ExceptHandler",
+                                          state->excepthandler_type,
+                                          ExceptHandler_fields, 3);
+    if (!state->ExceptHandler_type) return 0;
+    state->arguments_type = make_type("arguments", state->AST_type,
+                                      arguments_fields, 7);
+    if (!state->arguments_type) return 0;
+    if (!add_attributes(state->arguments_type, NULL, 0)) return 0;
+    state->arg_type = make_type("arg", state->AST_type, arg_fields, 3);
+    if (!state->arg_type) return 0;
+    if (!add_attributes(state->arg_type, arg_attributes, 4)) return 0;
+    state->keyword_type = make_type("keyword", state->AST_type, keyword_fields,
+                                    2);
+    if (!state->keyword_type) return 0;
+    if (!add_attributes(state->keyword_type, NULL, 0)) return 0;
+    state->alias_type = make_type("alias", state->AST_type, alias_fields, 2);
+    if (!state->alias_type) return 0;
+    if (!add_attributes(state->alias_type, NULL, 0)) return 0;
+    state->withitem_type = make_type("withitem", state->AST_type,
+                                     withitem_fields, 2);
+    if (!state->withitem_type) return 0;
+    if (!add_attributes(state->withitem_type, NULL, 0)) return 0;
+    state->type_ignore_type = make_type("type_ignore", state->AST_type, NULL,
+                                        0);
+    if (!state->type_ignore_type) return 0;
+    if (!add_attributes(state->type_ignore_type, NULL, 0)) return 0;
+    state->TypeIgnore_type = make_type("TypeIgnore", state->type_ignore_type,
+                                       TypeIgnore_fields, 2);
+    if (!state->TypeIgnore_type) return 0;
+    state->initialized = 1;
     return 1;
 }
 
@@ -2691,63 +3349,72 @@ ast2obj_mod(void* _o)
 {
     mod_ty o = (mod_ty)_o;
     PyObject *result = NULL, *value = NULL;
+    PyTypeObject *tp;
     if (!o) {
         Py_RETURN_NONE;
     }
 
     switch (o->kind) {
     case Module_kind:
-        result = PyType_GenericNew(Module_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Module_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.Module.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.Module.type_ignores, ast2obj_type_ignore);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_type_ignores, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->type_ignores,
+            value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Interactive_kind:
-        result = PyType_GenericNew(Interactive_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Interactive_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.Interactive.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Expression_kind:
-        result = PyType_GenericNew(Expression_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Expression_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Expression.body);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case FunctionType_kind:
-        result = PyType_GenericNew(FunctionType_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->FunctionType_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.FunctionType.argtypes, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_argtypes, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->argtypes, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.FunctionType.returns);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->returns, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Suite_kind:
-        result = PyType_GenericNew(Suite_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Suite_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.Suite.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
@@ -2764,461 +3431,517 @@ ast2obj_stmt(void* _o)
 {
     stmt_ty o = (stmt_ty)_o;
     PyObject *result = NULL, *value = NULL;
+    PyTypeObject *tp;
     if (!o) {
         Py_RETURN_NONE;
     }
 
     switch (o->kind) {
     case FunctionDef_kind:
-        result = PyType_GenericNew(FunctionDef_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->FunctionDef_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_identifier(o->v.FunctionDef.name);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_name, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->name, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_arguments(o->v.FunctionDef.args);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_args, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->args, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.FunctionDef.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.FunctionDef.decorator_list, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_decorator_list, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->decorator_list,
+            value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.FunctionDef.returns);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->returns, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_string(o->v.FunctionDef.type_comment);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->type_comment,
+            value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case AsyncFunctionDef_kind:
-        result = PyType_GenericNew(AsyncFunctionDef_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->AsyncFunctionDef_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_identifier(o->v.AsyncFunctionDef.name);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_name, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->name, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_arguments(o->v.AsyncFunctionDef.args);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_args, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->args, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.AsyncFunctionDef.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.AsyncFunctionDef.decorator_list,
                              ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_decorator_list, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->decorator_list,
+            value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.AsyncFunctionDef.returns);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->returns, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_string(o->v.AsyncFunctionDef.type_comment);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->type_comment,
+            value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case ClassDef_kind:
-        result = PyType_GenericNew(ClassDef_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->ClassDef_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_identifier(o->v.ClassDef.name);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_name, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->name, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.ClassDef.bases, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_bases, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->bases, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.ClassDef.keywords, ast2obj_keyword);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_keywords, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->keywords, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.ClassDef.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.ClassDef.decorator_list, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_decorator_list, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->decorator_list,
+            value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Return_kind:
-        result = PyType_GenericNew(Return_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Return_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Return.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Delete_kind:
-        result = PyType_GenericNew(Delete_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Delete_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.Delete.targets, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_targets, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->targets, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Assign_kind:
-        result = PyType_GenericNew(Assign_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Assign_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.Assign.targets, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_targets, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->targets, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.Assign.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_string(o->v.Assign.type_comment);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->type_comment,
+            value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case AugAssign_kind:
-        result = PyType_GenericNew(AugAssign_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->AugAssign_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.AugAssign.target);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_target, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->target, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_operator(o->v.AugAssign.op);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_op, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->op, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.AugAssign.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case AnnAssign_kind:
-        result = PyType_GenericNew(AnnAssign_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->AnnAssign_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.AnnAssign.target);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_target, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->target, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.AnnAssign.annotation);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_annotation, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->annotation, value)
+            == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.AnnAssign.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_int(o->v.AnnAssign.simple);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_simple, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->simple, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         break;
     case For_kind:
-        result = PyType_GenericNew(For_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->For_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.For.target);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_target, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->target, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.For.iter);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_iter, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->iter, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.For.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.For.orelse, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->orelse, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_string(o->v.For.type_comment);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->type_comment,
+            value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case AsyncFor_kind:
-        result = PyType_GenericNew(AsyncFor_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->AsyncFor_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.AsyncFor.target);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_target, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->target, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.AsyncFor.iter);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_iter, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->iter, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.AsyncFor.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.AsyncFor.orelse, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->orelse, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_string(o->v.AsyncFor.type_comment);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->type_comment,
+            value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case While_kind:
-        result = PyType_GenericNew(While_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->While_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.While.test);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_test, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->test, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.While.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.While.orelse, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->orelse, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         break;
     case If_kind:
-        result = PyType_GenericNew(If_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->If_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.If.test);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_test, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->test, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.If.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.If.orelse, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->orelse, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         break;
     case With_kind:
-        result = PyType_GenericNew(With_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->With_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.With.items, ast2obj_withitem);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_items, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->items, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.With.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_string(o->v.With.type_comment);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->type_comment,
+            value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case AsyncWith_kind:
-        result = PyType_GenericNew(AsyncWith_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->AsyncWith_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.AsyncWith.items, ast2obj_withitem);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_items, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->items, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.AsyncWith.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_string(o->v.AsyncWith.type_comment);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->type_comment,
+            value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Raise_kind:
-        result = PyType_GenericNew(Raise_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Raise_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Raise.exc);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_exc, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->exc, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.Raise.cause);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_cause, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->cause, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Try_kind:
-        result = PyType_GenericNew(Try_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Try_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.Try.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.Try.handlers, ast2obj_excepthandler);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_handlers, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->handlers, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.Try.orelse, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->orelse, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.Try.finalbody, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_finalbody, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->finalbody, value)
+            == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Assert_kind:
-        result = PyType_GenericNew(Assert_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Assert_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Assert.test);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_test, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->test, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.Assert.msg);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_msg, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->msg, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Import_kind:
-        result = PyType_GenericNew(Import_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Import_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.Import.names, ast2obj_alias);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_names, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->names, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case ImportFrom_kind:
-        result = PyType_GenericNew(ImportFrom_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->ImportFrom_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_identifier(o->v.ImportFrom.module);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_module, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->module, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.ImportFrom.names, ast2obj_alias);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_names, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->names, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_int(o->v.ImportFrom.level);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_level, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->level, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Global_kind:
-        result = PyType_GenericNew(Global_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Global_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.Global.names, ast2obj_identifier);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_names, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->names, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Nonlocal_kind:
-        result = PyType_GenericNew(Nonlocal_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Nonlocal_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.Nonlocal.names, ast2obj_identifier);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_names, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->names, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Expr_kind:
-        result = PyType_GenericNew(Expr_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Expr_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Expr.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Pass_kind:
-        result = PyType_GenericNew(Pass_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Pass_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         break;
     case Break_kind:
-        result = PyType_GenericNew(Break_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Break_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         break;
     case Continue_kind:
-        result = PyType_GenericNew(Continue_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Continue_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         break;
     }
     value = ast2obj_int(o->lineno);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_lineno, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->lineno, value) < 0)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->col_offset);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->col_offset, value) < 0)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->end_lineno);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->end_lineno, value) < 0)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->end_col_offset);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->end_col_offset, value)
+        < 0)
         goto failed;
     Py_DECREF(value);
     return result;
@@ -3233,223 +3956,249 @@ ast2obj_expr(void* _o)
 {
     expr_ty o = (expr_ty)_o;
     PyObject *result = NULL, *value = NULL;
+    PyTypeObject *tp;
     if (!o) {
         Py_RETURN_NONE;
     }
 
     switch (o->kind) {
     case BoolOp_kind:
-        result = PyType_GenericNew(BoolOp_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->BoolOp_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_boolop(o->v.BoolOp.op);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_op, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->op, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.BoolOp.values, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_values, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->values, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         break;
     case NamedExpr_kind:
-        result = PyType_GenericNew(NamedExpr_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->NamedExpr_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.NamedExpr.target);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_target, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->target, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.NamedExpr.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case BinOp_kind:
-        result = PyType_GenericNew(BinOp_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->BinOp_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.BinOp.left);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_left, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->left, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_operator(o->v.BinOp.op);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_op, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->op, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.BinOp.right);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_right, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->right, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case UnaryOp_kind:
-        result = PyType_GenericNew(UnaryOp_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->UnaryOp_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_unaryop(o->v.UnaryOp.op);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_op, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->op, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.UnaryOp.operand);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_operand, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->operand, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Lambda_kind:
-        result = PyType_GenericNew(Lambda_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Lambda_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_arguments(o->v.Lambda.args);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_args, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->args, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.Lambda.body);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case IfExp_kind:
-        result = PyType_GenericNew(IfExp_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->IfExp_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.IfExp.test);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_test, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->test, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.IfExp.body);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.IfExp.orelse);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->orelse, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Dict_kind:
-        result = PyType_GenericNew(Dict_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Dict_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.Dict.keys, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_keys, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->keys, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.Dict.values, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_values, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->values, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Set_kind:
-        result = PyType_GenericNew(Set_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Set_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.Set.elts, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_elts, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->elts, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case ListComp_kind:
-        result = PyType_GenericNew(ListComp_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->ListComp_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.ListComp.elt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_elt, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->elt, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.ListComp.generators, ast2obj_comprehension);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->generators, value)
+            == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case SetComp_kind:
-        result = PyType_GenericNew(SetComp_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->SetComp_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.SetComp.elt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_elt, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->elt, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.SetComp.generators, ast2obj_comprehension);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->generators, value)
+            == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case DictComp_kind:
-        result = PyType_GenericNew(DictComp_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->DictComp_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.DictComp.key);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_key, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->key, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.DictComp.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.DictComp.generators, ast2obj_comprehension);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->generators, value)
+            == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case GeneratorExp_kind:
-        result = PyType_GenericNew(GeneratorExp_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->GeneratorExp_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.GeneratorExp.elt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_elt, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->elt, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.GeneratorExp.generators,
                              ast2obj_comprehension);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->generators, value)
+            == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Await_kind:
-        result = PyType_GenericNew(Await_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Await_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Await.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Yield_kind:
-        result = PyType_GenericNew(Yield_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Yield_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Yield.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case YieldFrom_kind:
-        result = PyType_GenericNew(YieldFrom_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->YieldFrom_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.YieldFrom.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Compare_kind:
-        result = PyType_GenericNew(Compare_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Compare_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Compare.left);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_left, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->left, value) == -1)
             goto failed;
         Py_DECREF(value);
         {
@@ -3460,189 +4209,205 @@ ast2obj_expr(void* _o)
                 PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i)));
         }
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_ops, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->ops, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.Compare.comparators, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_comparators, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->comparators, value)
+            == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Call_kind:
-        result = PyType_GenericNew(Call_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Call_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Call.func);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_func, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->func, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.Call.args, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_args, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->args, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.Call.keywords, ast2obj_keyword);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_keywords, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->keywords, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         break;
     case FormattedValue_kind:
-        result = PyType_GenericNew(FormattedValue_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->FormattedValue_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.FormattedValue.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_int(o->v.FormattedValue.conversion);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_conversion, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->conversion, value)
+            == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.FormattedValue.format_spec);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_format_spec, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->format_spec, value)
+            == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case JoinedStr_kind:
-        result = PyType_GenericNew(JoinedStr_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->JoinedStr_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.JoinedStr.values, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_values, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->values, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Constant_kind:
-        result = PyType_GenericNew(Constant_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Constant_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_constant(o->v.Constant.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_string(o->v.Constant.kind);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_kind, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->kind, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Attribute_kind:
-        result = PyType_GenericNew(Attribute_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Attribute_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Attribute.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_identifier(o->v.Attribute.attr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_attr, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->attr, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr_context(o->v.Attribute.ctx);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->ctx, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Subscript_kind:
-        result = PyType_GenericNew(Subscript_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Subscript_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Subscript.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_slice(o->v.Subscript.slice);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_slice, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->slice, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr_context(o->v.Subscript.ctx);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->ctx, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Starred_kind:
-        result = PyType_GenericNew(Starred_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Starred_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Starred.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr_context(o->v.Starred.ctx);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->ctx, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Name_kind:
-        result = PyType_GenericNew(Name_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Name_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_identifier(o->v.Name.id);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_id, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->id, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr_context(o->v.Name.ctx);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->ctx, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case List_kind:
-        result = PyType_GenericNew(List_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->List_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.List.elts, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_elts, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->elts, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr_context(o->v.List.ctx);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->ctx, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Tuple_kind:
-        result = PyType_GenericNew(Tuple_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Tuple_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.Tuple.elts, ast2obj_expr);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_elts, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->elts, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr_context(o->v.Tuple.ctx);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->ctx, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     }
     value = ast2obj_int(o->lineno);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_lineno, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->lineno, value) < 0)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->col_offset);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->col_offset, value) < 0)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->end_lineno);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->end_lineno, value) < 0)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->end_col_offset);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->end_col_offset, value)
+        < 0)
         goto failed;
     Py_DECREF(value);
     return result;
@@ -3656,23 +4421,23 @@ PyObject* ast2obj_expr_context(expr_context_ty o)
 {
     switch(o) {
         case Load:
-            Py_INCREF(Load_singleton);
-            return Load_singleton;
+            Py_INCREF(astmodulestate_global->Load_singleton);
+            return astmodulestate_global->Load_singleton;
         case Store:
-            Py_INCREF(Store_singleton);
-            return Store_singleton;
+            Py_INCREF(astmodulestate_global->Store_singleton);
+            return astmodulestate_global->Store_singleton;
         case Del:
-            Py_INCREF(Del_singleton);
-            return Del_singleton;
+            Py_INCREF(astmodulestate_global->Del_singleton);
+            return astmodulestate_global->Del_singleton;
         case AugLoad:
-            Py_INCREF(AugLoad_singleton);
-            return AugLoad_singleton;
+            Py_INCREF(astmodulestate_global->AugLoad_singleton);
+            return astmodulestate_global->AugLoad_singleton;
         case AugStore:
-            Py_INCREF(AugStore_singleton);
-            return AugStore_singleton;
+            Py_INCREF(astmodulestate_global->AugStore_singleton);
+            return astmodulestate_global->AugStore_singleton;
         case Param:
-            Py_INCREF(Param_singleton);
-            return Param_singleton;
+            Py_INCREF(astmodulestate_global->Param_singleton);
+            return astmodulestate_global->Param_singleton;
         default:
             /* should never happen, but just in case ... */
             PyErr_Format(PyExc_SystemError, "unknown expr_context found");
@@ -3684,45 +4449,49 @@ ast2obj_slice(void* _o)
 {
     slice_ty o = (slice_ty)_o;
     PyObject *result = NULL, *value = NULL;
+    PyTypeObject *tp;
     if (!o) {
         Py_RETURN_NONE;
     }
 
     switch (o->kind) {
     case Slice_kind:
-        result = PyType_GenericNew(Slice_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Slice_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Slice.lower);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_lower, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->lower, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.Slice.upper);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_upper, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->upper, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_expr(o->v.Slice.step);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_step, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->step, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case ExtSlice_kind:
-        result = PyType_GenericNew(ExtSlice_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->ExtSlice_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_list(o->v.ExtSlice.dims, ast2obj_slice);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_dims, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->dims, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     case Index_kind:
-        result = PyType_GenericNew(Index_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->Index_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.Index.value);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
@@ -3738,11 +4507,11 @@ PyObject* ast2obj_boolop(boolop_ty o)
 {
     switch(o) {
         case And:
-            Py_INCREF(And_singleton);
-            return And_singleton;
+            Py_INCREF(astmodulestate_global->And_singleton);
+            return astmodulestate_global->And_singleton;
         case Or:
-            Py_INCREF(Or_singleton);
-            return Or_singleton;
+            Py_INCREF(astmodulestate_global->Or_singleton);
+            return astmodulestate_global->Or_singleton;
         default:
             /* should never happen, but just in case ... */
             PyErr_Format(PyExc_SystemError, "unknown boolop found");
@@ -3753,44 +4522,44 @@ PyObject* ast2obj_operator(operator_ty o)
 {
     switch(o) {
         case Add:
-            Py_INCREF(Add_singleton);
-            return Add_singleton;
+            Py_INCREF(astmodulestate_global->Add_singleton);
+            return astmodulestate_global->Add_singleton;
         case Sub:
-            Py_INCREF(Sub_singleton);
-            return Sub_singleton;
+            Py_INCREF(astmodulestate_global->Sub_singleton);
+            return astmodulestate_global->Sub_singleton;
         case Mult:
-            Py_INCREF(Mult_singleton);
-            return Mult_singleton;
+            Py_INCREF(astmodulestate_global->Mult_singleton);
+            return astmodulestate_global->Mult_singleton;
         case MatMult:
-            Py_INCREF(MatMult_singleton);
-            return MatMult_singleton;
+            Py_INCREF(astmodulestate_global->MatMult_singleton);
+            return astmodulestate_global->MatMult_singleton;
         case Div:
-            Py_INCREF(Div_singleton);
-            return Div_singleton;
+            Py_INCREF(astmodulestate_global->Div_singleton);
+            return astmodulestate_global->Div_singleton;
         case Mod:
-            Py_INCREF(Mod_singleton);
-            return Mod_singleton;
+            Py_INCREF(astmodulestate_global->Mod_singleton);
+            return astmodulestate_global->Mod_singleton;
         case Pow:
-            Py_INCREF(Pow_singleton);
-            return Pow_singleton;
+            Py_INCREF(astmodulestate_global->Pow_singleton);
+            return astmodulestate_global->Pow_singleton;
         case LShift:
-            Py_INCREF(LShift_singleton);
-            return LShift_singleton;
+            Py_INCREF(astmodulestate_global->LShift_singleton);
+            return astmodulestate_global->LShift_singleton;
         case RShift:
-            Py_INCREF(RShift_singleton);
-            return RShift_singleton;
+            Py_INCREF(astmodulestate_global->RShift_singleton);
+            return astmodulestate_global->RShift_singleton;
         case BitOr:
-            Py_INCREF(BitOr_singleton);
-            return BitOr_singleton;
+            Py_INCREF(astmodulestate_global->BitOr_singleton);
+            return astmodulestate_global->BitOr_singleton;
         case BitXor:
-            Py_INCREF(BitXor_singleton);
-            return BitXor_singleton;
+            Py_INCREF(astmodulestate_global->BitXor_singleton);
+            return astmodulestate_global->BitXor_singleton;
         case BitAnd:
-            Py_INCREF(BitAnd_singleton);
-            return BitAnd_singleton;
+            Py_INCREF(astmodulestate_global->BitAnd_singleton);
+            return astmodulestate_global->BitAnd_singleton;
         case FloorDiv:
-            Py_INCREF(FloorDiv_singleton);
-            return FloorDiv_singleton;
+            Py_INCREF(astmodulestate_global->FloorDiv_singleton);
+            return astmodulestate_global->FloorDiv_singleton;
         default:
             /* should never happen, but just in case ... */
             PyErr_Format(PyExc_SystemError, "unknown operator found");
@@ -3801,17 +4570,17 @@ PyObject* ast2obj_unaryop(unaryop_ty o)
 {
     switch(o) {
         case Invert:
-            Py_INCREF(Invert_singleton);
-            return Invert_singleton;
+            Py_INCREF(astmodulestate_global->Invert_singleton);
+            return astmodulestate_global->Invert_singleton;
         case Not:
-            Py_INCREF(Not_singleton);
-            return Not_singleton;
+            Py_INCREF(astmodulestate_global->Not_singleton);
+            return astmodulestate_global->Not_singleton;
         case UAdd:
-            Py_INCREF(UAdd_singleton);
-            return UAdd_singleton;
+            Py_INCREF(astmodulestate_global->UAdd_singleton);
+            return astmodulestate_global->UAdd_singleton;
         case USub:
-            Py_INCREF(USub_singleton);
-            return USub_singleton;
+            Py_INCREF(astmodulestate_global->USub_singleton);
+            return astmodulestate_global->USub_singleton;
         default:
             /* should never happen, but just in case ... */
             PyErr_Format(PyExc_SystemError, "unknown unaryop found");
@@ -3822,35 +4591,35 @@ PyObject* ast2obj_cmpop(cmpop_ty o)
 {
     switch(o) {
         case Eq:
-            Py_INCREF(Eq_singleton);
-            return Eq_singleton;
+            Py_INCREF(astmodulestate_global->Eq_singleton);
+            return astmodulestate_global->Eq_singleton;
         case NotEq:
-            Py_INCREF(NotEq_singleton);
-            return NotEq_singleton;
+            Py_INCREF(astmodulestate_global->NotEq_singleton);
+            return astmodulestate_global->NotEq_singleton;
         case Lt:
-            Py_INCREF(Lt_singleton);
-            return Lt_singleton;
+            Py_INCREF(astmodulestate_global->Lt_singleton);
+            return astmodulestate_global->Lt_singleton;
         case LtE:
-            Py_INCREF(LtE_singleton);
-            return LtE_singleton;
+            Py_INCREF(astmodulestate_global->LtE_singleton);
+            return astmodulestate_global->LtE_singleton;
         case Gt:
-            Py_INCREF(Gt_singleton);
-            return Gt_singleton;
+            Py_INCREF(astmodulestate_global->Gt_singleton);
+            return astmodulestate_global->Gt_singleton;
         case GtE:
-            Py_INCREF(GtE_singleton);
-            return GtE_singleton;
+            Py_INCREF(astmodulestate_global->GtE_singleton);
+            return astmodulestate_global->GtE_singleton;
         case Is:
-            Py_INCREF(Is_singleton);
-            return Is_singleton;
+            Py_INCREF(astmodulestate_global->Is_singleton);
+            return astmodulestate_global->Is_singleton;
         case IsNot:
-            Py_INCREF(IsNot_singleton);
-            return IsNot_singleton;
+            Py_INCREF(astmodulestate_global->IsNot_singleton);
+            return astmodulestate_global->IsNot_singleton;
         case In:
-            Py_INCREF(In_singleton);
-            return In_singleton;
+            Py_INCREF(astmodulestate_global->In_singleton);
+            return astmodulestate_global->In_singleton;
         case NotIn:
-            Py_INCREF(NotIn_singleton);
-            return NotIn_singleton;
+            Py_INCREF(astmodulestate_global->NotIn_singleton);
+            return astmodulestate_global->NotIn_singleton;
         default:
             /* should never happen, but just in case ... */
             PyErr_Format(PyExc_SystemError, "unknown cmpop found");
@@ -3862,30 +4631,32 @@ ast2obj_comprehension(void* _o)
 {
     comprehension_ty o = (comprehension_ty)_o;
     PyObject *result = NULL, *value = NULL;
+    PyTypeObject *tp;
     if (!o) {
         Py_RETURN_NONE;
     }
 
-    result = PyType_GenericNew(comprehension_type, NULL, NULL);
+    tp = (PyTypeObject *)astmodulestate_global->comprehension_type;
+    result = PyType_GenericNew(tp, NULL, NULL);
     if (!result) return NULL;
     value = ast2obj_expr(o->target);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_target, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->target, value) == -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_expr(o->iter);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_iter, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->iter, value) == -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_list(o->ifs, ast2obj_expr);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_ifs, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->ifs, value) == -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->is_async);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_is_async, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->is_async, value) == -1)
         goto failed;
     Py_DECREF(value);
     return result;
@@ -3900,49 +4671,52 @@ ast2obj_excepthandler(void* _o)
 {
     excepthandler_ty o = (excepthandler_ty)_o;
     PyObject *result = NULL, *value = NULL;
+    PyTypeObject *tp;
     if (!o) {
         Py_RETURN_NONE;
     }
 
     switch (o->kind) {
     case ExceptHandler_kind:
-        result = PyType_GenericNew(ExceptHandler_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->ExceptHandler_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_expr(o->v.ExceptHandler.type);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_type, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->type, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_identifier(o->v.ExceptHandler.name);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_name, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->name, value) == -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_list(o->v.ExceptHandler.body, ast2obj_stmt);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->body, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
     }
     value = ast2obj_int(o->lineno);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_lineno, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->lineno, value) < 0)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->col_offset);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->col_offset, value) < 0)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->end_lineno);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->end_lineno, value) < 0)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->end_col_offset);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->end_col_offset, value)
+        < 0)
         goto failed;
     Py_DECREF(value);
     return result;
@@ -3957,45 +4731,50 @@ ast2obj_arguments(void* _o)
 {
     arguments_ty o = (arguments_ty)_o;
     PyObject *result = NULL, *value = NULL;
+    PyTypeObject *tp;
     if (!o) {
         Py_RETURN_NONE;
     }
 
-    result = PyType_GenericNew(arguments_type, NULL, NULL);
+    tp = (PyTypeObject *)astmodulestate_global->arguments_type;
+    result = PyType_GenericNew(tp, NULL, NULL);
     if (!result) return NULL;
     value = ast2obj_list(o->posonlyargs, ast2obj_arg);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_posonlyargs, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->posonlyargs, value) ==
+        -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_list(o->args, ast2obj_arg);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_args, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->args, value) == -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_arg(o->vararg);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_vararg, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->vararg, value) == -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_list(o->kwonlyargs, ast2obj_arg);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_kwonlyargs, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->kwonlyargs, value) ==
+        -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_list(o->kw_defaults, ast2obj_expr);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_kw_defaults, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->kw_defaults, value) ==
+        -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_arg(o->kwarg);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_kwarg, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->kwarg, value) == -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_list(o->defaults, ast2obj_expr);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_defaults, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->defaults, value) == -1)
         goto failed;
     Py_DECREF(value);
     return result;
@@ -4010,45 +4789,50 @@ ast2obj_arg(void* _o)
 {
     arg_ty o = (arg_ty)_o;
     PyObject *result = NULL, *value = NULL;
+    PyTypeObject *tp;
     if (!o) {
         Py_RETURN_NONE;
     }
 
-    result = PyType_GenericNew(arg_type, NULL, NULL);
+    tp = (PyTypeObject *)astmodulestate_global->arg_type;
+    result = PyType_GenericNew(tp, NULL, NULL);
     if (!result) return NULL;
     value = ast2obj_identifier(o->arg);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_arg, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->arg, value) == -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_expr(o->annotation);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_annotation, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->annotation, value) ==
+        -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_string(o->type_comment);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->type_comment, value) ==
+        -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->lineno);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_lineno, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->lineno, value) < 0)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->col_offset);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->col_offset, value) < 0)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->end_lineno);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->end_lineno, value) < 0)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_int(o->end_col_offset);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0)
+    if (PyObject_SetAttr(result, astmodulestate_global->end_col_offset, value)
+        < 0)
         goto failed;
     Py_DECREF(value);
     return result;
@@ -4063,20 +4847,22 @@ ast2obj_keyword(void* _o)
 {
     keyword_ty o = (keyword_ty)_o;
     PyObject *result = NULL, *value = NULL;
+    PyTypeObject *tp;
     if (!o) {
         Py_RETURN_NONE;
     }
 
-    result = PyType_GenericNew(keyword_type, NULL, NULL);
+    tp = (PyTypeObject *)astmodulestate_global->keyword_type;
+    result = PyType_GenericNew(tp, NULL, NULL);
     if (!result) return NULL;
     value = ast2obj_identifier(o->arg);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_arg, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->arg, value) == -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_expr(o->value);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1)
         goto failed;
     Py_DECREF(value);
     return result;
@@ -4091,20 +4877,22 @@ ast2obj_alias(void* _o)
 {
     alias_ty o = (alias_ty)_o;
     PyObject *result = NULL, *value = NULL;
+    PyTypeObject *tp;
     if (!o) {
         Py_RETURN_NONE;
     }
 
-    result = PyType_GenericNew(alias_type, NULL, NULL);
+    tp = (PyTypeObject *)astmodulestate_global->alias_type;
+    result = PyType_GenericNew(tp, NULL, NULL);
     if (!result) return NULL;
     value = ast2obj_identifier(o->name);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_name, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->name, value) == -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_identifier(o->asname);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_asname, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->asname, value) == -1)
         goto failed;
     Py_DECREF(value);
     return result;
@@ -4119,20 +4907,24 @@ ast2obj_withitem(void* _o)
 {
     withitem_ty o = (withitem_ty)_o;
     PyObject *result = NULL, *value = NULL;
+    PyTypeObject *tp;
     if (!o) {
         Py_RETURN_NONE;
     }
 
-    result = PyType_GenericNew(withitem_type, NULL, NULL);
+    tp = (PyTypeObject *)astmodulestate_global->withitem_type;
+    result = PyType_GenericNew(tp, NULL, NULL);
     if (!result) return NULL;
     value = ast2obj_expr(o->context_expr);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_context_expr, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->context_expr, value) ==
+        -1)
         goto failed;
     Py_DECREF(value);
     value = ast2obj_expr(o->optional_vars);
     if (!value) goto failed;
-    if (_PyObject_SetAttrId(result, &PyId_optional_vars, value) == -1)
+    if (PyObject_SetAttr(result, astmodulestate_global->optional_vars, value)
+        == -1)
         goto failed;
     Py_DECREF(value);
     return result;
@@ -4147,22 +4939,25 @@ ast2obj_type_ignore(void* _o)
 {
     type_ignore_ty o = (type_ignore_ty)_o;
     PyObject *result = NULL, *value = NULL;
+    PyTypeObject *tp;
     if (!o) {
         Py_RETURN_NONE;
     }
 
     switch (o->kind) {
     case TypeIgnore_kind:
-        result = PyType_GenericNew(TypeIgnore_type, NULL, NULL);
+        tp = (PyTypeObject *)astmodulestate_global->TypeIgnore_type;
+        result = PyType_GenericNew(tp, NULL, NULL);
         if (!result) goto failed;
         value = ast2obj_int(o->v.TypeIgnore.lineno);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_lineno, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->lineno, value) ==
+            -1)
             goto failed;
         Py_DECREF(value);
         value = ast2obj_string(o->v.TypeIgnore.tag);
         if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_tag, value) == -1)
+        if (PyObject_SetAttr(result, astmodulestate_global->tag, value) == -1)
             goto failed;
         Py_DECREF(value);
         break;
@@ -4181,12 +4976,14 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
     int isinstance;
 
     PyObject *tmp = NULL;
+    PyObject *tp;
 
     if (obj == Py_None) {
         *out = NULL;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Module_type);
+    tp = astmodulestate_global->Module_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -4194,7 +4991,7 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
         asdl_seq* body;
         asdl_seq* type_ignores;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4206,7 +5003,7 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Module field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Module field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4227,7 +5024,8 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_type_ignores, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->type_ignores,
+            &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4239,7 +5037,7 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Module field \"type_ignores\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Module field \"type_ignores\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4264,14 +5062,15 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Interactive_type);
+    tp = astmodulestate_global->Interactive_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         asdl_seq* body;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4283,7 +5082,7 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Interactive field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Interactive field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4308,14 +5107,15 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Expression_type);
+    tp = astmodulestate_global->Expression_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         expr_ty body;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4332,7 +5132,8 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)FunctionType_type);
+    tp = astmodulestate_global->FunctionType_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -4340,7 +5141,8 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
         asdl_seq* argtypes;
         expr_ty returns;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_argtypes, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->argtypes, &tmp) <
+            0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4352,7 +5154,7 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "FunctionType field \"argtypes\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "FunctionType field \"argtypes\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4373,7 +5175,8 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_returns, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->returns, &tmp) <
+            0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4390,14 +5193,15 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Suite_type);
+    tp = astmodulestate_global->Suite_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         asdl_seq* body;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4409,7 +5213,7 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Suite field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Suite field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4447,6 +5251,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
     int isinstance;
 
     PyObject *tmp = NULL;
+    PyObject *tp;
     int lineno;
     int col_offset;
     int end_lineno;
@@ -4456,7 +5261,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         *out = NULL;
         return 0;
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_lineno, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->lineno, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -4469,7 +5274,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_col_offset, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->col_offset, &tmp) < 0)
+        {
         return 1;
     }
     if (tmp == NULL) {
@@ -4482,7 +5288,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->end_lineno, &tmp) < 0)
+        {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -4495,7 +5302,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->end_col_offset, &tmp)
+        < 0) {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -4508,7 +5316,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)FunctionDef_type);
+    tp = astmodulestate_global->FunctionDef_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -4520,7 +5329,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         expr_ty returns;
         string type_comment;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_name, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->name, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4533,7 +5342,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_args, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->args, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4546,7 +5355,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4558,7 +5367,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "FunctionDef field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "FunctionDef field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4579,7 +5388,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_decorator_list, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->decorator_list,
+            &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4591,7 +5401,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "FunctionDef field \"decorator_list\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "FunctionDef field \"decorator_list\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4612,7 +5422,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_returns, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->returns, &tmp) <
+            0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -4625,7 +5436,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->type_comment,
+            &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -4644,7 +5456,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)AsyncFunctionDef_type);
+    tp = astmodulestate_global->AsyncFunctionDef_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -4656,7 +5469,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         expr_ty returns;
         string type_comment;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_name, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->name, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4669,7 +5482,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_args, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->args, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4682,7 +5495,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4694,7 +5507,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4715,7 +5528,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_decorator_list, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->decorator_list,
+            &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4727,7 +5541,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"decorator_list\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"decorator_list\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4748,7 +5562,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_returns, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->returns, &tmp) <
+            0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -4761,7 +5576,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->type_comment,
+            &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -4780,7 +5596,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)ClassDef_type);
+    tp = astmodulestate_global->ClassDef_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -4791,7 +5608,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* body;
         asdl_seq* decorator_list;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_name, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->name, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4804,7 +5621,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_bases, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->bases, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4816,7 +5633,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "ClassDef field \"bases\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "ClassDef field \"bases\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4837,7 +5654,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_keywords, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->keywords, &tmp) <
+            0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4849,7 +5667,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "ClassDef field \"keywords\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "ClassDef field \"keywords\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4870,7 +5688,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4882,7 +5700,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "ClassDef field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "ClassDef field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4903,7 +5721,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_decorator_list, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->decorator_list,
+            &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4915,7 +5734,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "ClassDef field \"decorator_list\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "ClassDef field \"decorator_list\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -4941,14 +5760,15 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Return_type);
+    tp = astmodulestate_global->Return_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         expr_ty value;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -4966,14 +5786,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Delete_type);
+    tp = astmodulestate_global->Delete_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         asdl_seq* targets;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_targets, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->targets, &tmp) <
+            0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -4985,7 +5807,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Delete field \"targets\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Delete field \"targets\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5011,7 +5833,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Assign_type);
+    tp = astmodulestate_global->Assign_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -5020,7 +5843,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         expr_ty value;
         string type_comment;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_targets, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->targets, &tmp) <
+            0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5032,7 +5856,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Assign field \"targets\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Assign field \"targets\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5053,7 +5877,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5066,7 +5890,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->type_comment,
+            &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -5084,7 +5909,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)AugAssign_type);
+    tp = astmodulestate_global->AugAssign_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -5093,7 +5919,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         operator_ty op;
         expr_ty value;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_target, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->target, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -5106,7 +5933,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_op, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->op, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5119,7 +5946,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5137,7 +5964,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)AnnAssign_type);
+    tp = astmodulestate_global->AnnAssign_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -5147,7 +5975,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         expr_ty value;
         int simple;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_target, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->target, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -5160,7 +5989,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_annotation, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->annotation, &tmp)
+            < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5173,7 +6003,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -5186,7 +6016,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_simple, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->simple, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -5204,7 +6035,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)For_type);
+    tp = astmodulestate_global->For_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -5215,7 +6047,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* orelse;
         string type_comment;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_target, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->target, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -5228,7 +6061,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_iter, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->iter, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5241,7 +6074,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5253,7 +6086,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "For field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "For field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5274,7 +6107,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_orelse, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->orelse, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -5286,7 +6120,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "For field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "For field \"orelse\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5307,7 +6141,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->type_comment,
+            &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -5325,7 +6160,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)AsyncFor_type);
+    tp = astmodulestate_global->AsyncFor_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -5336,7 +6172,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* orelse;
         string type_comment;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_target, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->target, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -5349,7 +6186,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_iter, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->iter, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5362,7 +6199,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5374,7 +6211,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "AsyncFor field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "AsyncFor field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5395,7 +6232,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_orelse, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->orelse, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -5407,7 +6245,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "AsyncFor field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "AsyncFor field \"orelse\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5428,7 +6266,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->type_comment,
+            &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -5446,7 +6285,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)While_type);
+    tp = astmodulestate_global->While_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -5455,7 +6295,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* body;
         asdl_seq* orelse;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_test, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->test, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5468,7 +6308,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5480,7 +6320,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "While field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "While field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5501,7 +6341,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_orelse, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->orelse, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -5513,7 +6354,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "While field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "While field \"orelse\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5539,7 +6380,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)If_type);
+    tp = astmodulestate_global->If_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -5548,7 +6390,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* body;
         asdl_seq* orelse;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_test, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->test, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5561,7 +6403,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5573,7 +6415,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "If field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "If field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5594,7 +6436,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_orelse, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->orelse, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -5606,7 +6449,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "If field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "If field \"orelse\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5632,7 +6475,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)With_type);
+    tp = astmodulestate_global->With_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -5641,7 +6485,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* body;
         string type_comment;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_items, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->items, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5653,7 +6497,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "With field \"items\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "With field \"items\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5674,7 +6518,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5686,7 +6530,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "With field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "With field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5707,7 +6551,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->type_comment,
+            &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -5725,7 +6570,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)AsyncWith_type);
+    tp = astmodulestate_global->AsyncWith_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -5734,7 +6580,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* body;
         string type_comment;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_items, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->items, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5746,7 +6592,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "AsyncWith field \"items\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "AsyncWith field \"items\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5767,7 +6613,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5779,7 +6625,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "AsyncWith field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "AsyncWith field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5800,7 +6646,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->type_comment,
+            &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -5818,7 +6665,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Raise_type);
+    tp = astmodulestate_global->Raise_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -5826,7 +6674,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         expr_ty exc;
         expr_ty cause;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_exc, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->exc, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -5839,7 +6687,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_cause, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->cause, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -5857,7 +6705,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Try_type);
+    tp = astmodulestate_global->Try_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -5867,7 +6716,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* orelse;
         asdl_seq* finalbody;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5879,7 +6728,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Try field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Try field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5900,7 +6749,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_handlers, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->handlers, &tmp) <
+            0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5912,7 +6762,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Try field \"handlers\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Try field \"handlers\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5933,7 +6783,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_orelse, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->orelse, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -5945,7 +6796,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Try field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Try field \"orelse\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -5966,7 +6817,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_finalbody, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->finalbody, &tmp) <
+            0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -5978,7 +6830,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Try field \"finalbody\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Try field \"finalbody\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -6004,7 +6856,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Assert_type);
+    tp = astmodulestate_global->Assert_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6012,7 +6865,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         expr_ty test;
         expr_ty msg;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_test, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->test, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6025,7 +6878,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_msg, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->msg, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -6043,14 +6896,15 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Import_type);
+    tp = astmodulestate_global->Import_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         asdl_seq* names;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_names, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->names, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6062,7 +6916,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Import field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Import field \"names\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -6088,7 +6942,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)ImportFrom_type);
+    tp = astmodulestate_global->ImportFrom_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6097,7 +6952,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* names;
         int level;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_module, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->module, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -6110,7 +6966,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_names, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->names, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6122,7 +6978,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "ImportFrom field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "ImportFrom field \"names\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -6143,7 +6999,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_level, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->level, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -6161,14 +7017,15 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Global_type);
+    tp = astmodulestate_global->Global_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         asdl_seq* names;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_names, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->names, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6180,7 +7037,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Global field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Global field \"names\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -6206,14 +7063,15 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Nonlocal_type);
+    tp = astmodulestate_global->Nonlocal_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         asdl_seq* names;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_names, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->names, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6225,7 +7083,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Nonlocal field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Nonlocal field \"names\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -6251,14 +7109,15 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Expr_type);
+    tp = astmodulestate_global->Expr_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         expr_ty value;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6276,7 +7135,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Pass_type);
+    tp = astmodulestate_global->Pass_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6286,7 +7146,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Break_type);
+    tp = astmodulestate_global->Break_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6296,7 +7157,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Continue_type);
+    tp = astmodulestate_global->Continue_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6319,6 +7181,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
     int isinstance;
 
     PyObject *tmp = NULL;
+    PyObject *tp;
     int lineno;
     int col_offset;
     int end_lineno;
@@ -6328,7 +7191,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         *out = NULL;
         return 0;
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_lineno, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->lineno, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -6341,7 +7204,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_col_offset, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->col_offset, &tmp) < 0)
+        {
         return 1;
     }
     if (tmp == NULL) {
@@ -6354,7 +7218,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->end_lineno, &tmp) < 0)
+        {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -6367,7 +7232,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->end_col_offset, &tmp)
+        < 0) {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -6380,7 +7246,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)BoolOp_type);
+    tp = astmodulestate_global->BoolOp_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6388,7 +7255,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         boolop_ty op;
         asdl_seq* values;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_op, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->op, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6401,7 +7268,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_values, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->values, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -6413,7 +7281,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "BoolOp field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "BoolOp field \"values\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -6439,7 +7307,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)NamedExpr_type);
+    tp = astmodulestate_global->NamedExpr_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6447,7 +7316,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         expr_ty target;
         expr_ty value;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_target, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->target, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -6460,7 +7330,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6478,7 +7348,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)BinOp_type);
+    tp = astmodulestate_global->BinOp_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6487,7 +7358,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         operator_ty op;
         expr_ty right;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_left, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->left, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6500,7 +7371,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_op, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->op, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6513,7 +7384,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_right, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->right, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6531,7 +7402,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)UnaryOp_type);
+    tp = astmodulestate_global->UnaryOp_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6539,7 +7411,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         unaryop_ty op;
         expr_ty operand;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_op, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->op, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6552,7 +7424,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_operand, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->operand, &tmp) <
+            0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6570,7 +7443,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Lambda_type);
+    tp = astmodulestate_global->Lambda_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6578,7 +7452,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         arguments_ty args;
         expr_ty body;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_args, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->args, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6591,7 +7465,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6609,7 +7483,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)IfExp_type);
+    tp = astmodulestate_global->IfExp_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6618,7 +7493,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         expr_ty body;
         expr_ty orelse;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_test, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->test, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6631,7 +7506,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6644,7 +7519,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_orelse, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->orelse, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -6662,7 +7538,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Dict_type);
+    tp = astmodulestate_global->Dict_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6670,7 +7547,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         asdl_seq* keys;
         asdl_seq* values;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_keys, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->keys, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6682,7 +7559,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Dict field \"keys\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Dict field \"keys\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -6703,7 +7580,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_values, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->values, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -6715,7 +7593,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Dict field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Dict field \"values\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -6741,14 +7619,15 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Set_type);
+    tp = astmodulestate_global->Set_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         asdl_seq* elts;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_elts, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->elts, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6760,7 +7639,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Set field \"elts\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Set field \"elts\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -6785,7 +7664,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)ListComp_type);
+    tp = astmodulestate_global->ListComp_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6793,7 +7673,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         expr_ty elt;
         asdl_seq* generators;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_elt, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->elt, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6806,7 +7686,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_generators, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->generators, &tmp)
+            < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6818,7 +7699,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "ListComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "ListComp field \"generators\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -6844,7 +7725,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)SetComp_type);
+    tp = astmodulestate_global->SetComp_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6852,7 +7734,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         expr_ty elt;
         asdl_seq* generators;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_elt, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->elt, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6865,7 +7747,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_generators, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->generators, &tmp)
+            < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6877,7 +7760,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "SetComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "SetComp field \"generators\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -6903,7 +7786,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)DictComp_type);
+    tp = astmodulestate_global->DictComp_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6912,7 +7796,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         expr_ty value;
         asdl_seq* generators;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_key, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->key, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6925,7 +7809,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6938,7 +7822,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_generators, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->generators, &tmp)
+            < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6950,7 +7835,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "DictComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "DictComp field \"generators\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -6976,7 +7861,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)GeneratorExp_type);
+    tp = astmodulestate_global->GeneratorExp_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -6984,7 +7870,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         expr_ty elt;
         asdl_seq* generators;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_elt, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->elt, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -6997,7 +7883,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_generators, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->generators, &tmp)
+            < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7009,7 +7896,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "GeneratorExp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "GeneratorExp field \"generators\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -7035,14 +7922,15 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Await_type);
+    tp = astmodulestate_global->Await_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         expr_ty value;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7060,14 +7948,15 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Yield_type);
+    tp = astmodulestate_global->Yield_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         expr_ty value;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -7085,14 +7974,15 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)YieldFrom_type);
+    tp = astmodulestate_global->YieldFrom_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         expr_ty value;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7110,7 +8000,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Compare_type);
+    tp = astmodulestate_global->Compare_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -7119,7 +8010,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         asdl_int_seq* ops;
         asdl_seq* comparators;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_left, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->left, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7132,7 +8023,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_ops, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->ops, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7144,7 +8035,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Compare field \"ops\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Compare field \"ops\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -7165,7 +8056,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_comparators, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->comparators, &tmp)
+            < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7177,7 +8069,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Compare field \"comparators\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Compare field \"comparators\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -7203,7 +8095,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Call_type);
+    tp = astmodulestate_global->Call_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -7212,7 +8105,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         asdl_seq* args;
         asdl_seq* keywords;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_func, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->func, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7225,7 +8118,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_args, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->args, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7237,7 +8130,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Call field \"args\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Call field \"args\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -7258,7 +8151,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_keywords, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->keywords, &tmp) <
+            0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7270,7 +8164,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Call field \"keywords\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Call field \"keywords\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -7296,7 +8190,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)FormattedValue_type);
+    tp = astmodulestate_global->FormattedValue_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -7305,7 +8200,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         int conversion;
         expr_ty format_spec;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7318,7 +8213,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_conversion, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->conversion, &tmp)
+            < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -7331,7 +8227,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_format_spec, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->format_spec, &tmp)
+            < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -7349,14 +8246,16 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)JoinedStr_type);
+    tp = astmodulestate_global->JoinedStr_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         asdl_seq* values;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_values, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->values, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -7368,7 +8267,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "JoinedStr field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "JoinedStr field \"values\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -7394,7 +8293,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Constant_type);
+    tp = astmodulestate_global->Constant_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -7402,7 +8302,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         constant value;
         string kind;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7415,7 +8315,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_kind, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->kind, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -7433,7 +8333,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Attribute_type);
+    tp = astmodulestate_global->Attribute_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -7442,7 +8343,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         identifier attr;
         expr_context_ty ctx;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7455,7 +8356,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_attr, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->attr, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7468,7 +8369,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_ctx, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->ctx, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7486,7 +8387,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Subscript_type);
+    tp = astmodulestate_global->Subscript_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -7495,7 +8397,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         slice_ty slice;
         expr_context_ty ctx;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7508,7 +8410,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_slice, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->slice, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7521,7 +8423,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_ctx, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->ctx, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7539,7 +8441,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Starred_type);
+    tp = astmodulestate_global->Starred_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -7547,7 +8450,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         expr_ty value;
         expr_context_ty ctx;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7560,7 +8463,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_ctx, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->ctx, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7578,7 +8481,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Name_type);
+    tp = astmodulestate_global->Name_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -7586,7 +8490,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         identifier id;
         expr_context_ty ctx;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_id, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->id, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7599,7 +8503,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_ctx, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->ctx, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7617,7 +8521,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)List_type);
+    tp = astmodulestate_global->List_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -7625,7 +8530,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         asdl_seq* elts;
         expr_context_ty ctx;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_elts, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->elts, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7637,7 +8542,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "List field \"elts\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "List field \"elts\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -7658,7 +8563,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_ctx, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->ctx, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7676,7 +8581,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Tuple_type);
+    tp = astmodulestate_global->Tuple_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -7684,7 +8590,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         asdl_seq* elts;
         expr_context_ty ctx;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_elts, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->elts, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7696,7 +8602,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "Tuple field \"elts\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "Tuple field \"elts\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -7717,7 +8623,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_ctx, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->ctx, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7747,7 +8653,7 @@ obj2ast_expr_context(PyObject* obj, expr_context_ty* out, PyArena* arena)
 {
     int isinstance;
 
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Load_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Load_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -7755,7 +8661,7 @@ obj2ast_expr_context(PyObject* obj, expr_context_ty* out, PyArena* arena)
         *out = Load;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Store_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Store_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -7763,7 +8669,7 @@ obj2ast_expr_context(PyObject* obj, expr_context_ty* out, PyArena* arena)
         *out = Store;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Del_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Del_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -7771,7 +8677,7 @@ obj2ast_expr_context(PyObject* obj, expr_context_ty* out, PyArena* arena)
         *out = Del;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)AugLoad_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->AugLoad_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -7779,7 +8685,7 @@ obj2ast_expr_context(PyObject* obj, expr_context_ty* out, PyArena* arena)
         *out = AugLoad;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)AugStore_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->AugStore_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -7787,7 +8693,7 @@ obj2ast_expr_context(PyObject* obj, expr_context_ty* out, PyArena* arena)
         *out = AugStore;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Param_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Param_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -7806,12 +8712,14 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena)
     int isinstance;
 
     PyObject *tmp = NULL;
+    PyObject *tp;
 
     if (obj == Py_None) {
         *out = NULL;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Slice_type);
+    tp = astmodulestate_global->Slice_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -7820,7 +8728,7 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena)
         expr_ty upper;
         expr_ty step;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_lower, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->lower, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -7833,7 +8741,7 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_upper, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->upper, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -7846,7 +8754,7 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_step, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->step, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -7863,14 +8771,15 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)ExtSlice_type);
+    tp = astmodulestate_global->ExtSlice_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         asdl_seq* dims;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_dims, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->dims, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7882,7 +8791,7 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "ExtSlice field \"dims\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "ExtSlice field \"dims\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -7907,14 +8816,15 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)Index_type);
+    tp = astmodulestate_global->Index_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
     if (isinstance) {
         expr_ty value;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -7943,7 +8853,7 @@ obj2ast_boolop(PyObject* obj, boolop_ty* out, PyArena* arena)
 {
     int isinstance;
 
-    isinstance = PyObject_IsInstance(obj, (PyObject *)And_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->And_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -7951,7 +8861,7 @@ obj2ast_boolop(PyObject* obj, boolop_ty* out, PyArena* arena)
         *out = And;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Or_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Or_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -7969,7 +8879,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
 {
     int isinstance;
 
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Add_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Add_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -7977,7 +8887,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
         *out = Add;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Sub_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Sub_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -7985,7 +8895,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
         *out = Sub;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Mult_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Mult_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -7993,7 +8903,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
         *out = Mult;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)MatMult_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->MatMult_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8001,7 +8911,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
         *out = MatMult;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Div_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Div_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8009,7 +8919,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
         *out = Div;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Mod_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Mod_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8017,7 +8927,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
         *out = Mod;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Pow_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Pow_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8025,7 +8935,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
         *out = Pow;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)LShift_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->LShift_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8033,7 +8943,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
         *out = LShift;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)RShift_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->RShift_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8041,7 +8951,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
         *out = RShift;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)BitOr_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->BitOr_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8049,7 +8959,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
         *out = BitOr;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)BitXor_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->BitXor_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8057,7 +8967,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
         *out = BitXor;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)BitAnd_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->BitAnd_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8065,7 +8975,7 @@ obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
         *out = BitAnd;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)FloorDiv_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->FloorDiv_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8083,7 +8993,7 @@ obj2ast_unaryop(PyObject* obj, unaryop_ty* out, PyArena* arena)
 {
     int isinstance;
 
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Invert_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Invert_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8091,7 +9001,7 @@ obj2ast_unaryop(PyObject* obj, unaryop_ty* out, PyArena* arena)
         *out = Invert;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Not_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Not_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8099,7 +9009,7 @@ obj2ast_unaryop(PyObject* obj, unaryop_ty* out, PyArena* arena)
         *out = Not;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)UAdd_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->UAdd_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8107,7 +9017,7 @@ obj2ast_unaryop(PyObject* obj, unaryop_ty* out, PyArena* arena)
         *out = UAdd;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)USub_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->USub_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8125,7 +9035,7 @@ obj2ast_cmpop(PyObject* obj, cmpop_ty* out, PyArena* arena)
 {
     int isinstance;
 
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Eq_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Eq_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8133,7 +9043,7 @@ obj2ast_cmpop(PyObject* obj, cmpop_ty* out, PyArena* arena)
         *out = Eq;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)NotEq_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->NotEq_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8141,7 +9051,7 @@ obj2ast_cmpop(PyObject* obj, cmpop_ty* out, PyArena* arena)
         *out = NotEq;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Lt_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Lt_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8149,7 +9059,7 @@ obj2ast_cmpop(PyObject* obj, cmpop_ty* out, PyArena* arena)
         *out = Lt;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)LtE_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->LtE_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8157,7 +9067,7 @@ obj2ast_cmpop(PyObject* obj, cmpop_ty* out, PyArena* arena)
         *out = LtE;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Gt_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Gt_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8165,7 +9075,7 @@ obj2ast_cmpop(PyObject* obj, cmpop_ty* out, PyArena* arena)
         *out = Gt;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)GtE_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->GtE_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8173,7 +9083,7 @@ obj2ast_cmpop(PyObject* obj, cmpop_ty* out, PyArena* arena)
         *out = GtE;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)Is_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->Is_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8181,7 +9091,7 @@ obj2ast_cmpop(PyObject* obj, cmpop_ty* out, PyArena* arena)
         *out = Is;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)IsNot_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->IsNot_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8189,7 +9099,7 @@ obj2ast_cmpop(PyObject* obj, cmpop_ty* out, PyArena* arena)
         *out = IsNot;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)In_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->In_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8197,7 +9107,7 @@ obj2ast_cmpop(PyObject* obj, cmpop_ty* out, PyArena* arena)
         *out = In;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject *)NotIn_type);
+    isinstance = PyObject_IsInstance(obj, astmodulestate_global->NotIn_type);
     if (isinstance == -1) {
         return 1;
     }
@@ -8219,7 +9129,7 @@ obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena)
     asdl_seq* ifs;
     int is_async;
 
-    if (_PyObject_LookupAttrId(obj, &PyId_target, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->target, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8232,7 +9142,7 @@ obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_iter, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->iter, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8245,7 +9155,7 @@ obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_ifs, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->ifs, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8257,7 +9167,7 @@ obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena)
         Py_ssize_t len;
         Py_ssize_t i;
         if (!PyList_Check(tmp)) {
-            PyErr_Format(PyExc_TypeError, "comprehension field \"ifs\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+            PyErr_Format(PyExc_TypeError, "comprehension field \"ifs\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
             goto failed;
         }
         len = PyList_GET_SIZE(tmp);
@@ -8278,7 +9188,7 @@ obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena)
         }
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_is_async, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->is_async, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8304,6 +9214,7 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
     int isinstance;
 
     PyObject *tmp = NULL;
+    PyObject *tp;
     int lineno;
     int col_offset;
     int end_lineno;
@@ -8313,7 +9224,7 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
         *out = NULL;
         return 0;
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_lineno, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->lineno, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8326,7 +9237,8 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_col_offset, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->col_offset, &tmp) < 0)
+        {
         return 1;
     }
     if (tmp == NULL) {
@@ -8339,7 +9251,8 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->end_lineno, &tmp) < 0)
+        {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -8352,7 +9265,8 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->end_col_offset, &tmp)
+        < 0) {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -8365,7 +9279,8 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)ExceptHandler_type);
+    tp = astmodulestate_global->ExceptHandler_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -8374,7 +9289,7 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
         identifier name;
         asdl_seq* body;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_type, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->type, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -8387,7 +9302,7 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_name, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->name, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL || tmp == Py_None) {
@@ -8400,7 +9315,7 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->body, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -8412,7 +9327,7 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
             Py_ssize_t len;
             Py_ssize_t i;
             if (!PyList_Check(tmp)) {
-                PyErr_Format(PyExc_TypeError, "ExceptHandler field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                PyErr_Format(PyExc_TypeError, "ExceptHandler field \"body\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
                 goto failed;
             }
             len = PyList_GET_SIZE(tmp);
@@ -8457,7 +9372,8 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
     arg_ty kwarg;
     asdl_seq* defaults;
 
-    if (_PyObject_LookupAttrId(obj, &PyId_posonlyargs, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->posonlyargs, &tmp) <
+        0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8469,7 +9385,7 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         Py_ssize_t len;
         Py_ssize_t i;
         if (!PyList_Check(tmp)) {
-            PyErr_Format(PyExc_TypeError, "arguments field \"posonlyargs\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+            PyErr_Format(PyExc_TypeError, "arguments field \"posonlyargs\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
             goto failed;
         }
         len = PyList_GET_SIZE(tmp);
@@ -8490,7 +9406,7 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         }
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_args, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->args, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8502,7 +9418,7 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         Py_ssize_t len;
         Py_ssize_t i;
         if (!PyList_Check(tmp)) {
-            PyErr_Format(PyExc_TypeError, "arguments field \"args\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+            PyErr_Format(PyExc_TypeError, "arguments field \"args\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
             goto failed;
         }
         len = PyList_GET_SIZE(tmp);
@@ -8523,7 +9439,7 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         }
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_vararg, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->vararg, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -8536,7 +9452,8 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_kwonlyargs, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->kwonlyargs, &tmp) < 0)
+        {
         return 1;
     }
     if (tmp == NULL) {
@@ -8548,7 +9465,7 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         Py_ssize_t len;
         Py_ssize_t i;
         if (!PyList_Check(tmp)) {
-            PyErr_Format(PyExc_TypeError, "arguments field \"kwonlyargs\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+            PyErr_Format(PyExc_TypeError, "arguments field \"kwonlyargs\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
             goto failed;
         }
         len = PyList_GET_SIZE(tmp);
@@ -8569,7 +9486,8 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         }
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_kw_defaults, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->kw_defaults, &tmp) <
+        0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8581,7 +9499,7 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         Py_ssize_t len;
         Py_ssize_t i;
         if (!PyList_Check(tmp)) {
-            PyErr_Format(PyExc_TypeError, "arguments field \"kw_defaults\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+            PyErr_Format(PyExc_TypeError, "arguments field \"kw_defaults\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
             goto failed;
         }
         len = PyList_GET_SIZE(tmp);
@@ -8602,7 +9520,7 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         }
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_kwarg, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->kwarg, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -8615,7 +9533,7 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_defaults, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->defaults, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8627,7 +9545,7 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         Py_ssize_t len;
         Py_ssize_t i;
         if (!PyList_Check(tmp)) {
-            PyErr_Format(PyExc_TypeError, "arguments field \"defaults\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+            PyErr_Format(PyExc_TypeError, "arguments field \"defaults\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp)));
             goto failed;
         }
         len = PyList_GET_SIZE(tmp);
@@ -8668,7 +9586,7 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
     int end_lineno;
     int end_col_offset;
 
-    if (_PyObject_LookupAttrId(obj, &PyId_arg, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->arg, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8681,7 +9599,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_annotation, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->annotation, &tmp) < 0)
+        {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -8694,7 +9613,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->type_comment, &tmp) <
+        0) {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -8707,7 +9627,7 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_lineno, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->lineno, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8720,7 +9640,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_col_offset, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->col_offset, &tmp) < 0)
+        {
         return 1;
     }
     if (tmp == NULL) {
@@ -8733,7 +9654,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->end_lineno, &tmp) < 0)
+        {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -8746,7 +9668,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->end_col_offset, &tmp)
+        < 0) {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -8774,7 +9697,7 @@ obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena)
     identifier arg;
     expr_ty value;
 
-    if (_PyObject_LookupAttrId(obj, &PyId_arg, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->arg, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -8787,7 +9710,7 @@ obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8814,7 +9737,7 @@ obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena)
     identifier name;
     identifier asname;
 
-    if (_PyObject_LookupAttrId(obj, &PyId_name, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->name, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8827,7 +9750,7 @@ obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_asname, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->asname, &tmp) < 0) {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -8854,7 +9777,8 @@ obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena)
     expr_ty context_expr;
     expr_ty optional_vars;
 
-    if (_PyObject_LookupAttrId(obj, &PyId_context_expr, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->context_expr, &tmp) <
+        0) {
         return 1;
     }
     if (tmp == NULL) {
@@ -8867,7 +9791,8 @@ obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    if (_PyObject_LookupAttrId(obj, &PyId_optional_vars, &tmp) < 0) {
+    if (_PyObject_LookupAttr(obj, astmodulestate_global->optional_vars, &tmp) <
+        0) {
         return 1;
     }
     if (tmp == NULL || tmp == Py_None) {
@@ -8893,12 +9818,14 @@ obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* arena)
     int isinstance;
 
     PyObject *tmp = NULL;
+    PyObject *tp;
 
     if (obj == Py_None) {
         *out = NULL;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)TypeIgnore_type);
+    tp = astmodulestate_global->TypeIgnore_type;
+    isinstance = PyObject_IsInstance(obj, tp);
     if (isinstance == -1) {
         return 1;
     }
@@ -8906,7 +9833,8 @@ obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* arena)
         int lineno;
         string tag;
 
-        if (_PyObject_LookupAttrId(obj, &PyId_lineno, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->lineno, &tmp) < 0)
+            {
             return 1;
         }
         if (tmp == NULL) {
@@ -8919,7 +9847,7 @@ obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        if (_PyObject_LookupAttrId(obj, &PyId_tag, &tmp) < 0) {
+        if (_PyObject_LookupAttr(obj, astmodulestate_global->tag, &tmp) < 0) {
             return 1;
         }
         if (tmp == NULL) {
@@ -8944,215 +9872,361 @@ obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* arena)
 }
 
 
-static struct PyModuleDef _astmodule = {
-  PyModuleDef_HEAD_INIT, "_ast"
-};
 PyMODINIT_FUNC
 PyInit__ast(void)
 {
-    PyObject *m, *d;
+    PyObject *m;
     if (!init_types()) return NULL;
-    m = PyModule_Create(&_astmodule);
+    m = PyState_FindModule(&_astmodule);
     if (!m) return NULL;
-    d = PyModule_GetDict(m);
-    if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->AST_type);
+    if (PyModule_AddObject(m, "AST", astmodulestate_global->AST_type) < 0)
+        return NULL;
     if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0)
         return NULL;
     if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)
         return NULL;
     if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Module", (PyObject*)Module_type) < 0) return
+    if (PyModule_AddObject(m, "mod", astmodulestate_global->mod_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->mod_type);
+    if (PyModule_AddObject(m, "Module", astmodulestate_global->Module_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->Module_type);
+    if (PyModule_AddObject(m, "Interactive",
+        astmodulestate_global->Interactive_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->Interactive_type);
+    if (PyModule_AddObject(m, "Expression",
+        astmodulestate_global->Expression_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->Expression_type);
+    if (PyModule_AddObject(m, "FunctionType",
+        astmodulestate_global->FunctionType_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->FunctionType_type);
+    if (PyModule_AddObject(m, "Suite", astmodulestate_global->Suite_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->Suite_type);
+    if (PyModule_AddObject(m, "stmt", astmodulestate_global->stmt_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->stmt_type);
+    if (PyModule_AddObject(m, "FunctionDef",
+        astmodulestate_global->FunctionDef_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->FunctionDef_type);
+    if (PyModule_AddObject(m, "AsyncFunctionDef",
+        astmodulestate_global->AsyncFunctionDef_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->AsyncFunctionDef_type);
+    if (PyModule_AddObject(m, "ClassDef", astmodulestate_global->ClassDef_type)
+        < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->ClassDef_type);
+    if (PyModule_AddObject(m, "Return", astmodulestate_global->Return_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->Return_type);
+    if (PyModule_AddObject(m, "Delete", astmodulestate_global->Delete_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->Delete_type);
+    if (PyModule_AddObject(m, "Assign", astmodulestate_global->Assign_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->Assign_type);
+    if (PyModule_AddObject(m, "AugAssign",
+        astmodulestate_global->AugAssign_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->AugAssign_type);
+    if (PyModule_AddObject(m, "AnnAssign",
+        astmodulestate_global->AnnAssign_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->AnnAssign_type);
+    if (PyModule_AddObject(m, "For", astmodulestate_global->For_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->For_type);
+    if (PyModule_AddObject(m, "AsyncFor", astmodulestate_global->AsyncFor_type)
+        < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->AsyncFor_type);
+    if (PyModule_AddObject(m, "While", astmodulestate_global->While_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->While_type);
+    if (PyModule_AddObject(m, "If", astmodulestate_global->If_type) < 0) return
         NULL;
-    if (PyDict_SetItemString(d, "Interactive", (PyObject*)Interactive_type) <
+    Py_INCREF(astmodulestate(m)->If_type);
+    if (PyModule_AddObject(m, "With", astmodulestate_global->With_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->With_type);
+    if (PyModule_AddObject(m, "AsyncWith",
+        astmodulestate_global->AsyncWith_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->AsyncWith_type);
+    if (PyModule_AddObject(m, "Raise", astmodulestate_global->Raise_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->Raise_type);
+    if (PyModule_AddObject(m, "Try", astmodulestate_global->Try_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->Try_type);
+    if (PyModule_AddObject(m, "Assert", astmodulestate_global->Assert_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->Assert_type);
+    if (PyModule_AddObject(m, "Import", astmodulestate_global->Import_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->Import_type);
+    if (PyModule_AddObject(m, "ImportFrom",
+        astmodulestate_global->ImportFrom_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->ImportFrom_type);
+    if (PyModule_AddObject(m, "Global", astmodulestate_global->Global_type) <
         0) return NULL;
-    if (PyDict_SetItemString(d, "Expression", (PyObject*)Expression_type) < 0)
+    Py_INCREF(astmodulestate(m)->Global_type);
+    if (PyModule_AddObject(m, "Nonlocal", astmodulestate_global->Nonlocal_type)
+        < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->Nonlocal_type);
+    if (PyModule_AddObject(m, "Expr", astmodulestate_global->Expr_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->Expr_type);
+    if (PyModule_AddObject(m, "Pass", astmodulestate_global->Pass_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->Pass_type);
+    if (PyModule_AddObject(m, "Break", astmodulestate_global->Break_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->Break_type);
+    if (PyModule_AddObject(m, "Continue", astmodulestate_global->Continue_type)
+        < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->Continue_type);
+    if (PyModule_AddObject(m, "expr", astmodulestate_global->expr_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "FunctionType", (PyObject*)FunctionType_type) <
+    Py_INCREF(astmodulestate(m)->expr_type);
+    if (PyModule_AddObject(m, "BoolOp", astmodulestate_global->BoolOp_type) <
         0) return NULL;
-    if (PyDict_SetItemString(d, "Suite", (PyObject*)Suite_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "stmt", (PyObject*)stmt_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "FunctionDef", (PyObject*)FunctionDef_type) <
+    Py_INCREF(astmodulestate(m)->BoolOp_type);
+    if (PyModule_AddObject(m, "NamedExpr",
+        astmodulestate_global->NamedExpr_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->NamedExpr_type);
+    if (PyModule_AddObject(m, "BinOp", astmodulestate_global->BinOp_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->BinOp_type);
+    if (PyModule_AddObject(m, "UnaryOp", astmodulestate_global->UnaryOp_type) <
         0) return NULL;
-    if (PyDict_SetItemString(d, "AsyncFunctionDef",
-        (PyObject*)AsyncFunctionDef_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "ClassDef", (PyObject*)ClassDef_type) < 0)
+    Py_INCREF(astmodulestate(m)->UnaryOp_type);
+    if (PyModule_AddObject(m, "Lambda", astmodulestate_global->Lambda_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->Lambda_type);
+    if (PyModule_AddObject(m, "IfExp", astmodulestate_global->IfExp_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "Return", (PyObject*)Return_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Delete", (PyObject*)Delete_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Assign", (PyObject*)Assign_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "AugAssign", (PyObject*)AugAssign_type) < 0)
+    Py_INCREF(astmodulestate(m)->IfExp_type);
+    if (PyModule_AddObject(m, "Dict", astmodulestate_global->Dict_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "AnnAssign", (PyObject*)AnnAssign_type) < 0)
+    Py_INCREF(astmodulestate(m)->Dict_type);
+    if (PyModule_AddObject(m, "Set", astmodulestate_global->Set_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "For", (PyObject*)For_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "AsyncFor", (PyObject*)AsyncFor_type) < 0)
+    Py_INCREF(astmodulestate(m)->Set_type);
+    if (PyModule_AddObject(m, "ListComp", astmodulestate_global->ListComp_type)
+        < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->ListComp_type);
+    if (PyModule_AddObject(m, "SetComp", astmodulestate_global->SetComp_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->SetComp_type);
+    if (PyModule_AddObject(m, "DictComp", astmodulestate_global->DictComp_type)
+        < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->DictComp_type);
+    if (PyModule_AddObject(m, "GeneratorExp",
+        astmodulestate_global->GeneratorExp_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->GeneratorExp_type);
+    if (PyModule_AddObject(m, "Await", astmodulestate_global->Await_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->Await_type);
+    if (PyModule_AddObject(m, "Yield", astmodulestate_global->Yield_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->Yield_type);
+    if (PyModule_AddObject(m, "YieldFrom",
+        astmodulestate_global->YieldFrom_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->YieldFrom_type);
+    if (PyModule_AddObject(m, "Compare", astmodulestate_global->Compare_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->Compare_type);
+    if (PyModule_AddObject(m, "Call", astmodulestate_global->Call_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->Call_type);
+    if (PyModule_AddObject(m, "FormattedValue",
+        astmodulestate_global->FormattedValue_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->FormattedValue_type);
+    if (PyModule_AddObject(m, "JoinedStr",
+        astmodulestate_global->JoinedStr_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->JoinedStr_type);
+    if (PyModule_AddObject(m, "Constant", astmodulestate_global->Constant_type)
+        < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->Constant_type);
+    if (PyModule_AddObject(m, "Attribute",
+        astmodulestate_global->Attribute_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->Attribute_type);
+    if (PyModule_AddObject(m, "Subscript",
+        astmodulestate_global->Subscript_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->Subscript_type);
+    if (PyModule_AddObject(m, "Starred", astmodulestate_global->Starred_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->Starred_type);
+    if (PyModule_AddObject(m, "Name", astmodulestate_global->Name_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "While", (PyObject*)While_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "If", (PyObject*)If_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "With", (PyObject*)With_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "AsyncWith", (PyObject*)AsyncWith_type) < 0)
+    Py_INCREF(astmodulestate(m)->Name_type);
+    if (PyModule_AddObject(m, "List", astmodulestate_global->List_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "Raise", (PyObject*)Raise_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Try", (PyObject*)Try_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Assert", (PyObject*)Assert_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Import", (PyObject*)Import_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "ImportFrom", (PyObject*)ImportFrom_type) < 0)
+    Py_INCREF(astmodulestate(m)->List_type);
+    if (PyModule_AddObject(m, "Tuple", astmodulestate_global->Tuple_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "Global", (PyObject*)Global_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Nonlocal", (PyObject*)Nonlocal_type) < 0)
+    Py_INCREF(astmodulestate(m)->Tuple_type);
+    if (PyModule_AddObject(m, "expr_context",
+        astmodulestate_global->expr_context_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->expr_context_type);
+    if (PyModule_AddObject(m, "Load", astmodulestate_global->Load_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "Expr", (PyObject*)Expr_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Pass", (PyObject*)Pass_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Break", (PyObject*)Break_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Continue", (PyObject*)Continue_type) < 0)
+    Py_INCREF(astmodulestate(m)->Load_type);
+    if (PyModule_AddObject(m, "Store", astmodulestate_global->Store_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "expr", (PyObject*)expr_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "BoolOp", (PyObject*)BoolOp_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "NamedExpr", (PyObject*)NamedExpr_type) < 0)
+    Py_INCREF(astmodulestate(m)->Store_type);
+    if (PyModule_AddObject(m, "Del", astmodulestate_global->Del_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "BinOp", (PyObject*)BinOp_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "UnaryOp", (PyObject*)UnaryOp_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Lambda", (PyObject*)Lambda_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "IfExp", (PyObject*)IfExp_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Dict", (PyObject*)Dict_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Set", (PyObject*)Set_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "ListComp", (PyObject*)ListComp_type) < 0)
+    Py_INCREF(astmodulestate(m)->Del_type);
+    if (PyModule_AddObject(m, "AugLoad", astmodulestate_global->AugLoad_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->AugLoad_type);
+    if (PyModule_AddObject(m, "AugStore", astmodulestate_global->AugStore_type)
+        < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->AugStore_type);
+    if (PyModule_AddObject(m, "Param", astmodulestate_global->Param_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "SetComp", (PyObject*)SetComp_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "DictComp", (PyObject*)DictComp_type) < 0)
+    Py_INCREF(astmodulestate(m)->Param_type);
+    if (PyModule_AddObject(m, "slice", astmodulestate_global->slice_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->slice_type);
+    if (PyModule_AddObject(m, "Slice", astmodulestate_global->Slice_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->Slice_type);
+    if (PyModule_AddObject(m, "ExtSlice", astmodulestate_global->ExtSlice_type)
+        < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->ExtSlice_type);
+    if (PyModule_AddObject(m, "Index", astmodulestate_global->Index_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "GeneratorExp", (PyObject*)GeneratorExp_type) <
+    Py_INCREF(astmodulestate(m)->Index_type);
+    if (PyModule_AddObject(m, "boolop", astmodulestate_global->boolop_type) <
         0) return NULL;
-    if (PyDict_SetItemString(d, "Await", (PyObject*)Await_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "YieldFrom", (PyObject*)YieldFrom_type) < 0)
+    Py_INCREF(astmodulestate(m)->boolop_type);
+    if (PyModule_AddObject(m, "And", astmodulestate_global->And_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "Compare", (PyObject*)Compare_type) < 0) return
+    Py_INCREF(astmodulestate(m)->And_type);
+    if (PyModule_AddObject(m, "Or", astmodulestate_global->Or_type) < 0) return
         NULL;
-    if (PyDict_SetItemString(d, "Call", (PyObject*)Call_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "FormattedValue",
-        (PyObject*)FormattedValue_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "JoinedStr", (PyObject*)JoinedStr_type) < 0)
+    Py_INCREF(astmodulestate(m)->Or_type);
+    if (PyModule_AddObject(m, "operator", astmodulestate_global->operator_type)
+        < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->operator_type);
+    if (PyModule_AddObject(m, "Add", astmodulestate_global->Add_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->Add_type);
+    if (PyModule_AddObject(m, "Sub", astmodulestate_global->Sub_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "Constant", (PyObject*)Constant_type) < 0)
+    Py_INCREF(astmodulestate(m)->Sub_type);
+    if (PyModule_AddObject(m, "Mult", astmodulestate_global->Mult_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) < 0)
+    Py_INCREF(astmodulestate(m)->Mult_type);
+    if (PyModule_AddObject(m, "MatMult", astmodulestate_global->MatMult_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->MatMult_type);
+    if (PyModule_AddObject(m, "Div", astmodulestate_global->Div_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "Subscript", (PyObject*)Subscript_type) < 0)
+    Py_INCREF(astmodulestate(m)->Div_type);
+    if (PyModule_AddObject(m, "Mod", astmodulestate_global->Mod_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "Starred", (PyObject*)Starred_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Name", (PyObject*)Name_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "List", (PyObject*)List_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Tuple", (PyObject*)Tuple_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "expr_context", (PyObject*)expr_context_type) <
+    Py_INCREF(astmodulestate(m)->Mod_type);
+    if (PyModule_AddObject(m, "Pow", astmodulestate_global->Pow_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->Pow_type);
+    if (PyModule_AddObject(m, "LShift", astmodulestate_global->LShift_type) <
         0) return NULL;
-    if (PyDict_SetItemString(d, "Load", (PyObject*)Load_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Store", (PyObject*)Store_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Del", (PyObject*)Del_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "AugLoad", (PyObject*)AugLoad_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "AugStore", (PyObject*)AugStore_type) < 0)
+    Py_INCREF(astmodulestate(m)->LShift_type);
+    if (PyModule_AddObject(m, "RShift", astmodulestate_global->RShift_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->RShift_type);
+    if (PyModule_AddObject(m, "BitOr", astmodulestate_global->BitOr_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "Param", (PyObject*)Param_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "slice", (PyObject*)slice_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Slice", (PyObject*)Slice_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "ExtSlice", (PyObject*)ExtSlice_type) < 0)
+    Py_INCREF(astmodulestate(m)->BitOr_type);
+    if (PyModule_AddObject(m, "BitXor", astmodulestate_global->BitXor_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->BitXor_type);
+    if (PyModule_AddObject(m, "BitAnd", astmodulestate_global->BitAnd_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->BitAnd_type);
+    if (PyModule_AddObject(m, "FloorDiv", astmodulestate_global->FloorDiv_type)
+        < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->FloorDiv_type);
+    if (PyModule_AddObject(m, "unaryop", astmodulestate_global->unaryop_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->unaryop_type);
+    if (PyModule_AddObject(m, "Invert", astmodulestate_global->Invert_type) <
+        0) return NULL;
+    Py_INCREF(astmodulestate(m)->Invert_type);
+    if (PyModule_AddObject(m, "Not", astmodulestate_global->Not_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "Index", (PyObject*)Index_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "boolop", (PyObject*)boolop_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "And", (PyObject*)And_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Or", (PyObject*)Or_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "operator", (PyObject*)operator_type) < 0)
-        return NULL;
-    if (PyDict_SetItemString(d, "Add", (PyObject*)Add_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Sub", (PyObject*)Sub_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Mult", (PyObject*)Mult_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "MatMult", (PyObject*)MatMult_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Div", (PyObject*)Div_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Mod", (PyObject*)Mod_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Pow", (PyObject*)Pow_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "LShift", (PyObject*)LShift_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "RShift", (PyObject*)RShift_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "BitOr", (PyObject*)BitOr_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "BitXor", (PyObject*)BitXor_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "BitAnd", (PyObject*)BitAnd_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "FloorDiv", (PyObject*)FloorDiv_type) < 0)
+    Py_INCREF(astmodulestate(m)->Not_type);
+    if (PyModule_AddObject(m, "UAdd", astmodulestate_global->UAdd_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "unaryop", (PyObject*)unaryop_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Invert", (PyObject*)Invert_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Not", (PyObject*)Not_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "UAdd", (PyObject*)UAdd_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "USub", (PyObject*)USub_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "cmpop", (PyObject*)cmpop_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Eq", (PyObject*)Eq_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "NotEq", (PyObject*)NotEq_type) < 0) return
-        NULL;
-    if (PyDict_SetItemString(d, "Lt", (PyObject*)Lt_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "LtE", (PyObject*)LtE_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Gt", (PyObject*)Gt_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "GtE", (PyObject*)GtE_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "Is", (PyObject*)Is_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "IsNot", (PyObject*)IsNot_type) < 0) return
+    Py_INCREF(astmodulestate(m)->UAdd_type);
+    if (PyModule_AddObject(m, "USub", astmodulestate_global->USub_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->USub_type);
+    if (PyModule_AddObject(m, "cmpop", astmodulestate_global->cmpop_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->cmpop_type);
+    if (PyModule_AddObject(m, "Eq", astmodulestate_global->Eq_type) < 0) return
         NULL;
-    if (PyDict_SetItemString(d, "In", (PyObject*)In_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "NotIn", (PyObject*)NotIn_type) < 0) return
+    Py_INCREF(astmodulestate(m)->Eq_type);
+    if (PyModule_AddObject(m, "NotEq", astmodulestate_global->NotEq_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->NotEq_type);
+    if (PyModule_AddObject(m, "Lt", astmodulestate_global->Lt_type) < 0) return
         NULL;
-    if (PyDict_SetItemString(d, "comprehension", (PyObject*)comprehension_type)
-        < 0) return NULL;
-    if (PyDict_SetItemString(d, "excepthandler", (PyObject*)excepthandler_type)
-        < 0) return NULL;
-    if (PyDict_SetItemString(d, "ExceptHandler", (PyObject*)ExceptHandler_type)
-        < 0) return NULL;
-    if (PyDict_SetItemString(d, "arguments", (PyObject*)arguments_type) < 0)
+    Py_INCREF(astmodulestate(m)->Lt_type);
+    if (PyModule_AddObject(m, "LtE", astmodulestate_global->LtE_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "arg", (PyObject*)arg_type) < 0) return NULL;
-    if (PyDict_SetItemString(d, "keyword", (PyObject*)keyword_type) < 0) return
+    Py_INCREF(astmodulestate(m)->LtE_type);
+    if (PyModule_AddObject(m, "Gt", astmodulestate_global->Gt_type) < 0) return
         NULL;
-    if (PyDict_SetItemString(d, "alias", (PyObject*)alias_type) < 0) return
+    Py_INCREF(astmodulestate(m)->Gt_type);
+    if (PyModule_AddObject(m, "GtE", astmodulestate_global->GtE_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->GtE_type);
+    if (PyModule_AddObject(m, "Is", astmodulestate_global->Is_type) < 0) return
         NULL;
-    if (PyDict_SetItemString(d, "withitem", (PyObject*)withitem_type) < 0)
+    Py_INCREF(astmodulestate(m)->Is_type);
+    if (PyModule_AddObject(m, "IsNot", astmodulestate_global->IsNot_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "type_ignore", (PyObject*)type_ignore_type) <
+    Py_INCREF(astmodulestate(m)->IsNot_type);
+    if (PyModule_AddObject(m, "In", astmodulestate_global->In_type) < 0) return
+        NULL;
+    Py_INCREF(astmodulestate(m)->In_type);
+    if (PyModule_AddObject(m, "NotIn", astmodulestate_global->NotIn_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->NotIn_type);
+    if (PyModule_AddObject(m, "comprehension",
+        astmodulestate_global->comprehension_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->comprehension_type);
+    if (PyModule_AddObject(m, "excepthandler",
+        astmodulestate_global->excepthandler_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->excepthandler_type);
+    if (PyModule_AddObject(m, "ExceptHandler",
+        astmodulestate_global->ExceptHandler_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->ExceptHandler_type);
+    if (PyModule_AddObject(m, "arguments",
+        astmodulestate_global->arguments_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->arguments_type);
+    if (PyModule_AddObject(m, "arg", astmodulestate_global->arg_type) < 0)
+        return NULL;
+    Py_INCREF(astmodulestate(m)->arg_type);
+    if (PyModule_AddObject(m, "keyword", astmodulestate_global->keyword_type) <
         0) return NULL;
-    if (PyDict_SetItemString(d, "TypeIgnore", (PyObject*)TypeIgnore_type) < 0)
+    Py_INCREF(astmodulestate(m)->keyword_type);
+    if (PyModule_AddObject(m, "alias", astmodulestate_global->alias_type) < 0)
         return NULL;
+    Py_INCREF(astmodulestate(m)->alias_type);
+    if (PyModule_AddObject(m, "withitem", astmodulestate_global->withitem_type)
+        < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->withitem_type);
+    if (PyModule_AddObject(m, "type_ignore",
+        astmodulestate_global->type_ignore_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->type_ignore_type);
+    if (PyModule_AddObject(m, "TypeIgnore",
+        astmodulestate_global->TypeIgnore_type) < 0) return NULL;
+    Py_INCREF(astmodulestate(m)->TypeIgnore_type);
     return m;
 }
 
@@ -9176,9 +10250,9 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
         return NULL;
     }
 
-    req_type[0] = (PyObject*)Module_type;
-    req_type[1] = (PyObject*)Expression_type;
-    req_type[2] = (PyObject*)Interactive_type;
+    req_type[0] = astmodulestate_global->Module_type;
+    req_type[1] = astmodulestate_global->Expression_type;
+    req_type[2] = astmodulestate_global->Interactive_type;
 
     assert(0 <= mode && mode <= 2);
 
@@ -9190,7 +10264,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
         return NULL;
     if (!isinstance) {
         PyErr_Format(PyExc_TypeError, "expected %s node, got %.400s",
-                     req_name[mode], Py_TYPE(ast)->tp_name);
+                     req_name[mode], _PyType_Name(Py_TYPE(ast)));
         return NULL;
     }
     if (obj2ast_mod(ast, &res, arena) != 0)
@@ -9203,7 +10277,7 @@ int PyAST_Check(PyObject* obj)
 {
     if (!init_types())
         return -1;
-    return PyObject_IsInstance(obj, (PyObject*)&AST_type);
+    return PyObject_IsInstance(obj, astmodulestate_global->AST_type);
 }
 
 


From webhook-mailer at python.org  Wed Sep 11 13:17:09 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Wed, 11 Sep 2019 17:17:09 -0000
Subject: [Python-checkins] Doc: recursive glob ** follows symlinks to
 directories (GH-12918)
Message-ID: <mailman.605.1568222230.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e24594bfe75aff3e654665cb940ddc4d4acffd2f
commit: e24594bfe75aff3e654665cb940ddc4d4acffd2f
branch: master
author: Marc <Marc.Herbert+github at gmail.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-11T19:17:05+02:00
summary:

Doc: recursive glob ** follows symlinks to directories (GH-12918)

files:
M Doc/library/glob.rst

diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst
index 96584437cef1..92a8c4d1eb87 100644
--- a/Doc/library/glob.rst
+++ b/Doc/library/glob.rst
@@ -49,8 +49,9 @@ For example, ``'[?]'`` matches the character ``'?'``.
       single: **; in glob-style wildcards
 
    If *recursive* is true, the pattern "``**``" will match any files and zero or
-   more directories and subdirectories.  If the pattern is followed by an
-   ``os.sep``, only directories and subdirectories match.
+   more directories, subdirectories and symbolic links to directories. If the
+   pattern is followed by an :data:`os.sep` or :data:`os.altsep` then files will not
+   match.
 
    .. audit-event:: glob.glob pathname,recursive glob.glob
 


From webhook-mailer at python.org  Wed Sep 11 13:23:31 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 17:23:31 -0000
Subject: [Python-checkins] [2.7] bpo-35264: Modules/_ssl.c: fix build with
 OpenSSL 1.1.0 (GH-10570)
Message-ID: <mailman.606.1568222613.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0d63669e52dd7e95708ec14e9e3e07d7dc9cd913
commit: 0d63669e52dd7e95708ec14e9e3e07d7dc9cd913
branch: 2.7
author: Alexandru Ardelean <ardeleanalex at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T10:23:28-07:00
summary:

[2.7] bpo-35264: Modules/_ssl.c: fix build with OpenSSL 1.1.0 (GH-10570)



Fixes a build error with OpenSSL 1.1.0. There is already code in the
`_ssl.c` that handles all the weird cases of the NPN config macros (with
various OpenSSL & LibreSSL versions).
That code will provide a HAVE_NPN variable, which should be used in the
rest of the code to check whether (or what) to compile regarding NPN.

This change adds HAVE_NPN in the remaining places where it should have been
placed.

Signed-off-by: Alexandru Ardelean <ardeleanalex at gmail.com>


https://bugs.python.org/issue35264

files:
A Misc/NEWS.d/next/Build/2018-11-16-15-19-09.bpo-35264.h5GxH3.rst
M Modules/_ssl.c

diff --git a/Misc/NEWS.d/next/Build/2018-11-16-15-19-09.bpo-35264.h5GxH3.rst b/Misc/NEWS.d/next/Build/2018-11-16-15-19-09.bpo-35264.h5GxH3.rst
new file mode 100644
index 000000000000..d17968a52deb
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2018-11-16-15-19-09.bpo-35264.h5GxH3.rst
@@ -0,0 +1 @@
+Fix SSL module build with OpenSSL 1.1.0
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index 93b635cc4ac9..98c8a5a4a95c 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -1590,7 +1590,7 @@ static PyObject *PySSL_version(PySSLSocket *self)
     return PyUnicode_FromString(version);
 }
 
-#if defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG)
+#if HAVE_NPN
 static PyObject *PySSL_selected_npn_protocol(PySSLSocket *self) {
     const unsigned char *out;
     unsigned int outlen;
@@ -2118,7 +2118,7 @@ static PyMethodDef PySSLMethods[] = {
      PySSL_peercert_doc},
     {"cipher", (PyCFunction)PySSL_cipher, METH_NOARGS},
     {"version", (PyCFunction)PySSL_version, METH_NOARGS},
-#ifdef OPENSSL_NPN_NEGOTIATED
+#if HAVE_NPN
     {"selected_npn_protocol", (PyCFunction)PySSL_selected_npn_protocol, METH_NOARGS},
 #endif
 #if HAVE_ALPN


From webhook-mailer at python.org  Wed Sep 11 13:24:51 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Wed, 11 Sep 2019 17:24:51 -0000
Subject: [Python-checkins] bpo-34001: Fix test_ssl with LibreSSL (GH-13783)
Message-ID: <mailman.607.1568222692.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c9bc49c5f6e26a7c958307c2ac338951a7534d9a
commit: c9bc49c5f6e26a7c958307c2ac338951a7534d9a
branch: master
author: Christian Heimes <christian at python.org>
committer: T. Wouters <thomas at python.org>
date: 2019-09-11T10:24:47-07:00
summary:

bpo-34001: Fix test_ssl with LibreSSL (GH-13783)

files:
A Misc/NEWS.d/next/Tests/2019-06-03-20-47-10.bpo-34001.KvYx9z.rst
M Lib/test/test_ssl.py

diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index b71928368b04..54e271627638 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -1109,6 +1109,7 @@ def test_hostname_checks_common_name(self):
 
     @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'),
                          "required OpenSSL 1.1.0g")
+    @unittest.skipIf(IS_LIBRESSL, "see bpo-34001")
     def test_min_max_version(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         # OpenSSL default is MINIMUM_SUPPORTED, however some vendors like
@@ -3731,8 +3732,8 @@ def test_min_max_version(self):
                 self.assertEqual(s.version(), 'TLSv1.1')
 
         # client 1.0, server 1.2 (mismatch)
-        server_context.minimum_version = ssl.TLSVersion.TLSv1_2
         server_context.maximum_version = ssl.TLSVersion.TLSv1_2
+        server_context.minimum_version = ssl.TLSVersion.TLSv1_2
         client_context.maximum_version = ssl.TLSVersion.TLSv1
         client_context.maximum_version = ssl.TLSVersion.TLSv1
         with ThreadedEchoServer(context=server_context) as server:
diff --git a/Misc/NEWS.d/next/Tests/2019-06-03-20-47-10.bpo-34001.KvYx9z.rst b/Misc/NEWS.d/next/Tests/2019-06-03-20-47-10.bpo-34001.KvYx9z.rst
new file mode 100644
index 000000000000..35c65a8fbf0f
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-06-03-20-47-10.bpo-34001.KvYx9z.rst
@@ -0,0 +1,2 @@
+Make test_ssl pass with LibreSSL. LibreSSL handles minimum and maximum TLS
+version differently than OpenSSL.


From webhook-mailer at python.org  Wed Sep 11 13:25:58 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 17:25:58 -0000
Subject: [Python-checkins] bpo-37698: Update doc of PyBuffer_ToContiguous
 (GH-14992)
Message-ID: <mailman.608.1568222759.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/15f5a7527b87e11fcf23069c147fd4cb7d42cfb0
commit: 15f5a7527b87e11fcf23069c147fd4cb7d42cfb0
branch: master
author: Hai Shi <shihai1992 at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T10:25:55-07:00
summary:

bpo-37698: Update doc of PyBuffer_ToContiguous (GH-14992)



https://bugs.python.org/issue37698

files:
M Doc/c-api/buffer.rst

diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index fb9d47b16c9f..e660716856b6 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -493,8 +493,8 @@ Buffer-related functions
 .. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order)
 
    Copy *len* bytes from *src* to its contiguous representation in *buf*.
-   *order* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering).
-   ``0`` is returned on success, ``-1`` on error.
+   *order* can be ``'C'`` or ``'F'`` or ``'A'`` (for C-style or Fortran-style
+   ordering or either one). ``0`` is returned on success, ``-1`` on error.
 
    This function fails if *len* != *src->len*.
 


From webhook-mailer at python.org  Wed Sep 11 13:28:20 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 17:28:20 -0000
Subject: [Python-checkins] Doc: recursive glob ** follows symlinks to
 directories (GH-12918)
Message-ID: <mailman.609.1568222901.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9b46586124b1022f31f7b6d582a54dff49675ab3
commit: 9b46586124b1022f31f7b6d582a54dff49675ab3
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T10:28:16-07:00
summary:

Doc: recursive glob ** follows symlinks to directories (GH-12918)

(cherry picked from commit e24594bfe75aff3e654665cb940ddc4d4acffd2f)

Co-authored-by: Marc <Marc.Herbert+github at gmail.com>

files:
M Doc/library/glob.rst

diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst
index 0db10b5efc10..9ab79d7a6bab 100644
--- a/Doc/library/glob.rst
+++ b/Doc/library/glob.rst
@@ -48,8 +48,9 @@ For example, ``'[?]'`` matches the character ``'?'``.
       single: **; in glob-style wildcards
 
    If *recursive* is true, the pattern "``**``" will match any files and zero or
-   more directories and subdirectories.  If the pattern is followed by an
-   ``os.sep``, only directories and subdirectories match.
+   more directories, subdirectories and symbolic links to directories. If the
+   pattern is followed by an :data:`os.sep` or :data:`os.altsep` then files will not
+   match.
 
    .. note::
       Using the "``**``" pattern in large directory trees may consume


From webhook-mailer at python.org  Wed Sep 11 13:32:29 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 17:32:29 -0000
Subject: [Python-checkins] Doc: recursive glob ** follows symlinks to
 directories (GH-12918)
Message-ID: <mailman.610.1568223151.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d47993664e6e82e98190cd14030db75aff7e2437
commit: d47993664e6e82e98190cd14030db75aff7e2437
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T10:32:26-07:00
summary:

Doc: recursive glob ** follows symlinks to directories (GH-12918)

(cherry picked from commit e24594bfe75aff3e654665cb940ddc4d4acffd2f)

Co-authored-by: Marc <Marc.Herbert+github at gmail.com>

files:
M Doc/library/glob.rst

diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst
index 96584437cef1..92a8c4d1eb87 100644
--- a/Doc/library/glob.rst
+++ b/Doc/library/glob.rst
@@ -49,8 +49,9 @@ For example, ``'[?]'`` matches the character ``'?'``.
       single: **; in glob-style wildcards
 
    If *recursive* is true, the pattern "``**``" will match any files and zero or
-   more directories and subdirectories.  If the pattern is followed by an
-   ``os.sep``, only directories and subdirectories match.
+   more directories, subdirectories and symbolic links to directories. If the
+   pattern is followed by an :data:`os.sep` or :data:`os.altsep` then files will not
+   match.
 
    .. audit-event:: glob.glob pathname,recursive glob.glob
 


From webhook-mailer at python.org  Wed Sep 11 13:36:10 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 17:36:10 -0000
Subject: [Python-checkins] [3.7] bpo-38117: Test with OpenSSL 1.1.1d
 (GH-15983) (GH-15994)
Message-ID: <mailman.611.1568223373.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/cec68c31e8507159534b6eec3fb5d801cd3dbf8c
commit: cec68c31e8507159534b6eec3fb5d801cd3dbf8c
branch: 3.7
author: St?phane Wirtel <stephane at wirtel.be>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-11T10:36:06-07:00
summary:

[3.7] bpo-38117: Test with OpenSSL 1.1.1d (GH-15983) (GH-15994)



Signed-off-by: Christian Heimes <christian at python.org>.
(cherry picked from commit 58ab13479d854491ac9207bacfae25e8b18b044a)

Co-authored-by: Christian Heimes <christian at python.org>


https://bugs.python.org/issue38117



Automerge-Triggered-By: @matrixise

files:
A Misc/NEWS.d/next/Tests/2019-09-11-17-22-32.bpo-38117.X7LgGY.rst
M .azure-pipelines/ci.yml
M .azure-pipelines/pr.yml
M .travis.yml
M Tools/ssl/multissltests.py

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index 9d8da910acda..38d0cc3e47dd 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -59,7 +59,7 @@ jobs:
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1d
 
   steps:
   - template: ./posix-steps.yml
@@ -116,7 +116,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1d
 
   steps:
   - template: ./posix-steps.yml
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
index bafed733a75d..8661da93d7e7 100644
--- a/.azure-pipelines/pr.yml
+++ b/.azure-pipelines/pr.yml
@@ -59,7 +59,7 @@ jobs:
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1d
 
   steps:
   - template: ./posix-steps.yml
@@ -116,7 +116,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1d
 
   steps:
   - template: ./posix-steps.yml
diff --git a/.travis.yml b/.travis.yml
index 378f3e5eb733..1f67b70f799c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,7 +11,7 @@ cache:
 
 env:
   global:
-    - OPENSSL=1.1.1c
+    - OPENSSL=1.1.1d
     - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}"
     - PATH="${OPENSSL_DIR}/bin:$PATH"
     # Use -O3 because we don't use debugger on Travis-CI
diff --git a/Misc/NEWS.d/next/Tests/2019-09-11-17-22-32.bpo-38117.X7LgGY.rst b/Misc/NEWS.d/next/Tests/2019-09-11-17-22-32.bpo-38117.X7LgGY.rst
new file mode 100644
index 000000000000..d3280375516a
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-11-17-22-32.bpo-38117.X7LgGY.rst
@@ -0,0 +1 @@
+Test with OpenSSL 1.1.1d
diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py
index 07bd9b016d97..d816810d5707 100755
--- a/Tools/ssl/multissltests.py
+++ b/Tools/ssl/multissltests.py
@@ -45,16 +45,16 @@
 ]
 
 OPENSSL_RECENT_VERSIONS = [
-    "1.0.2s",
-    "1.1.0k",
-    "1.1.1c",
+    "1.0.2t",
+    "1.1.0l",
+    "1.1.1d",
 ]
 
 LIBRESSL_OLD_VERSIONS = [
 ]
 
 LIBRESSL_RECENT_VERSIONS = [
-    "2.7.4",
+    "2.9.2",
 ]
 
 # store files in ../multissl


From webhook-mailer at python.org  Wed Sep 11 13:49:02 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Wed, 11 Sep 2019 17:49:02 -0000
Subject: [Python-checkins] Correct typo in min version test (GH-16000)
Message-ID: <mailman.612.1568224144.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/de606ea169435fe4dd40dc3e3b2b591e11396a14
commit: de606ea169435fe4dd40dc3e3b2b591e11396a14
branch: master
author: Christian Heimes <christian at python.org>
committer: T. Wouters <thomas at python.org>
date: 2019-09-11T10:48:58-07:00
summary:

Correct typo in min version test (GH-16000)

Signed-off-by: Christian Heimes <christian at python.org>

files:
M Lib/test/test_ssl.py

diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 54e271627638..737a96989968 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -3735,7 +3735,7 @@ def test_min_max_version(self):
         server_context.maximum_version = ssl.TLSVersion.TLSv1_2
         server_context.minimum_version = ssl.TLSVersion.TLSv1_2
         client_context.maximum_version = ssl.TLSVersion.TLSv1
-        client_context.maximum_version = ssl.TLSVersion.TLSv1
+        client_context.minimum_version = ssl.TLSVersion.TLSv1
         with ThreadedEchoServer(context=server_context) as server:
             with client_context.wrap_socket(socket.socket(),
                                             server_hostname=hostname) as s:


From webhook-mailer at python.org  Wed Sep 11 13:55:05 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Wed, 11 Sep 2019 17:55:05 -0000
Subject: [Python-checkins] bpo-36270: Doc: add link to traceback object
 reference (GH-13119)
Message-ID: <mailman.613.1568224507.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9936371af298d465095ae70bc9c2943b4b16eac4
commit: 9936371af298d465095ae70bc9c2943b4b16eac4
branch: master
author: Bj?rn Meier <bjoern at opentrash.org>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-11T19:55:02+02:00
summary:

bpo-36270: Doc: add link to traceback object reference (GH-13119)

files:
M Doc/library/sys.rst

diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index a5528f72e4b7..ca43dc994cb7 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -345,7 +345,7 @@ always available.
    ``(type, value, traceback)``.  Their meaning is: *type* gets the type of the
    exception being handled (a subclass of :exc:`BaseException`); *value* gets
    the exception instance (an instance of the exception type); *traceback* gets
-   a traceback object (see the Reference Manual) which encapsulates the call
+   a :ref:`traceback object <traceback-objects>` which encapsulates the call
    stack at the point where the exception originally occurred.
 
 


From webhook-mailer at python.org  Wed Sep 11 14:12:39 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 11 Sep 2019 18:12:39 -0000
Subject: [Python-checkins] bpo-36270: Doc: add link to traceback object
 reference (GH-13119)
Message-ID: <mailman.614.1568225560.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f79a022d762edc749d0fecdc50c567d2bb910c53
commit: f79a022d762edc749d0fecdc50c567d2bb910c53
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T11:12:35-07:00
summary:

bpo-36270: Doc: add link to traceback object reference (GH-13119)

(cherry picked from commit 9936371af298d465095ae70bc9c2943b4b16eac4)

Co-authored-by: Bj?rn Meier <bjoern at opentrash.org>

files:
M Doc/library/sys.rst

diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index be1af371d428..5f8afa8d6ed4 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -345,7 +345,7 @@ always available.
    ``(type, value, traceback)``.  Their meaning is: *type* gets the type of the
    exception being handled (a subclass of :exc:`BaseException`); *value* gets
    the exception instance (an instance of the exception type); *traceback* gets
-   a traceback object (see the Reference Manual) which encapsulates the call
+   a :ref:`traceback object <traceback-objects>` which encapsulates the call
    stack at the point where the exception originally occurred.
 
 


From webhook-mailer at python.org  Wed Sep 11 14:19:16 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Wed, 11 Sep 2019 18:19:16 -0000
Subject: [Python-checkins] Correct typo in min version test (GH-16001)
Message-ID: <mailman.615.1568225957.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8af4e0c9942424b0dc6f5e882ce0bbd36f5fbb96
commit: 8af4e0c9942424b0dc6f5e882ce0bbd36f5fbb96
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-11T11:19:12-07:00
summary:

Correct typo in min version test (GH-16001)

Signed-off-by: Christian Heimes <christian at python.org>
(cherry picked from commit de606ea169435fe4dd40dc3e3b2b591e11396a14)

Co-authored-by: Christian Heimes <christian at python.org>

files:
M Lib/test/test_ssl.py

diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index a32b0bcc7e22..9f3365aa864c 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -3730,7 +3730,7 @@ def test_min_max_version(self):
         server_context.maximum_version = ssl.TLSVersion.TLSv1_2
         server_context.minimum_version = ssl.TLSVersion.TLSv1_2
         client_context.maximum_version = ssl.TLSVersion.TLSv1
-        client_context.maximum_version = ssl.TLSVersion.TLSv1
+        client_context.minimum_version = ssl.TLSVersion.TLSv1
         with ThreadedEchoServer(context=server_context) as server:
             with client_context.wrap_socket(socket.socket(),
                                             server_hostname=hostname) as s:


From webhook-mailer at python.org  Wed Sep 11 14:49:50 2019
From: webhook-mailer at python.org (Eric Snow)
Date: Wed, 11 Sep 2019 18:49:50 -0000
Subject: [Python-checkins] bpo-36876: Add a tool that identifies unsupported
 global C variables. (#15877)
Message-ID: <mailman.616.1568227792.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ee536b2020b1f0baad1286dbd4345e13870324af
commit: ee536b2020b1f0baad1286dbd4345e13870324af
branch: master
author: Eric Snow <ericsnowcurrently at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T19:49:45+01:00
summary:

bpo-36876: Add a tool that identifies unsupported global C variables. (#15877)

files:
A Lib/test/test_check_c_globals.py
A Lib/test/test_tools/test_c_analyzer/__init__.py
A Lib/test/test_tools/test_c_analyzer/__main__.py
A Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py
A Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_files.py
A Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_info.py
A Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py
A Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py
A Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py
A Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py
A Lib/test/test_tools/test_c_analyzer/test_c_globals/test_functional.py
A Lib/test/test_tools/test_c_analyzer/test_c_globals/test_show.py
A Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py
A Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py
A Lib/test/test_tools/test_c_analyzer/test_c_parser/test_declarations.py
A Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py
A Lib/test/test_tools/test_c_analyzer/test_c_parser/test_preprocessor.py
A Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py
A Lib/test/test_tools/test_c_analyzer/test_c_symbols/test_info.py
A Lib/test/test_tools/test_c_analyzer/util.py
A Tools/c-analyzer/README
A Tools/c-analyzer/c-globals.py
A Tools/c-analyzer/c_analyzer_common/__init__.py
A Tools/c-analyzer/c_analyzer_common/_generate.py
A Tools/c-analyzer/c_analyzer_common/files.py
A Tools/c-analyzer/c_analyzer_common/info.py
A Tools/c-analyzer/c_analyzer_common/known.py
A Tools/c-analyzer/c_analyzer_common/util.py
A Tools/c-analyzer/c_globals/README
A Tools/c-analyzer/c_globals/__init__.py
A Tools/c-analyzer/c_globals/__main__.py
A Tools/c-analyzer/c_globals/find.py
A Tools/c-analyzer/c_globals/show.py
A Tools/c-analyzer/c_globals/supported.py
A Tools/c-analyzer/c_parser/__init__.py
A Tools/c-analyzer/c_parser/declarations.py
A Tools/c-analyzer/c_parser/info.py
A Tools/c-analyzer/c_parser/naive.py
A Tools/c-analyzer/c_parser/preprocessor.py
A Tools/c-analyzer/c_parser/source.py
A Tools/c-analyzer/c_symbols/__init__.py
A Tools/c-analyzer/c_symbols/binary.py
A Tools/c-analyzer/c_symbols/info.py
A Tools/c-analyzer/c_symbols/resolve.py
A Tools/c-analyzer/c_symbols/source.py
A Tools/c-analyzer/check-c-globals.py
A Tools/c-analyzer/ignored-globals.txt
A Tools/c-analyzer/ignored.tsv
A Tools/c-analyzer/known.tsv
D Tools/c-globals/README
D Tools/c-globals/check-c-globals.py
D Tools/c-globals/ignored-globals.txt
M Lib/test/test_clinic.py
M Lib/test/test_tools/__init__.py

diff --git a/Lib/test/test_check_c_globals.py b/Lib/test/test_check_c_globals.py
new file mode 100644
index 000000000000..009560e8d98d
--- /dev/null
+++ b/Lib/test/test_check_c_globals.py
@@ -0,0 +1,22 @@
+import unittest
+import test.test_tools
+
+test.test_tools.skip_if_missing('c-analyzer')
+with test.test_tools.imports_under_tool('c-analyzer'):
+    from c_globals.__main__ import main
+
+
+class ActualChecks(unittest.TestCase):
+
+    # XXX Also run the check in "make check".
+    @unittest.expectedFailure
+    def test_check_c_globals(self):
+        try:
+            main('check', {})
+        except NotImplementedError:
+            raise unittest.SkipTest('not supported on this host')
+
+
+if __name__ == '__main__':
+    # Test needs to be a package, so we can do relative imports.
+    unittest.main()
diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py
index 244c5fecd349..3d5dc4759d50 100644
--- a/Lib/test/test_clinic.py
+++ b/Lib/test/test_clinic.py
@@ -2,7 +2,7 @@
 # Copyright 2012-2013 by Larry Hastings.
 # Licensed to the PSF under a contributor agreement.
 
-from test import support
+from test import support, test_tools
 from unittest import TestCase
 import collections
 import inspect
@@ -10,17 +10,10 @@
 import sys
 import unittest
 
-
-clinic_path = os.path.join(os.path.dirname(__file__), '..', '..', 'Tools', 'clinic')
-clinic_path = os.path.normpath(clinic_path)
-if not os.path.exists(clinic_path):
-    raise unittest.SkipTest(f'{clinic_path!r} path does not exist')
-sys.path.append(clinic_path)
-try:
+test_tools.skip_if_missing('clinic')
+with test_tools.imports_under_tool('clinic'):
     import clinic
     from clinic import DSLParser
-finally:
-    del sys.path[-1]
 
 
 class FakeConverter:
diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py
index 4d0fca330a11..eb9acad677d5 100644
--- a/Lib/test/test_tools/__init__.py
+++ b/Lib/test/test_tools/__init__.py
@@ -1,20 +1,33 @@
 """Support functions for testing scripts in the Tools directory."""
-import os
-import unittest
+import contextlib
 import importlib
+import os.path
+import unittest
 from test import support
 
-basepath = os.path.dirname(                 # <src/install dir>
-                os.path.dirname(                # Lib
-                    os.path.dirname(                # test
-                        os.path.dirname(__file__))))    # test_tools
+basepath = os.path.normpath(
+        os.path.dirname(                 # <src/install dir>
+            os.path.dirname(                # Lib
+                os.path.dirname(                # test
+                    os.path.dirname(__file__)))))    # test_tools
 
 toolsdir = os.path.join(basepath, 'Tools')
 scriptsdir = os.path.join(toolsdir, 'scripts')
 
-def skip_if_missing():
-    if not os.path.isdir(scriptsdir):
-        raise unittest.SkipTest('scripts directory could not be found')
+def skip_if_missing(tool=None):
+    if tool:
+        tooldir = os.path.join(toolsdir, tool)
+    else:
+        tool = 'scripts'
+        tooldir = scriptsdir
+    if not os.path.isdir(tooldir):
+        raise unittest.SkipTest(f'{tool} directory could not be found')
+
+ at contextlib.contextmanager
+def imports_under_tool(name, *subdirs):
+    tooldir = os.path.join(toolsdir, name, *subdirs)
+    with support.DirsOnSysPath(tooldir) as cm:
+        yield cm
 
 def import_tool(toolname):
     with support.DirsOnSysPath(scriptsdir):
diff --git a/Lib/test/test_tools/test_c_analyzer/__init__.py b/Lib/test/test_tools/test_c_analyzer/__init__.py
new file mode 100644
index 000000000000..d0b4c045104d
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/__init__.py
@@ -0,0 +1,15 @@
+import contextlib
+import os.path
+import test.test_tools
+from test.support import load_package_tests
+
+
+ at contextlib.contextmanager
+def tool_imports_for_tests():
+    test.test_tools.skip_if_missing('c-analyzer')
+    with test.test_tools.imports_under_tool('c-analyzer'):
+        yield
+
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)
diff --git a/Lib/test/test_tools/test_c_analyzer/__main__.py b/Lib/test/test_tools/test_c_analyzer/__main__.py
new file mode 100644
index 000000000000..b5b017de8a8a
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/__main__.py
@@ -0,0 +1,5 @@
+from . import load_tests
+import unittest
+
+
+unittest.main()
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_files.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_files.py
new file mode 100644
index 000000000000..6d14aea78a48
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_files.py
@@ -0,0 +1,470 @@
+import os.path
+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_analyzer_common.files import (
+            iter_files, _walk_tree, glob_tree,
+            )
+
+
+def fixpath(filename):
+    return filename.replace('/', os.path.sep)
+
+
+class IterFilesTests(unittest.TestCase):
+
+    maxDiff = None
+
+    _return_walk = None
+
+    @property
+    def calls(self):
+        try:
+            return self._calls
+        except AttributeError:
+            self._calls = []
+            return self._calls
+
+    def set_files(self, *filesperroot):
+        roots = []
+        result = []
+        for root, files in filesperroot:
+            root = fixpath(root)
+            roots.append(root)
+            result.append([os.path.join(root, fixpath(f))
+                           for f in files])
+        self._return_walk = result
+        return roots
+
+    def _walk(self, root, *, suffix=None, walk=None):
+        self.calls.append(('_walk', (root, suffix, walk)))
+        return iter(self._return_walk.pop(0))
+
+    def _glob(self, root, *, suffix=None):
+        self.calls.append(('_glob', (root, suffix)))
+        return iter(self._return_walk.pop(0))
+
+    def test_typical(self):
+        dirnames = self.set_files(
+            ('spam', ['file1.c', 'file2.c']),
+            ('eggs', ['ham/file3.h']),
+            )
+        suffixes = ('.c', '.h')
+
+        files = list(iter_files(dirnames, suffixes,
+                                _glob=self._glob,
+                                _walk=self._walk))
+
+        self.assertEqual(files, [
+            fixpath('spam/file1.c'),
+            fixpath('spam/file2.c'),
+            fixpath('eggs/ham/file3.h'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_walk', ('spam', None, _walk_tree)),
+            ('_walk', ('eggs', None, _walk_tree)),
+            ])
+
+    def test_single_root(self):
+        self._return_walk = [
+                [fixpath('spam/file1.c'), fixpath('spam/file2.c')],
+                ]
+
+        files = list(iter_files('spam', '.c',
+                                _glob=self._glob,
+                                _walk=self._walk))
+
+        self.assertEqual(files, [
+            fixpath('spam/file1.c'),
+            fixpath('spam/file2.c'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_walk', ('spam', '.c', _walk_tree)),
+            ])
+
+    def test_one_root(self):
+        self._return_walk = [
+                [fixpath('spam/file1.c'), fixpath('spam/file2.c')],
+                ]
+
+        files = list(iter_files(['spam'], '.c',
+                                _glob=self._glob,
+                                _walk=self._walk))
+
+        self.assertEqual(files, [
+            fixpath('spam/file1.c'),
+            fixpath('spam/file2.c'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_walk', ('spam', '.c', _walk_tree)),
+            ])
+
+    def test_multiple_roots(self):
+        dirnames = self.set_files(
+            ('spam', ['file1.c', 'file2.c']),
+            ('eggs', ['ham/file3.c']),
+            )
+
+        files = list(iter_files(dirnames, '.c',
+                                _glob=self._glob,
+                                _walk=self._walk))
+
+        self.assertEqual(files, [
+            fixpath('spam/file1.c'),
+            fixpath('spam/file2.c'),
+            fixpath('eggs/ham/file3.c'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_walk', ('spam', '.c', _walk_tree)),
+            ('_walk', ('eggs', '.c', _walk_tree)),
+            ])
+
+    def test_no_roots(self):
+        files = list(iter_files([], '.c',
+                                _glob=self._glob,
+                                _walk=self._walk))
+
+        self.assertEqual(files, [])
+        self.assertEqual(self.calls, [])
+
+    def test_single_suffix(self):
+        self._return_walk = [
+                [fixpath('spam/file1.c'),
+                 fixpath('spam/eggs/file3.c'),
+                 ],
+                ]
+
+        files = list(iter_files('spam', '.c',
+                                _glob=self._glob,
+                                _walk=self._walk))
+
+        self.assertEqual(files, [
+            fixpath('spam/file1.c'),
+            fixpath('spam/eggs/file3.c'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_walk', ('spam', '.c', _walk_tree)),
+            ])
+
+    def test_one_suffix(self):
+        self._return_walk = [
+                [fixpath('spam/file1.c'),
+                 fixpath('spam/file1.h'),
+                 fixpath('spam/file1.o'),
+                 fixpath('spam/eggs/file3.c'),
+                 ],
+                ]
+
+        files = list(iter_files('spam', ['.c'],
+                                _glob=self._glob,
+                                _walk=self._walk))
+
+        self.assertEqual(files, [
+            fixpath('spam/file1.c'),
+            fixpath('spam/eggs/file3.c'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_walk', ('spam', None, _walk_tree)),
+            ])
+
+    def test_multiple_suffixes(self):
+        self._return_walk = [
+                [fixpath('spam/file1.c'),
+                 fixpath('spam/file1.h'),
+                 fixpath('spam/file1.o'),
+                 fixpath('spam/eggs/file3.c'),
+                 ],
+                ]
+
+        files = list(iter_files('spam', ('.c', '.h'),
+                                _glob=self._glob,
+                                _walk=self._walk))
+
+        self.assertEqual(files, [
+            fixpath('spam/file1.c'),
+            fixpath('spam/file1.h'),
+            fixpath('spam/eggs/file3.c'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_walk', ('spam', None, _walk_tree)),
+            ])
+
+    def test_no_suffix(self):
+        expected = [fixpath('spam/file1.c'),
+                    fixpath('spam/file1.h'),
+                    fixpath('spam/file1.o'),
+                    fixpath('spam/eggs/file3.c'),
+                    ]
+        for suffix in (None, '', ()):
+            with self.subTest(suffix):
+                self.calls.clear()
+                self._return_walk = [list(expected)]
+
+                files = list(iter_files('spam', suffix,
+                                        _glob=self._glob,
+                                        _walk=self._walk))
+
+                self.assertEqual(files, expected)
+                self.assertEqual(self.calls, [
+                    ('_walk', ('spam', suffix, _walk_tree)),
+                    ])
+
+    def test_relparent(self):
+        dirnames = self.set_files(
+            ('/x/y/z/spam', ['file1.c', 'file2.c']),
+            ('/x/y/z/eggs', ['ham/file3.c']),
+            )
+
+        files = list(iter_files(dirnames, '.c', fixpath('/x/y'),
+                                _glob=self._glob,
+                                _walk=self._walk))
+
+        self.assertEqual(files, [
+            fixpath('z/spam/file1.c'),
+            fixpath('z/spam/file2.c'),
+            fixpath('z/eggs/ham/file3.c'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_walk', (fixpath('/x/y/z/spam'), '.c', _walk_tree)),
+            ('_walk', (fixpath('/x/y/z/eggs'), '.c', _walk_tree)),
+            ])
+
+    def test_glob(self):
+        dirnames = self.set_files(
+            ('spam', ['file1.c', 'file2.c']),
+            ('eggs', ['ham/file3.c']),
+            )
+
+        files = list(iter_files(dirnames, '.c',
+                                get_files=glob_tree,
+                                _walk=self._walk,
+                                _glob=self._glob))
+
+        self.assertEqual(files, [
+            fixpath('spam/file1.c'),
+            fixpath('spam/file2.c'),
+            fixpath('eggs/ham/file3.c'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_glob', ('spam', '.c')),
+            ('_glob', ('eggs', '.c')),
+            ])
+
+
+    def test_alt_walk_func(self):
+        dirnames = self.set_files(
+            ('spam', ['file1.c', 'file2.c']),
+            ('eggs', ['ham/file3.c']),
+            )
+        def get_files(root):
+            return None
+
+        files = list(iter_files(dirnames, '.c',
+                                get_files=get_files,
+                                _walk=self._walk,
+                                _glob=self._glob))
+
+        self.assertEqual(files, [
+            fixpath('spam/file1.c'),
+            fixpath('spam/file2.c'),
+            fixpath('eggs/ham/file3.c'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_walk', ('spam', '.c', get_files)),
+            ('_walk', ('eggs', '.c', get_files)),
+            ])
+
+
+
+
+
+
+#    def test_no_dirnames(self):
+#        dirnames = []
+#        filter_by_name = None
+#
+#        files = list(iter_files(dirnames, filter_by_name,
+#                                _walk=self._walk))
+#
+#        self.assertEqual(files, [])
+#        self.assertEqual(self.calls, [])
+#
+#    def test_no_filter(self):
+#        self._return_walk = [
+#                [('spam', (), ('file1', 'file2.c', 'file3.h', 'file4.o')),
+#                 ],
+#                ]
+#        dirnames = [
+#                'spam',
+#                ]
+#        filter_by_name = None
+#
+#        files = list(iter_files(dirnames, filter_by_name,
+#                                _walk=self._walk))
+#
+#        self.assertEqual(files, [
+#            fixpath('spam/file1'),
+#            fixpath('spam/file2.c'),
+#            fixpath('spam/file3.h'),
+#            fixpath('spam/file4.o'),
+#            ])
+#        self.assertEqual(self.calls, [
+#            ('_walk', ('spam',)),
+#            ])
+#
+#    def test_no_files(self):
+#        self._return_walk = [
+#                [('spam', (), ()),
+#                 ],
+#                [(fixpath('eggs/ham'), (), ()),
+#                 ],
+#                ]
+#        dirnames = [
+#                'spam',
+#                fixpath('eggs/ham'),
+#                ]
+#        filter_by_name = None
+#
+#        files = list(iter_files(dirnames, filter_by_name,
+#                                _walk=self._walk))
+#
+#        self.assertEqual(files, [])
+#        self.assertEqual(self.calls, [
+#            ('_walk', ('spam',)),
+#            ('_walk', (fixpath('eggs/ham'),)),
+#            ])
+#
+#    def test_tree(self):
+#        self._return_walk = [
+#                [('spam', ('sub1', 'sub2', 'sub3'), ('file1',)),
+#                 (fixpath('spam/sub1'), ('sub1sub1',), ('file2', 'file3')),
+#                 (fixpath('spam/sub1/sub1sub1'), (), ('file4',)),
+#                 (fixpath('spam/sub2'), (), ()),
+#                 (fixpath('spam/sub3'), (), ('file5',)),
+#                 ],
+#                [(fixpath('eggs/ham'), (), ('file6',)),
+#                 ],
+#                ]
+#        dirnames = [
+#                'spam',
+#                fixpath('eggs/ham'),
+#                ]
+#        filter_by_name = None
+#
+#        files = list(iter_files(dirnames, filter_by_name,
+#                                _walk=self._walk))
+#
+#        self.assertEqual(files, [
+#            fixpath('spam/file1'),
+#            fixpath('spam/sub1/file2'),
+#            fixpath('spam/sub1/file3'),
+#            fixpath('spam/sub1/sub1sub1/file4'),
+#            fixpath('spam/sub3/file5'),
+#            fixpath('eggs/ham/file6'),
+#            ])
+#        self.assertEqual(self.calls, [
+#            ('_walk', ('spam',)),
+#            ('_walk', (fixpath('eggs/ham'),)),
+#            ])
+#
+#    def test_filter_suffixes(self):
+#        self._return_walk = [
+#                [('spam', (), ('file1', 'file2.c', 'file3.h', 'file4.o')),
+#                 ],
+#                ]
+#        dirnames = [
+#                'spam',
+#                ]
+#        filter_by_name = ('.c', '.h')
+#
+#        files = list(iter_files(dirnames, filter_by_name,
+#                                _walk=self._walk))
+#
+#        self.assertEqual(files, [
+#            fixpath('spam/file2.c'),
+#            fixpath('spam/file3.h'),
+#            ])
+#        self.assertEqual(self.calls, [
+#            ('_walk', ('spam',)),
+#            ])
+#
+#    def test_some_filtered(self):
+#        self._return_walk = [
+#                [('spam', (), ('file1', 'file2', 'file3', 'file4')),
+#                 ],
+#                ]
+#        dirnames = [
+#                'spam',
+#                ]
+#        def filter_by_name(filename, results=[False, True, False, True]):
+#            self.calls.append(('filter_by_name', (filename,)))
+#            return results.pop(0)
+#
+#        files = list(iter_files(dirnames, filter_by_name,
+#                                _walk=self._walk))
+#
+#        self.assertEqual(files, [
+#            fixpath('spam/file2'),
+#            fixpath('spam/file4'),
+#            ])
+#        self.assertEqual(self.calls, [
+#            ('_walk', ('spam',)),
+#            ('filter_by_name', ('file1',)),
+#            ('filter_by_name', ('file2',)),
+#            ('filter_by_name', ('file3',)),
+#            ('filter_by_name', ('file4',)),
+#            ])
+#
+#    def test_none_filtered(self):
+#        self._return_walk = [
+#                [('spam', (), ('file1', 'file2', 'file3', 'file4')),
+#                 ],
+#                ]
+#        dirnames = [
+#                'spam',
+#                ]
+#        def filter_by_name(filename, results=[True, True, True, True]):
+#            self.calls.append(('filter_by_name', (filename,)))
+#            return results.pop(0)
+#
+#        files = list(iter_files(dirnames, filter_by_name,
+#                                _walk=self._walk))
+#
+#        self.assertEqual(files, [
+#            fixpath('spam/file1'),
+#            fixpath('spam/file2'),
+#            fixpath('spam/file3'),
+#            fixpath('spam/file4'),
+#            ])
+#        self.assertEqual(self.calls, [
+#            ('_walk', ('spam',)),
+#            ('filter_by_name', ('file1',)),
+#            ('filter_by_name', ('file2',)),
+#            ('filter_by_name', ('file3',)),
+#            ('filter_by_name', ('file4',)),
+#            ])
+#
+#    def test_all_filtered(self):
+#        self._return_walk = [
+#                [('spam', (), ('file1', 'file2', 'file3', 'file4')),
+#                 ],
+#                ]
+#        dirnames = [
+#                'spam',
+#                ]
+#        def filter_by_name(filename, results=[False, False, False, False]):
+#            self.calls.append(('filter_by_name', (filename,)))
+#            return results.pop(0)
+#
+#        files = list(iter_files(dirnames, filter_by_name,
+#                                _walk=self._walk))
+#
+#        self.assertEqual(files, [])
+#        self.assertEqual(self.calls, [
+#            ('_walk', ('spam',)),
+#            ('filter_by_name', ('file1',)),
+#            ('filter_by_name', ('file2',)),
+#            ('filter_by_name', ('file3',)),
+#            ('filter_by_name', ('file4',)),
+#            ])
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_info.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_info.py
new file mode 100644
index 000000000000..2d386713b998
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_info.py
@@ -0,0 +1,194 @@
+import string
+import unittest
+
+from ..util import PseudoStr, StrProxy, Object
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_analyzer_common.info import ID
+
+
+class IDTests(unittest.TestCase):
+
+    VALID_ARGS = (
+            'x/y/z/spam.c',
+            'func',
+            'eggs',
+            )
+    VALID_KWARGS = dict(zip(ID._fields, VALID_ARGS))
+    VALID_EXPECTED = VALID_ARGS
+
+    def test_from_raw(self):
+        tests = [
+            ('', None),
+            (None, None),
+            ('spam', (None, None, 'spam')),
+            (('spam',), (None, None, 'spam')),
+            (('x/y/z/spam.c', 'spam'), ('x/y/z/spam.c', None, 'spam')),
+            (self.VALID_ARGS, self.VALID_EXPECTED),
+            (self.VALID_KWARGS, self.VALID_EXPECTED),
+            ]
+        for raw, expected in tests:
+            with self.subTest(raw):
+                id = ID.from_raw(raw)
+
+                self.assertEqual(id, expected)
+
+    def test_minimal(self):
+        id = ID(
+                filename=None,
+                funcname=None,
+                name='eggs',
+                )
+
+        self.assertEqual(id, (
+                None,
+                None,
+                'eggs',
+                ))
+
+    def test_init_typical_global(self):
+        id = ID(
+                filename='x/y/z/spam.c',
+                funcname=None,
+                name='eggs',
+                )
+
+        self.assertEqual(id, (
+                'x/y/z/spam.c',
+                None,
+                'eggs',
+                ))
+
+    def test_init_typical_local(self):
+        id = ID(
+                filename='x/y/z/spam.c',
+                funcname='func',
+                name='eggs',
+                )
+
+        self.assertEqual(id, (
+                'x/y/z/spam.c',
+                'func',
+                'eggs',
+                ))
+
+    def test_init_all_missing(self):
+        for value in ('', None):
+            with self.subTest(repr(value)):
+                id = ID(
+                        filename=value,
+                        funcname=value,
+                        name=value,
+                        )
+
+                self.assertEqual(id, (
+                        None,
+                        None,
+                        None,
+                        ))
+
+    def test_init_all_coerced(self):
+        tests = [
+            ('str subclass',
+             dict(
+                 filename=PseudoStr('x/y/z/spam.c'),
+                 funcname=PseudoStr('func'),
+                 name=PseudoStr('eggs'),
+                 ),
+             ('x/y/z/spam.c',
+              'func',
+              'eggs',
+              )),
+            ('non-str',
+             dict(
+                 filename=StrProxy('x/y/z/spam.c'),
+                 funcname=Object(),
+                 name=('a', 'b', 'c'),
+                 ),
+             ('x/y/z/spam.c',
+              '<object>',
+              "('a', 'b', 'c')",
+              )),
+            ]
+        for summary, kwargs, expected in tests:
+            with self.subTest(summary):
+                id = ID(**kwargs)
+
+                for field in ID._fields:
+                    value = getattr(id, field)
+                    self.assertIs(type(value), str)
+                self.assertEqual(tuple(id), expected)
+
+    def test_iterable(self):
+        id = ID(**self.VALID_KWARGS)
+
+        filename, funcname, name = id
+
+        values = (filename, funcname, name)
+        for value, expected in zip(values, self.VALID_EXPECTED):
+            self.assertEqual(value, expected)
+
+    def test_fields(self):
+        id = ID('a', 'b', 'z')
+
+        self.assertEqual(id.filename, 'a')
+        self.assertEqual(id.funcname, 'b')
+        self.assertEqual(id.name, 'z')
+
+    def test_validate_typical(self):
+        id = ID(
+                filename='x/y/z/spam.c',
+                funcname='func',
+                name='eggs',
+                )
+
+        id.validate()  # This does not fail.
+
+    def test_validate_missing_field(self):
+        for field in ID._fields:
+            with self.subTest(field):
+                id = ID(**self.VALID_KWARGS)
+                id = id._replace(**{field: None})
+
+                if field == 'funcname':
+                    id.validate()  # The field can be missing (not set).
+                    id = id._replace(filename=None)
+                    id.validate()  # Both fields can be missing (not set).
+                    continue
+
+                with self.assertRaises(TypeError):
+                    id.validate()
+
+    def test_validate_bad_field(self):
+        badch = tuple(c for c in string.punctuation + string.digits)
+        notnames = (
+                '1a',
+                'a.b',
+                'a-b',
+                '&a',
+                'a++',
+                ) + badch
+        tests = [
+            ('filename', ()),  # Any non-empty str is okay.
+            ('funcname', notnames),
+            ('name', notnames),
+            ]
+        seen = set()
+        for field, invalid in tests:
+            for value in invalid:
+                seen.add(value)
+                with self.subTest(f'{field}={value!r}'):
+                    id = ID(**self.VALID_KWARGS)
+                    id = id._replace(**{field: value})
+
+                    with self.assertRaises(ValueError):
+                        id.validate()
+
+        for field, invalid in tests:
+            valid = seen - set(invalid)
+            for value in valid:
+                with self.subTest(f'{field}={value!r}'):
+                    id = ID(**self.VALID_KWARGS)
+                    id = id._replace(**{field: value})
+
+                    id.validate()  # This does not fail.
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py
new file mode 100644
index 000000000000..215023da577b
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py
@@ -0,0 +1,68 @@
+import re
+import textwrap
+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_parser.info import Variable
+    from c_analyzer_common.info import ID
+    from c_analyzer_common.known import from_file
+
+
+class FromFileTests(unittest.TestCase):
+
+    maxDiff = None
+
+    _return_read_tsv = ()
+
+    @property
+    def calls(self):
+        try:
+            return self._calls
+        except AttributeError:
+            self._calls = []
+            return self._calls
+
+    def _read_tsv(self, *args):
+        self.calls.append(('_read_tsv', args))
+        return self._return_read_tsv
+
+    def test_typical(self):
+        lines = textwrap.dedent('''
+            filename    funcname        name    kind    declaration
+            file1.c     -       var1    variable        static int
+            file1.c     func1   local1  variable        static int
+            file1.c     -       var2    variable        int
+            file1.c     func2   local2  variable        char *
+            file2.c     -       var1    variable        char *
+            ''').strip().splitlines()
+        lines = [re.sub(r'\s+', '\t', line, 4) for line in lines]
+        self._return_read_tsv = [tuple(v.strip() for v in line.split('\t'))
+                                 for line in lines[1:]]
+
+        known = from_file('spam.c', _read_tsv=self._read_tsv)
+
+        self.assertEqual(known, {
+            'variables': {v.id: v for v in [
+                Variable.from_parts('file1.c', '', 'var1', 'static int'),
+                Variable.from_parts('file1.c', 'func1', 'local1', 'static int'),
+                Variable.from_parts('file1.c', '', 'var2', 'int'),
+                Variable.from_parts('file1.c', 'func2', 'local2', 'char *'),
+                Variable.from_parts('file2.c', '', 'var1', 'char *'),
+                ]},
+            })
+        self.assertEqual(self.calls, [
+            ('_read_tsv', ('spam.c', 'filename\tfuncname\tname\tkind\tdeclaration')),
+            ])
+
+    def test_empty(self):
+        self._return_read_tsv = []
+
+        known = from_file('spam.c', _read_tsv=self._read_tsv)
+
+        self.assertEqual(known, {
+            'variables': {},
+            })
+        self.assertEqual(self.calls, [
+            ('_read_tsv', ('spam.c', 'filename\tfuncname\tname\tkind\tdeclaration')),
+            ])
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py
new file mode 100644
index 000000000000..5f52c588d7c8
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py
@@ -0,0 +1,296 @@
+import sys
+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_analyzer_common import SOURCE_DIRS
+    from c_analyzer_common.known import DATA_FILE as KNOWN_FILE
+    from c_parser import info
+    import c_globals as cg
+    from c_globals.supported import IGNORED_FILE
+    from c_globals.__main__ import cmd_check, cmd_show, parse_args, main
+
+
+TYPICAL = [
+        (info.Variable.from_parts('src1/spam.c', None, 'var1', 'const char *'),
+         True,
+         ),
+        (info.Variable.from_parts('src1/spam.c', 'ham', 'initialized', 'int'),
+         True,
+         ),
+        (info.Variable.from_parts('src1/spam.c', None, 'var2', 'PyObject *'),
+         False,
+         ),
+        (info.Variable.from_parts('src1/eggs.c', 'tofu', 'ready', 'int'),
+         True,
+         ),
+        (info.Variable.from_parts('src1/spam.c', None, 'freelist', '(PyTupleObject *)[10]'),
+         False,
+         ),
+        (info.Variable.from_parts('src1/sub/ham.c', None, 'var1', 'const char const *'),
+         True,
+         ),
+        (info.Variable.from_parts('src2/jam.c', None, 'var1', 'int'),
+         True,
+         ),
+        (info.Variable.from_parts('src2/jam.c', None, 'var2', 'MyObject *'),
+         False,
+         ),
+        (info.Variable.from_parts('Include/spam.h', None, 'data', 'const int'),
+         True,
+         ),
+        ]
+
+
+class CMDBase(unittest.TestCase):
+
+    maxDiff = None
+
+    _return_find = ()
+
+    @property
+    def calls(self):
+        try:
+            return self._calls
+        except AttributeError:
+            self._calls = []
+            return self._calls
+
+    def _find(self, *args):
+        self.calls.append(('_find', args))
+        return self._return_find
+
+    def _show(self, *args):
+        self.calls.append(('_show', args))
+
+    def _print(self, *args):
+        self.calls.append(('_print', args))
+
+
+class CheckTests(CMDBase):
+
+    def test_defaults(self):
+        self._return_find = []
+
+        cmd_check('check',
+                  _find=self._find,
+                  _show=self._show,
+                  _print=self._print,
+                  )
+
+        self.assertEqual(self.calls[0], (
+            '_find', (
+                SOURCE_DIRS,
+                KNOWN_FILE,
+                IGNORED_FILE,
+                ),
+            ))
+
+    def test_all_supported(self):
+        self._return_find = [(v, s) for v, s in TYPICAL if s]
+        dirs = ['src1', 'src2', 'Include']
+
+        cmd_check('check',
+                 dirs,
+                 ignored='ignored.tsv',
+                 known='known.tsv',
+                 _find=self._find,
+                 _show=self._show,
+                 _print=self._print,
+                 )
+
+        self.assertEqual(self.calls, [
+            ('_find', (dirs, 'known.tsv', 'ignored.tsv')),
+            #('_print', ('okay',)),
+            ])
+
+    def test_some_unsupported(self):
+        self._return_find = TYPICAL
+        dirs = ['src1', 'src2', 'Include']
+
+        with self.assertRaises(SystemExit) as cm:
+            cmd_check('check',
+                      dirs,
+                      ignored='ignored.tsv',
+                      known='known.tsv',
+                      _find=self._find,
+                      _show=self._show,
+                      _print=self._print,
+                      )
+
+        unsupported = [v for v, s in TYPICAL if not s]
+        self.assertEqual(self.calls, [
+            ('_find', (dirs, 'known.tsv', 'ignored.tsv')),
+            ('_print', ('ERROR: found unsupported global variables',)),
+            ('_print', ()),
+            ('_show', (sorted(unsupported),)),
+            ('_print', (' (3 total)',)),
+            ])
+        self.assertEqual(cm.exception.code, 1)
+
+
+class ShowTests(CMDBase):
+
+    def test_defaults(self):
+        self._return_find = []
+
+        cmd_show('show',
+                 _find=self._find,
+                 _show=self._show,
+                 _print=self._print,
+                 )
+
+        self.assertEqual(self.calls[0], (
+            '_find', (
+                SOURCE_DIRS,
+                KNOWN_FILE,
+                IGNORED_FILE,
+                ),
+            ))
+
+    def test_typical(self):
+        self._return_find = TYPICAL
+        dirs = ['src1', 'src2', 'Include']
+
+        cmd_show('show',
+                 dirs,
+                 known='known.tsv',
+                 ignored='ignored.tsv',
+                 _find=self._find,
+                 _show=self._show,
+                 _print=self._print,
+                 )
+
+        supported = [v for v, s in TYPICAL if s]
+        unsupported = [v for v, s in TYPICAL if not s]
+        self.assertEqual(self.calls, [
+            ('_find', (dirs, 'known.tsv', 'ignored.tsv')),
+            ('_print', ('supported:',)),
+            ('_print', ('----------',)),
+            ('_show', (sorted(supported),)),
+            ('_print', (' (6 total)',)),
+            ('_print', ()),
+            ('_print', ('unsupported:',)),
+            ('_print', ('------------',)),
+            ('_show', (sorted(unsupported),)),
+            ('_print', (' (3 total)',)),
+            ])
+
+
+class ParseArgsTests(unittest.TestCase):
+
+    maxDiff = None
+
+    def test_no_args(self):
+        self.errmsg = None
+        def fail(msg):
+            self.errmsg = msg
+            sys.exit(msg)
+
+        with self.assertRaises(SystemExit):
+            parse_args('cg', [], _fail=fail)
+
+        self.assertEqual(self.errmsg, 'missing command')
+
+    def test_check_no_args(self):
+        cmd, cmdkwargs = parse_args('cg', [
+            'check',
+            ])
+
+        self.assertEqual(cmd, 'check')
+        self.assertEqual(cmdkwargs, {
+            'ignored': IGNORED_FILE,
+            'known': KNOWN_FILE,
+            'dirs': SOURCE_DIRS,
+            })
+
+    def test_check_full_args(self):
+        cmd, cmdkwargs = parse_args('cg', [
+            'check',
+            '--ignored', 'spam.tsv',
+            '--known', 'eggs.tsv',
+            'dir1',
+            'dir2',
+            'dir3',
+            ])
+
+        self.assertEqual(cmd, 'check')
+        self.assertEqual(cmdkwargs, {
+            'ignored': 'spam.tsv',
+            'known': 'eggs.tsv',
+            'dirs': ['dir1', 'dir2', 'dir3']
+            })
+
+    def test_show_no_args(self):
+        cmd, cmdkwargs = parse_args('cg', [
+            'show',
+            ])
+
+        self.assertEqual(cmd, 'show')
+        self.assertEqual(cmdkwargs, {
+            'ignored': IGNORED_FILE,
+            'known': KNOWN_FILE,
+            'dirs': SOURCE_DIRS,
+            'skip_objects': False,
+            })
+
+    def test_show_full_args(self):
+        cmd, cmdkwargs = parse_args('cg', [
+            'show',
+            '--ignored', 'spam.tsv',
+            '--known', 'eggs.tsv',
+            'dir1',
+            'dir2',
+            'dir3',
+            ])
+
+        self.assertEqual(cmd, 'show')
+        self.assertEqual(cmdkwargs, {
+            'ignored': 'spam.tsv',
+            'known': 'eggs.tsv',
+            'dirs': ['dir1', 'dir2', 'dir3'],
+            'skip_objects': False,
+            })
+
+
+def new_stub_commands(*names):
+    calls = []
+    def cmdfunc(cmd, **kwargs):
+        calls.append((cmd, kwargs))
+    commands = {name: cmdfunc for name in names}
+    return commands, calls
+
+
+class MainTests(unittest.TestCase):
+
+    def test_no_command(self):
+        with self.assertRaises(ValueError):
+            main(None, {})
+
+    def test_check(self):
+        commands, calls = new_stub_commands('check', 'show')
+
+        cmdkwargs = {
+            'ignored': 'spam.tsv',
+            'known': 'eggs.tsv',
+            'dirs': ['dir1', 'dir2', 'dir3'],
+            }
+        main('check', cmdkwargs, _COMMANDS=commands)
+
+        self.assertEqual(calls, [
+            ('check', cmdkwargs),
+            ])
+
+    def test_show(self):
+        commands, calls = new_stub_commands('check', 'show')
+
+        cmdkwargs = {
+            'ignored': 'spam.tsv',
+            'known': 'eggs.tsv',
+            'dirs': ['dir1', 'dir2', 'dir3'],
+            }
+        main('show', cmdkwargs, _COMMANDS=commands)
+
+        self.assertEqual(calls, [
+            ('show', cmdkwargs),
+            ])
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py
new file mode 100644
index 000000000000..b29f966fd2ad
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py
@@ -0,0 +1,332 @@
+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_parser import info
+    from c_globals.find import globals_from_binary, globals
+
+
+class _Base(unittest.TestCase):
+
+    maxDiff = None
+
+    @property
+    def calls(self):
+        try:
+            return self._calls
+        except AttributeError:
+            self._calls = []
+            return self._calls
+
+
+class StaticsFromBinaryTests(_Base):
+
+    _return_iter_symbols = ()
+    _return_resolve_symbols = ()
+    _return_get_symbol_resolver = None
+
+    def setUp(self):
+        super().setUp()
+
+        self.kwargs = dict(
+                _iter_symbols=self._iter_symbols,
+                _resolve=self._resolve_symbols,
+                _get_symbol_resolver=self._get_symbol_resolver,
+                )
+
+    def _iter_symbols(self, binfile, find_local_symbol):
+        self.calls.append(('_iter_symbols', (binfile, find_local_symbol)))
+        return self._return_iter_symbols
+
+    def _resolve_symbols(self, symbols, resolve):
+        self.calls.append(('_resolve_symbols', (symbols, resolve,)))
+        return self._return_resolve_symbols
+
+    def _get_symbol_resolver(self, knownvars, dirnames=None):
+        self.calls.append(('_get_symbol_resolver', (knownvars, dirnames)))
+        return self._return_get_symbol_resolver
+
+    def test_typical(self):
+        symbols = self._return_iter_symbols = ()
+        resolver = self._return_get_symbol_resolver = object()
+        variables = self._return_resolve_symbols = [
+            info.Variable.from_parts('dir1/spam.c', None, 'var1', 'int'),
+            info.Variable.from_parts('dir1/spam.c', None, 'var2', 'static int'),
+            info.Variable.from_parts('dir1/spam.c', None, 'var3', 'char *'),
+            info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', 'const char *'),
+            info.Variable.from_parts('dir1/eggs.c', None, 'var1', 'static int'),
+            info.Variable.from_parts('dir1/eggs.c', 'func1', 'var2', 'static char *'),
+            ]
+        knownvars = object()
+
+        found = list(globals_from_binary('python',
+                                         knownvars=knownvars,
+                                         **self.kwargs))
+
+        self.assertEqual(found, [
+            info.Variable.from_parts('dir1/spam.c', None, 'var2', 'static int'),
+            info.Variable.from_parts('dir1/eggs.c', None, 'var1', 'static int'),
+            info.Variable.from_parts('dir1/eggs.c', 'func1', 'var2', 'static char *'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_iter_symbols', ('python', None)),
+            ('_get_symbol_resolver', (knownvars, None)),
+            ('_resolve_symbols', (symbols, resolver)),
+            ])
+
+#        self._return_iter_symbols = [
+#                s_info.Symbol(('dir1/spam.c', None, 'var1'), 'variable', False),
+#                s_info.Symbol(('dir1/spam.c', None, 'var2'), 'variable', False),
+#                s_info.Symbol(('dir1/spam.c', None, 'func1'), 'function', False),
+#                s_info.Symbol(('dir1/spam.c', None, 'func2'), 'function', True),
+#                s_info.Symbol(('dir1/spam.c', None, 'var3'), 'variable', False),
+#                s_info.Symbol(('dir1/spam.c', 'func2', 'var4'), 'variable', False),
+#                s_info.Symbol(('dir1/ham.c', None, 'var1'), 'variable', True),
+#                s_info.Symbol(('dir1/eggs.c', None, 'var1'), 'variable', False),
+#                s_info.Symbol(('dir1/eggs.c', None, 'xyz'), 'other', False),
+#                s_info.Symbol(('dir1/eggs.c', '???', 'var2'), 'variable', False),
+#                s_info.Symbol(('???', None, 'var_x'), 'variable', False),
+#                s_info.Symbol(('???', '???', 'var_y'), 'variable', False),
+#                s_info.Symbol((None, None, '???'), 'other', False),
+#                ]
+#        known = object()
+#
+#        globals_from_binary('python', knownvars=known, **this.kwargs)
+#        found = list(globals_from_symbols(['dir1'], self.iter_symbols))
+#
+#        self.assertEqual(found, [
+#            info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'),
+#            info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'),
+#            info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'),
+#            ])
+#        self.assertEqual(self.calls, [
+#            ('iter_symbols', (['dir1'],)),
+#            ])
+#
+#    def test_no_symbols(self):
+#        self._return_iter_symbols = []
+#
+#        found = list(globals_from_symbols(['dir1'], self.iter_symbols))
+#
+#        self.assertEqual(found, [])
+#        self.assertEqual(self.calls, [
+#            ('iter_symbols', (['dir1'],)),
+#            ])
+
+    # XXX need functional test
+
+
+#class StaticFromDeclarationsTests(_Base):
+#
+#    _return_iter_declarations = ()
+#
+#    def iter_declarations(self, dirnames):
+#        self.calls.append(('iter_declarations', (dirnames,)))
+#        return iter(self._return_iter_declarations)
+#
+#    def test_typical(self):
+#        self._return_iter_declarations = [
+#            None,
+#            info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'),
+#            object(),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'),
+#            object(),
+#            info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'),
+#            object(),
+#            info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'),
+#            object(),
+#            ]
+#
+#        found = list(globals_from_declarations(['dir1'], self.iter_declarations))
+#
+#        self.assertEqual(found, [
+#            info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'),
+#            info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'),
+#            info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'),
+#            ])
+#        self.assertEqual(self.calls, [
+#            ('iter_declarations', (['dir1'],)),
+#            ])
+#
+#    def test_no_declarations(self):
+#        self._return_iter_declarations = []
+#
+#        found = list(globals_from_declarations(['dir1'], self.iter_declarations))
+#
+#        self.assertEqual(found, [])
+#        self.assertEqual(self.calls, [
+#            ('iter_declarations', (['dir1'],)),
+#            ])
+
+
+#class IterVariablesTests(_Base):
+#
+#    _return_from_symbols = ()
+#    _return_from_declarations = ()
+#
+#    def _from_symbols(self, dirnames, iter_symbols):
+#        self.calls.append(('_from_symbols', (dirnames, iter_symbols)))
+#        return iter(self._return_from_symbols)
+#
+#    def _from_declarations(self, dirnames, iter_declarations):
+#        self.calls.append(('_from_declarations', (dirnames, iter_declarations)))
+#        return iter(self._return_from_declarations)
+#
+#    def test_typical(self):
+#        expected = [
+#            info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'),
+#            info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'),
+#            info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'),
+#            ]
+#        self._return_from_symbols = expected
+#
+#        found = list(iter_variables(['dir1'],
+#                                  _from_symbols=self._from_symbols,
+#                                  _from_declarations=self._from_declarations))
+#
+#        self.assertEqual(found, expected)
+#        self.assertEqual(self.calls, [
+#            ('_from_symbols', (['dir1'], b_symbols.iter_symbols)),
+#            ])
+#
+#    def test_no_symbols(self):
+#        self._return_from_symbols = []
+#
+#        found = list(iter_variables(['dir1'],
+#                                  _from_symbols=self._from_symbols,
+#                                  _from_declarations=self._from_declarations))
+#
+#        self.assertEqual(found, [])
+#        self.assertEqual(self.calls, [
+#            ('_from_symbols', (['dir1'], b_symbols.iter_symbols)),
+#            ])
+#
+#    def test_from_binary(self):
+#        expected = [
+#            info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'),
+#            info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'),
+#            info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'),
+#            ]
+#        self._return_from_symbols = expected
+#
+#        found = list(iter_variables(['dir1'], 'platform',
+#                                  _from_symbols=self._from_symbols,
+#                                  _from_declarations=self._from_declarations))
+#
+#        self.assertEqual(found, expected)
+#        self.assertEqual(self.calls, [
+#            ('_from_symbols', (['dir1'], b_symbols.iter_symbols)),
+#            ])
+#
+#    def test_from_symbols(self):
+#        expected = [
+#            info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'),
+#            info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'),
+#            info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'),
+#            ]
+#        self._return_from_symbols = expected
+#
+#        found = list(iter_variables(['dir1'], 'symbols',
+#                                  _from_symbols=self._from_symbols,
+#                                  _from_declarations=self._from_declarations))
+#
+#        self.assertEqual(found, expected)
+#        self.assertEqual(self.calls, [
+#            ('_from_symbols', (['dir1'], s_symbols.iter_symbols)),
+#            ])
+#
+#    def test_from_declarations(self):
+#        expected = [
+#            info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'),
+#            info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'),
+#            info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'),
+#            ]
+#        self._return_from_declarations = expected
+#
+#        found = list(iter_variables(['dir1'], 'declarations',
+#                                  _from_symbols=self._from_symbols,
+#                                  _from_declarations=self._from_declarations))
+#
+#        self.assertEqual(found, expected)
+#        self.assertEqual(self.calls, [
+#            ('_from_declarations', (['dir1'], declarations.iter_all)),
+#            ])
+#
+#    def test_from_preprocessed(self):
+#        expected = [
+#            info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'),
+#            info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'),
+#            info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'),
+#            info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'),
+#            ]
+#        self._return_from_declarations = expected
+#
+#        found = list(iter_variables(['dir1'], 'preprocessed',
+#                                  _from_symbols=self._from_symbols,
+#                                  _from_declarations=self._from_declarations))
+#
+#        self.assertEqual(found, expected)
+#        self.assertEqual(self.calls, [
+#            ('_from_declarations', (['dir1'], declarations.iter_preprocessed)),
+#            ])
+
+
+class StaticsTest(_Base):
+
+    _return_iter_variables = None
+
+    def _iter_variables(self, kind, *, known, dirnames):
+        self.calls.append(
+                ('_iter_variables', (kind, known, dirnames)))
+        return iter(self._return_iter_variables or ())
+
+    def test_typical(self):
+        self._return_iter_variables = [
+            info.Variable.from_parts('src1/spam.c', None, 'var1', 'static const char *'),
+            info.Variable.from_parts('src1/spam.c', None, 'var1b', 'const char *'),
+            info.Variable.from_parts('src1/spam.c', 'ham', 'initialized', 'static int'),
+            info.Variable.from_parts('src1/spam.c', 'ham', 'result', 'int'),
+            info.Variable.from_parts('src1/spam.c', None, 'var2', 'static PyObject *'),
+            info.Variable.from_parts('src1/eggs.c', 'tofu', 'ready', 'static int'),
+            info.Variable.from_parts('src1/spam.c', None, 'freelist', 'static (PyTupleObject *)[10]'),
+            info.Variable.from_parts('src1/sub/ham.c', None, 'var1', 'static const char const *'),
+            info.Variable.from_parts('src2/jam.c', None, 'var1', 'static int'),
+            info.Variable.from_parts('src2/jam.c', None, 'var2', 'static MyObject *'),
+            info.Variable.from_parts('Include/spam.h', None, 'data', 'static const int'),
+            ]
+        dirnames = object()
+        known = object()
+
+        found = list(globals(dirnames, known,
+                             kind='platform',
+                             _iter_variables=self._iter_variables,
+                             ))
+
+        self.assertEqual(found, [
+            info.Variable.from_parts('src1/spam.c', None, 'var1', 'static const char *'),
+            info.Variable.from_parts('src1/spam.c', 'ham', 'initialized', 'static int'),
+            info.Variable.from_parts('src1/spam.c', None, 'var2', 'static PyObject *'),
+            info.Variable.from_parts('src1/eggs.c', 'tofu', 'ready', 'static int'),
+            info.Variable.from_parts('src1/spam.c', None, 'freelist', 'static (PyTupleObject *)[10]'),
+            info.Variable.from_parts('src1/sub/ham.c', None, 'var1', 'static const char const *'),
+            info.Variable.from_parts('src2/jam.c', None, 'var1', 'static int'),
+            info.Variable.from_parts('src2/jam.c', None, 'var2', 'static MyObject *'),
+            info.Variable.from_parts('Include/spam.h', None, 'data', 'static const int'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_iter_variables', ('platform', known, dirnames)),
+            ])
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_functional.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_functional.py
new file mode 100644
index 000000000000..927979048448
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_functional.py
@@ -0,0 +1,34 @@
+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    pass
+
+
+class SelfCheckTests(unittest.TestCase):
+
+    @unittest.expectedFailure
+    def test_known(self):
+        # Make sure known macros & vartypes aren't hiding unknown local types.
+        # XXX finish!
+        raise NotImplementedError
+
+    @unittest.expectedFailure
+    def test_compare_nm_results(self):
+        # Make sure the "show" results match the statics found by "nm" command.
+        # XXX Skip if "nm" is not available.
+        # XXX finish!
+        raise NotImplementedError
+
+
+class DummySourceTests(unittest.TestCase):
+
+    @unittest.expectedFailure
+    def test_check(self):
+        # XXX finish!
+        raise NotImplementedError
+
+    @unittest.expectedFailure
+    def test_show(self):
+        # XXX finish!
+        raise NotImplementedError
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_show.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_show.py
new file mode 100644
index 000000000000..ce1dad85db1b
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_show.py
@@ -0,0 +1,52 @@
+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_parser import info
+    from c_globals.show import basic
+
+
+TYPICAL = [
+        info.Variable.from_parts('src1/spam.c', None, 'var1', 'static const char *'),
+        info.Variable.from_parts('src1/spam.c', 'ham', 'initialized', 'static int'),
+        info.Variable.from_parts('src1/spam.c', None, 'var2', 'static PyObject *'),
+        info.Variable.from_parts('src1/eggs.c', 'tofu', 'ready', 'static int'),
+        info.Variable.from_parts('src1/spam.c', None, 'freelist', 'static (PyTupleObject *)[10]'),
+        info.Variable.from_parts('src1/sub/ham.c', None, 'var1', 'static const char const *'),
+        info.Variable.from_parts('src2/jam.c', None, 'var1', 'static int'),
+        info.Variable.from_parts('src2/jam.c', None, 'var2', 'static MyObject *'),
+        info.Variable.from_parts('Include/spam.h', None, 'data', 'static const int'),
+        ]
+
+
+class BasicTests(unittest.TestCase):
+
+    maxDiff = None
+
+    def setUp(self):
+        self.lines = []
+
+    def print(self, line):
+        self.lines.append(line)
+
+    def test_typical(self):
+        basic(TYPICAL,
+              _print=self.print)
+
+        self.assertEqual(self.lines, [
+            'src1/spam.c:var1                                                 static const char *',
+            'src1/spam.c:ham():initialized                                    static int',
+            'src1/spam.c:var2                                                 static PyObject *',
+            'src1/eggs.c:tofu():ready                                         static int',
+            'src1/spam.c:freelist                                             static (PyTupleObject *)[10]',
+            'src1/sub/ham.c:var1                                              static const char const *',
+            'src2/jam.c:var1                                                  static int',
+            'src2/jam.c:var2                                                  static MyObject *',
+            'Include/spam.h:data                                              static const int',
+            ])
+
+    def test_no_rows(self):
+        basic([],
+              _print=self.print)
+
+        self.assertEqual(self.lines, [])
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py
new file mode 100644
index 000000000000..1e7d40e2afcb
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py
@@ -0,0 +1,96 @@
+import re
+import textwrap
+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_analyzer_common.info import ID
+    from c_parser import info
+    from c_globals.supported import is_supported, ignored_from_file
+
+
+class IsSupportedTests(unittest.TestCase):
+
+    @unittest.expectedFailure
+    def test_supported(self):
+        statics = [
+                info.StaticVar('src1/spam.c', None, 'var1', 'const char *'),
+                info.StaticVar('src1/spam.c', None, 'var1', 'int'),
+                ]
+        for static in statics:
+            with self.subTest(static):
+                result = is_supported(static)
+
+                self.assertTrue(result)
+
+    @unittest.expectedFailure
+    def test_not_supported(self):
+        statics = [
+                info.StaticVar('src1/spam.c', None, 'var1', 'PyObject *'),
+                info.StaticVar('src1/spam.c', None, 'var1', 'PyObject[10]'),
+                ]
+        for static in statics:
+            with self.subTest(static):
+                result = is_supported(static)
+
+                self.assertFalse(result)
+
+
+class IgnoredFromFileTests(unittest.TestCase):
+
+    maxDiff = None
+
+    _return_read_tsv = ()
+
+    @property
+    def calls(self):
+        try:
+            return self._calls
+        except AttributeError:
+            self._calls = []
+            return self._calls
+
+    def _read_tsv(self, *args):
+        self.calls.append(('_read_tsv', args))
+        return self._return_read_tsv
+
+    def test_typical(self):
+        lines = textwrap.dedent('''
+            filename    funcname        name    kind    reason
+            file1.c     -       var1    variable        ...
+            file1.c     func1   local1  variable        |
+            file1.c     -       var2    variable        ???
+            file1.c     func2   local2  variable           |
+            file2.c     -       var1    variable        reasons
+            ''').strip().splitlines()
+        lines = [re.sub(r'\s{1,8}', '\t', line, 4).replace('|', '')
+                 for line in lines]
+        self._return_read_tsv = [tuple(v.strip() for v in line.split('\t'))
+                                 for line in lines[1:]]
+
+        ignored = ignored_from_file('spam.c', _read_tsv=self._read_tsv)
+
+        self.assertEqual(ignored, {
+            'variables': {
+                ID('file1.c', '', 'var1'): '...',
+                ID('file1.c', 'func1', 'local1'): '',
+                ID('file1.c', '', 'var2'): '???',
+                ID('file1.c', 'func2', 'local2'): '',
+                ID('file2.c', '', 'var1'): 'reasons',
+                },
+            })
+        self.assertEqual(self.calls, [
+            ('_read_tsv', ('spam.c', 'filename\tfuncname\tname\tkind\treason')),
+            ])
+
+    def test_empty(self):
+        self._return_read_tsv = []
+
+        ignored = ignored_from_file('spam.c', _read_tsv=self._read_tsv)
+
+        self.assertEqual(ignored, {
+            'variables': {},
+            })
+        self.assertEqual(self.calls, [
+            ('_read_tsv', ('spam.c', 'filename\tfuncname\tname\tkind\treason')),
+            ])
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_declarations.py b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_declarations.py
new file mode 100644
index 000000000000..b68744ef0aba
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_declarations.py
@@ -0,0 +1,795 @@
+import textwrap
+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_parser.declarations import (
+        iter_global_declarations, iter_local_statements,
+        parse_func, parse_var, parse_compound,
+        iter_variables,
+        )
+
+
+class TestCaseBase(unittest.TestCase):
+
+    maxDiff = None
+
+    @property
+    def calls(self):
+        try:
+            return self._calls
+        except AttributeError:
+            self._calls = []
+            return self._calls
+
+
+class IterGlobalDeclarationsTests(TestCaseBase):
+
+    def test_functions(self):
+        tests = [
+            (textwrap.dedent('''
+                void func1() {
+                    return;
+                }
+                '''),
+             textwrap.dedent('''
+                void func1() {
+                return;
+                }
+                ''').strip(),
+             ),
+            (textwrap.dedent('''
+                static unsigned int * _func1(
+                    const char *arg1,
+                    int *arg2
+                    long long arg3
+                    )
+                {
+                    return _do_something(arg1, arg2, arg3);
+                }
+                '''),
+             textwrap.dedent('''
+                static unsigned int * _func1( const char *arg1, int *arg2 long long arg3 ) {
+                return _do_something(arg1, arg2, arg3);
+                }
+                ''').strip(),
+             ),
+            (textwrap.dedent('''
+                static PyObject *
+                _func1(const char *arg1, PyObject *arg2)
+                {
+                    static int initialized = 0;
+                    if (!initialized) {
+                        initialized = 1;
+                        _init(arg1);
+                    }
+
+                    PyObject *result = _do_something(arg1, arg2);
+                    Py_INCREF(result);
+                    return result;
+                }
+                '''),
+             textwrap.dedent('''
+                static PyObject * _func1(const char *arg1, PyObject *arg2) {
+                static int initialized = 0;
+                if (!initialized) {
+                initialized = 1;
+                _init(arg1);
+                }
+                PyObject *result = _do_something(arg1, arg2);
+                Py_INCREF(result);
+                return result;
+                }
+                ''').strip(),
+             ),
+            ]
+        for lines, expected in tests:
+            body = textwrap.dedent(
+                    expected.partition('{')[2].rpartition('}')[0]
+                    ).strip()
+            expected = (expected, body)
+            with self.subTest(lines):
+                lines = lines.splitlines()
+
+                stmts = list(iter_global_declarations(lines))
+
+                self.assertEqual(stmts, [expected])
+
+    @unittest.expectedFailure
+    def test_declarations(self):
+        tests = [
+            'int spam;',
+            'long long spam;',
+            'static const int const *spam;',
+            'int spam;',
+            'typedef int myint;',
+            'typedef PyObject * (*unaryfunc)(PyObject *);',
+            # typedef struct
+            # inline struct
+            # enum
+            # inline enum
+            ]
+        for text in tests:
+            expected = (text,
+                        ' '.join(l.strip() for l in text.splitlines()))
+            with self.subTest(lines):
+                lines = lines.splitlines()
+
+                stmts = list(iter_global_declarations(lines))
+
+                self.assertEqual(stmts, [expected])
+
+    @unittest.expectedFailure
+    def test_declaration_multiple_vars(self):
+        lines = ['static const int const *spam, *ham=NULL, eggs = 3;']
+
+        stmts = list(iter_global_declarations(lines))
+
+        self.assertEqual(stmts, [
+            ('static const int const *spam;', None),
+            ('static const int *ham=NULL;', None),
+            ('static const int eggs = 3;', None),
+            ])
+
+    def test_mixed(self):
+        lines = textwrap.dedent('''
+           int spam;
+           static const char const *eggs;
+
+           PyObject * start(void) {
+               static int initialized = 0;
+               if (initialized) {
+                   initialized = 1;
+                   init();
+               }
+               return _start();
+           }
+
+           char* ham;
+
+           static int stop(char *reason) {
+               ham = reason;
+               return _stop();
+           }
+           ''').splitlines()
+        expected = [
+            (textwrap.dedent('''
+                PyObject * start(void) {
+                static int initialized = 0;
+                if (initialized) {
+                initialized = 1;
+                init();
+                }
+                return _start();
+                }
+                ''').strip(),
+             textwrap.dedent('''
+                static int initialized = 0;
+                if (initialized) {
+                initialized = 1;
+                init();
+                }
+                return _start();
+                ''').strip(),
+             ),
+            (textwrap.dedent('''
+                static int stop(char *reason) {
+                ham = reason;
+                return _stop();
+                }
+                ''').strip(),
+             textwrap.dedent('''
+                ham = reason;
+                return _stop();
+                ''').strip(),
+             ),
+            ]
+
+        stmts = list(iter_global_declarations(lines))
+
+        self.assertEqual(stmts, expected)
+        #self.assertEqual([stmt for stmt, _ in stmts],
+        #                 [stmt for stmt, _ in expected])
+        #self.assertEqual([body for _, body in stmts],
+        #                 [body for _, body in expected])
+
+    def test_no_statements(self):
+        lines = []
+
+        stmts = list(iter_global_declarations(lines))
+
+        self.assertEqual(stmts, [])
+
+    def test_bogus(self):
+        tests = [
+                (textwrap.dedent('''
+                    int spam;
+                    static const char const *eggs;
+
+                    PyObject * start(void) {
+                        static int initialized = 0;
+                        if (initialized) {
+                            initialized = 1;
+                            init();
+                        }
+                        return _start();
+                    }
+
+                    char* ham;
+
+                    static int _stop(void) {
+                    // missing closing bracket
+
+                    static int stop(char *reason) {
+                        ham = reason;
+                        return _stop();
+                    }
+                    '''),
+                 [(textwrap.dedent('''
+                    PyObject * start(void) {
+                    static int initialized = 0;
+                    if (initialized) {
+                    initialized = 1;
+                    init();
+                    }
+                    return _start();
+                    }
+                    ''').strip(),
+                   textwrap.dedent('''
+                    static int initialized = 0;
+                    if (initialized) {
+                    initialized = 1;
+                    init();
+                    }
+                    return _start();
+                    ''').strip(),
+                   ),
+                   # Neither "stop()" nor "_stop()" are here.
+                  ],
+                 ),
+                ]
+        for lines, expected in tests:
+            with self.subTest(lines):
+                lines = lines.splitlines()
+
+                stmts = list(iter_global_declarations(lines))
+
+                self.assertEqual(stmts, expected)
+                #self.assertEqual([stmt for stmt, _ in stmts],
+                #                 [stmt for stmt, _ in expected])
+                #self.assertEqual([body for _, body in stmts],
+                #                 [body for _, body in expected])
+
+    def test_ignore_comments(self):
+        tests = [
+            ('// msg', None),
+            ('// int stmt;', None),
+            ('    // ...    ', None),
+            ('// /*', None),
+            ('/* int stmt; */', None),
+            ("""
+             /**
+              * ...
+              * int stmt;
+              */
+             """, None),
+            ]
+        for lines, expected in tests:
+            with self.subTest(lines):
+                lines = lines.splitlines()
+
+                stmts = list(iter_global_declarations(lines))
+
+                self.assertEqual(stmts, [expected] if expected else [])
+
+
+class IterLocalStatementsTests(TestCaseBase):
+
+    def test_vars(self):
+        tests = [
+            # POTS
+            'int spam;',
+            'unsigned int spam;',
+            'char spam;',
+            'float spam;',
+
+            # typedefs
+            'uint spam;',
+            'MyType spam;',
+
+            # complex
+            'struct myspam spam;',
+            'union choice spam;',
+            # inline struct
+            # inline union
+            # enum?
+            ]
+        # pointers
+        tests.extend([
+            # POTS
+            'int * spam;',
+            'unsigned int * spam;',
+            'char *spam;',
+            'char const *spam = "spamspamspam...";',
+            # typedefs
+            'MyType *spam;',
+            # complex
+            'struct myspam *spam;',
+            'union choice *spam;',
+            # packed with details
+            'const char const *spam;',
+            # void pointer
+            'void *data = NULL;',
+            # function pointers
+            'int (* func)(char *arg1);',
+            'char * (* func)(void);',
+            ])
+        # storage class
+        tests.extend([
+            'static int spam;',
+            'extern int spam;',
+            'static unsigned int spam;',
+            'static struct myspam spam;',
+            ])
+        # type qualifier
+        tests.extend([
+            'const int spam;',
+            'const unsigned int spam;',
+            'const struct myspam spam;',
+            ])
+        # combined
+        tests.extend([
+            'const char *spam = eggs;',
+            'static const char const *spam = "spamspamspam...";',
+            'extern const char const *spam;',
+            'static void *data = NULL;',
+            'static int (const * func)(char *arg1) = func1;',
+            'static char * (* func)(void);',
+            ])
+        for line in tests:
+            expected = line
+            with self.subTest(line):
+                stmts = list(iter_local_statements([line]))
+
+                self.assertEqual(stmts, [(expected, None)])
+
+    @unittest.expectedFailure
+    def test_vars_multiline_var(self):
+        lines = textwrap.dedent('''
+            PyObject *
+            spam
+            = NULL;
+            ''').splitlines()
+        expected = 'PyObject * spam = NULL;'
+
+        stmts = list(iter_local_statements(lines))
+
+        self.assertEqual(stmts, [(expected, None)])
+
+    @unittest.expectedFailure
+    def test_declaration_multiple_vars(self):
+        lines = ['static const int const *spam, *ham=NULL, ham2[]={1, 2, 3}, ham3[2]={1, 2}, eggs = 3;']
+
+        stmts = list(iter_global_declarations(lines))
+
+        self.assertEqual(stmts, [
+            ('static const int const *spam;', None),
+            ('static const int *ham=NULL;', None),
+            ('static const int ham[]={1, 2, 3};', None),
+            ('static const int ham[2]={1, 2};', None),
+            ('static const int eggs = 3;', None),
+            ])
+
+    @unittest.expectedFailure
+    def test_other_simple(self):
+        raise NotImplementedError
+
+    @unittest.expectedFailure
+    def test_compound(self):
+        raise NotImplementedError
+
+    @unittest.expectedFailure
+    def test_mixed(self):
+        raise NotImplementedError
+
+    def test_no_statements(self):
+        lines = []
+
+        stmts = list(iter_local_statements(lines))
+
+        self.assertEqual(stmts, [])
+
+    @unittest.expectedFailure
+    def test_bogus(self):
+        raise NotImplementedError
+
+    def test_ignore_comments(self):
+        tests = [
+            ('// msg', None),
+            ('// int stmt;', None),
+            ('    // ...    ', None),
+            ('// /*', None),
+            ('/* int stmt; */', None),
+            ("""
+             /**
+              * ...
+              * int stmt;
+              */
+             """, None),
+            # mixed with statements
+            ('int stmt; // ...', ('int stmt;', None)),
+            ( 'int stmt; /* ...  */', ('int stmt;', None)),
+            ( '/* ...  */ int stmt;', ('int stmt;', None)),
+            ]
+        for lines, expected in tests:
+            with self.subTest(lines):
+                lines = lines.splitlines()
+
+                stmts = list(iter_local_statements(lines))
+
+                self.assertEqual(stmts, [expected] if expected else [])
+
+
+class ParseFuncTests(TestCaseBase):
+
+    def test_typical(self):
+        tests = [
+            ('PyObject *\nspam(char *a)\n{\nreturn _spam(a);\n}',
+             'return _spam(a);',
+             ('spam', 'PyObject * spam(char *a)'),
+             ),
+            ]
+        for stmt, body, expected in tests:
+            with self.subTest(stmt):
+                name, signature = parse_func(stmt, body)
+
+                self.assertEqual((name, signature), expected)
+
+
+class ParseVarTests(TestCaseBase):
+
+    def test_typical(self):
+        tests = [
+            # POTS
+            ('int spam;', ('spam', 'int')),
+            ('unsigned int spam;', ('spam', 'unsigned int')),
+            ('char spam;', ('spam', 'char')),
+            ('float spam;', ('spam', 'float')),
+
+            # typedefs
+            ('uint spam;', ('spam', 'uint')),
+            ('MyType spam;', ('spam', 'MyType')),
+
+            # complex
+            ('struct myspam spam;', ('spam', 'struct myspam')),
+            ('union choice spam;', ('spam', 'union choice')),
+            # inline struct
+            # inline union
+            # enum?
+            ]
+        # pointers
+        tests.extend([
+            # POTS
+            ('int * spam;', ('spam', 'int *')),
+            ('unsigned int * spam;', ('spam', 'unsigned int *')),
+            ('char *spam;', ('spam', 'char *')),
+            ('char const *spam = "spamspamspam...";', ('spam', 'char const *')),
+            # typedefs
+            ('MyType *spam;', ('spam', 'MyType *')),
+            # complex
+            ('struct myspam *spam;', ('spam', 'struct myspam *')),
+            ('union choice *spam;', ('spam', 'union choice *')),
+            # packed with details
+            ('const char const *spam;', ('spam', 'const char const *')),
+            # void pointer
+            ('void *data = NULL;', ('data', 'void *')),
+            # function pointers
+            ('int (* func)(char *);', ('func', 'int (*)(char *)')),
+            ('char * (* func)(void);', ('func', 'char * (*)(void)')),
+            ])
+        # storage class
+        tests.extend([
+            ('static int spam;', ('spam', 'static int')),
+            ('extern int spam;', ('spam', 'extern int')),
+            ('static unsigned int spam;', ('spam', 'static unsigned int')),
+            ('static struct myspam spam;', ('spam', 'static struct myspam')),
+            ])
+        # type qualifier
+        tests.extend([
+            ('const int spam;', ('spam', 'const int')),
+            ('const unsigned int spam;', ('spam', 'const unsigned int')),
+            ('const struct myspam spam;', ('spam', 'const struct myspam')),
+            ])
+        # combined
+        tests.extend([
+            ('const char *spam = eggs;', ('spam', 'const char *')),
+            ('static const char const *spam = "spamspamspam...";',
+             ('spam', 'static const char const *')),
+            ('extern const char const *spam;',
+             ('spam', 'extern const char const *')),
+            ('static void *data = NULL;', ('data', 'static void *')),
+            ('static int (const * func)(char *) = func1;',
+             ('func', 'static int (const *)(char *)')),
+            ('static char * (* func)(void);',
+             ('func', 'static char * (*)(void)')),
+            ])
+        for stmt, expected in tests:
+            with self.subTest(stmt):
+                name, vartype = parse_var(stmt)
+
+                self.assertEqual((name, vartype), expected)
+
+
+ at unittest.skip('not finished')
+class ParseCompoundTests(TestCaseBase):
+
+    def test_typical(self):
+        headers, bodies = parse_compound(stmt, blocks)
+        ...
+
+
+class IterVariablesTests(TestCaseBase):
+
+    _return_iter_source_lines = None
+    _return_iter_global = None
+    _return_iter_local = None
+    _return_parse_func = None
+    _return_parse_var = None
+    _return_parse_compound = None
+
+    def _iter_source_lines(self, filename):
+        self.calls.append(
+                ('_iter_source_lines', (filename,)))
+        return self._return_iter_source_lines.splitlines()
+
+    def _iter_global(self, lines):
+        self.calls.append(
+                ('_iter_global', (lines,)))
+        try:
+            return self._return_iter_global.pop(0)
+        except IndexError:
+            return ('???', None)
+
+    def _iter_local(self, lines):
+        self.calls.append(
+                ('_iter_local', (lines,)))
+        try:
+            return self._return_iter_local.pop(0)
+        except IndexError:
+            return ('???', None)
+
+    def _parse_func(self, stmt, body):
+        self.calls.append(
+                ('_parse_func', (stmt, body)))
+        try:
+            return self._return_parse_func.pop(0)
+        except IndexError:
+            return ('???', '???')
+
+    def _parse_var(self, lines):
+        self.calls.append(
+                ('_parse_var', (lines,)))
+        try:
+            return self._return_parse_var.pop(0)
+        except IndexError:
+            return ('???', '???')
+
+    def _parse_compound(self, stmt, blocks):
+        self.calls.append(
+                ('_parse_compound', (stmt, blocks)))
+        try:
+            return self._return_parse_compound.pop(0)
+        except IndexError:
+            return (['???'], ['???'])
+
+    def test_empty_file(self):
+        self._return_iter_source_lines = ''
+        self._return_iter_global = [
+            [],
+            ]
+        self._return_parse_func = None
+        self._return_parse_var = None
+        self._return_parse_compound = None
+
+        srcvars = list(iter_variables('spam.c',
+                                      _iter_source_lines=self._iter_source_lines,
+                                      _iter_global=self._iter_global,
+                                      _iter_local=self._iter_local,
+                                      _parse_func=self._parse_func,
+                                      _parse_var=self._parse_var,
+                                      _parse_compound=self._parse_compound,
+                                      ))
+
+        self.assertEqual(srcvars, [])
+        self.assertEqual(self.calls, [
+            ('_iter_source_lines', ('spam.c',)),
+            ('_iter_global', ([],)),
+            ])
+
+    def test_no_statements(self):
+        content = textwrap.dedent('''
+        ...
+        ''')
+        self._return_iter_source_lines = content
+        self._return_iter_global = [
+            [],
+            ]
+        self._return_parse_func = None
+        self._return_parse_var = None
+        self._return_parse_compound = None
+
+        srcvars = list(iter_variables('spam.c',
+                                      _iter_source_lines=self._iter_source_lines,
+                                      _iter_global=self._iter_global,
+                                      _iter_local=self._iter_local,
+                                      _parse_func=self._parse_func,
+                                      _parse_var=self._parse_var,
+                                      _parse_compound=self._parse_compound,
+                                      ))
+
+        self.assertEqual(srcvars, [])
+        self.assertEqual(self.calls, [
+            ('_iter_source_lines', ('spam.c',)),
+            ('_iter_global', (content.splitlines(),)),
+            ])
+
+    def test_typical(self):
+        content = textwrap.dedent('''
+        ...
+        ''')
+        self._return_iter_source_lines = content
+        self._return_iter_global = [
+            [('<lines 1>', None),  # var1
+             ('<lines 2>', None),  # non-var
+             ('<lines 3>', None),  # var2
+             ('<lines 4>', '<body 1>'),  # func1
+             ('<lines 9>', None),  # var4
+             ],
+            ]
+        self._return_iter_local = [
+            # func1
+            [('<lines 5>', None),  # var3
+             ('<lines 6>', [('<header 1>', '<block 1>')]),  # if
+             ('<lines 8>', None),  # non-var
+             ],
+            # if
+            [('<lines 7>', None),  # var2 ("collision" with global var)
+             ],
+            ]
+        self._return_parse_func = [
+            ('func1', '<sig 1>'),
+            ]
+        self._return_parse_var = [
+            ('var1', '<vartype 1>'),
+            (None, None),
+            ('var2', '<vartype 2>'),
+            ('var3', '<vartype 3>'),
+            ('var2', '<vartype 2b>'),
+            ('var4', '<vartype 4>'),
+            (None, None),
+            (None, None),
+            (None, None),
+            ('var5', '<vartype 5>'),
+            ]
+        self._return_parse_compound = [
+            ([[
+                'if (',
+                '<simple>',
+                ')',
+                ],
+              ],
+             ['<block 1>']),
+            ]
+
+        srcvars = list(iter_variables('spam.c',
+                                      _iter_source_lines=self._iter_source_lines,
+                                      _iter_global=self._iter_global,
+                                      _iter_local=self._iter_local,
+                                      _parse_func=self._parse_func,
+                                      _parse_var=self._parse_var,
+                                      _parse_compound=self._parse_compound,
+                                      ))
+
+        self.assertEqual(srcvars, [
+            (None, 'var1', '<vartype 1>'),
+            (None, 'var2', '<vartype 2>'),
+            ('func1', 'var3', '<vartype 3>'),
+            ('func1', 'var2', '<vartype 2b>'),
+            ('func1', 'var4', '<vartype 4>'),
+            (None, 'var5', '<vartype 5>'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_iter_source_lines', ('spam.c',)),
+            ('_iter_global', (content.splitlines(),)),
+            ('_parse_var', ('<lines 1>',)),
+            ('_parse_var', ('<lines 2>',)),
+            ('_parse_var', ('<lines 3>',)),
+            ('_parse_func', ('<lines 4>', '<body 1>')),
+            ('_iter_local', (['<body 1>'],)),
+            ('_parse_var', ('<lines 5>',)),
+            ('_parse_compound', ('<lines 6>', [('<header 1>', '<block 1>')])),
+            ('_parse_var', ('if (',)),
+            ('_parse_var', ('<simple>',)),
+            ('_parse_var', (')',)),
+            ('_parse_var', ('<lines 8>',)),
+            ('_iter_local', (['<block 1>'],)),
+            ('_parse_var', ('<lines 7>',)),
+            ('_parse_var', ('<lines 9>',)),
+            ])
+
+    def test_no_locals(self):
+        content = textwrap.dedent('''
+        ...
+        ''')
+        self._return_iter_source_lines = content
+        self._return_iter_global = [
+            [('<lines 1>', None),  # var1
+             ('<lines 2>', None),  # non-var
+             ('<lines 3>', None),  # var2
+             ('<lines 4>', '<body 1>'),  # func1
+             ],
+            ]
+        self._return_iter_local = [
+            # func1
+            [('<lines 5>', None),  # non-var
+             ('<lines 6>', [('<header 1>', '<block 1>')]),  # if
+             ('<lines 8>', None),  # non-var
+             ],
+            # if
+            [('<lines 7>', None),  # non-var
+             ],
+            ]
+        self._return_parse_func = [
+            ('func1', '<sig 1>'),
+            ]
+        self._return_parse_var = [
+            ('var1', '<vartype 1>'),
+            (None, None),
+            ('var2', '<vartype 2>'),
+            (None, None),
+            (None, None),
+            (None, None),
+            (None, None),
+            (None, None),
+            (None, None),
+            ]
+        self._return_parse_compound = [
+            ([[
+                'if (',
+                '<simple>',
+                ')',
+                ],
+              ],
+             ['<block 1>']),
+            ]
+
+        srcvars = list(iter_variables('spam.c',
+                                      _iter_source_lines=self._iter_source_lines,
+                                      _iter_global=self._iter_global,
+                                      _iter_local=self._iter_local,
+                                      _parse_func=self._parse_func,
+                                      _parse_var=self._parse_var,
+                                      _parse_compound=self._parse_compound,
+                                      ))
+
+        self.assertEqual(srcvars, [
+            (None, 'var1', '<vartype 1>'),
+            (None, 'var2', '<vartype 2>'),
+            ])
+        self.assertEqual(self.calls, [
+            ('_iter_source_lines', ('spam.c',)),
+            ('_iter_global', (content.splitlines(),)),
+            ('_parse_var', ('<lines 1>',)),
+            ('_parse_var', ('<lines 2>',)),
+            ('_parse_var', ('<lines 3>',)),
+            ('_parse_func', ('<lines 4>', '<body 1>')),
+            ('_iter_local', (['<body 1>'],)),
+            ('_parse_var', ('<lines 5>',)),
+            ('_parse_compound', ('<lines 6>', [('<header 1>', '<block 1>')])),
+            ('_parse_var', ('if (',)),
+            ('_parse_var', ('<simple>',)),
+            ('_parse_var', (')',)),
+            ('_parse_var', ('<lines 8>',)),
+            ('_iter_local', (['<block 1>'],)),
+            ('_parse_var', ('<lines 7>',)),
+            ])
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py
new file mode 100644
index 000000000000..1dfe5d066a33
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py
@@ -0,0 +1,208 @@
+import string
+import unittest
+
+from ..util import PseudoStr, StrProxy, Object
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_analyzer_common.info import ID
+    from c_parser.info import (
+        normalize_vartype, Variable,
+        )
+
+
+class NormalizeVartypeTests(unittest.TestCase):
+
+    def test_basic(self):
+        tests = [
+                (None, None),
+                ('', ''),
+                ('int', 'int'),
+                (PseudoStr('int'), 'int'),
+                (StrProxy('int'), 'int'),
+                ]
+        for vartype, expected in tests:
+            with self.subTest(vartype):
+                normalized = normalize_vartype(vartype)
+
+                self.assertEqual(normalized, expected)
+
+
+class VariableTests(unittest.TestCase):
+
+    VALID_ARGS = (
+            ('x/y/z/spam.c', 'func', 'eggs'),
+            'int',
+            )
+    VALID_KWARGS = dict(zip(Variable._fields, VALID_ARGS))
+    VALID_EXPECTED = VALID_ARGS
+
+    def test_init_typical_global(self):
+        static = Variable(
+                id=ID(
+                    filename='x/y/z/spam.c',
+                    funcname=None,
+                    name='eggs',
+                    ),
+                vartype='int',
+                )
+
+        self.assertEqual(static, (
+                ('x/y/z/spam.c', None, 'eggs'),
+                'int',
+                ))
+
+    def test_init_typical_local(self):
+        static = Variable(
+                id=ID(
+                    filename='x/y/z/spam.c',
+                    funcname='func',
+                    name='eggs',
+                    ),
+                vartype='int',
+                )
+
+        self.assertEqual(static, (
+                ('x/y/z/spam.c', 'func', 'eggs'),
+                'int',
+                ))
+
+    def test_init_all_missing(self):
+        for value in ('', None):
+            with self.subTest(repr(value)):
+                static = Variable(
+                        id=value,
+                        vartype=value,
+                        )
+
+                self.assertEqual(static, (
+                        None,
+                        None,
+                        ))
+
+    def test_init_all_coerced(self):
+        id = ID('x/y/z/spam.c', 'func', 'spam')
+        tests = [
+            ('str subclass',
+             dict(
+                 id=(
+                    PseudoStr('x/y/z/spam.c'),
+                    PseudoStr('func'),
+                    PseudoStr('spam'),
+                    ),
+                 vartype=PseudoStr('int'),
+                 ),
+             (id,
+              'int',
+              )),
+            ('non-str 1',
+             dict(
+                 id=id,
+                 vartype=Object(),
+                 ),
+             (id,
+              '<object>',
+              )),
+            ('non-str 2',
+             dict(
+                 id=id,
+                 vartype=StrProxy('variable'),
+                 ),
+             (id,
+              'variable',
+              )),
+            ('non-str',
+             dict(
+                 id=id,
+                 vartype=('a', 'b', 'c'),
+                 ),
+             (id,
+              "('a', 'b', 'c')",
+              )),
+            ]
+        for summary, kwargs, expected in tests:
+            with self.subTest(summary):
+                static = Variable(**kwargs)
+
+                for field in Variable._fields:
+                    value = getattr(static, field)
+                    if field == 'id':
+                        self.assertIs(type(value), ID)
+                    else:
+                        self.assertIs(type(value), str)
+                self.assertEqual(tuple(static), expected)
+
+    def test_iterable(self):
+        static = Variable(**self.VALID_KWARGS)
+
+        id, vartype = static
+
+        values = (id, vartype)
+        for value, expected in zip(values, self.VALID_EXPECTED):
+            self.assertEqual(value, expected)
+
+    def test_fields(self):
+        static = Variable(('a', 'b', 'z'), 'x')
+
+        self.assertEqual(static.id, ('a', 'b', 'z'))
+        self.assertEqual(static.vartype, 'x')
+
+    def test___getattr__(self):
+        static = Variable(('a', 'b', 'z'), 'x')
+
+        self.assertEqual(static.filename, 'a')
+        self.assertEqual(static.funcname, 'b')
+        self.assertEqual(static.name, 'z')
+
+    def test_validate_typical(self):
+        static = Variable(
+                id=ID(
+                    filename='x/y/z/spam.c',
+                    funcname='func',
+                    name='eggs',
+                    ),
+                vartype='int',
+                )
+
+        static.validate()  # This does not fail.
+
+    def test_validate_missing_field(self):
+        for field in Variable._fields:
+            with self.subTest(field):
+                static = Variable(**self.VALID_KWARGS)
+                static = static._replace(**{field: None})
+
+                with self.assertRaises(TypeError):
+                    static.validate()
+
+    def test_validate_bad_field(self):
+        badch = tuple(c for c in string.punctuation + string.digits)
+        notnames = (
+                '1a',
+                'a.b',
+                'a-b',
+                '&a',
+                'a++',
+                ) + badch
+        tests = [
+            ('id', ()),  # Any non-empty str is okay.
+            ('vartype', ()),  # Any non-empty str is okay.
+            ]
+        seen = set()
+        for field, invalid in tests:
+            for value in invalid:
+                seen.add(value)
+                with self.subTest(f'{field}={value!r}'):
+                    static = Variable(**self.VALID_KWARGS)
+                    static = static._replace(**{field: value})
+
+                    with self.assertRaises(ValueError):
+                        static.validate()
+
+        for field, invalid in tests:
+            valid = seen - set(invalid)
+            for value in valid:
+                with self.subTest(f'{field}={value!r}'):
+                    static = Variable(**self.VALID_KWARGS)
+                    static = static._replace(**{field: value})
+
+                    static.validate()  # This does not fail.
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_preprocessor.py b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_preprocessor.py
new file mode 100644
index 000000000000..89e15570d653
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_preprocessor.py
@@ -0,0 +1,1562 @@
+import itertools
+import textwrap
+import unittest
+import sys
+
+from ..util import wrapped_arg_combos, StrProxy
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_parser.preprocessor import (
+        iter_lines,
+        # directives
+        parse_directive, PreprocessorDirective,
+        Constant, Macro, IfDirective, Include, OtherDirective,
+        )
+
+
+class TestCaseBase(unittest.TestCase):
+
+    maxDiff = None
+
+    def reset(self):
+        self._calls = []
+        self.errors = None
+
+    @property
+    def calls(self):
+        try:
+            return self._calls
+        except AttributeError:
+            self._calls = []
+            return self._calls
+
+    errors = None
+
+    def try_next_exc(self):
+        if not self.errors:
+            return
+        if exc := self.errors.pop(0):
+            raise exc
+
+    def check_calls(self, *expected):
+        self.assertEqual(self.calls, list(expected))
+        self.assertEqual(self.errors or [], [])
+
+
+class IterLinesTests(TestCaseBase):
+
+    parsed = None
+
+    def check_calls(self, *expected):
+        super().check_calls(*expected)
+        self.assertEqual(self.parsed or [], [])
+
+    def _parse_directive(self, line):
+        self.calls.append(
+                ('_parse_directive', line))
+        self.try_next_exc()
+        return self.parsed.pop(0)
+
+    def test_no_lines(self):
+        lines = []
+
+        results = list(
+                iter_lines(lines, _parse_directive=self._parse_directive))
+
+        self.assertEqual(results, [])
+        self.check_calls()
+
+    def test_no_directives(self):
+        lines = textwrap.dedent('''
+
+            // xyz
+            typedef enum {
+                SPAM
+                EGGS
+            } kind;
+
+            struct info {
+                kind kind;
+                int status;
+            };
+
+            typedef struct spam {
+                struct info info;
+            } myspam;
+
+            static int spam = 0;
+
+            /**
+             * ...
+             */
+            static char *
+            get_name(int arg,
+                     char *default,
+                     )
+            {
+                return default
+            }
+
+            int check(void) {
+                return 0;
+            }
+
+            ''')[1:-1].splitlines()
+        expected = [(lno, line, None, ())
+                    for lno, line in enumerate(lines, 1)]
+        expected[1] = (2, ' ', None, ())
+        expected[20] = (21, ' ', None, ())
+        del expected[19]
+        del expected[18]
+
+        results = list(
+                iter_lines(lines, _parse_directive=self._parse_directive))
+
+        self.assertEqual(results, expected)
+        self.check_calls()
+
+    def test_single_directives(self):
+        tests = [
+            ('#include <stdio>', Include('<stdio>')),
+            ('#define SPAM 1', Constant('SPAM', '1')),
+            ('#define SPAM() 1', Macro('SPAM', (), '1')),
+            ('#define SPAM(a, b) a = b;', Macro('SPAM', ('a', 'b'), 'a = b;')),
+            ('#if defined(SPAM)', IfDirective('if', 'defined(SPAM)')),
+            ('#ifdef SPAM', IfDirective('ifdef', 'SPAM')),
+            ('#ifndef SPAM', IfDirective('ifndef', 'SPAM')),
+            ('#elseif defined(SPAM)', IfDirective('elseif', 'defined(SPAM)')),
+            ('#else', OtherDirective('else', None)),
+            ('#endif', OtherDirective('endif', None)),
+            ('#error ...', OtherDirective('error', '...')),
+            ('#warning ...', OtherDirective('warning', '...')),
+            ('#__FILE__ ...', OtherDirective('__FILE__', '...')),
+            ('#__LINE__ ...', OtherDirective('__LINE__', '...')),
+            ('#__DATE__ ...', OtherDirective('__DATE__', '...')),
+            ('#__TIME__ ...', OtherDirective('__TIME__', '...')),
+            ('#__TIMESTAMP__ ...', OtherDirective('__TIMESTAMP__', '...')),
+            ]
+        for line, directive in tests:
+            with self.subTest(line):
+                self.reset()
+                self.parsed = [
+                    directive,
+                    ]
+                text = textwrap.dedent('''
+                    static int spam = 0;
+                    {}
+                    static char buffer[256];
+                    ''').strip().format(line)
+                lines = text.strip().splitlines()
+
+                results = list(
+                        iter_lines(lines, _parse_directive=self._parse_directive))
+
+                self.assertEqual(results, [
+                    (1, 'static int spam = 0;', None, ()),
+                    (2, line, directive, ()),
+                    ((3, 'static char buffer[256];', None, ('defined(SPAM)',))
+                     if directive.kind in ('if', 'ifdef', 'elseif')
+                     else (3, 'static char buffer[256];', None, ('! defined(SPAM)',))
+                     if directive.kind == 'ifndef'
+                     else (3, 'static char buffer[256];', None, ())),
+                    ])
+                self.check_calls(
+                        ('_parse_directive', line),
+                        )
+
+    def test_directive_whitespace(self):
+        line = ' # define  eggs  (  a  ,  b  )  {  a  =  b  ;  }  '
+        directive = Macro('eggs', ('a', 'b'), '{ a = b; }')
+        self.parsed = [
+            directive,
+            ]
+        lines = [line]
+
+        results = list(
+                iter_lines(lines, _parse_directive=self._parse_directive))
+
+        self.assertEqual(results, [
+            (1, line, directive, ()),
+            ])
+        self.check_calls(
+                ('_parse_directive', '#define eggs ( a , b ) { a = b ; }'),
+                )
+
+    @unittest.skipIf(sys.platform == 'win32', 'needs fix under Windows')
+    def test_split_lines(self):
+        directive = Macro('eggs', ('a', 'b'), '{ a = b; }')
+        self.parsed = [
+            directive,
+            ]
+        text = textwrap.dedent(r'''
+            static int spam = 0;
+            #define eggs(a, b) \
+                { \
+                    a = b; \
+                }
+            static char buffer[256];
+            ''').strip()
+        lines = [line + '\n' for line in text.splitlines()]
+        lines[-1] = lines[-1][:-1]
+
+        results = list(
+                iter_lines(lines, _parse_directive=self._parse_directive))
+
+        self.assertEqual(results, [
+            (1, 'static int spam = 0;\n', None, ()),
+            (5, '#define eggs(a, b)      {          a = b;      }\n', directive, ()),
+            (6, 'static char buffer[256];', None, ()),
+            ])
+        self.check_calls(
+                ('_parse_directive', '#define eggs(a, b) { a = b; }'),
+                )
+
+    def test_nested_conditions(self):
+        directives = [
+            IfDirective('ifdef', 'SPAM'),
+            IfDirective('if', 'SPAM == 1'),
+            IfDirective('elseif', 'SPAM == 2'),
+            OtherDirective('else', None),
+            OtherDirective('endif', None),
+            OtherDirective('endif', None),
+            ]
+        self.parsed = list(directives)
+        text = textwrap.dedent(r'''
+            static int spam = 0;
+
+            #ifdef SPAM
+            static int start = 0;
+            #  if SPAM == 1
+            static char buffer[10];
+            #  elif SPAM == 2
+            static char buffer[100];
+            #  else
+            static char buffer[256];
+            #  endif
+            static int end = 0;
+            #endif
+
+            static int eggs = 0;
+            ''').strip()
+        lines = [line for line in text.splitlines() if line.strip()]
+
+        results = list(
+                iter_lines(lines, _parse_directive=self._parse_directive))
+
+        self.assertEqual(results, [
+            (1, 'static int spam = 0;', None, ()),
+            (2, '#ifdef SPAM', directives[0], ()),
+            (3, 'static int start = 0;', None, ('defined(SPAM)',)),
+            (4, '#  if SPAM == 1', directives[1], ('defined(SPAM)',)),
+            (5, 'static char buffer[10];', None, ('defined(SPAM)', 'SPAM == 1')),
+            (6, '#  elif SPAM == 2', directives[2], ('defined(SPAM)', 'SPAM == 1')),
+            (7, 'static char buffer[100];', None, ('defined(SPAM)', '! (SPAM == 1)', 'SPAM == 2')),
+            (8, '#  else', directives[3], ('defined(SPAM)', '! (SPAM == 1)', 'SPAM == 2')),
+            (9, 'static char buffer[256];', None, ('defined(SPAM)', '! (SPAM == 1)', '! (SPAM == 2)')),
+            (10, '#  endif', directives[4], ('defined(SPAM)', '! (SPAM == 1)', '! (SPAM == 2)')),
+            (11, 'static int end = 0;', None, ('defined(SPAM)',)),
+            (12, '#endif', directives[5], ('defined(SPAM)',)),
+            (13, 'static int eggs = 0;', None, ()),
+            ])
+        self.check_calls(
+                ('_parse_directive', '#ifdef SPAM'),
+                ('_parse_directive', '#if SPAM == 1'),
+                ('_parse_directive', '#elif SPAM == 2'),
+                ('_parse_directive', '#else'),
+                ('_parse_directive', '#endif'),
+                ('_parse_directive', '#endif'),
+                )
+
+    def test_split_blocks(self):
+        directives = [
+            IfDirective('ifdef', 'SPAM'),
+            OtherDirective('else', None),
+            OtherDirective('endif', None),
+            ]
+        self.parsed = list(directives)
+        text = textwrap.dedent(r'''
+            void str_copy(char *buffer, *orig);
+
+            int init(char *name) {
+                static int initialized = 0;
+                if (initialized) {
+                    return 0;
+                }
+            #ifdef SPAM
+                static char buffer[10];
+                str_copy(buffer, char);
+            }
+
+            void copy(char *buffer, *orig) {
+                strncpy(buffer, orig, 9);
+                buffer[9] = 0;
+            }
+
+            #else
+                static char buffer[256];
+                str_copy(buffer, char);
+            }
+
+            void copy(char *buffer, *orig) {
+                strcpy(buffer, orig);
+            }
+
+            #endif
+            ''').strip()
+        lines = [line for line in text.splitlines() if line.strip()]
+
+        results = list(
+                iter_lines(lines, _parse_directive=self._parse_directive))
+
+        self.assertEqual(results, [
+            (1, 'void str_copy(char *buffer, *orig);', None, ()),
+            (2, 'int init(char *name) {', None, ()),
+            (3, '    static int initialized = 0;', None, ()),
+            (4, '    if (initialized) {', None, ()),
+            (5, '        return 0;', None, ()),
+            (6, '    }', None, ()),
+
+            (7, '#ifdef SPAM', directives[0], ()),
+
+            (8, '    static char buffer[10];', None, ('defined(SPAM)',)),
+            (9, '    str_copy(buffer, char);', None, ('defined(SPAM)',)),
+            (10, '}', None, ('defined(SPAM)',)),
+            (11, 'void copy(char *buffer, *orig) {', None, ('defined(SPAM)',)),
+            (12, '    strncpy(buffer, orig, 9);', None, ('defined(SPAM)',)),
+            (13, '    buffer[9] = 0;', None, ('defined(SPAM)',)),
+            (14, '}', None, ('defined(SPAM)',)),
+
+            (15, '#else', directives[1], ('defined(SPAM)',)),
+
+            (16, '    static char buffer[256];', None, ('! (defined(SPAM))',)),
+            (17, '    str_copy(buffer, char);', None, ('! (defined(SPAM))',)),
+            (18, '}', None, ('! (defined(SPAM))',)),
+            (19, 'void copy(char *buffer, *orig) {', None, ('! (defined(SPAM))',)),
+            (20, '    strcpy(buffer, orig);', None, ('! (defined(SPAM))',)),
+            (21, '}', None, ('! (defined(SPAM))',)),
+
+            (22, '#endif', directives[2], ('! (defined(SPAM))',)),
+            ])
+        self.check_calls(
+                ('_parse_directive', '#ifdef SPAM'),
+                ('_parse_directive', '#else'),
+                ('_parse_directive', '#endif'),
+                )
+
+    @unittest.skipIf(sys.platform == 'win32', 'needs fix under Windows')
+    def test_basic(self):
+        directives = [
+            Include('<stdio.h>'),
+            IfDirective('ifdef', 'SPAM'),
+            IfDirective('if', '! defined(HAM) || !HAM'),
+            Constant('HAM', '0'),
+            IfDirective('elseif', 'HAM < 0'),
+            Constant('HAM', '-1'),
+            OtherDirective('else', None),
+            OtherDirective('endif', None),
+            OtherDirective('endif', None),
+            IfDirective('if', 'defined(HAM) && (HAM < 0 || ! HAM)'),
+            OtherDirective('undef', 'HAM'),
+            OtherDirective('endif', None),
+            IfDirective('ifndef', 'HAM'),
+            OtherDirective('endif', None),
+            ]
+        self.parsed = list(directives)
+        text = textwrap.dedent(r'''
+            #include <stdio.h>
+            print("begin");
+            #ifdef SPAM
+               print("spam");
+               #if ! defined(HAM) || !HAM
+            #      DEFINE HAM 0
+               #elseif HAM < 0
+            #      DEFINE HAM -1
+               #else
+                   print("ham HAM");
+               #endif
+            #endif
+
+            #if defined(HAM) && \
+                (HAM < 0 || ! HAM)
+              print("ham?");
+              #undef HAM
+            # endif
+
+            #ifndef HAM
+               print("no ham");
+            #endif
+            print("end");
+            ''')[1:-1]
+        lines = [line + '\n' for line in text.splitlines()]
+        lines[-1] = lines[-1][:-1]
+
+        results = list(
+                iter_lines(lines, _parse_directive=self._parse_directive))
+
+        self.assertEqual(results, [
+            (1, '#include <stdio.h>\n', Include('<stdio.h>'), ()),
+            (2, 'print("begin");\n', None, ()),
+            #
+            (3, '#ifdef SPAM\n',
+                IfDirective('ifdef', 'SPAM'),
+                ()),
+            (4, '   print("spam");\n',
+                None,
+                ('defined(SPAM)',)),
+            (5, '   #if ! defined(HAM) || !HAM\n',
+                IfDirective('if', '! defined(HAM) || !HAM'),
+                ('defined(SPAM)',)),
+            (6, '#      DEFINE HAM 0\n',
+                Constant('HAM', '0'),
+                ('defined(SPAM)', '! defined(HAM) || !HAM')),
+            (7, '   #elseif HAM < 0\n',
+                IfDirective('elseif', 'HAM < 0'),
+                ('defined(SPAM)', '! defined(HAM) || !HAM')),
+            (8, '#      DEFINE HAM -1\n',
+                Constant('HAM', '-1'),
+                ('defined(SPAM)', '! (! defined(HAM) || !HAM)', 'HAM < 0')),
+            (9, '   #else\n',
+                OtherDirective('else', None),
+                ('defined(SPAM)', '! (! defined(HAM) || !HAM)', 'HAM < 0')),
+            (10, '       print("ham HAM");\n',
+                None,
+                ('defined(SPAM)', '! (! defined(HAM) || !HAM)', '! (HAM < 0)')),
+            (11, '   #endif\n',
+                OtherDirective('endif', None),
+                ('defined(SPAM)', '! (! defined(HAM) || !HAM)', '! (HAM < 0)')),
+            (12, '#endif\n',
+                OtherDirective('endif', None),
+                ('defined(SPAM)',)),
+            #
+            (13, '\n', None, ()),
+            #
+            (15, '#if defined(HAM) &&      (HAM < 0 || ! HAM)\n',
+                IfDirective('if', 'defined(HAM) && (HAM < 0 || ! HAM)'),
+                ()),
+            (16, '  print("ham?");\n',
+                None,
+                ('defined(HAM) && (HAM < 0 || ! HAM)',)),
+            (17, '  #undef HAM\n',
+                OtherDirective('undef', 'HAM'),
+                ('defined(HAM) && (HAM < 0 || ! HAM)',)),
+            (18, '# endif\n',
+                OtherDirective('endif', None),
+                ('defined(HAM) && (HAM < 0 || ! HAM)',)),
+            #
+            (19, '\n', None, ()),
+            #
+            (20, '#ifndef HAM\n',
+                IfDirective('ifndef', 'HAM'),
+                ()),
+            (21, '   print("no ham");\n',
+                None,
+                ('! defined(HAM)',)),
+            (22, '#endif\n',
+                OtherDirective('endif', None),
+                ('! defined(HAM)',)),
+            #
+            (23, 'print("end");', None, ()),
+            ])
+
+    @unittest.skipIf(sys.platform == 'win32', 'needs fix under Windows')
+    def test_typical(self):
+        # We use Include/compile.h from commit 66c4f3f38b86.  It has
+        # a good enough mix of code without being too large.
+        directives = [
+            IfDirective('ifndef', 'Py_COMPILE_H'),
+            Constant('Py_COMPILE_H', None),
+
+            IfDirective('ifndef', 'Py_LIMITED_API'),
+
+            Include('"code.h"'),
+
+            IfDirective('ifdef', '__cplusplus'),
+            OtherDirective('endif', None),
+
+            Constant('PyCF_MASK', '(CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS)'),
+            Constant('PyCF_MASK_OBSOLETE', '(CO_NESTED)'),
+            Constant('PyCF_SOURCE_IS_UTF8', ' 0x0100'),
+            Constant('PyCF_DONT_IMPLY_DEDENT', '0x0200'),
+            Constant('PyCF_ONLY_AST', '0x0400'),
+            Constant('PyCF_IGNORE_COOKIE', '0x0800'),
+            Constant('PyCF_TYPE_COMMENTS', '0x1000'),
+            Constant('PyCF_ALLOW_TOP_LEVEL_AWAIT', '0x2000'),
+
+            IfDirective('ifndef', 'Py_LIMITED_API'),
+            OtherDirective('endif', None),
+
+            Constant('FUTURE_NESTED_SCOPES', '"nested_scopes"'),
+            Constant('FUTURE_GENERATORS', '"generators"'),
+            Constant('FUTURE_DIVISION', '"division"'),
+            Constant('FUTURE_ABSOLUTE_IMPORT', '"absolute_import"'),
+            Constant('FUTURE_WITH_STATEMENT', '"with_statement"'),
+            Constant('FUTURE_PRINT_FUNCTION', '"print_function"'),
+            Constant('FUTURE_UNICODE_LITERALS', '"unicode_literals"'),
+            Constant('FUTURE_BARRY_AS_BDFL', '"barry_as_FLUFL"'),
+            Constant('FUTURE_GENERATOR_STOP', '"generator_stop"'),
+            Constant('FUTURE_ANNOTATIONS', '"annotations"'),
+
+            Macro('PyAST_Compile', ('mod', 's', 'f', 'ar'), 'PyAST_CompileEx(mod, s, f, -1, ar)'),
+
+            Constant('PY_INVALID_STACK_EFFECT', 'INT_MAX'),
+
+            IfDirective('ifdef', '__cplusplus'),
+            OtherDirective('endif', None),
+
+            OtherDirective('endif', None),  # ifndef Py_LIMITED_API
+
+            Constant('Py_single_input', '256'),
+            Constant('Py_file_input', '257'),
+            Constant('Py_eval_input', '258'),
+            Constant('Py_func_type_input', '345'),
+
+            OtherDirective('endif', None),  # ifndef Py_COMPILE_H
+            ]
+        self.parsed = list(directives)
+        text = textwrap.dedent(r'''
+            #ifndef Py_COMPILE_H
+            #define Py_COMPILE_H
+
+            #ifndef Py_LIMITED_API
+            #include "code.h"
+
+            #ifdef __cplusplus
+            extern "C" {
+            #endif
+
+            /* Public interface */
+            struct _node; /* Declare the existence of this type */
+            PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
+            /* XXX (ncoghlan): Unprefixed type name in a public API! */
+
+            #define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \
+                               CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \
+                               CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \
+                               CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS)
+            #define PyCF_MASK_OBSOLETE (CO_NESTED)
+            #define PyCF_SOURCE_IS_UTF8  0x0100
+            #define PyCF_DONT_IMPLY_DEDENT 0x0200
+            #define PyCF_ONLY_AST 0x0400
+            #define PyCF_IGNORE_COOKIE 0x0800
+            #define PyCF_TYPE_COMMENTS 0x1000
+            #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
+
+            #ifndef Py_LIMITED_API
+            typedef struct {
+                int cf_flags;  /* bitmask of CO_xxx flags relevant to future */
+                int cf_feature_version;  /* minor Python version (PyCF_ONLY_AST) */
+            } PyCompilerFlags;
+            #endif
+
+            /* Future feature support */
+
+            typedef struct {
+                int ff_features;      /* flags set by future statements */
+                int ff_lineno;        /* line number of last future statement */
+            } PyFutureFeatures;
+
+            #define FUTURE_NESTED_SCOPES "nested_scopes"
+            #define FUTURE_GENERATORS "generators"
+            #define FUTURE_DIVISION "division"
+            #define FUTURE_ABSOLUTE_IMPORT "absolute_import"
+            #define FUTURE_WITH_STATEMENT "with_statement"
+            #define FUTURE_PRINT_FUNCTION "print_function"
+            #define FUTURE_UNICODE_LITERALS "unicode_literals"
+            #define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL"
+            #define FUTURE_GENERATOR_STOP "generator_stop"
+            #define FUTURE_ANNOTATIONS "annotations"
+
+            struct _mod; /* Declare the existence of this type */
+            #define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar)
+            PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(
+                struct _mod *mod,
+                const char *filename,       /* decoded from the filesystem encoding */
+                PyCompilerFlags *flags,
+                int optimize,
+                PyArena *arena);
+            PyAPI_FUNC(PyCodeObject *) PyAST_CompileObject(
+                struct _mod *mod,
+                PyObject *filename,
+                PyCompilerFlags *flags,
+                int optimize,
+                PyArena *arena);
+            PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(
+                struct _mod * mod,
+                const char *filename        /* decoded from the filesystem encoding */
+                );
+            PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromASTObject(
+                struct _mod * mod,
+                PyObject *filename
+                );
+
+            /* _Py_Mangle is defined in compile.c */
+            PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
+
+            #define PY_INVALID_STACK_EFFECT INT_MAX
+            PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg);
+            PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump);
+
+            PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, int optimize);
+
+            #ifdef __cplusplus
+            }
+            #endif
+
+            #endif /* !Py_LIMITED_API */
+
+            /* These definitions must match corresponding definitions in graminit.h. */
+            #define Py_single_input 256
+            #define Py_file_input 257
+            #define Py_eval_input 258
+            #define Py_func_type_input 345
+
+            #endif /* !Py_COMPILE_H */
+            ''').strip()
+        lines = [line + '\n' for line in text.splitlines()]
+        lines[-1] = lines[-1][:-1]
+
+        results = list(
+                iter_lines(lines, _parse_directive=self._parse_directive))
+
+        self.assertEqual(results, [
+            (1, '#ifndef Py_COMPILE_H\n',
+                IfDirective('ifndef', 'Py_COMPILE_H'),
+                ()),
+            (2, '#define Py_COMPILE_H\n',
+                Constant('Py_COMPILE_H', None),
+                ('! defined(Py_COMPILE_H)',)),
+            (3, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)',)),
+            (4, '#ifndef Py_LIMITED_API\n',
+                IfDirective('ifndef', 'Py_LIMITED_API'),
+                ('! defined(Py_COMPILE_H)',)),
+            (5, '#include "code.h"\n',
+                Include('"code.h"'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (6, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (7, '#ifdef __cplusplus\n',
+                IfDirective('ifdef', '__cplusplus'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (8, 'extern "C" {\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)', 'defined(__cplusplus)')),
+            (9, '#endif\n',
+                OtherDirective('endif', None),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)', 'defined(__cplusplus)')),
+            (10, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (11, ' \n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (12, 'struct _node;  \n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (13, 'PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (14, ' \n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (15, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (19, '#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT |                     CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION |                     CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL |                     CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS)\n',
+                Constant('PyCF_MASK', '(CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS)'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (20, '#define PyCF_MASK_OBSOLETE (CO_NESTED)\n',
+                Constant('PyCF_MASK_OBSOLETE', '(CO_NESTED)'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (21, '#define PyCF_SOURCE_IS_UTF8  0x0100\n',
+                Constant('PyCF_SOURCE_IS_UTF8', ' 0x0100'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (22, '#define PyCF_DONT_IMPLY_DEDENT 0x0200\n',
+                Constant('PyCF_DONT_IMPLY_DEDENT', '0x0200'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (23, '#define PyCF_ONLY_AST 0x0400\n',
+                Constant('PyCF_ONLY_AST', '0x0400'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (24, '#define PyCF_IGNORE_COOKIE 0x0800\n',
+                Constant('PyCF_IGNORE_COOKIE', '0x0800'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (25, '#define PyCF_TYPE_COMMENTS 0x1000\n',
+                Constant('PyCF_TYPE_COMMENTS', '0x1000'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (26, '#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000\n',
+                Constant('PyCF_ALLOW_TOP_LEVEL_AWAIT', '0x2000'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (27, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (28, '#ifndef Py_LIMITED_API\n',
+                IfDirective('ifndef', 'Py_LIMITED_API'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (29, 'typedef struct {\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)', '! defined(Py_LIMITED_API)')),
+            (30, '    int cf_flags;   \n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)', '! defined(Py_LIMITED_API)')),
+            (31, '    int cf_feature_version;   \n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)', '! defined(Py_LIMITED_API)')),
+            (32, '} PyCompilerFlags;\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)', '! defined(Py_LIMITED_API)')),
+            (33, '#endif\n',
+                OtherDirective('endif', None),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)', '! defined(Py_LIMITED_API)')),
+            (34, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (35, ' \n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (36, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (37, 'typedef struct {\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (38, '    int ff_features;       \n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (39, '    int ff_lineno;         \n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (40, '} PyFutureFeatures;\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (41, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (42, '#define FUTURE_NESTED_SCOPES "nested_scopes"\n',
+                Constant('FUTURE_NESTED_SCOPES', '"nested_scopes"'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (43, '#define FUTURE_GENERATORS "generators"\n',
+                Constant('FUTURE_GENERATORS', '"generators"'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (44, '#define FUTURE_DIVISION "division"\n',
+                Constant('FUTURE_DIVISION', '"division"'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (45, '#define FUTURE_ABSOLUTE_IMPORT "absolute_import"\n',
+                Constant('FUTURE_ABSOLUTE_IMPORT', '"absolute_import"'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (46, '#define FUTURE_WITH_STATEMENT "with_statement"\n',
+                Constant('FUTURE_WITH_STATEMENT', '"with_statement"'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (47, '#define FUTURE_PRINT_FUNCTION "print_function"\n',
+                Constant('FUTURE_PRINT_FUNCTION', '"print_function"'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (48, '#define FUTURE_UNICODE_LITERALS "unicode_literals"\n',
+                Constant('FUTURE_UNICODE_LITERALS', '"unicode_literals"'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (49, '#define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL"\n',
+                Constant('FUTURE_BARRY_AS_BDFL', '"barry_as_FLUFL"'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (50, '#define FUTURE_GENERATOR_STOP "generator_stop"\n',
+                Constant('FUTURE_GENERATOR_STOP', '"generator_stop"'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (51, '#define FUTURE_ANNOTATIONS "annotations"\n',
+                Constant('FUTURE_ANNOTATIONS', '"annotations"'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (52, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (53, 'struct _mod;  \n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (54, '#define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar)\n',
+                Macro('PyAST_Compile', ('mod', 's', 'f', 'ar'), 'PyAST_CompileEx(mod, s, f, -1, ar)'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (55, 'PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (56, '    struct _mod *mod,\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (57, '    const char *filename,        \n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (58, '    PyCompilerFlags *flags,\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (59, '    int optimize,\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (60, '    PyArena *arena);\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (61, 'PyAPI_FUNC(PyCodeObject *) PyAST_CompileObject(\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (62, '    struct _mod *mod,\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (63, '    PyObject *filename,\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (64, '    PyCompilerFlags *flags,\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (65, '    int optimize,\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (66, '    PyArena *arena);\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (67, 'PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (68, '    struct _mod * mod,\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (69, '    const char *filename         \n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (70, '    );\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (71, 'PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromASTObject(\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (72, '    struct _mod * mod,\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (73, '    PyObject *filename\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (74, '    );\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (75, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (76, ' \n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (77, 'PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (78, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (79, '#define PY_INVALID_STACK_EFFECT INT_MAX\n',
+                Constant('PY_INVALID_STACK_EFFECT', 'INT_MAX'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (80, 'PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg);\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (81, 'PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump);\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (82, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (83, 'PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, int optimize);\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (84, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (85, '#ifdef __cplusplus\n',
+                IfDirective('ifdef', '__cplusplus'),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (86, '}\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)', 'defined(__cplusplus)')),
+            (87, '#endif\n',
+                OtherDirective('endif', None),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)', 'defined(__cplusplus)')),
+            (88, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (89, '#endif  \n',
+                OtherDirective('endif', None),
+                ('! defined(Py_COMPILE_H)', '! defined(Py_LIMITED_API)')),
+            (90, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)',)),
+            (91, ' \n',
+                None,
+                ('! defined(Py_COMPILE_H)',)),
+            (92, '#define Py_single_input 256\n',
+                Constant('Py_single_input', '256'),
+                ('! defined(Py_COMPILE_H)',)),
+            (93, '#define Py_file_input 257\n',
+                Constant('Py_file_input', '257'),
+                ('! defined(Py_COMPILE_H)',)),
+            (94, '#define Py_eval_input 258\n',
+                Constant('Py_eval_input', '258'),
+                ('! defined(Py_COMPILE_H)',)),
+            (95, '#define Py_func_type_input 345\n',
+                Constant('Py_func_type_input', '345'),
+                ('! defined(Py_COMPILE_H)',)),
+            (96, '\n',
+                None,
+                ('! defined(Py_COMPILE_H)',)),
+            (97, '#endif  ',
+                OtherDirective('endif', None),
+                ('! defined(Py_COMPILE_H)',)),
+            ])
+        self.check_calls(
+                ('_parse_directive', '#ifndef Py_COMPILE_H'),
+                ('_parse_directive', '#define Py_COMPILE_H'),
+                ('_parse_directive', '#ifndef Py_LIMITED_API'),
+                ('_parse_directive', '#include "code.h"'),
+                ('_parse_directive', '#ifdef __cplusplus'),
+                ('_parse_directive', '#endif'),
+                ('_parse_directive', '#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS)'),
+                ('_parse_directive', '#define PyCF_MASK_OBSOLETE (CO_NESTED)'),
+                ('_parse_directive', '#define PyCF_SOURCE_IS_UTF8 0x0100'),
+                ('_parse_directive', '#define PyCF_DONT_IMPLY_DEDENT 0x0200'),
+                ('_parse_directive', '#define PyCF_ONLY_AST 0x0400'),
+                ('_parse_directive', '#define PyCF_IGNORE_COOKIE 0x0800'),
+                ('_parse_directive', '#define PyCF_TYPE_COMMENTS 0x1000'),
+                ('_parse_directive', '#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000'),
+                ('_parse_directive', '#ifndef Py_LIMITED_API'),
+                ('_parse_directive', '#endif'),
+                ('_parse_directive', '#define FUTURE_NESTED_SCOPES "nested_scopes"'),
+                ('_parse_directive', '#define FUTURE_GENERATORS "generators"'),
+                ('_parse_directive', '#define FUTURE_DIVISION "division"'),
+                ('_parse_directive', '#define FUTURE_ABSOLUTE_IMPORT "absolute_import"'),
+                ('_parse_directive', '#define FUTURE_WITH_STATEMENT "with_statement"'),
+                ('_parse_directive', '#define FUTURE_PRINT_FUNCTION "print_function"'),
+                ('_parse_directive', '#define FUTURE_UNICODE_LITERALS "unicode_literals"'),
+                ('_parse_directive', '#define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL"'),
+                ('_parse_directive', '#define FUTURE_GENERATOR_STOP "generator_stop"'),
+                ('_parse_directive', '#define FUTURE_ANNOTATIONS "annotations"'),
+                ('_parse_directive', '#define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar)'),
+                ('_parse_directive', '#define PY_INVALID_STACK_EFFECT INT_MAX'),
+                ('_parse_directive', '#ifdef __cplusplus'),
+                ('_parse_directive', '#endif'),
+                ('_parse_directive', '#endif'),
+                ('_parse_directive', '#define Py_single_input 256'),
+                ('_parse_directive', '#define Py_file_input 257'),
+                ('_parse_directive', '#define Py_eval_input 258'),
+                ('_parse_directive', '#define Py_func_type_input 345'),
+                ('_parse_directive', '#endif'),
+                )
+
+
+class ParseDirectiveTests(unittest.TestCase):
+
+    def test_directives(self):
+        tests = [
+            # includes
+            ('#include "internal/pycore_pystate.h"', Include('"internal/pycore_pystate.h"')),
+            ('#include <stdio>', Include('<stdio>')),
+
+            # defines
+            ('#define SPAM int', Constant('SPAM', 'int')),
+            ('#define SPAM', Constant('SPAM', '')),
+            ('#define SPAM(x, y) run(x, y)', Macro('SPAM', ('x', 'y'), 'run(x, y)')),
+            ('#undef SPAM', None),
+
+            # conditionals
+            ('#if SPAM', IfDirective('if', 'SPAM')),
+            # XXX complex conditionls
+            ('#ifdef SPAM', IfDirective('ifdef', 'SPAM')),
+            ('#ifndef SPAM', IfDirective('ifndef', 'SPAM')),
+            ('#elseif SPAM', IfDirective('elseif', 'SPAM')),
+            # XXX complex conditionls
+            ('#else', OtherDirective('else', '')),
+            ('#endif', OtherDirective('endif', '')),
+
+            # other
+            ('#error oops!', None),
+            ('#warning oops!', None),
+            ('#pragma ...', None),
+            ('#__FILE__ ...', None),
+            ('#__LINE__ ...', None),
+            ('#__DATE__ ...', None),
+            ('#__TIME__ ...', None),
+            ('#__TIMESTAMP__ ...', None),
+
+            # extra whitespace
+            (' # include  <stdio> ', Include('<stdio>')),
+            ('#else  ', OtherDirective('else', '')),
+            ('#endif  ', OtherDirective('endif', '')),
+            ('#define SPAM int  ', Constant('SPAM', 'int')),
+            ('#define SPAM  ', Constant('SPAM', '')),
+            ]
+        for line, expected in tests:
+            if expected is None:
+                kind, _, text = line[1:].partition(' ')
+                expected = OtherDirective(kind, text)
+            with self.subTest(line):
+                directive = parse_directive(line)
+
+                self.assertEqual(directive, expected)
+
+    def test_bad_directives(self):
+        tests = [
+            # valid directives with bad text
+            '#define 123',
+            '#else spam',
+            '#endif spam',
+            ]
+        for kind in PreprocessorDirective.KINDS:
+            # missing leading "#"
+            tests.append(kind)
+            if kind in ('else', 'endif'):
+                continue
+            # valid directives with missing text
+            tests.append('#' + kind)
+            tests.append('#' + kind + ' ')
+        for line in tests:
+            with self.subTest(line):
+                with self.assertRaises(ValueError):
+                    parse_directive(line)
+
+    def test_not_directives(self):
+        tests = [
+            '',
+            ' ',
+            'directive',
+            'directive?',
+            '???',
+            ]
+        for line in tests:
+            with self.subTest(line):
+                with self.assertRaises(ValueError):
+                    parse_directive(line)
+
+
+class ConstantTests(unittest.TestCase):
+
+    def test_type(self):
+        directive = Constant('SPAM', '123')
+
+        self.assertIs(type(directive), Constant)
+        self.assertIsInstance(directive, PreprocessorDirective)
+
+    def test_attrs(self):
+        d = Constant('SPAM', '123')
+        kind, name, value = d.kind, d.name, d.value
+
+        self.assertEqual(kind, 'define')
+        self.assertEqual(name, 'SPAM')
+        self.assertEqual(value, '123')
+
+    def test_text(self):
+        tests = [
+            (('SPAM', '123'), 'SPAM 123'),
+            (('SPAM',), 'SPAM'),
+            ]
+        for args, expected in tests:
+            with self.subTest(args):
+                d = Constant(*args)
+                text = d.text
+
+                self.assertEqual(text, expected)
+
+    def test_iter(self):
+        kind, name, value = Constant('SPAM', '123')
+
+        self.assertEqual(kind, 'define')
+        self.assertEqual(name, 'SPAM')
+        self.assertEqual(value, '123')
+
+    def test_defaults(self):
+        kind, name, value = Constant('SPAM')
+
+        self.assertEqual(kind, 'define')
+        self.assertEqual(name, 'SPAM')
+        self.assertIs(value, None)
+
+    def test_coerce(self):
+        tests = []
+        # coerced name, value
+        for args in wrapped_arg_combos('SPAM', '123'):
+            tests.append((args, ('SPAM', '123')))
+        # missing name, value
+        for name in ('', ' ', None, StrProxy(' '), ()):
+            for value in ('', ' ', None, StrProxy(' '), ()):
+                tests.append(
+                        ((name, value), (None, None)))
+        # whitespace
+        tests.extend([
+            ((' SPAM ', ' 123 '), ('SPAM', '123')),
+            ])
+
+        for args, expected in tests:
+            with self.subTest(args):
+                d = Constant(*args)
+
+                self.assertEqual(d[1:], expected)
+                for i, exp in enumerate(expected, start=1):
+                    if exp is not None:
+                        self.assertIs(type(d[i]), str)
+
+    def test_valid(self):
+        tests = [
+            ('SPAM', '123'),
+            # unusual name
+            ('_SPAM_', '123'),
+            ('X_1', '123'),
+            # unusual value
+            ('SPAM', None),
+            ]
+        for args in tests:
+            with self.subTest(args):
+                directive = Constant(*args)
+
+                directive.validate()
+
+    def test_invalid(self):
+        tests = [
+            # invalid name
+            ((None, '123'), TypeError),
+            (('_', '123'), ValueError),
+            (('1', '123'), ValueError),
+            (('_1_', '123'), ValueError),
+            # There is no invalid value (including None).
+            ]
+        for args, exctype in tests:
+            with self.subTest(args):
+                directive = Constant(*args)
+
+                with self.assertRaises(exctype):
+                    directive.validate()
+
+
+class MacroTests(unittest.TestCase):
+
+    def test_type(self):
+        directive = Macro('SPAM', ('x', 'y'), '123')
+
+        self.assertIs(type(directive), Macro)
+        self.assertIsInstance(directive, PreprocessorDirective)
+
+    def test_attrs(self):
+        d = Macro('SPAM', ('x', 'y'), '123')
+        kind, name, args, body = d.kind, d.name, d.args, d.body
+
+        self.assertEqual(kind, 'define')
+        self.assertEqual(name, 'SPAM')
+        self.assertEqual(args, ('x', 'y'))
+        self.assertEqual(body, '123')
+
+    def test_text(self):
+        tests = [
+            (('SPAM', ('x', 'y'), '123'), 'SPAM(x, y) 123'),
+            (('SPAM', ('x', 'y'),), 'SPAM(x, y)'),
+            ]
+        for args, expected in tests:
+            with self.subTest(args):
+                d = Macro(*args)
+                text = d.text
+
+                self.assertEqual(text, expected)
+
+    def test_iter(self):
+        kind, name, args, body = Macro('SPAM', ('x', 'y'), '123')
+
+        self.assertEqual(kind, 'define')
+        self.assertEqual(name, 'SPAM')
+        self.assertEqual(args, ('x', 'y'))
+        self.assertEqual(body, '123')
+
+    def test_defaults(self):
+        kind, name, args, body = Macro('SPAM', ('x', 'y'))
+
+        self.assertEqual(kind, 'define')
+        self.assertEqual(name, 'SPAM')
+        self.assertEqual(args, ('x', 'y'))
+        self.assertIs(body, None)
+
+    def test_coerce(self):
+        tests = []
+        # coerce name and body
+        for args in wrapped_arg_combos('SPAM', ('x', 'y'), '123'):
+            tests.append(
+                    (args, ('SPAM', ('x', 'y'), '123')))
+        # coerce args
+        tests.extend([
+            (('SPAM', 'x', '123'),
+             ('SPAM', ('x',), '123')),
+            (('SPAM', 'x,y', '123'),
+             ('SPAM', ('x', 'y'), '123')),
+            ])
+        # coerce arg names
+        for argnames in wrapped_arg_combos('x', 'y'):
+            tests.append(
+                    (('SPAM', argnames, '123'),
+                     ('SPAM', ('x', 'y'), '123')))
+        # missing name, body
+        for name in ('', ' ', None, StrProxy(' '), ()):
+            for argnames in (None, ()):
+                for body in ('', ' ', None, StrProxy(' '), ()):
+                    tests.append(
+                            ((name, argnames, body),
+                             (None, (), None)))
+        # missing args
+        tests.extend([
+            (('SPAM', None, '123'),
+             ('SPAM', (), '123')),
+            (('SPAM', (), '123'),
+             ('SPAM', (), '123')),
+            ])
+        # missing arg names
+        for arg in ('', ' ', None, StrProxy(' '), ()):
+            tests.append(
+                    (('SPAM', (arg,), '123'),
+                     ('SPAM', (None,), '123')))
+        tests.extend([
+            (('SPAM', ('x', '', 'z'), '123'),
+             ('SPAM', ('x', None, 'z'), '123')),
+            ])
+        # whitespace
+        tests.extend([
+            ((' SPAM ', (' x ', ' y '), ' 123 '),
+             ('SPAM', ('x', 'y'), '123')),
+            (('SPAM', 'x, y', '123'),
+             ('SPAM', ('x', 'y'), '123')),
+            ])
+
+        for args, expected in tests:
+            with self.subTest(args):
+                d = Macro(*args)
+
+                self.assertEqual(d[1:], expected)
+                for i, exp in enumerate(expected, start=1):
+                    if i == 2:
+                        self.assertIs(type(d[i]), tuple)
+                    elif exp is not None:
+                        self.assertIs(type(d[i]), str)
+
+    def test_init_bad_args(self):
+        tests = [
+            ('SPAM', StrProxy('x'), '123'),
+            ('SPAM', object(), '123'),
+            ]
+        for args in tests:
+            with self.subTest(args):
+                with self.assertRaises(TypeError):
+                    Macro(*args)
+
+    def test_valid(self):
+        tests = [
+            # unusual name
+            ('SPAM', ('x', 'y'), 'run(x, y)'),
+            ('_SPAM_', ('x', 'y'), 'run(x, y)'),
+            ('X_1', ('x', 'y'), 'run(x, y)'),
+            # unusual args
+            ('SPAM', (), 'run(x, y)'),
+            ('SPAM', ('_x_', 'y_1'), 'run(x, y)'),
+            ('SPAM', 'x', 'run(x, y)'),
+            ('SPAM', 'x, y', 'run(x, y)'),
+            # unusual body
+            ('SPAM', ('x', 'y'), None),
+            ]
+        for args in tests:
+            with self.subTest(args):
+                directive = Macro(*args)
+
+                directive.validate()
+
+    def test_invalid(self):
+        tests = [
+            # invalid name
+            ((None, ('x', 'y'), '123'), TypeError),
+            (('_', ('x', 'y'), '123'), ValueError),
+            (('1', ('x', 'y'), '123'), ValueError),
+            (('_1', ('x', 'y'), '123'), ValueError),
+            # invalid args
+            (('SPAM', (None, 'y'), '123'), ValueError),
+            (('SPAM', ('x', '_'), '123'), ValueError),
+            (('SPAM', ('x', '1'), '123'), ValueError),
+            (('SPAM', ('x', '_1_'), '123'), ValueError),
+            # There is no invalid body (including None).
+            ]
+        for args, exctype in tests:
+            with self.subTest(args):
+                directive = Macro(*args)
+
+                with self.assertRaises(exctype):
+                    directive.validate()
+
+
+class IfDirectiveTests(unittest.TestCase):
+
+    def test_type(self):
+        directive = IfDirective('if', '1')
+
+        self.assertIs(type(directive), IfDirective)
+        self.assertIsInstance(directive, PreprocessorDirective)
+
+    def test_attrs(self):
+        d = IfDirective('if', '1')
+        kind, condition = d.kind, d.condition
+
+        self.assertEqual(kind, 'if')
+        self.assertEqual(condition, '1')
+        #self.assertEqual(condition, (ArithmeticCondition('1'),))
+
+    def test_text(self):
+        tests = [
+            (('if', 'defined(SPAM) && 1 || (EGGS > 3 && defined(HAM))'),
+             'defined(SPAM) && 1 || (EGGS > 3 && defined(HAM))'),
+            ]
+        for kind in IfDirective.KINDS:
+            tests.append(
+                    ((kind, 'SPAM'), 'SPAM'))
+        for args, expected in tests:
+            with self.subTest(args):
+                d = IfDirective(*args)
+                text = d.text
+
+                self.assertEqual(text, expected)
+
+    def test_iter(self):
+        kind, condition = IfDirective('if', '1')
+
+        self.assertEqual(kind, 'if')
+        self.assertEqual(condition, '1')
+        #self.assertEqual(condition, (ArithmeticCondition('1'),))
+
+    #def test_complex_conditions(self):
+    #    ...
+
+    def test_coerce(self):
+        tests = []
+        for kind in IfDirective.KINDS:
+            if kind == 'ifdef':
+                cond = 'defined(SPAM)'
+            elif kind == 'ifndef':
+                cond = '! defined(SPAM)'
+            else:
+                cond = 'SPAM'
+            for args in wrapped_arg_combos(kind, 'SPAM'):
+                tests.append((args, (kind, cond)))
+            tests.extend([
+                ((' ' + kind + ' ', ' SPAM '), (kind, cond)),
+                ])
+            for raw in ('', ' ', None, StrProxy(' '), ()):
+                tests.append(((kind, raw), (kind, None)))
+        for kind in ('', ' ', None, StrProxy(' '), ()):
+            tests.append(((kind, 'SPAM'), (None, 'SPAM')))
+        for args, expected in tests:
+            with self.subTest(args):
+                d = IfDirective(*args)
+
+                self.assertEqual(tuple(d), expected)
+                for i, exp in enumerate(expected):
+                    if exp is not None:
+                        self.assertIs(type(d[i]), str)
+
+    def test_valid(self):
+        tests = []
+        for kind in IfDirective.KINDS:
+            tests.extend([
+                (kind, 'SPAM'),
+                (kind, '_SPAM_'),
+                (kind, 'X_1'),
+                (kind, '()'),
+                (kind, '--'),
+                (kind, '???'),
+                ])
+        for args in tests:
+            with self.subTest(args):
+                directive = IfDirective(*args)
+
+                directive.validate()
+
+    def test_invalid(self):
+        tests = []
+        # kind
+        tests.extend([
+            ((None, 'SPAM'), TypeError),
+            (('_', 'SPAM'), ValueError),
+            (('-', 'SPAM'), ValueError),
+            (('spam', 'SPAM'), ValueError),
+            ])
+        for kind in PreprocessorDirective.KINDS:
+            if kind in IfDirective.KINDS:
+                continue
+            tests.append(
+                ((kind, 'SPAM'), ValueError))
+        # condition
+        for kind in IfDirective.KINDS:
+            tests.extend([
+                ((kind, None), TypeError),
+                # Any other condition is valid.
+                ])
+        for args, exctype in tests:
+            with self.subTest(args):
+                directive = IfDirective(*args)
+
+                with self.assertRaises(exctype):
+                    directive.validate()
+
+
+class IncludeTests(unittest.TestCase):
+
+    def test_type(self):
+        directive = Include('<stdio>')
+
+        self.assertIs(type(directive), Include)
+        self.assertIsInstance(directive, PreprocessorDirective)
+
+    def test_attrs(self):
+        d = Include('<stdio>')
+        kind, file, text = d.kind, d.file, d.text
+
+        self.assertEqual(kind, 'include')
+        self.assertEqual(file, '<stdio>')
+        self.assertEqual(text, '<stdio>')
+
+    def test_iter(self):
+        kind, file = Include('<stdio>')
+
+        self.assertEqual(kind, 'include')
+        self.assertEqual(file, '<stdio>')
+
+    def test_coerce(self):
+        tests = []
+        for arg, in wrapped_arg_combos('<stdio>'):
+            tests.append((arg, '<stdio>'))
+        tests.extend([
+            (' <stdio> ', '<stdio>'),
+            ])
+        for arg in ('', ' ', None, StrProxy(' '), ()):
+            tests.append((arg, None ))
+        for arg, expected in tests:
+            with self.subTest(arg):
+                _, file = Include(arg)
+
+                self.assertEqual(file, expected)
+                if expected is not None:
+                    self.assertIs(type(file), str)
+
+    def test_valid(self):
+        tests = [
+            '<stdio>',
+            '"spam.h"',
+            '"internal/pycore_pystate.h"',
+            ]
+        for arg in tests:
+            with self.subTest(arg):
+                directive = Include(arg)
+
+                directive.validate()
+
+    def test_invalid(self):
+        tests = [
+            (None, TypeError),
+            # We currently don't check the file.
+            ]
+        for arg, exctype in tests:
+            with self.subTest(arg):
+                directive = Include(arg)
+
+                with self.assertRaises(exctype):
+                    directive.validate()
+
+
+class OtherDirectiveTests(unittest.TestCase):
+
+    def test_type(self):
+        directive = OtherDirective('undef', 'SPAM')
+
+        self.assertIs(type(directive), OtherDirective)
+        self.assertIsInstance(directive, PreprocessorDirective)
+
+    def test_attrs(self):
+        d = OtherDirective('undef', 'SPAM')
+        kind, text = d.kind, d.text
+
+        self.assertEqual(kind, 'undef')
+        self.assertEqual(text, 'SPAM')
+
+    def test_iter(self):
+        kind, text = OtherDirective('undef', 'SPAM')
+
+        self.assertEqual(kind, 'undef')
+        self.assertEqual(text, 'SPAM')
+
+    def test_coerce(self):
+        tests = []
+        for kind in OtherDirective.KINDS:
+            if kind in ('else', 'endif'):
+                continue
+            for args in wrapped_arg_combos(kind, '...'):
+                tests.append((args, (kind, '...')))
+            tests.extend([
+                ((' ' + kind + ' ', ' ... '), (kind, '...')),
+                ])
+            for raw in ('', ' ', None, StrProxy(' '), ()):
+                tests.append(((kind, raw), (kind, None)))
+        for kind in ('else', 'endif'):
+            for args in wrapped_arg_combos(kind, None):
+                tests.append((args, (kind, None)))
+            tests.extend([
+                ((' ' + kind + ' ', None), (kind, None)),
+                ])
+        for kind in ('', ' ', None, StrProxy(' '), ()):
+            tests.append(((kind, '...'), (None, '...')))
+        for args, expected in tests:
+            with self.subTest(args):
+                d = OtherDirective(*args)
+
+                self.assertEqual(tuple(d), expected)
+                for i, exp in enumerate(expected):
+                    if exp is not None:
+                        self.assertIs(type(d[i]), str)
+
+    def test_valid(self):
+        tests = []
+        for kind in OtherDirective.KINDS:
+            if kind in ('else', 'endif'):
+                continue
+            tests.extend([
+                (kind, '...'),
+                (kind, '???'),
+                (kind, 'SPAM'),
+                (kind, '1 + 1'),
+                ])
+        for kind in ('else', 'endif'):
+            tests.append((kind, None))
+        for args in tests:
+            with self.subTest(args):
+                directive = OtherDirective(*args)
+
+                directive.validate()
+
+    def test_invalid(self):
+        tests = []
+        # kind
+        tests.extend([
+            ((None, '...'), TypeError),
+            (('_', '...'), ValueError),
+            (('-', '...'), ValueError),
+            (('spam', '...'), ValueError),
+            ])
+        for kind in PreprocessorDirective.KINDS:
+            if kind in OtherDirective.KINDS:
+                continue
+            tests.append(
+                ((kind, None), ValueError))
+        # text
+        for kind in OtherDirective.KINDS:
+            if kind in ('else', 'endif'):
+                tests.extend([
+                    # Any text is invalid.
+                    ((kind, 'SPAM'), ValueError),
+                    ((kind, '...'), ValueError),
+                    ])
+            else:
+                tests.extend([
+                    ((kind, None), TypeError),
+                    # Any other text is valid.
+                    ])
+        for args, exctype in tests:
+            with self.subTest(args):
+                directive = OtherDirective(*args)
+
+                with self.assertRaises(exctype):
+                    directive.validate()
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_symbols/test_info.py b/Lib/test/test_tools/test_c_analyzer/test_c_symbols/test_info.py
new file mode 100644
index 000000000000..e029dcf66124
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_symbols/test_info.py
@@ -0,0 +1,192 @@
+import string
+import unittest
+
+from ..util import PseudoStr, StrProxy, Object
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_analyzer_common.info import ID
+    from c_symbols.info import Symbol
+
+
+class SymbolTests(unittest.TestCase):
+
+    VALID_ARGS = (
+            ID('x/y/z/spam.c', 'func', 'eggs'),
+            Symbol.KIND.VARIABLE,
+            False,
+            )
+    VALID_KWARGS = dict(zip(Symbol._fields, VALID_ARGS))
+    VALID_EXPECTED = VALID_ARGS
+
+    def test_init_typical_binary_local(self):
+        id = ID(None, None, 'spam')
+        symbol = Symbol(
+                id=id,
+                kind=Symbol.KIND.VARIABLE,
+                external=False,
+                )
+
+        self.assertEqual(symbol, (
+            id,
+            Symbol.KIND.VARIABLE,
+            False,
+            ))
+
+    def test_init_typical_binary_global(self):
+        id = ID('Python/ceval.c', None, 'spam')
+        symbol = Symbol(
+                id=id,
+                kind=Symbol.KIND.VARIABLE,
+                external=False,
+                )
+
+        self.assertEqual(symbol, (
+            id,
+            Symbol.KIND.VARIABLE,
+            False,
+            ))
+
+    def test_init_coercion(self):
+        tests = [
+            ('str subclass',
+             dict(
+                 id=PseudoStr('eggs'),
+                 kind=PseudoStr('variable'),
+                 external=0,
+                 ),
+             (ID(None, None, 'eggs'),
+              Symbol.KIND.VARIABLE,
+              False,
+              )),
+            ('with filename',
+             dict(
+                 id=('x/y/z/spam.c', 'eggs'),
+                 kind=PseudoStr('variable'),
+                 external=0,
+                 ),
+             (ID('x/y/z/spam.c', None, 'eggs'),
+              Symbol.KIND.VARIABLE,
+              False,
+              )),
+            ('non-str 1',
+             dict(
+                 id=('a', 'b', 'c'),
+                 kind=StrProxy('variable'),
+                 external=0,
+                 ),
+             (ID('a', 'b', 'c'),
+              Symbol.KIND.VARIABLE,
+              False,
+              )),
+            ('non-str 2',
+             dict(
+                 id=('a', 'b', 'c'),
+                 kind=Object(),
+                 external=0,
+                 ),
+             (ID('a', 'b', 'c'),
+              '<object>',
+              False,
+              )),
+            ]
+        for summary, kwargs, expected in tests:
+            with self.subTest(summary):
+                symbol = Symbol(**kwargs)
+
+                for field in Symbol._fields:
+                    value = getattr(symbol, field)
+                    if field == 'external':
+                        self.assertIs(type(value), bool)
+                    elif field == 'id':
+                        self.assertIs(type(value), ID)
+                    else:
+                        self.assertIs(type(value), str)
+                self.assertEqual(tuple(symbol), expected)
+
+    def test_init_all_missing(self):
+        id = ID(None, None, 'spam')
+
+        symbol = Symbol(id)
+
+        self.assertEqual(symbol, (
+            id,
+            Symbol.KIND.VARIABLE,
+            None,
+            ))
+
+    def test_fields(self):
+        id = ID('z', 'x', 'a')
+
+        symbol = Symbol(id, 'b', False)
+
+        self.assertEqual(symbol.id, id)
+        self.assertEqual(symbol.kind, 'b')
+        self.assertIs(symbol.external, False)
+
+    def test___getattr__(self):
+        id = ID('z', 'x', 'a')
+        symbol = Symbol(id, 'b', False)
+
+        filename = symbol.filename
+        funcname = symbol.funcname
+        name = symbol.name
+
+        self.assertEqual(filename, 'z')
+        self.assertEqual(funcname, 'x')
+        self.assertEqual(name, 'a')
+
+    def test_validate_typical(self):
+        id = ID('z', 'x', 'a')
+
+        symbol = Symbol(
+                id=id,
+                kind=Symbol.KIND.VARIABLE,
+                external=False,
+                )
+
+        symbol.validate()  # This does not fail.
+
+    def test_validate_missing_field(self):
+        for field in Symbol._fields:
+            with self.subTest(field):
+                symbol = Symbol(**self.VALID_KWARGS)
+                symbol = symbol._replace(**{field: None})
+
+                with self.assertRaises(TypeError):
+                    symbol.validate()
+
+    def test_validate_bad_field(self):
+        badch = tuple(c for c in string.punctuation + string.digits)
+        notnames = (
+                '1a',
+                'a.b',
+                'a-b',
+                '&a',
+                'a++',
+                ) + badch
+        tests = [
+            ('id', notnames),
+            ('kind', ('bogus',)),
+            ]
+        seen = set()
+        for field, invalid in tests:
+            for value in invalid:
+                if field != 'kind':
+                    seen.add(value)
+                with self.subTest(f'{field}={value!r}'):
+                    symbol = Symbol(**self.VALID_KWARGS)
+                    symbol = symbol._replace(**{field: value})
+
+                    with self.assertRaises(ValueError):
+                        symbol.validate()
+
+        for field, invalid in tests:
+            if field == 'kind':
+                continue
+            valid = seen - set(invalid)
+            for value in valid:
+                with self.subTest(f'{field}={value!r}'):
+                    symbol = Symbol(**self.VALID_KWARGS)
+                    symbol = symbol._replace(**{field: value})
+
+                    symbol.validate()  # This does not fail.
diff --git a/Lib/test/test_tools/test_c_analyzer/util.py b/Lib/test/test_tools/test_c_analyzer/util.py
new file mode 100644
index 000000000000..ba73b0a4b5fc
--- /dev/null
+++ b/Lib/test/test_tools/test_c_analyzer/util.py
@@ -0,0 +1,60 @@
+import itertools
+
+
+class PseudoStr(str):
+    pass
+
+
+class StrProxy:
+    def __init__(self, value):
+        self.value = value
+    def __str__(self):
+        return self.value
+    def __bool__(self):
+        return bool(self.value)
+
+
+class Object:
+    def __repr__(self):
+        return '<object>'
+
+
+def wrapped_arg_combos(*args,
+                       wrappers=(PseudoStr, StrProxy),
+                       skip=(lambda w, i, v: not isinstance(v, str)),
+                       ):
+    """Yield every possible combination of wrapped items for the given args.
+
+    Effectively, the wrappers are applied to the args according to the
+    powerset of the args indicies.  So the result includes the args
+    completely unwrapped.
+
+    If "skip" is supplied (default is to skip all non-str values) and
+    it returns True for a given arg index/value then that arg will
+    remain unwrapped,
+
+    Only unique results are returned.  If an arg was skipped for one
+    of the combinations then it could end up matching one of the other
+    combinations.  In that case only one of them will be yielded.
+    """
+    if not args:
+        return
+    indices = list(range(len(args)))
+    # The powerset (from recipe in the itertools docs).
+    combos = itertools.chain.from_iterable(itertools.combinations(indices, r)
+                                           for r in range(len(indices)+1))
+    seen = set()
+    for combo in combos:
+        for wrap in wrappers:
+            indexes = []
+            applied = list(args)
+            for i in combo:
+                arg = args[i]
+                if skip and skip(wrap, i, arg):
+                    continue
+                indexes.append(i)
+                applied[i] = wrap(arg)
+            key = (wrap, tuple(indexes))
+            if key not in seen:
+                yield tuple(applied)
+                seen.add(key)
diff --git a/Tools/c-globals/README b/Tools/c-analyzer/README
similarity index 100%
rename from Tools/c-globals/README
rename to Tools/c-analyzer/README
diff --git a/Tools/c-analyzer/c-globals.py b/Tools/c-analyzer/c-globals.py
new file mode 100644
index 000000000000..9afe059b28c6
--- /dev/null
+++ b/Tools/c-analyzer/c-globals.py
@@ -0,0 +1,9 @@
+# This is a script equivalent of running "python -m test.test_c_globals.cg".
+
+from c_globals.__main__ import parse_args, main
+
+
+# This is effectively copied from cg/__main__.py:
+if __name__ == '__main__':
+    cmd, cmdkwargs = parse_args()
+    main(cmd, cmdkwargs)
diff --git a/Tools/c-analyzer/c_analyzer_common/__init__.py b/Tools/c-analyzer/c_analyzer_common/__init__.py
new file mode 100644
index 000000000000..888b16ff41d1
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer_common/__init__.py
@@ -0,0 +1,19 @@
+import os.path
+
+
+PKG_ROOT = os.path.dirname(__file__)
+DATA_DIR = os.path.dirname(PKG_ROOT)
+REPO_ROOT = os.path.dirname(
+        os.path.dirname(DATA_DIR))
+
+SOURCE_DIRS = [os.path.join(REPO_ROOT, name) for name in [
+        'Include',
+        'Python',
+        'Parser',
+        'Objects',
+        'Modules',
+        ]]
+
+
+# Clean up the namespace.
+del os
diff --git a/Tools/c-analyzer/c_analyzer_common/_generate.py b/Tools/c-analyzer/c_analyzer_common/_generate.py
new file mode 100644
index 000000000000..1629aa6b5210
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer_common/_generate.py
@@ -0,0 +1,328 @@
+# The code here consists of hacks for pre-populating the known.tsv file.
+
+from c_parser.preprocessor import _iter_clean_lines
+from c_parser.naive import (
+        iter_variables, parse_variable_declaration, find_variables,
+        )
+from c_parser.info import Variable
+
+from . import SOURCE_DIRS, REPO_ROOT
+from .known import DATA_FILE as KNOWN_FILE, HEADER as KNOWN_HEADER
+from .info import UNKNOWN, ID
+from .util import write_tsv
+from .files import iter_cpython_files
+
+
+POTS = ('char ', 'wchar_t ', 'int ', 'Py_ssize_t ')
+POTS += tuple('const ' + v for v in POTS)
+STRUCTS = ('PyTypeObject', 'PyObject', 'PyMethodDef', 'PyModuleDef', 'grammar')
+
+
+def _parse_global(line, funcname=None):
+    line = line.strip()
+    if line.startswith('static '):
+        if '(' in line and '[' not in line and ' = ' not in line:
+            return None, None
+        name, decl = parse_variable_declaration(line)
+    elif line.startswith(('Py_LOCAL(', 'Py_LOCAL_INLINE(')):
+        name, decl = parse_variable_declaration(line)
+    elif line.startswith('_Py_static_string('):
+        decl = line.strip(';').strip()
+        name = line.split('(')[1].split(',')[0].strip()
+    elif line.startswith('_Py_IDENTIFIER('):
+        decl = line.strip(';').strip()
+        name = 'PyId_' + line.split('(')[1].split(')')[0].strip()
+    elif funcname:
+        return None, None
+
+    # global-only
+    elif line.startswith('PyAPI_DATA('):  # only in .h files
+        name, decl = parse_variable_declaration(line)
+    elif line.startswith('extern '):  # only in .h files
+        name, decl = parse_variable_declaration(line)
+    elif line.startswith('PyDoc_VAR('):
+        decl = line.strip(';').strip()
+        name = line.split('(')[1].split(')')[0].strip()
+    elif line.startswith(POTS):  # implied static
+        if '(' in line and '[' not in line and ' = ' not in line:
+            return None, None
+        name, decl = parse_variable_declaration(line)
+    elif line.startswith(STRUCTS) and line.endswith(' = {'):  # implied static
+        name, decl = parse_variable_declaration(line)
+    elif line.startswith(STRUCTS) and line.endswith(' = NULL;'):  # implied static
+        name, decl = parse_variable_declaration(line)
+    elif line.startswith('struct '):
+        if not line.endswith(' = {'):
+            return None, None
+        if not line.partition(' ')[2].startswith(STRUCTS):
+            return None, None
+        # implied static
+        name, decl = parse_variable_declaration(line)
+
+    # file-specific
+    elif line.startswith(('SLOT1BINFULL(', 'SLOT1BIN(')):
+        # Objects/typeobject.c
+        funcname = line.split('(')[1].split(',')[0]
+        return [
+                ('op_id', funcname, '_Py_static_string(op_id, OPSTR)'),
+                ('rop_id', funcname, '_Py_static_string(op_id, OPSTR)'),
+                ]
+    elif line.startswith('WRAP_METHOD('):
+        # Objects/weakrefobject.c
+        funcname, name = (v.strip() for v in line.split('(')[1].split(')')[0].split(','))
+        return [
+                ('PyId_' + name, funcname, f'_Py_IDENTIFIER({name})'),
+                ]
+
+    else:
+        return None, None
+    return name, decl
+
+
+def _pop_cached(varcache, filename, funcname, name, *,
+                _iter_variables=iter_variables,
+                ):
+    # Look for the file.
+    try:
+        cached = varcache[filename]
+    except KeyError:
+        cached = varcache[filename] = {}
+        for variable in _iter_variables(filename,
+                                        parse_variable=_parse_global,
+                                        ):
+            variable._isglobal = True
+            cached[variable.id] = variable
+        for var in cached:
+            print(' ', var)
+
+    # Look for the variable.
+    if funcname == UNKNOWN:
+        for varid in cached:
+            if varid.name == name:
+                break
+        else:
+            return None
+        return cached.pop(varid)
+    else:
+        return cached.pop((filename, funcname, name), None)
+
+
+def find_matching_variable(varid, varcache, allfilenames, *,
+                           _pop_cached=_pop_cached,
+                           ):
+    if varid.filename and varid.filename != UNKNOWN:
+        filenames = [varid.filename]
+    else:
+        filenames = allfilenames
+    for filename in filenames:
+        variable = _pop_cached(varcache, filename, varid.funcname, varid.name)
+        if variable is not None:
+            return variable
+    else:
+        if varid.filename and varid.filename != UNKNOWN and varid.funcname is None:
+            for filename in allfilenames:
+                if not filename.endswith('.h'):
+                    continue
+                variable = _pop_cached(varcache, filename, None, varid.name)
+                if variable is not None:
+                    return variable
+        return None
+
+
+MULTILINE = {
+    # Python/Python-ast.c
+    'Load_singleton': 'PyObject *',
+    'Store_singleton': 'PyObject *',
+    'Del_singleton': 'PyObject *',
+    'AugLoad_singleton': 'PyObject *',
+    'AugStore_singleton': 'PyObject *',
+    'Param_singleton': 'PyObject *',
+    'And_singleton': 'PyObject *',
+    'Or_singleton': 'PyObject *',
+    'Add_singleton': 'static PyObject *',
+    'Sub_singleton': 'static PyObject *',
+    'Mult_singleton': 'static PyObject *',
+    'MatMult_singleton': 'static PyObject *',
+    'Div_singleton': 'static PyObject *',
+    'Mod_singleton': 'static PyObject *',
+    'Pow_singleton': 'static PyObject *',
+    'LShift_singleton': 'static PyObject *',
+    'RShift_singleton': 'static PyObject *',
+    'BitOr_singleton': 'static PyObject *',
+    'BitXor_singleton': 'static PyObject *',
+    'BitAnd_singleton': 'static PyObject *',
+    'FloorDiv_singleton': 'static PyObject *',
+    'Invert_singleton': 'static PyObject *',
+    'Not_singleton': 'static PyObject *',
+    'UAdd_singleton': 'static PyObject *',
+    'USub_singleton': 'static PyObject *',
+    'Eq_singleton': 'static PyObject *',
+    'NotEq_singleton': 'static PyObject *',
+    'Lt_singleton': 'static PyObject *',
+    'LtE_singleton': 'static PyObject *',
+    'Gt_singleton': 'static PyObject *',
+    'GtE_singleton': 'static PyObject *',
+    'Is_singleton': 'static PyObject *',
+    'IsNot_singleton': 'static PyObject *',
+    'In_singleton': 'static PyObject *',
+    'NotIn_singleton': 'static PyObject *',
+    # Python/symtable.c
+    'top': 'static identifier ',
+    'lambda': 'static identifier ',
+    'genexpr': 'static identifier ',
+    'listcomp': 'static identifier ',
+    'setcomp': 'static identifier ',
+    'dictcomp': 'static identifier ',
+    '__class__': 'static identifier ',
+    # Python/compile.c
+    '__doc__': 'static PyObject *',
+    '__annotations__': 'static PyObject *',
+    # Objects/floatobject.c
+    'double_format': 'static float_format_type ',
+    'float_format': 'static float_format_type ',
+    'detected_double_format': 'static float_format_type ',
+    'detected_float_format': 'static float_format_type ',
+    # Parser/listnode.c
+    'level': 'static int ',
+    'atbol': 'static int ',
+    # Python/dtoa.c
+    'private_mem': 'static double private_mem[PRIVATE_mem]',
+    'pmem_next': 'static double *',
+    # Modules/_weakref.c
+    'weakref_functions': 'static PyMethodDef ',
+}
+INLINE = {
+    # Modules/_tracemalloc.c
+    'allocators': 'static struct { PyMemAllocatorEx mem; PyMemAllocatorEx raw; PyMemAllocatorEx obj; } ',
+    # Modules/faulthandler.c
+    'fatal_error': 'static struct { int enabled; PyObject *file; int fd; int all_threads; PyInterpreterState *interp; void *exc_handler; } ',
+    'thread': 'static struct { PyObject *file; int fd; PY_TIMEOUT_T timeout_us; int repeat; PyInterpreterState *interp; int exit; char *header; size_t header_len; PyThread_type_lock cancel_event; PyThread_type_lock running; } ',
+    # Modules/signalmodule.c
+    'Handlers': 'static volatile struct { _Py_atomic_int tripped; PyObject *func; } Handlers[NSIG]',
+    'wakeup': 'static volatile struct { SOCKET_T fd; int warn_on_full_buffer; int use_send; } ',
+    # Python/dynload_shlib.c
+    'handles': 'static struct { dev_t dev; ino_t ino; void *handle; } handles[128]',
+    # Objects/obmalloc.c
+    '_PyMem_Debug': 'static struct { debug_alloc_api_t raw; debug_alloc_api_t mem; debug_alloc_api_t obj; } ',
+    # Python/bootstrap_hash.c
+    'urandom_cache': 'static struct { int fd; dev_t st_dev; ino_t st_ino; } ',
+    }
+FUNC = {
+    # Objects/object.c
+    '_Py_abstract_hack': 'Py_ssize_t (*_Py_abstract_hack)(PyObject *)',
+    # Parser/myreadline.c
+    'PyOS_InputHook': 'int (*PyOS_InputHook)(void)',
+    # Python/pylifecycle.c
+    '_PyOS_mystrnicmp_hack': 'int (*_PyOS_mystrnicmp_hack)(const char *, const char *, Py_ssize_t)',
+    # Parser/myreadline.c
+    'PyOS_ReadlineFunctionPointer': 'char *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *)',
+    }
+IMPLIED = {
+    # Objects/boolobject.c
+    '_Py_FalseStruct': 'static struct _longobject ',
+    '_Py_TrueStruct': 'static struct _longobject ',
+    # Modules/config.c
+    '_PyImport_Inittab': 'struct _inittab _PyImport_Inittab[]',
+    }
+GLOBALS = {}
+GLOBALS.update(MULTILINE)
+GLOBALS.update(INLINE)
+GLOBALS.update(FUNC)
+GLOBALS.update(IMPLIED)
+
+LOCALS = {
+    'buildinfo': ('Modules/getbuildinfo.c',
+                  'Py_GetBuildInfo',
+                  'static char buildinfo[50 + sizeof(GITVERSION) + ((sizeof(GITTAG) > sizeof(GITBRANCH)) ?  sizeof(GITTAG) : sizeof(GITBRANCH))]'),
+    'methods': ('Python/codecs.c',
+                '_PyCodecRegistry_Init',
+                'static struct { char *name; PyMethodDef def; } methods[]'),
+    }
+
+
+def _known(symbol):
+    if symbol.funcname:
+        if symbol.funcname != UNKNOWN or symbol.filename != UNKNOWN:
+            raise KeyError(symbol.name)
+        filename, funcname, decl = LOCALS[symbol.name]
+        varid = ID(filename, funcname, symbol.name)
+    elif not symbol.filename or symbol.filename == UNKNOWN:
+        raise KeyError(symbol.name)
+    else:
+        varid = symbol.id
+        try:
+            decl = GLOBALS[symbol.name]
+        except KeyError:
+
+            if symbol.name.endswith('_methods'):
+                decl = 'static PyMethodDef '
+            elif symbol.filename == 'Objects/exceptions.c' and symbol.name.startswith(('PyExc_', '_PyExc_')):
+                decl = 'static PyTypeObject '
+            else:
+                raise
+    if symbol.name not in decl:
+        decl = decl + symbol.name
+    return Variable(varid, decl)
+
+
+def known_row(varid, decl):
+    return (
+            varid.filename,
+            varid.funcname or '-',
+            varid.name,
+            'variable',
+            decl,
+            )
+
+
+def known_rows(symbols, *,
+               cached=True,
+               _get_filenames=iter_cpython_files,
+               _find_match=find_matching_variable,
+               _find_symbols=find_variables,
+               _as_known=known_row,
+               ):
+    filenames = list(_get_filenames())
+    cache = {}
+    if cached:
+        for symbol in symbols:
+            try:
+                found = _known(symbol)
+            except KeyError:
+                found = _find_match(symbol, cache, filenames)
+                if found is None:
+                    found = Variable(symbol.id, UNKNOWN)
+            yield _as_known(found.id, found.vartype)
+    else:
+        raise NotImplementedError  # XXX incorporate KNOWN
+        for variable in _find_symbols(symbols, filenames,
+                                      srccache=cache,
+                                      parse_variable=_parse_global,
+                                      ):
+            #variable = variable._replace(
+            #    filename=os.path.relpath(variable.filename, REPO_ROOT))
+            if variable.funcname == UNKNOWN:
+                print(variable)
+            if variable.vartype== UNKNOWN:
+                print(variable)
+            yield _as_known(variable.id, variable.vartype)
+
+
+def generate(symbols, filename=None, *,
+             _generate_rows=known_rows,
+             _write_tsv=write_tsv,
+             ):
+    if not filename:
+        filename = KNOWN_FILE + '.new'
+
+    rows = _generate_rows(symbols)
+    _write_tsv(filename, KNOWN_HEADER, rows)
+
+
+if __name__ == '__main__':
+    from c_symbols import binary
+    symbols = binary.iter_symbols(
+            binary.PYTHON,
+            find_local_symbol=None,
+            )
+    generate(symbols)
diff --git a/Tools/c-analyzer/c_analyzer_common/files.py b/Tools/c-analyzer/c_analyzer_common/files.py
new file mode 100644
index 000000000000..b3cd16c8dc00
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer_common/files.py
@@ -0,0 +1,138 @@
+import glob
+import os
+import os.path
+
+from . import SOURCE_DIRS, REPO_ROOT
+
+
+C_SOURCE_SUFFIXES = ('.c', '.h')
+
+
+def _walk_tree(root, *,
+               _walk=os.walk,
+               ):
+    # A wrapper around os.walk that resolves the filenames.
+    for parent, _, names in _walk(root):
+        for name in names:
+            yield os.path.join(parent, name)
+
+
+def walk_tree(root, *,
+              suffix=None,
+              walk=_walk_tree,
+              ):
+    """Yield each file in the tree under the given directory name.
+
+    If "suffix" is provided then only files with that suffix will
+    be included.
+    """
+    if suffix and not isinstance(suffix, str):
+        raise ValueError('suffix must be a string')
+
+    for filename in walk(root):
+        if suffix and not filename.endswith(suffix):
+            continue
+        yield filename
+
+
+def glob_tree(root, *,
+              suffix=None,
+              _glob=glob.iglob,
+              ):
+    """Yield each file in the tree under the given directory name.
+
+    If "suffix" is provided then only files with that suffix will
+    be included.
+    """
+    suffix = suffix or ''
+    if not isinstance(suffix, str):
+        raise ValueError('suffix must be a string')
+
+    for filename in _glob(f'{root}/*{suffix}'):
+        yield filename
+    for filename in _glob(f'{root}/**/*{suffix}'):
+        yield filename
+
+
+def iter_files(root, suffix=None, relparent=None, *,
+               get_files=os.walk,
+               _glob=glob_tree,
+               _walk=walk_tree,
+               ):
+    """Yield each file in the tree under the given directory name.
+
+    If "root" is a non-string iterable then do the same for each of
+    those trees.
+
+    If "suffix" is provided then only files with that suffix will
+    be included.
+
+    if "relparent" is provided then it is used to resolve each
+    filename as a relative path.
+    """
+    if not isinstance(root, str):
+        roots = root
+        for root in roots:
+            yield from iter_files(root, suffix, relparent,
+                                  get_files=get_files,
+                                  _glob=_glob, _walk=_walk)
+        return
+
+    # Use the right "walk" function.
+    if get_files in (glob.glob, glob.iglob, glob_tree):
+        get_files = _glob
+    else:
+        _files = _walk_tree if get_files in (os.walk, walk_tree) else get_files
+        get_files = (lambda *a, **k: _walk(*a, walk=_files, **k))
+
+    # Handle a single suffix.
+    if suffix and not isinstance(suffix, str):
+        filenames = get_files(root)
+        suffix = tuple(suffix)
+    else:
+        filenames = get_files(root, suffix=suffix)
+        suffix = None
+
+    for filename in filenames:
+        if suffix and not isinstance(suffix, str):  # multiple suffixes
+            if not filename.endswith(suffix):
+                continue
+        if relparent:
+            filename = os.path.relpath(filename, relparent)
+        yield filename
+
+
+def iter_files_by_suffix(root, suffixes, relparent=None, *,
+                         walk=walk_tree,
+                         _iter_files=iter_files,
+                         ):
+    """Yield each file in the tree that has the given suffixes.
+
+    Unlike iter_files(), the results are in the original suffix order.
+    """
+    if isinstance(suffixes, str):
+        suffixes = [suffixes]
+    # XXX Ignore repeated suffixes?
+    for suffix in suffixes:
+        yield from _iter_files(root, suffix, relparent)
+
+
+def iter_cpython_files(*,
+                       walk=walk_tree,
+                       _files=iter_files_by_suffix,
+                       ):
+    """Yield each file in the tree for each of the given directory names."""
+    excludedtrees = [
+        os.path.join('Include', 'cpython', ''),
+        ]
+    def is_excluded(filename):
+        for root in excludedtrees:
+            if filename.startswith(root):
+                return True
+        return False
+    for filename in _files(SOURCE_DIRS, C_SOURCE_SUFFIXES, REPO_ROOT,
+                           walk=walk,
+                           ):
+        if is_excluded(filename):
+            continue
+        yield filename
diff --git a/Tools/c-analyzer/c_analyzer_common/info.py b/Tools/c-analyzer/c_analyzer_common/info.py
new file mode 100644
index 000000000000..e21738040645
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer_common/info.py
@@ -0,0 +1,69 @@
+from collections import namedtuple
+import re
+
+from .util import classonly, _NTBase
+
+
+UNKNOWN = '???'
+
+NAME_RE = re.compile(r'^([a-zA-Z]|_\w*[a-zA-Z]\w*|[a-zA-Z]\w*)$')
+
+
+class ID(_NTBase, namedtuple('ID', 'filename funcname name')):
+    """A unique ID for a single symbol or declaration."""
+
+    __slots__ = ()
+    # XXX Add optional conditions (tuple of strings) field.
+    #conditions = Slot()
+
+    @classonly
+    def from_raw(cls, raw):
+        if not raw:
+            return None
+        if isinstance(raw, str):
+            return cls(None, None, raw)
+        try:
+            name, = raw
+            filename = None
+        except ValueError:
+            try:
+                filename, name = raw
+            except ValueError:
+                return super().from_raw(raw)
+        return cls(filename, None, name)
+
+    def __new__(cls, filename, funcname, name):
+        self = super().__new__(
+                cls,
+                filename=str(filename) if filename else None,
+                funcname=str(funcname) if funcname else None,
+                name=str(name) if name else None,
+                )
+        #cls.conditions.set(self, tuple(str(s) if s else None
+        #                               for s in conditions or ()))
+        return self
+
+    def validate(self):
+        """Fail if the object is invalid (i.e. init with bad data)."""
+        if not self.name:
+            raise TypeError('missing name')
+        else:
+            if not NAME_RE.match(self.name):
+                raise ValueError(
+                        f'name must be an identifier, got {self.name!r}')
+
+        # Symbols from a binary might not have filename/funcname info.
+
+        if self.funcname:
+            if not self.filename:
+                raise TypeError('missing filename')
+            if not NAME_RE.match(self.funcname) and self.funcname != UNKNOWN:
+                raise ValueError(
+                        f'name must be an identifier, got {self.funcname!r}')
+
+        # XXX Require the filename (at least UNKONWN)?
+        # XXX Check the filename?
+
+    @property
+    def islocal(self):
+        return self.funcname is not None
diff --git a/Tools/c-analyzer/c_analyzer_common/known.py b/Tools/c-analyzer/c_analyzer_common/known.py
new file mode 100644
index 000000000000..a0c6dfa5aa47
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer_common/known.py
@@ -0,0 +1,67 @@
+import csv
+import os.path
+
+from c_parser.info import Variable
+
+from . import DATA_DIR
+from .info import ID, UNKNOWN
+from .util import read_tsv
+
+
+DATA_FILE = os.path.join(DATA_DIR, 'known.tsv')
+
+COLUMNS = ('filename', 'funcname', 'name', 'kind', 'declaration')
+HEADER = '\t'.join(COLUMNS)
+
+
+# XXX need tests:
+# * from_file()
+
+def from_file(infile, *,
+              _read_tsv=read_tsv,
+              ):
+    """Return the info for known declarations in the given file."""
+    known = {
+        'variables': {},
+        #'types': {},
+        #'constants': {},
+        #'macros': {},
+        }
+    for row in _read_tsv(infile, HEADER):
+        filename, funcname, name, kind, declaration = row
+        if not funcname or funcname == '-':
+            funcname = None
+        id = ID(filename, funcname, name)
+        if kind == 'variable':
+            values = known['variables']
+            value = Variable(id, declaration)
+            value._isglobal = _is_global(declaration) or id.funcname is None
+        else:
+            raise ValueError(f'unsupported kind in row {row}')
+        if value.name == 'id' and declaration == UNKNOWN:
+            # None of these are variables.
+            declaration = 'int id';
+        else:
+            value.validate()
+        values[id] = value
+    return known
+
+
+def _is_global(vartype):
+    # statics
+    if vartype.startswith('static '):
+        return True
+    if vartype.startswith(('Py_LOCAL(', 'Py_LOCAL_INLINE(')):
+        return True
+    if vartype.startswith(('_Py_IDENTIFIER(', '_Py_static_string(')):
+        return True
+    if vartype.startswith('PyDoc_VAR('):
+        return True
+    if vartype.startswith(('SLOT1BINFULL(', 'SLOT1BIN(')):
+        return True
+    if vartype.startswith('WRAP_METHOD('):
+        return True
+    # public extern
+    if vartype.startswith('PyAPI_DATA('):
+        return True
+    return False
diff --git a/Tools/c-analyzer/c_analyzer_common/util.py b/Tools/c-analyzer/c_analyzer_common/util.py
new file mode 100644
index 000000000000..511c54f17834
--- /dev/null
+++ b/Tools/c-analyzer/c_analyzer_common/util.py
@@ -0,0 +1,214 @@
+import csv
+import subprocess
+
+
+_NOT_SET = object()
+
+
+def run_cmd(argv, **kwargs):
+    proc = subprocess.run(
+            argv,
+            #capture_output=True,
+            #stderr=subprocess.STDOUT,
+            stdout=subprocess.PIPE,
+            text=True,
+            check=True,
+            **kwargs
+            )
+    return proc.stdout
+
+
+def read_tsv(infile, header, *,
+             _open=open,
+             _get_reader=csv.reader,
+             ):
+    """Yield each row of the given TSV (tab-separated) file."""
+    if isinstance(infile, str):
+        with _open(infile, newline='') as infile:
+            yield from read_tsv(infile, header,
+                                _open=_open,
+                                _get_reader=_get_reader,
+                                )
+            return
+    lines = iter(infile)
+
+    # Validate the header.
+    try:
+        actualheader = next(lines).strip()
+    except StopIteration:
+        actualheader = ''
+    if actualheader != header:
+        raise ValueError(f'bad header {actualheader!r}')
+
+    for row in _get_reader(lines, delimiter='\t'):
+        yield tuple(v.strip() for v in row)
+
+
+def write_tsv(outfile, header, rows, *,
+             _open=open,
+             _get_writer=csv.writer,
+             ):
+    """Write each of the rows to the given TSV (tab-separated) file."""
+    if isinstance(outfile, str):
+        with _open(outfile, 'w', newline='') as outfile:
+            return write_tsv(outfile, header, rows,
+                            _open=_open,
+                            _get_writer=_get_writer,
+                            )
+
+    if isinstance(header, str):
+        header = header.split('\t')
+    writer = _get_writer(outfile, delimiter='\t')
+    writer.writerow(header)
+    for row in rows:
+        writer.writerow('' if v is None else str(v)
+                        for v in row)
+
+
+class Slot:
+    """A descriptor that provides a slot.
+
+    This is useful for types that can't have slots via __slots__,
+    e.g. tuple subclasses.
+    """
+
+    __slots__ = ('initial', 'default', 'readonly', 'instances', 'name')
+
+    def __init__(self, initial=_NOT_SET, *,
+                 default=_NOT_SET,
+                 readonly=False,
+                 ):
+        self.initial = initial
+        self.default = default
+        self.readonly = readonly
+
+        self.instances = {}
+        self.name = None
+
+    def __set_name__(self, cls, name):
+        if self.name is not None:
+            raise TypeError('already used')
+        self.name = name
+
+    def __get__(self, obj, cls):
+        if obj is None:  # called on the class
+            return self
+        try:
+            value = self.instances[id(obj)]
+        except KeyError:
+            if self.initial is _NOT_SET:
+                value = self.default
+            else:
+                value = self.initial
+            self.instances[id(obj)] = value
+        if value is _NOT_SET:
+            raise AttributeError(self.name)
+        # XXX Optionally make a copy?
+        return value
+
+    def __set__(self, obj, value):
+        if self.readonly:
+            raise AttributeError(f'{self.name} is readonly')
+        # XXX Optionally coerce?
+        self.instances[id(obj)] = value
+
+    def __delete__(self, obj):
+        if self.readonly:
+            raise AttributeError(f'{self.name} is readonly')
+        self.instances[id(obj)] = self.default
+
+    def set(self, obj, value):
+        """Update the cached value for an object.
+
+        This works even if the descriptor is read-only.  This is
+        particularly useful when initializing the object (e.g. in
+        its __new__ or __init__).
+        """
+        self.instances[id(obj)] = value
+
+
+class classonly:
+    """A non-data descriptor that makes a value only visible on the class.
+
+    This is like the "classmethod" builtin, but does not show up on
+    instances of the class.  It may be used as a decorator.
+    """
+
+    def __init__(self, value):
+        self.value = value
+        self.getter = classmethod(value).__get__
+        self.name = None
+
+    def __set_name__(self, cls, name):
+        if self.name is not None:
+            raise TypeError('already used')
+        self.name = name
+
+    def __get__(self, obj, cls):
+        if obj is not None:
+            raise AttributeError(self.name)
+        # called on the class
+        return self.getter(None, cls)
+
+
+class _NTBase:
+
+    __slots__ = ()
+
+    @classonly
+    def from_raw(cls, raw):
+        if not raw:
+            return None
+        elif isinstance(raw, cls):
+            return raw
+        elif isinstance(raw, str):
+            return cls.from_string(raw)
+        else:
+            if hasattr(raw, 'items'):
+                return cls(**raw)
+            try:
+                args = tuple(raw)
+            except TypeError:
+                pass
+            else:
+                return cls(*args)
+        raise NotImplementedError
+
+    @classonly
+    def from_string(cls, value):
+        """Return a new instance based on the given string."""
+        raise NotImplementedError
+
+    @classmethod
+    def _make(cls, iterable):  # The default _make() is not subclass-friendly.
+        return cls.__new__(cls, *iterable)
+
+    # XXX Always validate?
+    #def __init__(self, *args, **kwargs):
+    #    self.validate()
+
+    # XXX The default __repr__() is not subclass-friendly (where the name changes).
+    #def __repr__(self):
+    #    _, _, sig = super().__repr__().partition('(')
+    #    return f'{self.__class__.__name__}({sig}'
+
+    # To make sorting work with None:
+    def __lt__(self, other):
+        try:
+            return super().__lt__(other)
+        except TypeError:
+            if None in self:
+                return True
+            elif None in other:
+                return False
+            else:
+                raise
+
+    def validate(self):
+        return
+
+    # XXX Always validate?
+    #def _replace(self, **kwargs):
+    #    obj = super()._replace(**kwargs)
+    #    obj.validate()
+    #    return obj
diff --git a/Tools/c-analyzer/c_globals/README b/Tools/c-analyzer/c_globals/README
new file mode 100644
index 000000000000..772b8be27008
--- /dev/null
+++ b/Tools/c-analyzer/c_globals/README
@@ -0,0 +1,72 @@
+#######################################
+# C Globals and CPython Runtime State.
+
+CPython's C code makes extensive use of global variables (whether static
+globals or static locals).  Each such variable falls into one of several
+categories:
+
+* strictly const data
+* used exclusively in main or in the REPL
+* process-global state (e.g. managing process-level resources
+  like signals and file descriptors)
+* Python "global" runtime state
+* per-interpreter runtime state
+
+The last one can be a problem as soon as anyone creates a second
+interpreter (AKA "subinterpreter") in a process.  It is definitely a
+problem under subinterpreters if they are no longer sharing the GIL,
+since the GIL protects us from a lot of race conditions.  Keep in mind
+that ultimately *all* objects (PyObject) should be treated as
+per-interpreter state.  This includes "static types", freelists,
+_PyIdentifier, and singletons.  Take that in for a second.  It has
+significant implications on where we use static variables!
+
+Be aware that module-global state (stored in C statics) is a kind of
+per-interpreter state.  There have been efforts across many years, and
+still going, to provide extension module authors mechanisms to store
+that state safely (see PEPs 3121, 489, etc.).
+
+(Note that there has been discussion around support for running multiple
+Python runtimes in the same process.  That would ends up with the same
+problems, relative to static variables, that subinterpreters have.)
+
+Historically we have been bad at keeping per-interpreter state out of
+static variables, mostly because until recently subinterpreters were
+not widely used nor even factored in to solutions.  However, the
+feature is growing in popularity and use in the community.
+
+Mandate: "Eliminate use of static variables for per-interpreter state."
+
+The "c-statics.py" script in this directory, along with its accompanying
+data files, are part of the effort to resolve existing problems with
+our use of static variables and to prevent future problems.
+
+#-------------------------
+## statics for actually-global state (and runtime state consolidation)
+
+In general, holding any kind of state in static variables
+increases maintenance burden and increases the complexity of code (e.g.
+we use TSS to identify the active thread state).  So it is a good idea
+to avoid using statics for state even if for the "global" runtime or
+for process-global state.
+
+Relative to maintenance burden, one problem is where the runtime
+state is spread throughout the codebase in dozens of individual
+globals.  Unlike the other globals, the runtime state represents a set
+of values that are constantly shifting in a complex way.  When they are
+spread out it's harder to get a clear picture of what the runtime
+involves.  Furthermore, when they are spread out it complicates efforts
+that change the runtime.
+
+Consequently, the globals for Python's runtime state have been
+consolidated under a single top-level _PyRuntime global. No new globals
+should be added for runtime state.  Instead, they should be added to
+_PyRuntimeState or one of its sub-structs.  The tools in this directory
+are run as part of the test suite to ensure that no new globals have
+been added.  The script can be run manually as well:
+
+  ./python Lib/test/test_c_statics/c-statics.py check
+
+If it reports any globals then they should be resolved.  If the globals
+are runtime state then they should be folded into _PyRuntimeState.
+Otherwise they should be marked as ignored.
diff --git a/Tools/c-analyzer/c_globals/__init__.py b/Tools/c-analyzer/c_globals/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/Tools/c-analyzer/c_globals/__main__.py b/Tools/c-analyzer/c_globals/__main__.py
new file mode 100644
index 000000000000..9570fb6a14c4
--- /dev/null
+++ b/Tools/c-analyzer/c_globals/__main__.py
@@ -0,0 +1,209 @@
+import argparse
+import os.path
+import re
+import sys
+
+from c_analyzer_common import SOURCE_DIRS, REPO_ROOT
+from c_analyzer_common.info import UNKNOWN
+from c_analyzer_common.known import (
+    from_file as known_from_file,
+    DATA_FILE as KNOWN_FILE,
+    )
+from . import find, show
+from .supported import is_supported, ignored_from_file, IGNORED_FILE, _is_object
+
+
+def _match_unused_global(variable, knownvars, used):
+    found = []
+    for varid in knownvars:
+        if varid in used:
+            continue
+        if varid.funcname is not None:
+            continue
+        if varid.name != variable.name:
+            continue
+        if variable.filename and variable.filename != UNKNOWN:
+            if variable.filename == varid.filename:
+                found.append(varid)
+        else:
+            found.append(varid)
+    return found
+
+
+def _check_results(unknown, knownvars, used):
+    badknown = set()
+    for variable in sorted(unknown):
+        msg = None
+        if variable.funcname != UNKNOWN:
+            msg = f'could not find global symbol {variable.id}'
+        elif m := _match_unused_global(variable, knownvars, used):
+            assert isinstance(m, list)
+            badknown.update(m)
+        elif variable.name in ('completed', 'id'):  # XXX Figure out where these variables are.
+            unknown.remove(variable)
+        else:
+            msg = f'could not find local symbol {variable.id}'
+        if msg:
+            #raise Exception(msg)
+            print(msg)
+    if badknown:
+        print('---')
+        print(f'{len(badknown)} globals in known.tsv, but may actually be local:')
+        for varid in sorted(badknown):
+            print(f'{varid.filename:30} {varid.name}')
+    unused = sorted(varid
+                    for varid in set(knownvars) - used
+                    if varid.name != 'id')  # XXX Figure out where these variables are.
+    if unused:
+        print('---')
+        print(f'did not use {len(unused)} known vars:')
+        for varid in unused:
+            print(f'{varid.filename:30} {varid.funcname or "-":20} {varid.name}')
+        raise Exception('not all known symbols used')
+    if unknown:
+        print('---')
+        raise Exception('could not find all symbols')
+
+
+def _find_globals(dirnames, known, ignored):
+    if dirnames == SOURCE_DIRS:
+        dirnames = [os.path.relpath(d, REPO_ROOT) for d in dirnames]
+
+    ignored = ignored_from_file(ignored)
+    known = known_from_file(known)
+
+    used = set()
+    unknown = set()
+    knownvars = (known or {}).get('variables')
+    for variable in find.globals_from_binary(knownvars=knownvars,
+                                             dirnames=dirnames):
+    #for variable in find.globals(dirnames, known, kind='platform'):
+        if variable.vartype == UNKNOWN:
+            unknown.add(variable)
+            continue
+        yield variable, is_supported(variable, ignored, known)
+        used.add(variable.id)
+
+    #_check_results(unknown, knownvars, used)
+
+
+def cmd_check(cmd, dirs=SOURCE_DIRS, *,
+              ignored=IGNORED_FILE,
+              known=KNOWN_FILE,
+              _find=_find_globals,
+              _show=show.basic,
+              _print=print,
+              ):
+    """
+    Fail if there are unsupported globals variables.
+
+    In the failure case, the list of unsupported variables
+    will be printed out.
+    """
+    unsupported = [v for v, s in _find(dirs, known, ignored) if not s]
+    if not unsupported:
+        #_print('okay')
+        return
+
+    _print('ERROR: found unsupported global variables')
+    _print()
+    _show(sorted(unsupported))
+    _print(f' ({len(unsupported)} total)')
+    sys.exit(1)
+
+
+def cmd_show(cmd, dirs=SOURCE_DIRS, *,
+             ignored=IGNORED_FILE,
+             known=KNOWN_FILE,
+             skip_objects=False,
+              _find=_find_globals,
+             _show=show.basic,
+             _print=print,
+             ):
+    """
+    Print out the list of found global variables.
+
+    The variables will be distinguished as "supported" or "unsupported".
+    """
+    allsupported = []
+    allunsupported = []
+    for found, supported in _find(dirs, known, ignored):
+        if skip_objects:  # XXX Support proper filters instead.
+            if _is_object(found.vartype):
+                continue
+        (allsupported if supported else allunsupported
+         ).append(found)
+
+    _print('supported:')
+    _print('----------')
+    _show(sorted(allsupported))
+    _print(f' ({len(allsupported)} total)')
+    _print()
+    _print('unsupported:')
+    _print('------------')
+    _show(sorted(allunsupported))
+    _print(f' ({len(allunsupported)} total)')
+
+
+#############################
+# the script
+
+COMMANDS = {
+        'check': cmd_check,
+        'show': cmd_show,
+        }
+
+PROG = sys.argv[0]
+PROG = 'c-globals.py'
+
+
+def parse_args(prog=PROG, argv=sys.argv[1:], *, _fail=None):
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument('--ignored', metavar='FILE',
+                        default=IGNORED_FILE,
+                        help='path to file that lists ignored vars')
+    common.add_argument('--known', metavar='FILE',
+                        default=KNOWN_FILE,
+                        help='path to file that lists known types')
+    common.add_argument('dirs', metavar='DIR', nargs='*',
+                        default=SOURCE_DIRS,
+                        help='a directory to check')
+
+    parser = argparse.ArgumentParser(
+            prog=prog,
+            )
+    subs = parser.add_subparsers(dest='cmd')
+
+    check = subs.add_parser('check', parents=[common])
+
+    show = subs.add_parser('show', parents=[common])
+    show.add_argument('--skip-objects', action='store_true')
+
+    if _fail is None:
+        def _fail(msg):
+            parser.error(msg)
+
+    # Now parse the args.
+    args = parser.parse_args(argv)
+    ns = vars(args)
+
+    cmd = ns.pop('cmd')
+    if not cmd:
+        _fail('missing command')
+
+    return cmd, ns
+
+
+def main(cmd, cmdkwargs=None, *, _COMMANDS=COMMANDS):
+    try:
+        cmdfunc = _COMMANDS[cmd]
+    except KeyError:
+        raise ValueError(
+            f'unsupported cmd {cmd!r}' if cmd else 'missing cmd')
+
+    cmdfunc(cmd, **cmdkwargs or {})
+
+
+if __name__ == '__main__':
+    cmd, cmdkwargs = parse_args()
+    main(cmd, cmdkwargs)
diff --git a/Tools/c-analyzer/c_globals/find.py b/Tools/c-analyzer/c_globals/find.py
new file mode 100644
index 000000000000..a51b947cbdf1
--- /dev/null
+++ b/Tools/c-analyzer/c_globals/find.py
@@ -0,0 +1,95 @@
+from c_analyzer_common import SOURCE_DIRS
+from c_analyzer_common.info import UNKNOWN
+from c_symbols import (
+        info as s_info,
+        binary as b_symbols,
+        source as s_symbols,
+        resolve,
+        )
+from c_parser import info, declarations
+
+
+# XXX needs tests:
+# * iter_variables
+
+def globals_from_binary(binfile=b_symbols.PYTHON, *,
+                        knownvars=None,
+                        dirnames=None,
+                        _iter_symbols=b_symbols.iter_symbols,
+                        _resolve=resolve.symbols_to_variables,
+                        _get_symbol_resolver=resolve.get_resolver,
+                        ):
+    """Yield a Variable for each found Symbol.
+
+    Details are filled in from the given "known" variables and types.
+    """
+    symbols = _iter_symbols(binfile, find_local_symbol=None)
+    #symbols = list(symbols)
+    for variable in _resolve(symbols,
+                             resolve=_get_symbol_resolver(knownvars, dirnames),
+                             ):
+        # Skip each non-global variable (unless we couldn't find it).
+        # XXX Drop the "UNKNOWN" condition?
+        if not variable.isglobal and variable.vartype != UNKNOWN:
+            continue
+        yield variable
+
+
+def globals_from_declarations(dirnames=SOURCE_DIRS, *,
+                              known=None,
+                              ):
+    """Yield a Variable for each found declaration.
+
+    Details are filled in from the given "known" variables and types.
+    """
+    raise NotImplementedError
+
+
+def iter_variables(kind='platform', *,
+                   known=None,
+                   dirnames=None,
+                   _resolve_symbols=resolve.symbols_to_variables,
+                   _get_symbol_resolver=resolve.get_resolver,
+                   _symbols_from_binary=b_symbols.iter_symbols,
+                   _symbols_from_source=s_symbols.iter_symbols,
+                   _iter_raw=declarations.iter_all,
+                   _iter_preprocessed=declarations.iter_preprocessed,
+                   ):
+    """Yield a Variable for each one found (e.g. in files)."""
+    kind = kind or 'platform'
+
+    if kind == 'symbols':
+        knownvars = (known or {}).get('variables')
+        yield from _resolve_symbols(
+                _symbols_from_source(dirnames, known),
+                resolve=_get_symbol_resolver(knownvars, dirnames),
+                )
+    elif kind == 'platform':
+        knownvars = (known or {}).get('variables')
+        yield from _resolve_symbols(
+                _symbols_from_binary(find_local_symbol=None),
+                resolve=_get_symbol_resolver(knownvars, dirnames),
+                )
+    elif kind == 'declarations':
+        for decl in _iter_raw(dirnames):
+            if not isinstance(decl, info.Variable):
+                continue
+            yield decl
+    elif kind == 'preprocessed':
+        for decl in _iter_preprocessed(dirnames):
+            if not isinstance(decl, info.Variable):
+                continue
+            yield decl
+    else:
+        raise ValueError(f'unsupported kind {kind!r}')
+
+
+def globals(dirnames, known, *,
+            kind=None,  # Use the default.
+            _iter_variables=iter_variables,
+            ):
+    """Return a list of (StaticVar, <supported>) for each found global var."""
+    for found in _iter_variables(kind, known=known, dirnames=dirnames):
+        if not found.isglobal:
+            continue
+        yield found
diff --git a/Tools/c-analyzer/c_globals/show.py b/Tools/c-analyzer/c_globals/show.py
new file mode 100644
index 000000000000..f4298b17b678
--- /dev/null
+++ b/Tools/c-analyzer/c_globals/show.py
@@ -0,0 +1,16 @@
+
+def basic(globals, *,
+          _print=print):
+    """Print each row simply."""
+    for variable in globals:
+        if variable.funcname:
+            line = f'{variable.filename}:{variable.funcname}():{variable.name}'
+        else:
+            line = f'{variable.filename}:{variable.name}'
+        vartype = variable.vartype
+        #if vartype.startswith('static '):
+        #    vartype = vartype.partition(' ')[2]
+        #else:
+        #    vartype = '=' + vartype
+        line = f'{line:<64} {vartype}'
+        _print(line)
diff --git a/Tools/c-analyzer/c_globals/supported.py b/Tools/c-analyzer/c_globals/supported.py
new file mode 100644
index 000000000000..4643e4ef6e88
--- /dev/null
+++ b/Tools/c-analyzer/c_globals/supported.py
@@ -0,0 +1,368 @@
+import os.path
+import re
+
+from c_analyzer_common import DATA_DIR
+from c_analyzer_common.info import ID
+from c_analyzer_common.util import read_tsv, write_tsv
+
+
+IGNORED_FILE = os.path.join(DATA_DIR, 'ignored.tsv')
+
+IGNORED_COLUMNS = ('filename', 'funcname', 'name', 'kind', 'reason')
+IGNORED_HEADER = '\t'.join(IGNORED_COLUMNS)
+
+# XXX Move these to ignored.tsv.
+IGNORED = {
+        # global
+        'PyImport_FrozenModules': 'process-global',
+        'M___hello__': 'process-global',
+        'inittab_copy': 'process-global',
+        'PyHash_Func': 'process-global',
+        '_Py_HashSecret_Initialized': 'process-global',
+        '_TARGET_LOCALES': 'process-global',
+
+        # startup (only changed before/during)
+        '_PyRuntime': 'runtime startup',
+        'runtime_initialized': 'runtime startup',
+        'static_arg_parsers': 'runtime startup',
+        'orig_argv': 'runtime startup',
+        'opt_ptr': 'runtime startup',
+        '_preinit_warnoptions': 'runtime startup',
+        '_Py_StandardStreamEncoding': 'runtime startup',
+        'Py_FileSystemDefaultEncoding': 'runtime startup',
+        '_Py_StandardStreamErrors': 'runtime startup',
+        'Py_FileSystemDefaultEncodeErrors': 'runtime startup',
+        'Py_BytesWarningFlag': 'runtime startup',
+        'Py_DebugFlag': 'runtime startup',
+        'Py_DontWriteBytecodeFlag': 'runtime startup',
+        'Py_FrozenFlag': 'runtime startup',
+        'Py_HashRandomizationFlag': 'runtime startup',
+        'Py_IgnoreEnvironmentFlag': 'runtime startup',
+        'Py_InspectFlag': 'runtime startup',
+        'Py_InteractiveFlag': 'runtime startup',
+        'Py_IsolatedFlag': 'runtime startup',
+        'Py_NoSiteFlag': 'runtime startup',
+        'Py_NoUserSiteDirectory': 'runtime startup',
+        'Py_OptimizeFlag': 'runtime startup',
+        'Py_QuietFlag': 'runtime startup',
+        'Py_UTF8Mode': 'runtime startup',
+        'Py_UnbufferedStdioFlag': 'runtime startup',
+        'Py_VerboseFlag': 'runtime startup',
+        '_Py_path_config': 'runtime startup',
+        '_PyOS_optarg': 'runtime startup',
+        '_PyOS_opterr': 'runtime startup',
+        '_PyOS_optind': 'runtime startup',
+        '_Py_HashSecret': 'runtime startup',
+
+        # REPL
+        '_PyOS_ReadlineLock': 'repl',
+        '_PyOS_ReadlineTState': 'repl',
+
+        # effectively const
+        'tracemalloc_empty_traceback': 'const',
+        '_empty_bitmap_node': 'const',
+        'posix_constants_pathconf': 'const',
+        'posix_constants_confstr': 'const',
+        'posix_constants_sysconf': 'const',
+        '_PySys_ImplCacheTag': 'const',
+        '_PySys_ImplName': 'const',
+        'PyImport_Inittab': 'const',
+        '_PyImport_DynLoadFiletab': 'const',
+        '_PyParser_Grammar': 'const',
+        'Py_hexdigits': 'const',
+        '_PyImport_Inittab': 'const',
+        '_PyByteArray_empty_string': 'const',
+        '_PyLong_DigitValue': 'const',
+        '_Py_SwappedOp': 'const',
+        'PyStructSequence_UnnamedField': 'const',
+
+        # signals are main-thread only
+        'faulthandler_handlers': 'signals are main-thread only',
+        'user_signals': 'signals are main-thread only',
+        'wakeup': 'signals are main-thread only',
+
+        # hacks
+        '_PySet_Dummy': 'only used as a placeholder',
+        }
+
+BENIGN = 'races here are benign and unlikely'
+
+
+def is_supported(variable, ignored=None, known=None, *,
+                 _ignored=(lambda *a, **k: _is_ignored(*a, **k)),
+                 _vartype_okay=(lambda *a, **k: _is_vartype_okay(*a, **k)),
+                 ):
+    """Return True if the given global variable is okay in CPython."""
+    if _ignored(variable,
+                ignored and ignored.get('variables')):
+        return True
+    elif _vartype_okay(variable.vartype,
+                       ignored.get('types')):
+        return True
+    else:
+        return False
+
+
+def _is_ignored(variable, ignoredvars=None, *,
+                _IGNORED=IGNORED,
+                ):
+    """Return the reason if the variable is a supported global.
+
+    Return None if the variable is not a supported global.
+    """
+    if ignoredvars and (reason := ignoredvars.get(variable.id)):
+        return reason
+
+    if variable.funcname is None:
+        if reason := _IGNORED.get(variable.name):
+            return reason
+
+    # compiler
+    if variable.filename == 'Python/graminit.c':
+        if variable.vartype.startswith('static state '):
+            return 'compiler'
+    if variable.filename == 'Python/symtable.c':
+        if variable.vartype.startswith('static identifier '):
+            return 'compiler'
+    if variable.filename == 'Python/Python-ast.c':
+        # These should be const.
+        if variable.name.endswith('_field'):
+            return 'compiler'
+        if variable.name.endswith('_attribute'):
+            return 'compiler'
+
+    # other
+    if variable.filename == 'Python/dtoa.c':
+        # guarded by lock?
+        if variable.name in ('p5s', 'freelist'):
+            return 'dtoa is thread-safe?'
+        if variable.name in ('private_mem', 'pmem_next'):
+            return 'dtoa is thread-safe?'
+    if variable.filename == 'Python/thread.c':
+        # Threads do not become an issue until after these have been set
+        # and these never get changed after that.
+        if variable.name in ('initialized', 'thread_debug'):
+            return 'thread-safe'
+    if variable.filename == 'Python/getversion.c':
+        if variable.name == 'version':
+            # Races are benign here, as well as unlikely.
+            return BENIGN
+    if variable.filename == 'Python/fileutils.c':
+        if variable.name == 'force_ascii':
+            return BENIGN
+        if variable.name == 'ioctl_works':
+            return BENIGN
+        if variable.name == '_Py_open_cloexec_works':
+            return BENIGN
+    if variable.filename == 'Python/codecs.c':
+        if variable.name == 'ucnhash_CAPI':
+            return BENIGN
+    if variable.filename == 'Python/bootstrap_hash.c':
+        if variable.name == 'getrandom_works':
+            return BENIGN
+    if variable.filename == 'Objects/unicodeobject.c':
+        if variable.name == 'ucnhash_CAPI':
+            return BENIGN
+        if variable.name == 'bloom_linebreak':
+            # *mostly* benign
+            return BENIGN
+    if variable.filename == 'Modules/getbuildinfo.c':
+        if variable.name == 'buildinfo':
+            # The static is used for pre-allocation.
+            return BENIGN
+    if variable.filename == 'Modules/posixmodule.c':
+        if variable.name == 'ticks_per_second':
+            return BENIGN
+        if variable.name == 'dup3_works':
+            return BENIGN
+    if variable.filename == 'Modules/timemodule.c':
+        if variable.name == 'ticks_per_second':
+            return BENIGN
+    if variable.filename == 'Objects/longobject.c':
+        if variable.name == 'log_base_BASE':
+            return BENIGN
+        if variable.name == 'convwidth_base':
+            return BENIGN
+        if variable.name == 'convmultmax_base':
+            return BENIGN
+
+    return None
+
+
+def _is_vartype_okay(vartype, ignoredtypes=None):
+    if _is_object(vartype):
+        return None
+
+    if vartype.startswith('static const '):
+        return 'const'
+    if vartype.startswith('const '):
+        return 'const'
+
+    # components for TypeObject definitions
+    for name in ('PyMethodDef', 'PyGetSetDef', 'PyMemberDef'):
+        if name in vartype:
+            return 'const'
+    for name in ('PyNumberMethods', 'PySequenceMethods', 'PyMappingMethods',
+                 'PyBufferProcs', 'PyAsyncMethods'):
+        if name in vartype:
+            return 'const'
+    for name in ('slotdef', 'newfunc'):
+        if name in vartype:
+            return 'const'
+
+    # structseq
+    for name in ('PyStructSequence_Desc', 'PyStructSequence_Field'):
+        if name in vartype:
+            return 'const'
+
+    # other definiitions
+    if 'PyModuleDef' in vartype:
+        return 'const'
+
+    # thread-safe
+    if '_Py_atomic_int' in vartype:
+        return 'thread-safe'
+    if 'pthread_condattr_t' in vartype:
+        return 'thread-safe'
+
+    # startup
+    if '_Py_PreInitEntry' in vartype:
+        return 'startup'
+
+    # global
+#    if 'PyMemAllocatorEx' in vartype:
+#        return True
+
+    # others
+#    if 'PyThread_type_lock' in vartype:
+#        return True
+
+    # XXX ???
+    # _Py_tss_t
+    # _Py_hashtable_t
+    # stack_t
+    # _PyUnicode_Name_CAPI
+
+    # functions
+    if '(' in vartype and '[' not in vartype:
+        return 'function pointer'
+
+    # XXX finish!
+    # * allow const values?
+    #raise NotImplementedError
+    return None
+
+
+def _is_object(vartype):
+    if re.match(r'.*\bPy\w*Object\b', vartype):
+        return True
+    if '_PyArg_Parser ' in vartype:
+        return True
+    if vartype.startswith(('_Py_IDENTIFIER(', 'static _Py_Identifier',
+                           '_Py_static_string(')):
+        return True
+    if 'traceback_t' in vartype:
+        return True
+    if 'PyAsyncGenASend' in vartype:
+        return True
+    if '_PyAsyncGenWrappedValue' in vartype:
+        return True
+    if 'PyContext' in vartype:
+        return True
+    if 'method_cache_entry' in vartype:
+        return True
+    if vartype.startswith('static identifier '):
+        return True
+    if vartype.endswith((' _Py_FalseStruct', ' _Py_TrueStruct')):
+        return True
+
+    # XXX Add more?
+
+    #for part in vartype.split():
+    #    # XXX const is automatic True?
+    #    if part == 'PyObject' or part.startswith('PyObject['):
+    #        return True
+    return False
+
+
+def ignored_from_file(infile, *,
+                      _read_tsv=read_tsv,
+                      ):
+    """Yield a Variable for each ignored var in the file."""
+    ignored = {
+        'variables': {},
+        #'types': {},
+        #'constants': {},
+        #'macros': {},
+        }
+    for row in _read_tsv(infile, IGNORED_HEADER):
+        filename, funcname, name, kind, reason = row
+        if not funcname or funcname == '-':
+            funcname = None
+        id = ID(filename, funcname, name)
+        if kind == 'variable':
+            values = ignored['variables']
+        else:
+            raise ValueError(f'unsupported kind in row {row}')
+        values[id] = reason
+    return ignored
+
+
+##################################
+# generate
+
+def _get_row(varid, reason):
+    return (
+            varid.filename,
+            varid.funcname or '-',
+            varid.name,
+            'variable',
+            str(reason),
+            )
+
+
+def _get_rows(variables, ignored=None, *,
+              _as_row=_get_row,
+              _is_ignored=_is_ignored,
+              _vartype_okay=_is_vartype_okay,
+              ):
+    count = 0
+    for variable in variables:
+        reason = _is_ignored(variable,
+                             ignored and ignored.get('variables'),
+                             )
+        if not reason:
+            reason = _vartype_okay(variable.vartype,
+                                   ignored and ignored.get('types'))
+        if not reason:
+            continue
+
+        print(' ', variable, repr(reason))
+        yield _as_row(variable.id, reason)
+        count += 1
+    print(f'total: {count}')
+
+
+def _generate_ignored_file(variables, filename=None, *,
+                           _generate_rows=_get_rows,
+                           _write_tsv=write_tsv,
+                           ):
+    if not filename:
+        filename = IGNORED_FILE + '.new'
+    rows = _generate_rows(variables)
+    _write_tsv(filename, IGNORED_HEADER, rows)
+
+
+if __name__ == '__main__':
+    from c_analyzer_common import SOURCE_DIRS
+    from c_analyzer_common.known import (
+        from_file as known_from_file,
+        DATA_FILE as KNOWN_FILE,
+        )
+    from . import find
+    known = known_from_file(KNOWN_FILE)
+    knownvars = (known or {}).get('variables')
+    variables = find.globals_from_binary(knownvars=knownvars,
+                                         dirnames=SOURCE_DIRS)
+
+    _generate_ignored_file(variables)
diff --git a/Tools/c-analyzer/c_parser/__init__.py b/Tools/c-analyzer/c_parser/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/Tools/c-analyzer/c_parser/declarations.py b/Tools/c-analyzer/c_parser/declarations.py
new file mode 100644
index 000000000000..19fa3ff4e66b
--- /dev/null
+++ b/Tools/c-analyzer/c_parser/declarations.py
@@ -0,0 +1,295 @@
+import re
+import shlex
+import subprocess
+
+from . import source
+
+
+IDENTIFIER = r'(?:[a-zA-z]|_+[a-zA-Z0-9]\w*)'
+
+TYPE_QUAL = r'(?:const|volatile)'
+
+VAR_TYPE_SPEC = r'''(?:
+        void |
+        (?:
+         (?:(?:un)?signed\s+)?
+         (?:
+          char |
+          short |
+          int |
+          long |
+          long\s+int |
+          long\s+long
+          ) |
+         ) |
+        float |
+        double |
+        {IDENTIFIER} |
+        (?:struct|union)\s+{IDENTIFIER}
+        )'''
+
+POINTER = rf'''(?:
+        (?:\s+const)?\s*[*]
+        )'''
+
+#STRUCT = r'''(?:
+#        (?:struct|(struct\s+%s))\s*[{]
+#            [^}]*
+#        [}]
+#        )''' % (IDENTIFIER)
+#UNION = r'''(?:
+#        (?:union|(union\s+%s))\s*[{]
+#            [^}]*
+#        [}]
+#        )''' % (IDENTIFIER)
+#DECL_SPEC = rf'''(?:
+#        ({VAR_TYPE_SPEC}) |
+#        ({STRUCT}) |
+#        ({UNION})
+#        )'''
+
+FUNC_START = rf'''(?:
+        (?:
+          (?:
+            extern |
+            static |
+            static\s+inline
+           )\s+
+         )?
+        #(?:const\s+)?
+        {VAR_TYPE_SPEC}
+        )'''
+#GLOBAL_VAR_START = rf'''(?:
+#        (?:
+#          (?:
+#            extern |
+#            static
+#           )\s+
+#         )?
+#        (?:
+#           {TYPE_QUAL}
+#           (?:\s+{TYPE_QUAL})?
+#         )?\s+
+#        {VAR_TYPE_SPEC}
+#        )'''
+GLOBAL_DECL_START_RE = re.compile(rf'''
+        ^
+        (?:
+            ({FUNC_START})
+         )
+        ''', re.VERBOSE)
+
+LOCAL_VAR_START = rf'''(?:
+        (?:
+          (?:
+            register |
+            static
+           )\s+
+         )?
+        (?:
+          (?:
+            {TYPE_QUAL}
+            (?:\s+{TYPE_QUAL})?
+           )\s+
+         )?
+        {VAR_TYPE_SPEC}
+        {POINTER}?
+        )'''
+LOCAL_STMT_START_RE = re.compile(rf'''
+        ^
+        (?:
+            ({LOCAL_VAR_START})
+         )
+        ''', re.VERBOSE)
+
+
+def iter_global_declarations(lines):
+    """Yield (decl, body) for each global declaration in the given lines.
+
+    For function definitions the header is reduced to one line and
+    the body is provided as-is.  For other compound declarations (e.g.
+    struct) the entire declaration is reduced to one line and "body"
+    is None.  Likewise for simple declarations (e.g. variables).
+
+    Declarations inside function bodies are ignored, though their text
+    is provided in the function body.
+    """
+    # XXX Bail out upon bogus syntax.
+    lines = source.iter_clean_lines(lines)
+    for line in lines:
+        if not GLOBAL_DECL_START_RE.match(line):
+            continue
+        # We only need functions here, since we only need locals for now.
+        if line.endswith(';'):
+            continue
+        if line.endswith('{') and '(' not in line:
+            continue
+
+        # Capture the function.
+        # (assume no func is a one-liner)
+        decl = line
+        while '{' not in line:  # assume no inline structs, etc.
+            try:
+                line = next(lines)
+            except StopIteration:
+                return
+            decl += ' ' + line
+
+        body, end = _extract_block(lines)
+        if end is None:
+            return
+        assert end == '}'
+        yield (f'{decl}\n{body}\n{end}', body)
+
+
+def iter_local_statements(lines):
+    """Yield (lines, blocks) for each statement in the given lines.
+
+    For simple statements, "blocks" is None and the statement is reduced
+    to a single line.  For compound statements, "blocks" is a pair of
+    (header, body) for each block in the statement.  The headers are
+    reduced to a single line each, but the bpdies are provided as-is.
+    """
+    # XXX Bail out upon bogus syntax.
+    lines = source.iter_clean_lines(lines)
+    for line in lines:
+        if not LOCAL_STMT_START_RE.match(line):
+            continue
+
+        stmt = line
+        blocks = None
+        if not line.endswith(';'):
+            # XXX Support compound & multiline simple statements.
+            #blocks = []
+            continue
+
+        yield (stmt, blocks)
+
+
+def _extract_block(lines):
+    end = None
+    depth = 1
+    body = []
+    for line in lines:
+        depth += line.count('{') - line.count('}')
+        if depth == 0:
+            end = line
+            break
+        body.append(line)
+    return '\n'.join(body), end
+
+
+def parse_func(stmt, body):
+    """Return (name, signature) for the given function definition."""
+    header, _, end = stmt.partition(body)
+    assert end.strip() == '}'
+    assert header.strip().endswith('{')
+    header, _, _= header.rpartition('{')
+
+    signature = ' '.join(header.strip().splitlines())
+
+    _, _, name = signature.split('(')[0].strip().rpartition(' ')
+    assert name
+
+    return name, signature
+
+
+def parse_var(stmt):
+    """Return (name, vartype) for the given variable declaration."""
+    stmt = stmt.rstrip(';')
+    m = LOCAL_STMT_START_RE.match(stmt)
+    assert m
+    vartype = m.group(0)
+    name = stmt[len(vartype):].partition('=')[0].strip()
+
+    if name.startswith('('):
+        name, _, after = name[1:].partition(')')
+        assert after
+        name = name.replace('*', '* ')
+        inside, _, name = name.strip().rpartition(' ')
+        vartype = f'{vartype} ({inside.strip()}){after}'
+    else:
+        name = name.replace('*', '* ')
+        before, _, name = name.rpartition(' ')
+        vartype = f'{vartype} {before}'
+
+    vartype = vartype.strip()
+    while '  ' in vartype:
+        vartype = vartype.replace('  ', ' ')
+
+    return name, vartype
+
+
+def parse_compound(stmt, blocks):
+    """Return (headers, bodies) for the given compound statement."""
+    # XXX Identify declarations inside compound statements
+    # (if/switch/for/while).
+    raise NotImplementedError
+
+
+def iter_variables(filename, *,
+                   _iter_source_lines=source.iter_lines,
+                   _iter_global=iter_global_declarations,
+                   _iter_local=iter_local_statements,
+                   _parse_func=parse_func,
+                   _parse_var=parse_var,
+                   _parse_compound=parse_compound,
+                   ):
+    """Yield (funcname, name, vartype) for every variable in the given file."""
+    lines = _iter_source_lines(filename)
+    for stmt, body in _iter_global(lines):
+        # At the file top-level we only have to worry about vars & funcs.
+        if not body:
+            name, vartype = _parse_var(stmt)
+            if name:
+                yield (None, name, vartype)
+        else:
+            funcname, _ = _parse_func(stmt, body)
+            localvars = _iter_locals(body,
+                                     _iter_statements=_iter_local,
+                                     _parse_var=_parse_var,
+                                     _parse_compound=_parse_compound,
+                                     )
+            for name, vartype in localvars:
+                yield (funcname, name, vartype)
+
+
+def _iter_locals(lines, *,
+                 _iter_statements=iter_local_statements,
+                 _parse_var=parse_var,
+                 _parse_compound=parse_compound,
+                 ):
+    compound = [lines]
+    while compound:
+        body = compound.pop(0)
+        bodylines = body.splitlines()
+        for stmt, blocks in _iter_statements(bodylines):
+            if not blocks:
+                name, vartype = _parse_var(stmt)
+                if name:
+                    yield (name, vartype)
+            else:
+                headers, bodies = _parse_compound(stmt, blocks)
+                for header in headers:
+                    for line in header:
+                        name, vartype = _parse_var(line)
+                        if name:
+                            yield (name, vartype)
+                compound.extend(bodies)
+
+
+def iter_all(dirnames):
+    """Yield a Declaration for each one found.
+
+    If there are duplicates, due to preprocessor conditionals, then
+    they are checked to make sure they are the same.
+    """
+    raise NotImplementedError
+
+
+def iter_preprocessed(dirnames):
+    """Yield a Declaration for each one found.
+
+    All source files are run through the preprocessor first.
+    """
+    raise NotImplementedError
diff --git a/Tools/c-analyzer/c_parser/info.py b/Tools/c-analyzer/c_parser/info.py
new file mode 100644
index 000000000000..9ab697978638
--- /dev/null
+++ b/Tools/c-analyzer/c_parser/info.py
@@ -0,0 +1,78 @@
+from collections import namedtuple
+
+from c_analyzer_common import info, util
+from c_analyzer_common.util import classonly, _NTBase
+
+
+def normalize_vartype(vartype):
+    """Return the canonical form for a variable type (or func signature)."""
+    # We allow empty strring through for semantic reasons.
+    if vartype is None:
+        return None
+
+    # XXX finish!
+    # XXX Return (modifiers, type, pointer)?
+    return str(vartype)
+
+
+class Variable(_NTBase,
+               namedtuple('Variable', 'id vartype')):
+    """Information about a single variable declaration."""
+
+    __slots__ = ()
+    _isglobal = util.Slot()
+
+    @classonly
+    def from_parts(cls, filename, funcname, name, vartype, isglobal=False):
+        id = info.ID(filename, funcname, name)
+        self = cls(id, vartype)
+        if isglobal:
+            self._isglobal = True
+        return self
+
+    def __new__(cls, id, vartype):
+        self = super().__new__(
+                cls,
+                id=info.ID.from_raw(id),
+                vartype=normalize_vartype(vartype) if vartype else None,
+                )
+        return self
+
+    def __hash__(self):
+        return hash(self.id)
+
+    def __getattr__(self, name):
+        return getattr(self.id, name)
+
+    def _validate_id(self):
+        if not self.id:
+            raise TypeError('missing id')
+
+        if not self.filename or self.filename == info.UNKNOWN:
+            raise TypeError(f'id missing filename ({self.id})')
+
+        if self.funcname and self.funcname == info.UNKNOWN:
+            raise TypeError(f'id missing funcname ({self.id})')
+
+        self.id.validate()
+
+    def validate(self):
+        """Fail if the object is invalid (i.e. init with bad data)."""
+        self._validate_id()
+
+        if self.vartype is None or self.vartype == info.UNKNOWN:
+            raise TypeError('missing vartype')
+
+    @property
+    def isglobal(self):
+        try:
+            return self._isglobal
+        except AttributeError:
+            # XXX Include extern variables.
+            # XXX Ignore functions.
+            self._isglobal = ('static' in self.vartype.split())
+            return self._isglobal
+
+    @property
+    def isconst(self):
+        return 'const' in self.vartype.split()
diff --git a/Tools/c-analyzer/c_parser/naive.py b/Tools/c-analyzer/c_parser/naive.py
new file mode 100644
index 000000000000..e0370cc3d1d4
--- /dev/null
+++ b/Tools/c-analyzer/c_parser/naive.py
@@ -0,0 +1,180 @@
+import re
+
+from c_analyzer_common.info import UNKNOWN
+
+from .info import Variable
+from .preprocessor import _iter_clean_lines
+
+
+_NOT_SET = object()
+
+
+def get_srclines(filename, *,
+                 cache=None,
+                 _open=open,
+                 _iter_lines=_iter_clean_lines,
+                 ):
+    """Return the file's lines as a list.
+
+    Each line will have trailing whitespace removed (including newline).
+
+    If a cache is given the it is used.
+    """
+    if cache is not None:
+        try:
+            return cache[filename]
+        except KeyError:
+            pass
+
+    with _open(filename) as srcfile:
+        srclines = [line
+                    for _, line in _iter_lines(srcfile)
+                    if not line.startswith('#')]
+    for i, line in enumerate(srclines):
+        srclines[i] = line.rstrip()
+
+    if cache is not None:
+        cache[filename] = srclines
+    return srclines
+
+
+def parse_variable_declaration(srcline):
+    """Return (name, decl) for the given declaration line."""
+    # XXX possible false negatives...
+    decl, sep, _ = srcline.partition('=')
+    if not sep:
+        if not srcline.endswith(';'):
+            return None, None
+        decl = decl.strip(';')
+    decl = decl.strip()
+    m = re.match(r'.*\b(\w+)\s*(?:\[[^\]]*\])?$', decl)
+    if not m:
+        return None, None
+    name = m.group(1)
+    return name, decl
+
+
+def parse_variable(srcline, funcname=None):
+    """Return a Variable for the variable declared on the line (or None)."""
+    line = srcline.strip()
+
+    # XXX Handle more than just static variables.
+    if line.startswith('static '):
+        if '(' in line and '[' not in line:
+            # a function
+            return None, None
+        return parse_variable_declaration(line)
+    else:
+        return None, None
+
+
+def iter_variables(filename, *,
+                   srccache=None,
+                   parse_variable=None,
+                   _get_srclines=get_srclines,
+                   _default_parse_variable=parse_variable,
+                   ):
+    """Yield a Variable for each in the given source file."""
+    if parse_variable is None:
+        parse_variable = _default_parse_variable
+
+    indent = ''
+    prev = ''
+    funcname = None
+    for line in _get_srclines(filename, cache=srccache):
+        # remember current funcname
+        if funcname:
+            if line == indent + '}':
+                funcname = None
+                continue
+        else:
+            if '(' in prev and line == indent + '{':
+                if not prev.startswith('__attribute__'):
+                    funcname = prev.split('(')[0].split()[-1]
+                    prev = ''
+                    continue
+            indent = line[:-len(line.lstrip())]
+            prev = line
+
+        info = parse_variable(line, funcname)
+        if isinstance(info, list):
+            for name, _funcname, decl in info:
+                yield Variable.from_parts(filename, _funcname, name, decl)
+            continue
+        name, decl = info
+
+        if name is None:
+            continue
+        yield Variable.from_parts(filename, funcname, name, decl)
+
+
+def _match_varid(variable, name, funcname, ignored=None):
+    if ignored and variable in ignored:
+        return False
+
+    if variable.name != name:
+        return False
+
+    if funcname == UNKNOWN:
+        if not variable.funcname:
+            return False
+    elif variable.funcname != funcname:
+        return False
+
+    return True
+
+
+def find_variable(filename, funcname, name, *,
+                  ignored=None,
+                  srccache=None,  # {filename: lines}
+                  parse_variable=None,
+                  _iter_variables=iter_variables,
+                  ):
+    """Return the matching variable.
+
+    Return None if the variable is not found.
+    """
+    for variable in _iter_variables(filename,
+                                    srccache=srccache,
+                                    parse_variable=parse_variable,
+                                    ):
+        if _match_varid(variable, name, funcname, ignored):
+            return variable
+    else:
+        return None
+
+
+def find_variables(varids, filenames=None, *,
+                   srccache=_NOT_SET,
+                   parse_variable=None,
+                   _find_symbol=find_variable,
+                   ):
+    """Yield a Variable for each ID.
+
+    If the variable is not found then its decl will be UNKNOWN.  That
+    way there will be one resulting Variable per given ID.
+    """
+    if srccache is _NOT_SET:
+        srccache = {}
+
+    used = set()
+    for varid in varids:
+        if varid.filename and varid.filename != UNKNOWN:
+            srcfiles = [varid.filename]
+        else:
+            if not filenames:
+                yield Variable(varid, UNKNOWN)
+                continue
+            srcfiles = filenames
+        for filename in srcfiles:
+            found = _find_varid(filename, varid.funcname, varid.name,
+                                 ignored=used,
+                                 srccache=srccache,
+                                 parse_variable=parse_variable,
+                                 )
+            if found:
+                yield found
+                used.add(found)
+                break
+        else:
+            yield Variable(varid, UNKNOWN)
diff --git a/Tools/c-analyzer/c_parser/preprocessor.py b/Tools/c-analyzer/c_parser/preprocessor.py
new file mode 100644
index 000000000000..0e2866e4873e
--- /dev/null
+++ b/Tools/c-analyzer/c_parser/preprocessor.py
@@ -0,0 +1,512 @@
+from collections import namedtuple
+import shlex
+import os
+import re
+
+from c_analyzer_common import util
+from . import info
+
+
+CONTINUATION = '\\' + os.linesep
+
+IDENTIFIER = r'(?:\w*[a-zA-Z]\w*)'
+IDENTIFIER_RE = re.compile('^' + IDENTIFIER + '$')
+
+
+def _coerce_str(value):
+    if not value:
+        return ''
+    return str(value).strip()
+
+
+#############################
+# directives
+
+DIRECTIVE_START = r'''
+    (?:
+      ^ \s*
+      [#] \s*
+      )'''
+DIRECTIVE_TEXT = r'''
+    (?:
+      (?: \s+ ( .*\S ) )?
+      \s* $
+      )'''
+DIRECTIVE = rf'''
+    (?:
+      {DIRECTIVE_START}
+      (
+        include |
+        error | warning |
+        pragma |
+        define | undef |
+        if | ifdef | ifndef | elseif | else | endif |
+        __FILE__ | __LINE__ | __DATE __ | __TIME__ | __TIMESTAMP__
+        )
+      {DIRECTIVE_TEXT}
+      )'''
+#       (?:
+#        [^\\\n] |
+#        \\ [^\n] |
+#        \\ \n
+#        )+
+#      ) \n
+#     )'''
+DIRECTIVE_RE = re.compile(DIRECTIVE, re.VERBOSE)
+
+DEFINE = rf'''
+    (?:
+      {DIRECTIVE_START} define \s+
+      (?:
+        ( \w*[a-zA-Z]\w* )
+        (?: \s* [(] ([^)]*) [)] )?
+        )
+      {DIRECTIVE_TEXT}
+      )'''
+DEFINE_RE = re.compile(DEFINE, re.VERBOSE)
+
+
+def parse_directive(line):
+    """Return the appropriate directive for the given line."""
+    line = line.strip()
+    if line.startswith('#'):
+        line = line[1:].lstrip()
+        line = '#' + line
+    directive = line
+    #directive = '#' + line
+    while '  ' in directive:
+        directive = directive.replace('  ', ' ')
+    return _parse_directive(directive)
+
+
+def _parse_directive(line):
+    m = DEFINE_RE.match(line)
+    if m:
+        name, args, text = m.groups()
+        if args:
+            args = [a.strip() for a in args.split(',')]
+            return Macro(name, args, text)
+        else:
+            return Constant(name, text)
+
+    m = DIRECTIVE_RE.match(line)
+    if not m:
+        raise ValueError(f'unsupported directive {line!r}')
+    kind, text = m.groups()
+    if not text:
+        if kind not in ('else', 'endif'):
+            raise ValueError(f'missing text in directive {line!r}')
+    elif kind in ('else', 'endif', 'define'):
+        raise ValueError(f'unexpected text in directive {line!r}')
+    if kind == 'include':
+        directive = Include(text)
+    elif kind in IfDirective.KINDS:
+        directive = IfDirective(kind, text)
+    else:
+        directive = OtherDirective(kind, text)
+    directive.validate()
+    return directive
+
+
+class PreprocessorDirective(util._NTBase):
+    """The base class for directives."""
+
+    __slots__ = ()
+
+    KINDS = frozenset([
+            'include',
+            'pragma',
+            'error', 'warning',
+            'define', 'undef',
+            'if', 'ifdef', 'ifndef', 'elseif', 'else', 'endif',
+            '__FILE__', '__DATE__', '__LINE__', '__TIME__', '__TIMESTAMP__',
+            ])
+
+    @property
+    def text(self):
+        return ' '.join(v for v in self[1:] if v and v.strip()) or None
+
+    def validate(self):
+        """Fail if the object is invalid (i.e. init with bad data)."""
+        super().validate()
+
+        if not self.kind:
+            raise TypeError('missing kind')
+        elif self.kind not in self.KINDS:
+            raise ValueError
+
+        # text can be anything, including None.
+
+
+class Constant(PreprocessorDirective,
+               namedtuple('Constant', 'kind name value')):
+    """A single "constant" directive ("define")."""
+
+    __slots__ = ()
+
+    def __new__(cls, name, value=None):
+        self = super().__new__(
+                cls,
+                'define',
+                name=_coerce_str(name) or None,
+                value=_coerce_str(value) or None,
+                )
+        return self
+
+    def validate(self):
+        """Fail if the object is invalid (i.e. init with bad data)."""
+        super().validate()
+
+        if not self.name:
+            raise TypeError('missing name')
+        elif not IDENTIFIER_RE.match(self.name):
+            raise ValueError(f'name must be identifier, got {self.name!r}')
+
+        # value can be anything, including None
+
+
+class Macro(PreprocessorDirective,
+            namedtuple('Macro', 'kind name args body')):
+    """A single "macro" directive ("define")."""
+
+    __slots__ = ()
+
+    def __new__(cls, name, args, body=None):
+        # "args" must be a string or an iterable of strings (or "empty").
+        if isinstance(args, str):
+            args = [v.strip() for v in args.split(',')]
+        if args:
+            args = tuple(_coerce_str(a) or None for a in args)
+        self = super().__new__(
+                cls,
+                kind='define',
+                name=_coerce_str(name) or None,
+                args=args if args else (),
+                body=_coerce_str(body) or None,
+                )
+        return self
+
+    @property
+    def text(self):
+        if self.body:
+            return f'{self.name}({", ".join(self.args)}) {self.body}'
+        else:
+            return f'{self.name}({", ".join(self.args)})'
+
+    def validate(self):
+        """Fail if the object is invalid (i.e. init with bad data)."""
+        super().validate()
+
+        if not self.name:
+            raise TypeError('missing name')
+        elif not IDENTIFIER_RE.match(self.name):
+            raise ValueError(f'name must be identifier, got {self.name!r}')
+
+        for arg in self.args:
+            if not arg:
+                raise ValueError(f'missing arg in {self.args}')
+            elif not IDENTIFIER_RE.match(arg):
+                raise ValueError(f'arg must be identifier, got {arg!r}')
+
+        # body can be anything, including None
+
+
+class IfDirective(PreprocessorDirective,
+                  namedtuple('IfDirective', 'kind condition')):
+    """A single conditional directive (e.g. "if", "ifdef").
+
+    This only includes directives that actually provide conditions.  The
+    related directives "else" and "endif" are covered by OtherDirective
+    instead.
+    """
+
+    __slots__ = ()
+
+    KINDS = frozenset([
+            'if',
+            'ifdef',
+            'ifndef',
+            'elseif',
+            ])
+
+    @classmethod
+    def _condition_from_raw(cls, raw, kind):
+        #return Condition.from_raw(raw, _kind=kind)
+        condition = _coerce_str(raw)
+        if not condition:
+            return None
+
+        if kind == 'ifdef':
+            condition = f'defined({condition})'
+        elif kind == 'ifndef':
+            condition = f'! defined({condition})'
+
+        return condition
+
+    def __new__(cls, kind, condition):
+        kind = _coerce_str(kind)
+        self = super().__new__(
+                cls,
+                kind=kind or None,
+                condition=cls._condition_from_raw(condition, kind),
+                )
+        return self
+
+    @property
+    def text(self):
+        if self.kind == 'ifdef':
+            return self.condition[8:-1]  # strip "defined("
+        elif self.kind == 'ifndef':
+            return self.condition[10:-1]  # strip "! defined("
+        else:
+            return self.condition
+        #return str(self.condition)
+
+    def validate(self):
+        """Fail if the object is invalid (i.e. init with bad data)."""
+        super().validate()
+
+        if not self.condition:
+            raise TypeError('missing condition')
+        #else:
+        #    for cond in self.condition:
+        #        if not cond:
+        #            raise ValueError(f'missing condition in {self.condition}')
+        #        cond.validate()
+        #    if self.kind in ('ifdef', 'ifndef'):
+        #        if len(self.condition) != 1:
+        #            raise ValueError('too many condition')
+        #        if self.kind == 'ifdef':
+        #            if not self.condition[0].startswith('defined '):
+        #                raise ValueError('bad condition')
+        #        else:
+        #            if not self.condition[0].startswith('! defined '):
+        #                raise ValueError('bad condition')
+
+
+class Include(PreprocessorDirective,
+              namedtuple('Include', 'kind file')):
+    """A single "include" directive.
+
+    Supported "file" values are either follow the bracket style
+    (<stdio>) or double quotes ("spam.h").
+    """
+
+    __slots__ = ()
+
+    def __new__(cls, file):
+        self = super().__new__(
+                cls,
+                kind='include',
+                file=_coerce_str(file) or None,
+                )
+        return self
+
+    def validate(self):
+        """Fail if the object is invalid (i.e. init with bad data)."""
+        super().validate()
+
+        if not self.file:
+            raise TypeError('missing file')
+
+
+class OtherDirective(PreprocessorDirective,
+                     namedtuple('OtherDirective', 'kind text')):
+    """A single directive not covered by another class.
+
+    This includes the "else", "endif", and "undef" directives, which are
+    otherwise inherently related to the directives covered by the
+    Constant, Macro, and IfCondition classes.
+
+    Note that all directives must have a text value, except for "else"
+    and "endif" (which must have no text).
+    """
+
+    __slots__ = ()
+
+    KINDS = PreprocessorDirective.KINDS - {'include', 'define'} - IfDirective.KINDS
+
+    def __new__(cls, kind, text):
+        self = super().__new__(
+                cls,
+                kind=_coerce_str(kind) or None,
+                text=_coerce_str(text) or None,
+                )
+        return self
+
+    def validate(self):
+        """Fail if the object is invalid (i.e. init with bad data)."""
+        super().validate()
+
+        if self.text:
+            if self.kind in ('else', 'endif'):
+                raise ValueError('unexpected text in directive')
+        elif self.kind not in ('else', 'endif'):
+            raise TypeError('missing text')
+
+
+#############################
+# iterating lines
+
+def _recompute_conditions(directive, ifstack):
+    if directive.kind in ('if', 'ifdef', 'ifndef'):
+        ifstack.append(
+                ([], directive.condition))
+    elif directive.kind == 'elseif':
+        if ifstack:
+            negated, active = ifstack.pop()
+            if active:
+                negated.append(active)
+        else:
+            negated = []
+        ifstack.append(
+                (negated, directive.condition))
+    elif directive.kind == 'else':
+        if ifstack:
+            negated, active = ifstack.pop()
+            if active:
+                negated.append(active)
+            ifstack.append(
+                    (negated, None))
+    elif directive.kind == 'endif':
+        if ifstack:
+            ifstack.pop()
+
+    conditions = []
+    for negated, active in ifstack:
+        for condition in negated:
+            conditions.append(f'! ({condition})')
+        if active:
+            conditions.append(active)
+    return tuple(conditions)
+
+
+def _iter_clean_lines(lines):
+    lines = iter(enumerate(lines, 1))
+    for lno, line in lines:
+        # Handle line continuations.
+        while line.endswith(CONTINUATION):
+            try:
+                lno, _line = next(lines)
+            except StopIteration:
+                break
+            line = line[:-len(CONTINUATION)] + ' ' + _line
+
+        # Deal with comments.
+        after = line
+        line = ''
+        while True:
+            # Look for a comment.
+            before, begin, remainder = after.partition('/*')
+            if '//' in before:
+                before, _, _ = before.partition('//')
+                line += before + ' '  # per the C99 spec
+                break
+            line += before
+            if not begin:
+                break
+            line += ' '  # per the C99 spec
+
+            # Go until we find the end of the comment.
+            _, end, after = remainder.partition('*/')
+            while not end:
+                try:
+                    lno, remainder = next(lines)
+                except StopIteration:
+                    raise Exception('unterminated comment')
+                _, end, after = remainder.partition('*/')
+
+        yield lno, line
+
+
+def iter_lines(lines, *,
+                   _iter_clean_lines=_iter_clean_lines,
+                   _parse_directive=_parse_directive,
+                   _recompute_conditions=_recompute_conditions,
+                   ):
+    """Yield (lno, line, directive, active conditions) for each given line.
+
+    This is effectively a subset of the operations taking place in
+    translation phases 2-4 from the C99 spec (ISO/IEC 9899:TC2); see
+    section 5.1.1.2.  Line continuations are removed and comments
+    replaced with a single space.  (In both cases "lno" will be the last
+    line involved.)  Otherwise each line is returned as-is.
+
+    "lno" is the (1-indexed) line number for the line.
+
+    "directive" will be a PreprocessorDirective or None, depending on
+    whether or not there is a directive on the line.
+
+    "active conditions" is the set of preprocessor conditions (e.g.
+    "defined()") under which the current line of code will be included
+    in compilation.  That set is derived from every conditional
+    directive block (e.g. "if defined()", "ifdef", "else") containing
+    that line.  That includes nested directives.  Note that the
+    current line does not affect the active conditions for iteself.
+    It only impacts subsequent lines.  That applies to directives
+    that close blocks (e.g. "endif") just as much as conditional
+    directvies.  Also note that "else" and "elseif" directives
+    update the active conditions (for later lines), rather than
+    adding to them.
+    """
+    ifstack = []
+    conditions = ()
+    for lno, line in _iter_clean_lines(lines):
+        stripped = line.strip()
+        if not stripped.startswith('#'):
+            yield lno, line, None, conditions
+            continue
+
+        directive = '#' + stripped[1:].lstrip()
+        while '  ' in directive:
+            directive = directive.replace('  ', ' ')
+        directive = _parse_directive(directive)
+        yield lno, line, directive, conditions
+
+        if directive.kind in ('else', 'endif'):
+            conditions = _recompute_conditions(directive, ifstack)
+        elif isinstance(directive, IfDirective):
+            conditions = _recompute_conditions(directive, ifstack)
+
+
+#############################
+# running (platform-specific?)
+
+def _gcc(filename, *,
+         _get_argv=(lambda: _get_gcc_argv()),
+         _run=util.run_cmd,
+         ):
+    argv = _get_argv()
+    argv.extend([
+            '-E', filename,
+            ])
+    output = _run(argv)
+    return output
+
+
+def _get_gcc_argv(*,
+                  _open=open,
+                  _run=util.run_cmd,
+                  ):
+    with _open('/tmp/print.mk', 'w') as tmpfile:
+        tmpfile.write('print-%:\n')
+        #tmpfile.write('\t at echo $* = $($*)\n')
+        tmpfile.write('\t at echo $($*)\n')
+    argv = ['/usr/bin/make',
+            '-f', 'Makefile',
+            '-f', '/tmp/print.mk',
+            'print-CC',
+            'print-PY_CORE_CFLAGS',
+            ]
+    output = _run(argv)
+    gcc, cflags = output.strip().splitlines()
+    argv = shlex.split(gcc.strip())
+    cflags = shlex.split(cflags.strip())
+    return argv + cflags
+
+
+def run(filename, *,
+        _gcc=_gcc,
+        ):
+    """Return the text of the given file after running the preprocessor."""
+    return _gcc(filename)
diff --git a/Tools/c-analyzer/c_parser/source.py b/Tools/c-analyzer/c_parser/source.py
new file mode 100644
index 000000000000..f8998c8a338b
--- /dev/null
+++ b/Tools/c-analyzer/c_parser/source.py
@@ -0,0 +1,34 @@
+from . import preprocessor
+
+
+def iter_clean_lines(lines):
+    incomment = False
+    for line in lines:
+        # Deal with comments.
+        if incomment:
+            _, sep, line = line.partition('*/')
+            if sep:
+                incomment = False
+            continue
+        line, _, _ = line.partition('//')
+        line, sep, remainder = line.partition('/*')
+        if sep:
+            _, sep, after = remainder.partition('*/')
+            if not sep:
+                incomment = True
+                continue
+            line += ' ' + after
+
+        # Ignore blank lines and leading/trailing whitespace.
+        line = line.strip()
+        if not line:
+            continue
+
+        yield line
+
+
+def iter_lines(filename, *,
+               preprocess=preprocessor.run,
+               ):
+    content = preprocess(filename)
+    return iter(content.splitlines())
diff --git a/Tools/c-analyzer/c_symbols/__init__.py b/Tools/c-analyzer/c_symbols/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/Tools/c-analyzer/c_symbols/binary.py b/Tools/c-analyzer/c_symbols/binary.py
new file mode 100644
index 000000000000..e125dbd5b5ed
--- /dev/null
+++ b/Tools/c-analyzer/c_symbols/binary.py
@@ -0,0 +1,157 @@
+import os
+import os.path
+import shutil
+import sys
+
+from c_analyzer_common import util, info
+from . import source
+from .info import Symbol
+
+
+#PYTHON = os.path.join(REPO_ROOT, 'python')
+PYTHON = sys.executable
+
+
+def iter_symbols(binary=PYTHON, dirnames=None, *,
+                 # Alternately, use look_up_known_symbol()
+                 # from c_globals.supported.
+                 find_local_symbol=source.find_symbol,
+                 _file_exists=os.path.exists,
+                 _iter_symbols_nm=(lambda b, *a: _iter_symbols_nm(b, *a)),
+                 ):
+    """Yield a Symbol for each symbol found in the binary."""
+    if not _file_exists(binary):
+        raise Exception('executable missing (need to build it first?)')
+
+    if find_local_symbol:
+        cache = {}
+        def find_local_symbol(name, *, _find=find_local_symbol):
+            return _find(name, dirnames, _perfilecache=cache)
+    else:
+        find_local_symbol = None
+
+    if os.name == 'nt':
+        # XXX Support this.
+        raise NotImplementedError
+    else:
+        yield from _iter_symbols_nm(binary, find_local_symbol)
+
+
+#############################
+# binary format (e.g. ELF)
+
+SPECIAL_SYMBOLS = {
+        '__bss_start',
+        '__data_start',
+        '__dso_handle',
+        '_DYNAMIC',
+        '_edata',
+        '_end',
+        '__environ@@GLIBC_2.2.5',
+        '_GLOBAL_OFFSET_TABLE_',
+        '__JCR_END__',
+        '__JCR_LIST__',
+        '__TMC_END__',
+        }
+
+
+def _is_special_symbol(name):
+    if name in SPECIAL_SYMBOLS:
+        return True
+    if '@@GLIBC' in name:
+        return True
+    return False
+
+
+#############################
+# "nm"
+
+NM_KINDS = {
+        'b': Symbol.KIND.VARIABLE,  # uninitialized
+        'd': Symbol.KIND.VARIABLE,  # initialized
+        #'g': Symbol.KIND.VARIABLE,  # uninitialized
+        #'s': Symbol.KIND.VARIABLE,  # initialized
+        't': Symbol.KIND.FUNCTION,
+        }
+
+
+def _iter_symbols_nm(binary, find_local_symbol=None,
+                     *,
+                     _which=shutil.which,
+                     _run=util.run_cmd,
+                     ):
+    nm = _which('nm')
+    if not nm:
+        raise NotImplementedError
+    argv = [nm,
+            '--line-numbers',
+            binary,
+            ]
+    try:
+        output = _run(argv)
+    except Exception:
+        if nm is None:
+            # XXX Use dumpbin.exe /SYMBOLS on Windows.
+            raise NotImplementedError
+        raise
+    for line in output.splitlines():
+        (name, kind, external, filename, funcname, vartype,
+         ) = _parse_nm_line(line,
+                            _find_local_symbol=find_local_symbol,
+                            )
+        if kind != Symbol.KIND.VARIABLE:
+            continue
+        elif _is_special_symbol(name):
+            continue
+        assert vartype is None
+        yield Symbol(
+                id=(filename, funcname, name),
+                kind=kind,
+                external=external,
+                )
+
+
+def _parse_nm_line(line, *, _find_local_symbol=None):
+    _origline = line
+    _, _, line = line.partition(' ')  # strip off the address
+    line = line.strip()
+
+    kind, _, line = line.partition(' ')
+    line = line.strip()
+    external = kind.isupper()
+    kind = NM_KINDS.get(kind.lower(), Symbol.KIND.OTHER)
+
+    name, _, filename = line.partition('\t')
+    name = name.strip()
+    if filename:
+        filename = os.path.relpath(filename.partition(':')[0])
+    else:
+        filename = info.UNKNOWN
+
+    vartype = None
+    name, islocal = _parse_nm_name(name, kind)
+    if islocal:
+        funcname = info.UNKNOWN
+        if _find_local_symbol is not None:
+            filename, funcname, vartype = _find_local_symbol(name)
+            filename = filename or info.UNKNOWN
+            funcname = funcname or info.UNKNOWN
+    else:
+        funcname = None
+        # XXX fine filename and vartype?
+    return name, kind, external, filename, funcname, vartype
+
+
+def _parse_nm_name(name, kind):
+    if kind != Symbol.KIND.VARIABLE:
+        return name, None
+    if _is_special_symbol(name):
+        return name, None
+
+    actual, sep, digits = name.partition('.')
+    if not sep:
+        return name, False
+
+    if not digits.isdigit():
+        raise Exception(f'got bogus name {name}')
+    return actual, True
diff --git a/Tools/c-analyzer/c_symbols/info.py b/Tools/c-analyzer/c_symbols/info.py
new file mode 100644
index 000000000000..f6ed52c8f071
--- /dev/null
+++ b/Tools/c-analyzer/c_symbols/info.py
@@ -0,0 +1,51 @@
+from collections import namedtuple
+
+from c_analyzer_common.info import ID
+from c_analyzer_common.util import classonly, _NTBase
+
+
+class Symbol(_NTBase, namedtuple('Symbol', 'id kind external')):
+    """Info for a single compilation symbol."""
+
+    __slots__ = ()
+
+    class KIND:
+        VARIABLE = 'variable'
+        FUNCTION = 'function'
+        OTHER = 'other'
+
+    @classonly
+    def from_name(cls, name, filename=None, kind=KIND.VARIABLE, external=None):
+        """Return a new symbol based on the given name."""
+        id = ID(filename, None, name)
+        return cls(id, kind, external)
+
+    def __new__(cls, id, kind=KIND.VARIABLE, external=None):
+        self = super().__new__(
+                cls,
+                id=ID.from_raw(id),
+                kind=str(kind) if kind else None,
+                external=bool(external) if external is not None else None,
+                )
+        return self
+
+    def __hash__(self):
+        return hash(self.id)
+
+    def __getattr__(self, name):
+        return getattr(self.id, name)
+
+    def validate(self):
+        """Fail if the object is invalid (i.e. init with bad data)."""
+        if not self.id:
+            raise TypeError('missing id')
+        else:
+            self.id.validate()
+
+        if not self.kind:
+            raise TypeError('missing kind')
+        elif self.kind not in vars(self.KIND).values():
+            raise ValueError(f'unsupported kind {self.kind}')
+
+        if self.external is None:
+            raise TypeError('missing external')
diff --git a/Tools/c-analyzer/c_symbols/resolve.py b/Tools/c-analyzer/c_symbols/resolve.py
new file mode 100644
index 000000000000..dc876ae0b759
--- /dev/null
+++ b/Tools/c-analyzer/c_symbols/resolve.py
@@ -0,0 +1,149 @@
+import os.path
+
+from c_analyzer_common import files
+from c_analyzer_common.info import UNKNOWN
+from c_parser import declarations, info
+from .info import Symbol
+from .source import _find_symbol
+
+
+# XXX need tests:
+# * look_up_known_symbol()
+# * symbol_from_source()
+# * get_resolver()
+# * symbols_to_variables()
+
+def look_up_known_symbol(symbol, knownvars, *,
+                         match_files=(lambda f1, f2: f1 == f2),
+                         ):
+    """Return the known variable matching the given symbol.
+
+    "knownvars" is a mapping of common.ID to parser.Variable.
+
+    "match_files" is used to verify if two filenames point to
+    the same file.
+    """
+    if not knownvars:
+        return None
+
+    if symbol.funcname == UNKNOWN:
+        if not symbol.filename or symbol.filename == UNKNOWN:
+            for varid in knownvars:
+                if not varid.funcname:
+                    continue
+                if varid.name == symbol.name:
+                    return knownvars[varid]
+            else:
+                return None
+        else:
+            for varid in knownvars:
+                if not varid.funcname:
+                    continue
+                if not match_files(varid.filename, symbol.filename):
+                    continue
+                if varid.name == symbol.name:
+                    return knownvars[varid]
+            else:
+                return None
+    elif not symbol.filename or symbol.filename == UNKNOWN:
+        raise NotImplementedError
+    else:
+        return knownvars.get(symbol.id)
+
+
+def find_in_source(symbol, dirnames, *,
+                   _perfilecache={},
+                   _find_symbol=_find_symbol,
+                   _iter_files=files.iter_files_by_suffix,
+                   ):
+    """Return the Variable matching the given Symbol.
+
+    If there is no match then return None.
+    """
+    if symbol.filename and symbol.filename != UNKNOWN:
+        filenames = [symbol.filename]
+    else:
+        filenames = _iter_files(dirnames, ('.c', '.h'))
+
+    if symbol.funcname and symbol.funcname != UNKNOWN:
+        raise NotImplementedError
+
+    (filename, funcname, vartype
+     ) = _find_symbol(symbol.name, filenames, _perfilecache)
+    if filename == UNKNOWN:
+        return None
+    return info.Variable(
+            id=(filename, funcname, symbol.name),
+            vartype=vartype,
+            )
+
+
+def get_resolver(knownvars=None, dirnames=None, *,
+                 _look_up_known=look_up_known_symbol,
+                 _from_source=find_in_source,
+                 ):
+    """Return a "resolver" func for the given known vars and dirnames.
+
+    The func takes a single Symbol and returns a corresponding Variable.
+    If the symbol was located then the variable will be valid, populated
+    with the corresponding information.  Otherwise None is returned.
+    """
+    if knownvars:
+        knownvars = dict(knownvars)  # a copy
+        def resolve_known(symbol):
+            found = _look_up_known(symbol, knownvars)
+            if found is None:
+                return None
+            elif symbol.funcname == UNKNOWN:
+                knownvars.pop(found.id)
+            elif not symbol.filename or symbol.filename == UNKNOWN:
+                knownvars.pop(found.id)
+            return found
+        if dirnames:
+            def resolve(symbol):
+                found = resolve_known(symbol)
+                if found is None:
+                    return None
+                    #return _from_source(symbol, dirnames)
+                else:
+                    for dirname in dirnames:
+                        if not dirname.endswith(os.path.sep):
+                            dirname += os.path.sep
+                        if found.filename.startswith(dirname):
+                            break
+                    else:
+                        return None
+                    return found
+        else:
+            resolve = resolve_known
+    elif dirnames:
+        def resolve(symbol):
+            return _from_source(symbol, dirnames)
+    else:
+        def resolve(symbol):
+            return None
+    return resolve
+
+
+def symbols_to_variables(symbols, *,
+                         resolve=(lambda s: look_up_known_symbol(s, None)),
+                         ):
+    """Yield the variable the matches each given symbol.
+
+    Use get_resolver() for a "resolve" func to use.
+    """
+    for symbol in symbols:
+        if isinstance(symbol, info.Variable):
+            # XXX validate?
+            yield symbol
+            continue
+        if symbol.kind != Symbol.KIND.VARIABLE:
+            continue
+        resolved = resolve(symbol)
+        if resolved is None:
+            #raise NotImplementedError(symbol)
+            resolved = info.Variable(
+                    id=symbol.id,
+                    vartype=UNKNOWN,
+                    )
+        yield resolved
diff --git a/Tools/c-analyzer/c_symbols/source.py b/Tools/c-analyzer/c_symbols/source.py
new file mode 100644
index 000000000000..a7248104c94c
--- /dev/null
+++ b/Tools/c-analyzer/c_symbols/source.py
@@ -0,0 +1,58 @@
+from c_analyzer_common import files
+from c_analyzer_common.info import UNKNOWN
+from c_parser import declarations
+
+
+# XXX need tests:
+# * find_symbol()
+
+def find_symbol(name, dirnames, *,
+                _perfilecache,
+                _iter_files=files.iter_files_by_suffix,
+                **kwargs
+                ):
+    """Return (filename, funcname, vartype) for the matching Symbol."""
+    filenames = _iter_files(dirnames, ('.c', '.h'))
+    return _find_symbol(name, filenames, _perfilecache, **kwargs)
+
+
+def _get_symbols(filename, *,
+                 _iter_variables=declarations.iter_variables,
+                 ):
+    """Return the list of Symbols found in the given file."""
+    symbols = {}
+    for funcname, name, vartype in _iter_variables(filename):
+        if not funcname:
+            continue
+        try:
+            instances = symbols[name]
+        except KeyError:
+            instances = symbols[name] = []
+        instances.append((funcname, vartype))
+    return symbols
+
+
+def _find_symbol(name, filenames, _perfilecache, *,
+                _get_local_symbols=_get_symbols,
+                ):
+    for filename in filenames:
+        try:
+            symbols = _perfilecache[filename]
+        except KeyError:
+            symbols = _perfilecache[filename] = _get_local_symbols(filename)
+
+        try:
+            instances = symbols[name]
+        except KeyError:
+            continue
+
+        funcname, vartype = instances.pop(0)
+        if not instances:
+            symbols.pop(name)
+        return filename, funcname, vartype
+    else:
+        return UNKNOWN, UNKNOWN, UNKNOWN
+
+
+def iter_symbols():
+    raise NotImplementedError
diff --git a/Tools/c-globals/check-c-globals.py b/Tools/c-analyzer/check-c-globals.py
similarity index 100%
rename from Tools/c-globals/check-c-globals.py
rename to Tools/c-analyzer/check-c-globals.py
diff --git a/Tools/c-globals/ignored-globals.txt b/Tools/c-analyzer/ignored-globals.txt
similarity index 100%
rename from Tools/c-globals/ignored-globals.txt
rename to Tools/c-analyzer/ignored-globals.txt
diff --git a/Tools/c-analyzer/ignored.tsv b/Tools/c-analyzer/ignored.tsv
new file mode 100644
index 000000000000..a0e0e503da6a
--- /dev/null
+++ b/Tools/c-analyzer/ignored.tsv
@@ -0,0 +1 @@
+filename	funcname	name	kind	reason
diff --git a/Tools/c-analyzer/known.tsv b/Tools/c-analyzer/known.tsv
new file mode 100644
index 000000000000..ce2afcb9591e
--- /dev/null
+++ b/Tools/c-analyzer/known.tsv
@@ -0,0 +1,1922 @@
+filename	funcname	name	kind	declaration
+Modules/_abc.c	-	_abc_data_type	variable	static PyTypeObject _abc_data_type
+Modules/_abc.c	-	abc_invalidation_counter	variable	static unsigned long long abc_invalidation_counter
+Modules/_abc.c	-	_abcmodule	variable	static struct PyModuleDef _abcmodule
+Python/import.c	import_find_and_load	accumulated	variable	static _PyTime_t accumulated
+Modules/itertoolsmodule.c	-	accumulate_methods	variable	static PyMethodDef accumulate_methods
+Modules/itertoolsmodule.c	-	accumulate_type	variable	static PyTypeObject accumulate_type
+Python/Python-ast.c	-	Add_singleton	variable	static PyObject *Add_singleton
+Python/Python-ast.c	-	Add_type	variable	static PyTypeObject *Add_type
+Objects/genobject.c	-	ag_asend_freelist	variable	static PyAsyncGenASend *ag_asend_freelist[_PyAsyncGen_MAXFREELIST]
+Objects/genobject.c	-	ag_asend_freelist_free	variable	static int ag_asend_freelist_free
+Objects/genobject.c	-	ag_value_freelist	variable	static _PyAsyncGenWrappedValue *ag_value_freelist[_PyAsyncGen_MAXFREELIST]
+Objects/genobject.c	-	ag_value_freelist_free	variable	static int ag_value_freelist_free
+Python/Python-ast.c	-	alias_fields	variable	static const char *alias_fields[]
+Python/Python-ast.c	-	alias_type	variable	static PyTypeObject *alias_type
+Modules/_tracemalloc.c	-	allocators	variable	static struct { PyMemAllocatorEx mem; PyMemAllocatorEx raw; PyMemAllocatorEx obj; } allocators
+Python/Python-ast.c	-	And_singleton	variable	static PyObject *And_singleton
+Python/Python-ast.c	-	And_type	variable	static PyTypeObject *And_type
+Python/Python-ast.c	-	AnnAssign_fields	variable	static const char *AnnAssign_fields[]
+Python/Python-ast.c	-	AnnAssign_type	variable	static PyTypeObject *AnnAssign_type
+Python/compile.c	-	__annotations__	variable	static PyObject *__annotations__
+Objects/obmalloc.c	-	arenas	variable	static struct arena_object* arenas
+Python/Python-ast.c	-	arg_attributes	variable	static const char *arg_attributes[]
+Python/Python-ast.c	-	arg_fields	variable	static const char *arg_fields[]
+Python/Python-ast.c	-	arg_type	variable	static PyTypeObject *arg_type
+Python/Python-ast.c	-	arguments_fields	variable	static const char *arguments_fields[]
+Python/Python-ast.c	-	arguments_type	variable	static PyTypeObject *arguments_type
+Python/Python-ast.c	-	Assert_fields	variable	static const char *Assert_fields[]
+Python/compile.c	compiler_assert	assertion_error	variable	static PyObject *assertion_error
+Python/Python-ast.c	-	Assert_type	variable	static PyTypeObject *Assert_type
+Python/Python-ast.c	-	Assign_fields	variable	static const char *Assign_fields[]
+Python/Python-ast.c	-	Assign_type	variable	static PyTypeObject *Assign_type
+Python/Python-ast.c	-	_astmodule	variable	static struct PyModuleDef _astmodule
+Python/Python-ast.c	-	AST_type	variable	static PyTypeObject AST_type
+Python/Python-ast.c	-	ast_type_getsets	variable	static PyGetSetDef ast_type_getsets[]
+Python/Python-ast.c	-	ast_type_methods	variable	static PyMethodDef ast_type_methods
+Python/Python-ast.c	-	AsyncFor_fields	variable	static const char *AsyncFor_fields[]
+Python/Python-ast.c	-	AsyncFor_type	variable	static PyTypeObject *AsyncFor_type
+Python/Python-ast.c	-	AsyncFunctionDef_fields	variable	static const char *AsyncFunctionDef_fields[]
+Python/Python-ast.c	-	AsyncFunctionDef_type	variable	static PyTypeObject *AsyncFunctionDef_type
+Objects/genobject.c	-	async_gen_as_async	variable	static PyAsyncMethods async_gen_as_async
+Objects/genobject.c	-	async_gen_asend_as_async	variable	static PyAsyncMethods async_gen_asend_as_async
+Objects/genobject.c	-	async_gen_asend_methods	variable	static PyMethodDef async_gen_asend_methods
+Objects/genobject.c	-	async_gen_athrow_as_async	variable	static PyAsyncMethods async_gen_athrow_as_async
+Objects/genobject.c	-	async_gen_athrow_methods	variable	static PyMethodDef async_gen_athrow_methods
+Objects/genobject.c	-	async_gen_getsetlist	variable	static PyGetSetDef async_gen_getsetlist[]
+Python/sysmodule.c	-	asyncgen_hooks_desc	variable	static PyStructSequence_Desc asyncgen_hooks_desc
+Python/sysmodule.c	-	asyncgen_hooks_fields	variable	static PyStructSequence_Field asyncgen_hooks_fields[]
+Python/sysmodule.c	-	AsyncGenHooksType	variable	static PyTypeObject AsyncGenHooksType
+Objects/genobject.c	-	async_gen_memberlist	variable	static PyMemberDef async_gen_memberlist[]
+Objects/genobject.c	-	async_gen_methods	variable	static PyMethodDef async_gen_methods
+Python/Python-ast.c	-	AsyncWith_fields	variable	static const char *AsyncWith_fields[]
+Python/Python-ast.c	-	AsyncWith_type	variable	static PyTypeObject *AsyncWith_type
+Parser/listnode.c	-	atbol	variable	static int atbol
+Modules/atexitmodule.c	-	atexit_methods	variable	static PyMethodDef atexit_methods
+Modules/atexitmodule.c	-	atexitmodule	variable	static struct PyModuleDef atexitmodule
+Modules/atexitmodule.c	-	atexit_slots	variable	static PyModuleDef_Slot atexit_slots[]
+Modules/_operator.c	-	attrgetter_methods	variable	static PyMethodDef attrgetter_methods
+Modules/_operator.c	-	attrgetter_type	variable	static PyTypeObject attrgetter_type
+Python/Python-ast.c	-	Attribute_fields	variable	static const char *Attribute_fields[]
+Python/Python-ast.c	-	Attribute_type	variable	static PyTypeObject *Attribute_type
+Python/Python-ast.c	-	AugAssign_fields	variable	static const char *AugAssign_fields[]
+Python/Python-ast.c	-	AugAssign_type	variable	static PyTypeObject *AugAssign_type
+Python/Python-ast.c	-	AugLoad_singleton	variable	static PyObject *AugLoad_singleton
+Python/Python-ast.c	-	AugLoad_type	variable	static PyTypeObject *AugLoad_type
+Python/Python-ast.c	-	AugStore_singleton	variable	static PyObject *AugStore_singleton
+Python/Python-ast.c	-	AugStore_type	variable	static PyTypeObject *AugStore_type
+Python/Python-ast.c	-	Await_fields	variable	static const char *Await_fields[]
+Python/Python-ast.c	-	Await_type	variable	static PyTypeObject *Await_type
+Objects/exceptions.c	-	BaseException_getset	variable	static PyGetSetDef BaseException_getset[]
+Objects/exceptions.c	-	BaseException_members	variable	static struct PyMemberDef BaseException_members[]
+Objects/exceptions.c	-	BaseException_methods	variable	static PyMethodDef BaseException_methods
+Modules/posixmodule.c	-	billion	variable	static PyObject *billion
+Python/Python-ast.c	-	BinOp_fields	variable	static const char *BinOp_fields[]
+Python/Python-ast.c	-	BinOp_type	variable	static PyTypeObject *BinOp_type
+Python/Python-ast.c	-	BitAnd_singleton	variable	static PyObject *BitAnd_singleton
+Python/Python-ast.c	-	BitAnd_type	variable	static PyTypeObject *BitAnd_type
+Python/Python-ast.c	-	BitOr_singleton	variable	static PyObject *BitOr_singleton
+Python/Python-ast.c	-	BitOr_type	variable	static PyTypeObject *BitOr_type
+Python/Python-ast.c	-	BitXor_singleton	variable	static PyObject *BitXor_singleton
+Python/Python-ast.c	-	BitXor_type	variable	static PyTypeObject *BitXor_type
+Objects/unicodeobject.c	-	bloom_linebreak	variable	static BLOOM_MASK bloom_linebreak
+Objects/boolobject.c	-	bool_as_number	variable	static PyNumberMethods bool_as_number
+Python/Python-ast.c	-	BoolOp_fields	variable	static const char *BoolOp_fields[]
+Python/Python-ast.c	-	boolop_type	variable	static PyTypeObject *boolop_type
+Python/Python-ast.c	-	BoolOp_type	variable	static PyTypeObject *BoolOp_type
+Python/_warnings.c	is_internal_frame	bootstrap_string	variable	static PyObject *bootstrap_string
+Python/Python-ast.c	-	Break_type	variable	static PyTypeObject *Break_type
+Modules/_io/bufferedio.c	-	bufferediobase_methods	variable	static PyMethodDef bufferediobase_methods
+Modules/_io/bufferedio.c	-	bufferedrandom_getset	variable	static PyGetSetDef bufferedrandom_getset[]
+Modules/_io/bufferedio.c	-	bufferedrandom_members	variable	static PyMemberDef bufferedrandom_members[]
+Modules/_io/bufferedio.c	-	bufferedrandom_methods	variable	static PyMethodDef bufferedrandom_methods
+Modules/_io/bufferedio.c	-	bufferedreader_getset	variable	static PyGetSetDef bufferedreader_getset[]
+Modules/_io/bufferedio.c	-	bufferedreader_members	variable	static PyMemberDef bufferedreader_members[]
+Modules/_io/bufferedio.c	-	bufferedreader_methods	variable	static PyMethodDef bufferedreader_methods
+Modules/_io/bufferedio.c	-	bufferedrwpair_getset	variable	static PyGetSetDef bufferedrwpair_getset[]
+Modules/_io/bufferedio.c	-	bufferedrwpair_methods	variable	static PyMethodDef bufferedrwpair_methods
+Modules/_io/bufferedio.c	-	bufferedwriter_getset	variable	static PyGetSetDef bufferedwriter_getset[]
+Modules/_io/bufferedio.c	-	bufferedwriter_members	variable	static PyMemberDef bufferedwriter_members[]
+Modules/_io/bufferedio.c	-	bufferedwriter_methods	variable	static PyMethodDef bufferedwriter_methods
+Modules/getbuildinfo.c	Py_GetBuildInfo	buildinfo	variable	static char buildinfo[50 + sizeof(GITVERSION) + ((sizeof(GITTAG) > sizeof(GITBRANCH)) ?  sizeof(GITTAG) : sizeof(GITBRANCH))]
+Python/bltinmodule.c	-	builtin_methods	variable	static PyMethodDef builtin_methods
+Python/bltinmodule.c	-	builtinsmodule	variable	static struct PyModuleDef builtinsmodule
+Python/import.c	PyImport_Import	builtins_str	variable	static PyObject *builtins_str
+Python/ceval.c	make_pending_calls	busy	variable	static int busy
+Objects/bytearrayobject.c	-	bytearray_as_buffer	variable	static PyBufferProcs bytearray_as_buffer
+Objects/bytearrayobject.c	-	bytearray_as_mapping	variable	static PyMappingMethods bytearray_as_mapping
+Objects/bytearrayobject.c	-	bytearray_as_number	variable	static PyNumberMethods bytearray_as_number
+Objects/bytearrayobject.c	-	bytearray_as_sequence	variable	static PySequenceMethods bytearray_as_sequence
+Objects/bytearrayobject.c	-	bytearrayiter_methods	variable	static PyMethodDef bytearrayiter_methods
+Objects/bytearrayobject.c	-	bytearray_methods	variable	static PyMethodDef bytearray_methods
+Objects/bytesobject.c	-	bytes_as_buffer	variable	static PyBufferProcs bytes_as_buffer
+Objects/bytesobject.c	-	bytes_as_mapping	variable	static PyMappingMethods bytes_as_mapping
+Objects/bytesobject.c	-	bytes_as_number	variable	static PyNumberMethods bytes_as_number
+Objects/bytesobject.c	-	bytes_as_sequence	variable	static PySequenceMethods bytes_as_sequence
+Modules/_io/bytesio.c	-	bytesiobuf_as_buffer	variable	static PyBufferProcs bytesiobuf_as_buffer
+Modules/_io/bytesio.c	-	bytesio_getsetlist	variable	static PyGetSetDef bytesio_getsetlist[]
+Modules/_io/bytesio.c	-	bytesio_methods	variable	static PyMethodDef bytesio_methods
+Objects/bytesobject.c	-	bytes_methods	variable	static PyMethodDef bytes_methods
+Python/thread_pthread.h	init_condattr	ca	variable	static pthread_condattr_t ca
+Python/Python-ast.c	-	Call_fields	variable	static const char *Call_fields[]
+Objects/iterobject.c	-	calliter_methods	variable	static PyMethodDef calliter_methods
+Python/Python-ast.c	-	Call_type	variable	static PyTypeObject *Call_type
+Objects/cellobject.c	-	cell_getsetlist	variable	static PyGetSetDef cell_getsetlist[]
+Modules/itertoolsmodule.c	-	chain_methods	variable	static PyMethodDef chain_methods
+Modules/itertoolsmodule.c	-	chain_type	variable	static PyTypeObject chain_type
+Objects/bytesobject.c	-	characters	variable	static PyBytesObject *characters[UCHAR_MAX + 1]
+Python/symtable.c	-	__class__	variable	static identifier __class__
+Python/Python-ast.c	-	ClassDef_fields	variable	static const char *ClassDef_fields[]
+Python/Python-ast.c	-	ClassDef_type	variable	static PyTypeObject *ClassDef_type
+Objects/funcobject.c	-	cm_getsetlist	variable	static PyGetSetDef cm_getsetlist[]
+Objects/funcobject.c	-	cm_memberlist	variable	static PyMemberDef cm_memberlist[]
+Python/Python-ast.c	-	cmpop_type	variable	static PyTypeObject *cmpop_type
+Modules/_codecsmodule.c	-	_codecs_functions	variable	static PyMethodDef _codecs_functions[]
+Modules/_codecsmodule.c	-	codecsmodule	variable	static struct PyModuleDef codecsmodule
+Objects/codeobject.c	-	code_memberlist	variable	static PyMemberDef code_memberlist[]
+Objects/codeobject.c	-	code_methods	variable	static PyMethodDef code_methods
+Modules/_collectionsmodule.c	-	_collectionsmodule	variable	static struct PyModuleDef _collectionsmodule
+Modules/itertoolsmodule.c	-	combinations_methods	variable	static PyMethodDef combinations_methods
+Modules/itertoolsmodule.c	-	combinations_type	variable	static PyTypeObject combinations_type
+Objects/typeobject.c	object_new	comma_id	variable	_Py_static_string(comma_id, "", "")
+Python/Python-ast.c	-	Compare_fields	variable	static const char *Compare_fields[]
+Python/Python-ast.c	-	Compare_type	variable	static PyTypeObject *Compare_type
+Objects/complexobject.c	-	complex_as_number	variable	static PyNumberMethods complex_as_number
+Objects/complexobject.c	-	complex_members	variable	static PyMemberDef complex_members[]
+Objects/complexobject.c	-	complex_methods	variable	static PyMethodDef complex_methods
+Python/Python-ast.c	-	comprehension_fields	variable	static const char *comprehension_fields[]
+Python/Python-ast.c	-	comprehension_type	variable	static PyTypeObject *comprehension_type
+Modules/itertoolsmodule.c	-	compress_methods	variable	static PyMethodDef compress_methods
+Modules/itertoolsmodule.c	-	compress_type	variable	static PyTypeObject compress_type
+Python/thread_pthread.h	-	condattr_monotonic	variable	static pthread_condattr_t *condattr_monotonic
+Python/Python-ast.c	-	Constant_fields	variable	static const char *Constant_fields[]
+Python/Python-ast.c	-	Constant_type	variable	static PyTypeObject *Constant_type
+Python/Python-ast.c	-	Continue_type	variable	static PyTypeObject *Continue_type
+Objects/longobject.c	PyLong_FromString	convmultmax_base	variable	static twodigits convmultmax_base[37]
+Objects/longobject.c	PyLong_FromString	convwidth_base	variable	static int convwidth_base[37]
+Objects/genobject.c	-	coro_as_async	variable	static PyAsyncMethods coro_as_async
+Objects/genobject.c	-	coro_getsetlist	variable	static PyGetSetDef coro_getsetlist[]
+Objects/genobject.c	-	coro_memberlist	variable	static PyMemberDef coro_memberlist[]
+Objects/genobject.c	-	coro_methods	variable	static PyMethodDef coro_methods
+Objects/genobject.c	-	coro_wrapper_methods	variable	static PyMethodDef coro_wrapper_methods
+Modules/itertoolsmodule.c	-	count_methods	variable	static PyMethodDef count_methods
+Modules/itertoolsmodule.c	-	count_type	variable	static PyTypeObject count_type
+Python/context.c	-	ctx_freelist	variable	static PyContext *ctx_freelist
+Python/context.c	-	ctx_freelist_len	variable	static int ctx_freelist_len
+Modules/itertoolsmodule.c	-	cwr_methods	variable	static PyMethodDef cwr_methods
+Modules/itertoolsmodule.c	-	cwr_type	variable	static PyTypeObject cwr_type
+Modules/itertoolsmodule.c	-	cycle_methods	variable	static PyMethodDef cycle_methods
+Modules/itertoolsmodule.c	-	cycle_type	variable	static PyTypeObject cycle_type
+Objects/obmalloc.c	new_arena	debug_stats	variable	static int debug_stats
+Modules/signalmodule.c	-	DefaultHandler	variable	static PyObject *DefaultHandler
+Modules/_collectionsmodule.c	-	defdict_members	variable	static PyMemberDef defdict_members[]
+Modules/_collectionsmodule.c	-	defdict_methods	variable	static PyMethodDef defdict_methods
+Modules/_collectionsmodule.c	-	defdict_type	variable	static PyTypeObject defdict_type
+Python/Python-ast.c	-	Delete_fields	variable	static const char *Delete_fields[]
+Python/Python-ast.c	-	Delete_type	variable	static PyTypeObject *Delete_type
+Python/Python-ast.c	-	Del_singleton	variable	static PyObject *Del_singleton
+Python/Python-ast.c	-	Del_type	variable	static PyTypeObject *Del_type
+Modules/_collectionsmodule.c	-	deque_as_number	variable	static PyNumberMethods deque_as_number
+Modules/_collectionsmodule.c	-	deque_as_sequence	variable	static PySequenceMethods deque_as_sequence
+Modules/_collectionsmodule.c	-	deque_getset	variable	static PyGetSetDef deque_getset[]
+Modules/_collectionsmodule.c	-	dequeiter_methods	variable	static PyMethodDef dequeiter_methods
+Modules/_collectionsmodule.c	-	dequeiter_type	variable	static PyTypeObject dequeiter_type
+Modules/_collectionsmodule.c	-	deque_methods	variable	static PyMethodDef deque_methods
+Modules/_collectionsmodule.c	-	dequereviter_type	variable	static PyTypeObject dequereviter_type
+Modules/_collectionsmodule.c	-	deque_type	variable	static PyTypeObject deque_type
+Objects/descrobject.c	-	descr_members	variable	static PyMemberDef descr_members[]
+Objects/descrobject.c	-	descr_methods	variable	static PyMethodDef descr_methods
+Modules/_abc.c	-	_destroy_def	variable	static PyMethodDef _destroy_def
+Objects/floatobject.c	-	detected_double_format	variable	static float_format_type detected_double_format
+Objects/floatobject.c	-	detected_float_format	variable	static float_format_type detected_float_format
+Objects/dictobject.c	-	dict_as_mapping	variable	static PyMappingMethods dict_as_mapping
+Objects/dictobject.c	-	dict_as_sequence	variable	static PySequenceMethods dict_as_sequence
+Python/symtable.c	-	dictcomp	variable	static identifier dictcomp
+Python/Python-ast.c	-	DictComp_fields	variable	static const char *DictComp_fields[]
+Python/Python-ast.c	-	DictComp_type	variable	static PyTypeObject *DictComp_type
+Python/Python-ast.c	-	Dict_fields	variable	static const char *Dict_fields[]
+Objects/dictobject.c	-	dictitems_as_sequence	variable	static PySequenceMethods dictitems_as_sequence
+Objects/dictobject.c	-	dictitems_methods	variable	static PyMethodDef dictitems_methods
+Objects/dictobject.c	-	dictiter_methods	variable	static PyMethodDef dictiter_methods
+Objects/dictobject.c	-	dictkeys_as_sequence	variable	static PySequenceMethods dictkeys_as_sequence
+Objects/dictobject.c	-	dictkeys_methods	variable	static PyMethodDef dictkeys_methods
+Python/Python-ast.c	-	Dict_type	variable	static PyTypeObject *Dict_type
+Objects/dictobject.c	-	dictvalues_as_sequence	variable	static PySequenceMethods dictvalues_as_sequence
+Objects/dictobject.c	-	dictvalues_methods	variable	static PyMethodDef dictvalues_methods
+Objects/dictobject.c	-	dictviews_as_number	variable	static PyNumberMethods dictviews_as_number
+Modules/posixmodule.c	-	DirEntry_members	variable	static PyMemberDef DirEntry_members[]
+Modules/posixmodule.c	-	DirEntry_methods	variable	static PyMethodDef DirEntry_methods
+Modules/posixmodule.c	-	DirEntryType	variable	static PyTypeObject DirEntryType
+Python/Python-ast.c	-	Div_singleton	variable	static PyObject *Div_singleton
+Python/Python-ast.c	-	Div_type	variable	static PyTypeObject *Div_type
+Python/compile.c	-	__doc__	variable	static PyObject *__doc__
+Objects/classobject.c	method_get_doc	docstr	variable	static PyObject *docstr
+Objects/classobject.c	instancemethod_get_doc	docstr	variable	static PyObject *docstr
+Python/compile.c	compiler_set_qualname	dot	variable	_Py_static_string(dot, ""."")
+Python/compile.c	compiler_set_qualname	dot_locals	variable	_Py_static_string(dot_locals, "".<locals>"")
+Objects/floatobject.c	-	double_format	variable	static float_format_type double_format
+Modules/itertoolsmodule.c	-	dropwhile_methods	variable	static PyMethodDef dropwhile_methods
+Modules/itertoolsmodule.c	-	dropwhile_type	variable	static PyTypeObject dropwhile_type
+Objects/setobject.c	-	_dummy_struct	variable	static PyObject _dummy_struct
+Modules/posixmodule.c	os_dup2_impl	dup3_works	variable	static int dup3_works
+Modules/_io/bufferedio.c	_PyIO_trap_eintr	eintr_int	variable	static PyObject *eintr_int
+Objects/sliceobject.c	-	ellipsis_methods	variable	static PyMethodDef ellipsis_methods
+Python/hamt.c	-	_empty_bitmap_node	variable	static PyHamtNode_Bitmap *_empty_bitmap_node
+Objects/setobject.c	-	emptyfrozenset	variable	static PyObject *emptyfrozenset
+Python/hamt.c	-	_empty_hamt	variable	static PyHamtObject *_empty_hamt
+Objects/dictobject.c	-	empty_keys_struct	variable	static PyDictKeysObject empty_keys_struct
+Objects/codeobject.c	PyCode_NewEmpty	emptystring	variable	static PyObject *emptystring
+Python/compile.c	compiler_from_import	empty_string	variable	static PyObject *empty_string
+Objects/dictobject.c	-	empty_values	variable	static PyObject *empty_values[1]
+Objects/unicodeobject.c	-	encoding_map_methods	variable	static PyMethodDef encoding_map_methods
+Objects/unicodeobject.c	-	EncodingMapType	variable	static PyTypeObject EncodingMapType
+Objects/enumobject.c	-	enum_methods	variable	static PyMethodDef enum_methods
+Python/Python-ast.c	-	Eq_singleton	variable	static PyObject *Eq_singleton
+Python/Python-ast.c	-	Eq_type	variable	static PyTypeObject *Eq_type
+Objects/exceptions.c	-	errnomap	variable	static PyObject *errnomap
+Modules/errnomodule.c	-	errno_methods	variable	static PyMethodDef errno_methods
+Modules/errnomodule.c	-	errnomodule	variable	static struct PyModuleDef errnomodule
+Modules/_localemodule.c	-	Error	variable	static PyObject *Error
+Python/Python-ast.c	-	excepthandler_attributes	variable	static const char *excepthandler_attributes[]
+Python/Python-ast.c	-	ExceptHandler_fields	variable	static const char *ExceptHandler_fields[]
+Python/Python-ast.c	-	excepthandler_type	variable	static PyTypeObject *excepthandler_type
+Python/Python-ast.c	-	ExceptHandler_type	variable	static PyTypeObject *ExceptHandler_type
+Modules/_threadmodule.c	-	ExceptHookArgs_desc	variable	static PyStructSequence_Desc ExceptHookArgs_desc
+Modules/_threadmodule.c	-	ExceptHookArgs_fields	variable	static PyStructSequence_Field ExceptHookArgs_fields[]
+Modules/_threadmodule.c	-	ExceptHookArgsType	variable	static PyTypeObject ExceptHookArgsType
+Objects/exceptions.c	_check_for_legacy_statements	exec_prefix	variable	static PyObject *exec_prefix
+Python/Python-ast.c	-	expr_attributes	variable	static const char *expr_attributes[]
+Python/Python-ast.c	-	expr_context_type	variable	static PyTypeObject *expr_context_type
+Python/Python-ast.c	-	Expression_fields	variable	static const char *Expression_fields[]
+Python/Python-ast.c	-	Expression_type	variable	static PyTypeObject *Expression_type
+Python/Python-ast.c	-	Expr_fields	variable	static const char *Expr_fields[]
+Python/Python-ast.c	-	expr_type	variable	static PyTypeObject *expr_type
+Python/Python-ast.c	-	Expr_type	variable	static PyTypeObject *Expr_type
+Python/import.c	-	extensions	variable	static PyObject *extensions
+Python/Python-ast.c	-	ExtSlice_fields	variable	static const char *ExtSlice_fields[]
+Python/Python-ast.c	-	ExtSlice_type	variable	static PyTypeObject *ExtSlice_type
+Objects/boolobject.c	-	false_str	variable	static PyObject *false_str
+Modules/faulthandler.c	-	fatal_error	variable	static struct { int enabled; PyObject *file; int fd; int all_threads; PyInterpreterState *interp; void *exc_handler; } fatal_error
+Modules/faulthandler.c	-	faulthandler_handlers	variable	static fault_handler_t faulthandler_handlers[]
+Objects/stringlib/unicode_format.h	-	fieldnameiter_methods	variable	static PyMethodDef fieldnameiter_methods
+Modules/_io/fileio.c	-	fileio_getsetlist	variable	static PyGetSetDef fileio_getsetlist[]
+Modules/_io/fileio.c	-	fileio_members	variable	static PyMemberDef fileio_members[]
+Modules/_io/fileio.c	-	fileio_methods	variable	static PyMethodDef fileio_methods
+Modules/itertoolsmodule.c	-	filterfalse_methods	variable	static PyMethodDef filterfalse_methods
+Modules/itertoolsmodule.c	-	filterfalse_type	variable	static PyTypeObject filterfalse_type
+Python/bltinmodule.c	-	filter_methods	variable	static PyMethodDef filter_methods
+Python/sysmodule.c	-	flags_desc	variable	static PyStructSequence_Desc flags_desc
+Python/sysmodule.c	-	flags_fields	variable	static PyStructSequence_Field flags_fields[]
+Python/sysmodule.c	-	FlagsType	variable	static PyTypeObject FlagsType
+Objects/floatobject.c	-	float_as_number	variable	static PyNumberMethods float_as_number
+Objects/floatobject.c	-	float_format	variable	static float_format_type 
+Objects/floatobject.c	-	float_getset	variable	static PyGetSetDef float_getset[]
+Objects/floatobject.c	-	floatinfo_desc	variable	static PyStructSequence_Desc floatinfo_desc
+Objects/floatobject.c	-	floatinfo_fields	variable	static PyStructSequence_Field floatinfo_fields[]
+Objects/floatobject.c	-	FloatInfoType	variable	static PyTypeObject FloatInfoType
+Objects/floatobject.c	-	float_methods	variable	static PyMethodDef float_methods
+Python/Python-ast.c	-	FloorDiv_singleton	variable	static PyObject *FloorDiv_singleton
+Python/Python-ast.c	-	FloorDiv_type	variable	static PyTypeObject *FloorDiv_type
+Python/fileutils.c	-	force_ascii	variable	static int force_ascii
+Python/Python-ast.c	-	For_fields	variable	static const char *For_fields[]
+Python/Python-ast.c	-	FormattedValue_fields	variable	static const char *FormattedValue_fields[]
+Python/Python-ast.c	-	FormattedValue_type	variable	static PyTypeObject *FormattedValue_type
+Objects/stringlib/unicode_format.h	-	formatteriter_methods	variable	static PyMethodDef formatteriter_methods
+Python/Python-ast.c	-	For_type	variable	static PyTypeObject *For_type
+Objects/frameobject.c	-	frame_getsetlist	variable	static PyGetSetDef frame_getsetlist[]
+Objects/frameobject.c	-	frame_memberlist	variable	static PyMemberDef frame_memberlist[]
+Objects/frameobject.c	-	frame_methods	variable	static PyMethodDef frame_methods
+Modules/_collectionsmodule.c	-	freeblocks	variable	static block *freeblocks[MAXFREEBLOCKS]
+Python/dtoa.c	-	freelist	variable	static Bigint *freelist[Kmax+1]
+Objects/floatobject.c	-	free_list	variable	static PyFloatObject *free_list
+Objects/frameobject.c	-	free_list	variable	static PyFrameObject *free_list
+Objects/listobject.c	-	free_list	variable	static PyListObject *free_list[PyList_MAXFREELIST]
+Objects/dictobject.c	-	free_list	variable	static PyDictObject *free_list[PyDict_MAXFREELIST]
+Objects/methodobject.c	-	free_list	variable	static PyCFunctionObject *free_list
+Objects/tupleobject.c	-	free_list	variable	static PyTupleObject *free_list[PyTuple_MAXSAVESIZE]
+Objects/classobject.c	-	free_list	variable	static PyMethodObject *free_list
+Objects/setobject.c	-	frozenset_as_number	variable	static PyNumberMethods frozenset_as_number
+Objects/setobject.c	-	frozenset_methods	variable	static PyMethodDef frozenset_methods
+Objects/funcobject.c	-	func_getsetlist	variable	static PyGetSetDef func_getsetlist[]
+Objects/funcobject.c	-	func_memberlist	variable	static PyMemberDef func_memberlist[]
+Python/Python-ast.c	-	FunctionDef_fields	variable	static const char *FunctionDef_fields[]
+Python/Python-ast.c	-	FunctionDef_type	variable	static PyTypeObject *FunctionDef_type
+Modules/_sre.c	-	_functions	variable	static PyMethodDef _functions[]
+Python/Python-ast.c	-	FunctionType_fields	variable	static const char *FunctionType_fields[]
+Python/Python-ast.c	-	FunctionType_type	variable	static PyTypeObject *FunctionType_type
+Modules/_functoolsmodule.c	-	_functoolsmodule	variable	static struct PyModuleDef _functoolsmodule
+Modules/gcmodule.c	-	GcMethods	variable	static PyMethodDef GcMethods[]
+Modules/gcmodule.c	-	gcmodule	variable	static struct PyModuleDef gcmodule
+Modules/gcmodule.c	-	gc_str	variable	static PyObject *gc_str
+Python/Python-ast.c	-	GeneratorExp_fields	variable	static const char *GeneratorExp_fields[]
+Python/Python-ast.c	-	GeneratorExp_type	variable	static PyTypeObject *GeneratorExp_type
+Python/symtable.c	-	genexpr	variable	static identifier genexpr
+Objects/genobject.c	-	gen_getsetlist	variable	static PyGetSetDef gen_getsetlist[]
+Objects/genobject.c	-	gen_memberlist	variable	static PyMemberDef gen_memberlist[]
+Objects/genobject.c	-	gen_methods	variable	static PyMethodDef gen_methods
+Python/bootstrap_hash.c	py_getrandom	getrandom_works	variable	static int getrandom_works
+Objects/descrobject.c	-	getset_getset	variable	static PyGetSetDef getset_getset[]
+Python/Python-ast.c	-	Global_fields	variable	static const char *Global_fields[]
+Python/Python-ast.c	-	Global_type	variable	static PyTypeObject *Global_type
+Modules/itertoolsmodule.c	-	groupby_methods	variable	static PyMethodDef groupby_methods
+Modules/itertoolsmodule.c	-	groupby_type	variable	static PyTypeObject groupby_type
+Modules/itertoolsmodule.c	-	_grouper_methods	variable	static PyMethodDef _grouper_methods
+Modules/itertoolsmodule.c	-	_grouper_type	variable	static PyTypeObject _grouper_type
+Python/Python-ast.c	-	GtE_singleton	variable	static PyObject *GtE_singleton
+Python/Python-ast.c	-	GtE_type	variable	static PyTypeObject *GtE_type
+Python/Python-ast.c	-	Gt_singleton	variable	static PyObject *Gt_singleton
+Python/Python-ast.c	-	Gt_type	variable	static PyTypeObject *Gt_type
+Modules/signalmodule.c	-	Handlers	variable	static volatile struct { _Py_atomic_int tripped; PyObject *func; } Handlers[NSIG]
+Python/dynload_shlib.c	-	handles	variable	static struct { dev_t dev; ino_t ino; void *handle; } handles[128]
+Python/sysmodule.c	-	hash_info_desc	variable	static PyStructSequence_Desc hash_info_desc
+Python/sysmodule.c	-	hash_info_fields	variable	static PyStructSequence_Field hash_info_fields[]
+Python/sysmodule.c	-	Hash_InfoType	variable	static PyTypeObject Hash_InfoType
+Python/import.c	import_find_and_load	header	variable	static int header
+Python/Python-ast.c	-	IfExp_fields	variable	static const char *IfExp_fields[]
+Python/Python-ast.c	-	IfExp_type	variable	static PyTypeObject *IfExp_type
+Python/Python-ast.c	-	If_fields	variable	static const char *If_fields[]
+Python/Python-ast.c	-	If_type	variable	static PyTypeObject *If_type
+Modules/signalmodule.c	-	IgnoreHandler	variable	static PyObject *IgnoreHandler
+Python/import.c	-	imp_methods	variable	static PyMethodDef imp_methods
+Python/import.c	-	impmodule	variable	static struct PyModuleDef impmodule
+Objects/exceptions.c	-	ImportError_members	variable	static PyMemberDef ImportError_members[]
+Objects/exceptions.c	-	ImportError_methods	variable	static PyMethodDef ImportError_methods
+Python/Python-ast.c	-	Import_fields	variable	static const char *Import_fields[]
+Python/Python-ast.c	-	ImportFrom_fields	variable	static const char *ImportFrom_fields[]
+Python/Python-ast.c	-	ImportFrom_type	variable	static PyTypeObject *ImportFrom_type
+Python/import.c	import_find_and_load	import_level	variable	static int import_level
+Python/_warnings.c	is_internal_frame	importlib_string	variable	static PyObject *importlib_string
+Python/import.c	-	import_lock	variable	static PyThread_type_lock import_lock
+Python/import.c	-	import_lock_level	variable	static int import_lock_level
+Python/import.c	-	import_lock_thread	variable	static unsigned long import_lock_thread
+Python/import.c	PyImport_Import	import_str	variable	static PyObject *import_str
+Python/Python-ast.c	-	Import_type	variable	static PyTypeObject *Import_type
+Modules/_io/textio.c	-	incrementalnewlinedecoder_getset	variable	static PyGetSetDef incrementalnewlinedecoder_getset[]
+Modules/_io/textio.c	-	incrementalnewlinedecoder_methods	variable	static PyMethodDef incrementalnewlinedecoder_methods
+Objects/listobject.c	-	indexerr	variable	static PyObject *indexerr
+Python/Python-ast.c	-	Index_fields	variable	static const char *Index_fields[]
+Python/Python-ast.c	-	Index_type	variable	static PyTypeObject *Index_type
+Python/thread.c	-	initialized	variable	static int initialized
+Modules/posixmodule.c	-	initialized	variable	static int initialized
+Modules/pwdmodule.c	-	initialized	variable	static int initialized
+Modules/signalmodule.c	-	initialized	variable	static int initialized
+Modules/timemodule.c	-	initialized	variable	static int initialized
+Python/Python-ast.c	init_types	initialized	variable	static int initialized
+Objects/listobject.c	PyList_New	initialized	variable	static int initialized
+Python/import.c	-	inittab_copy	variable	static struct _inittab *inittab_copy
+Python/Python-ast.c	-	In_singleton	variable	static PyObject *In_singleton
+Objects/classobject.c	-	instancemethod_getset	variable	static PyGetSetDef instancemethod_getset[]
+Objects/classobject.c	-	instancemethod_memberlist	variable	static PyMemberDef instancemethod_memberlist[]
+Python/Python-ast.c	-	Interactive_fields	variable	static const char *Interactive_fields[]
+Python/Python-ast.c	-	Interactive_type	variable	static PyTypeObject *Interactive_type
+Objects/unicodeobject.c	-	interned	variable	static PyObject *interned
+Objects/interpreteridobject.c	-	interpid_as_number	variable	static PyNumberMethods interpid_as_number
+Modules/signalmodule.c	-	IntHandler	variable	static PyObject *IntHandler
+Objects/longobject.c	-	int_info_desc	variable	static PyStructSequence_Desc int_info_desc
+Objects/longobject.c	-	int_info_fields	variable	static PyStructSequence_Field int_info_fields[]
+Objects/longobject.c	-	Int_InfoType	variable	static PyTypeObject Int_InfoType
+Python/Python-ast.c	-	In_type	variable	static PyTypeObject *In_type
+Python/Python-ast.c	-	Invert_singleton	variable	static PyObject *Invert_singleton
+Python/Python-ast.c	-	Invert_type	variable	static PyTypeObject *Invert_type
+Modules/_io/iobase.c	-	iobase_getset	variable	static PyGetSetDef iobase_getset[]
+Modules/_io/iobase.c	-	iobase_methods	variable	static PyMethodDef iobase_methods
+Python/fileutils.c	set_inheritable	ioctl_works	variable	static int ioctl_works
+Modules/itertoolsmodule.c	-	islice_methods	variable	static PyMethodDef islice_methods
+Modules/itertoolsmodule.c	-	islice_type	variable	static PyTypeObject islice_type
+Python/Python-ast.c	-	IsNot_singleton	variable	static PyObject *IsNot_singleton
+Python/Python-ast.c	-	IsNot_type	variable	static PyTypeObject *IsNot_type
+Python/Python-ast.c	-	Is_singleton	variable	static PyObject *Is_singleton
+Modules/signalmodule.c	-	is_tripped	variable	static _Py_atomic_int is_tripped
+Python/Python-ast.c	-	Is_type	variable	static PyTypeObject *Is_type
+Modules/_operator.c	-	itemgetter_methods	variable	static PyMethodDef itemgetter_methods
+Modules/_operator.c	-	itemgetter_type	variable	static PyTypeObject itemgetter_type
+Modules/itertoolsmodule.c	-	itertoolsmodule	variable	static struct PyModuleDef itertoolsmodule
+Modules/signalmodule.c	-	ItimerError	variable	static PyObject *ItimerError
+Python/Python-ast.c	-	JoinedStr_fields	variable	static const char *JoinedStr_fields[]
+Python/Python-ast.c	-	JoinedStr_type	variable	static PyTypeObject *JoinedStr_type
+Modules/_functoolsmodule.c	-	keyobject_members	variable	static PyMemberDef keyobject_members[]
+Modules/_functoolsmodule.c	-	keyobject_type	variable	static PyTypeObject keyobject_type
+Objects/dictobject.c	-	keys_free_list	variable	static PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]
+Python/Python-ast.c	-	keyword_fields	variable	static const char *keyword_fields[]
+Python/sysmodule.c	sys_set_asyncgen_hooks	keywords	variable	static const char *keywords[]
+Modules/_bisectmodule.c	bisect_right	keywords	variable	static const char *keywords[]
+Modules/_bisectmodule.c	insort_right	keywords	variable	static const char *keywords[]
+Python/Python-ast.c	-	keyword_type	variable	static PyTypeObject *keyword_type
+Modules/_functoolsmodule.c	keyobject_call	kwargs	variable	static const char *kwargs[]
+Modules/_functoolsmodule.c	functools_cmp_to_key	kwargs	variable	static const char *kwargs[]
+Modules/itertoolsmodule.c	repeat_new	kwargs	variable	static const char *kwargs[]
+Python/_warnings.c	warnings_warn_explicit	kwd_list	variable	static const char *kwd_list[]
+Modules/_functoolsmodule.c	-	kwd_mark	variable	static PyObject *kwd_mark
+Python/bltinmodule.c	builtin___import__	kwlist	variable	static const char *kwlist[]
+Python/bltinmodule.c	min_max	kwlist	variable	static const char *kwlist[]
+Python/context.c	contextvar_tp_new	kwlist	variable	static const char *kwlist[]
+Python/sysmodule.c	sys_getsizeof	kwlist	variable	static const char *kwlist[]
+Objects/bytearrayobject.c	bytearray_init	kwlist	variable	static const char *kwlist[]
+Objects/bytesobject.c	bytes_new	kwlist	variable	static const char *kwlist[]
+Objects/exceptions.c	ImportError_init	kwlist	variable	static const char *kwlist[]
+Objects/interpreteridobject.c	interpid_new	kwlist	variable	static const char *kwlist[]
+Objects/memoryobject.c	memory_new	kwlist	variable	static const char *kwlist[]
+Objects/memoryobject.c	memory_cast	kwlist	variable	static const char *kwlist[]
+Objects/memoryobject.c	memory_tobytes	kwlist	variable	static const char *kwlist[]
+Objects/odictobject.c	odict_pop	kwlist	variable	static const char *kwlist[]
+Objects/unicodeobject.c	unicode_new	kwlist	variable	static const char *kwlist[]
+Objects/weakrefobject.c	weakref_call	kwlist	variable	static const char *kwlist[]
+Modules/_elementtree.c	element_setstate_from_Python	kwlist	variable	static const char *kwlist[]
+Modules/_json.c	scanner_call	kwlist	variable	static const char *kwlist[]
+Modules/_json.c	scanner_new	kwlist	variable	static const char *kwlist[]
+Modules/_json.c	encoder_new	kwlist	variable	static const char *kwlist[]
+Modules/_json.c	encoder_call	kwlist	variable	static const char *kwlist[]
+Python/symtable.c	-	lambda	variable	static identifier lambda
+Python/Python-ast.c	-	Lambda_fields	variable	static const char *Lambda_fields[]
+Python/Python-ast.c	-	Lambda_type	variable	static PyTypeObject *Lambda_type
+Parser/listnode.c	-	level	variable	static int level
+Objects/listobject.c	-	list_as_mapping	variable	static PyMappingMethods list_as_mapping
+Objects/listobject.c	-	list_as_sequence	variable	static PySequenceMethods list_as_sequence
+Python/symtable.c	-	listcomp	variable	static identifier listcomp
+Python/Python-ast.c	-	ListComp_fields	variable	static const char *ListComp_fields[]
+Python/Python-ast.c	-	ListComp_type	variable	static PyTypeObject *ListComp_type
+Python/Python-ast.c	-	List_fields	variable	static const char *List_fields[]
+Objects/listobject.c	-	listiter_methods	variable	static PyMethodDef listiter_methods
+Objects/listobject.c	-	list_methods	variable	static PyMethodDef list_methods
+Objects/listobject.c	-	listreviter_methods	variable	static PyMethodDef listreviter_methods
+Python/Python-ast.c	-	List_type	variable	static PyTypeObject *List_type
+Python/ceval.c	-	lltrace	variable	static int lltrace
+Python/Python-ast.c	-	Load_singleton	variable	static PyObject *Load_singleton
+Python/Python-ast.c	-	Load_type	variable	static PyTypeObject *Load_type
+Modules/_threadmodule.c	-	localdummytype	variable	static PyTypeObject localdummytype
+Modules/_localemodule.c	-	_localemodule	variable	static struct PyModuleDef _localemodule
+Modules/_threadmodule.c	-	localtype	variable	static PyTypeObject localtype
+Modules/_threadmodule.c	-	lock_methods	variable	static PyMethodDef lock_methods
+Modules/_threadmodule.c	-	Locktype	variable	static PyTypeObject Locktype
+Objects/longobject.c	PyLong_FromString	log_base_BASE	variable	static double log_base_BASE[37]
+Objects/longobject.c	-	long_as_number	variable	static PyNumberMethods long_as_number
+Objects/longobject.c	-	long_getset	variable	static PyGetSetDef long_getset[]
+Objects/longobject.c	-	long_methods	variable	static PyMethodDef long_methods
+Objects/rangeobject.c	-	longrangeiter_methods	variable	static PyMethodDef longrangeiter_methods
+Modules/_functoolsmodule.c	-	lru_cache_getsetlist	variable	static PyGetSetDef lru_cache_getsetlist[]
+Modules/_functoolsmodule.c	-	lru_cache_methods	variable	static PyMethodDef lru_cache_methods
+Modules/_functoolsmodule.c	-	lru_cache_type	variable	static PyTypeObject lru_cache_type
+Modules/_functoolsmodule.c	-	lru_list_elem_type	variable	static PyTypeObject lru_list_elem_type
+Python/Python-ast.c	-	LShift_singleton	variable	static PyObject *LShift_singleton
+Python/Python-ast.c	-	LShift_type	variable	static PyTypeObject *LShift_type
+Python/Python-ast.c	-	LtE_singleton	variable	static PyObject *LtE_singleton
+Python/Python-ast.c	-	LtE_type	variable	static PyTypeObject *LtE_type
+Python/Python-ast.c	-	Lt_singleton	variable	static PyObject *Lt_singleton
+Python/Python-ast.c	-	Lt_type	variable	static PyTypeObject *Lt_type
+Python/bltinmodule.c	-	map_methods	variable	static PyMethodDef map_methods
+Objects/descrobject.c	-	mappingproxy_as_mapping	variable	static PyMappingMethods mappingproxy_as_mapping
+Objects/descrobject.c	-	mappingproxy_as_sequence	variable	static PySequenceMethods mappingproxy_as_sequence
+Objects/descrobject.c	-	mappingproxy_methods	variable	static PyMethodDef mappingproxy_methods
+Objects/dictobject.c	-	mapp_methods	variable	static PyMethodDef mapp_methods
+Python/marshal.c	-	marshal_methods	variable	static PyMethodDef marshal_methods
+Python/marshal.c	-	marshalmodule	variable	static struct PyModuleDef marshalmodule
+Modules/_sre.c	-	match_as_mapping	variable	static PyMappingMethods match_as_mapping
+Modules/_sre.c	-	match_getset	variable	static PyGetSetDef match_getset[]
+Modules/_sre.c	-	match_members	variable	static PyMemberDef match_members[]
+Modules/_sre.c	-	match_methods	variable	static PyMethodDef match_methods
+Modules/_sre.c	-	Match_Type	variable	static PyTypeObject Match_Type
+Python/Python-ast.c	-	MatMult_singleton	variable	static PyObject *MatMult_singleton
+Python/Python-ast.c	-	MatMult_type	variable	static PyTypeObject *MatMult_type
+Objects/obmalloc.c	-	maxarenas	variable	static uint maxarenas
+Objects/moduleobject.c	-	max_module_number	variable	static Py_ssize_t max_module_number
+Objects/descrobject.c	-	member_getset	variable	static PyGetSetDef member_getset[]
+Objects/exceptions.c	-	memerrors_freelist	variable	static PyBaseExceptionObject *memerrors_freelist
+Objects/exceptions.c	-	memerrors_numfree	variable	static int memerrors_numfree
+Objects/memoryobject.c	-	memory_as_buffer	variable	static PyBufferProcs memory_as_buffer
+Objects/memoryobject.c	-	memory_as_mapping	variable	static PyMappingMethods memory_as_mapping
+Objects/memoryobject.c	-	memory_as_sequence	variable	static PySequenceMethods memory_as_sequence
+Objects/memoryobject.c	-	memory_getsetlist	variable	static PyGetSetDef memory_getsetlist[]
+Objects/memoryobject.c	-	memory_methods	variable	static PyMethodDef memory_methods
+Objects/methodobject.c	-	meth_getsets	variable	static PyGetSetDef meth_getsets []
+Objects/methodobject.c	-	meth_members	variable	static PyMemberDef meth_members[]
+Objects/methodobject.c	-	meth_methods	variable	static PyMethodDef meth_methods
+Objects/typeobject.c	-	method_cache	variable	static struct method_cache_entry method_cache[1 << MCACHE_SIZE_EXP]
+Modules/_operator.c	-	methodcaller_methods	variable	static PyMethodDef methodcaller_methods
+Modules/_operator.c	-	methodcaller_type	variable	static PyTypeObject methodcaller_type
+Objects/classobject.c	-	method_getset	variable	static PyGetSetDef method_getset[]
+Objects/descrobject.c	-	method_getset	variable	static PyGetSetDef method_getset[]
+Objects/classobject.c	-	method_memberlist	variable	static PyMemberDef method_memberlist[]
+Objects/classobject.c	-	method_methods	variable	static PyMethodDef method_methods
+Python/codecs.c	_PyCodecRegistry_Init	methods	variable	static struct { char *name; PyMethodDef def; } methods[]
+Python/frozen.c	-	M___hello__	variable	static unsigned char M___hello__[]
+Python/Python-ast.c	-	Mod_singleton	variable	static PyObject *Mod_singleton
+Python/Python-ast.c	-	mod_type	variable	static PyTypeObject *mod_type
+Python/Python-ast.c	-	Mod_type	variable	static PyTypeObject *Mod_type
+Modules/faulthandler.c	-	module_def	variable	static struct PyModuleDef module_def
+Modules/_tracemalloc.c	-	module_def	variable	static struct PyModuleDef module_def
+Python/Python-ast.c	-	Module_fields	variable	static const char *Module_fields[]
+Modules/_collectionsmodule.c	-	module_functions	variable	static struct PyMethodDef module_functions[]
+Modules/_abc.c	-	module_functions	variable	static struct PyMethodDef module_functions[]
+Objects/moduleobject.c	-	module_members	variable	static PyMemberDef module_members[]
+Objects/moduleobject.c	-	module_methods	variable	static PyMethodDef module_methods
+Modules/_functoolsmodule.c	-	module_methods	variable	static PyMethodDef module_methods
+Modules/itertoolsmodule.c	-	module_methods	variable	static PyMethodDef module_methods
+Modules/_io/_iomodule.c	-	module_methods	variable	static PyMethodDef module_methods
+Modules/faulthandler.c	-	module_methods	variable	static PyMethodDef module_methods
+Modules/_tracemalloc.c	-	module_methods	variable	static PyMethodDef module_methods
+Python/Python-ast.c	-	Module_type	variable	static PyTypeObject *Module_type
+Python/Python-ast.c	-	Mult_singleton	variable	static PyObject *Mult_singleton
+Python/Python-ast.c	-	Mult_type	variable	static PyTypeObject *Mult_type
+Objects/funcobject.c	PyFunction_NewWithQualName	__name__	variable	static PyObject *__name__
+Python/compile.c	compiler_lambda	name	variable	static identifier name
+Python/compile.c	compiler_genexp	name	variable	static identifier name
+Python/compile.c	compiler_listcomp	name	variable	static identifier name
+Python/compile.c	compiler_setcomp	name	variable	static identifier name
+Python/compile.c	compiler_dictcomp	name	variable	static identifier name
+Python/Python-ast.c	-	NamedExpr_fields	variable	static const char *NamedExpr_fields[]
+Python/Python-ast.c	-	NamedExpr_type	variable	static PyTypeObject *NamedExpr_type
+Python/Python-ast.c	-	Name_fields	variable	static const char *Name_fields[]
+Objects/typeobject.c	-	name_op	variable	static _Py_Identifier name_op[]
+Objects/namespaceobject.c	-	namespace_members	variable	static PyMemberDef namespace_members[]
+Objects/namespaceobject.c	-	namespace_methods	variable	static PyMethodDef namespace_methods
+Python/Python-ast.c	-	Name_type	variable	static PyTypeObject *Name_type
+Objects/obmalloc.c	-	narenas_currently_allocated	variable	static size_t narenas_currently_allocated
+Objects/obmalloc.c	-	narenas_highwater	variable	static size_t narenas_highwater
+Python/sysmodule.c	sys_displayhook	newline	variable	static PyObject *newline
+Objects/typeobject.c	-	next_version_tag	variable	static unsigned int next_version_tag
+Objects/obmalloc.c	-	nfp2lasta	variable	static struct arena_object* nfp2lasta[MAX_POOLS_IN_ARENA + 1]
+Python/dynload_shlib.c	-	nhandles	variable	static int nhandles
+Objects/object.c	-	none_as_number	variable	static PyNumberMethods none_as_number
+Python/Python-ast.c	-	Nonlocal_fields	variable	static const char *Nonlocal_fields[]
+Python/Python-ast.c	-	Nonlocal_type	variable	static PyTypeObject *Nonlocal_type
+Python/Python-ast.c	-	NotEq_singleton	variable	static PyObject *NotEq_singleton
+Python/Python-ast.c	-	NotEq_type	variable	static PyTypeObject *NotEq_type
+Objects/object.c	-	notimplemented_methods	variable	static PyMethodDef notimplemented_methods
+Python/Python-ast.c	-	NotIn_singleton	variable	static PyObject *NotIn_singleton
+Python/Python-ast.c	-	NotIn_type	variable	static PyTypeObject *NotIn_type
+Python/Python-ast.c	-	Not_singleton	variable	static PyObject *Not_singleton
+Python/Python-ast.c	-	Not_type	variable	static PyTypeObject *Not_type
+Objects/obmalloc.c	-	ntimes_arena_allocated	variable	static size_t ntimes_arena_allocated
+Objects/bytesobject.c	-	nullstring	variable	static PyBytesObject *nullstring
+Objects/codeobject.c	PyCode_NewEmpty	nulltuple	variable	static PyObject *nulltuple
+Objects/floatobject.c	-	numfree	variable	static int numfree
+Objects/frameobject.c	-	numfree	variable	static int numfree
+Objects/listobject.c	-	numfree	variable	static int numfree
+Objects/dictobject.c	-	numfree	variable	static int numfree
+Objects/methodobject.c	-	numfree	variable	static int numfree
+Objects/tupleobject.c	-	numfree	variable	static int numfree[PyTuple_MAXSAVESIZE]
+Objects/classobject.c	-	numfree	variable	static int numfree
+Modules/_collectionsmodule.c	-	numfreeblocks	variable	static Py_ssize_t numfreeblocks
+Objects/dictobject.c	-	numfreekeys	variable	static int numfreekeys
+Objects/typeobject.c	-	object_getsets	variable	static PyGetSetDef object_getsets[]
+Objects/typeobject.c	-	object_methods	variable	static PyMethodDef object_methods
+Objects/typeobject.c	object___reduce_ex___impl	objreduce	variable	static PyObject *objreduce
+Objects/odictobject.c	-	odict_as_mapping	variable	static PyMappingMethods odict_as_mapping
+Objects/odictobject.c	-	odict_getset	variable	static PyGetSetDef odict_getset[]
+Objects/odictobject.c	-	odictitems_methods	variable	static PyMethodDef odictitems_methods
+Objects/odictobject.c	-	odictiter_methods	variable	static PyMethodDef odictiter_methods
+Objects/odictobject.c	-	odictkeys_methods	variable	static PyMethodDef odictkeys_methods
+Objects/odictobject.c	-	odict_methods	variable	static PyMethodDef odict_methods
+Objects/odictobject.c	-	odictvalues_methods	variable	static PyMethodDef odictvalues_methods
+Modules/faulthandler.c	-	old_stack	variable	static stack_t old_stack
+Modules/_operator.c	-	operator_methods	variable	static PyMethodDef operator_methods
+Modules/_operator.c	-	operatormodule	variable	static struct PyModuleDef operatormodule
+Python/Python-ast.c	-	operator_type	variable	static PyTypeObject *operator_type
+Objects/typeobject.c	slot_nb_add	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_subtract	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_multiply	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_matrix_multiply	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_remainder	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_divmod	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_power_binary	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_lshift	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_rshift	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_and	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_xor	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_or	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_floor_divide	op_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_true_divide	op_id	variable	_Py_static_string(op_id, OPSTR)
+Python/getopt.c	-	opt_ptr	variable	static const wchar_t *opt_ptr
+Python/initconfig.c	-	orig_argv	variable	static PyWideStringList orig_argv
+Python/Python-ast.c	-	Or_singleton	variable	static PyObject *Or_singleton
+Python/Python-ast.c	-	Or_type	variable	static PyTypeObject *Or_type
+Objects/exceptions.c	-	OSError_getset	variable	static PyGetSetDef OSError_getset[]
+Objects/exceptions.c	-	OSError_members	variable	static PyMemberDef OSError_members[]
+Objects/exceptions.c	-	OSError_methods	variable	static PyMethodDef OSError_methods
+Python/dtoa.c	-	p5s	variable	static Bigint *p5s
+Python/Python-ast.c	-	Param_singleton	variable	static PyObject *Param_singleton
+Python/Python-ast.c	-	Param_type	variable	static PyTypeObject *Param_type
+Python/bltinmodule.c	builtin_print	_parser	variable	static struct _PyArg_Parser _parser
+Python/clinic/_warnings.c.h	warnings_warn	_parser	variable	static _PyArg_Parser _parser
+Python/clinic/bltinmodule.c.h	builtin_compile	_parser	variable	static _PyArg_Parser _parser
+Python/clinic/bltinmodule.c.h	builtin_round	_parser	variable	static _PyArg_Parser _parser
+Python/clinic/bltinmodule.c.h	builtin_sum	_parser	variable	static _PyArg_Parser _parser
+Python/clinic/import.c.h	_imp_source_hash	_parser	variable	static _PyArg_Parser _parser
+Python/clinic/sysmodule.c.h	sys_addaudithook	_parser	variable	static _PyArg_Parser _parser
+Python/clinic/sysmodule.c.h	sys_set_coroutine_origin_tracking_depth	_parser	variable	static _PyArg_Parser _parser
+Python/clinic/traceback.c.h	tb_new	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/bytearrayobject.c.h	bytearray_translate	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/bytearrayobject.c.h	bytearray_split	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/bytearrayobject.c.h	bytearray_rsplit	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/bytearrayobject.c.h	bytearray_decode	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/bytearrayobject.c.h	bytearray_splitlines	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/bytearrayobject.c.h	bytearray_hex	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/bytesobject.c.h	bytes_split	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/bytesobject.c.h	bytes_rsplit	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/bytesobject.c.h	bytes_translate	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/bytesobject.c.h	bytes_decode	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/bytesobject.c.h	bytes_splitlines	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/bytesobject.c.h	bytes_hex	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/codeobject.c.h	code_replace	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/complexobject.c.h	complex_new	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/descrobject.c.h	mappingproxy_new	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/descrobject.c.h	property_init	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/enumobject.c.h	enum_new	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/funcobject.c.h	func_new	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/listobject.c.h	list_sort	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/longobject.c.h	long_new	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/longobject.c.h	int_to_bytes	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/longobject.c.h	int_from_bytes	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/memoryobject.c.h	memoryview_hex	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/moduleobject.c.h	module___init__	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/odictobject.c.h	OrderedDict_fromkeys	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/odictobject.c.h	OrderedDict_setdefault	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/odictobject.c.h	OrderedDict_popitem	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/odictobject.c.h	OrderedDict_move_to_end	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/structseq.c.h	structseq_new	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/unicodeobject.c.h	unicode_encode	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/unicodeobject.c.h	unicode_expandtabs	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/unicodeobject.c.h	unicode_split	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/unicodeobject.c.h	unicode_rsplit	_parser	variable	static _PyArg_Parser _parser
+Objects/clinic/unicodeobject.c.h	unicode_splitlines	_parser	variable	static _PyArg_Parser _parser
+Objects/stringlib/clinic/transmogrify.h.h	stringlib_expandtabs	_parser	variable	static _PyArg_Parser _parser
+Modules/_blake2/clinic/blake2b_impl.c.h	py_blake2b_new	_parser	variable	static _PyArg_Parser _parser
+Modules/_blake2/clinic/blake2s_impl.c.h	py_blake2s_new	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/_iomodule.c.h	_io_open	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/_iomodule.c.h	_io_open_code	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/bufferedio.c.h	_io_BufferedReader___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/bufferedio.c.h	_io_BufferedWriter___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/bufferedio.c.h	_io_BufferedRandom___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/bytesio.c.h	_io_BytesIO___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/fileio.c.h	_io_FileIO___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/stringio.c.h	_io_StringIO___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/textio.c.h	_io_IncrementalNewlineDecoder___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/textio.c.h	_io_IncrementalNewlineDecoder_decode	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/textio.c.h	_io_TextIOWrapper___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/textio.c.h	_io_TextIOWrapper_reconfigure	_parser	variable	static _PyArg_Parser _parser
+Modules/_io/clinic/winconsoleio.c.h	_io__WindowsConsoleIO___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/_multiprocessing/clinic/posixshmem.c.h	_posixshmem_shm_open	_parser	variable	static _PyArg_Parser _parser
+Modules/_multiprocessing/clinic/posixshmem.c.h	_posixshmem_shm_unlink	_parser	variable	static _PyArg_Parser _parser
+Modules/cjkcodecs/clinic/multibytecodec.c.h	_multibytecodec_MultibyteCodec_encode	_parser	variable	static _PyArg_Parser _parser
+Modules/cjkcodecs/clinic/multibytecodec.c.h	_multibytecodec_MultibyteCodec_decode	_parser	variable	static _PyArg_Parser _parser
+Modules/cjkcodecs/clinic/multibytecodec.c.h	_multibytecodec_MultibyteIncrementalEncoder_encode	_parser	variable	static _PyArg_Parser _parser
+Modules/cjkcodecs/clinic/multibytecodec.c.h	_multibytecodec_MultibyteIncrementalDecoder_decode	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h	_asyncio_Future___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h	_asyncio_Future_add_done_callback	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h	_asyncio_Task___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h	_asyncio_Task_current_task	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h	_asyncio_Task_all_tasks	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h	_asyncio_Task_get_stack	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h	_asyncio_Task_print_stack	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h	_asyncio__register_task	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h	_asyncio__unregister_task	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h	_asyncio__enter_task	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h	_asyncio__leave_task	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_bz2module.c.h	_bz2_BZ2Decompressor_decompress	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_codecsmodule.c.h	_codecs_encode	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_codecsmodule.c.h	_codecs_decode	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_cursesmodule.c.h	_curses_setupterm	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_datetimemodule.c.h	datetime_datetime_now	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h	_elementtree_Element_find	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h	_elementtree_Element_findtext	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h	_elementtree_Element_findall	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h	_elementtree_Element_iterfind	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h	_elementtree_Element_get	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h	_elementtree_Element_iter	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h	_elementtree_Element_getiterator	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h	_elementtree_TreeBuilder___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h	_elementtree_XMLParser___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_hashopenssl.c.h	EVP_new	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_hashopenssl.c.h	pbkdf2_hmac	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_hashopenssl.c.h	_hashlib_scrypt	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_hashopenssl.c.h	_hashlib_hmac_digest	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_lzmamodule.c.h	_lzma_LZMADecompressor_decompress	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_lzmamodule.c.h	_lzma_LZMADecompressor___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_opcode.c.h	_opcode_stack_effect	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_pickle.c.h	_pickle_Pickler___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_pickle.c.h	_pickle_Unpickler___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_pickle.c.h	_pickle_dump	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_pickle.c.h	_pickle_dumps	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_pickle.c.h	_pickle_load	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_pickle.c.h	_pickle_loads	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_queuemodule.c.h	_queue_SimpleQueue_put	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_queuemodule.c.h	_queue_SimpleQueue_put_nowait	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_queuemodule.c.h	_queue_SimpleQueue_get	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_SRE_Pattern_match	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_SRE_Pattern_fullmatch	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_SRE_Pattern_search	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_SRE_Pattern_findall	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_SRE_Pattern_finditer	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_SRE_Pattern_scanner	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_SRE_Pattern_split	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_SRE_Pattern_sub	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_SRE_Pattern_subn	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_compile	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_SRE_Match_expand	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_SRE_Match_groups	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h	_sre_SRE_Match_groupdict	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h	_ssl__SSLSocket_get_channel_binding	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h	_ssl__SSLContext_load_cert_chain	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h	_ssl__SSLContext_load_verify_locations	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h	_ssl__SSLContext__wrap_socket	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h	_ssl__SSLContext__wrap_bio	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h	_ssl__SSLContext_get_ca_certs	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h	_ssl_txt2obj	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h	_ssl_enum_certificates	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h	_ssl_enum_crls	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_struct.c.h	Struct___init__	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_struct.c.h	Struct_unpack_from	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_struct.c.h	unpack_from	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_winapi.c.h	_winapi_ConnectNamedPipe	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_winapi.c.h	_winapi_ReadFile	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_winapi.c.h	_winapi_WriteFile	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/_winapi.c.h	_winapi_GetFileType	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/binascii.c.h	binascii_b2a_uu	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/binascii.c.h	binascii_b2a_base64	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/binascii.c.h	binascii_b2a_hex	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/binascii.c.h	binascii_hexlify	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/binascii.c.h	binascii_a2b_qp	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/binascii.c.h	binascii_b2a_qp	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/cmathmodule.c.h	cmath_isclose	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/gcmodule.c.h	gc_collect	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/gcmodule.c.h	gc_get_objects	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/grpmodule.c.h	grp_getgrgid	_parser	variable	static _PyArg_Parser _parser
+Modules/clinic/grpmodule.c.h	grp_getgrnam	_parser	variable	static _PyArg_Parser _parser
+Modules/_functoolsmodule.c	-	partial_getsetlist	variable	static PyGetSetDef partial_getsetlist[]
+Modules/_functoolsmodule.c	-	partial_memberlist	variable	static PyMemberDef partial_memberlist[]
+Modules/_functoolsmodule.c	-	partial_methods	variable	static PyMethodDef partial_methods
+Modules/_functoolsmodule.c	-	partial_type	variable	static PyTypeObject partial_type
+Python/Python-ast.c	-	Pass_type	variable	static PyTypeObject *Pass_type
+Modules/_sre.c	-	pattern_getset	variable	static PyGetSetDef pattern_getset[]
+Modules/_sre.c	-	pattern_members	variable	static PyMemberDef pattern_members[]
+Modules/_sre.c	-	pattern_methods	variable	static PyMethodDef pattern_methods
+Modules/_sre.c	-	Pattern_Type	variable	static PyTypeObject Pattern_Type
+Modules/itertoolsmodule.c	-	permuations_methods	variable	static PyMethodDef permuations_methods
+Modules/itertoolsmodule.c	-	permutations_type	variable	static PyTypeObject permutations_type
+Objects/picklebufobject.c	-	picklebuf_as_buffer	variable	static PyBufferProcs picklebuf_as_buffer
+Objects/picklebufobject.c	-	picklebuf_methods	variable	static PyMethodDef picklebuf_methods
+Python/dtoa.c	-	pmem_next	variable	static double *pmem_next
+Objects/typeobject.c	resolve_slotdups	pname	variable	static PyObject *pname
+Modules/posixmodule.c	-	posix_constants_confstr	variable	static struct constdef posix_constants_confstr[]
+Modules/posixmodule.c	-	posix_constants_pathconf	variable	static struct constdef  posix_constants_pathconf[]
+Modules/posixmodule.c	-	posix_constants_sysconf	variable	static struct constdef posix_constants_sysconf[]
+Modules/posixmodule.c	-	posix_methods	variable	static PyMethodDef posix_methods
+Modules/posixmodule.c	-	posixmodule	variable	static struct PyModuleDef posixmodule
+Modules/posixmodule.c	-	posix_putenv_garbage	variable	static PyObject *posix_putenv_garbage
+Python/Python-ast.c	-	Pow_singleton	variable	static PyObject *Pow_singleton
+Python/Python-ast.c	-	Pow_type	variable	static PyTypeObject *Pow_type
+Python/sysmodule.c	-	_preinit_warnoptions	variable	static _Py_PreInitEntry _preinit_warnoptions
+Python/sysmodule.c	-	_preinit_xoptions	variable	static _Py_PreInitEntry _preinit_xoptions
+Objects/exceptions.c	_check_for_legacy_statements	print_prefix	variable	static PyObject *print_prefix
+Python/dtoa.c	-	private_mem	variable	static double private_mem[PRIVATE_mem]
+Modules/itertoolsmodule.c	-	product_methods	variable	static PyMethodDef product_methods
+Modules/itertoolsmodule.c	-	product_type	variable	static PyTypeObject product_type
+Objects/descrobject.c	-	property_getsetlist	variable	static PyGetSetDef property_getsetlist[]
+Objects/descrobject.c	-	property_members	variable	static PyMemberDef property_members[]
+Objects/descrobject.c	-	property_methods	variable	static PyMethodDef property_methods
+Objects/weakrefobject.c	-	proxy_as_mapping	variable	static PyMappingMethods proxy_as_mapping
+Objects/weakrefobject.c	-	proxy_as_number	variable	static PyNumberMethods proxy_as_number
+Objects/weakrefobject.c	-	proxy_as_sequence	variable	static PySequenceMethods proxy_as_sequence
+Objects/weakrefobject.c	-	proxy_methods	variable	static PyMethodDef proxy_methods
+Objects/typeobject.c	resolve_slotdups	ptrs	variable	static slotdef *ptrs[MAX_EQUIV]
+Modules/pwdmodule.c	-	pwd_methods	variable	static PyMethodDef pwd_methods
+Modules/pwdmodule.c	-	pwdmodule	variable	static struct PyModuleDef pwdmodule
+Objects/obmalloc.c	-	_Py_AllocatedBlocks	variable	static Py_ssize_t _Py_AllocatedBlocks
+Objects/genobject.c	-	_PyAsyncGenASend_Type	variable	PyTypeObject _PyAsyncGenASend_Type
+Objects/genobject.c	-	_PyAsyncGenAThrow_Type	variable	PyTypeObject _PyAsyncGenAThrow_Type
+Objects/genobject.c	-	PyAsyncGen_Type	variable	PyTypeObject PyAsyncGen_Type
+Objects/genobject.c	-	_PyAsyncGenWrappedValue_Type	variable	PyTypeObject _PyAsyncGenWrappedValue_Type
+Objects/typeobject.c	-	PyBaseObject_Type	variable	PyTypeObject PyBaseObject_Type
+Objects/boolobject.c	-	PyBool_Type	variable	PyTypeObject PyBool_Type
+Modules/_io/bufferedio.c	-	PyBufferedIOBase_Type	variable	PyTypeObject PyBufferedIOBase_Type
+Modules/_io/bufferedio.c	-	PyBufferedRandom_Type	variable	PyTypeObject PyBufferedRandom_Type
+Modules/_io/bufferedio.c	-	PyBufferedReader_Type	variable	PyTypeObject PyBufferedReader_Type
+Modules/_io/bufferedio.c	-	PyBufferedRWPair_Type	variable	PyTypeObject PyBufferedRWPair_Type
+Modules/_io/bufferedio.c	-	PyBufferedWriter_Type	variable	PyTypeObject PyBufferedWriter_Type
+Objects/bytearrayobject.c	-	_PyByteArray_empty_string	variable	char _PyByteArray_empty_string[]
+Objects/bytearrayobject.c	-	PyByteArrayIter_Type	variable	PyTypeObject PyByteArrayIter_Type
+Objects/bytearrayobject.c	-	PyByteArray_Type	variable	PyTypeObject PyByteArray_Type
+Modules/_io/bytesio.c	-	_PyBytesIOBuffer_Type	variable	PyTypeObject _PyBytesIOBuffer_Type
+Modules/_io/bytesio.c	-	PyBytesIO_Type	variable	PyTypeObject PyBytesIO_Type
+Objects/bytesobject.c	-	PyBytesIter_Type	variable	PyTypeObject PyBytesIter_Type
+Objects/bytesobject.c	-	PyBytes_Type	variable	PyTypeObject PyBytes_Type
+Python/initconfig.c	-	Py_BytesWarningFlag	variable	int Py_BytesWarningFlag
+Objects/iterobject.c	-	PyCallIter_Type	variable	PyTypeObject PyCallIter_Type
+Objects/capsule.c	-	PyCapsule_Type	variable	PyTypeObject PyCapsule_Type
+Objects/cellobject.c	-	PyCell_Type	variable	PyTypeObject PyCell_Type
+Objects/methodobject.c	-	PyCFunction_Type	variable	PyTypeObject PyCFunction_Type
+Python/ceval.c	-	_Py_CheckRecursionLimit	variable	int _Py_CheckRecursionLimit
+Objects/descrobject.c	-	PyClassMethodDescr_Type	variable	PyTypeObject PyClassMethodDescr_Type
+Objects/funcobject.c	-	PyClassMethod_Type	variable	PyTypeObject PyClassMethod_Type
+Objects/codeobject.c	-	PyCode_Type	variable	PyTypeObject PyCode_Type
+Objects/complexobject.c	-	PyComplex_Type	variable	PyTypeObject PyComplex_Type
+Python/context.c	-	PyContext_as_mapping	variable	static PyMappingMethods PyContext_as_mapping
+Python/context.c	-	PyContext_as_sequence	variable	static PySequenceMethods PyContext_as_sequence
+Python/context.c	-	PyContext_methods	variable	static PyMethodDef PyContext_methods
+Python/context.c	-	PyContextTokenMissing_Type	variable	PyTypeObject PyContextTokenMissing_Type
+Python/context.c	-	PyContextToken_Type	variable	PyTypeObject PyContextToken_Type
+Python/context.c	-	PyContextTokenType_getsetlist	variable	static PyGetSetDef PyContextTokenType_getsetlist[]
+Python/context.c	-	PyContext_Type	variable	PyTypeObject PyContext_Type
+Python/context.c	-	PyContextVar_members	variable	static PyMemberDef PyContextVar_members[]
+Python/context.c	-	PyContextVar_methods	variable	static PyMethodDef PyContextVar_methods
+Python/context.c	-	PyContextVar_Type	variable	PyTypeObject PyContextVar_Type
+Objects/genobject.c	-	PyCoro_Type	variable	PyTypeObject PyCoro_Type
+Objects/genobject.c	-	_PyCoroWrapper_Type	variable	PyTypeObject _PyCoroWrapper_Type
+Python/initconfig.c	-	Py_DebugFlag	variable	int Py_DebugFlag
+Objects/dictobject.c	-	pydict_global_version	variable	static uint64_t pydict_global_version
+Objects/dictobject.c	-	PyDictItems_Type	variable	PyTypeObject PyDictItems_Type
+Objects/dictobject.c	-	PyDictIterItem_Type	variable	PyTypeObject PyDictIterItem_Type
+Objects/dictobject.c	-	PyDictIterKey_Type	variable	PyTypeObject PyDictIterKey_Type
+Objects/dictobject.c	-	PyDictIterValue_Type	variable	PyTypeObject PyDictIterValue_Type
+Objects/dictobject.c	-	PyDictKeys_Type	variable	PyTypeObject PyDictKeys_Type
+Objects/descrobject.c	-	PyDictProxy_Type	variable	PyTypeObject PyDictProxy_Type
+Objects/dictobject.c	-	PyDictRevIterItem_Type	variable	PyTypeObject PyDictRevIterItem_Type
+Objects/dictobject.c	-	PyDictRevIterKey_Type	variable	PyTypeObject PyDictRevIterKey_Type
+Objects/dictobject.c	-	PyDictRevIterValue_Type	variable	PyTypeObject PyDictRevIterValue_Type
+Objects/dictobject.c	-	PyDict_Type	variable	PyTypeObject PyDict_Type
+Objects/dictobject.c	-	PyDictValues_Type	variable	PyTypeObject PyDictValues_Type
+Python/initconfig.c	-	Py_DontWriteBytecodeFlag	variable	int Py_DontWriteBytecodeFlag
+Objects/sliceobject.c	-	_Py_EllipsisObject	variable	PyObject _Py_EllipsisObject
+Objects/sliceobject.c	-	PyEllipsis_Type	variable	PyTypeObject PyEllipsis_Type
+Objects/enumobject.c	-	PyEnum_Type	variable	PyTypeObject PyEnum_Type
+Objects/exceptions.c	-	_PyExc_ArithmeticError	variable	static PyTypeObject _PyExc_ArithmeticError
+Objects/exceptions.c	-	PyExc_ArithmeticError	variable	static PyTypeObject PyExc_ArithmeticError
+Objects/exceptions.c	-	_PyExc_AssertionError	variable	static PyTypeObject _PyExc_AssertionError
+Objects/exceptions.c	-	PyExc_AssertionError	variable	static PyTypeObject PyExc_AssertionError
+Objects/exceptions.c	-	_PyExc_AttributeError	variable	static PyTypeObject _PyExc_AttributeError
+Objects/exceptions.c	-	PyExc_AttributeError	variable	static PyTypeObject PyExc_AttributeError
+Objects/exceptions.c	-	_PyExc_BaseException	variable	static PyTypeObject _PyExc_BaseException
+Objects/exceptions.c	-	PyExc_BaseException	variable	static PyTypeObject PyExc_BaseException
+Objects/exceptions.c	-	_PyExc_BlockingIOError	variable	static PyTypeObject _PyExc_BlockingIOError
+Objects/exceptions.c	-	PyExc_BlockingIOError	variable	static PyTypeObject PyExc_BlockingIOError
+Objects/exceptions.c	-	_PyExc_BrokenPipeError	variable	static PyTypeObject _PyExc_BrokenPipeError
+Objects/exceptions.c	-	PyExc_BrokenPipeError	variable	static PyTypeObject PyExc_BrokenPipeError
+Objects/exceptions.c	-	_PyExc_BufferError	variable	static PyTypeObject _PyExc_BufferError
+Objects/exceptions.c	-	PyExc_BufferError	variable	static PyTypeObject PyExc_BufferError
+Objects/exceptions.c	-	_PyExc_BytesWarning	variable	static PyTypeObject _PyExc_BytesWarning
+Objects/exceptions.c	-	PyExc_BytesWarning	variable	static PyTypeObject PyExc_BytesWarning
+Objects/exceptions.c	-	_PyExc_ChildProcessError	variable	static PyTypeObject _PyExc_ChildProcessError
+Objects/exceptions.c	-	PyExc_ChildProcessError	variable	static PyTypeObject PyExc_ChildProcessError
+Objects/exceptions.c	-	_PyExc_ConnectionAbortedError	variable	static PyTypeObject _PyExc_ConnectionAbortedError
+Objects/exceptions.c	-	PyExc_ConnectionAbortedError	variable	static PyTypeObject PyExc_ConnectionAbortedError
+Objects/exceptions.c	-	_PyExc_ConnectionError	variable	static PyTypeObject _PyExc_ConnectionError
+Objects/exceptions.c	-	PyExc_ConnectionError	variable	static PyTypeObject PyExc_ConnectionError
+Objects/exceptions.c	-	_PyExc_ConnectionRefusedError	variable	static PyTypeObject _PyExc_ConnectionRefusedError
+Objects/exceptions.c	-	PyExc_ConnectionRefusedError	variable	static PyTypeObject PyExc_ConnectionRefusedError
+Objects/exceptions.c	-	_PyExc_ConnectionResetError	variable	static PyTypeObject _PyExc_ConnectionResetError
+Objects/exceptions.c	-	PyExc_ConnectionResetError	variable	static PyTypeObject PyExc_ConnectionResetError
+Objects/exceptions.c	-	_PyExc_DeprecationWarning	variable	static PyTypeObject _PyExc_DeprecationWarning
+Objects/exceptions.c	-	PyExc_DeprecationWarning	variable	static PyTypeObject PyExc_DeprecationWarning
+Objects/exceptions.c	-	PyExc_EnvironmentError	variable	static PyTypeObject PyExc_EnvironmentError
+Objects/exceptions.c	-	_PyExc_EOFError	variable	static PyTypeObject _PyExc_EOFError
+Objects/exceptions.c	-	PyExc_EOFError	variable	static PyTypeObject PyExc_EOFError
+Objects/exceptions.c	-	_PyExc_Exception	variable	static PyTypeObject _PyExc_Exception
+Objects/exceptions.c	-	PyExc_Exception	variable	static PyTypeObject PyExc_Exception
+Objects/exceptions.c	-	_PyExc_FileExistsError	variable	static PyTypeObject _PyExc_FileExistsError
+Objects/exceptions.c	-	PyExc_FileExistsError	variable	static PyTypeObject PyExc_FileExistsError
+Objects/exceptions.c	-	_PyExc_FileNotFoundError	variable	static PyTypeObject _PyExc_FileNotFoundError
+Objects/exceptions.c	-	PyExc_FileNotFoundError	variable	static PyTypeObject PyExc_FileNotFoundError
+Objects/exceptions.c	-	_PyExc_FloatingPointError	variable	static PyTypeObject _PyExc_FloatingPointError
+Objects/exceptions.c	-	PyExc_FloatingPointError	variable	static PyTypeObject PyExc_FloatingPointError
+Objects/exceptions.c	-	_PyExc_FutureWarning	variable	static PyTypeObject _PyExc_FutureWarning
+Objects/exceptions.c	-	PyExc_FutureWarning	variable	static PyTypeObject PyExc_FutureWarning
+Objects/exceptions.c	-	_PyExc_GeneratorExit	variable	static PyTypeObject _PyExc_GeneratorExit
+Objects/exceptions.c	-	PyExc_GeneratorExit	variable	static PyTypeObject PyExc_GeneratorExit
+Objects/exceptions.c	-	_PyExc_ImportError	variable	static PyTypeObject _PyExc_ImportError
+Objects/exceptions.c	-	PyExc_ImportError	variable	static PyTypeObject PyExc_ImportError
+Objects/exceptions.c	-	_PyExc_ImportWarning	variable	static PyTypeObject _PyExc_ImportWarning
+Objects/exceptions.c	-	PyExc_ImportWarning	variable	static PyTypeObject PyExc_ImportWarning
+Objects/exceptions.c	-	_PyExc_IndentationError	variable	static PyTypeObject _PyExc_IndentationError
+Objects/exceptions.c	-	PyExc_IndentationError	variable	static PyTypeObject PyExc_IndentationError
+Objects/exceptions.c	-	_PyExc_IndexError	variable	static PyTypeObject _PyExc_IndexError
+Objects/exceptions.c	-	PyExc_IndexError	variable	static PyTypeObject PyExc_IndexError
+Objects/exceptions.c	-	_PyExc_InterruptedError	variable	static PyTypeObject _PyExc_InterruptedError
+Objects/exceptions.c	-	PyExc_InterruptedError	variable	static PyTypeObject PyExc_InterruptedError
+Objects/exceptions.c	-	PyExc_IOError	variable	static PyTypeObject PyExc_IOError
+Objects/exceptions.c	-	_PyExc_IsADirectoryError	variable	static PyTypeObject _PyExc_IsADirectoryError
+Objects/exceptions.c	-	PyExc_IsADirectoryError	variable	static PyTypeObject PyExc_IsADirectoryError
+Objects/exceptions.c	-	_PyExc_KeyboardInterrupt	variable	static PyTypeObject _PyExc_KeyboardInterrupt
+Objects/exceptions.c	-	PyExc_KeyboardInterrupt	variable	static PyTypeObject PyExc_KeyboardInterrupt
+Objects/exceptions.c	-	_PyExc_KeyError	variable	static PyTypeObject _PyExc_KeyError
+Objects/exceptions.c	-	PyExc_KeyError	variable	static PyTypeObject PyExc_KeyError
+Objects/exceptions.c	-	_PyExc_LookupError	variable	static PyTypeObject _PyExc_LookupError
+Objects/exceptions.c	-	PyExc_LookupError	variable	static PyTypeObject PyExc_LookupError
+Objects/exceptions.c	-	_PyExc_MemoryError	variable	static PyTypeObject _PyExc_MemoryError
+Objects/exceptions.c	-	PyExc_MemoryError	variable	static PyTypeObject PyExc_MemoryError
+Objects/exceptions.c	-	_PyExc_ModuleNotFoundError	variable	static PyTypeObject _PyExc_ModuleNotFoundError
+Objects/exceptions.c	-	PyExc_ModuleNotFoundError	variable	static PyTypeObject PyExc_ModuleNotFoundError
+Objects/exceptions.c	-	_PyExc_NameError	variable	static PyTypeObject _PyExc_NameError
+Objects/exceptions.c	-	PyExc_NameError	variable	static PyTypeObject PyExc_NameError
+Objects/exceptions.c	-	_PyExc_NotADirectoryError	variable	static PyTypeObject _PyExc_NotADirectoryError
+Objects/exceptions.c	-	PyExc_NotADirectoryError	variable	static PyTypeObject PyExc_NotADirectoryError
+Objects/exceptions.c	-	_PyExc_NotImplementedError	variable	static PyTypeObject _PyExc_NotImplementedError
+Objects/exceptions.c	-	PyExc_NotImplementedError	variable	static PyTypeObject PyExc_NotImplementedError
+Objects/exceptions.c	-	_PyExc_OSError	variable	static PyTypeObject _PyExc_OSError
+Objects/exceptions.c	-	PyExc_OSError	variable	static PyTypeObject PyExc_OSError
+Objects/exceptions.c	-	_PyExc_OverflowError	variable	static PyTypeObject _PyExc_OverflowError
+Objects/exceptions.c	-	PyExc_OverflowError	variable	static PyTypeObject PyExc_OverflowError
+Objects/exceptions.c	-	_PyExc_PendingDeprecationWarning	variable	static PyTypeObject _PyExc_PendingDeprecationWarning
+Objects/exceptions.c	-	PyExc_PendingDeprecationWarning	variable	static PyTypeObject PyExc_PendingDeprecationWarning
+Objects/exceptions.c	-	_PyExc_PermissionError	variable	static PyTypeObject _PyExc_PermissionError
+Objects/exceptions.c	-	PyExc_PermissionError	variable	static PyTypeObject PyExc_PermissionError
+Objects/exceptions.c	-	_PyExc_ProcessLookupError	variable	static PyTypeObject _PyExc_ProcessLookupError
+Objects/exceptions.c	-	PyExc_ProcessLookupError	variable	static PyTypeObject PyExc_ProcessLookupError
+Objects/exceptions.c	-	_PyExc_RecursionError	variable	static PyTypeObject _PyExc_RecursionError
+Objects/exceptions.c	-	PyExc_RecursionError	variable	static PyTypeObject PyExc_RecursionError
+Objects/exceptions.c	-	_PyExc_ReferenceError	variable	static PyTypeObject _PyExc_ReferenceError
+Objects/exceptions.c	-	PyExc_ReferenceError	variable	static PyTypeObject PyExc_ReferenceError
+Objects/exceptions.c	-	_PyExc_ResourceWarning	variable	static PyTypeObject _PyExc_ResourceWarning
+Objects/exceptions.c	-	PyExc_ResourceWarning	variable	static PyTypeObject PyExc_ResourceWarning
+Objects/exceptions.c	-	_PyExc_RuntimeError	variable	static PyTypeObject _PyExc_RuntimeError
+Objects/exceptions.c	-	PyExc_RuntimeError	variable	static PyTypeObject PyExc_RuntimeError
+Objects/exceptions.c	-	_PyExc_RuntimeWarning	variable	static PyTypeObject _PyExc_RuntimeWarning
+Objects/exceptions.c	-	PyExc_RuntimeWarning	variable	static PyTypeObject PyExc_RuntimeWarning
+Objects/exceptions.c	-	_PyExc_StopAsyncIteration	variable	static PyTypeObject _PyExc_StopAsyncIteration
+Objects/exceptions.c	-	PyExc_StopAsyncIteration	variable	static PyTypeObject PyExc_StopAsyncIteration
+Objects/exceptions.c	-	_PyExc_StopIteration	variable	static PyTypeObject _PyExc_StopIteration
+Objects/exceptions.c	-	PyExc_StopIteration	variable	static PyTypeObject PyExc_StopIteration
+Objects/exceptions.c	-	_PyExc_SyntaxError	variable	static PyTypeObject _PyExc_SyntaxError
+Objects/exceptions.c	-	PyExc_SyntaxError	variable	static PyTypeObject PyExc_SyntaxError
+Objects/exceptions.c	-	_PyExc_SyntaxWarning	variable	static PyTypeObject _PyExc_SyntaxWarning
+Objects/exceptions.c	-	PyExc_SyntaxWarning	variable	static PyTypeObject PyExc_SyntaxWarning
+Objects/exceptions.c	-	_PyExc_SystemError	variable	static PyTypeObject _PyExc_SystemError
+Objects/exceptions.c	-	PyExc_SystemError	variable	static PyTypeObject PyExc_SystemError
+Objects/exceptions.c	-	_PyExc_SystemExit	variable	static PyTypeObject _PyExc_SystemExit
+Objects/exceptions.c	-	PyExc_SystemExit	variable	static PyTypeObject PyExc_SystemExit
+Objects/exceptions.c	-	_PyExc_TabError	variable	static PyTypeObject _PyExc_TabError
+Objects/exceptions.c	-	PyExc_TabError	variable	static PyTypeObject PyExc_TabError
+Objects/exceptions.c	-	_PyExc_TargetScopeError	variable	static PyTypeObject _PyExc_TargetScopeError
+Objects/exceptions.c	-	PyExc_TargetScopeError	variable	static PyTypeObject PyExc_TargetScopeError
+Objects/exceptions.c	-	_PyExc_TimeoutError	variable	static PyTypeObject _PyExc_TimeoutError
+Objects/exceptions.c	-	PyExc_TimeoutError	variable	static PyTypeObject PyExc_TimeoutError
+Objects/exceptions.c	-	_PyExc_TypeError	variable	static PyTypeObject _PyExc_TypeError
+Objects/exceptions.c	-	PyExc_TypeError	variable	static PyTypeObject PyExc_TypeError
+Objects/exceptions.c	-	_PyExc_UnboundLocalError	variable	static PyTypeObject _PyExc_UnboundLocalError
+Objects/exceptions.c	-	PyExc_UnboundLocalError	variable	static PyTypeObject PyExc_UnboundLocalError
+Objects/exceptions.c	-	_PyExc_UnicodeDecodeError	variable	static PyTypeObject _PyExc_UnicodeDecodeError
+Objects/exceptions.c	-	PyExc_UnicodeDecodeError	variable	static PyTypeObject PyExc_UnicodeDecodeError
+Objects/exceptions.c	-	_PyExc_UnicodeEncodeError	variable	static PyTypeObject _PyExc_UnicodeEncodeError
+Objects/exceptions.c	-	PyExc_UnicodeEncodeError	variable	static PyTypeObject PyExc_UnicodeEncodeError
+Objects/exceptions.c	-	_PyExc_UnicodeError	variable	static PyTypeObject _PyExc_UnicodeError
+Objects/exceptions.c	-	PyExc_UnicodeError	variable	static PyTypeObject PyExc_UnicodeError
+Objects/exceptions.c	-	_PyExc_UnicodeTranslateError	variable	static PyTypeObject _PyExc_UnicodeTranslateError
+Objects/exceptions.c	-	PyExc_UnicodeTranslateError	variable	static PyTypeObject PyExc_UnicodeTranslateError
+Objects/exceptions.c	-	_PyExc_UnicodeWarning	variable	static PyTypeObject _PyExc_UnicodeWarning
+Objects/exceptions.c	-	PyExc_UnicodeWarning	variable	static PyTypeObject PyExc_UnicodeWarning
+Objects/exceptions.c	-	_PyExc_UserWarning	variable	static PyTypeObject _PyExc_UserWarning
+Objects/exceptions.c	-	PyExc_UserWarning	variable	static PyTypeObject PyExc_UserWarning
+Objects/exceptions.c	-	_PyExc_ValueError	variable	static PyTypeObject _PyExc_ValueError
+Objects/exceptions.c	-	PyExc_ValueError	variable	static PyTypeObject PyExc_ValueError
+Objects/exceptions.c	-	_PyExc_Warning	variable	static PyTypeObject _PyExc_Warning
+Objects/exceptions.c	-	PyExc_Warning	variable	static PyTypeObject PyExc_Warning
+Objects/exceptions.c	-	_PyExc_ZeroDivisionError	variable	static PyTypeObject _PyExc_ZeroDivisionError
+Objects/exceptions.c	-	PyExc_ZeroDivisionError	variable	static PyTypeObject PyExc_ZeroDivisionError
+Objects/boolobject.c	-	_Py_FalseStruct	variable	static struct _longobject _Py_FalseStruct
+Objects/stringlib/unicode_format.h	-	PyFieldNameIter_Type	variable	static PyTypeObject PyFieldNameIter_Type
+Modules/_io/fileio.c	-	PyFileIO_Type	variable	PyTypeObject PyFileIO_Type
+Python/preconfig.c	-	Py_FileSystemDefaultEncodeErrors	variable	const char *Py_FileSystemDefaultEncodeErrors
+Python/preconfig.c	-	Py_FileSystemDefaultEncoding	variable	const char * Py_FileSystemDefaultEncoding
+Python/bltinmodule.c	-	PyFilter_Type	variable	PyTypeObject PyFilter_Type
+Objects/floatobject.c	-	PyFloat_Type	variable	PyTypeObject PyFloat_Type
+Objects/stringlib/unicode_format.h	-	PyFormatterIter_Type	variable	static PyTypeObject PyFormatterIter_Type
+Objects/frameobject.c	-	PyFrame_Type	variable	PyTypeObject PyFrame_Type
+Python/initconfig.c	-	Py_FrozenFlag	variable	int Py_FrozenFlag
+Objects/setobject.c	-	PyFrozenSet_Type	variable	PyTypeObject PyFrozenSet_Type
+Objects/funcobject.c	-	PyFunction_Type	variable	PyTypeObject PyFunction_Type
+Objects/genobject.c	-	PyGen_Type	variable	PyTypeObject PyGen_Type
+Objects/descrobject.c	-	PyGetSetDescr_Type	variable	PyTypeObject PyGetSetDescr_Type
+Python/hamt.c	-	_PyHamt_ArrayNode_Type	variable	PyTypeObject _PyHamt_ArrayNode_Type
+Python/hamt.c	-	PyHamt_as_mapping	variable	static PyMappingMethods PyHamt_as_mapping
+Python/hamt.c	-	PyHamt_as_sequence	variable	static PySequenceMethods PyHamt_as_sequence
+Python/hamt.c	-	_PyHamt_BitmapNode_Type	variable	PyTypeObject _PyHamt_BitmapNode_Type
+Python/hamt.c	-	_PyHamt_CollisionNode_Type	variable	PyTypeObject _PyHamt_CollisionNode_Type
+Python/hamt.c	-	_PyHamtItems_Type	variable	PyTypeObject _PyHamtItems_Type
+Python/hamt.c	-	PyHamtIterator_as_mapping	variable	static PyMappingMethods PyHamtIterator_as_mapping
+Python/hamt.c	-	_PyHamtKeys_Type	variable	PyTypeObject _PyHamtKeys_Type
+Python/hamt.c	-	PyHamt_methods	variable	static PyMethodDef PyHamt_methods
+Python/hamt.c	-	_PyHamt_Type	variable	PyTypeObject _PyHamt_Type
+Python/hamt.c	-	_PyHamtValues_Type	variable	PyTypeObject _PyHamtValues_Type
+Python/preconfig.c	-	_Py_HasFileSystemDefaultEncodeErrors	variable	const(int) _Py_HasFileSystemDefaultEncodeErrors
+Python/preconfig.c	-	Py_HasFileSystemDefaultEncoding	variable	const(int) Py_HasFileSystemDefaultEncoding
+Python/pyhash.c	-	PyHash_Func	variable	static PyHash_FuncDef PyHash_Func
+Python/initconfig.c	-	Py_HashRandomizationFlag	variable	int Py_HashRandomizationFlag
+Python/pyhash.c	-	_Py_HashSecret	variable	_Py_HashSecret_t _Py_HashSecret
+Python/bootstrap_hash.c	-	_Py_HashSecret_Initialized	variable	static int _Py_HashSecret_Initialized
+Python/codecs.c	-	Py_hexdigits	variable	const char * Py_hexdigits
+Python/sysmodule.c	-	PyId__	variable	_Py_IDENTIFIER(_)
+Modules/_abc.c	-	PyId__abc_impl	variable	_Py_IDENTIFIER(_abc_impl)
+Objects/typeobject.c	-	PyId___abstractmethods__	variable	_Py_IDENTIFIER(__abstractmethods__)
+Modules/_abc.c	-	PyId___abstractmethods__	variable	_Py_IDENTIFIER(__abstractmethods__)
+Python/ceval.c	_PyEval_EvalFrameDefault	PyId___aenter__	variable	_Py_IDENTIFIER(__aenter__)
+Python/ceval.c	_PyEval_EvalFrameDefault	PyId___aexit__	variable	_Py_IDENTIFIER(__aexit__)
+Objects/typeobject.c	slot_am_aiter	PyId___aiter__	variable	_Py_IDENTIFIER(__aiter__)
+Python/ceval.c	import_all_from	PyId___all__	variable	_Py_IDENTIFIER(__all__)
+Objects/typeobject.c	slot_am_anext	PyId___anext__	variable	_Py_IDENTIFIER(__anext__)
+Python/Python-ast.c	-	PyId_annotation	variable	_Py_IDENTIFIER(annotation)
+Python/ceval.c	_PyEval_EvalFrameDefault	PyId___annotations__	variable	_Py_IDENTIFIER(__annotations__)
+Python/Python-ast.c	-	PyId_arg	variable	_Py_IDENTIFIER(arg)
+Python/Python-ast.c	-	PyId_args	variable	_Py_IDENTIFIER(args)
+Python/Python-ast.c	-	PyId_argtypes	variable	_Py_IDENTIFIER(argtypes)
+Python/Python-ast.c	-	PyId_asname	variable	_Py_IDENTIFIER(asname)
+Python/Python-ast.c	make_type	PyId__ast	variable	_Py_IDENTIFIER(_ast)
+Python/Python-ast.c	-	PyId_attr	variable	_Py_IDENTIFIER(attr)
+Python/Python-ast.c	-	PyId__attributes	variable	_Py_IDENTIFIER(_attributes)
+Objects/typeobject.c	slot_am_await	PyId___await__	variable	_Py_IDENTIFIER(__await__)
+Python/Python-ast.c	-	PyId_bases	variable	_Py_IDENTIFIER(bases)
+Modules/_abc.c	-	PyId___bases__	variable	_Py_IDENTIFIER(__bases__)
+Objects/abstract.c	abstract_get_bases	PyId___bases__	variable	_Py_IDENTIFIER(__bases__)
+Objects/typeobject.c	merge_class_dict	PyId___bases__	variable	_Py_IDENTIFIER(__bases__)
+Objects/longobject.c	-	PyId_big	variable	_Py_IDENTIFIER(big)
+Modules/_io/_iomodule.c	_io_open_impl	PyId__blksize	variable	_Py_IDENTIFIER(_blksize)
+Python/Python-ast.c	-	PyId_body	variable	_Py_IDENTIFIER(body)
+Objects/typeobject.c	slot_nb_bool	PyId___bool__	variable	_Py_IDENTIFIER(__bool__)
+Python/sysmodule.c	-	PyId_buffer	variable	_Py_IDENTIFIER(buffer)
+Python/ceval.c	_PyEval_EvalFrameDefault	PyId___build_class__	variable	_Py_IDENTIFIER(__build_class__)
+Objects/typeobject.c	-	PyId_builtins	variable	_Py_IDENTIFIER(builtins)
+Python/errors.c	-	PyId_builtins	variable	_Py_IDENTIFIER(builtins)
+Python/pythonrun.c	-	PyId_builtins	variable	_Py_IDENTIFIER(builtins)
+Python/sysmodule.c	-	PyId_builtins	variable	_Py_IDENTIFIER(builtins)
+Objects/frameobject.c	-	PyId___builtins__	variable	_Py_IDENTIFIER(__builtins__)
+Python/bltinmodule.c	-	PyId___builtins__	variable	_Py_IDENTIFIER(__builtins__)
+Python/import.c	module_dict_for_exec	PyId___builtins__	variable	_Py_IDENTIFIER(__builtins__)
+Objects/object.c	-	PyId___bytes__	variable	_Py_IDENTIFIER(__bytes__)
+Objects/bytesobject.c	format_obj	PyId___bytes__	variable	_Py_IDENTIFIER(__bytes__)
+Objects/bytesobject.c	bytes_new	PyId___bytes__	variable	_Py_IDENTIFIER(__bytes__)
+Objects/weakrefobject.c	proxy_bytes	PyId___bytes__	variable	_Py_IDENTIFIER(__bytes__)
+Objects/typeobject.c	slot_tp_call	PyId___call__	variable	_Py_IDENTIFIER(__call__)
+Python/Python-ast.c	-	PyId_cause	variable	_Py_IDENTIFIER(cause)
+Objects/typeobject.c	-	PyId___class__	variable	_Py_IDENTIFIER(__class__)
+Modules/_abc.c	-	PyId___class__	variable	_Py_IDENTIFIER(__class__)
+Python/compile.c	compiler_enter_scope	PyId___class__	variable	_Py_IDENTIFIER(__class__)
+Objects/abstract.c	recursive_isinstance	PyId___class__	variable	_Py_IDENTIFIER(__class__)
+Objects/typeobject.c	type_new	PyId___classcell__	variable	_Py_IDENTIFIER(__classcell__)
+Objects/typeobject.c	-	PyId___class_getitem__	variable	_Py_IDENTIFIER(__class_getitem__)
+Objects/abstract.c	PyObject_GetItem	PyId___class_getitem__	variable	_Py_IDENTIFIER(__class_getitem__)
+Python/import.c	PyImport_Cleanup	PyId_clear	variable	_Py_IDENTIFIER(clear)
+Python/traceback.c	-	PyId_close	variable	_Py_IDENTIFIER(close)
+Modules/_io/bufferedio.c	-	PyId_close	variable	_Py_IDENTIFIER(close)
+Modules/_io/textio.c	-	PyId_close	variable	_Py_IDENTIFIER(close)
+Objects/genobject.c	gen_close_iter	PyId_close	variable	_Py_IDENTIFIER(close)
+Modules/_dbmmodule.c	dbm__exit__	PyId_close	variable	_Py_IDENTIFIER(close)
+Modules/_gdbmmodule.c	dbm__exit__	PyId_close	variable	_Py_IDENTIFIER(close)
+Python/pythonrun.c	_Py_HandleSystemExit	PyId_code	variable	_Py_IDENTIFIER(code)
+Python/Python-ast.c	-	PyId_col_offset	variable	_Py_IDENTIFIER(col_offset)
+Python/Python-ast.c	-	PyId_comparators	variable	_Py_IDENTIFIER(comparators)
+Objects/complexobject.c	try_complex_special_method	PyId___complex__	variable	_Py_IDENTIFIER(__complex__)
+Objects/typeobject.c	slot_sq_contains	PyId___contains__	variable	_Py_IDENTIFIER(__contains__)
+Python/Python-ast.c	-	PyId_context_expr	variable	_Py_IDENTIFIER(context_expr)
+Python/Python-ast.c	-	PyId_conversion	variable	_Py_IDENTIFIER(conversion)
+Modules/itertoolsmodule.c	itertools_tee_impl	PyId___copy__	variable	_Py_IDENTIFIER(__copy__)
+Objects/descrobject.c	mappingproxy_copy	PyId_copy	variable	_Py_IDENTIFIER(copy)
+Objects/typeobject.c	import_copyreg	PyId_copyreg	variable	_Py_IDENTIFIER(copyreg)
+Python/Python-ast.c	-	PyId_ctx	variable	_Py_IDENTIFIER(ctx)
+Modules/_io/bufferedio.c	-	PyId__dealloc_warn	variable	_Py_IDENTIFIER(_dealloc_warn)
+Modules/_io/textio.c	-	PyId__dealloc_warn	variable	_Py_IDENTIFIER(_dealloc_warn)
+Modules/_io/textio.c	-	PyId_decode	variable	_Py_IDENTIFIER(decode)
+Python/Python-ast.c	-	PyId_decorator_list	variable	_Py_IDENTIFIER(decorator_list)
+Python/_warnings.c	get_default_action	PyId_defaultaction	variable	_Py_IDENTIFIER(defaultaction)
+Python/Python-ast.c	-	PyId_defaults	variable	_Py_IDENTIFIER(defaults)
+Objects/typeobject.c	slot_tp_finalize	PyId___del__	variable	_Py_IDENTIFIER(__del__)
+Objects/typeobject.c	slot_tp_setattro	PyId___delattr__	variable	_Py_IDENTIFIER(__delattr__)
+Objects/typeobject.c	slot_tp_descr_set	PyId___delete__	variable	_Py_IDENTIFIER(__delete__)
+Objects/typeobject.c	-	PyId___delitem__	variable	_Py_IDENTIFIER(__delitem__)
+Objects/typeobject.c	-	PyId___dict__	variable	_Py_IDENTIFIER(__dict__)
+Modules/_abc.c	-	PyId___dict__	variable	_Py_IDENTIFIER(__dict__)
+Python/bltinmodule.c	-	PyId___dict__	variable	_Py_IDENTIFIER(__dict__)
+Python/Python-ast.c	ast_type_reduce	PyId___dict__	variable	_Py_IDENTIFIER(__dict__)
+Python/ceval.c	import_all_from	PyId___dict__	variable	_Py_IDENTIFIER(__dict__)
+Objects/bytearrayobject.c	_common_reduce	PyId___dict__	variable	_Py_IDENTIFIER(__dict__)
+Objects/moduleobject.c	module_dir	PyId___dict__	variable	_Py_IDENTIFIER(__dict__)
+Objects/odictobject.c	odict_reduce	PyId___dict__	variable	_Py_IDENTIFIER(__dict__)
+Objects/setobject.c	set_reduce	PyId___dict__	variable	_Py_IDENTIFIER(__dict__)
+Modules/_collectionsmodule.c	deque_reduce	PyId___dict__	variable	_Py_IDENTIFIER(__dict__)
+Objects/dictobject.c	dictviews_sub	PyId_difference_update	variable	_Py_IDENTIFIER(difference_update)
+Python/Python-ast.c	-	PyId_dims	variable	_Py_IDENTIFIER(dims)
+Objects/object.c	-	PyId___dir__	variable	_Py_IDENTIFIER(__dir__)
+Objects/moduleobject.c	module_dir	PyId___dir__	variable	_Py_IDENTIFIER(__dir__)
+Python/ceval.c	_PyEval_EvalFrameDefault	PyId_displayhook	variable	_Py_IDENTIFIER(displayhook)
+Objects/typeobject.c	-	PyId___doc__	variable	_Py_IDENTIFIER(__doc__)
+Objects/descrobject.c	property_init_impl	PyId___doc__	variable	_Py_IDENTIFIER(__doc__)
+Objects/moduleobject.c	module_init_dict	PyId___doc__	variable	_Py_IDENTIFIER(__doc__)
+Objects/moduleobject.c	PyModule_SetDocString	PyId___doc__	variable	_Py_IDENTIFIER(__doc__)
+Python/Python-ast.c	-	PyId_elt	variable	_Py_IDENTIFIER(elt)
+Python/Python-ast.c	-	PyId_elts	variable	_Py_IDENTIFIER(elts)
+Modules/faulthandler.c	-	PyId_enable	variable	_Py_IDENTIFIER(enable)
+Python/sysmodule.c	-	PyId_encoding	variable	_Py_IDENTIFIER(encoding)
+Python/bltinmodule.c	-	PyId_encoding	variable	_Py_IDENTIFIER(encoding)
+Python/pythonrun.c	PyRun_InteractiveOneObjectEx	PyId_encoding	variable	_Py_IDENTIFIER(encoding)
+Python/Python-ast.c	-	PyId_end_col_offset	variable	_Py_IDENTIFIER(end_col_offset)
+Python/Python-ast.c	-	PyId_end_lineno	variable	_Py_IDENTIFIER(end_lineno)
+Python/ceval.c	_PyEval_EvalFrameDefault	PyId___enter__	variable	_Py_IDENTIFIER(__enter__)
+Objects/typeobject.c	overrides_hash	PyId___eq__	variable	_Py_IDENTIFIER(__eq__)
+Python/bltinmodule.c	-	PyId_errors	variable	_Py_IDENTIFIER(errors)
+Python/Python-ast.c	-	PyId_exc	variable	_Py_IDENTIFIER(exc)
+Python/pythonrun.c	-	PyId_excepthook	variable	_Py_IDENTIFIER(excepthook)
+Python/ceval.c	_PyEval_EvalFrameDefault	PyId___exit__	variable	_Py_IDENTIFIER(__exit__)
+Modules/_pickle.c	do_append	PyId_extend	variable	_Py_IDENTIFIER(extend)
+Python/Python-ast.c	-	PyId__fields	variable	_Py_IDENTIFIER(_fields)
+Objects/moduleobject.c	PyModule_GetFilenameObject	PyId___file__	variable	_Py_IDENTIFIER(__file__)
+Python/errors.c	PyErr_SyntaxLocationObject	PyId_filename	variable	_Py_IDENTIFIER(filename)
+Python/pythonrun.c	parse_syntax_error	PyId_filename	variable	_Py_IDENTIFIER(filename)
+Modules/_io/textio.c	-	PyId_fileno	variable	_Py_IDENTIFIER(fileno)
+Modules/faulthandler.c	-	PyId_fileno	variable	_Py_IDENTIFIER(fileno)
+Python/bltinmodule.c	-	PyId_fileno	variable	_Py_IDENTIFIER(fileno)
+Objects/fileobject.c	PyObject_AsFileDescriptor	PyId_fileno	variable	_Py_IDENTIFIER(fileno)
+Modules/itertoolsmodule.c	zip_longest_new	PyId_fillvalue	variable	_Py_IDENTIFIER(fillvalue)
+Python/_warnings.c	get_filter	PyId_filters	variable	_Py_IDENTIFIER(filters)
+Python/Python-ast.c	-	PyId_finalbody	variable	_Py_IDENTIFIER(finalbody)
+Modules/_io/iobase.c	iobase_finalize	PyId__finalizing	variable	_Py_IDENTIFIER(_finalizing)
+Python/import.c	import_find_and_load	PyId__find_and_load	variable	_Py_IDENTIFIER(_find_and_load)
+Python/import.c	PyImport_ExecCodeModuleObject	PyId__fix_up_module	variable	_Py_IDENTIFIER(_fix_up_module)
+Python/errors.c	-	PyId_flush	variable	_Py_IDENTIFIER(flush)
+Python/pylifecycle.c	-	PyId_flush	variable	_Py_IDENTIFIER(flush)
+Python/pythonrun.c	-	PyId_flush	variable	_Py_IDENTIFIER(flush)
+Modules/_threadmodule.c	-	PyId_flush	variable	_Py_IDENTIFIER(flush)
+Modules/_io/bufferedio.c	-	PyId_flush	variable	_Py_IDENTIFIER(flush)
+Modules/_io/textio.c	-	PyId_flush	variable	_Py_IDENTIFIER(flush)
+Modules/faulthandler.c	-	PyId_flush	variable	_Py_IDENTIFIER(flush)
+Python/bltinmodule.c	-	PyId_flush	variable	_Py_IDENTIFIER(flush)
+Objects/abstract.c	PyObject_Format	PyId___format__	variable	_Py_IDENTIFIER(__format__)
+Python/Python-ast.c	-	PyId_format_spec	variable	_Py_IDENTIFIER(format_spec)
+Modules/posixmodule.c	path_converter	PyId___fspath__	variable	_Py_IDENTIFIER(__fspath__)
+Modules/posixmodule.c	PyOS_FSPath	PyId___fspath__	variable	_Py_IDENTIFIER(__fspath__)
+Python/Python-ast.c	-	PyId_func	variable	_Py_IDENTIFIER(func)
+Python/Python-ast.c	-	PyId_generators	variable	_Py_IDENTIFIER(generators)
+Objects/descrobject.c	mappingproxy_get	PyId_get	variable	_Py_IDENTIFIER(get)
+Modules/_collectionsmodule.c	_count_elements	PyId_get	variable	_Py_IDENTIFIER(get)
+Objects/typeobject.c	slot_tp_descr_get	PyId___get__	variable	_Py_IDENTIFIER(__get__)
+Objects/classobject.c	method_reduce	PyId_getattr	variable	_Py_IDENTIFIER(getattr)
+Objects/descrobject.c	descr_reduce	PyId_getattr	variable	_Py_IDENTIFIER(getattr)
+Objects/descrobject.c	wrapper_reduce	PyId_getattr	variable	_Py_IDENTIFIER(getattr)
+Objects/moduleobject.c	module_getattro	PyId___getattr__	variable	_Py_IDENTIFIER(__getattr__)
+Objects/methodobject.c	meth_reduce	PyId_getattr	variable	_Py_IDENTIFIER(getattr)
+Objects/typeobject.c	slot_tp_getattr_hook	PyId___getattr__	variable	_Py_IDENTIFIER(__getattr__)
+Objects/typeobject.c	-	PyId___getattribute__	variable	_Py_IDENTIFIER(__getattribute__)
+Objects/typeobject.c	-	PyId___getitem__	variable	_Py_IDENTIFIER(__getitem__)
+Objects/typeobject.c	_PyObject_GetNewArguments	PyId___getnewargs__	variable	_Py_IDENTIFIER(__getnewargs__)
+Objects/typeobject.c	_PyObject_GetNewArguments	PyId___getnewargs_ex__	variable	_Py_IDENTIFIER(__getnewargs_ex__)
+Modules/_io/textio.c	-	PyId_getpreferredencoding	variable	_Py_IDENTIFIER(getpreferredencoding)
+Python/_warnings.c	get_source_line	PyId_get_source	variable	_Py_IDENTIFIER(get_source)
+Python/import.c	PyImport_ExecCodeModuleWithPathnames	PyId__get_sourcefile	variable	_Py_IDENTIFIER(_get_sourcefile)
+Objects/typeobject.c	_PyObject_GetState	PyId___getstate__	variable	_Py_IDENTIFIER(__getstate__)
+Python/import.c	PyImport_ImportModuleLevelObject	PyId__handle_fromlist	variable	_Py_IDENTIFIER(_handle_fromlist)
+Python/Python-ast.c	-	PyId_handlers	variable	_Py_IDENTIFIER(handlers)
+Objects/typeobject.c	-	PyId___hash__	variable	_Py_IDENTIFIER(__hash__)
+Python/Python-ast.c	-	PyId_id	variable	_Py_IDENTIFIER(id)
+Python/Python-ast.c	-	PyId_ifs	variable	_Py_IDENTIFIER(ifs)
+Python/import.c	PyImport_ReloadModule	PyId_imp	variable	_Py_IDENTIFIER(imp)
+Python/ceval.c	import_name	PyId___import__	variable	_Py_IDENTIFIER(__import__)
+Objects/typeobject.c	slot_nb_index	PyId___index__	variable	_Py_IDENTIFIER(__index__)
+Objects/typeobject.c	slot_tp_init	PyId___init__	variable	_Py_IDENTIFIER(__init__)
+Objects/moduleobject.c	_PyModuleSpec_IsInitializing	PyId__initializing	variable	_Py_IDENTIFIER(_initializing)
+Objects/typeobject.c	-	PyId___init_subclass__	variable	_Py_IDENTIFIER(__init_subclass__)
+Objects/abstract.c	PyObject_IsInstance	PyId___instancecheck__	variable	_Py_IDENTIFIER(__instancecheck__)
+Objects/dictobject.c	_PyDictView_Intersect	PyId_intersection_update	variable	_Py_IDENTIFIER(intersection_update)
+Modules/_io/iobase.c	-	PyId___IOBase_closed	variable	_Py_IDENTIFIER(__IOBase_closed)
+Objects/typeobject.c	slot_nb_inplace_power	PyId___ipow__	variable	_Py_IDENTIFIER(__ipow__)
+Objects/object.c	-	PyId___isabstractmethod__	variable	_Py_IDENTIFIER(__isabstractmethod__)
+Python/Python-ast.c	-	PyId_is_async	variable	_Py_IDENTIFIER(is_async)
+Modules/_io/bufferedio.c	-	PyId_isatty	variable	_Py_IDENTIFIER(isatty)
+Modules/_io/textio.c	-	PyId_isatty	variable	_Py_IDENTIFIER(isatty)
+Python/pylifecycle.c	create_stdio	PyId_isatty	variable	_Py_IDENTIFIER(isatty)
+Modules/_io/_iomodule.c	_io_open_impl	PyId_isatty	variable	_Py_IDENTIFIER(isatty)
+Python/codecs.c	_PyCodec_LookupTextEncoding	PyId__is_text_encoding	variable	_Py_IDENTIFIER(_is_text_encoding)
+Python/Python-ast.c	-	PyId_items	variable	_Py_IDENTIFIER(items)
+Objects/abstract.c	PyMapping_Items	PyId_items	variable	_Py_IDENTIFIER(items)
+Objects/descrobject.c	mappingproxy_items	PyId_items	variable	_Py_IDENTIFIER(items)
+Objects/odictobject.c	odict_reduce	PyId_items	variable	_Py_IDENTIFIER(items)
+Objects/odictobject.c	odict_repr	PyId_items	variable	_Py_IDENTIFIER(items)
+Objects/odictobject.c	mutablemapping_update	PyId_items	variable	_Py_IDENTIFIER(items)
+Objects/typeobject.c	_PyObject_GetItemsIter	PyId_items	variable	_Py_IDENTIFIER(items)
+Modules/_collectionsmodule.c	defdict_reduce	PyId_items	variable	_Py_IDENTIFIER(items)
+Python/Python-ast.c	-	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/bytearrayobject.c	bytearrayiter_reduce	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/bytesobject.c	striter_reduce	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/dictobject.c	dictiter_reduce	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/iterobject.c	iter_reduce	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/iterobject.c	calliter_reduce	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/listobject.c	listiter_reduce_general	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/odictobject.c	odictiter_reduce	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/rangeobject.c	rangeiter_reduce	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/rangeobject.c	longrangeiter_reduce	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/setobject.c	setiter_reduce	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/tupleobject.c	tupleiter_reduce	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/unicodeobject.c	unicodeiter_reduce	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Objects/typeobject.c	slot_tp_iter	PyId___iter__	variable	_Py_IDENTIFIER(__iter__)
+Modules/arraymodule.c	array_arrayiterator___reduce___impl	PyId_iter	variable	_Py_IDENTIFIER(iter)
+Python/Python-ast.c	-	PyId_key	variable	_Py_IDENTIFIER(key)
+Python/Python-ast.c	-	PyId_keys	variable	_Py_IDENTIFIER(keys)
+Objects/abstract.c	PyMapping_Keys	PyId_keys	variable	_Py_IDENTIFIER(keys)
+Objects/descrobject.c	mappingproxy_keys	PyId_keys	variable	_Py_IDENTIFIER(keys)
+Objects/dictobject.c	dict_update_common	PyId_keys	variable	_Py_IDENTIFIER(keys)
+Objects/odictobject.c	mutablemapping_update	PyId_keys	variable	_Py_IDENTIFIER(keys)
+Python/Python-ast.c	-	PyId_keywords	variable	_Py_IDENTIFIER(keywords)
+Python/Python-ast.c	-	PyId_kind	variable	_Py_IDENTIFIER(kind)
+Python/Python-ast.c	-	PyId_kwarg	variable	_Py_IDENTIFIER(kwarg)
+Python/Python-ast.c	-	PyId_kw_defaults	variable	_Py_IDENTIFIER(kw_defaults)
+Python/Python-ast.c	-	PyId_kwonlyargs	variable	_Py_IDENTIFIER(kwonlyargs)
+Python/pythonrun.c	-	PyId_last_traceback	variable	_Py_IDENTIFIER(last_traceback)
+Python/pythonrun.c	-	PyId_last_type	variable	_Py_IDENTIFIER(last_type)
+Python/pythonrun.c	-	PyId_last_value	variable	_Py_IDENTIFIER(last_value)
+Python/Python-ast.c	-	PyId_left	variable	_Py_IDENTIFIER(left)
+Objects/typeobject.c	-	PyId___len__	variable	_Py_IDENTIFIER(__len__)
+Objects/abstract.c	PyObject_LengthHint	PyId___length_hint__	variable	_Py_IDENTIFIER(__length_hint__)
+Python/Python-ast.c	-	PyId_level	variable	_Py_IDENTIFIER(level)
+Python/Python-ast.c	-	PyId_lineno	variable	_Py_IDENTIFIER(lineno)
+Python/errors.c	PyErr_SyntaxLocationObject	PyId_lineno	variable	_Py_IDENTIFIER(lineno)
+Python/pythonrun.c	parse_syntax_error	PyId_lineno	variable	_Py_IDENTIFIER(lineno)
+Objects/longobject.c	-	PyId_little	variable	_Py_IDENTIFIER(little)
+Python/_warnings.c	get_source_line	PyId___loader__	variable	_Py_IDENTIFIER(__loader__)
+Objects/moduleobject.c	module_init_dict	PyId___loader__	variable	_Py_IDENTIFIER(__loader__)
+Python/import.c	PyImport_ImportModuleLevelObject	PyId__lock_unlock_module	variable	_Py_IDENTIFIER(_lock_unlock_module)
+Python/Python-ast.c	-	PyId_lower	variable	_Py_IDENTIFIER(lower)
+Python/ceval.c	_PyEval_EvalFrameDefault	PyId___ltrace__	variable	_Py_IDENTIFIER(__ltrace__)
+Python/pythonrun.c	PyRun_InteractiveOneObjectEx	PyId___main__	variable	_Py_IDENTIFIER(__main__)
+Python/_warnings.c	check_matched	PyId_match	variable	_Py_IDENTIFIER(match)
+Python/bltinmodule.c	-	PyId_metaclass	variable	_Py_IDENTIFIER(metaclass)
+Objects/dictobject.c	dict_subscript	PyId___missing__	variable	_Py_IDENTIFIER(__missing__)
+Modules/_io/bufferedio.c	-	PyId_mode	variable	_Py_IDENTIFIER(mode)
+Modules/_io/textio.c	-	PyId_mode	variable	_Py_IDENTIFIER(mode)
+Python/pylifecycle.c	create_stdio	PyId_mode	variable	_Py_IDENTIFIER(mode)
+Modules/_io/_iomodule.c	_io_open_impl	PyId_mode	variable	_Py_IDENTIFIER(mode)
+Python/Python-ast.c	-	PyId_module	variable	_Py_IDENTIFIER(module)
+Objects/typeobject.c	-	PyId___module__	variable	_Py_IDENTIFIER(__module__)
+Python/Python-ast.c	make_type	PyId___module__	variable	_Py_IDENTIFIER(__module__)
+Python/errors.c	PyErr_NewException	PyId___module__	variable	_Py_IDENTIFIER(__module__)
+Python/errors.c	PyErr_NewException	PyId___module__	variable	_Py_IDENTIFIER(__module__)
+Python/pythonrun.c	print_exception	PyId___module__	variable	_Py_IDENTIFIER(__module__)
+Modules/_pickle.c	whichmodule	PyId___module__	variable	_Py_IDENTIFIER(__module__)
+Objects/typeobject.c	type_mro_modified	PyId_mro	variable	_Py_IDENTIFIER(mro)
+Objects/typeobject.c	mro_invoke	PyId_mro	variable	_Py_IDENTIFIER(mro)
+Python/bltinmodule.c	-	PyId___mro_entries__	variable	_Py_IDENTIFIER(__mro_entries__)
+Objects/typeobject.c	type_new	PyId___mro_entries__	variable	_Py_IDENTIFIER(__mro_entries__)
+Python/Python-ast.c	-	PyId_msg	variable	_Py_IDENTIFIER(msg)
+Python/errors.c	PyErr_SyntaxLocationObject	PyId_msg	variable	_Py_IDENTIFIER(msg)
+Python/pythonrun.c	parse_syntax_error	PyId_msg	variable	_Py_IDENTIFIER(msg)
+Python/pylifecycle.c	-	PyId_name	variable	_Py_IDENTIFIER(name)
+Modules/_io/fileio.c	-	PyId_name	variable	_Py_IDENTIFIER(name)
+Modules/_io/bufferedio.c	-	PyId_name	variable	_Py_IDENTIFIER(name)
+Modules/_io/textio.c	-	PyId_name	variable	_Py_IDENTIFIER(name)
+Python/Python-ast.c	-	PyId_name	variable	_Py_IDENTIFIER(name)
+Objects/exceptions.c	ImportError_getstate	PyId_name	variable	_Py_IDENTIFIER(name)
+Objects/typeobject.c	-	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Objects/classobject.c	-	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Python/_warnings.c	setup_context	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Python/_warnings.c	get_source_line	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Python/_warnings.c	show_warning	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Python/ceval.c	import_from	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Python/ceval.c	import_all_from	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Python/import.c	resolve_name	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Objects/moduleobject.c	module_init_dict	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Objects/moduleobject.c	PyModule_GetNameObject	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Objects/moduleobject.c	module_getattro	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Objects/weakrefobject.c	weakref_repr	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Modules/_pickle.c	save_global	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Modules/_pickle.c	save_reduce	PyId___name__	variable	_Py_IDENTIFIER(__name__)
+Python/Python-ast.c	-	PyId_names	variable	_Py_IDENTIFIER(names)
+Objects/typeobject.c	-	PyId___new__	variable	_Py_IDENTIFIER(__new__)
+Objects/typeobject.c	reduce_newobj	PyId___newobj__	variable	_Py_IDENTIFIER(__newobj__)
+Objects/typeobject.c	reduce_newobj	PyId___newobj_ex__	variable	_Py_IDENTIFIER(__newobj_ex__)
+Objects/typeobject.c	slot_tp_iternext	PyId___next__	variable	_Py_IDENTIFIER(__next__)
+Objects/structseq.c	-	PyId_n_fields	variable	_Py_IDENTIFIER(n_fields)
+Python/ast.c	new_identifier	PyId_NFKC	variable	_Py_IDENTIFIER(NFKC)
+Objects/structseq.c	-	PyId_n_sequence_fields	variable	_Py_IDENTIFIER(n_sequence_fields)
+Objects/structseq.c	-	PyId_n_unnamed_fields	variable	_Py_IDENTIFIER(n_unnamed_fields)
+Python/errors.c	PyErr_SyntaxLocationObject	PyId_offset	variable	_Py_IDENTIFIER(offset)
+Python/pythonrun.c	parse_syntax_error	PyId_offset	variable	_Py_IDENTIFIER(offset)
+Python/_warnings.c	get_once_registry	PyId_onceregistry	variable	_Py_IDENTIFIER(onceregistry)
+Python/Python-ast.c	-	PyId_op	variable	_Py_IDENTIFIER(op)
+Python/traceback.c	-	PyId_open	variable	_Py_IDENTIFIER(open)
+Python/pylifecycle.c	create_stdio	PyId_open	variable	_Py_IDENTIFIER(open)
+Parser/tokenizer.c	fp_setreadl	PyId_open	variable	_Py_IDENTIFIER(open)
+Objects/fileobject.c	PyFile_FromFd	PyId_open	variable	_Py_IDENTIFIER(open)
+Objects/fileobject.c	PyFile_OpenCodeObject	PyId_open	variable	_Py_IDENTIFIER(open)
+Python/Python-ast.c	-	PyId_operand	variable	_Py_IDENTIFIER(operand)
+Python/Python-ast.c	-	PyId_ops	variable	_Py_IDENTIFIER(ops)
+Python/Python-ast.c	-	PyId_optional_vars	variable	_Py_IDENTIFIER(optional_vars)
+Python/Python-ast.c	-	PyId_orelse	variable	_Py_IDENTIFIER(orelse)
+Python/import.c	resolve_name	PyId___package__	variable	_Py_IDENTIFIER(__package__)
+Objects/moduleobject.c	module_init_dict	PyId___package__	variable	_Py_IDENTIFIER(__package__)
+Python/import.c	resolve_name	PyId_parent	variable	_Py_IDENTIFIER(parent)
+Modules/_operator.c	methodcaller_reduce	PyId_partial	variable	_Py_IDENTIFIER(partial)
+Python/sysmodule.c	-	PyId_path	variable	_Py_IDENTIFIER(path)
+Python/traceback.c	-	PyId_path	variable	_Py_IDENTIFIER(path)
+Objects/exceptions.c	ImportError_getstate	PyId_path	variable	_Py_IDENTIFIER(path)
+Modules/main.c	pymain_sys_path_add_path0	PyId_path	variable	_Py_IDENTIFIER(path)
+Python/import.c	resolve_name	PyId___path__	variable	_Py_IDENTIFIER(__path__)
+Python/import.c	PyImport_ImportModuleLevelObject	PyId___path__	variable	_Py_IDENTIFIER(__path__)
+Modules/_io/bufferedio.c	-	PyId_peek	variable	_Py_IDENTIFIER(peek)
+Python/Python-ast.c	-	PyId_posonlyargs	variable	_Py_IDENTIFIER(posonlyargs)
+Objects/typeobject.c	slot_nb_power	PyId___pow__	variable	_Py_IDENTIFIER(__pow__)
+Python/bltinmodule.c	-	PyId___prepare__	variable	_Py_IDENTIFIER(__prepare__)
+Python/errors.c	PyErr_SyntaxLocationObject	PyId_print_file_and_line	variable	_Py_IDENTIFIER(print_file_and_line)
+Python/pythonrun.c	print_exception	PyId_print_file_and_line	variable	_Py_IDENTIFIER(print_file_and_line)
+Python/pythonrun.c	-	PyId_ps1	variable	_Py_IDENTIFIER(ps1)
+Python/pythonrun.c	-	PyId_ps2	variable	_Py_IDENTIFIER(ps2)
+Objects/object.c	-	PyId_Py_Repr	variable	_Py_IDENTIFIER(Py_Repr)
+Objects/classobject.c	-	PyId___qualname__	variable	_Py_IDENTIFIER(__qualname__)
+Objects/descrobject.c	calculate_qualname	PyId___qualname__	variable	_Py_IDENTIFIER(__qualname__)
+Objects/methodobject.c	meth_get__qualname__	PyId___qualname__	variable	_Py_IDENTIFIER(__qualname__)
+Objects/typeobject.c	type_new	PyId___qualname__	variable	_Py_IDENTIFIER(__qualname__)
+Modules/_io/textio.c	-	PyId_raw	variable	_Py_IDENTIFIER(raw)
+Python/pylifecycle.c	create_stdio	PyId_raw	variable	_Py_IDENTIFIER(raw)
+Modules/_io/iobase.c	-	PyId_read	variable	_Py_IDENTIFIER(read)
+Modules/_io/bufferedio.c	-	PyId_read	variable	_Py_IDENTIFIER(read)
+Modules/_io/textio.c	-	PyId_read	variable	_Py_IDENTIFIER(read)
+Modules/_io/bufferedio.c	-	PyId_read1	variable	_Py_IDENTIFIER(read1)
+Python/marshal.c	marshal_load	PyId_read	variable	_Py_IDENTIFIER(read)
+Modules/_io/bufferedio.c	-	PyId_readable	variable	_Py_IDENTIFIER(readable)
+Modules/_io/textio.c	-	PyId_readable	variable	_Py_IDENTIFIER(readable)
+Modules/_io/iobase.c	_io__RawIOBase_read_impl	PyId_readall	variable	_Py_IDENTIFIER(readall)
+Modules/_io/bufferedio.c	-	PyId_readinto	variable	_Py_IDENTIFIER(readinto)
+Modules/_io/bufferedio.c	-	PyId_readinto1	variable	_Py_IDENTIFIER(readinto1)
+Python/marshal.c	r_string	PyId_readinto	variable	_Py_IDENTIFIER(readinto)
+Parser/tokenizer.c	fp_setreadl	PyId_readline	variable	_Py_IDENTIFIER(readline)
+Objects/fileobject.c	PyFile_GetLine	PyId_readline	variable	_Py_IDENTIFIER(readline)
+Objects/typeobject.c	object___reduce_ex___impl	PyId___reduce__	variable	_Py_IDENTIFIER(__reduce__)
+Python/import.c	PyImport_ReloadModule	PyId_reload	variable	_Py_IDENTIFIER(reload)
+Modules/_io/textio.c	-	PyId_replace	variable	_Py_IDENTIFIER(replace)
+Python/importdl.c	get_encoded_name	PyId_replace	variable	_Py_IDENTIFIER(replace)
+Objects/typeobject.c	slot_tp_repr	PyId___repr__	variable	_Py_IDENTIFIER(__repr__)
+Modules/_io/textio.c	-	PyId_reset	variable	_Py_IDENTIFIER(reset)
+Python/Python-ast.c	-	PyId_returns	variable	_Py_IDENTIFIER(returns)
+Objects/enumobject.c	reversed_new_impl	PyId___reversed__	variable	_Py_IDENTIFIER(__reversed__)
+Objects/listobject.c	listiter_reduce_general	PyId_reversed	variable	_Py_IDENTIFIER(reversed)
+Python/Python-ast.c	-	PyId_right	variable	_Py_IDENTIFIER(right)
+Python/bltinmodule.c	-	PyId___round__	variable	_Py_IDENTIFIER(__round__)
+Modules/_io/textio.c	-	PyId_seek	variable	_Py_IDENTIFIER(seek)
+Modules/_io/iobase.c	_io__IOBase_tell_impl	PyId_seek	variable	_Py_IDENTIFIER(seek)
+Modules/_io/textio.c	-	PyId_seekable	variable	_Py_IDENTIFIER(seekable)
+Python/ceval.c	_PyEval_EvalFrameDefault	PyId_send	variable	_Py_IDENTIFIER(send)
+Objects/typeobject.c	slot_tp_descr_set	PyId___set__	variable	_Py_IDENTIFIER(__set__)
+Objects/typeobject.c	slot_tp_setattro	PyId___setattr__	variable	_Py_IDENTIFIER(__setattr__)
+Objects/typeobject.c	-	PyId___setitem__	variable	_Py_IDENTIFIER(__setitem__)
+Modules/_collectionsmodule.c	_count_elements	PyId___setitem__	variable	_Py_IDENTIFIER(__setitem__)
+Objects/typeobject.c	-	PyId___set_name__	variable	_Py_IDENTIFIER(__set_name__)
+Modules/_io/textio.c	-	PyId_setstate	variable	_Py_IDENTIFIER(setstate)
+Modules/_pickle.c	load_build	PyId___setstate__	variable	_Py_IDENTIFIER(__setstate__)
+Python/_warnings.c	call_show_warning	PyId__showwarnmsg	variable	_Py_IDENTIFIER(_showwarnmsg)
+Python/pylifecycle.c	wait_for_thread_shutdown	PyId__shutdown	variable	_Py_IDENTIFIER(_shutdown)
+Python/Python-ast.c	-	PyId_simple	variable	_Py_IDENTIFIER(simple)
+Python/sysmodule.c	-	PyId___sizeof__	variable	_Py_IDENTIFIER(__sizeof__)
+Python/Python-ast.c	-	PyId_slice	variable	_Py_IDENTIFIER(slice)
+Objects/typeobject.c	_PyType_GetSlotNames	PyId___slotnames__	variable	_Py_IDENTIFIER(__slotnames__)
+Objects/typeobject.c	_PyType_GetSlotNames	PyId__slotnames	variable	_Py_IDENTIFIER(_slotnames)
+Objects/typeobject.c	type_new	PyId___slots__	variable	_Py_IDENTIFIER(__slots__)
+Python/bltinmodule.c	-	PyId_sort	variable	_Py_IDENTIFIER(sort)
+Python/import.c	resolve_name	PyId___spec__	variable	_Py_IDENTIFIER(__spec__)
+Python/import.c	PyImport_ImportModuleLevelObject	PyId___spec__	variable	_Py_IDENTIFIER(__spec__)
+Objects/moduleobject.c	module_init_dict	PyId___spec__	variable	_Py_IDENTIFIER(__spec__)
+Objects/moduleobject.c	module_getattro	PyId___spec__	variable	_Py_IDENTIFIER(__spec__)
+Python/_warnings.c	-	PyId_stderr	variable	_Py_IDENTIFIER(stderr)
+Python/errors.c	-	PyId_stderr	variable	_Py_IDENTIFIER(stderr)
+Python/pylifecycle.c	-	PyId_stderr	variable	_Py_IDENTIFIER(stderr)
+Python/pythonrun.c	-	PyId_stderr	variable	_Py_IDENTIFIER(stderr)
+Python/sysmodule.c	-	PyId_stderr	variable	_Py_IDENTIFIER(stderr)
+Modules/_threadmodule.c	-	PyId_stderr	variable	_Py_IDENTIFIER(stderr)
+Modules/faulthandler.c	-	PyId_stderr	variable	_Py_IDENTIFIER(stderr)
+Python/bltinmodule.c	-	PyId_stderr	variable	_Py_IDENTIFIER(stderr)
+Python/pylifecycle.c	-	PyId_stdin	variable	_Py_IDENTIFIER(stdin)
+Python/pythonrun.c	-	PyId_stdin	variable	_Py_IDENTIFIER(stdin)
+Python/bltinmodule.c	-	PyId_stdin	variable	_Py_IDENTIFIER(stdin)
+Python/pylifecycle.c	-	PyId_stdout	variable	_Py_IDENTIFIER(stdout)
+Python/pythonrun.c	-	PyId_stdout	variable	_Py_IDENTIFIER(stdout)
+Python/sysmodule.c	-	PyId_stdout	variable	_Py_IDENTIFIER(stdout)
+Python/bltinmodule.c	-	PyId_stdout	variable	_Py_IDENTIFIER(stdout)
+Python/Python-ast.c	-	PyId_step	variable	_Py_IDENTIFIER(step)
+Modules/posixmodule.c	DirEntry_test_mode	PyId_st_mode	variable	_Py_IDENTIFIER(st_mode)
+Modules/_io/textio.c	-	PyId_strict	variable	_Py_IDENTIFIER(strict)
+Python/pythonrun.c	-	PyId_string	variable	_Py_static_string(PyId_string, ""<string>"")
+Modules/timemodule.c	time_strptime	PyId__strptime_time	variable	_Py_IDENTIFIER(_strptime_time)
+Modules/posixmodule.c	wait_helper	PyId_struct_rusage	variable	_Py_IDENTIFIER(struct_rusage)
+Modules/_abc.c	-	PyId___subclasscheck__	variable	_Py_IDENTIFIER(__subclasscheck__)
+Objects/abstract.c	PyObject_IsSubclass	PyId___subclasscheck__	variable	_Py_IDENTIFIER(__subclasscheck__)
+Modules/_abc.c	-	PyId___subclasshook__	variable	_Py_IDENTIFIER(__subclasshook__)
+Objects/dictobject.c	dictviews_xor	PyId_symmetric_difference_update	variable	_Py_IDENTIFIER(symmetric_difference_update)
+Python/Python-ast.c	-	PyId_tag	variable	_Py_IDENTIFIER(tag)
+Python/Python-ast.c	-	PyId_target	variable	_Py_IDENTIFIER(target)
+Python/Python-ast.c	-	PyId_targets	variable	_Py_IDENTIFIER(targets)
+Modules/_io/textio.c	-	PyId_tell	variable	_Py_IDENTIFIER(tell)
+Python/Python-ast.c	-	PyId_test	variable	_Py_IDENTIFIER(test)
+Python/errors.c	PyErr_SyntaxLocationObject	PyId_text	variable	_Py_IDENTIFIER(text)
+Python/pythonrun.c	parse_syntax_error	PyId_text	variable	_Py_IDENTIFIER(text)
+Python/traceback.c	-	PyId_TextIOWrapper	variable	_Py_IDENTIFIER(TextIOWrapper)
+Python/pylifecycle.c	create_stdio	PyId_TextIOWrapper	variable	_Py_IDENTIFIER(TextIOWrapper)
+Python/pylifecycle.c	-	PyId_threading	variable	_Py_IDENTIFIER(threading)
+Objects/genobject.c	_gen_throw	PyId_throw	variable	_Py_IDENTIFIER(throw)
+Objects/abstract.c	PyNumber_Long	PyId___trunc__	variable	_Py_IDENTIFIER(__trunc__)
+Python/Python-ast.c	-	PyId_type	variable	_Py_IDENTIFIER(type)
+Python/Python-ast.c	-	PyId_type_comment	variable	_Py_IDENTIFIER(type_comment)
+Python/Python-ast.c	-	PyId_type_ignores	variable	_Py_IDENTIFIER(type_ignores)
+Python/errors.c	_PyErr_WriteUnraisableMsg	PyId_unraisablehook	variable	_Py_IDENTIFIER(unraisablehook)
+Objects/dictobject.c	dictviews_or	PyId_update	variable	_Py_IDENTIFIER(update)
+Python/Python-ast.c	-	PyId_upper	variable	_Py_IDENTIFIER(upper)
+Python/Python-ast.c	-	PyId_value	variable	_Py_IDENTIFIER(value)
+Python/Python-ast.c	-	PyId_values	variable	_Py_IDENTIFIER(values)
+Objects/abstract.c	PyMapping_Values	PyId_values	variable	_Py_IDENTIFIER(values)
+Objects/descrobject.c	mappingproxy_values	PyId_values	variable	_Py_IDENTIFIER(values)
+Python/Python-ast.c	-	PyId_vararg	variable	_Py_IDENTIFIER(vararg)
+Python/_warnings.c	already_warned	PyId_version	variable	_Py_IDENTIFIER(version)
+Python/_warnings.c	call_show_warning	PyId_WarningMessage	variable	_Py_IDENTIFIER(WarningMessage)
+Python/_warnings.c	setup_context	PyId___warningregistry__	variable	_Py_IDENTIFIER(__warningregistry__)
+Python/_warnings.c	get_warnings_attr	PyId_warnings	variable	_Py_IDENTIFIER(warnings)
+Python/sysmodule.c	-	PyId_warnoptions	variable	_Py_IDENTIFIER(warnoptions)
+Python/_warnings.c	_PyErr_WarnUnawaitedCoroutine	PyId__warn_unawaited_coroutine	variable	_Py_IDENTIFIER(_warn_unawaited_coroutine)
+Modules/_io/bufferedio.c	-	PyId_writable	variable	_Py_IDENTIFIER(writable)
+Modules/_io/textio.c	-	PyId_writable	variable	_Py_IDENTIFIER(writable)
+Python/sysmodule.c	-	PyId_write	variable	_Py_IDENTIFIER(write)
+Modules/_io/bufferedio.c	-	PyId_write	variable	_Py_IDENTIFIER(write)
+Python/marshal.c	marshal_dump_impl	PyId_write	variable	_Py_IDENTIFIER(write)
+Objects/fileobject.c	PyFile_WriteObject	PyId_write	variable	_Py_IDENTIFIER(write)
+Python/sysmodule.c	-	PyId__xoptions	variable	_Py_IDENTIFIER(_xoptions)
+Python/import.c	_PyImportZip_Init	PyId_zipimporter	variable	_Py_IDENTIFIER(zipimporter)
+Python/initconfig.c	-	Py_IgnoreEnvironmentFlag	variable	int Py_IgnoreEnvironmentFlag
+Python/dynload_shlib.c	-	_PyImport_DynLoadFiletab	variable	const char *_PyImport_DynLoadFiletab[]
+Python/frozen.c	-	PyImport_FrozenModules	variable	const struct _frozen * PyImport_FrozenModules
+Modules/config.c	-	_PyImport_Inittab	variable	struct _inittab _PyImport_Inittab[]
+Python/import.c	-	PyImport_Inittab	variable	struct _inittab * PyImport_Inittab
+Modules/_io/textio.c	-	PyIncrementalNewlineDecoder_Type	variable	PyTypeObject PyIncrementalNewlineDecoder_Type
+Python/initconfig.c	-	Py_InspectFlag	variable	int Py_InspectFlag
+Objects/classobject.c	-	PyInstanceMethod_Type	variable	PyTypeObject PyInstanceMethod_Type
+Python/initconfig.c	-	Py_InteractiveFlag	variable	int Py_InteractiveFlag
+Objects/interpreteridobject.c	-	_PyInterpreterID_Type	variable	PyTypeObject _PyInterpreterID_Type
+Modules/_io/iobase.c	-	PyIOBase_Type	variable	PyTypeObject PyIOBase_Type
+Modules/_io/_iomodule.c	-	_PyIO_empty_bytes	variable	PyObject *_PyIO_empty_bytes
+Modules/_io/_iomodule.c	-	_PyIO_empty_str	variable	PyObject *_PyIO_empty_str
+Modules/_io/_iomodule.c	-	_PyIO_Module	variable	struct PyModuleDef _PyIO_Module
+Modules/_io/_iomodule.c	-	_PyIO_str_close	variable	PyObject *_PyIO_str_close
+Modules/_io/_iomodule.c	-	_PyIO_str_closed	variable	PyObject *_PyIO_str_closed
+Modules/_io/_iomodule.c	-	_PyIO_str_decode	variable	PyObject *_PyIO_str_decode
+Modules/_io/_iomodule.c	-	_PyIO_str_encode	variable	PyObject *_PyIO_str_encode
+Modules/_io/_iomodule.c	-	_PyIO_str_fileno	variable	PyObject *_PyIO_str_fileno
+Modules/_io/_iomodule.c	-	_PyIO_str_flush	variable	PyObject *_PyIO_str_flush
+Modules/_io/_iomodule.c	-	_PyIO_str_getstate	variable	PyObject *_PyIO_str_getstate
+Modules/_io/_iomodule.c	-	_PyIO_str_isatty	variable	PyObject *_PyIO_str_isatty
+Modules/_io/_iomodule.c	-	_PyIO_str_newlines	variable	PyObject *_PyIO_str_newlines
+Modules/_io/_iomodule.c	-	_PyIO_str_nl	variable	PyObject *_PyIO_str_nl
+Modules/_io/_iomodule.c	-	_PyIO_str_peek	variable	PyObject *_PyIO_str_peek
+Modules/_io/_iomodule.c	-	_PyIO_str_read	variable	PyObject *_PyIO_str_read
+Modules/_io/_iomodule.c	-	_PyIO_str_read1	variable	PyObject *_PyIO_str_read1
+Modules/_io/_iomodule.c	-	_PyIO_str_readable	variable	PyObject *_PyIO_str_readable
+Modules/_io/_iomodule.c	-	_PyIO_str_readall	variable	PyObject *_PyIO_str_readall
+Modules/_io/_iomodule.c	-	_PyIO_str_readinto	variable	PyObject *_PyIO_str_readinto
+Modules/_io/_iomodule.c	-	_PyIO_str_readline	variable	PyObject *_PyIO_str_readline
+Modules/_io/_iomodule.c	-	_PyIO_str_reset	variable	PyObject *_PyIO_str_reset
+Modules/_io/_iomodule.c	-	_PyIO_str_seek	variable	PyObject *_PyIO_str_seek
+Modules/_io/_iomodule.c	-	_PyIO_str_seekable	variable	PyObject *_PyIO_str_seekable
+Modules/_io/_iomodule.c	-	_PyIO_str_setstate	variable	PyObject *_PyIO_str_setstate
+Modules/_io/_iomodule.c	-	_PyIO_str_tell	variable	PyObject *_PyIO_str_tell
+Modules/_io/_iomodule.c	-	_PyIO_str_truncate	variable	PyObject *_PyIO_str_truncate
+Modules/_io/_iomodule.c	-	_PyIO_str_writable	variable	PyObject *_PyIO_str_writable
+Modules/_io/_iomodule.c	-	_PyIO_str_write	variable	PyObject *_PyIO_str_write
+Python/initconfig.c	-	Py_IsolatedFlag	variable	int Py_IsolatedFlag
+Objects/listobject.c	-	PyListIter_Type	variable	PyTypeObject PyListIter_Type
+Objects/listobject.c	-	PyListRevIter_Type	variable	PyTypeObject PyListRevIter_Type
+Objects/listobject.c	-	PyList_Type	variable	PyTypeObject PyList_Type
+Modules/_localemodule.c	-	PyLocale_Methods	variable	static struct PyMethodDef PyLocale_Methods[]
+Objects/longobject.c	-	_PyLong_DigitValue	variable	unsigned char _PyLong_DigitValue[256]
+Objects/longobject.c	-	_PyLong_One	variable	PyObject *_PyLong_One
+Objects/rangeobject.c	-	PyLongRangeIter_Type	variable	PyTypeObject PyLongRangeIter_Type
+Objects/longobject.c	-	PyLong_Type	variable	PyTypeObject PyLong_Type
+Objects/longobject.c	-	_PyLong_Zero	variable	PyObject *_PyLong_Zero
+Objects/memoryobject.c	-	_PyManagedBuffer_Type	variable	PyTypeObject _PyManagedBuffer_Type
+Python/bltinmodule.c	-	PyMap_Type	variable	PyTypeObject PyMap_Type
+Objects/obmalloc.c	-	_PyMem	variable	static PyMemAllocatorEx _PyMem
+Objects/descrobject.c	-	PyMemberDescr_Type	variable	PyTypeObject PyMemberDescr_Type
+Objects/obmalloc.c	-	_PyMem_Debug	variable	static struct { debug_alloc_api_t raw; debug_alloc_api_t mem; debug_alloc_api_t obj; } _PyMem_Debug
+Objects/memoryobject.c	-	PyMemoryView_Type	variable	PyTypeObject PyMemoryView_Type
+Objects/obmalloc.c	-	_PyMem_Raw	variable	static PyMemAllocatorEx _PyMem_Raw
+Objects/descrobject.c	-	PyMethodDescr_Type	variable	PyTypeObject PyMethodDescr_Type
+Objects/classobject.c	-	PyMethod_Type	variable	PyTypeObject PyMethod_Type
+Objects/descrobject.c	-	_PyMethodWrapper_Type	variable	PyTypeObject _PyMethodWrapper_Type
+Objects/moduleobject.c	-	PyModuleDef_Type	variable	PyTypeObject PyModuleDef_Type
+Objects/moduleobject.c	-	PyModule_Type	variable	PyTypeObject PyModule_Type
+Objects/namespaceobject.c	-	_PyNamespace_Type	variable	PyTypeObject _PyNamespace_Type
+Objects/object.c	-	_Py_NoneStruct	variable	PyObject _Py_NoneStruct
+Objects/object.c	-	_PyNone_Type	variable	PyTypeObject _PyNone_Type
+Python/initconfig.c	-	Py_NoSiteFlag	variable	int Py_NoSiteFlag
+Objects/object.c	-	_Py_NotImplementedStruct	variable	PyObject _Py_NotImplementedStruct
+Objects/object.c	-	_PyNotImplemented_Type	variable	PyTypeObject _PyNotImplemented_Type
+Python/initconfig.c	-	Py_NoUserSiteDirectory	variable	int Py_NoUserSiteDirectory
+Objects/obmalloc.c	-	_PyObject	variable	static PyMemAllocatorEx _PyObject
+Objects/obmalloc.c	-	_PyObject_Arena	variable	static PyObjectArenaAllocator _PyObject_Arena
+Objects/odictobject.c	-	PyODictItems_Type	variable	PyTypeObject PyODictItems_Type
+Objects/odictobject.c	-	PyODictIter_Type	variable	PyTypeObject PyODictIter_Type
+Objects/odictobject.c	-	PyODictKeys_Type	variable	PyTypeObject PyODictKeys_Type
+Objects/odictobject.c	-	PyODict_Type	variable	PyTypeObject PyODict_Type
+Objects/odictobject.c	-	PyODictValues_Type	variable	PyTypeObject PyODictValues_Type
+Python/fileutils.c	-	_Py_open_cloexec_works	variable	int _Py_open_cloexec_works
+Python/initconfig.c	-	Py_OptimizeFlag	variable	int Py_OptimizeFlag
+Parser/myreadline.c	-	PyOS_InputHook	variable	int (*PyOS_InputHook)(void)
+Python/pylifecycle.c	-	_PyOS_mystrnicmp_hack	variable	int (*_PyOS_mystrnicmp_hack)(const char *, const char *, Py_ssize_t)
+Python/getopt.c	-	_PyOS_optarg	variable	const wchar_t *_PyOS_optarg
+Python/getopt.c	-	_PyOS_opterr	variable	int _PyOS_opterr
+Python/getopt.c	-	_PyOS_optind	variable	Py_ssize_t _PyOS_optind
+Parser/myreadline.c	-	PyOS_ReadlineFunctionPointer	variable	char *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *)
+Parser/myreadline.c	-	_PyOS_ReadlineLock	variable	static PyThread_type_lock _PyOS_ReadlineLock
+Parser/myreadline.c	-	_PyOS_ReadlineTState	variable	PyThreadState* _PyOS_ReadlineTState
+Python/modsupport.c	-	_Py_PackageContext	variable	const char *_Py_PackageContext
+Python/graminit.c	-	_PyParser_Grammar	variable	grammar _PyParser_Grammar
+Python/pathconfig.c	-	_Py_path_config	variable	_PyPathConfig _Py_path_config
+Objects/picklebufobject.c	-	PyPickleBuffer_Type	variable	PyTypeObject PyPickleBuffer_Type
+Objects/descrobject.c	-	PyProperty_Type	variable	PyTypeObject PyProperty_Type
+Python/initconfig.c	-	Py_QuietFlag	variable	int Py_QuietFlag
+Objects/rangeobject.c	-	PyRangeIter_Type	variable	PyTypeObject PyRangeIter_Type
+Objects/rangeobject.c	-	PyRange_Type	variable	PyTypeObject PyRange_Type
+Modules/_io/iobase.c	-	PyRawIOBase_Type	variable	PyTypeObject PyRawIOBase_Type
+Objects/object.c	-	_Py_RefTotal	variable	Py_ssize_t _Py_RefTotal
+Objects/enumobject.c	-	PyReversed_Type	variable	PyTypeObject PyReversed_Type
+Python/pylifecycle.c	-	_PyRuntime	variable	_PyRuntimeState _PyRuntime
+Objects/iterobject.c	-	PySeqIter_Type	variable	PyTypeObject PySeqIter_Type
+Objects/setobject.c	-	_PySet_Dummy	variable	PyObject * _PySet_Dummy
+Objects/setobject.c	-	_PySetDummy_Type	variable	static PyTypeObject _PySetDummy_Type
+Objects/setobject.c	-	PySetIter_Type	variable	PyTypeObject PySetIter_Type
+Objects/setobject.c	-	PySet_Type	variable	PyTypeObject PySet_Type
+Objects/sliceobject.c	-	PySlice_Type	variable	PyTypeObject PySlice_Type
+Python/initconfig.c	-	_Py_StandardStreamEncoding	variable	static char *_Py_StandardStreamEncoding
+Python/initconfig.c	-	_Py_StandardStreamErrors	variable	static char *_Py_StandardStreamErrors
+Objects/funcobject.c	-	PyStaticMethod_Type	variable	PyTypeObject PyStaticMethod_Type
+Objects/fileobject.c	-	PyStdPrinter_Type	variable	PyTypeObject PyStdPrinter_Type
+Python/symtable.c	-	PySTEntry_Type	variable	PyTypeObject PySTEntry_Type
+Modules/_io/stringio.c	-	PyStringIO_Type	variable	PyTypeObject PyStringIO_Type
+Objects/structseq.c	-	PyStructSequence_UnnamedField	variable	char *PyStructSequence_UnnamedField
+Objects/typeobject.c	-	PySuper_Type	variable	PyTypeObject PySuper_Type
+Objects/object.c	-	_Py_SwappedOp	variable	int _Py_SwappedOp[]
+Python/sysmodule.c	-	_PySys_ImplCacheTag	variable	const char *_PySys_ImplCacheTag
+Python/sysmodule.c	-	_PySys_ImplName	variable	const char *_PySys_ImplName
+Modules/_io/textio.c	-	PyTextIOBase_Type	variable	PyTypeObject PyTextIOBase_Type
+Modules/_io/textio.c	-	PyTextIOWrapper_Type	variable	PyTypeObject PyTextIOWrapper_Type
+Python/traceback.c	-	PyTraceBack_Type	variable	PyTypeObject PyTraceBack_Type
+Objects/obmalloc.c	-	_Py_tracemalloc_config	variable	struct _PyTraceMalloc_Config _Py_tracemalloc_config
+Objects/boolobject.c	-	_Py_TrueStruct	variable	static struct _longobject _Py_TrueStruct
+Objects/tupleobject.c	-	PyTupleIter_Type	variable	PyTypeObject PyTupleIter_Type
+Objects/tupleobject.c	-	PyTuple_Type	variable	PyTypeObject PyTuple_Type
+Objects/typeobject.c	-	PyType_Type	variable	PyTypeObject PyType_Type
+Python/initconfig.c	-	Py_UnbufferedStdioFlag	variable	int Py_UnbufferedStdioFlag
+Python/pylifecycle.c	-	_Py_UnhandledKeyboardInterrupt	variable	int _Py_UnhandledKeyboardInterrupt
+Objects/unicodeobject.c	-	PyUnicodeIter_Type	variable	PyTypeObject PyUnicodeIter_Type
+Objects/unicodeobject.c	-	PyUnicode_Type	variable	PyTypeObject PyUnicode_Type
+Python/initconfig.c	-	Py_UTF8Mode	variable	int Py_UTF8Mode
+Python/initconfig.c	-	Py_VerboseFlag	variable	int Py_VerboseFlag
+Objects/weakrefobject.c	-	_PyWeakref_CallableProxyType	variable	PyTypeObject _PyWeakref_CallableProxyType
+Objects/weakrefobject.c	-	_PyWeakref_ProxyType	variable	PyTypeObject _PyWeakref_ProxyType
+Objects/weakrefobject.c	-	_PyWeakref_RefType	variable	PyTypeObject _PyWeakref_RefType
+Objects/weakrefobject.c	-	_PyWeakref_RefType	variable	PyTypeObject _PyWeakref_RefType
+Objects/descrobject.c	-	PyWrapperDescr_Type	variable	PyTypeObject PyWrapperDescr_Type
+Python/bltinmodule.c	-	PyZip_Type	variable	PyTypeObject PyZip_Type
+Python/Python-ast.c	-	Raise_fields	variable	static const char *Raise_fields[]
+Python/Python-ast.c	-	Raise_type	variable	static PyTypeObject *Raise_type
+Objects/rangeobject.c	-	range_as_mapping	variable	static PyMappingMethods range_as_mapping
+Objects/rangeobject.c	-	range_as_number	variable	static PyNumberMethods range_as_number
+Objects/rangeobject.c	-	range_as_sequence	variable	static PySequenceMethods range_as_sequence
+Objects/rangeobject.c	-	rangeiter_methods	variable	static PyMethodDef rangeiter_methods
+Objects/rangeobject.c	-	range_members	variable	static PyMemberDef range_members[]
+Objects/rangeobject.c	-	range_methods	variable	static PyMethodDef range_methods
+Modules/_io/iobase.c	-	rawiobase_methods	variable	static PyMethodDef rawiobase_methods
+Python/pylifecycle.c	fatal_error	reentrant	variable	static int reentrant
+Modules/faulthandler.c	faulthandler_dump_traceback	reentrant	variable	static volatile int reentrant
+Modules/itertoolsmodule.c	-	repeat_methods	variable	static PyMethodDef repeat_methods
+Modules/itertoolsmodule.c	-	repeat_type	variable	static PyTypeObject repeat_type
+Python/Python-ast.c	-	Return_fields	variable	static const char *Return_fields[]
+Python/compile.c	compiler_visit_annotations	return_str	variable	static identifier return_str
+Python/Python-ast.c	-	Return_type	variable	static PyTypeObject *Return_type
+Objects/enumobject.c	-	reversediter_methods	variable	static PyMethodDef reversediter_methods
+Modules/_threadmodule.c	-	rlock_methods	variable	static PyMethodDef rlock_methods
+Modules/_threadmodule.c	-	RLocktype	variable	static PyTypeObject RLocktype
+Objects/typeobject.c	slot_nb_add	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_subtract	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_multiply	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_matrix_multiply	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_remainder	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_divmod	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_power_binary	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_lshift	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_rshift	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_and	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_xor	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_or	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_floor_divide	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Objects/typeobject.c	slot_nb_true_divide	rop_id	variable	_Py_static_string(op_id, OPSTR)
+Python/Python-ast.c	-	RShift_singleton	variable	static PyObject *RShift_singleton
+Python/Python-ast.c	-	RShift_type	variable	static PyTypeObject *RShift_type
+Python/pylifecycle.c	-	runtime_initialized	variable	static int runtime_initialized
+Modules/posixmodule.c	-	ScandirIterator_methods	variable	static PyMethodDef ScandirIterator_methods
+Modules/posixmodule.c	-	ScandirIteratorType	variable	static PyTypeObject ScandirIteratorType
+Modules/_sre.c	-	scanner_members	variable	static PyMemberDef scanner_members[]
+Modules/_sre.c	-	scanner_methods	variable	static PyMethodDef scanner_methods
+Modules/_sre.c	-	Scanner_Type	variable	static PyTypeObject Scanner_Type
+Modules/posixmodule.c	-	sched_param_desc	variable	static PyStructSequence_Desc sched_param_desc
+Modules/posixmodule.c	-	sched_param_fields	variable	static PyStructSequence_Field sched_param_fields[]
+Modules/posixmodule.c	-	SchedParamType	variable	static PyTypeObject* SchedParamType
+Objects/iterobject.c	-	seqiter_methods	variable	static PyMethodDef seqiter_methods
+Objects/setobject.c	-	set_as_number	variable	static PyNumberMethods set_as_number
+Objects/setobject.c	-	set_as_sequence	variable	static PySequenceMethods set_as_sequence
+Python/symtable.c	-	setcomp	variable	static identifier setcomp
+Python/Python-ast.c	-	SetComp_fields	variable	static const char *SetComp_fields[]
+Python/Python-ast.c	-	SetComp_type	variable	static PyTypeObject *SetComp_type
+Python/Python-ast.c	-	Set_fields	variable	static const char *Set_fields[]
+Objects/setobject.c	-	setiter_methods	variable	static PyMethodDef setiter_methods
+Objects/setobject.c	-	set_methods	variable	static PyMethodDef set_methods
+Python/Python-ast.c	-	Set_type	variable	static PyTypeObject *Set_type
+Modules/signalmodule.c	-	SiginfoType	variable	static PyTypeObject SiginfoType
+Modules/signalmodule.c	-	signal_methods	variable	static PyMethodDef signal_methods
+Modules/signalmodule.c	-	signalmodule	variable	static struct PyModuleDef signalmodule
+Python/import.c	PyImport_Import	silly_list	variable	static PyObject *silly_list
+Objects/sliceobject.c	-	slice_cache	variable	static PySliceObject *slice_cache
+Python/Python-ast.c	-	Slice_fields	variable	static const char *Slice_fields[]
+Objects/sliceobject.c	-	slice_members	variable	static PyMemberDef slice_members[]
+Objects/sliceobject.c	-	slice_methods	variable	static PyMethodDef slice_methods
+Python/Python-ast.c	-	slice_type	variable	static PyTypeObject *slice_type
+Python/Python-ast.c	-	Slice_type	variable	static PyTypeObject *Slice_type
+Objects/typeobject.c	-	slotdefs	variable	static slotdef slotdefs[]
+Objects/typeobject.c	-	slotdefs_initialized	variable	static int slotdefs_initialized
+Objects/longobject.c	-	small_ints	variable	static PyLongObject small_ints[NSMALLNEGINTS + NSMALLPOSINTS]
+Objects/funcobject.c	-	sm_getsetlist	variable	static PyGetSetDef sm_getsetlist[]
+Objects/funcobject.c	-	sm_memberlist	variable	static PyMemberDef sm_memberlist[]
+Modules/xxsubtype.c	-	spamdict_members	variable	static PyMemberDef spamdict_members[]
+Modules/xxsubtype.c	-	spamdict_methods	variable	static PyMethodDef spamdict_methods
+Modules/xxsubtype.c	-	spamdict_type	variable	static PyTypeObject spamdict_type
+Modules/xxsubtype.c	-	spamlist_getsets	variable	static PyGetSetDef spamlist_getsets[]
+Modules/xxsubtype.c	-	spamlist_methods	variable	static PyMethodDef spamlist_methods
+Modules/xxsubtype.c	-	spamlist_type	variable	static PyTypeObject spamlist_type
+Modules/_sre.c	-	sremodule	variable	static struct PyModuleDef sremodule
+Modules/faulthandler.c	-	stack	variable	static stack_t stack
+Modules/itertoolsmodule.c	-	starmap_methods	variable	static PyMethodDef starmap_methods
+Modules/itertoolsmodule.c	-	starmap_type	variable	static PyTypeObject starmap_type
+Python/Python-ast.c	-	Starred_fields	variable	static const char *Starred_fields[]
+Python/Python-ast.c	-	Starred_type	variable	static PyTypeObject *Starred_type
+Python/graminit.c	-	states_0	variable	static state states_0[3]
+Python/graminit.c	-	states_1	variable	static state states_1[2]
+Python/graminit.c	-	states_10	variable	static state states_10[4]
+Python/graminit.c	-	states_11	variable	static state states_11[34]
+Python/graminit.c	-	states_12	variable	static state states_12[2]
+Python/graminit.c	-	states_13	variable	static state states_13[2]
+Python/graminit.c	-	states_14	variable	static state states_14[4]
+Python/graminit.c	-	states_15	variable	static state states_15[2]
+Python/graminit.c	-	states_16	variable	static state states_16[6]
+Python/graminit.c	-	states_17	variable	static state states_17[5]
+Python/graminit.c	-	states_18	variable	static state states_18[3]
+Python/graminit.c	-	states_19	variable	static state states_19[2]
+Python/graminit.c	-	states_2	variable	static state states_2[3]
+Python/graminit.c	-	states_20	variable	static state states_20[3]
+Python/graminit.c	-	states_21	variable	static state states_21[2]
+Python/graminit.c	-	states_22	variable	static state states_22[2]
+Python/graminit.c	-	states_23	variable	static state states_23[2]
+Python/graminit.c	-	states_24	variable	static state states_24[2]
+Python/graminit.c	-	states_25	variable	static state states_25[3]
+Python/graminit.c	-	states_26	variable	static state states_26[2]
+Python/graminit.c	-	states_27	variable	static state states_27[5]
+Python/graminit.c	-	states_28	variable	static state states_28[2]
+Python/graminit.c	-	states_29	variable	static state states_29[3]
+Python/graminit.c	-	states_3	variable	static state states_3[7]
+Python/graminit.c	-	states_30	variable	static state states_30[8]
+Python/graminit.c	-	states_31	variable	static state states_31[4]
+Python/graminit.c	-	states_32	variable	static state states_32[4]
+Python/graminit.c	-	states_33	variable	static state states_33[3]
+Python/graminit.c	-	states_34	variable	static state states_34[2]
+Python/graminit.c	-	states_35	variable	static state states_35[2]
+Python/graminit.c	-	states_36	variable	static state states_36[3]
+Python/graminit.c	-	states_37	variable	static state states_37[3]
+Python/graminit.c	-	states_38	variable	static state states_38[5]
+Python/graminit.c	-	states_39	variable	static state states_39[2]
+Python/graminit.c	-	states_4	variable	static state states_4[2]
+Python/graminit.c	-	states_40	variable	static state states_40[3]
+Python/graminit.c	-	states_41	variable	static state states_41[8]
+Python/graminit.c	-	states_42	variable	static state states_42[8]
+Python/graminit.c	-	states_43	variable	static state states_43[11]
+Python/graminit.c	-	states_44	variable	static state states_44[13]
+Python/graminit.c	-	states_45	variable	static state states_45[6]
+Python/graminit.c	-	states_46	variable	static state states_46[4]
+Python/graminit.c	-	states_47	variable	static state states_47[5]
+Python/graminit.c	-	states_48	variable	static state states_48[5]
+Python/graminit.c	-	states_49	variable	static state states_49[4]
+Python/graminit.c	-	states_5	variable	static state states_5[3]
+Python/graminit.c	-	states_50	variable	static state states_50[6]
+Python/graminit.c	-	states_51	variable	static state states_51[2]
+Python/graminit.c	-	states_52	variable	static state states_52[5]
+Python/graminit.c	-	states_53	variable	static state states_53[5]
+Python/graminit.c	-	states_54	variable	static state states_54[2]
+Python/graminit.c	-	states_55	variable	static state states_55[2]
+Python/graminit.c	-	states_56	variable	static state states_56[3]
+Python/graminit.c	-	states_57	variable	static state states_57[2]
+Python/graminit.c	-	states_58	variable	static state states_58[4]
+Python/graminit.c	-	states_59	variable	static state states_59[3]
+Python/graminit.c	-	states_6	variable	static state states_6[3]
+Python/graminit.c	-	states_60	variable	static state states_60[2]
+Python/graminit.c	-	states_61	variable	static state states_61[2]
+Python/graminit.c	-	states_62	variable	static state states_62[2]
+Python/graminit.c	-	states_63	variable	static state states_63[2]
+Python/graminit.c	-	states_64	variable	static state states_64[2]
+Python/graminit.c	-	states_65	variable	static state states_65[2]
+Python/graminit.c	-	states_66	variable	static state states_66[3]
+Python/graminit.c	-	states_67	variable	static state states_67[4]
+Python/graminit.c	-	states_68	variable	static state states_68[3]
+Python/graminit.c	-	states_69	variable	static state states_69[9]
+Python/graminit.c	-	states_7	variable	static state states_7[9]
+Python/graminit.c	-	states_70	variable	static state states_70[5]
+Python/graminit.c	-	states_71	variable	static state states_71[7]
+Python/graminit.c	-	states_72	variable	static state states_72[3]
+Python/graminit.c	-	states_73	variable	static state states_73[5]
+Python/graminit.c	-	states_74	variable	static state states_74[3]
+Python/graminit.c	-	states_75	variable	static state states_75[3]
+Python/graminit.c	-	states_76	variable	static state states_76[3]
+Python/graminit.c	-	states_77	variable	static state states_77[14]
+Python/graminit.c	-	states_78	variable	static state states_78[8]
+Python/graminit.c	-	states_79	variable	static state states_79[3]
+Python/graminit.c	-	states_8	variable	static state states_8[4]
+Python/graminit.c	-	states_80	variable	static state states_80[4]
+Python/graminit.c	-	states_81	variable	static state states_81[2]
+Python/graminit.c	-	states_82	variable	static state states_82[6]
+Python/graminit.c	-	states_83	variable	static state states_83[3]
+Python/graminit.c	-	states_84	variable	static state states_84[4]
+Python/graminit.c	-	states_85	variable	static state states_85[2]
+Python/graminit.c	-	states_86	variable	static state states_86[3]
+Python/graminit.c	-	states_87	variable	static state states_87[3]
+Python/graminit.c	-	states_88	variable	static state states_88[7]
+Python/graminit.c	-	states_89	variable	static state states_89[3]
+Python/graminit.c	-	states_9	variable	static state states_9[42]
+Python/graminit.c	-	states_90	variable	static state states_90[6]
+Python/graminit.c	-	states_91	variable	static state states_91[11]
+Python/getargs.c	-	static_arg_parsers	variable	static struct _PyArg_Parser *static_arg_parsers
+Objects/unicodeobject.c	-	static_strings	variable	static _Py_Identifier *static_strings
+Modules/_stat.c	-	stat_methods	variable	static PyMethodDef stat_methods
+Modules/_stat.c	-	statmodule	variable	static struct PyModuleDef statmodule
+Modules/posixmodule.c	-	stat_result_desc	variable	static PyStructSequence_Desc stat_result_desc
+Modules/posixmodule.c	-	stat_result_fields	variable	static PyStructSequence_Field stat_result_fields[]
+Modules/posixmodule.c	-	StatResultType	variable	static PyTypeObject* StatResultType
+Modules/posixmodule.c	-	statvfs_result_desc	variable	static PyStructSequence_Desc statvfs_result_desc
+Modules/posixmodule.c	-	statvfs_result_fields	variable	static PyStructSequence_Field statvfs_result_fields[]
+Modules/posixmodule.c	-	StatVFSResultType	variable	static PyTypeObject* StatVFSResultType
+Objects/fileobject.c	-	stdprinter_getsetlist	variable	static PyGetSetDef stdprinter_getsetlist[]
+Objects/fileobject.c	-	stdprinter_methods	variable	static PyMethodDef stdprinter_methods
+Python/symtable.c	-	ste_memberlist	variable	static PyMemberDef ste_memberlist[]
+Python/Python-ast.c	-	stmt_attributes	variable	static const char *stmt_attributes[]
+Python/Python-ast.c	-	stmt_type	variable	static PyTypeObject *stmt_type
+Objects/exceptions.c	-	StopIteration_members	variable	static PyMemberDef StopIteration_members[]
+Python/Python-ast.c	-	Store_singleton	variable	static PyObject *Store_singleton
+Python/Python-ast.c	-	Store_type	variable	static PyTypeObject *Store_type
+Python/ast_unparse.c	-	_str_close_br	variable	static PyObject *_str_close_br
+Python/ast_unparse.c	-	_str_dbl_close_br	variable	static PyObject *_str_dbl_close_br
+Python/ast_unparse.c	-	_str_dbl_open_br	variable	static PyObject *_str_dbl_open_br
+Modules/_threadmodule.c	-	str_dict	variable	static PyObject *str_dict
+Modules/_io/stringio.c	-	stringio_getset	variable	static PyGetSetDef stringio_getset[]
+Modules/_io/stringio.c	-	stringio_methods	variable	static PyMethodDef stringio_methods
+Objects/unicodeobject.c	-	_string_methods	variable	static PyMethodDef _string_methods
+Objects/unicodeobject.c	-	_string_module	variable	static struct PyModuleDef _string_module
+Objects/bytesobject.c	-	striter_methods	variable	static PyMethodDef striter_methods
+Python/ast_unparse.c	-	_str_open_br	variable	static PyObject *_str_open_br
+Modules/pwdmodule.c	-	StructPwdType	variable	static PyTypeObject StructPwdType
+Modules/pwdmodule.c	-	struct_pwd_type_desc	variable	static PyStructSequence_Desc struct_pwd_type_desc
+Modules/pwdmodule.c	-	struct_pwd_type_fields	variable	static PyStructSequence_Field struct_pwd_type_fields[]
+Modules/posixmodule.c	wait_helper	struct_rusage	variable	static PyObject *struct_rusage
+Objects/structseq.c	-	structseq_methods	variable	static PyMethodDef structseq_methods
+Modules/posixmodule.c	-	structseq_new	variable	static newfunc structseq_new
+Modules/signalmodule.c	-	struct_siginfo_desc	variable	static PyStructSequence_Desc struct_siginfo_desc
+Modules/signalmodule.c	-	struct_siginfo_fields	variable	static PyStructSequence_Field struct_siginfo_fields[]
+Modules/timemodule.c	-	StructTimeType	variable	static PyTypeObject StructTimeType
+Modules/timemodule.c	-	struct_time_type_desc	variable	static PyStructSequence_Desc struct_time_type_desc
+Modules/timemodule.c	-	struct_time_type_fields	variable	static PyStructSequence_Field struct_time_type_fields[]
+Python/Python-ast.c	-	Subscript_fields	variable	static const char *Subscript_fields[]
+Python/Python-ast.c	-	Subscript_type	variable	static PyTypeObject *Subscript_type
+Python/Python-ast.c	-	Sub_singleton	variable	static PyObject *Sub_singleton
+Python/Python-ast.c	-	Sub_type	variable	static PyTypeObject *Sub_type
+Objects/typeobject.c	-	subtype_getsets_dict_only	variable	static PyGetSetDef subtype_getsets_dict_only[]
+Objects/typeobject.c	-	subtype_getsets_full	variable	static PyGetSetDef subtype_getsets_full[]
+Objects/typeobject.c	-	subtype_getsets_weakref_only	variable	static PyGetSetDef subtype_getsets_weakref_only[]
+Python/Python-ast.c	-	Suite_fields	variable	static const char *Suite_fields[]
+Python/Python-ast.c	-	Suite_type	variable	static PyTypeObject *Suite_type
+Objects/typeobject.c	-	super_members	variable	static PyMemberDef super_members[]
+Modules/symtablemodule.c	-	symtable_methods	variable	static PyMethodDef symtable_methods
+Modules/symtablemodule.c	-	symtablemodule	variable	static struct PyModuleDef symtablemodule
+Objects/exceptions.c	-	SyntaxError_members	variable	static PyMemberDef SyntaxError_members[]
+Python/sysmodule.c	-	sys_methods	variable	static PyMethodDef sys_methods
+Python/sysmodule.c	-	sysmodule	variable	static struct PyModuleDef sysmodule
+Objects/exceptions.c	-	SystemExit_members	variable	static PyMemberDef SystemExit_members[]
+Modules/_tracemalloc.c	-	tables_lock	variable	static PyThread_type_lock tables_lock
+Modules/itertoolsmodule.c	-	takewhile_reduce_methods	variable	static PyMethodDef takewhile_reduce_methods
+Modules/itertoolsmodule.c	-	takewhile_type	variable	static PyTypeObject takewhile_type
+Python/pylifecycle.c	-	_TARGET_LOCALES	variable	static _LocaleCoercionTarget _TARGET_LOCALES[]
+Python/traceback.c	-	tb_getsetters	variable	static PyGetSetDef tb_getsetters[]
+Python/traceback.c	-	tb_memberlist	variable	static PyMemberDef tb_memberlist[]
+Python/traceback.c	-	tb_methods	variable	static PyMethodDef tb_methods
+Modules/itertoolsmodule.c	-	teedataobject_methods	variable	static PyMethodDef teedataobject_methods
+Modules/itertoolsmodule.c	-	teedataobject_type	variable	static PyTypeObject teedataobject_type
+Modules/itertoolsmodule.c	-	tee_methods	variable	static PyMethodDef tee_methods
+Modules/itertoolsmodule.c	-	tee_type	variable	static PyTypeObject tee_type
+Modules/posixmodule.c	-	TerminalSize_desc	variable	static PyStructSequence_Desc TerminalSize_desc
+Modules/posixmodule.c	-	TerminalSize_fields	variable	static PyStructSequence_Field TerminalSize_fields[]
+Modules/posixmodule.c	-	TerminalSizeType	variable	static PyTypeObject* TerminalSizeType
+Modules/_io/textio.c	-	textiobase_getset	variable	static PyGetSetDef textiobase_getset[]
+Modules/_io/textio.c	-	textiobase_methods	variable	static PyMethodDef textiobase_methods
+Modules/_io/textio.c	-	textiowrapper_getset	variable	static PyGetSetDef textiowrapper_getset[]
+Modules/_io/textio.c	-	textiowrapper_members	variable	static PyMemberDef textiowrapper_members[]
+Modules/_io/textio.c	-	textiowrapper_methods	variable	static PyMethodDef textiowrapper_methods
+Modules/faulthandler.c	-	thread	variable	static struct { PyObject *file; int fd; PY_TIMEOUT_T timeout_us; int repeat; PyInterpreterState *interp; int exit; char *header; size_t header_len; PyThread_type_lock cancel_event; PyThread_type_lock running; } thread
+Python/thread.c	-	thread_debug	variable	static int thread_debug
+Modules/_threadmodule.c	-	ThreadError	variable	static PyObject *ThreadError
+Python/thread.c	-	threadinfo_desc	variable	static PyStructSequence_Desc threadinfo_desc
+Python/thread.c	-	threadinfo_fields	variable	static PyStructSequence_Field threadinfo_fields[]
+Python/thread.c	-	ThreadInfoType	variable	static PyTypeObject ThreadInfoType
+Modules/_threadmodule.c	-	thread_methods	variable	static PyMethodDef thread_methods
+Modules/_threadmodule.c	-	threadmodule	variable	static struct PyModuleDef threadmodule
+Modules/posixmodule.c	-	ticks_per_second	variable	static long ticks_per_second
+Modules/timemodule.c	_PyTime_GetProcessTimeWithInfo	ticks_per_second	variable	static long ticks_per_second
+Modules/timemodule.c	-	time_methods	variable	static PyMethodDef time_methods
+Modules/timemodule.c	-	timemodule	variable	static struct PyModuleDef timemodule
+Modules/posixmodule.c	-	times_result_desc	variable	static PyStructSequence_Desc times_result_desc
+Modules/posixmodule.c	-	times_result_fields	variable	static PyStructSequence_Field times_result_fields[]
+Modules/posixmodule.c	-	TimesResultType	variable	static PyTypeObject* TimesResultType
+Python/context.c	-	_token_missing	variable	static PyObject *_token_missing
+Python/symtable.c	-	top	variable	static identifier top
+Objects/typeobject.c	-	tp_new_methoddef	variable	static struct PyMethodDef tp_new_methoddef[]
+Modules/_tracemalloc.c	-	tracemalloc_empty_traceback	variable	static traceback_t tracemalloc_empty_traceback
+Modules/_tracemalloc.c	-	tracemalloc_filenames	variable	static _Py_hashtable_t *tracemalloc_filenames
+Modules/_tracemalloc.c	-	tracemalloc_peak_traced_memory	variable	static size_t tracemalloc_peak_traced_memory
+Modules/_tracemalloc.c	-	tracemalloc_reentrant_key	variable	static Py_tss_t tracemalloc_reentrant_key
+Modules/_tracemalloc.c	-	tracemalloc_traceback	variable	static traceback_t *tracemalloc_traceback
+Modules/_tracemalloc.c	-	tracemalloc_tracebacks	variable	static _Py_hashtable_t *tracemalloc_tracebacks
+Modules/_tracemalloc.c	-	tracemalloc_traced_memory	variable	static size_t tracemalloc_traced_memory
+Modules/_tracemalloc.c	-	tracemalloc_traces	variable	static _Py_hashtable_t *tracemalloc_traces
+Objects/boolobject.c	-	true_str	variable	static PyObject *true_str
+Python/Python-ast.c	-	Try_fields	variable	static const char *Try_fields[]
+Python/Python-ast.c	-	Try_type	variable	static PyTypeObject *Try_type
+Objects/tupleobject.c	-	tuple_as_mapping	variable	static PyMappingMethods tuple_as_mapping
+Objects/tupleobject.c	-	tuple_as_sequence	variable	static PySequenceMethods tuple_as_sequence
+Python/Python-ast.c	-	Tuple_fields	variable	static const char *Tuple_fields[]
+Modules/_collectionsmodule.c	-	tuplegetter_members	variable	static PyMemberDef tuplegetter_members[]
+Modules/_collectionsmodule.c	-	tuplegetter_methods	variable	static PyMethodDef tuplegetter_methods
+Modules/_collectionsmodule.c	-	tuplegetter_type	variable	static PyTypeObject tuplegetter_type
+Objects/tupleobject.c	-	tupleiter_methods	variable	static PyMethodDef tupleiter_methods
+Objects/tupleobject.c	-	tuple_methods	variable	static PyMethodDef tuple_methods
+Python/Python-ast.c	-	Tuple_type	variable	static PyTypeObject *Tuple_type
+Objects/typeobject.c	-	type_getsets	variable	static PyGetSetDef type_getsets[]
+Python/Python-ast.c	-	TypeIgnore_fields	variable	static const char *TypeIgnore_fields[]
+Python/Python-ast.c	-	type_ignore_type	variable	static PyTypeObject *type_ignore_type
+Python/Python-ast.c	-	TypeIgnore_type	variable	static PyTypeObject *TypeIgnore_type
+Objects/typeobject.c	-	type_members	variable	static PyMemberDef type_members[]
+Objects/typeobject.c	-	type_methods	variable	static PyMethodDef type_methods
+Python/Python-ast.c	-	UAdd_singleton	variable	static PyObject *UAdd_singleton
+Python/Python-ast.c	-	UAdd_type	variable	static PyTypeObject *UAdd_type
+Objects/unicodeobject.c	-	ucnhash_CAPI	variable	static _PyUnicode_Name_CAPI *ucnhash_CAPI
+Python/codecs.c	-	ucnhash_CAPI	variable	static _PyUnicode_Name_CAPI *ucnhash_CAPI
+Python/ast.c	-	u_kind	variable	static PyObject *u_kind
+Modules/posixmodule.c	-	uname_result_desc	variable	static PyStructSequence_Desc uname_result_desc
+Modules/posixmodule.c	-	uname_result_fields	variable	static PyStructSequence_Field uname_result_fields[]
+Modules/posixmodule.c	-	UnameResultType	variable	static PyTypeObject* UnameResultType
+Python/Python-ast.c	-	UnaryOp_fields	variable	static const char *UnaryOp_fields[]
+Python/Python-ast.c	-	unaryop_type	variable	static PyTypeObject *unaryop_type
+Python/Python-ast.c	-	UnaryOp_type	variable	static PyTypeObject *UnaryOp_type
+Objects/unicodeobject.c	-	unicode_as_mapping	variable	static PyMappingMethods unicode_as_mapping
+Objects/unicodeobject.c	-	unicode_as_number	variable	static PyNumberMethods unicode_as_number
+Objects/unicodeobject.c	-	unicode_as_sequence	variable	static PySequenceMethods unicode_as_sequence
+Objects/unicodeobject.c	-	unicode_empty	variable	static PyObject *unicode_empty
+Objects/exceptions.c	-	UnicodeError_members	variable	static PyMemberDef UnicodeError_members[]
+Objects/unicodeobject.c	-	unicodeiter_methods	variable	static PyMethodDef unicodeiter_methods
+Objects/unicodeobject.c	-	unicode_latin1	variable	static PyObject *unicode_latin1[256]
+Objects/unicodeobject.c	-	unicode_methods	variable	static PyMethodDef unicode_methods
+Modules/_tracemalloc.c	-	unknown_filename	variable	static PyObject *unknown_filename
+Python/errors.c	-	UnraisableHookArgs_desc	variable	static PyStructSequence_Desc UnraisableHookArgs_desc
+Python/errors.c	-	UnraisableHookArgs_fields	variable	static PyStructSequence_Field UnraisableHookArgs_fields[]
+Python/errors.c	-	UnraisableHookArgsType	variable	static PyTypeObject UnraisableHookArgsType
+Objects/obmalloc.c	-	unused_arena_objects	variable	static struct arena_object* unused_arena_objects
+Python/bootstrap_hash.c	-	urandom_cache	variable	static struct { int fd; dev_t st_dev; ino_t st_ino; } urandom_cache
+Objects/obmalloc.c	-	usable_arenas	variable	static struct arena_object* usable_arenas
+Objects/obmalloc.c	-	usedpools	variable	static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8]
+Modules/faulthandler.c	-	user_signals	variable	static user_signal_t *user_signals
+Python/Python-ast.c	-	USub_singleton	variable	static PyObject *USub_singleton
+Python/Python-ast.c	-	USub_type	variable	static PyTypeObject *USub_type
+Python/getversion.c	Py_GetVersion	version	variable	static char version[250]
+Python/sysmodule.c	-	version_info_desc	variable	static PyStructSequence_Desc version_info_desc
+Python/sysmodule.c	-	version_info_fields	variable	static PyStructSequence_Field version_info_fields[]
+Python/sysmodule.c	-	VersionInfoType	variable	static PyTypeObject VersionInfoType
+Modules/posixmodule.c	-	waitid_result_desc	variable	static PyStructSequence_Desc waitid_result_desc
+Modules/posixmodule.c	-	waitid_result_fields	variable	static PyStructSequence_Field waitid_result_fields[]
+Modules/posixmodule.c	-	WaitidResultType	variable	static PyTypeObject* WaitidResultType
+Modules/signalmodule.c	-	wakeup	variable	static volatile struct { SOCKET_T fd; int warn_on_full_buffer; int use_send; } wakeup
+Python/_warnings.c	-	warnings_functions	variable	static PyMethodDef warnings_functions[]
+Python/_warnings.c	-	warningsmodule	variable	static struct PyModuleDef warningsmodule
+Modules/_weakref.c	-	weakref_functions	variable	static PyMethodDef weakref_functions
+Objects/weakrefobject.c	-	weakref_members	variable	static PyMemberDef weakref_members[]
+Modules/_weakref.c	-	weakrefmodule	variable	static struct PyModuleDef weakrefmodule
+Python/sysmodule.c	-	whatstrings	variable	static PyObject *whatstrings[8]
+Python/Python-ast.c	-	While_fields	variable	static const char *While_fields[]
+Python/Python-ast.c	-	While_type	variable	static PyTypeObject *While_type
+Python/Python-ast.c	-	With_fields	variable	static const char *With_fields[]
+Python/Python-ast.c	-	withitem_fields	variable	static const char *withitem_fields[]
+Python/Python-ast.c	-	withitem_type	variable	static PyTypeObject *withitem_type
+Python/Python-ast.c	-	With_type	variable	static PyTypeObject *With_type
+Objects/descrobject.c	-	wrapperdescr_getset	variable	static PyGetSetDef wrapperdescr_getset[]
+Objects/descrobject.c	-	wrapper_getsets	variable	static PyGetSetDef wrapper_getsets[]
+Objects/descrobject.c	-	wrapper_members	variable	static PyMemberDef wrapper_members[]
+Objects/descrobject.c	-	wrapper_methods	variable	static PyMethodDef wrapper_methods
+Modules/_threadmodule.c	local_new	wr_callback_def	variable	static PyMethodDef wr_callback_def
+Modules/xxsubtype.c	-	xxsubtype_functions	variable	static PyMethodDef xxsubtype_functions[]
+Modules/xxsubtype.c	-	xxsubtypemodule	variable	static struct PyModuleDef xxsubtypemodule
+Modules/xxsubtype.c	-	xxsubtype_slots	variable	static struct PyModuleDef_Slot xxsubtype_slots[]
+Python/Python-ast.c	-	Yield_fields	variable	static const char *Yield_fields[]
+Python/Python-ast.c	-	YieldFrom_fields	variable	static const char *YieldFrom_fields[]
+Python/Python-ast.c	-	YieldFrom_type	variable	static PyTypeObject *YieldFrom_type
+Python/Python-ast.c	-	Yield_type	variable	static PyTypeObject *Yield_type
+Modules/itertoolsmodule.c	-	zip_longest_methods	variable	static PyMethodDef zip_longest_methods
+Modules/itertoolsmodule.c	-	ziplongest_type	variable	static PyTypeObject ziplongest_type
+Python/bltinmodule.c	-	zip_methods	variable	static PyMethodDef zip_methods


From webhook-mailer at python.org  Wed Sep 11 19:43:26 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Wed, 11 Sep 2019 23:43:26 -0000
Subject: [Python-checkins] closes bpo-38124: Fix bounds check in
 PyState_AddModule. (GH-16007)
Message-ID: <mailman.617.1568245407.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/39de95b746c990e6a2fe9af5fad01747f58b2e5f
commit: 39de95b746c990e6a2fe9af5fad01747f58b2e5f
branch: master
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-12T00:43:22+01:00
summary:

closes bpo-38124: Fix bounds check in PyState_AddModule. (GH-16007)

The >=, checking whether a module index was in already in the module-by-index list, needed to be strict.

Also, fold nested ifs into one and fix some bad spacing.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-12-00-14-01.bpo-38124.n6E0H7.rst
M Python/pystate.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-12-00-14-01.bpo-38124.n6E0H7.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-12-00-14-01.bpo-38124.n6E0H7.rst
new file mode 100644
index 000000000000..dca0ba5bc0cc
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-12-00-14-01.bpo-38124.n6E0H7.rst	
@@ -0,0 +1,2 @@
+Fix an off-by-one error in PyState_AddModule that could cause out-of-bounds
+memory access.
diff --git a/Python/pystate.c b/Python/pystate.c
index 02bc90394361..7dd8b7f866b5 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -684,7 +684,7 @@ _PyState_AddModule(PyObject* module, struct PyModuleDef* def)
         if (!state->modules_by_index)
             return -1;
     }
-    while(PyList_GET_SIZE(state->modules_by_index) <= def->m_base.m_index)
+    while (PyList_GET_SIZE(state->modules_by_index) <= def->m_base.m_index)
         if (PyList_Append(state->modules_by_index, Py_None) < 0)
             return -1;
     Py_INCREF(module);
@@ -702,13 +702,11 @@ PyState_AddModule(PyObject* module, struct PyModuleDef* def)
         return -1;
     }
     index = def->m_base.m_index;
-    if (state->modules_by_index) {
-        if(PyList_GET_SIZE(state->modules_by_index) >= index) {
-            if(module == PyList_GET_ITEM(state->modules_by_index, index)) {
-                Py_FatalError("PyState_AddModule: Module already added!");
-                return -1;
-            }
-        }
+    if (state->modules_by_index &&
+        index < PyList_GET_SIZE(state->modules_by_index) &&
+        module == PyList_GET_ITEM(state->modules_by_index, index)) {
+        Py_FatalError("PyState_AddModule: Module already added!");
+        return -1;
     }
     return _PyState_AddModule(module, def);
 }


From webhook-mailer at python.org  Wed Sep 11 20:02:54 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 00:02:54 -0000
Subject: [Python-checkins] closes bpo-38124: Fix bounds check in
 PyState_AddModule. (GH-16007)
Message-ID: <mailman.618.1568246575.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8892a1d685e4898b28961308b3c1447fe9ad3269
commit: 8892a1d685e4898b28961308b3c1447fe9ad3269
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T17:02:50-07:00
summary:

closes bpo-38124: Fix bounds check in PyState_AddModule. (GH-16007)


The >=, checking whether a module index was in already in the module-by-index list, needed to be strict.

Also, fold nested ifs into one and fix some bad spacing.
(cherry picked from commit 39de95b746c990e6a2fe9af5fad01747f58b2e5f)

Co-authored-by: Benjamin Peterson <benjamin at python.org>

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-12-00-14-01.bpo-38124.n6E0H7.rst
M Python/pystate.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-12-00-14-01.bpo-38124.n6E0H7.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-12-00-14-01.bpo-38124.n6E0H7.rst
new file mode 100644
index 000000000000..dca0ba5bc0cc
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-12-00-14-01.bpo-38124.n6E0H7.rst	
@@ -0,0 +1,2 @@
+Fix an off-by-one error in PyState_AddModule that could cause out-of-bounds
+memory access.
diff --git a/Python/pystate.c b/Python/pystate.c
index fc695c62a330..90cd56bb05f4 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -484,7 +484,7 @@ _PyState_AddModule(PyObject* module, struct PyModuleDef* def)
         if (!state->modules_by_index)
             return -1;
     }
-    while(PyList_GET_SIZE(state->modules_by_index) <= def->m_base.m_index)
+    while (PyList_GET_SIZE(state->modules_by_index) <= def->m_base.m_index)
         if (PyList_Append(state->modules_by_index, Py_None) < 0)
             return -1;
     Py_INCREF(module);
@@ -502,13 +502,11 @@ PyState_AddModule(PyObject* module, struct PyModuleDef* def)
         return -1;
     }
     index = def->m_base.m_index;
-    if (state->modules_by_index) {
-        if(PyList_GET_SIZE(state->modules_by_index) >= index) {
-            if(module == PyList_GET_ITEM(state->modules_by_index, index)) {
-                Py_FatalError("PyState_AddModule: Module already added!");
-                return -1;
-            }
-        }
+    if (state->modules_by_index &&
+        index < PyList_GET_SIZE(state->modules_by_index) &&
+        module == PyList_GET_ITEM(state->modules_by_index, index)) {
+        Py_FatalError("PyState_AddModule: Module already added!");
+        return -1;
     }
     return _PyState_AddModule(module, def);
 }


From webhook-mailer at python.org  Wed Sep 11 20:04:31 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 00:04:31 -0000
Subject: [Python-checkins] closes bpo-38124: Fix bounds check in
 PyState_AddModule. (GH-16007)
Message-ID: <mailman.619.1568246672.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a5a7102636de82e0687af7131357762337d49c7c
commit: a5a7102636de82e0687af7131357762337d49c7c
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-11T17:04:27-07:00
summary:

closes bpo-38124: Fix bounds check in PyState_AddModule. (GH-16007)


The >=, checking whether a module index was in already in the module-by-index list, needed to be strict.

Also, fold nested ifs into one and fix some bad spacing.
(cherry picked from commit 39de95b746c990e6a2fe9af5fad01747f58b2e5f)

Co-authored-by: Benjamin Peterson <benjamin at python.org>

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-12-00-14-01.bpo-38124.n6E0H7.rst
M Python/pystate.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-12-00-14-01.bpo-38124.n6E0H7.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-12-00-14-01.bpo-38124.n6E0H7.rst
new file mode 100644
index 000000000000..dca0ba5bc0cc
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-12-00-14-01.bpo-38124.n6E0H7.rst	
@@ -0,0 +1,2 @@
+Fix an off-by-one error in PyState_AddModule that could cause out-of-bounds
+memory access.
diff --git a/Python/pystate.c b/Python/pystate.c
index 8f30c94096b0..f2924a8ff263 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -685,7 +685,7 @@ _PyState_AddModule(PyObject* module, struct PyModuleDef* def)
         if (!state->modules_by_index)
             return -1;
     }
-    while(PyList_GET_SIZE(state->modules_by_index) <= def->m_base.m_index)
+    while (PyList_GET_SIZE(state->modules_by_index) <= def->m_base.m_index)
         if (PyList_Append(state->modules_by_index, Py_None) < 0)
             return -1;
     Py_INCREF(module);
@@ -703,13 +703,11 @@ PyState_AddModule(PyObject* module, struct PyModuleDef* def)
         return -1;
     }
     index = def->m_base.m_index;
-    if (state->modules_by_index) {
-        if(PyList_GET_SIZE(state->modules_by_index) >= index) {
-            if(module == PyList_GET_ITEM(state->modules_by_index, index)) {
-                Py_FatalError("PyState_AddModule: Module already added!");
-                return -1;
-            }
-        }
+    if (state->modules_by_index &&
+        index < PyList_GET_SIZE(state->modules_by_index) &&
+        module == PyList_GET_ITEM(state->modules_by_index, index)) {
+        Py_FatalError("PyState_AddModule: Module already added!");
+        return -1;
     }
     return _PyState_AddModule(module, def);
 }


From webhook-mailer at python.org  Wed Sep 11 22:50:34 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Thu, 12 Sep 2019 02:50:34 -0000
Subject: [Python-checkins] bpo-38096: Clean up the "struct sequence" /
 "named tuple" docs (GH-15895) (GH-15961)
Message-ID: <mailman.620.1568256635.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2bb6bf0c8cf863c057027b10751c0fb74189871f
commit: 2bb6bf0c8cf863c057027b10751c0fb74189871f
branch: 3.8
author: Paul Ganssle <paul at ganssle.io>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-11T19:50:29-07:00
summary:

bpo-38096: Clean up the "struct sequence" / "named tuple" docs (GH-15895) (GH-15961)

files:
M Doc/glossary.rst
M Doc/library/sys.rst
M Doc/whatsnew/3.3.rst
M Doc/whatsnew/3.4.rst
M Objects/floatobject.c
M Objects/longobject.c
M Python/sysmodule.c
M Python/thread.c

diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 0f2a3a1fdf05..84d0fcab9d75 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -739,17 +739,28 @@ Glossary
       also :term:`immutable`.
 
    named tuple
-      Any tuple-like class whose indexable elements are also accessible using
-      named attributes (for example, :func:`time.localtime` returns a
-      tuple-like object where the *year* is accessible either with an
-      index such as ``t[0]`` or with a named attribute like ``t.tm_year``).
-
-      A named tuple can be a built-in type such as :class:`time.struct_time`,
-      or it can be created with a regular class definition.  A full featured
-      named tuple can also be created with the factory function
-      :func:`collections.namedtuple`.  The latter approach automatically
-      provides extra features such as a self-documenting representation like
-      ``Employee(name='jones', title='programmer')``.
+      The term "named tuple" applies to any type or class that inherits from
+      tuple and whose indexable elements are also accessible using named
+      attributes.  The type or class may have other features as well.
+
+      Several built-in types are named tuples, including the values returned
+      by :func:`time.localtime` and :func:`os.stat`.  Another example is
+      :data:`sys.float_info`::
+
+           >>> sys.float_info[1]                   # indexed access
+           1024
+           >>> sys.float_info.max_exp              # named field access
+           1024
+           >>> isinstance(sys.float_info, tuple)   # kind of tuple
+           True
+
+      Some named tuples are built-in types (such as the above examples).
+      Alternatively, a named tuple can be created from a regular class
+      definition that inherits from :class:`tuple` and that defines named
+      fields.  Such as class can be written by hand or it can be created with
+      the factory function :func:`collections.namedtuple`.  The latter
+      technique also adds some extra methods that may not be found in
+      hand-written or built-in named tuples.
 
    namespace
       The place where a variable is stored.  Namespaces are implemented as
@@ -1032,14 +1043,6 @@ Glossary
       an :term:`expression` or one of several constructs with a keyword, such
       as :keyword:`if`, :keyword:`while` or :keyword:`for`.
 
-   struct sequence
-      A tuple with named elements. Struct sequences expose an interface similar
-      to :term:`named tuple` in that elements can be accessed either by
-      index or as an attribute. However, they do not have any of the named tuple
-      methods like :meth:`~collections.somenamedtuple._make` or
-      :meth:`~collections.somenamedtuple._asdict`. Examples of struct sequences
-      include :data:`sys.float_info` and the return value of :func:`os.stat`.
-
    text encoding
       A codec which encodes Unicode strings to bytes.
 
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 5f8afa8d6ed4..975c14c88824 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -408,7 +408,7 @@ always available.
 
 .. data:: flags
 
-   The :term:`struct sequence` *flags* exposes the status of command line
+   The :term:`named tuple` *flags* exposes the status of command line
    flags. The attributes are read only.
 
    ============================= =============================
@@ -450,7 +450,7 @@ always available.
 
 .. data:: float_info
 
-   A :term:`struct sequence` holding information about the float type. It
+   A :term:`named tuple` holding information about the float type. It
    contains low level information about the precision and internal
    representation.  The values correspond to the various floating-point
    constants defined in the standard header file :file:`float.h` for the 'C'
@@ -790,7 +790,7 @@ always available.
 
 .. data:: hash_info
 
-   A :term:`struct sequence` giving parameters of the numeric hash
+   A :term:`named tuple` giving parameters of the numeric hash
    implementation.  For more details about hashing of numeric types, see
    :ref:`numeric-hash`.
 
@@ -838,7 +838,7 @@ always available.
 
    This is called ``hexversion`` since it only really looks meaningful when viewed
    as the result of passing it to the built-in :func:`hex` function.  The
-   :term:`struct sequence`  :data:`sys.version_info` may be used for a more
+   :term:`named tuple`  :data:`sys.version_info` may be used for a more
    human-friendly encoding of the same information.
 
    More details of ``hexversion`` can be found at :ref:`apiabiversion`.
@@ -890,7 +890,7 @@ always available.
 
 .. data:: int_info
 
-   A :term:`struct sequence` that holds information about Python's internal
+   A :term:`named tuple` that holds information about Python's internal
    representation of integers.  The attributes are read only.
 
    .. tabularcolumns:: |l|L|
@@ -1480,7 +1480,7 @@ always available.
 
 .. data:: thread_info
 
-   A :term:`struct sequence` holding information about the thread
+   A :term:`named tuple` holding information about the thread
    implementation.
 
    .. tabularcolumns:: |l|p{0.7\linewidth}|
diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst
index ea036840d671..f1a033c6dae6 100644
--- a/Doc/whatsnew/3.3.rst
+++ b/Doc/whatsnew/3.3.rst
@@ -2002,8 +2002,8 @@ platform-independent fashion.  (Contributed by Ross Lagerwall in
 sys
 ---
 
-The :mod:`sys` module has a new :data:`~sys.thread_info` :term:`struct
-sequence` holding information about the thread implementation
+The :mod:`sys` module has a new :data:`~sys.thread_info` :term:`named
+tuple` holding information about the thread implementation
 (:issue:`11223`).
 
 
diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst
index 845e327cabc8..822ba81be8a7 100644
--- a/Doc/whatsnew/3.4.rst
+++ b/Doc/whatsnew/3.4.rst
@@ -1849,7 +1849,7 @@ Python's default implementation to a SipHash implementation on platforms that
 have a 64 bit data type.  Any performance differences in comparison with the
 older FNV algorithm are trivial.
 
-The PEP adds additional fields to the :attr:`sys.hash_info` struct sequence to
+The PEP adds additional fields to the :attr:`sys.hash_info` named tuple to
 describe the hash algorithm in use by the currently executing binary.  Otherwise,
 the PEP does not alter any existing CPython APIs.
 
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
index 15cbe5c9d8ba..d909e415c127 100644
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -43,7 +43,7 @@ static PyTypeObject FloatInfoType;
 PyDoc_STRVAR(floatinfo__doc__,
 "sys.float_info\n\
 \n\
-A structseq holding information about the float type. It contains low level\n\
+A named tuple holding information about the float type. It contains low level\n\
 information about the precision and internal representation. Please study\n\
 your system's :file:`float.h` for more information.");
 
diff --git a/Objects/longobject.c b/Objects/longobject.c
index 50ea2a4e54a4..708934c51fe2 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -5771,7 +5771,7 @@ static PyTypeObject Int_InfoType;
 PyDoc_STRVAR(int_info__doc__,
 "sys.int_info\n\
 \n\
-A struct sequence that holds information about Python's\n\
+A named tuple that holds information about Python's\n\
 internal representation of integers.  The attributes are read only.");
 
 static PyStructSequence_Field int_info_fields[] = {
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 11c20cf3c713..57f7a9a5a750 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -1162,7 +1162,7 @@ static PyTypeObject AsyncGenHooksType;
 PyDoc_STRVAR(asyncgen_hooks_doc,
 "asyncgen_hooks\n\
 \n\
-A struct sequence providing information about asynhronous\n\
+A named tuple providing information about asynchronous\n\
 generators hooks.  The attributes are read only.");
 
 static PyStructSequence_Field asyncgen_hooks_fields[] = {
@@ -1270,7 +1270,7 @@ static PyTypeObject Hash_InfoType;
 PyDoc_STRVAR(hash_info_doc,
 "hash_info\n\
 \n\
-A struct sequence providing parameters used for computing\n\
+A named tuple providing parameters used for computing\n\
 hashes. The attributes are read only.");
 
 static PyStructSequence_Field hash_info_fields[] = {
@@ -2321,17 +2321,17 @@ builtin_module_names -- tuple of module names built into this interpreter\n\
 copyright -- copyright notice pertaining to this interpreter\n\
 exec_prefix -- prefix used to find the machine-specific Python library\n\
 executable -- absolute path of the executable binary of the Python interpreter\n\
-float_info -- a struct sequence with information about the float implementation.\n\
+float_info -- a named tuple with information about the float implementation.\n\
 float_repr_style -- string indicating the style of repr() output for floats\n\
-hash_info -- a struct sequence with information about the hash algorithm.\n\
+hash_info -- a named tuple with information about the hash algorithm.\n\
 hexversion -- version information encoded as a single integer\n\
 implementation -- Python implementation information.\n\
-int_info -- a struct sequence with information about the int implementation.\n\
+int_info -- a named tuple with information about the int implementation.\n\
 maxsize -- the largest supported length of containers.\n\
 maxunicode -- the value of the largest Unicode code point\n\
 platform -- platform identifier\n\
 prefix -- prefix used to find the Python library\n\
-thread_info -- a struct sequence with information about the thread implementation.\n\
+thread_info -- a named tuple with information about the thread implementation.\n\
 version -- the version of this interpreter as a string\n\
 version_info -- version information as a named tuple\n\
 "
diff --git a/Python/thread.c b/Python/thread.c
index c5364f91948f..c36ce6ff9835 100644
--- a/Python/thread.c
+++ b/Python/thread.c
@@ -147,7 +147,7 @@ PyThread_tss_is_created(Py_tss_t *key)
 PyDoc_STRVAR(threadinfo__doc__,
 "sys.thread_info\n\
 \n\
-A struct sequence holding information about the thread implementation.");
+A named tuple holding information about the thread implementation.");
 
 static PyStructSequence_Field threadinfo_fields[] = {
     {"name",    "name of the thread implementation"},


From webhook-mailer at python.org  Wed Sep 11 22:50:57 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Thu, 12 Sep 2019 02:50:57 -0000
Subject: [Python-checkins] bpo-38096: Clean up the "struct sequence" /
 "named tuple" docs (GH-15895) (GH-15962)
Message-ID: <mailman.621.1568256658.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b7a310d865347eabc60d387dbec82ee408315050
commit: b7a310d865347eabc60d387dbec82ee408315050
branch: 3.7
author: Paul Ganssle <paul at ganssle.io>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-11T19:50:51-07:00
summary:

bpo-38096: Clean up the "struct sequence" / "named tuple" docs (GH-15895) (GH-15962)

files:
M Doc/glossary.rst
M Doc/library/sys.rst
M Doc/whatsnew/3.3.rst
M Doc/whatsnew/3.4.rst
M Objects/floatobject.c
M Objects/longobject.c
M Python/sysmodule.c
M Python/thread.c

diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index df6f6b646295..3aa54b9eff9e 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -735,17 +735,28 @@ Glossary
       also :term:`immutable`.
 
    named tuple
-      Any tuple-like class whose indexable elements are also accessible using
-      named attributes (for example, :func:`time.localtime` returns a
-      tuple-like object where the *year* is accessible either with an
-      index such as ``t[0]`` or with a named attribute like ``t.tm_year``).
-
-      A named tuple can be a built-in type such as :class:`time.struct_time`,
-      or it can be created with a regular class definition.  A full featured
-      named tuple can also be created with the factory function
-      :func:`collections.namedtuple`.  The latter approach automatically
-      provides extra features such as a self-documenting representation like
-      ``Employee(name='jones', title='programmer')``.
+      The term "named tuple" applies to any type or class that inherits from
+      tuple and whose indexable elements are also accessible using named
+      attributes.  The type or class may have other features as well.
+
+      Several built-in types are named tuples, including the values returned
+      by :func:`time.localtime` and :func:`os.stat`.  Another example is
+      :data:`sys.float_info`::
+
+           >>> sys.float_info[1]                   # indexed access
+           1024
+           >>> sys.float_info.max_exp              # named field access
+           1024
+           >>> isinstance(sys.float_info, tuple)   # kind of tuple
+           True
+
+      Some named tuples are built-in types (such as the above examples).
+      Alternatively, a named tuple can be created from a regular class
+      definition that inherits from :class:`tuple` and that defines named
+      fields.  Such as class can be written by hand or it can be created with
+      the factory function :func:`collections.namedtuple`.  The latter
+      technique also adds some extra methods that may not be found in
+      hand-written or built-in named tuples.
 
    namespace
       The place where a variable is stored.  Namespaces are implemented as
@@ -1028,14 +1039,6 @@ Glossary
       an :term:`expression` or one of several constructs with a keyword, such
       as :keyword:`if`, :keyword:`while` or :keyword:`for`.
 
-   struct sequence
-      A tuple with named elements. Struct sequences expose an interface similar
-      to :term:`named tuple` in that elements can be accessed either by
-      index or as an attribute. However, they do not have any of the named tuple
-      methods like :meth:`~collections.somenamedtuple._make` or
-      :meth:`~collections.somenamedtuple._asdict`. Examples of struct sequences
-      include :data:`sys.float_info` and the return value of :func:`os.stat`.
-
    text encoding
       A codec which encodes Unicode strings to bytes.
 
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 121464f9470b..ae946e830f6b 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -324,7 +324,7 @@ always available.
 
 .. data:: flags
 
-   The :term:`struct sequence` *flags* exposes the status of command line
+   The :term:`named tuple` *flags* exposes the status of command line
    flags. The attributes are read only.
 
    ============================= =============================
@@ -366,7 +366,7 @@ always available.
 
 .. data:: float_info
 
-   A :term:`struct sequence` holding information about the float type. It
+   A :term:`named tuple` holding information about the float type. It
    contains low level information about the precision and internal
    representation.  The values correspond to the various floating-point
    constants defined in the standard header file :file:`float.h` for the 'C'
@@ -716,7 +716,7 @@ always available.
 
 .. data:: hash_info
 
-   A :term:`struct sequence` giving parameters of the numeric hash
+   A :term:`named tuple` giving parameters of the numeric hash
    implementation.  For more details about hashing of numeric types, see
    :ref:`numeric-hash`.
 
@@ -764,7 +764,7 @@ always available.
 
    This is called ``hexversion`` since it only really looks meaningful when viewed
    as the result of passing it to the built-in :func:`hex` function.  The
-   :term:`struct sequence`  :data:`sys.version_info` may be used for a more
+   :term:`named tuple`  :data:`sys.version_info` may be used for a more
    human-friendly encoding of the same information.
 
    More details of ``hexversion`` can be found at :ref:`apiabiversion`.
@@ -816,7 +816,7 @@ always available.
 
 .. data:: int_info
 
-   A :term:`struct sequence` that holds information about Python's internal
+   A :term:`named tuple` that holds information about Python's internal
    representation of integers.  The attributes are read only.
 
    .. tabularcolumns:: |l|L|
@@ -1423,7 +1423,7 @@ always available.
 
 .. data:: thread_info
 
-   A :term:`struct sequence` holding information about the thread
+   A :term:`named tuple` holding information about the thread
    implementation.
 
    .. tabularcolumns:: |l|p{0.7\linewidth}|
diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst
index 8862b37e0628..eac200b2cc5d 100644
--- a/Doc/whatsnew/3.3.rst
+++ b/Doc/whatsnew/3.3.rst
@@ -2002,8 +2002,8 @@ platform-independent fashion.  (Contributed by Ross Lagerwall in
 sys
 ---
 
-The :mod:`sys` module has a new :data:`~sys.thread_info` :term:`struct
-sequence` holding information about the thread implementation
+The :mod:`sys` module has a new :data:`~sys.thread_info` :term:`named
+tuple` holding information about the thread implementation
 (:issue:`11223`).
 
 
diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst
index 845e327cabc8..822ba81be8a7 100644
--- a/Doc/whatsnew/3.4.rst
+++ b/Doc/whatsnew/3.4.rst
@@ -1849,7 +1849,7 @@ Python's default implementation to a SipHash implementation on platforms that
 have a 64 bit data type.  Any performance differences in comparison with the
 older FNV algorithm are trivial.
 
-The PEP adds additional fields to the :attr:`sys.hash_info` struct sequence to
+The PEP adds additional fields to the :attr:`sys.hash_info` named tuple to
 describe the hash algorithm in use by the currently executing binary.  Otherwise,
 the PEP does not alter any existing CPython APIs.
 
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
index 67f9e5d5b4ef..b4b1e5cff6bc 100644
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -43,7 +43,7 @@ static PyTypeObject FloatInfoType;
 PyDoc_STRVAR(floatinfo__doc__,
 "sys.float_info\n\
 \n\
-A structseq holding information about the float type. It contains low level\n\
+A named tuple holding information about the float type. It contains low level\n\
 information about the precision and internal representation. Please study\n\
 your system's :file:`float.h` for more information.");
 
diff --git a/Objects/longobject.c b/Objects/longobject.c
index 7628a114e384..202f652fc6df 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -5424,7 +5424,7 @@ static PyTypeObject Int_InfoType;
 PyDoc_STRVAR(int_info__doc__,
 "sys.int_info\n\
 \n\
-A struct sequence that holds information about Python's\n\
+A named tuple that holds information about Python's\n\
 internal representation of integers.  The attributes are read only.");
 
 static PyStructSequence_Field int_info_fields[] = {
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 942a8b6becd9..b953a0009775 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -836,7 +836,7 @@ static PyTypeObject AsyncGenHooksType;
 PyDoc_STRVAR(asyncgen_hooks_doc,
 "asyncgen_hooks\n\
 \n\
-A struct sequence providing information about asynhronous\n\
+A named tuple providing information about asynchronous\n\
 generators hooks.  The attributes are read only.");
 
 static PyStructSequence_Field asyncgen_hooks_fields[] = {
@@ -943,7 +943,7 @@ static PyTypeObject Hash_InfoType;
 PyDoc_STRVAR(hash_info_doc,
 "hash_info\n\
 \n\
-A struct sequence providing parameters used for computing\n\
+A named tuple providing parameters used for computing\n\
 hashes. The attributes are read only.");
 
 static PyStructSequence_Field hash_info_fields[] = {
@@ -1967,17 +1967,17 @@ builtin_module_names -- tuple of module names built into this interpreter\n\
 copyright -- copyright notice pertaining to this interpreter\n\
 exec_prefix -- prefix used to find the machine-specific Python library\n\
 executable -- absolute path of the executable binary of the Python interpreter\n\
-float_info -- a struct sequence with information about the float implementation.\n\
+float_info -- a named tuple with information about the float implementation.\n\
 float_repr_style -- string indicating the style of repr() output for floats\n\
-hash_info -- a struct sequence with information about the hash algorithm.\n\
+hash_info -- a named tuple with information about the hash algorithm.\n\
 hexversion -- version information encoded as a single integer\n\
 implementation -- Python implementation information.\n\
-int_info -- a struct sequence with information about the int implementation.\n\
+int_info -- a named tuple with information about the int implementation.\n\
 maxsize -- the largest supported length of containers.\n\
 maxunicode -- the value of the largest Unicode code point\n\
 platform -- platform identifier\n\
 prefix -- prefix used to find the Python library\n\
-thread_info -- a struct sequence with information about the thread implementation.\n\
+thread_info -- a named tuple with information about the thread implementation.\n\
 version -- the version of this interpreter as a string\n\
 version_info -- version information as a named tuple\n\
 "
diff --git a/Python/thread.c b/Python/thread.c
index 07743840693b..e0cbdeef8ba2 100644
--- a/Python/thread.c
+++ b/Python/thread.c
@@ -147,7 +147,7 @@ PyThread_tss_is_created(Py_tss_t *key)
 PyDoc_STRVAR(threadinfo__doc__,
 "sys.thread_info\n\
 \n\
-A struct sequence holding information about the thread implementation.");
+A named tuple holding information about the thread implementation.");
 
 static PyStructSequence_Field threadinfo_fields[] = {
     {"name",    "name of the thread implementation"},


From webhook-mailer at python.org  Thu Sep 12 03:41:36 2019
From: webhook-mailer at python.org (Carol Willing)
Date: Thu, 12 Sep 2019 07:41:36 -0000
Subject: [Python-checkins] Correct typos in the codecs module documentation
 (#15135)
Message-ID: <mailman.622.1568274098.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/891e9e3b44c99d643dc309a4e63082451271b136
commit: 891e9e3b44c99d643dc309a4e63082451271b136
branch: master
author: G?ry Ogam <gery.ogam at gmail.com>
committer: Carol Willing <carolcode at willingconsulting.com>
date: 2019-09-12T08:41:32+01:00
summary:

Correct typos in the codecs module documentation (#15135)

files:
M Doc/library/codecs.rst

diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
index 5048621bf0ad..f071057293ee 100644
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -292,7 +292,7 @@ Error Handlers
 
 To simplify and standardize error handling,
 codecs may implement different error handling schemes by
-accepting the *errors* string argument.  The following string values are
+accepting the *errors* string argument. The following string values are
 defined and implemented by all standard Python codecs:
 
 .. tabularcolumns:: |l|L|
@@ -301,11 +301,11 @@ defined and implemented by all standard Python codecs:
 | Value                   | Meaning                                       |
 +=========================+===============================================+
 | ``'strict'``            | Raise :exc:`UnicodeError` (or a subclass);    |
-|                         | this is the default.  Implemented in          |
+|                         | this is the default. Implemented in           |
 |                         | :func:`strict_errors`.                        |
 +-------------------------+-----------------------------------------------+
 | ``'ignore'``            | Ignore the malformed data and continue        |
-|                         | without further notice.  Implemented in       |
+|                         | without further notice. Implemented in        |
 |                         | :func:`ignore_errors`.                        |
 +-------------------------+-----------------------------------------------+
 
@@ -327,11 +327,11 @@ The following error handlers are only applicable to
 |                         | marker; Python will use the official          |
 |                         | ``U+FFFD`` REPLACEMENT CHARACTER for the      |
 |                         | built-in codecs on decoding, and '?' on       |
-|                         | encoding.  Implemented in                     |
+|                         | encoding. Implemented in                      |
 |                         | :func:`replace_errors`.                       |
 +-------------------------+-----------------------------------------------+
 | ``'xmlcharrefreplace'`` | Replace with the appropriate XML character    |
-|                         | reference (only for encoding).  Implemented   |
+|                         | reference (only for encoding). Implemented    |
 |                         | in :func:`xmlcharrefreplace_errors`.          |
 +-------------------------+-----------------------------------------------+
 | ``'backslashreplace'``  | Replace with backslashed escape sequences.    |
@@ -339,15 +339,15 @@ The following error handlers are only applicable to
 |                         | :func:`backslashreplace_errors`.              |
 +-------------------------+-----------------------------------------------+
 | ``'namereplace'``       | Replace with ``\N{...}`` escape sequences     |
-|                         | (only for encoding).  Implemented in          |
+|                         | (only for encoding). Implemented in           |
 |                         | :func:`namereplace_errors`.                   |
 +-------------------------+-----------------------------------------------+
 | ``'surrogateescape'``   | On decoding, replace byte with individual     |
 |                         | surrogate code ranging from ``U+DC80`` to     |
-|                         | ``U+DCFF``.  This code will then be turned    |
+|                         | ``U+DCFF``. This code will then be turned     |
 |                         | back into the same byte when the              |
 |                         | ``'surrogateescape'`` error handler is used   |
-|                         | when encoding the data.  (See :pep:`383` for  |
+|                         | when encoding the data. (See :pep:`383` for   |
 |                         | more.)                                        |
 +-------------------------+-----------------------------------------------+
 
@@ -357,7 +357,7 @@ In addition, the following error handler is specific to the given codecs:
 | Value             | Codecs                 | Meaning                                   |
 +===================+========================+===========================================+
 |``'surrogatepass'``| utf-8, utf-16, utf-32, | Allow encoding and decoding of surrogate  |
-|                   | utf-16-be, utf-16-le,  | codes.  These codecs normally treat the   |
+|                   | utf-16-be, utf-16-le,  | codes. These codecs normally treat the    |
 |                   | utf-32-be, utf-32-le   | presence of surrogates as an error.       |
 +-------------------+------------------------+-------------------------------------------+
 
@@ -388,9 +388,9 @@ handler:
    error handler must either raise this or a different exception, or return a
    tuple with a replacement for the unencodable part of the input and a position
    where encoding should continue. The replacement may be either :class:`str` or
-   :class:`bytes`.  If the replacement is bytes, the encoder will simply copy
+   :class:`bytes`. If the replacement is bytes, the encoder will simply copy
    them into the output buffer. If the replacement is a string, the encoder will
-   encode the replacement.  Encoding continues on original input at the
+   encode the replacement. Encoding continues on original input at the
    specified position. Negative position values will be treated as being
    relative to the end of the input string. If the resulting position is out of
    bound an :exc:`IndexError` will be raised.
@@ -484,7 +484,7 @@ function interfaces of the stateless encoder and decoder:
 .. method:: Codec.decode(input[, errors])
 
    Decodes the object *input* and returns a tuple (output object, length
-   consumed).  For instance, for a :term:`text encoding`, decoding converts
+   consumed). For instance, for a :term:`text encoding`, decoding converts
    a bytes object encoded using a particular
    character set encoding to a string object.
 
@@ -568,7 +568,7 @@ define in order to be compatible with the Python codec registry.
       implementation should make sure that ``0`` is the most common
       state. (States that are more complicated than integers can be converted
       into an integer by marshaling/pickling the state and encoding the bytes
-      of the resulting string into an integer).
+      of the resulting string into an integer.)
 
 
    .. method:: setstate(state)
@@ -751,7 +751,7 @@ compatible with the Python codec registry.
       number of encoded bytes or code points to read
       for decoding. The decoder can modify this setting as
       appropriate. The default value -1 indicates to read and decode as much as
-      possible.  This parameter is intended to
+      possible. This parameter is intended to
       prevent having to decode huge files in one step.
 
       The *firstline* flag indicates that
@@ -780,8 +780,8 @@ compatible with the Python codec registry.
       Read all lines available on the input stream and return them as a list of
       lines.
 
-      Line-endings are implemented using the codec's decoder method and are
-      included in the list entries if *keepends* is true.
+      Line-endings are implemented using the codec's :meth:`decode` method and
+      are included in the list entries if *keepends* is true.
 
       *sizehint*, if given, is passed as the *size* argument to the stream's
       :meth:`read` method.
@@ -791,7 +791,7 @@ compatible with the Python codec registry.
 
       Resets the codec buffers used for keeping state.
 
-      Note that no stream repositioning should take place.  This method is
+      Note that no stream repositioning should take place. This method is
       primarily intended to be able to recover from decoding errors.
 
 
@@ -841,7 +841,7 @@ The design is such that one can use the factory functions returned by the
    code calling :meth:`read` and :meth:`write`, while *Reader* and *Writer*
    work on the backend?? the data in *stream*.
 
-   You can use these objects to do transparent transcodings from e.g. Latin-1
+   You can use these objects to do transparent transcodings, e.g., from Latin-1
    to UTF-8 and back.
 
    The *stream* argument must be a file-like object.
@@ -866,10 +866,10 @@ Encodings and Unicode
 ---------------------
 
 Strings are stored internally as sequences of code points in
-range ``0x0``--``0x10FFFF``.  (See :pep:`393` for
+range ``0x0``--``0x10FFFF``. (See :pep:`393` for
 more details about the implementation.)
 Once a string object is used outside of CPU and memory, endianness
-and how these arrays are stored as bytes become an issue.  As with other
+and how these arrays are stored as bytes become an issue. As with other
 codecs, serialising a string into a sequence of bytes is known as *encoding*,
 and recreating the string from the sequence of bytes is known as *decoding*.
 There are a variety of different text serialisation codecs, which are
@@ -964,7 +964,7 @@ to determine the byte order used for generating the byte sequence, but as a
 signature that helps in guessing the encoding. On encoding the utf-8-sig codec
 will write ``0xef``, ``0xbb``, ``0xbf`` as the first three bytes to the file. On
 decoding ``utf-8-sig`` will skip those three bytes if they appear as the first
-three bytes in the file.  In UTF-8, the use of the BOM is discouraged and
+three bytes in the file. In UTF-8, the use of the BOM is discouraged and
 should generally be avoided.
 
 
@@ -984,7 +984,7 @@ e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec.
 .. impl-detail::
 
    Some common encodings can bypass the codecs lookup machinery to
-   improve performance.  These optimization opportunities are only
+   improve performance. These optimization opportunities are only
    recognized by CPython for a limited set of (case insensitive)
    aliases: utf-8, utf8, latin-1, latin1, iso-8859-1, iso8859-1, mbcs
    (Windows only), ascii, us-ascii, utf-16, utf16, utf-32, utf32, and
@@ -1145,7 +1145,7 @@ particular, the following variants typically exist:
 | iso2022_kr      | csiso2022kr, iso2022kr,        | Korean                         |
 |                 | iso-2022-kr                    |                                |
 +-----------------+--------------------------------+--------------------------------+
-| latin_1         | iso-8859-1, iso8859-1, 8859,   | West Europe                    |
+| latin_1         | iso-8859-1, iso8859-1, 8859,   | Western Europe                 |
 |                 | cp819, latin, latin1, L1       |                                |
 +-----------------+--------------------------------+--------------------------------+
 | iso8859_2       | iso-8859-2, latin2, L2         | Central and Eastern Europe     |
@@ -1250,11 +1250,11 @@ Python Specific Encodings
 -------------------------
 
 A number of predefined codecs are specific to Python, so their codec names have
-no meaning outside Python.  These are listed in the tables below based on the
+no meaning outside Python. These are listed in the tables below based on the
 expected input and output types (note that while text encodings are the most
 common use case for codecs, the underlying codec infrastructure supports
-arbitrary data transforms rather than just text encodings).  For asymmetric
-codecs, the stated purpose describes the encoding direction.
+arbitrary data transforms rather than just text encodings). For asymmetric
+codecs, the stated meaning describes the encoding direction.
 
 Text Encodings
 ^^^^^^^^^^^^^^
@@ -1266,27 +1266,27 @@ encodings.
 .. tabularcolumns:: |l|p{0.3\linewidth}|p{0.3\linewidth}|
 
 +--------------------+---------+---------------------------+
-| Codec              | Aliases | Purpose                   |
+| Codec              | Aliases | Meaning                   |
 +====================+=========+===========================+
-| idna               |         | Implements :rfc:`3490`,   |
+| idna               |         | Implement :rfc:`3490`,    |
 |                    |         | see also                  |
 |                    |         | :mod:`encodings.idna`.    |
 |                    |         | Only ``errors='strict'``  |
 |                    |         | is supported.             |
 +--------------------+---------+---------------------------+
-| mbcs               | ansi,   | Windows only: Encode      |
+| mbcs               | ansi,   | Windows only: Encode the  |
 |                    | dbcs    | operand according to the  |
-|                    |         | ANSI codepage (CP_ACP)    |
+|                    |         | ANSI codepage (CP_ACP).   |
 +--------------------+---------+---------------------------+
-| oem                |         | Windows only: Encode      |
+| oem                |         | Windows only: Encode the  |
 |                    |         | operand according to the  |
-|                    |         | OEM codepage (CP_OEMCP)   |
+|                    |         | OEM codepage (CP_OEMCP).  |
 |                    |         |                           |
 |                    |         | .. versionadded:: 3.6     |
 +--------------------+---------+---------------------------+
-| palmos             |         | Encoding of PalmOS 3.5    |
+| palmos             |         | Encoding of PalmOS 3.5.   |
 +--------------------+---------+---------------------------+
-| punycode           |         | Implements :rfc:`3492`.   |
+| punycode           |         | Implement :rfc:`3492`.    |
 |                    |         | Stateful codecs are not   |
 |                    |         | supported.                |
 +--------------------+---------+---------------------------+
@@ -1309,8 +1309,8 @@ encodings.
 |                    |         | literal in ASCII-encoded  |
 |                    |         | Python source code,       |
 |                    |         | except that quotes are    |
-|                    |         | not escaped. Decodes from |
-|                    |         | Latin-1 source code.      |
+|                    |         | not escaped. Decode       |
+|                    |         | from Latin-1 source code. |
 |                    |         | Beware that Python source |
 |                    |         | code actually uses UTF-8  |
 |                    |         | by default.               |
@@ -1326,19 +1326,19 @@ Binary Transforms
 ^^^^^^^^^^^^^^^^^
 
 The following codecs provide binary transforms: :term:`bytes-like object`
-to :class:`bytes` mappings.  They are not supported by :meth:`bytes.decode`
+to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode`
 (which only produces :class:`str` output).
 
 
 .. tabularcolumns:: |l|L|L|L|
 
 +----------------------+------------------+------------------------------+------------------------------+
-| Codec                | Aliases          | Purpose                      | Encoder / decoder            |
+| Codec                | Aliases          | Meaning                      | Encoder / decoder            |
 +======================+==================+==============================+==============================+
-| base64_codec [#b64]_ | base64, base_64  | Convert operand to multiline | :meth:`base64.encodebytes` / |
-|                      |                  | MIME base64 (the result      | :meth:`base64.decodebytes`   |
-|                      |                  | always includes a trailing   |                              |
-|                      |                  | ``'\n'``)                    |                              |
+| base64_codec [#b64]_ | base64, base_64  | Convert the operand to       | :meth:`base64.encodebytes` / |
+|                      |                  | multiline MIME base64 (the   | :meth:`base64.decodebytes`   |
+|                      |                  | result always includes a     |                              |
+|                      |                  | trailing ``'\n'``).          |                              |
 |                      |                  |                              |                              |
 |                      |                  | .. versionchanged:: 3.4      |                              |
 |                      |                  |    accepts any               |                              |
@@ -1346,23 +1346,23 @@ to :class:`bytes` mappings.  They are not supported by :meth:`bytes.decode`
 |                      |                  |    as input for encoding and |                              |
 |                      |                  |    decoding                  |                              |
 +----------------------+------------------+------------------------------+------------------------------+
-| bz2_codec            | bz2              | Compress the operand         | :meth:`bz2.compress` /       |
-|                      |                  | using bz2                    | :meth:`bz2.decompress`       |
+| bz2_codec            | bz2              | Compress the operand using   | :meth:`bz2.compress` /       |
+|                      |                  | bz2.                         | :meth:`bz2.decompress`       |
 +----------------------+------------------+------------------------------+------------------------------+
-| hex_codec            | hex              | Convert operand to           | :meth:`binascii.b2a_hex` /   |
+| hex_codec            | hex              | Convert the operand to       | :meth:`binascii.b2a_hex` /   |
 |                      |                  | hexadecimal                  | :meth:`binascii.a2b_hex`     |
 |                      |                  | representation, with two     |                              |
-|                      |                  | digits per byte              |                              |
+|                      |                  | digits per byte.             |                              |
 +----------------------+------------------+------------------------------+------------------------------+
-| quopri_codec         | quopri,          | Convert operand to MIME      | :meth:`quopri.encode` with   |
-|                      | quotedprintable, | quoted printable             | ``quotetabs=True`` /         |
+| quopri_codec         | quopri,          | Convert the operand to MIME  | :meth:`quopri.encode` with   |
+|                      | quotedprintable, | quoted printable.            | ``quotetabs=True`` /         |
 |                      | quoted_printable |                              | :meth:`quopri.decode`        |
 +----------------------+------------------+------------------------------+------------------------------+
 | uu_codec             | uu               | Convert the operand using    | :meth:`uu.encode` /          |
-|                      |                  | uuencode                     | :meth:`uu.decode`            |
+|                      |                  | uuencode.                    | :meth:`uu.decode`            |
 +----------------------+------------------+------------------------------+------------------------------+
-| zlib_codec           | zip, zlib        | Compress the operand         | :meth:`zlib.compress` /      |
-|                      |                  | using gzip                   | :meth:`zlib.decompress`      |
+| zlib_codec           | zip, zlib        | Compress the operand using   | :meth:`zlib.compress` /      |
+|                      |                  | gzip.                        | :meth:`zlib.decompress`      |
 +----------------------+------------------+------------------------------+------------------------------+
 
 .. [#b64] In addition to :term:`bytes-like objects <bytes-like object>`,
@@ -1382,16 +1382,17 @@ Text Transforms
 ^^^^^^^^^^^^^^^
 
 The following codec provides a text transform: a :class:`str` to :class:`str`
-mapping.  It is not supported by :meth:`str.encode` (which only produces
+mapping. It is not supported by :meth:`str.encode` (which only produces
 :class:`bytes` output).
 
 .. tabularcolumns:: |l|l|L|
 
 +--------------------+---------+---------------------------+
-| Codec              | Aliases | Purpose                   |
+| Codec              | Aliases | Meaning                   |
 +====================+=========+===========================+
-| rot_13             | rot13   | Returns the Caesar-cypher |
-|                    |         | encryption of the operand |
+| rot_13             | rot13   | Return the Caesar-cypher  |
+|                    |         | encryption of the         |
+|                    |         | operand.                  |
 +--------------------+---------+---------------------------+
 
 .. versionadded:: 3.2
@@ -1429,7 +1430,7 @@ conversion between Unicode and ACE, separating an input string into labels
 based on the separator characters defined in :rfc:`section 3.1 of RFC 3490 <3490#section-3.1>`
 and converting each label to ACE as required, and conversely separating an input
 byte string into labels based on the ``.`` separator and converting any ACE
-labels found into unicode.  Furthermore, the :mod:`socket` module
+labels found into unicode. Furthermore, the :mod:`socket` module
 transparently converts Unicode host names to ACE, so that applications need not
 be concerned about converting host names themselves when they pass them to the
 socket module. On top of that, modules that have host names as function
@@ -1438,7 +1439,7 @@ names (:mod:`http.client` then also transparently sends an IDNA hostname in the
 :mailheader:`Host` field if it sends that field at all).
 
 When receiving host names from the wire (such as in reverse name lookup), no
-automatic conversion to Unicode is performed: Applications wishing to present
+automatic conversion to Unicode is performed: applications wishing to present
 such host names to the user should decode them to Unicode.
 
 The module :mod:`encodings.idna` also implements the nameprep procedure, which
@@ -1470,7 +1471,7 @@ functions can be used directly if desired.
 .. module:: encodings.mbcs
    :synopsis: Windows ANSI codepage
 
-Encode operand according to the ANSI codepage (CP_ACP).
+This module implements the ANSI codepage (CP_ACP).
 
 .. availability:: Windows only.
 
@@ -1489,7 +1490,7 @@ Encode operand according to the ANSI codepage (CP_ACP).
    :synopsis: UTF-8 codec with BOM signature
 .. moduleauthor:: Walter D?rwald
 
-This module implements a variant of the UTF-8 codec: On encoding a UTF-8 encoded
+This module implements a variant of the UTF-8 codec. On encoding, a UTF-8 encoded
 BOM will be prepended to the UTF-8 encoded bytes. For the stateful encoder this
-is only done once (on the first write to the byte stream).  For decoding an
+is only done once (on the first write to the byte stream). On decoding, an
 optional UTF-8 encoded BOM at the start of the data will be skipped.


From webhook-mailer at python.org  Thu Sep 12 04:13:26 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Thu, 12 Sep 2019 08:13:26 -0000
Subject: [Python-checkins] bpo-38120: Fix DeprecationWarning in test_random
 for invalid type of arguments to random.seed. (GH-15987)
Message-ID: <mailman.623.1568276008.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a06d683d7fd88721eaf59abcf5b02eb82045c7b1
commit: a06d683d7fd88721eaf59abcf5b02eb82045c7b1
branch: master
author: Xtreak <tir.karthi at gmail.com>
committer: Serhiy Storchaka <storchaka at gmail.com>
date: 2019-09-12T11:13:20+03:00
summary:

bpo-38120: Fix DeprecationWarning in test_random for invalid type of arguments to random.seed. (GH-15987)

files:
M Lib/test/test_random.py

diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py
index 899ca108c65d..f59c5652b5dd 100644
--- a/Lib/test/test_random.py
+++ b/Lib/test/test_random.py
@@ -41,11 +41,17 @@ def test_seedargs(self):
         class MySeed(object):
             def __hash__(self):
                 return -1729
-        for arg in [None, 0, 0, 1, 1, -1, -1, 10**20, -(10**20),
-                    3.14, 1+2j, 'a', tuple('abc'), MySeed()]:
+        for arg in [None, 0, 1, -1, 10**20, -(10**20),
+                    3.14, 'a']:
             self.gen.seed(arg)
+
+        for arg in [1+2j, tuple('abc'), MySeed()]:
+            with self.assertWarns(DeprecationWarning):
+                self.gen.seed(arg)
+
         for arg in [list(range(3)), dict(one=1)]:
-            self.assertRaises(TypeError, self.gen.seed, arg)
+            with self.assertWarns(DeprecationWarning):
+                self.assertRaises(TypeError, self.gen.seed, arg)
         self.assertRaises(TypeError, self.gen.seed, 1, 2, 3, 4)
         self.assertRaises(TypeError, type(self.gen), [])
 


From webhook-mailer at python.org  Thu Sep 12 05:03:35 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Thu, 12 Sep 2019 09:03:35 -0000
Subject: [Python-checkins] bpo-32820: __format__ method for ipaddress (#5627)
Message-ID: <mailman.624.1568279015.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f9c95a4ba24c52eb1c052e3052d677e90a429a9a
commit: f9c95a4ba24c52eb1c052e3052d677e90a429a9a
branch: master
author: ewosborne <ewosborne at users.noreply.github.com>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-12T10:03:31+01:00
summary:

bpo-32820: __format__ method for ipaddress (#5627)

* bits method and test_bits

* Cleaned up assert string

* blurb

* added docstring

* Faster method, per Eric Smith

* redoing as __format__

* added ipv6 method

* test cases and cleanup

* updated news

* cleanup and NEWS.d

* cleaned up old NEWS

* removed cut and paste leftover

* one more cleanup

* moved to regexp, moved away from v4- and v6-specific versions of __format__

* More cleanup, added ipv6 test cases

* more cleanup

* more cleanup

* cleanup

* cleanup

* cleanup per review, part 1

* addressed review comments around help string and regexp matching

* wrapped v6 test strings. contiguous integers: break at 72char. with underscores: break so that it looks clean.

*  's' and '' tests for pv4 and ipv6

* whitespace cleanup

* Remove trailing whitespace

* Remove more trailing whitespace

* Remove an excess blank line

files:
A Misc/NEWS.d/next/Library/2018-02-13-12-25-43.bpo-32820.0stF0u.rst
M Lib/ipaddress.py
M Lib/test/test_ipaddress.py

diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
index 873c7644081a..c389f0528823 100644
--- a/Lib/ipaddress.py
+++ b/Lib/ipaddress.py
@@ -618,6 +618,78 @@ def _get_address_key(self):
     def __reduce__(self):
         return self.__class__, (self._ip,)
 
+    def __format__(self, fmt):
+        """Returns an IP address as a formatted string.
+
+        Supported presentation types are:
+        's': returns the IP address as a string (default)
+        'b' or 'n': converts to binary and returns a zero-padded string
+        'X' or 'x': converts to upper- or lower-case hex and returns a zero-padded string
+
+        For binary and hex presentation types, the alternate form specifier
+        '#' and the grouping option '_' are supported.
+        """
+
+
+        # Support string formatting
+        if not fmt or fmt[-1] == 's':
+            # let format() handle it
+            return format(str(self), fmt)
+
+        # From here on down, support for 'bnXx'
+
+        import re
+        fmt_re = '^(?P<alternate>#?)(?P<grouping>_?)(?P<fmt_base>[xbnX]){1}$'
+        m = re.match(fmt_re, fmt)
+        if not m:
+            return super().__format__(fmt)
+
+        groupdict = m.groupdict()
+        alternate = groupdict['alternate']
+        grouping = groupdict['grouping']
+        fmt_base = groupdict['fmt_base']
+
+        # Set some defaults
+        if fmt_base == 'n':
+            if self._version == 4:
+                fmt_base = 'b'  # Binary is default for ipv4
+            if self._version == 6:
+                fmt_base = 'x'  # Hex is default for ipv6
+
+        # Handle binary formatting
+        if fmt_base == 'b':
+            if self._version == 4:
+                # resulting string is '0b' + 32 bits
+                #  plus 7 _ if needed
+                padlen = IPV4LENGTH+2 + (7*len(grouping))
+            elif self._version == 6:
+                # resulting string is '0b' + 128 bits
+                #  plus 31 _ if needed
+                padlen = IPV6LENGTH+2 + (31*len(grouping))
+
+        # Handle hex formatting
+        elif fmt_base in 'Xx':
+            if self._version == 4:
+                # resulting string is '0x' + 8 hex digits
+                # plus a single _ if needed
+                padlen = int(IPV4LENGTH/4)+2 + len(grouping)
+            elif self._version == 6:
+                # resulting string is '0x' + 32 hex digits
+                # plus 7 _ if needed
+                padlen = int(IPV6LENGTH/4)+2 + (7*len(grouping))
+
+        retstr = f'{int(self):#0{padlen}{grouping}{fmt_base}}'
+
+        if fmt_base == 'X':
+            retstr = retstr.upper()
+
+        # If alternate is not set, strip the two leftmost
+        #  characters ('0b')
+        if not alternate:
+            retstr = retstr[2:]
+
+        return retstr
+
 
 @functools.total_ordering
 class _BaseNetwork(_IPAddressBase):
diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py
index de77111705b6..3a59a6102f4c 100644
--- a/Lib/test/test_ipaddress.py
+++ b/Lib/test/test_ipaddress.py
@@ -174,6 +174,31 @@ def assertBadLength(length):
 class AddressTestCase_v4(BaseTestCase, CommonTestMixin_v4):
     factory = ipaddress.IPv4Address
 
+    def test_format(self):
+        v4 = ipaddress.IPv4Address("1.2.3.42")
+        v4_pairs  = [
+            ("b" ,"00000001000000100000001100101010"),
+            ("n" ,"00000001000000100000001100101010"),
+            ("x" ,"0102032a"),
+            ("X" ,"0102032A"),
+            ("_b" ,"0000_0001_0000_0010_0000_0011_0010_1010"),
+            ("_n" ,"0000_0001_0000_0010_0000_0011_0010_1010"),
+            ("_x" ,"0102_032a"),
+            ("_X" ,"0102_032A"),
+            ("#b" ,"0b00000001000000100000001100101010"),
+            ("#n" ,"0b00000001000000100000001100101010"),
+            ("#x" ,"0x0102032a"),
+            ("#X" ,"0X0102032A"),
+            ("#_b" ,"0b0000_0001_0000_0010_0000_0011_0010_1010"),
+            ("#_n" ,"0b0000_0001_0000_0010_0000_0011_0010_1010"),
+            ("#_x" ,"0x0102_032a"),
+            ("#_X" ,"0X0102_032A"),
+            ("s" ,"1.2.3.42"),
+            ("" ,"1.2.3.42"),
+        ]
+        for (fmt, txt) in v4_pairs:
+            self.assertEqual(txt, format(v4, fmt))
+
     def test_network_passed_as_address(self):
         addr = "127.0.0.1/24"
         with self.assertAddressError("Unexpected '/' in %r", addr):
@@ -261,6 +286,47 @@ def test_weakref(self):
 class AddressTestCase_v6(BaseTestCase, CommonTestMixin_v6):
     factory = ipaddress.IPv6Address
 
+    def test_format(self):
+
+        v6 = ipaddress.IPv6Address("::1.2.3.42")
+        v6_pairs = [
+            ("b",
+                "000000000000000000000000000000000000000000000000000000"
+                "000000000000000000000000000000000000000000000000010000"
+                "00100000001100101010"),
+            ("n", "0000000000000000000000000102032a"),
+            ("x", "0000000000000000000000000102032a"),
+            ("X", "0000000000000000000000000102032A"),
+            ("_b",
+                "0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000"
+                "_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000"
+                "_0000_0000_0000_0000_0001_0000_0010_0000_0011_0010"
+                "_1010"),
+            ("_n", "0000_0000_0000_0000_0000_0000_0102_032a"),
+            ("_x", "0000_0000_0000_0000_0000_0000_0102_032a"),
+            ("_X", "0000_0000_0000_0000_0000_0000_0102_032A"),
+            ("#b",
+                "0b0000000000000000000000000000000000000000000000000000"
+                "000000000000000000000000000000000000000000000000000100"
+                "0000100000001100101010"),
+            ("#n", "0x0000000000000000000000000102032a"),
+            ("#x", "0x0000000000000000000000000102032a"),
+            ("#X", "0X0000000000000000000000000102032A"),
+            ("#_b",
+                "0b0000_0000_0000_0000_0000_0000_0000_0000_0000_0000"
+                "_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000"
+                "_0000_0000_0000_0000_0000_0001_0000_0010_0000_0011"
+                "_0010_1010"),
+            ("#_n", "0x0000_0000_0000_0000_0000_0000_0102_032a"),
+            ("#_x", "0x0000_0000_0000_0000_0000_0000_0102_032a"),
+            ("#_X", "0X0000_0000_0000_0000_0000_0000_0102_032A"),
+            ("s", "::102:32a"),
+            ("", "::102:32a"),
+        ]
+
+        for (fmt, txt) in v6_pairs:
+            self.assertEqual(txt, format(v6, fmt))
+
     def test_network_passed_as_address(self):
         addr = "::1/24"
         with self.assertAddressError("Unexpected '/' in %r", addr):
diff --git a/Misc/NEWS.d/next/Library/2018-02-13-12-25-43.bpo-32820.0stF0u.rst b/Misc/NEWS.d/next/Library/2018-02-13-12-25-43.bpo-32820.0stF0u.rst
new file mode 100644
index 000000000000..dd5bd269cbf7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-02-13-12-25-43.bpo-32820.0stF0u.rst
@@ -0,0 +1,4 @@
+Added __format__ to IPv4 and IPv6 classes.  Always outputs a fully zero-
+padded string. Supports b/x/n modifiers (bin/hex/native format).  Native
+format for IPv4 is bin, native format for IPv6 is hex. Also supports '#' and
+'_' modifiers.


From webhook-mailer at python.org  Thu Sep 12 05:12:57 2019
From: webhook-mailer at python.org (Petr Viktorin)
Date: Thu, 12 Sep 2019 09:12:57 -0000
Subject: [Python-checkins] bpo-37879: Fix warnings in _testcapimodule
 (GH-16004)
Message-ID: <mailman.625.1568279578.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5e9caeec76119a0d61c25f1466c27b7dbd5115bd
commit: 5e9caeec76119a0d61c25f1466c27b7dbd5115bd
branch: master
author: Petr Viktorin <encukou at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T10:12:53+01:00
summary:

bpo-37879: Fix warnings in _testcapimodule (GH-16004)

files:
M Modules/_testcapimodule.c

diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 21061a2171a0..dee06de107a1 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -6205,7 +6205,7 @@ static PyType_Slot HeapGcCType_slots[] = {
     {Py_tp_init, heapctype_init},
     {Py_tp_members, heapctype_members},
     {Py_tp_dealloc, heapgcctype_dealloc},
-    {Py_tp_doc, heapgctype__doc__},
+    {Py_tp_doc, (char*)heapgctype__doc__},
     {0, 0},
 };
 
@@ -6233,7 +6233,7 @@ static PyType_Slot HeapCType_slots[] = {
     {Py_tp_init, heapctype_init},
     {Py_tp_members, heapctype_members},
     {Py_tp_dealloc, heapctype_dealloc},
-    {Py_tp_doc, heapctype__doc__},
+    {Py_tp_doc, (char*)heapctype__doc__},
     {0, 0},
 };
 
@@ -6274,7 +6274,7 @@ static struct PyMemberDef heapctypesubclass_members[] = {
 static PyType_Slot HeapCTypeSubclass_slots[] = {
     {Py_tp_init, heapctypesubclass_init},
     {Py_tp_members, heapctypesubclass_members},
-    {Py_tp_doc, heapctypesubclass__doc__},
+    {Py_tp_doc, (char*)heapctypesubclass__doc__},
     {0, 0},
 };
 
@@ -6303,7 +6303,8 @@ heapctypesubclasswithfinalizer_init(PyObject *self, PyObject *args, PyObject *kw
 static void
 heapctypesubclasswithfinalizer_finalize(PyObject *self)
 {
-    PyObject *error_type, *error_value, *error_traceback, *m, *oldtype, *newtype;
+    PyObject *error_type, *error_value, *error_traceback, *m;
+    PyObject *oldtype = NULL, *newtype = NULL;
 
     /* Save the current exception, if any. */
     PyErr_Fetch(&error_type, &error_value, &error_traceback);
@@ -6342,7 +6343,7 @@ static PyType_Slot HeapCTypeSubclassWithFinalizer_slots[] = {
     {Py_tp_init, heapctypesubclasswithfinalizer_init},
     {Py_tp_members, heapctypesubclass_members},
     {Py_tp_finalize, heapctypesubclasswithfinalizer_finalize},
-    {Py_tp_doc, heapctypesubclasswithfinalizer__doc__},
+    {Py_tp_doc, (char*)heapctypesubclasswithfinalizer__doc__},
     {0, 0},
 };
 
@@ -6372,7 +6373,7 @@ static PyTypeObject MethInstance_Type = {
     .tp_new = PyType_GenericNew,
     .tp_flags = Py_TPFLAGS_DEFAULT,
     .tp_methods = meth_instance_methods,
-    .tp_doc = PyDoc_STR(
+    .tp_doc = (char*)PyDoc_STR(
         "Class with normal (instance) methods to test calling conventions"),
 };
 


From webhook-mailer at python.org  Thu Sep 12 05:23:46 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 12 Sep 2019 09:23:46 -0000
Subject: [Python-checkins] bpo-37760: Convert from length-18 lists to a
 dataclass, in makeunicodedata. (GH-15265)
Message-ID: <mailman.626.1568280228.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a65678c5c90002c5e40fa82746de07e6217df625
commit: a65678c5c90002c5e40fa82746de07e6217df625
branch: master
author: Greg Price <gnprice at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-12T10:23:43+01:00
summary:

bpo-37760: Convert from length-18 lists to a dataclass, in makeunicodedata. (GH-15265)

Now the fields have names!  Much easier to keep straight as a
reader than the elements of an 18-tuple.

Runs about 10-15% slower: from 10.8s to 12.3s, on my laptop.
Fortunately that's perfectly fine for this maintenance script.

files:
A Misc/NEWS.d/next/Build/2019-08-24-17-39-09.bpo-37760.f3jXuH.rst
M Tools/unicode/makeunicodedata.py

diff --git a/Misc/NEWS.d/next/Build/2019-08-24-17-39-09.bpo-37760.f3jXuH.rst b/Misc/NEWS.d/next/Build/2019-08-24-17-39-09.bpo-37760.f3jXuH.rst
new file mode 100644
index 000000000000..04981733dbe9
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-08-24-17-39-09.bpo-37760.f3jXuH.rst
@@ -0,0 +1,6 @@
+The :file:`Tools/unicode/makeunicodedata.py` script, which is used for
+converting information from the Unicode Character Database into generated
+code and data used by the methods of :class:`str` and by the
+:mod:`unicodedata` module, now handles each character's data as a
+``dataclass`` with named attributes, rather than a length-18 list of
+different fields.
diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py
index 464a4ebf7722..a8e92be49629 100644
--- a/Tools/unicode/makeunicodedata.py
+++ b/Tools/unicode/makeunicodedata.py
@@ -26,13 +26,14 @@
 # written by Fredrik Lundh (fredrik at pythonware.com)
 #
 
+import dataclasses
 import os
 import sys
 import zipfile
 
 from functools import partial
 from textwrap import dedent
-from typing import Iterator, List, Tuple
+from typing import Iterator, List, Optional, Set, Tuple
 
 SCRIPT = sys.argv[0]
 VERSION = "3.3"
@@ -148,12 +149,12 @@ def makeunicodedata(unicode, trace):
         record = unicode.table[char]
         if record:
             # extract database properties
-            category = CATEGORY_NAMES.index(record[2])
-            combining = int(record[3])
-            bidirectional = BIDIRECTIONAL_NAMES.index(record[4])
-            mirrored = record[9] == "Y"
-            eastasianwidth = EASTASIANWIDTH_NAMES.index(record[15])
-            normalizationquickcheck = record[17]
+            category = CATEGORY_NAMES.index(record.general_category)
+            combining = int(record.canonical_combining_class)
+            bidirectional = BIDIRECTIONAL_NAMES.index(record.bidi_class)
+            mirrored = record.bidi_mirrored == "Y"
+            eastasianwidth = EASTASIANWIDTH_NAMES.index(record.east_asian_width)
+            normalizationquickcheck = record.quick_check
             item = (
                 category, combining, bidirectional, mirrored, eastasianwidth,
                 normalizationquickcheck
@@ -179,8 +180,8 @@ def makeunicodedata(unicode, trace):
     for char in unicode.chars:
         record = unicode.table[char]
         if record:
-            if record[5]:
-                decomp = record[5].split()
+            if record.decomposition_type:
+                decomp = record.decomposition_type.split()
                 if len(decomp) > 19:
                     raise Exception("character %x has a decomposition too large for nfd_nfkd" % char)
                 # prefix
@@ -200,7 +201,7 @@ def makeunicodedata(unicode, trace):
                 # Collect NFC pairs
                 if not prefix and len(decomp) == 3 and \
                    char not in unicode.exclusions and \
-                   unicode.table[decomp[1]][3] == "0":
+                   unicode.table[decomp[1]].canonical_combining_class == "0":
                     p, l, r = decomp
                     comp_first[l] = 1
                     comp_last[r] = 1
@@ -404,9 +405,9 @@ def makeunicodetype(unicode, trace):
         record = unicode.table[char]
         if record:
             # extract database properties
-            category = record[2]
-            bidirectional = record[4]
-            properties = record[16]
+            category = record.general_category
+            bidirectional = record.bidi_class
+            properties = record.binary_properties
             flags = 0
             if category in ["Lm", "Lt", "Lu", "Ll", "Lo"]:
                 flags |= ALPHA_MASK
@@ -434,16 +435,16 @@ def makeunicodetype(unicode, trace):
                 flags |= CASE_IGNORABLE_MASK
             sc = unicode.special_casing.get(char)
             cf = unicode.case_folding.get(char, [char])
-            if record[12]:
-                upper = int(record[12], 16)
+            if record.simple_uppercase_mapping:
+                upper = int(record.simple_uppercase_mapping, 16)
             else:
                 upper = char
-            if record[13]:
-                lower = int(record[13], 16)
+            if record.simple_lowercase_mapping:
+                lower = int(record.simple_lowercase_mapping, 16)
             else:
                 lower = char
-            if record[14]:
-                title = int(record[14], 16)
+            if record.simple_titlecase_mapping:
+                title = int(record.simple_titlecase_mapping, 16)
             else:
                 title = upper
             if sc is None and cf != [lower]:
@@ -480,16 +481,16 @@ def makeunicodetype(unicode, trace):
                     extra_casing.extend(sc[1])
             # decimal digit, integer digit
             decimal = 0
-            if record[6]:
+            if record.decomposition_mapping:
                 flags |= DECIMAL_MASK
-                decimal = int(record[6])
+                decimal = int(record.decomposition_mapping)
             digit = 0
-            if record[7]:
+            if record.numeric_type:
                 flags |= DIGIT_MASK
-                digit = int(record[7])
-            if record[8]:
+                digit = int(record.numeric_type)
+            if record.numeric_value:
                 flags |= NUMERIC_MASK
-                numeric.setdefault(record[8], []).append(char)
+                numeric.setdefault(record.numeric_value, []).append(char)
             item = (
                 upper, lower, title, decimal, digit, flags
                 )
@@ -609,7 +610,7 @@ def makeunicodename(unicode, trace):
     for char in unicode.chars:
         record = unicode.table[char]
         if record:
-            name = record[1].strip()
+            name = record.name.strip()
             if name and name[0] != "<":
                 names[char] = name + chr(0)
 
@@ -719,7 +720,7 @@ def word_key(a):
     for char in unicode.chars:
         record = unicode.table[char]
         if record:
-            name = record[1].strip()
+            name = record.name.strip()
             if name and name[0] != "<":
                 data.append((name, char))
 
@@ -819,31 +820,27 @@ def merge_old_version(version, new, old):
             continue
         # check characters that differ
         if old.table[i] != new.table[i]:
-            for k in range(len(old.table[i])):
-                if old.table[i][k] != new.table[i][k]:
-                    value = old.table[i][k]
+            for k, field in enumerate(dataclasses.fields(UcdRecord)):
+                value = getattr(old.table[i], field.name)
+                new_value = getattr(new.table[i], field.name)
+                if value != new_value:
                     if k == 1 and i in PUA_15:
                         # the name is not set in the old.table, but in the
                         # new.table we are using it for aliases and named seq
                         assert value == ''
                     elif k == 2:
-                        #print "CATEGORY",hex(i), old.table[i][k], new.table[i][k]
                         category_changes[i] = CATEGORY_NAMES.index(value)
                     elif k == 4:
-                        #print "BIDIR",hex(i), old.table[i][k], new.table[i][k]
                         bidir_changes[i] = BIDIRECTIONAL_NAMES.index(value)
                     elif k == 5:
-                        #print "DECOMP",hex(i), old.table[i][k], new.table[i][k]
                         # We assume that all normalization changes are in 1:1 mappings
                         assert " " not in value
                         normalization_changes.append((i, value))
                     elif k == 6:
-                        #print "DECIMAL",hex(i), old.table[i][k], new.table[i][k]
                         # we only support changes where the old value is a single digit
                         assert value in "0123456789"
                         decimal_changes[i] = int(value)
                     elif k == 8:
-                        # print "NUMERIC",hex(i), `old.table[i][k]`, new.table[i][k]
                         # Since 0 encodes "no change", the old value is better not 0
                         if not value:
                             numeric_changes[i] = -1
@@ -952,6 +949,45 @@ def expanded(self) -> Iterator[Tuple[int, List[str]]]:
                 yield char, rest
 
 
+ at dataclasses.dataclass
+class UcdRecord:
+    # 15 fields from UnicodeData.txt .  See:
+    #   https://www.unicode.org/reports/tr44/#UnicodeData.txt
+    codepoint: str
+    name: str
+    general_category: str
+    canonical_combining_class: str
+    bidi_class: str
+    decomposition_type: str
+    decomposition_mapping: str
+    numeric_type: str
+    numeric_value: str
+    bidi_mirrored: str
+    unicode_1_name: str  # obsolete
+    iso_comment: str  # obsolete
+    simple_uppercase_mapping: str
+    simple_lowercase_mapping: str
+    simple_titlecase_mapping: str
+
+    # https://www.unicode.org/reports/tr44/#EastAsianWidth.txt
+    east_asian_width: Optional[str]
+
+    # Binary properties, as a set of those that are true.
+    # Taken from multiple files:
+    #   https://www.unicode.org/reports/tr44/#DerivedCoreProperties.txt
+    #   https://www.unicode.org/reports/tr44/#LineBreak.txt
+    binary_properties: Set[str]
+
+    # The Quick_Check properties related to normalization:
+    #   https://www.unicode.org/reports/tr44/#Decompositions_and_Normalization
+    # We store them as a bitmask.
+    quick_check: int
+
+
+def from_row(row: List[str]) -> UcdRecord:
+    return UcdRecord(*row, None, set(), 0)
+
+
 # --------------------------------------------------------------------
 # the following support code is taken from the unidb utilities
 # Copyright (c) 1999-2000 by Secret Labs AB
@@ -959,18 +995,14 @@ def expanded(self) -> Iterator[Tuple[int, List[str]]]:
 # load a unicode-data file from disk
 
 class UnicodeData:
-    # Record structure:
-    # [ID, name, category, combining, bidi, decomp,  (6)
-    #  decimal, digit, numeric, bidi-mirrored, Unicode-1-name, (11)
-    #  ISO-comment, uppercase, lowercase, titlecase, ea-width, (16)
-    #  derived-props] (17)
+    # table: List[Optional[UcdRecord]]  # index is codepoint; None means unassigned
 
     def __init__(self, version, cjk_check=True):
         self.changed = []
         table = [None] * 0x110000
         for s in UcdFile(UNICODE_DATA, version):
             char = int(s[0], 16)
-            table[char] = s
+            table[char] = from_row(s)
 
         cjk_ranges_found = []
 
@@ -982,19 +1014,17 @@ def __init__(self, version, cjk_check=True):
             #   https://www.unicode.org/reports/tr44/#Code_Point_Ranges
             s = table[i]
             if s:
-                if s[1][-6:] == "First>":
-                    s[1] = ""
-                    field = s
-                elif s[1][-5:] == "Last>":
-                    if s[1].startswith("<CJK Ideograph"):
+                if s.name[-6:] == "First>":
+                    s.name = ""
+                    field = dataclasses.astuple(s)[:15]
+                elif s.name[-5:] == "Last>":
+                    if s.name.startswith("<CJK Ideograph"):
                         cjk_ranges_found.append((field[0],
-                                                 s[0]))
-                    s[1] = ""
+                                                 s.codepoint))
+                    s.name = ""
                     field = None
             elif field:
-                f2 = field[:]
-                f2[0] = "%X" % i
-                table[i] = f2
+                table[i] = from_row(('%X' % i,) + field[1:])
         if cjk_check and cjk_ranges != cjk_ranges_found:
             raise ValueError("CJK ranges deviate: have %r" % cjk_ranges_found)
 
@@ -1015,7 +1045,7 @@ def __init__(self, version, cjk_check=True):
                 char = int(char, 16)
                 self.aliases.append((name, char))
                 # also store the name in the PUA 1
-                self.table[pua_index][1] = name
+                self.table[pua_index].name = name
                 pua_index += 1
             assert pua_index - NAME_ALIASES_START == len(self.aliases)
 
@@ -1034,7 +1064,7 @@ def __init__(self, version, cjk_check=True):
                     "the NamedSequence struct and in unicodedata_lookup")
                 self.named_sequences.append((name, chars))
                 # also store these in the PUA 1
-                self.table[pua_index][1] = name
+                self.table[pua_index].name = name
                 pua_index += 1
             assert pua_index - NAMED_SEQUENCES_START == len(self.named_sequences)
 
@@ -1049,23 +1079,19 @@ def __init__(self, version, cjk_check=True):
 
         for i in range(0, 0x110000):
             if table[i] is not None:
-                table[i].append(widths[i])
-
-        for i in range(0, 0x110000):
-            if table[i] is not None:
-                table[i].append(set())
+                table[i].east_asian_width = widths[i]
 
         for char, (p,) in UcdFile(DERIVED_CORE_PROPERTIES, version).expanded():
             if table[char]:
                 # Some properties (e.g. Default_Ignorable_Code_Point)
                 # apply to unassigned code points; ignore them
-                table[char][-1].add(p)
+                table[char].binary_properties.add(p)
 
         for char_range, value in UcdFile(LINE_BREAK, version):
             if value not in MANDATORY_LINE_BREAKS:
                 continue
             for char in expand_range(char_range):
-                table[char][-1].add('Line_Break')
+                table[char].binary_properties.add('Line_Break')
 
         # We only want the quickcheck properties
         # Format: NF?_QC; Y(es)/N(o)/M(aybe)
@@ -1087,7 +1113,7 @@ def __init__(self, version, cjk_check=True):
                 quickchecks[char] |= quickcheck
         for i in range(0, 0x110000):
             if table[i] is not None:
-                table[i].append(quickchecks[i])
+                table[i].quick_check = quickchecks[i]
 
         with open_data(UNIHAN, version) as file:
             zip = zipfile.ZipFile(file)
@@ -1106,7 +1132,7 @@ def __init__(self, version, cjk_check=True):
             i = int(code[2:], 16)
             # Patch the numeric field
             if table[i] is not None:
-                table[i][8] = value
+                table[i].numeric_value = value
 
         sc = self.special_casing = {}
         for data in UcdFile(SPECIAL_CASING, version):


From webhook-mailer at python.org  Thu Sep 12 05:25:29 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 12 Sep 2019 09:25:29 -0000
Subject: [Python-checkins] closes bpo-37758: Extend unicodedata checksum
 tests to cover all of Unicode. (GH-15125)
Message-ID: <mailman.627.1568280330.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6954be815a16fad11d1d66be576865bbbeb2b97d
commit: 6954be815a16fad11d1d66be576865bbbeb2b97d
branch: master
author: Greg Price <gnprice at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-12T10:25:25+01:00
summary:

closes bpo-37758: Extend unicodedata checksum tests to cover all of Unicode. (GH-15125)

Unicode has grown since Python first gained support for it,
when Unicode itself was still rather new.

This pair of test cases was added in commit 6a20ee7de back in 2000,
and they haven't needed to change much since then.  But do change
them to look beyond the Basic Multilingual Plane (range(0x10000))
and cover all 17 planes of Unicode's final form.

This adds about 5 seconds to the test suite's runtime.  Mark the
tests as CPU-using accordingly.

files:
M Lib/test/test_unicodedata.py

diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py
index 40c38c1c4262..4b5c89276181 100644
--- a/Lib/test/test_unicodedata.py
+++ b/Lib/test/test_unicodedata.py
@@ -11,17 +11,18 @@
 import sys
 import unicodedata
 import unittest
-from test.support import open_urlresource, script_helper
+from test.support import open_urlresource, requires_resource, script_helper
 
 
 class UnicodeMethodsTest(unittest.TestCase):
 
     # update this, if the database changes
-    expectedchecksum = '9129d6f2bdf008a81c2476e5b5127014a62130c1'
+    expectedchecksum = 'e728278035eb76cf92d86f07852266b0433f16a5'
 
+    @requires_resource('cpu')
     def test_method_checksum(self):
         h = hashlib.sha1()
-        for i in range(0x10000):
+        for i in range(sys.maxunicode + 1):
             char = chr(i)
             data = [
                 # Predicates (single char)
@@ -69,12 +70,14 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest):
 
     # Update this if the database changes. Make sure to do a full rebuild
     # (e.g. 'make distclean && make') to get the correct checksum.
-    expectedchecksum = 'c44a49ca7c5cb6441640fe174ede604b45028652'
+    expectedchecksum = '4bcbf9df344114b1ebc95b904f4352dd250dff7e'
+
+    @requires_resource('cpu')
     def test_function_checksum(self):
         data = []
         h = hashlib.sha1()
 
-        for i in range(0x10000):
+        for i in range(sys.maxunicode + 1):
             char = chr(i)
             data = [
                 # Properties


From webhook-mailer at python.org  Thu Sep 12 05:27:18 2019
From: webhook-mailer at python.org (Mark Shannon)
Date: Thu, 12 Sep 2019 09:27:18 -0000
Subject: [Python-checkins] bpo-21120: Exclude Python-ast.h,
 ast.h and asdl.h from the limited API (#14634)
Message-ID: <mailman.628.1568280439.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/421a72af4deaec96a49a79951b9c2546a2faa13d
commit: 421a72af4deaec96a49a79951b9c2546a2faa13d
branch: master
author: Zackery Spytz <zspytz at gmail.com>
committer: Mark Shannon <mark at hotpy.org>
date: 2019-09-12T10:27:14+01:00
summary:

bpo-21120: Exclude Python-ast.h, ast.h and asdl.h from the limited API (#14634)

The PyArena type is not part of the limited API, so these headers
shouldn't be part of it either.

files:
A Misc/NEWS.d/next/C API/2019-07-06-23-56-47.bpo-21120.lXHqlT.rst
M Include/Python-ast.h
M Include/asdl.h
M Include/ast.h
M Parser/asdl_c.py

diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index 5fe4f2b4314c..b9232b1d4630 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -6,6 +6,7 @@
 extern "C" {
 #endif
 
+#ifndef Py_LIMITED_API
 #include "asdl.h"
 
 #undef Yield   /* undefine macro conflicting with <winbase.h> */
@@ -708,6 +709,7 @@ type_ignore_ty _Py_TypeIgnore(int lineno, string tag, PyArena *arena);
 PyObject* PyAST_mod2obj(mod_ty t);
 mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);
 int PyAST_Check(PyObject* obj);
+#endif /* !Py_LIMITED_API */
 
 #ifdef __cplusplus
 }
diff --git a/Include/asdl.h b/Include/asdl.h
index fc6d22371b63..549df2ace755 100644
--- a/Include/asdl.h
+++ b/Include/asdl.h
@@ -1,3 +1,4 @@
+#ifndef Py_LIMITED_API
 #ifndef Py_ASDL_H
 #define Py_ASDL_H
 
@@ -44,3 +45,4 @@ asdl_int_seq *_Py_asdl_int_seq_new(Py_ssize_t size, PyArena *arena);
 #endif
 
 #endif /* !Py_ASDL_H */
+#endif /* Py_LIMITED_API */
diff --git a/Include/ast.h b/Include/ast.h
index 1347fcffcd75..a8c52af786b1 100644
--- a/Include/ast.h
+++ b/Include/ast.h
@@ -1,3 +1,4 @@
+#ifndef Py_LIMITED_API
 #ifndef Py_AST_H
 #define Py_AST_H
 #ifdef __cplusplus
@@ -19,8 +20,6 @@ PyAPI_FUNC(mod_ty) PyAST_FromNodeObject(
     PyObject *filename,
     PyArena *arena);
 
-#ifndef Py_LIMITED_API
-
 /* _PyAST_ExprAsUnicode is defined in ast_unparse.c */
 PyAPI_FUNC(PyObject *) _PyAST_ExprAsUnicode(expr_ty);
 
@@ -29,9 +28,8 @@ PyAPI_FUNC(PyObject *) _PyAST_ExprAsUnicode(expr_ty);
    Doesn't set exception. */
 PyAPI_FUNC(PyObject *) _PyAST_GetDocString(asdl_seq *);
 
-#endif /* !Py_LIMITED_API */
-
 #ifdef __cplusplus
 }
 #endif
 #endif /* !Py_AST_H */
+#endif /* !Py_LIMITED_API */
diff --git a/Misc/NEWS.d/next/C API/2019-07-06-23-56-47.bpo-21120.lXHqlT.rst b/Misc/NEWS.d/next/C API/2019-07-06-23-56-47.bpo-21120.lXHqlT.rst
new file mode 100644
index 000000000000..088a6a8614f3
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-07-06-23-56-47.bpo-21120.lXHqlT.rst	
@@ -0,0 +1 @@
+Exclude Python-ast.h, ast.h and asdl.h from the limited API.
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 85d96a33f3ff..6b31ffd8315d 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -1339,6 +1339,7 @@ def main(srcfile, dump_module=False):
             f.write('extern "C" {\n')
             f.write('#endif\n')
             f.write('\n')
+            f.write('#ifndef Py_LIMITED_API\n')
             f.write('#include "asdl.h"\n')
             f.write('\n')
             f.write('#undef Yield   /* undefine macro conflicting with <winbase.h> */\n')
@@ -1353,6 +1354,7 @@ def main(srcfile, dump_module=False):
             f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
             f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
             f.write("int PyAST_Check(PyObject* obj);\n")
+            f.write("#endif /* !Py_LIMITED_API */\n")
             f.write('\n')
             f.write('#ifdef __cplusplus\n')
             f.write('}\n')


From webhook-mailer at python.org  Thu Sep 12 05:29:15 2019
From: webhook-mailer at python.org (Paul Ganssle)
Date: Thu, 12 Sep 2019 09:29:15 -0000
Subject: [Python-checkins] bpo-38121: Sync importlib.metadata with 0.22
 backport (GH-15993)
Message-ID: <mailman.629.1568280557.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8ed6503eca4e3ea4949479d8d7fd9ffd54f81038
commit: 8ed6503eca4e3ea4949479d8d7fd9ffd54f81038
branch: master
author: Jason R. Coombs <jaraco at jaraco.com>
committer: Paul Ganssle <paul at ganssle.io>
date: 2019-09-12T10:29:11+01:00
summary:

bpo-38121: Sync importlib.metadata with 0.22 backport (GH-15993)

* bpo-38121: Sync importlib.metadata with 0.22 backport

* ?? Added by blurb_it.

files:
A Misc/NEWS.d/next/Library/2019-09-11-16-54-57.bpo-38121.SrSDzB.rst
M Lib/importlib/_bootstrap_external.py
M Lib/importlib/metadata.py
M Python/importlib_external.h

diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index ec4bbec41e5a..825ed77c3d4b 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -1370,7 +1370,7 @@ def find_module(cls, fullname, path=None):
         return spec.loader
 
     @classmethod
-    def find_distributions(self, context=None):
+    def find_distributions(cls, *args, **kwargs):
         """
         Find distributions.
 
@@ -1379,54 +1379,8 @@ def find_distributions(self, context=None):
         (or all names if ``None`` indicated) along the paths in the list
         of directories ``context.path``.
         """
-        from importlib.metadata import PathDistribution, DistributionFinder
-        if context is None:
-            context = DistributionFinder.Context()
-        found = self._search_paths(context.pattern, context.path)
-        return map(PathDistribution, found)
-
-    @classmethod
-    def _search_paths(cls, pattern, paths):
-        """Find metadata directories in paths heuristically."""
-        import itertools
-        return itertools.chain.from_iterable(
-            cls._search_path(path, pattern)
-            for path in map(cls._switch_path, paths)
-            )
-
-    @staticmethod
-    def _switch_path(path):
-        from contextlib import suppress
-        import zipfile
-        import pathlib
-        PYPY_OPEN_BUG = False
-        if not PYPY_OPEN_BUG or os.path.isfile(path):  # pragma: no branch
-            with suppress(Exception):
-                return zipfile.Path(path)
-        return pathlib.Path(path)
-
-    @classmethod
-    def _matches_info(cls, normalized, item):
-        import re
-        template = r'{pattern}(-.*)?\.(dist|egg)-info'
-        manifest = template.format(pattern=normalized)
-        return re.match(manifest, item.name, flags=re.IGNORECASE)
-
-    @classmethod
-    def _matches_legacy(cls, normalized, item):
-        import re
-        template = r'{pattern}-.*\.egg[\\/]EGG-INFO'
-        manifest = template.format(pattern=normalized)
-        return re.search(manifest, str(item), flags=re.IGNORECASE)
-
-    @classmethod
-    def _search_path(cls, root, pattern):
-        if not root.is_dir():
-            return ()
-        normalized = pattern.replace('-', '_')
-        return (item for item in root.iterdir()
-                if cls._matches_info(normalized, item)
-                or cls._matches_legacy(normalized, item))
+        from importlib.metadata import MetadataPathFinder
+        return MetadataPathFinder.find_distributions(*args, **kwargs)
 
 
 class FileFinder:
diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py
index e23076654fe4..8cb45ec1ef3a 100644
--- a/Lib/importlib/metadata.py
+++ b/Lib/importlib/metadata.py
@@ -1,10 +1,12 @@
 import io
+import os
 import re
 import abc
 import csv
 import sys
 import email
 import pathlib
+import zipfile
 import operator
 import functools
 import itertools
@@ -363,6 +365,58 @@ def find_distributions(self, context=Context()):
         """
 
 
+class MetadataPathFinder(DistributionFinder):
+    @classmethod
+    def find_distributions(cls, context=DistributionFinder.Context()):
+        """
+        Find distributions.
+
+        Return an iterable of all Distribution instances capable of
+        loading the metadata for packages matching ``context.name``
+        (or all names if ``None`` indicated) along the paths in the list
+        of directories ``context.path``.
+        """
+        found = cls._search_paths(context.pattern, context.path)
+        return map(PathDistribution, found)
+
+    @classmethod
+    def _search_paths(cls, pattern, paths):
+        """Find metadata directories in paths heuristically."""
+        return itertools.chain.from_iterable(
+            cls._search_path(path, pattern)
+            for path in map(cls._switch_path, paths)
+            )
+
+    @staticmethod
+    def _switch_path(path):
+        PYPY_OPEN_BUG = False
+        if not PYPY_OPEN_BUG or os.path.isfile(path):  # pragma: no branch
+            with suppress(Exception):
+                return zipfile.Path(path)
+        return pathlib.Path(path)
+
+    @classmethod
+    def _matches_info(cls, normalized, item):
+        template = r'{pattern}(-.*)?\.(dist|egg)-info'
+        manifest = template.format(pattern=normalized)
+        return re.match(manifest, item.name, flags=re.IGNORECASE)
+
+    @classmethod
+    def _matches_legacy(cls, normalized, item):
+        template = r'{pattern}-.*\.egg[\\/]EGG-INFO'
+        manifest = template.format(pattern=normalized)
+        return re.search(manifest, str(item), flags=re.IGNORECASE)
+
+    @classmethod
+    def _search_path(cls, root, pattern):
+        if not root.is_dir():
+            return ()
+        normalized = pattern.replace('-', '_')
+        return (item for item in root.iterdir()
+                if cls._matches_info(normalized, item)
+                or cls._matches_legacy(normalized, item))
+
+
 class PathDistribution(Distribution):
     def __init__(self, path):
         """Construct a distribution from a path to the metadata directory.
@@ -382,13 +436,13 @@ def locate_file(self, path):
         return self._path.parent / path
 
 
-def distribution(package):
-    """Get the ``Distribution`` instance for the given package.
+def distribution(distribution_name):
+    """Get the ``Distribution`` instance for the named package.
 
-    :param package: The name of the package as a string.
+    :param distribution_name: The name of the distribution package as a string.
     :return: A ``Distribution`` instance (or subclass thereof).
     """
-    return Distribution.from_name(package)
+    return Distribution.from_name(distribution_name)
 
 
 def distributions(**kwargs):
@@ -399,23 +453,23 @@ def distributions(**kwargs):
     return Distribution.discover(**kwargs)
 
 
-def metadata(package):
-    """Get the metadata for the package.
+def metadata(distribution_name):
+    """Get the metadata for the named package.
 
-    :param package: The name of the distribution package to query.
+    :param distribution_name: The name of the distribution package to query.
     :return: An email.Message containing the parsed metadata.
     """
-    return Distribution.from_name(package).metadata
+    return Distribution.from_name(distribution_name).metadata
 
 
-def version(package):
+def version(distribution_name):
     """Get the version string for the named package.
 
-    :param package: The name of the distribution package to query.
+    :param distribution_name: The name of the distribution package to query.
     :return: The version string for the package as defined in the package's
         "Version" metadata key.
     """
-    return distribution(package).version
+    return distribution(distribution_name).version
 
 
 def entry_points():
@@ -434,15 +488,20 @@ def entry_points():
         }
 
 
-def files(package):
-    return distribution(package).files
+def files(distribution_name):
+    """Return a list of files for the named package.
+
+    :param distribution_name: The name of the distribution package to query.
+    :return: List of files composing the distribution.
+    """
+    return distribution(distribution_name).files
 
 
-def requires(package):
+def requires(distribution_name):
     """
-    Return a list of requirements for the indicated distribution.
+    Return a list of requirements for the named package.
 
     :return: An iterator of requirements, suitable for
     packaging.requirement.Requirement.
     """
-    return distribution(package).requires
+    return distribution(distribution_name).requires
diff --git a/Misc/NEWS.d/next/Library/2019-09-11-16-54-57.bpo-38121.SrSDzB.rst b/Misc/NEWS.d/next/Library/2019-09-11-16-54-57.bpo-38121.SrSDzB.rst
new file mode 100644
index 000000000000..93eabcfaab39
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-11-16-54-57.bpo-38121.SrSDzB.rst
@@ -0,0 +1 @@
+Update parameter names on functions in importlib.metadata matching the changes in the 0.22 release of importlib_metadata.
\ No newline at end of file
diff --git a/Python/importlib_external.h b/Python/importlib_external.h
index b9d14d64b402..054fc8468452 100644
--- a/Python/importlib_external.h
+++ b/Python/importlib_external.h
@@ -570,7 +570,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,104,97,110,100,108,101,32,37,115,169,1,218,4,110,97,
     109,101,41,2,114,116,0,0,0,218,11,73,109,112,111,114,
     116,69,114,114,111,114,41,4,218,4,115,101,108,102,114,116,
-    0,0,0,218,4,97,114,103,115,90,6,107,119,97,114,103,
+    0,0,0,218,4,97,114,103,115,218,6,107,119,97,114,103,
     115,169,1,218,6,109,101,116,104,111,100,114,3,0,0,0,
     114,6,0,0,0,218,19,95,99,104,101,99,107,95,110,97,
     109,101,95,119,114,97,112,112,101,114,201,1,0,0,115,18,
@@ -595,13 +595,13 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,115,8,0,0,0,0,1,8,1,10,1,20,1,122,26,
     95,99,104,101,99,107,95,110,97,109,101,46,60,108,111,99,
     97,108,115,62,46,95,119,114,97,112,41,1,78,41,3,218,
-    10,95,98,111,111,116,115,116,114,97,112,114,132,0,0,0,
-    218,9,78,97,109,101,69,114,114,111,114,41,3,114,121,0,
-    0,0,114,122,0,0,0,114,132,0,0,0,114,3,0,0,
-    0,114,120,0,0,0,114,6,0,0,0,218,11,95,99,104,
+    10,95,98,111,111,116,115,116,114,97,112,114,133,0,0,0,
+    218,9,78,97,109,101,69,114,114,111,114,41,3,114,122,0,
+    0,0,114,123,0,0,0,114,133,0,0,0,114,3,0,0,
+    0,114,121,0,0,0,114,6,0,0,0,218,11,95,99,104,
     101,99,107,95,110,97,109,101,193,1,0,0,115,14,0,0,
     0,0,8,14,7,2,1,10,1,14,2,14,5,10,1,114,
-    135,0,0,0,99,2,0,0,0,0,0,0,0,0,0,0,
+    136,0,0,0,99,2,0,0,0,0,0,0,0,0,0,0,
     0,5,0,0,0,6,0,0,0,67,0,0,0,115,60,0,
     0,0,124,0,160,0,124,1,161,1,92,2,125,2,125,3,
     124,2,100,1,107,8,114,56,116,1,124,3,131,1,114,56,
@@ -629,7 +629,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     114,3,0,0,0,114,6,0,0,0,218,17,95,102,105,110,
     100,95,109,111,100,117,108,101,95,115,104,105,109,221,1,0,
     0,115,10,0,0,0,0,10,14,1,16,1,4,1,22,1,
-    114,142,0,0,0,99,3,0,0,0,0,0,0,0,0,0,
+    114,143,0,0,0,99,3,0,0,0,0,0,0,0,0,0,
     0,0,6,0,0,0,4,0,0,0,67,0,0,0,115,158,
     0,0,0,124,0,100,1,100,2,133,2,25,0,125,3,124,
     3,116,0,107,3,114,60,100,3,124,1,155,2,100,4,124,
@@ -687,7 +687,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     8,0,0,0,233,252,255,255,255,122,14,105,110,118,97,108,
     105,100,32,102,108,97,103,115,32,122,4,32,105,110,32,41,
     7,218,12,77,65,71,73,67,95,78,85,77,66,69,82,114,
-    133,0,0,0,218,16,95,118,101,114,98,111,115,101,95,109,
+    134,0,0,0,218,16,95,118,101,114,98,111,115,101,95,109,
     101,115,115,97,103,101,114,117,0,0,0,114,22,0,0,0,
     218,8,69,79,70,69,114,114,111,114,114,26,0,0,0,41,
     6,114,25,0,0,0,114,116,0,0,0,218,11,101,120,99,
@@ -696,7 +696,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,114,6,0,0,0,218,13,95,99,108,97,115,115,
     105,102,121,95,112,121,99,238,1,0,0,115,28,0,0,0,
     0,16,12,1,8,1,16,1,12,1,12,1,12,1,10,1,
-    12,1,8,1,16,2,8,1,16,1,12,1,114,151,0,0,
+    12,1,8,1,16,2,8,1,16,1,12,1,114,152,0,0,
     0,99,5,0,0,0,0,0,0,0,0,0,0,0,6,0,
     0,0,4,0,0,0,67,0,0,0,115,112,0,0,0,116,
     0,124,0,100,1,100,2,133,2,25,0,131,1,124,1,100,
@@ -738,19 +738,19 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     10,32,32,32,32,65,110,32,73,109,112,111,114,116,69,114,
     114,111,114,32,105,115,32,114,97,105,115,101,100,32,105,102,
     32,116,104,101,32,98,121,116,101,99,111,100,101,32,105,115,
-    32,115,116,97,108,101,46,10,10,32,32,32,32,114,145,0,
+    32,115,116,97,108,101,46,10,10,32,32,32,32,114,146,0,
     0,0,233,12,0,0,0,114,14,0,0,0,122,22,98,121,
     116,101,99,111,100,101,32,105,115,32,115,116,97,108,101,32,
-    102,111,114,32,114,143,0,0,0,78,114,144,0,0,0,41,
-    4,114,26,0,0,0,114,133,0,0,0,114,148,0,0,0,
+    102,111,114,32,114,144,0,0,0,78,114,145,0,0,0,41,
+    4,114,26,0,0,0,114,134,0,0,0,114,149,0,0,0,
     114,117,0,0,0,41,6,114,25,0,0,0,218,12,115,111,
     117,114,99,101,95,109,116,105,109,101,218,11,115,111,117,114,
-    99,101,95,115,105,122,101,114,116,0,0,0,114,150,0,0,
+    99,101,95,115,105,122,101,114,116,0,0,0,114,151,0,0,
     0,114,92,0,0,0,114,3,0,0,0,114,3,0,0,0,
     114,6,0,0,0,218,23,95,118,97,108,105,100,97,116,101,
     95,116,105,109,101,115,116,97,109,112,95,112,121,99,15,2,
     0,0,115,16,0,0,0,0,19,24,1,10,1,12,1,12,
-    1,8,1,22,255,2,2,114,155,0,0,0,99,4,0,0,
+    1,8,1,22,255,2,2,114,156,0,0,0,99,4,0,0,
     0,0,0,0,0,0,0,0,0,4,0,0,0,3,0,0,
     0,67,0,0,0,115,38,0,0,0,124,0,100,1,100,2,
     133,2,25,0,124,1,107,3,114,34,116,0,100,3,124,2,
@@ -786,17 +786,17 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,32,65,110,32,73,109,112,111,114,116,69,114,114,111,114,
     32,105,115,32,114,97,105,115,101,100,32,105,102,32,116,104,
     101,32,98,121,116,101,99,111,100,101,32,105,115,32,115,116,
-    97,108,101,46,10,10,32,32,32,32,114,145,0,0,0,114,
-    144,0,0,0,122,46,104,97,115,104,32,105,110,32,98,121,
+    97,108,101,46,10,10,32,32,32,32,114,146,0,0,0,114,
+    145,0,0,0,122,46,104,97,115,104,32,105,110,32,98,121,
     116,101,99,111,100,101,32,100,111,101,115,110,39,116,32,109,
     97,116,99,104,32,104,97,115,104,32,111,102,32,115,111,117,
     114,99,101,32,78,41,1,114,117,0,0,0,41,4,114,25,
     0,0,0,218,11,115,111,117,114,99,101,95,104,97,115,104,
-    114,116,0,0,0,114,150,0,0,0,114,3,0,0,0,114,
+    114,116,0,0,0,114,151,0,0,0,114,3,0,0,0,114,
     3,0,0,0,114,6,0,0,0,218,18,95,118,97,108,105,
     100,97,116,101,95,104,97,115,104,95,112,121,99,43,2,0,
     0,115,12,0,0,0,0,17,16,1,2,1,8,255,2,2,
-    2,254,114,157,0,0,0,99,4,0,0,0,0,0,0,0,
+    2,254,114,158,0,0,0,99,4,0,0,0,0,0,0,0,
     0,0,0,0,5,0,0,0,5,0,0,0,67,0,0,0,
     115,80,0,0,0,116,0,160,1,124,0,161,1,125,4,116,
     2,124,4,116,3,131,2,114,56,116,4,160,5,100,1,124,
@@ -812,7 +812,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     116,0,0,0,114,43,0,0,0,41,10,218,7,109,97,114,
     115,104,97,108,90,5,108,111,97,100,115,218,10,105,115,105,
     110,115,116,97,110,99,101,218,10,95,99,111,100,101,95,116,
-    121,112,101,114,133,0,0,0,114,148,0,0,0,218,4,95,
+    121,112,101,114,134,0,0,0,114,149,0,0,0,218,4,95,
     105,109,112,90,16,95,102,105,120,95,99,111,95,102,105,108,
     101,110,97,109,101,114,117,0,0,0,114,61,0,0,0,41,
     5,114,25,0,0,0,114,116,0,0,0,114,106,0,0,0,
@@ -820,7 +820,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     114,3,0,0,0,114,6,0,0,0,218,17,95,99,111,109,
     112,105,108,101,95,98,121,116,101,99,111,100,101,67,2,0,
     0,115,20,0,0,0,0,2,10,1,10,1,12,1,8,1,
-    12,1,4,2,10,1,2,0,2,255,114,164,0,0,0,114,
+    12,1,4,2,10,1,2,0,2,255,114,165,0,0,0,114,
     72,0,0,0,99,3,0,0,0,0,0,0,0,0,0,0,
     0,4,0,0,0,5,0,0,0,67,0,0,0,115,70,0,
     0,0,116,0,116,1,131,1,125,3,124,3,160,2,116,3,
@@ -831,14 +831,14 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     117,99,101,32,116,104,101,32,100,97,116,97,32,102,111,114,
     32,97,32,116,105,109,101,115,116,97,109,112,45,98,97,115,
     101,100,32,112,121,99,46,114,72,0,0,0,41,6,218,9,
-    98,121,116,101,97,114,114,97,121,114,147,0,0,0,218,6,
-    101,120,116,101,110,100,114,20,0,0,0,114,159,0,0,0,
-    218,5,100,117,109,112,115,41,4,114,163,0,0,0,218,5,
-    109,116,105,109,101,114,154,0,0,0,114,25,0,0,0,114,
+    98,121,116,101,97,114,114,97,121,114,148,0,0,0,218,6,
+    101,120,116,101,110,100,114,20,0,0,0,114,160,0,0,0,
+    218,5,100,117,109,112,115,41,4,114,164,0,0,0,218,5,
+    109,116,105,109,101,114,155,0,0,0,114,25,0,0,0,114,
     3,0,0,0,114,3,0,0,0,114,6,0,0,0,218,22,
     95,99,111,100,101,95,116,111,95,116,105,109,101,115,116,97,
     109,112,95,112,121,99,80,2,0,0,115,12,0,0,0,0,
-    2,8,1,14,1,14,1,14,1,16,1,114,169,0,0,0,
+    2,8,1,14,1,14,1,14,1,16,1,114,170,0,0,0,
     84,99,3,0,0,0,0,0,0,0,0,0,0,0,5,0,
     0,0,5,0,0,0,67,0,0,0,115,80,0,0,0,116,
     0,116,1,131,1,125,3,100,1,124,2,100,1,62,0,66,
@@ -848,16 +848,16 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     5,160,6,124,0,161,1,161,1,1,0,124,3,83,0,41,
     3,122,38,80,114,111,100,117,99,101,32,116,104,101,32,100,
     97,116,97,32,102,111,114,32,97,32,104,97,115,104,45,98,
-    97,115,101,100,32,112,121,99,46,114,38,0,0,0,114,145,
-    0,0,0,41,7,114,165,0,0,0,114,147,0,0,0,114,
-    166,0,0,0,114,20,0,0,0,114,22,0,0,0,114,159,
-    0,0,0,114,167,0,0,0,41,5,114,163,0,0,0,114,
-    156,0,0,0,90,7,99,104,101,99,107,101,100,114,25,0,
+    97,115,101,100,32,112,121,99,46,114,38,0,0,0,114,146,
+    0,0,0,41,7,114,166,0,0,0,114,148,0,0,0,114,
+    167,0,0,0,114,20,0,0,0,114,22,0,0,0,114,160,
+    0,0,0,114,168,0,0,0,41,5,114,164,0,0,0,114,
+    157,0,0,0,90,7,99,104,101,99,107,101,100,114,25,0,
     0,0,114,82,0,0,0,114,3,0,0,0,114,3,0,0,
     0,114,6,0,0,0,218,17,95,99,111,100,101,95,116,111,
     95,104,97,115,104,95,112,121,99,90,2,0,0,115,14,0,
     0,0,0,2,8,1,12,1,14,1,16,1,10,1,16,1,
-    114,170,0,0,0,99,1,0,0,0,0,0,0,0,0,0,
+    114,171,0,0,0,99,1,0,0,0,0,0,0,0,0,0,
     0,0,5,0,0,0,6,0,0,0,67,0,0,0,115,62,
     0,0,0,100,1,100,2,108,0,125,1,116,1,160,2,124,
     0,161,1,106,3,125,2,124,1,160,4,124,2,161,1,125,
@@ -877,14 +877,14 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     116,95,101,110,99,111,100,105,110,103,90,25,73,110,99,114,
     101,109,101,110,116,97,108,78,101,119,108,105,110,101,68,101,
     99,111,100,101,114,218,6,100,101,99,111,100,101,41,5,218,
-    12,115,111,117,114,99,101,95,98,121,116,101,115,114,171,0,
+    12,115,111,117,114,99,101,95,98,121,116,101,115,114,172,0,
     0,0,90,21,115,111,117,114,99,101,95,98,121,116,101,115,
     95,114,101,97,100,108,105,110,101,218,8,101,110,99,111,100,
     105,110,103,90,15,110,101,119,108,105,110,101,95,100,101,99,
     111,100,101,114,114,3,0,0,0,114,3,0,0,0,114,6,
     0,0,0,218,13,100,101,99,111,100,101,95,115,111,117,114,
     99,101,101,2,0,0,115,10,0,0,0,0,5,8,1,12,
-    1,10,1,12,1,114,175,0,0,0,169,2,114,139,0,0,
+    1,10,1,12,1,114,176,0,0,0,169,2,114,140,0,0,
     0,218,26,115,117,98,109,111,100,117,108,101,95,115,101,97,
     114,99,104,95,108,111,99,97,116,105,111,110,115,99,2,0,
     0,0,0,0,0,0,2,0,0,0,9,0,0,0,8,0,
@@ -929,19 +929,19 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     122,9,60,117,110,107,110,111,119,110,62,218,12,103,101,116,
     95,102,105,108,101,110,97,109,101,169,1,218,6,111,114,105,
     103,105,110,84,218,10,105,115,95,112,97,99,107,97,103,101,
-    114,72,0,0,0,41,17,114,127,0,0,0,114,178,0,0,
+    114,72,0,0,0,41,17,114,128,0,0,0,114,179,0,0,
     0,114,117,0,0,0,114,2,0,0,0,114,78,0,0,0,
-    114,133,0,0,0,218,10,77,111,100,117,108,101,83,112,101,
+    114,134,0,0,0,218,10,77,111,100,117,108,101,83,112,101,
     99,90,13,95,115,101,116,95,102,105,108,101,97,116,116,114,
     218,27,95,103,101,116,95,115,117,112,112,111,114,116,101,100,
     95,102,105,108,101,95,108,111,97,100,101,114,115,114,110,0,
-    0,0,114,111,0,0,0,114,139,0,0,0,218,9,95,80,
-    79,80,85,76,65,84,69,114,181,0,0,0,114,177,0,0,
+    0,0,114,111,0,0,0,114,140,0,0,0,218,9,95,80,
+    79,80,85,76,65,84,69,114,182,0,0,0,114,178,0,0,
     0,114,46,0,0,0,218,6,97,112,112,101,110,100,41,9,
     114,116,0,0,0,90,8,108,111,99,97,116,105,111,110,114,
-    139,0,0,0,114,177,0,0,0,218,4,115,112,101,99,218,
+    140,0,0,0,114,178,0,0,0,218,4,115,112,101,99,218,
     12,108,111,97,100,101,114,95,99,108,97,115,115,218,8,115,
-    117,102,102,105,120,101,115,114,181,0,0,0,90,7,100,105,
+    117,102,102,105,120,101,115,114,182,0,0,0,90,7,100,105,
     114,110,97,109,101,114,3,0,0,0,114,3,0,0,0,114,
     6,0,0,0,218,23,115,112,101,99,95,102,114,111,109,95,
     102,105,108,101,95,108,111,99,97,116,105,111,110,118,2,0,
@@ -949,7 +949,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     14,1,14,1,8,2,10,8,16,1,6,3,8,1,14,1,
     14,1,10,1,6,1,6,2,4,3,8,2,10,1,2,1,
     14,1,14,1,6,2,4,1,8,2,6,1,12,1,6,1,
-    12,1,12,2,114,189,0,0,0,99,0,0,0,0,0,0,
+    12,1,12,2,114,190,0,0,0,99,0,0,0,0,0,0,
     0,0,0,0,0,0,0,0,0,0,4,0,0,0,64,0,
     0,0,115,80,0,0,0,101,0,90,1,100,0,90,2,100,
     1,90,3,100,2,90,4,100,3,90,5,100,4,90,6,101,
@@ -996,15 +996,15 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     87,0,110,22,4,0,116,9,107,10,114,108,1,0,1,0,
     1,0,89,0,100,0,83,0,88,0,124,5,83,0,41,5,
     78,122,5,37,100,46,37,100,114,27,0,0,0,41,2,114,
-    138,0,0,0,90,11,115,121,115,95,118,101,114,115,105,111,
+    139,0,0,0,90,11,115,121,115,95,118,101,114,115,105,111,
     110,114,39,0,0,0,41,10,218,11,68,69,66,85,71,95,
     66,85,73,76,68,218,18,82,69,71,73,83,84,82,89,95,
     75,69,89,95,68,69,66,85,71,218,12,82,69,71,73,83,
     84,82,89,95,75,69,89,114,61,0,0,0,114,8,0,0,
     0,218,12,118,101,114,115,105,111,110,95,105,110,102,111,114,
-    193,0,0,0,114,191,0,0,0,90,10,81,117,101,114,121,
-    86,97,108,117,101,114,49,0,0,0,41,6,114,192,0,0,
-    0,114,138,0,0,0,90,12,114,101,103,105,115,116,114,121,
+    194,0,0,0,114,192,0,0,0,90,10,81,117,101,114,121,
+    86,97,108,117,101,114,49,0,0,0,41,6,114,193,0,0,
+    0,114,139,0,0,0,90,12,114,101,103,105,115,116,114,121,
     95,107,101,121,114,5,0,0,0,90,4,104,107,101,121,218,
     8,102,105,108,101,112,97,116,104,114,3,0,0,0,114,3,
     0,0,0,114,6,0,0,0,218,16,95,115,101,97,114,99,
@@ -1023,13 +1023,13 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     124,6,131,1,161,1,114,64,116,6,106,7,124,1,124,5,
     124,1,124,4,131,2,124,4,100,1,141,3,125,7,124,7,
     2,0,1,0,83,0,113,64,100,0,83,0,41,2,78,114,
-    179,0,0,0,41,8,114,199,0,0,0,114,48,0,0,0,
-    114,49,0,0,0,114,183,0,0,0,114,110,0,0,0,114,
-    111,0,0,0,114,133,0,0,0,218,16,115,112,101,99,95,
-    102,114,111,109,95,108,111,97,100,101,114,41,8,114,192,0,
-    0,0,114,138,0,0,0,114,43,0,0,0,218,6,116,97,
-    114,103,101,116,114,198,0,0,0,114,139,0,0,0,114,188,
-    0,0,0,114,186,0,0,0,114,3,0,0,0,114,3,0,
+    180,0,0,0,41,8,114,200,0,0,0,114,48,0,0,0,
+    114,49,0,0,0,114,184,0,0,0,114,110,0,0,0,114,
+    111,0,0,0,114,134,0,0,0,218,16,115,112,101,99,95,
+    102,114,111,109,95,108,111,97,100,101,114,41,8,114,193,0,
+    0,0,114,139,0,0,0,114,43,0,0,0,218,6,116,97,
+    114,103,101,116,114,199,0,0,0,114,140,0,0,0,114,189,
+    0,0,0,114,187,0,0,0,114,3,0,0,0,114,3,0,
     0,0,114,6,0,0,0,218,9,102,105,110,100,95,115,112,
     101,99,220,2,0,0,115,28,0,0,0,0,2,10,1,8,
     1,4,1,2,1,12,1,14,1,8,1,14,1,14,1,6,
@@ -1046,23 +1046,23 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101,
     100,46,32,32,85,115,101,32,101,120,101,99,95,109,111,100,
     117,108,101,40,41,32,105,110,115,116,101,97,100,46,10,10,
-    32,32,32,32,32,32,32,32,78,169,2,114,202,0,0,0,
-    114,139,0,0,0,169,4,114,192,0,0,0,114,138,0,0,
-    0,114,43,0,0,0,114,186,0,0,0,114,3,0,0,0,
+    32,32,32,32,32,32,32,32,78,169,2,114,203,0,0,0,
+    114,140,0,0,0,169,4,114,193,0,0,0,114,139,0,0,
+    0,114,43,0,0,0,114,187,0,0,0,114,3,0,0,0,
     114,3,0,0,0,114,6,0,0,0,218,11,102,105,110,100,
     95,109,111,100,117,108,101,236,2,0,0,115,8,0,0,0,
     0,7,12,1,8,1,6,2,122,33,87,105,110,100,111,119,
     115,82,101,103,105,115,116,114,121,70,105,110,100,101,114,46,
     102,105,110,100,95,109,111,100,117,108,101,41,2,78,78,41,
-    1,78,41,12,114,124,0,0,0,114,123,0,0,0,114,125,
-    0,0,0,114,126,0,0,0,114,196,0,0,0,114,195,0,
-    0,0,114,194,0,0,0,218,11,99,108,97,115,115,109,101,
-    116,104,111,100,114,193,0,0,0,114,199,0,0,0,114,202,
-    0,0,0,114,205,0,0,0,114,3,0,0,0,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,190,0,0,
+    1,78,41,12,114,125,0,0,0,114,124,0,0,0,114,126,
+    0,0,0,114,127,0,0,0,114,197,0,0,0,114,196,0,
+    0,0,114,195,0,0,0,218,11,99,108,97,115,115,109,101,
+    116,104,111,100,114,194,0,0,0,114,200,0,0,0,114,203,
+    0,0,0,114,206,0,0,0,114,3,0,0,0,114,3,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,114,191,0,0,
     0,186,2,0,0,115,28,0,0,0,8,2,4,3,2,255,
     2,4,2,255,2,3,4,2,2,1,10,6,2,1,10,14,
-    2,1,12,15,2,1,114,190,0,0,0,99,0,0,0,0,
+    2,1,12,15,2,1,114,191,0,0,0,99,0,0,0,0,
     0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,
     64,0,0,0,115,48,0,0,0,101,0,90,1,100,0,90,
     2,100,1,90,3,100,2,100,3,132,0,90,4,100,4,100,
@@ -1091,11 +1091,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     39,95,95,105,110,105,116,95,95,46,112,121,39,46,114,38,
     0,0,0,114,70,0,0,0,114,72,0,0,0,114,27,0,
     0,0,218,8,95,95,105,110,105,116,95,95,41,4,114,46,
-    0,0,0,114,178,0,0,0,114,42,0,0,0,114,40,0,
-    0,0,41,5,114,118,0,0,0,114,138,0,0,0,114,96,
+    0,0,0,114,179,0,0,0,114,42,0,0,0,114,40,0,
+    0,0,41,5,114,118,0,0,0,114,139,0,0,0,114,96,
     0,0,0,90,13,102,105,108,101,110,97,109,101,95,98,97,
     115,101,90,9,116,97,105,108,95,110,97,109,101,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,181,0,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,114,182,0,0,
     0,255,2,0,0,115,8,0,0,0,0,3,18,1,16,1,
     14,1,122,24,95,76,111,97,100,101,114,66,97,115,105,99,
     115,46,105,115,95,112,97,99,107,97,103,101,99,2,0,0,
@@ -1104,7 +1104,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     122,42,85,115,101,32,100,101,102,97,117,108,116,32,115,101,
     109,97,110,116,105,99,115,32,102,111,114,32,109,111,100,117,
     108,101,32,99,114,101,97,116,105,111,110,46,78,114,3,0,
-    0,0,169,2,114,118,0,0,0,114,186,0,0,0,114,3,
+    0,0,169,2,114,118,0,0,0,114,187,0,0,0,114,3,
     0,0,0,114,3,0,0,0,114,6,0,0,0,218,13,99,
     114,101,97,116,101,95,109,111,100,117,108,101,7,3,0,0,
     115,2,0,0,0,0,1,122,27,95,76,111,97,100,101,114,
@@ -1120,11 +1120,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     109,111,100,117,108,101,32,123,33,114,125,32,119,104,101,110,
     32,103,101,116,95,99,111,100,101,40,41,32,114,101,116,117,
     114,110,115,32,78,111,110,101,41,8,218,8,103,101,116,95,
-    99,111,100,101,114,124,0,0,0,114,117,0,0,0,114,61,
-    0,0,0,114,133,0,0,0,218,25,95,99,97,108,108,95,
+    99,111,100,101,114,125,0,0,0,114,117,0,0,0,114,61,
+    0,0,0,114,134,0,0,0,218,25,95,99,97,108,108,95,
     119,105,116,104,95,102,114,97,109,101,115,95,114,101,109,111,
-    118,101,100,218,4,101,120,101,99,114,130,0,0,0,41,3,
-    114,118,0,0,0,218,6,109,111,100,117,108,101,114,163,0,
+    118,101,100,218,4,101,120,101,99,114,131,0,0,0,41,3,
+    114,118,0,0,0,218,6,109,111,100,117,108,101,114,164,0,
     0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
     0,218,11,101,120,101,99,95,109,111,100,117,108,101,10,3,
     0,0,115,12,0,0,0,0,2,12,1,8,1,6,1,4,
@@ -1134,19 +1134,19 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,67,0,0,0,115,12,0,0,0,116,0,160,1,
     124,0,124,1,161,2,83,0,41,1,122,26,84,104,105,115,
     32,109,111,100,117,108,101,32,105,115,32,100,101,112,114,101,
-    99,97,116,101,100,46,41,2,114,133,0,0,0,218,17,95,
+    99,97,116,101,100,46,41,2,114,134,0,0,0,218,17,95,
     108,111,97,100,95,109,111,100,117,108,101,95,115,104,105,109,
-    169,2,114,118,0,0,0,114,138,0,0,0,114,3,0,0,
+    169,2,114,118,0,0,0,114,139,0,0,0,114,3,0,0,
     0,114,3,0,0,0,114,6,0,0,0,218,11,108,111,97,
     100,95,109,111,100,117,108,101,18,3,0,0,115,2,0,0,
     0,0,2,122,25,95,76,111,97,100,101,114,66,97,115,105,
     99,115,46,108,111,97,100,95,109,111,100,117,108,101,78,41,
-    8,114,124,0,0,0,114,123,0,0,0,114,125,0,0,0,
-    114,126,0,0,0,114,181,0,0,0,114,211,0,0,0,114,
-    216,0,0,0,114,219,0,0,0,114,3,0,0,0,114,3,
-    0,0,0,114,3,0,0,0,114,6,0,0,0,114,207,0,
+    8,114,125,0,0,0,114,124,0,0,0,114,126,0,0,0,
+    114,127,0,0,0,114,182,0,0,0,114,212,0,0,0,114,
+    217,0,0,0,114,220,0,0,0,114,3,0,0,0,114,3,
+    0,0,0,114,3,0,0,0,114,6,0,0,0,114,208,0,
     0,0,250,2,0,0,115,10,0,0,0,8,2,4,3,8,
-    8,8,3,8,8,114,207,0,0,0,99,0,0,0,0,0,
+    8,8,3,8,8,114,208,0,0,0,99,0,0,0,0,0,
     0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,64,
     0,0,0,115,74,0,0,0,101,0,90,1,100,0,90,2,
     100,1,100,2,132,0,90,3,100,3,100,4,132,0,90,4,
@@ -1201,8 +1201,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     115,101,115,32,79,83,69,114,114,111,114,32,119,104,101,110,
     32,116,104,101,32,112,97,116,104,32,99,97,110,110,111,116,
     32,98,101,32,104,97,110,100,108,101,100,46,10,32,32,32,
-    32,32,32,32,32,114,168,0,0,0,41,1,114,222,0,0,
-    0,114,221,0,0,0,114,3,0,0,0,114,3,0,0,0,
+    32,32,32,32,32,114,169,0,0,0,41,1,114,223,0,0,
+    0,114,222,0,0,0,114,3,0,0,0,114,3,0,0,0,
     114,6,0,0,0,218,10,112,97,116,104,95,115,116,97,116,
     115,33,3,0,0,115,2,0,0,0,0,12,122,23,83,111,
     117,114,99,101,76,111,97,100,101,114,46,112,97,116,104,95,
@@ -1244,7 +1244,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     101,99,111,100,101,32,102,105,108,101,115,46,10,32,32,32,
     32,32,32,32,32,78,114,3,0,0,0,41,3,114,118,0,
     0,0,114,43,0,0,0,114,25,0,0,0,114,3,0,0,
-    0,114,3,0,0,0,114,6,0,0,0,114,224,0,0,0,
+    0,114,3,0,0,0,114,6,0,0,0,114,225,0,0,0,
     57,3,0,0,115,2,0,0,0,0,1,122,21,83,111,117,
     114,99,101,76,111,97,100,101,114,46,115,101,116,95,100,97,
     116,97,99,2,0,0,0,0,0,0,0,0,0,0,0,5,
@@ -1260,10 +1260,10 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     101,116,95,115,111,117,114,99,101,46,122,39,115,111,117,114,
     99,101,32,110,111,116,32,97,118,97,105,108,97,98,108,101,
     32,116,104,114,111,117,103,104,32,103,101,116,95,100,97,116,
-    97,40,41,114,115,0,0,0,78,41,5,114,178,0,0,0,
+    97,40,41,114,115,0,0,0,78,41,5,114,179,0,0,0,
     218,8,103,101,116,95,100,97,116,97,114,49,0,0,0,114,
-    117,0,0,0,114,175,0,0,0,41,5,114,118,0,0,0,
-    114,138,0,0,0,114,43,0,0,0,114,173,0,0,0,218,
+    117,0,0,0,114,176,0,0,0,41,5,114,118,0,0,0,
+    114,139,0,0,0,114,43,0,0,0,114,174,0,0,0,218,
     3,101,120,99,114,3,0,0,0,114,3,0,0,0,114,6,
     0,0,0,218,10,103,101,116,95,115,111,117,114,99,101,64,
     3,0,0,115,20,0,0,0,0,2,10,1,2,1,14,1,
@@ -1282,11 +1282,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     97,110,121,32,111,98,106,101,99,116,32,116,121,112,101,32,
     116,104,97,116,32,99,111,109,112,105,108,101,40,41,32,115,
     117,112,112,111,114,116,115,46,10,32,32,32,32,32,32,32,
-    32,114,214,0,0,0,84,41,2,218,12,100,111,110,116,95,
-    105,110,104,101,114,105,116,114,83,0,0,0,41,3,114,133,
-    0,0,0,114,213,0,0,0,218,7,99,111,109,112,105,108,
+    32,114,215,0,0,0,84,41,2,218,12,100,111,110,116,95,
+    105,110,104,101,114,105,116,114,83,0,0,0,41,3,114,134,
+    0,0,0,114,214,0,0,0,218,7,99,111,109,112,105,108,
     101,41,4,114,118,0,0,0,114,25,0,0,0,114,43,0,
-    0,0,114,229,0,0,0,114,3,0,0,0,114,3,0,0,
+    0,0,114,230,0,0,0,114,3,0,0,0,114,3,0,0,
     0,114,6,0,0,0,218,14,115,111,117,114,99,101,95,116,
     111,95,99,111,100,101,74,3,0,0,115,8,0,0,0,0,
     5,12,1,2,0,2,255,122,27,83,111,117,114,99,101,76,
@@ -1339,33 +1339,33 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     98,121,116,101,99,111,100,101,44,32,115,101,116,95,100,97,
     116,97,32,109,117,115,116,32,97,108,115,111,32,98,101,32,
     105,109,112,108,101,109,101,110,116,101,100,46,10,10,32,32,
-    32,32,32,32,32,32,78,70,84,114,168,0,0,0,114,158,
-    0,0,0,114,144,0,0,0,114,38,0,0,0,114,72,0,
+    32,32,32,32,32,32,78,70,84,114,169,0,0,0,114,159,
+    0,0,0,114,145,0,0,0,114,38,0,0,0,114,72,0,
     0,0,114,27,0,0,0,90,5,110,101,118,101,114,90,6,
     97,108,119,97,121,115,218,4,115,105,122,101,122,13,123,125,
     32,109,97,116,99,104,101,115,32,123,125,41,3,114,116,0,
     0,0,114,106,0,0,0,114,107,0,0,0,122,19,99,111,
     100,101,32,111,98,106,101,99,116,32,102,114,111,109,32,123,
-    125,41,27,114,178,0,0,0,114,97,0,0,0,114,81,0,
-    0,0,114,223,0,0,0,114,49,0,0,0,114,17,0,0,
-    0,114,226,0,0,0,114,151,0,0,0,218,10,109,101,109,
-    111,114,121,118,105,101,119,114,162,0,0,0,90,21,99,104,
+    125,41,27,114,179,0,0,0,114,97,0,0,0,114,81,0,
+    0,0,114,224,0,0,0,114,49,0,0,0,114,17,0,0,
+    0,114,227,0,0,0,114,152,0,0,0,218,10,109,101,109,
+    111,114,121,118,105,101,119,114,163,0,0,0,90,21,99,104,
     101,99,107,95,104,97,115,104,95,98,97,115,101,100,95,112,
-    121,99,115,114,156,0,0,0,218,17,95,82,65,87,95,77,
-    65,71,73,67,95,78,85,77,66,69,82,114,157,0,0,0,
-    114,155,0,0,0,114,117,0,0,0,114,149,0,0,0,114,
-    133,0,0,0,114,148,0,0,0,114,164,0,0,0,114,232,
+    121,99,115,114,157,0,0,0,218,17,95,82,65,87,95,77,
+    65,71,73,67,95,78,85,77,66,69,82,114,158,0,0,0,
+    114,156,0,0,0,114,117,0,0,0,114,150,0,0,0,114,
+    134,0,0,0,114,149,0,0,0,114,165,0,0,0,114,233,
     0,0,0,114,8,0,0,0,218,19,100,111,110,116,95,119,
-    114,105,116,101,95,98,121,116,101,99,111,100,101,114,170,0,
-    0,0,114,169,0,0,0,114,22,0,0,0,114,225,0,0,
-    0,41,15,114,118,0,0,0,114,138,0,0,0,114,107,0,
-    0,0,114,153,0,0,0,114,173,0,0,0,114,156,0,0,
+    114,105,116,101,95,98,121,116,101,99,111,100,101,114,171,0,
+    0,0,114,170,0,0,0,114,22,0,0,0,114,226,0,0,
+    0,41,15,114,118,0,0,0,114,139,0,0,0,114,107,0,
+    0,0,114,154,0,0,0,114,174,0,0,0,114,157,0,0,
     0,90,10,104,97,115,104,95,98,97,115,101,100,90,12,99,
     104,101,99,107,95,115,111,117,114,99,101,114,106,0,0,0,
-    218,2,115,116,114,25,0,0,0,114,150,0,0,0,114,82,
+    218,2,115,116,114,25,0,0,0,114,151,0,0,0,114,82,
     0,0,0,90,10,98,121,116,101,115,95,100,97,116,97,90,
     11,99,111,100,101,95,111,98,106,101,99,116,114,3,0,0,
-    0,114,3,0,0,0,114,6,0,0,0,114,212,0,0,0,
+    0,114,3,0,0,0,114,6,0,0,0,114,213,0,0,0,
     82,3,0,0,115,152,0,0,0,0,7,10,1,4,1,4,
     1,4,1,4,1,4,1,2,1,12,1,14,1,12,2,2,
     1,14,1,14,1,8,2,12,1,2,1,14,1,14,1,6,
@@ -1377,13 +1377,13 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     1,12,1,12,1,18,1,6,255,4,2,6,1,10,1,10,
     1,14,2,6,1,6,255,4,2,2,1,18,1,16,1,6,
     1,122,21,83,111,117,114,99,101,76,111,97,100,101,114,46,
-    103,101,116,95,99,111,100,101,78,41,10,114,124,0,0,0,
-    114,123,0,0,0,114,125,0,0,0,114,222,0,0,0,114,
-    223,0,0,0,114,225,0,0,0,114,224,0,0,0,114,228,
-    0,0,0,114,232,0,0,0,114,212,0,0,0,114,3,0,
+    103,101,116,95,99,111,100,101,78,41,10,114,125,0,0,0,
+    114,124,0,0,0,114,126,0,0,0,114,223,0,0,0,114,
+    224,0,0,0,114,226,0,0,0,114,225,0,0,0,114,229,
+    0,0,0,114,233,0,0,0,114,213,0,0,0,114,3,0,
     0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
-    0,114,220,0,0,0,23,3,0,0,115,14,0,0,0,8,
-    2,8,8,8,14,8,10,8,7,8,10,14,8,114,220,0,
+    0,114,221,0,0,0,23,3,0,0,115,14,0,0,0,8,
+    2,8,8,8,14,8,10,8,7,8,10,14,8,114,221,0,
     0,0,99,0,0,0,0,0,0,0,0,0,0,0,0,0,
     0,0,0,4,0,0,0,0,0,0,0,115,124,0,0,0,
     101,0,90,1,100,0,90,2,100,1,90,3,100,2,100,3,
@@ -1409,16 +1409,16 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     104,101,32,112,97,116,104,32,116,111,32,116,104,101,32,102,
     105,108,101,32,102,111,117,110,100,32,98,121,32,116,104,101,
     10,32,32,32,32,32,32,32,32,102,105,110,100,101,114,46,
-    78,114,158,0,0,0,41,3,114,118,0,0,0,114,138,0,
+    78,114,159,0,0,0,41,3,114,118,0,0,0,114,139,0,
     0,0,114,43,0,0,0,114,3,0,0,0,114,3,0,0,
-    0,114,6,0,0,0,114,208,0,0,0,172,3,0,0,115,
+    0,114,6,0,0,0,114,209,0,0,0,172,3,0,0,115,
     4,0,0,0,0,3,6,1,122,19,70,105,108,101,76,111,
     97,100,101,114,46,95,95,105,110,105,116,95,95,99,2,0,
     0,0,0,0,0,0,0,0,0,0,2,0,0,0,2,0,
     0,0,67,0,0,0,115,24,0,0,0,124,0,106,0,124,
     1,106,0,107,2,111,22,124,0,106,1,124,1,106,1,107,
     2,83,0,114,109,0,0,0,169,2,218,9,95,95,99,108,
-    97,115,115,95,95,114,130,0,0,0,169,2,114,118,0,0,
+    97,115,115,95,95,114,131,0,0,0,169,2,114,118,0,0,
     0,90,5,111,116,104,101,114,114,3,0,0,0,114,3,0,
     0,0,114,6,0,0,0,218,6,95,95,101,113,95,95,178,
     3,0,0,115,6,0,0,0,0,1,12,1,10,255,122,17,
@@ -1441,9 +1441,9 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     101,112,114,101,99,97,116,101,100,46,32,32,85,115,101,32,
     101,120,101,99,95,109,111,100,117,108,101,40,41,32,105,110,
     115,116,101,97,100,46,10,10,32,32,32,32,32,32,32,32,
-    41,3,218,5,115,117,112,101,114,114,238,0,0,0,114,219,
-    0,0,0,114,218,0,0,0,169,1,114,240,0,0,0,114,
-    3,0,0,0,114,6,0,0,0,114,219,0,0,0,185,3,
+    41,3,218,5,115,117,112,101,114,114,239,0,0,0,114,220,
+    0,0,0,114,219,0,0,0,169,1,114,241,0,0,0,114,
+    3,0,0,0,114,6,0,0,0,114,220,0,0,0,185,3,
     0,0,115,2,0,0,0,0,10,122,22,70,105,108,101,76,
     111,97,100,101,114,46,108,111,97,100,95,109,111,100,117,108,
     101,99,2,0,0,0,0,0,0,0,0,0,0,0,2,0,
@@ -1452,8 +1452,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     116,104,101,32,112,97,116,104,32,116,111,32,116,104,101,32,
     115,111,117,114,99,101,32,102,105,108,101,32,97,115,32,102,
     111,117,110,100,32,98,121,32,116,104,101,32,102,105,110,100,
-    101,114,46,114,47,0,0,0,114,218,0,0,0,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,178,0,0,
+    101,114,46,114,47,0,0,0,114,219,0,0,0,114,3,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,114,179,0,0,
     0,197,3,0,0,115,2,0,0,0,0,3,122,23,70,105,
     108,101,76,111,97,100,101,114,46,103,101,116,95,102,105,108,
     101,110,97,109,101,99,2,0,0,0,0,0,0,0,0,0,
@@ -1467,20 +1467,20 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,82,0,88,0,100,2,83,0,41,3,122,39,82,101,116,
     117,114,110,32,116,104,101,32,100,97,116,97,32,102,114,111,
     109,32,112,97,116,104,32,97,115,32,114,97,119,32,98,121,
-    116,101,115,46,218,1,114,78,41,8,114,160,0,0,0,114,
-    220,0,0,0,218,19,69,120,116,101,110,115,105,111,110,70,
+    116,101,115,46,218,1,114,78,41,8,114,161,0,0,0,114,
+    221,0,0,0,218,19,69,120,116,101,110,115,105,111,110,70,
     105,108,101,76,111,97,100,101,114,114,63,0,0,0,90,9,
     111,112,101,110,95,99,111,100,101,114,84,0,0,0,90,4,
     114,101,97,100,114,64,0,0,0,41,3,114,118,0,0,0,
     114,43,0,0,0,114,67,0,0,0,114,3,0,0,0,114,
-    3,0,0,0,114,6,0,0,0,114,226,0,0,0,202,3,
+    3,0,0,0,114,6,0,0,0,114,227,0,0,0,202,3,
     0,0,115,10,0,0,0,0,2,14,1,16,1,28,2,14,
     1,122,19,70,105,108,101,76,111,97,100,101,114,46,103,101,
     116,95,100,97,116,97,99,2,0,0,0,0,0,0,0,0,
     0,0,0,2,0,0,0,3,0,0,0,67,0,0,0,115,
     18,0,0,0,124,0,160,0,124,1,161,1,114,14,124,0,
-    83,0,100,0,83,0,114,109,0,0,0,41,1,114,181,0,
-    0,0,169,2,114,118,0,0,0,114,215,0,0,0,114,3,
+    83,0,100,0,83,0,114,109,0,0,0,41,1,114,182,0,
+    0,0,169,2,114,118,0,0,0,114,216,0,0,0,114,3,
     0,0,0,114,3,0,0,0,114,6,0,0,0,218,19,103,
     101,116,95,114,101,115,111,117,114,99,101,95,114,101,97,100,
     101,114,213,3,0,0,115,6,0,0,0,0,2,10,1,4,
@@ -1490,7 +1490,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,4,0,0,0,67,0,0,0,115,32,0,0,0,116,
     0,116,1,124,0,106,2,131,1,100,1,25,0,124,1,131,
     2,125,2,116,3,160,4,124,2,100,2,161,2,83,0,41,
-    3,78,114,72,0,0,0,114,250,0,0,0,41,5,114,37,
+    3,78,114,72,0,0,0,114,251,0,0,0,41,5,114,37,
     0,0,0,114,46,0,0,0,114,43,0,0,0,114,63,0,
     0,0,114,64,0,0,0,169,3,114,118,0,0,0,90,8,
     114,101,115,111,117,114,99,101,114,43,0,0,0,114,3,0,
@@ -1506,7 +1506,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     218,11,105,115,95,114,101,115,111,117,114,99,101,218,17,70,
     105,108,101,78,111,116,70,111,117,110,100,69,114,114,111,114,
     114,37,0,0,0,114,46,0,0,0,114,43,0,0,0,114,
-    254,0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
+    255,0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
     0,0,0,218,13,114,101,115,111,117,114,99,101,95,112,97,
     116,104,223,3,0,0,115,8,0,0,0,0,1,10,1,4,
     1,20,1,122,24,70,105,108,101,76,111,97,100,101,114,46,
@@ -1519,30 +1519,30 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,114,37,0,0,0,114,46,0,0,0,114,43,0,
     0,0,114,53,0,0,0,169,3,114,118,0,0,0,114,116,
     0,0,0,114,43,0,0,0,114,3,0,0,0,114,3,0,
-    0,0,114,6,0,0,0,114,1,1,0,0,229,3,0,0,
+    0,0,114,6,0,0,0,114,2,1,0,0,229,3,0,0,
     115,8,0,0,0,0,1,8,1,4,1,20,1,122,22,70,
     105,108,101,76,111,97,100,101,114,46,105,115,95,114,101,115,
     111,117,114,99,101,99,1,0,0,0,0,0,0,0,0,0,
     0,0,1,0,0,0,5,0,0,0,67,0,0,0,115,24,
     0,0,0,116,0,116,1,160,2,116,3,124,0,106,4,131,
-    1,100,1,25,0,161,1,131,1,83,0,114,0,1,0,0,
+    1,100,1,25,0,161,1,131,1,83,0,114,1,1,0,0,
     41,5,218,4,105,116,101,114,114,2,0,0,0,218,7,108,
     105,115,116,100,105,114,114,46,0,0,0,114,43,0,0,0,
-    114,245,0,0,0,114,3,0,0,0,114,3,0,0,0,114,
+    114,246,0,0,0,114,3,0,0,0,114,3,0,0,0,114,
     6,0,0,0,218,8,99,111,110,116,101,110,116,115,235,3,
     0,0,115,2,0,0,0,0,1,122,19,70,105,108,101,76,
     111,97,100,101,114,46,99,111,110,116,101,110,116,115,41,17,
-    114,124,0,0,0,114,123,0,0,0,114,125,0,0,0,114,
-    126,0,0,0,114,208,0,0,0,114,242,0,0,0,114,246,
-    0,0,0,114,135,0,0,0,114,219,0,0,0,114,178,0,
-    0,0,114,226,0,0,0,114,253,0,0,0,114,255,0,0,
-    0,114,3,1,0,0,114,1,1,0,0,114,7,1,0,0,
+    114,125,0,0,0,114,124,0,0,0,114,126,0,0,0,114,
+    127,0,0,0,114,209,0,0,0,114,243,0,0,0,114,247,
+    0,0,0,114,136,0,0,0,114,220,0,0,0,114,179,0,
+    0,0,114,227,0,0,0,114,254,0,0,0,114,0,1,0,
+    0,114,4,1,0,0,114,2,1,0,0,114,8,1,0,0,
     90,13,95,95,99,108,97,115,115,99,101,108,108,95,95,114,
-    3,0,0,0,114,3,0,0,0,114,248,0,0,0,114,6,
-    0,0,0,114,238,0,0,0,167,3,0,0,115,30,0,0,
+    3,0,0,0,114,3,0,0,0,114,249,0,0,0,114,6,
+    0,0,0,114,239,0,0,0,167,3,0,0,115,30,0,0,
     0,8,2,4,3,8,6,8,4,8,3,2,1,14,11,2,
     1,10,4,8,11,2,1,10,5,8,4,8,6,8,6,114,
-    238,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0,
+    239,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0,
     0,0,0,0,0,3,0,0,0,64,0,0,0,115,46,0,
     0,0,101,0,90,1,100,0,90,2,100,1,90,3,100,2,
     100,3,132,0,90,4,100,4,100,5,132,0,90,5,100,6,
@@ -1558,25 +1558,25 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     106,2,100,1,156,2,83,0,41,2,122,33,82,101,116,117,
     114,110,32,116,104,101,32,109,101,116,97,100,97,116,97,32,
     102,111,114,32,116,104,101,32,112,97,116,104,46,41,2,114,
-    168,0,0,0,114,233,0,0,0,41,3,114,48,0,0,0,
+    169,0,0,0,114,234,0,0,0,41,3,114,48,0,0,0,
     218,8,115,116,95,109,116,105,109,101,90,7,115,116,95,115,
     105,122,101,41,3,114,118,0,0,0,114,43,0,0,0,114,
-    237,0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
-    0,0,0,114,223,0,0,0,243,3,0,0,115,4,0,0,
+    238,0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
+    0,0,0,114,224,0,0,0,243,3,0,0,115,4,0,0,
     0,0,2,8,1,122,27,83,111,117,114,99,101,70,105,108,
     101,76,111,97,100,101,114,46,112,97,116,104,95,115,116,97,
     116,115,99,4,0,0,0,0,0,0,0,0,0,0,0,5,
     0,0,0,5,0,0,0,67,0,0,0,115,24,0,0,0,
     116,0,124,1,131,1,125,4,124,0,106,1,124,2,124,3,
     124,4,100,1,141,3,83,0,41,2,78,169,1,218,5,95,
-    109,111,100,101,41,2,114,114,0,0,0,114,224,0,0,0,
+    109,111,100,101,41,2,114,114,0,0,0,114,225,0,0,0,
     41,5,114,118,0,0,0,114,107,0,0,0,114,106,0,0,
     0,114,25,0,0,0,114,51,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,225,0,0,0,248,
+    114,3,0,0,0,114,6,0,0,0,114,226,0,0,0,248,
     3,0,0,115,4,0,0,0,0,2,8,1,122,32,83,111,
     117,114,99,101,70,105,108,101,76,111,97,100,101,114,46,95,
     99,97,99,104,101,95,98,121,116,101,99,111,100,101,114,59,
-    0,0,0,114,10,1,0,0,99,3,0,0,0,0,0,0,
+    0,0,0,114,11,1,0,0,99,3,0,0,0,0,0,0,
     0,1,0,0,0,9,0,0,0,11,0,0,0,67,0,0,
     0,115,252,0,0,0,116,0,124,1,131,1,92,2,125,4,
     125,5,103,0,125,6,124,4,114,52,116,1,124,4,131,1,
@@ -1599,26 +1599,26 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     46,122,27,99,111,117,108,100,32,110,111,116,32,99,114,101,
     97,116,101,32,123,33,114,125,58,32,123,33,114,125,78,122,
     12,99,114,101,97,116,101,100,32,123,33,114,125,41,12,114,
-    46,0,0,0,114,55,0,0,0,114,185,0,0,0,114,41,
+    46,0,0,0,114,55,0,0,0,114,186,0,0,0,114,41,
     0,0,0,114,37,0,0,0,114,2,0,0,0,90,5,109,
     107,100,105,114,218,15,70,105,108,101,69,120,105,115,116,115,
-    69,114,114,111,114,114,49,0,0,0,114,133,0,0,0,114,
-    148,0,0,0,114,68,0,0,0,41,9,114,118,0,0,0,
-    114,43,0,0,0,114,25,0,0,0,114,11,1,0,0,218,
+    69,114,114,111,114,114,49,0,0,0,114,134,0,0,0,114,
+    149,0,0,0,114,68,0,0,0,41,9,114,118,0,0,0,
+    114,43,0,0,0,114,25,0,0,0,114,12,1,0,0,218,
     6,112,97,114,101,110,116,114,96,0,0,0,114,36,0,0,
-    0,114,32,0,0,0,114,227,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,224,0,0,0,253,
+    0,114,32,0,0,0,114,228,0,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,225,0,0,0,253,
     3,0,0,115,48,0,0,0,0,2,12,1,4,2,12,1,
     12,1,12,2,12,1,10,1,2,1,14,1,14,2,8,1,
     16,3,6,1,2,0,2,255,4,2,28,1,2,1,12,1,
     16,1,16,2,8,1,2,255,122,25,83,111,117,114,99,101,
     70,105,108,101,76,111,97,100,101,114,46,115,101,116,95,100,
-    97,116,97,78,41,7,114,124,0,0,0,114,123,0,0,0,
-    114,125,0,0,0,114,126,0,0,0,114,223,0,0,0,114,
-    225,0,0,0,114,224,0,0,0,114,3,0,0,0,114,3,
-    0,0,0,114,3,0,0,0,114,6,0,0,0,114,8,1,
+    97,116,97,78,41,7,114,125,0,0,0,114,124,0,0,0,
+    114,126,0,0,0,114,127,0,0,0,114,224,0,0,0,114,
+    226,0,0,0,114,225,0,0,0,114,3,0,0,0,114,3,
+    0,0,0,114,3,0,0,0,114,6,0,0,0,114,9,1,
     0,0,239,3,0,0,115,8,0,0,0,8,2,4,2,8,
-    5,8,5,114,8,1,0,0,99,0,0,0,0,0,0,0,
+    5,8,5,114,9,1,0,0,99,0,0,0,0,0,0,0,
     0,0,0,0,0,0,0,0,0,2,0,0,0,64,0,0,
     0,115,32,0,0,0,101,0,90,1,100,0,90,2,100,1,
     90,3,100,2,100,3,132,0,90,4,100,4,100,5,132,0,
@@ -1633,12 +1633,12 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     1,124,2,100,1,156,2,125,4,116,2,124,3,124,1,124,
     4,131,3,1,0,116,3,116,4,124,3,131,1,100,2,100,
     0,133,2,25,0,124,1,124,2,100,3,141,3,83,0,41,
-    4,78,114,158,0,0,0,114,144,0,0,0,41,2,114,116,
-    0,0,0,114,106,0,0,0,41,5,114,178,0,0,0,114,
-    226,0,0,0,114,151,0,0,0,114,164,0,0,0,114,234,
-    0,0,0,41,5,114,118,0,0,0,114,138,0,0,0,114,
-    43,0,0,0,114,25,0,0,0,114,150,0,0,0,114,3,
-    0,0,0,114,3,0,0,0,114,6,0,0,0,114,212,0,
+    4,78,114,159,0,0,0,114,145,0,0,0,41,2,114,116,
+    0,0,0,114,106,0,0,0,41,5,114,179,0,0,0,114,
+    227,0,0,0,114,152,0,0,0,114,165,0,0,0,114,235,
+    0,0,0,41,5,114,118,0,0,0,114,139,0,0,0,114,
+    43,0,0,0,114,25,0,0,0,114,151,0,0,0,114,3,
+    0,0,0,114,3,0,0,0,114,6,0,0,0,114,213,0,
     0,0,32,4,0,0,115,22,0,0,0,0,1,10,1,10,
     4,2,1,2,254,6,4,12,1,2,1,14,1,2,1,2,
     253,122,29,83,111,117,114,99,101,108,101,115,115,70,105,108,
@@ -1648,16 +1648,16 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     83,0,41,2,122,39,82,101,116,117,114,110,32,78,111,110,
     101,32,97,115,32,116,104,101,114,101,32,105,115,32,110,111,
     32,115,111,117,114,99,101,32,99,111,100,101,46,78,114,3,
-    0,0,0,114,218,0,0,0,114,3,0,0,0,114,3,0,
-    0,0,114,6,0,0,0,114,228,0,0,0,48,4,0,0,
+    0,0,0,114,219,0,0,0,114,3,0,0,0,114,3,0,
+    0,0,114,6,0,0,0,114,229,0,0,0,48,4,0,0,
     115,2,0,0,0,0,2,122,31,83,111,117,114,99,101,108,
     101,115,115,70,105,108,101,76,111,97,100,101,114,46,103,101,
-    116,95,115,111,117,114,99,101,78,41,6,114,124,0,0,0,
-    114,123,0,0,0,114,125,0,0,0,114,126,0,0,0,114,
-    212,0,0,0,114,228,0,0,0,114,3,0,0,0,114,3,
-    0,0,0,114,3,0,0,0,114,6,0,0,0,114,14,1,
+    116,95,115,111,117,114,99,101,78,41,6,114,125,0,0,0,
+    114,124,0,0,0,114,126,0,0,0,114,127,0,0,0,114,
+    213,0,0,0,114,229,0,0,0,114,3,0,0,0,114,3,
+    0,0,0,114,3,0,0,0,114,6,0,0,0,114,15,1,
     0,0,28,4,0,0,115,6,0,0,0,8,2,4,2,8,
-    16,114,14,1,0,0,99,0,0,0,0,0,0,0,0,0,
+    16,114,15,1,0,0,99,0,0,0,0,0,0,0,0,0,
     0,0,0,0,0,0,0,3,0,0,0,64,0,0,0,115,
     92,0,0,0,101,0,90,1,100,0,90,2,100,1,90,3,
     100,2,100,3,132,0,90,4,100,4,100,5,132,0,90,5,
@@ -1665,7 +1665,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     100,10,100,11,132,0,90,8,100,12,100,13,132,0,90,9,
     100,14,100,15,132,0,90,10,100,16,100,17,132,0,90,11,
     101,12,100,18,100,19,132,0,131,1,90,13,100,20,83,0,
-    41,21,114,251,0,0,0,122,93,76,111,97,100,101,114,32,
+    41,21,114,252,0,0,0,122,93,76,111,97,100,101,114,32,
     102,111,114,32,101,120,116,101,110,115,105,111,110,32,109,111,
     100,117,108,101,115,46,10,10,32,32,32,32,84,104,101,32,
     99,111,110,115,116,114,117,99,116,111,114,32,105,115,32,100,
@@ -1674,25 +1674,25 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     10,10,32,32,32,32,99,3,0,0,0,0,0,0,0,0,
     0,0,0,3,0,0,0,2,0,0,0,67,0,0,0,115,
     16,0,0,0,124,1,124,0,95,0,124,2,124,0,95,1,
-    100,0,83,0,114,109,0,0,0,114,158,0,0,0,114,4,
+    100,0,83,0,114,109,0,0,0,114,159,0,0,0,114,5,
     1,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,114,208,0,0,0,65,4,0,0,115,4,0,0,0,
+    0,0,114,209,0,0,0,65,4,0,0,115,4,0,0,0,
     0,1,6,1,122,28,69,120,116,101,110,115,105,111,110,70,
     105,108,101,76,111,97,100,101,114,46,95,95,105,110,105,116,
     95,95,99,2,0,0,0,0,0,0,0,0,0,0,0,2,
     0,0,0,2,0,0,0,67,0,0,0,115,24,0,0,0,
     124,0,106,0,124,1,106,0,107,2,111,22,124,0,106,1,
-    124,1,106,1,107,2,83,0,114,109,0,0,0,114,239,0,
-    0,0,114,241,0,0,0,114,3,0,0,0,114,3,0,0,
-    0,114,6,0,0,0,114,242,0,0,0,69,4,0,0,115,
+    124,1,106,1,107,2,83,0,114,109,0,0,0,114,240,0,
+    0,0,114,242,0,0,0,114,3,0,0,0,114,3,0,0,
+    0,114,6,0,0,0,114,243,0,0,0,69,4,0,0,115,
     6,0,0,0,0,1,12,1,10,255,122,26,69,120,116,101,
     110,115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,
     95,95,101,113,95,95,99,1,0,0,0,0,0,0,0,0,
     0,0,0,1,0,0,0,3,0,0,0,67,0,0,0,115,
     20,0,0,0,116,0,124,0,106,1,131,1,116,0,124,0,
-    106,2,131,1,65,0,83,0,114,109,0,0,0,114,243,0,
-    0,0,114,245,0,0,0,114,3,0,0,0,114,3,0,0,
-    0,114,6,0,0,0,114,246,0,0,0,73,4,0,0,115,
+    106,2,131,1,65,0,83,0,114,109,0,0,0,114,244,0,
+    0,0,114,246,0,0,0,114,3,0,0,0,114,3,0,0,
+    0,114,6,0,0,0,114,247,0,0,0,73,4,0,0,115,
     2,0,0,0,0,1,122,28,69,120,116,101,110,115,105,111,
     110,70,105,108,101,76,111,97,100,101,114,46,95,95,104,97,
     115,104,95,95,99,2,0,0,0,0,0,0,0,0,0,0,
@@ -1704,12 +1704,12 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     101,120,116,101,110,115,105,111,110,32,109,111,100,117,108,101,
     122,38,101,120,116,101,110,115,105,111,110,32,109,111,100,117,
     108,101,32,123,33,114,125,32,108,111,97,100,101,100,32,102,
-    114,111,109,32,123,33,114,125,41,7,114,133,0,0,0,114,
-    213,0,0,0,114,162,0,0,0,90,14,99,114,101,97,116,
-    101,95,100,121,110,97,109,105,99,114,148,0,0,0,114,116,
+    114,111,109,32,123,33,114,125,41,7,114,134,0,0,0,114,
+    214,0,0,0,114,163,0,0,0,90,14,99,114,101,97,116,
+    101,95,100,121,110,97,109,105,99,114,149,0,0,0,114,116,
     0,0,0,114,43,0,0,0,41,3,114,118,0,0,0,114,
-    186,0,0,0,114,215,0,0,0,114,3,0,0,0,114,3,
-    0,0,0,114,6,0,0,0,114,211,0,0,0,76,4,0,
+    187,0,0,0,114,216,0,0,0,114,3,0,0,0,114,3,
+    0,0,0,114,6,0,0,0,114,212,0,0,0,76,4,0,
     0,115,18,0,0,0,0,2,4,1,4,0,2,255,4,2,
     6,1,4,0,4,255,4,2,122,33,69,120,116,101,110,115,
     105,111,110,70,105,108,101,76,111,97,100,101,114,46,99,114,
@@ -1722,11 +1722,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     120,116,101,110,115,105,111,110,32,109,111,100,117,108,101,122,
     40,101,120,116,101,110,115,105,111,110,32,109,111,100,117,108,
     101,32,123,33,114,125,32,101,120,101,99,117,116,101,100,32,
-    102,114,111,109,32,123,33,114,125,78,41,7,114,133,0,0,
-    0,114,213,0,0,0,114,162,0,0,0,90,12,101,120,101,
-    99,95,100,121,110,97,109,105,99,114,148,0,0,0,114,116,
-    0,0,0,114,43,0,0,0,114,252,0,0,0,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,216,0,0,
+    102,114,111,109,32,123,33,114,125,78,41,7,114,134,0,0,
+    0,114,214,0,0,0,114,163,0,0,0,90,12,101,120,101,
+    99,95,100,121,110,97,109,105,99,114,149,0,0,0,114,116,
+    0,0,0,114,43,0,0,0,114,253,0,0,0,114,3,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,114,217,0,0,
     0,84,4,0,0,115,10,0,0,0,0,2,14,1,6,1,
     4,0,4,255,122,31,69,120,116,101,110,115,105,111,110,70,
     105,108,101,76,111,97,100,101,114,46,101,120,101,99,95,109,
@@ -1741,7 +1741,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,99,1,0,0,0,0,0,0,0,0,0,0,0,2,0,
     0,0,4,0,0,0,51,0,0,0,115,26,0,0,0,124,
     0,93,18,125,1,136,0,100,0,124,1,23,0,107,2,86,
-    0,1,0,113,2,100,1,83,0,41,2,114,208,0,0,0,
+    0,1,0,113,2,100,1,83,0,41,2,114,209,0,0,0,
     78,114,3,0,0,0,169,2,114,31,0,0,0,218,6,115,
     117,102,102,105,120,169,1,90,9,102,105,108,101,95,110,97,
     109,101,114,3,0,0,0,114,6,0,0,0,218,9,60,103,
@@ -1751,9 +1751,9 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     107,97,103,101,46,60,108,111,99,97,108,115,62,46,60,103,
     101,110,101,120,112,114,62,41,4,114,46,0,0,0,114,43,
     0,0,0,218,3,97,110,121,218,18,69,88,84,69,78,83,
-    73,79,78,95,83,85,70,70,73,88,69,83,114,218,0,0,
-    0,114,3,0,0,0,114,17,1,0,0,114,6,0,0,0,
-    114,181,0,0,0,90,4,0,0,115,8,0,0,0,0,2,
+    73,79,78,95,83,85,70,70,73,88,69,83,114,219,0,0,
+    0,114,3,0,0,0,114,18,1,0,0,114,6,0,0,0,
+    114,182,0,0,0,90,4,0,0,115,8,0,0,0,0,2,
     14,1,12,1,2,255,122,30,69,120,116,101,110,115,105,111,
     110,70,105,108,101,76,111,97,100,101,114,46,105,115,95,112,
     97,99,107,97,103,101,99,2,0,0,0,0,0,0,0,0,
@@ -1763,8 +1763,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     116,101,110,115,105,111,110,32,109,111,100,117,108,101,32,99,
     97,110,110,111,116,32,99,114,101,97,116,101,32,97,32,99,
     111,100,101,32,111,98,106,101,99,116,46,78,114,3,0,0,
-    0,114,218,0,0,0,114,3,0,0,0,114,3,0,0,0,
-    114,6,0,0,0,114,212,0,0,0,96,4,0,0,115,2,
+    0,114,219,0,0,0,114,3,0,0,0,114,3,0,0,0,
+    114,6,0,0,0,114,213,0,0,0,96,4,0,0,115,2,
     0,0,0,0,2,122,28,69,120,116,101,110,115,105,111,110,
     70,105,108,101,76,111,97,100,101,114,46,103,101,116,95,99,
     111,100,101,99,2,0,0,0,0,0,0,0,0,0,0,0,
@@ -1773,26 +1773,26 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     78,111,110,101,32,97,115,32,101,120,116,101,110,115,105,111,
     110,32,109,111,100,117,108,101,115,32,104,97,118,101,32,110,
     111,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114,
-    3,0,0,0,114,218,0,0,0,114,3,0,0,0,114,3,
-    0,0,0,114,6,0,0,0,114,228,0,0,0,100,4,0,
+    3,0,0,0,114,219,0,0,0,114,3,0,0,0,114,3,
+    0,0,0,114,6,0,0,0,114,229,0,0,0,100,4,0,
     0,115,2,0,0,0,0,2,122,30,69,120,116,101,110,115,
     105,111,110,70,105,108,101,76,111,97,100,101,114,46,103,101,
     116,95,115,111,117,114,99,101,99,2,0,0,0,0,0,0,
     0,0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,
-    0,115,6,0,0,0,124,0,106,0,83,0,114,249,0,0,
-    0,114,47,0,0,0,114,218,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,178,0,0,0,104,
+    0,115,6,0,0,0,124,0,106,0,83,0,114,250,0,0,
+    0,114,47,0,0,0,114,219,0,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,179,0,0,0,104,
     4,0,0,115,2,0,0,0,0,3,122,32,69,120,116,101,
     110,115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,
     103,101,116,95,102,105,108,101,110,97,109,101,78,41,14,114,
-    124,0,0,0,114,123,0,0,0,114,125,0,0,0,114,126,
-    0,0,0,114,208,0,0,0,114,242,0,0,0,114,246,0,
-    0,0,114,211,0,0,0,114,216,0,0,0,114,181,0,0,
-    0,114,212,0,0,0,114,228,0,0,0,114,135,0,0,0,
-    114,178,0,0,0,114,3,0,0,0,114,3,0,0,0,114,
-    3,0,0,0,114,6,0,0,0,114,251,0,0,0,57,4,
+    125,0,0,0,114,124,0,0,0,114,126,0,0,0,114,127,
+    0,0,0,114,209,0,0,0,114,243,0,0,0,114,247,0,
+    0,0,114,212,0,0,0,114,217,0,0,0,114,182,0,0,
+    0,114,213,0,0,0,114,229,0,0,0,114,136,0,0,0,
+    114,179,0,0,0,114,3,0,0,0,114,3,0,0,0,114,
+    3,0,0,0,114,6,0,0,0,114,252,0,0,0,57,4,
     0,0,115,22,0,0,0,8,2,4,6,8,4,8,4,8,
-    3,8,8,8,6,8,6,8,4,8,4,2,1,114,251,0,
+    3,8,8,8,6,8,6,8,4,8,4,2,1,114,252,0,
     0,0,99,0,0,0,0,0,0,0,0,0,0,0,0,0,
     0,0,0,2,0,0,0,64,0,0,0,115,104,0,0,0,
     101,0,90,1,100,0,90,2,100,1,90,3,100,2,100,3,
@@ -1833,7 +1833,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     110,100,101,114,169,4,114,118,0,0,0,114,116,0,0,0,
     114,43,0,0,0,90,11,112,97,116,104,95,102,105,110,100,
     101,114,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
-    0,114,208,0,0,0,117,4,0,0,115,8,0,0,0,0,
+    0,114,209,0,0,0,117,4,0,0,115,8,0,0,0,0,
     1,6,1,6,1,14,1,122,23,95,78,97,109,101,115,112,
     97,99,101,80,97,116,104,46,95,95,105,110,105,116,95,95,
     99,1,0,0,0,0,0,0,0,0,0,0,0,4,0,0,
@@ -1846,8 +1846,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     112,97,114,101,110,116,45,112,97,116,104,45,97,116,116,114,
     45,110,97,109,101,41,114,70,0,0,0,114,39,0,0,0,
     41,2,114,8,0,0,0,114,43,0,0,0,90,8,95,95,
-    112,97,116,104,95,95,41,2,114,22,1,0,0,114,40,0,
-    0,0,41,4,114,118,0,0,0,114,13,1,0,0,218,3,
+    112,97,116,104,95,95,41,2,114,23,1,0,0,114,40,0,
+    0,0,41,4,114,118,0,0,0,114,14,1,0,0,218,3,
     100,111,116,90,2,109,101,114,3,0,0,0,114,3,0,0,
     0,114,6,0,0,0,218,23,95,102,105,110,100,95,112,97,
     114,101,110,116,95,112,97,116,104,95,110,97,109,101,115,123,
@@ -1858,12 +1858,12 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,0,0,3,0,0,0,3,0,0,0,67,0,0,
     0,115,28,0,0,0,124,0,160,0,161,0,92,2,125,1,
     125,2,116,1,116,2,106,3,124,1,25,0,124,2,131,2,
-    83,0,114,109,0,0,0,41,4,114,29,1,0,0,114,129,
+    83,0,114,109,0,0,0,41,4,114,30,1,0,0,114,130,
     0,0,0,114,8,0,0,0,218,7,109,111,100,117,108,101,
     115,41,3,114,118,0,0,0,90,18,112,97,114,101,110,116,
     95,109,111,100,117,108,101,95,110,97,109,101,90,14,112,97,
     116,104,95,97,116,116,114,95,110,97,109,101,114,3,0,0,
-    0,114,3,0,0,0,114,6,0,0,0,114,24,1,0,0,
+    0,114,3,0,0,0,114,6,0,0,0,114,25,1,0,0,
     133,4,0,0,115,4,0,0,0,0,1,12,1,122,31,95,
     78,97,109,101,115,112,97,99,101,80,97,116,104,46,95,103,
     101,116,95,112,97,114,101,110,116,95,112,97,116,104,99,1,
@@ -1874,11 +1874,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     124,2,100,0,107,9,114,68,124,2,106,5,100,0,107,8,
     114,68,124,2,106,6,114,68,124,2,106,6,124,0,95,7,
     124,1,124,0,95,2,124,0,106,7,83,0,114,109,0,0,
-    0,41,8,114,111,0,0,0,114,24,1,0,0,114,25,1,
-    0,0,114,26,1,0,0,114,22,1,0,0,114,139,0,0,
-    0,114,177,0,0,0,114,23,1,0,0,41,3,114,118,0,
+    0,41,8,114,111,0,0,0,114,25,1,0,0,114,26,1,
+    0,0,114,27,1,0,0,114,23,1,0,0,114,140,0,0,
+    0,114,178,0,0,0,114,24,1,0,0,41,3,114,118,0,
     0,0,90,11,112,97,114,101,110,116,95,112,97,116,104,114,
-    186,0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
+    187,0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
     0,0,0,218,12,95,114,101,99,97,108,99,117,108,97,116,
     101,137,4,0,0,115,16,0,0,0,0,2,12,1,10,1,
     14,3,18,1,6,1,8,1,6,1,122,27,95,78,97,109,
@@ -1886,15 +1886,15 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     108,99,117,108,97,116,101,99,1,0,0,0,0,0,0,0,
     0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0,
     115,12,0,0,0,116,0,124,0,160,1,161,0,131,1,83,
-    0,114,109,0,0,0,41,2,114,5,1,0,0,114,31,1,
-    0,0,114,245,0,0,0,114,3,0,0,0,114,3,0,0,
+    0,114,109,0,0,0,41,2,114,6,1,0,0,114,32,1,
+    0,0,114,246,0,0,0,114,3,0,0,0,114,3,0,0,
     0,114,6,0,0,0,218,8,95,95,105,116,101,114,95,95,
     150,4,0,0,115,2,0,0,0,0,1,122,23,95,78,97,
     109,101,115,112,97,99,101,80,97,116,104,46,95,95,105,116,
     101,114,95,95,99,2,0,0,0,0,0,0,0,0,0,0,
     0,2,0,0,0,2,0,0,0,67,0,0,0,115,12,0,
     0,0,124,0,160,0,161,0,124,1,25,0,83,0,114,109,
-    0,0,0,169,1,114,31,1,0,0,41,2,114,118,0,0,
+    0,0,0,169,1,114,32,1,0,0,41,2,114,118,0,0,
     0,218,5,105,110,100,101,120,114,3,0,0,0,114,3,0,
     0,0,114,6,0,0,0,218,11,95,95,103,101,116,105,116,
     101,109,95,95,153,4,0,0,115,2,0,0,0,0,1,122,
@@ -1902,8 +1902,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     95,95,103,101,116,105,116,101,109,95,95,99,3,0,0,0,
     0,0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,
     67,0,0,0,115,14,0,0,0,124,2,124,0,106,0,124,
-    1,60,0,100,0,83,0,114,109,0,0,0,41,1,114,23,
-    1,0,0,41,3,114,118,0,0,0,114,34,1,0,0,114,
+    1,60,0,100,0,83,0,114,109,0,0,0,41,1,114,24,
+    1,0,0,41,3,114,118,0,0,0,114,35,1,0,0,114,
     43,0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
     0,0,0,218,11,95,95,115,101,116,105,116,101,109,95,95,
     156,4,0,0,115,2,0,0,0,0,1,122,26,95,78,97,
@@ -1911,8 +1911,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     116,105,116,101,109,95,95,99,1,0,0,0,0,0,0,0,
     0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0,
     115,12,0,0,0,116,0,124,0,160,1,161,0,131,1,83,
-    0,114,109,0,0,0,41,2,114,22,0,0,0,114,31,1,
-    0,0,114,245,0,0,0,114,3,0,0,0,114,3,0,0,
+    0,114,109,0,0,0,41,2,114,22,0,0,0,114,32,1,
+    0,0,114,246,0,0,0,114,3,0,0,0,114,3,0,0,
     0,114,6,0,0,0,218,7,95,95,108,101,110,95,95,159,
     4,0,0,115,2,0,0,0,0,1,122,22,95,78,97,109,
     101,115,112,97,99,101,80,97,116,104,46,95,95,108,101,110,
@@ -1920,15 +1920,15 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,3,0,0,0,67,0,0,0,115,12,0,0,0,
     100,1,160,0,124,0,106,1,161,1,83,0,41,2,78,122,
     20,95,78,97,109,101,115,112,97,99,101,80,97,116,104,40,
-    123,33,114,125,41,41,2,114,61,0,0,0,114,23,1,0,
-    0,114,245,0,0,0,114,3,0,0,0,114,3,0,0,0,
+    123,33,114,125,41,41,2,114,61,0,0,0,114,24,1,0,
+    0,114,246,0,0,0,114,3,0,0,0,114,3,0,0,0,
     114,6,0,0,0,218,8,95,95,114,101,112,114,95,95,162,
     4,0,0,115,2,0,0,0,0,1,122,23,95,78,97,109,
     101,115,112,97,99,101,80,97,116,104,46,95,95,114,101,112,
     114,95,95,99,2,0,0,0,0,0,0,0,0,0,0,0,
     2,0,0,0,3,0,0,0,67,0,0,0,115,12,0,0,
     0,124,1,124,0,160,0,161,0,107,6,83,0,114,109,0,
-    0,0,114,33,1,0,0,169,2,114,118,0,0,0,218,4,
+    0,0,114,34,1,0,0,169,2,114,118,0,0,0,218,4,
     105,116,101,109,114,3,0,0,0,114,3,0,0,0,114,6,
     0,0,0,218,12,95,95,99,111,110,116,97,105,110,115,95,
     95,165,4,0,0,115,2,0,0,0,0,1,122,27,95,78,
@@ -1936,20 +1936,20 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     111,110,116,97,105,110,115,95,95,99,2,0,0,0,0,0,
     0,0,0,0,0,0,2,0,0,0,3,0,0,0,67,0,
     0,0,115,16,0,0,0,124,0,106,0,160,1,124,1,161,
-    1,1,0,100,0,83,0,114,109,0,0,0,41,2,114,23,
-    1,0,0,114,185,0,0,0,114,39,1,0,0,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,185,0,0,
+    1,1,0,100,0,83,0,114,109,0,0,0,41,2,114,24,
+    1,0,0,114,186,0,0,0,114,40,1,0,0,114,3,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,114,186,0,0,
     0,168,4,0,0,115,2,0,0,0,0,1,122,21,95,78,
     97,109,101,115,112,97,99,101,80,97,116,104,46,97,112,112,
-    101,110,100,78,41,15,114,124,0,0,0,114,123,0,0,0,
-    114,125,0,0,0,114,126,0,0,0,114,208,0,0,0,114,
-    29,1,0,0,114,24,1,0,0,114,31,1,0,0,114,32,
-    1,0,0,114,35,1,0,0,114,36,1,0,0,114,37,1,
-    0,0,114,38,1,0,0,114,41,1,0,0,114,185,0,0,
+    101,110,100,78,41,15,114,125,0,0,0,114,124,0,0,0,
+    114,126,0,0,0,114,127,0,0,0,114,209,0,0,0,114,
+    30,1,0,0,114,25,1,0,0,114,32,1,0,0,114,33,
+    1,0,0,114,36,1,0,0,114,37,1,0,0,114,38,1,
+    0,0,114,39,1,0,0,114,42,1,0,0,114,186,0,0,
     0,114,3,0,0,0,114,3,0,0,0,114,3,0,0,0,
-    114,6,0,0,0,114,21,1,0,0,110,4,0,0,115,24,
+    114,6,0,0,0,114,22,1,0,0,110,4,0,0,115,24,
     0,0,0,8,1,4,6,8,6,8,10,8,4,8,13,8,
-    3,8,3,8,3,8,3,8,3,8,3,114,21,1,0,0,
+    3,8,3,8,3,8,3,8,3,8,3,114,22,1,0,0,
     99,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
     0,3,0,0,0,64,0,0,0,115,80,0,0,0,101,0,
     90,1,100,0,90,2,100,1,100,2,132,0,90,3,101,4,
@@ -1961,9 +1961,9 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     101,114,99,4,0,0,0,0,0,0,0,0,0,0,0,4,
     0,0,0,4,0,0,0,67,0,0,0,115,18,0,0,0,
     116,0,124,1,124,2,124,3,131,3,124,0,95,1,100,0,
-    83,0,114,109,0,0,0,41,2,114,21,1,0,0,114,23,
-    1,0,0,114,27,1,0,0,114,3,0,0,0,114,3,0,
-    0,0,114,6,0,0,0,114,208,0,0,0,174,4,0,0,
+    83,0,114,109,0,0,0,41,2,114,22,1,0,0,114,24,
+    1,0,0,114,28,1,0,0,114,3,0,0,0,114,3,0,
+    0,0,114,6,0,0,0,114,209,0,0,0,174,4,0,0,
     115,2,0,0,0,0,1,122,25,95,78,97,109,101,115,112,
     97,99,101,76,111,97,100,101,114,46,95,95,105,110,105,116,
     95,95,99,2,0,0,0,0,0,0,0,0,0,0,0,2,
@@ -1978,46 +1978,46 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     98,32,105,116,115,101,108,102,46,10,10,32,32,32,32,32,
     32,32,32,122,25,60,109,111,100,117,108,101,32,123,33,114,
     125,32,40,110,97,109,101,115,112,97,99,101,41,62,41,2,
-    114,61,0,0,0,114,124,0,0,0,41,2,114,192,0,0,
-    0,114,215,0,0,0,114,3,0,0,0,114,3,0,0,0,
+    114,61,0,0,0,114,125,0,0,0,41,2,114,193,0,0,
+    0,114,216,0,0,0,114,3,0,0,0,114,3,0,0,0,
     114,6,0,0,0,218,11,109,111,100,117,108,101,95,114,101,
     112,114,177,4,0,0,115,2,0,0,0,0,7,122,28,95,
     78,97,109,101,115,112,97,99,101,76,111,97,100,101,114,46,
     109,111,100,117,108,101,95,114,101,112,114,99,2,0,0,0,
     0,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,
     67,0,0,0,115,4,0,0,0,100,1,83,0,41,2,78,
-    84,114,3,0,0,0,114,218,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,181,0,0,0,186,
+    84,114,3,0,0,0,114,219,0,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,182,0,0,0,186,
     4,0,0,115,2,0,0,0,0,1,122,27,95,78,97,109,
     101,115,112,97,99,101,76,111,97,100,101,114,46,105,115,95,
     112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0,
     0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,
     115,4,0,0,0,100,1,83,0,41,2,78,114,39,0,0,
-    0,114,3,0,0,0,114,218,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,228,0,0,0,189,
+    0,114,3,0,0,0,114,219,0,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,229,0,0,0,189,
     4,0,0,115,2,0,0,0,0,1,122,27,95,78,97,109,
     101,115,112,97,99,101,76,111,97,100,101,114,46,103,101,116,
     95,115,111,117,114,99,101,99,2,0,0,0,0,0,0,0,
     0,0,0,0,2,0,0,0,6,0,0,0,67,0,0,0,
     115,16,0,0,0,116,0,100,1,100,2,100,3,100,4,100,
     5,141,4,83,0,41,6,78,114,39,0,0,0,122,8,60,
-    115,116,114,105,110,103,62,114,214,0,0,0,84,41,1,114,
-    230,0,0,0,41,1,114,231,0,0,0,114,218,0,0,0,
+    115,116,114,105,110,103,62,114,215,0,0,0,84,41,1,114,
+    231,0,0,0,41,1,114,232,0,0,0,114,219,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    212,0,0,0,192,4,0,0,115,2,0,0,0,0,1,122,
+    213,0,0,0,192,4,0,0,115,2,0,0,0,0,1,122,
     25,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101,
     114,46,103,101,116,95,99,111,100,101,99,2,0,0,0,0,
     0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,67,
-    0,0,0,115,4,0,0,0,100,1,83,0,114,209,0,0,
-    0,114,3,0,0,0,114,210,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,211,0,0,0,195,
+    0,0,0,115,4,0,0,0,100,1,83,0,114,210,0,0,
+    0,114,3,0,0,0,114,211,0,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,212,0,0,0,195,
     4,0,0,115,2,0,0,0,0,1,122,30,95,78,97,109,
     101,115,112,97,99,101,76,111,97,100,101,114,46,99,114,101,
     97,116,101,95,109,111,100,117,108,101,99,2,0,0,0,0,
     0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,67,
     0,0,0,115,4,0,0,0,100,0,83,0,114,109,0,0,
-    0,114,3,0,0,0,114,252,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,216,0,0,0,198,
+    0,114,3,0,0,0,114,253,0,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,217,0,0,0,198,
     4,0,0,115,2,0,0,0,0,1,122,28,95,78,97,109,
     101,115,112,97,99,101,76,111,97,100,101,114,46,101,120,101,
     99,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0,
@@ -2033,215 +2033,209 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,32,32,32,32,32,122,38,110,97,109,101,115,112,97,99,
     101,32,109,111,100,117,108,101,32,108,111,97,100,101,100,32,
     119,105,116,104,32,112,97,116,104,32,123,33,114,125,41,4,
-    114,133,0,0,0,114,148,0,0,0,114,23,1,0,0,114,
-    217,0,0,0,114,218,0,0,0,114,3,0,0,0,114,3,
-    0,0,0,114,6,0,0,0,114,219,0,0,0,201,4,0,
+    114,134,0,0,0,114,149,0,0,0,114,24,1,0,0,114,
+    218,0,0,0,114,219,0,0,0,114,3,0,0,0,114,3,
+    0,0,0,114,6,0,0,0,114,220,0,0,0,201,4,0,
     0,115,8,0,0,0,0,7,6,1,4,255,4,2,122,28,
     95,78,97,109,101,115,112,97,99,101,76,111,97,100,101,114,
     46,108,111,97,100,95,109,111,100,117,108,101,78,41,12,114,
-    124,0,0,0,114,123,0,0,0,114,125,0,0,0,114,208,
-    0,0,0,114,206,0,0,0,114,43,1,0,0,114,181,0,
-    0,0,114,228,0,0,0,114,212,0,0,0,114,211,0,0,
-    0,114,216,0,0,0,114,219,0,0,0,114,3,0,0,0,
+    125,0,0,0,114,124,0,0,0,114,126,0,0,0,114,209,
+    0,0,0,114,207,0,0,0,114,44,1,0,0,114,182,0,
+    0,0,114,229,0,0,0,114,213,0,0,0,114,212,0,0,
+    0,114,217,0,0,0,114,220,0,0,0,114,3,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    42,1,0,0,173,4,0,0,115,18,0,0,0,8,1,8,
+    43,1,0,0,173,4,0,0,115,18,0,0,0,8,1,8,
     3,2,1,10,8,8,3,8,3,8,3,8,3,8,3,114,
-    42,1,0,0,99,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,4,0,0,0,64,0,0,0,115,180,0,
+    43,1,0,0,99,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,4,0,0,0,64,0,0,0,115,118,0,
     0,0,101,0,90,1,100,0,90,2,100,1,90,3,101,4,
     100,2,100,3,132,0,131,1,90,5,101,4,100,4,100,5,
     132,0,131,1,90,6,101,4,100,6,100,7,132,0,131,1,
     90,7,101,4,100,8,100,9,132,0,131,1,90,8,101,4,
-    100,29,100,11,100,12,132,1,131,1,90,9,101,4,100,30,
-    100,13,100,14,132,1,131,1,90,10,101,4,100,31,100,15,
-    100,16,132,1,131,1,90,11,101,4,100,32,100,17,100,18,
-    132,1,131,1,90,12,101,4,100,19,100,20,132,0,131,1,
-    90,13,101,14,100,21,100,22,132,0,131,1,90,15,101,4,
-    100,23,100,24,132,0,131,1,90,16,101,4,100,25,100,26,
-    132,0,131,1,90,17,101,4,100,27,100,28,132,0,131,1,
-    90,18,100,10,83,0,41,33,218,10,80,97,116,104,70,105,
-    110,100,101,114,122,62,77,101,116,97,32,112,97,116,104,32,
-    102,105,110,100,101,114,32,102,111,114,32,115,121,115,46,112,
-    97,116,104,32,97,110,100,32,112,97,99,107,97,103,101,32,
-    95,95,112,97,116,104,95,95,32,97,116,116,114,105,98,117,
-    116,101,115,46,99,1,0,0,0,0,0,0,0,0,0,0,
-    0,3,0,0,0,4,0,0,0,67,0,0,0,115,64,0,
-    0,0,116,0,116,1,106,2,160,3,161,0,131,1,68,0,
-    93,44,92,2,125,1,125,2,124,2,100,1,107,8,114,40,
-    116,1,106,2,124,1,61,0,113,14,116,4,124,2,100,2,
-    131,2,114,14,124,2,160,5,161,0,1,0,113,14,100,1,
-    83,0,41,3,122,125,67,97,108,108,32,116,104,101,32,105,
-    110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115,
-    40,41,32,109,101,116,104,111,100,32,111,110,32,97,108,108,
-    32,112,97,116,104,32,101,110,116,114,121,32,102,105,110,100,
-    101,114,115,10,32,32,32,32,32,32,32,32,115,116,111,114,
-    101,100,32,105,110,32,115,121,115,46,112,97,116,104,95,105,
-    109,112,111,114,116,101,114,95,99,97,99,104,101,115,32,40,
-    119,104,101,114,101,32,105,109,112,108,101,109,101,110,116,101,
-    100,41,46,78,218,17,105,110,118,97,108,105,100,97,116,101,
-    95,99,97,99,104,101,115,41,6,218,4,108,105,115,116,114,
-    8,0,0,0,218,19,112,97,116,104,95,105,109,112,111,114,
-    116,101,114,95,99,97,99,104,101,218,5,105,116,101,109,115,
-    114,127,0,0,0,114,45,1,0,0,41,3,114,192,0,0,
-    0,114,116,0,0,0,218,6,102,105,110,100,101,114,114,3,
-    0,0,0,114,3,0,0,0,114,6,0,0,0,114,45,1,
-    0,0,219,4,0,0,115,10,0,0,0,0,4,22,1,8,
-    1,10,1,10,1,122,28,80,97,116,104,70,105,110,100,101,
-    114,46,105,110,118,97,108,105,100,97,116,101,95,99,97,99,
-    104,101,115,99,2,0,0,0,0,0,0,0,0,0,0,0,
-    3,0,0,0,9,0,0,0,67,0,0,0,115,84,0,0,
-    0,116,0,106,1,100,1,107,9,114,28,116,0,106,1,115,
-    28,116,2,160,3,100,2,116,4,161,2,1,0,116,0,106,
-    1,68,0,93,44,125,2,122,14,124,2,124,1,131,1,87,
-    0,2,0,1,0,83,0,4,0,116,5,107,10,114,76,1,
-    0,1,0,1,0,89,0,113,34,89,0,113,34,88,0,113,
-    34,100,1,83,0,41,3,122,46,83,101,97,114,99,104,32,
-    115,121,115,46,112,97,116,104,95,104,111,111,107,115,32,102,
-    111,114,32,97,32,102,105,110,100,101,114,32,102,111,114,32,
-    39,112,97,116,104,39,46,78,122,23,115,121,115,46,112,97,
-    116,104,95,104,111,111,107,115,32,105,115,32,101,109,112,116,
-    121,41,6,114,8,0,0,0,218,10,112,97,116,104,95,104,
-    111,111,107,115,114,74,0,0,0,114,75,0,0,0,114,137,
-    0,0,0,114,117,0,0,0,41,3,114,192,0,0,0,114,
-    43,0,0,0,90,4,104,111,111,107,114,3,0,0,0,114,
-    3,0,0,0,114,6,0,0,0,218,11,95,112,97,116,104,
-    95,104,111,111,107,115,229,4,0,0,115,16,0,0,0,0,
-    3,16,1,12,1,10,1,2,1,14,1,14,1,12,2,122,
-    22,80,97,116,104,70,105,110,100,101,114,46,95,112,97,116,
-    104,95,104,111,111,107,115,99,2,0,0,0,0,0,0,0,
-    0,0,0,0,3,0,0,0,8,0,0,0,67,0,0,0,
-    115,104,0,0,0,124,1,100,1,107,2,114,44,122,12,116,
-    0,160,1,161,0,125,1,87,0,110,22,4,0,116,2,107,
-    10,114,42,1,0,1,0,1,0,89,0,100,2,83,0,88,
-    0,122,14,116,3,106,4,124,1,25,0,125,2,87,0,110,
-    40,4,0,116,5,107,10,114,98,1,0,1,0,1,0,124,
-    0,160,6,124,1,161,1,125,2,124,2,116,3,106,4,124,
-    1,60,0,89,0,110,2,88,0,124,2,83,0,41,3,122,
-    210,71,101,116,32,116,104,101,32,102,105,110,100,101,114,32,
-    102,111,114,32,116,104,101,32,112,97,116,104,32,101,110,116,
-    114,121,32,102,114,111,109,32,115,121,115,46,112,97,116,104,
+    100,19,100,11,100,12,132,1,131,1,90,9,101,4,100,20,
+    100,13,100,14,132,1,131,1,90,10,101,4,100,21,100,15,
+    100,16,132,1,131,1,90,11,101,4,100,17,100,18,132,0,
+    131,1,90,12,100,10,83,0,41,22,218,10,80,97,116,104,
+    70,105,110,100,101,114,122,62,77,101,116,97,32,112,97,116,
+    104,32,102,105,110,100,101,114,32,102,111,114,32,115,121,115,
+    46,112,97,116,104,32,97,110,100,32,112,97,99,107,97,103,
+    101,32,95,95,112,97,116,104,95,95,32,97,116,116,114,105,
+    98,117,116,101,115,46,99,1,0,0,0,0,0,0,0,0,
+    0,0,0,3,0,0,0,4,0,0,0,67,0,0,0,115,
+    64,0,0,0,116,0,116,1,106,2,160,3,161,0,131,1,
+    68,0,93,44,92,2,125,1,125,2,124,2,100,1,107,8,
+    114,40,116,1,106,2,124,1,61,0,113,14,116,4,124,2,
+    100,2,131,2,114,14,124,2,160,5,161,0,1,0,113,14,
+    100,1,83,0,41,3,122,125,67,97,108,108,32,116,104,101,
+    32,105,110,118,97,108,105,100,97,116,101,95,99,97,99,104,
+    101,115,40,41,32,109,101,116,104,111,100,32,111,110,32,97,
+    108,108,32,112,97,116,104,32,101,110,116,114,121,32,102,105,
+    110,100,101,114,115,10,32,32,32,32,32,32,32,32,115,116,
+    111,114,101,100,32,105,110,32,115,121,115,46,112,97,116,104,
+    95,105,109,112,111,114,116,101,114,95,99,97,99,104,101,115,
+    32,40,119,104,101,114,101,32,105,109,112,108,101,109,101,110,
+    116,101,100,41,46,78,218,17,105,110,118,97,108,105,100,97,
+    116,101,95,99,97,99,104,101,115,41,6,218,4,108,105,115,
+    116,114,8,0,0,0,218,19,112,97,116,104,95,105,109,112,
+    111,114,116,101,114,95,99,97,99,104,101,218,5,105,116,101,
+    109,115,114,128,0,0,0,114,46,1,0,0,41,3,114,193,
+    0,0,0,114,116,0,0,0,218,6,102,105,110,100,101,114,
+    114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
+    46,1,0,0,219,4,0,0,115,10,0,0,0,0,4,22,
+    1,8,1,10,1,10,1,122,28,80,97,116,104,70,105,110,
+    100,101,114,46,105,110,118,97,108,105,100,97,116,101,95,99,
+    97,99,104,101,115,99,2,0,0,0,0,0,0,0,0,0,
+    0,0,3,0,0,0,9,0,0,0,67,0,0,0,115,84,
+    0,0,0,116,0,106,1,100,1,107,9,114,28,116,0,106,
+    1,115,28,116,2,160,3,100,2,116,4,161,2,1,0,116,
+    0,106,1,68,0,93,44,125,2,122,14,124,2,124,1,131,
+    1,87,0,2,0,1,0,83,0,4,0,116,5,107,10,114,
+    76,1,0,1,0,1,0,89,0,113,34,89,0,113,34,88,
+    0,113,34,100,1,83,0,41,3,122,46,83,101,97,114,99,
+    104,32,115,121,115,46,112,97,116,104,95,104,111,111,107,115,
+    32,102,111,114,32,97,32,102,105,110,100,101,114,32,102,111,
+    114,32,39,112,97,116,104,39,46,78,122,23,115,121,115,46,
+    112,97,116,104,95,104,111,111,107,115,32,105,115,32,101,109,
+    112,116,121,41,6,114,8,0,0,0,218,10,112,97,116,104,
+    95,104,111,111,107,115,114,74,0,0,0,114,75,0,0,0,
+    114,138,0,0,0,114,117,0,0,0,41,3,114,193,0,0,
+    0,114,43,0,0,0,90,4,104,111,111,107,114,3,0,0,
+    0,114,3,0,0,0,114,6,0,0,0,218,11,95,112,97,
+    116,104,95,104,111,111,107,115,229,4,0,0,115,16,0,0,
+    0,0,3,16,1,12,1,10,1,2,1,14,1,14,1,12,
+    2,122,22,80,97,116,104,70,105,110,100,101,114,46,95,112,
+    97,116,104,95,104,111,111,107,115,99,2,0,0,0,0,0,
+    0,0,0,0,0,0,3,0,0,0,8,0,0,0,67,0,
+    0,0,115,104,0,0,0,124,1,100,1,107,2,114,44,122,
+    12,116,0,160,1,161,0,125,1,87,0,110,22,4,0,116,
+    2,107,10,114,42,1,0,1,0,1,0,89,0,100,2,83,
+    0,88,0,122,14,116,3,106,4,124,1,25,0,125,2,87,
+    0,110,40,4,0,116,5,107,10,114,98,1,0,1,0,1,
+    0,124,0,160,6,124,1,161,1,125,2,124,2,116,3,106,
+    4,124,1,60,0,89,0,110,2,88,0,124,2,83,0,41,
+    3,122,210,71,101,116,32,116,104,101,32,102,105,110,100,101,
+    114,32,102,111,114,32,116,104,101,32,112,97,116,104,32,101,
+    110,116,114,121,32,102,114,111,109,32,115,121,115,46,112,97,
+    116,104,95,105,109,112,111,114,116,101,114,95,99,97,99,104,
+    101,46,10,10,32,32,32,32,32,32,32,32,73,102,32,116,
+    104,101,32,112,97,116,104,32,101,110,116,114,121,32,105,115,
+    32,110,111,116,32,105,110,32,116,104,101,32,99,97,99,104,
+    101,44,32,102,105,110,100,32,116,104,101,32,97,112,112,114,
+    111,112,114,105,97,116,101,32,102,105,110,100,101,114,10,32,
+    32,32,32,32,32,32,32,97,110,100,32,99,97,99,104,101,
+    32,105,116,46,32,73,102,32,110,111,32,102,105,110,100,101,
+    114,32,105,115,32,97,118,97,105,108,97,98,108,101,44,32,
+    115,116,111,114,101,32,78,111,110,101,46,10,10,32,32,32,
+    32,32,32,32,32,114,39,0,0,0,78,41,7,114,2,0,
+    0,0,114,54,0,0,0,114,3,1,0,0,114,8,0,0,
+    0,114,48,1,0,0,218,8,75,101,121,69,114,114,111,114,
+    114,52,1,0,0,41,3,114,193,0,0,0,114,43,0,0,
+    0,114,50,1,0,0,114,3,0,0,0,114,3,0,0,0,
+    114,6,0,0,0,218,20,95,112,97,116,104,95,105,109,112,
+    111,114,116,101,114,95,99,97,99,104,101,242,4,0,0,115,
+    22,0,0,0,0,8,8,1,2,1,12,1,14,3,8,1,
+    2,1,14,1,14,1,10,1,16,1,122,31,80,97,116,104,
+    70,105,110,100,101,114,46,95,112,97,116,104,95,105,109,112,
+    111,114,116,101,114,95,99,97,99,104,101,99,3,0,0,0,
+    0,0,0,0,0,0,0,0,6,0,0,0,4,0,0,0,
+    67,0,0,0,115,82,0,0,0,116,0,124,2,100,1,131,
+    2,114,26,124,2,160,1,124,1,161,1,92,2,125,3,125,
+    4,110,14,124,2,160,2,124,1,161,1,125,3,103,0,125,
+    4,124,3,100,0,107,9,114,60,116,3,160,4,124,1,124,
+    3,161,2,83,0,116,3,160,5,124,1,100,0,161,2,125,
+    5,124,4,124,5,95,6,124,5,83,0,41,2,78,114,137,
+    0,0,0,41,7,114,128,0,0,0,114,137,0,0,0,114,
+    206,0,0,0,114,134,0,0,0,114,201,0,0,0,114,183,
+    0,0,0,114,178,0,0,0,41,6,114,193,0,0,0,114,
+    139,0,0,0,114,50,1,0,0,114,140,0,0,0,114,141,
+    0,0,0,114,187,0,0,0,114,3,0,0,0,114,3,0,
+    0,0,114,6,0,0,0,218,16,95,108,101,103,97,99,121,
+    95,103,101,116,95,115,112,101,99,8,5,0,0,115,18,0,
+    0,0,0,4,10,1,16,2,10,1,4,1,8,1,12,1,
+    12,1,6,1,122,27,80,97,116,104,70,105,110,100,101,114,
+    46,95,108,101,103,97,99,121,95,103,101,116,95,115,112,101,
+    99,78,99,4,0,0,0,0,0,0,0,0,0,0,0,9,
+    0,0,0,5,0,0,0,67,0,0,0,115,166,0,0,0,
+    103,0,125,4,124,2,68,0,93,134,125,5,116,0,124,5,
+    116,1,116,2,102,2,131,2,115,28,113,8,124,0,160,3,
+    124,5,161,1,125,6,124,6,100,1,107,9,114,8,116,4,
+    124,6,100,2,131,2,114,70,124,6,160,5,124,1,124,3,
+    161,2,125,7,110,12,124,0,160,6,124,1,124,6,161,2,
+    125,7,124,7,100,1,107,8,114,92,113,8,124,7,106,7,
+    100,1,107,9,114,110,124,7,2,0,1,0,83,0,124,7,
+    106,8,125,8,124,8,100,1,107,8,114,132,116,9,100,3,
+    131,1,130,1,124,4,160,10,124,8,161,1,1,0,113,8,
+    116,11,160,12,124,1,100,1,161,2,125,7,124,4,124,7,
+    95,8,124,7,83,0,41,4,122,63,70,105,110,100,32,116,
+    104,101,32,108,111,97,100,101,114,32,111,114,32,110,97,109,
+    101,115,112,97,99,101,95,112,97,116,104,32,102,111,114,32,
+    116,104,105,115,32,109,111,100,117,108,101,47,112,97,99,107,
+    97,103,101,32,110,97,109,101,46,78,114,203,0,0,0,122,
+    19,115,112,101,99,32,109,105,115,115,105,110,103,32,108,111,
+    97,100,101,114,41,13,114,161,0,0,0,114,84,0,0,0,
+    218,5,98,121,116,101,115,114,54,1,0,0,114,128,0,0,
+    0,114,203,0,0,0,114,55,1,0,0,114,140,0,0,0,
+    114,178,0,0,0,114,117,0,0,0,114,167,0,0,0,114,
+    134,0,0,0,114,183,0,0,0,41,9,114,193,0,0,0,
+    114,139,0,0,0,114,43,0,0,0,114,202,0,0,0,218,
+    14,110,97,109,101,115,112,97,99,101,95,112,97,116,104,90,
+    5,101,110,116,114,121,114,50,1,0,0,114,187,0,0,0,
+    114,141,0,0,0,114,3,0,0,0,114,3,0,0,0,114,
+    6,0,0,0,218,9,95,103,101,116,95,115,112,101,99,23,
+    5,0,0,115,40,0,0,0,0,5,4,1,8,1,14,1,
+    2,1,10,1,8,1,10,1,14,2,12,1,8,1,2,1,
+    10,1,8,1,6,1,8,1,8,5,12,2,12,1,6,1,
+    122,20,80,97,116,104,70,105,110,100,101,114,46,95,103,101,
+    116,95,115,112,101,99,99,4,0,0,0,0,0,0,0,0,
+    0,0,0,6,0,0,0,5,0,0,0,67,0,0,0,115,
+    100,0,0,0,124,2,100,1,107,8,114,14,116,0,106,1,
+    125,2,124,0,160,2,124,1,124,2,124,3,161,3,125,4,
+    124,4,100,1,107,8,114,40,100,1,83,0,124,4,106,3,
+    100,1,107,8,114,92,124,4,106,4,125,5,124,5,114,86,
+    100,1,124,4,95,5,116,6,124,1,124,5,124,0,106,2,
+    131,3,124,4,95,4,124,4,83,0,100,1,83,0,110,4,
+    124,4,83,0,100,1,83,0,41,2,122,141,84,114,121,32,
+    116,111,32,102,105,110,100,32,97,32,115,112,101,99,32,102,
+    111,114,32,39,102,117,108,108,110,97,109,101,39,32,111,110,
+    32,115,121,115,46,112,97,116,104,32,111,114,32,39,112,97,
+    116,104,39,46,10,10,32,32,32,32,32,32,32,32,84,104,
+    101,32,115,101,97,114,99,104,32,105,115,32,98,97,115,101,
+    100,32,111,110,32,115,121,115,46,112,97,116,104,95,104,111,
+    111,107,115,32,97,110,100,32,115,121,115,46,112,97,116,104,
     95,105,109,112,111,114,116,101,114,95,99,97,99,104,101,46,
-    10,10,32,32,32,32,32,32,32,32,73,102,32,116,104,101,
-    32,112,97,116,104,32,101,110,116,114,121,32,105,115,32,110,
-    111,116,32,105,110,32,116,104,101,32,99,97,99,104,101,44,
-    32,102,105,110,100,32,116,104,101,32,97,112,112,114,111,112,
-    114,105,97,116,101,32,102,105,110,100,101,114,10,32,32,32,
-    32,32,32,32,32,97,110,100,32,99,97,99,104,101,32,105,
-    116,46,32,73,102,32,110,111,32,102,105,110,100,101,114,32,
-    105,115,32,97,118,97,105,108,97,98,108,101,44,32,115,116,
-    111,114,101,32,78,111,110,101,46,10,10,32,32,32,32,32,
-    32,32,32,114,39,0,0,0,78,41,7,114,2,0,0,0,
-    114,54,0,0,0,114,2,1,0,0,114,8,0,0,0,114,
-    47,1,0,0,218,8,75,101,121,69,114,114,111,114,114,51,
-    1,0,0,41,3,114,192,0,0,0,114,43,0,0,0,114,
-    49,1,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
-    0,0,0,218,20,95,112,97,116,104,95,105,109,112,111,114,
-    116,101,114,95,99,97,99,104,101,242,4,0,0,115,22,0,
-    0,0,0,8,8,1,2,1,12,1,14,3,8,1,2,1,
-    14,1,14,1,10,1,16,1,122,31,80,97,116,104,70,105,
-    110,100,101,114,46,95,112,97,116,104,95,105,109,112,111,114,
-    116,101,114,95,99,97,99,104,101,99,3,0,0,0,0,0,
-    0,0,0,0,0,0,6,0,0,0,4,0,0,0,67,0,
-    0,0,115,82,0,0,0,116,0,124,2,100,1,131,2,114,
-    26,124,2,160,1,124,1,161,1,92,2,125,3,125,4,110,
-    14,124,2,160,2,124,1,161,1,125,3,103,0,125,4,124,
-    3,100,0,107,9,114,60,116,3,160,4,124,1,124,3,161,
-    2,83,0,116,3,160,5,124,1,100,0,161,2,125,5,124,
-    4,124,5,95,6,124,5,83,0,41,2,78,114,136,0,0,
-    0,41,7,114,127,0,0,0,114,136,0,0,0,114,205,0,
-    0,0,114,133,0,0,0,114,200,0,0,0,114,182,0,0,
-    0,114,177,0,0,0,41,6,114,192,0,0,0,114,138,0,
-    0,0,114,49,1,0,0,114,139,0,0,0,114,140,0,0,
-    0,114,186,0,0,0,114,3,0,0,0,114,3,0,0,0,
-    114,6,0,0,0,218,16,95,108,101,103,97,99,121,95,103,
-    101,116,95,115,112,101,99,8,5,0,0,115,18,0,0,0,
-    0,4,10,1,16,2,10,1,4,1,8,1,12,1,12,1,
-    6,1,122,27,80,97,116,104,70,105,110,100,101,114,46,95,
-    108,101,103,97,99,121,95,103,101,116,95,115,112,101,99,78,
-    99,4,0,0,0,0,0,0,0,0,0,0,0,9,0,0,
-    0,5,0,0,0,67,0,0,0,115,166,0,0,0,103,0,
-    125,4,124,2,68,0,93,134,125,5,116,0,124,5,116,1,
-    116,2,102,2,131,2,115,28,113,8,124,0,160,3,124,5,
-    161,1,125,6,124,6,100,1,107,9,114,8,116,4,124,6,
-    100,2,131,2,114,70,124,6,160,5,124,1,124,3,161,2,
-    125,7,110,12,124,0,160,6,124,1,124,6,161,2,125,7,
-    124,7,100,1,107,8,114,92,113,8,124,7,106,7,100,1,
-    107,9,114,110,124,7,2,0,1,0,83,0,124,7,106,8,
-    125,8,124,8,100,1,107,8,114,132,116,9,100,3,131,1,
-    130,1,124,4,160,10,124,8,161,1,1,0,113,8,116,11,
-    160,12,124,1,100,1,161,2,125,7,124,4,124,7,95,8,
-    124,7,83,0,41,4,122,63,70,105,110,100,32,116,104,101,
-    32,108,111,97,100,101,114,32,111,114,32,110,97,109,101,115,
-    112,97,99,101,95,112,97,116,104,32,102,111,114,32,116,104,
-    105,115,32,109,111,100,117,108,101,47,112,97,99,107,97,103,
-    101,32,110,97,109,101,46,78,114,202,0,0,0,122,19,115,
-    112,101,99,32,109,105,115,115,105,110,103,32,108,111,97,100,
-    101,114,41,13,114,160,0,0,0,114,84,0,0,0,218,5,
-    98,121,116,101,115,114,53,1,0,0,114,127,0,0,0,114,
-    202,0,0,0,114,54,1,0,0,114,139,0,0,0,114,177,
-    0,0,0,114,117,0,0,0,114,166,0,0,0,114,133,0,
-    0,0,114,182,0,0,0,41,9,114,192,0,0,0,114,138,
-    0,0,0,114,43,0,0,0,114,201,0,0,0,218,14,110,
-    97,109,101,115,112,97,99,101,95,112,97,116,104,90,5,101,
-    110,116,114,121,114,49,1,0,0,114,186,0,0,0,114,140,
-    0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,218,9,95,103,101,116,95,115,112,101,99,23,5,0,
-    0,115,40,0,0,0,0,5,4,1,8,1,14,1,2,1,
-    10,1,8,1,10,1,14,2,12,1,8,1,2,1,10,1,
-    8,1,6,1,8,1,8,5,12,2,12,1,6,1,122,20,
-    80,97,116,104,70,105,110,100,101,114,46,95,103,101,116,95,
-    115,112,101,99,99,4,0,0,0,0,0,0,0,0,0,0,
-    0,6,0,0,0,5,0,0,0,67,0,0,0,115,100,0,
-    0,0,124,2,100,1,107,8,114,14,116,0,106,1,125,2,
-    124,0,160,2,124,1,124,2,124,3,161,3,125,4,124,4,
-    100,1,107,8,114,40,100,1,83,0,124,4,106,3,100,1,
-    107,8,114,92,124,4,106,4,125,5,124,5,114,86,100,1,
-    124,4,95,5,116,6,124,1,124,5,124,0,106,2,131,3,
-    124,4,95,4,124,4,83,0,100,1,83,0,110,4,124,4,
-    83,0,100,1,83,0,41,2,122,141,84,114,121,32,116,111,
-    32,102,105,110,100,32,97,32,115,112,101,99,32,102,111,114,
-    32,39,102,117,108,108,110,97,109,101,39,32,111,110,32,115,
+    10,32,32,32,32,32,32,32,32,78,41,7,114,8,0,0,
+    0,114,43,0,0,0,114,58,1,0,0,114,140,0,0,0,
+    114,178,0,0,0,114,181,0,0,0,114,22,1,0,0,41,
+    6,114,193,0,0,0,114,139,0,0,0,114,43,0,0,0,
+    114,202,0,0,0,114,187,0,0,0,114,57,1,0,0,114,
+    3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,203,
+    0,0,0,55,5,0,0,115,26,0,0,0,0,6,8,1,
+    6,1,14,1,8,1,4,1,10,1,6,1,4,3,6,1,
+    16,1,4,2,6,2,122,20,80,97,116,104,70,105,110,100,
+    101,114,46,102,105,110,100,95,115,112,101,99,99,3,0,0,
+    0,0,0,0,0,0,0,0,0,4,0,0,0,4,0,0,
+    0,67,0,0,0,115,30,0,0,0,124,0,160,0,124,1,
+    124,2,161,2,125,3,124,3,100,1,107,8,114,24,100,1,
+    83,0,124,3,106,1,83,0,41,2,122,170,102,105,110,100,
+    32,116,104,101,32,109,111,100,117,108,101,32,111,110,32,115,
     121,115,46,112,97,116,104,32,111,114,32,39,112,97,116,104,
-    39,46,10,10,32,32,32,32,32,32,32,32,84,104,101,32,
-    115,101,97,114,99,104,32,105,115,32,98,97,115,101,100,32,
-    111,110,32,115,121,115,46,112,97,116,104,95,104,111,111,107,
-    115,32,97,110,100,32,115,121,115,46,112,97,116,104,95,105,
-    109,112,111,114,116,101,114,95,99,97,99,104,101,46,10,32,
-    32,32,32,32,32,32,32,78,41,7,114,8,0,0,0,114,
-    43,0,0,0,114,57,1,0,0,114,139,0,0,0,114,177,
-    0,0,0,114,180,0,0,0,114,21,1,0,0,41,6,114,
-    192,0,0,0,114,138,0,0,0,114,43,0,0,0,114,201,
-    0,0,0,114,186,0,0,0,114,56,1,0,0,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,202,0,0,
-    0,55,5,0,0,115,26,0,0,0,0,6,8,1,6,1,
-    14,1,8,1,4,1,10,1,6,1,4,3,6,1,16,1,
-    4,2,6,2,122,20,80,97,116,104,70,105,110,100,101,114,
-    46,102,105,110,100,95,115,112,101,99,99,3,0,0,0,0,
-    0,0,0,0,0,0,0,4,0,0,0,4,0,0,0,67,
-    0,0,0,115,30,0,0,0,124,0,160,0,124,1,124,2,
-    161,2,125,3,124,3,100,1,107,8,114,24,100,1,83,0,
-    124,3,106,1,83,0,41,2,122,170,102,105,110,100,32,116,
-    104,101,32,109,111,100,117,108,101,32,111,110,32,115,121,115,
-    46,112,97,116,104,32,111,114,32,39,112,97,116,104,39,32,
-    98,97,115,101,100,32,111,110,32,115,121,115,46,112,97,116,
-    104,95,104,111,111,107,115,32,97,110,100,10,32,32,32,32,
-    32,32,32,32,115,121,115,46,112,97,116,104,95,105,109,112,
-    111,114,116,101,114,95,99,97,99,104,101,46,10,10,32,32,
-    32,32,32,32,32,32,84,104,105,115,32,109,101,116,104,111,
-    100,32,105,115,32,100,101,112,114,101,99,97,116,101,100,46,
-    32,32,85,115,101,32,102,105,110,100,95,115,112,101,99,40,
-    41,32,105,110,115,116,101,97,100,46,10,10,32,32,32,32,
-    32,32,32,32,78,114,203,0,0,0,114,204,0,0,0,114,
-    3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,205,
-    0,0,0,79,5,0,0,115,8,0,0,0,0,8,12,1,
-    8,1,4,1,122,22,80,97,116,104,70,105,110,100,101,114,
-    46,102,105,110,100,95,109,111,100,117,108,101,99,2,0,0,
-    0,0,0,0,0,0,0,0,0,5,0,0,0,4,0,0,
-    0,67,0,0,0,115,58,0,0,0,100,1,100,2,108,0,
-    109,1,125,2,109,2,125,3,1,0,124,1,100,3,107,8,
-    114,32,124,3,160,3,161,0,125,1,124,0,160,4,124,1,
-    106,5,124,1,106,6,161,2,125,4,116,7,124,2,124,4,
-    131,2,83,0,41,4,97,32,1,0,0,10,32,32,32,32,
+    39,32,98,97,115,101,100,32,111,110,32,115,121,115,46,112,
+    97,116,104,95,104,111,111,107,115,32,97,110,100,10,32,32,
+    32,32,32,32,32,32,115,121,115,46,112,97,116,104,95,105,
+    109,112,111,114,116,101,114,95,99,97,99,104,101,46,10,10,
+    32,32,32,32,32,32,32,32,84,104,105,115,32,109,101,116,
+    104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101,
+    100,46,32,32,85,115,101,32,102,105,110,100,95,115,112,101,
+    99,40,41,32,105,110,115,116,101,97,100,46,10,10,32,32,
+    32,32,32,32,32,32,78,114,204,0,0,0,114,205,0,0,
+    0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
+    114,206,0,0,0,79,5,0,0,115,8,0,0,0,0,8,
+    12,1,8,1,4,1,122,22,80,97,116,104,70,105,110,100,
+    101,114,46,102,105,110,100,95,109,111,100,117,108,101,99,1,
+    0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,3,
+    0,0,0,79,0,0,0,115,24,0,0,0,100,1,100,2,
+    108,0,109,1,125,3,1,0,124,3,106,2,124,1,124,2,
+    142,1,83,0,41,3,97,32,1,0,0,10,32,32,32,32,
     32,32,32,32,70,105,110,100,32,100,105,115,116,114,105,98,
     117,116,105,111,110,115,46,10,10,32,32,32,32,32,32,32,
     32,82,101,116,117,114,110,32,97,110,32,105,116,101,114,97,
@@ -2260,593 +2254,479 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,32,111,102,32,100,105,114,101,99,116,111,114,105,101,115,
     32,96,96,99,111,110,116,101,120,116,46,112,97,116,104,96,
     96,46,10,32,32,32,32,32,32,32,32,114,72,0,0,0,
-    41,2,218,16,80,97,116,104,68,105,115,116,114,105,98,117,
-    116,105,111,110,218,18,68,105,115,116,114,105,98,117,116,105,
-    111,110,70,105,110,100,101,114,78,41,8,90,18,105,109,112,
-    111,114,116,108,105,98,46,109,101,116,97,100,97,116,97,114,
-    58,1,0,0,114,59,1,0,0,90,7,67,111,110,116,101,
-    120,116,218,13,95,115,101,97,114,99,104,95,112,97,116,104,
-    115,218,7,112,97,116,116,101,114,110,114,43,0,0,0,218,
-    3,109,97,112,41,5,114,118,0,0,0,90,7,99,111,110,
-    116,101,120,116,114,58,1,0,0,114,59,1,0,0,90,5,
-    102,111,117,110,100,114,3,0,0,0,114,3,0,0,0,114,
-    6,0,0,0,218,18,102,105,110,100,95,100,105,115,116,114,
-    105,98,117,116,105,111,110,115,92,5,0,0,115,10,0,0,
-    0,0,10,16,1,8,1,8,1,16,1,122,29,80,97,116,
-    104,70,105,110,100,101,114,46,102,105,110,100,95,100,105,115,
-    116,114,105,98,117,116,105,111,110,115,99,3,0,0,0,0,
-    0,0,0,0,0,0,0,4,0,0,0,6,0,0,0,3,
-    0,0,0,115,44,0,0,0,100,1,100,2,108,0,125,3,
-    124,3,106,1,160,2,135,0,135,1,102,2,100,3,100,4,
-    132,8,116,3,136,0,106,4,124,2,131,2,68,0,131,1,
-    161,1,83,0,41,5,122,49,70,105,110,100,32,109,101,116,
-    97,100,97,116,97,32,100,105,114,101,99,116,111,114,105,101,
-    115,32,105,110,32,112,97,116,104,115,32,104,101,117,114,105,
-    115,116,105,99,97,108,108,121,46,114,72,0,0,0,78,99,
-    1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,
-    5,0,0,0,51,0,0,0,115,26,0,0,0,124,0,93,
-    18,125,1,136,0,160,0,124,1,136,1,161,2,86,0,1,
-    0,113,2,100,0,83,0,114,109,0,0,0,41,1,218,12,
-    95,115,101,97,114,99,104,95,112,97,116,104,41,2,114,31,
-    0,0,0,114,43,0,0,0,169,2,114,192,0,0,0,114,
-    61,1,0,0,114,3,0,0,0,114,6,0,0,0,114,18,
-    1,0,0,112,5,0,0,115,4,0,0,0,4,2,2,255,
-    122,43,80,97,116,104,70,105,110,100,101,114,46,95,115,101,
-    97,114,99,104,95,112,97,116,104,115,46,60,108,111,99,97,
-    108,115,62,46,60,103,101,110,101,120,112,114,62,41,5,218,
-    9,105,116,101,114,116,111,111,108,115,90,5,99,104,97,105,
-    110,90,13,102,114,111,109,95,105,116,101,114,97,98,108,101,
-    114,62,1,0,0,218,12,95,115,119,105,116,99,104,95,112,
-    97,116,104,41,4,114,192,0,0,0,114,61,1,0,0,90,
-    5,112,97,116,104,115,114,66,1,0,0,114,3,0,0,0,
-    114,65,1,0,0,114,6,0,0,0,114,60,1,0,0,108,
-    5,0,0,115,8,0,0,0,0,3,8,1,18,2,10,254,
-    122,24,80,97,116,104,70,105,110,100,101,114,46,95,115,101,
-    97,114,99,104,95,112,97,116,104,115,99,1,0,0,0,0,
-    0,0,0,0,0,0,0,5,0,0,0,10,0,0,0,67,
-    0,0,0,115,96,0,0,0,100,1,100,2,108,0,109,1,
-    125,1,1,0,100,1,100,0,108,2,125,2,100,1,100,0,
-    108,3,125,3,100,3,125,4,124,4,114,48,116,4,106,5,
-    160,6,124,0,161,1,114,86,124,1,116,7,131,1,143,24,
-    1,0,124,2,160,8,124,0,161,1,87,0,2,0,53,0,
-    81,0,82,0,163,0,83,0,81,0,82,0,88,0,124,3,
-    160,8,124,0,161,1,83,0,41,4,78,114,72,0,0,0,
-    41,1,218,8,115,117,112,112,114,101,115,115,70,41,9,90,
-    10,99,111,110,116,101,120,116,108,105,98,114,68,1,0,0,
-    218,7,122,105,112,102,105,108,101,218,7,112,97,116,104,108,
-    105,98,90,2,111,115,114,43,0,0,0,90,6,105,115,102,
-    105,108,101,218,9,69,120,99,101,112,116,105,111,110,90,4,
-    80,97,116,104,41,5,114,43,0,0,0,114,68,1,0,0,
-    114,69,1,0,0,114,70,1,0,0,90,13,80,89,80,89,
-    95,79,80,69,78,95,66,85,71,114,3,0,0,0,114,3,
-    0,0,0,114,6,0,0,0,114,67,1,0,0,117,5,0,
-    0,115,16,0,0,0,0,2,12,1,8,1,8,1,4,1,
-    16,1,10,1,28,1,122,23,80,97,116,104,70,105,110,100,
-    101,114,46,95,115,119,105,116,99,104,95,112,97,116,104,99,
-    3,0,0,0,0,0,0,0,0,0,0,0,6,0,0,0,
-    5,0,0,0,67,0,0,0,115,44,0,0,0,100,1,100,
-    0,108,0,125,3,100,2,125,4,124,4,106,1,124,1,100,
-    3,141,1,125,5,124,3,106,2,124,5,124,2,106,3,124,
-    3,106,4,100,4,141,3,83,0,41,5,78,114,72,0,0,
-    0,122,32,123,112,97,116,116,101,114,110,125,40,45,46,42,
-    41,63,92,46,40,100,105,115,116,124,101,103,103,41,45,105,
-    110,102,111,169,1,114,61,1,0,0,169,1,114,82,0,0,
-    0,41,5,218,2,114,101,114,61,0,0,0,90,5,109,97,
-    116,99,104,114,116,0,0,0,218,10,73,71,78,79,82,69,
-    67,65,83,69,169,6,114,192,0,0,0,218,10,110,111,114,
-    109,97,108,105,122,101,100,114,40,1,0,0,114,74,1,0,
-    0,90,8,116,101,109,112,108,97,116,101,90,8,109,97,110,
-    105,102,101,115,116,114,3,0,0,0,114,3,0,0,0,114,
-    6,0,0,0,218,13,95,109,97,116,99,104,101,115,95,105,
-    110,102,111,128,5,0,0,115,8,0,0,0,0,2,8,1,
-    4,1,12,1,122,24,80,97,116,104,70,105,110,100,101,114,
-    46,95,109,97,116,99,104,101,115,95,105,110,102,111,99,3,
-    0,0,0,0,0,0,0,0,0,0,0,6,0,0,0,5,
-    0,0,0,67,0,0,0,115,46,0,0,0,100,1,100,0,
-    108,0,125,3,100,2,125,4,124,4,106,1,124,1,100,3,
-    141,1,125,5,124,3,106,2,124,5,116,3,124,2,131,1,
-    124,3,106,4,100,4,141,3,83,0,41,5,78,114,72,0,
-    0,0,122,30,123,112,97,116,116,101,114,110,125,45,46,42,
-    92,46,101,103,103,91,92,92,47,93,69,71,71,45,73,78,
-    70,79,114,72,1,0,0,114,73,1,0,0,41,5,114,74,
-    1,0,0,114,61,0,0,0,90,6,115,101,97,114,99,104,
-    114,84,0,0,0,114,75,1,0,0,114,76,1,0,0,114,
-    3,0,0,0,114,3,0,0,0,114,6,0,0,0,218,15,
-    95,109,97,116,99,104,101,115,95,108,101,103,97,99,121,135,
-    5,0,0,115,8,0,0,0,0,2,8,1,4,1,12,1,
-    122,26,80,97,116,104,70,105,110,100,101,114,46,95,109,97,
-    116,99,104,101,115,95,108,101,103,97,99,121,99,3,0,0,
-    0,0,0,0,0,0,0,0,0,3,0,0,0,4,0,0,
-    0,3,0,0,0,115,48,0,0,0,124,1,160,0,161,0,
-    115,12,100,1,83,0,124,2,160,1,100,2,100,3,161,2,
-    137,1,135,0,135,1,102,2,100,4,100,5,132,8,124,1,
-    160,2,161,0,68,0,131,1,83,0,41,6,78,114,3,0,
-    0,0,250,1,45,114,44,0,0,0,99,1,0,0,0,0,
-    0,0,0,0,0,0,0,2,0,0,0,5,0,0,0,51,
-    0,0,0,115,42,0,0,0,124,0,93,34,125,1,136,0,
-    160,0,136,1,124,1,161,2,115,30,136,0,160,1,136,1,
-    124,1,161,2,114,2,124,1,86,0,1,0,113,2,100,0,
-    83,0,114,109,0,0,0,41,2,114,78,1,0,0,114,79,
-    1,0,0,41,2,114,31,0,0,0,114,40,1,0,0,169,
-    2,114,192,0,0,0,114,77,1,0,0,114,3,0,0,0,
-    114,6,0,0,0,114,18,1,0,0,147,5,0,0,115,8,
-    0,0,0,4,0,2,1,12,1,12,254,122,42,80,97,116,
-    104,70,105,110,100,101,114,46,95,115,101,97,114,99,104,95,
-    112,97,116,104,46,60,108,111,99,97,108,115,62,46,60,103,
-    101,110,101,120,112,114,62,41,3,90,6,105,115,95,100,105,
-    114,114,66,0,0,0,90,7,105,116,101,114,100,105,114,41,
-    3,114,192,0,0,0,90,4,114,111,111,116,114,61,1,0,
-    0,114,3,0,0,0,114,81,1,0,0,114,6,0,0,0,
-    114,64,1,0,0,142,5,0,0,115,8,0,0,0,0,2,
-    8,1,4,1,12,1,122,23,80,97,116,104,70,105,110,100,
-    101,114,46,95,115,101,97,114,99,104,95,112,97,116,104,41,
-    1,78,41,2,78,78,41,1,78,41,1,78,41,19,114,124,
-    0,0,0,114,123,0,0,0,114,125,0,0,0,114,126,0,
-    0,0,114,206,0,0,0,114,45,1,0,0,114,51,1,0,
-    0,114,53,1,0,0,114,54,1,0,0,114,57,1,0,0,
-    114,202,0,0,0,114,205,0,0,0,114,63,1,0,0,114,
-    60,1,0,0,218,12,115,116,97,116,105,99,109,101,116,104,
-    111,100,114,67,1,0,0,114,78,1,0,0,114,79,1,0,
-    0,114,64,1,0,0,114,3,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,44,1,0,0,215,
-    4,0,0,115,54,0,0,0,8,2,4,2,2,1,10,9,
-    2,1,10,12,2,1,10,21,2,1,10,14,2,1,12,31,
-    2,1,12,23,2,1,12,12,2,1,12,15,2,1,10,8,
-    2,1,10,10,2,1,10,6,2,1,10,6,2,1,114,44,
-    1,0,0,99,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,3,0,0,0,64,0,0,0,115,90,0,0,
-    0,101,0,90,1,100,0,90,2,100,1,90,3,100,2,100,
-    3,132,0,90,4,100,4,100,5,132,0,90,5,101,6,90,
-    7,100,6,100,7,132,0,90,8,100,8,100,9,132,0,90,
-    9,100,19,100,11,100,12,132,1,90,10,100,13,100,14,132,
-    0,90,11,101,12,100,15,100,16,132,0,131,1,90,13,100,
-    17,100,18,132,0,90,14,100,10,83,0,41,20,218,10,70,
-    105,108,101,70,105,110,100,101,114,122,172,70,105,108,101,45,
-    98,97,115,101,100,32,102,105,110,100,101,114,46,10,10,32,
-    32,32,32,73,110,116,101,114,97,99,116,105,111,110,115,32,
-    119,105,116,104,32,116,104,101,32,102,105,108,101,32,115,121,
-    115,116,101,109,32,97,114,101,32,99,97,99,104,101,100,32,
-    102,111,114,32,112,101,114,102,111,114,109,97,110,99,101,44,
-    32,98,101,105,110,103,10,32,32,32,32,114,101,102,114,101,
-    115,104,101,100,32,119,104,101,110,32,116,104,101,32,100,105,
-    114,101,99,116,111,114,121,32,116,104,101,32,102,105,110,100,
-    101,114,32,105,115,32,104,97,110,100,108,105,110,103,32,104,
-    97,115,32,98,101,101,110,32,109,111,100,105,102,105,101,100,
-    46,10,10,32,32,32,32,99,2,0,0,0,0,0,0,0,
-    0,0,0,0,5,0,0,0,6,0,0,0,7,0,0,0,
-    115,84,0,0,0,103,0,125,3,124,2,68,0,93,32,92,
-    2,137,0,125,4,124,3,160,0,135,0,102,1,100,1,100,
-    2,132,8,124,4,68,0,131,1,161,1,1,0,113,8,124,
-    3,124,0,95,1,124,1,112,54,100,3,124,0,95,2,100,
-    4,124,0,95,3,116,4,131,0,124,0,95,5,116,4,131,
-    0,124,0,95,6,100,5,83,0,41,6,122,154,73,110,105,
-    116,105,97,108,105,122,101,32,119,105,116,104,32,116,104,101,
-    32,112,97,116,104,32,116,111,32,115,101,97,114,99,104,32,
-    111,110,32,97,110,100,32,97,32,118,97,114,105,97,98,108,
-    101,32,110,117,109,98,101,114,32,111,102,10,32,32,32,32,
-    32,32,32,32,50,45,116,117,112,108,101,115,32,99,111,110,
-    116,97,105,110,105,110,103,32,116,104,101,32,108,111,97,100,
-    101,114,32,97,110,100,32,116,104,101,32,102,105,108,101,32,
-    115,117,102,102,105,120,101,115,32,116,104,101,32,108,111,97,
-    100,101,114,10,32,32,32,32,32,32,32,32,114,101,99,111,
-    103,110,105,122,101,115,46,99,1,0,0,0,0,0,0,0,
-    0,0,0,0,2,0,0,0,3,0,0,0,51,0,0,0,
-    115,22,0,0,0,124,0,93,14,125,1,124,1,136,0,102,
-    2,86,0,1,0,113,2,100,0,83,0,114,109,0,0,0,
-    114,3,0,0,0,114,15,1,0,0,169,1,114,139,0,0,
-    0,114,3,0,0,0,114,6,0,0,0,114,18,1,0,0,
-    167,5,0,0,115,4,0,0,0,4,0,2,0,122,38,70,
-    105,108,101,70,105,110,100,101,114,46,95,95,105,110,105,116,
-    95,95,46,60,108,111,99,97,108,115,62,46,60,103,101,110,
-    101,120,112,114,62,114,70,0,0,0,114,104,0,0,0,78,
-    41,7,114,166,0,0,0,218,8,95,108,111,97,100,101,114,
-    115,114,43,0,0,0,218,11,95,112,97,116,104,95,109,116,
-    105,109,101,218,3,115,101,116,218,11,95,112,97,116,104,95,
-    99,97,99,104,101,218,19,95,114,101,108,97,120,101,100,95,
-    112,97,116,104,95,99,97,99,104,101,41,5,114,118,0,0,
-    0,114,43,0,0,0,218,14,108,111,97,100,101,114,95,100,
-    101,116,97,105,108,115,90,7,108,111,97,100,101,114,115,114,
-    188,0,0,0,114,3,0,0,0,114,84,1,0,0,114,6,
-    0,0,0,114,208,0,0,0,161,5,0,0,115,16,0,0,
-    0,0,4,4,1,12,1,26,1,6,2,10,1,6,1,8,
+    41,1,218,18,77,101,116,97,100,97,116,97,80,97,116,104,
+    70,105,110,100,101,114,41,3,90,18,105,109,112,111,114,116,
+    108,105,98,46,109,101,116,97,100,97,116,97,114,59,1,0,
+    0,218,18,102,105,110,100,95,100,105,115,116,114,105,98,117,
+    116,105,111,110,115,41,4,114,193,0,0,0,114,119,0,0,
+    0,114,120,0,0,0,114,59,1,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,60,1,0,0,92,
+    5,0,0,115,4,0,0,0,0,10,12,1,122,29,80,97,
+    116,104,70,105,110,100,101,114,46,102,105,110,100,95,100,105,
+    115,116,114,105,98,117,116,105,111,110,115,41,1,78,41,2,
+    78,78,41,1,78,41,13,114,125,0,0,0,114,124,0,0,
+    0,114,126,0,0,0,114,127,0,0,0,114,207,0,0,0,
+    114,46,1,0,0,114,52,1,0,0,114,54,1,0,0,114,
+    55,1,0,0,114,58,1,0,0,114,203,0,0,0,114,206,
+    0,0,0,114,60,1,0,0,114,3,0,0,0,114,3,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,114,45,1,0,
+    0,215,4,0,0,115,34,0,0,0,8,2,4,2,2,1,
+    10,9,2,1,10,12,2,1,10,21,2,1,10,14,2,1,
+    12,31,2,1,12,23,2,1,12,12,2,1,114,45,1,0,
+    0,99,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,3,0,0,0,64,0,0,0,115,90,0,0,0,101,
+    0,90,1,100,0,90,2,100,1,90,3,100,2,100,3,132,
+    0,90,4,100,4,100,5,132,0,90,5,101,6,90,7,100,
+    6,100,7,132,0,90,8,100,8,100,9,132,0,90,9,100,
+    19,100,11,100,12,132,1,90,10,100,13,100,14,132,0,90,
+    11,101,12,100,15,100,16,132,0,131,1,90,13,100,17,100,
+    18,132,0,90,14,100,10,83,0,41,20,218,10,70,105,108,
+    101,70,105,110,100,101,114,122,172,70,105,108,101,45,98,97,
+    115,101,100,32,102,105,110,100,101,114,46,10,10,32,32,32,
+    32,73,110,116,101,114,97,99,116,105,111,110,115,32,119,105,
+    116,104,32,116,104,101,32,102,105,108,101,32,115,121,115,116,
+    101,109,32,97,114,101,32,99,97,99,104,101,100,32,102,111,
+    114,32,112,101,114,102,111,114,109,97,110,99,101,44,32,98,
+    101,105,110,103,10,32,32,32,32,114,101,102,114,101,115,104,
+    101,100,32,119,104,101,110,32,116,104,101,32,100,105,114,101,
+    99,116,111,114,121,32,116,104,101,32,102,105,110,100,101,114,
+    32,105,115,32,104,97,110,100,108,105,110,103,32,104,97,115,
+    32,98,101,101,110,32,109,111,100,105,102,105,101,100,46,10,
+    10,32,32,32,32,99,2,0,0,0,0,0,0,0,0,0,
+    0,0,5,0,0,0,6,0,0,0,7,0,0,0,115,84,
+    0,0,0,103,0,125,3,124,2,68,0,93,32,92,2,137,
+    0,125,4,124,3,160,0,135,0,102,1,100,1,100,2,132,
+    8,124,4,68,0,131,1,161,1,1,0,113,8,124,3,124,
+    0,95,1,124,1,112,54,100,3,124,0,95,2,100,4,124,
+    0,95,3,116,4,131,0,124,0,95,5,116,4,131,0,124,
+    0,95,6,100,5,83,0,41,6,122,154,73,110,105,116,105,
+    97,108,105,122,101,32,119,105,116,104,32,116,104,101,32,112,
+    97,116,104,32,116,111,32,115,101,97,114,99,104,32,111,110,
+    32,97,110,100,32,97,32,118,97,114,105,97,98,108,101,32,
+    110,117,109,98,101,114,32,111,102,10,32,32,32,32,32,32,
+    32,32,50,45,116,117,112,108,101,115,32,99,111,110,116,97,
+    105,110,105,110,103,32,116,104,101,32,108,111,97,100,101,114,
+    32,97,110,100,32,116,104,101,32,102,105,108,101,32,115,117,
+    102,102,105,120,101,115,32,116,104,101,32,108,111,97,100,101,
+    114,10,32,32,32,32,32,32,32,32,114,101,99,111,103,110,
+    105,122,101,115,46,99,1,0,0,0,0,0,0,0,0,0,
+    0,0,2,0,0,0,3,0,0,0,51,0,0,0,115,22,
+    0,0,0,124,0,93,14,125,1,124,1,136,0,102,2,86,
+    0,1,0,113,2,100,0,83,0,114,109,0,0,0,114,3,
+    0,0,0,114,16,1,0,0,169,1,114,140,0,0,0,114,
+    3,0,0,0,114,6,0,0,0,114,19,1,0,0,121,5,
+    0,0,115,4,0,0,0,4,0,2,0,122,38,70,105,108,
+    101,70,105,110,100,101,114,46,95,95,105,110,105,116,95,95,
+    46,60,108,111,99,97,108,115,62,46,60,103,101,110,101,120,
+    112,114,62,114,70,0,0,0,114,104,0,0,0,78,41,7,
+    114,167,0,0,0,218,8,95,108,111,97,100,101,114,115,114,
+    43,0,0,0,218,11,95,112,97,116,104,95,109,116,105,109,
+    101,218,3,115,101,116,218,11,95,112,97,116,104,95,99,97,
+    99,104,101,218,19,95,114,101,108,97,120,101,100,95,112,97,
+    116,104,95,99,97,99,104,101,41,5,114,118,0,0,0,114,
+    43,0,0,0,218,14,108,111,97,100,101,114,95,100,101,116,
+    97,105,108,115,90,7,108,111,97,100,101,114,115,114,189,0,
+    0,0,114,3,0,0,0,114,62,1,0,0,114,6,0,0,
+    0,114,209,0,0,0,115,5,0,0,115,16,0,0,0,0,
+    4,4,1,12,1,26,1,6,2,10,1,6,1,8,1,122,
+    19,70,105,108,101,70,105,110,100,101,114,46,95,95,105,110,
+    105,116,95,95,99,1,0,0,0,0,0,0,0,0,0,0,
+    0,1,0,0,0,2,0,0,0,67,0,0,0,115,10,0,
+    0,0,100,1,124,0,95,0,100,2,83,0,41,3,122,31,
+    73,110,118,97,108,105,100,97,116,101,32,116,104,101,32,100,
+    105,114,101,99,116,111,114,121,32,109,116,105,109,101,46,114,
+    104,0,0,0,78,41,1,114,64,1,0,0,114,246,0,0,
+    0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
+    114,46,1,0,0,129,5,0,0,115,2,0,0,0,0,2,
+    122,28,70,105,108,101,70,105,110,100,101,114,46,105,110,118,
+    97,108,105,100,97,116,101,95,99,97,99,104,101,115,99,2,
+    0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,3,
+    0,0,0,67,0,0,0,115,42,0,0,0,124,0,160,0,
+    124,1,161,1,125,2,124,2,100,1,107,8,114,26,100,1,
+    103,0,102,2,83,0,124,2,106,1,124,2,106,2,112,38,
+    103,0,102,2,83,0,41,2,122,197,84,114,121,32,116,111,
+    32,102,105,110,100,32,97,32,108,111,97,100,101,114,32,102,
+    111,114,32,116,104,101,32,115,112,101,99,105,102,105,101,100,
+    32,109,111,100,117,108,101,44,32,111,114,32,116,104,101,32,
+    110,97,109,101,115,112,97,99,101,10,32,32,32,32,32,32,
+    32,32,112,97,99,107,97,103,101,32,112,111,114,116,105,111,
+    110,115,46,32,82,101,116,117,114,110,115,32,40,108,111,97,
+    100,101,114,44,32,108,105,115,116,45,111,102,45,112,111,114,
+    116,105,111,110,115,41,46,10,10,32,32,32,32,32,32,32,
+    32,84,104,105,115,32,109,101,116,104,111,100,32,105,115,32,
+    100,101,112,114,101,99,97,116,101,100,46,32,32,85,115,101,
+    32,102,105,110,100,95,115,112,101,99,40,41,32,105,110,115,
+    116,101,97,100,46,10,10,32,32,32,32,32,32,32,32,78,
+    41,3,114,203,0,0,0,114,140,0,0,0,114,178,0,0,
+    0,41,3,114,118,0,0,0,114,139,0,0,0,114,187,0,
+    0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
+    0,114,137,0,0,0,135,5,0,0,115,8,0,0,0,0,
+    7,10,1,8,1,8,1,122,22,70,105,108,101,70,105,110,
+    100,101,114,46,102,105,110,100,95,108,111,97,100,101,114,99,
+    6,0,0,0,0,0,0,0,0,0,0,0,7,0,0,0,
+    6,0,0,0,67,0,0,0,115,26,0,0,0,124,1,124,
+    2,124,3,131,2,125,6,116,0,124,2,124,3,124,6,124,
+    4,100,1,141,4,83,0,41,2,78,114,177,0,0,0,41,
+    1,114,190,0,0,0,41,7,114,118,0,0,0,114,188,0,
+    0,0,114,139,0,0,0,114,43,0,0,0,90,4,115,109,
+    115,108,114,202,0,0,0,114,140,0,0,0,114,3,0,0,
+    0,114,3,0,0,0,114,6,0,0,0,114,58,1,0,0,
+    147,5,0,0,115,8,0,0,0,0,1,10,1,8,1,2,
+    255,122,20,70,105,108,101,70,105,110,100,101,114,46,95,103,
+    101,116,95,115,112,101,99,78,99,3,0,0,0,0,0,0,
+    0,0,0,0,0,14,0,0,0,8,0,0,0,67,0,0,
+    0,115,98,1,0,0,100,1,125,3,124,1,160,0,100,2,
+    161,1,100,3,25,0,125,4,122,24,116,1,124,0,106,2,
+    112,34,116,3,160,4,161,0,131,1,106,5,125,5,87,0,
+    110,24,4,0,116,6,107,10,114,66,1,0,1,0,1,0,
+    100,4,125,5,89,0,110,2,88,0,124,5,124,0,106,7,
+    107,3,114,92,124,0,160,8,161,0,1,0,124,5,124,0,
+    95,7,116,9,131,0,114,114,124,0,106,10,125,6,124,4,
+    160,11,161,0,125,7,110,10,124,0,106,12,125,6,124,4,
+    125,7,124,7,124,6,107,6,114,218,116,13,124,0,106,2,
+    124,4,131,2,125,8,124,0,106,14,68,0,93,58,92,2,
+    125,9,125,10,100,5,124,9,23,0,125,11,116,13,124,8,
+    124,11,131,2,125,12,116,15,124,12,131,1,114,150,124,0,
+    160,16,124,10,124,1,124,12,124,8,103,1,124,2,161,5,
+    2,0,1,0,83,0,113,150,116,17,124,8,131,1,125,3,
+    124,0,106,14,68,0,93,82,92,2,125,9,125,10,116,13,
+    124,0,106,2,124,4,124,9,23,0,131,2,125,12,116,18,
+    106,19,100,6,124,12,100,3,100,7,141,3,1,0,124,7,
+    124,9,23,0,124,6,107,6,114,224,116,15,124,12,131,1,
+    114,224,124,0,160,16,124,10,124,1,124,12,100,8,124,2,
+    161,5,2,0,1,0,83,0,113,224,124,3,144,1,114,94,
+    116,18,160,19,100,9,124,8,161,2,1,0,116,18,160,20,
+    124,1,100,8,161,2,125,13,124,8,103,1,124,13,95,21,
+    124,13,83,0,100,8,83,0,41,10,122,111,84,114,121,32,
+    116,111,32,102,105,110,100,32,97,32,115,112,101,99,32,102,
+    111,114,32,116,104,101,32,115,112,101,99,105,102,105,101,100,
+    32,109,111,100,117,108,101,46,10,10,32,32,32,32,32,32,
+    32,32,82,101,116,117,114,110,115,32,116,104,101,32,109,97,
+    116,99,104,105,110,103,32,115,112,101,99,44,32,111,114,32,
+    78,111,110,101,32,105,102,32,110,111,116,32,102,111,117,110,
+    100,46,10,32,32,32,32,32,32,32,32,70,114,70,0,0,
+    0,114,27,0,0,0,114,104,0,0,0,114,209,0,0,0,
+    122,9,116,114,121,105,110,103,32,123,125,41,1,90,9,118,
+    101,114,98,111,115,105,116,121,78,122,25,112,111,115,115,105,
+    98,108,101,32,110,97,109,101,115,112,97,99,101,32,102,111,
+    114,32,123,125,41,22,114,40,0,0,0,114,48,0,0,0,
+    114,43,0,0,0,114,2,0,0,0,114,54,0,0,0,114,
+    10,1,0,0,114,49,0,0,0,114,64,1,0,0,218,11,
+    95,102,105,108,108,95,99,97,99,104,101,114,7,0,0,0,
+    114,67,1,0,0,114,105,0,0,0,114,66,1,0,0,114,
+    37,0,0,0,114,63,1,0,0,114,53,0,0,0,114,58,
+    1,0,0,114,55,0,0,0,114,134,0,0,0,114,149,0,
+    0,0,114,183,0,0,0,114,178,0,0,0,41,14,114,118,
+    0,0,0,114,139,0,0,0,114,202,0,0,0,90,12,105,
+    115,95,110,97,109,101,115,112,97,99,101,90,11,116,97,105,
+    108,95,109,111,100,117,108,101,114,169,0,0,0,90,5,99,
+    97,99,104,101,90,12,99,97,99,104,101,95,109,111,100,117,
+    108,101,90,9,98,97,115,101,95,112,97,116,104,114,17,1,
+    0,0,114,188,0,0,0,90,13,105,110,105,116,95,102,105,
+    108,101,110,97,109,101,90,9,102,117,108,108,95,112,97,116,
+    104,114,187,0,0,0,114,3,0,0,0,114,3,0,0,0,
+    114,6,0,0,0,114,203,0,0,0,152,5,0,0,115,74,
+    0,0,0,0,5,4,1,14,1,2,1,24,1,14,1,10,
+    1,10,1,8,1,6,2,6,1,6,1,10,2,6,1,4,
+    2,8,1,12,1,14,1,8,1,10,1,8,1,26,4,8,
+    2,14,1,16,1,16,1,12,1,8,1,10,1,2,0,2,
+    255,10,2,6,1,12,1,12,1,8,1,4,1,122,20,70,
+    105,108,101,70,105,110,100,101,114,46,102,105,110,100,95,115,
+    112,101,99,99,1,0,0,0,0,0,0,0,0,0,0,0,
+    9,0,0,0,10,0,0,0,67,0,0,0,115,190,0,0,
+    0,124,0,106,0,125,1,122,22,116,1,160,2,124,1,112,
+    22,116,1,160,3,161,0,161,1,125,2,87,0,110,30,4,
+    0,116,4,116,5,116,6,102,3,107,10,114,58,1,0,1,
+    0,1,0,103,0,125,2,89,0,110,2,88,0,116,7,106,
+    8,160,9,100,1,161,1,115,84,116,10,124,2,131,1,124,
+    0,95,11,110,74,116,10,131,0,125,3,124,2,68,0,93,
+    56,125,4,124,4,160,12,100,2,161,1,92,3,125,5,125,
+    6,125,7,124,6,114,136,100,3,160,13,124,5,124,7,160,
+    14,161,0,161,2,125,8,110,4,124,5,125,8,124,3,160,
+    15,124,8,161,1,1,0,113,94,124,3,124,0,95,11,116,
+    7,106,8,160,9,116,16,161,1,114,186,100,4,100,5,132,
+    0,124,2,68,0,131,1,124,0,95,17,100,6,83,0,41,
+    7,122,68,70,105,108,108,32,116,104,101,32,99,97,99,104,
+    101,32,111,102,32,112,111,116,101,110,116,105,97,108,32,109,
+    111,100,117,108,101,115,32,97,110,100,32,112,97,99,107,97,
+    103,101,115,32,102,111,114,32,116,104,105,115,32,100,105,114,
+    101,99,116,111,114,121,46,114,0,0,0,0,114,70,0,0,
+    0,114,60,0,0,0,99,1,0,0,0,0,0,0,0,0,
+    0,0,0,2,0,0,0,4,0,0,0,83,0,0,0,115,
+    20,0,0,0,104,0,124,0,93,12,125,1,124,1,160,0,
+    161,0,146,2,113,4,83,0,114,3,0,0,0,41,1,114,
+    105,0,0,0,41,2,114,31,0,0,0,90,2,102,110,114,
+    3,0,0,0,114,3,0,0,0,114,6,0,0,0,218,9,
+    60,115,101,116,99,111,109,112,62,229,5,0,0,115,4,0,
+    0,0,6,0,2,0,122,41,70,105,108,101,70,105,110,100,
+    101,114,46,95,102,105,108,108,95,99,97,99,104,101,46,60,
+    108,111,99,97,108,115,62,46,60,115,101,116,99,111,109,112,
+    62,78,41,18,114,43,0,0,0,114,2,0,0,0,114,7,
+    1,0,0,114,54,0,0,0,114,3,1,0,0,218,15,80,
+    101,114,109,105,115,115,105,111,110,69,114,114,111,114,218,18,
+    78,111,116,65,68,105,114,101,99,116,111,114,121,69,114,114,
+    111,114,114,8,0,0,0,114,9,0,0,0,114,10,0,0,
+    0,114,65,1,0,0,114,66,1,0,0,114,100,0,0,0,
+    114,61,0,0,0,114,105,0,0,0,218,3,97,100,100,114,
+    11,0,0,0,114,67,1,0,0,41,9,114,118,0,0,0,
+    114,43,0,0,0,114,8,1,0,0,90,21,108,111,119,101,
+    114,95,115,117,102,102,105,120,95,99,111,110,116,101,110,116,
+    115,114,41,1,0,0,114,116,0,0,0,114,29,1,0,0,
+    114,17,1,0,0,90,8,110,101,119,95,110,97,109,101,114,
+    3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,69,
+    1,0,0,200,5,0,0,115,34,0,0,0,0,2,6,1,
+    2,1,22,1,20,3,10,3,12,1,12,7,6,1,8,1,
+    16,1,4,1,18,2,4,1,12,1,6,1,12,1,122,22,
+    70,105,108,101,70,105,110,100,101,114,46,95,102,105,108,108,
+    95,99,97,99,104,101,99,1,0,0,0,0,0,0,0,0,
+    0,0,0,3,0,0,0,3,0,0,0,7,0,0,0,115,
+    18,0,0,0,135,0,135,1,102,2,100,1,100,2,132,8,
+    125,2,124,2,83,0,41,3,97,20,1,0,0,65,32,99,
+    108,97,115,115,32,109,101,116,104,111,100,32,119,104,105,99,
+    104,32,114,101,116,117,114,110,115,32,97,32,99,108,111,115,
+    117,114,101,32,116,111,32,117,115,101,32,111,110,32,115,121,
+    115,46,112,97,116,104,95,104,111,111,107,10,32,32,32,32,
+    32,32,32,32,119,104,105,99,104,32,119,105,108,108,32,114,
+    101,116,117,114,110,32,97,110,32,105,110,115,116,97,110,99,
+    101,32,117,115,105,110,103,32,116,104,101,32,115,112,101,99,
+    105,102,105,101,100,32,108,111,97,100,101,114,115,32,97,110,
+    100,32,116,104,101,32,112,97,116,104,10,32,32,32,32,32,
+    32,32,32,99,97,108,108,101,100,32,111,110,32,116,104,101,
+    32,99,108,111,115,117,114,101,46,10,10,32,32,32,32,32,
+    32,32,32,73,102,32,116,104,101,32,112,97,116,104,32,99,
+    97,108,108,101,100,32,111,110,32,116,104,101,32,99,108,111,
+    115,117,114,101,32,105,115,32,110,111,116,32,97,32,100,105,
+    114,101,99,116,111,114,121,44,32,73,109,112,111,114,116,69,
+    114,114,111,114,32,105,115,10,32,32,32,32,32,32,32,32,
+    114,97,105,115,101,100,46,10,10,32,32,32,32,32,32,32,
+    32,99,1,0,0,0,0,0,0,0,0,0,0,0,1,0,
+    0,0,4,0,0,0,19,0,0,0,115,34,0,0,0,116,
+    0,124,0,131,1,115,20,116,1,100,1,124,0,100,2,141,
+    2,130,1,136,0,124,0,102,1,136,1,158,2,142,0,83,
+    0,41,3,122,45,80,97,116,104,32,104,111,111,107,32,102,
+    111,114,32,105,109,112,111,114,116,108,105,98,46,109,97,99,
+    104,105,110,101,114,121,46,70,105,108,101,70,105,110,100,101,
+    114,46,122,30,111,110,108,121,32,100,105,114,101,99,116,111,
+    114,105,101,115,32,97,114,101,32,115,117,112,112,111,114,116,
+    101,100,114,47,0,0,0,41,2,114,55,0,0,0,114,117,
+    0,0,0,114,47,0,0,0,169,2,114,193,0,0,0,114,
+    68,1,0,0,114,3,0,0,0,114,6,0,0,0,218,24,
+    112,97,116,104,95,104,111,111,107,95,102,111,114,95,70,105,
+    108,101,70,105,110,100,101,114,241,5,0,0,115,6,0,0,
+    0,0,2,8,1,12,1,122,54,70,105,108,101,70,105,110,
+    100,101,114,46,112,97,116,104,95,104,111,111,107,46,60,108,
+    111,99,97,108,115,62,46,112,97,116,104,95,104,111,111,107,
+    95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,114,
+    3,0,0,0,41,3,114,193,0,0,0,114,68,1,0,0,
+    114,75,1,0,0,114,3,0,0,0,114,74,1,0,0,114,
+    6,0,0,0,218,9,112,97,116,104,95,104,111,111,107,231,
+    5,0,0,115,4,0,0,0,0,10,14,6,122,20,70,105,
+    108,101,70,105,110,100,101,114,46,112,97,116,104,95,104,111,
+    111,107,99,1,0,0,0,0,0,0,0,0,0,0,0,1,
+    0,0,0,3,0,0,0,67,0,0,0,115,12,0,0,0,
+    100,1,160,0,124,0,106,1,161,1,83,0,41,2,78,122,
+    16,70,105,108,101,70,105,110,100,101,114,40,123,33,114,125,
+    41,41,2,114,61,0,0,0,114,43,0,0,0,114,246,0,
+    0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
+    0,114,39,1,0,0,249,5,0,0,115,2,0,0,0,0,
     1,122,19,70,105,108,101,70,105,110,100,101,114,46,95,95,
-    105,110,105,116,95,95,99,1,0,0,0,0,0,0,0,0,
-    0,0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,
-    10,0,0,0,100,1,124,0,95,0,100,2,83,0,41,3,
-    122,31,73,110,118,97,108,105,100,97,116,101,32,116,104,101,
-    32,100,105,114,101,99,116,111,114,121,32,109,116,105,109,101,
-    46,114,104,0,0,0,78,41,1,114,86,1,0,0,114,245,
-    0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,114,45,1,0,0,175,5,0,0,115,2,0,0,0,
-    0,2,122,28,70,105,108,101,70,105,110,100,101,114,46,105,
-    110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115,
-    99,2,0,0,0,0,0,0,0,0,0,0,0,3,0,0,
-    0,3,0,0,0,67,0,0,0,115,42,0,0,0,124,0,
-    160,0,124,1,161,1,125,2,124,2,100,1,107,8,114,26,
-    100,1,103,0,102,2,83,0,124,2,106,1,124,2,106,2,
-    112,38,103,0,102,2,83,0,41,2,122,197,84,114,121,32,
-    116,111,32,102,105,110,100,32,97,32,108,111,97,100,101,114,
-    32,102,111,114,32,116,104,101,32,115,112,101,99,105,102,105,
-    101,100,32,109,111,100,117,108,101,44,32,111,114,32,116,104,
-    101,32,110,97,109,101,115,112,97,99,101,10,32,32,32,32,
-    32,32,32,32,112,97,99,107,97,103,101,32,112,111,114,116,
-    105,111,110,115,46,32,82,101,116,117,114,110,115,32,40,108,
-    111,97,100,101,114,44,32,108,105,115,116,45,111,102,45,112,
-    111,114,116,105,111,110,115,41,46,10,10,32,32,32,32,32,
-    32,32,32,84,104,105,115,32,109,101,116,104,111,100,32,105,
-    115,32,100,101,112,114,101,99,97,116,101,100,46,32,32,85,
-    115,101,32,102,105,110,100,95,115,112,101,99,40,41,32,105,
-    110,115,116,101,97,100,46,10,10,32,32,32,32,32,32,32,
-    32,78,41,3,114,202,0,0,0,114,139,0,0,0,114,177,
-    0,0,0,41,3,114,118,0,0,0,114,138,0,0,0,114,
-    186,0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
-    0,0,0,114,136,0,0,0,181,5,0,0,115,8,0,0,
-    0,0,7,10,1,8,1,8,1,122,22,70,105,108,101,70,
-    105,110,100,101,114,46,102,105,110,100,95,108,111,97,100,101,
-    114,99,6,0,0,0,0,0,0,0,0,0,0,0,7,0,
-    0,0,6,0,0,0,67,0,0,0,115,26,0,0,0,124,
-    1,124,2,124,3,131,2,125,6,116,0,124,2,124,3,124,
-    6,124,4,100,1,141,4,83,0,41,2,78,114,176,0,0,
-    0,41,1,114,189,0,0,0,41,7,114,118,0,0,0,114,
-    187,0,0,0,114,138,0,0,0,114,43,0,0,0,90,4,
-    115,109,115,108,114,201,0,0,0,114,139,0,0,0,114,3,
-    0,0,0,114,3,0,0,0,114,6,0,0,0,114,57,1,
-    0,0,193,5,0,0,115,8,0,0,0,0,1,10,1,8,
-    1,2,255,122,20,70,105,108,101,70,105,110,100,101,114,46,
-    95,103,101,116,95,115,112,101,99,78,99,3,0,0,0,0,
-    0,0,0,0,0,0,0,14,0,0,0,8,0,0,0,67,
-    0,0,0,115,98,1,0,0,100,1,125,3,124,1,160,0,
-    100,2,161,1,100,3,25,0,125,4,122,24,116,1,124,0,
-    106,2,112,34,116,3,160,4,161,0,131,1,106,5,125,5,
-    87,0,110,24,4,0,116,6,107,10,114,66,1,0,1,0,
-    1,0,100,4,125,5,89,0,110,2,88,0,124,5,124,0,
-    106,7,107,3,114,92,124,0,160,8,161,0,1,0,124,5,
-    124,0,95,7,116,9,131,0,114,114,124,0,106,10,125,6,
-    124,4,160,11,161,0,125,7,110,10,124,0,106,12,125,6,
-    124,4,125,7,124,7,124,6,107,6,114,218,116,13,124,0,
-    106,2,124,4,131,2,125,8,124,0,106,14,68,0,93,58,
-    92,2,125,9,125,10,100,5,124,9,23,0,125,11,116,13,
-    124,8,124,11,131,2,125,12,116,15,124,12,131,1,114,150,
-    124,0,160,16,124,10,124,1,124,12,124,8,103,1,124,2,
-    161,5,2,0,1,0,83,0,113,150,116,17,124,8,131,1,
-    125,3,124,0,106,14,68,0,93,82,92,2,125,9,125,10,
-    116,13,124,0,106,2,124,4,124,9,23,0,131,2,125,12,
-    116,18,106,19,100,6,124,12,100,3,100,7,141,3,1,0,
-    124,7,124,9,23,0,124,6,107,6,114,224,116,15,124,12,
-    131,1,114,224,124,0,160,16,124,10,124,1,124,12,100,8,
-    124,2,161,5,2,0,1,0,83,0,113,224,124,3,144,1,
-    114,94,116,18,160,19,100,9,124,8,161,2,1,0,116,18,
-    160,20,124,1,100,8,161,2,125,13,124,8,103,1,124,13,
-    95,21,124,13,83,0,100,8,83,0,41,10,122,111,84,114,
-    121,32,116,111,32,102,105,110,100,32,97,32,115,112,101,99,
-    32,102,111,114,32,116,104,101,32,115,112,101,99,105,102,105,
-    101,100,32,109,111,100,117,108,101,46,10,10,32,32,32,32,
-    32,32,32,32,82,101,116,117,114,110,115,32,116,104,101,32,
-    109,97,116,99,104,105,110,103,32,115,112,101,99,44,32,111,
-    114,32,78,111,110,101,32,105,102,32,110,111,116,32,102,111,
-    117,110,100,46,10,32,32,32,32,32,32,32,32,70,114,70,
-    0,0,0,114,27,0,0,0,114,104,0,0,0,114,208,0,
-    0,0,122,9,116,114,121,105,110,103,32,123,125,41,1,90,
-    9,118,101,114,98,111,115,105,116,121,78,122,25,112,111,115,
-    115,105,98,108,101,32,110,97,109,101,115,112,97,99,101,32,
-    102,111,114,32,123,125,41,22,114,40,0,0,0,114,48,0,
-    0,0,114,43,0,0,0,114,2,0,0,0,114,54,0,0,
-    0,114,9,1,0,0,114,49,0,0,0,114,86,1,0,0,
-    218,11,95,102,105,108,108,95,99,97,99,104,101,114,7,0,
-    0,0,114,89,1,0,0,114,105,0,0,0,114,88,1,0,
-    0,114,37,0,0,0,114,85,1,0,0,114,53,0,0,0,
-    114,57,1,0,0,114,55,0,0,0,114,133,0,0,0,114,
-    148,0,0,0,114,182,0,0,0,114,177,0,0,0,41,14,
-    114,118,0,0,0,114,138,0,0,0,114,201,0,0,0,90,
-    12,105,115,95,110,97,109,101,115,112,97,99,101,90,11,116,
-    97,105,108,95,109,111,100,117,108,101,114,168,0,0,0,90,
-    5,99,97,99,104,101,90,12,99,97,99,104,101,95,109,111,
-    100,117,108,101,90,9,98,97,115,101,95,112,97,116,104,114,
-    16,1,0,0,114,187,0,0,0,90,13,105,110,105,116,95,
-    102,105,108,101,110,97,109,101,90,9,102,117,108,108,95,112,
-    97,116,104,114,186,0,0,0,114,3,0,0,0,114,3,0,
-    0,0,114,6,0,0,0,114,202,0,0,0,198,5,0,0,
-    115,74,0,0,0,0,5,4,1,14,1,2,1,24,1,14,
-    1,10,1,10,1,8,1,6,2,6,1,6,1,10,2,6,
-    1,4,2,8,1,12,1,14,1,8,1,10,1,8,1,26,
-    4,8,2,14,1,16,1,16,1,12,1,8,1,10,1,2,
-    0,2,255,10,2,6,1,12,1,12,1,8,1,4,1,122,
-    20,70,105,108,101,70,105,110,100,101,114,46,102,105,110,100,
-    95,115,112,101,99,99,1,0,0,0,0,0,0,0,0,0,
-    0,0,9,0,0,0,10,0,0,0,67,0,0,0,115,190,
-    0,0,0,124,0,106,0,125,1,122,22,116,1,160,2,124,
-    1,112,22,116,1,160,3,161,0,161,1,125,2,87,0,110,
-    30,4,0,116,4,116,5,116,6,102,3,107,10,114,58,1,
-    0,1,0,1,0,103,0,125,2,89,0,110,2,88,0,116,
-    7,106,8,160,9,100,1,161,1,115,84,116,10,124,2,131,
-    1,124,0,95,11,110,74,116,10,131,0,125,3,124,2,68,
-    0,93,56,125,4,124,4,160,12,100,2,161,1,92,3,125,
-    5,125,6,125,7,124,6,114,136,100,3,160,13,124,5,124,
-    7,160,14,161,0,161,2,125,8,110,4,124,5,125,8,124,
-    3,160,15,124,8,161,1,1,0,113,94,124,3,124,0,95,
-    11,116,7,106,8,160,9,116,16,161,1,114,186,100,4,100,
-    5,132,0,124,2,68,0,131,1,124,0,95,17,100,6,83,
-    0,41,7,122,68,70,105,108,108,32,116,104,101,32,99,97,
-    99,104,101,32,111,102,32,112,111,116,101,110,116,105,97,108,
-    32,109,111,100,117,108,101,115,32,97,110,100,32,112,97,99,
-    107,97,103,101,115,32,102,111,114,32,116,104,105,115,32,100,
-    105,114,101,99,116,111,114,121,46,114,0,0,0,0,114,70,
-    0,0,0,114,60,0,0,0,99,1,0,0,0,0,0,0,
-    0,0,0,0,0,2,0,0,0,4,0,0,0,83,0,0,
-    0,115,20,0,0,0,104,0,124,0,93,12,125,1,124,1,
-    160,0,161,0,146,2,113,4,83,0,114,3,0,0,0,41,
-    1,114,105,0,0,0,41,2,114,31,0,0,0,90,2,102,
-    110,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
-    218,9,60,115,101,116,99,111,109,112,62,19,6,0,0,115,
-    4,0,0,0,6,0,2,0,122,41,70,105,108,101,70,105,
-    110,100,101,114,46,95,102,105,108,108,95,99,97,99,104,101,
-    46,60,108,111,99,97,108,115,62,46,60,115,101,116,99,111,
-    109,112,62,78,41,18,114,43,0,0,0,114,2,0,0,0,
-    114,6,1,0,0,114,54,0,0,0,114,2,1,0,0,218,
-    15,80,101,114,109,105,115,115,105,111,110,69,114,114,111,114,
-    218,18,78,111,116,65,68,105,114,101,99,116,111,114,121,69,
-    114,114,111,114,114,8,0,0,0,114,9,0,0,0,114,10,
-    0,0,0,114,87,1,0,0,114,88,1,0,0,114,100,0,
-    0,0,114,61,0,0,0,114,105,0,0,0,218,3,97,100,
-    100,114,11,0,0,0,114,89,1,0,0,41,9,114,118,0,
-    0,0,114,43,0,0,0,114,7,1,0,0,90,21,108,111,
-    119,101,114,95,115,117,102,102,105,120,95,99,111,110,116,101,
-    110,116,115,114,40,1,0,0,114,116,0,0,0,114,28,1,
-    0,0,114,16,1,0,0,90,8,110,101,119,95,110,97,109,
+    114,101,112,114,95,95,41,1,78,41,15,114,125,0,0,0,
+    114,124,0,0,0,114,126,0,0,0,114,127,0,0,0,114,
+    209,0,0,0,114,46,1,0,0,114,143,0,0,0,114,206,
+    0,0,0,114,137,0,0,0,114,58,1,0,0,114,203,0,
+    0,0,114,69,1,0,0,114,207,0,0,0,114,76,1,0,
+    0,114,39,1,0,0,114,3,0,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,61,1,0,0,106,
+    5,0,0,115,22,0,0,0,8,2,4,7,8,14,8,4,
+    4,2,8,12,8,5,10,48,8,31,2,1,10,17,114,61,
+    1,0,0,99,4,0,0,0,0,0,0,0,0,0,0,0,
+    6,0,0,0,8,0,0,0,67,0,0,0,115,146,0,0,
+    0,124,0,160,0,100,1,161,1,125,4,124,0,160,0,100,
+    2,161,1,125,5,124,4,115,66,124,5,114,36,124,5,106,
+    1,125,4,110,30,124,2,124,3,107,2,114,56,116,2,124,
+    1,124,2,131,2,125,4,110,10,116,3,124,1,124,2,131,
+    2,125,4,124,5,115,84,116,4,124,1,124,2,124,4,100,
+    3,141,3,125,5,122,36,124,5,124,0,100,2,60,0,124,
+    4,124,0,100,1,60,0,124,2,124,0,100,4,60,0,124,
+    3,124,0,100,5,60,0,87,0,110,20,4,0,116,5,107,
+    10,114,140,1,0,1,0,1,0,89,0,110,2,88,0,100,
+    0,83,0,41,6,78,218,10,95,95,108,111,97,100,101,114,
+    95,95,218,8,95,95,115,112,101,99,95,95,114,62,1,0,
+    0,90,8,95,95,102,105,108,101,95,95,90,10,95,95,99,
+    97,99,104,101,100,95,95,41,6,218,3,103,101,116,114,140,
+    0,0,0,114,15,1,0,0,114,9,1,0,0,114,190,0,
+    0,0,218,9,69,120,99,101,112,116,105,111,110,41,6,90,
+    2,110,115,114,116,0,0,0,90,8,112,97,116,104,110,97,
+    109,101,90,9,99,112,97,116,104,110,97,109,101,114,140,0,
+    0,0,114,187,0,0,0,114,3,0,0,0,114,3,0,0,
+    0,114,6,0,0,0,218,14,95,102,105,120,95,117,112,95,
+    109,111,100,117,108,101,255,5,0,0,115,34,0,0,0,0,
+    2,10,1,10,1,4,1,4,1,8,1,8,1,12,2,10,
+    1,4,1,14,1,2,1,8,1,8,1,8,1,12,1,14,
+    2,114,81,1,0,0,99,0,0,0,0,0,0,0,0,0,
+    0,0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,
+    38,0,0,0,116,0,116,1,160,2,161,0,102,2,125,0,
+    116,3,116,4,102,2,125,1,116,5,116,6,102,2,125,2,
+    124,0,124,1,124,2,103,3,83,0,41,1,122,95,82,101,
+    116,117,114,110,115,32,97,32,108,105,115,116,32,111,102,32,
+    102,105,108,101,45,98,97,115,101,100,32,109,111,100,117,108,
+    101,32,108,111,97,100,101,114,115,46,10,10,32,32,32,32,
+    69,97,99,104,32,105,116,101,109,32,105,115,32,97,32,116,
+    117,112,108,101,32,40,108,111,97,100,101,114,44,32,115,117,
+    102,102,105,120,101,115,41,46,10,32,32,32,32,41,7,114,
+    252,0,0,0,114,163,0,0,0,218,18,101,120,116,101,110,
+    115,105,111,110,95,115,117,102,102,105,120,101,115,114,9,1,
+    0,0,114,101,0,0,0,114,15,1,0,0,114,88,0,0,
+    0,41,3,90,10,101,120,116,101,110,115,105,111,110,115,90,
+    6,115,111,117,114,99,101,90,8,98,121,116,101,99,111,100,
     101,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
-    114,91,1,0,0,246,5,0,0,115,34,0,0,0,0,2,
-    6,1,2,1,22,1,20,3,10,3,12,1,12,7,6,1,
-    8,1,16,1,4,1,18,2,4,1,12,1,6,1,12,1,
-    122,22,70,105,108,101,70,105,110,100,101,114,46,95,102,105,
-    108,108,95,99,97,99,104,101,99,1,0,0,0,0,0,0,
-    0,0,0,0,0,3,0,0,0,3,0,0,0,7,0,0,
-    0,115,18,0,0,0,135,0,135,1,102,2,100,1,100,2,
-    132,8,125,2,124,2,83,0,41,3,97,20,1,0,0,65,
-    32,99,108,97,115,115,32,109,101,116,104,111,100,32,119,104,
-    105,99,104,32,114,101,116,117,114,110,115,32,97,32,99,108,
-    111,115,117,114,101,32,116,111,32,117,115,101,32,111,110,32,
-    115,121,115,46,112,97,116,104,95,104,111,111,107,10,32,32,
-    32,32,32,32,32,32,119,104,105,99,104,32,119,105,108,108,
-    32,114,101,116,117,114,110,32,97,110,32,105,110,115,116,97,
-    110,99,101,32,117,115,105,110,103,32,116,104,101,32,115,112,
-    101,99,105,102,105,101,100,32,108,111,97,100,101,114,115,32,
-    97,110,100,32,116,104,101,32,112,97,116,104,10,32,32,32,
-    32,32,32,32,32,99,97,108,108,101,100,32,111,110,32,116,
-    104,101,32,99,108,111,115,117,114,101,46,10,10,32,32,32,
-    32,32,32,32,32,73,102,32,116,104,101,32,112,97,116,104,
-    32,99,97,108,108,101,100,32,111,110,32,116,104,101,32,99,
-    108,111,115,117,114,101,32,105,115,32,110,111,116,32,97,32,
-    100,105,114,101,99,116,111,114,121,44,32,73,109,112,111,114,
-    116,69,114,114,111,114,32,105,115,10,32,32,32,32,32,32,
-    32,32,114,97,105,115,101,100,46,10,10,32,32,32,32,32,
-    32,32,32,99,1,0,0,0,0,0,0,0,0,0,0,0,
-    1,0,0,0,4,0,0,0,19,0,0,0,115,34,0,0,
-    0,116,0,124,0,131,1,115,20,116,1,100,1,124,0,100,
-    2,141,2,130,1,136,0,124,0,102,1,136,1,158,2,142,
-    0,83,0,41,3,122,45,80,97,116,104,32,104,111,111,107,
-    32,102,111,114,32,105,109,112,111,114,116,108,105,98,46,109,
-    97,99,104,105,110,101,114,121,46,70,105,108,101,70,105,110,
-    100,101,114,46,122,30,111,110,108,121,32,100,105,114,101,99,
-    116,111,114,105,101,115,32,97,114,101,32,115,117,112,112,111,
-    114,116,101,100,114,47,0,0,0,41,2,114,55,0,0,0,
-    114,117,0,0,0,114,47,0,0,0,169,2,114,192,0,0,
-    0,114,90,1,0,0,114,3,0,0,0,114,6,0,0,0,
-    218,24,112,97,116,104,95,104,111,111,107,95,102,111,114,95,
-    70,105,108,101,70,105,110,100,101,114,31,6,0,0,115,6,
-    0,0,0,0,2,8,1,12,1,122,54,70,105,108,101,70,
-    105,110,100,101,114,46,112,97,116,104,95,104,111,111,107,46,
-    60,108,111,99,97,108,115,62,46,112,97,116,104,95,104,111,
-    111,107,95,102,111,114,95,70,105,108,101,70,105,110,100,101,
-    114,114,3,0,0,0,41,3,114,192,0,0,0,114,90,1,
-    0,0,114,97,1,0,0,114,3,0,0,0,114,96,1,0,
-    0,114,6,0,0,0,218,9,112,97,116,104,95,104,111,111,
-    107,21,6,0,0,115,4,0,0,0,0,10,14,6,122,20,
-    70,105,108,101,70,105,110,100,101,114,46,112,97,116,104,95,
-    104,111,111,107,99,1,0,0,0,0,0,0,0,0,0,0,
-    0,1,0,0,0,3,0,0,0,67,0,0,0,115,12,0,
-    0,0,100,1,160,0,124,0,106,1,161,1,83,0,41,2,
-    78,122,16,70,105,108,101,70,105,110,100,101,114,40,123,33,
-    114,125,41,41,2,114,61,0,0,0,114,43,0,0,0,114,
-    245,0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
-    0,0,0,114,38,1,0,0,39,6,0,0,115,2,0,0,
-    0,0,1,122,19,70,105,108,101,70,105,110,100,101,114,46,
-    95,95,114,101,112,114,95,95,41,1,78,41,15,114,124,0,
-    0,0,114,123,0,0,0,114,125,0,0,0,114,126,0,0,
-    0,114,208,0,0,0,114,45,1,0,0,114,142,0,0,0,
-    114,205,0,0,0,114,136,0,0,0,114,57,1,0,0,114,
-    202,0,0,0,114,91,1,0,0,114,206,0,0,0,114,98,
-    1,0,0,114,38,1,0,0,114,3,0,0,0,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,83,1,0,
-    0,152,5,0,0,115,22,0,0,0,8,2,4,7,8,14,
-    8,4,4,2,8,12,8,5,10,48,8,31,2,1,10,17,
-    114,83,1,0,0,99,4,0,0,0,0,0,0,0,0,0,
-    0,0,6,0,0,0,8,0,0,0,67,0,0,0,115,146,
-    0,0,0,124,0,160,0,100,1,161,1,125,4,124,0,160,
-    0,100,2,161,1,125,5,124,4,115,66,124,5,114,36,124,
-    5,106,1,125,4,110,30,124,2,124,3,107,2,114,56,116,
-    2,124,1,124,2,131,2,125,4,110,10,116,3,124,1,124,
-    2,131,2,125,4,124,5,115,84,116,4,124,1,124,2,124,
-    4,100,3,141,3,125,5,122,36,124,5,124,0,100,2,60,
-    0,124,4,124,0,100,1,60,0,124,2,124,0,100,4,60,
-    0,124,3,124,0,100,5,60,0,87,0,110,20,4,0,116,
-    5,107,10,114,140,1,0,1,0,1,0,89,0,110,2,88,
-    0,100,0,83,0,41,6,78,218,10,95,95,108,111,97,100,
-    101,114,95,95,218,8,95,95,115,112,101,99,95,95,114,84,
-    1,0,0,90,8,95,95,102,105,108,101,95,95,90,10,95,
-    95,99,97,99,104,101,100,95,95,41,6,218,3,103,101,116,
-    114,139,0,0,0,114,14,1,0,0,114,8,1,0,0,114,
-    189,0,0,0,114,71,1,0,0,41,6,90,2,110,115,114,
-    116,0,0,0,90,8,112,97,116,104,110,97,109,101,90,9,
-    99,112,97,116,104,110,97,109,101,114,139,0,0,0,114,186,
-    0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,218,14,95,102,105,120,95,117,112,95,109,111,100,117,
-    108,101,45,6,0,0,115,34,0,0,0,0,2,10,1,10,
-    1,4,1,4,1,8,1,8,1,12,2,10,1,4,1,14,
-    1,2,1,8,1,8,1,8,1,12,1,14,2,114,102,1,
-    0,0,99,0,0,0,0,0,0,0,0,0,0,0,0,3,
-    0,0,0,3,0,0,0,67,0,0,0,115,38,0,0,0,
-    116,0,116,1,160,2,161,0,102,2,125,0,116,3,116,4,
-    102,2,125,1,116,5,116,6,102,2,125,2,124,0,124,1,
-    124,2,103,3,83,0,41,1,122,95,82,101,116,117,114,110,
-    115,32,97,32,108,105,115,116,32,111,102,32,102,105,108,101,
-    45,98,97,115,101,100,32,109,111,100,117,108,101,32,108,111,
-    97,100,101,114,115,46,10,10,32,32,32,32,69,97,99,104,
-    32,105,116,101,109,32,105,115,32,97,32,116,117,112,108,101,
-    32,40,108,111,97,100,101,114,44,32,115,117,102,102,105,120,
-    101,115,41,46,10,32,32,32,32,41,7,114,251,0,0,0,
-    114,162,0,0,0,218,18,101,120,116,101,110,115,105,111,110,
-    95,115,117,102,102,105,120,101,115,114,8,1,0,0,114,101,
-    0,0,0,114,14,1,0,0,114,88,0,0,0,41,3,90,
-    10,101,120,116,101,110,115,105,111,110,115,90,6,115,111,117,
-    114,99,101,90,8,98,121,116,101,99,111,100,101,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,183,0,0,
-    0,68,6,0,0,115,8,0,0,0,0,5,12,1,8,1,
-    8,1,114,183,0,0,0,99,1,0,0,0,0,0,0,0,
-    0,0,0,0,12,0,0,0,9,0,0,0,67,0,0,0,
-    115,178,1,0,0,124,0,97,0,116,0,106,1,97,1,116,
-    0,106,2,97,2,116,1,106,3,116,4,25,0,125,1,100,
-    1,68,0,93,48,125,2,124,2,116,1,106,3,107,7,114,
-    56,116,0,160,5,124,2,161,1,125,3,110,10,116,1,106,
-    3,124,2,25,0,125,3,116,6,124,1,124,2,124,3,131,
-    3,1,0,113,30,100,2,100,3,103,1,102,2,100,4,100,
-    5,100,3,103,2,102,2,102,2,125,4,124,4,68,0,93,
-    110,92,2,125,5,125,6,116,7,100,6,100,7,132,0,124,
-    6,68,0,131,1,131,1,115,136,74,0,130,1,124,6,100,
-    8,25,0,125,7,124,5,116,1,106,3,107,6,114,170,116,
-    1,106,3,124,5,25,0,125,8,1,0,113,226,113,106,122,
-    20,116,0,160,5,124,5,161,1,125,8,87,0,1,0,113,
-    226,87,0,113,106,4,0,116,8,107,10,114,214,1,0,1,
-    0,1,0,89,0,113,106,89,0,113,106,88,0,113,106,116,
-    8,100,9,131,1,130,1,116,6,124,1,100,10,124,8,131,
-    3,1,0,116,6,124,1,100,11,124,7,131,3,1,0,116,
-    6,124,1,100,12,100,13,160,9,124,6,161,1,131,3,1,
-    0,116,6,124,1,100,14,100,15,100,16,132,0,124,6,68,
-    0,131,1,131,3,1,0,116,0,160,5,100,17,161,1,125,
-    9,116,6,124,1,100,17,124,9,131,3,1,0,116,0,160,
-    5,100,18,161,1,125,10,116,6,124,1,100,18,124,10,131,
-    3,1,0,124,5,100,4,107,2,144,1,114,110,116,0,160,
-    5,100,19,161,1,125,11,116,6,124,1,100,20,124,11,131,
-    3,1,0,116,6,124,1,100,21,116,10,131,0,131,3,1,
-    0,116,11,160,12,116,2,160,13,161,0,161,1,1,0,124,
-    5,100,4,107,2,144,1,114,174,116,14,160,15,100,22,161,
-    1,1,0,100,23,116,11,107,6,144,1,114,174,100,24,116,
-    16,95,17,100,25,83,0,41,26,122,205,83,101,116,117,112,
-    32,116,104,101,32,112,97,116,104,45,98,97,115,101,100,32,
-    105,109,112,111,114,116,101,114,115,32,102,111,114,32,105,109,
-    112,111,114,116,108,105,98,32,98,121,32,105,109,112,111,114,
-    116,105,110,103,32,110,101,101,100,101,100,10,32,32,32,32,
-    98,117,105,108,116,45,105,110,32,109,111,100,117,108,101,115,
-    32,97,110,100,32,105,110,106,101,99,116,105,110,103,32,116,
-    104,101,109,32,105,110,116,111,32,116,104,101,32,103,108,111,
-    98,97,108,32,110,97,109,101,115,112,97,99,101,46,10,10,
-    32,32,32,32,79,116,104,101,114,32,99,111,109,112,111,110,
-    101,110,116,115,32,97,114,101,32,101,120,116,114,97,99,116,
-    101,100,32,102,114,111,109,32,116,104,101,32,99,111,114,101,
-    32,98,111,111,116,115,116,114,97,112,32,109,111,100,117,108,
-    101,46,10,10,32,32,32,32,41,4,114,63,0,0,0,114,
-    74,0,0,0,218,8,98,117,105,108,116,105,110,115,114,159,
-    0,0,0,90,5,112,111,115,105,120,250,1,47,90,2,110,
-    116,250,1,92,99,1,0,0,0,0,0,0,0,0,0,0,
-    0,2,0,0,0,3,0,0,0,115,0,0,0,115,26,0,
-    0,0,124,0,93,18,125,1,116,0,124,1,131,1,100,0,
-    107,2,86,0,1,0,113,2,100,1,83,0,41,2,114,38,
-    0,0,0,78,41,1,114,22,0,0,0,41,2,114,31,0,
-    0,0,114,94,0,0,0,114,3,0,0,0,114,3,0,0,
-    0,114,6,0,0,0,114,18,1,0,0,104,6,0,0,115,
-    4,0,0,0,4,0,2,0,122,25,95,115,101,116,117,112,
-    46,60,108,111,99,97,108,115,62,46,60,103,101,110,101,120,
-    112,114,62,114,72,0,0,0,122,30,105,109,112,111,114,116,
-    108,105,98,32,114,101,113,117,105,114,101,115,32,112,111,115,
-    105,120,32,111,114,32,110,116,114,2,0,0,0,114,34,0,
-    0,0,114,30,0,0,0,114,39,0,0,0,114,57,0,0,
-    0,99,1,0,0,0,0,0,0,0,0,0,0,0,2,0,
-    0,0,4,0,0,0,83,0,0,0,115,22,0,0,0,104,
-    0,124,0,93,14,125,1,100,0,124,1,155,0,157,2,146,
-    2,113,4,83,0,41,1,114,73,0,0,0,114,3,0,0,
-    0,41,2,114,31,0,0,0,218,1,115,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,92,1,0,0,120,
-    6,0,0,115,4,0,0,0,6,0,2,0,122,25,95,115,
-    101,116,117,112,46,60,108,111,99,97,108,115,62,46,60,115,
-    101,116,99,111,109,112,62,90,7,95,116,104,114,101,97,100,
-    90,8,95,119,101,97,107,114,101,102,90,6,119,105,110,114,
-    101,103,114,191,0,0,0,114,7,0,0,0,122,4,46,112,
-    121,119,122,6,95,100,46,112,121,100,84,78,41,18,114,133,
-    0,0,0,114,8,0,0,0,114,162,0,0,0,114,30,1,
-    0,0,114,124,0,0,0,90,18,95,98,117,105,108,116,105,
-    110,95,102,114,111,109,95,110,97,109,101,114,128,0,0,0,
-    218,3,97,108,108,114,117,0,0,0,114,35,0,0,0,114,
-    13,0,0,0,114,20,1,0,0,114,166,0,0,0,114,103,
-    1,0,0,114,101,0,0,0,114,185,0,0,0,114,190,0,
-    0,0,114,194,0,0,0,41,12,218,17,95,98,111,111,116,
-    115,116,114,97,112,95,109,111,100,117,108,101,90,11,115,101,
-    108,102,95,109,111,100,117,108,101,90,12,98,117,105,108,116,
-    105,110,95,110,97,109,101,90,14,98,117,105,108,116,105,110,
-    95,109,111,100,117,108,101,90,10,111,115,95,100,101,116,97,
-    105,108,115,90,10,98,117,105,108,116,105,110,95,111,115,114,
-    30,0,0,0,114,34,0,0,0,90,9,111,115,95,109,111,
-    100,117,108,101,90,13,116,104,114,101,97,100,95,109,111,100,
-    117,108,101,90,14,119,101,97,107,114,101,102,95,109,111,100,
-    117,108,101,90,13,119,105,110,114,101,103,95,109,111,100,117,
-    108,101,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
-    0,218,6,95,115,101,116,117,112,79,6,0,0,115,78,0,
-    0,0,0,8,4,1,6,1,6,3,10,1,8,1,10,1,
-    12,2,10,1,14,3,22,1,12,2,22,1,8,1,10,1,
-    10,1,6,2,2,1,10,1,10,1,14,1,12,2,8,1,
-    12,1,12,1,18,1,22,3,10,1,12,3,10,1,12,3,
-    10,1,10,1,12,3,14,1,14,1,10,1,10,1,10,1,
-    114,110,1,0,0,99,1,0,0,0,0,0,0,0,0,0,
-    0,0,2,0,0,0,4,0,0,0,67,0,0,0,115,50,
-    0,0,0,116,0,124,0,131,1,1,0,116,1,131,0,125,
-    1,116,2,106,3,160,4,116,5,106,6,124,1,142,0,103,
-    1,161,1,1,0,116,2,106,7,160,8,116,9,161,1,1,
-    0,100,1,83,0,41,2,122,41,73,110,115,116,97,108,108,
-    32,116,104,101,32,112,97,116,104,45,98,97,115,101,100,32,
-    105,109,112,111,114,116,32,99,111,109,112,111,110,101,110,116,
-    115,46,78,41,10,114,110,1,0,0,114,183,0,0,0,114,
-    8,0,0,0,114,50,1,0,0,114,166,0,0,0,114,83,
-    1,0,0,114,98,1,0,0,218,9,109,101,116,97,95,112,
-    97,116,104,114,185,0,0,0,114,44,1,0,0,41,2,114,
-    109,1,0,0,90,17,115,117,112,112,111,114,116,101,100,95,
-    108,111,97,100,101,114,115,114,3,0,0,0,114,3,0,0,
-    0,114,6,0,0,0,218,8,95,105,110,115,116,97,108,108,
-    144,6,0,0,115,8,0,0,0,0,2,8,1,6,1,20,
-    1,114,112,1,0,0,41,1,114,59,0,0,0,41,1,78,
-    41,3,78,78,78,41,2,114,72,0,0,0,114,72,0,0,
-    0,41,1,84,41,1,78,41,1,78,41,63,114,126,0,0,
-    0,114,12,0,0,0,90,37,95,67,65,83,69,95,73,78,
-    83,69,78,83,73,84,73,86,69,95,80,76,65,84,70,79,
-    82,77,83,95,66,89,84,69,83,95,75,69,89,114,11,0,
-    0,0,114,13,0,0,0,114,20,0,0,0,114,26,0,0,
-    0,114,28,0,0,0,114,37,0,0,0,114,46,0,0,0,
-    114,48,0,0,0,114,52,0,0,0,114,53,0,0,0,114,
-    55,0,0,0,114,58,0,0,0,114,68,0,0,0,218,4,
-    116,121,112,101,218,8,95,95,99,111,100,101,95,95,114,161,
-    0,0,0,114,18,0,0,0,114,147,0,0,0,114,17,0,
-    0,0,114,23,0,0,0,114,235,0,0,0,114,91,0,0,
-    0,114,87,0,0,0,114,101,0,0,0,114,88,0,0,0,
-    90,23,68,69,66,85,71,95,66,89,84,69,67,79,68,69,
-    95,83,85,70,70,73,88,69,83,90,27,79,80,84,73,77,
-    73,90,69,68,95,66,89,84,69,67,79,68,69,95,83,85,
-    70,70,73,88,69,83,114,97,0,0,0,114,102,0,0,0,
-    114,108,0,0,0,114,112,0,0,0,114,114,0,0,0,114,
-    135,0,0,0,114,142,0,0,0,114,151,0,0,0,114,155,
-    0,0,0,114,157,0,0,0,114,164,0,0,0,114,169,0,
-    0,0,114,170,0,0,0,114,175,0,0,0,218,6,111,98,
-    106,101,99,116,114,184,0,0,0,114,189,0,0,0,114,190,
-    0,0,0,114,207,0,0,0,114,220,0,0,0,114,238,0,
-    0,0,114,8,1,0,0,114,14,1,0,0,114,20,1,0,
-    0,114,251,0,0,0,114,21,1,0,0,114,42,1,0,0,
-    114,44,1,0,0,114,83,1,0,0,114,102,1,0,0,114,
-    183,0,0,0,114,110,1,0,0,114,112,1,0,0,114,3,
-    0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,218,8,60,109,111,100,117,108,101,62,1,0,0,0,
-    115,126,0,0,0,4,22,4,1,4,1,2,1,2,255,4,
-    4,8,17,8,5,8,5,8,6,8,6,8,12,8,10,8,
-    9,8,5,8,7,8,9,10,22,10,127,0,14,16,1,12,
-    2,4,1,4,2,6,2,6,2,8,2,16,71,8,40,8,
-    19,8,12,8,12,8,28,8,17,8,33,8,28,8,24,10,
-    13,10,10,10,11,8,14,6,3,4,1,2,255,12,68,14,
-    64,14,29,16,127,0,17,14,72,18,45,18,26,4,3,18,
-    53,14,63,14,42,14,127,0,66,14,127,0,22,10,23,8,
-    11,8,65,
+    114,184,0,0,0,22,6,0,0,115,8,0,0,0,0,5,
+    12,1,8,1,8,1,114,184,0,0,0,99,1,0,0,0,
+    0,0,0,0,0,0,0,0,12,0,0,0,9,0,0,0,
+    67,0,0,0,115,178,1,0,0,124,0,97,0,116,0,106,
+    1,97,1,116,0,106,2,97,2,116,1,106,3,116,4,25,
+    0,125,1,100,1,68,0,93,48,125,2,124,2,116,1,106,
+    3,107,7,114,56,116,0,160,5,124,2,161,1,125,3,110,
+    10,116,1,106,3,124,2,25,0,125,3,116,6,124,1,124,
+    2,124,3,131,3,1,0,113,30,100,2,100,3,103,1,102,
+    2,100,4,100,5,100,3,103,2,102,2,102,2,125,4,124,
+    4,68,0,93,110,92,2,125,5,125,6,116,7,100,6,100,
+    7,132,0,124,6,68,0,131,1,131,1,115,136,74,0,130,
+    1,124,6,100,8,25,0,125,7,124,5,116,1,106,3,107,
+    6,114,170,116,1,106,3,124,5,25,0,125,8,1,0,113,
+    226,113,106,122,20,116,0,160,5,124,5,161,1,125,8,87,
+    0,1,0,113,226,87,0,113,106,4,0,116,8,107,10,114,
+    214,1,0,1,0,1,0,89,0,113,106,89,0,113,106,88,
+    0,113,106,116,8,100,9,131,1,130,1,116,6,124,1,100,
+    10,124,8,131,3,1,0,116,6,124,1,100,11,124,7,131,
+    3,1,0,116,6,124,1,100,12,100,13,160,9,124,6,161,
+    1,131,3,1,0,116,6,124,1,100,14,100,15,100,16,132,
+    0,124,6,68,0,131,1,131,3,1,0,116,0,160,5,100,
+    17,161,1,125,9,116,6,124,1,100,17,124,9,131,3,1,
+    0,116,0,160,5,100,18,161,1,125,10,116,6,124,1,100,
+    18,124,10,131,3,1,0,124,5,100,4,107,2,144,1,114,
+    110,116,0,160,5,100,19,161,1,125,11,116,6,124,1,100,
+    20,124,11,131,3,1,0,116,6,124,1,100,21,116,10,131,
+    0,131,3,1,0,116,11,160,12,116,2,160,13,161,0,161,
+    1,1,0,124,5,100,4,107,2,144,1,114,174,116,14,160,
+    15,100,22,161,1,1,0,100,23,116,11,107,6,144,1,114,
+    174,100,24,116,16,95,17,100,25,83,0,41,26,122,205,83,
+    101,116,117,112,32,116,104,101,32,112,97,116,104,45,98,97,
+    115,101,100,32,105,109,112,111,114,116,101,114,115,32,102,111,
+    114,32,105,109,112,111,114,116,108,105,98,32,98,121,32,105,
+    109,112,111,114,116,105,110,103,32,110,101,101,100,101,100,10,
+    32,32,32,32,98,117,105,108,116,45,105,110,32,109,111,100,
+    117,108,101,115,32,97,110,100,32,105,110,106,101,99,116,105,
+    110,103,32,116,104,101,109,32,105,110,116,111,32,116,104,101,
+    32,103,108,111,98,97,108,32,110,97,109,101,115,112,97,99,
+    101,46,10,10,32,32,32,32,79,116,104,101,114,32,99,111,
+    109,112,111,110,101,110,116,115,32,97,114,101,32,101,120,116,
+    114,97,99,116,101,100,32,102,114,111,109,32,116,104,101,32,
+    99,111,114,101,32,98,111,111,116,115,116,114,97,112,32,109,
+    111,100,117,108,101,46,10,10,32,32,32,32,41,4,114,63,
+    0,0,0,114,74,0,0,0,218,8,98,117,105,108,116,105,
+    110,115,114,160,0,0,0,90,5,112,111,115,105,120,250,1,
+    47,90,2,110,116,250,1,92,99,1,0,0,0,0,0,0,
+    0,0,0,0,0,2,0,0,0,3,0,0,0,115,0,0,
+    0,115,26,0,0,0,124,0,93,18,125,1,116,0,124,1,
+    131,1,100,0,107,2,86,0,1,0,113,2,100,1,83,0,
+    41,2,114,38,0,0,0,78,41,1,114,22,0,0,0,41,
+    2,114,31,0,0,0,114,94,0,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,19,1,0,0,58,
+    6,0,0,115,4,0,0,0,4,0,2,0,122,25,95,115,
+    101,116,117,112,46,60,108,111,99,97,108,115,62,46,60,103,
+    101,110,101,120,112,114,62,114,72,0,0,0,122,30,105,109,
+    112,111,114,116,108,105,98,32,114,101,113,117,105,114,101,115,
+    32,112,111,115,105,120,32,111,114,32,110,116,114,2,0,0,
+    0,114,34,0,0,0,114,30,0,0,0,114,39,0,0,0,
+    114,57,0,0,0,99,1,0,0,0,0,0,0,0,0,0,
+    0,0,2,0,0,0,4,0,0,0,83,0,0,0,115,22,
+    0,0,0,104,0,124,0,93,14,125,1,100,0,124,1,155,
+    0,157,2,146,2,113,4,83,0,41,1,114,73,0,0,0,
+    114,3,0,0,0,41,2,114,31,0,0,0,218,1,115,114,
+    3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,70,
+    1,0,0,74,6,0,0,115,4,0,0,0,6,0,2,0,
+    122,25,95,115,101,116,117,112,46,60,108,111,99,97,108,115,
+    62,46,60,115,101,116,99,111,109,112,62,90,7,95,116,104,
+    114,101,97,100,90,8,95,119,101,97,107,114,101,102,90,6,
+    119,105,110,114,101,103,114,192,0,0,0,114,7,0,0,0,
+    122,4,46,112,121,119,122,6,95,100,46,112,121,100,84,78,
+    41,18,114,134,0,0,0,114,8,0,0,0,114,163,0,0,
+    0,114,31,1,0,0,114,125,0,0,0,90,18,95,98,117,
+    105,108,116,105,110,95,102,114,111,109,95,110,97,109,101,114,
+    129,0,0,0,218,3,97,108,108,114,117,0,0,0,114,35,
+    0,0,0,114,13,0,0,0,114,21,1,0,0,114,167,0,
+    0,0,114,82,1,0,0,114,101,0,0,0,114,186,0,0,
+    0,114,191,0,0,0,114,195,0,0,0,41,12,218,17,95,
+    98,111,111,116,115,116,114,97,112,95,109,111,100,117,108,101,
+    90,11,115,101,108,102,95,109,111,100,117,108,101,90,12,98,
+    117,105,108,116,105,110,95,110,97,109,101,90,14,98,117,105,
+    108,116,105,110,95,109,111,100,117,108,101,90,10,111,115,95,
+    100,101,116,97,105,108,115,90,10,98,117,105,108,116,105,110,
+    95,111,115,114,30,0,0,0,114,34,0,0,0,90,9,111,
+    115,95,109,111,100,117,108,101,90,13,116,104,114,101,97,100,
+    95,109,111,100,117,108,101,90,14,119,101,97,107,114,101,102,
+    95,109,111,100,117,108,101,90,13,119,105,110,114,101,103,95,
+    109,111,100,117,108,101,114,3,0,0,0,114,3,0,0,0,
+    114,6,0,0,0,218,6,95,115,101,116,117,112,33,6,0,
+    0,115,78,0,0,0,0,8,4,1,6,1,6,3,10,1,
+    8,1,10,1,12,2,10,1,14,3,22,1,12,2,22,1,
+    8,1,10,1,10,1,6,2,2,1,10,1,10,1,14,1,
+    12,2,8,1,12,1,12,1,18,1,22,3,10,1,12,3,
+    10,1,12,3,10,1,10,1,12,3,14,1,14,1,10,1,
+    10,1,10,1,114,89,1,0,0,99,1,0,0,0,0,0,
+    0,0,0,0,0,0,2,0,0,0,4,0,0,0,67,0,
+    0,0,115,50,0,0,0,116,0,124,0,131,1,1,0,116,
+    1,131,0,125,1,116,2,106,3,160,4,116,5,106,6,124,
+    1,142,0,103,1,161,1,1,0,116,2,106,7,160,8,116,
+    9,161,1,1,0,100,1,83,0,41,2,122,41,73,110,115,
+    116,97,108,108,32,116,104,101,32,112,97,116,104,45,98,97,
+    115,101,100,32,105,109,112,111,114,116,32,99,111,109,112,111,
+    110,101,110,116,115,46,78,41,10,114,89,1,0,0,114,184,
+    0,0,0,114,8,0,0,0,114,51,1,0,0,114,167,0,
+    0,0,114,61,1,0,0,114,76,1,0,0,218,9,109,101,
+    116,97,95,112,97,116,104,114,186,0,0,0,114,45,1,0,
+    0,41,2,114,88,1,0,0,90,17,115,117,112,112,111,114,
+    116,101,100,95,108,111,97,100,101,114,115,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,218,8,95,105,110,115,
+    116,97,108,108,98,6,0,0,115,8,0,0,0,0,2,8,
+    1,6,1,20,1,114,91,1,0,0,41,1,114,59,0,0,
+    0,41,1,78,41,3,78,78,78,41,2,114,72,0,0,0,
+    114,72,0,0,0,41,1,84,41,1,78,41,1,78,41,63,
+    114,127,0,0,0,114,12,0,0,0,90,37,95,67,65,83,
+    69,95,73,78,83,69,78,83,73,84,73,86,69,95,80,76,
+    65,84,70,79,82,77,83,95,66,89,84,69,83,95,75,69,
+    89,114,11,0,0,0,114,13,0,0,0,114,20,0,0,0,
+    114,26,0,0,0,114,28,0,0,0,114,37,0,0,0,114,
+    46,0,0,0,114,48,0,0,0,114,52,0,0,0,114,53,
+    0,0,0,114,55,0,0,0,114,58,0,0,0,114,68,0,
+    0,0,218,4,116,121,112,101,218,8,95,95,99,111,100,101,
+    95,95,114,162,0,0,0,114,18,0,0,0,114,148,0,0,
+    0,114,17,0,0,0,114,23,0,0,0,114,236,0,0,0,
+    114,91,0,0,0,114,87,0,0,0,114,101,0,0,0,114,
+    88,0,0,0,90,23,68,69,66,85,71,95,66,89,84,69,
+    67,79,68,69,95,83,85,70,70,73,88,69,83,90,27,79,
+    80,84,73,77,73,90,69,68,95,66,89,84,69,67,79,68,
+    69,95,83,85,70,70,73,88,69,83,114,97,0,0,0,114,
+    102,0,0,0,114,108,0,0,0,114,112,0,0,0,114,114,
+    0,0,0,114,136,0,0,0,114,143,0,0,0,114,152,0,
+    0,0,114,156,0,0,0,114,158,0,0,0,114,165,0,0,
+    0,114,170,0,0,0,114,171,0,0,0,114,176,0,0,0,
+    218,6,111,98,106,101,99,116,114,185,0,0,0,114,190,0,
+    0,0,114,191,0,0,0,114,208,0,0,0,114,221,0,0,
+    0,114,239,0,0,0,114,9,1,0,0,114,15,1,0,0,
+    114,21,1,0,0,114,252,0,0,0,114,22,1,0,0,114,
+    43,1,0,0,114,45,1,0,0,114,61,1,0,0,114,81,
+    1,0,0,114,184,0,0,0,114,89,1,0,0,114,91,1,
+    0,0,114,3,0,0,0,114,3,0,0,0,114,3,0,0,
+    0,114,6,0,0,0,218,8,60,109,111,100,117,108,101,62,
+    1,0,0,0,115,126,0,0,0,4,22,4,1,4,1,2,
+    1,2,255,4,4,8,17,8,5,8,5,8,6,8,6,8,
+    12,8,10,8,9,8,5,8,7,8,9,10,22,10,127,0,
+    14,16,1,12,2,4,1,4,2,6,2,6,2,8,2,16,
+    71,8,40,8,19,8,12,8,12,8,28,8,17,8,33,8,
+    28,8,24,10,13,10,10,10,11,8,14,6,3,4,1,2,
+    255,12,68,14,64,14,29,16,127,0,17,14,72,18,45,18,
+    26,4,3,18,53,14,63,14,42,14,127,0,20,14,127,0,
+    22,10,23,8,11,8,65,
 };


From webhook-mailer at python.org  Thu Sep 12 05:29:40 2019
From: webhook-mailer at python.org (Brett Cannon)
Date: Thu, 12 Sep 2019 09:29:40 -0000
Subject: [Python-checkins] Fix the ImportWarning regarding __spec__ and
 __package__ being None (GH-16003)
Message-ID: <mailman.630.1568280581.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6e1a30b15e73ebc82e6790495fd54cc8971723ec
commit: 6e1a30b15e73ebc82e6790495fd54cc8971723ec
branch: master
author: Xtreak <tir.karthi at gmail.com>
committer: Brett Cannon <54418+brettcannon at users.noreply.github.com>
date: 2019-09-12T10:29:36+01:00
summary:

Fix the ImportWarning regarding __spec__ and __package__ being None (GH-16003)

files:
M Lib/test/test_builtin.py

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 185d8f794b25..e1d2fffb36ea 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -161,9 +161,10 @@ def test_import(self):
         self.assertRaises(ValueError, __import__, '')
         self.assertRaises(TypeError, __import__, 'sys', name='sys')
         # Relative import outside of a package with no __package__ or __spec__ (bpo-37409).
-        self.assertRaises(ImportError, __import__, '',
-                          {'__package__': None, '__spec__': None, '__name__': '__main__'},
-                          locals={}, fromlist=('foo',), level=1)
+        with self.assertWarns(ImportWarning):
+            self.assertRaises(ImportError, __import__, '',
+                              {'__package__': None, '__spec__': None, '__name__': '__main__'},
+                              locals={}, fromlist=('foo',), level=1)
         # embedded null character
         self.assertRaises(ModuleNotFoundError, __import__, 'string\x00')
 


From webhook-mailer at python.org  Thu Sep 12 05:44:50 2019
From: webhook-mailer at python.org (Petr Viktorin)
Date: Thu, 12 Sep 2019 09:44:50 -0000
Subject: [Python-checkins] [3.8] bpo-37879: Suppress subtype_dealloc decref
 when base type is a C heap type (GH-15323, GH-16004) (GH-15966)
Message-ID: <mailman.631.1568281491.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3562ae25402aad36583bc27d4d82c67554323d5e
commit: 3562ae25402aad36583bc27d4d82c67554323d5e
branch: 3.8
author: Petr Viktorin <encukou at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T10:44:46+01:00
summary:

[3.8] bpo-37879: Suppress subtype_dealloc decref when base type is a C heap type (GH-15323, GH-16004) (GH-15966)

The instance destructor for a type is responsible for preparing
an instance for deallocation by decrementing the reference counts
of its referents.

If an instance belongs to a heap type, the type object of an instance
has its reference count decremented while for static types, which
are permanently allocated, the type object is unaffected by the
instance destructor.

Previously, the default instance destructor searched the class
hierarchy for an inherited instance destructor and, if present,
would invoke it.

Then, if the instance type is a heap type, it would decrement the
reference count of that heap type.  However, this could result in the
premature destruction of a type because the inherited instance
destructor should have already decremented the reference count
of the type object.

This change avoids the premature destruction of the type object
by suppressing the decrement of its reference count when an
inherited, non-default instance destructor has been invoked.

Finally, an assertion on the Py_SIZE of a type was deleted.  Heap
types have a non zero size, making this into an incorrect assertion.

https://github.com/python/cpython/pull/15323.
(cherry picked from commit ff023ed36ea260ab64be5895f1f1f087c798987a)
Fixup: https://github.com/python/cpython/pull/16004.
(cherry picked from commit 5e9caeec76119a0d61c25f1466c27b7dbd5115bd)

Co-authored-by: Eddie Elizondo <eduardo.elizondorueda at gmail.com>

files:
A Misc/NEWS.d/next/C API/2019-08-17-13-50-21.bpo-37879.CZeUem.rst
M Lib/test/test_capi.py
M Modules/_testcapimodule.c
M Objects/typeobject.c

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 7b35ba60b53a..4d6e2f21551a 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -383,6 +383,92 @@ def __del__(self):
             del L
             self.assertEqual(PyList.num, 0)
 
+    def test_subclass_of_heap_gc_ctype_with_tpdealloc_decrefs_once(self):
+        class HeapGcCTypeSubclass(_testcapi.HeapGcCType):
+            def __init__(self):
+                self.value2 = 20
+                super().__init__()
+
+        subclass_instance = HeapGcCTypeSubclass()
+        type_refcnt = sys.getrefcount(HeapGcCTypeSubclass)
+
+        # Test that subclass instance was fully created
+        self.assertEqual(subclass_instance.value, 10)
+        self.assertEqual(subclass_instance.value2, 20)
+
+        # Test that the type reference count is only decremented once
+        del subclass_instance
+        self.assertEqual(type_refcnt - 1, sys.getrefcount(HeapGcCTypeSubclass))
+
+    def test_subclass_of_heap_gc_ctype_with_del_modifying_dunder_class_only_decrefs_once(self):
+        class A(_testcapi.HeapGcCType):
+            def __init__(self):
+                self.value2 = 20
+                super().__init__()
+
+        class B(A):
+            def __init__(self):
+                super().__init__()
+
+            def __del__(self):
+                self.__class__ = A
+                A.refcnt_in_del = sys.getrefcount(A)
+                B.refcnt_in_del = sys.getrefcount(B)
+
+        subclass_instance = B()
+        type_refcnt = sys.getrefcount(B)
+        new_type_refcnt = sys.getrefcount(A)
+
+        # Test that subclass instance was fully created
+        self.assertEqual(subclass_instance.value, 10)
+        self.assertEqual(subclass_instance.value2, 20)
+
+        del subclass_instance
+
+        # Test that setting __class__ modified the reference counts of the types
+        self.assertEqual(type_refcnt - 1, B.refcnt_in_del)
+        self.assertEqual(new_type_refcnt + 1, A.refcnt_in_del)
+
+        # Test that the original type already has decreased its refcnt
+        self.assertEqual(type_refcnt - 1, sys.getrefcount(B))
+
+        # Test that subtype_dealloc decref the newly assigned __class__ only once
+        self.assertEqual(new_type_refcnt, sys.getrefcount(A))
+
+    def test_c_subclass_of_heap_ctype_with_tpdealloc_decrefs_once(self):
+        subclass_instance = _testcapi.HeapCTypeSubclass()
+        type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass)
+
+        # Test that subclass instance was fully created
+        self.assertEqual(subclass_instance.value, 10)
+        self.assertEqual(subclass_instance.value2, 20)
+
+        # Test that the type reference count is only decremented once
+        del subclass_instance
+        self.assertEqual(type_refcnt - 1, sys.getrefcount(_testcapi.HeapCTypeSubclass))
+
+    def test_c_subclass_of_heap_ctype_with_del_modifying_dunder_class_only_decrefs_once(self):
+        subclass_instance = _testcapi.HeapCTypeSubclassWithFinalizer()
+        type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclassWithFinalizer)
+        new_type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass)
+
+        # Test that subclass instance was fully created
+        self.assertEqual(subclass_instance.value, 10)
+        self.assertEqual(subclass_instance.value2, 20)
+
+        # The tp_finalize slot will set __class__ to HeapCTypeSubclass
+        del subclass_instance
+
+        # Test that setting __class__ modified the reference counts of the types
+        self.assertEqual(type_refcnt - 1, _testcapi.HeapCTypeSubclassWithFinalizer.refcnt_in_del)
+        self.assertEqual(new_type_refcnt + 1, _testcapi.HeapCTypeSubclass.refcnt_in_del)
+
+        # Test that the original type already has decreased its refcnt
+        self.assertEqual(type_refcnt - 1, sys.getrefcount(_testcapi.HeapCTypeSubclassWithFinalizer))
+
+        # Test that subtype_dealloc decref the newly assigned __class__ only once
+        self.assertEqual(new_type_refcnt, sys.getrefcount(_testcapi.HeapCTypeSubclass))
+
 
 class TestPendingCalls(unittest.TestCase):
 
diff --git a/Misc/NEWS.d/next/C API/2019-08-17-13-50-21.bpo-37879.CZeUem.rst b/Misc/NEWS.d/next/C API/2019-08-17-13-50-21.bpo-37879.CZeUem.rst
new file mode 100644
index 000000000000..87322fbf5f23
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-08-17-13-50-21.bpo-37879.CZeUem.rst	
@@ -0,0 +1,2 @@
+Fix subtype_dealloc to suppress the type decref when the base type is a C
+heap type
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 15bfe26a044f..cb9206ca4eba 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -32,6 +32,8 @@
 #  error "_testcapi must test the public Python C API, not CPython internal C API"
 #endif
 
+static struct PyModuleDef _testcapimodule;
+
 static PyObject *TestError;     /* set to exception object in init */
 
 /* Raise TestError with test_name + ": " + msg, and return NULL. */
@@ -5955,6 +5957,190 @@ static PyTypeObject MethodDescriptor2_Type = {
     .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | _Py_TPFLAGS_HAVE_VECTORCALL,
 };
 
+PyDoc_STRVAR(heapgctype__doc__,
+"A heap type with GC, and with overridden dealloc.\n\n"
+"The 'value' attribute is set to 10 in __init__.");
+
+typedef struct {
+    PyObject_HEAD
+    int value;
+} HeapCTypeObject;
+
+static struct PyMemberDef heapctype_members[] = {
+    {"value", T_INT, offsetof(HeapCTypeObject, value)},
+    {NULL} /* Sentinel */
+};
+
+static int
+heapctype_init(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+    ((HeapCTypeObject *)self)->value = 10;
+    return 0;
+}
+
+static void
+heapgcctype_dealloc(HeapCTypeObject *self)
+{
+    PyTypeObject *tp = Py_TYPE(self);
+    PyObject_GC_UnTrack(self);
+    PyObject_GC_Del(self);
+    Py_DECREF(tp);
+}
+
+static PyType_Slot HeapGcCType_slots[] = {
+    {Py_tp_init, heapctype_init},
+    {Py_tp_members, heapctype_members},
+    {Py_tp_dealloc, heapgcctype_dealloc},
+    {Py_tp_doc, (char*)heapgctype__doc__},
+    {0, 0},
+};
+
+static PyType_Spec HeapGcCType_spec = {
+    "_testcapi.HeapGcCType",
+    sizeof(HeapCTypeObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+    HeapGcCType_slots
+};
+
+PyDoc_STRVAR(heapctype__doc__,
+"A heap type without GC, but with overridden dealloc.\n\n"
+"The 'value' attribute is set to 10 in __init__.");
+
+static void
+heapctype_dealloc(HeapCTypeObject *self)
+{
+    PyTypeObject *tp = Py_TYPE(self);
+    PyObject_Del(self);
+    Py_DECREF(tp);
+}
+
+static PyType_Slot HeapCType_slots[] = {
+    {Py_tp_init, heapctype_init},
+    {Py_tp_members, heapctype_members},
+    {Py_tp_dealloc, heapctype_dealloc},
+    {Py_tp_doc, (char*)heapctype__doc__},
+    {0, 0},
+};
+
+static PyType_Spec HeapCType_spec = {
+    "_testcapi.HeapCType",
+    sizeof(HeapCTypeObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    HeapCType_slots
+};
+
+PyDoc_STRVAR(heapctypesubclass__doc__,
+"Subclass of HeapCType, without GC.\n\n"
+"__init__ sets the 'value' attribute to 10 and 'value2' to 20.");
+
+typedef struct {
+    HeapCTypeObject base;
+    int value2;
+} HeapCTypeSubclassObject;
+
+static int
+heapctypesubclass_init(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+    /* Call __init__ of the superclass */
+    if (heapctype_init(self, args, kwargs) < 0) {
+        return -1;
+    }
+    /* Initialize additional element */
+    ((HeapCTypeSubclassObject *)self)->value2 = 20;
+    return 0;
+}
+
+static struct PyMemberDef heapctypesubclass_members[] = {
+    {"value2", T_INT, offsetof(HeapCTypeSubclassObject, value2)},
+    {NULL} /* Sentinel */
+};
+
+static PyType_Slot HeapCTypeSubclass_slots[] = {
+    {Py_tp_init, heapctypesubclass_init},
+    {Py_tp_members, heapctypesubclass_members},
+    {Py_tp_doc, (char*)heapctypesubclass__doc__},
+    {0, 0},
+};
+
+static PyType_Spec HeapCTypeSubclass_spec = {
+    "_testcapi.HeapCTypeSubclass",
+    sizeof(HeapCTypeSubclassObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    HeapCTypeSubclass_slots
+};
+
+PyDoc_STRVAR(heapctypesubclasswithfinalizer__doc__,
+"Subclass of HeapCType with a finalizer that reassigns __class__.\n\n"
+"__class__ is set to plain HeapCTypeSubclass during finalization.\n"
+"__init__ sets the 'value' attribute to 10 and 'value2' to 20.");
+
+static int
+heapctypesubclasswithfinalizer_init(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+    PyTypeObject *base = (PyTypeObject *)PyType_GetSlot(Py_TYPE(self), Py_tp_base);
+    initproc base_init = PyType_GetSlot(base, Py_tp_init);
+    base_init(self, args, kwargs);
+    return 0;
+}
+
+static void
+heapctypesubclasswithfinalizer_finalize(PyObject *self)
+{
+    PyObject *error_type, *error_value, *error_traceback, *m;
+    PyObject *oldtype = NULL, *newtype = NULL;
+
+    /* Save the current exception, if any. */
+    PyErr_Fetch(&error_type, &error_value, &error_traceback);
+
+    m = PyState_FindModule(&_testcapimodule);
+    if (m == NULL) {
+        goto cleanup_finalize;
+    }
+    oldtype = PyObject_GetAttrString(m, "HeapCTypeSubclassWithFinalizer");
+    newtype = PyObject_GetAttrString(m, "HeapCTypeSubclass");
+    if (oldtype == NULL || newtype == NULL) {
+        goto cleanup_finalize;
+    }
+
+    if (PyObject_SetAttrString(self, "__class__", newtype) < 0) {
+        goto cleanup_finalize;
+    }
+    if (PyObject_SetAttrString(
+        oldtype, "refcnt_in_del", PyLong_FromSsize_t(Py_REFCNT(oldtype))) < 0) {
+        goto cleanup_finalize;
+    }
+    if (PyObject_SetAttrString(
+        newtype, "refcnt_in_del", PyLong_FromSsize_t(Py_REFCNT(newtype))) < 0) {
+        goto cleanup_finalize;
+    }
+
+cleanup_finalize:
+    Py_XDECREF(oldtype);
+    Py_XDECREF(newtype);
+
+    /* Restore the saved exception. */
+    PyErr_Restore(error_type, error_value, error_traceback);
+}
+
+static PyType_Slot HeapCTypeSubclassWithFinalizer_slots[] = {
+    {Py_tp_init, heapctypesubclasswithfinalizer_init},
+    {Py_tp_members, heapctypesubclass_members},
+    {Py_tp_finalize, heapctypesubclasswithfinalizer_finalize},
+    {Py_tp_doc, (char*)heapctypesubclasswithfinalizer__doc__},
+    {0, 0},
+};
+
+static PyType_Spec HeapCTypeSubclassWithFinalizer_spec = {
+    "_testcapi.HeapCTypeSubclassWithFinalizer",
+    sizeof(HeapCTypeSubclassObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_FINALIZE,
+    HeapCTypeSubclassWithFinalizer_slots
+};
+
 
 static struct PyModuleDef _testcapimodule = {
     PyModuleDef_HEAD_INIT,
@@ -6089,5 +6275,40 @@ PyInit__testcapi(void)
     TestError = PyErr_NewException("_testcapi.error", NULL, NULL);
     Py_INCREF(TestError);
     PyModule_AddObject(m, "error", TestError);
+
+    PyObject *HeapGcCType = PyType_FromSpec(&HeapGcCType_spec);
+    if (HeapGcCType == NULL) {
+        return NULL;
+    }
+    PyModule_AddObject(m, "HeapGcCType", HeapGcCType);
+
+    PyObject *HeapCType = PyType_FromSpec(&HeapCType_spec);
+    if (HeapCType == NULL) {
+        return NULL;
+    }
+    PyObject *subclass_bases = PyTuple_Pack(1, HeapCType);
+    if (subclass_bases == NULL) {
+        return NULL;
+    }
+    PyObject *HeapCTypeSubclass = PyType_FromSpecWithBases(&HeapCTypeSubclass_spec, subclass_bases);
+    if (HeapCTypeSubclass == NULL) {
+        return NULL;
+    }
+    Py_DECREF(subclass_bases);
+    PyModule_AddObject(m, "HeapCTypeSubclass", HeapCTypeSubclass);
+
+    PyObject *subclass_with_finalizer_bases = PyTuple_Pack(1, HeapCTypeSubclass);
+    if (subclass_with_finalizer_bases == NULL) {
+        return NULL;
+    }
+    PyObject *HeapCTypeSubclassWithFinalizer = PyType_FromSpecWithBases(
+        &HeapCTypeSubclassWithFinalizer_spec, subclass_with_finalizer_bases);
+    if (HeapCTypeSubclassWithFinalizer == NULL) {
+        return NULL;
+    }
+    Py_DECREF(subclass_with_finalizer_bases);
+    PyModule_AddObject(m, "HeapCTypeSubclassWithFinalizer", HeapCTypeSubclassWithFinalizer);
+
+    PyState_AddModule(m, &_testcapimodule);
     return m;
 }
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index f42caedf6cc2..e635b718d6a9 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1157,11 +1157,9 @@ subtype_dealloc(PyObject *self)
     /* Test whether the type has GC exactly once */
 
     if (!PyType_IS_GC(type)) {
-        /* It's really rare to find a dynamic type that doesn't have
-           GC; it can only happen when deriving from 'object' and not
-           adding any slots or instance variables.  This allows
-           certain simplifications: there's no need to call
-           clear_slots(), or DECREF the dict, or clear weakrefs. */
+        /* A non GC dynamic type allows certain simplifications:
+           there's no need to call clear_slots(), or DECREF the dict,
+           or clear weakrefs. */
 
         /* Maybe call finalizer; exit early if resurrected */
         if (type->tp_finalize) {
@@ -1177,7 +1175,6 @@ subtype_dealloc(PyObject *self)
         /* Find the nearest base with a different tp_dealloc */
         base = type;
         while ((basedealloc = base->tp_dealloc) == subtype_dealloc) {
-            assert(Py_SIZE(base) == 0);
             base = base->tp_base;
             assert(base);
         }
@@ -1189,8 +1186,10 @@ subtype_dealloc(PyObject *self)
         assert(basedealloc);
         basedealloc(self);
 
-        /* Can't reference self beyond this point */
-        Py_DECREF(type);
+       /* Only decref if the base type is not already a heap allocated type.
+          Otherwise, basedealloc should have decref'd it already */
+        if (type->tp_flags & Py_TPFLAGS_HEAPTYPE && !(base->tp_flags & Py_TPFLAGS_HEAPTYPE))
+            Py_DECREF(type);
 
         /* Done */
         return;
@@ -1289,8 +1288,9 @@ subtype_dealloc(PyObject *self)
 
     /* Can't reference self beyond this point. It's possible tp_del switched
        our type from a HEAPTYPE to a non-HEAPTYPE, so be careful about
-       reference counting. */
-    if (type->tp_flags & Py_TPFLAGS_HEAPTYPE)
+       reference counting. Only decref if the base type is not already a heap
+       allocated type. Otherwise, basedealloc should have decref'd it already */
+    if (type->tp_flags & Py_TPFLAGS_HEAPTYPE && !(base->tp_flags & Py_TPFLAGS_HEAPTYPE))
       Py_DECREF(type);
 
   endlabel:


From webhook-mailer at python.org  Thu Sep 12 05:51:04 2019
From: webhook-mailer at python.org (Eric Snow)
Date: Thu, 12 Sep 2019 09:51:04 -0000
Subject: [Python-checkins] bpo-36876: Skip test_check_c_globals for now.
 (gh-16017)
Message-ID: <mailman.632.1568281865.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/64535fc6c0712caef0bc46be30e661f7ccf8280e
commit: 64535fc6c0712caef0bc46be30e661f7ccf8280e
branch: master
author: Eric Snow <ericsnowcurrently at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T10:51:00+01:00
summary:

bpo-36876: Skip test_check_c_globals for now. (gh-16017)

files:
M Lib/test/test_check_c_globals.py

diff --git a/Lib/test/test_check_c_globals.py b/Lib/test/test_check_c_globals.py
index 009560e8d98d..a3925f0ca887 100644
--- a/Lib/test/test_check_c_globals.py
+++ b/Lib/test/test_check_c_globals.py
@@ -9,7 +9,9 @@
 class ActualChecks(unittest.TestCase):
 
     # XXX Also run the check in "make check".
-    @unittest.expectedFailure
+    #@unittest.expectedFailure
+    # Failing on one of the buildbots (see https://bugs.python.org/issue36876).
+    @unittest.skip('activate this once all the globals have been resolved')
     def test_check_c_globals(self):
         try:
             main('check', {})


From webhook-mailer at python.org  Thu Sep 12 06:02:53 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 12 Sep 2019 10:02:53 -0000
Subject: [Python-checkins] closes bpo-37405: Make socket.getsockname()
 always return a tuple for AF_CAN. (GH-14392)
Message-ID: <mailman.633.1568282574.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/954900a3f98a8c0dea14dd575490237f3f8626b3
commit: 954900a3f98a8c0dea14dd575490237f3f8626b3
branch: master
author: bggardner <brent at ebrent.net>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-12T11:02:48+01:00
summary:

closes bpo-37405: Make socket.getsockname() always return a tuple for AF_CAN. (GH-14392)

This fixes a regression from 3.5. In recent releases, `getsockname()` in the AF_CAN case has returned a string.

files:
A Misc/NEWS.d/next/Library/2019-09-11-20-27-41.bpo-37405.MG5xiY.rst
M Lib/test/test_socket.py
M Modules/socketmodule.c

diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
index b855c523f080..b74549024b78 100755
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -1948,7 +1948,9 @@ def testCreateBCMSocket(self):
 
     def testBindAny(self):
         with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s:
-            s.bind(('', ))
+            address = ('', )
+            s.bind(address)
+            self.assertEqual(s.getsockname(), address)
 
     def testTooLongInterfaceName(self):
         # most systems limit IFNAMSIZ to 16, take 1024 to be sure
diff --git a/Misc/NEWS.d/next/Library/2019-09-11-20-27-41.bpo-37405.MG5xiY.rst b/Misc/NEWS.d/next/Library/2019-09-11-20-27-41.bpo-37405.MG5xiY.rst
new file mode 100644
index 000000000000..09e109734221
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-11-20-27-41.bpo-37405.MG5xiY.rst
@@ -0,0 +1,2 @@
+Fixed regression bug for socket.getsockname() for non-CAN_ISOTP AF_CAN
+address family sockets by returning a 1-tuple instead of string.
diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c
index 3548b0c87875..c649fa3c982f 100644
--- a/Modules/socketmodule.c
+++ b/Modules/socketmodule.c
@@ -1563,7 +1563,7 @@ makesockaddr(SOCKET_T sockfd, struct sockaddr *addr, size_t addrlen, int proto)
 #endif /* CAN_ISOTP */
           default:
           {
-              return Py_BuildValue("O&", PyUnicode_DecodeFSDefault,
+              return Py_BuildValue("(O&)", PyUnicode_DecodeFSDefault,
                                         ifname);
           }
         }


From webhook-mailer at python.org  Thu Sep 12 06:09:36 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 12 Sep 2019 10:09:36 -0000
Subject: [Python-checkins] closes bpo-38127: _ctypes: PyObject_IsSubclass()
 should be checked for failure. (GH-16011)
Message-ID: <mailman.634.1568282977.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ea683deccc505a78bbbb1eb8c6a88b0835ad5151
commit: ea683deccc505a78bbbb1eb8c6a88b0835ad5151
branch: master
author: Zackery Spytz <zspytz at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-12T11:09:32+01:00
summary:

closes bpo-38127: _ctypes: PyObject_IsSubclass() should be checked for failure. (GH-16011)

An exception may occur during a PyObject_IsSubclass() call.

files:
M Modules/_ctypes/_ctypes.c

diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index d2f6391fa6db..16a0cfe8dd4d 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -1168,7 +1168,11 @@ PyCPointerType_from_param(PyObject *type, PyObject *value)
         */
         StgDictObject *v = PyObject_stgdict(value);
         assert(v); /* Cannot be NULL for pointer or array objects */
-        if (PyObject_IsSubclass(v->proto, typedict->proto)) {
+        int ret = PyObject_IsSubclass(v->proto, typedict->proto);
+        if (ret < 0) {
+            return NULL;
+        }
+        if (ret) {
             Py_INCREF(value);
             return value;
         }


From webhook-mailer at python.org  Thu Sep 12 06:13:55 2019
From: webhook-mailer at python.org (Ivan Levkivskyi)
Date: Thu, 12 Sep 2019 10:13:55 -0000
Subject: [Python-checkins] bpo-38008: Move builtin protocol whitelist to
 mapping instead of list (GH-15647)
Message-ID: <mailman.635.1568283235.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/692a0dc91597b7fb350383b633dc4d044cbd360e
commit: 692a0dc91597b7fb350383b633dc4d044cbd360e
branch: master
author: Divij Rajkumar <drajkuma1 at gmail.com>
committer: Ivan Levkivskyi <levkivskyi at gmail.com>
date: 2019-09-12T11:13:51+01:00
summary:

bpo-38008: Move builtin protocol whitelist to mapping instead of list (GH-15647)

Fixes https://bugs.python.org/issue38008

files:
A Misc/NEWS.d/next/Library/2019-09-12-10-47-34.bpo-38008.sH74Iy.rst
M Lib/test/test_typing.py
M Lib/typing.py

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index ba0800fae90e..3402996b2d2e 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -1381,6 +1381,14 @@ def close(self):
         self.assertIsSubclass(B, Custom)
         self.assertNotIsSubclass(A, Custom)
 
+    def test_builtin_protocol_whitelist(self):
+        with self.assertRaises(TypeError):
+            class CustomProtocol(TestCase, Protocol):
+                pass
+
+        class CustomContextManager(typing.ContextManager, Protocol):
+            pass
+
 class GenericTests(BaseTestCase):
 
     def test_basics(self):
diff --git a/Lib/typing.py b/Lib/typing.py
index 5f1a0ad3d637..7b07112c3abf 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -989,10 +989,13 @@ def _allow_reckless_class_cheks():
         return True
 
 
-_PROTO_WHITELIST = ['Callable', 'Awaitable',
-                    'Iterable', 'Iterator', 'AsyncIterable', 'AsyncIterator',
-                    'Hashable', 'Sized', 'Container', 'Collection', 'Reversible',
-                    'ContextManager', 'AsyncContextManager']
+_PROTO_WHITELIST = {
+    'collections.abc': [
+        'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable',
+        'Hashable', 'Sized', 'Container', 'Collection', 'Reversible',
+    ],
+    'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'],
+}
 
 
 class _ProtocolMeta(ABCMeta):
@@ -1105,7 +1108,8 @@ def _proto_hook(other):
         # ... otherwise check consistency of bases, and prohibit instantiation.
         for base in cls.__bases__:
             if not (base in (object, Generic) or
-                    base.__module__ == 'collections.abc' and base.__name__ in _PROTO_WHITELIST or
+                    base.__module__ in _PROTO_WHITELIST and
+                    base.__name__ in _PROTO_WHITELIST[base.__module__] or
                     issubclass(base, Generic) and base._is_protocol):
                 raise TypeError('Protocols can only inherit from other'
                                 ' protocols, got %r' % base)
diff --git a/Misc/NEWS.d/next/Library/2019-09-12-10-47-34.bpo-38008.sH74Iy.rst b/Misc/NEWS.d/next/Library/2019-09-12-10-47-34.bpo-38008.sH74Iy.rst
new file mode 100644
index 000000000000..f696707eadf7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-12-10-47-34.bpo-38008.sH74Iy.rst
@@ -0,0 +1,3 @@
+Fix parent class check in protocols to correctly identify the module that
+provides a builtin protocol, instead of assuming they all come from the
+:mod:`collections.abc` module


From webhook-mailer at python.org  Thu Sep 12 06:22:34 2019
From: webhook-mailer at python.org (Paul Ganssle)
Date: Thu, 12 Sep 2019 10:22:34 -0000
Subject: [Python-checkins] bpo-13927: time.ctime and time.asctime return
 string explantion (GH-11303)
Message-ID: <mailman.636.1568283755.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2d32bf1ef23c9e468b2e8afab3c24e7a2047ac36
commit: 2d32bf1ef23c9e468b2e8afab3c24e7a2047ac36
branch: master
author: Harmandeep Singh <harmandeep.singh1 at delhivery.com>
committer: Paul Ganssle <paul at ganssle.io>
date: 2019-09-12T11:22:30+01:00
summary:

bpo-13927: time.ctime and time.asctime return string explantion (GH-11303)

* bpo-13927: time.ctime and time.asctime return string explantion

* Add note explaining that time.ctime and time.asctime returns a space padded date value in case it contains a single digit date

* Reformat linebreaks

files:
M Doc/library/time.rst

diff --git a/Doc/library/time.rst b/Doc/library/time.rst
index 4faa0bbebc5b..65ab679669eb 100644
--- a/Doc/library/time.rst
+++ b/Doc/library/time.rst
@@ -127,9 +127,12 @@ Functions
 
    Convert a tuple or :class:`struct_time` representing a time as returned by
    :func:`gmtime` or :func:`localtime` to a string of the following
-   form: ``'Sun Jun 20 23:21:05 1993'``.  If *t* is not provided, the current time
-   as returned by :func:`localtime` is used. Locale information is not used by
-   :func:`asctime`.
+   form: ``'Sun Jun 20 23:21:05 1993'``. The day field is two characters long
+   and is space padded if the day is a single digit,
+   e.g.: ``'Wed Jun  9 04:26:40 1993'``.
+
+   If *t* is not provided, the current time as returned by :func:`localtime`
+   is used. Locale information is not used by :func:`asctime`.
 
    .. note::
 
@@ -203,10 +206,15 @@ Functions
 
 .. function:: ctime([secs])
 
-   Convert a time expressed in seconds since the epoch to a string representing
-   local time. If *secs* is not provided or :const:`None`, the current time as
-   returned by :func:`.time` is used.  ``ctime(secs)`` is equivalent to
-   ``asctime(localtime(secs))``. Locale information is not used by :func:`ctime`.
+   Convert a time expressed in seconds since the epoch to a string of a form:
+   ``'Sun Jun 20 23:21:05 1993'`` representing local time. The day field
+   is two characters long and is space padded if the day is a single digit,
+   e.g.: ``'Wed Jun  9 04:26:40 1993'``.
+
+   If *secs* is not provided or :const:`None`, the current time as
+   returned by :func:`.time` is used. ``ctime(secs)`` is equivalent to
+   ``asctime(localtime(secs))``. Locale information is not used by
+   :func:`ctime`.
 
 
 .. function:: get_clock_info(name)


From webhook-mailer at python.org  Thu Sep 12 06:28:09 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 10:28:09 -0000
Subject: [Python-checkins] closes bpo-38127: _ctypes: PyObject_IsSubclass()
 should be checked for failure. (GH-16011)
Message-ID: <mailman.637.1568284090.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/79cbaf50ac9530d507149402de5c84fa590d9cfb
commit: 79cbaf50ac9530d507149402de5c84fa590d9cfb
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T03:28:05-07:00
summary:

closes bpo-38127: _ctypes: PyObject_IsSubclass() should be checked for failure. (GH-16011)


An exception may occur during a PyObject_IsSubclass() call.
(cherry picked from commit ea683deccc505a78bbbb1eb8c6a88b0835ad5151)

Co-authored-by: Zackery Spytz <zspytz at gmail.com>

files:
M Modules/_ctypes/_ctypes.c

diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index 47288740063e..0f11e01e1110 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -1168,7 +1168,11 @@ PyCPointerType_from_param(PyObject *type, PyObject *value)
         */
         StgDictObject *v = PyObject_stgdict(value);
         assert(v); /* Cannot be NULL for pointer or array objects */
-        if (PyObject_IsSubclass(v->proto, typedict->proto)) {
+        int ret = PyObject_IsSubclass(v->proto, typedict->proto);
+        if (ret < 0) {
+            return NULL;
+        }
+        if (ret) {
             Py_INCREF(value);
             return value;
         }


From webhook-mailer at python.org  Thu Sep 12 06:30:58 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 10:30:58 -0000
Subject: [Python-checkins] closes bpo-38127: _ctypes: PyObject_IsSubclass()
 should be checked for failure. (GH-16011)
Message-ID: <mailman.638.1568284259.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/197ac1ad1c03243c647aa9100edbaa16bebbb7e9
commit: 197ac1ad1c03243c647aa9100edbaa16bebbb7e9
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T03:30:55-07:00
summary:

closes bpo-38127: _ctypes: PyObject_IsSubclass() should be checked for failure. (GH-16011)


An exception may occur during a PyObject_IsSubclass() call.
(cherry picked from commit ea683deccc505a78bbbb1eb8c6a88b0835ad5151)

Co-authored-by: Zackery Spytz <zspytz at gmail.com>

files:
M Modules/_ctypes/_ctypes.c

diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index 8008a481e275..f186e5082afb 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -1094,7 +1094,11 @@ PyCPointerType_from_param(PyObject *type, PyObject *value)
         */
         StgDictObject *v = PyObject_stgdict(value);
         assert(v); /* Cannot be NULL for pointer or array objects */
-        if (PyObject_IsSubclass(v->proto, typedict->proto)) {
+        int ret = PyObject_IsSubclass(v->proto, typedict->proto);
+        if (ret < 0) {
+            return NULL;
+        }
+        if (ret) {
             Py_INCREF(value);
             return value;
         }


From webhook-mailer at python.org  Thu Sep 12 06:31:47 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 10:31:47 -0000
Subject: [Python-checkins] bpo-13927: time.ctime and time.asctime return
 string explantion (GH-11303)
Message-ID: <mailman.639.1568284309.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1e17c4d414049ad2b5b6a0a53685efc7a550c5c5
commit: 1e17c4d414049ad2b5b6a0a53685efc7a550c5c5
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T03:31:43-07:00
summary:

bpo-13927: time.ctime and time.asctime return string explantion (GH-11303)


* bpo-13927: time.ctime and time.asctime return string explantion

* Add note explaining that time.ctime and time.asctime returns a space padded date value in case it contains a single digit date

* Reformat linebreaks
(cherry picked from commit 2d32bf1ef23c9e468b2e8afab3c24e7a2047ac36)

Co-authored-by: Harmandeep Singh <harmandeep.singh1 at delhivery.com>

files:
M Doc/library/time.rst

diff --git a/Doc/library/time.rst b/Doc/library/time.rst
index 4faa0bbebc5b..65ab679669eb 100644
--- a/Doc/library/time.rst
+++ b/Doc/library/time.rst
@@ -127,9 +127,12 @@ Functions
 
    Convert a tuple or :class:`struct_time` representing a time as returned by
    :func:`gmtime` or :func:`localtime` to a string of the following
-   form: ``'Sun Jun 20 23:21:05 1993'``.  If *t* is not provided, the current time
-   as returned by :func:`localtime` is used. Locale information is not used by
-   :func:`asctime`.
+   form: ``'Sun Jun 20 23:21:05 1993'``. The day field is two characters long
+   and is space padded if the day is a single digit,
+   e.g.: ``'Wed Jun  9 04:26:40 1993'``.
+
+   If *t* is not provided, the current time as returned by :func:`localtime`
+   is used. Locale information is not used by :func:`asctime`.
 
    .. note::
 
@@ -203,10 +206,15 @@ Functions
 
 .. function:: ctime([secs])
 
-   Convert a time expressed in seconds since the epoch to a string representing
-   local time. If *secs* is not provided or :const:`None`, the current time as
-   returned by :func:`.time` is used.  ``ctime(secs)`` is equivalent to
-   ``asctime(localtime(secs))``. Locale information is not used by :func:`ctime`.
+   Convert a time expressed in seconds since the epoch to a string of a form:
+   ``'Sun Jun 20 23:21:05 1993'`` representing local time. The day field
+   is two characters long and is space padded if the day is a single digit,
+   e.g.: ``'Wed Jun  9 04:26:40 1993'``.
+
+   If *secs* is not provided or :const:`None`, the current time as
+   returned by :func:`.time` is used. ``ctime(secs)`` is equivalent to
+   ``asctime(localtime(secs))``. Locale information is not used by
+   :func:`ctime`.
 
 
 .. function:: get_clock_info(name)


From webhook-mailer at python.org  Thu Sep 12 06:32:41 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 10:32:41 -0000
Subject: [Python-checkins] bpo-38008: Move builtin protocol whitelist to
 mapping instead of list (GH-15647)
Message-ID: <mailman.640.1568284362.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/52baf90a74f3a4573f4266bb4ca8569534db42a9
commit: 52baf90a74f3a4573f4266bb4ca8569534db42a9
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T03:32:36-07:00
summary:

bpo-38008: Move builtin protocol whitelist to mapping instead of list (GH-15647)


Fixes https://bugs.python.org/issue38008
(cherry picked from commit 692a0dc91597b7fb350383b633dc4d044cbd360e)

Co-authored-by: Divij Rajkumar <drajkuma1 at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-12-10-47-34.bpo-38008.sH74Iy.rst
M Lib/test/test_typing.py
M Lib/typing.py

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index ba0800fae90e..3402996b2d2e 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -1381,6 +1381,14 @@ def close(self):
         self.assertIsSubclass(B, Custom)
         self.assertNotIsSubclass(A, Custom)
 
+    def test_builtin_protocol_whitelist(self):
+        with self.assertRaises(TypeError):
+            class CustomProtocol(TestCase, Protocol):
+                pass
+
+        class CustomContextManager(typing.ContextManager, Protocol):
+            pass
+
 class GenericTests(BaseTestCase):
 
     def test_basics(self):
diff --git a/Lib/typing.py b/Lib/typing.py
index 5f1a0ad3d637..7b07112c3abf 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -989,10 +989,13 @@ def _allow_reckless_class_cheks():
         return True
 
 
-_PROTO_WHITELIST = ['Callable', 'Awaitable',
-                    'Iterable', 'Iterator', 'AsyncIterable', 'AsyncIterator',
-                    'Hashable', 'Sized', 'Container', 'Collection', 'Reversible',
-                    'ContextManager', 'AsyncContextManager']
+_PROTO_WHITELIST = {
+    'collections.abc': [
+        'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable',
+        'Hashable', 'Sized', 'Container', 'Collection', 'Reversible',
+    ],
+    'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'],
+}
 
 
 class _ProtocolMeta(ABCMeta):
@@ -1105,7 +1108,8 @@ def _proto_hook(other):
         # ... otherwise check consistency of bases, and prohibit instantiation.
         for base in cls.__bases__:
             if not (base in (object, Generic) or
-                    base.__module__ == 'collections.abc' and base.__name__ in _PROTO_WHITELIST or
+                    base.__module__ in _PROTO_WHITELIST and
+                    base.__name__ in _PROTO_WHITELIST[base.__module__] or
                     issubclass(base, Generic) and base._is_protocol):
                 raise TypeError('Protocols can only inherit from other'
                                 ' protocols, got %r' % base)
diff --git a/Misc/NEWS.d/next/Library/2019-09-12-10-47-34.bpo-38008.sH74Iy.rst b/Misc/NEWS.d/next/Library/2019-09-12-10-47-34.bpo-38008.sH74Iy.rst
new file mode 100644
index 000000000000..f696707eadf7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-12-10-47-34.bpo-38008.sH74Iy.rst
@@ -0,0 +1,3 @@
+Fix parent class check in protocols to correctly identify the module that
+provides a builtin protocol, instead of assuming they all come from the
+:mod:`collections.abc` module


From webhook-mailer at python.org  Thu Sep 12 06:34:32 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 12 Sep 2019 10:34:32 -0000
Subject: [Python-checkins] closes bpo-37405: Make socket.getsockname()
 always return a tuple for AF_CAN. (GH-14392) (GH-16018)
Message-ID: <mailman.641.1568284473.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f60fd95dcc189ace8c0a2177a394b9cc20389a1e
commit: f60fd95dcc189ace8c0a2177a394b9cc20389a1e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-12T11:34:28+01:00
summary:

closes bpo-37405: Make socket.getsockname() always return a tuple for AF_CAN. (GH-14392) (GH-16018)

This fixes a regression from 3.5. In recent releases, `getsockname()` in the AF_CAN case has returned a string.
(cherry picked from commit 954900a3f98a8c0dea14dd575490237f3f8626b3)

Co-authored-by: bggardner <brent at ebrent.net>

files:
A Misc/NEWS.d/next/Library/2019-09-11-20-27-41.bpo-37405.MG5xiY.rst
M Lib/test/test_socket.py
M Modules/socketmodule.c

diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
index c472be135e39..1bf562a03d55 100644
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -1897,7 +1897,9 @@ def testCreateBCMSocket(self):
 
     def testBindAny(self):
         with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s:
-            s.bind(('', ))
+            address = ('', )
+            s.bind(address)
+            self.assertEqual(s.getsockname(), address)
 
     def testTooLongInterfaceName(self):
         # most systems limit IFNAMSIZ to 16, take 1024 to be sure
diff --git a/Misc/NEWS.d/next/Library/2019-09-11-20-27-41.bpo-37405.MG5xiY.rst b/Misc/NEWS.d/next/Library/2019-09-11-20-27-41.bpo-37405.MG5xiY.rst
new file mode 100644
index 000000000000..09e109734221
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-11-20-27-41.bpo-37405.MG5xiY.rst
@@ -0,0 +1,2 @@
+Fixed regression bug for socket.getsockname() for non-CAN_ISOTP AF_CAN
+address family sockets by returning a 1-tuple instead of string.
diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c
index 910e2bdd1319..594a0d6efad1 100644
--- a/Modules/socketmodule.c
+++ b/Modules/socketmodule.c
@@ -1524,7 +1524,7 @@ makesockaddr(SOCKET_T sockfd, struct sockaddr *addr, size_t addrlen, int proto)
 #endif /* CAN_ISOTP */
           default:
           {
-              return Py_BuildValue("O&", PyUnicode_DecodeFSDefault,
+              return Py_BuildValue("(O&)", PyUnicode_DecodeFSDefault,
                                         ifname);
           }
         }


From webhook-mailer at python.org  Thu Sep 12 06:52:53 2019
From: webhook-mailer at python.org (Michael Foord)
Date: Thu, 12 Sep 2019 10:52:53 -0000
Subject: [Python-checkins] bpo-37972: unittest.mock._Call now passes on
 __getitem__ to the __getattr__ chaining so that call() can be subscriptable
 (GH-15565) (GH-15965)
Message-ID: <mailman.642.1568285574.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/db0d8a5b2c803d30d9df436e00b6627ec8e09a13
commit: db0d8a5b2c803d30d9df436e00b6627ec8e09a13
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Michael Foord <voidspace at users.noreply.github.com>
date: 2019-09-12T12:52:49+02:00
summary:

bpo-37972: unittest.mock._Call now passes on __getitem__ to the __getattr__ chaining so that call() can be subscriptable (GH-15565) (GH-15965)

* bpo-37972: unittest.mock._Call now passes on __getitem__ to the __getattr__ chaining so that call() can be subscriptable

* ?? Added by blurb_it.

* Update 2019-08-28-21-40-12.bpo-37972.kP-n4L.rst

added name of the contributor

* bpo-37972: made all dunder methods chainable for _Call

* bpo-37972: delegate only attributes of tuple instead to __getattr__
(cherry picked from commit 72c359912d36705a94fca8b63d80451905a14ae4)

Co-authored-by: blhsing <github at ydooby.com>

files:
A Misc/NEWS.d/next/Library/2019-08-28-21-40-12.bpo-37972.kP-n4L.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testhelpers.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 7bfbfc92eef9..cbc4d76b8513 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -2464,6 +2464,12 @@ def __getattr__(self, attr):
         return _Call(name=name, parent=self, from_kall=False)
 
 
+    def __getattribute__(self, attr):
+        if attr in tuple.__dict__:
+            raise AttributeError
+        return tuple.__getattribute__(self, attr)
+
+
     def count(self, /, *args, **kwargs):
         return self.__getattr__('count')(*args, **kwargs)
 
diff --git a/Lib/unittest/test/testmock/testhelpers.py b/Lib/unittest/test/testmock/testhelpers.py
index 301bca430c13..f3c7acb98c21 100644
--- a/Lib/unittest/test/testmock/testhelpers.py
+++ b/Lib/unittest/test/testmock/testhelpers.py
@@ -334,6 +334,26 @@ def test_call_with_name(self):
         self.assertEqual(_Call((('bar', 'barz'),),)[0], '')
         self.assertEqual(_Call((('bar', 'barz'), {'hello': 'world'}),)[0], '')
 
+    def test_dunder_call(self):
+        m = MagicMock()
+        m().foo()['bar']()
+        self.assertEqual(
+            m.mock_calls,
+            [call(), call().foo(), call().foo().__getitem__('bar'), call().foo().__getitem__()()]
+        )
+        m = MagicMock()
+        m().foo()['bar'] = 1
+        self.assertEqual(
+            m.mock_calls,
+            [call(), call().foo(), call().foo().__setitem__('bar', 1)]
+        )
+        m = MagicMock()
+        iter(m().foo())
+        self.assertEqual(
+            m.mock_calls,
+            [call(), call().foo(), call().foo().__iter__()]
+        )
+
 
 class SpecSignatureTest(unittest.TestCase):
 
diff --git a/Misc/NEWS.d/next/Library/2019-08-28-21-40-12.bpo-37972.kP-n4L.rst b/Misc/NEWS.d/next/Library/2019-08-28-21-40-12.bpo-37972.kP-n4L.rst
new file mode 100644
index 000000000000..73d9ef77762b
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-28-21-40-12.bpo-37972.kP-n4L.rst
@@ -0,0 +1,5 @@
+Subscripts to the `unittest.mock.call` objects now receive the same chaining mechanism as any other custom attributes, so that the following usage no longer raises a `TypeError`:
+
+    call().foo().__getitem__('bar')
+
+Patch by blhsing


From webhook-mailer at python.org  Thu Sep 12 06:56:09 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 10:56:09 -0000
Subject: [Python-checkins] bpo-9938: Add optional keyword argument
 exit_on_error to argparse.ArgumentParser (GH-15362)
Message-ID: <mailman.643.1568285770.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f545638b5701652ffbe1774989533cdf5bc6631e
commit: f545638b5701652ffbe1774989533cdf5bc6631e
branch: master
author: Hai Shi <shihai1992 at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-12T03:56:05-07:00
summary:

bpo-9938: Add optional keyword argument exit_on_error to argparse.ArgumentParser (GH-15362)



Co-Authored-by: Xuanji Li <xuanji at gmail.com>


https://bugs.python.org/issue9938



Automerge-Triggered-By: @matrixise

files:
A Misc/NEWS.d/next/Library/2019-08-21-16-38-56.bpo-9938.t3G7N9.rst
M Doc/library/argparse.rst
M Lib/argparse.py
M Lib/test/test_argparse.py

diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index 368b1cfebf05..c2cf7d320b3f 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -142,7 +142,7 @@ ArgumentParser objects
                           formatter_class=argparse.HelpFormatter, \
                           prefix_chars='-', fromfile_prefix_chars=None, \
                           argument_default=None, conflict_handler='error', \
-                          add_help=True, allow_abbrev=True)
+                          add_help=True, allow_abbrev=True, exit_on_error=True)
 
    Create a new :class:`ArgumentParser` object. All parameters should be passed
    as keyword arguments. Each parameter has its own more detailed description
@@ -179,6 +179,9 @@ ArgumentParser objects
    * allow_abbrev_ - Allows long options to be abbreviated if the
      abbreviation is unambiguous. (default: ``True``)
 
+   * exit_on_error_ - Determines whether or not ArgumentParser exits with
+     error info when an error occurs. (default: ``True``)
+
    .. versionchanged:: 3.5
       *allow_abbrev* parameter was added.
 
@@ -186,6 +189,9 @@ ArgumentParser objects
       In previous versions, *allow_abbrev* also disabled grouping of short
       flags such as ``-vv`` to mean ``-v -v``.
 
+   .. versionchanged:: 3.9
+      *exit_on_error* parameter was added.
+
 The following sections describe how each of these are used.
 
 
@@ -647,6 +653,28 @@ the help options::
      +h, ++help  show this help message and exit
 
 
+exit_on_error
+^^^^^^^^^^^^^
+
+Normally, when you pass an invalid argument list to the :meth:`~ArgumentParser.parse_args`
+method of an :class:`ArgumentParser`, it will exit with error info.
+
+If the user would like catch errors manually, the feature can be enable by setting
+``exit_on_error`` to ``False``::
+
+   >>> parser = argparse.ArgumentParser(exit_on_error=False)
+   >>> parser.add_argument('--integers', type=int)
+   _StoreAction(option_strings=['--integers'], dest='integers', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, help=None, metavar=None)
+   >>> try:
+   ...     parser.parse_args('--integers a'.split())
+   ... except argparse.ArgumentError:
+   ...     print('Catching an argumentError')
+   ...
+   Catching an argumentError
+
+.. versionadded:: 3.9
+
+
 The add_argument() method
 -------------------------
 
diff --git a/Lib/argparse.py b/Lib/argparse.py
index ddfd772c1126..2e46e76a4997 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -1630,6 +1630,8 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         - conflict_handler -- String indicating how to handle conflicts
         - add_help -- Add a -h/-help option
         - allow_abbrev -- Allow long options to be abbreviated unambiguously
+        - exit_on_error -- Determines whether or not ArgumentParser exits with
+            error info when an error occurs
     """
 
     def __init__(self,
@@ -1644,7 +1646,8 @@ def __init__(self,
                  argument_default=None,
                  conflict_handler='error',
                  add_help=True,
-                 allow_abbrev=True):
+                 allow_abbrev=True,
+                 exit_on_error=True):
 
         superinit = super(ArgumentParser, self).__init__
         superinit(description=description,
@@ -1663,6 +1666,7 @@ def __init__(self,
         self.fromfile_prefix_chars = fromfile_prefix_chars
         self.add_help = add_help
         self.allow_abbrev = allow_abbrev
+        self.exit_on_error = exit_on_error
 
         add_group = self.add_argument_group
         self._positionals = add_group(_('positional arguments'))
@@ -1793,15 +1797,19 @@ def parse_known_args(self, args=None, namespace=None):
                 setattr(namespace, dest, self._defaults[dest])
 
         # parse the arguments and exit if there are any errors
-        try:
+        if self.exit_on_error:
+            try:
+                namespace, args = self._parse_known_args(args, namespace)
+            except ArgumentError:
+                err = _sys.exc_info()[1]
+                self.error(str(err))
+        else:
             namespace, args = self._parse_known_args(args, namespace)
-            if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
-                args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
-                delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
-            return namespace, args
-        except ArgumentError:
-            err = _sys.exc_info()[1]
-            self.error(str(err))
+
+        if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
+            args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
+            delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
+        return namespace, args
 
     def _parse_known_args(self, arg_strings, namespace):
         # replace arg strings that are file references
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index 86ec6cca51ac..464db2906998 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -5262,6 +5262,21 @@ def test_help_with_metavar(self):
             '''))
 
 
+class TestExitOnError(TestCase):
+
+    def setUp(self):
+        self.parser = argparse.ArgumentParser(exit_on_error=False)
+        self.parser.add_argument('--integers', metavar='N', type=int)
+
+    def test_exit_on_error_with_good_args(self):
+        ns = self.parser.parse_args('--integers 4'.split())
+        self.assertEqual(ns, argparse.Namespace(integers=4))
+
+    def test_exit_on_error_with_bad_args(self):
+        with self.assertRaises(argparse.ArgumentError):
+            self.parser.parse_args('--integers a'.split())
+
+
 def test_main():
     support.run_unittest(__name__)
     # Remove global references to avoid looking like we have refleaks.
diff --git a/Misc/NEWS.d/next/Library/2019-08-21-16-38-56.bpo-9938.t3G7N9.rst b/Misc/NEWS.d/next/Library/2019-08-21-16-38-56.bpo-9938.t3G7N9.rst
new file mode 100644
index 000000000000..4cb89893159e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-21-16-38-56.bpo-9938.t3G7N9.rst
@@ -0,0 +1 @@
+Add optional keyword argument ``exit_on_error`` for :class:`ArgumentParser`.


From webhook-mailer at python.org  Thu Sep 12 07:19:25 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 11:19:25 -0000
Subject: [Python-checkins] bpo-38110: Use fdwalk for os.closerange() when
 available. (GH-15224)
Message-ID: <mailman.644.1568287166.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/84eb42e3c90c03f13262f19185b14566e10941ac
commit: 84eb42e3c90c03f13262f19185b14566e10941ac
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T04:19:21-07:00
summary:

bpo-38110: Use fdwalk for os.closerange() when available. (GH-15224)


Use fdwalk() on platforms that support it to implement os.closerange().
(cherry picked from commit e20134f889a0cfcc37a46979f31a1c98b800de07)

Co-authored-by: Jakub Kul?k <Kulikjak at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-11-14-49-20.bpo-38110.A19Y-q.rst
M Modules/posixmodule.c
M configure
M configure.ac
M pyconfig.h.in

diff --git a/Misc/NEWS.d/next/Library/2019-09-11-14-49-20.bpo-38110.A19Y-q.rst b/Misc/NEWS.d/next/Library/2019-09-11-14-49-20.bpo-38110.A19Y-q.rst
new file mode 100644
index 000000000000..7b94a2e90a50
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-11-14-49-20.bpo-38110.A19Y-q.rst
@@ -0,0 +1,2 @@
+The os.closewalk() implementation now uses the libc fdwalk() API on
+platforms where it is available.
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index cf6c00b7d6ae..a066eb1e66d4 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -8419,6 +8419,21 @@ os_close_impl(PyObject *module, int fd)
 }
 
 
+#ifdef HAVE_FDWALK
+static int
+_fdwalk_close_func(void *lohi, int fd)
+{
+    int lo = ((int *)lohi)[0];
+    int hi = ((int *)lohi)[1];
+
+    if (fd >= hi)
+        return 1;
+    else if (fd >= lo)
+        close(fd);
+    return 0;
+}
+#endif /* HAVE_FDWALK */
+
 /*[clinic input]
 os.closerange
 
@@ -8433,11 +8448,21 @@ static PyObject *
 os_closerange_impl(PyObject *module, int fd_low, int fd_high)
 /*[clinic end generated code: output=0ce5c20fcda681c2 input=5855a3d053ebd4ec]*/
 {
+#ifdef HAVE_FDWALK
+    int lohi[2];
+#else
     int i;
+#endif
     Py_BEGIN_ALLOW_THREADS
     _Py_BEGIN_SUPPRESS_IPH
+#ifdef HAVE_FDWALK
+    lohi[0] = Py_MAX(fd_low, 0);
+    lohi[1] = fd_high;
+    fdwalk(_fdwalk_close_func, lohi);
+#else
     for (i = Py_MAX(fd_low, 0); i < fd_high; i++)
         close(i);
+#endif
     _Py_END_SUPPRESS_IPH
     Py_END_ALLOW_THREADS
     Py_RETURN_NONE;
diff --git a/configure b/configure
index 59466c6123d6..4beaf7a9c5e8 100755
--- a/configure
+++ b/configure
@@ -11500,7 +11500,7 @@ fi
 for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
  clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \
  faccessat fchmod fchmodat fchown fchownat \
- fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
+ fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
  futimens futimes gai_strerror getentropy \
  getgrgid_r getgrnam_r \
  getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
diff --git a/configure.ac b/configure.ac
index 0689c70e1e99..f7e0d80f02e3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3538,7 +3538,7 @@ fi
 AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
  clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \
  faccessat fchmod fchmodat fchown fchownat \
- fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
+ fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
  futimens futimes gai_strerror getentropy \
  getgrgid_r getgrnam_r \
  getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
diff --git a/pyconfig.h.in b/pyconfig.h.in
index 840fbcdb2a6f..5f952d2cfc36 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -341,6 +341,9 @@
 /* Define to 1 if you have the `fdopendir' function. */
 #undef HAVE_FDOPENDIR
 
+/* Define to 1 if you have the `fdwalk' function. */
+#undef HAVE_FDWALK
+
 /* Define to 1 if you have the `fexecve' function. */
 #undef HAVE_FEXECVE
 


From webhook-mailer at python.org  Thu Sep 12 07:23:58 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Thu, 12 Sep 2019 11:23:58 -0000
Subject: [Python-checkins] bpo-36675: Remove obsolete code. (GH-16024)
Message-ID: <mailman.645.1568287438.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2c910c1e732c9a3ec4c67a7c43d789d6c729304a
commit: 2c910c1e732c9a3ec4c67a7c43d789d6c729304a
branch: master
author: Julien Palard <julien at palard.fr>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-12T12:23:53+01:00
summary:

bpo-36675: Remove obsolete code. (GH-16024)

Does no longer work since Sphinx moved the trim_doctest_flag option in
the configuration.

files:
M Doc/library/doctest.rst
M Doc/tools/extensions/pyspecific.py

diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst
index e7c0033eb6bc4..7fc471fe2ab7d 100644
--- a/Doc/library/doctest.rst
+++ b/Doc/library/doctest.rst
@@ -1,5 +1,3 @@
-:keepdoctest:
-
 :mod:`doctest` --- Test interactive Python examples
 ===================================================
 
diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index 4e2ad7fc2a1d1..bc51555fa0512 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -29,7 +29,6 @@
 from sphinx.locale import translators
 from sphinx.util import status_iterator, logging
 from sphinx.util.nodes import split_explicit_title
-from sphinx.writers.html import HTMLTranslator
 from sphinx.writers.text import TextWriter, TextTranslator
 from sphinx.writers.latex import LaTeXTranslator
 from sphinx.domains.python import PyModulelevel, PyClassmember
@@ -49,37 +48,6 @@
     Body.enum.converters['lowerroman'] = \
     Body.enum.converters['upperroman'] = lambda x: None
 
-# monkey-patch HTML and LaTeX translators to keep doctest blocks in the
-# doctest docs themselves
-orig_visit_literal_block = HTMLTranslator.visit_literal_block
-orig_depart_literal_block = LaTeXTranslator.depart_literal_block
-
-
-def new_visit_literal_block(self, node):
-    meta = self.builder.env.metadata[self.builder.current_docname]
-    old_trim_doctest_flags = self.highlighter.trim_doctest_flags
-    if 'keepdoctest' in meta:
-        self.highlighter.trim_doctest_flags = False
-    try:
-        orig_visit_literal_block(self, node)
-    finally:
-        self.highlighter.trim_doctest_flags = old_trim_doctest_flags
-
-
-def new_depart_literal_block(self, node):
-    meta = self.builder.env.metadata[self.curfilestack[-1]]
-    old_trim_doctest_flags = self.highlighter.trim_doctest_flags
-    if 'keepdoctest' in meta:
-        self.highlighter.trim_doctest_flags = False
-    try:
-        orig_depart_literal_block(self, node)
-    finally:
-        self.highlighter.trim_doctest_flags = old_trim_doctest_flags
-
-
-HTMLTranslator.visit_literal_block = new_visit_literal_block
-LaTeXTranslator.depart_literal_block = new_depart_literal_block
-
 
 # Support for marking up and linking to bugs.python.org issues
 


From webhook-mailer at python.org  Thu Sep 12 07:29:58 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Thu, 12 Sep 2019 11:29:58 -0000
Subject: [Python-checkins] bpo-35685: Add examples of
 unittest.mock.patch.dict usage (GH-11456)
Message-ID: <mailman.646.1568287799.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/31a82e25b6044a5b5ee25246bad3eb7b873cf5ec
commit: 31a82e25b6044a5b5ee25246bad3eb7b873cf5ec
branch: master
author: Emmanuel Arias <emmanuelarias30 at gmail.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-12T13:29:54+02:00
summary:

bpo-35685: Add examples of unittest.mock.patch.dict usage (GH-11456)

files:
M Doc/library/unittest.mock.rst

diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index 04ff8a61da3c..b446ddb3598f 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -1572,15 +1572,36 @@ patch.dict
     :func:`patch.dict` can also be called with arbitrary keyword arguments to set
     values in the dictionary.
 
-    :func:`patch.dict` can be used as a context manager, decorator or class
-    decorator. When used as a class decorator :func:`patch.dict` honours
-    ``patch.TEST_PREFIX`` for choosing which methods to wrap.
-
     .. versionchanged:: 3.8
 
         :func:`patch.dict` now returns the patched dictionary when used as a context
         manager.
 
+:func:`patch.dict` can be used as a context manager, decorator or class
+decorator:
+
+    >>> foo = {}
+    >>> @patch.dict(foo, {'newkey': 'newvalue'})
+    ... def test():
+    ...     assert foo == {'newkey': 'newvalue'}
+    >>> test()
+    >>> assert foo == {}
+
+When used as a class decorator :func:`patch.dict` honours
+``patch.TEST_PREFIX`` (default to ``'test'``) for choosing which methods to wrap:
+
+    >>> import os
+    >>> import unittest
+    >>> from unittest.mock import patch
+    >>> @patch.dict('os.environ', {'newkey': 'newvalue'})
+    ... class TestSample(unittest.TestCase):
+    ...     def test_sample(self):
+    ...         self.assertEqual(os.environ['newkey'], 'newvalue')
+
+If you want to use a different prefix for your test, you can inform the
+patchers of the different prefix by setting ``patch.TEST_PREFIX``. For
+more details about how to change the value of see :ref:`test-prefix`.
+
 :func:`patch.dict` can be used to add members to a dictionary, or simply let a test
 change a dictionary, and ensure the dictionary is restored when the test
 ends.
@@ -1793,6 +1814,8 @@ builtin :func:`ord`::
     101
 
 
+.. _test-prefix:
+
 TEST_PREFIX
 ~~~~~~~~~~~
 


From webhook-mailer at python.org  Thu Sep 12 07:31:21 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 11:31:21 -0000
Subject: [Python-checkins] bpo-36675: Remove obsolete code. (GH-16024)
Message-ID: <mailman.647.1568287883.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/94a684734f669eab02b5c915394b749ccf936449
commit: 94a684734f669eab02b5c915394b749ccf936449
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T04:31:17-07:00
summary:

bpo-36675: Remove obsolete code. (GH-16024)


Does no longer work since Sphinx moved the trim_doctest_flag option in
the configuration.
(cherry picked from commit 2c910c1e732c9a3ec4c67a7c43d789d6c729304a)

Co-authored-by: Julien Palard <julien at palard.fr>

files:
M Doc/library/doctest.rst
M Doc/tools/extensions/pyspecific.py

diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst
index e7c0033eb6bc..7fc471fe2ab7 100644
--- a/Doc/library/doctest.rst
+++ b/Doc/library/doctest.rst
@@ -1,5 +1,3 @@
-:keepdoctest:
-
 :mod:`doctest` --- Test interactive Python examples
 ===================================================
 
diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index 82dfc6bb263b..2d9561241594 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -29,7 +29,6 @@
 from sphinx.locale import translators
 from sphinx.util import status_iterator, logging
 from sphinx.util.nodes import split_explicit_title
-from sphinx.writers.html import HTMLTranslator
 from sphinx.writers.text import TextWriter, TextTranslator
 from sphinx.writers.latex import LaTeXTranslator
 from sphinx.domains.python import PyModulelevel, PyClassmember
@@ -49,37 +48,6 @@
     Body.enum.converters['lowerroman'] = \
     Body.enum.converters['upperroman'] = lambda x: None
 
-# monkey-patch HTML and LaTeX translators to keep doctest blocks in the
-# doctest docs themselves
-orig_visit_literal_block = HTMLTranslator.visit_literal_block
-orig_depart_literal_block = LaTeXTranslator.depart_literal_block
-
-
-def new_visit_literal_block(self, node):
-    meta = self.builder.env.metadata[self.builder.current_docname]
-    old_trim_doctest_flags = self.highlighter.trim_doctest_flags
-    if 'keepdoctest' in meta:
-        self.highlighter.trim_doctest_flags = False
-    try:
-        orig_visit_literal_block(self, node)
-    finally:
-        self.highlighter.trim_doctest_flags = old_trim_doctest_flags
-
-
-def new_depart_literal_block(self, node):
-    meta = self.builder.env.metadata[self.curfilestack[-1]]
-    old_trim_doctest_flags = self.highlighter.trim_doctest_flags
-    if 'keepdoctest' in meta:
-        self.highlighter.trim_doctest_flags = False
-    try:
-        orig_depart_literal_block(self, node)
-    finally:
-        self.highlighter.trim_doctest_flags = old_trim_doctest_flags
-
-
-HTMLTranslator.visit_literal_block = new_visit_literal_block
-LaTeXTranslator.depart_literal_block = new_depart_literal_block
-
 
 # Support for marking up and linking to bugs.python.org issues
 


From webhook-mailer at python.org  Thu Sep 12 07:37:09 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 11:37:09 -0000
Subject: [Python-checkins] bpo-35685: Add examples of
 unittest.mock.patch.dict usage (GH-11456)
Message-ID: <mailman.648.1568288230.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/648494b44aad273590382c8db5a9d1a6c96ee67a
commit: 648494b44aad273590382c8db5a9d1a6c96ee67a
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T04:37:05-07:00
summary:

bpo-35685: Add examples of unittest.mock.patch.dict usage (GH-11456)

(cherry picked from commit 31a82e25b6044a5b5ee25246bad3eb7b873cf5ec)

Co-authored-by: Emmanuel Arias <emmanuelarias30 at gmail.com>

files:
M Doc/library/unittest.mock.rst

diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index 04ff8a61da3c..b446ddb3598f 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -1572,15 +1572,36 @@ patch.dict
     :func:`patch.dict` can also be called with arbitrary keyword arguments to set
     values in the dictionary.
 
-    :func:`patch.dict` can be used as a context manager, decorator or class
-    decorator. When used as a class decorator :func:`patch.dict` honours
-    ``patch.TEST_PREFIX`` for choosing which methods to wrap.
-
     .. versionchanged:: 3.8
 
         :func:`patch.dict` now returns the patched dictionary when used as a context
         manager.
 
+:func:`patch.dict` can be used as a context manager, decorator or class
+decorator:
+
+    >>> foo = {}
+    >>> @patch.dict(foo, {'newkey': 'newvalue'})
+    ... def test():
+    ...     assert foo == {'newkey': 'newvalue'}
+    >>> test()
+    >>> assert foo == {}
+
+When used as a class decorator :func:`patch.dict` honours
+``patch.TEST_PREFIX`` (default to ``'test'``) for choosing which methods to wrap:
+
+    >>> import os
+    >>> import unittest
+    >>> from unittest.mock import patch
+    >>> @patch.dict('os.environ', {'newkey': 'newvalue'})
+    ... class TestSample(unittest.TestCase):
+    ...     def test_sample(self):
+    ...         self.assertEqual(os.environ['newkey'], 'newvalue')
+
+If you want to use a different prefix for your test, you can inform the
+patchers of the different prefix by setting ``patch.TEST_PREFIX``. For
+more details about how to change the value of see :ref:`test-prefix`.
+
 :func:`patch.dict` can be used to add members to a dictionary, or simply let a test
 change a dictionary, and ensure the dictionary is restored when the test
 ends.
@@ -1793,6 +1814,8 @@ builtin :func:`ord`::
     101
 
 
+.. _test-prefix:
+
 TEST_PREFIX
 ~~~~~~~~~~~
 


From webhook-mailer at python.org  Thu Sep 12 07:46:41 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Thu, 12 Sep 2019 11:46:41 -0000
Subject: [Python-checkins] bpo-38134: Remove PKBDF2_HMAC_fast from
 _hashopenssl (GH-16028)
Message-ID: <mailman.649.1568288802.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/64117e059b79236c7345bc9afc1cc707162411de
commit: 64117e059b79236c7345bc9afc1cc707162411de
branch: master
author: Christian Heimes <christian at python.org>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-12T12:46:37+01:00
summary:

bpo-38134: Remove PKBDF2_HMAC_fast from _hashopenssl (GH-16028)

Signed-off-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Library/2019-09-12-13-18-55.bpo-38134.gXJTbP.rst
M Modules/_hashopenssl.c
M Modules/clinic/_hashopenssl.c.h

diff --git a/Misc/NEWS.d/next/Library/2019-09-12-13-18-55.bpo-38134.gXJTbP.rst b/Misc/NEWS.d/next/Library/2019-09-12-13-18-55.bpo-38134.gXJTbP.rst
new file mode 100644
index 000000000000..fc134c203fdd
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-12-13-18-55.bpo-38134.gXJTbP.rst
@@ -0,0 +1,2 @@
+Remove obsolete copy of PBKDF2_HMAC_fast. All supported OpenSSL versions
+contain a fast implementation.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index a806241897fc..5e76853b3411 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -28,18 +28,6 @@
 
 #define MUNCH_SIZE INT_MAX
 
-#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
-/* OpenSSL < 1.1.0 */
-#define EVP_MD_CTX_new EVP_MD_CTX_create
-#define EVP_MD_CTX_free EVP_MD_CTX_destroy
-#define HAS_FAST_PKCS5_PBKDF2_HMAC 0
-#include <openssl/hmac.h>
-#else
-/* OpenSSL >= 1.1.0 */
-#define HAS_FAST_PKCS5_PBKDF2_HMAC 1
-#endif
-
-
 typedef struct {
     PyObject_HEAD
     PyObject            *name;  /* name of this hash algorithm */
@@ -481,97 +469,6 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj)
     return ret_obj;
 }
 
-#if (OPENSSL_VERSION_NUMBER >= 0x10000000 && !defined(OPENSSL_NO_HMAC) \
-     && !defined(OPENSSL_NO_SHA))
-
-#define PY_PBKDF2_HMAC 1
-
-#if !HAS_FAST_PKCS5_PBKDF2_HMAC
-/* Improved implementation of PKCS5_PBKDF2_HMAC()
- *
- * PKCS5_PBKDF2_HMAC_fast() hashes the password exactly one time instead of
- * `iter` times. Today (2013) the iteration count is typically 100,000 or
- * more. The improved algorithm is not subject to a Denial-of-Service
- * vulnerability with overly large passwords.
- *
- * Also OpenSSL < 1.0 don't provide PKCS5_PBKDF2_HMAC(), only
- * PKCS5_PBKDF2_SHA1.
- */
-static int
-PKCS5_PBKDF2_HMAC_fast(const char *pass, int passlen,
-                       const unsigned char *salt, int saltlen,
-                       int iter, const EVP_MD *digest,
-                       int keylen, unsigned char *out)
-{
-    unsigned char digtmp[EVP_MAX_MD_SIZE], *p, itmp[4];
-    int cplen, j, k, tkeylen, mdlen;
-    unsigned long i = 1;
-    HMAC_CTX hctx_tpl, hctx;
-
-    mdlen = EVP_MD_size(digest);
-    if (mdlen < 0)
-        return 0;
-
-    HMAC_CTX_init(&hctx_tpl);
-    HMAC_CTX_init(&hctx);
-    p = out;
-    tkeylen = keylen;
-    if (!HMAC_Init_ex(&hctx_tpl, pass, passlen, digest, NULL)) {
-        HMAC_CTX_cleanup(&hctx_tpl);
-        return 0;
-    }
-    while (tkeylen) {
-        if (tkeylen > mdlen)
-            cplen = mdlen;
-        else
-            cplen = tkeylen;
-        /* We are unlikely to ever use more than 256 blocks (5120 bits!)
-         * but just in case...
-         */
-        itmp[0] = (unsigned char)((i >> 24) & 0xff);
-        itmp[1] = (unsigned char)((i >> 16) & 0xff);
-        itmp[2] = (unsigned char)((i >> 8) & 0xff);
-        itmp[3] = (unsigned char)(i & 0xff);
-        if (!HMAC_CTX_copy(&hctx, &hctx_tpl)) {
-            HMAC_CTX_cleanup(&hctx_tpl);
-            return 0;
-        }
-        if (!HMAC_Update(&hctx, salt, saltlen)
-                || !HMAC_Update(&hctx, itmp, 4)
-                || !HMAC_Final(&hctx, digtmp, NULL)) {
-            HMAC_CTX_cleanup(&hctx_tpl);
-            HMAC_CTX_cleanup(&hctx);
-            return 0;
-        }
-        HMAC_CTX_cleanup(&hctx);
-        memcpy(p, digtmp, cplen);
-        for (j = 1; j < iter; j++) {
-            if (!HMAC_CTX_copy(&hctx, &hctx_tpl)) {
-                HMAC_CTX_cleanup(&hctx_tpl);
-                return 0;
-            }
-            if (!HMAC_Update(&hctx, digtmp, mdlen)
-                    || !HMAC_Final(&hctx, digtmp, NULL)) {
-                HMAC_CTX_cleanup(&hctx_tpl);
-                HMAC_CTX_cleanup(&hctx);
-                return 0;
-            }
-            HMAC_CTX_cleanup(&hctx);
-            for (k = 0; k < cplen; k++) {
-                p[k] ^= digtmp[k];
-            }
-        }
-        tkeylen-= cplen;
-        i++;
-        p+= cplen;
-    }
-    HMAC_CTX_cleanup(&hctx_tpl);
-    return 1;
-}
-#endif
-
-
-
 /*[clinic input]
 _hashlib.pbkdf2_hmac as pbkdf2_hmac
 
@@ -652,17 +549,10 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name,
     key = PyBytes_AS_STRING(key_obj);
 
     Py_BEGIN_ALLOW_THREADS
-#if HAS_FAST_PKCS5_PBKDF2_HMAC
     retval = PKCS5_PBKDF2_HMAC((char*)password->buf, (int)password->len,
                                (unsigned char *)salt->buf, (int)salt->len,
                                iterations, digest, dklen,
                                (unsigned char *)key);
-#else
-    retval = PKCS5_PBKDF2_HMAC_fast((char*)password->buf, (int)password->len,
-                                    (unsigned char *)salt->buf, (int)salt->len,
-                                    iterations, digest, dklen,
-                                    (unsigned char *)key);
-#endif
     Py_END_ALLOW_THREADS
 
     if (!retval) {
@@ -675,8 +565,6 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name,
     return key_obj;
 }
 
-#endif
-
 #if OPENSSL_VERSION_NUMBER > 0x10100000L && !defined(OPENSSL_NO_SCRYPT) && !defined(LIBRESSL_VERSION_NUMBER)
 #define PY_SCRYPT 1
 
@@ -986,9 +874,7 @@ GEN_CONSTRUCTOR(sha512)
 
 static struct PyMethodDef EVP_functions[] = {
     EVP_NEW_METHODDEF
-#ifdef PY_PBKDF2_HMAC
     PBKDF2_HMAC_METHODDEF
-#endif
     _HASHLIB_SCRYPT_METHODDEF
     _HASHLIB_HMAC_DIGEST_METHODDEF
     CONSTRUCTOR_METH_DEF(md5),
diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h
index 6377e18ece7a..854d92537736 100644
--- a/Modules/clinic/_hashopenssl.c.h
+++ b/Modules/clinic/_hashopenssl.c.h
@@ -109,8 +109,6 @@ EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn
     return return_value;
 }
 
-#if ((OPENSSL_VERSION_NUMBER >= 0x10000000 && !defined(OPENSSL_NO_HMAC) && !defined(OPENSSL_NO_SHA)))
-
 PyDoc_STRVAR(pbkdf2_hmac__doc__,
 "pbkdf2_hmac($module, /, hash_name, password, salt, iterations,\n"
 "            dklen=None)\n"
@@ -200,8 +198,6 @@ pbkdf2_hmac(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject
     return return_value;
 }
 
-#endif /* ((OPENSSL_VERSION_NUMBER >= 0x10000000 && !defined(OPENSSL_NO_HMAC) && !defined(OPENSSL_NO_SHA))) */
-
 #if (OPENSSL_VERSION_NUMBER > 0x10100000L && !defined(OPENSSL_NO_SCRYPT) && !defined(LIBRESSL_VERSION_NUMBER))
 
 PyDoc_STRVAR(_hashlib_scrypt__doc__,
@@ -402,11 +398,7 @@ _hashlib_hmac_digest(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
     return return_value;
 }
 
-#ifndef PBKDF2_HMAC_METHODDEF
-    #define PBKDF2_HMAC_METHODDEF
-#endif /* !defined(PBKDF2_HMAC_METHODDEF) */
-
 #ifndef _HASHLIB_SCRYPT_METHODDEF
     #define _HASHLIB_SCRYPT_METHODDEF
 #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */
-/*[clinic end generated code: output=565dcbe3452e71f4 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=cfe686cb2fa042e1 input=a9049054013a1b77]*/


From webhook-mailer at python.org  Thu Sep 12 08:10:54 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Thu, 12 Sep 2019 12:10:54 -0000
Subject: [Python-checkins] bpo-35325: Doc: imp.find_module() return value
 documentation discrepancy (GH-11040)
Message-ID: <mailman.650.1568290255.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/967b84c913c7b09ae2fc86272cb9373415e2beaf
commit: 967b84c913c7b09ae2fc86272cb9373415e2beaf
branch: master
author: Windson yang <wiwindson at outlook.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-12T14:10:50+02:00
summary:

bpo-35325: Doc: imp.find_module() return value documentation discrepancy (GH-11040)

files:
M Doc/library/imp.rst

diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst
index 04f207f58d38..f5ad8c722964 100644
--- a/Doc/library/imp.rst
+++ b/Doc/library/imp.rst
@@ -66,12 +66,11 @@ This module provides an interface to the mechanisms used to implement the
    contained in the list returned by :func:`get_suffixes` describing the kind of
    module found.
 
-   If the module does not live in a file, the returned *file* is ``None``,
-   *pathname* is the empty string, and the *description* tuple contains empty
-   strings for its suffix and mode; the module type is indicated as given in
-   parentheses above.  If the search is unsuccessful, :exc:`ImportError` is
-   raised.  Other exceptions indicate problems with the arguments or
-   environment.
+   If the module is built-in or frozen then *file* and *pathname* are both ``None``
+   and the *description* tuple contains empty strings for its suffix and mode;
+   the module type is indicated as given in parentheses above.  If the search
+   is unsuccessful, :exc:`ImportError` is raised.  Other exceptions indicate
+   problems with the arguments or environment.
 
    If the module is a package, *file* is ``None``, *pathname* is the package
    path and the last item in the *description* tuple is :const:`PKG_DIRECTORY`.


From webhook-mailer at python.org  Thu Sep 12 08:18:42 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 12:18:42 -0000
Subject: [Python-checkins] bpo-38134: Remove PKBDF2_HMAC_fast from
 _hashopenssl (GH-16028)
Message-ID: <mailman.651.1568290723.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/80e33655a264d75f9a6ffedfc60697c30f92bdfe
commit: 80e33655a264d75f9a6ffedfc60697c30f92bdfe
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T05:18:38-07:00
summary:

bpo-38134: Remove PKBDF2_HMAC_fast from _hashopenssl (GH-16028)


Signed-off-by: Christian Heimes <christian at python.org>
(cherry picked from commit 64117e059b79236c7345bc9afc1cc707162411de)

Co-authored-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Library/2019-09-12-13-18-55.bpo-38134.gXJTbP.rst
M Modules/_hashopenssl.c
M Modules/clinic/_hashopenssl.c.h

diff --git a/Misc/NEWS.d/next/Library/2019-09-12-13-18-55.bpo-38134.gXJTbP.rst b/Misc/NEWS.d/next/Library/2019-09-12-13-18-55.bpo-38134.gXJTbP.rst
new file mode 100644
index 000000000000..fc134c203fdd
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-12-13-18-55.bpo-38134.gXJTbP.rst
@@ -0,0 +1,2 @@
+Remove obsolete copy of PBKDF2_HMAC_fast. All supported OpenSSL versions
+contain a fast implementation.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index a806241897fc..5e76853b3411 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -28,18 +28,6 @@
 
 #define MUNCH_SIZE INT_MAX
 
-#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
-/* OpenSSL < 1.1.0 */
-#define EVP_MD_CTX_new EVP_MD_CTX_create
-#define EVP_MD_CTX_free EVP_MD_CTX_destroy
-#define HAS_FAST_PKCS5_PBKDF2_HMAC 0
-#include <openssl/hmac.h>
-#else
-/* OpenSSL >= 1.1.0 */
-#define HAS_FAST_PKCS5_PBKDF2_HMAC 1
-#endif
-
-
 typedef struct {
     PyObject_HEAD
     PyObject            *name;  /* name of this hash algorithm */
@@ -481,97 +469,6 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj)
     return ret_obj;
 }
 
-#if (OPENSSL_VERSION_NUMBER >= 0x10000000 && !defined(OPENSSL_NO_HMAC) \
-     && !defined(OPENSSL_NO_SHA))
-
-#define PY_PBKDF2_HMAC 1
-
-#if !HAS_FAST_PKCS5_PBKDF2_HMAC
-/* Improved implementation of PKCS5_PBKDF2_HMAC()
- *
- * PKCS5_PBKDF2_HMAC_fast() hashes the password exactly one time instead of
- * `iter` times. Today (2013) the iteration count is typically 100,000 or
- * more. The improved algorithm is not subject to a Denial-of-Service
- * vulnerability with overly large passwords.
- *
- * Also OpenSSL < 1.0 don't provide PKCS5_PBKDF2_HMAC(), only
- * PKCS5_PBKDF2_SHA1.
- */
-static int
-PKCS5_PBKDF2_HMAC_fast(const char *pass, int passlen,
-                       const unsigned char *salt, int saltlen,
-                       int iter, const EVP_MD *digest,
-                       int keylen, unsigned char *out)
-{
-    unsigned char digtmp[EVP_MAX_MD_SIZE], *p, itmp[4];
-    int cplen, j, k, tkeylen, mdlen;
-    unsigned long i = 1;
-    HMAC_CTX hctx_tpl, hctx;
-
-    mdlen = EVP_MD_size(digest);
-    if (mdlen < 0)
-        return 0;
-
-    HMAC_CTX_init(&hctx_tpl);
-    HMAC_CTX_init(&hctx);
-    p = out;
-    tkeylen = keylen;
-    if (!HMAC_Init_ex(&hctx_tpl, pass, passlen, digest, NULL)) {
-        HMAC_CTX_cleanup(&hctx_tpl);
-        return 0;
-    }
-    while (tkeylen) {
-        if (tkeylen > mdlen)
-            cplen = mdlen;
-        else
-            cplen = tkeylen;
-        /* We are unlikely to ever use more than 256 blocks (5120 bits!)
-         * but just in case...
-         */
-        itmp[0] = (unsigned char)((i >> 24) & 0xff);
-        itmp[1] = (unsigned char)((i >> 16) & 0xff);
-        itmp[2] = (unsigned char)((i >> 8) & 0xff);
-        itmp[3] = (unsigned char)(i & 0xff);
-        if (!HMAC_CTX_copy(&hctx, &hctx_tpl)) {
-            HMAC_CTX_cleanup(&hctx_tpl);
-            return 0;
-        }
-        if (!HMAC_Update(&hctx, salt, saltlen)
-                || !HMAC_Update(&hctx, itmp, 4)
-                || !HMAC_Final(&hctx, digtmp, NULL)) {
-            HMAC_CTX_cleanup(&hctx_tpl);
-            HMAC_CTX_cleanup(&hctx);
-            return 0;
-        }
-        HMAC_CTX_cleanup(&hctx);
-        memcpy(p, digtmp, cplen);
-        for (j = 1; j < iter; j++) {
-            if (!HMAC_CTX_copy(&hctx, &hctx_tpl)) {
-                HMAC_CTX_cleanup(&hctx_tpl);
-                return 0;
-            }
-            if (!HMAC_Update(&hctx, digtmp, mdlen)
-                    || !HMAC_Final(&hctx, digtmp, NULL)) {
-                HMAC_CTX_cleanup(&hctx_tpl);
-                HMAC_CTX_cleanup(&hctx);
-                return 0;
-            }
-            HMAC_CTX_cleanup(&hctx);
-            for (k = 0; k < cplen; k++) {
-                p[k] ^= digtmp[k];
-            }
-        }
-        tkeylen-= cplen;
-        i++;
-        p+= cplen;
-    }
-    HMAC_CTX_cleanup(&hctx_tpl);
-    return 1;
-}
-#endif
-
-
-
 /*[clinic input]
 _hashlib.pbkdf2_hmac as pbkdf2_hmac
 
@@ -652,17 +549,10 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name,
     key = PyBytes_AS_STRING(key_obj);
 
     Py_BEGIN_ALLOW_THREADS
-#if HAS_FAST_PKCS5_PBKDF2_HMAC
     retval = PKCS5_PBKDF2_HMAC((char*)password->buf, (int)password->len,
                                (unsigned char *)salt->buf, (int)salt->len,
                                iterations, digest, dklen,
                                (unsigned char *)key);
-#else
-    retval = PKCS5_PBKDF2_HMAC_fast((char*)password->buf, (int)password->len,
-                                    (unsigned char *)salt->buf, (int)salt->len,
-                                    iterations, digest, dklen,
-                                    (unsigned char *)key);
-#endif
     Py_END_ALLOW_THREADS
 
     if (!retval) {
@@ -675,8 +565,6 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name,
     return key_obj;
 }
 
-#endif
-
 #if OPENSSL_VERSION_NUMBER > 0x10100000L && !defined(OPENSSL_NO_SCRYPT) && !defined(LIBRESSL_VERSION_NUMBER)
 #define PY_SCRYPT 1
 
@@ -986,9 +874,7 @@ GEN_CONSTRUCTOR(sha512)
 
 static struct PyMethodDef EVP_functions[] = {
     EVP_NEW_METHODDEF
-#ifdef PY_PBKDF2_HMAC
     PBKDF2_HMAC_METHODDEF
-#endif
     _HASHLIB_SCRYPT_METHODDEF
     _HASHLIB_HMAC_DIGEST_METHODDEF
     CONSTRUCTOR_METH_DEF(md5),
diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h
index 6377e18ece7a..854d92537736 100644
--- a/Modules/clinic/_hashopenssl.c.h
+++ b/Modules/clinic/_hashopenssl.c.h
@@ -109,8 +109,6 @@ EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn
     return return_value;
 }
 
-#if ((OPENSSL_VERSION_NUMBER >= 0x10000000 && !defined(OPENSSL_NO_HMAC) && !defined(OPENSSL_NO_SHA)))
-
 PyDoc_STRVAR(pbkdf2_hmac__doc__,
 "pbkdf2_hmac($module, /, hash_name, password, salt, iterations,\n"
 "            dklen=None)\n"
@@ -200,8 +198,6 @@ pbkdf2_hmac(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject
     return return_value;
 }
 
-#endif /* ((OPENSSL_VERSION_NUMBER >= 0x10000000 && !defined(OPENSSL_NO_HMAC) && !defined(OPENSSL_NO_SHA))) */
-
 #if (OPENSSL_VERSION_NUMBER > 0x10100000L && !defined(OPENSSL_NO_SCRYPT) && !defined(LIBRESSL_VERSION_NUMBER))
 
 PyDoc_STRVAR(_hashlib_scrypt__doc__,
@@ -402,11 +398,7 @@ _hashlib_hmac_digest(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
     return return_value;
 }
 
-#ifndef PBKDF2_HMAC_METHODDEF
-    #define PBKDF2_HMAC_METHODDEF
-#endif /* !defined(PBKDF2_HMAC_METHODDEF) */
-
 #ifndef _HASHLIB_SCRYPT_METHODDEF
     #define _HASHLIB_SCRYPT_METHODDEF
 #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */
-/*[clinic end generated code: output=565dcbe3452e71f4 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=cfe686cb2fa042e1 input=a9049054013a1b77]*/


From webhook-mailer at python.org  Thu Sep 12 08:25:00 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 12:25:00 -0000
Subject: [Python-checkins] bpo-35325: Doc: imp.find_module() return value
 documentation discrepancy (GH-11040)
Message-ID: <mailman.652.1568291101.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/01b47c94ca8ef89c5ac5fda0ab1074e73234a738
commit: 01b47c94ca8ef89c5ac5fda0ab1074e73234a738
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T05:24:56-07:00
summary:

bpo-35325: Doc: imp.find_module() return value documentation discrepancy (GH-11040)

(cherry picked from commit 967b84c913c7b09ae2fc86272cb9373415e2beaf)

Co-authored-by: Windson yang <wiwindson at outlook.com>

files:
M Doc/library/imp.rst

diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst
index 04f207f58d38..f5ad8c722964 100644
--- a/Doc/library/imp.rst
+++ b/Doc/library/imp.rst
@@ -66,12 +66,11 @@ This module provides an interface to the mechanisms used to implement the
    contained in the list returned by :func:`get_suffixes` describing the kind of
    module found.
 
-   If the module does not live in a file, the returned *file* is ``None``,
-   *pathname* is the empty string, and the *description* tuple contains empty
-   strings for its suffix and mode; the module type is indicated as given in
-   parentheses above.  If the search is unsuccessful, :exc:`ImportError` is
-   raised.  Other exceptions indicate problems with the arguments or
-   environment.
+   If the module is built-in or frozen then *file* and *pathname* are both ``None``
+   and the *description* tuple contains empty strings for its suffix and mode;
+   the module type is indicated as given in parentheses above.  If the search
+   is unsuccessful, :exc:`ImportError` is raised.  Other exceptions indicate
+   problems with the arguments or environment.
 
    If the module is a package, *file* is ``None``, *pathname* is the package
    path and the last item in the *description* tuple is :const:`PKG_DIRECTORY`.


From webhook-mailer at python.org  Thu Sep 12 08:25:58 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 12:25:58 -0000
Subject: [Python-checkins] bpo-35325: Doc: imp.find_module() return value
 documentation discrepancy (GH-11040)
Message-ID: <mailman.653.1568291159.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/14afe203d6937069c66b7e9c4a9fc0db49b32c19
commit: 14afe203d6937069c66b7e9c4a9fc0db49b32c19
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T05:25:54-07:00
summary:

bpo-35325: Doc: imp.find_module() return value documentation discrepancy (GH-11040)

(cherry picked from commit 967b84c913c7b09ae2fc86272cb9373415e2beaf)

Co-authored-by: Windson yang <wiwindson at outlook.com>

files:
M Doc/library/imp.rst

diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst
index 04f207f58d38..f5ad8c722964 100644
--- a/Doc/library/imp.rst
+++ b/Doc/library/imp.rst
@@ -66,12 +66,11 @@ This module provides an interface to the mechanisms used to implement the
    contained in the list returned by :func:`get_suffixes` describing the kind of
    module found.
 
-   If the module does not live in a file, the returned *file* is ``None``,
-   *pathname* is the empty string, and the *description* tuple contains empty
-   strings for its suffix and mode; the module type is indicated as given in
-   parentheses above.  If the search is unsuccessful, :exc:`ImportError` is
-   raised.  Other exceptions indicate problems with the arguments or
-   environment.
+   If the module is built-in or frozen then *file* and *pathname* are both ``None``
+   and the *description* tuple contains empty strings for its suffix and mode;
+   the module type is indicated as given in parentheses above.  If the search
+   is unsuccessful, :exc:`ImportError` is raised.  Other exceptions indicate
+   problems with the arguments or environment.
 
    If the module is a package, *file* is ``None``, *pathname* is the package
    path and the last item in the *description* tuple is :const:`PKG_DIRECTORY`.


From webhook-mailer at python.org  Thu Sep 12 08:26:50 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 12:26:50 -0000
Subject: [Python-checkins] bpo-26868: Fix example usage of
 PyModule_AddObject. (GH-15725)
Message-ID: <mailman.654.1568291212.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/535863e3f599a6ad829204d83f144c91e44de443
commit: 535863e3f599a6ad829204d83f144c91e44de443
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T05:26:46-07:00
summary:

bpo-26868: Fix example usage of PyModule_AddObject. (GH-15725)


* Add a note to the PyModule_AddObject docs.

* Correct example usages of PyModule_AddObject.

* Whitespace.

* Clean up wording.

* ?? Added by blurb_it.

* First code review.

* Add < 0 in the tests with PyModule_AddObject
(cherry picked from commit 224b8aaa7e8f67f748e8b7b6a4a77a25f6554651)

Co-authored-by: Brandt Bucher <brandtbucher at gmail.com>

files:
A Misc/NEWS.d/next/Documentation/2019-09-07-15-55-46.bpo-26868.Raw0Gd.rst
M Doc/c-api/module.rst
M Doc/extending/extending.rst
M Doc/extending/newtypes_tutorial.rst
M Doc/includes/custom.c
M Doc/includes/custom2.c
M Doc/includes/custom3.c
M Doc/includes/custom4.c
M Doc/includes/sublist.c

diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index 68cbda2938f3..feca1ec2a057 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -417,7 +417,22 @@ state:
 
    Add an object to *module* as *name*.  This is a convenience function which can
    be used from the module's initialization function.  This steals a reference to
-   *value*.  Return ``-1`` on error, ``0`` on success.
+   *value* on success.  Return ``-1`` on error, ``0`` on success.
+
+   .. note::
+
+      Unlike other functions that steal references, ``PyModule_AddObject()`` only
+      decrements the reference count of *value* **on success**.
+
+      This means that its return value must be checked, and calling code must
+      :c:func:`Py_DECREF` *value* manually on error. Example usage::
+
+         Py_INCREF(spam);
+         if (PyModule_AddObject(module, "spam", spam) < 0) {
+             Py_DECREF(module);
+             Py_DECREF(spam);
+             return NULL;
+         }
 
 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value)
 
diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst
index e459514b2b58..5b4ea8220e23 100644
--- a/Doc/extending/extending.rst
+++ b/Doc/extending/extending.rst
@@ -209,7 +209,7 @@ usually declare a static object variable at the beginning of your file::
    static PyObject *SpamError;
 
 and initialize it in your module's initialization function (:c:func:`PyInit_spam`)
-with an exception object (leaving out the error checking for now)::
+with an exception object::
 
    PyMODINIT_FUNC
    PyInit_spam(void)
@@ -221,8 +221,14 @@ with an exception object (leaving out the error checking for now)::
            return NULL;
 
        SpamError = PyErr_NewException("spam.error", NULL, NULL);
-       Py_INCREF(SpamError);
-       PyModule_AddObject(m, "error", SpamError);
+       Py_XINCREF(SpamError);
+       if (PyModule_AddObject(m, "error", SpamError) < 0) {
+           Py_XDECREF(SpamError);
+           Py_CLEAR(SpamError);
+           Py_DECREF(m);
+           return NULL;
+       }
+
        return m;
    }
 
@@ -1261,8 +1267,12 @@ function must take care of initializing the C API pointer array::
        /* Create a Capsule containing the API pointer array's address */
        c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL);
 
-       if (c_api_object != NULL)
-           PyModule_AddObject(m, "_C_API", c_api_object);
+       if (PyModule_AddObject(m, "_C_API", c_api_object) < 0) {
+           Py_XDECREF(c_api_object);
+           Py_DECREF(m);
+           return NULL;
+       }
+
        return m;
    }
 
diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst
index 59c8cc01222b..94ca747c7aeb 100644
--- a/Doc/extending/newtypes_tutorial.rst
+++ b/Doc/extending/newtypes_tutorial.rst
@@ -179,7 +179,12 @@ This initializes the :class:`Custom` type, filling in a number of members
 to the appropriate default values, including :attr:`ob_type` that we initially
 set to *NULL*. ::
 
-   PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+   Py_INCREF(&CustomType);
+   if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+       Py_DECREF(&CustomType);
+       PY_DECREF(m);
+       return NULL;
+   }
 
 This adds the type to the module dictionary.  This allows us to create
 :class:`Custom` instances by calling the :class:`Custom` class:
@@ -864,7 +869,12 @@ function::
            return NULL;
 
        Py_INCREF(&SubListType);
-       PyModule_AddObject(m, "SubList", (PyObject *) &SubListType);
+       if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) {
+           Py_DECREF(&SubListType);
+           Py_DECREF(m);
+           return NULL;
+       }
+
        return m;
    }
 
diff --git a/Doc/includes/custom.c b/Doc/includes/custom.c
index 13d16f5424ae..bda32e2ad81d 100644
--- a/Doc/includes/custom.c
+++ b/Doc/includes/custom.c
@@ -35,6 +35,11 @@ PyInit_custom(void)
         return NULL;
 
     Py_INCREF(&CustomType);
-    PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+    if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+        Py_DECREF(&CustomType);
+        PY_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Doc/includes/custom2.c b/Doc/includes/custom2.c
index 6477a19dafed..5bacab7a2a97 100644
--- a/Doc/includes/custom2.c
+++ b/Doc/includes/custom2.c
@@ -128,6 +128,11 @@ PyInit_custom2(void)
         return NULL;
 
     Py_INCREF(&CustomType);
-    PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+    if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+        Py_DECREF(&CustomType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Doc/includes/custom3.c b/Doc/includes/custom3.c
index 213d0864ce1c..2b7a99ecf96c 100644
--- a/Doc/includes/custom3.c
+++ b/Doc/includes/custom3.c
@@ -179,6 +179,11 @@ PyInit_custom3(void)
         return NULL;
 
     Py_INCREF(&CustomType);
-    PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+    if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+        Py_DECREF(&CustomType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Doc/includes/custom4.c b/Doc/includes/custom4.c
index b0b2906dbdc8..584992fc5f1a 100644
--- a/Doc/includes/custom4.c
+++ b/Doc/includes/custom4.c
@@ -193,6 +193,11 @@ PyInit_custom4(void)
         return NULL;
 
     Py_INCREF(&CustomType);
-    PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+    if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+        Py_DECREF(&CustomType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Doc/includes/sublist.c b/Doc/includes/sublist.c
index 76ff93948cfd..b2c26e73ebaf 100644
--- a/Doc/includes/sublist.c
+++ b/Doc/includes/sublist.c
@@ -59,6 +59,11 @@ PyInit_sublist(void)
         return NULL;
 
     Py_INCREF(&SubListType);
-    PyModule_AddObject(m, "SubList", (PyObject *) &SubListType);
+    if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) {
+        Py_DECREF(&SubListType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Misc/NEWS.d/next/Documentation/2019-09-07-15-55-46.bpo-26868.Raw0Gd.rst b/Misc/NEWS.d/next/Documentation/2019-09-07-15-55-46.bpo-26868.Raw0Gd.rst
new file mode 100644
index 000000000000..c668092b41a6
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-09-07-15-55-46.bpo-26868.Raw0Gd.rst
@@ -0,0 +1 @@
+Fix example usage of :c:func:`PyModule_AddObject` to properly handle errors.
\ No newline at end of file


From webhook-mailer at python.org  Thu Sep 12 08:29:13 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 12:29:13 -0000
Subject: [Python-checkins] bpo-26868: Fix example usage of
 PyModule_AddObject. (GH-15725)
Message-ID: <mailman.655.1568291355.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c8d1338441114fbc504625bc66607e7996018a5d
commit: c8d1338441114fbc504625bc66607e7996018a5d
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T05:29:09-07:00
summary:

bpo-26868: Fix example usage of PyModule_AddObject. (GH-15725)


* Add a note to the PyModule_AddObject docs.

* Correct example usages of PyModule_AddObject.

* Whitespace.

* Clean up wording.

* ?? Added by blurb_it.

* First code review.

* Add < 0 in the tests with PyModule_AddObject
(cherry picked from commit 224b8aaa7e8f67f748e8b7b6a4a77a25f6554651)

Co-authored-by: Brandt Bucher <brandtbucher at gmail.com>

files:
A Misc/NEWS.d/next/Documentation/2019-09-07-15-55-46.bpo-26868.Raw0Gd.rst
M Doc/c-api/module.rst
M Doc/extending/extending.rst
M Doc/extending/newtypes_tutorial.rst
M Doc/includes/custom.c
M Doc/includes/custom2.c
M Doc/includes/custom3.c
M Doc/includes/custom4.c
M Doc/includes/sublist.c

diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index 017b656854a8..7168dc7363ca 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -417,7 +417,22 @@ state:
 
    Add an object to *module* as *name*.  This is a convenience function which can
    be used from the module's initialization function.  This steals a reference to
-   *value*.  Return ``-1`` on error, ``0`` on success.
+   *value* on success.  Return ``-1`` on error, ``0`` on success.
+
+   .. note::
+
+      Unlike other functions that steal references, ``PyModule_AddObject()`` only
+      decrements the reference count of *value* **on success**.
+
+      This means that its return value must be checked, and calling code must
+      :c:func:`Py_DECREF` *value* manually on error. Example usage::
+
+         Py_INCREF(spam);
+         if (PyModule_AddObject(module, "spam", spam) < 0) {
+             Py_DECREF(module);
+             Py_DECREF(spam);
+             return NULL;
+         }
 
 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value)
 
diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst
index afed3aabb79a..73a77c781bc2 100644
--- a/Doc/extending/extending.rst
+++ b/Doc/extending/extending.rst
@@ -209,7 +209,7 @@ usually declare a static object variable at the beginning of your file::
    static PyObject *SpamError;
 
 and initialize it in your module's initialization function (:c:func:`PyInit_spam`)
-with an exception object (leaving out the error checking for now)::
+with an exception object::
 
    PyMODINIT_FUNC
    PyInit_spam(void)
@@ -221,8 +221,14 @@ with an exception object (leaving out the error checking for now)::
            return NULL;
 
        SpamError = PyErr_NewException("spam.error", NULL, NULL);
-       Py_INCREF(SpamError);
-       PyModule_AddObject(m, "error", SpamError);
+       Py_XINCREF(SpamError);
+       if (PyModule_AddObject(m, "error", SpamError) < 0) {
+           Py_XDECREF(SpamError);
+           Py_CLEAR(SpamError);
+           Py_DECREF(m);
+           return NULL;
+       }
+
        return m;
    }
 
@@ -1261,8 +1267,12 @@ function must take care of initializing the C API pointer array::
        /* Create a Capsule containing the API pointer array's address */
        c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL);
 
-       if (c_api_object != NULL)
-           PyModule_AddObject(m, "_C_API", c_api_object);
+       if (PyModule_AddObject(m, "_C_API", c_api_object) < 0) {
+           Py_XDECREF(c_api_object);
+           Py_DECREF(m);
+           return NULL;
+       }
+
        return m;
    }
 
diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst
index 07c2ef718aed..e6448e46411b 100644
--- a/Doc/extending/newtypes_tutorial.rst
+++ b/Doc/extending/newtypes_tutorial.rst
@@ -179,7 +179,12 @@ This initializes the :class:`Custom` type, filling in a number of members
 to the appropriate default values, including :attr:`ob_type` that we initially
 set to *NULL*. ::
 
-   PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+   Py_INCREF(&CustomType);
+   if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+       Py_DECREF(&CustomType);
+       PY_DECREF(m);
+       return NULL;
+   }
 
 This adds the type to the module dictionary.  This allows us to create
 :class:`Custom` instances by calling the :class:`Custom` class:
@@ -864,7 +869,12 @@ function::
            return NULL;
 
        Py_INCREF(&SubListType);
-       PyModule_AddObject(m, "SubList", (PyObject *) &SubListType);
+       if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) {
+           Py_DECREF(&SubListType);
+           Py_DECREF(m);
+           return NULL;
+       }
+
        return m;
    }
 
diff --git a/Doc/includes/custom.c b/Doc/includes/custom.c
index 13d16f5424ae..bda32e2ad81d 100644
--- a/Doc/includes/custom.c
+++ b/Doc/includes/custom.c
@@ -35,6 +35,11 @@ PyInit_custom(void)
         return NULL;
 
     Py_INCREF(&CustomType);
-    PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+    if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+        Py_DECREF(&CustomType);
+        PY_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Doc/includes/custom2.c b/Doc/includes/custom2.c
index 6477a19dafed..5bacab7a2a97 100644
--- a/Doc/includes/custom2.c
+++ b/Doc/includes/custom2.c
@@ -128,6 +128,11 @@ PyInit_custom2(void)
         return NULL;
 
     Py_INCREF(&CustomType);
-    PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+    if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+        Py_DECREF(&CustomType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Doc/includes/custom3.c b/Doc/includes/custom3.c
index 213d0864ce1c..2b7a99ecf96c 100644
--- a/Doc/includes/custom3.c
+++ b/Doc/includes/custom3.c
@@ -179,6 +179,11 @@ PyInit_custom3(void)
         return NULL;
 
     Py_INCREF(&CustomType);
-    PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+    if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+        Py_DECREF(&CustomType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Doc/includes/custom4.c b/Doc/includes/custom4.c
index b0b2906dbdc8..584992fc5f1a 100644
--- a/Doc/includes/custom4.c
+++ b/Doc/includes/custom4.c
@@ -193,6 +193,11 @@ PyInit_custom4(void)
         return NULL;
 
     Py_INCREF(&CustomType);
-    PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+    if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+        Py_DECREF(&CustomType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Doc/includes/sublist.c b/Doc/includes/sublist.c
index 76ff93948cfd..b2c26e73ebaf 100644
--- a/Doc/includes/sublist.c
+++ b/Doc/includes/sublist.c
@@ -59,6 +59,11 @@ PyInit_sublist(void)
         return NULL;
 
     Py_INCREF(&SubListType);
-    PyModule_AddObject(m, "SubList", (PyObject *) &SubListType);
+    if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) {
+        Py_DECREF(&SubListType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Misc/NEWS.d/next/Documentation/2019-09-07-15-55-46.bpo-26868.Raw0Gd.rst b/Misc/NEWS.d/next/Documentation/2019-09-07-15-55-46.bpo-26868.Raw0Gd.rst
new file mode 100644
index 000000000000..c668092b41a6
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-09-07-15-55-46.bpo-26868.Raw0Gd.rst
@@ -0,0 +1 @@
+Fix example usage of :c:func:`PyModule_AddObject` to properly handle errors.
\ No newline at end of file


From webhook-mailer at python.org  Thu Sep 12 08:35:52 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Thu, 12 Sep 2019 12:35:52 -0000
Subject: [Python-checkins] Enhance Py_UNREACHABLE macro (GH-16032)
Message-ID: <mailman.656.1568291753.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3ab61473ba7f3dca32d779ec2766a4faa0657923
commit: 3ab61473ba7f3dca32d779ec2766a4faa0657923
branch: master
author: Zachary Ware <zachary.ware at gmail.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-12T13:35:48+01:00
summary:

Enhance Py_UNREACHABLE macro (GH-16032)

files:
M Include/pymacro.h

diff --git a/Include/pymacro.h b/Include/pymacro.h
index 1890619099a3..ae09063e10d8 100644
--- a/Include/pymacro.h
+++ b/Include/pymacro.h
@@ -100,6 +100,32 @@
 #  define Py_UNUSED(name) _unused_ ## name
 #endif
 
+#if defined(RANDALL_WAS_HERE)
+#define Py_UNREACHABLE() do { \
+        fputs( \
+            "ERROR:\n\n" \
+            "If you're seeing this, the code is in what I thought was\n" \
+            "an unreachable state.\n\n" \
+            "I could give you advice for what to do, but honestly, why\n" \
+            "should you trust me?  I clearly screwed this up.  I'm writing\n" \
+            "a message that should never appear, yet I know it will\n" \
+            "probably appear someday.\n\n" \
+            "On a deep level, I know I'm not up to this task.\n" \
+            "I'm so sorry.\n\n" \
+            "https://xkcd.com/2200\n", stderr); \
+        abort(); \
+    } while(0)
+#elif defined(Py_DEBUG)
+#define Py_UNREACHABLE() do { \
+        fputs( \
+            "ERROR:\n\n" \
+            "We've reached an unreachable state. Anything is possible.\n" \
+            "The limits were in our heads all along. Follow your dreams.\n\n" \
+            "https://xkcd.com/2200\n", stderr); \
+        abort(); \
+    } while(0)
+#else
 #define Py_UNREACHABLE() abort()
+#endif
 
 #endif /* Py_PYMACRO_H */


From webhook-mailer at python.org  Thu Sep 12 08:40:45 2019
From: webhook-mailer at python.org (Andrew Svetlov)
Date: Thu, 12 Sep 2019 12:40:45 -0000
Subject: [Python-checkins] bpo-36373: Deprecate explicit loop in task and
 subprocess API (GH-16033)
Message-ID: <mailman.657.1568292046.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a488879cbaf4b8b52699cadccf73bb4c271bcb29
commit: a488879cbaf4b8b52699cadccf73bb4c271bcb29
branch: master
author: Andrew Svetlov <andrew.svetlov at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T15:40:40+03:00
summary:

bpo-36373: Deprecate explicit loop in task and subprocess API (GH-16033)

files:
M Doc/library/asyncio-subprocess.rst
M Doc/library/asyncio-task.rst
M Lib/asyncio/subprocess.py
M Lib/asyncio/tasks.py
M Lib/test/test_asyncio/test_streams.py
M Lib/test/test_asyncio/test_subprocess.py
M Lib/test/test_asyncio/test_tasks.py

diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst
index 444fb6361b5e..bd92257c82ef 100644
--- a/Doc/library/asyncio-subprocess.rst
+++ b/Doc/library/asyncio-subprocess.rst
@@ -71,6 +71,10 @@ Creating Subprocesses
    See the documentation of :meth:`loop.subprocess_exec` for other
    parameters.
 
+   .. deprecated-removed:: 3.8 3.10
+
+      The *loop* parameter.
+
 .. coroutinefunction:: create_subprocess_shell(cmd, stdin=None, \
                           stdout=None, stderr=None, loop=None, \
                           limit=None, \*\*kwds)
@@ -95,6 +99,10 @@ Creating Subprocesses
    escape whitespace and special shell characters in strings that are going
    to be used to construct shell commands.
 
+   .. deprecated-removed:: 3.8 3.10
+
+      The *loop* parameter.
+
 .. note::
 
    The default asyncio event loop implementation on **Windows** does not
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index 1fcdcb985d88..57e0e07ad3bb 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -334,6 +334,9 @@ Running Tasks Concurrently
    cancellation of one submitted Task/Future to cause other
    Tasks/Futures to be cancelled.
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    .. _asyncio_example_gather:
 
    Example::
@@ -411,6 +414,9 @@ Shielding From Cancellation
        except CancelledError:
            res = None
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
 
 Timeouts
 ========
@@ -478,22 +484,12 @@ Waiting Primitives
    set concurrently and block until the condition specified
    by *return_when*.
 
-   .. deprecated:: 3.8
-
-      If any awaitable in *aws* is a coroutine, it is automatically
-      scheduled as a Task.  Passing coroutines objects to
-      ``wait()`` directly is deprecated as it leads to
-      :ref:`confusing behavior <asyncio_example_wait_coroutine>`.
-
    Returns two sets of Tasks/Futures: ``(done, pending)``.
 
    Usage::
 
         done, pending = await asyncio.wait(aws)
 
-   .. deprecated-removed:: 3.8 3.10
-      The *loop* parameter.
-
    *timeout* (a float or int), if specified, can be used to control
    the maximum number of seconds to wait before returning.
 
@@ -525,6 +521,17 @@ Waiting Primitives
    Unlike :func:`~asyncio.wait_for`, ``wait()`` does not cancel the
    futures when a timeout occurs.
 
+   .. deprecated:: 3.8
+
+      If any awaitable in *aws* is a coroutine, it is automatically
+      scheduled as a Task.  Passing coroutines objects to
+      ``wait()`` directly is deprecated as it leads to
+      :ref:`confusing behavior <asyncio_example_wait_coroutine>`.
+
+   .. deprecated-removed:: 3.8 3.10
+
+      The *loop* parameter.
+
    .. _asyncio_example_wait_coroutine:
    .. note::
 
@@ -568,6 +575,9 @@ Waiting Primitives
    Raises :exc:`asyncio.TimeoutError` if the timeout occurs before
    all Futures are done.
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    Example::
 
        for f in as_completed(aws):
@@ -694,6 +704,9 @@ Task Object
    .. versionchanged:: 3.8
       Added the ``name`` parameter.
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    .. method:: cancel()
 
       Request the Task to be cancelled.
diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py
index e4f9e5297a19..bddfb0146402 100644
--- a/Lib/asyncio/subprocess.py
+++ b/Lib/asyncio/subprocess.py
@@ -224,6 +224,13 @@ def kill(self):
                                   **kwds):
     if loop is None:
         loop = events.get_event_loop()
+    else:
+        warnings.warn("The loop argument is deprecated since Python 3.8 "
+                      "and scheduled for removal in Python 3.10.",
+                      DeprecationWarning,
+                      stacklevel=2
+        )
+
     protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
                                                         loop=loop,
                                                         _asyncio_internal=True)
@@ -239,6 +246,12 @@ def kill(self):
                                  limit=streams._DEFAULT_LIMIT, **kwds):
     if loop is None:
         loop = events.get_event_loop()
+    else:
+        warnings.warn("The loop argument is deprecated since Python 3.8 "
+                      "and scheduled for removal in Python 3.10.",
+                      DeprecationWarning,
+                      stacklevel=2
+        )
     protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
                                                         loop=loop,
                                                         _asyncio_internal=True)
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
index cd4832cfee24..a0cb884eacab 100644
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -573,10 +573,17 @@ def as_completed(fs, *, loop=None, timeout=None):
     """
     if futures.isfuture(fs) or coroutines.iscoroutine(fs):
         raise TypeError(f"expect a list of futures, not {type(fs).__name__}")
-    loop = loop if loop is not None else events.get_event_loop()
-    todo = {ensure_future(f, loop=loop) for f in set(fs)}
+
     from .queues import Queue  # Import here to avoid circular import problem.
     done = Queue(loop=loop)
+
+    if loop is None:
+        loop = events.get_event_loop()
+    else:
+        warnings.warn("The loop argument is deprecated since Python 3.8, "
+                      "and scheduled for removal in Python 3.10.",
+                      DeprecationWarning, stacklevel=2)
+    todo = {ensure_future(f, loop=loop) for f in set(fs)}
     timeout_handle = None
 
     def _on_timeout():
@@ -733,6 +740,10 @@ def gather(*coros_or_futures, loop=None, return_exceptions=False):
     if not coros_or_futures:
         if loop is None:
             loop = events.get_event_loop()
+        else:
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
         outer = loop.create_future()
         outer.set_result([])
         return outer
@@ -842,6 +853,10 @@ def shield(arg, *, loop=None):
         except CancelledError:
             res = None
     """
+    if loop is not None:
+        warnings.warn("The loop argument is deprecated since Python 3.8, "
+                      "and scheduled for removal in Python 3.10.",
+                      DeprecationWarning, stacklevel=2)
     inner = ensure_future(arg, loop=loop)
     if inner.done():
         # Shortcut.
diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py
index 60fe52fa4b37..6325ee3983cc 100644
--- a/Lib/test/test_asyncio/test_streams.py
+++ b/Lib/test/test_asyncio/test_streams.py
@@ -855,9 +855,10 @@ def test_read_all_from_pipe_reader(self):
         watcher.attach_loop(self.loop)
         try:
             asyncio.set_child_watcher(watcher)
-            create = asyncio.create_subprocess_exec(*args,
-                                                    pass_fds={wfd},
-                                                    loop=self.loop)
+            create = asyncio.create_subprocess_exec(
+                *args,
+                pass_fds={wfd},
+            )
             proc = self.loop.run_until_complete(create)
             self.loop.run_until_complete(proc.wait())
         finally:
diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py
index 2cfe6576d623..3ad18e5c5166 100644
--- a/Lib/test/test_asyncio/test_subprocess.py
+++ b/Lib/test/test_asyncio/test_subprocess.py
@@ -109,10 +109,10 @@ def test_stdin_stdout(self):
 
         async def run(data):
             proc = await asyncio.create_subprocess_exec(
-                                          *args,
-                                          stdin=subprocess.PIPE,
-                                          stdout=subprocess.PIPE,
-                                          loop=self.loop)
+                *args,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+            )
 
             # feed data
             proc.stdin.write(data)
@@ -135,10 +135,10 @@ def test_communicate(self):
 
         async def run(data):
             proc = await asyncio.create_subprocess_exec(
-                                          *args,
-                                          stdin=subprocess.PIPE,
-                                          stdout=subprocess.PIPE,
-                                          loop=self.loop)
+                *args,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+            )
             stdout, stderr = await proc.communicate(data)
             return proc.returncode, stdout
 
@@ -149,25 +149,28 @@ def test_communicate(self):
         self.assertEqual(stdout, b'some data')
 
     def test_shell(self):
-        create = asyncio.create_subprocess_shell('exit 7',
-                                                 loop=self.loop)
-        proc = self.loop.run_until_complete(create)
+        proc = self.loop.run_until_complete(
+            asyncio.create_subprocess_shell('exit 7')
+        )
         exitcode = self.loop.run_until_complete(proc.wait())
         self.assertEqual(exitcode, 7)
 
     def test_start_new_session(self):
         # start the new process in a new session
-        create = asyncio.create_subprocess_shell('exit 8',
-                                                 start_new_session=True,
-                                                 loop=self.loop)
-        proc = self.loop.run_until_complete(create)
+        proc = self.loop.run_until_complete(
+            asyncio.create_subprocess_shell(
+                'exit 8',
+                start_new_session=True,
+            )
+        )
         exitcode = self.loop.run_until_complete(proc.wait())
         self.assertEqual(exitcode, 8)
 
     def test_kill(self):
         args = PROGRAM_BLOCKED
-        create = asyncio.create_subprocess_exec(*args, loop=self.loop)
-        proc = self.loop.run_until_complete(create)
+        proc = self.loop.run_until_complete(
+            asyncio.create_subprocess_exec(*args)
+        )
         proc.kill()
         returncode = self.loop.run_until_complete(proc.wait())
         if sys.platform == 'win32':
@@ -178,8 +181,9 @@ def test_kill(self):
 
     def test_terminate(self):
         args = PROGRAM_BLOCKED
-        create = asyncio.create_subprocess_exec(*args, loop=self.loop)
-        proc = self.loop.run_until_complete(create)
+        proc = self.loop.run_until_complete(
+            asyncio.create_subprocess_exec(*args)
+        )
         proc.terminate()
         returncode = self.loop.run_until_complete(proc.wait())
         if sys.platform == 'win32':
@@ -197,10 +201,12 @@ def test_send_signal(self):
         try:
             code = 'import time; print("sleeping", flush=True); time.sleep(3600)'
             args = [sys.executable, '-c', code]
-            create = asyncio.create_subprocess_exec(*args,
-                                                    stdout=subprocess.PIPE,
-                                                    loop=self.loop)
-            proc = self.loop.run_until_complete(create)
+            proc = self.loop.run_until_complete(
+                asyncio.create_subprocess_exec(
+                    *args,
+                    stdout=subprocess.PIPE,
+                )
+            )
 
             async def send_signal(proc):
                 # basic synchronization to wait until the program is sleeping
@@ -221,11 +227,13 @@ def prepare_broken_pipe_test(self):
         large_data = b'x' * support.PIPE_MAX_SIZE
 
         # the program ends before the stdin can be feeded
-        create = asyncio.create_subprocess_exec(
-                             sys.executable, '-c', 'pass',
-                             stdin=subprocess.PIPE,
-                             loop=self.loop)
-        proc = self.loop.run_until_complete(create)
+        proc = self.loop.run_until_complete(
+            asyncio.create_subprocess_exec(
+                sys.executable, '-c', 'pass',
+                stdin=subprocess.PIPE,
+            )
+        )
+
         return (proc, large_data)
 
     def test_stdin_broken_pipe(self):
@@ -273,11 +281,11 @@ def test_pause_reading(self):
             self.loop.connect_read_pipe = connect_read_pipe_mock
 
             proc = await asyncio.create_subprocess_exec(
-                                         sys.executable, '-c', code,
-                                         stdin=asyncio.subprocess.PIPE,
-                                         stdout=asyncio.subprocess.PIPE,
-                                         limit=limit,
-                                         loop=self.loop)
+                sys.executable, '-c', code,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                limit=limit,
+            )
             stdout_transport = proc._transport.get_pipe_transport(1)
 
             stdout, stderr = await proc.communicate()
@@ -301,12 +309,12 @@ def test_stdin_not_inheritable(self):
         async def len_message(message):
             code = 'import sys; data = sys.stdin.read(); print(len(data))'
             proc = await asyncio.create_subprocess_exec(
-                                          sys.executable, '-c', code,
-                                          stdin=asyncio.subprocess.PIPE,
-                                          stdout=asyncio.subprocess.PIPE,
-                                          stderr=asyncio.subprocess.PIPE,
-                                          close_fds=False,
-                                          loop=self.loop)
+                sys.executable, '-c', code,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                close_fds=False,
+            )
             stdout, stderr = await proc.communicate(message)
             exitcode = await proc.wait()
             return (stdout, exitcode)
@@ -320,12 +328,12 @@ def test_empty_input(self):
         async def empty_input():
             code = 'import sys; data = sys.stdin.read(); print(len(data))'
             proc = await asyncio.create_subprocess_exec(
-                                          sys.executable, '-c', code,
-                                          stdin=asyncio.subprocess.PIPE,
-                                          stdout=asyncio.subprocess.PIPE,
-                                          stderr=asyncio.subprocess.PIPE,
-                                          close_fds=False,
-                                          loop=self.loop)
+                sys.executable, '-c', code,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                close_fds=False,
+            )
             stdout, stderr = await proc.communicate(b'')
             exitcode = await proc.wait()
             return (stdout, exitcode)
@@ -339,12 +347,12 @@ def test_devnull_input(self):
         async def empty_input():
             code = 'import sys; data = sys.stdin.read(); print(len(data))'
             proc = await asyncio.create_subprocess_exec(
-                                          sys.executable, '-c', code,
-                                          stdin=asyncio.subprocess.DEVNULL,
-                                          stdout=asyncio.subprocess.PIPE,
-                                          stderr=asyncio.subprocess.PIPE,
-                                          close_fds=False,
-                                          loop=self.loop)
+                sys.executable, '-c', code,
+                stdin=asyncio.subprocess.DEVNULL,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                close_fds=False,
+            )
             stdout, stderr = await proc.communicate()
             exitcode = await proc.wait()
             return (stdout, exitcode)
@@ -358,12 +366,12 @@ def test_devnull_output(self):
         async def empty_output():
             code = 'import sys; data = sys.stdin.read(); print(len(data))'
             proc = await asyncio.create_subprocess_exec(
-                                          sys.executable, '-c', code,
-                                          stdin=asyncio.subprocess.PIPE,
-                                          stdout=asyncio.subprocess.DEVNULL,
-                                          stderr=asyncio.subprocess.PIPE,
-                                          close_fds=False,
-                                          loop=self.loop)
+                sys.executable, '-c', code,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.DEVNULL,
+                stderr=asyncio.subprocess.PIPE,
+                close_fds=False,
+            )
             stdout, stderr = await proc.communicate(b"abc")
             exitcode = await proc.wait()
             return (stdout, exitcode)
@@ -377,12 +385,12 @@ def test_devnull_error(self):
         async def empty_error():
             code = 'import sys; data = sys.stdin.read(); print(len(data))'
             proc = await asyncio.create_subprocess_exec(
-                                          sys.executable, '-c', code,
-                                          stdin=asyncio.subprocess.PIPE,
-                                          stdout=asyncio.subprocess.PIPE,
-                                          stderr=asyncio.subprocess.DEVNULL,
-                                          close_fds=False,
-                                          loop=self.loop)
+                sys.executable, '-c', code,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.DEVNULL,
+                close_fds=False,
+            )
             stdout, stderr = await proc.communicate(b"abc")
             exitcode = await proc.wait()
             return (stderr, exitcode)
@@ -395,9 +403,7 @@ def test_cancel_process_wait(self):
         # Issue #23140: cancel Process.wait()
 
         async def cancel_wait():
-            proc = await asyncio.create_subprocess_exec(
-                                          *PROGRAM_BLOCKED,
-                                          loop=self.loop)
+            proc = await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED)
 
             # Create an internal future waiting on the process exit
             task = self.loop.create_task(proc.wait())
@@ -419,8 +425,7 @@ def test_cancel_process_wait(self):
     def test_cancel_make_subprocess_transport_exec(self):
 
         async def cancel_make_transport():
-            coro = asyncio.create_subprocess_exec(*PROGRAM_BLOCKED,
-                                                  loop=self.loop)
+            coro = asyncio.create_subprocess_exec(*PROGRAM_BLOCKED)
             task = self.loop.create_task(coro)
 
             self.loop.call_soon(task.cancel)
@@ -524,7 +529,7 @@ def test_close_dont_kill_finished(self):
                 isinstance(self, SubprocessFastWatcherTests)):
             asyncio.get_child_watcher()._callbacks.clear()
 
-    def _test_popen_error(self, stdin):
+    async def _test_popen_error(self, stdin):
         if sys.platform == 'win32':
             target = 'asyncio.windows_utils.Popen'
         else:
@@ -533,23 +538,26 @@ def _test_popen_error(self, stdin):
             exc = ZeroDivisionError
             popen.side_effect = exc
 
-            create = asyncio.create_subprocess_exec(sys.executable, '-c',
-                                                    'pass', stdin=stdin,
-                                                    loop=self.loop)
             with warnings.catch_warnings(record=True) as warns:
                 with self.assertRaises(exc):
-                    self.loop.run_until_complete(create)
+                    await asyncio.create_subprocess_exec(
+                        sys.executable,
+                        '-c',
+                        'pass',
+                        stdin=stdin
+                    )
                 self.assertEqual(warns, [])
 
     def test_popen_error(self):
         # Issue #24763: check that the subprocess transport is closed
         # when BaseSubprocessTransport fails
-        self._test_popen_error(stdin=None)
+        self.loop.run_until_complete(self._test_popen_error(stdin=None))
 
     def test_popen_error_with_stdin_pipe(self):
         # Issue #35721: check that newly created socket pair is closed when
         # Popen fails
-        self._test_popen_error(stdin=subprocess.PIPE)
+        self.loop.run_until_complete(
+            self._test_popen_error(stdin=subprocess.PIPE))
 
     def test_read_stdout_after_process_exit(self):
 
@@ -560,12 +568,11 @@ def test_read_stdout_after_process_exit(self):
                               'sys.stdout.flush()',
                               'sys.exit(1)'])
 
-            fut = asyncio.create_subprocess_exec(
+            process = await asyncio.create_subprocess_exec(
                 sys.executable, '-c', code,
                 stdout=asyncio.subprocess.PIPE,
-                loop=self.loop)
+            )
 
-            process = await fut
             while True:
                 data = await process.stdout.read(65536)
                 if data:
@@ -620,7 +627,6 @@ def test_create_subprocess_shell_text_mode_fails(self):
 
         self.loop.run_until_complete(execute())
 
-
     def test_create_subprocess_exec_with_path(self):
         async def execute():
             p = await subprocess.create_subprocess_exec(
@@ -632,6 +638,26 @@ def test_create_subprocess_exec_with_path(self):
 
         self.assertIsNone(self.loop.run_until_complete(execute()))
 
+    def test_exec_loop_deprecated(self):
+        async def go():
+            with self.assertWarns(DeprecationWarning):
+                proc = await asyncio.create_subprocess_exec(
+                    sys.executable, '-c', 'pass',
+                    loop=self.loop,
+                )
+            await proc.wait()
+        self.loop.run_until_complete(go())
+
+    def test_shell_loop_deprecated(self):
+        async def go():
+            with self.assertWarns(DeprecationWarning):
+                proc = await asyncio.create_subprocess_shell(
+                    "exit 0",
+                    loop=self.loop,
+                )
+            await proc.wait()
+        self.loop.run_until_complete(go())
+
 
 if sys.platform != 'win32':
     # Unix
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
index 3fbb2a1a1629..6e832eab6d58 100644
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -1810,7 +1810,7 @@ def test_shield_effect(self):
 
         async def outer():
             nonlocal proof
-            await asyncio.shield(inner(), loop=self.loop)
+            await asyncio.shield(inner())
             proof += 100
 
         f = asyncio.ensure_future(outer(), loop=self.loop)
@@ -1825,8 +1825,8 @@ def test_shield_effect(self):
     def test_shield_gather(self):
         child1 = self.new_future(self.loop)
         child2 = self.new_future(self.loop)
-        parent = asyncio.gather(child1, child2, loop=self.loop)
-        outer = asyncio.shield(parent, loop=self.loop)
+        parent = asyncio.gather(child1, child2)
+        outer = asyncio.shield(parent)
         test_utils.run_briefly(self.loop)
         outer.cancel()
         test_utils.run_briefly(self.loop)
@@ -1839,9 +1839,9 @@ def test_shield_gather(self):
     def test_gather_shield(self):
         child1 = self.new_future(self.loop)
         child2 = self.new_future(self.loop)
-        inner1 = asyncio.shield(child1, loop=self.loop)
-        inner2 = asyncio.shield(child2, loop=self.loop)
-        parent = asyncio.gather(inner1, inner2, loop=self.loop)
+        inner1 = asyncio.shield(child1)
+        inner2 = asyncio.shield(child2)
+        parent = asyncio.gather(inner1, inner2)
         test_utils.run_briefly(self.loop)
         parent.cancel()
         # This should cancel inner1 and inner2 but bot child1 and child2.
@@ -2981,7 +2981,8 @@ def _check_empty_sequence(self, seq_or_iter):
         self._run_loop(self.one_loop)
         self.assertTrue(fut.done())
         self.assertEqual(fut.result(), [])
-        fut = asyncio.gather(*seq_or_iter, loop=self.other_loop)
+        with self.assertWarns(DeprecationWarning):
+            fut = asyncio.gather(*seq_or_iter, loop=self.other_loop)
         self.assertIs(fut._loop, self.other_loop)
 
     def test_constructor_empty_sequence(self):


From webhook-mailer at python.org  Thu Sep 12 08:54:52 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Thu, 12 Sep 2019 12:54:52 -0000
Subject: [Python-checkins] bpo-37935: Added tests for os.walk(),
 glob.iglob() and Path.glob() (GH-15956)
Message-ID: <mailman.658.1568292893.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f9dc2ad89032201427ed5f08061c703794627ad9
commit: f9dc2ad89032201427ed5f08061c703794627ad9
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-12T13:54:48+01:00
summary:

bpo-37935: Added tests for os.walk(), glob.iglob() and Path.glob() (GH-15956)

Test that they do not keep too many file descriptors open for the host OS in a reasonable test scenario.

See [bpo-37935](https://bugs.python.org/issue37935).

files:
M Lib/test/test_glob.py
M Lib/test/test_os.py
M Lib/test/test_pathlib.py

diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py
index 767bd3764b89..cba8c7c60e21 100644
--- a/Lib/test/test_glob.py
+++ b/Lib/test/test_glob.py
@@ -264,6 +264,23 @@ def test_recursive_glob(self):
                 expect += [join('sym3', 'EF')]
             eq(glob.glob(join('**', 'EF'), recursive=True), expect)
 
+    def test_glob_many_open_files(self):
+        depth = 30
+        base = os.path.join(self.tempdir, 'deep')
+        p = os.path.join(base, *(['d']*depth))
+        os.makedirs(p)
+        pattern = os.path.join(base, *(['*']*depth))
+        iters = [glob.iglob(pattern, recursive=True) for j in range(100)]
+        for it in iters:
+            self.assertEqual(next(it), p)
+        pattern = os.path.join(base, '**', 'd')
+        iters = [glob.iglob(pattern, recursive=True) for j in range(100)]
+        p = base
+        for i in range(depth):
+            p = os.path.join(p, 'd')
+            for it in iters:
+                self.assertEqual(next(it), p)
+
 
 @skip_unless_symlink
 class SymlinkLoopGlobTests(unittest.TestCase):
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 8ff0296fad0b..4a076e3bbf54 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -1178,6 +1178,27 @@ def test_walk_bad_dir(self):
         finally:
             os.rename(path1new, path1)
 
+    def test_walk_many_open_files(self):
+        depth = 30
+        base = os.path.join(support.TESTFN, 'deep')
+        p = os.path.join(base, *(['d']*depth))
+        os.makedirs(p)
+
+        iters = [self.walk(base, topdown=False) for j in range(100)]
+        for i in range(depth + 1):
+            expected = (p, ['d'] if i else [], [])
+            for it in iters:
+                self.assertEqual(next(it), expected)
+            p = os.path.dirname(p)
+
+        iters = [self.walk(base, topdown=True) for j in range(100)]
+        p = base
+        for i in range(depth + 1):
+            expected = (p, ['d'] if i < depth else [], [])
+            for it in iters:
+                self.assertEqual(next(it), expected)
+            p = os.path.join(p, 'd')
+
 
 @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()")
 class FwalkTests(WalkTests):
@@ -1247,6 +1268,10 @@ def test_fd_leak(self):
         self.addCleanup(os.close, newfd)
         self.assertEqual(newfd, minfd)
 
+    # fwalk() keeps file descriptors open
+    test_walk_many_open_files = None
+
+
 class BytesWalkTests(WalkTests):
     """Tests for os.walk() with bytes."""
     def walk(self, top, **kwargs):
diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py
index f3b385fbf367..f3304f01b4be 100644
--- a/Lib/test/test_pathlib.py
+++ b/Lib/test/test_pathlib.py
@@ -1562,6 +1562,23 @@ def test_rglob_symlink_loop(self):
                   }
         self.assertEqual(given, {p / x for x in expect})
 
+    def test_glob_many_open_files(self):
+        depth = 30
+        P = self.cls
+        base = P(BASE) / 'deep'
+        p = P(base, *(['d']*depth))
+        p.mkdir(parents=True)
+        pattern = '/'.join(['*'] * depth)
+        iters = [base.glob(pattern) for j in range(100)]
+        for it in iters:
+            self.assertEqual(next(it), p)
+        iters = [base.rglob('d') for j in range(100)]
+        p = base
+        for i in range(depth):
+            p = p / 'd'
+            for it in iters:
+                self.assertEqual(next(it), p)
+
     def test_glob_dotdot(self):
         # ".." is not special in globs.
         P = self.cls


From webhook-mailer at python.org  Thu Sep 12 08:59:54 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 12:59:54 -0000
Subject: [Python-checkins] bpo-36373: Deprecate explicit loop in task and
 subprocess API (GH-16033)
Message-ID: <mailman.659.1568293196.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/345bfc990f5f3e873774051d43136b06bfed82cb
commit: 345bfc990f5f3e873774051d43136b06bfed82cb
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T05:59:50-07:00
summary:

bpo-36373: Deprecate explicit loop in task and subprocess API (GH-16033)

(cherry picked from commit a488879cbaf4b8b52699cadccf73bb4c271bcb29)

Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com>

files:
M Doc/library/asyncio-subprocess.rst
M Doc/library/asyncio-task.rst
M Lib/asyncio/subprocess.py
M Lib/asyncio/tasks.py
M Lib/test/test_asyncio/test_streams.py
M Lib/test/test_asyncio/test_subprocess.py
M Lib/test/test_asyncio/test_tasks.py

diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst
index 444fb6361b5e..bd92257c82ef 100644
--- a/Doc/library/asyncio-subprocess.rst
+++ b/Doc/library/asyncio-subprocess.rst
@@ -71,6 +71,10 @@ Creating Subprocesses
    See the documentation of :meth:`loop.subprocess_exec` for other
    parameters.
 
+   .. deprecated-removed:: 3.8 3.10
+
+      The *loop* parameter.
+
 .. coroutinefunction:: create_subprocess_shell(cmd, stdin=None, \
                           stdout=None, stderr=None, loop=None, \
                           limit=None, \*\*kwds)
@@ -95,6 +99,10 @@ Creating Subprocesses
    escape whitespace and special shell characters in strings that are going
    to be used to construct shell commands.
 
+   .. deprecated-removed:: 3.8 3.10
+
+      The *loop* parameter.
+
 .. note::
 
    The default asyncio event loop implementation on **Windows** does not
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index 1fcdcb985d88..57e0e07ad3bb 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -334,6 +334,9 @@ Running Tasks Concurrently
    cancellation of one submitted Task/Future to cause other
    Tasks/Futures to be cancelled.
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    .. _asyncio_example_gather:
 
    Example::
@@ -411,6 +414,9 @@ Shielding From Cancellation
        except CancelledError:
            res = None
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
 
 Timeouts
 ========
@@ -478,22 +484,12 @@ Waiting Primitives
    set concurrently and block until the condition specified
    by *return_when*.
 
-   .. deprecated:: 3.8
-
-      If any awaitable in *aws* is a coroutine, it is automatically
-      scheduled as a Task.  Passing coroutines objects to
-      ``wait()`` directly is deprecated as it leads to
-      :ref:`confusing behavior <asyncio_example_wait_coroutine>`.
-
    Returns two sets of Tasks/Futures: ``(done, pending)``.
 
    Usage::
 
         done, pending = await asyncio.wait(aws)
 
-   .. deprecated-removed:: 3.8 3.10
-      The *loop* parameter.
-
    *timeout* (a float or int), if specified, can be used to control
    the maximum number of seconds to wait before returning.
 
@@ -525,6 +521,17 @@ Waiting Primitives
    Unlike :func:`~asyncio.wait_for`, ``wait()`` does not cancel the
    futures when a timeout occurs.
 
+   .. deprecated:: 3.8
+
+      If any awaitable in *aws* is a coroutine, it is automatically
+      scheduled as a Task.  Passing coroutines objects to
+      ``wait()`` directly is deprecated as it leads to
+      :ref:`confusing behavior <asyncio_example_wait_coroutine>`.
+
+   .. deprecated-removed:: 3.8 3.10
+
+      The *loop* parameter.
+
    .. _asyncio_example_wait_coroutine:
    .. note::
 
@@ -568,6 +575,9 @@ Waiting Primitives
    Raises :exc:`asyncio.TimeoutError` if the timeout occurs before
    all Futures are done.
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    Example::
 
        for f in as_completed(aws):
@@ -694,6 +704,9 @@ Task Object
    .. versionchanged:: 3.8
       Added the ``name`` parameter.
 
+   .. deprecated-removed:: 3.8 3.10
+      The *loop* parameter.
+
    .. method:: cancel()
 
       Request the Task to be cancelled.
diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py
index e6c1c8b61cc3..ce504b8b0cee 100644
--- a/Lib/asyncio/subprocess.py
+++ b/Lib/asyncio/subprocess.py
@@ -224,6 +224,13 @@ def kill(self):
                                   **kwds):
     if loop is None:
         loop = events.get_event_loop()
+    else:
+        warnings.warn("The loop argument is deprecated since Python 3.8 "
+                      "and scheduled for removal in Python 3.10.",
+                      DeprecationWarning,
+                      stacklevel=2
+        )
+
     protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
                                                         loop=loop,
                                                         _asyncio_internal=True)
@@ -239,6 +246,12 @@ def kill(self):
                                  limit=streams._DEFAULT_LIMIT, **kwds):
     if loop is None:
         loop = events.get_event_loop()
+    else:
+        warnings.warn("The loop argument is deprecated since Python 3.8 "
+                      "and scheduled for removal in Python 3.10.",
+                      DeprecationWarning,
+                      stacklevel=2
+        )
     protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
                                                         loop=loop,
                                                         _asyncio_internal=True)
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
index cd4832cfee24..a0cb884eacab 100644
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -573,10 +573,17 @@ def as_completed(fs, *, loop=None, timeout=None):
     """
     if futures.isfuture(fs) or coroutines.iscoroutine(fs):
         raise TypeError(f"expect a list of futures, not {type(fs).__name__}")
-    loop = loop if loop is not None else events.get_event_loop()
-    todo = {ensure_future(f, loop=loop) for f in set(fs)}
+
     from .queues import Queue  # Import here to avoid circular import problem.
     done = Queue(loop=loop)
+
+    if loop is None:
+        loop = events.get_event_loop()
+    else:
+        warnings.warn("The loop argument is deprecated since Python 3.8, "
+                      "and scheduled for removal in Python 3.10.",
+                      DeprecationWarning, stacklevel=2)
+    todo = {ensure_future(f, loop=loop) for f in set(fs)}
     timeout_handle = None
 
     def _on_timeout():
@@ -733,6 +740,10 @@ def gather(*coros_or_futures, loop=None, return_exceptions=False):
     if not coros_or_futures:
         if loop is None:
             loop = events.get_event_loop()
+        else:
+            warnings.warn("The loop argument is deprecated since Python 3.8, "
+                          "and scheduled for removal in Python 3.10.",
+                          DeprecationWarning, stacklevel=2)
         outer = loop.create_future()
         outer.set_result([])
         return outer
@@ -842,6 +853,10 @@ def shield(arg, *, loop=None):
         except CancelledError:
             res = None
     """
+    if loop is not None:
+        warnings.warn("The loop argument is deprecated since Python 3.8, "
+                      "and scheduled for removal in Python 3.10.",
+                      DeprecationWarning, stacklevel=2)
     inner = ensure_future(arg, loop=loop)
     if inner.done():
         # Shortcut.
diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py
index 60fe52fa4b37..6325ee3983cc 100644
--- a/Lib/test/test_asyncio/test_streams.py
+++ b/Lib/test/test_asyncio/test_streams.py
@@ -855,9 +855,10 @@ def test_read_all_from_pipe_reader(self):
         watcher.attach_loop(self.loop)
         try:
             asyncio.set_child_watcher(watcher)
-            create = asyncio.create_subprocess_exec(*args,
-                                                    pass_fds={wfd},
-                                                    loop=self.loop)
+            create = asyncio.create_subprocess_exec(
+                *args,
+                pass_fds={wfd},
+            )
             proc = self.loop.run_until_complete(create)
             self.loop.run_until_complete(proc.wait())
         finally:
diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py
index 2cfe6576d623..3ad18e5c5166 100644
--- a/Lib/test/test_asyncio/test_subprocess.py
+++ b/Lib/test/test_asyncio/test_subprocess.py
@@ -109,10 +109,10 @@ def test_stdin_stdout(self):
 
         async def run(data):
             proc = await asyncio.create_subprocess_exec(
-                                          *args,
-                                          stdin=subprocess.PIPE,
-                                          stdout=subprocess.PIPE,
-                                          loop=self.loop)
+                *args,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+            )
 
             # feed data
             proc.stdin.write(data)
@@ -135,10 +135,10 @@ def test_communicate(self):
 
         async def run(data):
             proc = await asyncio.create_subprocess_exec(
-                                          *args,
-                                          stdin=subprocess.PIPE,
-                                          stdout=subprocess.PIPE,
-                                          loop=self.loop)
+                *args,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+            )
             stdout, stderr = await proc.communicate(data)
             return proc.returncode, stdout
 
@@ -149,25 +149,28 @@ def test_communicate(self):
         self.assertEqual(stdout, b'some data')
 
     def test_shell(self):
-        create = asyncio.create_subprocess_shell('exit 7',
-                                                 loop=self.loop)
-        proc = self.loop.run_until_complete(create)
+        proc = self.loop.run_until_complete(
+            asyncio.create_subprocess_shell('exit 7')
+        )
         exitcode = self.loop.run_until_complete(proc.wait())
         self.assertEqual(exitcode, 7)
 
     def test_start_new_session(self):
         # start the new process in a new session
-        create = asyncio.create_subprocess_shell('exit 8',
-                                                 start_new_session=True,
-                                                 loop=self.loop)
-        proc = self.loop.run_until_complete(create)
+        proc = self.loop.run_until_complete(
+            asyncio.create_subprocess_shell(
+                'exit 8',
+                start_new_session=True,
+            )
+        )
         exitcode = self.loop.run_until_complete(proc.wait())
         self.assertEqual(exitcode, 8)
 
     def test_kill(self):
         args = PROGRAM_BLOCKED
-        create = asyncio.create_subprocess_exec(*args, loop=self.loop)
-        proc = self.loop.run_until_complete(create)
+        proc = self.loop.run_until_complete(
+            asyncio.create_subprocess_exec(*args)
+        )
         proc.kill()
         returncode = self.loop.run_until_complete(proc.wait())
         if sys.platform == 'win32':
@@ -178,8 +181,9 @@ def test_kill(self):
 
     def test_terminate(self):
         args = PROGRAM_BLOCKED
-        create = asyncio.create_subprocess_exec(*args, loop=self.loop)
-        proc = self.loop.run_until_complete(create)
+        proc = self.loop.run_until_complete(
+            asyncio.create_subprocess_exec(*args)
+        )
         proc.terminate()
         returncode = self.loop.run_until_complete(proc.wait())
         if sys.platform == 'win32':
@@ -197,10 +201,12 @@ def test_send_signal(self):
         try:
             code = 'import time; print("sleeping", flush=True); time.sleep(3600)'
             args = [sys.executable, '-c', code]
-            create = asyncio.create_subprocess_exec(*args,
-                                                    stdout=subprocess.PIPE,
-                                                    loop=self.loop)
-            proc = self.loop.run_until_complete(create)
+            proc = self.loop.run_until_complete(
+                asyncio.create_subprocess_exec(
+                    *args,
+                    stdout=subprocess.PIPE,
+                )
+            )
 
             async def send_signal(proc):
                 # basic synchronization to wait until the program is sleeping
@@ -221,11 +227,13 @@ def prepare_broken_pipe_test(self):
         large_data = b'x' * support.PIPE_MAX_SIZE
 
         # the program ends before the stdin can be feeded
-        create = asyncio.create_subprocess_exec(
-                             sys.executable, '-c', 'pass',
-                             stdin=subprocess.PIPE,
-                             loop=self.loop)
-        proc = self.loop.run_until_complete(create)
+        proc = self.loop.run_until_complete(
+            asyncio.create_subprocess_exec(
+                sys.executable, '-c', 'pass',
+                stdin=subprocess.PIPE,
+            )
+        )
+
         return (proc, large_data)
 
     def test_stdin_broken_pipe(self):
@@ -273,11 +281,11 @@ def test_pause_reading(self):
             self.loop.connect_read_pipe = connect_read_pipe_mock
 
             proc = await asyncio.create_subprocess_exec(
-                                         sys.executable, '-c', code,
-                                         stdin=asyncio.subprocess.PIPE,
-                                         stdout=asyncio.subprocess.PIPE,
-                                         limit=limit,
-                                         loop=self.loop)
+                sys.executable, '-c', code,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                limit=limit,
+            )
             stdout_transport = proc._transport.get_pipe_transport(1)
 
             stdout, stderr = await proc.communicate()
@@ -301,12 +309,12 @@ def test_stdin_not_inheritable(self):
         async def len_message(message):
             code = 'import sys; data = sys.stdin.read(); print(len(data))'
             proc = await asyncio.create_subprocess_exec(
-                                          sys.executable, '-c', code,
-                                          stdin=asyncio.subprocess.PIPE,
-                                          stdout=asyncio.subprocess.PIPE,
-                                          stderr=asyncio.subprocess.PIPE,
-                                          close_fds=False,
-                                          loop=self.loop)
+                sys.executable, '-c', code,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                close_fds=False,
+            )
             stdout, stderr = await proc.communicate(message)
             exitcode = await proc.wait()
             return (stdout, exitcode)
@@ -320,12 +328,12 @@ def test_empty_input(self):
         async def empty_input():
             code = 'import sys; data = sys.stdin.read(); print(len(data))'
             proc = await asyncio.create_subprocess_exec(
-                                          sys.executable, '-c', code,
-                                          stdin=asyncio.subprocess.PIPE,
-                                          stdout=asyncio.subprocess.PIPE,
-                                          stderr=asyncio.subprocess.PIPE,
-                                          close_fds=False,
-                                          loop=self.loop)
+                sys.executable, '-c', code,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                close_fds=False,
+            )
             stdout, stderr = await proc.communicate(b'')
             exitcode = await proc.wait()
             return (stdout, exitcode)
@@ -339,12 +347,12 @@ def test_devnull_input(self):
         async def empty_input():
             code = 'import sys; data = sys.stdin.read(); print(len(data))'
             proc = await asyncio.create_subprocess_exec(
-                                          sys.executable, '-c', code,
-                                          stdin=asyncio.subprocess.DEVNULL,
-                                          stdout=asyncio.subprocess.PIPE,
-                                          stderr=asyncio.subprocess.PIPE,
-                                          close_fds=False,
-                                          loop=self.loop)
+                sys.executable, '-c', code,
+                stdin=asyncio.subprocess.DEVNULL,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                close_fds=False,
+            )
             stdout, stderr = await proc.communicate()
             exitcode = await proc.wait()
             return (stdout, exitcode)
@@ -358,12 +366,12 @@ def test_devnull_output(self):
         async def empty_output():
             code = 'import sys; data = sys.stdin.read(); print(len(data))'
             proc = await asyncio.create_subprocess_exec(
-                                          sys.executable, '-c', code,
-                                          stdin=asyncio.subprocess.PIPE,
-                                          stdout=asyncio.subprocess.DEVNULL,
-                                          stderr=asyncio.subprocess.PIPE,
-                                          close_fds=False,
-                                          loop=self.loop)
+                sys.executable, '-c', code,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.DEVNULL,
+                stderr=asyncio.subprocess.PIPE,
+                close_fds=False,
+            )
             stdout, stderr = await proc.communicate(b"abc")
             exitcode = await proc.wait()
             return (stdout, exitcode)
@@ -377,12 +385,12 @@ def test_devnull_error(self):
         async def empty_error():
             code = 'import sys; data = sys.stdin.read(); print(len(data))'
             proc = await asyncio.create_subprocess_exec(
-                                          sys.executable, '-c', code,
-                                          stdin=asyncio.subprocess.PIPE,
-                                          stdout=asyncio.subprocess.PIPE,
-                                          stderr=asyncio.subprocess.DEVNULL,
-                                          close_fds=False,
-                                          loop=self.loop)
+                sys.executable, '-c', code,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.DEVNULL,
+                close_fds=False,
+            )
             stdout, stderr = await proc.communicate(b"abc")
             exitcode = await proc.wait()
             return (stderr, exitcode)
@@ -395,9 +403,7 @@ def test_cancel_process_wait(self):
         # Issue #23140: cancel Process.wait()
 
         async def cancel_wait():
-            proc = await asyncio.create_subprocess_exec(
-                                          *PROGRAM_BLOCKED,
-                                          loop=self.loop)
+            proc = await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED)
 
             # Create an internal future waiting on the process exit
             task = self.loop.create_task(proc.wait())
@@ -419,8 +425,7 @@ def test_cancel_process_wait(self):
     def test_cancel_make_subprocess_transport_exec(self):
 
         async def cancel_make_transport():
-            coro = asyncio.create_subprocess_exec(*PROGRAM_BLOCKED,
-                                                  loop=self.loop)
+            coro = asyncio.create_subprocess_exec(*PROGRAM_BLOCKED)
             task = self.loop.create_task(coro)
 
             self.loop.call_soon(task.cancel)
@@ -524,7 +529,7 @@ def test_close_dont_kill_finished(self):
                 isinstance(self, SubprocessFastWatcherTests)):
             asyncio.get_child_watcher()._callbacks.clear()
 
-    def _test_popen_error(self, stdin):
+    async def _test_popen_error(self, stdin):
         if sys.platform == 'win32':
             target = 'asyncio.windows_utils.Popen'
         else:
@@ -533,23 +538,26 @@ def _test_popen_error(self, stdin):
             exc = ZeroDivisionError
             popen.side_effect = exc
 
-            create = asyncio.create_subprocess_exec(sys.executable, '-c',
-                                                    'pass', stdin=stdin,
-                                                    loop=self.loop)
             with warnings.catch_warnings(record=True) as warns:
                 with self.assertRaises(exc):
-                    self.loop.run_until_complete(create)
+                    await asyncio.create_subprocess_exec(
+                        sys.executable,
+                        '-c',
+                        'pass',
+                        stdin=stdin
+                    )
                 self.assertEqual(warns, [])
 
     def test_popen_error(self):
         # Issue #24763: check that the subprocess transport is closed
         # when BaseSubprocessTransport fails
-        self._test_popen_error(stdin=None)
+        self.loop.run_until_complete(self._test_popen_error(stdin=None))
 
     def test_popen_error_with_stdin_pipe(self):
         # Issue #35721: check that newly created socket pair is closed when
         # Popen fails
-        self._test_popen_error(stdin=subprocess.PIPE)
+        self.loop.run_until_complete(
+            self._test_popen_error(stdin=subprocess.PIPE))
 
     def test_read_stdout_after_process_exit(self):
 
@@ -560,12 +568,11 @@ def test_read_stdout_after_process_exit(self):
                               'sys.stdout.flush()',
                               'sys.exit(1)'])
 
-            fut = asyncio.create_subprocess_exec(
+            process = await asyncio.create_subprocess_exec(
                 sys.executable, '-c', code,
                 stdout=asyncio.subprocess.PIPE,
-                loop=self.loop)
+            )
 
-            process = await fut
             while True:
                 data = await process.stdout.read(65536)
                 if data:
@@ -620,7 +627,6 @@ def test_create_subprocess_shell_text_mode_fails(self):
 
         self.loop.run_until_complete(execute())
 
-
     def test_create_subprocess_exec_with_path(self):
         async def execute():
             p = await subprocess.create_subprocess_exec(
@@ -632,6 +638,26 @@ def test_create_subprocess_exec_with_path(self):
 
         self.assertIsNone(self.loop.run_until_complete(execute()))
 
+    def test_exec_loop_deprecated(self):
+        async def go():
+            with self.assertWarns(DeprecationWarning):
+                proc = await asyncio.create_subprocess_exec(
+                    sys.executable, '-c', 'pass',
+                    loop=self.loop,
+                )
+            await proc.wait()
+        self.loop.run_until_complete(go())
+
+    def test_shell_loop_deprecated(self):
+        async def go():
+            with self.assertWarns(DeprecationWarning):
+                proc = await asyncio.create_subprocess_shell(
+                    "exit 0",
+                    loop=self.loop,
+                )
+            await proc.wait()
+        self.loop.run_until_complete(go())
+
 
 if sys.platform != 'win32':
     # Unix
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
index 3fbb2a1a1629..6e832eab6d58 100644
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -1810,7 +1810,7 @@ def test_shield_effect(self):
 
         async def outer():
             nonlocal proof
-            await asyncio.shield(inner(), loop=self.loop)
+            await asyncio.shield(inner())
             proof += 100
 
         f = asyncio.ensure_future(outer(), loop=self.loop)
@@ -1825,8 +1825,8 @@ def test_shield_effect(self):
     def test_shield_gather(self):
         child1 = self.new_future(self.loop)
         child2 = self.new_future(self.loop)
-        parent = asyncio.gather(child1, child2, loop=self.loop)
-        outer = asyncio.shield(parent, loop=self.loop)
+        parent = asyncio.gather(child1, child2)
+        outer = asyncio.shield(parent)
         test_utils.run_briefly(self.loop)
         outer.cancel()
         test_utils.run_briefly(self.loop)
@@ -1839,9 +1839,9 @@ def test_shield_gather(self):
     def test_gather_shield(self):
         child1 = self.new_future(self.loop)
         child2 = self.new_future(self.loop)
-        inner1 = asyncio.shield(child1, loop=self.loop)
-        inner2 = asyncio.shield(child2, loop=self.loop)
-        parent = asyncio.gather(inner1, inner2, loop=self.loop)
+        inner1 = asyncio.shield(child1)
+        inner2 = asyncio.shield(child2)
+        parent = asyncio.gather(inner1, inner2)
         test_utils.run_briefly(self.loop)
         parent.cancel()
         # This should cancel inner1 and inner2 but bot child1 and child2.
@@ -2981,7 +2981,8 @@ def _check_empty_sequence(self, seq_or_iter):
         self._run_loop(self.one_loop)
         self.assertTrue(fut.done())
         self.assertEqual(fut.result(), [])
-        fut = asyncio.gather(*seq_or_iter, loop=self.other_loop)
+        with self.assertWarns(DeprecationWarning):
+            fut = asyncio.gather(*seq_or_iter, loop=self.other_loop)
         self.assertIs(fut._loop, self.other_loop)
 
     def test_constructor_empty_sequence(self):


From webhook-mailer at python.org  Thu Sep 12 09:13:05 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 13:13:05 -0000
Subject: [Python-checkins] bpo-37363: Document internal audit events
 (GH-14663)
Message-ID: <mailman.660.1568293987.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ed4b3216e59df4806e5ef63b3357fc577a9443d7
commit: ed4b3216e59df4806e5ef63b3357fc577a9443d7
branch: master
author: Christian Heimes <christian at python.org>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-12T06:13:02-07:00
summary:

bpo-37363: Document internal audit events (GH-14663)



Three internal cpython events were not documented, yet.

Signed-off-by: Christian Heimes <christian at python.org>



https://bugs.python.org/issue37363

files:
M Doc/c-api/init.rst

diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index e0558f426dce..0f8ff3b0dde8 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -301,8 +301,9 @@ Initializing and finalizing the interpreter
    than once; this can happen if an application calls :c:func:`Py_Initialize` and
    :c:func:`Py_FinalizeEx` more than once.
 
-   .. versionadded:: 3.6
+   .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx
 
+   .. versionadded:: 3.6
 
 .. c:function:: void Py_Finalize()
 
@@ -992,12 +993,16 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
    be held, but may be held if it is necessary to serialize calls to this
    function.
 
+   .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New
+
 
 .. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp)
 
    Reset all information in an interpreter state object.  The global interpreter
    lock must be held.
 
+   .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear
+
 
 .. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp)
 


From webhook-mailer at python.org  Thu Sep 12 09:20:29 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 13:20:29 -0000
Subject: [Python-checkins] bpo-37363: Document internal audit events
 (GH-14663)
Message-ID: <mailman.661.1568294429.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2f01cf61874aa42eeb9999e49d51d4881792d9c4
commit: 2f01cf61874aa42eeb9999e49d51d4881792d9c4
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T06:20:26-07:00
summary:

bpo-37363: Document internal audit events (GH-14663)


Three internal cpython events were not documented, yet.

Signed-off-by: Christian Heimes <christian at python.org>

https://bugs.python.org/issue37363
(cherry picked from commit ed4b3216e59df4806e5ef63b3357fc577a9443d7)

Co-authored-by: Christian Heimes <christian at python.org>

files:
M Doc/c-api/init.rst

diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index e0558f426dce..0f8ff3b0dde8 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -301,8 +301,9 @@ Initializing and finalizing the interpreter
    than once; this can happen if an application calls :c:func:`Py_Initialize` and
    :c:func:`Py_FinalizeEx` more than once.
 
-   .. versionadded:: 3.6
+   .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx
 
+   .. versionadded:: 3.6
 
 .. c:function:: void Py_Finalize()
 
@@ -992,12 +993,16 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
    be held, but may be held if it is necessary to serialize calls to this
    function.
 
+   .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New
+
 
 .. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp)
 
    Reset all information in an interpreter state object.  The global interpreter
    lock must be held.
 
+   .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear
+
 
 .. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp)
 


From webhook-mailer at python.org  Thu Sep 12 09:42:27 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Thu, 12 Sep 2019 13:42:27 -0000
Subject: [Python-checkins] Fix depth-first-search computation in compile.c
 (GH-16042)
Message-ID: <mailman.662.1568295748.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/355f3e1e5caf16198255df573a1f5e8b98b30105
commit: 355f3e1e5caf16198255df573a1f5e8b98b30105
branch: master
author: Mark Shannon <mark at hotpy.org>
committer: T. Wouters <thomas at python.org>
date: 2019-09-12T06:42:23-07:00
summary:

Fix depth-first-search computation in compile.c (GH-16042)

files:
M Python/compile.c

diff --git a/Python/compile.c b/Python/compile.c
index edd8625cba91..4906ea8fac1a 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -5373,7 +5373,7 @@ compiler_visit_slice(struct compiler *c, slice_ty s, expr_context_ty ctx)
 /* End of the compiler section, beginning of the assembler section */
 
 /* do depth-first search of basic block graph, starting with block.
-   post records the block indices in post-order.
+   a_order records the block indices in order.
 
    XXX must handle implicit jumps from one block to next
 */
@@ -5382,7 +5382,7 @@ struct assembler {
     PyObject *a_bytecode;  /* string containing bytecode */
     int a_offset;              /* offset into bytecode */
     int a_nblocks;             /* number of reachable blocks */
-    basicblock **a_postorder; /* list of blocks in dfs postorder */
+    basicblock **a_order; /* list of blocks in dfs order */
     PyObject *a_lnotab;    /* string containing lnotab */
     int a_lnotab_off;      /* offset into lnotab */
     int a_lineno;              /* last lineno of emitted instruction */
@@ -5390,28 +5390,21 @@ struct assembler {
 };
 
 static void
-dfs(struct compiler *c, basicblock *b, struct assembler *a, int end)
+dfs(struct compiler *c, basicblock *entry, struct assembler *a)
 {
-    int i, j;
-
-    /* Get rid of recursion for normal control flow.
-       Since the number of blocks is limited, unused space in a_postorder
-       (from a_nblocks to end) can be used as a stack for still not ordered
-       blocks. */
-    for (j = end; b && !b->b_seen; b = b->b_next) {
+    /* Avoid excessive recursion by following 'next' links to the 
+     * end of the chain before handling any branches */
+    for (basicblock *b = entry; b; b = b->b_next) {
         b->b_seen = 1;
-        assert(a->a_nblocks < j);
-        a->a_postorder[--j] = b;
+        a->a_order[a->a_nblocks++] = b;
     }
-    while (j < end) {
-        b = a->a_postorder[j++];
-        for (i = 0; i < b->b_iused; i++) {
-            struct instr *instr = &b->b_instr[i];
-            if (instr->i_jrel || instr->i_jabs)
-                dfs(c, instr->i_target, a, j);
+    for (basicblock *b = entry; b; b = b->b_next) {
+        for (int i = 0; i < b->b_iused; i++) {
+            basicblock *target = b->b_instr[i].i_target;
+            if (target && !target->b_seen) {
+                dfs(c, target, a);
+            }
         }
-        assert(a->a_nblocks < j);
-        a->a_postorder[a->a_nblocks++] = b;
     }
 }
 
@@ -5517,9 +5510,9 @@ assemble_init(struct assembler *a, int nblocks, int firstlineno)
         PyErr_NoMemory();
         return 0;
     }
-    a->a_postorder = (basicblock **)PyObject_Malloc(
+    a->a_order = (basicblock **)PyObject_Malloc(
                                         sizeof(basicblock *) * nblocks);
-    if (!a->a_postorder) {
+    if (!a->a_order) {
         PyErr_NoMemory();
         return 0;
     }
@@ -5531,8 +5524,8 @@ assemble_free(struct assembler *a)
 {
     Py_XDECREF(a->a_bytecode);
     Py_XDECREF(a->a_lnotab);
-    if (a->a_postorder)
-        PyObject_Free(a->a_postorder);
+    if (a->a_order)
+        PyObject_Free(a->a_order);
 }
 
 static int
@@ -5693,8 +5686,8 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c)
        Replace block pointer with position in bytecode. */
     do {
         totsize = 0;
-        for (i = a->a_nblocks - 1; i >= 0; i--) {
-            b = a->a_postorder[i];
+        for (i = 0; i < a->a_nblocks; i++) {
+            b = a->a_order[i];
             bsize = blocksize(b);
             b->b_offset = totsize;
             totsize += bsize;
@@ -5998,14 +5991,15 @@ assemble(struct compiler *c, int addNone)
     }
     if (!assemble_init(&a, nblocks, c->u->u_firstlineno))
         goto error;
-    dfs(c, entryblock, &a, nblocks);
+    dfs(c, entryblock, &a);
+    assert(a.a_nblocks <= nblocks);
 
     /* Can't modify the bytecode after computing jump offsets. */
     assemble_jump_offsets(&a, c);
 
-    /* Emit code in reverse postorder from dfs. */
-    for (i = a.a_nblocks - 1; i >= 0; i--) {
-        b = a.a_postorder[i];
+    /* Emit code in order from dfs. */
+    for (i = 0; i < a.a_nblocks; i++) {
+        b = a.a_order[i];
         for (j = 0; j < b->b_iused; j++)
             if (!assemble_emit(&a, &b->b_instr[j]))
                 goto error;


From webhook-mailer at python.org  Thu Sep 12 09:50:50 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 13:50:50 -0000
Subject: [Python-checkins] bpo-38132: Check EVP_DigestUpdate for error
 (GH-16041)
Message-ID: <mailman.663.1568296251.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0d7cb5bb291d8645824581ae570a394e3d221e5e
commit: 0d7cb5bb291d8645824581ae570a394e3d221e5e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T06:50:46-07:00
summary:

bpo-38132: Check EVP_DigestUpdate for error (GH-16041)

(cherry picked from commit 8c74574e0aaf1a00719fbc9acbdc27a3923520aa)

Co-authored-by: Christian Heimes <christian at python.org>

files:
M Modules/_hashopenssl.c

diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 798317fb58b9..bea7e5ed3123 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -96,7 +96,7 @@ newEVPobject(void)
     return retval;
 }
 
-static void
+static int
 EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len)
 {
     unsigned int process;
@@ -108,11 +108,12 @@ EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len)
             process = Py_SAFE_DOWNCAST(len, Py_ssize_t, unsigned int);
         if (!EVP_DigestUpdate(self->ctx, (const void*)cp, process)) {
             _setException(PyExc_ValueError);
-            break;
+            return -1;
         }
         len -= process;
         cp += process;
     }
+    return 0;
 }
 
 /* Internal methods for a hash object */
@@ -243,6 +244,7 @@ static PyObject *
 EVP_update(EVPobject *self, PyObject *obj)
 /*[clinic end generated code: output=ec1d55ed2432e966 input=9b30ec848f015501]*/
 {
+    int result;
     Py_buffer view;
 
     GET_BUFFER_VIEW_OR_ERROUT(obj, &view);
@@ -255,14 +257,17 @@ EVP_update(EVPobject *self, PyObject *obj)
     if (self->lock != NULL) {
         Py_BEGIN_ALLOW_THREADS
         PyThread_acquire_lock(self->lock, 1);
-        EVP_hash(self, view.buf, view.len);
+        result = EVP_hash(self, view.buf, view.len);
         PyThread_release_lock(self->lock);
         Py_END_ALLOW_THREADS
     } else {
-        EVP_hash(self, view.buf, view.len);
+        result = EVP_hash(self, view.buf, view.len);
     }
 
     PyBuffer_Release(&view);
+
+    if (result == -1)
+        return NULL;
     Py_RETURN_NONE;
 }
 
@@ -396,6 +401,7 @@ static PyObject *
 EVPnew(const EVP_MD *digest,
        const unsigned char *cp, Py_ssize_t len)
 {
+    int result = 0;
     EVPobject *self;
 
     if (!digest) {
@@ -415,10 +421,14 @@ EVPnew(const EVP_MD *digest,
     if (cp && len) {
         if (len >= HASHLIB_GIL_MINSIZE) {
             Py_BEGIN_ALLOW_THREADS
-            EVP_hash(self, cp, len);
+            result = EVP_hash(self, cp, len);
             Py_END_ALLOW_THREADS
         } else {
-            EVP_hash(self, cp, len);
+            result = EVP_hash(self, cp, len);
+        }
+        if (result == -1) {
+            Py_DECREF(self);
+            return NULL;
         }
     }
 


From webhook-mailer at python.org  Thu Sep 12 10:05:37 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Thu, 12 Sep 2019 14:05:37 -0000
Subject: [Python-checkins] Revert "Fix depth-first-search computation in
 compile.c (GH-16042)" (GH-16050)
Message-ID: <mailman.664.1568297137.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/99b54d68172ad64ba3d0fdc0137f0df88c28ea2b
commit: 99b54d68172ad64ba3d0fdc0137f0df88c28ea2b
branch: master
author: T. Wouters <thomas at python.org>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-12T15:05:33+01:00
summary:

Revert "Fix depth-first-search computation in compile.c (GH-16042)" (GH-16050)

This reverts commit 355f3e1e5caf16198255df573a1f5e8b98b30105.

bpo-38135

files:
M Python/compile.c

diff --git a/Python/compile.c b/Python/compile.c
index 4906ea8fac1a..edd8625cba91 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -5373,7 +5373,7 @@ compiler_visit_slice(struct compiler *c, slice_ty s, expr_context_ty ctx)
 /* End of the compiler section, beginning of the assembler section */
 
 /* do depth-first search of basic block graph, starting with block.
-   a_order records the block indices in order.
+   post records the block indices in post-order.
 
    XXX must handle implicit jumps from one block to next
 */
@@ -5382,7 +5382,7 @@ struct assembler {
     PyObject *a_bytecode;  /* string containing bytecode */
     int a_offset;              /* offset into bytecode */
     int a_nblocks;             /* number of reachable blocks */
-    basicblock **a_order; /* list of blocks in dfs order */
+    basicblock **a_postorder; /* list of blocks in dfs postorder */
     PyObject *a_lnotab;    /* string containing lnotab */
     int a_lnotab_off;      /* offset into lnotab */
     int a_lineno;              /* last lineno of emitted instruction */
@@ -5390,21 +5390,28 @@ struct assembler {
 };
 
 static void
-dfs(struct compiler *c, basicblock *entry, struct assembler *a)
+dfs(struct compiler *c, basicblock *b, struct assembler *a, int end)
 {
-    /* Avoid excessive recursion by following 'next' links to the 
-     * end of the chain before handling any branches */
-    for (basicblock *b = entry; b; b = b->b_next) {
+    int i, j;
+
+    /* Get rid of recursion for normal control flow.
+       Since the number of blocks is limited, unused space in a_postorder
+       (from a_nblocks to end) can be used as a stack for still not ordered
+       blocks. */
+    for (j = end; b && !b->b_seen; b = b->b_next) {
         b->b_seen = 1;
-        a->a_order[a->a_nblocks++] = b;
+        assert(a->a_nblocks < j);
+        a->a_postorder[--j] = b;
     }
-    for (basicblock *b = entry; b; b = b->b_next) {
-        for (int i = 0; i < b->b_iused; i++) {
-            basicblock *target = b->b_instr[i].i_target;
-            if (target && !target->b_seen) {
-                dfs(c, target, a);
-            }
+    while (j < end) {
+        b = a->a_postorder[j++];
+        for (i = 0; i < b->b_iused; i++) {
+            struct instr *instr = &b->b_instr[i];
+            if (instr->i_jrel || instr->i_jabs)
+                dfs(c, instr->i_target, a, j);
         }
+        assert(a->a_nblocks < j);
+        a->a_postorder[a->a_nblocks++] = b;
     }
 }
 
@@ -5510,9 +5517,9 @@ assemble_init(struct assembler *a, int nblocks, int firstlineno)
         PyErr_NoMemory();
         return 0;
     }
-    a->a_order = (basicblock **)PyObject_Malloc(
+    a->a_postorder = (basicblock **)PyObject_Malloc(
                                         sizeof(basicblock *) * nblocks);
-    if (!a->a_order) {
+    if (!a->a_postorder) {
         PyErr_NoMemory();
         return 0;
     }
@@ -5524,8 +5531,8 @@ assemble_free(struct assembler *a)
 {
     Py_XDECREF(a->a_bytecode);
     Py_XDECREF(a->a_lnotab);
-    if (a->a_order)
-        PyObject_Free(a->a_order);
+    if (a->a_postorder)
+        PyObject_Free(a->a_postorder);
 }
 
 static int
@@ -5686,8 +5693,8 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c)
        Replace block pointer with position in bytecode. */
     do {
         totsize = 0;
-        for (i = 0; i < a->a_nblocks; i++) {
-            b = a->a_order[i];
+        for (i = a->a_nblocks - 1; i >= 0; i--) {
+            b = a->a_postorder[i];
             bsize = blocksize(b);
             b->b_offset = totsize;
             totsize += bsize;
@@ -5991,15 +5998,14 @@ assemble(struct compiler *c, int addNone)
     }
     if (!assemble_init(&a, nblocks, c->u->u_firstlineno))
         goto error;
-    dfs(c, entryblock, &a);
-    assert(a.a_nblocks <= nblocks);
+    dfs(c, entryblock, &a, nblocks);
 
     /* Can't modify the bytecode after computing jump offsets. */
     assemble_jump_offsets(&a, c);
 
-    /* Emit code in order from dfs. */
-    for (i = 0; i < a.a_nblocks; i++) {
-        b = a.a_order[i];
+    /* Emit code in reverse postorder from dfs. */
+    for (i = a.a_nblocks - 1; i >= 0; i--) {
+        b = a.a_postorder[i];
         for (j = 0; j < b->b_iused; j++)
             if (!assemble_emit(&a, &b->b_instr[j]))
                 goto error;


From webhook-mailer at python.org  Thu Sep 12 10:13:48 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Thu, 12 Sep 2019 14:13:48 -0000
Subject: [Python-checkins] bpo-36991: Fix incorrect exception escaping
 ZipFile.extract() (GH-13632)
Message-ID: <mailman.665.1568297628.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2f1b857562b0f1601c9019db74c29b7d7e21ac9f
commit: 2f1b857562b0f1601c9019db74c29b7d7e21ac9f
branch: master
author: Berker Peksag <berker.peksag at gmail.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-12T15:13:44+01:00
summary:

bpo-36991: Fix incorrect exception escaping ZipFile.extract() (GH-13632)

files:
A Misc/NEWS.d/next/Library/2019-09-12-14-52-38.bpo-36991.1OcSm8.rst
M Lib/test/test_zipfile.py
M Lib/zipfile.py

diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
index 99d599eac79a..f812188a7108 100644
--- a/Lib/test/test_zipfile.py
+++ b/Lib/test/test_zipfile.py
@@ -9,6 +9,7 @@
 import sys
 import time
 import unittest
+import unittest.mock as mock
 import zipfile
 
 
@@ -1766,6 +1767,16 @@ def test_seek_tell(self):
                 fp.seek(0, os.SEEK_SET)
                 self.assertEqual(fp.tell(), 0)
 
+    @requires_bz2
+    def test_decompress_without_3rd_party_library(self):
+        data = b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+        zip_file = io.BytesIO(data)
+        with zipfile.ZipFile(zip_file, 'w', compression=zipfile.ZIP_BZIP2) as zf:
+            zf.writestr('a.txt', b'a')
+        with mock.patch('zipfile.bz2', None):
+            with zipfile.ZipFile(zip_file) as zf:
+                self.assertRaises(RuntimeError, zf.extract, 'a.txt')
+
     def tearDown(self):
         unlink(TESTFN)
         unlink(TESTFN2)
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
index c2c5b6a2a44a..97a5ebf753b5 100644
--- a/Lib/zipfile.py
+++ b/Lib/zipfile.py
@@ -703,6 +703,7 @@ def _get_compressor(compress_type, compresslevel=None):
 
 
 def _get_decompressor(compress_type):
+    _check_compression(compress_type)
     if compress_type == ZIP_STORED:
         return None
     elif compress_type == ZIP_DEFLATED:
diff --git a/Misc/NEWS.d/next/Library/2019-09-12-14-52-38.bpo-36991.1OcSm8.rst b/Misc/NEWS.d/next/Library/2019-09-12-14-52-38.bpo-36991.1OcSm8.rst
new file mode 100644
index 000000000000..c6fa852e94ad
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-12-14-52-38.bpo-36991.1OcSm8.rst
@@ -0,0 +1,2 @@
+Fixes a potential incorrect AttributeError exception escaping
+ZipFile.extract() in some unsupported input error situations.


From webhook-mailer at python.org  Thu Sep 12 10:20:42 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Thu, 12 Sep 2019 14:20:42 -0000
Subject: [Python-checkins] bpo-21872: fix lzma library decompresses data
 incompletely (GH-14048)
Message-ID: <mailman.666.1568298044.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4ffd05d7ec47cfd0d7fc95dce851633be9663255
commit: 4ffd05d7ec47cfd0d7fc95dce851633be9663255
branch: master
author: animalize <animalize at users.noreply.github.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-12T15:20:37+01:00
summary:

bpo-21872: fix lzma library decompresses data incompletely (GH-14048)

* 1. add test case with wrong behavior
* 2. fix bug when max_length == -1
* 3. allow b"" as valid input data for decompress_buf()
* 4. when max_length >= 0, let needs_input mechanism works
* add more asserts to test case

files:
A Misc/NEWS.d/next/Library/2019-06-12-08-56-22.bpo-21872.V9QGGN.rst
M Lib/test/test_lzma.py
M Modules/_lzmamodule.c

diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py
index 117de0a1b554..f24ed3ca3d43 100644
--- a/Lib/test/test_lzma.py
+++ b/Lib/test/test_lzma.py
@@ -1197,6 +1197,36 @@ def test_tell_bad_args(self):
         f.close()
         self.assertRaises(ValueError, f.tell)
 
+    def test_issue21872(self):
+        # sometimes decompress data incompletely
+
+        # ---------------------
+        # when max_length == -1
+        # ---------------------
+        d1 = LZMADecompressor()
+        entire = d1.decompress(ISSUE_21872_DAT, max_length=-1)
+        self.assertEqual(len(entire), 13160)
+        self.assertTrue(d1.eof)
+
+        # ---------------------
+        # when max_length > 0
+        # ---------------------
+        d2 = LZMADecompressor()
+
+        # When this value of max_length is used, the input and output
+        # buffers are exhausted at the same time, and lzs's internal
+        # state still have 11 bytes can be output.
+        out1 = d2.decompress(ISSUE_21872_DAT, max_length=13149)
+        self.assertFalse(d2.needs_input) # ensure needs_input mechanism works
+        self.assertFalse(d2.eof)
+
+        # simulate needs_input mechanism
+        # output internal state's 11 bytes
+        out2 = d2.decompress(b'')
+        self.assertEqual(len(out2), 11)
+        self.assertTrue(d2.eof)
+        self.assertEqual(out1 + out2, entire)
+
 
 class OpenTestCase(unittest.TestCase):
 
@@ -1763,6 +1793,139 @@ def test_filter_properties_roundtrip(self):
     b"\x00"
 )
 
+ISSUE_21872_DAT = (
+    b']\x00\x00@\x00h3\x00\x00\x00\x00\x00\x00\x00\x00`D\x0c\x99\xc8'
+    b'\xd1\xbbZ^\xc43+\x83\xcd\xf1\xc6g\xec-\x061F\xb1\xbb\xc7\x17%-\xea'
+    b'\xfap\xfb\x8fs\x128\xb2,\x88\xe4\xc0\x12|*x\xd0\xa2\xc4b\x1b!\x02c'
+    b'\xab\xd9\x87U\xb8n \xfaVJ\x9a"\xb78\xff%_\x17`?@*\xc2\x82'
+    b"\xf2^\x1b\xb8\x04&\xc0\xbb\x03g\x9d\xca\xe9\xa4\xc9\xaf'\xe5\x8e}"
+    b'F\xdd\x11\xf3\x86\xbe\x1fN\x95\\\xef\xa2Mz-\xcb\x9a\xe3O@'
+    b"\x19\x07\xf6\xee\x9e\x9ag\xc6\xa5w\rnG'\x99\xfd\xfeGI\xb0"
+    b'\xbb\xf9\xc2\xe1\xff\xc5r\xcf\x85y[\x01\xa1\xbd\xcc/\xa3\x1b\x83\xaa'
+    b'\xc6\xf9\x99\x0c\xb6_\xc9MQ+x\xa2F\xda]\xdd\xe8\xfb\x1a&'
+    b',\xc4\x19\x1df\x81\x1e\x90\xf3\xb8Hgr\x85v\xbe\xa3qx\x01Y\xb5\x9fF'
+    b"\x13\x18\x01\xe69\x9b\xc8'\x1e\x9d\xd6\xe4F\x84\xac\xf8d<\x11\xd5"
+    b'\\\x0b\xeb\x0e\x82\xab\xb1\xe6\x1fka\xe1i\xc4 C\xb1"4)\xd6\xa7`\x02'
+    b'\xec\x11\x8c\xf0\x14\xb0\x1d\x1c\xecy\xf0\xb7|\x11j\x85X\xb2!\x1c'
+    b'\xac\xb5N\xc7\x85j\x9ev\xf5\xe6\x0b\xc1]c\xc15\x16\x9f\xd5\x99'
+    b"\xfei^\xd2G\x9b\xbdl\xab:\xbe,\xa9'4\x82\xe5\xee\xb3\xc1"
+    b'$\x93\x95\xa8Y\x16\xf5\xbf\xacw\x91\x04\x1d\x18\x06\xe9\xc5\xfdk\x06'
+    b'\xe8\xfck\xc5\x86>\x8b~\xa4\xcb\xf1\xb3\x04\xf1\x04G5\xe2\xcc]'
+    b'\x16\xbf\x140d\x18\xe2\xedw#(3\xca\xa1\x80bX\x7f\xb3\x84'
+    b'\x9d\xdb\xe7\x08\x97\xcd\x16\xb9\xf1\xd5r+m\x1e\xcb3q\xc5\x9e\x92'
+    b"\x7f\x8e*\xc7\xde\xe9\xe26\xcds\xb1\x10-\xf6r\x02?\x9d\xddCgJN'"
+    b'\x11M\xfa\nQ\n\xe6`m\xb8N\xbbq\x8el\x0b\x02\xc7:q\x04G\xa1T'
+    b'\xf1\xfe!0\x85~\xe5\x884\xe9\x89\xfb\x13J8\x15\xe42\xb6\xad'
+    b'\x877A\x9a\xa6\xbft]\xd0\xe35M\xb0\x0cK\xc8\xf6\x88\xae\xed\xa9,j7'
+    b'\x81\x13\xa0(\xcb\xe1\xe9l2\x7f\xcd\xda\x95(\xa70B\xbd\xf4\xe3'
+    b'hp\x94\xbdJ\xd7\t\xc7g\xffo?\x89?\xf8}\x7f\xbc\x1c\x87'
+    b'\x14\xc0\xcf\x8cV:\x9a\x0e\xd0\xb2\x1ck\xffk\xb9\xe0=\xc7\x8d/'
+    b'\xb8\xff\x7f\x1d\x87`\x19.\x98X*~\xa7j\xb9\x0b"\xf4\xe4;V`\xb9\xd7'
+    b'\x03\x1e\xd0t0\xd3\xde\x1fd\xb9\xe2)\x16\x81}\xb1\\b\x7fJ'
+    b'\x92\xf4\xff\n+V!\xe9\xde\x98\xa0\x8fK\xdf7\xb9\xc0\x12\x1f\xe2'
+    b'\xe9\xb0`\xae\x14\r\xa7\xc4\x81~\xd8\x8d\xc5\x06\xd8m\xb0Y\x8a)'
+    b'\x06/\xbb\xf9\xac\xeaP\xe0\x91\x05m[\xe5z\xe6Z\xf3\x9f\xc7\xd0'
+    b'\xd3\x8b\xf3\x8a\x1b\xfa\xe4Pf\xbc0\x17\x10\xa9\xd0\x95J{\xb3\xc3'
+    b'\xfdW\x9bop\x0f\xbe\xaee\xa3]\x93\x9c\xda\xb75<\xf6g!\xcc\xb1\xfc\\'
+    b'7\x152Mc\x17\x84\x9d\xcd35\r0\xacL-\xf3\xfb\xcb\x96\x1e\xe9U\x7f'
+    b'\xd7\xca\xb0\xcc\x89\x0c*\xce\x14\xd1P\xf1\x03\xb6.~9o?\xe8'
+    b'\r\x86\xe0\x92\x87}\xa3\x84\x03P\xe0\xc2\x7f\n;m\x9d\x9e\xb4|'
+    b'\x8c\x18\xc0#0\xfe3\x07<\xda\xd8\xcf^\xd4Hi\xd6\xb3\x0bT'
+    b'\x1dF\x88\x85q}\x02\xc6&\xc4\xae\xce\x9cU\xfa\x0f\xcc\xb6\x1f\x11'
+    b'drw\x9eN\x19\xbd\xffz\x0f\xf0\x04s\xadR\xc1\xc0\xbfl\xf1\xba\xf95^'
+    b'e\xb1\xfbVY\xd9\x9f\x1c\xbf*\xc4\xa86\x08+\xd6\x88[\xc4_rc\xf0f'
+    b'\xb8\xd4\xec\x1dx\x19|\xbf\xa7\xe0\x82\x0b\x8c~\x10L/\x90\xd6\xfb'
+    b'\x81\xdb\x98\xcc\x02\x14\xa5C\xb2\xa7i\xfd\xcd\x1fO\xf7\xe9\x89t\xf0'
+    b'\x17\xa5\x1c\xad\xfe<Q`%\x075k\n7\x9eI\x82<#)&\x04\xc2\xf0C\xd4`!'
+    b'\xcb\xa9\xf9\xb3F\x86\xb5\xc3M\xbeu\x12\xb2\xca\x95e\x10\x0b\xb1\xcc'
+    b'\x01b\x9bXa\x1b[B\x8c\x07\x11Of;\xeaC\xebr\x8eb\xd9\x9c\xe4i]<z\x9a'
+    b'\x03T\x8b9pF\x10\x8c\x84\xc7\x0e\xeaPw\xe5\xa0\x94\x1f\x84\xdd'
+    b'a\xe8\x85\xc2\x00\xebq\xe7&Wo5q8\xc2t\x98\xab\xb7\x7f\xe64-H'
+    b'\t\xb4d\xbe\x06\xe3Q\x8b\xa9J\xb0\x00\xd7s.\x85"\xc0p\x05'
+    b'\x1c\x06N\x87\xa5\xf8\xc3g\x1b}\x0f\x0f\xc3|\x90\xea\xefd3X'
+    b'[\xab\x04E\xf2\xf2\xc9\x08\x8a\xa8+W\xa2v\xec\x15G\x08/I<L\\1'
+    b'\xff\x15O\xaa\x89{\xd1mW\x13\xbd~\xe1\x90^\xc4@\r\xed\xb5D@\xb4\x08'
+    b'A\x90\xe69;\xc7BO\xdb\xda\xebu\x9e\xa9tN\xae\x8aJ5\xcd\x11\x1d\xea'
+    b"\xe5\xa7\x04\xe6\x82Z\xc7O\xe46[7\xdco*[\xbe\x0b\xc9\xb7a\xab'\xf6"
+    b"\xd1u\xdb\xd9q\xf5+y\x1b\x00\xb4\xf3a\xae\xf1M\xc4\xbc\xd00'\x06pQ"
+    b'\x8dH\xaa\xaa\xc4\xd2K\x9b\xc0\xe9\xec=n\xa9\x1a\x8a\xc2\xe8\x18\xbc'
+    b'\x93\xb8F\xa1\x8fOY\xe7\xda\xcf0\t\xff|\xd9\xe5\xcf\xe7\xf6\xbe'
+    b'\xf8\x04\x17\xf2\xe5P\xa7y~\xce\x11h0\x81\x80d[\x00_v\xbbc\xdbI'
+    b'3\xbc`W\xc0yrkB\xf5\x9f\xe9i\xc5\x8a^\x8d\xd4\x81\xd9\x05\xc1\xfc>'
+    b'"\xd1v`\x82\xd5$\x89\xcf^\xd52.\xafd\xe8d@\xaa\xd5Y|\x90\x84'
+    b'j\xdb}\x84riV\x8e\xf0X4rB\xf2NPS[\x8e\x88\xd4\x0fI\xb8'
+    b'\xdd\xcb\x1d\xf2(\xdf;9\x9e|\xef^0;.*[\x9fl\x7f\xa2_X\xaff!\xbb\x03'
+    b'\xff\x19\x8f\x88\xb5\xb6\x884\xa3\x05\xde3D{\xe3\xcb\xce\xe4t]'
+    b'\x875\xe3Uf\xae\xea\x88\x1c\x03b\n\xb1,Q\xec\xcf\x08\t\xde@\x83\xaa<'
+    b',-\xe4\xee\x9b\x843\xe5\x007\tK\xac\x057\xd6*X\xa3\xc6~\xba\xe6O'
+    b'\x81kz"\xbe\xe43sL\xf1\xfa;\xf4^\x1e\xb4\x80\xe2\xbd\xaa\x17Z\xe1f'
+    b'\xda\xa6\xb9\x07:]}\x9fa\x0b?\xba\xe7\xf15\x04M\xe3\n}M\xa4\xcb\r'
+    b'2\x8a\x88\xa9\xa7\x92\x93\x84\x81Yo\x00\xcc\xc4\xab\x9aT\x96\x0b\xbe'
+    b'U\xac\x1d\x8d\x1b\x98"\xf8\x8f\xf1u\xc1n\xcc\xfcA\xcc\x90\xb7i'
+    b'\x83\x9c\x9c~\x1d4\xa2\xf0*J\xe7t\x12\xb4\xe3\xa0u\xd7\x95Z'
+    b'\xf7\xafG\x96~ST,\xa7\rC\x06\xf4\xf0\xeb`2\x9e>Q\x0e\xf6\xf5\xc5'
+    b'\x9b\xb5\xaf\xbe\xa3\x8f\xc0\xa3hu\x14\x12 \x97\x99\x04b\x8e\xc7\x1b'
+    b'VKc\xc1\xf3 \xde\x85-:\xdc\x1f\xac\xce*\x06\xb3\x80;`'
+    b'\xdb\xdd\x97\xfdg\xbf\xe7\xa8S\x08}\xf55e7\xb8/\xf0!\xc8'
+    b"Y\xa8\x9a\x07'\xe2\xde\r\x02\xe1\xb2\x0c\xf4C\xcd\xf9\xcb(\xe8\x90"
+    b'\xd3bTD\x15_\xf6\xc3\xfb\xb3E\xfc\xd6\x98{\xc6\\fz\x81\xa99\x85\xcb'
+    b'\xa5\xb1\x1d\x94bqW\x1a!;z~\x18\x88\xe8i\xdb\x1b\x8d\x8d'
+    b'\x06\xaa\x0e\x99s+5k\x00\xe4\xffh\xfe\xdbt\xa6\x1bU\xde\xa3'
+    b'\xef\xcb\x86\x9e\x81\x16j\n\x9d\xbc\xbbC\x80?\x010\xc7Jj;'
+    b'\xc4\xe5\x86\xd5\x0e0d#\xc6;\xb8\xd1\xc7c\xb5&8?\xd9J\xe5\xden\xb9'
+    b'\xe9cb4\xbb\xe6\x14\xe0\xe7l\x1b\x85\x94\x1fh\xf1n\xdeZ\xbe'
+    b'\x88\xff\xc2e\xca\xdc,B-\x8ac\xc9\xdf\xf5|&\xe4LL\xf0\x1f\xaa8\xbd'
+    b'\xc26\x94bVi\xd3\x0c\x1c\xb6\xbb\x99F\x8f\x0e\xcc\x8e4\xc6/^W\xf5?'
+    b'\xdc\x84(\x14dO\x9aD6\x0f4\xa3,\x0c\x0bS\x9fJ\xe1\xacc^\x8a0\t\x80D['
+    b'\xb8\xe6\x86\xb0\xe8\xd4\xf9\x1en\xf1\xf5^\xeb\xb8\xb8\xf8'
+    b')\xa8\xbf\xaa\x84\x86\xb1a \x95\x16\x08\x1c\xbb@\xbd+\r/\xfb'
+    b'\x92\xfbh\xf1\x8d3\xf9\x92\xde`\xf1\x86\x03\xaa+\xd9\xd9\xc6P\xaf'
+    b'\xe3-\xea\xa5\x0fB\xca\xde\xd5n^\xe3/\xbf\xa6w\xc8\x0e<M'
+    b'\xc2\x1e!\xd4\xc6E\xf2\xad\x0c\xbc\x1d\x88Y\x03\x98<\x92\xd9\xa6B'
+    b'\xc7\x83\xb5"\x97D|&\xc4\xd4\xfad\x0e\xde\x06\xa3\xc2\x9c`\xf2'
+    b'7\x03\x1a\xed\xd80\x10\xe9\x0co\x10\xcf\x18\x16\xa7\x1c'
+    b"\xe5\x96\xa4\xd9\xe1\xa5v;]\xb7\xa9\xdc'hA\xe3\x9c&\x98\x0b9\xdf~@"
+    b'\xf8\xact\x87<\xf94\x0c\x9d\x93\xb0)\xe1\xa2\x0f\x1e=:&\xd56\xa5A+'
+    b'\xab\xc4\x00\x8d\x81\x93\xd4\xd8<\x82k\\d\xd8v\xab\xbd^l5C?\xd4\xa0'
+    b'M\x12C\xc8\x80\r\xc83\xe8\xc0\xf5\xdf\xca\x05\xf4BPjy\xbe\x91\x9bzE'
+    b"\xd8[\x93oT\r\x13\x16'\x1a\xbd*H\xd6\xfe\r\xf3\x91M\x8b\xee\x8f7f"
+    b"\x0b;\xaa\x85\xf2\xdd'\x0fwM \xbd\x13\xb9\xe5\xb8\xb7 D+P\x1c\xe4g"
+    b'n\xd2\xf1kc\x15\xaf\xc6\x90V\x03\xc2UovfZ\xcc\xd23^\xb3\xe7\xbf'
+    b'\xacv\x1d\x82\xedx\xa3J\xa9\xb7\xcf\x0c\xe6j\x96n*o\x18>'
+    b'\xc6\xfd\x97_+D{\x03\x15\xe8s\xb1\xc8HAG\xcf\xf4\x1a\xdd'
+    b'\xad\x11\xbf\x157q+\xdeW\x89g"X\x82\xfd~\xf7\xab4\xf6`\xab\xf1q'
+    b')\x82\x10K\xe9sV\xfe\xe45\xafs*\x14\xa7;\xac{\x06\x9d<@\x93G'
+    b'j\x1d\xefL\xe9\xd8\x92\x19&\xa1\x16\x19\x04\tu5\x01]\xf6\xf4'
+    b'\xcd\\\xd8A|I\xd4\xeb\x05\x88C\xc6e\xacQ\xe9*\x97~\x9au\xf8Xy'
+    b"\x17P\x10\x9f\n\x8c\xe2fZEu>\x9b\x1e\x91\x0b'`\xbd\xc0\xa8\x86c\x1d"
+    b'Z\xe2\xdc8j\x95\xffU\x90\x1e\xf4o\xbc\xe5\xe3e>\xd2R\xc0b#\xbc\x15'
+    b'H-\xb9!\xde\x9d\x90k\xdew\x9b{\x99\xde\xf7/K)A\xfd\xf5\xe6:\xda'
+    b'UM\xcc\xbb\xa2\x0b\x9a\x93\xf5{9\xc0 \xd2((6i\xc0\xbbu\xd8\x9e\x8d'
+    b'\xf8\x04q\x10\xd4\x14\x9e7-\xb9B\xea\x01Q8\xc8v\x9a\x12A\x88Cd\x92'
+    b"\x1c\x8c!\xf4\x94\x87'\xe3\xcd\xae\xf7\xd8\x93\xfa\xde\xa8b\x9e\xee2"
+    b'K\xdb\x00l\x9d\t\xb1|D\x05U\xbb\xf4>\xf1w\x887\xd1}W\x9d|g|1\xb0\x13'
+    b"\xa3 \xe5\xbfm@\xc06+\xb7\t\xcf\x15D\x9a \x1fM\x1f\xd2\xb5'\xa9\xbb"
+    b'~Co\x82\xfa\xc2\t\xe6f\xfc\xbeI\xae1\x8e\xbe\xb8\xcf\x86\x17'
+    b'\x9f\xe2`\xbd\xaf\xba\xb9\xbc\x1b\xa3\xcd\x82\x8fwc\xefd\xa9\xd5\x14'
+    b'\xe2C\xafUE\xb6\x11MJH\xd0=\x05\xd4*I\xff"\r\x1b^\xcaS6=\xec@\xd5'
+    b'\x11,\xe0\x87Gr\xaa[\xb8\xbc>n\xbd\x81\x0c\x07<\xe9\x92('
+    b'\xb2\xff\xac}\xe7\xb6\x15\x90\x9f~4\x9a\xe6\xd6\xd8s\xed\x99tf'
+    b'\xa0f\xf8\xf1\x87\t\x96/)\x85\xb6\n\xd7\xb2w\x0b\xbc\xba\x99\xee'
+    b'Q\xeen\x1d\xad\x03\xc3s\xd1\xfd\xa2\xc6\xb7\x9a\x9c(G<6\xad[~H '
+    b'\x16\x89\x89\xd0\xc3\xd2\xca~\xac\xea\xa5\xed\xe5\xfb\r:'
+    b'\x8e\xa6\xf1e\xbb\xba\xbd\xe0(\xa3\x89_\x01(\xb5c\xcc\x9f\x1fg'
+    b'v\xfd\x17\xb3\x08S=S\xee\xfc\x85>\x91\x8d\x8d\nYR\xb3G\xd1A\xa2\xb1'
+    b'\xec\xb0\x01\xd2\xcd\xf9\xfe\x82\x06O\xb3\xecd\xa9c\xe0\x8eP\x90\xce'
+    b'\xe0\xcd\xd8\xd8\xdc\x9f\xaa\x01"[Q~\xe4\x88\xa1#\xc1\x12C\xcf'
+    b'\xbe\x80\x11H\xbf\x86\xd8\xbem\xcfWFQ(X\x01DK\xdfB\xaa\x10.-'
+    b'\xd5\x9e|\x86\x15\x86N]\xc7Z\x17\xcd=\xd7)M\xde\x15\xa4LTi\xa0\x15'
+    b'\xd1\xe7\xbdN\xa4?\xd1\xe7\x02\xfe4\xe4O\x89\x98&\x96\x0f\x02\x9c'
+    b'\x9e\x19\xaa\x13u7\xbd0\xdc\xd8\x93\xf4BNE\x1d\x93\x82\x81\x16'
+    b'\xe5y\xcf\x98D\xca\x9a\xe2\xfd\xcdL\xcc\xd1\xfc_\x0b\x1c\xa0]\xdc'
+    b'\xa91 \xc9c\xd8\xbf\x97\xcfp\xe6\x19-\xad\xff\xcc\xd1N(\xe8'
+    b'\xeb#\x182\x96I\xf7l\xf3r\x00'
+)
+
 
 def test_main():
     run_unittest(
diff --git a/Misc/NEWS.d/next/Library/2019-06-12-08-56-22.bpo-21872.V9QGGN.rst b/Misc/NEWS.d/next/Library/2019-06-12-08-56-22.bpo-21872.V9QGGN.rst
new file mode 100644
index 000000000000..90df756f9a81
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-12-08-56-22.bpo-21872.V9QGGN.rst
@@ -0,0 +1,3 @@
+Fix :mod:`lzma`: module decompresses data incompletely. When decompressing a
+FORMAT_ALONE format file, and it doesn't have the end marker, sometimes the
+last one to dozens bytes can't be output. Patch by Ma Lin.
diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c
index 9e68cbb78af5..1ab67f306b73 100644
--- a/Modules/_lzmamodule.c
+++ b/Modules/_lzmamodule.c
@@ -872,9 +872,6 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length)
     PyObject *result;
     lzma_stream *lzs = &d->lzs;
 
-    if (lzs->avail_in == 0)
-        return PyBytes_FromStringAndSize(NULL, 0);
-
     if (max_length < 0 || max_length >= INITIAL_BUFFER_SIZE)
         result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE);
     else
@@ -891,7 +888,10 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length)
         Py_BEGIN_ALLOW_THREADS
         lzret = lzma_code(lzs, LZMA_RUN);
         data_size = (char *)lzs->next_out - PyBytes_AS_STRING(result);
+        if (lzret == LZMA_BUF_ERROR && lzs->avail_in == 0 && lzs->avail_out > 0)
+            lzret = LZMA_OK; /* That wasn't a real error */
         Py_END_ALLOW_THREADS
+
         if (catch_lzma_error(lzret))
             goto error;
         if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK)
@@ -899,15 +899,19 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length)
         if (lzret == LZMA_STREAM_END) {
             d->eof = 1;
             break;
-        } else if (lzs->avail_in == 0) {
-            break;
         } else if (lzs->avail_out == 0) {
+            /* Need to check lzs->avail_out before lzs->avail_in.
+               Maybe lzs's internal state still have a few bytes
+               can be output, grow the output buffer and continue
+               if max_lengh < 0. */
             if (data_size == max_length)
                 break;
             if (grow_buffer(&result, max_length) == -1)
                 goto error;
             lzs->next_out = (uint8_t *)PyBytes_AS_STRING(result) + data_size;
             lzs->avail_out = PyBytes_GET_SIZE(result) - data_size;
+        } else if (lzs->avail_in == 0) {
+            break;
         }
     }
     if (data_size != PyBytes_GET_SIZE(result))
@@ -990,7 +994,19 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length)
     }
     else if (lzs->avail_in == 0) {
         lzs->next_in = NULL;
-        d->needs_input = 1;
+
+        if (lzs->avail_out == 0) {
+            /* (avail_in==0 && avail_out==0)
+               Maybe lzs's internal state still have a few bytes can
+               be output, try to output them next time. */
+            d->needs_input = 0;
+
+            /* if max_length < 0, lzs->avail_out always > 0 */
+            assert(max_length >= 0);
+        } else {
+            /* Input buffer exhausted, output buffer has space. */
+            d->needs_input = 1;
+        }
     }
     else {
         d->needs_input = 0;


From webhook-mailer at python.org  Thu Sep 12 10:33:30 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 14:33:30 -0000
Subject: [Python-checkins] bpo-38137: Re-add OpenSSL 1.0.2 compat (GH-16051)
Message-ID: <mailman.667.1568298811.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9a4963b932a087b78596cca0b8394ac898faa490
commit: 9a4963b932a087b78596cca0b8394ac898faa490
branch: master
author: Christian Heimes <christian at python.org>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-12T07:33:26-07:00
summary:

bpo-38137: Re-add OpenSSL 1.0.2 compat (GH-16051)



The defines are required for OpenSSL 1.0.2 and LibreSSL.


https://bugs.python.org/issue38134



Automerge-Triggered-By: @tiran

files:
M Modules/_hashopenssl.c

diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index bea7e5ed3123..c685279c7a7b 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -26,6 +26,12 @@
 #include <openssl/objects.h>
 #include "openssl/err.h"
 
+#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
+/* OpenSSL < 1.1.0 */
+#define EVP_MD_CTX_new EVP_MD_CTX_create
+#define EVP_MD_CTX_free EVP_MD_CTX_destroy
+#endif
+
 #define MUNCH_SIZE INT_MAX
 
 typedef struct {


From webhook-mailer at python.org  Thu Sep 12 10:33:57 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 14:33:57 -0000
Subject: [Python-checkins] bpo-36991: Fix incorrect exception escaping
 ZipFile.extract() (GH-13632)
Message-ID: <mailman.668.1568298838.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/717cc61ed103684b8c73e0e8af10ace345f39f16
commit: 717cc61ed103684b8c73e0e8af10ace345f39f16
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T07:33:53-07:00
summary:

bpo-36991: Fix incorrect exception escaping ZipFile.extract() (GH-13632)

(cherry picked from commit 2f1b857562b0f1601c9019db74c29b7d7e21ac9f)

Co-authored-by: Berker Peksag <berker.peksag at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-12-14-52-38.bpo-36991.1OcSm8.rst
M Lib/test/test_zipfile.py
M Lib/zipfile.py

diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
index 955e540f96e6..76e2f647c607 100644
--- a/Lib/test/test_zipfile.py
+++ b/Lib/test/test_zipfile.py
@@ -9,6 +9,7 @@
 import sys
 import time
 import unittest
+import unittest.mock as mock
 import zipfile
 
 
@@ -1739,6 +1740,16 @@ def test_seek_tell(self):
                 fp.seek(0, os.SEEK_SET)
                 self.assertEqual(fp.tell(), 0)
 
+    @requires_bz2
+    def test_decompress_without_3rd_party_library(self):
+        data = b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+        zip_file = io.BytesIO(data)
+        with zipfile.ZipFile(zip_file, 'w', compression=zipfile.ZIP_BZIP2) as zf:
+            zf.writestr('a.txt', b'a')
+        with mock.patch('zipfile.bz2', None):
+            with zipfile.ZipFile(zip_file) as zf:
+                self.assertRaises(RuntimeError, zf.extract, 'a.txt')
+
     def tearDown(self):
         unlink(TESTFN)
         unlink(TESTFN2)
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
index dfd090795019..59435064eabf 100644
--- a/Lib/zipfile.py
+++ b/Lib/zipfile.py
@@ -703,6 +703,7 @@ def _get_compressor(compress_type, compresslevel=None):
 
 
 def _get_decompressor(compress_type):
+    _check_compression(compress_type)
     if compress_type == ZIP_STORED:
         return None
     elif compress_type == ZIP_DEFLATED:
diff --git a/Misc/NEWS.d/next/Library/2019-09-12-14-52-38.bpo-36991.1OcSm8.rst b/Misc/NEWS.d/next/Library/2019-09-12-14-52-38.bpo-36991.1OcSm8.rst
new file mode 100644
index 000000000000..c6fa852e94ad
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-12-14-52-38.bpo-36991.1OcSm8.rst
@@ -0,0 +1,2 @@
+Fixes a potential incorrect AttributeError exception escaping
+ZipFile.extract() in some unsupported input error situations.


From webhook-mailer at python.org  Thu Sep 12 10:37:40 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 14:37:40 -0000
Subject: [Python-checkins] Overhaul datetime documentation (GH-13410)
Message-ID: <mailman.669.1568299061.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8976359c598f56825648f4e5397b78c5d14ea13c
commit: 8976359c598f56825648f4e5397b78c5d14ea13c
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T07:37:36-07:00
summary:

Overhaul datetime documentation (GH-13410)


This is a restructuring of the datetime documentation to hopefully make
them more user-friendly and approachable to new users without losing any
of the detail.

Changes include:
 - Creating dedicated subsections for some concepts such as:
    - "Constants"
    - "Naive vs Aware"
    - "Determining if an Object is Aware"
 - Give 'naive vs aware' its own subsection
 - Give 'constants' their own subsection
 - Overhauling the strftime-strptime section by:
    - Breaking it into logical, linkable, and digestable parts
    - Adding a high-level comparison table
    - Moving the technical detail to bottom: readers come to this
      section primarily to remind themselves to things:
      - How do I write the format code for X?
      - strptime/strftime: which one is which again?
 - Touching up fromisoformat + isoformat sections by:
    - Revising fromisoformat + isoformat for date, time, and
      datetime
    - Adding basic examples
    - Enforcing consistency about putting formats (i.e. ``HH:MM``)
      in double backticks.  This was previously done in some places
      but not all
    - Putting long 'supported formats', on their own line to improve
      readability
 - Moving the 'seealso' section to the top and add a link to dateutil
    Rationale: This doesn't really belong nested under the
    'constants' section.  Let readers know right away that
    datetime is one of several related tools.
 - Moving common features of several types into one place:
    Previously, each type went out of its way to note separately
    that it was hashable and picklable.  These can be brought
    into one single place that is more prominent.
 - Reducing some verbose explanations to improve readability
 - Breaking up long paragraphs into digestable chunks
 - Displaying longer "equivalent to" examples, as short code blocks
 - Using the dot notation for datetime/time classes:
    Use :class:`.time` and :class:`.datetime` rather than :class:`time` and
    :class:`datetime`; otherwise, the generated links will route to the
    respective modules, not classes.
 - Rewording the tzinfo class description
    The top paragraph should get straight to the point of telling the reader
    what subclasses of tzinfo _do_.  Previously, that was hidden in a later
    paragraph.
 - Adding a note on .today() versus .now()
 - Rearranging and expanding example blocks, including:
    - Moved long, multiline inline examples to standalone examples
    - Simplified the example block for timedelta arithmetic:
        - Broke the example into two logical sections:
          1. normalization/parameter 'merging'
          2. timedelta arithmetic
        - Reduced the complexity of the some of the examples.  Show
          reasonable, real-world uses cases that are easy to follow
          along with and progres in difficult slightly.
    - Broke up the example sections for date and datetime sections by putting
      the easy examples first, progressing to more esoteric situations and
      breaking it up into logical sections based on what the methods are
      doing at a high level.
    - Simplified the KabulTz example:
        - Put the class definition itself into a non-REPL block since there is
          no interactive output involved there
        - Briefly explained what's happening before launching into the code
        - Broke the example section into visually separate chunks
 - Various whitespace, formatting, style and grammar fixes including:
    - Consistently using backctics for 'date_string' formats
    - Consistently using one space after periods.
    - Consistently using bold for vocab terms
    - Consistently using italics when referring to params:
      See https://devguide.python.org/documenting/GH-id4
    - Using '::' to lead into code blocks
        Per https://devguide.python.org/documenting/GH-source-code, this will
        let the reader use the 'expand/collapse' top-right button for REPL
        blocks to hide or show the prompt.
    - Using consistent captialization schemes
    - Removing use of the default role
    - Put 'example' blocks in Markdown subsections

This is a combination of 66 commits.

See bpo-36960: https://bugs.python.org/issue36960
(cherry picked from commit 3fb1363fe87a24cdb2ee1dd9746f1c49046af958)

Co-authored-by: Brad <brad.solomon.1124 at gmail.com>

files:
A Misc/NEWS.d/next/Documentation/2019-05-18-16-25-44.bpo-36960.xEKHXj.rst
M Doc/library/datetime.rst

diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index 50f1b9e74949..882d4e16e1f6 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -14,40 +14,59 @@
 
 .. XXX what order should the types be discussed in?
 
-The :mod:`datetime` module supplies classes for manipulating dates and times in
-both simple and complex ways.  While date and time arithmetic is supported, the
-focus of the implementation is on efficient attribute extraction for output
-formatting and manipulation. For related functionality, see also the
-:mod:`time` and :mod:`calendar` modules.
-
-There are two kinds of date and time objects: "naive" and "aware".
-
-An aware object has sufficient knowledge of applicable algorithmic and
-political time adjustments, such as time zone and daylight saving time
-information, to locate itself relative to other aware objects.  An aware object
-is used to represent a specific moment in time that is not open to
-interpretation [#]_.
-
-A naive object does not contain enough information to unambiguously locate
-itself relative to other date/time objects.  Whether a naive object represents
+The :mod:`datetime` module supplies classes for manipulating dates and times.
+
+While date and time arithmetic is supported, the focus of the implementation is
+on efficient attribute extraction for output formatting and manipulation.
+
+.. seealso::
+
+   Module :mod:`calendar`
+      General calendar related functions.
+
+   Module :mod:`time`
+      Time access and conversions.
+
+   Package `dateutil <https://dateutil.readthedocs.io/en/stable/>`_
+      Third-party library with expanded time zone and parsing support.
+
+.. _datetime-naive-aware:
+
+Aware and Naive Objects
+-----------------------
+
+Date and time objects may be categorized as "aware" or "naive."
+
+With sufficient knowledge of applicable algorithmic and political time
+adjustments, such as time zone and daylight saving time information,
+an **aware** object can locate itself relative to other aware objects.
+An aware object represents a specific moment in time that is not open to
+interpretation. [#]_
+
+A **naive** object does not contain enough information to unambiguously locate
+itself relative to other date/time objects. Whether a naive object represents
 Coordinated Universal Time (UTC), local time, or time in some other timezone is
 purely up to the program, just like it is up to the program whether a
-particular number represents metres, miles, or mass.  Naive objects are easy to
+particular number represents metres, miles, or mass. Naive objects are easy to
 understand and to work with, at the cost of ignoring some aspects of reality.
 
 For applications requiring aware objects, :class:`.datetime` and :class:`.time`
 objects have an optional time zone information attribute, :attr:`!tzinfo`, that
 can be set to an instance of a subclass of the abstract :class:`tzinfo` class.
 These :class:`tzinfo` objects capture information about the offset from UTC
-time, the time zone name, and whether Daylight Saving Time is in effect.  Note
-that only one concrete :class:`tzinfo` class, the :class:`timezone` class, is
-supplied by the :mod:`datetime` module.  The :class:`timezone` class can
-represent simple timezones with fixed offset from UTC, such as UTC itself or
-North American EST and EDT timezones.  Supporting timezones at deeper levels of
-detail is up to the application.  The rules for time adjustment across the
+time, the time zone name, and whether daylight saving time is in effect.
+
+Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is
+supplied by the :mod:`datetime` module. The :class:`timezone` class can
+represent simple timezones with fixed offsets from UTC, such as UTC itself or
+North American EST and EDT timezones. Supporting timezones at deeper levels of
+detail is up to the application. The rules for time adjustment across the
 world are more political than rational, change frequently, and there is no
 standard suitable for every application aside from UTC.
 
+Constants
+---------
+
 The :mod:`datetime` module exports the following constants:
 
 .. data:: MINYEAR
@@ -61,16 +80,6 @@ The :mod:`datetime` module exports the following constants:
    The largest year number allowed in a :class:`date` or :class:`.datetime` object.
    :const:`MAXYEAR` is ``9999``.
 
-
-.. seealso::
-
-   Module :mod:`calendar`
-      General calendar related functions.
-
-   Module :mod:`time`
-      Time access and conversions.
-
-
 Available Types
 ---------------
 
@@ -86,7 +95,7 @@ Available Types
    :noindex:
 
    An idealized time, independent of any particular day, assuming that every day
-   has exactly 24\*60\*60 seconds (there is no notion of "leap seconds" here).
+   has exactly 24\*60\*60 seconds.  (There is no notion of "leap seconds" here.)
    Attributes: :attr:`hour`, :attr:`minute`, :attr:`second`, :attr:`microsecond`,
    and :attr:`.tzinfo`.
 
@@ -109,7 +118,7 @@ Available Types
 .. class:: tzinfo
    :noindex:
 
-   An abstract base class for time zone information objects.  These are used by the
+   An abstract base class for time zone information objects. These are used by the
    :class:`.datetime` and :class:`.time` classes to provide a customizable notion of
    time adjustment (for example, to account for time zone and/or daylight saving
    time).
@@ -122,22 +131,8 @@ Available Types
 
    .. versionadded:: 3.2
 
-
 Objects of these types are immutable.
 
-Objects of the :class:`date` type are always naive.
-
-An object of type :class:`.time` or :class:`.datetime` may be naive or aware.
-A :class:`.datetime` object *d* is aware if ``d.tzinfo`` is not ``None`` and
-``d.tzinfo.utcoffset(d)`` does not return ``None``.  If ``d.tzinfo`` is
-``None``, or if ``d.tzinfo`` is not ``None`` but ``d.tzinfo.utcoffset(d)``
-returns ``None``, *d* is naive.  A :class:`.time` object *t* is aware
-if ``t.tzinfo`` is not ``None`` and ``t.tzinfo.utcoffset(None)`` does not return
-``None``.  Otherwise, *t* is naive.
-
-The distinction between naive and aware doesn't apply to :class:`timedelta`
-objects.
-
 Subclass relationships::
 
    object
@@ -148,6 +143,40 @@ Subclass relationships::
        date
            datetime
 
+Common Properties
+^^^^^^^^^^^^^^^^^
+
+The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` types
+share these common features:
+
+- Objects of these types are immutable.
+- Objects of these types are hashable, meaning that they can be used as
+  dictionary keys.
+- Objects of these types support efficient pickling via the :mod:`pickle` module.
+
+Determining if an Object is Aware or Naive
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Objects of the :class:`date` type are always naive.
+
+An object of type :class:`.time` or :class:`.datetime` may be aware or naive.
+
+A :class:`.datetime` object *d* is aware if both of the following hold:
+
+1. ``d.tzinfo`` is not ``None``
+2. ``d.tzinfo.utcoffset(d)`` does not return ``None``
+
+Otherwise, *d* is naive.
+
+A :class:`.time` object *t* is aware if both of the following hold:
+
+1. ``t.tzinfo`` is not ``None``
+2. ``t.tzinfo.utcoffset(None)`` does not return ``None``.
+
+Otherwise, *t* is naive.
+
+The distinction between aware and naive doesn't apply to :class:`timedelta`
+objects.
 
 .. _datetime-timedelta:
 
@@ -159,11 +188,11 @@ dates or times.
 
 .. class:: timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0)
 
-   All arguments are optional and default to ``0``.  Arguments may be integers
+   All arguments are optional and default to ``0``. Arguments may be integers
    or floats, and may be positive or negative.
 
-   Only *days*, *seconds* and *microseconds* are stored internally.  Arguments are
-   converted to those units:
+   Only *days*, *seconds* and *microseconds* are stored internally.
+   Arguments are converted to those units:
 
    * A millisecond is converted to 1000 microseconds.
    * A minute is converted to 60 seconds.
@@ -177,10 +206,28 @@ dates or times.
    * ``0 <= seconds < 3600*24`` (the number of seconds in one day)
    * ``-999999999 <= days <= 999999999``
 
+   The following example illustrates how any arguments besides
+   *days*, *seconds* and *microseconds* are "merged" and normalized into those
+   three resulting attributes::
+
+       >>> from datetime import timedelta
+       >>> delta = timedelta(
+       ...     days=50,
+       ...     seconds=27,
+       ...     microseconds=10,
+       ...     milliseconds=29000,
+       ...     minutes=5,
+       ...     hours=8,
+       ...     weeks=2
+       ... )
+       >>> # Only days, seconds, and microseconds remain
+       >>> delta
+       datetime.timedelta(days=64, seconds=29156, microseconds=10)
+
    If any argument is a float and there are fractional microseconds,
    the fractional microseconds left over from all arguments are
    combined and their sum is rounded to the nearest microsecond using
-   round-half-to-even tiebreaker.  If no argument is a float, the
+   round-half-to-even tiebreaker. If no argument is a float, the
    conversion and normalization processes are exact (no information is
    lost).
 
@@ -188,7 +235,7 @@ dates or times.
    :exc:`OverflowError` is raised.
 
    Note that normalization of negative values may be surprising at first. For
-   example,
+   example::
 
       >>> from datetime import timedelta
       >>> d = timedelta(microseconds=-1)
@@ -196,7 +243,7 @@ dates or times.
       (-1, 86399, 999999)
 
 
-Class attributes are:
+Class attributes:
 
 .. attribute:: timedelta.min
 
@@ -263,7 +310,7 @@ Supported operations:
 |                                | timedelta.resolution using round-half-to-even.|
 +--------------------------------+-----------------------------------------------+
 | ``t1 = t2 // i`` or            | The floor is computed and the remainder (if   |
-| ``t1 = t2 // t3``              | any) is thrown away.  In the second case, an  |
+| ``t1 = t2 // t3``              | any) is thrown away. In the second case, an   |
 |                                | integer is returned. (3)                      |
 +--------------------------------+-----------------------------------------------+
 | ``t1 = t2 % t3``               | The remainder is computed as a                |
@@ -282,8 +329,8 @@ Supported operations:
 |                                | -*t1.seconds*, -*t1.microseconds*),           |
 |                                | and to *t1*\* -1. (1)(4)                      |
 +--------------------------------+-----------------------------------------------+
-| ``abs(t)``                     | equivalent to +\ *t* when ``t.days >= 0``, and|
-|                                | to -*t* when ``t.days < 0``. (2)              |
+| ``abs(t)``                     | equivalent to +\ *t* when ``t.days >= 0``,    |
+|                                | and to -*t* when ``t.days < 0``. (2)          |
 +--------------------------------+-----------------------------------------------+
 | ``str(t)``                     | Returns a string in the form                  |
 |                                | ``[D day[s], ][H]H:MM:SS[.UUUUUU]``, where D  |
@@ -298,10 +345,10 @@ Supported operations:
 Notes:
 
 (1)
-   This is exact, but may overflow.
+   This is exact but may overflow.
 
 (2)
-   This is exact, and cannot overflow.
+   This is exact and cannot overflow.
 
 (3)
    Division by 0 raises :exc:`ZeroDivisionError`.
@@ -310,41 +357,56 @@ Notes:
    -*timedelta.max* is not representable as a :class:`timedelta` object.
 
 (5)
-  String representations of :class:`timedelta` objects are normalized
-  similarly to their internal representation.  This leads to somewhat
-  unusual results for negative timedeltas.  For example:
+   String representations of :class:`timedelta` objects are normalized
+   similarly to their internal representation. This leads to somewhat
+   unusual results for negative timedeltas. For example::
 
-  >>> timedelta(hours=-5)
-  datetime.timedelta(days=-1, seconds=68400)
-  >>> print(_)
-  -1 day, 19:00:00
+      >>> timedelta(hours=-5)
+      datetime.timedelta(days=-1, seconds=68400)
+      >>> print(_)
+      -1 day, 19:00:00
 
 (6)
    The expression ``t2 - t3`` will always be equal to the expression ``t2 + (-t3)`` except
    when t3 is equal to ``timedelta.max``; in that case the former will produce a result
    while the latter will overflow.
 
-In addition to the operations listed above :class:`timedelta` objects support
+In addition to the operations listed above, :class:`timedelta` objects support
 certain additions and subtractions with :class:`date` and :class:`.datetime`
 objects (see below).
 
 .. versionchanged:: 3.2
    Floor division and true division of a :class:`timedelta` object by another
    :class:`timedelta` object are now supported, as are remainder operations and
-   the :func:`divmod` function.  True division and multiplication of a
+   the :func:`divmod` function. True division and multiplication of a
    :class:`timedelta` object by a :class:`float` object are now supported.
 
 
-Comparisons of :class:`timedelta` objects are supported with the
-:class:`timedelta` object representing the smaller duration considered to be the
-smaller timedelta. In order to stop mixed-type comparisons from falling back to
-the default comparison by object address, when a :class:`timedelta` object is
-compared to an object of a different type, :exc:`TypeError` is raised unless the
-comparison is ``==`` or ``!=``.  The latter cases return :const:`False` or
-:const:`True`, respectively.
+Comparisons of :class:`timedelta` objects are supported, with some caveats.
+
+The comparisons ``==`` or ``!=`` *always* return a :class:`bool`, no matter
+the type of the compared object::
 
-:class:`timedelta` objects are :term:`hashable` (usable as dictionary keys), support
-efficient pickling, and in Boolean contexts, a :class:`timedelta` object is
+    >>> from datetime import timedelta
+    >>> delta1 = timedelta(seconds=57)
+    >>> delta2 = timedelta(hours=25, seconds=2)
+    >>> delta2 != delta1
+    True
+    >>> delta2 == 5
+    False
+
+For all other comparisons (such as ``<`` and ``>``), when a :class:`timedelta`
+object is compared to an object of a different type, :exc:`TypeError`
+is raised::
+
+    >>> delta2 > delta1
+    True
+    >>> delta2 > 5
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+    TypeError: '>' not supported between instances of 'datetime.timedelta' and 'int'
+
+In Boolean contexts, a :class:`timedelta` object is
 considered to be true if and only if it isn't equal to ``timedelta(0)``.
 
 Instance methods:
@@ -360,29 +422,36 @@ Instance methods:
 
    .. versionadded:: 3.2
 
+Examples of usage: :class:`timedelta`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Example usage:
+An additional example of normalization::
 
+    >>> # Components of another_year add up to exactly 365 days
     >>> from datetime import timedelta
     >>> year = timedelta(days=365)
     >>> another_year = timedelta(weeks=40, days=84, hours=23,
-    ...                          minutes=50, seconds=600)  # adds up to 365 days
-    >>> year.total_seconds()
-    31536000.0
+    ...                          minutes=50, seconds=600)
     >>> year == another_year
     True
+    >>> year.total_seconds()
+    31536000.0
+
+Examples of :class:`timedelta` arithmetic::
+
+    >>> from datetime import timedelta
+    >>> year = timedelta(days=365)
     >>> ten_years = 10 * year
-    >>> ten_years, ten_years.days // 365
-    (datetime.timedelta(days=3650), 10)
+    >>> ten_years
+    datetime.timedelta(days=3650)
+    >>> ten_years.days // 365
+    10
     >>> nine_years = ten_years - year
-    >>> nine_years, nine_years.days // 365
-    (datetime.timedelta(days=3285), 9)
+    >>> nine_years
+    datetime.timedelta(days=3285)
     >>> three_years = nine_years // 3
     >>> three_years, three_years.days // 365
     (datetime.timedelta(days=1095), 3)
-    >>> abs(three_years - ten_years) == 2 * three_years + year
-    True
-
 
 .. _datetime-date:
 
@@ -391,17 +460,14 @@ Example usage:
 
 A :class:`date` object represents a date (year, month and day) in an idealized
 calendar, the current Gregorian calendar indefinitely extended in both
-directions.  January 1 of year 1 is called day number 1, January 2 of year 1 is
-called day number 2, and so on.  This matches the definition of the "proleptic
-Gregorian" calendar in Dershowitz and Reingold's book Calendrical Calculations,
-where it's the base calendar for all computations.  See the book for algorithms
-for converting between proleptic Gregorian ordinals and many other calendar
-systems.
+directions.
 
+January 1 of year 1 is called day number 1, January 2 of year 1 is
+called day number 2, and so on. [#]_
 
 .. class:: date(year, month, day)
 
-   All arguments are required.  Arguments must be integers in the following
+   All arguments are required. Arguments must be integers, in the following
    ranges:
 
    * ``MINYEAR <= year <= MAXYEAR``
@@ -415,17 +481,19 @@ Other constructors, all class methods:
 
 .. classmethod:: date.today()
 
-   Return the current local date.  This is equivalent to
-   ``date.fromtimestamp(time.time())``.
+   Return the current local date.
 
+   This is equivalent to ``date.fromtimestamp(time.time())``.
 
 .. classmethod:: date.fromtimestamp(timestamp)
 
-   Return the local date corresponding to the POSIX timestamp, such as is returned
-   by :func:`time.time`.  This may raise :exc:`OverflowError`, if the timestamp is out
-   of the range of values supported by the platform C :c:func:`localtime` function,
-   and :exc:`OSError` on :c:func:`localtime` failure.
-   It's common for this to be restricted to years from 1970 through 2038.  Note
+   Return the local date corresponding to the POSIX timestamp, such as is
+   returned by :func:`time.time`.
+
+   This may raise :exc:`OverflowError`, if the timestamp is out
+   of the range of values supported by the platform C :c:func:`localtime`
+   function, and :exc:`OSError` on :c:func:`localtime` failure.
+   It's common for this to be restricted to years from 1970 through 2038. Note
    that on non-POSIX systems that include leap seconds in their notion of a
    timestamp, leap seconds are ignored by :meth:`fromtimestamp`.
 
@@ -438,24 +506,27 @@ Other constructors, all class methods:
 
 .. classmethod:: date.fromordinal(ordinal)
 
-   Return the date corresponding to the proleptic Gregorian ordinal, where January
-   1 of year 1 has ordinal 1.  :exc:`ValueError` is raised unless ``1 <= ordinal <=
-   date.max.toordinal()``. For any date *d*, ``date.fromordinal(d.toordinal()) ==
-   d``.
+   Return the date corresponding to the proleptic Gregorian ordinal, where
+   January 1 of year 1 has ordinal 1.
+
+   :exc:`ValueError` is raised unless ``1 <= ordinal <=
+   date.max.toordinal()``. For any date *d*,
+   ``date.fromordinal(d.toordinal()) == d``.
 
 
 .. classmethod:: date.fromisoformat(date_string)
 
-  Return a :class:`date` corresponding to a *date_string* in the format emitted
-  by :meth:`date.isoformat`. Specifically, this function supports strings in
-  the format(s) ``YYYY-MM-DD``.
+   Return a :class:`date` corresponding to a *date_string* given in the format
+   ``YYYY-MM-DD``::
 
-  .. caution::
+      >>> from datetime import date
+      >>> date.fromisoformat('2019-12-04')
+      datetime.date(2019, 12, 4)
 
-    This does not support parsing arbitrary ISO 8601 strings - it is only intended
-    as the inverse operation of :meth:`date.isoformat`.
+   This is the inverse of :meth:`date.isoformat`. It only supports the format
+   ``YYYY-MM-DD``.
 
-  .. versionadded:: 3.7
+   .. versionadded:: 3.7
 
 
 .. classmethod:: date.fromisocalendar(year, week, day)
@@ -507,7 +578,7 @@ Supported operations:
 | Operation                     | Result                                       |
 +===============================+==============================================+
 | ``date2 = date1 + timedelta`` | *date2* is ``timedelta.days`` days removed   |
-|                               | from *date1*.  (1)                           |
+|                               | from *date1*. (1)                            |
 +-------------------------------+----------------------------------------------+
 | ``date2 = date1 - timedelta`` | Computes *date2* such that ``date2 +         |
 |                               | timedelta == date1``. (2)                    |
@@ -522,7 +593,7 @@ Notes:
 
 (1)
    *date2* is moved forward in time if ``timedelta.days > 0``, or backward if
-   ``timedelta.days < 0``.  Afterward ``date2 - date1 == timedelta.days``.
+   ``timedelta.days < 0``. Afterward ``date2 - date1 == timedelta.days``.
    ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored.
    :exc:`OverflowError` is raised if ``date2.year`` would be smaller than
    :const:`MINYEAR` or larger than :const:`MAXYEAR`.
@@ -531,7 +602,7 @@ Notes:
    ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored.
 
 (3)
-   This is exact, and cannot overflow.  timedelta.seconds and
+   This is exact, and cannot overflow. timedelta.seconds and
    timedelta.microseconds are 0, and date2 + timedelta == date1 after.
 
 (4)
@@ -539,38 +610,47 @@ Notes:
    date2.toordinal()``. Date comparison raises :exc:`TypeError` if
    the other comparand isn't also a :class:`date` object. However,
    ``NotImplemented`` is returned instead if the other comparand has a
-   :meth:`timetuple` attribute.  This hook gives other kinds of date objects a
+   :meth:`timetuple` attribute. This hook gives other kinds of date objects a
    chance at implementing mixed-type comparison. If not, when a :class:`date`
    object is compared to an object of a different type, :exc:`TypeError` is raised
-   unless the comparison is ``==`` or ``!=``.  The latter cases return
+   unless the comparison is ``==`` or ``!=``. The latter cases return
    :const:`False` or :const:`True`, respectively.
 
-Dates can be used as dictionary keys. In Boolean contexts, all :class:`date`
-objects are considered to be true.
+In Boolean contexts, all :class:`date` objects are considered to be true.
 
 Instance methods:
 
 .. method:: date.replace(year=self.year, month=self.month, day=self.day)
 
    Return a date with the same value, except for those parameters given new
-   values by whichever keyword arguments are specified.  For example, if ``d ==
-   date(2002, 12, 31)``, then ``d.replace(day=26) == date(2002, 12, 26)``.
+   values by whichever keyword arguments are specified.
+
+   Example::
+
+       >>> from datetime import date
+       >>> d = date(2002, 12, 31)
+       >>> d.replace(day=26)
+       datetime.date(2002, 12, 26)
 
 
 .. method:: date.timetuple()
 
    Return a :class:`time.struct_time` such as returned by :func:`time.localtime`.
-   The hours, minutes and seconds are 0, and the DST flag is -1. ``d.timetuple()``
-   is equivalent to ``time.struct_time((d.year, d.month, d.day, 0, 0, 0,
-   d.weekday(), yday, -1))``, where ``yday = d.toordinal() - date(d.year, 1,
-   1).toordinal() + 1`` is the day number within the current year starting with
-   ``1`` for January 1st.
+
+   The hours, minutes and seconds are 0, and the DST flag is -1.
+
+   ``d.timetuple()`` is equivalent to::
+
+     time.struct_time((d.year, d.month, d.day, 0, 0, 0, d.weekday(), yday, -1))
+
+   where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1``
+   is the day number within the current year starting with ``1`` for January 1st.
 
 
 .. method:: date.toordinal()
 
    Return the proleptic Gregorian ordinal of the date, where January 1 of year 1
-   has ordinal 1.  For any :class:`date` object *d*,
+   has ordinal 1. For any :class:`date` object *d*,
    ``date.fromordinal(d.toordinal()) == d``.
 
 
@@ -592,26 +672,31 @@ Instance methods:
 
    Return a 3-tuple, (ISO year, ISO week number, ISO weekday).
 
-   The ISO calendar is a widely used variant of the Gregorian calendar. See
-   https://www.staff.science.uu.nl/~gent0113/calendar/isocalendar.htm for a good
-   explanation.
+   The ISO calendar is a widely used variant of the Gregorian calendar. [#]_
 
    The ISO year consists of 52 or 53 full weeks, and where a week starts on a
-   Monday and ends on a Sunday.  The first week of an ISO year is the first
+   Monday and ends on a Sunday. The first week of an ISO year is the first
    (Gregorian) calendar week of a year containing a Thursday. This is called week
    number 1, and the ISO year of that Thursday is the same as its Gregorian year.
 
    For example, 2004 begins on a Thursday, so the first week of ISO year 2004
-   begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004, so that
-   ``date(2003, 12, 29).isocalendar() == (2004, 1, 1)`` and ``date(2004, 1,
-   4).isocalendar() == (2004, 1, 7)``.
+   begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004::
 
+       >>> from datetime import date
+       >>> date(2003, 12, 29).isocalendar()
+       (2004, 1, 1)
+       >>> date(2004, 1, 4).isocalendar()
+       (2004, 1, 7)
 
 .. method:: date.isoformat()
 
-   Return a string representing the date in ISO 8601 format, 'YYYY-MM-DD'.  For
-   example, ``date(2002, 12, 4).isoformat() == '2002-12-04'``.
+   Return a string representing the date in ISO 8601 format, ``YYYY-MM-DD``::
 
+       >>> from datetime import date
+       >>> date(2002, 12, 4).isoformat()
+       '2002-12-04'
+
+   This is the inverse of :meth:`date.fromisoformat`.
 
 .. method:: date.__str__()
 
@@ -620,9 +705,17 @@ Instance methods:
 
 .. method:: date.ctime()
 
-   Return a string representing the date, for example ``date(2002, 12,
-   4).ctime() == 'Wed Dec 4 00:00:00 2002'``. ``d.ctime()`` is equivalent to
-   ``time.ctime(time.mktime(d.timetuple()))`` on platforms where the native C
+   Return a string representing the date::
+
+       >>> from datetime import date
+       >>> date(2002, 12, 4).ctime()
+       'Wed Dec  4 00:00:00 2002'
+
+   ``d.ctime()`` is equivalent to::
+
+     time.ctime(time.mktime(d.timetuple()))
+
+   on platforms where the native C
    :c:func:`ctime` function (which :func:`time.ctime` invokes, but which
    :meth:`date.ctime` does not invoke) conforms to the C standard.
 
@@ -643,6 +736,8 @@ Instance methods:
    complete list of formatting directives, see
    :ref:`strftime-strptime-behavior`.
 
+Examples of Usage: :class:`date`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Example of counting days to an event::
 
@@ -662,7 +757,7 @@ Example of counting days to an event::
     >>> time_to_birthday.days
     202
 
-Example of working with :class:`date`:
+More examples of working with :class:`date`:
 
 .. doctest::
 
@@ -670,6 +765,20 @@ Example of working with :class:`date`:
     >>> d = date.fromordinal(730920) # 730920th day after 1. 1. 0001
     >>> d
     datetime.date(2002, 3, 11)
+
+    >>> # Methods related to formatting string output
+    >>> d.isoformat()
+    '2002-03-11'
+    >>> d.strftime("%d/%m/%y")
+    '11/03/02'
+    >>> d.strftime("%A %d. %B %Y")
+    'Monday 11. March 2002'
+    >>> d.ctime()
+    'Mon Mar 11 00:00:00 2002'
+    >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month")
+    'The day is 11, the month is March.'
+
+    >>> # Methods for to extracting 'components' under different calendars
     >>> t = d.timetuple()
     >>> for i in t:     # doctest: +SKIP
     ...     print(i)
@@ -688,14 +797,10 @@ Example of working with :class:`date`:
     2002                # ISO year
     11                  # ISO week number
     1                   # ISO day number ( 1 = Monday )
-    >>> d.isoformat()
-    '2002-03-11'
-    >>> d.strftime("%d/%m/%y")
-    '11/03/02'
-    >>> d.strftime("%A %d. %B %Y")
-    'Monday 11. March 2002'
-    >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month")
-    'The day is 11, the month is March.'
+
+    >>> # A date object is immutable; all operations produce a new object
+    >>> d.replace(year=2005)
+    datetime.date(2005, 3, 11)
 
 
 .. _datetime-datetime:
@@ -704,17 +809,18 @@ Example of working with :class:`date`:
 --------------------------
 
 A :class:`.datetime` object is a single object containing all the information
-from a :class:`date` object and a :class:`.time` object.  Like a :class:`date`
-object, :class:`.datetime` assumes the current Gregorian calendar extended in
-both directions; like a time object, :class:`.datetime` assumes there are exactly
-3600\*24 seconds in every day.
+from a :class:`date` object and a :class:`.time` object.
+
+Like a :class:`date` object, :class:`.datetime` assumes the current Gregorian
+calendar extended in both directions; like a :class:`.time` object,
+:class:`.datetime` assumes there are exactly 3600\*24 seconds in every day.
 
 Constructor:
 
 .. class:: datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0)
 
-   The year, month and day arguments are required.  *tzinfo* may be ``None``, or an
-   instance of a :class:`tzinfo` subclass.  The remaining arguments must be integers
+   The *year*, *month* and *day* arguments are required. *tzinfo* may be ``None``, or an
+   instance of a :class:`tzinfo` subclass. The remaining arguments must be integers
    in the following ranges:
 
    * ``MINYEAR <= year <= MAXYEAR``,
@@ -735,31 +841,43 @@ Other constructors, all class methods:
 
 .. classmethod:: datetime.today()
 
-   Return the current local datetime, with :attr:`.tzinfo` ``None``. This is
-   equivalent to ``datetime.fromtimestamp(time.time())``. See also :meth:`now`,
-   :meth:`fromtimestamp`.
+   Return the current local datetime, with :attr:`.tzinfo` ``None``.
+
+   Equivalent to::
+
+     datetime.fromtimestamp(time.time())
+
+   See also :meth:`now`, :meth:`fromtimestamp`.
 
+   This method is functionally equivalent to :meth:`now`, but without a
+   ``tz`` parameter.
 
 .. classmethod:: datetime.now(tz=None)
 
-   Return the current local date and time.  If optional argument *tz* is ``None``
+   Return the current local date and time.
+
+   If optional argument *tz* is ``None``
    or not specified, this is like :meth:`today`, but, if possible, supplies more
    precision than can be gotten from going through a :func:`time.time` timestamp
    (for example, this may be possible on platforms supplying the C
    :c:func:`gettimeofday` function).
 
    If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the
-   current date and time are converted to *tz*?s time zone.  In this case the
-   result is equivalent to ``tz.fromutc(datetime.utcnow().replace(tzinfo=tz))``.
+   current date and time are converted to *tz*?s time zone. In this case the
+   result is equivalent to::
+
+     tz.fromutc(datetime.utcnow().replace(tzinfo=tz))
+
    See also :meth:`today`, :meth:`utcnow`.
 
 
 .. classmethod:: datetime.utcnow()
 
-   Return the current UTC date and time, with :attr:`.tzinfo` ``None``. This is like
-   :meth:`now`, but returns the current UTC date and time, as a naive
-   :class:`.datetime` object.  An aware current UTC datetime can be obtained by
-   calling ``datetime.now(timezone.utc)``.  See also :meth:`now`.
+   Return the current UTC date and time, with :attr:`.tzinfo` ``None``.
+
+   This is like :meth:`now`, but returns the current UTC date and time, as a naive
+   :class:`.datetime` object. An aware current UTC datetime can be obtained by
+   calling ``datetime.now(timezone.utc)``. See also :meth:`now`.
 
 .. classmethod:: datetime.fromtimestamp(timestamp, tz=None)
 
@@ -769,9 +887,11 @@ Other constructors, all class methods:
    the returned :class:`.datetime` object is naive.
 
    If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the
-   timestamp is converted to *tz*?s time zone.  In this case the result is
-   equivalent to
-   ``tz.fromutc(datetime.utcfromtimestamp(timestamp).replace(tzinfo=tz))``.
+   timestamp is converted to *tz*?s time zone.
+
+   In this case the result is equivalent to::
+
+     tz.fromutc(datetime.utcfromtimestamp(timestamp).replace(tzinfo=tz))
 
    :meth:`fromtimestamp` may raise :exc:`OverflowError`, if the timestamp is out of
    the range of values supported by the platform C :c:func:`localtime` or
@@ -796,7 +916,9 @@ Other constructors, all class methods:
 .. classmethod:: datetime.utcfromtimestamp(timestamp)
 
    Return the UTC :class:`.datetime` corresponding to the POSIX timestamp, with
-   :attr:`.tzinfo` ``None``. This may raise :exc:`OverflowError`, if the timestamp is
+   :attr:`.tzinfo` ``None``.  (The resulting object is naive.)
+
+   This may raise :exc:`OverflowError`, if the timestamp is
    out of the range of values supported by the platform C :c:func:`gmtime` function,
    and :exc:`OSError` on :c:func:`gmtime` failure.
    It's common for this to be restricted to years in 1970 through 2038.
@@ -824,7 +946,7 @@ Other constructors, all class methods:
 
    Return the :class:`.datetime` corresponding to the proleptic Gregorian ordinal,
    where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1
-   <= ordinal <= datetime.max.toordinal()``.  The hour, minute, second and
+   <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and
    microsecond of the result are all 0, and :attr:`.tzinfo` is ``None``.
 
 
@@ -832,13 +954,13 @@ Other constructors, all class methods:
 
    Return a new :class:`.datetime` object whose date components are equal to the
    given :class:`date` object's, and whose time components
-   are equal to the given :class:`.time` object's.  If the *tzinfo*
+   are equal to the given :class:`.time` object's. If the *tzinfo*
    argument is provided, its value is used to set the :attr:`.tzinfo` attribute
    of the result, otherwise the :attr:`~.time.tzinfo` attribute of the *time* argument
    is used.
 
    For any :class:`.datetime` object *d*,
-   ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``.  If date is a
+   ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``. If date is a
    :class:`.datetime` object, its time components and :attr:`.tzinfo` attributes
    are ignored.
 
@@ -848,25 +970,46 @@ Other constructors, all class methods:
 
 .. classmethod:: datetime.fromisoformat(date_string)
 
-  Return a :class:`datetime` corresponding to a *date_string* in one of the
-  formats emitted by :meth:`date.isoformat` and :meth:`datetime.isoformat`.
-  Specifically, this function supports strings in the format(s)
-  ``YYYY-MM-DD[*HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]]``,
-  where ``*`` can match any single character.
+   Return a :class:`.datetime` corresponding to a *date_string* in one of the
+   formats emitted by :meth:`date.isoformat` and :meth:`datetime.isoformat`.
+
+   Specifically, this function supports strings in the format:
+
+   .. code-block:: none
+
+      YYYY-MM-DD[*HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]]
 
-  .. caution::
+   where ``*`` can match any single character.
 
-    This does not support parsing arbitrary ISO 8601 strings - it is only intended
-    as the inverse operation of :meth:`datetime.isoformat`. A more full-featured
-    ISO 8601 parser, ``dateutil.parser.isoparse`` is available in the third-party package
-    `dateutil <https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.isoparse>`_.
+   .. caution::
 
-  .. versionadded:: 3.7
+     This does *not* support parsing arbitrary ISO 8601 strings - it is only intended
+     as the inverse operation of :meth:`datetime.isoformat`. A more full-featured
+     ISO 8601 parser, ``dateutil.parser.isoparse`` is available in the third-party package
+     `dateutil <https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.isoparse>`__.
+     This does not support parsing arbitrary ISO 8601 strings - it is only intended
+     as the inverse operation of :meth:`datetime.isoformat`.
 
+   Examples::
+
+       >>> from datetime import datetime
+       >>> datetime.fromisoformat('2011-11-04')
+       datetime.datetime(2011, 11, 4, 0, 0)
+       >>> datetime.fromisoformat('2011-11-04T00:05:23')
+       datetime.datetime(2011, 11, 4, 0, 5, 23)
+       >>> datetime.fromisoformat('2011-11-04 00:05:23.283')
+       datetime.datetime(2011, 11, 4, 0, 5, 23, 283000)
+       >>> datetime.fromisoformat('2011-11-04 00:05:23.283+00:00')
+       datetime.datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=datetime.timezone.utc)
+       >>> datetime.fromisoformat('2011-11-04T00:05:23+04:00')   # doctest: +NORMALIZE_WHITESPACE
+       datetime.datetime(2011, 11, 4, 0, 5, 23,
+           tzinfo=datetime.timezone(datetime.timedelta(seconds=14400)))
+
+   .. versionadded:: 3.7
 
 .. classmethod:: datetime.fromisocalendar(year, week, day)
 
-   Return a :class:`datetime` corresponding to the ISO calendar date specified
+   Return a :class:`.datetime` corresponding to the ISO calendar date specified
    by year, week and day. The non-date components of the datetime are populated
    with their normal default values. This is the inverse of the function
    :meth:`datetime.isocalendar`.
@@ -876,8 +1019,13 @@ Other constructors, all class methods:
 .. classmethod:: datetime.strptime(date_string, format)
 
    Return a :class:`.datetime` corresponding to *date_string*, parsed according to
-   *format*.  This is equivalent to ``datetime(*(time.strptime(date_string,
-   format)[0:6]))``. :exc:`ValueError` is raised if the date_string and format
+   *format*.
+
+   This is equivalent to::
+
+     datetime(*(time.strptime(date_string, format)[0:6]))
+
+   :exc:`ValueError` is raised if the date_string and format
    can't be parsed by :func:`time.strptime` or if it returns a value which isn't a
    time tuple. For a complete list of formatting directives, see
    :ref:`strftime-strptime-behavior`.
@@ -949,7 +1097,7 @@ Instance attributes (read-only):
 
 .. attribute:: datetime.fold
 
-   In ``[0, 1]``.  Used to disambiguate wall times during a repeated interval.  (A
+   In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A
    repeated interval occurs when clocks are rolled back at the end of daylight saving
    time or when the UTC offset for the current zone is decreased for political reasons.)
    The value 0 (1) represents the earlier (later) of the two moments with the same wall
@@ -974,7 +1122,7 @@ Supported operations:
 
 (1)
    datetime2 is a duration of timedelta removed from datetime1, moving forward in
-   time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0.  The
+   time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0. The
    result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and
    datetime2 - datetime1 == timedelta after. :exc:`OverflowError` is raised if
    datetime2.year would be smaller than :const:`MINYEAR` or larger than
@@ -988,16 +1136,16 @@ Supported operations:
 
 (3)
    Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if
-   both operands are naive, or if both are aware.  If one is aware and the other is
+   both operands are naive, or if both are aware. If one is aware and the other is
    naive, :exc:`TypeError` is raised.
 
    If both are naive, or both are aware and have the same :attr:`~.datetime.tzinfo` attribute,
    the :attr:`~.datetime.tzinfo` attributes are ignored, and the result is a :class:`timedelta`
-   object *t* such that ``datetime2 + t == datetime1``.  No time zone adjustments
+   object *t* such that ``datetime2 + t == datetime1``. No time zone adjustments
    are done in this case.
 
    If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts
-   as if *a* and *b* were first converted to naive UTC datetimes first.  The
+   as if *a* and *b* were first converted to naive UTC datetimes first. The
    result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None)
    - b.utcoffset())`` except that the implementation never overflows.
 
@@ -1006,34 +1154,31 @@ Supported operations:
    *datetime2* in time.
 
    If one comparand is naive and the other is aware, :exc:`TypeError`
-   is raised if an order comparison is attempted.  For equality
+   is raised if an order comparison is attempted. For equality
    comparisons, naive instances are never equal to aware instances.
 
    If both comparands are aware, and have the same :attr:`~.datetime.tzinfo` attribute, the
    common :attr:`~.datetime.tzinfo` attribute is ignored and the base datetimes are
-   compared.  If both comparands are aware and have different :attr:`~.datetime.tzinfo`
+   compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo`
    attributes, the comparands are first adjusted by subtracting their UTC
    offsets (obtained from ``self.utcoffset()``).
 
    .. versionchanged:: 3.3
-      Equality comparisons between naive and aware :class:`.datetime`
+      Equality comparisons between aware and naive :class:`.datetime`
       instances don't raise :exc:`TypeError`.
 
    .. note::
 
       In order to stop comparison from falling back to the default scheme of comparing
       object addresses, datetime comparison normally raises :exc:`TypeError` if the
-      other comparand isn't also a :class:`.datetime` object.  However,
+      other comparand isn't also a :class:`.datetime` object. However,
       ``NotImplemented`` is returned instead if the other comparand has a
-      :meth:`timetuple` attribute.  This hook gives other kinds of date objects a
-      chance at implementing mixed-type comparison.  If not, when a :class:`.datetime`
+      :meth:`timetuple` attribute. This hook gives other kinds of date objects a
+      chance at implementing mixed-type comparison. If not, when a :class:`.datetime`
       object is compared to an object of a different type, :exc:`TypeError` is raised
-      unless the comparison is ``==`` or ``!=``.  The latter cases return
+      unless the comparison is ``==`` or ``!=``. The latter cases return
       :const:`False` or :const:`True`, respectively.
 
-:class:`.datetime` objects can be used as dictionary keys. In Boolean contexts,
-all :class:`.datetime` objects are considered to be true.
-
 Instance methods:
 
 .. method:: datetime.date()
@@ -1044,7 +1189,7 @@ Instance methods:
 .. method:: datetime.time()
 
    Return :class:`.time` object with same hour, minute, second, microsecond and fold.
-   :attr:`.tzinfo` is ``None``.  See also method :meth:`timetz`.
+   :attr:`.tzinfo` is ``None``. See also method :meth:`timetz`.
 
    .. versionchanged:: 3.6
       The fold value is copied to the returned :class:`.time` object.
@@ -1053,7 +1198,7 @@ Instance methods:
 .. method:: datetime.timetz()
 
    Return :class:`.time` object with same hour, minute, second, microsecond, fold, and
-   tzinfo attributes.  See also method :meth:`time`.
+   tzinfo attributes. See also method :meth:`time`.
 
    .. versionchanged:: 3.6
       The fold value is copied to the returned :class:`.time` object.
@@ -1064,7 +1209,7 @@ Instance methods:
    tzinfo=self.tzinfo, * fold=0)
 
    Return a datetime with the same attributes, except for those attributes given
-   new values by whichever keyword arguments are specified.  Note that
+   new values by whichever keyword arguments are specified. Note that
    ``tzinfo=None`` can be specified to create a naive datetime from an aware
    datetime with no conversion of date and time data.
 
@@ -1079,11 +1224,11 @@ Instance methods:
    *self*, but in *tz*'s local time.
 
    If provided, *tz* must be an instance of a :class:`tzinfo` subclass, and its
-   :meth:`utcoffset` and :meth:`dst` methods must not return ``None``.  If *self*
+   :meth:`utcoffset` and :meth:`dst` methods must not return ``None``. If *self*
    is naive, it is presumed to represent time in the system timezone.
 
    If called without arguments (or with ``tz=None``) the system local
-   timezone is assumed for the target timezone.  The ``.tzinfo`` attribute of the converted
+   timezone is assumed for the target timezone. The ``.tzinfo`` attribute of the converted
    datetime instance will be set to an instance of :class:`timezone`
    with the zone name and offset obtained from the OS.
 
@@ -1094,7 +1239,7 @@ Instance methods:
    the same date and time data as ``dt - dt.utcoffset()``.
 
    If you merely want to attach a time zone object *tz* to a datetime *dt* without
-   adjustment of date and time data, use ``dt.replace(tzinfo=tz)``.  If you
+   adjustment of date and time data, use ``dt.replace(tzinfo=tz)``. If you
    merely want to remove the time zone object from an aware datetime *dt* without
    conversion of date and time data, use ``dt.replace(tzinfo=None)``.
 
@@ -1148,25 +1293,31 @@ Instance methods:
 .. method:: datetime.timetuple()
 
    Return a :class:`time.struct_time` such as returned by :func:`time.localtime`.
-   ``d.timetuple()`` is equivalent to ``time.struct_time((d.year, d.month, d.day,
-   d.hour, d.minute, d.second, d.weekday(), yday, dst))``, where ``yday =
-   d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` is the day number within
-   the current year starting with ``1`` for January 1st. The :attr:`tm_isdst` flag
-   of the result is set according to the :meth:`dst` method: :attr:`.tzinfo` is
-   ``None`` or :meth:`dst` returns ``None``, :attr:`tm_isdst` is set to ``-1``;
-   else if :meth:`dst` returns a non-zero value, :attr:`tm_isdst` is set to ``1``;
-   else :attr:`tm_isdst` is set to ``0``.
+
+   ``d.timetuple()`` is equivalent to::
+
+     time.struct_time((d.year, d.month, d.day,
+                       d.hour, d.minute, d.second,
+                       d.weekday(), yday, dst))
+
+   where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1``
+   is the day number within the current year starting with ``1`` for January
+   1st. The :attr:`tm_isdst` flag of the result is set according to the
+   :meth:`dst` method: :attr:`.tzinfo` is ``None`` or :meth:`dst` returns
+   ``None``, :attr:`tm_isdst` is set to ``-1``; else if :meth:`dst` returns a
+   non-zero value, :attr:`tm_isdst` is set to ``1``; else :attr:`tm_isdst` is
+   set to ``0``.
 
 
 .. method:: datetime.utctimetuple()
 
    If :class:`.datetime` instance *d* is naive, this is the same as
    ``d.timetuple()`` except that :attr:`tm_isdst` is forced to 0 regardless of what
-   ``d.dst()`` returns.  DST is never in effect for a UTC time.
+   ``d.dst()`` returns. DST is never in effect for a UTC time.
 
    If *d* is aware, *d* is normalized to UTC time, by subtracting
    ``d.utcoffset()``, and a :class:`time.struct_time` for the
-   normalized time is returned.  :attr:`tm_isdst` is forced to 0. Note
+   normalized time is returned. :attr:`tm_isdst` is forced to 0. Note
    that an :exc:`OverflowError` may be raised if *d*.year was
    ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year
    boundary.
@@ -1174,18 +1325,18 @@ Instance methods:
 
 .. method:: datetime.toordinal()
 
-   Return the proleptic Gregorian ordinal of the date.  The same as
+   Return the proleptic Gregorian ordinal of the date. The same as
    ``self.date().toordinal()``.
 
 .. method:: datetime.timestamp()
 
    Return POSIX timestamp corresponding to the :class:`.datetime`
-   instance.  The return value is a :class:`float` similar to that
+   instance. The return value is a :class:`float` similar to that
    returned by :func:`time.time`.
 
    Naive :class:`.datetime` instances are assumed to represent local
    time and this method relies on the platform C :c:func:`mktime`
-   function to perform the conversion.  Since :class:`.datetime`
+   function to perform the conversion. Since :class:`.datetime`
    supports wider range of values than :c:func:`mktime` on many
    platforms, this method may raise :exc:`OverflowError` for times far
    in the past or far in the future.
@@ -1204,7 +1355,7 @@ Instance methods:
    .. note::
 
       There is no method to obtain the POSIX timestamp directly from a
-      naive :class:`.datetime` instance representing UTC time.  If your
+      naive :class:`.datetime` instance representing UTC time. If your
       application uses this convention and your system timezone is not
       set to UTC, you can obtain the POSIX timestamp by supplying
       ``tzinfo=timezone.utc``::
@@ -1230,30 +1381,45 @@ Instance methods:
 
 .. method:: datetime.isocalendar()
 
-   Return a 3-tuple, (ISO year, ISO week number, ISO weekday).  The same as
+   Return a 3-tuple, (ISO year, ISO week number, ISO weekday). The same as
    ``self.date().isocalendar()``.
 
 
 .. method:: datetime.isoformat(sep='T', timespec='auto')
 
-   Return a string representing the date and time in ISO 8601 format,
-   YYYY-MM-DDTHH:MM:SS.ffffff or, if :attr:`microsecond` is 0,
-   YYYY-MM-DDTHH:MM:SS
+   Return a string representing the date and time in ISO 8601 format:
+
+   - ``YYYY-MM-DDTHH:MM:SS.ffffff``, if :attr:`microsecond` is not 0
+   - ``YYYY-MM-DDTHH:MM:SS``, if :attr:`microsecond` is 0
 
    If :meth:`utcoffset` does not return ``None``, a string is
    appended, giving the UTC offset:
-   YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]] or, if :attr:`microsecond`
-   is 0 YYYY-MM-DDTHH:MM:SS+HH:MM[:SS[.ffffff]].
+
+   - ``YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond`
+     is not 0
+   - ``YYYY-MM-DDTHH:MM:SS+HH:MM[:SS[.ffffff]]``,  if :attr:`microsecond` is 0
+
+   Examples::
+
+       >>> from datetime import datetime, timezone
+       >>> datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat()
+       '2019-05-18T15:17:08.132263'
+       >>> datetime(2019, 5, 18, 15, 17, tzinfo=timezone.utc).isoformat()
+       '2019-05-18T15:17:00+00:00'
 
    The optional argument *sep* (default ``'T'``) is a one-character separator,
-   placed between the date and time portions of the result.  For example,
+   placed between the date and time portions of the result. For example::
 
       >>> from datetime import tzinfo, timedelta, datetime
       >>> class TZ(tzinfo):
-      ...     def utcoffset(self, dt): return timedelta(minutes=-399)
+      ...     """A time zone with an arbitrary, constant -06:39 offset."""
+      ...     def utcoffset(self, dt):
+      ...         return timedelta(hours=-6, minutes=-39)
       ...
       >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ')
       '2002-12-25 00:00:00-06:39'
+      >>> datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat()
+      '2009-11-27T00:00:00.000100-06:39'
 
    The optional argument *timespec* specifies the number of additional
    components of the time to include (the default is ``'auto'``).
@@ -1261,19 +1427,19 @@ Instance methods:
 
    - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0,
      same as ``'microseconds'`` otherwise.
-   - ``'hours'``: Include the :attr:`hour` in the two-digit HH format.
-   - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format.
+   - ``'hours'``: Include the :attr:`hour` in the two-digit ``HH`` format.
+   - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in ``HH:MM`` format.
    - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second`
-     in HH:MM:SS format.
+     in ``HH:MM:SS`` format.
    - ``'milliseconds'``: Include full time, but truncate fractional second
-     part to milliseconds. HH:MM:SS.sss format.
-   - ``'microseconds'``: Include full time in HH:MM:SS.ffffff format.
+     part to milliseconds. ``HH:MM:SS.sss`` format.
+   - ``'microseconds'``: Include full time in ``HH:MM:SS.ffffff`` format.
 
    .. note::
 
       Excluded time components are truncated, not rounded.
 
-   :exc:`ValueError` will be raised on an invalid *timespec* argument.
+   :exc:`ValueError` will be raised on an invalid *timespec* argument::
 
 
       >>> from datetime import datetime
@@ -1295,48 +1461,64 @@ Instance methods:
 
 .. method:: datetime.ctime()
 
-   Return a string representing the date and time, for example ``datetime(2002, 12,
-   4, 20, 30, 40).ctime() == 'Wed Dec  4 20:30:40 2002'``. ``d.ctime()`` is
-   equivalent to ``time.ctime(time.mktime(d.timetuple()))`` on platforms where the
-   native C :c:func:`ctime` function (which :func:`time.ctime` invokes, but which
-   :meth:`datetime.ctime` does not invoke) conforms to the C standard.
+   Return a string representing the date and time::
+
+       >>> from datetime import datetime
+       >>> datetime(2002, 12, 4, 20, 30, 40).ctime()
+       'Wed Dec  4 20:30:40 2002'
+
+   The output string will *not* include time zone information, regardless
+   of whether the input is aware or naive.
 
+   ``d.ctime()`` is equivalent to::
+
+     time.ctime(time.mktime(d.timetuple()))
+
+   on platforms where the native C :c:func:`ctime` function
+   (which :func:`time.ctime` invokes, but which
+   :meth:`datetime.ctime` does not invoke) conforms to the C standard.
 
 .. method:: datetime.strftime(format)
 
    Return a string representing the date and time, controlled by an explicit format
-   string.  For a complete list of formatting directives, see
+   string. For a complete list of formatting directives, see
    :ref:`strftime-strptime-behavior`.
 
 
 .. method:: datetime.__format__(format)
 
-   Same as :meth:`.datetime.strftime`.  This makes it possible to specify a format
+   Same as :meth:`.datetime.strftime`. This makes it possible to specify a format
    string for a :class:`.datetime` object in :ref:`formatted string
-   literals <f-strings>` and when using :meth:`str.format`.  For a
+   literals <f-strings>` and when using :meth:`str.format`. For a
    complete list of formatting directives, see
    :ref:`strftime-strptime-behavior`.
 
+Examples of Usage: :class:`.datetime`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Examples of working with datetime objects:
+Examples of working with :class:`~datetime.datetime` objects:
 
 .. doctest::
 
     >>> from datetime import datetime, date, time
+
     >>> # Using datetime.combine()
     >>> d = date(2005, 7, 14)
     >>> t = time(12, 30)
     >>> datetime.combine(d, t)
     datetime.datetime(2005, 7, 14, 12, 30)
+
     >>> # Using datetime.now() or datetime.utcnow()
     >>> datetime.now()   # doctest: +SKIP
     datetime.datetime(2007, 12, 6, 16, 29, 43, 79043)   # GMT +1
     >>> datetime.utcnow()   # doctest: +SKIP
     datetime.datetime(2007, 12, 6, 15, 29, 43, 79060)
+
     >>> # Using datetime.strptime()
     >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M")
     >>> dt
     datetime.datetime(2006, 11, 21, 16, 30)
+
     >>> # Using datetime.timetuple() to get tuple of all attributes
     >>> tt = dt.timetuple()
     >>> for it in tt:   # doctest: +SKIP
@@ -1351,6 +1533,7 @@ Examples of working with datetime objects:
     1       # weekday (0 = Monday)
     325     # number of days since 1st January
     -1      # dst - method tzinfo.dst() returned None
+
     >>> # Date in ISO format
     >>> ic = dt.isocalendar()
     >>> for it in ic:   # doctest: +SKIP
@@ -1359,89 +1542,95 @@ Examples of working with datetime objects:
     2006    # ISO year
     47      # ISO week
     2       # ISO weekday
-    >>> # Formatting datetime
+
+    >>> # Formatting a datetime
     >>> dt.strftime("%A, %d. %B %Y %I:%M%p")
     'Tuesday, 21. November 2006 04:30PM'
     >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time")
     'The day is 21, the month is November, the time is 04:30PM.'
 
-Using datetime with tzinfo:
-
-    >>> from datetime import timedelta, datetime, tzinfo, timezone
-    >>> class KabulTz(tzinfo):
-    ...     # Kabul used +4 until 1945, when they moved to +4:30
-    ...     UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc)
-    ...     def utcoffset(self, dt):
-    ...         if dt.year < 1945:
-    ...             return timedelta(hours=4)
-    ...         elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30):
-    ...             # If dt falls in the imaginary range, use fold to decide how
-    ...             # to resolve. See PEP495
-    ...             return timedelta(hours=4, minutes=(30 if dt.fold else 0))
-    ...         else:
-    ...             return timedelta(hours=4, minutes=30)
-    ...
-    ...     def fromutc(self, dt):
-    ...         # A custom implementation is required for fromutc as
-    ...         # the input to this function is a datetime with utc values
-    ...         # but with a tzinfo set to self
-    ...         # See datetime.astimezone or fromtimestamp
-    ...
-    ...         # Follow same validations as in datetime.tzinfo
-    ...         if not isinstance(dt, datetime):
-    ...             raise TypeError("fromutc() requires a datetime argument")
-    ...         if dt.tzinfo is not self:
-    ...             raise ValueError("dt.tzinfo is not self")
-    ...
-    ...         if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE:
-    ...             return dt + timedelta(hours=4, minutes=30)
-    ...         else:
-    ...             return dt + timedelta(hours=4)
-    ...
-    ...     def dst(self, dt):
-    ...         return timedelta(0)
-    ...
-    ...     def tzname(self, dt):
-    ...         if dt >= self.UTC_MOVE_DATE:
-    ...             return "+04:30"
-    ...         else:
-    ...             return "+04"
-    ...
-    ...     def  __repr__(self):
-    ...         return f"{self.__class__.__name__}()"
-    ...
-    >>> tz1 = KabulTz()
-    >>> # Datetime before the change
-    >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1)
-    >>> print(dt1.utcoffset())
-    4:00:00
-    >>> # Datetime after the change
-    >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1)
-    >>> print(dt2.utcoffset())
-    4:30:00
-    >>> # Convert datetime to another time zone
-    >>> dt3 = dt2.astimezone(timezone.utc)
-    >>> dt3
-    datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc)
-    >>> dt2
-    datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz())
-    >>> dt2.utctimetuple() == dt3.utctimetuple()
-    True
-
-
+The example below defines a :class:`tzinfo` subclass capturing time zone
+information for Kabul, Afghanistan, which used +4 UTC until 1945
+and then +4:30 UTC thereafter::
+
+   from datetime import timedelta, datetime, tzinfo, timezone
+
+   class KabulTz(tzinfo):
+       # Kabul used +4 until 1945, when they moved to +4:30
+       UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc)
+
+       def utcoffset(self, dt):
+           if dt.year < 1945:
+               return timedelta(hours=4)
+           elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30):
+               # An ambiguous ("imaginary") half-hour range representing
+               # a 'fold' in time due to the shift from +4 to +4:30.
+               # If dt falls in the imaginary range, use fold to decide how
+               # to resolve. See PEP495.
+               return timedelta(hours=4, minutes=(30 if dt.fold else 0))
+           else:
+               return timedelta(hours=4, minutes=30)
+
+       def fromutc(self, dt):
+           # Follow same validations as in datetime.tzinfo
+           if not isinstance(dt, datetime):
+               raise TypeError("fromutc() requires a datetime argument")
+           if dt.tzinfo is not self:
+               raise ValueError("dt.tzinfo is not self")
+
+           # A custom implementation is required for fromutc as
+           # the input to this function is a datetime with utc values
+           # but with a tzinfo set to self.
+           # See datetime.astimezone or fromtimestamp.
+           if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE:
+               return dt + timedelta(hours=4, minutes=30)
+           else:
+               return dt + timedelta(hours=4)
+
+       def dst(self, dt):
+           # Kabul does not observe daylight saving time.
+           return timedelta(0)
+
+       def tzname(self, dt):
+           if dt >= self.UTC_MOVE_DATE:
+               return "+04:30"
+           return "+04"
+
+Usage of ``KabulTz`` from above::
+
+   >>> tz1 = KabulTz()
+
+   >>> # Datetime before the change
+   >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1)
+   >>> print(dt1.utcoffset())
+   4:00:00
+
+   >>> # Datetime after the change
+   >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1)
+   >>> print(dt2.utcoffset())
+   4:30:00
+
+   >>> # Convert datetime to another time zone
+   >>> dt3 = dt2.astimezone(timezone.utc)
+   >>> dt3
+   datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc)
+   >>> dt2
+   datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz())
+   >>> dt2.utctimetuple() == dt3.utctimetuple()
+   True
 
 .. _datetime-time:
 
 :class:`.time` Objects
 ----------------------
 
-A time object represents a (local) time of day, independent of any particular
+A :class:`time` object represents a (local) time of day, independent of any particular
 day, and subject to adjustment via a :class:`tzinfo` object.
 
 .. class:: time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0)
 
-   All arguments are optional.  *tzinfo* may be ``None``, or an instance of a
-   :class:`tzinfo` subclass.  The remaining arguments must be integers in the
+   All arguments are optional. *tzinfo* may be ``None``, or an instance of a
+   :class:`tzinfo` subclass. The remaining arguments must be integers in the
    following ranges:
 
    * ``0 <= hour < 24``,
@@ -1450,7 +1639,7 @@ day, and subject to adjustment via a :class:`tzinfo` object.
    * ``0 <= microsecond < 1000000``,
    * ``fold in [0, 1]``.
 
-   If an argument outside those ranges is given, :exc:`ValueError` is raised.  All
+   If an argument outside those ranges is given, :exc:`ValueError` is raised. All
    default to ``0`` except *tzinfo*, which defaults to :const:`None`.
 
 Class attributes:
@@ -1503,7 +1692,7 @@ Instance attributes (read-only):
 
 .. attribute:: time.fold
 
-   In ``[0, 1]``.  Used to disambiguate wall times during a repeated interval.  (A
+   In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A
    repeated interval occurs when clocks are rolled back at the end of daylight saving
    time or when the UTC offset for the current zone is decreased for political reasons.)
    The value 0 (1) represents the earlier (later) of the two moments with the same wall
@@ -1511,38 +1700,32 @@ Instance attributes (read-only):
 
    .. versionadded:: 3.6
 
+:class:`.time` objects support comparison of :class:`.time` to :class:`.time`,
+where *a* is considered less
+than *b* when *a* precedes *b* in time. If one comparand is naive and the other
+is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality
+comparisons, naive instances are never equal to aware instances.
 
-Supported operations:
-
-* comparison of :class:`.time` to :class:`.time`, where *a* is considered less
-  than *b* when *a* precedes *b* in time.  If one comparand is naive and the other
-  is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality
-  comparisons, naive instances are never equal to aware instances.
+If both comparands are aware, and have
+the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is
+ignored and the base times are compared. If both comparands are aware and
+have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by
+subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order
+to stop mixed-type comparisons from falling back to the default comparison by
+object address, when a :class:`.time` object is compared to an object of a
+different type, :exc:`TypeError` is raised unless the comparison is ``==`` or
+``!=``. The latter cases return :const:`False` or :const:`True`, respectively.
 
-  If both comparands are aware, and have
-  the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is
-  ignored and the base times are compared.  If both comparands are aware and
-  have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by
-  subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order
-  to stop mixed-type comparisons from falling back to the default comparison by
-  object address, when a :class:`.time` object is compared to an object of a
-  different type, :exc:`TypeError` is raised unless the comparison is ``==`` or
-  ``!=``.  The latter cases return :const:`False` or :const:`True`, respectively.
+.. versionchanged:: 3.3
+  Equality comparisons between aware and naive :class:`~datetime.time` instances
+  don't raise :exc:`TypeError`.
 
-  .. versionchanged:: 3.3
-     Equality comparisons between naive and aware :class:`~datetime.time` instances
-     don't raise :exc:`TypeError`.
-
-* hash, use as dict key
-
-* efficient pickling
-
-In boolean contexts, a :class:`.time` object is always considered to be true.
+In Boolean contexts, a :class:`.time` object is always considered to be true.
 
 .. versionchanged:: 3.5
    Before Python 3.5, a :class:`.time` object was considered to be false if it
-   represented midnight in UTC.  This behavior was considered obscure and
-   error-prone and has been removed in Python 3.5.  See :issue:`13936` for full
+   represented midnight in UTC. This behavior was considered obscure and
+   error-prone and has been removed in Python 3.5. See :issue:`13936` for full
    details.
 
 
@@ -1550,16 +1733,30 @@ Other constructor:
 
 .. classmethod:: time.fromisoformat(time_string)
 
-  Return a :class:`time` corresponding to a *time_string* in one of the
-  formats emitted by :meth:`time.isoformat`. Specifically, this function supports
-  strings in the format(s) ``HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]``.
+   Return a :class:`.time` corresponding to a *time_string* in one of the
+   formats emitted by :meth:`time.isoformat`. Specifically, this function supports
+   strings in the format:
+
+   .. code-block:: none
+
+      HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]
+
+   .. caution::
 
-  .. caution::
+     This does *not* support parsing arbitrary ISO 8601 strings. It is only
+     intended as the inverse operation of :meth:`time.isoformat`.
 
-    This does not support parsing arbitrary ISO 8601 strings - it is only intended
-    as the inverse operation of :meth:`time.isoformat`.
+   Examples::
 
-  .. versionadded:: 3.7
+       >>> from datetime import time
+       >>> time.fromisoformat('04:23:01')
+       datetime.time(4, 23, 1)
+       >>> time.fromisoformat('04:23:01.000384')
+       datetime.time(4, 23, 1, 384)
+       >>> time.fromisoformat('04:23:01+04:00')
+       datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400)))
+
+   .. versionadded:: 3.7
 
 
 Instance methods:
@@ -1568,7 +1765,7 @@ Instance methods:
    microsecond=self.microsecond, tzinfo=self.tzinfo, * fold=0)
 
    Return a :class:`.time` with the same value, except for those attributes given
-   new values by whichever keyword arguments are specified.  Note that
+   new values by whichever keyword arguments are specified. Note that
    ``tzinfo=None`` can be specified to create a naive :class:`.time` from an
    aware :class:`.time`, without conversion of the time data.
 
@@ -1578,10 +1775,12 @@ Instance methods:
 
 .. method:: time.isoformat(timespec='auto')
 
-   Return a string representing the time in ISO 8601 format, HH:MM:SS.ffffff or, if
-   :attr:`microsecond` is 0, HH:MM:SS If :meth:`utcoffset` does not return ``None``, a
-   string is appended, giving the UTC offset: HH:MM:SS.ffffff+HH:MM[:SS[.ffffff]]
-   or, if self.microsecond is 0, HH:MM:SS+HH:MM[:SS[.ffffff]].
+   Return a string representing the time in ISO 8601 format, one of:
+
+   - ``HH:MM:SS.ffffff``, if :attr:`microsecond` is not 0
+   - ``HH:MM:SS``, if :attr:`microsecond` is 0
+   - ``HH:MM:SS.ffffff+HH:MM[:SS[.ffffff]]``, if :meth:`utcoffset` does not return ``None``
+   - ``HH:MM:SS+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond` is 0 and :meth:`utcoffset` does not return ``None``
 
    The optional argument *timespec* specifies the number of additional
    components of the time to include (the default is ``'auto'``).
@@ -1589,13 +1788,13 @@ Instance methods:
 
    - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0,
      same as ``'microseconds'`` otherwise.
-   - ``'hours'``: Include the :attr:`hour` in the two-digit HH format.
-   - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format.
+   - ``'hours'``: Include the :attr:`hour` in the two-digit ``HH`` format.
+   - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in ``HH:MM`` format.
    - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second`
-     in HH:MM:SS format.
+     in ``HH:MM:SS`` format.
    - ``'milliseconds'``: Include full time, but truncate fractional second
-     part to milliseconds. HH:MM:SS.sss format.
-   - ``'microseconds'``: Include full time in HH:MM:SS.ffffff format.
+     part to milliseconds. ``HH:MM:SS.sss`` format.
+   - ``'microseconds'``: Include full time in ``HH:MM:SS.ffffff`` format.
 
    .. note::
 
@@ -1603,6 +1802,7 @@ Instance methods:
 
    :exc:`ValueError` will be raised on an invalid *timespec* argument.
 
+   Example::
 
       >>> from datetime import time
       >>> time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes')
@@ -1625,7 +1825,7 @@ Instance methods:
 .. method:: time.strftime(format)
 
    Return a string representing the time, controlled by an explicit format
-   string.  For a complete list of formatting directives, see
+   string. For a complete list of formatting directives, see
    :ref:`strftime-strptime-behavior`.
 
 
@@ -1633,7 +1833,7 @@ Instance methods:
 
    Same as :meth:`.time.strftime`. This makes it possible to specify a format string
    for a :class:`.time` object in :ref:`formatted string
-   literals <f-strings>` and when using :meth:`str.format`.  For a
+   literals <f-strings>` and when using :meth:`str.format`. For a
    complete list of formatting directives, see
    :ref:`strftime-strptime-behavior`.
 
@@ -1663,7 +1863,10 @@ Instance methods:
    ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't
    return ``None`` or a string object.
 
-Example:
+Examples of Usage: :class:`.time`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Examples of working with a :class:`.time` object::
 
     >>> from datetime import time, tzinfo, timedelta
     >>> class TZ1(tzinfo):
@@ -1699,12 +1902,8 @@ Example:
 .. class:: tzinfo()
 
    This is an abstract base class, meaning that this class should not be
-   instantiated directly.  You need to derive a concrete subclass, and (at least)
-   supply implementations of the standard :class:`tzinfo` methods needed by the
-   :class:`.datetime` methods you use.  The :mod:`datetime` module supplies
-   a simple concrete subclass of :class:`tzinfo`, :class:`timezone`, which can represent
-   timezones with fixed offset from UTC such as UTC itself or North American EST and
-   EDT.
+   instantiated directly.  Define a subclass of :class:`tzinfo` to capture
+   information about a particular time zone.
 
    An instance of (a concrete subclass of) :class:`tzinfo` can be passed to the
    constructors for :class:`.datetime` and :class:`.time` objects. The latter objects
@@ -1712,28 +1911,35 @@ Example:
    supports methods revealing offset of local time from UTC, the name of the time
    zone, and DST offset, all relative to a date or time object passed to them.
 
+   You need to derive a concrete subclass, and (at least)
+   supply implementations of the standard :class:`tzinfo` methods needed by the
+   :class:`.datetime` methods you use. The :mod:`datetime` module provides
+   :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can
+   represent timezones with fixed offset from UTC such as UTC itself or North
+   American EST and EDT.
+
    Special requirement for pickling:  A :class:`tzinfo` subclass must have an
-   :meth:`__init__` method that can be called with no arguments, else it can be
-   pickled but possibly not unpickled again.  This is a technical requirement that
+   :meth:`__init__` method that can be called with no arguments, otherwise it can be
+   pickled but possibly not unpickled again. This is a technical requirement that
    may be relaxed in the future.
 
    A concrete subclass of :class:`tzinfo` may need to implement the following
-   methods.  Exactly which methods are needed depends on the uses made of aware
-   :mod:`datetime` objects.  If in doubt, simply implement all of them.
+   methods. Exactly which methods are needed depends on the uses made of aware
+   :mod:`datetime` objects. If in doubt, simply implement all of them.
 
 
 .. method:: tzinfo.utcoffset(dt)
 
    Return offset of local time from UTC, as a :class:`timedelta` object that is
-   positive east of UTC.  If local time is
-   west of UTC, this should be negative.  Note that this is intended to be the
-   total offset from UTC; for example, if a :class:`tzinfo` object represents both
-   time zone and DST adjustments, :meth:`utcoffset` should return their sum.  If
-   the UTC offset isn't known, return ``None``.  Else the value returned must be a
-   :class:`timedelta` object strictly between ``-timedelta(hours=24)`` and
-   ``timedelta(hours=24)`` (the magnitude of the offset must be less
-   than one day).  Most implementations of :meth:`utcoffset` will probably look
-   like one of these two::
+   positive east of UTC. If local time is west of UTC, this should be negative.
+
+   This represents the *total* offset from UTC; for example, if a
+   :class:`tzinfo` object represents both time zone and DST adjustments,
+   :meth:`utcoffset` should return their sum. If the UTC offset isn't known,
+   return ``None``. Else the value returned must be a :class:`timedelta` object
+   strictly between ``-timedelta(hours=24)`` and ``timedelta(hours=24)``
+   (the magnitude of the offset must be less than one day). Most implementations
+   of :meth:`utcoffset` will probably look like one of these two::
 
       return CONSTANT                 # fixed-offset class
       return CONSTANT + self.dst(dt)  # daylight-aware class
@@ -1752,12 +1958,14 @@ Example:
 
    Return the daylight saving time (DST) adjustment, as a :class:`timedelta`
    object or
-   ``None`` if DST information isn't known.  Return ``timedelta(0)`` if DST is not
-   in effect. If DST is in effect, return the offset as a :class:`timedelta` object
+   ``None`` if DST information isn't known.
+
+   Return ``timedelta(0)`` if DST is not in effect.
+   If DST is in effect, return the offset as a :class:`timedelta` object
    (see :meth:`utcoffset` for details). Note that DST offset, if applicable, has
    already been added to the UTC offset returned by :meth:`utcoffset`, so there's
    no need to consult :meth:`dst` unless you're interested in obtaining DST info
-   separately.  For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo`
+   separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo`
    attribute's :meth:`dst` method to determine how the :attr:`tm_isdst` flag
    should be set, and :meth:`tzinfo.fromutc` calls :meth:`dst` to account for
    DST changes when crossing time zones.
@@ -1770,9 +1978,9 @@ Example:
    must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo ==
    tz``  For sane :class:`tzinfo` subclasses, this expression yields the time
    zone's "standard offset", which should not depend on the date or the time, but
-   only on geographic location.  The implementation of :meth:`datetime.astimezone`
+   only on geographic location. The implementation of :meth:`datetime.astimezone`
    relies on this, but cannot detect violations; it's the programmer's
-   responsibility to ensure it.  If a :class:`tzinfo` subclass cannot guarantee
+   responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee
    this, it may be able to override the default implementation of
    :meth:`tzinfo.fromutc` to work correctly with :meth:`astimezone` regardless.
 
@@ -1782,12 +1990,12 @@ Example:
           # a fixed-offset class:  doesn't account for DST
           return timedelta(0)
 
-   or ::
+   or::
 
       def dst(self, dt):
           # Code to set dston and dstoff to the time zone's DST
           # transition times based on the input dt.year, and expressed
-          # in standard local time.  Then
+          # in standard local time.
 
           if dston <= dt.replace(tzinfo=None) < dstoff:
               return timedelta(hours=1)
@@ -1804,9 +2012,9 @@ Example:
 
    Return the time zone name corresponding to the :class:`.datetime` object *dt*, as
    a string. Nothing about string names is defined by the :mod:`datetime` module,
-   and there's no requirement that it mean anything in particular.  For example,
+   and there's no requirement that it mean anything in particular. For example,
    "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all
-   valid replies.  Return ``None`` if a string name isn't known.  Note that this is
+   valid replies. Return ``None`` if a string name isn't known. Note that this is
    a method rather than a fixed string primarily because some :class:`tzinfo`
    subclasses will wish to return different names depending on the specific value
    of *dt* passed, especially if the :class:`tzinfo` class is accounting for
@@ -1816,20 +2024,20 @@ Example:
 
 
 These methods are called by a :class:`.datetime` or :class:`.time` object, in
-response to their methods of the same names.  A :class:`.datetime` object passes
+response to their methods of the same names. A :class:`.datetime` object passes
 itself as the argument, and a :class:`.time` object passes ``None`` as the
-argument.  A :class:`tzinfo` subclass's methods should therefore be prepared to
+argument. A :class:`tzinfo` subclass's methods should therefore be prepared to
 accept a *dt* argument of ``None``, or of class :class:`.datetime`.
 
 When ``None`` is passed, it's up to the class designer to decide the best
-response.  For example, returning ``None`` is appropriate if the class wishes to
-say that time objects don't participate in the :class:`tzinfo` protocols.  It
+response. For example, returning ``None`` is appropriate if the class wishes to
+say that time objects don't participate in the :class:`tzinfo` protocols. It
 may be more useful for ``utcoffset(None)`` to return the standard UTC offset, as
 there is no other convention for discovering the standard offset.
 
 When a :class:`.datetime` object is passed in response to a :class:`.datetime`
-method, ``dt.tzinfo`` is the same object as *self*.  :class:`tzinfo` methods can
-rely on this, unless user code calls :class:`tzinfo` methods directly.  The
+method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can
+rely on this, unless user code calls :class:`tzinfo` methods directly. The
 intent is that the :class:`tzinfo` methods interpret *dt* as being in local
 time, and not need worry about objects in other timezones.
 
@@ -1839,16 +2047,16 @@ There is one more :class:`tzinfo` method that a subclass may wish to override:
 .. method:: tzinfo.fromutc(dt)
 
    This is called from the default :class:`datetime.astimezone()`
-   implementation.  When called from that, ``dt.tzinfo`` is *self*, and *dt*'s
-   date and time data are to be viewed as expressing a UTC time.  The purpose
+   implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s
+   date and time data are to be viewed as expressing a UTC time. The purpose
    of :meth:`fromutc` is to adjust the date and time data, returning an
    equivalent datetime in *self*'s local time.
 
    Most :class:`tzinfo` subclasses should be able to inherit the default
-   :meth:`fromutc` implementation without problems.  It's strong enough to handle
+   :meth:`fromutc` implementation without problems. It's strong enough to handle
    fixed-offset time zones, and time zones accounting for both standard and
    daylight time, and the latter even if the DST transition times differ in
-   different years.  An example of a time zone the default :meth:`fromutc`
+   different years. An example of a time zone the default :meth:`fromutc`
    implementation may not handle correctly in all cases is one where the standard
    offset (from UTC) depends on the specific date and time passed, which can happen
    for political reasons. The default implementations of :meth:`astimezone` and
@@ -1881,7 +2089,7 @@ In the following :download:`tzinfo_examples.py
 
 Note that there are unavoidable subtleties twice per year in a :class:`tzinfo`
 subclass accounting for both standard and daylight time, at the DST transition
-points.  For concreteness, consider US Eastern (UTC -0500), where EDT begins the
+points. For concreteness, consider US Eastern (UTC -0500), where EDT begins the
 minute after 1:59 (EST) on the second Sunday in March, and ends the minute after
 1:59 (EDT) on the first Sunday in November::
 
@@ -1894,9 +2102,9 @@ minute after 1:59 (EST) on the second Sunday in March, and ends the minute after
      end  23:MM  0:MM  1:MM  1:MM  2:MM  3:MM
 
 When DST starts (the "start" line), the local wall clock leaps from 1:59 to
-3:00.  A wall time of the form 2:MM doesn't really make sense on that day, so
+3:00. A wall time of the form 2:MM doesn't really make sense on that day, so
 ``astimezone(Eastern)`` won't deliver a result with ``hour == 2`` on the day DST
-begins.  For example, at the Spring forward transition of 2016, we get
+begins. For example, at the Spring forward transition of 2016, we get::
 
     >>> from datetime import datetime, timezone
     >>> from tzinfo_examples import HOUR, Eastern
@@ -1914,14 +2122,14 @@ begins.  For example, at the Spring forward transition of 2016, we get
 
 When DST ends (the "end" line), there's a potentially worse problem: there's an
 hour that can't be spelled unambiguously in local wall time: the last hour of
-daylight time.  In Eastern, that's times of the form 5:MM UTC on the day
-daylight time ends.  The local wall clock leaps from 1:59 (daylight time) back
+daylight time. In Eastern, that's times of the form 5:MM UTC on the day
+daylight time ends. The local wall clock leaps from 1:59 (daylight time) back
 to 1:00 (standard time) again. Local times of the form 1:MM are ambiguous.
 :meth:`astimezone` mimics the local clock's behavior by mapping two adjacent UTC
-hours into the same local hour then.  In the Eastern example, UTC times of the
+hours into the same local hour then. In the Eastern example, UTC times of the
 form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times
 have the :attr:`~datetime.fold` attribute set to 0 and the later times have it set to 1.
-For example, at the Fall back transition of 2016, we get
+For example, at the Fall back transition of 2016, we get::
 
     >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc)
     >>> for i in range(4):
@@ -1934,7 +2142,7 @@ For example, at the Fall back transition of 2016, we get
     06:00:00 UTC = 01:00:00 EST 1
     07:00:00 UTC = 02:00:00 EST 0
 
-Note that the :class:`datetime` instances that differ only by the value of the
+Note that the :class:`.datetime` instances that differ only by the value of the
 :attr:`~datetime.fold` attribute are considered equal in comparisons.
 
 Applications that can't bear wall-time ambiguities should explicitly check the
@@ -1946,15 +2154,17 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)).
 .. seealso::
 
    `dateutil.tz <https://dateutil.readthedocs.io/en/stable/tz.html>`_
-      The standard library has :class:`timezone` class for handling arbitrary
-      fixed offsets from UTC and :attr:`timezone.utc` as UTC timezone instance.
+      The :mod:`datetime` module has a basic :class:`timezone` class (for
+      handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc`
+      attribute (a UTC timezone instance).
 
-      *dateutil.tz* library brings the *IANA timezone database* (also known as the
-      Olson database) to Python and its usage is recommended.
+      *dateutil.tz* library brings the *IANA timezone database*
+      (also known as the Olson database) to Python, and its usage is
+      recommended.
 
    `IANA timezone database <https://www.iana.org/time-zones>`_
-      The Time Zone Database (often called tz, tzdata or zoneinfo) contains code and
-      data that represent the history of local time for many representative
+      The Time Zone Database (often called tz, tzdata or zoneinfo) contains code
+      and data that represent the history of local time for many representative
       locations around the globe. It is updated periodically to reflect changes
       made by political bodies to time zone boundaries, UTC offsets, and
       daylight-saving rules.
@@ -1967,20 +2177,21 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)).
 
 The :class:`timezone` class is a subclass of :class:`tzinfo`, each
 instance of which represents a timezone defined by a fixed offset from
-UTC.  Note that objects of this class cannot be used to represent
-timezone information in the locations where different offsets are used
-in different days of the year or where historical changes have been
-made to civil time.
+UTC.
+
+Objects of this class cannot be used to represent timezone information in the
+locations where different offsets are used in different days of the year or
+where historical changes have been made to civil time.
 
 
 .. class:: timezone(offset, name=None)
 
   The *offset* argument must be specified as a :class:`timedelta`
-  object representing the difference between the local time and UTC.  It must
+  object representing the difference between the local time and UTC. It must
   be strictly between ``-timedelta(hours=24)`` and
   ``timedelta(hours=24)``, otherwise :exc:`ValueError` is raised.
 
-  The *name* argument is optional.  If specified it must be a string that
+  The *name* argument is optional. If specified it must be a string that
   will be used as the value returned by the :meth:`datetime.tzname` method.
 
   .. versionadded:: 3.2
@@ -1992,9 +2203,10 @@ made to civil time.
 .. method:: timezone.utcoffset(dt)
 
   Return the fixed value specified when the :class:`timezone` instance is
-  constructed.  The *dt* argument is ignored.  The return value is a
-  :class:`timedelta` instance equal to the difference between the
-  local time and UTC.
+  constructed.
+
+  The *dt* argument is ignored. The return value is a :class:`timedelta`
+  instance equal to the difference between the local time and UTC.
 
   .. versionchanged:: 3.7
      The UTC offset is not restricted to a whole number of minutes.
@@ -2002,16 +2214,17 @@ made to civil time.
 .. method:: timezone.tzname(dt)
 
   Return the fixed value specified when the :class:`timezone` instance
-  is constructed.  If *name* is not provided in the constructor, the
-  name returned by ``tzname(dt)`` is generated from the value of the
-  ``offset`` as follows.  If *offset* is ``timedelta(0)``, the name
-  is "UTC", otherwise it is a string 'UTC?HH:MM', where ? is the sign
-  of ``offset``, HH and MM are two digits of ``offset.hours`` and
-  ``offset.minutes`` respectively.
+  is constructed.
+
+  If *name* is not provided in the constructor, the name returned by
+  ``tzname(dt)`` is generated from the value of the ``offset`` as follows. If
+  *offset* is ``timedelta(0)``, the name is "UTC", otherwise it is a string in
+  the format ``UTC?HH:MM``, where ? is the sign of ``offset``, HH and MM are
+  two digits of ``offset.hours`` and ``offset.minutes`` respectively.
 
   .. versionchanged:: 3.6
-     Name generated from ``offset=timedelta(0)`` is now plain 'UTC', not
-     'UTC+00:00'.
+     Name generated from ``offset=timedelta(0)`` is now plain `'UTC'`, not
+     ``'UTC+00:00'``.
 
 
 .. method:: timezone.dst(dt)
@@ -2020,7 +2233,7 @@ made to civil time.
 
 .. method:: timezone.fromutc(dt)
 
-  Return ``dt + offset``.  The *dt* argument must be an aware
+  Return ``dt + offset``. The *dt* argument must be an aware
   :class:`.datetime` instance, with ``tzinfo`` set to ``self``.
 
 Class attributes:
@@ -2040,43 +2253,33 @@ Class attributes:
 
 :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a
 ``strftime(format)`` method, to create a string representing the time under the
-control of an explicit format string.  Broadly speaking, ``d.strftime(fmt)``
-acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())``
-although not all objects support a :meth:`timetuple` method.
+control of an explicit format string.
 
 Conversely, the :meth:`datetime.strptime` class method creates a
 :class:`.datetime` object from a string representing a date and time and a
-corresponding format string. ``datetime.strptime(date_string, format)`` is
-equivalent to ``datetime(*(time.strptime(date_string, format)[0:6]))``, except
-when the format includes sub-second components or timezone offset information,
-which are supported in ``datetime.strptime`` but are discarded by ``time.strptime``.
-
-For :class:`.time` objects, the format codes for year, month, and day should not
-be used, as time objects have no such values.  If they're used anyway, ``1900``
-is substituted for the year, and ``1`` for the month and day.
+corresponding format string.
 
-For :class:`date` objects, the format codes for hours, minutes, seconds, and
-microseconds should not be used, as :class:`date` objects have no such
-values.  If they're used anyway, ``0`` is substituted for them.
+The table below provides a high-level comparison of :meth:`strftime`
+versus :meth:`strptime`:
 
-For the :meth:`datetime.strptime` class method, the default value is ``1900-01-01T00:00:00.000``:
-any components not specified in the format string will be pulled from the default value. [#]_
++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+
+|                | ``strftime``                                           | ``strptime``                                                                 |
++================+========================================================+==============================================================================+
+| Usage          | Convert object to a string according to a given format | Parse a string into a :class:`.datetime` object given a corresponding format |
++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+
+| Type of method | Instance method                                        | Class method                                                                 |
++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+
+| Method of      | :class:`date`; :class:`.datetime`; :class:`.time`      | :class:`.datetime`                                                           |
++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+
+| Signature      | ``strftime(format)``                                   | ``strptime(date_string, format)``                                            |
++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+
 
-The full set of format codes supported varies across platforms, because Python
-calls the platform C library's :func:`strftime` function, and platform
-variations are common.  To see the full set of format codes supported on your
-platform, consult the :manpage:`strftime(3)` documentation.
 
-For the same reason, handling of format strings containing Unicode code points
-that can't be represented in the charset of the current locale is also
-platform-dependent. On some platforms such code points are preserved intact in
-the output, while on others ``strftime`` may raise :exc:`UnicodeError` or return
-an empty string instead.
+:meth:`strftime` and :meth:`strptime` Format Codes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The following is a list of all the format codes that the C standard (1989
-version) requires, and these work on all platforms with a standard C
-implementation.  Note that the 1999 version of the C standard added additional
-format codes.
+The following is a list of all the format codes that the 1989 C standard
+requires, and these work on all platforms with a standard C implementation.
 
 +-----------+--------------------------------+------------------------+-------+
 | Directive | Meaning                        | Example                | Notes |
@@ -2137,7 +2340,7 @@ format codes.
 |           | left.                          |                        |       |
 +-----------+--------------------------------+------------------------+-------+
 | ``%z``    | UTC offset in the form         | (empty), +0000,        | \(6)  |
-|           | ?HHMM[SS[.ffffff]] (empty      | -0400, +1030,          |       |
+|           | ``?HHMM[SS[.ffffff]]`` (empty  | -0400, +1030,          |       |
 |           | string if the object is        | +063415,               |       |
 |           | naive).                        | -030712.345216         |       |
 +-----------+--------------------------------+------------------------+-------+
@@ -2179,11 +2382,7 @@ format codes.
 +-----------+--------------------------------+------------------------+-------+
 
 Several additional directives not required by the C89 standard are included for
-convenience. These parameters all correspond to ISO 8601 date values. These
-may not be available on all platforms when used with the :meth:`strftime`
-method. The ISO 8601 year and ISO 8601 week directives are not interchangeable
-with the year and week number directives above. Calling :meth:`strptime` with
-incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`.
+convenience. These parameters all correspond to ISO 8601 date values.
 
 +-----------+--------------------------------+------------------------+-------+
 | Directive | Meaning                        | Example                | Notes |
@@ -2203,9 +2402,52 @@ incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`.
 |           | Jan 4.                         |                        |       |
 +-----------+--------------------------------+------------------------+-------+
 
+These may not be available on all platforms when used with the :meth:`strftime`
+method. The ISO 8601 year and ISO 8601 week directives are not interchangeable
+with the year and week number directives above. Calling :meth:`strptime` with
+incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`.
+
+The full set of format codes supported varies across platforms, because Python
+calls the platform C library's :func:`strftime` function, and platform
+variations are common. To see the full set of format codes supported on your
+platform, consult the :manpage:`strftime(3)` documentation.
+
 .. versionadded:: 3.6
    ``%G``, ``%u`` and ``%V`` were added.
 
+Technical Detail
+^^^^^^^^^^^^^^^^
+
+Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's
+``time.strftime(fmt, d.timetuple())`` although not all objects support a
+:meth:`timetuple` method.
+
+For the :meth:`datetime.strptime` class method, the default value is
+``1900-01-01T00:00:00.000``: any components not specified in the format string
+will be pulled from the default value. [#]_
+
+Using ``datetime.strptime(date_string, format)`` is equivalent to::
+
+  datetime(*(time.strptime(date_string, format)[0:6]))
+
+except when the format includes sub-second components or timezone offset
+information, which are supported in ``datetime.strptime`` but are discarded by
+``time.strptime``.
+
+For :class:`.time` objects, the format codes for year, month, and day should not
+be used, as :class:`time` objects have no such values. If they're used anyway,
+``1900`` is substituted for the year, and ``1`` for the month and day.
+
+For :class:`date` objects, the format codes for hours, minutes, seconds, and
+microseconds should not be used, as :class:`date` objects have no such
+values. If they're used anyway, ``0`` is substituted for them.
+
+For the same reason, handling of format strings containing Unicode code points
+that can't be represented in the charset of the current locale is also
+platform-dependent. On some platforms such code points are preserved intact in
+the output, while on others ``strftime`` may raise :exc:`UnicodeError` or return
+an empty string instead.
+
 Notes:
 
 (1)
@@ -2239,7 +2481,7 @@ Notes:
 
 (5)
    When used with the :meth:`strptime` method, the ``%f`` directive
-   accepts from one to six digits and zero pads on the right.  ``%f`` is
+   accepts from one to six digits and zero pads on the right. ``%f`` is
    an extension to the set of format characters in the C standard (but
    implemented separately in datetime objects, and therefore always
    available).
@@ -2252,13 +2494,13 @@ Notes:
 
    ``%z``
       :meth:`utcoffset` is transformed into a string of the form
-      ?HHMM[SS[.ffffff]], where HH is a 2-digit string giving the number of UTC
-      offset hours, MM is a 2-digit string giving the number of UTC offset
-      minutes, SS is a 2-digit string giving the number of UTC offset
-      seconds and ffffff is a 6-digit string giving the number of UTC
-      offset microseconds.  The ffffff part is omitted when the offset is a
-      whole number of seconds and both the ffffff and the SS part is omitted
-      when the offset is a whole number of minutes.  For example, if
+      ``?HHMM[SS[.ffffff]]``, where ``HH`` is a 2-digit string giving the number
+      of UTC offset hours, ``MM`` is a 2-digit string giving the number of UTC
+      offset minutes, SS is a 2-digit string giving the number of UTC offset
+      seconds and ``ffffff`` is a 6-digit string giving the number of UTC
+      offset microseconds. The ``ffffff`` part is omitted when the offset is a
+      whole number of seconds and both the ``ffffff`` and the ``SS`` part is
+      omitted when the offset is a whole number of minutes. For example, if
       :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is
       replaced with the string ``'-0330'``.
 
@@ -2274,12 +2516,12 @@ Notes:
 
    ``%Z``
       If :meth:`tzname` returns ``None``, ``%Z`` is replaced by an empty
-      string.  Otherwise ``%Z`` is replaced by the returned value, which must
+      string. Otherwise ``%Z`` is replaced by the returned value, which must
       be a string.
 
    .. versionchanged:: 3.2
       When the ``%z`` directive is provided to the :meth:`strptime` method, an
-      aware :class:`.datetime` object will be produced.  The ``tzinfo`` of the
+      aware :class:`.datetime` object will be produced. The ``tzinfo`` of the
       result will be set to a :class:`timezone` instance.
 
 (7)
@@ -2301,4 +2543,15 @@ Notes:
 .. rubric:: Footnotes
 
 .. [#] If, that is, we ignore the effects of Relativity
+
+.. [#] This matches the definition of the "proleptic Gregorian" calendar in
+       Dershowitz and Reingold's book *Calendrical Calculations*,
+       where it's the base calendar for all computations. See the book for
+       algorithms for converting between proleptic Gregorian ordinals and
+       many other calendar systems.
+
+.. [#] See R. H. van Gent's `guide to the mathematics of the ISO 8601 calendar
+       <https://www.staff.science.uu.nl/~gent0113/calendar/isocalendar.htm>`_
+       for a good explanation.
+
 .. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since ``1900`` is not a leap year.
diff --git a/Misc/NEWS.d/next/Documentation/2019-05-18-16-25-44.bpo-36960.xEKHXj.rst b/Misc/NEWS.d/next/Documentation/2019-05-18-16-25-44.bpo-36960.xEKHXj.rst
new file mode 100644
index 000000000000..1d4b213367e7
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-05-18-16-25-44.bpo-36960.xEKHXj.rst
@@ -0,0 +1,2 @@
+Restructured the :mod:`datetime` docs in the interest of making them more
+user-friendly and improving readability. Patch by Brad Solomon.


From webhook-mailer at python.org  Thu Sep 12 10:38:18 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 14:38:18 -0000
Subject: [Python-checkins] bpo-38138: Fix memory leak introduced by interned
 strings (GH-16053)
Message-ID: <mailman.670.1568299100.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8d88e8c662a80c186c7a0e67f1c1f3d4010276f8
commit: 8d88e8c662a80c186c7a0e67f1c1f3d4010276f8
branch: master
author: Dino Viehland <dinoviehland at fb.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-12T07:38:13-07:00
summary:

bpo-38138: Fix memory leak introduced by interned strings (GH-16053)



Interned string needs to be decref'd



https://bugs.python.org/issue38138



Automerge-Triggered-By: @matrixise

files:
M Python/ast.c

diff --git a/Python/ast.c b/Python/ast.c
index 46815c271b62..e6f71671c18e 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -630,6 +630,7 @@ new_identifier(const char *n, struct compiling *c)
         PyObject *args[2] = {form, id};
         id2 = _PyObject_FastCall(c->c_normalize, args, 2);
         Py_DECREF(id);
+        Py_DECREF(form);
         if (!id2)
             return NULL;
         if (!PyUnicode_Check(id2)) {


From webhook-mailer at python.org  Thu Sep 12 10:41:15 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 14:41:15 -0000
Subject: [Python-checkins] bpo-21872: fix lzma library decompresses data
 incompletely (GH-14048)
Message-ID: <mailman.671.1568299276.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/824407f76e211a2a19c94d5cb1f39fc422ab62ee
commit: 824407f76e211a2a19c94d5cb1f39fc422ab62ee
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T07:41:11-07:00
summary:

bpo-21872: fix lzma library decompresses data incompletely (GH-14048)


* 1. add test case with wrong behavior
* 2. fix bug when max_length == -1
* 3. allow b"" as valid input data for decompress_buf()
* 4. when max_length >= 0, let needs_input mechanism works
* add more asserts to test case
(cherry picked from commit 4ffd05d7ec47cfd0d7fc95dce851633be9663255)

Co-authored-by: animalize <animalize at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Library/2019-06-12-08-56-22.bpo-21872.V9QGGN.rst
M Lib/test/test_lzma.py
M Modules/_lzmamodule.c

diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py
index 117de0a1b554..f24ed3ca3d43 100644
--- a/Lib/test/test_lzma.py
+++ b/Lib/test/test_lzma.py
@@ -1197,6 +1197,36 @@ def test_tell_bad_args(self):
         f.close()
         self.assertRaises(ValueError, f.tell)
 
+    def test_issue21872(self):
+        # sometimes decompress data incompletely
+
+        # ---------------------
+        # when max_length == -1
+        # ---------------------
+        d1 = LZMADecompressor()
+        entire = d1.decompress(ISSUE_21872_DAT, max_length=-1)
+        self.assertEqual(len(entire), 13160)
+        self.assertTrue(d1.eof)
+
+        # ---------------------
+        # when max_length > 0
+        # ---------------------
+        d2 = LZMADecompressor()
+
+        # When this value of max_length is used, the input and output
+        # buffers are exhausted at the same time, and lzs's internal
+        # state still have 11 bytes can be output.
+        out1 = d2.decompress(ISSUE_21872_DAT, max_length=13149)
+        self.assertFalse(d2.needs_input) # ensure needs_input mechanism works
+        self.assertFalse(d2.eof)
+
+        # simulate needs_input mechanism
+        # output internal state's 11 bytes
+        out2 = d2.decompress(b'')
+        self.assertEqual(len(out2), 11)
+        self.assertTrue(d2.eof)
+        self.assertEqual(out1 + out2, entire)
+
 
 class OpenTestCase(unittest.TestCase):
 
@@ -1763,6 +1793,139 @@ def test_filter_properties_roundtrip(self):
     b"\x00"
 )
 
+ISSUE_21872_DAT = (
+    b']\x00\x00@\x00h3\x00\x00\x00\x00\x00\x00\x00\x00`D\x0c\x99\xc8'
+    b'\xd1\xbbZ^\xc43+\x83\xcd\xf1\xc6g\xec-\x061F\xb1\xbb\xc7\x17%-\xea'
+    b'\xfap\xfb\x8fs\x128\xb2,\x88\xe4\xc0\x12|*x\xd0\xa2\xc4b\x1b!\x02c'
+    b'\xab\xd9\x87U\xb8n \xfaVJ\x9a"\xb78\xff%_\x17`?@*\xc2\x82'
+    b"\xf2^\x1b\xb8\x04&\xc0\xbb\x03g\x9d\xca\xe9\xa4\xc9\xaf'\xe5\x8e}"
+    b'F\xdd\x11\xf3\x86\xbe\x1fN\x95\\\xef\xa2Mz-\xcb\x9a\xe3O@'
+    b"\x19\x07\xf6\xee\x9e\x9ag\xc6\xa5w\rnG'\x99\xfd\xfeGI\xb0"
+    b'\xbb\xf9\xc2\xe1\xff\xc5r\xcf\x85y[\x01\xa1\xbd\xcc/\xa3\x1b\x83\xaa'
+    b'\xc6\xf9\x99\x0c\xb6_\xc9MQ+x\xa2F\xda]\xdd\xe8\xfb\x1a&'
+    b',\xc4\x19\x1df\x81\x1e\x90\xf3\xb8Hgr\x85v\xbe\xa3qx\x01Y\xb5\x9fF'
+    b"\x13\x18\x01\xe69\x9b\xc8'\x1e\x9d\xd6\xe4F\x84\xac\xf8d<\x11\xd5"
+    b'\\\x0b\xeb\x0e\x82\xab\xb1\xe6\x1fka\xe1i\xc4 C\xb1"4)\xd6\xa7`\x02'
+    b'\xec\x11\x8c\xf0\x14\xb0\x1d\x1c\xecy\xf0\xb7|\x11j\x85X\xb2!\x1c'
+    b'\xac\xb5N\xc7\x85j\x9ev\xf5\xe6\x0b\xc1]c\xc15\x16\x9f\xd5\x99'
+    b"\xfei^\xd2G\x9b\xbdl\xab:\xbe,\xa9'4\x82\xe5\xee\xb3\xc1"
+    b'$\x93\x95\xa8Y\x16\xf5\xbf\xacw\x91\x04\x1d\x18\x06\xe9\xc5\xfdk\x06'
+    b'\xe8\xfck\xc5\x86>\x8b~\xa4\xcb\xf1\xb3\x04\xf1\x04G5\xe2\xcc]'
+    b'\x16\xbf\x140d\x18\xe2\xedw#(3\xca\xa1\x80bX\x7f\xb3\x84'
+    b'\x9d\xdb\xe7\x08\x97\xcd\x16\xb9\xf1\xd5r+m\x1e\xcb3q\xc5\x9e\x92'
+    b"\x7f\x8e*\xc7\xde\xe9\xe26\xcds\xb1\x10-\xf6r\x02?\x9d\xddCgJN'"
+    b'\x11M\xfa\nQ\n\xe6`m\xb8N\xbbq\x8el\x0b\x02\xc7:q\x04G\xa1T'
+    b'\xf1\xfe!0\x85~\xe5\x884\xe9\x89\xfb\x13J8\x15\xe42\xb6\xad'
+    b'\x877A\x9a\xa6\xbft]\xd0\xe35M\xb0\x0cK\xc8\xf6\x88\xae\xed\xa9,j7'
+    b'\x81\x13\xa0(\xcb\xe1\xe9l2\x7f\xcd\xda\x95(\xa70B\xbd\xf4\xe3'
+    b'hp\x94\xbdJ\xd7\t\xc7g\xffo?\x89?\xf8}\x7f\xbc\x1c\x87'
+    b'\x14\xc0\xcf\x8cV:\x9a\x0e\xd0\xb2\x1ck\xffk\xb9\xe0=\xc7\x8d/'
+    b'\xb8\xff\x7f\x1d\x87`\x19.\x98X*~\xa7j\xb9\x0b"\xf4\xe4;V`\xb9\xd7'
+    b'\x03\x1e\xd0t0\xd3\xde\x1fd\xb9\xe2)\x16\x81}\xb1\\b\x7fJ'
+    b'\x92\xf4\xff\n+V!\xe9\xde\x98\xa0\x8fK\xdf7\xb9\xc0\x12\x1f\xe2'
+    b'\xe9\xb0`\xae\x14\r\xa7\xc4\x81~\xd8\x8d\xc5\x06\xd8m\xb0Y\x8a)'
+    b'\x06/\xbb\xf9\xac\xeaP\xe0\x91\x05m[\xe5z\xe6Z\xf3\x9f\xc7\xd0'
+    b'\xd3\x8b\xf3\x8a\x1b\xfa\xe4Pf\xbc0\x17\x10\xa9\xd0\x95J{\xb3\xc3'
+    b'\xfdW\x9bop\x0f\xbe\xaee\xa3]\x93\x9c\xda\xb75<\xf6g!\xcc\xb1\xfc\\'
+    b'7\x152Mc\x17\x84\x9d\xcd35\r0\xacL-\xf3\xfb\xcb\x96\x1e\xe9U\x7f'
+    b'\xd7\xca\xb0\xcc\x89\x0c*\xce\x14\xd1P\xf1\x03\xb6.~9o?\xe8'
+    b'\r\x86\xe0\x92\x87}\xa3\x84\x03P\xe0\xc2\x7f\n;m\x9d\x9e\xb4|'
+    b'\x8c\x18\xc0#0\xfe3\x07<\xda\xd8\xcf^\xd4Hi\xd6\xb3\x0bT'
+    b'\x1dF\x88\x85q}\x02\xc6&\xc4\xae\xce\x9cU\xfa\x0f\xcc\xb6\x1f\x11'
+    b'drw\x9eN\x19\xbd\xffz\x0f\xf0\x04s\xadR\xc1\xc0\xbfl\xf1\xba\xf95^'
+    b'e\xb1\xfbVY\xd9\x9f\x1c\xbf*\xc4\xa86\x08+\xd6\x88[\xc4_rc\xf0f'
+    b'\xb8\xd4\xec\x1dx\x19|\xbf\xa7\xe0\x82\x0b\x8c~\x10L/\x90\xd6\xfb'
+    b'\x81\xdb\x98\xcc\x02\x14\xa5C\xb2\xa7i\xfd\xcd\x1fO\xf7\xe9\x89t\xf0'
+    b'\x17\xa5\x1c\xad\xfe<Q`%\x075k\n7\x9eI\x82<#)&\x04\xc2\xf0C\xd4`!'
+    b'\xcb\xa9\xf9\xb3F\x86\xb5\xc3M\xbeu\x12\xb2\xca\x95e\x10\x0b\xb1\xcc'
+    b'\x01b\x9bXa\x1b[B\x8c\x07\x11Of;\xeaC\xebr\x8eb\xd9\x9c\xe4i]<z\x9a'
+    b'\x03T\x8b9pF\x10\x8c\x84\xc7\x0e\xeaPw\xe5\xa0\x94\x1f\x84\xdd'
+    b'a\xe8\x85\xc2\x00\xebq\xe7&Wo5q8\xc2t\x98\xab\xb7\x7f\xe64-H'
+    b'\t\xb4d\xbe\x06\xe3Q\x8b\xa9J\xb0\x00\xd7s.\x85"\xc0p\x05'
+    b'\x1c\x06N\x87\xa5\xf8\xc3g\x1b}\x0f\x0f\xc3|\x90\xea\xefd3X'
+    b'[\xab\x04E\xf2\xf2\xc9\x08\x8a\xa8+W\xa2v\xec\x15G\x08/I<L\\1'
+    b'\xff\x15O\xaa\x89{\xd1mW\x13\xbd~\xe1\x90^\xc4@\r\xed\xb5D@\xb4\x08'
+    b'A\x90\xe69;\xc7BO\xdb\xda\xebu\x9e\xa9tN\xae\x8aJ5\xcd\x11\x1d\xea'
+    b"\xe5\xa7\x04\xe6\x82Z\xc7O\xe46[7\xdco*[\xbe\x0b\xc9\xb7a\xab'\xf6"
+    b"\xd1u\xdb\xd9q\xf5+y\x1b\x00\xb4\xf3a\xae\xf1M\xc4\xbc\xd00'\x06pQ"
+    b'\x8dH\xaa\xaa\xc4\xd2K\x9b\xc0\xe9\xec=n\xa9\x1a\x8a\xc2\xe8\x18\xbc'
+    b'\x93\xb8F\xa1\x8fOY\xe7\xda\xcf0\t\xff|\xd9\xe5\xcf\xe7\xf6\xbe'
+    b'\xf8\x04\x17\xf2\xe5P\xa7y~\xce\x11h0\x81\x80d[\x00_v\xbbc\xdbI'
+    b'3\xbc`W\xc0yrkB\xf5\x9f\xe9i\xc5\x8a^\x8d\xd4\x81\xd9\x05\xc1\xfc>'
+    b'"\xd1v`\x82\xd5$\x89\xcf^\xd52.\xafd\xe8d@\xaa\xd5Y|\x90\x84'
+    b'j\xdb}\x84riV\x8e\xf0X4rB\xf2NPS[\x8e\x88\xd4\x0fI\xb8'
+    b'\xdd\xcb\x1d\xf2(\xdf;9\x9e|\xef^0;.*[\x9fl\x7f\xa2_X\xaff!\xbb\x03'
+    b'\xff\x19\x8f\x88\xb5\xb6\x884\xa3\x05\xde3D{\xe3\xcb\xce\xe4t]'
+    b'\x875\xe3Uf\xae\xea\x88\x1c\x03b\n\xb1,Q\xec\xcf\x08\t\xde@\x83\xaa<'
+    b',-\xe4\xee\x9b\x843\xe5\x007\tK\xac\x057\xd6*X\xa3\xc6~\xba\xe6O'
+    b'\x81kz"\xbe\xe43sL\xf1\xfa;\xf4^\x1e\xb4\x80\xe2\xbd\xaa\x17Z\xe1f'
+    b'\xda\xa6\xb9\x07:]}\x9fa\x0b?\xba\xe7\xf15\x04M\xe3\n}M\xa4\xcb\r'
+    b'2\x8a\x88\xa9\xa7\x92\x93\x84\x81Yo\x00\xcc\xc4\xab\x9aT\x96\x0b\xbe'
+    b'U\xac\x1d\x8d\x1b\x98"\xf8\x8f\xf1u\xc1n\xcc\xfcA\xcc\x90\xb7i'
+    b'\x83\x9c\x9c~\x1d4\xa2\xf0*J\xe7t\x12\xb4\xe3\xa0u\xd7\x95Z'
+    b'\xf7\xafG\x96~ST,\xa7\rC\x06\xf4\xf0\xeb`2\x9e>Q\x0e\xf6\xf5\xc5'
+    b'\x9b\xb5\xaf\xbe\xa3\x8f\xc0\xa3hu\x14\x12 \x97\x99\x04b\x8e\xc7\x1b'
+    b'VKc\xc1\xf3 \xde\x85-:\xdc\x1f\xac\xce*\x06\xb3\x80;`'
+    b'\xdb\xdd\x97\xfdg\xbf\xe7\xa8S\x08}\xf55e7\xb8/\xf0!\xc8'
+    b"Y\xa8\x9a\x07'\xe2\xde\r\x02\xe1\xb2\x0c\xf4C\xcd\xf9\xcb(\xe8\x90"
+    b'\xd3bTD\x15_\xf6\xc3\xfb\xb3E\xfc\xd6\x98{\xc6\\fz\x81\xa99\x85\xcb'
+    b'\xa5\xb1\x1d\x94bqW\x1a!;z~\x18\x88\xe8i\xdb\x1b\x8d\x8d'
+    b'\x06\xaa\x0e\x99s+5k\x00\xe4\xffh\xfe\xdbt\xa6\x1bU\xde\xa3'
+    b'\xef\xcb\x86\x9e\x81\x16j\n\x9d\xbc\xbbC\x80?\x010\xc7Jj;'
+    b'\xc4\xe5\x86\xd5\x0e0d#\xc6;\xb8\xd1\xc7c\xb5&8?\xd9J\xe5\xden\xb9'
+    b'\xe9cb4\xbb\xe6\x14\xe0\xe7l\x1b\x85\x94\x1fh\xf1n\xdeZ\xbe'
+    b'\x88\xff\xc2e\xca\xdc,B-\x8ac\xc9\xdf\xf5|&\xe4LL\xf0\x1f\xaa8\xbd'
+    b'\xc26\x94bVi\xd3\x0c\x1c\xb6\xbb\x99F\x8f\x0e\xcc\x8e4\xc6/^W\xf5?'
+    b'\xdc\x84(\x14dO\x9aD6\x0f4\xa3,\x0c\x0bS\x9fJ\xe1\xacc^\x8a0\t\x80D['
+    b'\xb8\xe6\x86\xb0\xe8\xd4\xf9\x1en\xf1\xf5^\xeb\xb8\xb8\xf8'
+    b')\xa8\xbf\xaa\x84\x86\xb1a \x95\x16\x08\x1c\xbb@\xbd+\r/\xfb'
+    b'\x92\xfbh\xf1\x8d3\xf9\x92\xde`\xf1\x86\x03\xaa+\xd9\xd9\xc6P\xaf'
+    b'\xe3-\xea\xa5\x0fB\xca\xde\xd5n^\xe3/\xbf\xa6w\xc8\x0e<M'
+    b'\xc2\x1e!\xd4\xc6E\xf2\xad\x0c\xbc\x1d\x88Y\x03\x98<\x92\xd9\xa6B'
+    b'\xc7\x83\xb5"\x97D|&\xc4\xd4\xfad\x0e\xde\x06\xa3\xc2\x9c`\xf2'
+    b'7\x03\x1a\xed\xd80\x10\xe9\x0co\x10\xcf\x18\x16\xa7\x1c'
+    b"\xe5\x96\xa4\xd9\xe1\xa5v;]\xb7\xa9\xdc'hA\xe3\x9c&\x98\x0b9\xdf~@"
+    b'\xf8\xact\x87<\xf94\x0c\x9d\x93\xb0)\xe1\xa2\x0f\x1e=:&\xd56\xa5A+'
+    b'\xab\xc4\x00\x8d\x81\x93\xd4\xd8<\x82k\\d\xd8v\xab\xbd^l5C?\xd4\xa0'
+    b'M\x12C\xc8\x80\r\xc83\xe8\xc0\xf5\xdf\xca\x05\xf4BPjy\xbe\x91\x9bzE'
+    b"\xd8[\x93oT\r\x13\x16'\x1a\xbd*H\xd6\xfe\r\xf3\x91M\x8b\xee\x8f7f"
+    b"\x0b;\xaa\x85\xf2\xdd'\x0fwM \xbd\x13\xb9\xe5\xb8\xb7 D+P\x1c\xe4g"
+    b'n\xd2\xf1kc\x15\xaf\xc6\x90V\x03\xc2UovfZ\xcc\xd23^\xb3\xe7\xbf'
+    b'\xacv\x1d\x82\xedx\xa3J\xa9\xb7\xcf\x0c\xe6j\x96n*o\x18>'
+    b'\xc6\xfd\x97_+D{\x03\x15\xe8s\xb1\xc8HAG\xcf\xf4\x1a\xdd'
+    b'\xad\x11\xbf\x157q+\xdeW\x89g"X\x82\xfd~\xf7\xab4\xf6`\xab\xf1q'
+    b')\x82\x10K\xe9sV\xfe\xe45\xafs*\x14\xa7;\xac{\x06\x9d<@\x93G'
+    b'j\x1d\xefL\xe9\xd8\x92\x19&\xa1\x16\x19\x04\tu5\x01]\xf6\xf4'
+    b'\xcd\\\xd8A|I\xd4\xeb\x05\x88C\xc6e\xacQ\xe9*\x97~\x9au\xf8Xy'
+    b"\x17P\x10\x9f\n\x8c\xe2fZEu>\x9b\x1e\x91\x0b'`\xbd\xc0\xa8\x86c\x1d"
+    b'Z\xe2\xdc8j\x95\xffU\x90\x1e\xf4o\xbc\xe5\xe3e>\xd2R\xc0b#\xbc\x15'
+    b'H-\xb9!\xde\x9d\x90k\xdew\x9b{\x99\xde\xf7/K)A\xfd\xf5\xe6:\xda'
+    b'UM\xcc\xbb\xa2\x0b\x9a\x93\xf5{9\xc0 \xd2((6i\xc0\xbbu\xd8\x9e\x8d'
+    b'\xf8\x04q\x10\xd4\x14\x9e7-\xb9B\xea\x01Q8\xc8v\x9a\x12A\x88Cd\x92'
+    b"\x1c\x8c!\xf4\x94\x87'\xe3\xcd\xae\xf7\xd8\x93\xfa\xde\xa8b\x9e\xee2"
+    b'K\xdb\x00l\x9d\t\xb1|D\x05U\xbb\xf4>\xf1w\x887\xd1}W\x9d|g|1\xb0\x13'
+    b"\xa3 \xe5\xbfm@\xc06+\xb7\t\xcf\x15D\x9a \x1fM\x1f\xd2\xb5'\xa9\xbb"
+    b'~Co\x82\xfa\xc2\t\xe6f\xfc\xbeI\xae1\x8e\xbe\xb8\xcf\x86\x17'
+    b'\x9f\xe2`\xbd\xaf\xba\xb9\xbc\x1b\xa3\xcd\x82\x8fwc\xefd\xa9\xd5\x14'
+    b'\xe2C\xafUE\xb6\x11MJH\xd0=\x05\xd4*I\xff"\r\x1b^\xcaS6=\xec@\xd5'
+    b'\x11,\xe0\x87Gr\xaa[\xb8\xbc>n\xbd\x81\x0c\x07<\xe9\x92('
+    b'\xb2\xff\xac}\xe7\xb6\x15\x90\x9f~4\x9a\xe6\xd6\xd8s\xed\x99tf'
+    b'\xa0f\xf8\xf1\x87\t\x96/)\x85\xb6\n\xd7\xb2w\x0b\xbc\xba\x99\xee'
+    b'Q\xeen\x1d\xad\x03\xc3s\xd1\xfd\xa2\xc6\xb7\x9a\x9c(G<6\xad[~H '
+    b'\x16\x89\x89\xd0\xc3\xd2\xca~\xac\xea\xa5\xed\xe5\xfb\r:'
+    b'\x8e\xa6\xf1e\xbb\xba\xbd\xe0(\xa3\x89_\x01(\xb5c\xcc\x9f\x1fg'
+    b'v\xfd\x17\xb3\x08S=S\xee\xfc\x85>\x91\x8d\x8d\nYR\xb3G\xd1A\xa2\xb1'
+    b'\xec\xb0\x01\xd2\xcd\xf9\xfe\x82\x06O\xb3\xecd\xa9c\xe0\x8eP\x90\xce'
+    b'\xe0\xcd\xd8\xd8\xdc\x9f\xaa\x01"[Q~\xe4\x88\xa1#\xc1\x12C\xcf'
+    b'\xbe\x80\x11H\xbf\x86\xd8\xbem\xcfWFQ(X\x01DK\xdfB\xaa\x10.-'
+    b'\xd5\x9e|\x86\x15\x86N]\xc7Z\x17\xcd=\xd7)M\xde\x15\xa4LTi\xa0\x15'
+    b'\xd1\xe7\xbdN\xa4?\xd1\xe7\x02\xfe4\xe4O\x89\x98&\x96\x0f\x02\x9c'
+    b'\x9e\x19\xaa\x13u7\xbd0\xdc\xd8\x93\xf4BNE\x1d\x93\x82\x81\x16'
+    b'\xe5y\xcf\x98D\xca\x9a\xe2\xfd\xcdL\xcc\xd1\xfc_\x0b\x1c\xa0]\xdc'
+    b'\xa91 \xc9c\xd8\xbf\x97\xcfp\xe6\x19-\xad\xff\xcc\xd1N(\xe8'
+    b'\xeb#\x182\x96I\xf7l\xf3r\x00'
+)
+
 
 def test_main():
     run_unittest(
diff --git a/Misc/NEWS.d/next/Library/2019-06-12-08-56-22.bpo-21872.V9QGGN.rst b/Misc/NEWS.d/next/Library/2019-06-12-08-56-22.bpo-21872.V9QGGN.rst
new file mode 100644
index 000000000000..90df756f9a81
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-12-08-56-22.bpo-21872.V9QGGN.rst
@@ -0,0 +1,3 @@
+Fix :mod:`lzma`: module decompresses data incompletely. When decompressing a
+FORMAT_ALONE format file, and it doesn't have the end marker, sometimes the
+last one to dozens bytes can't be output. Patch by Ma Lin.
diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c
index 9e68cbb78af5..1ab67f306b73 100644
--- a/Modules/_lzmamodule.c
+++ b/Modules/_lzmamodule.c
@@ -872,9 +872,6 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length)
     PyObject *result;
     lzma_stream *lzs = &d->lzs;
 
-    if (lzs->avail_in == 0)
-        return PyBytes_FromStringAndSize(NULL, 0);
-
     if (max_length < 0 || max_length >= INITIAL_BUFFER_SIZE)
         result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE);
     else
@@ -891,7 +888,10 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length)
         Py_BEGIN_ALLOW_THREADS
         lzret = lzma_code(lzs, LZMA_RUN);
         data_size = (char *)lzs->next_out - PyBytes_AS_STRING(result);
+        if (lzret == LZMA_BUF_ERROR && lzs->avail_in == 0 && lzs->avail_out > 0)
+            lzret = LZMA_OK; /* That wasn't a real error */
         Py_END_ALLOW_THREADS
+
         if (catch_lzma_error(lzret))
             goto error;
         if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK)
@@ -899,15 +899,19 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length)
         if (lzret == LZMA_STREAM_END) {
             d->eof = 1;
             break;
-        } else if (lzs->avail_in == 0) {
-            break;
         } else if (lzs->avail_out == 0) {
+            /* Need to check lzs->avail_out before lzs->avail_in.
+               Maybe lzs's internal state still have a few bytes
+               can be output, grow the output buffer and continue
+               if max_lengh < 0. */
             if (data_size == max_length)
                 break;
             if (grow_buffer(&result, max_length) == -1)
                 goto error;
             lzs->next_out = (uint8_t *)PyBytes_AS_STRING(result) + data_size;
             lzs->avail_out = PyBytes_GET_SIZE(result) - data_size;
+        } else if (lzs->avail_in == 0) {
+            break;
         }
     }
     if (data_size != PyBytes_GET_SIZE(result))
@@ -990,7 +994,19 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length)
     }
     else if (lzs->avail_in == 0) {
         lzs->next_in = NULL;
-        d->needs_input = 1;
+
+        if (lzs->avail_out == 0) {
+            /* (avail_in==0 && avail_out==0)
+               Maybe lzs's internal state still have a few bytes can
+               be output, try to output them next time. */
+            d->needs_input = 0;
+
+            /* if max_length < 0, lzs->avail_out always > 0 */
+            assert(max_length >= 0);
+        } else {
+            /* Input buffer exhausted, output buffer has space. */
+            d->needs_input = 1;
+        }
     }
     else {
         d->needs_input = 0;


From webhook-mailer at python.org  Thu Sep 12 10:41:18 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Thu, 12 Sep 2019 14:41:18 -0000
Subject: [Python-checkins] bpo-37802: Slightly improve perfomance of
 PyLong_FromUnsigned*() (GH-15192)
Message-ID: <mailman.672.1568299280.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c6734ee7c55add5fdc2c821729ed5f67e237a096
commit: c6734ee7c55add5fdc2c821729ed5f67e237a096
branch: master
author: Sergey Fedoseev <fedoseev.sergey at gmail.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-12T15:41:14+01:00
summary:

bpo-37802: Slightly improve perfomance of PyLong_FromUnsigned*() (GH-15192)

files:
A Misc/NEWS.d/next/Core and Builtins/2019-08-09-18-28-57.bpo-37802.pKxcAW.rst
M Objects/longobject.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-09-18-28-57.bpo-37802.pKxcAW.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-09-18-28-57.bpo-37802.pKxcAW.rst
new file mode 100644
index 000000000000..dc0a23851eaf
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-09-18-28-57.bpo-37802.pKxcAW.rst	
@@ -0,0 +1,3 @@
+Slightly improve performance of :c:func:`PyLong_FromUnsignedLong`,
+:c:func:`PyLong_FromUnsignedLongLong` and :c:func:`PyLong_FromSize_t`.
+Patch by Sergey Fedoseev.
diff --git a/Objects/longobject.c b/Objects/longobject.c
index 6afec180a56e..5da6951b2589 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -43,6 +43,7 @@ PyObject *_PyLong_One = NULL;
 static PyLongObject small_ints[NSMALLNEGINTS + NSMALLPOSINTS];
 
 #define IS_SMALL_INT(ival) (-NSMALLNEGINTS <= (ival) && (ival) < NSMALLPOSINTS)
+#define IS_SMALL_UINT(ival) ((ival) < NSMALLPOSINTS)
 
 #ifdef COUNT_ALLOCS
 Py_ssize_t _Py_quick_int_allocs, _Py_quick_neg_int_allocs;
@@ -78,6 +79,7 @@ maybe_small_long(PyLongObject *v)
 }
 #else
 #define IS_SMALL_INT(ival) 0
+#define IS_SMALL_UINT(ival) 0
 #define get_small_int(ival) (Py_UNREACHABLE(), NULL)
 #define maybe_small_long(val) (val)
 #endif
@@ -378,32 +380,52 @@ PyLong_FromLong(long ival)
     return (PyObject *)v;
 }
 
+#define PYLONG_FROM_UINT(INT_TYPE, ival) \
+    do { \
+        if (IS_SMALL_UINT(ival)) { \
+            return get_small_int((ival)); \
+        } \
+        /* Count the number of Python digits. */ \
+        Py_ssize_t ndigits = 0; \
+        INT_TYPE t = (ival); \
+        while (t) { \
+            ++ndigits; \
+            t >>= PyLong_SHIFT; \
+        } \
+        PyLongObject *v = _PyLong_New(ndigits); \
+        if (v == NULL) { \
+            return NULL; \
+        } \
+        digit *p = v->ob_digit; \
+        while ((ival)) { \
+            *p++ = (digit)((ival) & PyLong_MASK); \
+            (ival) >>= PyLong_SHIFT; \
+        } \
+        return (PyObject *)v; \
+    } while(0)
+
 /* Create a new int object from a C unsigned long int */
 
 PyObject *
 PyLong_FromUnsignedLong(unsigned long ival)
 {
-    PyLongObject *v;
-    unsigned long t;
-    int ndigits = 0;
+    PYLONG_FROM_UINT(unsigned long, ival);
+}
 
-    if (ival < PyLong_BASE)
-        return PyLong_FromLong(ival);
-    /* Count the number of Python digits. */
-    t = ival;
-    while (t) {
-        ++ndigits;
-        t >>= PyLong_SHIFT;
-    }
-    v = _PyLong_New(ndigits);
-    if (v != NULL) {
-        digit *p = v->ob_digit;
-        while (ival) {
-            *p++ = (digit)(ival & PyLong_MASK);
-            ival >>= PyLong_SHIFT;
-        }
-    }
-    return (PyObject *)v;
+/* Create a new int object from a C unsigned long long int. */
+
+PyObject *
+PyLong_FromUnsignedLongLong(unsigned long long ival)
+{
+    PYLONG_FROM_UINT(unsigned long long, ival);
+}
+
+/* Create a new int object from a C size_t. */
+
+PyObject *
+PyLong_FromSize_t(size_t ival)
+{
+    PYLONG_FROM_UINT(size_t, ival);
 }
 
 /* Create a new int object from a C double */
@@ -1186,34 +1208,6 @@ PyLong_FromLongLong(long long ival)
     return (PyObject *)v;
 }
 
-/* Create a new int object from a C unsigned long long int. */
-
-PyObject *
-PyLong_FromUnsignedLongLong(unsigned long long ival)
-{
-    PyLongObject *v;
-    unsigned long long t;
-    int ndigits = 0;
-
-    if (ival < PyLong_BASE)
-        return PyLong_FromLong((long)ival);
-    /* Count the number of Python digits. */
-    t = ival;
-    while (t) {
-        ++ndigits;
-        t >>= PyLong_SHIFT;
-    }
-    v = _PyLong_New(ndigits);
-    if (v != NULL) {
-        digit *p = v->ob_digit;
-        while (ival) {
-            *p++ = (digit)(ival & PyLong_MASK);
-            ival >>= PyLong_SHIFT;
-        }
-    }
-    return (PyObject *)v;
-}
-
 /* Create a new int object from a C Py_ssize_t. */
 
 PyObject *
@@ -1257,35 +1251,6 @@ PyLong_FromSsize_t(Py_ssize_t ival)
     return (PyObject *)v;
 }
 
-/* Create a new int object from a C size_t. */
-
-PyObject *
-PyLong_FromSize_t(size_t ival)
-{
-    PyLongObject *v;
-    size_t t;
-    int ndigits = 0;
-
-    if (ival < PyLong_BASE)
-        return PyLong_FromLong((long)ival);
-    /* Count the number of Python digits. */
-    t = ival;
-    while (t) {
-        ++ndigits;
-        t >>= PyLong_SHIFT;
-    }
-    v = _PyLong_New(ndigits);
-    if (v != NULL) {
-        digit *p = v->ob_digit;
-        Py_SIZE(v) = ndigits;
-        while (ival) {
-            *p++ = (digit)(ival & PyLong_MASK);
-            ival >>= PyLong_SHIFT;
-        }
-    }
-    return (PyObject *)v;
-}
-
 /* Get a C long long int from an int object or any object that has an
    __int__ method.  Return -1 and set an error if overflow occurs. */
 


From webhook-mailer at python.org  Thu Sep 12 10:41:34 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 14:41:34 -0000
Subject: [Python-checkins] bpo-21872: fix lzma library decompresses data
 incompletely (GH-14048)
Message-ID: <mailman.673.1568299295.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a3c53a1b45b05bcb69660eac5a271443b37ecc42
commit: a3c53a1b45b05bcb69660eac5a271443b37ecc42
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T07:41:30-07:00
summary:

bpo-21872: fix lzma library decompresses data incompletely (GH-14048)


* 1. add test case with wrong behavior
* 2. fix bug when max_length == -1
* 3. allow b"" as valid input data for decompress_buf()
* 4. when max_length >= 0, let needs_input mechanism works
* add more asserts to test case
(cherry picked from commit 4ffd05d7ec47cfd0d7fc95dce851633be9663255)

Co-authored-by: animalize <animalize at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Library/2019-06-12-08-56-22.bpo-21872.V9QGGN.rst
M Lib/test/test_lzma.py
M Modules/_lzmamodule.c

diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py
index 3dc2c1e7e3b7..49758ac7e780 100644
--- a/Lib/test/test_lzma.py
+++ b/Lib/test/test_lzma.py
@@ -1195,6 +1195,36 @@ def test_tell_bad_args(self):
         f.close()
         self.assertRaises(ValueError, f.tell)
 
+    def test_issue21872(self):
+        # sometimes decompress data incompletely
+
+        # ---------------------
+        # when max_length == -1
+        # ---------------------
+        d1 = LZMADecompressor()
+        entire = d1.decompress(ISSUE_21872_DAT, max_length=-1)
+        self.assertEqual(len(entire), 13160)
+        self.assertTrue(d1.eof)
+
+        # ---------------------
+        # when max_length > 0
+        # ---------------------
+        d2 = LZMADecompressor()
+
+        # When this value of max_length is used, the input and output
+        # buffers are exhausted at the same time, and lzs's internal
+        # state still have 11 bytes can be output.
+        out1 = d2.decompress(ISSUE_21872_DAT, max_length=13149)
+        self.assertFalse(d2.needs_input) # ensure needs_input mechanism works
+        self.assertFalse(d2.eof)
+
+        # simulate needs_input mechanism
+        # output internal state's 11 bytes
+        out2 = d2.decompress(b'')
+        self.assertEqual(len(out2), 11)
+        self.assertTrue(d2.eof)
+        self.assertEqual(out1 + out2, entire)
+
 
 class OpenTestCase(unittest.TestCase):
 
@@ -1761,6 +1791,139 @@ def test_filter_properties_roundtrip(self):
     b"\x00"
 )
 
+ISSUE_21872_DAT = (
+    b']\x00\x00@\x00h3\x00\x00\x00\x00\x00\x00\x00\x00`D\x0c\x99\xc8'
+    b'\xd1\xbbZ^\xc43+\x83\xcd\xf1\xc6g\xec-\x061F\xb1\xbb\xc7\x17%-\xea'
+    b'\xfap\xfb\x8fs\x128\xb2,\x88\xe4\xc0\x12|*x\xd0\xa2\xc4b\x1b!\x02c'
+    b'\xab\xd9\x87U\xb8n \xfaVJ\x9a"\xb78\xff%_\x17`?@*\xc2\x82'
+    b"\xf2^\x1b\xb8\x04&\xc0\xbb\x03g\x9d\xca\xe9\xa4\xc9\xaf'\xe5\x8e}"
+    b'F\xdd\x11\xf3\x86\xbe\x1fN\x95\\\xef\xa2Mz-\xcb\x9a\xe3O@'
+    b"\x19\x07\xf6\xee\x9e\x9ag\xc6\xa5w\rnG'\x99\xfd\xfeGI\xb0"
+    b'\xbb\xf9\xc2\xe1\xff\xc5r\xcf\x85y[\x01\xa1\xbd\xcc/\xa3\x1b\x83\xaa'
+    b'\xc6\xf9\x99\x0c\xb6_\xc9MQ+x\xa2F\xda]\xdd\xe8\xfb\x1a&'
+    b',\xc4\x19\x1df\x81\x1e\x90\xf3\xb8Hgr\x85v\xbe\xa3qx\x01Y\xb5\x9fF'
+    b"\x13\x18\x01\xe69\x9b\xc8'\x1e\x9d\xd6\xe4F\x84\xac\xf8d<\x11\xd5"
+    b'\\\x0b\xeb\x0e\x82\xab\xb1\xe6\x1fka\xe1i\xc4 C\xb1"4)\xd6\xa7`\x02'
+    b'\xec\x11\x8c\xf0\x14\xb0\x1d\x1c\xecy\xf0\xb7|\x11j\x85X\xb2!\x1c'
+    b'\xac\xb5N\xc7\x85j\x9ev\xf5\xe6\x0b\xc1]c\xc15\x16\x9f\xd5\x99'
+    b"\xfei^\xd2G\x9b\xbdl\xab:\xbe,\xa9'4\x82\xe5\xee\xb3\xc1"
+    b'$\x93\x95\xa8Y\x16\xf5\xbf\xacw\x91\x04\x1d\x18\x06\xe9\xc5\xfdk\x06'
+    b'\xe8\xfck\xc5\x86>\x8b~\xa4\xcb\xf1\xb3\x04\xf1\x04G5\xe2\xcc]'
+    b'\x16\xbf\x140d\x18\xe2\xedw#(3\xca\xa1\x80bX\x7f\xb3\x84'
+    b'\x9d\xdb\xe7\x08\x97\xcd\x16\xb9\xf1\xd5r+m\x1e\xcb3q\xc5\x9e\x92'
+    b"\x7f\x8e*\xc7\xde\xe9\xe26\xcds\xb1\x10-\xf6r\x02?\x9d\xddCgJN'"
+    b'\x11M\xfa\nQ\n\xe6`m\xb8N\xbbq\x8el\x0b\x02\xc7:q\x04G\xa1T'
+    b'\xf1\xfe!0\x85~\xe5\x884\xe9\x89\xfb\x13J8\x15\xe42\xb6\xad'
+    b'\x877A\x9a\xa6\xbft]\xd0\xe35M\xb0\x0cK\xc8\xf6\x88\xae\xed\xa9,j7'
+    b'\x81\x13\xa0(\xcb\xe1\xe9l2\x7f\xcd\xda\x95(\xa70B\xbd\xf4\xe3'
+    b'hp\x94\xbdJ\xd7\t\xc7g\xffo?\x89?\xf8}\x7f\xbc\x1c\x87'
+    b'\x14\xc0\xcf\x8cV:\x9a\x0e\xd0\xb2\x1ck\xffk\xb9\xe0=\xc7\x8d/'
+    b'\xb8\xff\x7f\x1d\x87`\x19.\x98X*~\xa7j\xb9\x0b"\xf4\xe4;V`\xb9\xd7'
+    b'\x03\x1e\xd0t0\xd3\xde\x1fd\xb9\xe2)\x16\x81}\xb1\\b\x7fJ'
+    b'\x92\xf4\xff\n+V!\xe9\xde\x98\xa0\x8fK\xdf7\xb9\xc0\x12\x1f\xe2'
+    b'\xe9\xb0`\xae\x14\r\xa7\xc4\x81~\xd8\x8d\xc5\x06\xd8m\xb0Y\x8a)'
+    b'\x06/\xbb\xf9\xac\xeaP\xe0\x91\x05m[\xe5z\xe6Z\xf3\x9f\xc7\xd0'
+    b'\xd3\x8b\xf3\x8a\x1b\xfa\xe4Pf\xbc0\x17\x10\xa9\xd0\x95J{\xb3\xc3'
+    b'\xfdW\x9bop\x0f\xbe\xaee\xa3]\x93\x9c\xda\xb75<\xf6g!\xcc\xb1\xfc\\'
+    b'7\x152Mc\x17\x84\x9d\xcd35\r0\xacL-\xf3\xfb\xcb\x96\x1e\xe9U\x7f'
+    b'\xd7\xca\xb0\xcc\x89\x0c*\xce\x14\xd1P\xf1\x03\xb6.~9o?\xe8'
+    b'\r\x86\xe0\x92\x87}\xa3\x84\x03P\xe0\xc2\x7f\n;m\x9d\x9e\xb4|'
+    b'\x8c\x18\xc0#0\xfe3\x07<\xda\xd8\xcf^\xd4Hi\xd6\xb3\x0bT'
+    b'\x1dF\x88\x85q}\x02\xc6&\xc4\xae\xce\x9cU\xfa\x0f\xcc\xb6\x1f\x11'
+    b'drw\x9eN\x19\xbd\xffz\x0f\xf0\x04s\xadR\xc1\xc0\xbfl\xf1\xba\xf95^'
+    b'e\xb1\xfbVY\xd9\x9f\x1c\xbf*\xc4\xa86\x08+\xd6\x88[\xc4_rc\xf0f'
+    b'\xb8\xd4\xec\x1dx\x19|\xbf\xa7\xe0\x82\x0b\x8c~\x10L/\x90\xd6\xfb'
+    b'\x81\xdb\x98\xcc\x02\x14\xa5C\xb2\xa7i\xfd\xcd\x1fO\xf7\xe9\x89t\xf0'
+    b'\x17\xa5\x1c\xad\xfe<Q`%\x075k\n7\x9eI\x82<#)&\x04\xc2\xf0C\xd4`!'
+    b'\xcb\xa9\xf9\xb3F\x86\xb5\xc3M\xbeu\x12\xb2\xca\x95e\x10\x0b\xb1\xcc'
+    b'\x01b\x9bXa\x1b[B\x8c\x07\x11Of;\xeaC\xebr\x8eb\xd9\x9c\xe4i]<z\x9a'
+    b'\x03T\x8b9pF\x10\x8c\x84\xc7\x0e\xeaPw\xe5\xa0\x94\x1f\x84\xdd'
+    b'a\xe8\x85\xc2\x00\xebq\xe7&Wo5q8\xc2t\x98\xab\xb7\x7f\xe64-H'
+    b'\t\xb4d\xbe\x06\xe3Q\x8b\xa9J\xb0\x00\xd7s.\x85"\xc0p\x05'
+    b'\x1c\x06N\x87\xa5\xf8\xc3g\x1b}\x0f\x0f\xc3|\x90\xea\xefd3X'
+    b'[\xab\x04E\xf2\xf2\xc9\x08\x8a\xa8+W\xa2v\xec\x15G\x08/I<L\\1'
+    b'\xff\x15O\xaa\x89{\xd1mW\x13\xbd~\xe1\x90^\xc4@\r\xed\xb5D@\xb4\x08'
+    b'A\x90\xe69;\xc7BO\xdb\xda\xebu\x9e\xa9tN\xae\x8aJ5\xcd\x11\x1d\xea'
+    b"\xe5\xa7\x04\xe6\x82Z\xc7O\xe46[7\xdco*[\xbe\x0b\xc9\xb7a\xab'\xf6"
+    b"\xd1u\xdb\xd9q\xf5+y\x1b\x00\xb4\xf3a\xae\xf1M\xc4\xbc\xd00'\x06pQ"
+    b'\x8dH\xaa\xaa\xc4\xd2K\x9b\xc0\xe9\xec=n\xa9\x1a\x8a\xc2\xe8\x18\xbc'
+    b'\x93\xb8F\xa1\x8fOY\xe7\xda\xcf0\t\xff|\xd9\xe5\xcf\xe7\xf6\xbe'
+    b'\xf8\x04\x17\xf2\xe5P\xa7y~\xce\x11h0\x81\x80d[\x00_v\xbbc\xdbI'
+    b'3\xbc`W\xc0yrkB\xf5\x9f\xe9i\xc5\x8a^\x8d\xd4\x81\xd9\x05\xc1\xfc>'
+    b'"\xd1v`\x82\xd5$\x89\xcf^\xd52.\xafd\xe8d@\xaa\xd5Y|\x90\x84'
+    b'j\xdb}\x84riV\x8e\xf0X4rB\xf2NPS[\x8e\x88\xd4\x0fI\xb8'
+    b'\xdd\xcb\x1d\xf2(\xdf;9\x9e|\xef^0;.*[\x9fl\x7f\xa2_X\xaff!\xbb\x03'
+    b'\xff\x19\x8f\x88\xb5\xb6\x884\xa3\x05\xde3D{\xe3\xcb\xce\xe4t]'
+    b'\x875\xe3Uf\xae\xea\x88\x1c\x03b\n\xb1,Q\xec\xcf\x08\t\xde@\x83\xaa<'
+    b',-\xe4\xee\x9b\x843\xe5\x007\tK\xac\x057\xd6*X\xa3\xc6~\xba\xe6O'
+    b'\x81kz"\xbe\xe43sL\xf1\xfa;\xf4^\x1e\xb4\x80\xe2\xbd\xaa\x17Z\xe1f'
+    b'\xda\xa6\xb9\x07:]}\x9fa\x0b?\xba\xe7\xf15\x04M\xe3\n}M\xa4\xcb\r'
+    b'2\x8a\x88\xa9\xa7\x92\x93\x84\x81Yo\x00\xcc\xc4\xab\x9aT\x96\x0b\xbe'
+    b'U\xac\x1d\x8d\x1b\x98"\xf8\x8f\xf1u\xc1n\xcc\xfcA\xcc\x90\xb7i'
+    b'\x83\x9c\x9c~\x1d4\xa2\xf0*J\xe7t\x12\xb4\xe3\xa0u\xd7\x95Z'
+    b'\xf7\xafG\x96~ST,\xa7\rC\x06\xf4\xf0\xeb`2\x9e>Q\x0e\xf6\xf5\xc5'
+    b'\x9b\xb5\xaf\xbe\xa3\x8f\xc0\xa3hu\x14\x12 \x97\x99\x04b\x8e\xc7\x1b'
+    b'VKc\xc1\xf3 \xde\x85-:\xdc\x1f\xac\xce*\x06\xb3\x80;`'
+    b'\xdb\xdd\x97\xfdg\xbf\xe7\xa8S\x08}\xf55e7\xb8/\xf0!\xc8'
+    b"Y\xa8\x9a\x07'\xe2\xde\r\x02\xe1\xb2\x0c\xf4C\xcd\xf9\xcb(\xe8\x90"
+    b'\xd3bTD\x15_\xf6\xc3\xfb\xb3E\xfc\xd6\x98{\xc6\\fz\x81\xa99\x85\xcb'
+    b'\xa5\xb1\x1d\x94bqW\x1a!;z~\x18\x88\xe8i\xdb\x1b\x8d\x8d'
+    b'\x06\xaa\x0e\x99s+5k\x00\xe4\xffh\xfe\xdbt\xa6\x1bU\xde\xa3'
+    b'\xef\xcb\x86\x9e\x81\x16j\n\x9d\xbc\xbbC\x80?\x010\xc7Jj;'
+    b'\xc4\xe5\x86\xd5\x0e0d#\xc6;\xb8\xd1\xc7c\xb5&8?\xd9J\xe5\xden\xb9'
+    b'\xe9cb4\xbb\xe6\x14\xe0\xe7l\x1b\x85\x94\x1fh\xf1n\xdeZ\xbe'
+    b'\x88\xff\xc2e\xca\xdc,B-\x8ac\xc9\xdf\xf5|&\xe4LL\xf0\x1f\xaa8\xbd'
+    b'\xc26\x94bVi\xd3\x0c\x1c\xb6\xbb\x99F\x8f\x0e\xcc\x8e4\xc6/^W\xf5?'
+    b'\xdc\x84(\x14dO\x9aD6\x0f4\xa3,\x0c\x0bS\x9fJ\xe1\xacc^\x8a0\t\x80D['
+    b'\xb8\xe6\x86\xb0\xe8\xd4\xf9\x1en\xf1\xf5^\xeb\xb8\xb8\xf8'
+    b')\xa8\xbf\xaa\x84\x86\xb1a \x95\x16\x08\x1c\xbb@\xbd+\r/\xfb'
+    b'\x92\xfbh\xf1\x8d3\xf9\x92\xde`\xf1\x86\x03\xaa+\xd9\xd9\xc6P\xaf'
+    b'\xe3-\xea\xa5\x0fB\xca\xde\xd5n^\xe3/\xbf\xa6w\xc8\x0e<M'
+    b'\xc2\x1e!\xd4\xc6E\xf2\xad\x0c\xbc\x1d\x88Y\x03\x98<\x92\xd9\xa6B'
+    b'\xc7\x83\xb5"\x97D|&\xc4\xd4\xfad\x0e\xde\x06\xa3\xc2\x9c`\xf2'
+    b'7\x03\x1a\xed\xd80\x10\xe9\x0co\x10\xcf\x18\x16\xa7\x1c'
+    b"\xe5\x96\xa4\xd9\xe1\xa5v;]\xb7\xa9\xdc'hA\xe3\x9c&\x98\x0b9\xdf~@"
+    b'\xf8\xact\x87<\xf94\x0c\x9d\x93\xb0)\xe1\xa2\x0f\x1e=:&\xd56\xa5A+'
+    b'\xab\xc4\x00\x8d\x81\x93\xd4\xd8<\x82k\\d\xd8v\xab\xbd^l5C?\xd4\xa0'
+    b'M\x12C\xc8\x80\r\xc83\xe8\xc0\xf5\xdf\xca\x05\xf4BPjy\xbe\x91\x9bzE'
+    b"\xd8[\x93oT\r\x13\x16'\x1a\xbd*H\xd6\xfe\r\xf3\x91M\x8b\xee\x8f7f"
+    b"\x0b;\xaa\x85\xf2\xdd'\x0fwM \xbd\x13\xb9\xe5\xb8\xb7 D+P\x1c\xe4g"
+    b'n\xd2\xf1kc\x15\xaf\xc6\x90V\x03\xc2UovfZ\xcc\xd23^\xb3\xe7\xbf'
+    b'\xacv\x1d\x82\xedx\xa3J\xa9\xb7\xcf\x0c\xe6j\x96n*o\x18>'
+    b'\xc6\xfd\x97_+D{\x03\x15\xe8s\xb1\xc8HAG\xcf\xf4\x1a\xdd'
+    b'\xad\x11\xbf\x157q+\xdeW\x89g"X\x82\xfd~\xf7\xab4\xf6`\xab\xf1q'
+    b')\x82\x10K\xe9sV\xfe\xe45\xafs*\x14\xa7;\xac{\x06\x9d<@\x93G'
+    b'j\x1d\xefL\xe9\xd8\x92\x19&\xa1\x16\x19\x04\tu5\x01]\xf6\xf4'
+    b'\xcd\\\xd8A|I\xd4\xeb\x05\x88C\xc6e\xacQ\xe9*\x97~\x9au\xf8Xy'
+    b"\x17P\x10\x9f\n\x8c\xe2fZEu>\x9b\x1e\x91\x0b'`\xbd\xc0\xa8\x86c\x1d"
+    b'Z\xe2\xdc8j\x95\xffU\x90\x1e\xf4o\xbc\xe5\xe3e>\xd2R\xc0b#\xbc\x15'
+    b'H-\xb9!\xde\x9d\x90k\xdew\x9b{\x99\xde\xf7/K)A\xfd\xf5\xe6:\xda'
+    b'UM\xcc\xbb\xa2\x0b\x9a\x93\xf5{9\xc0 \xd2((6i\xc0\xbbu\xd8\x9e\x8d'
+    b'\xf8\x04q\x10\xd4\x14\x9e7-\xb9B\xea\x01Q8\xc8v\x9a\x12A\x88Cd\x92'
+    b"\x1c\x8c!\xf4\x94\x87'\xe3\xcd\xae\xf7\xd8\x93\xfa\xde\xa8b\x9e\xee2"
+    b'K\xdb\x00l\x9d\t\xb1|D\x05U\xbb\xf4>\xf1w\x887\xd1}W\x9d|g|1\xb0\x13'
+    b"\xa3 \xe5\xbfm@\xc06+\xb7\t\xcf\x15D\x9a \x1fM\x1f\xd2\xb5'\xa9\xbb"
+    b'~Co\x82\xfa\xc2\t\xe6f\xfc\xbeI\xae1\x8e\xbe\xb8\xcf\x86\x17'
+    b'\x9f\xe2`\xbd\xaf\xba\xb9\xbc\x1b\xa3\xcd\x82\x8fwc\xefd\xa9\xd5\x14'
+    b'\xe2C\xafUE\xb6\x11MJH\xd0=\x05\xd4*I\xff"\r\x1b^\xcaS6=\xec@\xd5'
+    b'\x11,\xe0\x87Gr\xaa[\xb8\xbc>n\xbd\x81\x0c\x07<\xe9\x92('
+    b'\xb2\xff\xac}\xe7\xb6\x15\x90\x9f~4\x9a\xe6\xd6\xd8s\xed\x99tf'
+    b'\xa0f\xf8\xf1\x87\t\x96/)\x85\xb6\n\xd7\xb2w\x0b\xbc\xba\x99\xee'
+    b'Q\xeen\x1d\xad\x03\xc3s\xd1\xfd\xa2\xc6\xb7\x9a\x9c(G<6\xad[~H '
+    b'\x16\x89\x89\xd0\xc3\xd2\xca~\xac\xea\xa5\xed\xe5\xfb\r:'
+    b'\x8e\xa6\xf1e\xbb\xba\xbd\xe0(\xa3\x89_\x01(\xb5c\xcc\x9f\x1fg'
+    b'v\xfd\x17\xb3\x08S=S\xee\xfc\x85>\x91\x8d\x8d\nYR\xb3G\xd1A\xa2\xb1'
+    b'\xec\xb0\x01\xd2\xcd\xf9\xfe\x82\x06O\xb3\xecd\xa9c\xe0\x8eP\x90\xce'
+    b'\xe0\xcd\xd8\xd8\xdc\x9f\xaa\x01"[Q~\xe4\x88\xa1#\xc1\x12C\xcf'
+    b'\xbe\x80\x11H\xbf\x86\xd8\xbem\xcfWFQ(X\x01DK\xdfB\xaa\x10.-'
+    b'\xd5\x9e|\x86\x15\x86N]\xc7Z\x17\xcd=\xd7)M\xde\x15\xa4LTi\xa0\x15'
+    b'\xd1\xe7\xbdN\xa4?\xd1\xe7\x02\xfe4\xe4O\x89\x98&\x96\x0f\x02\x9c'
+    b'\x9e\x19\xaa\x13u7\xbd0\xdc\xd8\x93\xf4BNE\x1d\x93\x82\x81\x16'
+    b'\xe5y\xcf\x98D\xca\x9a\xe2\xfd\xcdL\xcc\xd1\xfc_\x0b\x1c\xa0]\xdc'
+    b'\xa91 \xc9c\xd8\xbf\x97\xcfp\xe6\x19-\xad\xff\xcc\xd1N(\xe8'
+    b'\xeb#\x182\x96I\xf7l\xf3r\x00'
+)
+
 
 def test_main():
     run_unittest(
diff --git a/Misc/NEWS.d/next/Library/2019-06-12-08-56-22.bpo-21872.V9QGGN.rst b/Misc/NEWS.d/next/Library/2019-06-12-08-56-22.bpo-21872.V9QGGN.rst
new file mode 100644
index 000000000000..90df756f9a81
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-12-08-56-22.bpo-21872.V9QGGN.rst
@@ -0,0 +1,3 @@
+Fix :mod:`lzma`: module decompresses data incompletely. When decompressing a
+FORMAT_ALONE format file, and it doesn't have the end marker, sometimes the
+last one to dozens bytes can't be output. Patch by Ma Lin.
diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c
index f93c6c668e5d..ec3708974950 100644
--- a/Modules/_lzmamodule.c
+++ b/Modules/_lzmamodule.c
@@ -881,9 +881,6 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length)
     PyObject *result;
     lzma_stream *lzs = &d->lzs;
 
-    if (lzs->avail_in == 0)
-        return PyBytes_FromStringAndSize(NULL, 0);
-
     if (max_length < 0 || max_length >= INITIAL_BUFFER_SIZE)
         result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE);
     else
@@ -900,7 +897,10 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length)
         Py_BEGIN_ALLOW_THREADS
         lzret = lzma_code(lzs, LZMA_RUN);
         data_size = (char *)lzs->next_out - PyBytes_AS_STRING(result);
+        if (lzret == LZMA_BUF_ERROR && lzs->avail_in == 0 && lzs->avail_out > 0)
+            lzret = LZMA_OK; /* That wasn't a real error */
         Py_END_ALLOW_THREADS
+
         if (catch_lzma_error(lzret))
             goto error;
         if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK)
@@ -908,15 +908,19 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length)
         if (lzret == LZMA_STREAM_END) {
             d->eof = 1;
             break;
-        } else if (lzs->avail_in == 0) {
-            break;
         } else if (lzs->avail_out == 0) {
+            /* Need to check lzs->avail_out before lzs->avail_in.
+               Maybe lzs's internal state still have a few bytes
+               can be output, grow the output buffer and continue
+               if max_lengh < 0. */
             if (data_size == max_length)
                 break;
             if (grow_buffer(&result, max_length) == -1)
                 goto error;
             lzs->next_out = (uint8_t *)PyBytes_AS_STRING(result) + data_size;
             lzs->avail_out = PyBytes_GET_SIZE(result) - data_size;
+        } else if (lzs->avail_in == 0) {
+            break;
         }
     }
     if (data_size != PyBytes_GET_SIZE(result))
@@ -999,7 +1003,19 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length)
     }
     else if (lzs->avail_in == 0) {
         lzs->next_in = NULL;
-        d->needs_input = 1;
+
+        if (lzs->avail_out == 0) {
+            /* (avail_in==0 && avail_out==0)
+               Maybe lzs's internal state still have a few bytes can
+               be output, try to output them next time. */
+            d->needs_input = 0;
+
+            /* if max_length < 0, lzs->avail_out always > 0 */
+            assert(max_length >= 0);
+        } else {
+            /* Input buffer exhausted, output buffer has space. */
+            d->needs_input = 1;
+        }
     }
     else {
         d->needs_input = 0;


From webhook-mailer at python.org  Thu Sep 12 10:55:52 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 14:55:52 -0000
Subject: [Python-checkins] bpo-37488 : Document a warning for
 datetime.utcnow() and utcfromtimestamp() (GH-15773)
Message-ID: <mailman.674.1568300153.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/307c5fe9428b175ff3871a1fdc19bdd7562cfee5
commit: 307c5fe9428b175ff3871a1fdc19bdd7562cfee5
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T07:55:48-07:00
summary:

bpo-37488 : Document a warning for datetime.utcnow() and utcfromtimestamp() (GH-15773)


https://bugs.python.org/issue37488

Automerge-Triggered-By: @pganssle
(cherry picked from commit 1a53c785e62e00bad87ae19466c3a32ebcebb915)

Co-authored-by: Joannah Nanjekye <33177550+nanjekyejoannah at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Library/2019-09-11-11-44-16.bpo-37488.S8CJUL.rst
M Doc/library/datetime.rst

diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index 882d4e16e1f6..b1e1b25691d8 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -862,13 +862,10 @@ Other constructors, all class methods:
    (for example, this may be possible on platforms supplying the C
    :c:func:`gettimeofday` function).
 
-   If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the
-   current date and time are converted to *tz*?s time zone. In this case the
-   result is equivalent to::
-
-     tz.fromutc(datetime.utcnow().replace(tzinfo=tz))
+   If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass,
+   and the current date and time are converted to *tz*?s time zone.
 
-   See also :meth:`today`, :meth:`utcnow`.
+   This function is preferred over :meth:`today` and :meth:`utcnow`.
 
 
 .. classmethod:: datetime.utcnow()
@@ -879,6 +876,14 @@ Other constructors, all class methods:
    :class:`.datetime` object. An aware current UTC datetime can be obtained by
    calling ``datetime.now(timezone.utc)``. See also :meth:`now`.
 
+   .. warning::
+
+      Because naive ``datetime`` objects are treated by many ``datetime`` methods
+      as local times, it is preferred to use aware datetimes to represent times
+      in UTC. As such, the recommended way to create an object representing the
+      current time in UTC  by calling ``datetime.now(timezone.utc)``.
+
+
 .. classmethod:: datetime.fromtimestamp(timestamp, tz=None)
 
    Return the local date and time corresponding to the POSIX timestamp, such as is
@@ -889,10 +894,6 @@ Other constructors, all class methods:
    If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the
    timestamp is converted to *tz*?s time zone.
 
-   In this case the result is equivalent to::
-
-     tz.fromutc(datetime.utcfromtimestamp(timestamp).replace(tzinfo=tz))
-
    :meth:`fromtimestamp` may raise :exc:`OverflowError`, if the timestamp is out of
    the range of values supported by the platform C :c:func:`localtime` or
    :c:func:`gmtime` functions, and :exc:`OSError` on :c:func:`localtime` or
@@ -901,7 +902,8 @@ Other constructors, all class methods:
    1970 through 2038. Note that on non-POSIX systems that include leap seconds in
    their notion of a timestamp, leap seconds are ignored by :meth:`fromtimestamp`,
    and then it's possible to have two timestamps differing by a second that yield
-   identical :class:`.datetime` objects. See also :meth:`utcfromtimestamp`.
+   identical :class:`.datetime` objects. This method is preferred over
+   :meth:`utcfromtimestamp`.
 
    .. versionchanged:: 3.3
       Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp
@@ -935,6 +937,14 @@ Other constructors, all class methods:
    except the latter formula always supports the full years range: between
    :const:`MINYEAR` and :const:`MAXYEAR` inclusive.
 
+   .. warning::
+
+      Because naive ``datetime`` objects are treated by many ``datetime`` methods
+      as local times, it is preferred to use aware datetimes to represent times
+      in UTC. As such, the recommended way to create an object representing a
+      specific timestamp in UTC  by calling
+      ``datetime.fromtimestamp(timestamp, tz=timezone.utc)``.
+
    .. versionchanged:: 3.3
       Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp
       is out of the range of values supported by the platform C
@@ -1322,6 +1332,14 @@ Instance methods:
    ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year
    boundary.
 
+   .. warning::
+
+      Because naive ``datetime`` objects are treated by many ``datetime`` methods
+      as local times, it is preferred to use aware datetimes to represent times
+      in UTC; as a result, using ``utcfromtimetuple`` may give misleading
+      results. If you have a naive ``datetime`` representing UTC, use
+      ``datetime.replace(tzinfo=timezone.utc)`` to make it aware, at which point
+      you can use :meth:`.datetime.timetuple`.
 
 .. method:: datetime.toordinal()
 
@@ -1500,7 +1518,7 @@ Examples of working with :class:`~datetime.datetime` objects:
 
 .. doctest::
 
-    >>> from datetime import datetime, date, time
+    >>> from datetime import datetime, date, time, timezone
 
     >>> # Using datetime.combine()
     >>> d = date(2005, 7, 14)
@@ -1508,11 +1526,11 @@ Examples of working with :class:`~datetime.datetime` objects:
     >>> datetime.combine(d, t)
     datetime.datetime(2005, 7, 14, 12, 30)
 
-    >>> # Using datetime.now() or datetime.utcnow()
+    >>> # Using datetime.now()
     >>> datetime.now()   # doctest: +SKIP
     datetime.datetime(2007, 12, 6, 16, 29, 43, 79043)   # GMT +1
-    >>> datetime.utcnow()   # doctest: +SKIP
-    datetime.datetime(2007, 12, 6, 15, 29, 43, 79060)
+    >>> datetime.now(timezone.utc)   # doctest: +SKIP
+    datetime.datetime(2007, 12, 6, 15, 29, 43, 79060, tzinfo=datetime.timezone.utc)
 
     >>> # Using datetime.strptime()
     >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M")
@@ -1616,7 +1634,7 @@ Usage of ``KabulTz`` from above::
    datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc)
    >>> dt2
    datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz())
-   >>> dt2.utctimetuple() == dt3.utctimetuple()
+   >>> dt2 == dt3
    True
 
 .. _datetime-time:
diff --git a/Misc/NEWS.d/next/Library/2019-09-11-11-44-16.bpo-37488.S8CJUL.rst b/Misc/NEWS.d/next/Library/2019-09-11-11-44-16.bpo-37488.S8CJUL.rst
new file mode 100644
index 000000000000..4f9041502a91
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-11-11-44-16.bpo-37488.S8CJUL.rst
@@ -0,0 +1 @@
+Add warning to :meth:`datetime.utctimetuple`,  :meth:`datetime.utcnow` and :meth:`datetime.utcfromtimestamp` .
\ No newline at end of file


From webhook-mailer at python.org  Thu Sep 12 10:56:31 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Thu, 12 Sep 2019 14:56:31 -0000
Subject: [Python-checkins] bpo-38096: Complete the "structseq" and "named
 tuple" cleanup (GH-16010)
Message-ID: <mailman.675.1568300194.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4210ad5ebd5769f585035e022876e161cd0e9a3e
commit: 4210ad5ebd5769f585035e022876e161cd0e9a3e
branch: master
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T07:56:28-07:00
summary:

bpo-38096:  Complete the "structseq" and "named tuple" cleanup (GH-16010)

files:
M Doc/glossary.rst
M Objects/structseq.c

diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 84d0fcab9d75..e601e8b36984 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -757,7 +757,7 @@ Glossary
       Some named tuples are built-in types (such as the above examples).
       Alternatively, a named tuple can be created from a regular class
       definition that inherits from :class:`tuple` and that defines named
-      fields.  Such as class can be written by hand or it can be created with
+      fields.  Such a class can be written by hand or it can be created with
       the factory function :func:`collections.namedtuple`.  The latter
       technique also adds some extra methods that may not be found in
       hand-written or built-in named tuples.
diff --git a/Objects/structseq.c b/Objects/structseq.c
index 2c25e1646a2a..320bf080d7e3 100644
--- a/Objects/structseq.c
+++ b/Objects/structseq.c
@@ -1,5 +1,11 @@
-/* Implementation helper: a struct that looks like a tuple.  See timemodule
-   and posixmodule for example uses. */
+/* Implementation helper: a struct that looks like a tuple.
+   See timemodule and posixmodule for example uses.
+
+   The structseq helper is considered an internal CPython implementation
+   detail.  Docs for modules using structseqs should call them
+   "named tuples" (be sure to include a space between the two
+   words and add a link back to the term in Docs/glossary.rst).
+*/
 
 #include "Python.h"
 #include "pycore_tupleobject.h"


From webhook-mailer at python.org  Thu Sep 12 10:57:06 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 14:57:06 -0000
Subject: [Python-checkins] [3.8] bpo-38137: Re-add OpenSSL 1.0.2 compat
 (GH-16051) (GH-16057)
Message-ID: <mailman.676.1568300227.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/36c29e444d6bc0e31b46509610d3fb7cff74f2c2
commit: 36c29e444d6bc0e31b46509610d3fb7cff74f2c2
branch: 3.8
author: St?phane Wirtel <stephane at wirtel.be>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-12T07:57:03-07:00
summary:

[3.8] bpo-38137: Re-add OpenSSL 1.0.2 compat (GH-16051) (GH-16057)



The defines are required for OpenSSL 1.0.2 and LibreSSL.

https://bugs.python.org/issue38134

Automerge-Triggered-By: @tiran
(cherry picked from commit 9a4963b932a087b78596cca0b8394ac898faa490)

Co-authored-by: Christian Heimes <christian at python.org>


https://bugs.python.org/issue38137



Automerge-Triggered-By: @matrixise

files:
M Modules/_hashopenssl.c

diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index bea7e5ed3123..c685279c7a7b 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -26,6 +26,12 @@
 #include <openssl/objects.h>
 #include "openssl/err.h"
 
+#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
+/* OpenSSL < 1.1.0 */
+#define EVP_MD_CTX_new EVP_MD_CTX_create
+#define EVP_MD_CTX_free EVP_MD_CTX_destroy
+#endif
+
 #define MUNCH_SIZE INT_MAX
 
 typedef struct {


From webhook-mailer at python.org  Thu Sep 12 11:07:50 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Thu, 12 Sep 2019 15:07:50 -0000
Subject: [Python-checkins] bpo-37935: Added tests for os.walk(),
 glob.iglob() and Path.glob() (GH-15956) (GH-16043)
Message-ID: <mailman.677.1568300871.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/98a4a713d001cf2dfb04a9e318e4aea899bc8fbd
commit: 98a4a713d001cf2dfb04a9e318e4aea899bc8fbd
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-12T16:07:47+01:00
summary:

bpo-37935: Added tests for os.walk(), glob.iglob() and Path.glob() (GH-15956) (GH-16043)

Test that they do not keep too many file descriptors open for the host OS in a reasonable test scenario.

See [bpo-37935](https://bugs.python.org/issue37935).
(cherry picked from commit f9dc2ad89032201427ed5f08061c703794627ad9)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
M Lib/test/test_glob.py
M Lib/test/test_os.py
M Lib/test/test_pathlib.py

diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py
index 767bd3764b89..cba8c7c60e21 100644
--- a/Lib/test/test_glob.py
+++ b/Lib/test/test_glob.py
@@ -264,6 +264,23 @@ def test_recursive_glob(self):
                 expect += [join('sym3', 'EF')]
             eq(glob.glob(join('**', 'EF'), recursive=True), expect)
 
+    def test_glob_many_open_files(self):
+        depth = 30
+        base = os.path.join(self.tempdir, 'deep')
+        p = os.path.join(base, *(['d']*depth))
+        os.makedirs(p)
+        pattern = os.path.join(base, *(['*']*depth))
+        iters = [glob.iglob(pattern, recursive=True) for j in range(100)]
+        for it in iters:
+            self.assertEqual(next(it), p)
+        pattern = os.path.join(base, '**', 'd')
+        iters = [glob.iglob(pattern, recursive=True) for j in range(100)]
+        p = base
+        for i in range(depth):
+            p = os.path.join(p, 'd')
+            for it in iters:
+                self.assertEqual(next(it), p)
+
 
 @skip_unless_symlink
 class SymlinkLoopGlobTests(unittest.TestCase):
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 8ff0296fad0b..4a076e3bbf54 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -1178,6 +1178,27 @@ def test_walk_bad_dir(self):
         finally:
             os.rename(path1new, path1)
 
+    def test_walk_many_open_files(self):
+        depth = 30
+        base = os.path.join(support.TESTFN, 'deep')
+        p = os.path.join(base, *(['d']*depth))
+        os.makedirs(p)
+
+        iters = [self.walk(base, topdown=False) for j in range(100)]
+        for i in range(depth + 1):
+            expected = (p, ['d'] if i else [], [])
+            for it in iters:
+                self.assertEqual(next(it), expected)
+            p = os.path.dirname(p)
+
+        iters = [self.walk(base, topdown=True) for j in range(100)]
+        p = base
+        for i in range(depth + 1):
+            expected = (p, ['d'] if i < depth else [], [])
+            for it in iters:
+                self.assertEqual(next(it), expected)
+            p = os.path.join(p, 'd')
+
 
 @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()")
 class FwalkTests(WalkTests):
@@ -1247,6 +1268,10 @@ def test_fd_leak(self):
         self.addCleanup(os.close, newfd)
         self.assertEqual(newfd, minfd)
 
+    # fwalk() keeps file descriptors open
+    test_walk_many_open_files = None
+
+
 class BytesWalkTests(WalkTests):
     """Tests for os.walk() with bytes."""
     def walk(self, top, **kwargs):
diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py
index ebc71ec219c8..95d0b09882f6 100644
--- a/Lib/test/test_pathlib.py
+++ b/Lib/test/test_pathlib.py
@@ -1475,6 +1475,23 @@ def test_rglob_symlink_loop(self):
                   }
         self.assertEqual(given, {p / x for x in expect})
 
+    def test_glob_many_open_files(self):
+        depth = 30
+        P = self.cls
+        base = P(BASE) / 'deep'
+        p = P(base, *(['d']*depth))
+        p.mkdir(parents=True)
+        pattern = '/'.join(['*'] * depth)
+        iters = [base.glob(pattern) for j in range(100)]
+        for it in iters:
+            self.assertEqual(next(it), p)
+        iters = [base.rglob('d') for j in range(100)]
+        p = base
+        for i in range(depth):
+            p = p / 'd'
+            for it in iters:
+                self.assertEqual(next(it), p)
+
     def test_glob_dotdot(self):
         # ".." is not special in globs.
         P = self.cls


From webhook-mailer at python.org  Thu Sep 12 11:20:11 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Thu, 12 Sep 2019 15:20:11 -0000
Subject: [Python-checkins] bpo-38096: Complete the "structseq" and "named
 tuple" cleanup (GH-16010) (GH-16063)
Message-ID: <mailman.678.1568301611.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e3c25fc902eedcd5c593fac58f35645961f55bf4
commit: e3c25fc902eedcd5c593fac58f35645961f55bf4
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-12T08:20:07-07:00
summary:

bpo-38096:  Complete the "structseq" and "named tuple" cleanup (GH-16010) (GH-16063)

(cherry picked from commit 4210ad5ebd5769f585035e022876e161cd0e9a3e)

Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com>

files:
M Doc/glossary.rst
M Objects/structseq.c

diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 3aa54b9eff9e..8af4373b11d7 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -753,7 +753,7 @@ Glossary
       Some named tuples are built-in types (such as the above examples).
       Alternatively, a named tuple can be created from a regular class
       definition that inherits from :class:`tuple` and that defines named
-      fields.  Such as class can be written by hand or it can be created with
+      fields.  Such a class can be written by hand or it can be created with
       the factory function :func:`collections.namedtuple`.  The latter
       technique also adds some extra methods that may not be found in
       hand-written or built-in named tuples.
diff --git a/Objects/structseq.c b/Objects/structseq.c
index e48165dcd008..c71898fd79c0 100644
--- a/Objects/structseq.c
+++ b/Objects/structseq.c
@@ -1,5 +1,11 @@
-/* Implementation helper: a struct that looks like a tuple.  See timemodule
-   and posixmodule for example uses. */
+/* Implementation helper: a struct that looks like a tuple.
+   See timemodule and posixmodule for example uses.
+
+   The structseq helper is considered an internal CPython implementation
+   detail.  Docs for modules using structseqs should call them
+   "named tuples" (be sure to include a space between the two
+   words and add a link back to the term in Docs/glossary.rst).
+*/
 
 #include "Python.h"
 #include "structmember.h"


From webhook-mailer at python.org  Thu Sep 12 11:20:35 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Thu, 12 Sep 2019 15:20:35 -0000
Subject: [Python-checkins] bpo-38096: Complete the "structseq" and "named
 tuple" cleanup (GH-16010) (GH-16062)
Message-ID: <mailman.679.1568301636.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d04c85f5a74e61637ee17e8d4870c6a1d3b07d3e
commit: d04c85f5a74e61637ee17e8d4870c6a1d3b07d3e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-12T08:20:31-07:00
summary:

bpo-38096:  Complete the "structseq" and "named tuple" cleanup (GH-16010) (GH-16062)

(cherry picked from commit 4210ad5ebd5769f585035e022876e161cd0e9a3e)

Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com>

files:
M Doc/glossary.rst
M Objects/structseq.c

diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 84d0fcab9d75..e601e8b36984 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -757,7 +757,7 @@ Glossary
       Some named tuples are built-in types (such as the above examples).
       Alternatively, a named tuple can be created from a regular class
       definition that inherits from :class:`tuple` and that defines named
-      fields.  Such as class can be written by hand or it can be created with
+      fields.  Such a class can be written by hand or it can be created with
       the factory function :func:`collections.namedtuple`.  The latter
       technique also adds some extra methods that may not be found in
       hand-written or built-in named tuples.
diff --git a/Objects/structseq.c b/Objects/structseq.c
index 2c25e1646a2a..320bf080d7e3 100644
--- a/Objects/structseq.c
+++ b/Objects/structseq.c
@@ -1,5 +1,11 @@
-/* Implementation helper: a struct that looks like a tuple.  See timemodule
-   and posixmodule for example uses. */
+/* Implementation helper: a struct that looks like a tuple.
+   See timemodule and posixmodule for example uses.
+
+   The structseq helper is considered an internal CPython implementation
+   detail.  Docs for modules using structseqs should call them
+   "named tuples" (be sure to include a space between the two
+   words and add a link back to the term in Docs/glossary.rst).
+*/
 
 #include "Python.h"
 #include "pycore_tupleobject.h"


From webhook-mailer at python.org  Thu Sep 12 11:41:35 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Thu, 12 Sep 2019 15:41:35 -0000
Subject: [Python-checkins] [3.8] bpo-38121: Sync importlib.metadata with
 0.22 backport (GH-15993) (GH-16064)
Message-ID: <mailman.680.1568302897.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a0d4aac50b362f4c33594a86e0e299d3f812e7b1
commit: a0d4aac50b362f4c33594a86e0e299d3f812e7b1
branch: 3.8
author: Jason R. Coombs <jaraco at jaraco.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T16:41:31+01:00
summary:

[3.8] bpo-38121: Sync importlib.metadata with 0.22 backport (GH-15993) (GH-16064)

* bpo-38121: Sync importlib.metadata with 0.22 backport

* ?? Added by blurb_it..
(cherry picked from commit 8ed6503eca4e3ea4949479d8d7fd9ffd54f81038)

Co-authored-by: Jason R. Coombs <jaraco at jaraco.com>

files:
A Misc/NEWS.d/next/Library/2019-09-11-16-54-57.bpo-38121.SrSDzB.rst
M Lib/importlib/_bootstrap_external.py
M Lib/importlib/metadata.py
M Python/importlib_external.h

diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index ba78c57eebc3..b8ac4829949a 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -1369,7 +1369,7 @@ def find_module(cls, fullname, path=None):
         return spec.loader
 
     @classmethod
-    def find_distributions(self, context=None):
+    def find_distributions(cls, *args, **kwargs):
         """
         Find distributions.
 
@@ -1378,54 +1378,8 @@ def find_distributions(self, context=None):
         (or all names if ``None`` indicated) along the paths in the list
         of directories ``context.path``.
         """
-        from importlib.metadata import PathDistribution, DistributionFinder
-        if context is None:
-            context = DistributionFinder.Context()
-        found = self._search_paths(context.pattern, context.path)
-        return map(PathDistribution, found)
-
-    @classmethod
-    def _search_paths(cls, pattern, paths):
-        """Find metadata directories in paths heuristically."""
-        import itertools
-        return itertools.chain.from_iterable(
-            cls._search_path(path, pattern)
-            for path in map(cls._switch_path, paths)
-            )
-
-    @staticmethod
-    def _switch_path(path):
-        from contextlib import suppress
-        import zipfile
-        import pathlib
-        PYPY_OPEN_BUG = False
-        if not PYPY_OPEN_BUG or os.path.isfile(path):  # pragma: no branch
-            with suppress(Exception):
-                return zipfile.Path(path)
-        return pathlib.Path(path)
-
-    @classmethod
-    def _matches_info(cls, normalized, item):
-        import re
-        template = r'{pattern}(-.*)?\.(dist|egg)-info'
-        manifest = template.format(pattern=normalized)
-        return re.match(manifest, item.name, flags=re.IGNORECASE)
-
-    @classmethod
-    def _matches_legacy(cls, normalized, item):
-        import re
-        template = r'{pattern}-.*\.egg[\\/]EGG-INFO'
-        manifest = template.format(pattern=normalized)
-        return re.search(manifest, str(item), flags=re.IGNORECASE)
-
-    @classmethod
-    def _search_path(cls, root, pattern):
-        if not root.is_dir():
-            return ()
-        normalized = pattern.replace('-', '_')
-        return (item for item in root.iterdir()
-                if cls._matches_info(normalized, item)
-                or cls._matches_legacy(normalized, item))
+        from importlib.metadata import MetadataPathFinder
+        return MetadataPathFinder.find_distributions(*args, **kwargs)
 
 
 class FileFinder:
diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py
index e23076654fe4..8cb45ec1ef3a 100644
--- a/Lib/importlib/metadata.py
+++ b/Lib/importlib/metadata.py
@@ -1,10 +1,12 @@
 import io
+import os
 import re
 import abc
 import csv
 import sys
 import email
 import pathlib
+import zipfile
 import operator
 import functools
 import itertools
@@ -363,6 +365,58 @@ def find_distributions(self, context=Context()):
         """
 
 
+class MetadataPathFinder(DistributionFinder):
+    @classmethod
+    def find_distributions(cls, context=DistributionFinder.Context()):
+        """
+        Find distributions.
+
+        Return an iterable of all Distribution instances capable of
+        loading the metadata for packages matching ``context.name``
+        (or all names if ``None`` indicated) along the paths in the list
+        of directories ``context.path``.
+        """
+        found = cls._search_paths(context.pattern, context.path)
+        return map(PathDistribution, found)
+
+    @classmethod
+    def _search_paths(cls, pattern, paths):
+        """Find metadata directories in paths heuristically."""
+        return itertools.chain.from_iterable(
+            cls._search_path(path, pattern)
+            for path in map(cls._switch_path, paths)
+            )
+
+    @staticmethod
+    def _switch_path(path):
+        PYPY_OPEN_BUG = False
+        if not PYPY_OPEN_BUG or os.path.isfile(path):  # pragma: no branch
+            with suppress(Exception):
+                return zipfile.Path(path)
+        return pathlib.Path(path)
+
+    @classmethod
+    def _matches_info(cls, normalized, item):
+        template = r'{pattern}(-.*)?\.(dist|egg)-info'
+        manifest = template.format(pattern=normalized)
+        return re.match(manifest, item.name, flags=re.IGNORECASE)
+
+    @classmethod
+    def _matches_legacy(cls, normalized, item):
+        template = r'{pattern}-.*\.egg[\\/]EGG-INFO'
+        manifest = template.format(pattern=normalized)
+        return re.search(manifest, str(item), flags=re.IGNORECASE)
+
+    @classmethod
+    def _search_path(cls, root, pattern):
+        if not root.is_dir():
+            return ()
+        normalized = pattern.replace('-', '_')
+        return (item for item in root.iterdir()
+                if cls._matches_info(normalized, item)
+                or cls._matches_legacy(normalized, item))
+
+
 class PathDistribution(Distribution):
     def __init__(self, path):
         """Construct a distribution from a path to the metadata directory.
@@ -382,13 +436,13 @@ def locate_file(self, path):
         return self._path.parent / path
 
 
-def distribution(package):
-    """Get the ``Distribution`` instance for the given package.
+def distribution(distribution_name):
+    """Get the ``Distribution`` instance for the named package.
 
-    :param package: The name of the package as a string.
+    :param distribution_name: The name of the distribution package as a string.
     :return: A ``Distribution`` instance (or subclass thereof).
     """
-    return Distribution.from_name(package)
+    return Distribution.from_name(distribution_name)
 
 
 def distributions(**kwargs):
@@ -399,23 +453,23 @@ def distributions(**kwargs):
     return Distribution.discover(**kwargs)
 
 
-def metadata(package):
-    """Get the metadata for the package.
+def metadata(distribution_name):
+    """Get the metadata for the named package.
 
-    :param package: The name of the distribution package to query.
+    :param distribution_name: The name of the distribution package to query.
     :return: An email.Message containing the parsed metadata.
     """
-    return Distribution.from_name(package).metadata
+    return Distribution.from_name(distribution_name).metadata
 
 
-def version(package):
+def version(distribution_name):
     """Get the version string for the named package.
 
-    :param package: The name of the distribution package to query.
+    :param distribution_name: The name of the distribution package to query.
     :return: The version string for the package as defined in the package's
         "Version" metadata key.
     """
-    return distribution(package).version
+    return distribution(distribution_name).version
 
 
 def entry_points():
@@ -434,15 +488,20 @@ def entry_points():
         }
 
 
-def files(package):
-    return distribution(package).files
+def files(distribution_name):
+    """Return a list of files for the named package.
+
+    :param distribution_name: The name of the distribution package to query.
+    :return: List of files composing the distribution.
+    """
+    return distribution(distribution_name).files
 
 
-def requires(package):
+def requires(distribution_name):
     """
-    Return a list of requirements for the indicated distribution.
+    Return a list of requirements for the named package.
 
     :return: An iterator of requirements, suitable for
     packaging.requirement.Requirement.
     """
-    return distribution(package).requires
+    return distribution(distribution_name).requires
diff --git a/Misc/NEWS.d/next/Library/2019-09-11-16-54-57.bpo-38121.SrSDzB.rst b/Misc/NEWS.d/next/Library/2019-09-11-16-54-57.bpo-38121.SrSDzB.rst
new file mode 100644
index 000000000000..93eabcfaab39
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-11-16-54-57.bpo-38121.SrSDzB.rst
@@ -0,0 +1 @@
+Update parameter names on functions in importlib.metadata matching the changes in the 0.22 release of importlib_metadata.
\ No newline at end of file
diff --git a/Python/importlib_external.h b/Python/importlib_external.h
index 1e3415e2efbd..74d98bdebab5 100644
--- a/Python/importlib_external.h
+++ b/Python/importlib_external.h
@@ -571,7 +571,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,104,97,110,100,108,101,32,37,115,169,1,218,4,110,97,
     109,101,41,2,114,117,0,0,0,218,11,73,109,112,111,114,
     116,69,114,114,111,114,41,4,218,4,115,101,108,102,114,117,
-    0,0,0,218,4,97,114,103,115,90,6,107,119,97,114,103,
+    0,0,0,218,4,97,114,103,115,218,6,107,119,97,114,103,
     115,169,1,218,6,109,101,116,104,111,100,114,3,0,0,0,
     114,6,0,0,0,218,19,95,99,104,101,99,107,95,110,97,
     109,101,95,119,114,97,112,112,101,114,200,1,0,0,115,18,
@@ -596,13 +596,13 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,115,8,0,0,0,0,1,8,1,10,1,20,1,122,26,
     95,99,104,101,99,107,95,110,97,109,101,46,60,108,111,99,
     97,108,115,62,46,95,119,114,97,112,41,1,78,41,3,218,
-    10,95,98,111,111,116,115,116,114,97,112,114,133,0,0,0,
-    218,9,78,97,109,101,69,114,114,111,114,41,3,114,122,0,
-    0,0,114,123,0,0,0,114,133,0,0,0,114,3,0,0,
-    0,114,121,0,0,0,114,6,0,0,0,218,11,95,99,104,
+    10,95,98,111,111,116,115,116,114,97,112,114,134,0,0,0,
+    218,9,78,97,109,101,69,114,114,111,114,41,3,114,123,0,
+    0,0,114,124,0,0,0,114,134,0,0,0,114,3,0,0,
+    0,114,122,0,0,0,114,6,0,0,0,218,11,95,99,104,
     101,99,107,95,110,97,109,101,192,1,0,0,115,14,0,0,
     0,0,8,14,7,2,1,10,1,14,2,14,5,10,1,114,
-    136,0,0,0,99,2,0,0,0,0,0,0,0,0,0,0,
+    137,0,0,0,99,2,0,0,0,0,0,0,0,0,0,0,
     0,5,0,0,0,6,0,0,0,67,0,0,0,115,60,0,
     0,0,124,0,160,0,124,1,161,1,92,2,125,2,125,3,
     124,2,100,1,107,8,114,56,116,1,124,3,131,1,114,56,
@@ -630,7 +630,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     114,3,0,0,0,114,6,0,0,0,218,17,95,102,105,110,
     100,95,109,111,100,117,108,101,95,115,104,105,109,220,1,0,
     0,115,10,0,0,0,0,10,14,1,16,1,4,1,22,1,
-    114,143,0,0,0,99,3,0,0,0,0,0,0,0,0,0,
+    114,144,0,0,0,99,3,0,0,0,0,0,0,0,0,0,
     0,0,6,0,0,0,4,0,0,0,67,0,0,0,115,158,
     0,0,0,124,0,100,1,100,2,133,2,25,0,125,3,124,
     3,116,0,107,3,114,60,100,3,124,1,155,2,100,4,124,
@@ -688,7 +688,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     8,0,0,0,233,252,255,255,255,122,14,105,110,118,97,108,
     105,100,32,102,108,97,103,115,32,122,4,32,105,110,32,41,
     7,218,12,77,65,71,73,67,95,78,85,77,66,69,82,114,
-    134,0,0,0,218,16,95,118,101,114,98,111,115,101,95,109,
+    135,0,0,0,218,16,95,118,101,114,98,111,115,101,95,109,
     101,115,115,97,103,101,114,118,0,0,0,114,22,0,0,0,
     218,8,69,79,70,69,114,114,111,114,114,27,0,0,0,41,
     6,114,26,0,0,0,114,117,0,0,0,218,11,101,120,99,
@@ -697,7 +697,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,114,6,0,0,0,218,13,95,99,108,97,115,115,
     105,102,121,95,112,121,99,237,1,0,0,115,28,0,0,0,
     0,16,12,1,8,1,16,1,12,1,12,1,12,1,10,1,
-    12,1,8,1,16,2,8,1,16,1,12,1,114,152,0,0,
+    12,1,8,1,16,2,8,1,16,1,12,1,114,153,0,0,
     0,99,5,0,0,0,0,0,0,0,0,0,0,0,6,0,
     0,0,4,0,0,0,67,0,0,0,115,112,0,0,0,116,
     0,124,0,100,1,100,2,133,2,25,0,131,1,124,1,100,
@@ -739,19 +739,19 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     10,32,32,32,32,65,110,32,73,109,112,111,114,116,69,114,
     114,111,114,32,105,115,32,114,97,105,115,101,100,32,105,102,
     32,116,104,101,32,98,121,116,101,99,111,100,101,32,105,115,
-    32,115,116,97,108,101,46,10,10,32,32,32,32,114,146,0,
+    32,115,116,97,108,101,46,10,10,32,32,32,32,114,147,0,
     0,0,233,12,0,0,0,114,14,0,0,0,122,22,98,121,
     116,101,99,111,100,101,32,105,115,32,115,116,97,108,101,32,
-    102,111,114,32,114,144,0,0,0,78,114,145,0,0,0,41,
-    4,114,27,0,0,0,114,134,0,0,0,114,149,0,0,0,
+    102,111,114,32,114,145,0,0,0,78,114,146,0,0,0,41,
+    4,114,27,0,0,0,114,135,0,0,0,114,150,0,0,0,
     114,118,0,0,0,41,6,114,26,0,0,0,218,12,115,111,
     117,114,99,101,95,109,116,105,109,101,218,11,115,111,117,114,
-    99,101,95,115,105,122,101,114,117,0,0,0,114,151,0,0,
+    99,101,95,115,105,122,101,114,117,0,0,0,114,152,0,0,
     0,114,93,0,0,0,114,3,0,0,0,114,3,0,0,0,
     114,6,0,0,0,218,23,95,118,97,108,105,100,97,116,101,
     95,116,105,109,101,115,116,97,109,112,95,112,121,99,14,2,
     0,0,115,16,0,0,0,0,19,24,1,10,1,12,1,12,
-    1,8,1,22,255,2,2,114,156,0,0,0,99,4,0,0,
+    1,8,1,22,255,2,2,114,157,0,0,0,99,4,0,0,
     0,0,0,0,0,0,0,0,0,4,0,0,0,3,0,0,
     0,67,0,0,0,115,38,0,0,0,124,0,100,1,100,2,
     133,2,25,0,124,1,107,3,114,34,116,0,100,3,124,2,
@@ -787,17 +787,17 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,32,65,110,32,73,109,112,111,114,116,69,114,114,111,114,
     32,105,115,32,114,97,105,115,101,100,32,105,102,32,116,104,
     101,32,98,121,116,101,99,111,100,101,32,105,115,32,115,116,
-    97,108,101,46,10,10,32,32,32,32,114,146,0,0,0,114,
-    145,0,0,0,122,46,104,97,115,104,32,105,110,32,98,121,
+    97,108,101,46,10,10,32,32,32,32,114,147,0,0,0,114,
+    146,0,0,0,122,46,104,97,115,104,32,105,110,32,98,121,
     116,101,99,111,100,101,32,100,111,101,115,110,39,116,32,109,
     97,116,99,104,32,104,97,115,104,32,111,102,32,115,111,117,
     114,99,101,32,78,41,1,114,118,0,0,0,41,4,114,26,
     0,0,0,218,11,115,111,117,114,99,101,95,104,97,115,104,
-    114,117,0,0,0,114,151,0,0,0,114,3,0,0,0,114,
+    114,117,0,0,0,114,152,0,0,0,114,3,0,0,0,114,
     3,0,0,0,114,6,0,0,0,218,18,95,118,97,108,105,
     100,97,116,101,95,104,97,115,104,95,112,121,99,42,2,0,
     0,115,12,0,0,0,0,17,16,1,2,1,8,255,2,2,
-    2,254,114,158,0,0,0,99,4,0,0,0,0,0,0,0,
+    2,254,114,159,0,0,0,99,4,0,0,0,0,0,0,0,
     0,0,0,0,5,0,0,0,5,0,0,0,67,0,0,0,
     115,80,0,0,0,116,0,160,1,124,0,161,1,125,4,116,
     2,124,4,116,3,131,2,114,56,116,4,160,5,100,1,124,
@@ -813,7 +813,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     117,0,0,0,114,44,0,0,0,41,10,218,7,109,97,114,
     115,104,97,108,90,5,108,111,97,100,115,218,10,105,115,105,
     110,115,116,97,110,99,101,218,10,95,99,111,100,101,95,116,
-    121,112,101,114,134,0,0,0,114,149,0,0,0,218,4,95,
+    121,112,101,114,135,0,0,0,114,150,0,0,0,218,4,95,
     105,109,112,90,16,95,102,105,120,95,99,111,95,102,105,108,
     101,110,97,109,101,114,118,0,0,0,114,62,0,0,0,41,
     5,114,26,0,0,0,114,117,0,0,0,114,107,0,0,0,
@@ -821,7 +821,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     114,3,0,0,0,114,6,0,0,0,218,17,95,99,111,109,
     112,105,108,101,95,98,121,116,101,99,111,100,101,66,2,0,
     0,115,20,0,0,0,0,2,10,1,10,1,12,1,8,1,
-    12,1,4,2,10,1,2,0,2,255,114,165,0,0,0,114,
+    12,1,4,2,10,1,2,0,2,255,114,166,0,0,0,114,
     73,0,0,0,99,3,0,0,0,0,0,0,0,0,0,0,
     0,4,0,0,0,5,0,0,0,67,0,0,0,115,70,0,
     0,0,116,0,116,1,131,1,125,3,124,3,160,2,116,3,
@@ -832,14 +832,14 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     117,99,101,32,116,104,101,32,100,97,116,97,32,102,111,114,
     32,97,32,116,105,109,101,115,116,97,109,112,45,98,97,115,
     101,100,32,112,121,99,46,114,73,0,0,0,41,6,218,9,
-    98,121,116,101,97,114,114,97,121,114,148,0,0,0,218,6,
-    101,120,116,101,110,100,114,20,0,0,0,114,160,0,0,0,
-    218,5,100,117,109,112,115,41,4,114,164,0,0,0,218,5,
-    109,116,105,109,101,114,155,0,0,0,114,26,0,0,0,114,
+    98,121,116,101,97,114,114,97,121,114,149,0,0,0,218,6,
+    101,120,116,101,110,100,114,20,0,0,0,114,161,0,0,0,
+    218,5,100,117,109,112,115,41,4,114,165,0,0,0,218,5,
+    109,116,105,109,101,114,156,0,0,0,114,26,0,0,0,114,
     3,0,0,0,114,3,0,0,0,114,6,0,0,0,218,22,
     95,99,111,100,101,95,116,111,95,116,105,109,101,115,116,97,
     109,112,95,112,121,99,79,2,0,0,115,12,0,0,0,0,
-    2,8,1,14,1,14,1,14,1,16,1,114,170,0,0,0,
+    2,8,1,14,1,14,1,14,1,16,1,114,171,0,0,0,
     84,99,3,0,0,0,0,0,0,0,0,0,0,0,5,0,
     0,0,5,0,0,0,67,0,0,0,115,80,0,0,0,116,
     0,116,1,131,1,125,3,100,1,124,2,100,1,62,0,66,
@@ -849,16 +849,16 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     6,160,7,124,0,161,1,161,1,1,0,124,3,83,0,41,
     3,122,38,80,114,111,100,117,99,101,32,116,104,101,32,100,
     97,116,97,32,102,111,114,32,97,32,104,97,115,104,45,98,
-    97,115,101,100,32,112,121,99,46,114,39,0,0,0,114,146,
-    0,0,0,41,8,114,166,0,0,0,114,148,0,0,0,114,
-    167,0,0,0,114,20,0,0,0,114,22,0,0,0,114,23,
-    0,0,0,114,160,0,0,0,114,168,0,0,0,41,5,114,
-    164,0,0,0,114,157,0,0,0,90,7,99,104,101,99,107,
+    97,115,101,100,32,112,121,99,46,114,39,0,0,0,114,147,
+    0,0,0,41,8,114,167,0,0,0,114,149,0,0,0,114,
+    168,0,0,0,114,20,0,0,0,114,22,0,0,0,114,23,
+    0,0,0,114,161,0,0,0,114,169,0,0,0,41,5,114,
+    165,0,0,0,114,158,0,0,0,90,7,99,104,101,99,107,
     101,100,114,26,0,0,0,114,83,0,0,0,114,3,0,0,
     0,114,3,0,0,0,114,6,0,0,0,218,17,95,99,111,
     100,101,95,116,111,95,104,97,115,104,95,112,121,99,89,2,
     0,0,115,14,0,0,0,0,2,8,1,12,1,14,1,16,
-    1,10,1,16,1,114,171,0,0,0,99,1,0,0,0,0,
+    1,10,1,16,1,114,172,0,0,0,99,1,0,0,0,0,
     0,0,0,0,0,0,0,5,0,0,0,6,0,0,0,67,
     0,0,0,115,62,0,0,0,100,1,100,2,108,0,125,1,
     116,1,160,2,124,0,161,1,106,3,125,2,124,1,160,4,
@@ -879,14 +879,14 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     25,73,110,99,114,101,109,101,110,116,97,108,78,101,119,108,
     105,110,101,68,101,99,111,100,101,114,218,6,100,101,99,111,
     100,101,41,5,218,12,115,111,117,114,99,101,95,98,121,116,
-    101,115,114,172,0,0,0,90,21,115,111,117,114,99,101,95,
+    101,115,114,173,0,0,0,90,21,115,111,117,114,99,101,95,
     98,121,116,101,115,95,114,101,97,100,108,105,110,101,218,8,
     101,110,99,111,100,105,110,103,90,15,110,101,119,108,105,110,
     101,95,100,101,99,111,100,101,114,114,3,0,0,0,114,3,
     0,0,0,114,6,0,0,0,218,13,100,101,99,111,100,101,
     95,115,111,117,114,99,101,100,2,0,0,115,10,0,0,0,
-    0,5,8,1,12,1,10,1,12,1,114,176,0,0,0,169,
-    2,114,140,0,0,0,218,26,115,117,98,109,111,100,117,108,
+    0,5,8,1,12,1,10,1,12,1,114,177,0,0,0,169,
+    2,114,141,0,0,0,218,26,115,117,98,109,111,100,117,108,
     101,95,115,101,97,114,99,104,95,108,111,99,97,116,105,111,
     110,115,99,2,0,0,0,0,0,0,0,2,0,0,0,9,
     0,0,0,8,0,0,0,67,0,0,0,115,16,1,0,0,
@@ -930,19 +930,19 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,32,32,32,78,122,9,60,117,110,107,110,111,119,110,62,
     218,12,103,101,116,95,102,105,108,101,110,97,109,101,169,1,
     218,6,111,114,105,103,105,110,84,218,10,105,115,95,112,97,
-    99,107,97,103,101,114,73,0,0,0,41,17,114,128,0,0,
-    0,114,179,0,0,0,114,118,0,0,0,114,2,0,0,0,
-    114,79,0,0,0,114,134,0,0,0,218,10,77,111,100,117,
+    99,107,97,103,101,114,73,0,0,0,41,17,114,129,0,0,
+    0,114,180,0,0,0,114,118,0,0,0,114,2,0,0,0,
+    114,79,0,0,0,114,135,0,0,0,218,10,77,111,100,117,
     108,101,83,112,101,99,90,13,95,115,101,116,95,102,105,108,
     101,97,116,116,114,218,27,95,103,101,116,95,115,117,112,112,
     111,114,116,101,100,95,102,105,108,101,95,108,111,97,100,101,
-    114,115,114,111,0,0,0,114,112,0,0,0,114,140,0,0,
-    0,218,9,95,80,79,80,85,76,65,84,69,114,182,0,0,
-    0,114,178,0,0,0,114,47,0,0,0,218,6,97,112,112,
+    114,115,114,111,0,0,0,114,112,0,0,0,114,141,0,0,
+    0,218,9,95,80,79,80,85,76,65,84,69,114,183,0,0,
+    0,114,179,0,0,0,114,47,0,0,0,218,6,97,112,112,
     101,110,100,41,9,114,117,0,0,0,90,8,108,111,99,97,
-    116,105,111,110,114,140,0,0,0,114,178,0,0,0,218,4,
+    116,105,111,110,114,141,0,0,0,114,179,0,0,0,218,4,
     115,112,101,99,218,12,108,111,97,100,101,114,95,99,108,97,
-    115,115,218,8,115,117,102,102,105,120,101,115,114,182,0,0,
+    115,115,218,8,115,117,102,102,105,120,101,115,114,183,0,0,
     0,90,7,100,105,114,110,97,109,101,114,3,0,0,0,114,
     3,0,0,0,114,6,0,0,0,218,23,115,112,101,99,95,
     102,114,111,109,95,102,105,108,101,95,108,111,99,97,116,105,
@@ -950,7 +950,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     1,10,2,2,1,14,1,14,1,8,2,10,8,16,1,6,
     3,8,1,14,1,14,1,10,1,6,1,6,2,4,3,8,
     2,10,1,2,1,14,1,14,1,6,2,4,1,8,2,6,
-    1,12,1,6,1,12,1,12,2,114,190,0,0,0,99,0,
+    1,12,1,6,1,12,1,12,2,114,191,0,0,0,99,0,
     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,
     0,0,0,64,0,0,0,115,80,0,0,0,101,0,90,1,
     100,0,90,2,100,1,90,3,100,2,90,4,100,3,90,5,
@@ -997,15 +997,15 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,82,0,88,0,87,0,110,22,4,0,116,9,107,10,114,
     108,1,0,1,0,1,0,89,0,100,0,83,0,88,0,124,
     5,83,0,41,5,78,122,5,37,100,46,37,100,114,28,0,
-    0,0,41,2,114,139,0,0,0,90,11,115,121,115,95,118,
+    0,0,41,2,114,140,0,0,0,90,11,115,121,115,95,118,
     101,114,115,105,111,110,114,40,0,0,0,41,10,218,11,68,
     69,66,85,71,95,66,85,73,76,68,218,18,82,69,71,73,
     83,84,82,89,95,75,69,89,95,68,69,66,85,71,218,12,
     82,69,71,73,83,84,82,89,95,75,69,89,114,62,0,0,
     0,114,8,0,0,0,218,12,118,101,114,115,105,111,110,95,
-    105,110,102,111,114,194,0,0,0,114,192,0,0,0,90,10,
+    105,110,102,111,114,195,0,0,0,114,193,0,0,0,90,10,
     81,117,101,114,121,86,97,108,117,101,114,50,0,0,0,41,
-    6,114,193,0,0,0,114,139,0,0,0,90,12,114,101,103,
+    6,114,194,0,0,0,114,140,0,0,0,90,12,114,101,103,
     105,115,116,114,121,95,107,101,121,114,5,0,0,0,90,4,
     104,107,101,121,218,8,102,105,108,101,112,97,116,104,114,3,
     0,0,0,114,3,0,0,0,114,6,0,0,0,218,16,95,
@@ -1024,13 +1024,13 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     4,160,4,116,5,124,6,131,1,161,1,114,64,116,6,106,
     7,124,1,124,5,124,1,124,4,131,2,124,4,100,1,141,
     3,125,7,124,7,2,0,1,0,83,0,113,64,100,0,83,
-    0,41,2,78,114,180,0,0,0,41,8,114,200,0,0,0,
-    114,49,0,0,0,114,50,0,0,0,114,184,0,0,0,114,
-    111,0,0,0,114,112,0,0,0,114,134,0,0,0,218,16,
+    0,41,2,78,114,181,0,0,0,41,8,114,201,0,0,0,
+    114,49,0,0,0,114,50,0,0,0,114,185,0,0,0,114,
+    111,0,0,0,114,112,0,0,0,114,135,0,0,0,218,16,
     115,112,101,99,95,102,114,111,109,95,108,111,97,100,101,114,
-    41,8,114,193,0,0,0,114,139,0,0,0,114,44,0,0,
-    0,218,6,116,97,114,103,101,116,114,199,0,0,0,114,140,
-    0,0,0,114,189,0,0,0,114,187,0,0,0,114,3,0,
+    41,8,114,194,0,0,0,114,140,0,0,0,114,44,0,0,
+    0,218,6,116,97,114,103,101,116,114,200,0,0,0,114,141,
+    0,0,0,114,190,0,0,0,114,188,0,0,0,114,3,0,
     0,0,114,3,0,0,0,114,6,0,0,0,218,9,102,105,
     110,100,95,115,112,101,99,219,2,0,0,115,28,0,0,0,
     0,2,10,1,8,1,4,1,2,1,12,1,14,1,8,1,
@@ -1048,22 +1048,22 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     101,99,97,116,101,100,46,32,32,85,115,101,32,101,120,101,
     99,95,109,111,100,117,108,101,40,41,32,105,110,115,116,101,
     97,100,46,10,10,32,32,32,32,32,32,32,32,78,169,2,
-    114,203,0,0,0,114,140,0,0,0,169,4,114,193,0,0,
-    0,114,139,0,0,0,114,44,0,0,0,114,187,0,0,0,
+    114,204,0,0,0,114,141,0,0,0,169,4,114,194,0,0,
+    0,114,140,0,0,0,114,44,0,0,0,114,188,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,218,
     11,102,105,110,100,95,109,111,100,117,108,101,235,2,0,0,
     115,8,0,0,0,0,7,12,1,8,1,6,2,122,33,87,
     105,110,100,111,119,115,82,101,103,105,115,116,114,121,70,105,
     110,100,101,114,46,102,105,110,100,95,109,111,100,117,108,101,
-    41,2,78,78,41,1,78,41,12,114,125,0,0,0,114,124,
-    0,0,0,114,126,0,0,0,114,127,0,0,0,114,197,0,
-    0,0,114,196,0,0,0,114,195,0,0,0,218,11,99,108,
-    97,115,115,109,101,116,104,111,100,114,194,0,0,0,114,200,
-    0,0,0,114,203,0,0,0,114,206,0,0,0,114,3,0,
+    41,2,78,78,41,1,78,41,12,114,126,0,0,0,114,125,
+    0,0,0,114,127,0,0,0,114,128,0,0,0,114,198,0,
+    0,0,114,197,0,0,0,114,196,0,0,0,218,11,99,108,
+    97,115,115,109,101,116,104,111,100,114,195,0,0,0,114,201,
+    0,0,0,114,204,0,0,0,114,207,0,0,0,114,3,0,
     0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
-    0,114,191,0,0,0,185,2,0,0,115,28,0,0,0,8,
+    0,114,192,0,0,0,185,2,0,0,115,28,0,0,0,8,
     2,4,3,2,255,2,4,2,255,2,3,4,2,2,1,10,
-    6,2,1,10,14,2,1,12,15,2,1,114,191,0,0,0,
+    6,2,1,10,14,2,1,12,15,2,1,114,192,0,0,0,
     99,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
     0,2,0,0,0,64,0,0,0,115,48,0,0,0,101,0,
     90,1,100,0,90,2,100,1,90,3,100,2,100,3,132,0,
@@ -1092,12 +1092,12 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     101,32,111,102,32,39,95,95,105,110,105,116,95,95,46,112,
     121,39,46,114,39,0,0,0,114,71,0,0,0,114,73,0,
     0,0,114,28,0,0,0,218,8,95,95,105,110,105,116,95,
-    95,41,4,114,47,0,0,0,114,179,0,0,0,114,43,0,
-    0,0,114,41,0,0,0,41,5,114,119,0,0,0,114,139,
+    95,41,4,114,47,0,0,0,114,180,0,0,0,114,43,0,
+    0,0,114,41,0,0,0,41,5,114,119,0,0,0,114,140,
     0,0,0,114,97,0,0,0,90,13,102,105,108,101,110,97,
     109,101,95,98,97,115,101,90,9,116,97,105,108,95,110,97,
     109,101,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
-    0,114,182,0,0,0,254,2,0,0,115,8,0,0,0,0,
+    0,114,183,0,0,0,254,2,0,0,115,8,0,0,0,0,
     3,18,1,16,1,14,1,122,24,95,76,111,97,100,101,114,
     66,97,115,105,99,115,46,105,115,95,112,97,99,107,97,103,
     101,99,2,0,0,0,0,0,0,0,0,0,0,0,2,0,
@@ -1105,7 +1105,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     1,83,0,169,2,122,42,85,115,101,32,100,101,102,97,117,
     108,116,32,115,101,109,97,110,116,105,99,115,32,102,111,114,
     32,109,111,100,117,108,101,32,99,114,101,97,116,105,111,110,
-    46,78,114,3,0,0,0,169,2,114,119,0,0,0,114,187,
+    46,78,114,3,0,0,0,169,2,114,119,0,0,0,114,188,
     0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
     0,0,218,13,99,114,101,97,116,101,95,109,111,100,117,108,
     101,6,3,0,0,115,2,0,0,0,0,1,122,27,95,76,
@@ -1121,12 +1121,12 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     108,111,97,100,32,109,111,100,117,108,101,32,123,33,114,125,
     32,119,104,101,110,32,103,101,116,95,99,111,100,101,40,41,
     32,114,101,116,117,114,110,115,32,78,111,110,101,41,8,218,
-    8,103,101,116,95,99,111,100,101,114,125,0,0,0,114,118,
-    0,0,0,114,62,0,0,0,114,134,0,0,0,218,25,95,
+    8,103,101,116,95,99,111,100,101,114,126,0,0,0,114,118,
+    0,0,0,114,62,0,0,0,114,135,0,0,0,218,25,95,
     99,97,108,108,95,119,105,116,104,95,102,114,97,109,101,115,
-    95,114,101,109,111,118,101,100,218,4,101,120,101,99,114,131,
+    95,114,101,109,111,118,101,100,218,4,101,120,101,99,114,132,
     0,0,0,41,3,114,119,0,0,0,218,6,109,111,100,117,
-    108,101,114,164,0,0,0,114,3,0,0,0,114,3,0,0,
+    108,101,114,165,0,0,0,114,3,0,0,0,114,3,0,0,
     0,114,6,0,0,0,218,11,101,120,101,99,95,109,111,100,
     117,108,101,9,3,0,0,115,12,0,0,0,0,2,12,1,
     8,1,6,1,4,255,6,2,122,25,95,76,111,97,100,101,
@@ -1135,19 +1135,19 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     2,0,0,0,4,0,0,0,67,0,0,0,115,12,0,0,
     0,116,0,160,1,124,0,124,1,161,2,83,0,41,1,122,
     26,84,104,105,115,32,109,111,100,117,108,101,32,105,115,32,
-    100,101,112,114,101,99,97,116,101,100,46,41,2,114,134,0,
+    100,101,112,114,101,99,97,116,101,100,46,41,2,114,135,0,
     0,0,218,17,95,108,111,97,100,95,109,111,100,117,108,101,
-    95,115,104,105,109,169,2,114,119,0,0,0,114,139,0,0,
+    95,115,104,105,109,169,2,114,119,0,0,0,114,140,0,0,
     0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
     218,11,108,111,97,100,95,109,111,100,117,108,101,17,3,0,
     0,115,2,0,0,0,0,2,122,25,95,76,111,97,100,101,
     114,66,97,115,105,99,115,46,108,111,97,100,95,109,111,100,
-    117,108,101,78,41,8,114,125,0,0,0,114,124,0,0,0,
-    114,126,0,0,0,114,127,0,0,0,114,182,0,0,0,114,
-    212,0,0,0,114,217,0,0,0,114,220,0,0,0,114,3,
+    117,108,101,78,41,8,114,126,0,0,0,114,125,0,0,0,
+    114,127,0,0,0,114,128,0,0,0,114,183,0,0,0,114,
+    213,0,0,0,114,218,0,0,0,114,221,0,0,0,114,3,
     0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,114,208,0,0,0,249,2,0,0,115,10,0,0,0,
-    8,2,4,3,8,8,8,3,8,8,114,208,0,0,0,99,
+    0,0,114,209,0,0,0,249,2,0,0,115,10,0,0,0,
+    8,2,4,3,8,8,8,3,8,8,114,209,0,0,0,99,
     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
     3,0,0,0,64,0,0,0,115,74,0,0,0,101,0,90,
     1,100,0,90,2,100,1,100,2,132,0,90,3,100,3,100,
@@ -1202,8 +1202,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,32,82,97,105,115,101,115,32,79,83,69,114,114,111,114,
     32,119,104,101,110,32,116,104,101,32,112,97,116,104,32,99,
     97,110,110,111,116,32,98,101,32,104,97,110,100,108,101,100,
-    46,10,32,32,32,32,32,32,32,32,114,169,0,0,0,41,
-    1,114,223,0,0,0,114,222,0,0,0,114,3,0,0,0,
+    46,10,32,32,32,32,32,32,32,32,114,170,0,0,0,41,
+    1,114,224,0,0,0,114,223,0,0,0,114,3,0,0,0,
     114,3,0,0,0,114,6,0,0,0,218,10,112,97,116,104,
     95,115,116,97,116,115,32,3,0,0,115,2,0,0,0,0,
     12,122,23,83,111,117,114,99,101,76,111,97,100,101,114,46,
@@ -1246,7 +1246,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     46,10,32,32,32,32,32,32,32,32,78,114,3,0,0,0,
     41,3,114,119,0,0,0,114,44,0,0,0,114,26,0,0,
     0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
-    114,225,0,0,0,56,3,0,0,115,2,0,0,0,0,1,
+    114,226,0,0,0,56,3,0,0,115,2,0,0,0,0,1,
     122,21,83,111,117,114,99,101,76,111,97,100,101,114,46,115,
     101,116,95,100,97,116,97,99,2,0,0,0,0,0,0,0,
     0,0,0,0,5,0,0,0,10,0,0,0,67,0,0,0,
@@ -1262,10 +1262,10 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     39,115,111,117,114,99,101,32,110,111,116,32,97,118,97,105,
     108,97,98,108,101,32,116,104,114,111,117,103,104,32,103,101,
     116,95,100,97,116,97,40,41,114,116,0,0,0,78,41,5,
-    114,179,0,0,0,218,8,103,101,116,95,100,97,116,97,114,
-    50,0,0,0,114,118,0,0,0,114,176,0,0,0,41,5,
-    114,119,0,0,0,114,139,0,0,0,114,44,0,0,0,114,
-    174,0,0,0,218,3,101,120,99,114,3,0,0,0,114,3,
+    114,180,0,0,0,218,8,103,101,116,95,100,97,116,97,114,
+    50,0,0,0,114,118,0,0,0,114,177,0,0,0,41,5,
+    114,119,0,0,0,114,140,0,0,0,114,44,0,0,0,114,
+    175,0,0,0,218,3,101,120,99,114,3,0,0,0,114,3,
     0,0,0,114,6,0,0,0,218,10,103,101,116,95,115,111,
     117,114,99,101,63,3,0,0,115,20,0,0,0,0,2,10,
     1,2,1,14,1,16,1,4,1,2,255,4,1,2,255,20,
@@ -1283,11 +1283,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     110,32,98,101,32,97,110,121,32,111,98,106,101,99,116,32,
     116,121,112,101,32,116,104,97,116,32,99,111,109,112,105,108,
     101,40,41,32,115,117,112,112,111,114,116,115,46,10,32,32,
-    32,32,32,32,32,32,114,215,0,0,0,84,41,2,218,12,
+    32,32,32,32,32,32,114,216,0,0,0,84,41,2,218,12,
     100,111,110,116,95,105,110,104,101,114,105,116,114,84,0,0,
-    0,41,3,114,134,0,0,0,114,214,0,0,0,218,7,99,
+    0,41,3,114,135,0,0,0,114,215,0,0,0,218,7,99,
     111,109,112,105,108,101,41,4,114,119,0,0,0,114,26,0,
-    0,0,114,44,0,0,0,114,230,0,0,0,114,3,0,0,
+    0,0,114,44,0,0,0,114,231,0,0,0,114,3,0,0,
     0,114,3,0,0,0,114,6,0,0,0,218,14,115,111,117,
     114,99,101,95,116,111,95,99,111,100,101,73,3,0,0,115,
     8,0,0,0,0,5,12,1,2,0,2,255,122,27,83,111,
@@ -1340,34 +1340,34 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,32,32,32,32,98,121,116,101,99,111,100,101,44,32,115,
     101,116,95,100,97,116,97,32,109,117,115,116,32,97,108,115,
     111,32,98,101,32,105,109,112,108,101,109,101,110,116,101,100,
-    46,10,10,32,32,32,32,32,32,32,32,78,70,84,114,169,
-    0,0,0,114,159,0,0,0,114,145,0,0,0,114,39,0,
+    46,10,10,32,32,32,32,32,32,32,32,78,70,84,114,170,
+    0,0,0,114,160,0,0,0,114,146,0,0,0,114,39,0,
     0,0,114,73,0,0,0,114,28,0,0,0,90,5,110,101,
     118,101,114,90,6,97,108,119,97,121,115,218,4,115,105,122,
     101,122,13,123,125,32,109,97,116,99,104,101,115,32,123,125,
     41,3,114,117,0,0,0,114,107,0,0,0,114,108,0,0,
     0,122,19,99,111,100,101,32,111,98,106,101,99,116,32,102,
-    114,111,109,32,123,125,41,27,114,179,0,0,0,114,98,0,
-    0,0,114,82,0,0,0,114,224,0,0,0,114,50,0,0,
-    0,114,17,0,0,0,114,227,0,0,0,114,152,0,0,0,
-    218,10,109,101,109,111,114,121,118,105,101,119,114,163,0,0,
+    114,111,109,32,123,125,41,27,114,180,0,0,0,114,98,0,
+    0,0,114,82,0,0,0,114,225,0,0,0,114,50,0,0,
+    0,114,17,0,0,0,114,228,0,0,0,114,153,0,0,0,
+    218,10,109,101,109,111,114,121,118,105,101,119,114,164,0,0,
     0,90,21,99,104,101,99,107,95,104,97,115,104,95,98,97,
-    115,101,100,95,112,121,99,115,114,157,0,0,0,218,17,95,
+    115,101,100,95,112,121,99,115,114,158,0,0,0,218,17,95,
     82,65,87,95,77,65,71,73,67,95,78,85,77,66,69,82,
-    114,158,0,0,0,114,156,0,0,0,114,118,0,0,0,114,
-    150,0,0,0,114,134,0,0,0,114,149,0,0,0,114,165,
-    0,0,0,114,233,0,0,0,114,8,0,0,0,218,19,100,
+    114,159,0,0,0,114,157,0,0,0,114,118,0,0,0,114,
+    151,0,0,0,114,135,0,0,0,114,150,0,0,0,114,166,
+    0,0,0,114,234,0,0,0,114,8,0,0,0,218,19,100,
     111,110,116,95,119,114,105,116,101,95,98,121,116,101,99,111,
-    100,101,114,171,0,0,0,114,170,0,0,0,114,22,0,0,
-    0,114,226,0,0,0,41,15,114,119,0,0,0,114,139,0,
-    0,0,114,108,0,0,0,114,154,0,0,0,114,174,0,0,
-    0,114,157,0,0,0,90,10,104,97,115,104,95,98,97,115,
+    100,101,114,172,0,0,0,114,171,0,0,0,114,22,0,0,
+    0,114,227,0,0,0,41,15,114,119,0,0,0,114,140,0,
+    0,0,114,108,0,0,0,114,155,0,0,0,114,175,0,0,
+    0,114,158,0,0,0,90,10,104,97,115,104,95,98,97,115,
     101,100,90,12,99,104,101,99,107,95,115,111,117,114,99,101,
-    114,107,0,0,0,218,2,115,116,114,26,0,0,0,114,151,
+    114,107,0,0,0,218,2,115,116,114,26,0,0,0,114,152,
     0,0,0,114,83,0,0,0,90,10,98,121,116,101,115,95,
     100,97,116,97,90,11,99,111,100,101,95,111,98,106,101,99,
     116,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
-    114,213,0,0,0,81,3,0,0,115,152,0,0,0,0,7,
+    114,214,0,0,0,81,3,0,0,115,152,0,0,0,0,7,
     10,1,4,1,4,1,4,1,4,1,4,1,2,1,12,1,
     14,1,12,2,2,1,14,1,14,1,8,2,12,1,2,1,
     14,1,14,1,6,3,2,1,2,254,6,4,2,1,12,1,
@@ -1379,13 +1379,13 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     6,1,10,1,10,1,14,2,6,1,6,255,4,2,2,1,
     18,1,16,1,6,1,122,21,83,111,117,114,99,101,76,111,
     97,100,101,114,46,103,101,116,95,99,111,100,101,78,41,10,
-    114,125,0,0,0,114,124,0,0,0,114,126,0,0,0,114,
-    223,0,0,0,114,224,0,0,0,114,226,0,0,0,114,225,
-    0,0,0,114,229,0,0,0,114,233,0,0,0,114,213,0,
+    114,126,0,0,0,114,125,0,0,0,114,127,0,0,0,114,
+    224,0,0,0,114,225,0,0,0,114,227,0,0,0,114,226,
+    0,0,0,114,230,0,0,0,114,234,0,0,0,114,214,0,
     0,0,114,3,0,0,0,114,3,0,0,0,114,3,0,0,
-    0,114,6,0,0,0,114,221,0,0,0,22,3,0,0,115,
+    0,114,6,0,0,0,114,222,0,0,0,22,3,0,0,115,
     14,0,0,0,8,2,8,8,8,14,8,10,8,7,8,10,
-    14,8,114,221,0,0,0,99,0,0,0,0,0,0,0,0,
+    14,8,114,222,0,0,0,99,0,0,0,0,0,0,0,0,
     0,0,0,0,0,0,0,0,4,0,0,0,0,0,0,0,
     115,124,0,0,0,101,0,90,1,100,0,90,2,100,1,90,
     3,100,2,100,3,132,0,90,4,100,4,100,5,132,0,90,
@@ -1410,16 +1410,16 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     97,110,100,32,116,104,101,32,112,97,116,104,32,116,111,32,
     116,104,101,32,102,105,108,101,32,102,111,117,110,100,32,98,
     121,32,116,104,101,10,32,32,32,32,32,32,32,32,102,105,
-    110,100,101,114,46,78,114,159,0,0,0,41,3,114,119,0,
-    0,0,114,139,0,0,0,114,44,0,0,0,114,3,0,0,
-    0,114,3,0,0,0,114,6,0,0,0,114,209,0,0,0,
+    110,100,101,114,46,78,114,160,0,0,0,41,3,114,119,0,
+    0,0,114,140,0,0,0,114,44,0,0,0,114,3,0,0,
+    0,114,3,0,0,0,114,6,0,0,0,114,210,0,0,0,
     171,3,0,0,115,4,0,0,0,0,3,6,1,122,19,70,
     105,108,101,76,111,97,100,101,114,46,95,95,105,110,105,116,
     95,95,99,2,0,0,0,0,0,0,0,0,0,0,0,2,
     0,0,0,2,0,0,0,67,0,0,0,115,24,0,0,0,
     124,0,106,0,124,1,106,0,107,2,111,22,124,0,106,1,
     124,1,106,1,107,2,83,0,114,110,0,0,0,169,2,218,
-    9,95,95,99,108,97,115,115,95,95,114,131,0,0,0,169,
+    9,95,95,99,108,97,115,115,95,95,114,132,0,0,0,169,
     2,114,119,0,0,0,90,5,111,116,104,101,114,114,3,0,
     0,0,114,3,0,0,0,114,6,0,0,0,218,6,95,95,
     101,113,95,95,177,3,0,0,115,6,0,0,0,0,1,12,
@@ -1442,9 +1442,9 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,105,115,32,100,101,112,114,101,99,97,116,101,100,46,32,
     32,85,115,101,32,101,120,101,99,95,109,111,100,117,108,101,
     40,41,32,105,110,115,116,101,97,100,46,10,10,32,32,32,
-    32,32,32,32,32,41,3,218,5,115,117,112,101,114,114,239,
-    0,0,0,114,220,0,0,0,114,219,0,0,0,169,1,114,
-    241,0,0,0,114,3,0,0,0,114,6,0,0,0,114,220,
+    32,32,32,32,32,41,3,218,5,115,117,112,101,114,114,240,
+    0,0,0,114,221,0,0,0,114,220,0,0,0,169,1,114,
+    242,0,0,0,114,3,0,0,0,114,6,0,0,0,114,221,
     0,0,0,184,3,0,0,115,2,0,0,0,0,10,122,22,
     70,105,108,101,76,111,97,100,101,114,46,108,111,97,100,95,
     109,111,100,117,108,101,99,2,0,0,0,0,0,0,0,0,
@@ -1453,9 +1453,9 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     116,117,114,110,32,116,104,101,32,112,97,116,104,32,116,111,
     32,116,104,101,32,115,111,117,114,99,101,32,102,105,108,101,
     32,97,115,32,102,111,117,110,100,32,98,121,32,116,104,101,
-    32,102,105,110,100,101,114,46,114,48,0,0,0,114,219,0,
+    32,102,105,110,100,101,114,46,114,48,0,0,0,114,220,0,
     0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
-    0,114,179,0,0,0,196,3,0,0,115,2,0,0,0,0,
+    0,114,180,0,0,0,196,3,0,0,115,2,0,0,0,0,
     3,122,23,70,105,108,101,76,111,97,100,101,114,46,103,101,
     116,95,102,105,108,101,110,97,109,101,99,2,0,0,0,0,
     0,0,0,0,0,0,0,3,0,0,0,10,0,0,0,67,
@@ -1469,19 +1469,19 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     122,39,82,101,116,117,114,110,32,116,104,101,32,100,97,116,
     97,32,102,114,111,109,32,112,97,116,104,32,97,115,32,114,
     97,119,32,98,121,116,101,115,46,218,1,114,78,41,8,114,
-    161,0,0,0,114,221,0,0,0,218,19,69,120,116,101,110,
+    162,0,0,0,114,222,0,0,0,218,19,69,120,116,101,110,
     115,105,111,110,70,105,108,101,76,111,97,100,101,114,114,64,
     0,0,0,90,9,111,112,101,110,95,99,111,100,101,114,85,
     0,0,0,90,4,114,101,97,100,114,65,0,0,0,41,3,
     114,119,0,0,0,114,44,0,0,0,114,68,0,0,0,114,
-    3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,227,
+    3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,228,
     0,0,0,201,3,0,0,115,10,0,0,0,0,2,14,1,
     16,1,28,2,14,1,122,19,70,105,108,101,76,111,97,100,
     101,114,46,103,101,116,95,100,97,116,97,99,2,0,0,0,
     0,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,
     67,0,0,0,115,18,0,0,0,124,0,160,0,124,1,161,
     1,114,14,124,0,83,0,100,0,83,0,114,110,0,0,0,
-    41,1,114,182,0,0,0,169,2,114,119,0,0,0,114,216,
+    41,1,114,183,0,0,0,169,2,114,119,0,0,0,114,217,
     0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
     0,0,218,19,103,101,116,95,114,101,115,111,117,114,99,101,
     95,114,101,97,100,101,114,212,3,0,0,115,6,0,0,0,
@@ -1491,7 +1491,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,3,0,0,0,4,0,0,0,67,0,0,0,115,
     32,0,0,0,116,0,116,1,124,0,106,2,131,1,100,1,
     25,0,124,1,131,2,125,2,116,3,160,4,124,2,100,2,
-    161,2,83,0,41,3,78,114,73,0,0,0,114,251,0,0,
+    161,2,83,0,41,3,78,114,73,0,0,0,114,252,0,0,
     0,41,5,114,38,0,0,0,114,47,0,0,0,114,44,0,
     0,0,114,64,0,0,0,114,65,0,0,0,169,3,114,119,
     0,0,0,90,8,114,101,115,111,117,114,99,101,114,44,0,
@@ -1507,7 +1507,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,41,5,218,11,105,115,95,114,101,115,111,117,114,
     99,101,218,17,70,105,108,101,78,111,116,70,111,117,110,100,
     69,114,114,111,114,114,38,0,0,0,114,47,0,0,0,114,
-    44,0,0,0,114,255,0,0,0,114,3,0,0,0,114,3,
+    44,0,0,0,114,0,1,0,0,114,3,0,0,0,114,3,
     0,0,0,114,6,0,0,0,218,13,114,101,115,111,117,114,
     99,101,95,112,97,116,104,222,3,0,0,115,8,0,0,0,
     0,1,10,1,4,1,20,1,122,24,70,105,108,101,76,111,
@@ -1520,30 +1520,30 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,41,5,114,35,0,0,0,114,38,0,0,0,114,47,0,
     0,0,114,44,0,0,0,114,54,0,0,0,169,3,114,119,
     0,0,0,114,117,0,0,0,114,44,0,0,0,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,2,1,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,114,3,1,0,
     0,228,3,0,0,115,8,0,0,0,0,1,8,1,4,1,
     20,1,122,22,70,105,108,101,76,111,97,100,101,114,46,105,
     115,95,114,101,115,111,117,114,99,101,99,1,0,0,0,0,
     0,0,0,0,0,0,0,1,0,0,0,5,0,0,0,67,
     0,0,0,115,24,0,0,0,116,0,116,1,160,2,116,3,
     124,0,106,4,131,1,100,1,25,0,161,1,131,1,83,0,
-    114,1,1,0,0,41,5,218,4,105,116,101,114,114,2,0,
+    114,2,1,0,0,41,5,218,4,105,116,101,114,114,2,0,
     0,0,218,7,108,105,115,116,100,105,114,114,47,0,0,0,
-    114,44,0,0,0,114,246,0,0,0,114,3,0,0,0,114,
+    114,44,0,0,0,114,247,0,0,0,114,3,0,0,0,114,
     3,0,0,0,114,6,0,0,0,218,8,99,111,110,116,101,
     110,116,115,234,3,0,0,115,2,0,0,0,0,1,122,19,
     70,105,108,101,76,111,97,100,101,114,46,99,111,110,116,101,
-    110,116,115,41,17,114,125,0,0,0,114,124,0,0,0,114,
-    126,0,0,0,114,127,0,0,0,114,209,0,0,0,114,243,
-    0,0,0,114,247,0,0,0,114,136,0,0,0,114,220,0,
-    0,0,114,179,0,0,0,114,227,0,0,0,114,254,0,0,
-    0,114,0,1,0,0,114,4,1,0,0,114,2,1,0,0,
-    114,8,1,0,0,90,13,95,95,99,108,97,115,115,99,101,
-    108,108,95,95,114,3,0,0,0,114,3,0,0,0,114,249,
-    0,0,0,114,6,0,0,0,114,239,0,0,0,166,3,0,
+    110,116,115,41,17,114,126,0,0,0,114,125,0,0,0,114,
+    127,0,0,0,114,128,0,0,0,114,210,0,0,0,114,244,
+    0,0,0,114,248,0,0,0,114,137,0,0,0,114,221,0,
+    0,0,114,180,0,0,0,114,228,0,0,0,114,255,0,0,
+    0,114,1,1,0,0,114,5,1,0,0,114,3,1,0,0,
+    114,9,1,0,0,90,13,95,95,99,108,97,115,115,99,101,
+    108,108,95,95,114,3,0,0,0,114,3,0,0,0,114,250,
+    0,0,0,114,6,0,0,0,114,240,0,0,0,166,3,0,
     0,115,30,0,0,0,8,2,4,3,8,6,8,4,8,3,
     2,1,14,11,2,1,10,4,8,11,2,1,10,5,8,4,
-    8,6,8,6,114,239,0,0,0,99,0,0,0,0,0,0,
+    8,6,8,6,114,240,0,0,0,99,0,0,0,0,0,0,
     0,0,0,0,0,0,0,0,0,0,3,0,0,0,64,0,
     0,0,115,46,0,0,0,101,0,90,1,100,0,90,2,100,
     1,90,3,100,2,100,3,132,0,90,4,100,4,100,5,132,
@@ -1559,11 +1559,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     2,106,1,124,2,106,2,100,1,156,2,83,0,41,2,122,
     33,82,101,116,117,114,110,32,116,104,101,32,109,101,116,97,
     100,97,116,97,32,102,111,114,32,116,104,101,32,112,97,116,
-    104,46,41,2,114,169,0,0,0,114,234,0,0,0,41,3,
+    104,46,41,2,114,170,0,0,0,114,235,0,0,0,41,3,
     114,49,0,0,0,218,8,115,116,95,109,116,105,109,101,90,
     7,115,116,95,115,105,122,101,41,3,114,119,0,0,0,114,
-    44,0,0,0,114,238,0,0,0,114,3,0,0,0,114,3,
-    0,0,0,114,6,0,0,0,114,224,0,0,0,242,3,0,
+    44,0,0,0,114,239,0,0,0,114,3,0,0,0,114,3,
+    0,0,0,114,6,0,0,0,114,225,0,0,0,242,3,0,
     0,115,4,0,0,0,0,2,8,1,122,27,83,111,117,114,
     99,101,70,105,108,101,76,111,97,100,101,114,46,112,97,116,
     104,95,115,116,97,116,115,99,4,0,0,0,0,0,0,0,
@@ -1571,13 +1571,13 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     115,24,0,0,0,116,0,124,1,131,1,125,4,124,0,106,
     1,124,2,124,3,124,4,100,1,141,3,83,0,41,2,78,
     169,1,218,5,95,109,111,100,101,41,2,114,115,0,0,0,
-    114,225,0,0,0,41,5,114,119,0,0,0,114,108,0,0,
+    114,226,0,0,0,41,5,114,119,0,0,0,114,108,0,0,
     0,114,107,0,0,0,114,26,0,0,0,114,52,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    226,0,0,0,247,3,0,0,115,4,0,0,0,0,2,8,
+    227,0,0,0,247,3,0,0,115,4,0,0,0,0,2,8,
     1,122,32,83,111,117,114,99,101,70,105,108,101,76,111,97,
     100,101,114,46,95,99,97,99,104,101,95,98,121,116,101,99,
-    111,100,101,114,60,0,0,0,114,11,1,0,0,99,3,0,
+    111,100,101,114,60,0,0,0,114,12,1,0,0,99,3,0,
     0,0,0,0,0,0,1,0,0,0,9,0,0,0,11,0,
     0,0,67,0,0,0,115,252,0,0,0,116,0,124,1,131,
     1,92,2,125,4,125,5,103,0,125,6,124,4,114,52,116,
@@ -1600,26 +1600,26 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,102,105,108,101,46,122,27,99,111,117,108,100,32,110,111,
     116,32,99,114,101,97,116,101,32,123,33,114,125,58,32,123,
     33,114,125,78,122,12,99,114,101,97,116,101,100,32,123,33,
-    114,125,41,12,114,47,0,0,0,114,56,0,0,0,114,186,
+    114,125,41,12,114,47,0,0,0,114,56,0,0,0,114,187,
     0,0,0,114,42,0,0,0,114,38,0,0,0,114,2,0,
     0,0,90,5,109,107,100,105,114,218,15,70,105,108,101,69,
     120,105,115,116,115,69,114,114,111,114,114,50,0,0,0,114,
-    134,0,0,0,114,149,0,0,0,114,69,0,0,0,41,9,
+    135,0,0,0,114,150,0,0,0,114,69,0,0,0,41,9,
     114,119,0,0,0,114,44,0,0,0,114,26,0,0,0,114,
-    12,1,0,0,218,6,112,97,114,101,110,116,114,97,0,0,
-    0,114,37,0,0,0,114,33,0,0,0,114,228,0,0,0,
+    13,1,0,0,218,6,112,97,114,101,110,116,114,97,0,0,
+    0,114,37,0,0,0,114,33,0,0,0,114,229,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    225,0,0,0,252,3,0,0,115,48,0,0,0,0,2,12,
+    226,0,0,0,252,3,0,0,115,48,0,0,0,0,2,12,
     1,4,2,12,1,12,1,12,2,12,1,10,1,2,1,14,
     1,14,2,8,1,16,3,6,1,2,0,2,255,4,2,28,
     1,2,1,12,1,16,1,16,2,8,1,2,255,122,25,83,
     111,117,114,99,101,70,105,108,101,76,111,97,100,101,114,46,
-    115,101,116,95,100,97,116,97,78,41,7,114,125,0,0,0,
-    114,124,0,0,0,114,126,0,0,0,114,127,0,0,0,114,
-    224,0,0,0,114,226,0,0,0,114,225,0,0,0,114,3,
+    115,101,116,95,100,97,116,97,78,41,7,114,126,0,0,0,
+    114,125,0,0,0,114,127,0,0,0,114,128,0,0,0,114,
+    225,0,0,0,114,227,0,0,0,114,226,0,0,0,114,3,
     0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,114,9,1,0,0,238,3,0,0,115,8,0,0,0,
-    8,2,4,2,8,5,8,5,114,9,1,0,0,99,0,0,
+    0,0,114,10,1,0,0,238,3,0,0,115,8,0,0,0,
+    8,2,4,2,8,5,8,5,114,10,1,0,0,99,0,0,
     0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,
     0,0,64,0,0,0,115,32,0,0,0,101,0,90,1,100,
     0,90,2,100,1,90,3,100,2,100,3,132,0,90,4,100,
@@ -1634,13 +1634,13 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     161,1,125,3,124,1,124,2,100,1,156,2,125,4,116,2,
     124,3,124,1,124,4,131,3,1,0,116,3,116,4,124,3,
     131,1,100,2,100,0,133,2,25,0,124,1,124,2,100,3,
-    141,3,83,0,41,4,78,114,159,0,0,0,114,145,0,0,
+    141,3,83,0,41,4,78,114,160,0,0,0,114,146,0,0,
     0,41,2,114,117,0,0,0,114,107,0,0,0,41,5,114,
-    179,0,0,0,114,227,0,0,0,114,152,0,0,0,114,165,
-    0,0,0,114,235,0,0,0,41,5,114,119,0,0,0,114,
-    139,0,0,0,114,44,0,0,0,114,26,0,0,0,114,151,
+    180,0,0,0,114,228,0,0,0,114,153,0,0,0,114,166,
+    0,0,0,114,236,0,0,0,41,5,114,119,0,0,0,114,
+    140,0,0,0,114,44,0,0,0,114,26,0,0,0,114,152,
     0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,114,213,0,0,0,31,4,0,0,115,22,0,0,0,
+    0,0,114,214,0,0,0,31,4,0,0,115,22,0,0,0,
     0,1,10,1,10,4,2,1,2,254,6,4,12,1,2,1,
     14,1,2,1,2,253,122,29,83,111,117,114,99,101,108,101,
     115,115,70,105,108,101,76,111,97,100,101,114,46,103,101,116,
@@ -1649,16 +1649,16 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,100,1,83,0,41,2,122,39,82,101,116,117,114,
     110,32,78,111,110,101,32,97,115,32,116,104,101,114,101,32,
     105,115,32,110,111,32,115,111,117,114,99,101,32,99,111,100,
-    101,46,78,114,3,0,0,0,114,219,0,0,0,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,229,0,0,
+    101,46,78,114,3,0,0,0,114,220,0,0,0,114,3,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,114,230,0,0,
     0,47,4,0,0,115,2,0,0,0,0,2,122,31,83,111,
     117,114,99,101,108,101,115,115,70,105,108,101,76,111,97,100,
     101,114,46,103,101,116,95,115,111,117,114,99,101,78,41,6,
-    114,125,0,0,0,114,124,0,0,0,114,126,0,0,0,114,
-    127,0,0,0,114,213,0,0,0,114,229,0,0,0,114,3,
+    114,126,0,0,0,114,125,0,0,0,114,127,0,0,0,114,
+    128,0,0,0,114,214,0,0,0,114,230,0,0,0,114,3,
     0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,114,15,1,0,0,27,4,0,0,115,6,0,0,0,
-    8,2,4,2,8,16,114,15,1,0,0,99,0,0,0,0,
+    0,0,114,16,1,0,0,27,4,0,0,115,6,0,0,0,
+    8,2,4,2,8,16,114,16,1,0,0,99,0,0,0,0,
     0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,
     64,0,0,0,115,92,0,0,0,101,0,90,1,100,0,90,
     2,100,1,90,3,100,2,100,3,132,0,90,4,100,4,100,
@@ -1666,7 +1666,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     9,132,0,90,7,100,10,100,11,132,0,90,8,100,12,100,
     13,132,0,90,9,100,14,100,15,132,0,90,10,100,16,100,
     17,132,0,90,11,101,12,100,18,100,19,132,0,131,1,90,
-    13,100,20,83,0,41,21,114,252,0,0,0,122,93,76,111,
+    13,100,20,83,0,41,21,114,253,0,0,0,122,93,76,111,
     97,100,101,114,32,102,111,114,32,101,120,116,101,110,115,105,
     111,110,32,109,111,100,117,108,101,115,46,10,10,32,32,32,
     32,84,104,101,32,99,111,110,115,116,114,117,99,116,111,114,
@@ -1675,25 +1675,25 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     110,100,101,114,46,10,10,32,32,32,32,99,3,0,0,0,
     0,0,0,0,0,0,0,0,3,0,0,0,2,0,0,0,
     67,0,0,0,115,16,0,0,0,124,1,124,0,95,0,124,
-    2,124,0,95,1,100,0,83,0,114,110,0,0,0,114,159,
-    0,0,0,114,5,1,0,0,114,3,0,0,0,114,3,0,
-    0,0,114,6,0,0,0,114,209,0,0,0,64,4,0,0,
+    2,124,0,95,1,100,0,83,0,114,110,0,0,0,114,160,
+    0,0,0,114,6,1,0,0,114,3,0,0,0,114,3,0,
+    0,0,114,6,0,0,0,114,210,0,0,0,64,4,0,0,
     115,4,0,0,0,0,1,6,1,122,28,69,120,116,101,110,
     115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,95,
     95,105,110,105,116,95,95,99,2,0,0,0,0,0,0,0,
     0,0,0,0,2,0,0,0,2,0,0,0,67,0,0,0,
     115,24,0,0,0,124,0,106,0,124,1,106,0,107,2,111,
     22,124,0,106,1,124,1,106,1,107,2,83,0,114,110,0,
-    0,0,114,240,0,0,0,114,242,0,0,0,114,3,0,0,
-    0,114,3,0,0,0,114,6,0,0,0,114,243,0,0,0,
+    0,0,114,241,0,0,0,114,243,0,0,0,114,3,0,0,
+    0,114,3,0,0,0,114,6,0,0,0,114,244,0,0,0,
     68,4,0,0,115,6,0,0,0,0,1,12,1,10,255,122,
     26,69,120,116,101,110,115,105,111,110,70,105,108,101,76,111,
     97,100,101,114,46,95,95,101,113,95,95,99,1,0,0,0,
     0,0,0,0,0,0,0,0,1,0,0,0,3,0,0,0,
     67,0,0,0,115,20,0,0,0,116,0,124,0,106,1,131,
     1,116,0,124,0,106,2,131,1,65,0,83,0,114,110,0,
-    0,0,114,244,0,0,0,114,246,0,0,0,114,3,0,0,
-    0,114,3,0,0,0,114,6,0,0,0,114,247,0,0,0,
+    0,0,114,245,0,0,0,114,247,0,0,0,114,3,0,0,
+    0,114,3,0,0,0,114,6,0,0,0,114,248,0,0,0,
     72,4,0,0,115,2,0,0,0,0,1,122,28,69,120,116,
     101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114,
     46,95,95,104,97,115,104,95,95,99,2,0,0,0,0,0,
@@ -1706,11 +1706,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     111,100,117,108,101,122,38,101,120,116,101,110,115,105,111,110,
     32,109,111,100,117,108,101,32,123,33,114,125,32,108,111,97,
     100,101,100,32,102,114,111,109,32,123,33,114,125,41,7,114,
-    134,0,0,0,114,214,0,0,0,114,163,0,0,0,90,14,
-    99,114,101,97,116,101,95,100,121,110,97,109,105,99,114,149,
+    135,0,0,0,114,215,0,0,0,114,164,0,0,0,90,14,
+    99,114,101,97,116,101,95,100,121,110,97,109,105,99,114,150,
     0,0,0,114,117,0,0,0,114,44,0,0,0,41,3,114,
-    119,0,0,0,114,187,0,0,0,114,216,0,0,0,114,3,
-    0,0,0,114,3,0,0,0,114,6,0,0,0,114,212,0,
+    119,0,0,0,114,188,0,0,0,114,217,0,0,0,114,3,
+    0,0,0,114,3,0,0,0,114,6,0,0,0,114,213,0,
     0,0,75,4,0,0,115,18,0,0,0,0,2,4,1,4,
     0,2,255,4,2,6,1,4,0,4,255,4,2,122,33,69,
     120,116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,
@@ -1724,11 +1724,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     100,117,108,101,122,40,101,120,116,101,110,115,105,111,110,32,
     109,111,100,117,108,101,32,123,33,114,125,32,101,120,101,99,
     117,116,101,100,32,102,114,111,109,32,123,33,114,125,78,41,
-    7,114,134,0,0,0,114,214,0,0,0,114,163,0,0,0,
-    90,12,101,120,101,99,95,100,121,110,97,109,105,99,114,149,
-    0,0,0,114,117,0,0,0,114,44,0,0,0,114,253,0,
+    7,114,135,0,0,0,114,215,0,0,0,114,164,0,0,0,
+    90,12,101,120,101,99,95,100,121,110,97,109,105,99,114,150,
+    0,0,0,114,117,0,0,0,114,44,0,0,0,114,254,0,
     0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
-    0,114,217,0,0,0,83,4,0,0,115,10,0,0,0,0,
+    0,114,218,0,0,0,83,4,0,0,115,10,0,0,0,0,
     2,14,1,6,1,4,0,4,255,122,31,69,120,116,101,110,
     115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,101,
     120,101,99,95,109,111,100,117,108,101,99,2,0,0,0,0,
@@ -1743,7 +1743,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,2,0,0,0,4,0,0,0,51,0,0,0,115,
     26,0,0,0,124,0,93,18,125,1,136,0,100,0,124,1,
     23,0,107,2,86,0,1,0,113,2,100,1,83,0,41,2,
-    114,209,0,0,0,78,114,3,0,0,0,169,2,114,32,0,
+    114,210,0,0,0,78,114,3,0,0,0,169,2,114,32,0,
     0,0,218,6,115,117,102,102,105,120,169,1,90,9,102,105,
     108,101,95,110,97,109,101,114,3,0,0,0,114,6,0,0,
     0,218,9,60,103,101,110,101,120,112,114,62,92,4,0,0,
@@ -1753,8 +1753,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     115,62,46,60,103,101,110,101,120,112,114,62,41,4,114,47,
     0,0,0,114,44,0,0,0,218,3,97,110,121,218,18,69,
     88,84,69,78,83,73,79,78,95,83,85,70,70,73,88,69,
-    83,114,219,0,0,0,114,3,0,0,0,114,18,1,0,0,
-    114,6,0,0,0,114,182,0,0,0,89,4,0,0,115,8,
+    83,114,220,0,0,0,114,3,0,0,0,114,19,1,0,0,
+    114,6,0,0,0,114,183,0,0,0,89,4,0,0,115,8,
     0,0,0,0,2,14,1,12,1,2,255,122,30,69,120,116,
     101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114,
     46,105,115,95,112,97,99,107,97,103,101,99,2,0,0,0,
@@ -1764,8 +1764,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     97,110,32,101,120,116,101,110,115,105,111,110,32,109,111,100,
     117,108,101,32,99,97,110,110,111,116,32,99,114,101,97,116,
     101,32,97,32,99,111,100,101,32,111,98,106,101,99,116,46,
-    78,114,3,0,0,0,114,219,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,213,0,0,0,95,
+    78,114,3,0,0,0,114,220,0,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,214,0,0,0,95,
     4,0,0,115,2,0,0,0,0,2,122,28,69,120,116,101,
     110,115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,
     103,101,116,95,99,111,100,101,99,2,0,0,0,0,0,0,
@@ -1774,27 +1774,27 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     116,117,114,110,32,78,111,110,101,32,97,115,32,101,120,116,
     101,110,115,105,111,110,32,109,111,100,117,108,101,115,32,104,
     97,118,101,32,110,111,32,115,111,117,114,99,101,32,99,111,
-    100,101,46,78,114,3,0,0,0,114,219,0,0,0,114,3,
-    0,0,0,114,3,0,0,0,114,6,0,0,0,114,229,0,
+    100,101,46,78,114,3,0,0,0,114,220,0,0,0,114,3,
+    0,0,0,114,3,0,0,0,114,6,0,0,0,114,230,0,
     0,0,99,4,0,0,115,2,0,0,0,0,2,122,30,69,
     120,116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,
     101,114,46,103,101,116,95,115,111,117,114,99,101,99,2,0,
     0,0,0,0,0,0,0,0,0,0,2,0,0,0,1,0,
     0,0,67,0,0,0,115,6,0,0,0,124,0,106,0,83,
-    0,114,250,0,0,0,114,48,0,0,0,114,219,0,0,0,
+    0,114,251,0,0,0,114,48,0,0,0,114,220,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    179,0,0,0,103,4,0,0,115,2,0,0,0,0,3,122,
+    180,0,0,0,103,4,0,0,115,2,0,0,0,0,3,122,
     32,69,120,116,101,110,115,105,111,110,70,105,108,101,76,111,
     97,100,101,114,46,103,101,116,95,102,105,108,101,110,97,109,
-    101,78,41,14,114,125,0,0,0,114,124,0,0,0,114,126,
-    0,0,0,114,127,0,0,0,114,209,0,0,0,114,243,0,
-    0,0,114,247,0,0,0,114,212,0,0,0,114,217,0,0,
-    0,114,182,0,0,0,114,213,0,0,0,114,229,0,0,0,
-    114,136,0,0,0,114,179,0,0,0,114,3,0,0,0,114,
-    3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,252,
+    101,78,41,14,114,126,0,0,0,114,125,0,0,0,114,127,
+    0,0,0,114,128,0,0,0,114,210,0,0,0,114,244,0,
+    0,0,114,248,0,0,0,114,213,0,0,0,114,218,0,0,
+    0,114,183,0,0,0,114,214,0,0,0,114,230,0,0,0,
+    114,137,0,0,0,114,180,0,0,0,114,3,0,0,0,114,
+    3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,253,
     0,0,0,56,4,0,0,115,22,0,0,0,8,2,4,6,
     8,4,8,4,8,3,8,8,8,6,8,6,8,4,8,4,
-    2,1,114,252,0,0,0,99,0,0,0,0,0,0,0,0,
+    2,1,114,253,0,0,0,99,0,0,0,0,0,0,0,0,
     0,0,0,0,0,0,0,0,2,0,0,0,64,0,0,0,
     115,104,0,0,0,101,0,90,1,100,0,90,2,100,1,90,
     3,100,2,100,3,132,0,90,4,100,4,100,5,132,0,90,
@@ -1834,7 +1834,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     116,104,95,102,105,110,100,101,114,169,4,114,119,0,0,0,
     114,117,0,0,0,114,44,0,0,0,90,11,112,97,116,104,
     95,102,105,110,100,101,114,114,3,0,0,0,114,3,0,0,
-    0,114,6,0,0,0,114,209,0,0,0,116,4,0,0,115,
+    0,114,6,0,0,0,114,210,0,0,0,116,4,0,0,115,
     8,0,0,0,0,1,6,1,6,1,14,1,122,23,95,78,
     97,109,101,115,112,97,99,101,80,97,116,104,46,95,95,105,
     110,105,116,95,95,99,1,0,0,0,0,0,0,0,0,0,
@@ -1847,8 +1847,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     97,109,101,44,32,112,97,114,101,110,116,45,112,97,116,104,
     45,97,116,116,114,45,110,97,109,101,41,114,71,0,0,0,
     114,40,0,0,0,41,2,114,8,0,0,0,114,44,0,0,
-    0,90,8,95,95,112,97,116,104,95,95,41,2,114,23,1,
-    0,0,114,41,0,0,0,41,4,114,119,0,0,0,114,14,
+    0,90,8,95,95,112,97,116,104,95,95,41,2,114,24,1,
+    0,0,114,41,0,0,0,41,4,114,119,0,0,0,114,15,
     1,0,0,218,3,100,111,116,90,2,109,101,114,3,0,0,
     0,114,3,0,0,0,114,6,0,0,0,218,23,95,102,105,
     110,100,95,112,97,114,101,110,116,95,112,97,116,104,95,110,
@@ -1859,13 +1859,13 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,0,0,0,0,0,0,0,3,0,0,0,3,0,
     0,0,67,0,0,0,115,28,0,0,0,124,0,160,0,161,
     0,92,2,125,1,125,2,116,1,116,2,106,3,124,1,25,
-    0,124,2,131,2,83,0,114,110,0,0,0,41,4,114,30,
-    1,0,0,114,130,0,0,0,114,8,0,0,0,218,7,109,
+    0,124,2,131,2,83,0,114,110,0,0,0,41,4,114,31,
+    1,0,0,114,131,0,0,0,114,8,0,0,0,218,7,109,
     111,100,117,108,101,115,41,3,114,119,0,0,0,90,18,112,
     97,114,101,110,116,95,109,111,100,117,108,101,95,110,97,109,
     101,90,14,112,97,116,104,95,97,116,116,114,95,110,97,109,
     101,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
-    114,25,1,0,0,132,4,0,0,115,4,0,0,0,0,1,
+    114,26,1,0,0,132,4,0,0,115,4,0,0,0,0,1,
     12,1,122,31,95,78,97,109,101,115,112,97,99,101,80,97,
     116,104,46,95,103,101,116,95,112,97,114,101,110,116,95,112,
     97,116,104,99,1,0,0,0,0,0,0,0,0,0,0,0,
@@ -1875,11 +1875,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     1,161,2,125,2,124,2,100,0,107,9,114,68,124,2,106,
     5,100,0,107,8,114,68,124,2,106,6,114,68,124,2,106,
     6,124,0,95,7,124,1,124,0,95,2,124,0,106,7,83,
-    0,114,110,0,0,0,41,8,114,112,0,0,0,114,25,1,
-    0,0,114,26,1,0,0,114,27,1,0,0,114,23,1,0,
-    0,114,140,0,0,0,114,178,0,0,0,114,24,1,0,0,
+    0,114,110,0,0,0,41,8,114,112,0,0,0,114,26,1,
+    0,0,114,27,1,0,0,114,28,1,0,0,114,24,1,0,
+    0,114,141,0,0,0,114,179,0,0,0,114,25,1,0,0,
     41,3,114,119,0,0,0,90,11,112,97,114,101,110,116,95,
-    112,97,116,104,114,187,0,0,0,114,3,0,0,0,114,3,
+    112,97,116,104,114,188,0,0,0,114,3,0,0,0,114,3,
     0,0,0,114,6,0,0,0,218,12,95,114,101,99,97,108,
     99,117,108,97,116,101,136,4,0,0,115,16,0,0,0,0,
     2,12,1,10,1,14,3,18,1,6,1,8,1,6,1,122,
@@ -1887,15 +1887,15 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     95,114,101,99,97,108,99,117,108,97,116,101,99,1,0,0,
     0,0,0,0,0,0,0,0,0,1,0,0,0,3,0,0,
     0,67,0,0,0,115,12,0,0,0,116,0,124,0,160,1,
-    161,0,131,1,83,0,114,110,0,0,0,41,2,114,6,1,
-    0,0,114,32,1,0,0,114,246,0,0,0,114,3,0,0,
+    161,0,131,1,83,0,114,110,0,0,0,41,2,114,7,1,
+    0,0,114,33,1,0,0,114,247,0,0,0,114,3,0,0,
     0,114,3,0,0,0,114,6,0,0,0,218,8,95,95,105,
     116,101,114,95,95,149,4,0,0,115,2,0,0,0,0,1,
     122,23,95,78,97,109,101,115,112,97,99,101,80,97,116,104,
     46,95,95,105,116,101,114,95,95,99,2,0,0,0,0,0,
     0,0,0,0,0,0,2,0,0,0,2,0,0,0,67,0,
     0,0,115,12,0,0,0,124,0,160,0,161,0,124,1,25,
-    0,83,0,114,110,0,0,0,169,1,114,32,1,0,0,41,
+    0,83,0,114,110,0,0,0,169,1,114,33,1,0,0,41,
     2,114,119,0,0,0,218,5,105,110,100,101,120,114,3,0,
     0,0,114,3,0,0,0,114,6,0,0,0,218,11,95,95,
     103,101,116,105,116,101,109,95,95,152,4,0,0,115,2,0,
@@ -1904,8 +1904,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     99,3,0,0,0,0,0,0,0,0,0,0,0,3,0,0,
     0,3,0,0,0,67,0,0,0,115,14,0,0,0,124,2,
     124,0,106,0,124,1,60,0,100,0,83,0,114,110,0,0,
-    0,41,1,114,24,1,0,0,41,3,114,119,0,0,0,114,
-    35,1,0,0,114,44,0,0,0,114,3,0,0,0,114,3,
+    0,41,1,114,25,1,0,0,41,3,114,119,0,0,0,114,
+    36,1,0,0,114,44,0,0,0,114,3,0,0,0,114,3,
     0,0,0,114,6,0,0,0,218,11,95,95,115,101,116,105,
     116,101,109,95,95,155,4,0,0,115,2,0,0,0,0,1,
     122,26,95,78,97,109,101,115,112,97,99,101,80,97,116,104,
@@ -1913,7 +1913,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,0,0,0,0,0,0,1,0,0,0,3,0,0,
     0,67,0,0,0,115,12,0,0,0,116,0,124,0,160,1,
     161,0,131,1,83,0,114,110,0,0,0,41,2,114,22,0,
-    0,0,114,32,1,0,0,114,246,0,0,0,114,3,0,0,
+    0,0,114,33,1,0,0,114,247,0,0,0,114,3,0,0,
     0,114,3,0,0,0,114,6,0,0,0,218,7,95,95,108,
     101,110,95,95,158,4,0,0,115,2,0,0,0,0,1,122,
     22,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46,
@@ -1922,14 +1922,14 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     115,12,0,0,0,100,1,160,0,124,0,106,1,161,1,83,
     0,41,2,78,122,20,95,78,97,109,101,115,112,97,99,101,
     80,97,116,104,40,123,33,114,125,41,41,2,114,62,0,0,
-    0,114,24,1,0,0,114,246,0,0,0,114,3,0,0,0,
+    0,114,25,1,0,0,114,247,0,0,0,114,3,0,0,0,
     114,3,0,0,0,114,6,0,0,0,218,8,95,95,114,101,
     112,114,95,95,161,4,0,0,115,2,0,0,0,0,1,122,
     23,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46,
     95,95,114,101,112,114,95,95,99,2,0,0,0,0,0,0,
     0,0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,
     0,115,12,0,0,0,124,1,124,0,160,0,161,0,107,6,
-    83,0,114,110,0,0,0,114,34,1,0,0,169,2,114,119,
+    83,0,114,110,0,0,0,114,35,1,0,0,169,2,114,119,
     0,0,0,218,4,105,116,101,109,114,3,0,0,0,114,3,
     0,0,0,114,6,0,0,0,218,12,95,95,99,111,110,116,
     97,105,110,115,95,95,164,4,0,0,115,2,0,0,0,0,
@@ -1938,20 +1938,20 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,3,
     0,0,0,67,0,0,0,115,16,0,0,0,124,0,106,0,
     160,1,124,1,161,1,1,0,100,0,83,0,114,110,0,0,
-    0,41,2,114,24,1,0,0,114,186,0,0,0,114,40,1,
+    0,41,2,114,25,1,0,0,114,187,0,0,0,114,41,1,
     0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
-    0,114,186,0,0,0,167,4,0,0,115,2,0,0,0,0,
+    0,114,187,0,0,0,167,4,0,0,115,2,0,0,0,0,
     1,122,21,95,78,97,109,101,115,112,97,99,101,80,97,116,
-    104,46,97,112,112,101,110,100,78,41,15,114,125,0,0,0,
-    114,124,0,0,0,114,126,0,0,0,114,127,0,0,0,114,
-    209,0,0,0,114,30,1,0,0,114,25,1,0,0,114,32,
-    1,0,0,114,33,1,0,0,114,36,1,0,0,114,37,1,
-    0,0,114,38,1,0,0,114,39,1,0,0,114,42,1,0,
-    0,114,186,0,0,0,114,3,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,22,1,0,0,109,
+    104,46,97,112,112,101,110,100,78,41,15,114,126,0,0,0,
+    114,125,0,0,0,114,127,0,0,0,114,128,0,0,0,114,
+    210,0,0,0,114,31,1,0,0,114,26,1,0,0,114,33,
+    1,0,0,114,34,1,0,0,114,37,1,0,0,114,38,1,
+    0,0,114,39,1,0,0,114,40,1,0,0,114,43,1,0,
+    0,114,187,0,0,0,114,3,0,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,23,1,0,0,109,
     4,0,0,115,24,0,0,0,8,1,4,6,8,6,8,10,
     8,4,8,13,8,3,8,3,8,3,8,3,8,3,8,3,
-    114,22,1,0,0,99,0,0,0,0,0,0,0,0,0,0,
+    114,23,1,0,0,99,0,0,0,0,0,0,0,0,0,0,
     0,0,0,0,0,0,3,0,0,0,64,0,0,0,115,80,
     0,0,0,101,0,90,1,100,0,90,2,100,1,100,2,132,
     0,90,3,101,4,100,3,100,4,132,0,131,1,90,5,100,
@@ -1962,9 +1962,9 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     101,76,111,97,100,101,114,99,4,0,0,0,0,0,0,0,
     0,0,0,0,4,0,0,0,4,0,0,0,67,0,0,0,
     115,18,0,0,0,116,0,124,1,124,2,124,3,131,3,124,
-    0,95,1,100,0,83,0,114,110,0,0,0,41,2,114,22,
-    1,0,0,114,24,1,0,0,114,28,1,0,0,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,209,0,0,
+    0,95,1,100,0,83,0,114,110,0,0,0,41,2,114,23,
+    1,0,0,114,25,1,0,0,114,29,1,0,0,114,3,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,114,210,0,0,
     0,173,4,0,0,115,2,0,0,0,0,1,122,25,95,78,
     97,109,101,115,112,97,99,101,76,111,97,100,101,114,46,95,
     95,105,110,105,116,95,95,99,2,0,0,0,0,0,0,0,
@@ -1979,47 +1979,47 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     104,101,32,106,111,98,32,105,116,115,101,108,102,46,10,10,
     32,32,32,32,32,32,32,32,122,25,60,109,111,100,117,108,
     101,32,123,33,114,125,32,40,110,97,109,101,115,112,97,99,
-    101,41,62,41,2,114,62,0,0,0,114,125,0,0,0,41,
-    2,114,193,0,0,0,114,216,0,0,0,114,3,0,0,0,
+    101,41,62,41,2,114,62,0,0,0,114,126,0,0,0,41,
+    2,114,194,0,0,0,114,217,0,0,0,114,3,0,0,0,
     114,3,0,0,0,114,6,0,0,0,218,11,109,111,100,117,
     108,101,95,114,101,112,114,176,4,0,0,115,2,0,0,0,
     0,7,122,28,95,78,97,109,101,115,112,97,99,101,76,111,
     97,100,101,114,46,109,111,100,117,108,101,95,114,101,112,114,
     99,2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,
     0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1,
-    83,0,41,2,78,84,114,3,0,0,0,114,219,0,0,0,
+    83,0,41,2,78,84,114,3,0,0,0,114,220,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    182,0,0,0,185,4,0,0,115,2,0,0,0,0,1,122,
+    183,0,0,0,185,4,0,0,115,2,0,0,0,0,1,122,
     27,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101,
     114,46,105,115,95,112,97,99,107,97,103,101,99,2,0,0,
     0,0,0,0,0,0,0,0,0,2,0,0,0,1,0,0,
     0,67,0,0,0,115,4,0,0,0,100,1,83,0,41,2,
-    78,114,40,0,0,0,114,3,0,0,0,114,219,0,0,0,
+    78,114,40,0,0,0,114,3,0,0,0,114,220,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    229,0,0,0,188,4,0,0,115,2,0,0,0,0,1,122,
+    230,0,0,0,188,4,0,0,115,2,0,0,0,0,1,122,
     27,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101,
     114,46,103,101,116,95,115,111,117,114,99,101,99,2,0,0,
     0,0,0,0,0,0,0,0,0,2,0,0,0,6,0,0,
     0,67,0,0,0,115,16,0,0,0,116,0,100,1,100,2,
     100,3,100,4,100,5,141,4,83,0,41,6,78,114,40,0,
-    0,0,122,8,60,115,116,114,105,110,103,62,114,215,0,0,
-    0,84,41,1,114,231,0,0,0,41,1,114,232,0,0,0,
-    114,219,0,0,0,114,3,0,0,0,114,3,0,0,0,114,
-    6,0,0,0,114,213,0,0,0,191,4,0,0,115,2,0,
+    0,0,122,8,60,115,116,114,105,110,103,62,114,216,0,0,
+    0,84,41,1,114,232,0,0,0,41,1,114,233,0,0,0,
+    114,220,0,0,0,114,3,0,0,0,114,3,0,0,0,114,
+    6,0,0,0,114,214,0,0,0,191,4,0,0,115,2,0,
     0,0,0,1,122,25,95,78,97,109,101,115,112,97,99,101,
     76,111,97,100,101,114,46,103,101,116,95,99,111,100,101,99,
     2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,
     1,0,0,0,67,0,0,0,115,4,0,0,0,100,1,83,
-    0,114,210,0,0,0,114,3,0,0,0,114,211,0,0,0,
+    0,114,211,0,0,0,114,3,0,0,0,114,212,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    212,0,0,0,194,4,0,0,115,2,0,0,0,0,1,122,
+    213,0,0,0,194,4,0,0,115,2,0,0,0,0,1,122,
     30,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101,
     114,46,99,114,101,97,116,101,95,109,111,100,117,108,101,99,
     2,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,
     1,0,0,0,67,0,0,0,115,4,0,0,0,100,0,83,
-    0,114,110,0,0,0,114,3,0,0,0,114,253,0,0,0,
+    0,114,110,0,0,0,114,3,0,0,0,114,254,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    217,0,0,0,197,4,0,0,115,2,0,0,0,0,1,122,
+    218,0,0,0,197,4,0,0,115,2,0,0,0,0,1,122,
     28,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101,
     114,46,101,120,101,99,95,109,111,100,117,108,101,99,2,0,
     0,0,0,0,0,0,0,0,0,0,2,0,0,0,4,0,
@@ -2034,215 +2034,209 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     46,10,10,32,32,32,32,32,32,32,32,122,38,110,97,109,
     101,115,112,97,99,101,32,109,111,100,117,108,101,32,108,111,
     97,100,101,100,32,119,105,116,104,32,112,97,116,104,32,123,
-    33,114,125,41,4,114,134,0,0,0,114,149,0,0,0,114,
-    24,1,0,0,114,218,0,0,0,114,219,0,0,0,114,3,
-    0,0,0,114,3,0,0,0,114,6,0,0,0,114,220,0,
+    33,114,125,41,4,114,135,0,0,0,114,150,0,0,0,114,
+    25,1,0,0,114,219,0,0,0,114,220,0,0,0,114,3,
+    0,0,0,114,3,0,0,0,114,6,0,0,0,114,221,0,
     0,0,200,4,0,0,115,8,0,0,0,0,7,6,1,4,
     255,4,2,122,28,95,78,97,109,101,115,112,97,99,101,76,
     111,97,100,101,114,46,108,111,97,100,95,109,111,100,117,108,
-    101,78,41,12,114,125,0,0,0,114,124,0,0,0,114,126,
-    0,0,0,114,209,0,0,0,114,207,0,0,0,114,44,1,
-    0,0,114,182,0,0,0,114,229,0,0,0,114,213,0,0,
-    0,114,212,0,0,0,114,217,0,0,0,114,220,0,0,0,
+    101,78,41,12,114,126,0,0,0,114,125,0,0,0,114,127,
+    0,0,0,114,210,0,0,0,114,208,0,0,0,114,45,1,
+    0,0,114,183,0,0,0,114,230,0,0,0,114,214,0,0,
+    0,114,213,0,0,0,114,218,0,0,0,114,221,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,3,0,0,0,114,
-    6,0,0,0,114,43,1,0,0,172,4,0,0,115,18,0,
+    6,0,0,0,114,44,1,0,0,172,4,0,0,115,18,0,
     0,0,8,1,8,3,2,1,10,8,8,3,8,3,8,3,
-    8,3,8,3,114,43,1,0,0,99,0,0,0,0,0,0,
+    8,3,8,3,114,44,1,0,0,99,0,0,0,0,0,0,
     0,0,0,0,0,0,0,0,0,0,4,0,0,0,64,0,
-    0,0,115,180,0,0,0,101,0,90,1,100,0,90,2,100,
+    0,0,115,118,0,0,0,101,0,90,1,100,0,90,2,100,
     1,90,3,101,4,100,2,100,3,132,0,131,1,90,5,101,
     4,100,4,100,5,132,0,131,1,90,6,101,4,100,6,100,
     7,132,0,131,1,90,7,101,4,100,8,100,9,132,0,131,
-    1,90,8,101,4,100,29,100,11,100,12,132,1,131,1,90,
-    9,101,4,100,30,100,13,100,14,132,1,131,1,90,10,101,
-    4,100,31,100,15,100,16,132,1,131,1,90,11,101,4,100,
-    32,100,17,100,18,132,1,131,1,90,12,101,4,100,19,100,
-    20,132,0,131,1,90,13,101,14,100,21,100,22,132,0,131,
-    1,90,15,101,4,100,23,100,24,132,0,131,1,90,16,101,
-    4,100,25,100,26,132,0,131,1,90,17,101,4,100,27,100,
-    28,132,0,131,1,90,18,100,10,83,0,41,33,218,10,80,
-    97,116,104,70,105,110,100,101,114,122,62,77,101,116,97,32,
-    112,97,116,104,32,102,105,110,100,101,114,32,102,111,114,32,
-    115,121,115,46,112,97,116,104,32,97,110,100,32,112,97,99,
-    107,97,103,101,32,95,95,112,97,116,104,95,95,32,97,116,
-    116,114,105,98,117,116,101,115,46,99,1,0,0,0,0,0,
-    0,0,0,0,0,0,3,0,0,0,4,0,0,0,67,0,
-    0,0,115,64,0,0,0,116,0,116,1,106,2,160,3,161,
-    0,131,1,68,0,93,44,92,2,125,1,125,2,124,2,100,
-    1,107,8,114,40,116,1,106,2,124,1,61,0,113,14,116,
-    4,124,2,100,2,131,2,114,14,124,2,160,5,161,0,1,
-    0,113,14,100,1,83,0,41,3,122,125,67,97,108,108,32,
-    116,104,101,32,105,110,118,97,108,105,100,97,116,101,95,99,
-    97,99,104,101,115,40,41,32,109,101,116,104,111,100,32,111,
-    110,32,97,108,108,32,112,97,116,104,32,101,110,116,114,121,
-    32,102,105,110,100,101,114,115,10,32,32,32,32,32,32,32,
-    32,115,116,111,114,101,100,32,105,110,32,115,121,115,46,112,
-    97,116,104,95,105,109,112,111,114,116,101,114,95,99,97,99,
-    104,101,115,32,40,119,104,101,114,101,32,105,109,112,108,101,
-    109,101,110,116,101,100,41,46,78,218,17,105,110,118,97,108,
-    105,100,97,116,101,95,99,97,99,104,101,115,41,6,218,4,
-    108,105,115,116,114,8,0,0,0,218,19,112,97,116,104,95,
-    105,109,112,111,114,116,101,114,95,99,97,99,104,101,218,5,
-    105,116,101,109,115,114,128,0,0,0,114,46,1,0,0,41,
-    3,114,193,0,0,0,114,117,0,0,0,218,6,102,105,110,
-    100,101,114,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,114,46,1,0,0,218,4,0,0,115,10,0,0,0,
-    0,4,22,1,8,1,10,1,10,1,122,28,80,97,116,104,
-    70,105,110,100,101,114,46,105,110,118,97,108,105,100,97,116,
-    101,95,99,97,99,104,101,115,99,2,0,0,0,0,0,0,
-    0,0,0,0,0,3,0,0,0,9,0,0,0,67,0,0,
-    0,115,84,0,0,0,116,0,106,1,100,1,107,9,114,28,
-    116,0,106,1,115,28,116,2,160,3,100,2,116,4,161,2,
-    1,0,116,0,106,1,68,0,93,44,125,2,122,14,124,2,
-    124,1,131,1,87,0,2,0,1,0,83,0,4,0,116,5,
-    107,10,114,76,1,0,1,0,1,0,89,0,113,34,89,0,
-    113,34,88,0,113,34,100,1,83,0,41,3,122,46,83,101,
-    97,114,99,104,32,115,121,115,46,112,97,116,104,95,104,111,
-    111,107,115,32,102,111,114,32,97,32,102,105,110,100,101,114,
-    32,102,111,114,32,39,112,97,116,104,39,46,78,122,23,115,
-    121,115,46,112,97,116,104,95,104,111,111,107,115,32,105,115,
-    32,101,109,112,116,121,41,6,114,8,0,0,0,218,10,112,
-    97,116,104,95,104,111,111,107,115,114,75,0,0,0,114,76,
-    0,0,0,114,138,0,0,0,114,118,0,0,0,41,3,114,
-    193,0,0,0,114,44,0,0,0,90,4,104,111,111,107,114,
-    3,0,0,0,114,3,0,0,0,114,6,0,0,0,218,11,
-    95,112,97,116,104,95,104,111,111,107,115,228,4,0,0,115,
-    16,0,0,0,0,3,16,1,12,1,10,1,2,1,14,1,
-    14,1,12,2,122,22,80,97,116,104,70,105,110,100,101,114,
-    46,95,112,97,116,104,95,104,111,111,107,115,99,2,0,0,
-    0,0,0,0,0,0,0,0,0,3,0,0,0,8,0,0,
-    0,67,0,0,0,115,104,0,0,0,124,1,100,1,107,2,
-    114,44,122,12,116,0,160,1,161,0,125,1,87,0,110,22,
-    4,0,116,2,107,10,114,42,1,0,1,0,1,0,89,0,
-    100,2,83,0,88,0,122,14,116,3,106,4,124,1,25,0,
-    125,2,87,0,110,40,4,0,116,5,107,10,114,98,1,0,
-    1,0,1,0,124,0,160,6,124,1,161,1,125,2,124,2,
-    116,3,106,4,124,1,60,0,89,0,110,2,88,0,124,2,
-    83,0,41,3,122,210,71,101,116,32,116,104,101,32,102,105,
-    110,100,101,114,32,102,111,114,32,116,104,101,32,112,97,116,
-    104,32,101,110,116,114,121,32,102,114,111,109,32,115,121,115,
+    1,90,8,101,4,100,19,100,11,100,12,132,1,131,1,90,
+    9,101,4,100,20,100,13,100,14,132,1,131,1,90,10,101,
+    4,100,21,100,15,100,16,132,1,131,1,90,11,101,4,100,
+    17,100,18,132,0,131,1,90,12,100,10,83,0,41,22,218,
+    10,80,97,116,104,70,105,110,100,101,114,122,62,77,101,116,
+    97,32,112,97,116,104,32,102,105,110,100,101,114,32,102,111,
+    114,32,115,121,115,46,112,97,116,104,32,97,110,100,32,112,
+    97,99,107,97,103,101,32,95,95,112,97,116,104,95,95,32,
+    97,116,116,114,105,98,117,116,101,115,46,99,1,0,0,0,
+    0,0,0,0,0,0,0,0,3,0,0,0,4,0,0,0,
+    67,0,0,0,115,64,0,0,0,116,0,116,1,106,2,160,
+    3,161,0,131,1,68,0,93,44,92,2,125,1,125,2,124,
+    2,100,1,107,8,114,40,116,1,106,2,124,1,61,0,113,
+    14,116,4,124,2,100,2,131,2,114,14,124,2,160,5,161,
+    0,1,0,113,14,100,1,83,0,41,3,122,125,67,97,108,
+    108,32,116,104,101,32,105,110,118,97,108,105,100,97,116,101,
+    95,99,97,99,104,101,115,40,41,32,109,101,116,104,111,100,
+    32,111,110,32,97,108,108,32,112,97,116,104,32,101,110,116,
+    114,121,32,102,105,110,100,101,114,115,10,32,32,32,32,32,
+    32,32,32,115,116,111,114,101,100,32,105,110,32,115,121,115,
+    46,112,97,116,104,95,105,109,112,111,114,116,101,114,95,99,
+    97,99,104,101,115,32,40,119,104,101,114,101,32,105,109,112,
+    108,101,109,101,110,116,101,100,41,46,78,218,17,105,110,118,
+    97,108,105,100,97,116,101,95,99,97,99,104,101,115,41,6,
+    218,4,108,105,115,116,114,8,0,0,0,218,19,112,97,116,
+    104,95,105,109,112,111,114,116,101,114,95,99,97,99,104,101,
+    218,5,105,116,101,109,115,114,129,0,0,0,114,47,1,0,
+    0,41,3,114,194,0,0,0,114,117,0,0,0,218,6,102,
+    105,110,100,101,114,114,3,0,0,0,114,3,0,0,0,114,
+    6,0,0,0,114,47,1,0,0,218,4,0,0,115,10,0,
+    0,0,0,4,22,1,8,1,10,1,10,1,122,28,80,97,
+    116,104,70,105,110,100,101,114,46,105,110,118,97,108,105,100,
+    97,116,101,95,99,97,99,104,101,115,99,2,0,0,0,0,
+    0,0,0,0,0,0,0,3,0,0,0,9,0,0,0,67,
+    0,0,0,115,84,0,0,0,116,0,106,1,100,1,107,9,
+    114,28,116,0,106,1,115,28,116,2,160,3,100,2,116,4,
+    161,2,1,0,116,0,106,1,68,0,93,44,125,2,122,14,
+    124,2,124,1,131,1,87,0,2,0,1,0,83,0,4,0,
+    116,5,107,10,114,76,1,0,1,0,1,0,89,0,113,34,
+    89,0,113,34,88,0,113,34,100,1,83,0,41,3,122,46,
+    83,101,97,114,99,104,32,115,121,115,46,112,97,116,104,95,
+    104,111,111,107,115,32,102,111,114,32,97,32,102,105,110,100,
+    101,114,32,102,111,114,32,39,112,97,116,104,39,46,78,122,
+    23,115,121,115,46,112,97,116,104,95,104,111,111,107,115,32,
+    105,115,32,101,109,112,116,121,41,6,114,8,0,0,0,218,
+    10,112,97,116,104,95,104,111,111,107,115,114,75,0,0,0,
+    114,76,0,0,0,114,139,0,0,0,114,118,0,0,0,41,
+    3,114,194,0,0,0,114,44,0,0,0,90,4,104,111,111,
+    107,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
+    218,11,95,112,97,116,104,95,104,111,111,107,115,228,4,0,
+    0,115,16,0,0,0,0,3,16,1,12,1,10,1,2,1,
+    14,1,14,1,12,2,122,22,80,97,116,104,70,105,110,100,
+    101,114,46,95,112,97,116,104,95,104,111,111,107,115,99,2,
+    0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,8,
+    0,0,0,67,0,0,0,115,104,0,0,0,124,1,100,1,
+    107,2,114,44,122,12,116,0,160,1,161,0,125,1,87,0,
+    110,22,4,0,116,2,107,10,114,42,1,0,1,0,1,0,
+    89,0,100,2,83,0,88,0,122,14,116,3,106,4,124,1,
+    25,0,125,2,87,0,110,40,4,0,116,5,107,10,114,98,
+    1,0,1,0,1,0,124,0,160,6,124,1,161,1,125,2,
+    124,2,116,3,106,4,124,1,60,0,89,0,110,2,88,0,
+    124,2,83,0,41,3,122,210,71,101,116,32,116,104,101,32,
+    102,105,110,100,101,114,32,102,111,114,32,116,104,101,32,112,
+    97,116,104,32,101,110,116,114,121,32,102,114,111,109,32,115,
+    121,115,46,112,97,116,104,95,105,109,112,111,114,116,101,114,
+    95,99,97,99,104,101,46,10,10,32,32,32,32,32,32,32,
+    32,73,102,32,116,104,101,32,112,97,116,104,32,101,110,116,
+    114,121,32,105,115,32,110,111,116,32,105,110,32,116,104,101,
+    32,99,97,99,104,101,44,32,102,105,110,100,32,116,104,101,
+    32,97,112,112,114,111,112,114,105,97,116,101,32,102,105,110,
+    100,101,114,10,32,32,32,32,32,32,32,32,97,110,100,32,
+    99,97,99,104,101,32,105,116,46,32,73,102,32,110,111,32,
+    102,105,110,100,101,114,32,105,115,32,97,118,97,105,108,97,
+    98,108,101,44,32,115,116,111,114,101,32,78,111,110,101,46,
+    10,10,32,32,32,32,32,32,32,32,114,40,0,0,0,78,
+    41,7,114,2,0,0,0,114,55,0,0,0,114,4,1,0,
+    0,114,8,0,0,0,114,49,1,0,0,218,8,75,101,121,
+    69,114,114,111,114,114,53,1,0,0,41,3,114,194,0,0,
+    0,114,44,0,0,0,114,51,1,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,218,20,95,112,97,116,
+    104,95,105,109,112,111,114,116,101,114,95,99,97,99,104,101,
+    241,4,0,0,115,22,0,0,0,0,8,8,1,2,1,12,
+    1,14,3,8,1,2,1,14,1,14,1,10,1,16,1,122,
+    31,80,97,116,104,70,105,110,100,101,114,46,95,112,97,116,
+    104,95,105,109,112,111,114,116,101,114,95,99,97,99,104,101,
+    99,3,0,0,0,0,0,0,0,0,0,0,0,6,0,0,
+    0,4,0,0,0,67,0,0,0,115,82,0,0,0,116,0,
+    124,2,100,1,131,2,114,26,124,2,160,1,124,1,161,1,
+    92,2,125,3,125,4,110,14,124,2,160,2,124,1,161,1,
+    125,3,103,0,125,4,124,3,100,0,107,9,114,60,116,3,
+    160,4,124,1,124,3,161,2,83,0,116,3,160,5,124,1,
+    100,0,161,2,125,5,124,4,124,5,95,6,124,5,83,0,
+    41,2,78,114,138,0,0,0,41,7,114,129,0,0,0,114,
+    138,0,0,0,114,207,0,0,0,114,135,0,0,0,114,202,
+    0,0,0,114,184,0,0,0,114,179,0,0,0,41,6,114,
+    194,0,0,0,114,140,0,0,0,114,51,1,0,0,114,141,
+    0,0,0,114,142,0,0,0,114,188,0,0,0,114,3,0,
+    0,0,114,3,0,0,0,114,6,0,0,0,218,16,95,108,
+    101,103,97,99,121,95,103,101,116,95,115,112,101,99,7,5,
+    0,0,115,18,0,0,0,0,4,10,1,16,2,10,1,4,
+    1,8,1,12,1,12,1,6,1,122,27,80,97,116,104,70,
+    105,110,100,101,114,46,95,108,101,103,97,99,121,95,103,101,
+    116,95,115,112,101,99,78,99,4,0,0,0,0,0,0,0,
+    0,0,0,0,9,0,0,0,5,0,0,0,67,0,0,0,
+    115,166,0,0,0,103,0,125,4,124,2,68,0,93,134,125,
+    5,116,0,124,5,116,1,116,2,102,2,131,2,115,28,113,
+    8,124,0,160,3,124,5,161,1,125,6,124,6,100,1,107,
+    9,114,8,116,4,124,6,100,2,131,2,114,70,124,6,160,
+    5,124,1,124,3,161,2,125,7,110,12,124,0,160,6,124,
+    1,124,6,161,2,125,7,124,7,100,1,107,8,114,92,113,
+    8,124,7,106,7,100,1,107,9,114,110,124,7,2,0,1,
+    0,83,0,124,7,106,8,125,8,124,8,100,1,107,8,114,
+    132,116,9,100,3,131,1,130,1,124,4,160,10,124,8,161,
+    1,1,0,113,8,116,11,160,12,124,1,100,1,161,2,125,
+    7,124,4,124,7,95,8,124,7,83,0,41,4,122,63,70,
+    105,110,100,32,116,104,101,32,108,111,97,100,101,114,32,111,
+    114,32,110,97,109,101,115,112,97,99,101,95,112,97,116,104,
+    32,102,111,114,32,116,104,105,115,32,109,111,100,117,108,101,
+    47,112,97,99,107,97,103,101,32,110,97,109,101,46,78,114,
+    204,0,0,0,122,19,115,112,101,99,32,109,105,115,115,105,
+    110,103,32,108,111,97,100,101,114,41,13,114,162,0,0,0,
+    114,85,0,0,0,218,5,98,121,116,101,115,114,55,1,0,
+    0,114,129,0,0,0,114,204,0,0,0,114,56,1,0,0,
+    114,141,0,0,0,114,179,0,0,0,114,118,0,0,0,114,
+    168,0,0,0,114,135,0,0,0,114,184,0,0,0,41,9,
+    114,194,0,0,0,114,140,0,0,0,114,44,0,0,0,114,
+    203,0,0,0,218,14,110,97,109,101,115,112,97,99,101,95,
+    112,97,116,104,90,5,101,110,116,114,121,114,51,1,0,0,
+    114,188,0,0,0,114,142,0,0,0,114,3,0,0,0,114,
+    3,0,0,0,114,6,0,0,0,218,9,95,103,101,116,95,
+    115,112,101,99,22,5,0,0,115,40,0,0,0,0,5,4,
+    1,8,1,14,1,2,1,10,1,8,1,10,1,14,2,12,
+    1,8,1,2,1,10,1,8,1,6,1,8,1,8,5,12,
+    2,12,1,6,1,122,20,80,97,116,104,70,105,110,100,101,
+    114,46,95,103,101,116,95,115,112,101,99,99,4,0,0,0,
+    0,0,0,0,0,0,0,0,6,0,0,0,5,0,0,0,
+    67,0,0,0,115,100,0,0,0,124,2,100,1,107,8,114,
+    14,116,0,106,1,125,2,124,0,160,2,124,1,124,2,124,
+    3,161,3,125,4,124,4,100,1,107,8,114,40,100,1,83,
+    0,124,4,106,3,100,1,107,8,114,92,124,4,106,4,125,
+    5,124,5,114,86,100,1,124,4,95,5,116,6,124,1,124,
+    5,124,0,106,2,131,3,124,4,95,4,124,4,83,0,100,
+    1,83,0,110,4,124,4,83,0,100,1,83,0,41,2,122,
+    141,84,114,121,32,116,111,32,102,105,110,100,32,97,32,115,
+    112,101,99,32,102,111,114,32,39,102,117,108,108,110,97,109,
+    101,39,32,111,110,32,115,121,115,46,112,97,116,104,32,111,
+    114,32,39,112,97,116,104,39,46,10,10,32,32,32,32,32,
+    32,32,32,84,104,101,32,115,101,97,114,99,104,32,105,115,
+    32,98,97,115,101,100,32,111,110,32,115,121,115,46,112,97,
+    116,104,95,104,111,111,107,115,32,97,110,100,32,115,121,115,
     46,112,97,116,104,95,105,109,112,111,114,116,101,114,95,99,
-    97,99,104,101,46,10,10,32,32,32,32,32,32,32,32,73,
-    102,32,116,104,101,32,112,97,116,104,32,101,110,116,114,121,
-    32,105,115,32,110,111,116,32,105,110,32,116,104,101,32,99,
-    97,99,104,101,44,32,102,105,110,100,32,116,104,101,32,97,
-    112,112,114,111,112,114,105,97,116,101,32,102,105,110,100,101,
-    114,10,32,32,32,32,32,32,32,32,97,110,100,32,99,97,
-    99,104,101,32,105,116,46,32,73,102,32,110,111,32,102,105,
-    110,100,101,114,32,105,115,32,97,118,97,105,108,97,98,108,
-    101,44,32,115,116,111,114,101,32,78,111,110,101,46,10,10,
-    32,32,32,32,32,32,32,32,114,40,0,0,0,78,41,7,
-    114,2,0,0,0,114,55,0,0,0,114,3,1,0,0,114,
-    8,0,0,0,114,48,1,0,0,218,8,75,101,121,69,114,
-    114,111,114,114,52,1,0,0,41,3,114,193,0,0,0,114,
-    44,0,0,0,114,50,1,0,0,114,3,0,0,0,114,3,
-    0,0,0,114,6,0,0,0,218,20,95,112,97,116,104,95,
-    105,109,112,111,114,116,101,114,95,99,97,99,104,101,241,4,
-    0,0,115,22,0,0,0,0,8,8,1,2,1,12,1,14,
-    3,8,1,2,1,14,1,14,1,10,1,16,1,122,31,80,
-    97,116,104,70,105,110,100,101,114,46,95,112,97,116,104,95,
-    105,109,112,111,114,116,101,114,95,99,97,99,104,101,99,3,
-    0,0,0,0,0,0,0,0,0,0,0,6,0,0,0,4,
-    0,0,0,67,0,0,0,115,82,0,0,0,116,0,124,2,
-    100,1,131,2,114,26,124,2,160,1,124,1,161,1,92,2,
-    125,3,125,4,110,14,124,2,160,2,124,1,161,1,125,3,
-    103,0,125,4,124,3,100,0,107,9,114,60,116,3,160,4,
-    124,1,124,3,161,2,83,0,116,3,160,5,124,1,100,0,
-    161,2,125,5,124,4,124,5,95,6,124,5,83,0,41,2,
-    78,114,137,0,0,0,41,7,114,128,0,0,0,114,137,0,
-    0,0,114,206,0,0,0,114,134,0,0,0,114,201,0,0,
-    0,114,183,0,0,0,114,178,0,0,0,41,6,114,193,0,
-    0,0,114,139,0,0,0,114,50,1,0,0,114,140,0,0,
-    0,114,141,0,0,0,114,187,0,0,0,114,3,0,0,0,
-    114,3,0,0,0,114,6,0,0,0,218,16,95,108,101,103,
-    97,99,121,95,103,101,116,95,115,112,101,99,7,5,0,0,
-    115,18,0,0,0,0,4,10,1,16,2,10,1,4,1,8,
-    1,12,1,12,1,6,1,122,27,80,97,116,104,70,105,110,
-    100,101,114,46,95,108,101,103,97,99,121,95,103,101,116,95,
-    115,112,101,99,78,99,4,0,0,0,0,0,0,0,0,0,
-    0,0,9,0,0,0,5,0,0,0,67,0,0,0,115,166,
-    0,0,0,103,0,125,4,124,2,68,0,93,134,125,5,116,
-    0,124,5,116,1,116,2,102,2,131,2,115,28,113,8,124,
-    0,160,3,124,5,161,1,125,6,124,6,100,1,107,9,114,
-    8,116,4,124,6,100,2,131,2,114,70,124,6,160,5,124,
-    1,124,3,161,2,125,7,110,12,124,0,160,6,124,1,124,
-    6,161,2,125,7,124,7,100,1,107,8,114,92,113,8,124,
-    7,106,7,100,1,107,9,114,110,124,7,2,0,1,0,83,
-    0,124,7,106,8,125,8,124,8,100,1,107,8,114,132,116,
-    9,100,3,131,1,130,1,124,4,160,10,124,8,161,1,1,
-    0,113,8,116,11,160,12,124,1,100,1,161,2,125,7,124,
-    4,124,7,95,8,124,7,83,0,41,4,122,63,70,105,110,
-    100,32,116,104,101,32,108,111,97,100,101,114,32,111,114,32,
-    110,97,109,101,115,112,97,99,101,95,112,97,116,104,32,102,
-    111,114,32,116,104,105,115,32,109,111,100,117,108,101,47,112,
-    97,99,107,97,103,101,32,110,97,109,101,46,78,114,203,0,
-    0,0,122,19,115,112,101,99,32,109,105,115,115,105,110,103,
-    32,108,111,97,100,101,114,41,13,114,161,0,0,0,114,85,
-    0,0,0,218,5,98,121,116,101,115,114,54,1,0,0,114,
-    128,0,0,0,114,203,0,0,0,114,55,1,0,0,114,140,
-    0,0,0,114,178,0,0,0,114,118,0,0,0,114,167,0,
-    0,0,114,134,0,0,0,114,183,0,0,0,41,9,114,193,
-    0,0,0,114,139,0,0,0,114,44,0,0,0,114,202,0,
-    0,0,218,14,110,97,109,101,115,112,97,99,101,95,112,97,
-    116,104,90,5,101,110,116,114,121,114,50,1,0,0,114,187,
-    0,0,0,114,141,0,0,0,114,3,0,0,0,114,3,0,
-    0,0,114,6,0,0,0,218,9,95,103,101,116,95,115,112,
-    101,99,22,5,0,0,115,40,0,0,0,0,5,4,1,8,
-    1,14,1,2,1,10,1,8,1,10,1,14,2,12,1,8,
-    1,2,1,10,1,8,1,6,1,8,1,8,5,12,2,12,
-    1,6,1,122,20,80,97,116,104,70,105,110,100,101,114,46,
-    95,103,101,116,95,115,112,101,99,99,4,0,0,0,0,0,
-    0,0,0,0,0,0,6,0,0,0,5,0,0,0,67,0,
-    0,0,115,100,0,0,0,124,2,100,1,107,8,114,14,116,
-    0,106,1,125,2,124,0,160,2,124,1,124,2,124,3,161,
-    3,125,4,124,4,100,1,107,8,114,40,100,1,83,0,124,
-    4,106,3,100,1,107,8,114,92,124,4,106,4,125,5,124,
-    5,114,86,100,1,124,4,95,5,116,6,124,1,124,5,124,
-    0,106,2,131,3,124,4,95,4,124,4,83,0,100,1,83,
-    0,110,4,124,4,83,0,100,1,83,0,41,2,122,141,84,
-    114,121,32,116,111,32,102,105,110,100,32,97,32,115,112,101,
-    99,32,102,111,114,32,39,102,117,108,108,110,97,109,101,39,
+    97,99,104,101,46,10,32,32,32,32,32,32,32,32,78,41,
+    7,114,8,0,0,0,114,44,0,0,0,114,59,1,0,0,
+    114,141,0,0,0,114,179,0,0,0,114,182,0,0,0,114,
+    23,1,0,0,41,6,114,194,0,0,0,114,140,0,0,0,
+    114,44,0,0,0,114,203,0,0,0,114,188,0,0,0,114,
+    58,1,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
+    0,0,0,114,204,0,0,0,54,5,0,0,115,26,0,0,
+    0,0,6,8,1,6,1,14,1,8,1,4,1,10,1,6,
+    1,4,3,6,1,16,1,4,2,6,2,122,20,80,97,116,
+    104,70,105,110,100,101,114,46,102,105,110,100,95,115,112,101,
+    99,99,3,0,0,0,0,0,0,0,0,0,0,0,4,0,
+    0,0,4,0,0,0,67,0,0,0,115,30,0,0,0,124,
+    0,160,0,124,1,124,2,161,2,125,3,124,3,100,1,107,
+    8,114,24,100,1,83,0,124,3,106,1,83,0,41,2,122,
+    170,102,105,110,100,32,116,104,101,32,109,111,100,117,108,101,
     32,111,110,32,115,121,115,46,112,97,116,104,32,111,114,32,
-    39,112,97,116,104,39,46,10,10,32,32,32,32,32,32,32,
-    32,84,104,101,32,115,101,97,114,99,104,32,105,115,32,98,
-    97,115,101,100,32,111,110,32,115,121,115,46,112,97,116,104,
-    95,104,111,111,107,115,32,97,110,100,32,115,121,115,46,112,
+    39,112,97,116,104,39,32,98,97,115,101,100,32,111,110,32,
+    115,121,115,46,112,97,116,104,95,104,111,111,107,115,32,97,
+    110,100,10,32,32,32,32,32,32,32,32,115,121,115,46,112,
     97,116,104,95,105,109,112,111,114,116,101,114,95,99,97,99,
-    104,101,46,10,32,32,32,32,32,32,32,32,78,41,7,114,
-    8,0,0,0,114,44,0,0,0,114,58,1,0,0,114,140,
-    0,0,0,114,178,0,0,0,114,181,0,0,0,114,22,1,
-    0,0,41,6,114,193,0,0,0,114,139,0,0,0,114,44,
-    0,0,0,114,202,0,0,0,114,187,0,0,0,114,57,1,
-    0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
-    0,114,203,0,0,0,54,5,0,0,115,26,0,0,0,0,
-    6,8,1,6,1,14,1,8,1,4,1,10,1,6,1,4,
-    3,6,1,16,1,4,2,6,2,122,20,80,97,116,104,70,
-    105,110,100,101,114,46,102,105,110,100,95,115,112,101,99,99,
-    3,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,
-    4,0,0,0,67,0,0,0,115,30,0,0,0,124,0,160,
-    0,124,1,124,2,161,2,125,3,124,3,100,1,107,8,114,
-    24,100,1,83,0,124,3,106,1,83,0,41,2,122,170,102,
-    105,110,100,32,116,104,101,32,109,111,100,117,108,101,32,111,
-    110,32,115,121,115,46,112,97,116,104,32,111,114,32,39,112,
-    97,116,104,39,32,98,97,115,101,100,32,111,110,32,115,121,
-    115,46,112,97,116,104,95,104,111,111,107,115,32,97,110,100,
-    10,32,32,32,32,32,32,32,32,115,121,115,46,112,97,116,
-    104,95,105,109,112,111,114,116,101,114,95,99,97,99,104,101,
-    46,10,10,32,32,32,32,32,32,32,32,84,104,105,115,32,
-    109,101,116,104,111,100,32,105,115,32,100,101,112,114,101,99,
-    97,116,101,100,46,32,32,85,115,101,32,102,105,110,100,95,
-    115,112,101,99,40,41,32,105,110,115,116,101,97,100,46,10,
-    10,32,32,32,32,32,32,32,32,78,114,204,0,0,0,114,
-    205,0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
-    0,0,0,114,206,0,0,0,78,5,0,0,115,8,0,0,
-    0,0,8,12,1,8,1,4,1,122,22,80,97,116,104,70,
-    105,110,100,101,114,46,102,105,110,100,95,109,111,100,117,108,
-    101,99,2,0,0,0,0,0,0,0,0,0,0,0,5,0,
-    0,0,4,0,0,0,67,0,0,0,115,58,0,0,0,100,
-    1,100,2,108,0,109,1,125,2,109,2,125,3,1,0,124,
-    1,100,3,107,8,114,32,124,3,160,3,161,0,125,1,124,
-    0,160,4,124,1,106,5,124,1,106,6,161,2,125,4,116,
-    7,124,2,124,4,131,2,83,0,41,4,97,32,1,0,0,
+    104,101,46,10,10,32,32,32,32,32,32,32,32,84,104,105,
+    115,32,109,101,116,104,111,100,32,105,115,32,100,101,112,114,
+    101,99,97,116,101,100,46,32,32,85,115,101,32,102,105,110,
+    100,95,115,112,101,99,40,41,32,105,110,115,116,101,97,100,
+    46,10,10,32,32,32,32,32,32,32,32,78,114,205,0,0,
+    0,114,206,0,0,0,114,3,0,0,0,114,3,0,0,0,
+    114,6,0,0,0,114,207,0,0,0,78,5,0,0,115,8,
+    0,0,0,0,8,12,1,8,1,4,1,122,22,80,97,116,
+    104,70,105,110,100,101,114,46,102,105,110,100,95,109,111,100,
+    117,108,101,99,1,0,0,0,0,0,0,0,0,0,0,0,
+    4,0,0,0,3,0,0,0,79,0,0,0,115,24,0,0,
+    0,100,1,100,2,108,0,109,1,125,3,1,0,124,3,106,
+    2,124,1,124,2,142,1,83,0,41,3,97,32,1,0,0,
     10,32,32,32,32,32,32,32,32,70,105,110,100,32,100,105,
     115,116,114,105,98,117,116,105,111,110,115,46,10,10,32,32,
     32,32,32,32,32,32,82,101,116,117,114,110,32,97,110,32,
@@ -2261,593 +2255,480 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,32,32,32,32,32,32,111,102,32,100,105,114,101,99,116,
     111,114,105,101,115,32,96,96,99,111,110,116,101,120,116,46,
     112,97,116,104,96,96,46,10,32,32,32,32,32,32,32,32,
-    114,73,0,0,0,41,2,218,16,80,97,116,104,68,105,115,
-    116,114,105,98,117,116,105,111,110,218,18,68,105,115,116,114,
-    105,98,117,116,105,111,110,70,105,110,100,101,114,78,41,8,
-    90,18,105,109,112,111,114,116,108,105,98,46,109,101,116,97,
-    100,97,116,97,114,59,1,0,0,114,60,1,0,0,90,7,
-    67,111,110,116,101,120,116,218,13,95,115,101,97,114,99,104,
-    95,112,97,116,104,115,218,7,112,97,116,116,101,114,110,114,
-    44,0,0,0,218,3,109,97,112,41,5,114,119,0,0,0,
-    90,7,99,111,110,116,101,120,116,114,59,1,0,0,114,60,
-    1,0,0,90,5,102,111,117,110,100,114,3,0,0,0,114,
-    3,0,0,0,114,6,0,0,0,218,18,102,105,110,100,95,
-    100,105,115,116,114,105,98,117,116,105,111,110,115,91,5,0,
-    0,115,10,0,0,0,0,10,16,1,8,1,8,1,16,1,
-    122,29,80,97,116,104,70,105,110,100,101,114,46,102,105,110,
-    100,95,100,105,115,116,114,105,98,117,116,105,111,110,115,99,
-    3,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,
-    6,0,0,0,3,0,0,0,115,44,0,0,0,100,1,100,
-    2,108,0,125,3,124,3,106,1,160,2,135,0,135,1,102,
-    2,100,3,100,4,132,8,116,3,136,0,106,4,124,2,131,
-    2,68,0,131,1,161,1,83,0,41,5,122,49,70,105,110,
-    100,32,109,101,116,97,100,97,116,97,32,100,105,114,101,99,
-    116,111,114,105,101,115,32,105,110,32,112,97,116,104,115,32,
-    104,101,117,114,105,115,116,105,99,97,108,108,121,46,114,73,
-    0,0,0,78,99,1,0,0,0,0,0,0,0,0,0,0,
-    0,2,0,0,0,5,0,0,0,51,0,0,0,115,26,0,
-    0,0,124,0,93,18,125,1,136,0,160,0,124,1,136,1,
-    161,2,86,0,1,0,113,2,100,0,83,0,114,110,0,0,
-    0,41,1,218,12,95,115,101,97,114,99,104,95,112,97,116,
-    104,41,2,114,32,0,0,0,114,44,0,0,0,169,2,114,
-    193,0,0,0,114,62,1,0,0,114,3,0,0,0,114,6,
-    0,0,0,114,19,1,0,0,111,5,0,0,115,4,0,0,
-    0,4,2,2,255,122,43,80,97,116,104,70,105,110,100,101,
-    114,46,95,115,101,97,114,99,104,95,112,97,116,104,115,46,
-    60,108,111,99,97,108,115,62,46,60,103,101,110,101,120,112,
-    114,62,41,5,218,9,105,116,101,114,116,111,111,108,115,90,
-    5,99,104,97,105,110,90,13,102,114,111,109,95,105,116,101,
-    114,97,98,108,101,114,63,1,0,0,218,12,95,115,119,105,
-    116,99,104,95,112,97,116,104,41,4,114,193,0,0,0,114,
-    62,1,0,0,90,5,112,97,116,104,115,114,67,1,0,0,
-    114,3,0,0,0,114,66,1,0,0,114,6,0,0,0,114,
-    61,1,0,0,107,5,0,0,115,8,0,0,0,0,3,8,
-    1,18,2,10,254,122,24,80,97,116,104,70,105,110,100,101,
-    114,46,95,115,101,97,114,99,104,95,112,97,116,104,115,99,
-    1,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,
-    10,0,0,0,67,0,0,0,115,96,0,0,0,100,1,100,
-    2,108,0,109,1,125,1,1,0,100,1,100,0,108,2,125,
-    2,100,1,100,0,108,3,125,3,100,3,125,4,124,4,114,
-    48,116,4,106,5,160,6,124,0,161,1,114,86,124,1,116,
-    7,131,1,143,24,1,0,124,2,160,8,124,0,161,1,87,
-    0,2,0,53,0,81,0,82,0,163,0,83,0,81,0,82,
-    0,88,0,124,3,160,8,124,0,161,1,83,0,41,4,78,
-    114,73,0,0,0,41,1,218,8,115,117,112,112,114,101,115,
-    115,70,41,9,90,10,99,111,110,116,101,120,116,108,105,98,
-    114,69,1,0,0,218,7,122,105,112,102,105,108,101,218,7,
-    112,97,116,104,108,105,98,90,2,111,115,114,44,0,0,0,
-    90,6,105,115,102,105,108,101,218,9,69,120,99,101,112,116,
-    105,111,110,90,4,80,97,116,104,41,5,114,44,0,0,0,
-    114,69,1,0,0,114,70,1,0,0,114,71,1,0,0,90,
-    13,80,89,80,89,95,79,80,69,78,95,66,85,71,114,3,
-    0,0,0,114,3,0,0,0,114,6,0,0,0,114,68,1,
-    0,0,116,5,0,0,115,16,0,0,0,0,2,12,1,8,
-    1,8,1,4,1,16,1,10,1,28,1,122,23,80,97,116,
-    104,70,105,110,100,101,114,46,95,115,119,105,116,99,104,95,
-    112,97,116,104,99,3,0,0,0,0,0,0,0,0,0,0,
-    0,6,0,0,0,5,0,0,0,67,0,0,0,115,44,0,
-    0,0,100,1,100,0,108,0,125,3,100,2,125,4,124,4,
-    106,1,124,1,100,3,141,1,125,5,124,3,106,2,124,5,
-    124,2,106,3,124,3,106,4,100,4,141,3,83,0,41,5,
-    78,114,73,0,0,0,122,32,123,112,97,116,116,101,114,110,
-    125,40,45,46,42,41,63,92,46,40,100,105,115,116,124,101,
-    103,103,41,45,105,110,102,111,169,1,114,62,1,0,0,169,
-    1,114,83,0,0,0,41,5,218,2,114,101,114,62,0,0,
-    0,90,5,109,97,116,99,104,114,117,0,0,0,218,10,73,
-    71,78,79,82,69,67,65,83,69,169,6,114,193,0,0,0,
-    218,10,110,111,114,109,97,108,105,122,101,100,114,41,1,0,
-    0,114,75,1,0,0,90,8,116,101,109,112,108,97,116,101,
-    90,8,109,97,110,105,102,101,115,116,114,3,0,0,0,114,
-    3,0,0,0,114,6,0,0,0,218,13,95,109,97,116,99,
-    104,101,115,95,105,110,102,111,127,5,0,0,115,8,0,0,
-    0,0,2,8,1,4,1,12,1,122,24,80,97,116,104,70,
-    105,110,100,101,114,46,95,109,97,116,99,104,101,115,95,105,
-    110,102,111,99,3,0,0,0,0,0,0,0,0,0,0,0,
-    6,0,0,0,5,0,0,0,67,0,0,0,115,46,0,0,
-    0,100,1,100,0,108,0,125,3,100,2,125,4,124,4,106,
-    1,124,1,100,3,141,1,125,5,124,3,106,2,124,5,116,
-    3,124,2,131,1,124,3,106,4,100,4,141,3,83,0,41,
-    5,78,114,73,0,0,0,122,30,123,112,97,116,116,101,114,
-    110,125,45,46,42,92,46,101,103,103,91,92,92,47,93,69,
-    71,71,45,73,78,70,79,114,73,1,0,0,114,74,1,0,
-    0,41,5,114,75,1,0,0,114,62,0,0,0,90,6,115,
-    101,97,114,99,104,114,85,0,0,0,114,76,1,0,0,114,
-    77,1,0,0,114,3,0,0,0,114,3,0,0,0,114,6,
-    0,0,0,218,15,95,109,97,116,99,104,101,115,95,108,101,
-    103,97,99,121,134,5,0,0,115,8,0,0,0,0,2,8,
-    1,4,1,12,1,122,26,80,97,116,104,70,105,110,100,101,
-    114,46,95,109,97,116,99,104,101,115,95,108,101,103,97,99,
-    121,99,3,0,0,0,0,0,0,0,0,0,0,0,3,0,
-    0,0,4,0,0,0,3,0,0,0,115,48,0,0,0,124,
-    1,160,0,161,0,115,12,100,1,83,0,124,2,160,1,100,
-    2,100,3,161,2,137,1,135,0,135,1,102,2,100,4,100,
-    5,132,8,124,1,160,2,161,0,68,0,131,1,83,0,41,
-    6,78,114,3,0,0,0,250,1,45,114,45,0,0,0,99,
-    1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,
-    5,0,0,0,51,0,0,0,115,42,0,0,0,124,0,93,
-    34,125,1,136,0,160,0,136,1,124,1,161,2,115,30,136,
-    0,160,1,136,1,124,1,161,2,114,2,124,1,86,0,1,
-    0,113,2,100,0,83,0,114,110,0,0,0,41,2,114,79,
-    1,0,0,114,80,1,0,0,41,2,114,32,0,0,0,114,
-    41,1,0,0,169,2,114,193,0,0,0,114,78,1,0,0,
-    114,3,0,0,0,114,6,0,0,0,114,19,1,0,0,146,
-    5,0,0,115,8,0,0,0,4,0,2,1,12,1,12,254,
-    122,42,80,97,116,104,70,105,110,100,101,114,46,95,115,101,
-    97,114,99,104,95,112,97,116,104,46,60,108,111,99,97,108,
-    115,62,46,60,103,101,110,101,120,112,114,62,41,3,90,6,
-    105,115,95,100,105,114,114,67,0,0,0,90,7,105,116,101,
-    114,100,105,114,41,3,114,193,0,0,0,90,4,114,111,111,
-    116,114,62,1,0,0,114,3,0,0,0,114,82,1,0,0,
-    114,6,0,0,0,114,65,1,0,0,141,5,0,0,115,8,
-    0,0,0,0,2,8,1,4,1,12,1,122,23,80,97,116,
-    104,70,105,110,100,101,114,46,95,115,101,97,114,99,104,95,
-    112,97,116,104,41,1,78,41,2,78,78,41,1,78,41,1,
-    78,41,19,114,125,0,0,0,114,124,0,0,0,114,126,0,
-    0,0,114,127,0,0,0,114,207,0,0,0,114,46,1,0,
-    0,114,52,1,0,0,114,54,1,0,0,114,55,1,0,0,
-    114,58,1,0,0,114,203,0,0,0,114,206,0,0,0,114,
-    64,1,0,0,114,61,1,0,0,218,12,115,116,97,116,105,
-    99,109,101,116,104,111,100,114,68,1,0,0,114,79,1,0,
-    0,114,80,1,0,0,114,65,1,0,0,114,3,0,0,0,
+    114,73,0,0,0,41,1,218,18,77,101,116,97,100,97,116,
+    97,80,97,116,104,70,105,110,100,101,114,41,3,90,18,105,
+    109,112,111,114,116,108,105,98,46,109,101,116,97,100,97,116,
+    97,114,60,1,0,0,218,18,102,105,110,100,95,100,105,115,
+    116,114,105,98,117,116,105,111,110,115,41,4,114,194,0,0,
+    0,114,120,0,0,0,114,121,0,0,0,114,60,1,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    45,1,0,0,214,4,0,0,115,54,0,0,0,8,2,4,
-    2,2,1,10,9,2,1,10,12,2,1,10,21,2,1,10,
-    14,2,1,12,31,2,1,12,23,2,1,12,12,2,1,12,
-    15,2,1,10,8,2,1,10,10,2,1,10,6,2,1,10,
-    6,2,1,114,45,1,0,0,99,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,3,0,0,0,64,0,0,
-    0,115,90,0,0,0,101,0,90,1,100,0,90,2,100,1,
-    90,3,100,2,100,3,132,0,90,4,100,4,100,5,132,0,
-    90,5,101,6,90,7,100,6,100,7,132,0,90,8,100,8,
-    100,9,132,0,90,9,100,19,100,11,100,12,132,1,90,10,
-    100,13,100,14,132,0,90,11,101,12,100,15,100,16,132,0,
-    131,1,90,13,100,17,100,18,132,0,90,14,100,10,83,0,
-    41,20,218,10,70,105,108,101,70,105,110,100,101,114,122,172,
-    70,105,108,101,45,98,97,115,101,100,32,102,105,110,100,101,
-    114,46,10,10,32,32,32,32,73,110,116,101,114,97,99,116,
-    105,111,110,115,32,119,105,116,104,32,116,104,101,32,102,105,
-    108,101,32,115,121,115,116,101,109,32,97,114,101,32,99,97,
-    99,104,101,100,32,102,111,114,32,112,101,114,102,111,114,109,
-    97,110,99,101,44,32,98,101,105,110,103,10,32,32,32,32,
-    114,101,102,114,101,115,104,101,100,32,119,104,101,110,32,116,
-    104,101,32,100,105,114,101,99,116,111,114,121,32,116,104,101,
-    32,102,105,110,100,101,114,32,105,115,32,104,97,110,100,108,
-    105,110,103,32,104,97,115,32,98,101,101,110,32,109,111,100,
-    105,102,105,101,100,46,10,10,32,32,32,32,99,2,0,0,
-    0,0,0,0,0,0,0,0,0,5,0,0,0,6,0,0,
-    0,7,0,0,0,115,84,0,0,0,103,0,125,3,124,2,
-    68,0,93,32,92,2,137,0,125,4,124,3,160,0,135,0,
-    102,1,100,1,100,2,132,8,124,4,68,0,131,1,161,1,
-    1,0,113,8,124,3,124,0,95,1,124,1,112,54,100,3,
-    124,0,95,2,100,4,124,0,95,3,116,4,131,0,124,0,
-    95,5,116,4,131,0,124,0,95,6,100,5,83,0,41,6,
-    122,154,73,110,105,116,105,97,108,105,122,101,32,119,105,116,
-    104,32,116,104,101,32,112,97,116,104,32,116,111,32,115,101,
-    97,114,99,104,32,111,110,32,97,110,100,32,97,32,118,97,
-    114,105,97,98,108,101,32,110,117,109,98,101,114,32,111,102,
-    10,32,32,32,32,32,32,32,32,50,45,116,117,112,108,101,
-    115,32,99,111,110,116,97,105,110,105,110,103,32,116,104,101,
-    32,108,111,97,100,101,114,32,97,110,100,32,116,104,101,32,
-    102,105,108,101,32,115,117,102,102,105,120,101,115,32,116,104,
-    101,32,108,111,97,100,101,114,10,32,32,32,32,32,32,32,
-    32,114,101,99,111,103,110,105,122,101,115,46,99,1,0,0,
-    0,0,0,0,0,0,0,0,0,2,0,0,0,3,0,0,
-    0,51,0,0,0,115,22,0,0,0,124,0,93,14,125,1,
-    124,1,136,0,102,2,86,0,1,0,113,2,100,0,83,0,
-    114,110,0,0,0,114,3,0,0,0,114,16,1,0,0,169,
-    1,114,140,0,0,0,114,3,0,0,0,114,6,0,0,0,
-    114,19,1,0,0,166,5,0,0,115,4,0,0,0,4,0,
-    2,0,122,38,70,105,108,101,70,105,110,100,101,114,46,95,
-    95,105,110,105,116,95,95,46,60,108,111,99,97,108,115,62,
-    46,60,103,101,110,101,120,112,114,62,114,71,0,0,0,114,
-    105,0,0,0,78,41,7,114,167,0,0,0,218,8,95,108,
-    111,97,100,101,114,115,114,44,0,0,0,218,11,95,112,97,
-    116,104,95,109,116,105,109,101,218,3,115,101,116,218,11,95,
-    112,97,116,104,95,99,97,99,104,101,218,19,95,114,101,108,
-    97,120,101,100,95,112,97,116,104,95,99,97,99,104,101,41,
-    5,114,119,0,0,0,114,44,0,0,0,218,14,108,111,97,
-    100,101,114,95,100,101,116,97,105,108,115,90,7,108,111,97,
-    100,101,114,115,114,189,0,0,0,114,3,0,0,0,114,85,
-    1,0,0,114,6,0,0,0,114,209,0,0,0,160,5,0,
-    0,115,16,0,0,0,0,4,4,1,12,1,26,1,6,2,
-    10,1,6,1,8,1,122,19,70,105,108,101,70,105,110,100,
-    101,114,46,95,95,105,110,105,116,95,95,99,1,0,0,0,
-    0,0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,
-    67,0,0,0,115,10,0,0,0,100,1,124,0,95,0,100,
-    2,83,0,41,3,122,31,73,110,118,97,108,105,100,97,116,
-    101,32,116,104,101,32,100,105,114,101,99,116,111,114,121,32,
-    109,116,105,109,101,46,114,105,0,0,0,78,41,1,114,87,
-    1,0,0,114,246,0,0,0,114,3,0,0,0,114,3,0,
-    0,0,114,6,0,0,0,114,46,1,0,0,174,5,0,0,
-    115,2,0,0,0,0,2,122,28,70,105,108,101,70,105,110,
-    100,101,114,46,105,110,118,97,108,105,100,97,116,101,95,99,
-    97,99,104,101,115,99,2,0,0,0,0,0,0,0,0,0,
-    0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,42,
-    0,0,0,124,0,160,0,124,1,161,1,125,2,124,2,100,
-    1,107,8,114,26,100,1,103,0,102,2,83,0,124,2,106,
-    1,124,2,106,2,112,38,103,0,102,2,83,0,41,2,122,
-    197,84,114,121,32,116,111,32,102,105,110,100,32,97,32,108,
-    111,97,100,101,114,32,102,111,114,32,116,104,101,32,115,112,
-    101,99,105,102,105,101,100,32,109,111,100,117,108,101,44,32,
-    111,114,32,116,104,101,32,110,97,109,101,115,112,97,99,101,
-    10,32,32,32,32,32,32,32,32,112,97,99,107,97,103,101,
-    32,112,111,114,116,105,111,110,115,46,32,82,101,116,117,114,
-    110,115,32,40,108,111,97,100,101,114,44,32,108,105,115,116,
-    45,111,102,45,112,111,114,116,105,111,110,115,41,46,10,10,
-    32,32,32,32,32,32,32,32,84,104,105,115,32,109,101,116,
-    104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101,
-    100,46,32,32,85,115,101,32,102,105,110,100,95,115,112,101,
-    99,40,41,32,105,110,115,116,101,97,100,46,10,10,32,32,
-    32,32,32,32,32,32,78,41,3,114,203,0,0,0,114,140,
-    0,0,0,114,178,0,0,0,41,3,114,119,0,0,0,114,
-    139,0,0,0,114,187,0,0,0,114,3,0,0,0,114,3,
-    0,0,0,114,6,0,0,0,114,137,0,0,0,180,5,0,
-    0,115,8,0,0,0,0,7,10,1,8,1,8,1,122,22,
-    70,105,108,101,70,105,110,100,101,114,46,102,105,110,100,95,
-    108,111,97,100,101,114,99,6,0,0,0,0,0,0,0,0,
-    0,0,0,7,0,0,0,6,0,0,0,67,0,0,0,115,
-    26,0,0,0,124,1,124,2,124,3,131,2,125,6,116,0,
-    124,2,124,3,124,6,124,4,100,1,141,4,83,0,41,2,
-    78,114,177,0,0,0,41,1,114,190,0,0,0,41,7,114,
-    119,0,0,0,114,188,0,0,0,114,139,0,0,0,114,44,
-    0,0,0,90,4,115,109,115,108,114,202,0,0,0,114,140,
-    0,0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,
-    0,0,114,58,1,0,0,192,5,0,0,115,8,0,0,0,
-    0,1,10,1,8,1,2,255,122,20,70,105,108,101,70,105,
-    110,100,101,114,46,95,103,101,116,95,115,112,101,99,78,99,
-    3,0,0,0,0,0,0,0,0,0,0,0,14,0,0,0,
-    8,0,0,0,67,0,0,0,115,98,1,0,0,100,1,125,
-    3,124,1,160,0,100,2,161,1,100,3,25,0,125,4,122,
-    24,116,1,124,0,106,2,112,34,116,3,160,4,161,0,131,
-    1,106,5,125,5,87,0,110,24,4,0,116,6,107,10,114,
-    66,1,0,1,0,1,0,100,4,125,5,89,0,110,2,88,
-    0,124,5,124,0,106,7,107,3,114,92,124,0,160,8,161,
-    0,1,0,124,5,124,0,95,7,116,9,131,0,114,114,124,
-    0,106,10,125,6,124,4,160,11,161,0,125,7,110,10,124,
-    0,106,12,125,6,124,4,125,7,124,7,124,6,107,6,114,
-    218,116,13,124,0,106,2,124,4,131,2,125,8,124,0,106,
-    14,68,0,93,58,92,2,125,9,125,10,100,5,124,9,23,
-    0,125,11,116,13,124,8,124,11,131,2,125,12,116,15,124,
-    12,131,1,114,150,124,0,160,16,124,10,124,1,124,12,124,
-    8,103,1,124,2,161,5,2,0,1,0,83,0,113,150,116,
-    17,124,8,131,1,125,3,124,0,106,14,68,0,93,82,92,
-    2,125,9,125,10,116,13,124,0,106,2,124,4,124,9,23,
-    0,131,2,125,12,116,18,106,19,100,6,124,12,100,3,100,
-    7,141,3,1,0,124,7,124,9,23,0,124,6,107,6,114,
-    224,116,15,124,12,131,1,114,224,124,0,160,16,124,10,124,
-    1,124,12,100,8,124,2,161,5,2,0,1,0,83,0,113,
-    224,124,3,144,1,114,94,116,18,160,19,100,9,124,8,161,
-    2,1,0,116,18,160,20,124,1,100,8,161,2,125,13,124,
-    8,103,1,124,13,95,21,124,13,83,0,100,8,83,0,41,
-    10,122,111,84,114,121,32,116,111,32,102,105,110,100,32,97,
-    32,115,112,101,99,32,102,111,114,32,116,104,101,32,115,112,
-    101,99,105,102,105,101,100,32,109,111,100,117,108,101,46,10,
-    10,32,32,32,32,32,32,32,32,82,101,116,117,114,110,115,
-    32,116,104,101,32,109,97,116,99,104,105,110,103,32,115,112,
-    101,99,44,32,111,114,32,78,111,110,101,32,105,102,32,110,
-    111,116,32,102,111,117,110,100,46,10,32,32,32,32,32,32,
-    32,32,70,114,71,0,0,0,114,28,0,0,0,114,105,0,
-    0,0,114,209,0,0,0,122,9,116,114,121,105,110,103,32,
-    123,125,41,1,90,9,118,101,114,98,111,115,105,116,121,78,
-    122,25,112,111,115,115,105,98,108,101,32,110,97,109,101,115,
-    112,97,99,101,32,102,111,114,32,123,125,41,22,114,41,0,
-    0,0,114,49,0,0,0,114,44,0,0,0,114,2,0,0,
-    0,114,55,0,0,0,114,10,1,0,0,114,50,0,0,0,
-    114,87,1,0,0,218,11,95,102,105,108,108,95,99,97,99,
-    104,101,114,7,0,0,0,114,90,1,0,0,114,106,0,0,
-    0,114,89,1,0,0,114,38,0,0,0,114,86,1,0,0,
-    114,54,0,0,0,114,58,1,0,0,114,56,0,0,0,114,
-    134,0,0,0,114,149,0,0,0,114,183,0,0,0,114,178,
-    0,0,0,41,14,114,119,0,0,0,114,139,0,0,0,114,
-    202,0,0,0,90,12,105,115,95,110,97,109,101,115,112,97,
-    99,101,90,11,116,97,105,108,95,109,111,100,117,108,101,114,
-    169,0,0,0,90,5,99,97,99,104,101,90,12,99,97,99,
-    104,101,95,109,111,100,117,108,101,90,9,98,97,115,101,95,
-    112,97,116,104,114,17,1,0,0,114,188,0,0,0,90,13,
-    105,110,105,116,95,102,105,108,101,110,97,109,101,90,9,102,
-    117,108,108,95,112,97,116,104,114,187,0,0,0,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,114,203,0,0,
-    0,197,5,0,0,115,74,0,0,0,0,5,4,1,14,1,
-    2,1,24,1,14,1,10,1,10,1,8,1,6,2,6,1,
-    6,1,10,2,6,1,4,2,8,1,12,1,14,1,8,1,
-    10,1,8,1,26,4,8,2,14,1,16,1,16,1,12,1,
-    8,1,10,1,2,0,2,255,10,2,6,1,12,1,12,1,
-    8,1,4,1,122,20,70,105,108,101,70,105,110,100,101,114,
-    46,102,105,110,100,95,115,112,101,99,99,1,0,0,0,0,
-    0,0,0,0,0,0,0,9,0,0,0,10,0,0,0,67,
-    0,0,0,115,190,0,0,0,124,0,106,0,125,1,122,22,
-    116,1,160,2,124,1,112,22,116,1,160,3,161,0,161,1,
-    125,2,87,0,110,30,4,0,116,4,116,5,116,6,102,3,
-    107,10,114,58,1,0,1,0,1,0,103,0,125,2,89,0,
-    110,2,88,0,116,7,106,8,160,9,100,1,161,1,115,84,
-    116,10,124,2,131,1,124,0,95,11,110,74,116,10,131,0,
-    125,3,124,2,68,0,93,56,125,4,124,4,160,12,100,2,
-    161,1,92,3,125,5,125,6,125,7,124,6,114,136,100,3,
-    160,13,124,5,124,7,160,14,161,0,161,2,125,8,110,4,
-    124,5,125,8,124,3,160,15,124,8,161,1,1,0,113,94,
-    124,3,124,0,95,11,116,7,106,8,160,9,116,16,161,1,
-    114,186,100,4,100,5,132,0,124,2,68,0,131,1,124,0,
-    95,17,100,6,83,0,41,7,122,68,70,105,108,108,32,116,
-    104,101,32,99,97,99,104,101,32,111,102,32,112,111,116,101,
-    110,116,105,97,108,32,109,111,100,117,108,101,115,32,97,110,
-    100,32,112,97,99,107,97,103,101,115,32,102,111,114,32,116,
-    104,105,115,32,100,105,114,101,99,116,111,114,121,46,114,0,
-    0,0,0,114,71,0,0,0,114,61,0,0,0,99,1,0,
-    0,0,0,0,0,0,0,0,0,0,2,0,0,0,4,0,
-    0,0,83,0,0,0,115,20,0,0,0,104,0,124,0,93,
-    12,125,1,124,1,160,0,161,0,146,2,113,4,83,0,114,
-    3,0,0,0,41,1,114,106,0,0,0,41,2,114,32,0,
-    0,0,90,2,102,110,114,3,0,0,0,114,3,0,0,0,
-    114,6,0,0,0,218,9,60,115,101,116,99,111,109,112,62,
-    18,6,0,0,115,4,0,0,0,6,0,2,0,122,41,70,
-    105,108,101,70,105,110,100,101,114,46,95,102,105,108,108,95,
-    99,97,99,104,101,46,60,108,111,99,97,108,115,62,46,60,
-    115,101,116,99,111,109,112,62,78,41,18,114,44,0,0,0,
-    114,2,0,0,0,114,7,1,0,0,114,55,0,0,0,114,
-    3,1,0,0,218,15,80,101,114,109,105,115,115,105,111,110,
-    69,114,114,111,114,218,18,78,111,116,65,68,105,114,101,99,
-    116,111,114,121,69,114,114,111,114,114,8,0,0,0,114,9,
-    0,0,0,114,10,0,0,0,114,88,1,0,0,114,89,1,
-    0,0,114,101,0,0,0,114,62,0,0,0,114,106,0,0,
-    0,218,3,97,100,100,114,11,0,0,0,114,90,1,0,0,
-    41,9,114,119,0,0,0,114,44,0,0,0,114,8,1,0,
-    0,90,21,108,111,119,101,114,95,115,117,102,102,105,120,95,
-    99,111,110,116,101,110,116,115,114,41,1,0,0,114,117,0,
-    0,0,114,29,1,0,0,114,17,1,0,0,90,8,110,101,
-    119,95,110,97,109,101,114,3,0,0,0,114,3,0,0,0,
-    114,6,0,0,0,114,92,1,0,0,245,5,0,0,115,34,
-    0,0,0,0,2,6,1,2,1,22,1,20,3,10,3,12,
-    1,12,7,6,1,8,1,16,1,4,1,18,2,4,1,12,
-    1,6,1,12,1,122,22,70,105,108,101,70,105,110,100,101,
-    114,46,95,102,105,108,108,95,99,97,99,104,101,99,1,0,
-    0,0,0,0,0,0,0,0,0,0,3,0,0,0,3,0,
-    0,0,7,0,0,0,115,18,0,0,0,135,0,135,1,102,
-    2,100,1,100,2,132,8,125,2,124,2,83,0,41,3,97,
-    20,1,0,0,65,32,99,108,97,115,115,32,109,101,116,104,
-    111,100,32,119,104,105,99,104,32,114,101,116,117,114,110,115,
-    32,97,32,99,108,111,115,117,114,101,32,116,111,32,117,115,
-    101,32,111,110,32,115,121,115,46,112,97,116,104,95,104,111,
-    111,107,10,32,32,32,32,32,32,32,32,119,104,105,99,104,
-    32,119,105,108,108,32,114,101,116,117,114,110,32,97,110,32,
-    105,110,115,116,97,110,99,101,32,117,115,105,110,103,32,116,
-    104,101,32,115,112,101,99,105,102,105,101,100,32,108,111,97,
-    100,101,114,115,32,97,110,100,32,116,104,101,32,112,97,116,
-    104,10,32,32,32,32,32,32,32,32,99,97,108,108,101,100,
-    32,111,110,32,116,104,101,32,99,108,111,115,117,114,101,46,
-    10,10,32,32,32,32,32,32,32,32,73,102,32,116,104,101,
-    32,112,97,116,104,32,99,97,108,108,101,100,32,111,110,32,
-    116,104,101,32,99,108,111,115,117,114,101,32,105,115,32,110,
-    111,116,32,97,32,100,105,114,101,99,116,111,114,121,44,32,
-    73,109,112,111,114,116,69,114,114,111,114,32,105,115,10,32,
-    32,32,32,32,32,32,32,114,97,105,115,101,100,46,10,10,
-    32,32,32,32,32,32,32,32,99,1,0,0,0,0,0,0,
-    0,0,0,0,0,1,0,0,0,4,0,0,0,19,0,0,
-    0,115,34,0,0,0,116,0,124,0,131,1,115,20,116,1,
-    100,1,124,0,100,2,141,2,130,1,136,0,124,0,102,1,
-    136,1,158,2,142,0,83,0,41,3,122,45,80,97,116,104,
-    32,104,111,111,107,32,102,111,114,32,105,109,112,111,114,116,
-    108,105,98,46,109,97,99,104,105,110,101,114,121,46,70,105,
-    108,101,70,105,110,100,101,114,46,122,30,111,110,108,121,32,
-    100,105,114,101,99,116,111,114,105,101,115,32,97,114,101,32,
-    115,117,112,112,111,114,116,101,100,114,48,0,0,0,41,2,
-    114,56,0,0,0,114,118,0,0,0,114,48,0,0,0,169,
-    2,114,193,0,0,0,114,91,1,0,0,114,3,0,0,0,
-    114,6,0,0,0,218,24,112,97,116,104,95,104,111,111,107,
-    95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,30,
-    6,0,0,115,6,0,0,0,0,2,8,1,12,1,122,54,
-    70,105,108,101,70,105,110,100,101,114,46,112,97,116,104,95,
-    104,111,111,107,46,60,108,111,99,97,108,115,62,46,112,97,
-    116,104,95,104,111,111,107,95,102,111,114,95,70,105,108,101,
-    70,105,110,100,101,114,114,3,0,0,0,41,3,114,193,0,
-    0,0,114,91,1,0,0,114,98,1,0,0,114,3,0,0,
-    0,114,97,1,0,0,114,6,0,0,0,218,9,112,97,116,
-    104,95,104,111,111,107,20,6,0,0,115,4,0,0,0,0,
-    10,14,6,122,20,70,105,108,101,70,105,110,100,101,114,46,
-    112,97,116,104,95,104,111,111,107,99,1,0,0,0,0,0,
-    0,0,0,0,0,0,1,0,0,0,3,0,0,0,67,0,
-    0,0,115,12,0,0,0,100,1,160,0,124,0,106,1,161,
-    1,83,0,41,2,78,122,16,70,105,108,101,70,105,110,100,
-    101,114,40,123,33,114,125,41,41,2,114,62,0,0,0,114,
-    44,0,0,0,114,246,0,0,0,114,3,0,0,0,114,3,
-    0,0,0,114,6,0,0,0,114,39,1,0,0,38,6,0,
-    0,115,2,0,0,0,0,1,122,19,70,105,108,101,70,105,
-    110,100,101,114,46,95,95,114,101,112,114,95,95,41,1,78,
-    41,15,114,125,0,0,0,114,124,0,0,0,114,126,0,0,
-    0,114,127,0,0,0,114,209,0,0,0,114,46,1,0,0,
-    114,143,0,0,0,114,206,0,0,0,114,137,0,0,0,114,
-    58,1,0,0,114,203,0,0,0,114,92,1,0,0,114,207,
-    0,0,0,114,99,1,0,0,114,39,1,0,0,114,3,0,
+    61,1,0,0,91,5,0,0,115,4,0,0,0,0,10,12,
+    1,122,29,80,97,116,104,70,105,110,100,101,114,46,102,105,
+    110,100,95,100,105,115,116,114,105,98,117,116,105,111,110,115,
+    41,1,78,41,2,78,78,41,1,78,41,13,114,126,0,0,
+    0,114,125,0,0,0,114,127,0,0,0,114,128,0,0,0,
+    114,208,0,0,0,114,47,1,0,0,114,53,1,0,0,114,
+    55,1,0,0,114,56,1,0,0,114,59,1,0,0,114,204,
+    0,0,0,114,207,0,0,0,114,61,1,0,0,114,3,0,
     0,0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
-    0,114,84,1,0,0,151,5,0,0,115,22,0,0,0,8,
-    2,4,7,8,14,8,4,4,2,8,12,8,5,10,48,8,
-    31,2,1,10,17,114,84,1,0,0,99,4,0,0,0,0,
-    0,0,0,0,0,0,0,6,0,0,0,8,0,0,0,67,
-    0,0,0,115,146,0,0,0,124,0,160,0,100,1,161,1,
-    125,4,124,0,160,0,100,2,161,1,125,5,124,4,115,66,
-    124,5,114,36,124,5,106,1,125,4,110,30,124,2,124,3,
-    107,2,114,56,116,2,124,1,124,2,131,2,125,4,110,10,
-    116,3,124,1,124,2,131,2,125,4,124,5,115,84,116,4,
-    124,1,124,2,124,4,100,3,141,3,125,5,122,36,124,5,
-    124,0,100,2,60,0,124,4,124,0,100,1,60,0,124,2,
-    124,0,100,4,60,0,124,3,124,0,100,5,60,0,87,0,
-    110,20,4,0,116,5,107,10,114,140,1,0,1,0,1,0,
-    89,0,110,2,88,0,100,0,83,0,41,6,78,218,10,95,
-    95,108,111,97,100,101,114,95,95,218,8,95,95,115,112,101,
-    99,95,95,114,85,1,0,0,90,8,95,95,102,105,108,101,
-    95,95,90,10,95,95,99,97,99,104,101,100,95,95,41,6,
-    218,3,103,101,116,114,140,0,0,0,114,15,1,0,0,114,
-    9,1,0,0,114,190,0,0,0,114,72,1,0,0,41,6,
-    90,2,110,115,114,117,0,0,0,90,8,112,97,116,104,110,
-    97,109,101,90,9,99,112,97,116,104,110,97,109,101,114,140,
-    0,0,0,114,187,0,0,0,114,3,0,0,0,114,3,0,
-    0,0,114,6,0,0,0,218,14,95,102,105,120,95,117,112,
-    95,109,111,100,117,108,101,44,6,0,0,115,34,0,0,0,
-    0,2,10,1,10,1,4,1,4,1,8,1,8,1,12,2,
-    10,1,4,1,14,1,2,1,8,1,8,1,8,1,12,1,
-    14,2,114,103,1,0,0,99,0,0,0,0,0,0,0,0,
-    0,0,0,0,3,0,0,0,3,0,0,0,67,0,0,0,
-    115,38,0,0,0,116,0,116,1,160,2,161,0,102,2,125,
-    0,116,3,116,4,102,2,125,1,116,5,116,6,102,2,125,
-    2,124,0,124,1,124,2,103,3,83,0,41,1,122,95,82,
-    101,116,117,114,110,115,32,97,32,108,105,115,116,32,111,102,
-    32,102,105,108,101,45,98,97,115,101,100,32,109,111,100,117,
-    108,101,32,108,111,97,100,101,114,115,46,10,10,32,32,32,
-    32,69,97,99,104,32,105,116,101,109,32,105,115,32,97,32,
-    116,117,112,108,101,32,40,108,111,97,100,101,114,44,32,115,
-    117,102,102,105,120,101,115,41,46,10,32,32,32,32,41,7,
-    114,252,0,0,0,114,163,0,0,0,218,18,101,120,116,101,
-    110,115,105,111,110,95,115,117,102,102,105,120,101,115,114,9,
-    1,0,0,114,102,0,0,0,114,15,1,0,0,114,89,0,
-    0,0,41,3,90,10,101,120,116,101,110,115,105,111,110,115,
-    90,6,115,111,117,114,99,101,90,8,98,121,116,101,99,111,
-    100,101,114,3,0,0,0,114,3,0,0,0,114,6,0,0,
-    0,114,184,0,0,0,67,6,0,0,115,8,0,0,0,0,
-    5,12,1,8,1,8,1,114,184,0,0,0,99,1,0,0,
-    0,0,0,0,0,0,0,0,0,12,0,0,0,9,0,0,
-    0,67,0,0,0,115,178,1,0,0,124,0,97,0,116,0,
-    106,1,97,1,116,0,106,2,97,2,116,1,106,3,116,4,
-    25,0,125,1,100,1,68,0,93,48,125,2,124,2,116,1,
-    106,3,107,7,114,56,116,0,160,5,124,2,161,1,125,3,
-    110,10,116,1,106,3,124,2,25,0,125,3,116,6,124,1,
-    124,2,124,3,131,3,1,0,113,30,100,2,100,3,103,1,
-    102,2,100,4,100,5,100,3,103,2,102,2,102,2,125,4,
-    124,4,68,0,93,110,92,2,125,5,125,6,116,7,100,6,
-    100,7,132,0,124,6,68,0,131,1,131,1,115,136,116,8,
-    130,1,124,6,100,8,25,0,125,7,124,5,116,1,106,3,
-    107,6,114,170,116,1,106,3,124,5,25,0,125,8,1,0,
-    113,226,113,106,122,20,116,0,160,5,124,5,161,1,125,8,
-    87,0,1,0,113,226,87,0,113,106,4,0,116,9,107,10,
-    114,214,1,0,1,0,1,0,89,0,113,106,89,0,113,106,
-    88,0,113,106,116,9,100,9,131,1,130,1,116,6,124,1,
-    100,10,124,8,131,3,1,0,116,6,124,1,100,11,124,7,
-    131,3,1,0,116,6,124,1,100,12,100,13,160,10,124,6,
-    161,1,131,3,1,0,116,6,124,1,100,14,100,15,100,16,
-    132,0,124,6,68,0,131,1,131,3,1,0,116,0,160,5,
-    100,17,161,1,125,9,116,6,124,1,100,17,124,9,131,3,
-    1,0,116,0,160,5,100,18,161,1,125,10,116,6,124,1,
-    100,18,124,10,131,3,1,0,124,5,100,4,107,2,144,1,
-    114,110,116,0,160,5,100,19,161,1,125,11,116,6,124,1,
-    100,20,124,11,131,3,1,0,116,6,124,1,100,21,116,11,
-    131,0,131,3,1,0,116,12,160,13,116,2,160,14,161,0,
-    161,1,1,0,124,5,100,4,107,2,144,1,114,174,116,15,
-    160,16,100,22,161,1,1,0,100,23,116,12,107,6,144,1,
-    114,174,100,24,116,17,95,18,100,25,83,0,41,26,122,205,
-    83,101,116,117,112,32,116,104,101,32,112,97,116,104,45,98,
-    97,115,101,100,32,105,109,112,111,114,116,101,114,115,32,102,
-    111,114,32,105,109,112,111,114,116,108,105,98,32,98,121,32,
-    105,109,112,111,114,116,105,110,103,32,110,101,101,100,101,100,
-    10,32,32,32,32,98,117,105,108,116,45,105,110,32,109,111,
-    100,117,108,101,115,32,97,110,100,32,105,110,106,101,99,116,
-    105,110,103,32,116,104,101,109,32,105,110,116,111,32,116,104,
-    101,32,103,108,111,98,97,108,32,110,97,109,101,115,112,97,
-    99,101,46,10,10,32,32,32,32,79,116,104,101,114,32,99,
-    111,109,112,111,110,101,110,116,115,32,97,114,101,32,101,120,
-    116,114,97,99,116,101,100,32,102,114,111,109,32,116,104,101,
-    32,99,111,114,101,32,98,111,111,116,115,116,114,97,112,32,
-    109,111,100,117,108,101,46,10,10,32,32,32,32,41,4,114,
-    64,0,0,0,114,75,0,0,0,218,8,98,117,105,108,116,
-    105,110,115,114,160,0,0,0,90,5,112,111,115,105,120,250,
-    1,47,90,2,110,116,250,1,92,99,1,0,0,0,0,0,
-    0,0,0,0,0,0,2,0,0,0,3,0,0,0,115,0,
-    0,0,115,26,0,0,0,124,0,93,18,125,1,116,0,124,
-    1,131,1,100,0,107,2,86,0,1,0,113,2,100,1,83,
-    0,41,2,114,39,0,0,0,78,41,1,114,22,0,0,0,
-    41,2,114,32,0,0,0,114,95,0,0,0,114,3,0,0,
-    0,114,3,0,0,0,114,6,0,0,0,114,19,1,0,0,
-    103,6,0,0,115,4,0,0,0,4,0,2,0,122,25,95,
-    115,101,116,117,112,46,60,108,111,99,97,108,115,62,46,60,
-    103,101,110,101,120,112,114,62,114,73,0,0,0,122,30,105,
-    109,112,111,114,116,108,105,98,32,114,101,113,117,105,114,101,
-    115,32,112,111,115,105,120,32,111,114,32,110,116,114,2,0,
-    0,0,114,35,0,0,0,114,31,0,0,0,114,40,0,0,
-    0,114,58,0,0,0,99,1,0,0,0,0,0,0,0,0,
-    0,0,0,2,0,0,0,4,0,0,0,83,0,0,0,115,
-    22,0,0,0,104,0,124,0,93,14,125,1,100,0,124,1,
-    155,0,157,2,146,2,113,4,83,0,41,1,114,74,0,0,
-    0,114,3,0,0,0,41,2,114,32,0,0,0,218,1,115,
+    0,114,46,1,0,0,214,4,0,0,115,34,0,0,0,8,
+    2,4,2,2,1,10,9,2,1,10,12,2,1,10,21,2,
+    1,10,14,2,1,12,31,2,1,12,23,2,1,12,12,2,
+    1,114,46,1,0,0,99,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,3,0,0,0,64,0,0,0,115,
+    90,0,0,0,101,0,90,1,100,0,90,2,100,1,90,3,
+    100,2,100,3,132,0,90,4,100,4,100,5,132,0,90,5,
+    101,6,90,7,100,6,100,7,132,0,90,8,100,8,100,9,
+    132,0,90,9,100,19,100,11,100,12,132,1,90,10,100,13,
+    100,14,132,0,90,11,101,12,100,15,100,16,132,0,131,1,
+    90,13,100,17,100,18,132,0,90,14,100,10,83,0,41,20,
+    218,10,70,105,108,101,70,105,110,100,101,114,122,172,70,105,
+    108,101,45,98,97,115,101,100,32,102,105,110,100,101,114,46,
+    10,10,32,32,32,32,73,110,116,101,114,97,99,116,105,111,
+    110,115,32,119,105,116,104,32,116,104,101,32,102,105,108,101,
+    32,115,121,115,116,101,109,32,97,114,101,32,99,97,99,104,
+    101,100,32,102,111,114,32,112,101,114,102,111,114,109,97,110,
+    99,101,44,32,98,101,105,110,103,10,32,32,32,32,114,101,
+    102,114,101,115,104,101,100,32,119,104,101,110,32,116,104,101,
+    32,100,105,114,101,99,116,111,114,121,32,116,104,101,32,102,
+    105,110,100,101,114,32,105,115,32,104,97,110,100,108,105,110,
+    103,32,104,97,115,32,98,101,101,110,32,109,111,100,105,102,
+    105,101,100,46,10,10,32,32,32,32,99,2,0,0,0,0,
+    0,0,0,0,0,0,0,5,0,0,0,6,0,0,0,7,
+    0,0,0,115,84,0,0,0,103,0,125,3,124,2,68,0,
+    93,32,92,2,137,0,125,4,124,3,160,0,135,0,102,1,
+    100,1,100,2,132,8,124,4,68,0,131,1,161,1,1,0,
+    113,8,124,3,124,0,95,1,124,1,112,54,100,3,124,0,
+    95,2,100,4,124,0,95,3,116,4,131,0,124,0,95,5,
+    116,4,131,0,124,0,95,6,100,5,83,0,41,6,122,154,
+    73,110,105,116,105,97,108,105,122,101,32,119,105,116,104,32,
+    116,104,101,32,112,97,116,104,32,116,111,32,115,101,97,114,
+    99,104,32,111,110,32,97,110,100,32,97,32,118,97,114,105,
+    97,98,108,101,32,110,117,109,98,101,114,32,111,102,10,32,
+    32,32,32,32,32,32,32,50,45,116,117,112,108,101,115,32,
+    99,111,110,116,97,105,110,105,110,103,32,116,104,101,32,108,
+    111,97,100,101,114,32,97,110,100,32,116,104,101,32,102,105,
+    108,101,32,115,117,102,102,105,120,101,115,32,116,104,101,32,
+    108,111,97,100,101,114,10,32,32,32,32,32,32,32,32,114,
+    101,99,111,103,110,105,122,101,115,46,99,1,0,0,0,0,
+    0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,51,
+    0,0,0,115,22,0,0,0,124,0,93,14,125,1,124,1,
+    136,0,102,2,86,0,1,0,113,2,100,0,83,0,114,110,
+    0,0,0,114,3,0,0,0,114,17,1,0,0,169,1,114,
+    141,0,0,0,114,3,0,0,0,114,6,0,0,0,114,20,
+    1,0,0,120,5,0,0,115,4,0,0,0,4,0,2,0,
+    122,38,70,105,108,101,70,105,110,100,101,114,46,95,95,105,
+    110,105,116,95,95,46,60,108,111,99,97,108,115,62,46,60,
+    103,101,110,101,120,112,114,62,114,71,0,0,0,114,105,0,
+    0,0,78,41,7,114,168,0,0,0,218,8,95,108,111,97,
+    100,101,114,115,114,44,0,0,0,218,11,95,112,97,116,104,
+    95,109,116,105,109,101,218,3,115,101,116,218,11,95,112,97,
+    116,104,95,99,97,99,104,101,218,19,95,114,101,108,97,120,
+    101,100,95,112,97,116,104,95,99,97,99,104,101,41,5,114,
+    119,0,0,0,114,44,0,0,0,218,14,108,111,97,100,101,
+    114,95,100,101,116,97,105,108,115,90,7,108,111,97,100,101,
+    114,115,114,190,0,0,0,114,3,0,0,0,114,63,1,0,
+    0,114,6,0,0,0,114,210,0,0,0,114,5,0,0,115,
+    16,0,0,0,0,4,4,1,12,1,26,1,6,2,10,1,
+    6,1,8,1,122,19,70,105,108,101,70,105,110,100,101,114,
+    46,95,95,105,110,105,116,95,95,99,1,0,0,0,0,0,
+    0,0,0,0,0,0,1,0,0,0,2,0,0,0,67,0,
+    0,0,115,10,0,0,0,100,1,124,0,95,0,100,2,83,
+    0,41,3,122,31,73,110,118,97,108,105,100,97,116,101,32,
+    116,104,101,32,100,105,114,101,99,116,111,114,121,32,109,116,
+    105,109,101,46,114,105,0,0,0,78,41,1,114,65,1,0,
+    0,114,247,0,0,0,114,3,0,0,0,114,3,0,0,0,
+    114,6,0,0,0,114,47,1,0,0,128,5,0,0,115,2,
+    0,0,0,0,2,122,28,70,105,108,101,70,105,110,100,101,
+    114,46,105,110,118,97,108,105,100,97,116,101,95,99,97,99,
+    104,101,115,99,2,0,0,0,0,0,0,0,0,0,0,0,
+    3,0,0,0,3,0,0,0,67,0,0,0,115,42,0,0,
+    0,124,0,160,0,124,1,161,1,125,2,124,2,100,1,107,
+    8,114,26,100,1,103,0,102,2,83,0,124,2,106,1,124,
+    2,106,2,112,38,103,0,102,2,83,0,41,2,122,197,84,
+    114,121,32,116,111,32,102,105,110,100,32,97,32,108,111,97,
+    100,101,114,32,102,111,114,32,116,104,101,32,115,112,101,99,
+    105,102,105,101,100,32,109,111,100,117,108,101,44,32,111,114,
+    32,116,104,101,32,110,97,109,101,115,112,97,99,101,10,32,
+    32,32,32,32,32,32,32,112,97,99,107,97,103,101,32,112,
+    111,114,116,105,111,110,115,46,32,82,101,116,117,114,110,115,
+    32,40,108,111,97,100,101,114,44,32,108,105,115,116,45,111,
+    102,45,112,111,114,116,105,111,110,115,41,46,10,10,32,32,
+    32,32,32,32,32,32,84,104,105,115,32,109,101,116,104,111,
+    100,32,105,115,32,100,101,112,114,101,99,97,116,101,100,46,
+    32,32,85,115,101,32,102,105,110,100,95,115,112,101,99,40,
+    41,32,105,110,115,116,101,97,100,46,10,10,32,32,32,32,
+    32,32,32,32,78,41,3,114,204,0,0,0,114,141,0,0,
+    0,114,179,0,0,0,41,3,114,119,0,0,0,114,140,0,
+    0,0,114,188,0,0,0,114,3,0,0,0,114,3,0,0,
+    0,114,6,0,0,0,114,138,0,0,0,134,5,0,0,115,
+    8,0,0,0,0,7,10,1,8,1,8,1,122,22,70,105,
+    108,101,70,105,110,100,101,114,46,102,105,110,100,95,108,111,
+    97,100,101,114,99,6,0,0,0,0,0,0,0,0,0,0,
+    0,7,0,0,0,6,0,0,0,67,0,0,0,115,26,0,
+    0,0,124,1,124,2,124,3,131,2,125,6,116,0,124,2,
+    124,3,124,6,124,4,100,1,141,4,83,0,41,2,78,114,
+    178,0,0,0,41,1,114,191,0,0,0,41,7,114,119,0,
+    0,0,114,189,0,0,0,114,140,0,0,0,114,44,0,0,
+    0,90,4,115,109,115,108,114,203,0,0,0,114,141,0,0,
+    0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
+    114,59,1,0,0,146,5,0,0,115,8,0,0,0,0,1,
+    10,1,8,1,2,255,122,20,70,105,108,101,70,105,110,100,
+    101,114,46,95,103,101,116,95,115,112,101,99,78,99,3,0,
+    0,0,0,0,0,0,0,0,0,0,14,0,0,0,8,0,
+    0,0,67,0,0,0,115,98,1,0,0,100,1,125,3,124,
+    1,160,0,100,2,161,1,100,3,25,0,125,4,122,24,116,
+    1,124,0,106,2,112,34,116,3,160,4,161,0,131,1,106,
+    5,125,5,87,0,110,24,4,0,116,6,107,10,114,66,1,
+    0,1,0,1,0,100,4,125,5,89,0,110,2,88,0,124,
+    5,124,0,106,7,107,3,114,92,124,0,160,8,161,0,1,
+    0,124,5,124,0,95,7,116,9,131,0,114,114,124,0,106,
+    10,125,6,124,4,160,11,161,0,125,7,110,10,124,0,106,
+    12,125,6,124,4,125,7,124,7,124,6,107,6,114,218,116,
+    13,124,0,106,2,124,4,131,2,125,8,124,0,106,14,68,
+    0,93,58,92,2,125,9,125,10,100,5,124,9,23,0,125,
+    11,116,13,124,8,124,11,131,2,125,12,116,15,124,12,131,
+    1,114,150,124,0,160,16,124,10,124,1,124,12,124,8,103,
+    1,124,2,161,5,2,0,1,0,83,0,113,150,116,17,124,
+    8,131,1,125,3,124,0,106,14,68,0,93,82,92,2,125,
+    9,125,10,116,13,124,0,106,2,124,4,124,9,23,0,131,
+    2,125,12,116,18,106,19,100,6,124,12,100,3,100,7,141,
+    3,1,0,124,7,124,9,23,0,124,6,107,6,114,224,116,
+    15,124,12,131,1,114,224,124,0,160,16,124,10,124,1,124,
+    12,100,8,124,2,161,5,2,0,1,0,83,0,113,224,124,
+    3,144,1,114,94,116,18,160,19,100,9,124,8,161,2,1,
+    0,116,18,160,20,124,1,100,8,161,2,125,13,124,8,103,
+    1,124,13,95,21,124,13,83,0,100,8,83,0,41,10,122,
+    111,84,114,121,32,116,111,32,102,105,110,100,32,97,32,115,
+    112,101,99,32,102,111,114,32,116,104,101,32,115,112,101,99,
+    105,102,105,101,100,32,109,111,100,117,108,101,46,10,10,32,
+    32,32,32,32,32,32,32,82,101,116,117,114,110,115,32,116,
+    104,101,32,109,97,116,99,104,105,110,103,32,115,112,101,99,
+    44,32,111,114,32,78,111,110,101,32,105,102,32,110,111,116,
+    32,102,111,117,110,100,46,10,32,32,32,32,32,32,32,32,
+    70,114,71,0,0,0,114,28,0,0,0,114,105,0,0,0,
+    114,210,0,0,0,122,9,116,114,121,105,110,103,32,123,125,
+    41,1,90,9,118,101,114,98,111,115,105,116,121,78,122,25,
+    112,111,115,115,105,98,108,101,32,110,97,109,101,115,112,97,
+    99,101,32,102,111,114,32,123,125,41,22,114,41,0,0,0,
+    114,49,0,0,0,114,44,0,0,0,114,2,0,0,0,114,
+    55,0,0,0,114,11,1,0,0,114,50,0,0,0,114,65,
+    1,0,0,218,11,95,102,105,108,108,95,99,97,99,104,101,
+    114,7,0,0,0,114,68,1,0,0,114,106,0,0,0,114,
+    67,1,0,0,114,38,0,0,0,114,64,1,0,0,114,54,
+    0,0,0,114,59,1,0,0,114,56,0,0,0,114,135,0,
+    0,0,114,150,0,0,0,114,184,0,0,0,114,179,0,0,
+    0,41,14,114,119,0,0,0,114,140,0,0,0,114,203,0,
+    0,0,90,12,105,115,95,110,97,109,101,115,112,97,99,101,
+    90,11,116,97,105,108,95,109,111,100,117,108,101,114,170,0,
+    0,0,90,5,99,97,99,104,101,90,12,99,97,99,104,101,
+    95,109,111,100,117,108,101,90,9,98,97,115,101,95,112,97,
+    116,104,114,18,1,0,0,114,189,0,0,0,90,13,105,110,
+    105,116,95,102,105,108,101,110,97,109,101,90,9,102,117,108,
+    108,95,112,97,116,104,114,188,0,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,6,0,0,0,114,204,0,0,0,151,
+    5,0,0,115,74,0,0,0,0,5,4,1,14,1,2,1,
+    24,1,14,1,10,1,10,1,8,1,6,2,6,1,6,1,
+    10,2,6,1,4,2,8,1,12,1,14,1,8,1,10,1,
+    8,1,26,4,8,2,14,1,16,1,16,1,12,1,8,1,
+    10,1,2,0,2,255,10,2,6,1,12,1,12,1,8,1,
+    4,1,122,20,70,105,108,101,70,105,110,100,101,114,46,102,
+    105,110,100,95,115,112,101,99,99,1,0,0,0,0,0,0,
+    0,0,0,0,0,9,0,0,0,10,0,0,0,67,0,0,
+    0,115,190,0,0,0,124,0,106,0,125,1,122,22,116,1,
+    160,2,124,1,112,22,116,1,160,3,161,0,161,1,125,2,
+    87,0,110,30,4,0,116,4,116,5,116,6,102,3,107,10,
+    114,58,1,0,1,0,1,0,103,0,125,2,89,0,110,2,
+    88,0,116,7,106,8,160,9,100,1,161,1,115,84,116,10,
+    124,2,131,1,124,0,95,11,110,74,116,10,131,0,125,3,
+    124,2,68,0,93,56,125,4,124,4,160,12,100,2,161,1,
+    92,3,125,5,125,6,125,7,124,6,114,136,100,3,160,13,
+    124,5,124,7,160,14,161,0,161,2,125,8,110,4,124,5,
+    125,8,124,3,160,15,124,8,161,1,1,0,113,94,124,3,
+    124,0,95,11,116,7,106,8,160,9,116,16,161,1,114,186,
+    100,4,100,5,132,0,124,2,68,0,131,1,124,0,95,17,
+    100,6,83,0,41,7,122,68,70,105,108,108,32,116,104,101,
+    32,99,97,99,104,101,32,111,102,32,112,111,116,101,110,116,
+    105,97,108,32,109,111,100,117,108,101,115,32,97,110,100,32,
+    112,97,99,107,97,103,101,115,32,102,111,114,32,116,104,105,
+    115,32,100,105,114,101,99,116,111,114,121,46,114,0,0,0,
+    0,114,71,0,0,0,114,61,0,0,0,99,1,0,0,0,
+    0,0,0,0,0,0,0,0,2,0,0,0,4,0,0,0,
+    83,0,0,0,115,20,0,0,0,104,0,124,0,93,12,125,
+    1,124,1,160,0,161,0,146,2,113,4,83,0,114,3,0,
+    0,0,41,1,114,106,0,0,0,41,2,114,32,0,0,0,
+    90,2,102,110,114,3,0,0,0,114,3,0,0,0,114,6,
+    0,0,0,218,9,60,115,101,116,99,111,109,112,62,228,5,
+    0,0,115,4,0,0,0,6,0,2,0,122,41,70,105,108,
+    101,70,105,110,100,101,114,46,95,102,105,108,108,95,99,97,
+    99,104,101,46,60,108,111,99,97,108,115,62,46,60,115,101,
+    116,99,111,109,112,62,78,41,18,114,44,0,0,0,114,2,
+    0,0,0,114,8,1,0,0,114,55,0,0,0,114,4,1,
+    0,0,218,15,80,101,114,109,105,115,115,105,111,110,69,114,
+    114,111,114,218,18,78,111,116,65,68,105,114,101,99,116,111,
+    114,121,69,114,114,111,114,114,8,0,0,0,114,9,0,0,
+    0,114,10,0,0,0,114,66,1,0,0,114,67,1,0,0,
+    114,101,0,0,0,114,62,0,0,0,114,106,0,0,0,218,
+    3,97,100,100,114,11,0,0,0,114,68,1,0,0,41,9,
+    114,119,0,0,0,114,44,0,0,0,114,9,1,0,0,90,
+    21,108,111,119,101,114,95,115,117,102,102,105,120,95,99,111,
+    110,116,101,110,116,115,114,42,1,0,0,114,117,0,0,0,
+    114,30,1,0,0,114,18,1,0,0,90,8,110,101,119,95,
+    110,97,109,101,114,3,0,0,0,114,3,0,0,0,114,6,
+    0,0,0,114,70,1,0,0,199,5,0,0,115,34,0,0,
+    0,0,2,6,1,2,1,22,1,20,3,10,3,12,1,12,
+    7,6,1,8,1,16,1,4,1,18,2,4,1,12,1,6,
+    1,12,1,122,22,70,105,108,101,70,105,110,100,101,114,46,
+    95,102,105,108,108,95,99,97,99,104,101,99,1,0,0,0,
+    0,0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,
+    7,0,0,0,115,18,0,0,0,135,0,135,1,102,2,100,
+    1,100,2,132,8,125,2,124,2,83,0,41,3,97,20,1,
+    0,0,65,32,99,108,97,115,115,32,109,101,116,104,111,100,
+    32,119,104,105,99,104,32,114,101,116,117,114,110,115,32,97,
+    32,99,108,111,115,117,114,101,32,116,111,32,117,115,101,32,
+    111,110,32,115,121,115,46,112,97,116,104,95,104,111,111,107,
+    10,32,32,32,32,32,32,32,32,119,104,105,99,104,32,119,
+    105,108,108,32,114,101,116,117,114,110,32,97,110,32,105,110,
+    115,116,97,110,99,101,32,117,115,105,110,103,32,116,104,101,
+    32,115,112,101,99,105,102,105,101,100,32,108,111,97,100,101,
+    114,115,32,97,110,100,32,116,104,101,32,112,97,116,104,10,
+    32,32,32,32,32,32,32,32,99,97,108,108,101,100,32,111,
+    110,32,116,104,101,32,99,108,111,115,117,114,101,46,10,10,
+    32,32,32,32,32,32,32,32,73,102,32,116,104,101,32,112,
+    97,116,104,32,99,97,108,108,101,100,32,111,110,32,116,104,
+    101,32,99,108,111,115,117,114,101,32,105,115,32,110,111,116,
+    32,97,32,100,105,114,101,99,116,111,114,121,44,32,73,109,
+    112,111,114,116,69,114,114,111,114,32,105,115,10,32,32,32,
+    32,32,32,32,32,114,97,105,115,101,100,46,10,10,32,32,
+    32,32,32,32,32,32,99,1,0,0,0,0,0,0,0,0,
+    0,0,0,1,0,0,0,4,0,0,0,19,0,0,0,115,
+    34,0,0,0,116,0,124,0,131,1,115,20,116,1,100,1,
+    124,0,100,2,141,2,130,1,136,0,124,0,102,1,136,1,
+    158,2,142,0,83,0,41,3,122,45,80,97,116,104,32,104,
+    111,111,107,32,102,111,114,32,105,109,112,111,114,116,108,105,
+    98,46,109,97,99,104,105,110,101,114,121,46,70,105,108,101,
+    70,105,110,100,101,114,46,122,30,111,110,108,121,32,100,105,
+    114,101,99,116,111,114,105,101,115,32,97,114,101,32,115,117,
+    112,112,111,114,116,101,100,114,48,0,0,0,41,2,114,56,
+    0,0,0,114,118,0,0,0,114,48,0,0,0,169,2,114,
+    194,0,0,0,114,69,1,0,0,114,3,0,0,0,114,6,
+    0,0,0,218,24,112,97,116,104,95,104,111,111,107,95,102,
+    111,114,95,70,105,108,101,70,105,110,100,101,114,240,5,0,
+    0,115,6,0,0,0,0,2,8,1,12,1,122,54,70,105,
+    108,101,70,105,110,100,101,114,46,112,97,116,104,95,104,111,
+    111,107,46,60,108,111,99,97,108,115,62,46,112,97,116,104,
+    95,104,111,111,107,95,102,111,114,95,70,105,108,101,70,105,
+    110,100,101,114,114,3,0,0,0,41,3,114,194,0,0,0,
+    114,69,1,0,0,114,76,1,0,0,114,3,0,0,0,114,
+    75,1,0,0,114,6,0,0,0,218,9,112,97,116,104,95,
+    104,111,111,107,230,5,0,0,115,4,0,0,0,0,10,14,
+    6,122,20,70,105,108,101,70,105,110,100,101,114,46,112,97,
+    116,104,95,104,111,111,107,99,1,0,0,0,0,0,0,0,
+    0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0,
+    115,12,0,0,0,100,1,160,0,124,0,106,1,161,1,83,
+    0,41,2,78,122,16,70,105,108,101,70,105,110,100,101,114,
+    40,123,33,114,125,41,41,2,114,62,0,0,0,114,44,0,
+    0,0,114,247,0,0,0,114,3,0,0,0,114,3,0,0,
+    0,114,6,0,0,0,114,40,1,0,0,248,5,0,0,115,
+    2,0,0,0,0,1,122,19,70,105,108,101,70,105,110,100,
+    101,114,46,95,95,114,101,112,114,95,95,41,1,78,41,15,
+    114,126,0,0,0,114,125,0,0,0,114,127,0,0,0,114,
+    128,0,0,0,114,210,0,0,0,114,47,1,0,0,114,144,
+    0,0,0,114,207,0,0,0,114,138,0,0,0,114,59,1,
+    0,0,114,204,0,0,0,114,70,1,0,0,114,208,0,0,
+    0,114,77,1,0,0,114,40,1,0,0,114,3,0,0,0,
+    114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
+    62,1,0,0,105,5,0,0,115,22,0,0,0,8,2,4,
+    7,8,14,8,4,4,2,8,12,8,5,10,48,8,31,2,
+    1,10,17,114,62,1,0,0,99,4,0,0,0,0,0,0,
+    0,0,0,0,0,6,0,0,0,8,0,0,0,67,0,0,
+    0,115,146,0,0,0,124,0,160,0,100,1,161,1,125,4,
+    124,0,160,0,100,2,161,1,125,5,124,4,115,66,124,5,
+    114,36,124,5,106,1,125,4,110,30,124,2,124,3,107,2,
+    114,56,116,2,124,1,124,2,131,2,125,4,110,10,116,3,
+    124,1,124,2,131,2,125,4,124,5,115,84,116,4,124,1,
+    124,2,124,4,100,3,141,3,125,5,122,36,124,5,124,0,
+    100,2,60,0,124,4,124,0,100,1,60,0,124,2,124,0,
+    100,4,60,0,124,3,124,0,100,5,60,0,87,0,110,20,
+    4,0,116,5,107,10,114,140,1,0,1,0,1,0,89,0,
+    110,2,88,0,100,0,83,0,41,6,78,218,10,95,95,108,
+    111,97,100,101,114,95,95,218,8,95,95,115,112,101,99,95,
+    95,114,63,1,0,0,90,8,95,95,102,105,108,101,95,95,
+    90,10,95,95,99,97,99,104,101,100,95,95,41,6,218,3,
+    103,101,116,114,141,0,0,0,114,16,1,0,0,114,10,1,
+    0,0,114,191,0,0,0,218,9,69,120,99,101,112,116,105,
+    111,110,41,6,90,2,110,115,114,117,0,0,0,90,8,112,
+    97,116,104,110,97,109,101,90,9,99,112,97,116,104,110,97,
+    109,101,114,141,0,0,0,114,188,0,0,0,114,3,0,0,
+    0,114,3,0,0,0,114,6,0,0,0,218,14,95,102,105,
+    120,95,117,112,95,109,111,100,117,108,101,254,5,0,0,115,
+    34,0,0,0,0,2,10,1,10,1,4,1,4,1,8,1,
+    8,1,12,2,10,1,4,1,14,1,2,1,8,1,8,1,
+    8,1,12,1,14,2,114,82,1,0,0,99,0,0,0,0,
+    0,0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,
+    67,0,0,0,115,38,0,0,0,116,0,116,1,160,2,161,
+    0,102,2,125,0,116,3,116,4,102,2,125,1,116,5,116,
+    6,102,2,125,2,124,0,124,1,124,2,103,3,83,0,41,
+    1,122,95,82,101,116,117,114,110,115,32,97,32,108,105,115,
+    116,32,111,102,32,102,105,108,101,45,98,97,115,101,100,32,
+    109,111,100,117,108,101,32,108,111,97,100,101,114,115,46,10,
+    10,32,32,32,32,69,97,99,104,32,105,116,101,109,32,105,
+    115,32,97,32,116,117,112,108,101,32,40,108,111,97,100,101,
+    114,44,32,115,117,102,102,105,120,101,115,41,46,10,32,32,
+    32,32,41,7,114,253,0,0,0,114,164,0,0,0,218,18,
+    101,120,116,101,110,115,105,111,110,95,115,117,102,102,105,120,
+    101,115,114,10,1,0,0,114,102,0,0,0,114,16,1,0,
+    0,114,89,0,0,0,41,3,90,10,101,120,116,101,110,115,
+    105,111,110,115,90,6,115,111,117,114,99,101,90,8,98,121,
+    116,101,99,111,100,101,114,3,0,0,0,114,3,0,0,0,
+    114,6,0,0,0,114,185,0,0,0,21,6,0,0,115,8,
+    0,0,0,0,5,12,1,8,1,8,1,114,185,0,0,0,
+    99,1,0,0,0,0,0,0,0,0,0,0,0,12,0,0,
+    0,9,0,0,0,67,0,0,0,115,178,1,0,0,124,0,
+    97,0,116,0,106,1,97,1,116,0,106,2,97,2,116,1,
+    106,3,116,4,25,0,125,1,100,1,68,0,93,48,125,2,
+    124,2,116,1,106,3,107,7,114,56,116,0,160,5,124,2,
+    161,1,125,3,110,10,116,1,106,3,124,2,25,0,125,3,
+    116,6,124,1,124,2,124,3,131,3,1,0,113,30,100,2,
+    100,3,103,1,102,2,100,4,100,5,100,3,103,2,102,2,
+    102,2,125,4,124,4,68,0,93,110,92,2,125,5,125,6,
+    116,7,100,6,100,7,132,0,124,6,68,0,131,1,131,1,
+    115,136,116,8,130,1,124,6,100,8,25,0,125,7,124,5,
+    116,1,106,3,107,6,114,170,116,1,106,3,124,5,25,0,
+    125,8,1,0,113,226,113,106,122,20,116,0,160,5,124,5,
+    161,1,125,8,87,0,1,0,113,226,87,0,113,106,4,0,
+    116,9,107,10,114,214,1,0,1,0,1,0,89,0,113,106,
+    89,0,113,106,88,0,113,106,116,9,100,9,131,1,130,1,
+    116,6,124,1,100,10,124,8,131,3,1,0,116,6,124,1,
+    100,11,124,7,131,3,1,0,116,6,124,1,100,12,100,13,
+    160,10,124,6,161,1,131,3,1,0,116,6,124,1,100,14,
+    100,15,100,16,132,0,124,6,68,0,131,1,131,3,1,0,
+    116,0,160,5,100,17,161,1,125,9,116,6,124,1,100,17,
+    124,9,131,3,1,0,116,0,160,5,100,18,161,1,125,10,
+    116,6,124,1,100,18,124,10,131,3,1,0,124,5,100,4,
+    107,2,144,1,114,110,116,0,160,5,100,19,161,1,125,11,
+    116,6,124,1,100,20,124,11,131,3,1,0,116,6,124,1,
+    100,21,116,11,131,0,131,3,1,0,116,12,160,13,116,2,
+    160,14,161,0,161,1,1,0,124,5,100,4,107,2,144,1,
+    114,174,116,15,160,16,100,22,161,1,1,0,100,23,116,12,
+    107,6,144,1,114,174,100,24,116,17,95,18,100,25,83,0,
+    41,26,122,205,83,101,116,117,112,32,116,104,101,32,112,97,
+    116,104,45,98,97,115,101,100,32,105,109,112,111,114,116,101,
+    114,115,32,102,111,114,32,105,109,112,111,114,116,108,105,98,
+    32,98,121,32,105,109,112,111,114,116,105,110,103,32,110,101,
+    101,100,101,100,10,32,32,32,32,98,117,105,108,116,45,105,
+    110,32,109,111,100,117,108,101,115,32,97,110,100,32,105,110,
+    106,101,99,116,105,110,103,32,116,104,101,109,32,105,110,116,
+    111,32,116,104,101,32,103,108,111,98,97,108,32,110,97,109,
+    101,115,112,97,99,101,46,10,10,32,32,32,32,79,116,104,
+    101,114,32,99,111,109,112,111,110,101,110,116,115,32,97,114,
+    101,32,101,120,116,114,97,99,116,101,100,32,102,114,111,109,
+    32,116,104,101,32,99,111,114,101,32,98,111,111,116,115,116,
+    114,97,112,32,109,111,100,117,108,101,46,10,10,32,32,32,
+    32,41,4,114,64,0,0,0,114,75,0,0,0,218,8,98,
+    117,105,108,116,105,110,115,114,161,0,0,0,90,5,112,111,
+    115,105,120,250,1,47,90,2,110,116,250,1,92,99,1,0,
+    0,0,0,0,0,0,0,0,0,0,2,0,0,0,3,0,
+    0,0,115,0,0,0,115,26,0,0,0,124,0,93,18,125,
+    1,116,0,124,1,131,1,100,0,107,2,86,0,1,0,113,
+    2,100,1,83,0,41,2,114,39,0,0,0,78,41,1,114,
+    22,0,0,0,41,2,114,32,0,0,0,114,95,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    93,1,0,0,119,6,0,0,115,4,0,0,0,6,0,2,
+    20,1,0,0,57,6,0,0,115,4,0,0,0,4,0,2,
     0,122,25,95,115,101,116,117,112,46,60,108,111,99,97,108,
-    115,62,46,60,115,101,116,99,111,109,112,62,90,7,95,116,
-    104,114,101,97,100,90,8,95,119,101,97,107,114,101,102,90,
-    6,119,105,110,114,101,103,114,192,0,0,0,114,7,0,0,
-    0,122,4,46,112,121,119,122,6,95,100,46,112,121,100,84,
-    78,41,19,114,134,0,0,0,114,8,0,0,0,114,163,0,
-    0,0,114,31,1,0,0,114,125,0,0,0,90,18,95,98,
-    117,105,108,116,105,110,95,102,114,111,109,95,110,97,109,101,
-    114,129,0,0,0,218,3,97,108,108,114,23,0,0,0,114,
-    118,0,0,0,114,36,0,0,0,114,13,0,0,0,114,21,
-    1,0,0,114,167,0,0,0,114,104,1,0,0,114,102,0,
-    0,0,114,186,0,0,0,114,191,0,0,0,114,195,0,0,
-    0,41,12,218,17,95,98,111,111,116,115,116,114,97,112,95,
-    109,111,100,117,108,101,90,11,115,101,108,102,95,109,111,100,
-    117,108,101,90,12,98,117,105,108,116,105,110,95,110,97,109,
-    101,90,14,98,117,105,108,116,105,110,95,109,111,100,117,108,
-    101,90,10,111,115,95,100,101,116,97,105,108,115,90,10,98,
-    117,105,108,116,105,110,95,111,115,114,31,0,0,0,114,35,
-    0,0,0,90,9,111,115,95,109,111,100,117,108,101,90,13,
-    116,104,114,101,97,100,95,109,111,100,117,108,101,90,14,119,
-    101,97,107,114,101,102,95,109,111,100,117,108,101,90,13,119,
-    105,110,114,101,103,95,109,111,100,117,108,101,114,3,0,0,
-    0,114,3,0,0,0,114,6,0,0,0,218,6,95,115,101,
-    116,117,112,78,6,0,0,115,78,0,0,0,0,8,4,1,
-    6,1,6,3,10,1,8,1,10,1,12,2,10,1,14,3,
-    22,1,12,2,22,1,8,1,10,1,10,1,6,2,2,1,
-    10,1,10,1,14,1,12,2,8,1,12,1,12,1,18,1,
-    22,3,10,1,12,3,10,1,12,3,10,1,10,1,12,3,
-    14,1,14,1,10,1,10,1,10,1,114,111,1,0,0,99,
-    1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,
-    4,0,0,0,67,0,0,0,115,50,0,0,0,116,0,124,
-    0,131,1,1,0,116,1,131,0,125,1,116,2,106,3,160,
-    4,116,5,106,6,124,1,142,0,103,1,161,1,1,0,116,
-    2,106,7,160,8,116,9,161,1,1,0,100,1,83,0,41,
-    2,122,41,73,110,115,116,97,108,108,32,116,104,101,32,112,
-    97,116,104,45,98,97,115,101,100,32,105,109,112,111,114,116,
-    32,99,111,109,112,111,110,101,110,116,115,46,78,41,10,114,
-    111,1,0,0,114,184,0,0,0,114,8,0,0,0,114,51,
-    1,0,0,114,167,0,0,0,114,84,1,0,0,114,99,1,
-    0,0,218,9,109,101,116,97,95,112,97,116,104,114,186,0,
-    0,0,114,45,1,0,0,41,2,114,110,1,0,0,90,17,
-    115,117,112,112,111,114,116,101,100,95,108,111,97,100,101,114,
-    115,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
-    218,8,95,105,110,115,116,97,108,108,143,6,0,0,115,8,
-    0,0,0,0,2,8,1,6,1,20,1,114,113,1,0,0,
-    41,1,114,60,0,0,0,41,1,78,41,3,78,78,78,41,
-    2,114,73,0,0,0,114,73,0,0,0,41,1,84,41,1,
-    78,41,1,78,41,63,114,127,0,0,0,114,12,0,0,0,
-    90,37,95,67,65,83,69,95,73,78,83,69,78,83,73,84,
-    73,86,69,95,80,76,65,84,70,79,82,77,83,95,66,89,
-    84,69,83,95,75,69,89,114,11,0,0,0,114,13,0,0,
-    0,114,20,0,0,0,114,27,0,0,0,114,29,0,0,0,
-    114,38,0,0,0,114,47,0,0,0,114,49,0,0,0,114,
-    53,0,0,0,114,54,0,0,0,114,56,0,0,0,114,59,
-    0,0,0,114,69,0,0,0,218,4,116,121,112,101,218,8,
-    95,95,99,111,100,101,95,95,114,162,0,0,0,114,18,0,
-    0,0,114,148,0,0,0,114,17,0,0,0,114,24,0,0,
-    0,114,236,0,0,0,114,92,0,0,0,114,88,0,0,0,
-    114,102,0,0,0,114,89,0,0,0,90,23,68,69,66,85,
-    71,95,66,89,84,69,67,79,68,69,95,83,85,70,70,73,
-    88,69,83,90,27,79,80,84,73,77,73,90,69,68,95,66,
-    89,84,69,67,79,68,69,95,83,85,70,70,73,88,69,83,
-    114,98,0,0,0,114,103,0,0,0,114,109,0,0,0,114,
-    113,0,0,0,114,115,0,0,0,114,136,0,0,0,114,143,
-    0,0,0,114,152,0,0,0,114,156,0,0,0,114,158,0,
-    0,0,114,165,0,0,0,114,170,0,0,0,114,171,0,0,
-    0,114,176,0,0,0,218,6,111,98,106,101,99,116,114,185,
-    0,0,0,114,190,0,0,0,114,191,0,0,0,114,208,0,
-    0,0,114,221,0,0,0,114,239,0,0,0,114,9,1,0,
-    0,114,15,1,0,0,114,21,1,0,0,114,252,0,0,0,
-    114,22,1,0,0,114,43,1,0,0,114,45,1,0,0,114,
-    84,1,0,0,114,103,1,0,0,114,184,0,0,0,114,111,
-    1,0,0,114,113,1,0,0,114,3,0,0,0,114,3,0,
-    0,0,114,3,0,0,0,114,6,0,0,0,218,8,60,109,
-    111,100,117,108,101,62,1,0,0,0,115,126,0,0,0,4,
-    22,4,1,4,1,2,1,2,255,4,4,8,17,8,5,8,
-    5,8,6,8,6,8,12,8,10,8,9,8,5,8,7,8,
-    9,10,22,10,127,0,13,16,1,12,2,4,1,4,2,6,
-    2,6,2,8,2,16,71,8,40,8,19,8,12,8,12,8,
-    28,8,17,8,33,8,28,8,24,10,13,10,10,10,11,8,
-    14,6,3,4,1,2,255,12,68,14,64,14,29,16,127,0,
-    17,14,72,18,45,18,26,4,3,18,53,14,63,14,42,14,
-    127,0,66,14,127,0,22,10,23,8,11,8,65,
+    115,62,46,60,103,101,110,101,120,112,114,62,114,73,0,0,
+    0,122,30,105,109,112,111,114,116,108,105,98,32,114,101,113,
+    117,105,114,101,115,32,112,111,115,105,120,32,111,114,32,110,
+    116,114,2,0,0,0,114,35,0,0,0,114,31,0,0,0,
+    114,40,0,0,0,114,58,0,0,0,99,1,0,0,0,0,
+    0,0,0,0,0,0,0,2,0,0,0,4,0,0,0,83,
+    0,0,0,115,22,0,0,0,104,0,124,0,93,14,125,1,
+    100,0,124,1,155,0,157,2,146,2,113,4,83,0,41,1,
+    114,74,0,0,0,114,3,0,0,0,41,2,114,32,0,0,
+    0,218,1,115,114,3,0,0,0,114,3,0,0,0,114,6,
+    0,0,0,114,71,1,0,0,73,6,0,0,115,4,0,0,
+    0,6,0,2,0,122,25,95,115,101,116,117,112,46,60,108,
+    111,99,97,108,115,62,46,60,115,101,116,99,111,109,112,62,
+    90,7,95,116,104,114,101,97,100,90,8,95,119,101,97,107,
+    114,101,102,90,6,119,105,110,114,101,103,114,193,0,0,0,
+    114,7,0,0,0,122,4,46,112,121,119,122,6,95,100,46,
+    112,121,100,84,78,41,19,114,135,0,0,0,114,8,0,0,
+    0,114,164,0,0,0,114,32,1,0,0,114,126,0,0,0,
+    90,18,95,98,117,105,108,116,105,110,95,102,114,111,109,95,
+    110,97,109,101,114,130,0,0,0,218,3,97,108,108,114,23,
+    0,0,0,114,118,0,0,0,114,36,0,0,0,114,13,0,
+    0,0,114,22,1,0,0,114,168,0,0,0,114,83,1,0,
+    0,114,102,0,0,0,114,187,0,0,0,114,192,0,0,0,
+    114,196,0,0,0,41,12,218,17,95,98,111,111,116,115,116,
+    114,97,112,95,109,111,100,117,108,101,90,11,115,101,108,102,
+    95,109,111,100,117,108,101,90,12,98,117,105,108,116,105,110,
+    95,110,97,109,101,90,14,98,117,105,108,116,105,110,95,109,
+    111,100,117,108,101,90,10,111,115,95,100,101,116,97,105,108,
+    115,90,10,98,117,105,108,116,105,110,95,111,115,114,31,0,
+    0,0,114,35,0,0,0,90,9,111,115,95,109,111,100,117,
+    108,101,90,13,116,104,114,101,97,100,95,109,111,100,117,108,
+    101,90,14,119,101,97,107,114,101,102,95,109,111,100,117,108,
+    101,90,13,119,105,110,114,101,103,95,109,111,100,117,108,101,
+    114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,218,
+    6,95,115,101,116,117,112,32,6,0,0,115,78,0,0,0,
+    0,8,4,1,6,1,6,3,10,1,8,1,10,1,12,2,
+    10,1,14,3,22,1,12,2,22,1,8,1,10,1,10,1,
+    6,2,2,1,10,1,10,1,14,1,12,2,8,1,12,1,
+    12,1,18,1,22,3,10,1,12,3,10,1,12,3,10,1,
+    10,1,12,3,14,1,14,1,10,1,10,1,10,1,114,90,
+    1,0,0,99,1,0,0,0,0,0,0,0,0,0,0,0,
+    2,0,0,0,4,0,0,0,67,0,0,0,115,50,0,0,
+    0,116,0,124,0,131,1,1,0,116,1,131,0,125,1,116,
+    2,106,3,160,4,116,5,106,6,124,1,142,0,103,1,161,
+    1,1,0,116,2,106,7,160,8,116,9,161,1,1,0,100,
+    1,83,0,41,2,122,41,73,110,115,116,97,108,108,32,116,
+    104,101,32,112,97,116,104,45,98,97,115,101,100,32,105,109,
+    112,111,114,116,32,99,111,109,112,111,110,101,110,116,115,46,
+    78,41,10,114,90,1,0,0,114,185,0,0,0,114,8,0,
+    0,0,114,52,1,0,0,114,168,0,0,0,114,62,1,0,
+    0,114,77,1,0,0,218,9,109,101,116,97,95,112,97,116,
+    104,114,187,0,0,0,114,46,1,0,0,41,2,114,89,1,
+    0,0,90,17,115,117,112,112,111,114,116,101,100,95,108,111,
+    97,100,101,114,115,114,3,0,0,0,114,3,0,0,0,114,
+    6,0,0,0,218,8,95,105,110,115,116,97,108,108,97,6,
+    0,0,115,8,0,0,0,0,2,8,1,6,1,20,1,114,
+    92,1,0,0,41,1,114,60,0,0,0,41,1,78,41,3,
+    78,78,78,41,2,114,73,0,0,0,114,73,0,0,0,41,
+    1,84,41,1,78,41,1,78,41,63,114,128,0,0,0,114,
+    12,0,0,0,90,37,95,67,65,83,69,95,73,78,83,69,
+    78,83,73,84,73,86,69,95,80,76,65,84,70,79,82,77,
+    83,95,66,89,84,69,83,95,75,69,89,114,11,0,0,0,
+    114,13,0,0,0,114,20,0,0,0,114,27,0,0,0,114,
+    29,0,0,0,114,38,0,0,0,114,47,0,0,0,114,49,
+    0,0,0,114,53,0,0,0,114,54,0,0,0,114,56,0,
+    0,0,114,59,0,0,0,114,69,0,0,0,218,4,116,121,
+    112,101,218,8,95,95,99,111,100,101,95,95,114,163,0,0,
+    0,114,18,0,0,0,114,149,0,0,0,114,17,0,0,0,
+    114,24,0,0,0,114,237,0,0,0,114,92,0,0,0,114,
+    88,0,0,0,114,102,0,0,0,114,89,0,0,0,90,23,
+    68,69,66,85,71,95,66,89,84,69,67,79,68,69,95,83,
+    85,70,70,73,88,69,83,90,27,79,80,84,73,77,73,90,
+    69,68,95,66,89,84,69,67,79,68,69,95,83,85,70,70,
+    73,88,69,83,114,98,0,0,0,114,103,0,0,0,114,109,
+    0,0,0,114,113,0,0,0,114,115,0,0,0,114,137,0,
+    0,0,114,144,0,0,0,114,153,0,0,0,114,157,0,0,
+    0,114,159,0,0,0,114,166,0,0,0,114,171,0,0,0,
+    114,172,0,0,0,114,177,0,0,0,218,6,111,98,106,101,
+    99,116,114,186,0,0,0,114,191,0,0,0,114,192,0,0,
+    0,114,209,0,0,0,114,222,0,0,0,114,240,0,0,0,
+    114,10,1,0,0,114,16,1,0,0,114,22,1,0,0,114,
+    253,0,0,0,114,23,1,0,0,114,44,1,0,0,114,46,
+    1,0,0,114,62,1,0,0,114,82,1,0,0,114,185,0,
+    0,0,114,90,1,0,0,114,92,1,0,0,114,3,0,0,
+    0,114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,
+    218,8,60,109,111,100,117,108,101,62,1,0,0,0,115,126,
+    0,0,0,4,22,4,1,4,1,2,1,2,255,4,4,8,
+    17,8,5,8,5,8,6,8,6,8,12,8,10,8,9,8,
+    5,8,7,8,9,10,22,10,127,0,13,16,1,12,2,4,
+    1,4,2,6,2,6,2,8,2,16,71,8,40,8,19,8,
+    12,8,12,8,28,8,17,8,33,8,28,8,24,10,13,10,
+    10,10,11,8,14,6,3,4,1,2,255,12,68,14,64,14,
+    29,16,127,0,17,14,72,18,45,18,26,4,3,18,53,14,
+    63,14,42,14,127,0,20,14,127,0,22,10,23,8,11,8,
+    65,
 };


From webhook-mailer at python.org  Thu Sep 12 11:43:18 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 15:43:18 -0000
Subject: [Python-checkins] bpo-37908: Add an example of
 ArgumentParser.exit() (GH-15455)
Message-ID: <mailman.681.1568302998.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6dc3e61c511d3e13ce4eac86c8b6abdff58e3617
commit: 6dc3e61c511d3e13ce4eac86c8b6abdff58e3617
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T08:43:14-07:00
summary:

bpo-37908: Add an example of ArgumentParser.exit() (GH-15455)


Co-Authored-By: Brandt Bucher <brandtbucher at gmail.com>
(cherry picked from commit b1a2abdb06408ffc4f13d6ff50351ad49c99afc0)

Co-authored-by: Hai Shi <shihai1992 at gmail.com>

files:
M Doc/library/argparse.rst

diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index 368b1cfebf05..56bd64172f80 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -2001,7 +2001,14 @@ Exiting methods
 .. method:: ArgumentParser.exit(status=0, message=None)
 
    This method terminates the program, exiting with the specified *status*
-   and, if given, it prints a *message* before that.
+   and, if given, it prints a *message* before that. The user can override
+   this method to handle these steps differently::
+
+    class ErrorCatchingArgumentParser(argparse.ArgumentParser):
+        def exit(self, status=0, message=None):
+            if status:
+                raise Exception(f'Exiting because of an error: {message}')
+            exit(status)
 
 .. method:: ArgumentParser.error(message)
 


From webhook-mailer at python.org  Thu Sep 12 11:46:41 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 12 Sep 2019 15:46:41 -0000
Subject: [Python-checkins] closes bpo-37803: pdb: fix handling of options
 (--help / --version) (GH-15193)
Message-ID: <mailman.682.1568303202.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/855df7f273c3988c72f01e51ba57091887ec38b2
commit: 855df7f273c3988c72f01e51ba57091887ec38b2
branch: master
author: Daniel Hahler <github at thequod.de>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-12T16:46:37+01:00
summary:

closes bpo-37803: pdb: fix handling of options (--help / --version) (GH-15193)

The "--" should not be included with long options passed to
getopt.getopt.

Fixes https://bugs.python.org/issue37803

files:
A Misc/NEWS.d/next/Tools-Demos/2019-09-12-16-15-55.bpo-37803.chEizy.rst
M Lib/pdb.py

diff --git a/Lib/pdb.py b/Lib/pdb.py
index 8c1c96163ed9..8639204891cb 100755
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -1660,7 +1660,7 @@ def help():
 def main():
     import getopt
 
-    opts, args = getopt.getopt(sys.argv[1:], 'mhc:', ['--help', '--command='])
+    opts, args = getopt.getopt(sys.argv[1:], 'mhc:', ['help', 'command='])
 
     if not args:
         print(_usage)
diff --git a/Misc/NEWS.d/next/Tools-Demos/2019-09-12-16-15-55.bpo-37803.chEizy.rst b/Misc/NEWS.d/next/Tools-Demos/2019-09-12-16-15-55.bpo-37803.chEizy.rst
new file mode 100644
index 000000000000..5c0eedb93709
--- /dev/null
+++ b/Misc/NEWS.d/next/Tools-Demos/2019-09-12-16-15-55.bpo-37803.chEizy.rst
@@ -0,0 +1 @@
+pdb's ``--help`` and ``--version`` long options now work.


From webhook-mailer at python.org  Thu Sep 12 11:59:42 2019
From: webhook-mailer at python.org (Eric Snow)
Date: Thu, 12 Sep 2019 15:59:42 -0000
Subject: [Python-checkins] bpo-36876: Fix the globals checker tool.
 (gh-16058)
Message-ID: <mailman.683.1568303982.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/088b63ea7a8331a3e34bc93c3b873c60354b4fad
commit: 088b63ea7a8331a3e34bc93c3b873c60354b4fad
branch: master
author: Eric Snow <ericsnowcurrently at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T16:59:38+01:00
summary:

bpo-36876: Fix the globals checker tool. (gh-16058)

files:
A TODO
M Tools/c-analyzer/c_globals/supported.py
M Tools/c-analyzer/known.tsv

diff --git a/TODO b/TODO
new file mode 100644
index 000000000000..829daba29146
--- /dev/null
+++ b/TODO
@@ -0,0 +1,1072 @@
+
+# allocator (16)
+Objects/obmalloc.c:_PyMem                                        static PyMemAllocatorEx _PyMem
+Objects/obmalloc.c:_PyMem_Debug                                  static struct { debug_alloc_api_t raw; debug_alloc_api_t mem; debug_alloc_api_t obj; } _PyMem_Debug
+Objects/obmalloc.c:_PyMem_Raw                                    static PyMemAllocatorEx _PyMem_Raw
+Objects/obmalloc.c:_PyObject                                     static PyMemAllocatorEx _PyObject
+Objects/obmalloc.c:_PyObject_Arena                               static PyObjectArenaAllocator _PyObject_Arena
+Objects/obmalloc.c:_Py_tracemalloc_config                        struct _PyTraceMalloc_Config _Py_tracemalloc_config
+Objects/obmalloc.c:arenas                                        static struct arena_object* arenas
+Objects/obmalloc.c:maxarenas                                     static uint maxarenas
+Objects/obmalloc.c:narenas_currently_allocated                   static size_t narenas_currently_allocated
+Objects/obmalloc.c:narenas_highwater                             static size_t narenas_highwater
+Objects/obmalloc.c:nfp2lasta                                     static struct arena_object* nfp2lasta[MAX_POOLS_IN_ARENA + 1]
+Objects/obmalloc.c:ntimes_arena_allocated                        static size_t ntimes_arena_allocated
+Objects/obmalloc.c:unused_arena_objects                          static struct arena_object* unused_arena_objects
+Objects/obmalloc.c:usable_arenas                                 static struct arena_object* usable_arenas
+Objects/obmalloc.c:usedpools                                     static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8]
+Objects/obmalloc.c:new_arena():debug_stats                       static int debug_stats
+
+
+# counters
+Modules/_abc.c:abc_invalidation_counter                          static unsigned long long abc_invalidation_counter
+Objects/bytesobject.c:_Py_null_strings                           Py_ssize_t _Py_null_strings
+Objects/bytesobject.c:_Py_onel_strings                           Py_ssize_t _Py_one_strings
+
+
+# constants (effectively)
+Objects/dictobject.c:empty_keys_struct                           static PyDictKeysObject empty_keys_struct
+
+
+# "initialized"
+Python/fileutils.c:_Py_open_cloexec_works                        int _Py_open_cloexec_works
+
+
+# freelists
+Objects/dictobject.c:keys_free_list                              static PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]
+Objects/dictobject.c:numfreekeys                                 static int numfreekeys
+
+
+# other non-object (43)
+Modules/_tracemalloc.c:allocators                                static struct { PyMemAllocatorEx mem; PyMemAllocatorEx raw; PyMemAllocatorEx obj; } allocators
+Modules/_tracemalloc.c:tables_lock                               static PyThread_type_lock tables_lock
+Modules/_tracemalloc.c:tracemalloc_filenames                     static _Py_hashtable_t *tracemalloc_filenames
+Modules/_tracemalloc.c:tracemalloc_peak_traced_memory            static size_t tracemalloc_peak_traced_memory
+Modules/_tracemalloc.c:tracemalloc_reentrant_key                 static Py_tss_t tracemalloc_reentrant_key
+Modules/_tracemalloc.c:tracemalloc_tracebacks                    static _Py_hashtable_t *tracemalloc_tracebacks
+Modules/_tracemalloc.c:tracemalloc_traced_memory                 static size_t tracemalloc_traced_memory
+Modules/_tracemalloc.c:tracemalloc_traces                        static _Py_hashtable_t *tracemalloc_traces
+Modules/faulthandler.c:old_stack                                 static stack_t old_stack
+Modules/faulthandler.c:stack                                     static stack_t stack
+Modules/faulthandler.c:faulthandler_dump_traceback():reentrant   static volatile int reentrant
+Modules/posixmodule.c:initialized                                static int initialized
+Modules/signalmodule.c:initialized                               static int initialized
+Modules/timemodule.c:initialized                                 static int initialized
+Objects/dictobject.c:pydict_global_version                       static uint64_t pydict_global_version
+Objects/floatobject.c:detected_double_format                     static float_format_type detected_double_format
+Objects/floatobject.c:detected_float_format                      static float_format_type detected_float_format
+Objects/floatobject.c:double_format                              static float_format_type double_format
+Objects/floatobject.c:float_format                               static float_format_type
+Objects.longobject.c:_Py_quick_int_allocs                        Py_ssize_t _Py_quick_int_allocs
+Objects.longobject.c:_Py_quick_neg_int_allocs                    Py_ssize_t _Py_quick_neg_int_allocs
+Objects/moduleobject.c:max_module_number                         static Py_ssize_t max_module_number
+Objects/object.c:_Py_RefTotal                                    Py_ssize_t _Py_RefTotal
+Objects/tupleobject.c:_Py_fast_tuple_allocs                      Py_ssize_t _Py_fast_tuple_allocs
+Objects/tupleobject.c:_Py_tuple_zero_allocs                      Py_ssize_t _Py_tuple_zero_allocs
+Objects/typeobject.c:next_version_tag                            static unsigned int next_version_tag
+Parser/listnode.c:atbol                                          static int atbol
+Parser/listnode.c:level                                          static int level
+Python/Python-ast.c:init_types():initialized                     static int initialized
+Python/bootstrap_hash.c:urandom_cache                            static struct { int fd; dev_t st_dev; ino_t st_ino; } urandom_cache
+Python/ceval.c:_Py_CheckRecursionLimit                           int _Py_CheckRecursionLimit
+Python/ceval.c:lltrace                                           static int lltrace
+Python/ceval.c:make_pending_calls():busy                         static int busy
+Python/dynload_shlib.c:handles                                   static struct { dev_t dev; ino_t ino; void *handle; } handles[128]
+Python/dynload_shlib.c:nhandles                                  static int nhandles
+Python/import.c:import_lock                                      static PyThread_type_lock import_lock
+Python/import.c:import_lock_level                                static int import_lock_level
+Python/import.c:import_lock_thread                               static unsigned long import_lock_thread
+Python/import.c:import_find_and_load():accumulated               static _PyTime_t accumulated
+Python/import.c:import_find_and_load():header                    static int header
+Python/import.c:import_find_and_load():import_level              static int import_level
+Python/pylifecycle.c:_Py_UnhandledKeyboardInterrupt              int _Py_UnhandledKeyboardInterrupt
+Python/pylifecycle.c:fatal_error():reentrant                     static int reentrant
+
+
+#######################################
+# PyObject (960)
+
+# freelists (10 + 10)
+Modules/_collectionsmodule.c:freeblocks                          static block *freeblocks[MAXFREEBLOCKS]
+Modules/_collectionsmodule.c:numfreeblocks                       static Py_ssize_t numfreeblocks
+Objects/dictobject.c:free_list                                   static PyDictObject *free_list[PyDict_MAXFREELIST]
+Objects/dictobject.c:numfree                                     static int numfree
+Objects/exceptions.c:memerrors_freelist                          static PyBaseExceptionObject *memerrors_freelist
+Objects/exceptions.c:memerrors_numfree                           static int memerrors_numfree
+Objects/floatobject.c:free_list                                  static PyFloatObject *free_list
+Objects/floatobject.c:numfree                                    static int numfree
+Objects/frameobject.c:free_list                                  static PyFrameObject *free_list
+Objects/frameobject.c:numfree                                    static int numfree
+Objects/genobject.c:ag_asend_freelist                            static PyAsyncGenASend *ag_asend_freelist[_PyAsyncGen_MAXFREELIST]
+Objects/genobject.c:ag_asend_freelist_free                       static int ag_asend_freelist_free
+Objects/genobject.c:ag_value_freelist                            static _PyAsyncGenWrappedValue *ag_value_freelist[_PyAsyncGen_MAXFREELIST]
+Objects/genobject.c:ag_value_freelist_free                       static int ag_value_freelist_free
+Objects/listobject.c:free_list                                   static PyListObject *free_list[PyList_MAXFREELIST]
+Objects/listobject.c:numfree                                     static int numfree
+Objects/tupleobject.c:free_list                                  static PyTupleObject *free_list[PyTuple_MAXSAVESIZE]
+Objects/tupleobject.c:numfree                                    static int numfree[PyTuple_MAXSAVESIZE]
+Python/context.c:ctx_freelist                                    static PyContext *ctx_freelist
+Python/context.c:ctx_freelist_len                                static int ctx_freelist_len
+
+
+# singletons (7)
+Objects/boolobject.c:_Py_FalseStruct                             static struct _longobject _Py_FalseStruct
+Objects/boolobject.c:_Py_TrueStruct                              static struct _longobject _Py_TrueStruct
+Objects/boolobject.c:false_str                                   static PyObject *false_str
+Objects/boolobject.c:true_str                                    static PyObject *true_str
+Objects/object.c:_Py_NoneStruct                                  PyObject _Py_NoneStruct
+Objects/object.c:_Py_NotImplementedStruct                        PyObject _Py_NotImplementedStruct
+Objects/sliceobject.c:_Py_EllipsisObject                         PyObject _Py_EllipsisObject
+
+
+# module vars (9)
+Modules/_functoolsmodule.c:kwd_mark                              static PyObject *kwd_mark
+Modules/_localemodule.c:Error                                    static PyObject *Error
+Modules/_threadmodule.c:ThreadError                              static PyObject *ThreadError
+Modules/_tracemalloc.c:unknown_filename                          static PyObject *unknown_filename
+Modules/signalmodule.c:DefaultHandler                            static PyObject *DefaultHandler
+Modules/signalmodule.c:IgnoreHandler                             static PyObject *IgnoreHandler
+Modules/signalmodule.c:IntHandler                                static PyObject *IntHandler
+Modules/signalmodule.c:ItimerError                               static PyObject *ItimerError
+Objects/exceptions.c:errnomap                                    static PyObject *errnomap
+
+
+# other (non-cache) (5)
+Modules/_tracemalloc.c:tracemalloc_traceback                     static traceback_t *tracemalloc_traceback
+Modules/faulthandler.c:fatal_error                               static struct { int enabled; PyObject *file; int fd; int all_threads; PyInterpreterState *interp; void *exc_handler; } fatal_error
+Modules/faulthandler.c:thread                                    static struct { PyObject *file; int fd; PY_TIMEOUT_T timeout_us; int repeat; PyInterpreterState *interp; int exit; char *header; size_t header_len; PyThread_type_lock cancel_event; PyThread_type_lock running; } thread
+Modules/signalmodule.c:Handlers                                  static volatile struct { _Py_atomic_int tripped; PyObject *func; } Handlers[NSIG]
+Objects/setobject.c:_dummy_struct                                static PyObject _dummy_struct
+
+
+# caches (5)
+Modules/posixmodule.c:posix_putenv_garbage                       static PyObject *posix_putenv_garbage
+Objects/sliceobject.c:slice_cache                                static PySliceObject *slice_cache
+Objects/typeobject.c:method_cache                                static struct method_cache_entry method_cache[1 << MCACHE_SIZE_EXP]
+Objects/unicodeobject.c:interned                                 static PyObject *interned
+Python/import.c:extensions                                       static PyObject *extensions
+
+
+# cached constants - non-str (15)
+Modules/_io/_iomodule.c:_PyIO_empty_bytes                        PyObject *_PyIO_empty_bytes
+Modules/_io/bufferedio.c:_PyIO_trap_eintr():eintr_int            static PyObject *eintr_int
+Modules/posixmodule.c:billion                                    static PyObject *billion
+Modules/posixmodule.c:wait_helper():struct_rusage                static PyObject *struct_rusage
+Objects/bytesobject.c:characters                                 static PyBytesObject *characters[UCHAR_MAX + 1]
+Objects/bytesobject.c:nullstring                                 static PyBytesObject *nullstring
+Objects/codeobject.c:PyCode_NewEmpty():nulltuple                 static PyObject *nulltuple
+Objects/dictobject.c:empty_values                                static PyObject *empty_values[1]
+Objects/listobject.c:indexerr                                    static PyObject *indexerr
+Objects/longobject.c:_PyLong_One                                 PyObject *_PyLong_One
+Objects/longobject.c:_PyLong_Zero                                PyObject *_PyLong_Zero
+Objects/longobject.c:small_ints                                  static PyLongObject small_ints[NSMALLNEGINTS + NSMALLPOSINTS]
+Objects/setobject.c:emptyfrozenset                               static PyObject *emptyfrozenset
+Python/context.c:_token_missing                                  static PyObject *_token_missing
+Python/hamt.c:_empty_hamt                                        static PyHamtObject *_empty_hamt
+
+
+# cached constants - str (441)
+Modules/_io/_iomodule.c:_PyIO_empty_str                          PyObject *_PyIO_empty_str
+Modules/_io/_iomodule.c:_PyIO_str_close                          PyObject *_PyIO_str_close
+Modules/_io/_iomodule.c:_PyIO_str_closed                         PyObject *_PyIO_str_closed
+Modules/_io/_iomodule.c:_PyIO_str_decode                         PyObject *_PyIO_str_decode
+Modules/_io/_iomodule.c:_PyIO_str_encode                         PyObject *_PyIO_str_encode
+Modules/_io/_iomodule.c:_PyIO_str_fileno                         PyObject *_PyIO_str_fileno
+Modules/_io/_iomodule.c:_PyIO_str_flush                          PyObject *_PyIO_str_flush
+Modules/_io/_iomodule.c:_PyIO_str_getstate                       PyObject *_PyIO_str_getstate
+Modules/_io/_iomodule.c:_PyIO_str_isatty                         PyObject *_PyIO_str_isatty
+Modules/_io/_iomodule.c:_PyIO_str_newlines                       PyObject *_PyIO_str_newlines
+Modules/_io/_iomodule.c:_PyIO_str_nl                             PyObject *_PyIO_str_nl
+Modules/_io/_iomodule.c:_PyIO_str_peek                           PyObject *_PyIO_str_peek
+Modules/_io/_iomodule.c:_PyIO_str_read                           PyObject *_PyIO_str_read
+Modules/_io/_iomodule.c:_PyIO_str_read1                          PyObject *_PyIO_str_read1
+Modules/_io/_iomodule.c:_PyIO_str_readable                       PyObject *_PyIO_str_readable
+Modules/_io/_iomodule.c:_PyIO_str_readall                        PyObject *_PyIO_str_readall
+Modules/_io/_iomodule.c:_PyIO_str_readinto                       PyObject *_PyIO_str_readinto
+Modules/_io/_iomodule.c:_PyIO_str_readline                       PyObject *_PyIO_str_readline
+Modules/_io/_iomodule.c:_PyIO_str_reset                          PyObject *_PyIO_str_reset
+Modules/_io/_iomodule.c:_PyIO_str_seek                           PyObject *_PyIO_str_seek
+Modules/_io/_iomodule.c:_PyIO_str_seekable                       PyObject *_PyIO_str_seekable
+Modules/_io/_iomodule.c:_PyIO_str_setstate                       PyObject *_PyIO_str_setstate
+Modules/_io/_iomodule.c:_PyIO_str_tell                           PyObject *_PyIO_str_tell
+Modules/_io/_iomodule.c:_PyIO_str_truncate                       PyObject *_PyIO_str_truncate
+Modules/_io/_iomodule.c:_PyIO_str_writable                       PyObject *_PyIO_str_writable
+Modules/_io/_iomodule.c:_PyIO_str_write                          PyObject *_PyIO_str_write
+Modules/_threadmodule.c:str_dict                                 static PyObject *str_dict
+Modules/gcmodule.c:gc_str                                        static PyObject *gc_str
+Objects/classobject.c:instancemethod_get_doc():docstr            static PyObject *docstr
+Objects/classobject.c:method_get_doc():docstr                    static PyObject *docstr
+Objects/codeobject.c:PyCode_NewEmpty():emptystring               static PyObject *emptystring
+Objects/exceptions.c:_check_for_legacy_statements():exec_prefix  static PyObject *exec_prefix
+Objects/exceptions.c:_check_for_legacy_statements():print_prefix static PyObject *print_prefix
+Objects/funcobject.c:PyFunction_NewWithQualName():__name__       static PyObject *__name__
+Objects/typeobject.c:object___reduce_ex___impl():objreduce       static PyObject *objreduce
+Objects/typeobject.c:resolve_slotdups():pname                    static PyObject *pname
+Objects/unicodeobject.c:unicode_empty                            static PyObject *unicode_empty
+Objects/unicodeobject.c:unicode_latin1                           static PyObject *unicode_latin1[256]
+Python/_warnings.c:is_internal_frame():bootstrap_string          static PyObject *bootstrap_string
+Python/_warnings.c:is_internal_frame():importlib_string          static PyObject *importlib_string
+Python/ast.c:u_kind                                              static PyObject *u_kind
+Python/ast_unparse.c:_str_close_br                               static PyObject *_str_close_br
+Python/ast_unparse.c:_str_dbl_close_br                           static PyObject *_str_dbl_close_br
+Python/ast_unparse.c:_str_dbl_open_br                            static PyObject *_str_dbl_open_br
+Python/ast_unparse.c:_str_open_br                                static PyObject *_str_open_br
+Python/compile.c:__annotations__                                 static PyObject *__annotations__
+Python/compile.c:__doc__                                         static PyObject *__doc__
+Python/compile.c:compiler_dictcomp():name                        static identifier name
+Python/compile.c:compiler_from_import():empty_string             static PyObject *empty_string
+Python/compile.c:compiler_genexp():name                          static identifier name
+Python/compile.c:compiler_lambda():name                          static identifier name
+Python/compile.c:compiler_listcomp():name                        static identifier name
+Python/compile.c:compiler_setcomp():name                         static identifier name
+Python/compile.c:compiler_visit_annotations():return_str         static identifier return_str
+Python/import.c:PyImport_Import():builtins_str                   static PyObject *builtins_str
+Python/import.c:PyImport_Import():import_str                     static PyObject *import_str
+Python/import.c:PyImport_Import():silly_list                     static PyObject *silly_list
+Python/sysmodule.c:whatstrings                                   static PyObject *whatstrings[8]
+Python/sysmodule.c:sys_displayhook():newline                     static PyObject *newline
+Objects/typeobject.c:object_new():comma_id                       _Py_static_string(comma_id, "", "")
+Objects/typeobject.c:slot_nb_add():op_id                         _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_add():rop_id                        _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_and():op_id                         _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_and():rop_id                        _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_divmod():op_id                      _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_divmod():rop_id                     _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_floor_divide():op_id                _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_floor_divide():rop_id               _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_lshift():op_id                      _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_lshift():rop_id                     _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_matrix_multiply():op_id             _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_matrix_multiply():rop_id            _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_multiply():op_id                    _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_multiply():rop_id                   _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_or():op_id                          _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_or():rop_id                         _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_power_binary():op_id                _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_power_binary():rop_id               _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_remainder():op_id                   _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_remainder():rop_id                  _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_rshift():op_id                      _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_rshift():rop_id                     _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_subtract():op_id                    _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_subtract():rop_id                   _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_true_divide():op_id                 _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_true_divide():rop_id                _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_xor():op_id                         _Py_static_string(op_id, OPSTR)
+Objects/typeobject.c:slot_nb_xor():rop_id                        _Py_static_string(op_id, OPSTR)
+Python/compile.c:compiler_set_qualname():dot                     _Py_static_string(dot, ""."")
+Python/compile.c:compiler_set_qualname():dot_locals              _Py_static_string(dot_locals, "".<locals>"")
+Python/pythonrun.c:PyId_string                                   _Py_static_string(PyId_string, ""<string>"")
+Modules/_abc.c:PyId___abstractmethods__                          _Py_IDENTIFIER(__abstractmethods__)
+Modules/_abc.c:PyId___bases__                                    _Py_IDENTIFIER(__bases__)
+Modules/_abc.c:PyId___class__                                    _Py_IDENTIFIER(__class__)
+Modules/_abc.c:PyId___dict__                                     _Py_IDENTIFIER(__dict__)
+Modules/_abc.c:PyId___subclasscheck__                            _Py_IDENTIFIER(__subclasscheck__)
+Modules/_abc.c:PyId___subclasshook__                             _Py_IDENTIFIER(__subclasshook__)
+Modules/_abc.c:PyId__abc_impl                                    _Py_IDENTIFIER(_abc_impl)
+Modules/_collectionsmodule.c:_count_elements():PyId___setitem__  _Py_IDENTIFIER(__setitem__)
+Modules/_collectionsmodule.c:_count_elements():PyId_get          _Py_IDENTIFIER(get)
+Modules/_collectionsmodule.c:defdict_reduce():PyId_items         _Py_IDENTIFIER(items)
+Modules/_dbmmodule.c:dbm__exit__():PyId_close                    _Py_IDENTIFIER(close)
+Modules/_gdbmmodule.c:dbm__exit__():PyId_close                   _Py_IDENTIFIER(close)
+Modules/_io/_iomodule.c:_io_open_impl():PyId__blksize            _Py_IDENTIFIER(_blksize)
+Modules/_io/_iomodule.c:_io_open_impl():PyId_isatty              _Py_IDENTIFIER(isatty)
+Modules/_io/_iomodule.c:_io_open_impl():PyId_mode                _Py_IDENTIFIER(mode)
+Modules/_io/bufferedio.c:PyId__dealloc_warn                      _Py_IDENTIFIER(_dealloc_warn)
+Modules/_io/bufferedio.c:PyId_close                              _Py_IDENTIFIER(close)
+Modules/_io/bufferedio.c:PyId_flush                              _Py_IDENTIFIER(flush)
+Modules/_io/bufferedio.c:PyId_isatty                             _Py_IDENTIFIER(isatty)
+Modules/_io/bufferedio.c:PyId_mode                               _Py_IDENTIFIER(mode)
+Modules/_io/bufferedio.c:PyId_name                               _Py_IDENTIFIER(name)
+Modules/_io/bufferedio.c:PyId_peek                               _Py_IDENTIFIER(peek)
+Modules/_io/bufferedio.c:PyId_read                               _Py_IDENTIFIER(read)
+Modules/_io/bufferedio.c:PyId_read1                              _Py_IDENTIFIER(read1)
+Modules/_io/bufferedio.c:PyId_readable                           _Py_IDENTIFIER(readable)
+Modules/_io/bufferedio.c:PyId_readinto                           _Py_IDENTIFIER(readinto)
+Modules/_io/bufferedio.c:PyId_readinto1                          _Py_IDENTIFIER(readinto1)
+Modules/_io/bufferedio.c:PyId_writable                           _Py_IDENTIFIER(writable)
+Modules/_io/bufferedio.c:PyId_write                              _Py_IDENTIFIER(write)
+Modules/_io/fileio.c:PyId_name                                   _Py_IDENTIFIER(name)
+Modules/_io/iobase.c:PyId___IOBase_closed                        _Py_IDENTIFIER(__IOBase_closed)
+Modules/_io/iobase.c:PyId_read                                   _Py_IDENTIFIER(read)
+Modules/_io/iobase.c:_io__IOBase_tell_impl():PyId_seek           _Py_IDENTIFIER(seek)
+Modules/_io/iobase.c:_io__RawIOBase_read_impl():PyId_readall     _Py_IDENTIFIER(readall)
+Modules/_io/iobase.c:iobase_finalize():PyId__finalizing          _Py_IDENTIFIER(_finalizing)
+Modules/_io/textio.c:PyId__dealloc_warn                          _Py_IDENTIFIER(_dealloc_warn)
+Modules/_io/textio.c:PyId_close                                  _Py_IDENTIFIER(close)
+Modules/_io/textio.c:PyId_decode                                 _Py_IDENTIFIER(decode)
+Modules/_io/textio.c:PyId_fileno                                 _Py_IDENTIFIER(fileno)
+Modules/_io/textio.c:PyId_flush                                  _Py_IDENTIFIER(flush)
+Modules/_io/textio.c:PyId_getpreferredencoding                   _Py_IDENTIFIER(getpreferredencoding)
+Modules/_io/textio.c:PyId_isatty                                 _Py_IDENTIFIER(isatty)
+Modules/_io/textio.c:PyId_mode                                   _Py_IDENTIFIER(mode)
+Modules/_io/textio.c:PyId_name                                   _Py_IDENTIFIER(name)
+Modules/_io/textio.c:PyId_raw                                    _Py_IDENTIFIER(raw)
+Modules/_io/textio.c:PyId_read                                   _Py_IDENTIFIER(read)
+Modules/_io/textio.c:PyId_readable                               _Py_IDENTIFIER(readable)
+Modules/_io/textio.c:PyId_replace                                _Py_IDENTIFIER(replace)
+Modules/_io/textio.c:PyId_reset                                  _Py_IDENTIFIER(reset)
+Modules/_io/textio.c:PyId_seek                                   _Py_IDENTIFIER(seek)
+Modules/_io/textio.c:PyId_seekable                               _Py_IDENTIFIER(seekable)
+Modules/_io/textio.c:PyId_setstate                               _Py_IDENTIFIER(setstate)
+Modules/_io/textio.c:PyId_strict                                 _Py_IDENTIFIER(strict)
+Modules/_io/textio.c:PyId_tell                                   _Py_IDENTIFIER(tell)
+Modules/_io/textio.c:PyId_writable                               _Py_IDENTIFIER(writable)
+Modules/_operator.c:methodcaller_reduce():PyId_partial           _Py_IDENTIFIER(partial)
+Modules/_pickle.c:do_append():PyId_extend                        _Py_IDENTIFIER(extend)
+Modules/_pickle.c:load_build():PyId___setstate__                 _Py_IDENTIFIER(__setstate__)
+Modules/_threadmodule.c:PyId_flush                               _Py_IDENTIFIER(flush)
+Modules/_threadmodule.c:PyId_stderr                              _Py_IDENTIFIER(stderr)
+Modules/arraymodule.c:array_arrayiterator___reduce___impl():PyId_iter _Py_IDENTIFIER(iter)
+Modules/faulthandler.c:PyId_enable                               _Py_IDENTIFIER(enable)
+Modules/faulthandler.c:PyId_fileno                               _Py_IDENTIFIER(fileno)
+Modules/faulthandler.c:PyId_flush                                _Py_IDENTIFIER(flush)
+Modules/faulthandler.c:PyId_stderr                               _Py_IDENTIFIER(stderr)
+Modules/itertoolsmodule.c:itertools_tee_impl():PyId___copy__     _Py_IDENTIFIER(__copy__)
+Modules/itertoolsmodule.c:zip_longest_new():PyId_fillvalue       _Py_IDENTIFIER(fillvalue)
+Modules/main.c:pymain_sys_path_add_path0():PyId_path             _Py_IDENTIFIER(path)
+Modules/posixmodule.c:DirEntry_test_mode():PyId_st_mode          _Py_IDENTIFIER(st_mode)
+Modules/posixmodule.c:PyOS_FSPath():PyId___fspath__              _Py_IDENTIFIER(__fspath__)
+Modules/posixmodule.c:path_converter():PyId___fspath__           _Py_IDENTIFIER(__fspath__)
+Modules/posixmodule.c:wait_helper():PyId_struct_rusage           _Py_IDENTIFIER(struct_rusage)
+Modules/timemodule.c:time_strptime():PyId__strptime_time         _Py_IDENTIFIER(_strptime_time)
+Objects/abstract.c:PyMapping_Items():PyId_items                  _Py_IDENTIFIER(items)
+Objects/abstract.c:PyMapping_Keys():PyId_keys                    _Py_IDENTIFIER(keys)
+Objects/abstract.c:PyMapping_Values():PyId_values                _Py_IDENTIFIER(values)
+Objects/abstract.c:PyNumber_Long():PyId___trunc__                _Py_IDENTIFIER(__trunc__)
+Objects/abstract.c:PyObject_Format():PyId___format__             _Py_IDENTIFIER(__format__)
+Objects/abstract.c:PyObject_GetItem():PyId___class_getitem__     _Py_IDENTIFIER(__class_getitem__)
+Objects/abstract.c:PyObject_IsInstance():PyId___instancecheck__  _Py_IDENTIFIER(__instancecheck__)
+Objects/abstract.c:PyObject_IsSubclass():PyId___subclasscheck__  _Py_IDENTIFIER(__subclasscheck__)
+Objects/abstract.c:PyObject_LengthHint():PyId___length_hint__    _Py_IDENTIFIER(__length_hint__)
+Objects/abstract.c:abstract_get_bases():PyId___bases__           _Py_IDENTIFIER(__bases__)
+Objects/abstract.c:recursive_isinstance():PyId___class__         _Py_IDENTIFIER(__class__)
+Objects/bytearrayobject.c:_common_reduce():PyId___dict__         _Py_IDENTIFIER(__dict__)
+Objects/bytearrayobject.c:bytearrayiter_reduce():PyId_iter       _Py_IDENTIFIER(iter)
+Objects/bytesobject.c:bytes_new():PyId___bytes__                 _Py_IDENTIFIER(__bytes__)
+Objects/bytesobject.c:format_obj():PyId___bytes__                _Py_IDENTIFIER(__bytes__)
+Objects/bytesobject.c:striter_reduce():PyId_iter                 _Py_IDENTIFIER(iter)
+Objects/classobject.c:PyId___name__                              _Py_IDENTIFIER(__name__)
+Objects/classobject.c:PyId___qualname__                          _Py_IDENTIFIER(__qualname__)
+Objects/classobject.c:method_reduce():PyId_getattr               _Py_IDENTIFIER(getattr)
+Objects/complexobject.c:try_complex_special_method():PyId___complex__ _Py_IDENTIFIER(__complex__)
+Objects/descrobject.c:calculate_qualname():PyId___qualname__     _Py_IDENTIFIER(__qualname__)
+Objects/descrobject.c:descr_reduce():PyId_getattr                _Py_IDENTIFIER(getattr)
+Objects/descrobject.c:mappingproxy_copy():PyId_copy              _Py_IDENTIFIER(copy)
+Objects/descrobject.c:mappingproxy_get():PyId_get                _Py_IDENTIFIER(get)
+Objects/descrobject.c:mappingproxy_items():PyId_items            _Py_IDENTIFIER(items)
+Objects/descrobject.c:mappingproxy_keys():PyId_keys              _Py_IDENTIFIER(keys)
+Objects/descrobject.c:mappingproxy_values():PyId_values          _Py_IDENTIFIER(values)
+Objects/descrobject.c:property_init_impl():PyId___doc__          _Py_IDENTIFIER(__doc__)
+Objects/descrobject.c:wrapper_reduce():PyId_getattr              _Py_IDENTIFIER(getattr)
+Objects/dictobject.c:_PyDictView_Intersect():PyId_intersection_update _Py_IDENTIFIER(intersection_update)
+Objects/dictobject.c:dict_subscript():PyId___missing__           _Py_IDENTIFIER(__missing__)
+Objects/dictobject.c:dict_update_common():PyId_keys              _Py_IDENTIFIER(keys)
+Objects/dictobject.c:dictiter_reduce():PyId_iter                 _Py_IDENTIFIER(iter)
+Objects/dictobject.c:dictviews_or():PyId_update                  _Py_IDENTIFIER(update)
+Objects/dictobject.c:dictviews_sub():PyId_difference_update      _Py_IDENTIFIER(difference_update)
+Objects/dictobject.c:dictviews_xor():PyId_symmetric_difference_update _Py_IDENTIFIER(symmetric_difference_update)
+Objects/enumobject.c:reversed_new_impl():PyId___reversed__       _Py_IDENTIFIER(__reversed__)
+Objects/exceptions.c:ImportError_getstate():PyId_name            _Py_IDENTIFIER(name)
+Objects/exceptions.c:ImportError_getstate():PyId_path            _Py_IDENTIFIER(path)
+Objects/fileobject.c:PyFile_FromFd():PyId_open                   _Py_IDENTIFIER(open)
+Objects/fileobject.c:PyFile_GetLine():PyId_readline              _Py_IDENTIFIER(readline)
+Objects/fileobject.c:PyFile_OpenCodeObject():PyId_open           _Py_IDENTIFIER(open)
+Objects/fileobject.c:PyFile_WriteObject():PyId_write             _Py_IDENTIFIER(write)
+Objects/fileobject.c:PyObject_AsFileDescriptor():PyId_fileno     _Py_IDENTIFIER(fileno)
+Objects/frameobject.c:PyId___builtins__                          _Py_IDENTIFIER(__builtins__)
+Objects/genobject.c:_gen_throw():PyId_throw                      _Py_IDENTIFIER(throw)
+Objects/genobject.c:gen_close_iter():PyId_close                  _Py_IDENTIFIER(close)
+Objects/iterobject.c:calliter_reduce():PyId_iter                 _Py_IDENTIFIER(iter)
+Objects/iterobject.c:iter_reduce():PyId_iter                     _Py_IDENTIFIER(iter)
+Objects/listobject.c:listiter_reduce_general():PyId_iter         _Py_IDENTIFIER(iter)
+Objects/listobject.c:listiter_reduce_general():PyId_reversed     _Py_IDENTIFIER(reversed)
+Objects/longobject.c:PyId_big                                    _Py_IDENTIFIER(big)
+Objects/longobject.c:PyId_little                                 _Py_IDENTIFIER(little)
+Objects/methodobject.c:meth_get__qualname__():PyId___qualname__  _Py_IDENTIFIER(__qualname__)
+Objects/methodobject.c:meth_reduce():PyId_getattr                _Py_IDENTIFIER(getattr)
+Objects/moduleobject.c:PyModule_GetFilenameObject():PyId___file__ _Py_IDENTIFIER(__file__)
+Objects/moduleobject.c:PyModule_GetNameObject():PyId___name__    _Py_IDENTIFIER(__name__)
+Objects/moduleobject.c:PyModule_SetDocString():PyId___doc__      _Py_IDENTIFIER(__doc__)
+Objects/moduleobject.c:_PyModuleSpec_IsInitializing():PyId__initializing _Py_IDENTIFIER(_initializing)
+Objects/moduleobject.c:module_dir():PyId___dict__                _Py_IDENTIFIER(__dict__)
+Objects/moduleobject.c:module_dir():PyId___dir__                 _Py_IDENTIFIER(__dir__)
+Objects/moduleobject.c:module_getattro():PyId___getattr__        _Py_IDENTIFIER(__getattr__)
+Objects/moduleobject.c:module_getattro():PyId___name__           _Py_IDENTIFIER(__name__)
+Objects/moduleobject.c:module_getattro():PyId___spec__           _Py_IDENTIFIER(__spec__)
+Objects/moduleobject.c:module_init_dict():PyId___doc__           _Py_IDENTIFIER(__doc__)
+Objects/moduleobject.c:module_init_dict():PyId___loader__        _Py_IDENTIFIER(__loader__)
+Objects/moduleobject.c:module_init_dict():PyId___name__          _Py_IDENTIFIER(__name__)
+Objects/moduleobject.c:module_init_dict():PyId___package__       _Py_IDENTIFIER(__package__)
+Objects/moduleobject.c:module_init_dict():PyId___spec__          _Py_IDENTIFIER(__spec__)
+Objects/object.c:PyId_Py_Repr                                    _Py_IDENTIFIER(Py_Repr)
+Objects/object.c:PyId___bytes__                                  _Py_IDENTIFIER(__bytes__)
+Objects/object.c:PyId___dir__                                    _Py_IDENTIFIER(__dir__)
+Objects/object.c:PyId___isabstractmethod__                       _Py_IDENTIFIER(__isabstractmethod__)
+Objects/odictobject.c:mutablemapping_update():PyId_items         _Py_IDENTIFIER(items)
+Objects/odictobject.c:mutablemapping_update():PyId_keys          _Py_IDENTIFIER(keys)
+Objects/odictobject.c:odict_reduce():PyId___dict__               _Py_IDENTIFIER(__dict__)
+Objects/odictobject.c:odict_reduce():PyId_items                  _Py_IDENTIFIER(items)
+Objects/odictobject.c:odict_repr():PyId_items                    _Py_IDENTIFIER(items)
+Objects/odictobject.c:odictiter_reduce():PyId_iter               _Py_IDENTIFIER(iter)
+Objects/rangeobject.c:longrangeiter_reduce():PyId_iter           _Py_IDENTIFIER(iter)
+Objects/rangeobject.c:rangeiter_reduce():PyId_iter               _Py_IDENTIFIER(iter)
+Objects/setobject.c:set_reduce():PyId___dict__                   _Py_IDENTIFIER(__dict__)
+Objects/setobject.c:setiter_reduce():PyId_iter                   _Py_IDENTIFIER(iter)
+Objects/structseq.c:PyId_n_fields                                _Py_IDENTIFIER(n_fields)
+Objects/structseq.c:PyId_n_sequence_fields                       _Py_IDENTIFIER(n_sequence_fields)
+Objects/structseq.c:PyId_n_unnamed_fields                        _Py_IDENTIFIER(n_unnamed_fields)
+Objects/tupleobject.c:tupleiter_reduce():PyId_iter               _Py_IDENTIFIER(iter)
+Objects/typeobject.c:PyId___abstractmethods__                    _Py_IDENTIFIER(__abstractmethods__)
+Objects/typeobject.c:PyId___class__                              _Py_IDENTIFIER(__class__)
+Objects/typeobject.c:PyId___class_getitem__                      _Py_IDENTIFIER(__class_getitem__)
+Objects/typeobject.c:PyId___delitem__                            _Py_IDENTIFIER(__delitem__)
+Objects/typeobject.c:PyId___dict__                               _Py_IDENTIFIER(__dict__)
+Objects/typeobject.c:PyId___doc__                                _Py_IDENTIFIER(__doc__)
+Objects/typeobject.c:PyId___getattribute__                       _Py_IDENTIFIER(__getattribute__)
+Objects/typeobject.c:PyId___getitem__                            _Py_IDENTIFIER(__getitem__)
+Objects/typeobject.c:PyId___hash__                               _Py_IDENTIFIER(__hash__)
+Objects/typeobject.c:PyId___init_subclass__                      _Py_IDENTIFIER(__init_subclass__)
+Objects/typeobject.c:PyId___len__                                _Py_IDENTIFIER(__len__)
+Objects/typeobject.c:PyId___module__                             _Py_IDENTIFIER(__module__)
+Objects/typeobject.c:PyId___name__                               _Py_IDENTIFIER(__name__)
+Objects/typeobject.c:PyId___new__                                _Py_IDENTIFIER(__new__)
+Objects/typeobject.c:PyId___set_name__                           _Py_IDENTIFIER(__set_name__)
+Objects/typeobject.c:PyId___setitem__                            _Py_IDENTIFIER(__setitem__)
+Objects/typeobject.c:PyId_builtins                               _Py_IDENTIFIER(builtins)
+Objects/typeobject.c:_PyObject_GetItemsIter():PyId_items         _Py_IDENTIFIER(items)
+Objects/typeobject.c:_PyObject_GetNewArguments():PyId___getnewargs__ _Py_IDENTIFIER(__getnewargs__)
+Objects/typeobject.c:_PyObject_GetNewArguments():PyId___getnewargs_ex__ _Py_IDENTIFIER(__getnewargs_ex__)
+Objects/typeobject.c:_PyObject_GetState():PyId___getstate__      _Py_IDENTIFIER(__getstate__)
+Objects/typeobject.c:_PyType_GetSlotNames():PyId___slotnames__   _Py_IDENTIFIER(__slotnames__)
+Objects/typeobject.c:_PyType_GetSlotNames():PyId__slotnames      _Py_IDENTIFIER(_slotnames)
+Objects/typeobject.c:import_copyreg():PyId_copyreg               _Py_IDENTIFIER(copyreg)
+Objects/typeobject.c:merge_class_dict():PyId___bases__           _Py_IDENTIFIER(__bases__)
+Objects/typeobject.c:mro_invoke():PyId_mro                       _Py_IDENTIFIER(mro)
+Objects/typeobject.c:object___reduce_ex___impl():PyId___reduce__ _Py_IDENTIFIER(__reduce__)
+Objects/typeobject.c:overrides_hash():PyId___eq__                _Py_IDENTIFIER(__eq__)
+Objects/typeobject.c:reduce_newobj():PyId___newobj__             _Py_IDENTIFIER(__newobj__)
+Objects/typeobject.c:reduce_newobj():PyId___newobj_ex__          _Py_IDENTIFIER(__newobj_ex__)
+Objects/typeobject.c:slot_am_aiter():PyId___aiter__              _Py_IDENTIFIER(__aiter__)
+Objects/typeobject.c:slot_am_anext():PyId___anext__              _Py_IDENTIFIER(__anext__)
+Objects/typeobject.c:slot_am_await():PyId___await__              _Py_IDENTIFIER(__await__)
+Objects/typeobject.c:slot_nb_bool():PyId___bool__                _Py_IDENTIFIER(__bool__)
+Objects/typeobject.c:slot_nb_index():PyId___index__              _Py_IDENTIFIER(__index__)
+Objects/typeobject.c:slot_nb_inplace_power():PyId___ipow__       _Py_IDENTIFIER(__ipow__)
+Objects/typeobject.c:slot_nb_power():PyId___pow__                _Py_IDENTIFIER(__pow__)
+Objects/typeobject.c:slot_sq_contains():PyId___contains__        _Py_IDENTIFIER(__contains__)
+Objects/typeobject.c:slot_tp_call():PyId___call__                _Py_IDENTIFIER(__call__)
+Objects/typeobject.c:slot_tp_descr_get():PyId___get__            _Py_IDENTIFIER(__get__)
+Objects/typeobject.c:slot_tp_descr_set():PyId___delete__         _Py_IDENTIFIER(__delete__)
+Objects/typeobject.c:slot_tp_descr_set():PyId___set__            _Py_IDENTIFIER(__set__)
+Objects/typeobject.c:slot_tp_finalize():PyId___del__             _Py_IDENTIFIER(__del__)
+Objects/typeobject.c:slot_tp_getattr_hook():PyId___getattr__     _Py_IDENTIFIER(__getattr__)
+Objects/typeobject.c:slot_tp_init():PyId___init__                _Py_IDENTIFIER(__init__)
+Objects/typeobject.c:slot_tp_iter():PyId___iter__                _Py_IDENTIFIER(__iter__)
+Objects/typeobject.c:slot_tp_iternext():PyId___next__            _Py_IDENTIFIER(__next__)
+Objects/typeobject.c:slot_tp_repr():PyId___repr__                _Py_IDENTIFIER(__repr__)
+Objects/typeobject.c:slot_tp_setattro():PyId___delattr__         _Py_IDENTIFIER(__delattr__)
+Objects/typeobject.c:slot_tp_setattro():PyId___setattr__         _Py_IDENTIFIER(__setattr__)
+Objects/typeobject.c:type_mro_modified():PyId_mro                _Py_IDENTIFIER(mro)
+Objects/typeobject.c:type_new():PyId___classcell__               _Py_IDENTIFIER(__classcell__)
+Objects/typeobject.c:type_new():PyId___mro_entries__             _Py_IDENTIFIER(__mro_entries__)
+Objects/typeobject.c:type_new():PyId___qualname__                _Py_IDENTIFIER(__qualname__)
+Objects/typeobject.c:type_new():PyId___slots__                   _Py_IDENTIFIER(__slots__)
+Objects/unicodeobject.c:unicodeiter_reduce():PyId_iter           _Py_IDENTIFIER(iter)
+Objects/weakrefobject.c:proxy_bytes():PyId___bytes__             _Py_IDENTIFIER(__bytes__)
+Objects/weakrefobject.c:weakref_repr():PyId___name__             _Py_IDENTIFIER(__name__)
+Parser/tokenizer.c:fp_setreadl():PyId_open                       _Py_IDENTIFIER(open)
+Parser/tokenizer.c:fp_setreadl():PyId_readline                   _Py_IDENTIFIER(readline)
+Python/Python-ast.c:ast_type_reduce():PyId___dict__              _Py_IDENTIFIER(__dict__)
+Python/Python-ast.c:make_type():PyId___module__                  _Py_IDENTIFIER(__module__)
+Python/_warnings.c:PyId_stderr                                   _Py_IDENTIFIER(stderr)
+Python/_warnings.c:_PyErr_WarnUnawaitedCoroutine():PyId__warn_unawaited_coroutine _Py_IDENTIFIER(_warn_unawaited_coroutine)
+Python/_warnings.c:already_warned():PyId_version                 _Py_IDENTIFIER(version)
+Python/_warnings.c:call_show_warning():PyId_WarningMessage       _Py_IDENTIFIER(WarningMessage)
+Python/_warnings.c:call_show_warning():PyId__showwarnmsg         _Py_IDENTIFIER(_showwarnmsg)
+Python/_warnings.c:check_matched():PyId_match                    _Py_IDENTIFIER(match)
+Python/_warnings.c:get_default_action():PyId_defaultaction       _Py_IDENTIFIER(defaultaction)
+Python/_warnings.c:get_filter():PyId_filters                     _Py_IDENTIFIER(filters)
+Python/_warnings.c:get_once_registry():PyId_onceregistry         _Py_IDENTIFIER(onceregistry)
+Python/_warnings.c:get_source_line():PyId___loader__             _Py_IDENTIFIER(__loader__)
+Python/_warnings.c:get_source_line():PyId___name__               _Py_IDENTIFIER(__name__)
+Python/_warnings.c:get_source_line():PyId_get_source             _Py_IDENTIFIER(get_source)
+Python/_warnings.c:get_warnings_attr():PyId_warnings             _Py_IDENTIFIER(warnings)
+Python/_warnings.c:setup_context():PyId___name__                 _Py_IDENTIFIER(__name__)
+Python/_warnings.c:setup_context():PyId___warningregistry__      _Py_IDENTIFIER(__warningregistry__)
+Python/_warnings.c:show_warning():PyId___name__                  _Py_IDENTIFIER(__name__)
+Python/bltinmodule.c:PyId___builtins__                           _Py_IDENTIFIER(__builtins__)
+Python/bltinmodule.c:PyId___dict__                               _Py_IDENTIFIER(__dict__)
+Python/bltinmodule.c:PyId___mro_entries__                        _Py_IDENTIFIER(__mro_entries__)
+Python/bltinmodule.c:PyId___prepare__                            _Py_IDENTIFIER(__prepare__)
+Python/bltinmodule.c:PyId___round__                              _Py_IDENTIFIER(__round__)
+Python/bltinmodule.c:PyId_encoding                               _Py_IDENTIFIER(encoding)
+Python/bltinmodule.c:PyId_errors                                 _Py_IDENTIFIER(errors)
+Python/bltinmodule.c:PyId_fileno                                 _Py_IDENTIFIER(fileno)
+Python/bltinmodule.c:PyId_flush                                  _Py_IDENTIFIER(flush)
+Python/bltinmodule.c:PyId_metaclass                              _Py_IDENTIFIER(metaclass)
+Python/bltinmodule.c:PyId_sort                                   _Py_IDENTIFIER(sort)
+Python/bltinmodule.c:PyId_stderr                                 _Py_IDENTIFIER(stderr)
+Python/bltinmodule.c:PyId_stdin                                  _Py_IDENTIFIER(stdin)
+Python/bltinmodule.c:PyId_stdout                                 _Py_IDENTIFIER(stdout)
+Python/ceval.c:_PyEval_EvalFrameDefault():PyId___aenter__        _Py_IDENTIFIER(__aenter__)
+Python/ceval.c:_PyEval_EvalFrameDefault():PyId___aexit__         _Py_IDENTIFIER(__aexit__)
+Python/ceval.c:_PyEval_EvalFrameDefault():PyId___annotations__   _Py_IDENTIFIER(__annotations__)
+Python/ceval.c:_PyEval_EvalFrameDefault():PyId___build_class__   _Py_IDENTIFIER(__build_class__)
+Python/ceval.c:_PyEval_EvalFrameDefault():PyId___enter__         _Py_IDENTIFIER(__enter__)
+Python/ceval.c:_PyEval_EvalFrameDefault():PyId___exit__          _Py_IDENTIFIER(__exit__)
+Python/ceval.c:_PyEval_EvalFrameDefault():PyId___ltrace__        _Py_IDENTIFIER(__ltrace__)
+Python/ceval.c:_PyEval_EvalFrameDefault():PyId_displayhook       _Py_IDENTIFIER(displayhook)
+Python/ceval.c:_PyEval_EvalFrameDefault():PyId_send              _Py_IDENTIFIER(send)
+Python/ceval.c:import_all_from():PyId___all__                    _Py_IDENTIFIER(__all__)
+Python/ceval.c:import_all_from():PyId___dict__                   _Py_IDENTIFIER(__dict__)
+Python/ceval.c:import_all_from():PyId___name__                   _Py_IDENTIFIER(__name__)
+Python/ceval.c:import_from():PyId___name__                       _Py_IDENTIFIER(__name__)
+Python/ceval.c:import_name():PyId___import__                     _Py_IDENTIFIER(__import__)
+Python/codecs.c:_PyCodec_LookupTextEncoding():PyId__is_text_encoding _Py_IDENTIFIER(_is_text_encoding)
+Python/compile.c:compiler_enter_scope():PyId___class__           _Py_IDENTIFIER(__class__)
+Python/errors.c:PyId_builtins                                    _Py_IDENTIFIER(builtins)
+Python/errors.c:PyId_flush                                       _Py_IDENTIFIER(flush)
+Python/errors.c:PyId_stderr                                      _Py_IDENTIFIER(stderr)
+Python/errors.c:PyErr_NewException():PyId___module__             _Py_IDENTIFIER(__module__)
+Python/errors.c:PyErr_SyntaxLocationObject():PyId_filename       _Py_IDENTIFIER(filename)
+Python/errors.c:PyErr_SyntaxLocationObject():PyId_lineno         _Py_IDENTIFIER(lineno)
+Python/errors.c:PyErr_SyntaxLocationObject():PyId_msg            _Py_IDENTIFIER(msg)
+Python/errors.c:PyErr_SyntaxLocationObject():PyId_offset         _Py_IDENTIFIER(offset)
+Python/errors.c:PyErr_SyntaxLocationObject():PyId_print_file_and_line _Py_IDENTIFIER(print_file_and_line)
+Python/errors.c:PyErr_SyntaxLocationObject():PyId_text           _Py_IDENTIFIER(text)
+Python/errors.c:_PyErr_WriteUnraisableMsg():PyId_unraisablehook  _Py_IDENTIFIER(unraisablehook)
+Python/import.c:PyImport_Cleanup():PyId_clear                    _Py_IDENTIFIER(clear)
+Python/import.c:PyImport_ExecCodeModuleObject():PyId__fix_up_module _Py_IDENTIFIER(_fix_up_module)
+Python/import.c:PyImport_ExecCodeModuleWithPathnames():PyId__get_sourcefile _Py_IDENTIFIER(_get_sourcefile)
+Python/import.c:PyImport_ImportModuleLevelObject():PyId___path__ _Py_IDENTIFIER(__path__)
+Python/import.c:PyImport_ImportModuleLevelObject():PyId___spec__ _Py_IDENTIFIER(__spec__)
+Python/import.c:PyImport_ImportModuleLevelObject():PyId__handle_fromlist _Py_IDENTIFIER(_handle_fromlist)
+Python/import.c:PyImport_ImportModuleLevelObject():PyId__lock_unlock_module _Py_IDENTIFIER(_lock_unlock_module)
+Python/import.c:PyImport_ReloadModule():PyId_imp                 _Py_IDENTIFIER(imp)
+Python/import.c:PyImport_ReloadModule():PyId_reload              _Py_IDENTIFIER(reload)
+Python/import.c:_PyImportZip_Init():PyId_zipimporter             _Py_IDENTIFIER(zipimporter)
+Python/import.c:import_find_and_load():PyId__find_and_load       _Py_IDENTIFIER(_find_and_load)
+Python/import.c:module_dict_for_exec():PyId___builtins__         _Py_IDENTIFIER(__builtins__)
+Python/import.c:resolve_name():PyId___name__                     _Py_IDENTIFIER(__name__)
+Python/import.c:resolve_name():PyId___package__                  _Py_IDENTIFIER(__package__)
+Python/import.c:resolve_name():PyId___path__                     _Py_IDENTIFIER(__path__)
+Python/import.c:resolve_name():PyId___spec__                     _Py_IDENTIFIER(__spec__)
+Python/import.c:resolve_name():PyId_parent                       _Py_IDENTIFIER(parent)
+Python/importdl.c:get_encoded_name():PyId_replace                _Py_IDENTIFIER(replace)
+Python/marshal.c:marshal_dump_impl():PyId_write                  _Py_IDENTIFIER(write)
+Python/marshal.c:marshal_load():PyId_read                        _Py_IDENTIFIER(read)
+Python/marshal.c:r_string():PyId_readinto                        _Py_IDENTIFIER(readinto)
+Python/pylifecycle.c:PyId_flush                                  _Py_IDENTIFIER(flush)
+Python/pylifecycle.c:PyId_name                                   _Py_IDENTIFIER(name)
+Python/pylifecycle.c:PyId_stderr                                 _Py_IDENTIFIER(stderr)
+Python/pylifecycle.c:PyId_stdin                                  _Py_IDENTIFIER(stdin)
+Python/pylifecycle.c:PyId_stdout                                 _Py_IDENTIFIER(stdout)
+Python/pylifecycle.c:PyId_threading                              _Py_IDENTIFIER(threading)
+Python/pylifecycle.c:create_stdio():PyId_TextIOWrapper           _Py_IDENTIFIER(TextIOWrapper)
+Python/pylifecycle.c:create_stdio():PyId_isatty                  _Py_IDENTIFIER(isatty)
+Python/pylifecycle.c:create_stdio():PyId_mode                    _Py_IDENTIFIER(mode)
+Python/pylifecycle.c:create_stdio():PyId_open                    _Py_IDENTIFIER(open)
+Python/pylifecycle.c:create_stdio():PyId_raw                     _Py_IDENTIFIER(raw)
+Python/pylifecycle.c:wait_for_thread_shutdown():PyId__shutdown   _Py_IDENTIFIER(_shutdown)
+Python/pythonrun.c:PyId_builtins                                 _Py_IDENTIFIER(builtins)
+Python/pythonrun.c:PyId_excepthook                               _Py_IDENTIFIER(excepthook)
+Python/pythonrun.c:PyId_flush                                    _Py_IDENTIFIER(flush)
+Python/pythonrun.c:PyId_last_traceback                           _Py_IDENTIFIER(last_traceback)
+Python/pythonrun.c:PyId_last_type                                _Py_IDENTIFIER(last_type)
+Python/pythonrun.c:PyId_last_value                               _Py_IDENTIFIER(last_value)
+Python/pythonrun.c:PyId_ps1                                      _Py_IDENTIFIER(ps1)
+Python/pythonrun.c:PyId_ps2                                      _Py_IDENTIFIER(ps2)
+Python/pythonrun.c:PyId_stderr                                   _Py_IDENTIFIER(stderr)
+Python/pythonrun.c:PyId_stdin                                    _Py_IDENTIFIER(stdin)
+Python/pythonrun.c:PyId_stdout                                   _Py_IDENTIFIER(stdout)
+Python/pythonrun.c:PyRun_InteractiveOneObjectEx():PyId___main__  _Py_IDENTIFIER(__main__)
+Python/pythonrun.c:PyRun_InteractiveOneObjectEx():PyId_encoding  _Py_IDENTIFIER(encoding)
+Python/pythonrun.c:_Py_HandleSystemExit():PyId_code              _Py_IDENTIFIER(code)
+Python/pythonrun.c:parse_syntax_error():PyId_filename            _Py_IDENTIFIER(filename)
+Python/pythonrun.c:parse_syntax_error():PyId_lineno              _Py_IDENTIFIER(lineno)
+Python/pythonrun.c:parse_syntax_error():PyId_msg                 _Py_IDENTIFIER(msg)
+Python/pythonrun.c:parse_syntax_error():PyId_offset              _Py_IDENTIFIER(offset)
+Python/pythonrun.c:parse_syntax_error():PyId_text                _Py_IDENTIFIER(text)
+Python/pythonrun.c:print_exception():PyId___module__             _Py_IDENTIFIER(__module__)
+Python/pythonrun.c:print_exception():PyId_print_file_and_line    _Py_IDENTIFIER(print_file_and_line)
+Python/sysmodule.c:PyId__                                        _Py_IDENTIFIER(_)
+Python/sysmodule.c:PyId___sizeof__                               _Py_IDENTIFIER(__sizeof__)
+Python/sysmodule.c:PyId__xoptions                                _Py_IDENTIFIER(_xoptions)
+Python/sysmodule.c:PyId_buffer                                   _Py_IDENTIFIER(buffer)
+Python/sysmodule.c:PyId_builtins                                 _Py_IDENTIFIER(builtins)
+Python/sysmodule.c:PyId_encoding                                 _Py_IDENTIFIER(encoding)
+Python/sysmodule.c:PyId_path                                     _Py_IDENTIFIER(path)
+Python/sysmodule.c:PyId_stderr                                   _Py_IDENTIFIER(stderr)
+Python/sysmodule.c:PyId_stdout                                   _Py_IDENTIFIER(stdout)
+Python/sysmodule.c:PyId_warnoptions                              _Py_IDENTIFIER(warnoptions)
+Python/sysmodule.c:PyId_write                                    _Py_IDENTIFIER(write)
+Python/traceback.c:PyId_TextIOWrapper                            _Py_IDENTIFIER(TextIOWrapper)
+Python/traceback.c:PyId_close                                    _Py_IDENTIFIER(close)
+Python/traceback.c:PyId_open                                     _Py_IDENTIFIER(open)
+Python/traceback.c:PyId_path                                     _Py_IDENTIFIER(path)
+Objects/typeobject.c:name_op                                     static _Py_Identifier name_op[]
+Objects/unicodeobject.c:static_strings                           static _Py_Identifier *static_strings
+
+
+# PyTypeObject (311)
+Modules/_abc.c:_abc_data_type                                    static PyTypeObject _abc_data_type
+Modules/_blake2/blake2b_impl.c:PyBlake2_BLAKE2bType              PyTypeObject PyBlake2_BLAKE2bType
+Modules/_blake2/blake2s_impl.c:PyBlake2_BLAKE2sType              PyTypeObject PyBlake2_BLAKE2sType
+Modules/_collectionsmodule.c:defdict_type                        static PyTypeObject defdict_type
+Modules/_collectionsmodule.c:deque_type                          static PyTypeObject deque_type
+Modules/_collectionsmodule.c:dequeiter_type                      static PyTypeObject dequeiter_type
+Modules/_collectionsmodule.c:dequereviter_type                   static PyTypeObject dequereviter_type
+Modules/_collectionsmodule.c:tuplegetter_type                    static PyTypeObject tuplegetter_type
+Modules/_functoolsmodule.c:keyobject_type                        static PyTypeObject keyobject_type
+Modules/_functoolsmodule.c:lru_cache_type                        static PyTypeObject lru_cache_type
+Modules/_functoolsmodule.c:lru_list_elem_type                    static PyTypeObject lru_list_elem_type
+Modules/_functoolsmodule.c:partial_type                          static PyTypeObject partial_type
+Modules/_io/bufferedio.c:PyBufferedIOBase_Type                   PyTypeObject PyBufferedIOBase_Type
+Modules/_io/bufferedio.c:PyBufferedRWPair_Type                   PyTypeObject PyBufferedRWPair_Type
+Modules/_io/bufferedio.c:PyBufferedRandom_Type                   PyTypeObject PyBufferedRandom_Type
+Modules/_io/bufferedio.c:PyBufferedReader_Type                   PyTypeObject PyBufferedReader_Type
+Modules/_io/bufferedio.c:PyBufferedWriter_Type                   PyTypeObject PyBufferedWriter_Type
+Modules/_io/bytesio.c:PyBytesIO_Type                             PyTypeObject PyBytesIO_Type
+Modules/_io/bytesio.c:_PyBytesIOBuffer_Type                      PyTypeObject _PyBytesIOBuffer_Type
+Modules/_io/fileio.c:PyFileIO_Type                               PyTypeObject PyFileIO_Type
+Modules/_io/iobase.c:PyIOBase_Type                               PyTypeObject PyIOBase_Type
+Modules/_io/iobase.c:PyRawIOBase_Type                            PyTypeObject PyRawIOBase_Type
+Modules/_io/stringio.c:PyStringIO_Type                           PyTypeObject PyStringIO_Type
+Modules/_io/textio.c:PyIncrementalNewlineDecoder_Type            PyTypeObject PyIncrementalNewlineDecoder_Type
+Modules/_io/textio.c:PyTextIOBase_Type                           PyTypeObject PyTextIOBase_Type
+Modules/_io/textio.c:PyTextIOWrapper_Type                        PyTypeObject PyTextIOWrapper_Type
+Modules/_operator.c:attrgetter_type                              static PyTypeObject attrgetter_type
+Modules/_operator.c:itemgetter_type                              static PyTypeObject itemgetter_type
+Modules/_operator.c:methodcaller_type                            static PyTypeObject methodcaller_type
+Modules/_sre.c:Match_Type                                        static PyTypeObject Match_Type
+Modules/_sre.c:Pattern_Type                                      static PyTypeObject Pattern_Type
+Modules/_sre.c:Scanner_Type                                      static PyTypeObject Scanner_Type
+Modules/_threadmodule.c:ExceptHookArgsType                       static PyTypeObject ExceptHookArgsType
+Modules/_threadmodule.c:Locktype                                 static PyTypeObject Locktype
+Modules/_threadmodule.c:RLocktype                                static PyTypeObject RLocktype
+Modules/_threadmodule.c:localdummytype                           static PyTypeObject localdummytype
+Modules/_threadmodule.c:localtype                                static PyTypeObject localtype
+Modules/itertoolsmodule.c:_grouper_type                          static PyTypeObject _grouper_type
+Modules/itertoolsmodule.c:accumulate_type                        static PyTypeObject accumulate_type
+Modules/itertoolsmodule.c:chain_type                             static PyTypeObject chain_type
+Modules/itertoolsmodule.c:combinations_type                      static PyTypeObject combinations_type
+Modules/itertoolsmodule.c:compress_type                          static PyTypeObject compress_type
+Modules/itertoolsmodule.c:count_type                             static PyTypeObject count_type
+Modules/itertoolsmodule.c:cwr_type                               static PyTypeObject cwr_type
+Modules/itertoolsmodule.c:cycle_type                             static PyTypeObject cycle_type
+Modules/itertoolsmodule.c:dropwhile_type                         static PyTypeObject dropwhile_type
+Modules/itertoolsmodule.c:filterfalse_type                       static PyTypeObject filterfalse_type
+Modules/itertoolsmodule.c:groupby_type                           static PyTypeObject groupby_type
+Modules/itertoolsmodule.c:islice_type                            static PyTypeObject islice_type
+Modules/itertoolsmodule.c:permutations_type                      static PyTypeObject permutations_type
+Modules/itertoolsmodule.c:product_type                           static PyTypeObject product_type
+Modules/itertoolsmodule.c:repeat_type                            static PyTypeObject repeat_type
+Modules/itertoolsmodule.c:starmap_type                           static PyTypeObject starmap_type
+Modules/itertoolsmodule.c:takewhile_type                         static PyTypeObject takewhile_type
+Modules/itertoolsmodule.c:tee_type                               static PyTypeObject tee_type
+Modules/itertoolsmodule.c:teedataobject_type                     static PyTypeObject teedataobject_type
+Modules/itertoolsmodule.c:ziplongest_type                        static PyTypeObject ziplongest_type
+Modules/posixmodule.c:DirEntryType                               static PyTypeObject DirEntryType
+Modules/posixmodule.c:ScandirIteratorType                        static PyTypeObject ScandirIteratorType
+Modules/posixmodule.c:SchedParamType                             static PyTypeObject* SchedParamType
+Modules/posixmodule.c:StatResultType                             static PyTypeObject* StatResultType
+Modules/posixmodule.c:StatVFSResultType                          static PyTypeObject* StatVFSResultType
+Modules/posixmodule.c:TerminalSizeType                           static PyTypeObject* TerminalSizeType
+Modules/posixmodule.c:TimesResultType                            static PyTypeObject* TimesResultType
+Modules/posixmodule.c:UnameResultType                            static PyTypeObject* UnameResultType
+Modules/posixmodule.c:WaitidResultType                           static PyTypeObject* WaitidResultType
+Modules/signalmodule.c:SiginfoType                               static PyTypeObject SiginfoType
+Modules/timemodule.c:StructTimeType                              static PyTypeObject StructTimeType
+Modules/xxsubtype.c:spamdict_type                                static PyTypeObject spamdict_type
+Modules/xxsubtype.c:spamlist_type                                static PyTypeObject spamlist_type
+Objects/boolobject.c:PyBool_Type                                 PyTypeObject PyBool_Type
+Objects/bytearrayobject.c:PyByteArrayIter_Type                   PyTypeObject PyByteArrayIter_Type
+Objects/bytearrayobject.c:PyByteArray_Type                       PyTypeObject PyByteArray_Type
+Objects/bytesobject.c:PyBytesIter_Type                           PyTypeObject PyBytesIter_Type
+Objects/bytesobject.c:PyBytes_Type                               PyTypeObject PyBytes_Type
+Objects/capsule.c:PyCapsule_Type                                 PyTypeObject PyCapsule_Type
+Objects/cellobject.c:PyCell_Type                                 PyTypeObject PyCell_Type
+Objects/classobject.c:PyInstanceMethod_Type                      PyTypeObject PyInstanceMethod_Type
+Objects/classobject.c:PyMethod_Type                              PyTypeObject PyMethod_Type
+Objects/codeobject.c:PyCode_Type                                 PyTypeObject PyCode_Type
+Objects/complexobject.c:PyComplex_Type                           PyTypeObject PyComplex_Type
+Objects/descrobject.c:PyClassMethodDescr_Type                    PyTypeObject PyClassMethodDescr_Type
+Objects/descrobject.c:PyDictProxy_Type                           PyTypeObject PyDictProxy_Type
+Objects/descrobject.c:PyGetSetDescr_Type                         PyTypeObject PyGetSetDescr_Type
+Objects/descrobject.c:PyMemberDescr_Type                         PyTypeObject PyMemberDescr_Type
+Objects/descrobject.c:PyMethodDescr_Type                         PyTypeObject PyMethodDescr_Type
+Objects/descrobject.c:PyProperty_Type                            PyTypeObject PyProperty_Type
+Objects/descrobject.c:PyWrapperDescr_Type                        PyTypeObject PyWrapperDescr_Type
+Objects/descrobject.c:_PyMethodWrapper_Type                      PyTypeObject _PyMethodWrapper_Type
+Objects/dictobject.c:PyDictItems_Type                            PyTypeObject PyDictItems_Type
+Objects/dictobject.c:PyDictIterItem_Type                         PyTypeObject PyDictIterItem_Type
+Objects/dictobject.c:PyDictIterKey_Type                          PyTypeObject PyDictIterKey_Type
+Objects/dictobject.c:PyDictIterValue_Type                        PyTypeObject PyDictIterValue_Type
+Objects/dictobject.c:PyDictKeys_Type                             PyTypeObject PyDictKeys_Type
+Objects/dictobject.c:PyDictRevIterItem_Type                      PyTypeObject PyDictRevIterItem_Type
+Objects/dictobject.c:PyDictRevIterKey_Type                       PyTypeObject PyDictRevIterKey_Type
+Objects/dictobject.c:PyDictRevIterValue_Type                     PyTypeObject PyDictRevIterValue_Type
+Objects/dictobject.c:PyDictValues_Type                           PyTypeObject PyDictValues_Type
+Objects/dictobject.c:PyDict_Type                                 PyTypeObject PyDict_Type
+Objects/enumobject.c:PyEnum_Type                                 PyTypeObject PyEnum_Type
+Objects/enumobject.c:PyReversed_Type                             PyTypeObject PyReversed_Type
+Objects/exceptions.c:PyExc_ArithmeticError                       static PyTypeObject PyExc_ArithmeticError
+Objects/exceptions.c:PyExc_AssertionError                        static PyTypeObject PyExc_AssertionError
+Objects/exceptions.c:PyExc_AttributeError                        static PyTypeObject PyExc_AttributeError
+Objects/exceptions.c:PyExc_BaseException                         static PyTypeObject PyExc_BaseException
+Objects/exceptions.c:PyExc_BlockingIOError                       static PyTypeObject PyExc_BlockingIOError
+Objects/exceptions.c:PyExc_BrokenPipeError                       static PyTypeObject PyExc_BrokenPipeError
+Objects/exceptions.c:PyExc_BufferError                           static PyTypeObject PyExc_BufferError
+Objects/exceptions.c:PyExc_BytesWarning                          static PyTypeObject PyExc_BytesWarning
+Objects/exceptions.c:PyExc_ChildProcessError                     static PyTypeObject PyExc_ChildProcessError
+Objects/exceptions.c:PyExc_ConnectionAbortedError                static PyTypeObject PyExc_ConnectionAbortedError
+Objects/exceptions.c:PyExc_ConnectionError                       static PyTypeObject PyExc_ConnectionError
+Objects/exceptions.c:PyExc_ConnectionRefusedError                static PyTypeObject PyExc_ConnectionRefusedError
+Objects/exceptions.c:PyExc_ConnectionResetError                  static PyTypeObject PyExc_ConnectionResetError
+Objects/exceptions.c:PyExc_DeprecationWarning                    static PyTypeObject PyExc_DeprecationWarning
+Objects/exceptions.c:PyExc_EOFError                              static PyTypeObject PyExc_EOFError
+Objects/exceptions.c:PyExc_EnvironmentError                      static PyTypeObject PyExc_EnvironmentError
+Objects/exceptions.c:PyExc_Exception                             static PyTypeObject PyExc_Exception
+Objects/exceptions.c:PyExc_FileExistsError                       static PyTypeObject PyExc_FileExistsError
+Objects/exceptions.c:PyExc_FileNotFoundError                     static PyTypeObject PyExc_FileNotFoundError
+Objects/exceptions.c:PyExc_FloatingPointError                    static PyTypeObject PyExc_FloatingPointError
+Objects/exceptions.c:PyExc_FutureWarning                         static PyTypeObject PyExc_FutureWarning
+Objects/exceptions.c:PyExc_GeneratorExit                         static PyTypeObject PyExc_GeneratorExit
+Objects/exceptions.c:PyExc_IOError                               static PyTypeObject PyExc_IOError
+Objects/exceptions.c:PyExc_ImportError                           static PyTypeObject PyExc_ImportError
+Objects/exceptions.c:PyExc_ImportWarning                         static PyTypeObject PyExc_ImportWarning
+Objects/exceptions.c:PyExc_IndentationError                      static PyTypeObject PyExc_IndentationError
+Objects/exceptions.c:PyExc_IndexError                            static PyTypeObject PyExc_IndexError
+Objects/exceptions.c:PyExc_InterruptedError                      static PyTypeObject PyExc_InterruptedError
+Objects/exceptions.c:PyExc_IsADirectoryError                     static PyTypeObject PyExc_IsADirectoryError
+Objects/exceptions.c:PyExc_KeyError                              static PyTypeObject PyExc_KeyError
+Objects/exceptions.c:PyExc_KeyboardInterrupt                     static PyTypeObject PyExc_KeyboardInterrupt
+Objects/exceptions.c:PyExc_LookupError                           static PyTypeObject PyExc_LookupError
+Objects/exceptions.c:PyExc_MemoryError                           static PyTypeObject PyExc_MemoryError
+Objects/exceptions.c:PyExc_ModuleNotFoundError                   static PyTypeObject PyExc_ModuleNotFoundError
+Objects/exceptions.c:PyExc_NameError                             static PyTypeObject PyExc_NameError
+Objects/exceptions.c:PyExc_NotADirectoryError                    static PyTypeObject PyExc_NotADirectoryError
+Objects/exceptions.c:PyExc_NotImplementedError                   static PyTypeObject PyExc_NotImplementedError
+Objects/exceptions.c:PyExc_OSError                               static PyTypeObject PyExc_OSError
+Objects/exceptions.c:PyExc_OverflowError                         static PyTypeObject PyExc_OverflowError
+Objects/exceptions.c:PyExc_PendingDeprecationWarning             static PyTypeObject PyExc_PendingDeprecationWarning
+Objects/exceptions.c:PyExc_PermissionError                       static PyTypeObject PyExc_PermissionError
+Objects/exceptions.c:PyExc_ProcessLookupError                    static PyTypeObject PyExc_ProcessLookupError
+Objects/exceptions.c:PyExc_RecursionError                        static PyTypeObject PyExc_RecursionError
+Objects/exceptions.c:PyExc_ReferenceError                        static PyTypeObject PyExc_ReferenceError
+Objects/exceptions.c:PyExc_ResourceWarning                       static PyTypeObject PyExc_ResourceWarning
+Objects/exceptions.c:PyExc_RuntimeError                          static PyTypeObject PyExc_RuntimeError
+Objects/exceptions.c:PyExc_RuntimeWarning                        static PyTypeObject PyExc_RuntimeWarning
+Objects/exceptions.c:PyExc_StopAsyncIteration                    static PyTypeObject PyExc_StopAsyncIteration
+Objects/exceptions.c:PyExc_StopIteration                         static PyTypeObject PyExc_StopIteration
+Objects/exceptions.c:PyExc_SyntaxError                           static PyTypeObject PyExc_SyntaxError
+Objects/exceptions.c:PyExc_SyntaxWarning                         static PyTypeObject PyExc_SyntaxWarning
+Objects/exceptions.c:PyExc_SystemError                           static PyTypeObject PyExc_SystemError
+Objects/exceptions.c:PyExc_SystemExit                            static PyTypeObject PyExc_SystemExit
+Objects/exceptions.c:PyExc_TabError                              static PyTypeObject PyExc_TabError
+Objects/exceptions.c:PyExc_TimeoutError                          static PyTypeObject PyExc_TimeoutError
+Objects/exceptions.c:PyExc_TypeError                             static PyTypeObject PyExc_TypeError
+Objects/exceptions.c:PyExc_UnboundLocalError                     static PyTypeObject PyExc_UnboundLocalError
+Objects/exceptions.c:PyExc_UnicodeDecodeError                    static PyTypeObject PyExc_UnicodeDecodeError
+Objects/exceptions.c:PyExc_UnicodeEncodeError                    static PyTypeObject PyExc_UnicodeEncodeError
+Objects/exceptions.c:PyExc_UnicodeError                          static PyTypeObject PyExc_UnicodeError
+Objects/exceptions.c:PyExc_UnicodeTranslateError                 static PyTypeObject PyExc_UnicodeTranslateError
+Objects/exceptions.c:PyExc_UnicodeWarning                        static PyTypeObject PyExc_UnicodeWarning
+Objects/exceptions.c:PyExc_UserWarning                           static PyTypeObject PyExc_UserWarning
+Objects/exceptions.c:PyExc_ValueError                            static PyTypeObject PyExc_ValueError
+Objects/exceptions.c:PyExc_Warning                               static PyTypeObject PyExc_Warning
+Objects/exceptions.c:PyExc_ZeroDivisionError                     static PyTypeObject PyExc_ZeroDivisionError
+Objects/exceptions.c:_PyExc_ArithmeticError                      static PyTypeObject _PyExc_ArithmeticError
+Objects/exceptions.c:_PyExc_AssertionError                       static PyTypeObject _PyExc_AssertionError
+Objects/exceptions.c:_PyExc_AttributeError                       static PyTypeObject _PyExc_AttributeError
+Objects/exceptions.c:_PyExc_BaseException                        static PyTypeObject _PyExc_BaseException
+Objects/exceptions.c:_PyExc_BlockingIOError                      static PyTypeObject _PyExc_BlockingIOError
+Objects/exceptions.c:_PyExc_BrokenPipeError                      static PyTypeObject _PyExc_BrokenPipeError
+Objects/exceptions.c:_PyExc_BufferError                          static PyTypeObject _PyExc_BufferError
+Objects/exceptions.c:_PyExc_BytesWarning                         static PyTypeObject _PyExc_BytesWarning
+Objects/exceptions.c:_PyExc_ChildProcessError                    static PyTypeObject _PyExc_ChildProcessError
+Objects/exceptions.c:_PyExc_ConnectionAbortedError               static PyTypeObject _PyExc_ConnectionAbortedError
+Objects/exceptions.c:_PyExc_ConnectionError                      static PyTypeObject _PyExc_ConnectionError
+Objects/exceptions.c:_PyExc_ConnectionRefusedError               static PyTypeObject _PyExc_ConnectionRefusedError
+Objects/exceptions.c:_PyExc_ConnectionResetError                 static PyTypeObject _PyExc_ConnectionResetError
+Objects/exceptions.c:_PyExc_DeprecationWarning                   static PyTypeObject _PyExc_DeprecationWarning
+Objects/exceptions.c:_PyExc_EOFError                             static PyTypeObject _PyExc_EOFError
+Objects/exceptions.c:_PyExc_Exception                            static PyTypeObject _PyExc_Exception
+Objects/exceptions.c:_PyExc_FileExistsError                      static PyTypeObject _PyExc_FileExistsError
+Objects/exceptions.c:_PyExc_FileNotFoundError                    static PyTypeObject _PyExc_FileNotFoundError
+Objects/exceptions.c:_PyExc_FloatingPointError                   static PyTypeObject _PyExc_FloatingPointError
+Objects/exceptions.c:_PyExc_FutureWarning                        static PyTypeObject _PyExc_FutureWarning
+Objects/exceptions.c:_PyExc_GeneratorExit                        static PyTypeObject _PyExc_GeneratorExit
+Objects/exceptions.c:_PyExc_ImportError                          static PyTypeObject _PyExc_ImportError
+Objects/exceptions.c:_PyExc_ImportWarning                        static PyTypeObject _PyExc_ImportWarning
+Objects/exceptions.c:_PyExc_IndentationError                     static PyTypeObject _PyExc_IndentationError
+Objects/exceptions.c:_PyExc_IndexError                           static PyTypeObject _PyExc_IndexError
+Objects/exceptions.c:_PyExc_InterruptedError                     static PyTypeObject _PyExc_InterruptedError
+Objects/exceptions.c:_PyExc_IsADirectoryError                    static PyTypeObject _PyExc_IsADirectoryError
+Objects/exceptions.c:_PyExc_KeyError                             static PyTypeObject _PyExc_KeyError
+Objects/exceptions.c:_PyExc_KeyboardInterrupt                    static PyTypeObject _PyExc_KeyboardInterrupt
+Objects/exceptions.c:_PyExc_LookupError                          static PyTypeObject _PyExc_LookupError
+Objects/exceptions.c:_PyExc_MemoryError                          static PyTypeObject _PyExc_MemoryError
+Objects/exceptions.c:_PyExc_ModuleNotFoundError                  static PyTypeObject _PyExc_ModuleNotFoundError
+Objects/exceptions.c:_PyExc_NameError                            static PyTypeObject _PyExc_NameError
+Objects/exceptions.c:_PyExc_NotADirectoryError                   static PyTypeObject _PyExc_NotADirectoryError
+Objects/exceptions.c:_PyExc_NotImplementedError                  static PyTypeObject _PyExc_NotImplementedError
+Objects/exceptions.c:_PyExc_OSError                              static PyTypeObject _PyExc_OSError
+Objects/exceptions.c:_PyExc_OverflowError                        static PyTypeObject _PyExc_OverflowError
+Objects/exceptions.c:_PyExc_PendingDeprecationWarning            static PyTypeObject _PyExc_PendingDeprecationWarning
+Objects/exceptions.c:_PyExc_PermissionError                      static PyTypeObject _PyExc_PermissionError
+Objects/exceptions.c:_PyExc_ProcessLookupError                   static PyTypeObject _PyExc_ProcessLookupError
+Objects/exceptions.c:_PyExc_RecursionError                       static PyTypeObject _PyExc_RecursionError
+Objects/exceptions.c:_PyExc_ReferenceError                       static PyTypeObject _PyExc_ReferenceError
+Objects/exceptions.c:_PyExc_ResourceWarning                      static PyTypeObject _PyExc_ResourceWarning
+Objects/exceptions.c:_PyExc_RuntimeError                         static PyTypeObject _PyExc_RuntimeError
+Objects/exceptions.c:_PyExc_RuntimeWarning                       static PyTypeObject _PyExc_RuntimeWarning
+Objects/exceptions.c:_PyExc_StopAsyncIteration                   static PyTypeObject _PyExc_StopAsyncIteration
+Objects/exceptions.c:_PyExc_StopIteration                        static PyTypeObject _PyExc_StopIteration
+Objects/exceptions.c:_PyExc_SyntaxError                          static PyTypeObject _PyExc_SyntaxError
+Objects/exceptions.c:_PyExc_SyntaxWarning                        static PyTypeObject _PyExc_SyntaxWarning
+Objects/exceptions.c:_PyExc_SystemError                          static PyTypeObject _PyExc_SystemError
+Objects/exceptions.c:_PyExc_SystemExit                           static PyTypeObject _PyExc_SystemExit
+Objects/exceptions.c:_PyExc_TabError                             static PyTypeObject _PyExc_TabError
+Objects/exceptions.c:_PyExc_TimeoutError                         static PyTypeObject _PyExc_TimeoutError
+Objects/exceptions.c:_PyExc_TypeError                            static PyTypeObject _PyExc_TypeError
+Objects/exceptions.c:_PyExc_UnboundLocalError                    static PyTypeObject _PyExc_UnboundLocalError
+Objects/exceptions.c:_PyExc_UnicodeDecodeError                   static PyTypeObject _PyExc_UnicodeDecodeError
+Objects/exceptions.c:_PyExc_UnicodeEncodeError                   static PyTypeObject _PyExc_UnicodeEncodeError
+Objects/exceptions.c:_PyExc_UnicodeError                         static PyTypeObject _PyExc_UnicodeError
+Objects/exceptions.c:_PyExc_UnicodeTranslateError                static PyTypeObject _PyExc_UnicodeTranslateError
+Objects/exceptions.c:_PyExc_UnicodeWarning                       static PyTypeObject _PyExc_UnicodeWarning
+Objects/exceptions.c:_PyExc_UserWarning                          static PyTypeObject _PyExc_UserWarning
+Objects/exceptions.c:_PyExc_ValueError                           static PyTypeObject _PyExc_ValueError
+Objects/exceptions.c:_PyExc_Warning                              static PyTypeObject _PyExc_Warning
+Objects/exceptions.c:_PyExc_ZeroDivisionError                    static PyTypeObject _PyExc_ZeroDivisionError
+Objects/fileobject.c:PyStdPrinter_Type                           PyTypeObject PyStdPrinter_Type
+Objects/floatobject.c:FloatInfoType                              static PyTypeObject FloatInfoType
+Objects/floatobject.c:PyFloat_Type                               PyTypeObject PyFloat_Type
+Objects/frameobject.c:PyFrame_Type                               PyTypeObject PyFrame_Type
+Objects/funcobject.c:PyClassMethod_Type                          PyTypeObject PyClassMethod_Type
+Objects/funcobject.c:PyFunction_Type                             PyTypeObject PyFunction_Type
+Objects/funcobject.c:PyStaticMethod_Type                         PyTypeObject PyStaticMethod_Type
+Objects/genobject.c:PyAsyncGen_Type                              PyTypeObject PyAsyncGen_Type
+Objects/genobject.c:PyCoro_Type                                  PyTypeObject PyCoro_Type
+Objects/genobject.c:PyGen_Type                                   PyTypeObject PyGen_Type
+Objects/genobject.c:_PyAsyncGenASend_Type                        PyTypeObject _PyAsyncGenASend_Type
+Objects/genobject.c:_PyAsyncGenAThrow_Type                       PyTypeObject _PyAsyncGenAThrow_Type
+Objects/genobject.c:_PyAsyncGenWrappedValue_Type                 PyTypeObject _PyAsyncGenWrappedValue_Type
+Objects/genobject.c:_PyCoroWrapper_Type                          PyTypeObject _PyCoroWrapper_Type
+Objects/interpreteridobject.c:_PyInterpreterID_Type              PyTypeObject _PyInterpreterID_Type
+Objects/iterobject.c:PyCallIter_Type                             PyTypeObject PyCallIter_Type
+Objects/iterobject.c:PySeqIter_Type                              PyTypeObject PySeqIter_Type
+Objects/listobject.c:PyListIter_Type                             PyTypeObject PyListIter_Type
+Objects/listobject.c:PyListRevIter_Type                          PyTypeObject PyListRevIter_Type
+Objects/listobject.c:PyList_Type                                 PyTypeObject PyList_Type
+Objects/longobject.c:Int_InfoType                                static PyTypeObject Int_InfoType
+Objects/longobject.c:PyLong_Type                                 PyTypeObject PyLong_Type
+Objects/memoryobject.c:PyMemoryView_Type                         PyTypeObject PyMemoryView_Type
+Objects/memoryobject.c:_PyManagedBuffer_Type                     PyTypeObject _PyManagedBuffer_Type
+Objects/methodobject.c:PyCFunction_Type                          PyTypeObject PyCFunction_Type
+Objects/moduleobject.c:PyModuleDef_Type                          PyTypeObject PyModuleDef_Type
+Objects/moduleobject.c:PyModule_Type                             PyTypeObject PyModule_Type
+Objects/namespaceobject.c:_PyNamespace_Type                      PyTypeObject _PyNamespace_Type
+Objects/object.c:_PyNone_Type                                    PyTypeObject _PyNone_Type
+Objects/object.c:_PyNotImplemented_Type                          PyTypeObject _PyNotImplemented_Type
+Objects/odictobject.c:PyODictItems_Type                          PyTypeObject PyODictItems_Type
+Objects/odictobject.c:PyODictIter_Type                           PyTypeObject PyODictIter_Type
+Objects/odictobject.c:PyODictKeys_Type                           PyTypeObject PyODictKeys_Type
+Objects/odictobject.c:PyODictValues_Type                         PyTypeObject PyODictValues_Type
+Objects/odictobject.c:PyODict_Type                               PyTypeObject PyODict_Type
+Objects/picklebufobject.c:PyPickleBuffer_Type                    PyTypeObject PyPickleBuffer_Type
+Objects/rangeobject.c:PyLongRangeIter_Type                       PyTypeObject PyLongRangeIter_Type
+Objects/rangeobject.c:PyRangeIter_Type                           PyTypeObject PyRangeIter_Type
+Objects/rangeobject.c:PyRange_Type                               PyTypeObject PyRange_Type
+Objects/setobject.c:PyFrozenSet_Type                             PyTypeObject PyFrozenSet_Type
+Objects/setobject.c:PySetIter_Type                               PyTypeObject PySetIter_Type
+Objects/setobject.c:PySet_Type                                   PyTypeObject PySet_Type
+Objects/setobject.c:_PySetDummy_Type                             static PyTypeObject _PySetDummy_Type
+Objects/sliceobject.c:PyEllipsis_Type                            PyTypeObject PyEllipsis_Type
+Objects/sliceobject.c:PySlice_Type                               PyTypeObject PySlice_Type
+Objects/stringlib/unicode_format.h:PyFieldNameIter_Type          static PyTypeObject PyFieldNameIter_Type
+Objects/stringlib/unicode_format.h:PyFormatterIter_Type          static PyTypeObject PyFormatterIter_Type
+Objects/tupleobject.c:PyTupleIter_Type                           PyTypeObject PyTupleIter_Type
+Objects/tupleobject.c:PyTuple_Type                               PyTypeObject PyTuple_Type
+Objects/typeobject.c:PyBaseObject_Type                           PyTypeObject PyBaseObject_Type
+Objects/typeobject.c:PySuper_Type                                PyTypeObject PySuper_Type
+Objects/typeobject.c:PyType_Type                                 PyTypeObject PyType_Type
+Objects/unicodeobject.c:EncodingMapType                          static PyTypeObject EncodingMapType
+Objects/unicodeobject.c:PyUnicodeIter_Type                       PyTypeObject PyUnicodeIter_Type
+Objects/unicodeobject.c:PyUnicode_Type                           PyTypeObject PyUnicode_Type
+Objects/weakrefobject.c:_PyWeakref_CallableProxyType             PyTypeObject _PyWeakref_CallableProxyType
+Objects/weakrefobject.c:_PyWeakref_ProxyType                     PyTypeObject _PyWeakref_ProxyType
+Objects/weakrefobject.c:_PyWeakref_RefType                       PyTypeObject _PyWeakref_RefType
+Python/bltinmodule.c:PyFilter_Type                               PyTypeObject PyFilter_Type
+Python/bltinmodule.c:PyMap_Type                                  PyTypeObject PyMap_Type
+Python/bltinmodule.c:PyZip_Type                                  PyTypeObject PyZip_Type
+Python/context.c:PyContextTokenMissing_Type                      PyTypeObject PyContextTokenMissing_Type
+Python/context.c:PyContextToken_Type                             PyTypeObject PyContextToken_Type
+Python/context.c:PyContextVar_Type                               PyTypeObject PyContextVar_Type
+Python/context.c:PyContext_Type                                  PyTypeObject PyContext_Type
+Python/errors.c:UnraisableHookArgsType                           static PyTypeObject UnraisableHookArgsType
+Python/hamt.c:_PyHamtItems_Type                                  PyTypeObject _PyHamtItems_Type
+Python/hamt.c:_PyHamtKeys_Type                                   PyTypeObject _PyHamtKeys_Type
+Python/hamt.c:_PyHamtValues_Type                                 PyTypeObject _PyHamtValues_Type
+Python/hamt.c:_PyHamt_ArrayNode_Type                             PyTypeObject _PyHamt_ArrayNode_Type
+Python/hamt.c:_PyHamt_BitmapNode_Type                            PyTypeObject _PyHamt_BitmapNode_Type
+Python/hamt.c:_PyHamt_CollisionNode_Type                         PyTypeObject _PyHamt_CollisionNode_Type
+Python/hamt.c:_PyHamt_Type                                       PyTypeObject _PyHamt_Type
+Python/symtable.c:PySTEntry_Type                                 PyTypeObject PySTEntry_Type
+Python/sysmodule.c:AsyncGenHooksType                             static PyTypeObject AsyncGenHooksType
+Python/sysmodule.c:FlagsType                                     static PyTypeObject FlagsType
+Python/sysmodule.c:Hash_InfoType                                 static PyTypeObject Hash_InfoType
+Python/sysmodule.c:VersionInfoType                               static PyTypeObject VersionInfoType
+Python/thread.c:ThreadInfoType                                   static PyTypeObject ThreadInfoType
+Python/traceback.c:PyTraceBack_Type                              PyTypeObject PyTraceBack_Type
+
+
+# _PyArg_Parser (147)
+Modules/_blake2/clinic/blake2b_impl.c.h:py_blake2b_new():_parser static _PyArg_Parser _parser
+Modules/_blake2/clinic/blake2s_impl.c.h:py_blake2s_new():_parser static _PyArg_Parser _parser
+Modules/_io/clinic/_iomodule.c.h:_io_open():_parser              static _PyArg_Parser _parser
+Modules/_io/clinic/_iomodule.c.h:_io_open_code():_parser         static _PyArg_Parser _parser
+Modules/_io/clinic/bufferedio.c.h:_io_BufferedRandom___init__():_parser static _PyArg_Parser _parser
+Modules/_io/clinic/bufferedio.c.h:_io_BufferedReader___init__():_parser static _PyArg_Parser _parser
+Modules/_io/clinic/bufferedio.c.h:_io_BufferedWriter___init__():_parser static _PyArg_Parser _parser
+Modules/_io/clinic/bytesio.c.h:_io_BytesIO___init__():_parser    static _PyArg_Parser _parser
+Modules/_io/clinic/fileio.c.h:_io_FileIO___init__():_parser      static _PyArg_Parser _parser
+Modules/_io/clinic/stringio.c.h:_io_StringIO___init__():_parser  static _PyArg_Parser _parser
+Modules/_io/clinic/textio.c.h:_io_IncrementalNewlineDecoder___init__():_parser static _PyArg_Parser _parser
+Modules/_io/clinic/textio.c.h:_io_IncrementalNewlineDecoder_decode():_parser static _PyArg_Parser _parser
+Modules/_io/clinic/textio.c.h:_io_TextIOWrapper___init__():_parser static _PyArg_Parser _parser
+Modules/_io/clinic/textio.c.h:_io_TextIOWrapper_reconfigure():_parser static _PyArg_Parser _parser
+Modules/_io/clinic/winconsoleio.c.h:_io__WindowsConsoleIO___init__():_parser static _PyArg_Parser _parser
+Modules/_multiprocessing/clinic/posixshmem.c.h:_posixshmem_shm_open():_parser static _PyArg_Parser _parser
+Modules/_multiprocessing/clinic/posixshmem.c.h:_posixshmem_shm_unlink():_parser static _PyArg_Parser _parser
+Modules/cjkcodecs/clinic/multibytecodec.c.h:_multibytecodec_MultibyteCodec_decode():_parser static _PyArg_Parser _parser
+Modules/cjkcodecs/clinic/multibytecodec.c.h:_multibytecodec_MultibyteCodec_encode():_parser static _PyArg_Parser _parser
+Modules/cjkcodecs/clinic/multibytecodec.c.h:_multibytecodec_MultibyteIncrementalDecoder_decode():_parser static _PyArg_Parser _parser
+Modules/cjkcodecs/clinic/multibytecodec.c.h:_multibytecodec_MultibyteIncrementalEncoder_encode():_parser static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h:_asyncio_Future___init__():_parser static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h:_asyncio_Future_add_done_callback():_parser static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h:_asyncio_Task___init__():_parser static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h:_asyncio_Task_all_tasks():_parser static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h:_asyncio_Task_current_task():_parser static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h:_asyncio_Task_get_stack():_parser static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h:_asyncio_Task_print_stack():_parser static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h:_asyncio__enter_task():_parser static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h:_asyncio__leave_task():_parser static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h:_asyncio__register_task():_parser static _PyArg_Parser _parser
+Modules/clinic/_asynciomodule.c.h:_asyncio__unregister_task():_parser static _PyArg_Parser _parser
+Modules/clinic/_bz2module.c.h:_bz2_BZ2Decompressor_decompress():_parser static _PyArg_Parser _parser
+Modules/clinic/_codecsmodule.c.h:_codecs_decode():_parser        static _PyArg_Parser _parser
+Modules/clinic/_codecsmodule.c.h:_codecs_encode():_parser        static _PyArg_Parser _parser
+Modules/clinic/_cursesmodule.c.h:_curses_setupterm():_parser     static _PyArg_Parser _parser
+Modules/clinic/_datetimemodule.c.h:datetime_datetime_now():_parser static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h:_elementtree_Element_find():_parser static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h:_elementtree_Element_findall():_parser static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h:_elementtree_Element_findtext():_parser static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h:_elementtree_Element_get():_parser static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h:_elementtree_Element_getiterator():_parser static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h:_elementtree_Element_iter():_parser static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h:_elementtree_Element_iterfind():_parser static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h:_elementtree_TreeBuilder___init__():_parser static _PyArg_Parser _parser
+Modules/clinic/_elementtree.c.h:_elementtree_XMLParser___init__():_parser static _PyArg_Parser _parser
+Modules/clinic/_hashopenssl.c.h:EVP_new():_parser                static _PyArg_Parser _parser
+Modules/clinic/_hashopenssl.c.h:_hashlib_hmac_digest():_parser   static _PyArg_Parser _parser
+Modules/clinic/_hashopenssl.c.h:_hashlib_scrypt():_parser        static _PyArg_Parser _parser
+Modules/clinic/_hashopenssl.c.h:pbkdf2_hmac():_parser            static _PyArg_Parser _parser
+Modules/clinic/_lzmamodule.c.h:_lzma_LZMADecompressor___init__():_parser static _PyArg_Parser _parser
+Modules/clinic/_lzmamodule.c.h:_lzma_LZMADecompressor_decompress():_parser static _PyArg_Parser _parser
+Modules/clinic/_opcode.c.h:_opcode_stack_effect():_parser        static _PyArg_Parser _parser
+Modules/clinic/_pickle.c.h:_pickle_Pickler___init__():_parser    static _PyArg_Parser _parser
+Modules/clinic/_pickle.c.h:_pickle_Unpickler___init__():_parser  static _PyArg_Parser _parser
+Modules/clinic/_pickle.c.h:_pickle_dump():_parser                static _PyArg_Parser _parser
+Modules/clinic/_pickle.c.h:_pickle_dumps():_parser               static _PyArg_Parser _parser
+Modules/clinic/_pickle.c.h:_pickle_load():_parser                static _PyArg_Parser _parser
+Modules/clinic/_pickle.c.h:_pickle_loads():_parser               static _PyArg_Parser _parser
+Modules/clinic/_queuemodule.c.h:_queue_SimpleQueue_get():_parser static _PyArg_Parser _parser
+Modules/clinic/_queuemodule.c.h:_queue_SimpleQueue_put():_parser static _PyArg_Parser _parser
+Modules/clinic/_queuemodule.c.h:_queue_SimpleQueue_put_nowait():_parser static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_SRE_Match_expand():_parser          static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_SRE_Match_groupdict():_parser       static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_SRE_Match_groups():_parser          static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_SRE_Pattern_findall():_parser       static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_SRE_Pattern_finditer():_parser      static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_SRE_Pattern_fullmatch():_parser     static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_SRE_Pattern_match():_parser         static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_SRE_Pattern_scanner():_parser       static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_SRE_Pattern_search():_parser        static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_SRE_Pattern_split():_parser         static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_SRE_Pattern_sub():_parser           static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_SRE_Pattern_subn():_parser          static _PyArg_Parser _parser
+Modules/clinic/_sre.c.h:_sre_compile():_parser                   static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h:_ssl__SSLContext__wrap_bio():_parser     static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h:_ssl__SSLContext__wrap_socket():_parser  static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h:_ssl__SSLContext_get_ca_certs():_parser  static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h:_ssl__SSLContext_load_cert_chain():_parser static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h:_ssl__SSLContext_load_verify_locations():_parser static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h:_ssl__SSLSocket_get_channel_binding():_parser static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h:_ssl_enum_certificates():_parser         static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h:_ssl_enum_crls():_parser                 static _PyArg_Parser _parser
+Modules/clinic/_ssl.c.h:_ssl_txt2obj():_parser                   static _PyArg_Parser _parser
+Modules/clinic/_struct.c.h:Struct___init__():_parser             static _PyArg_Parser _parser
+Modules/clinic/_struct.c.h:Struct_unpack_from():_parser          static _PyArg_Parser _parser
+Modules/clinic/_struct.c.h:unpack_from():_parser                 static _PyArg_Parser _parser
+Modules/clinic/_winapi.c.h:_winapi_ConnectNamedPipe():_parser    static _PyArg_Parser _parser
+Modules/clinic/_winapi.c.h:_winapi_GetFileType():_parser         static _PyArg_Parser _parser
+Modules/clinic/_winapi.c.h:_winapi_ReadFile():_parser            static _PyArg_Parser _parser
+Modules/clinic/_winapi.c.h:_winapi_WriteFile():_parser           static _PyArg_Parser _parser
+Modules/clinic/binascii.c.h:binascii_a2b_qp():_parser            static _PyArg_Parser _parser
+Modules/clinic/binascii.c.h:binascii_b2a_base64():_parser        static _PyArg_Parser _parser
+Modules/clinic/binascii.c.h:binascii_b2a_hex():_parser           static _PyArg_Parser _parser
+Modules/clinic/binascii.c.h:binascii_b2a_qp():_parser            static _PyArg_Parser _parser
+Modules/clinic/binascii.c.h:binascii_b2a_uu():_parser            static _PyArg_Parser _parser
+Modules/clinic/binascii.c.h:binascii_hexlify():_parser           static _PyArg_Parser _parser
+Modules/clinic/cmathmodule.c.h:cmath_isclose():_parser           static _PyArg_Parser _parser
+Modules/clinic/gcmodule.c.h:gc_collect():_parser                 static _PyArg_Parser _parser
+Modules/clinic/gcmodule.c.h:gc_get_objects():_parser             static _PyArg_Parser _parser
+Modules/clinic/grpmodule.c.h:grp_getgrgid():_parser              static _PyArg_Parser _parser
+Modules/clinic/grpmodule.c.h:grp_getgrnam():_parser              static _PyArg_Parser _parser
+Objects/clinic/bytearrayobject.c.h:bytearray_decode():_parser    static _PyArg_Parser _parser
+Objects/clinic/bytearrayobject.c.h:bytearray_hex():_parser       static _PyArg_Parser _parser
+Objects/clinic/bytearrayobject.c.h:bytearray_rsplit():_parser    static _PyArg_Parser _parser
+Objects/clinic/bytearrayobject.c.h:bytearray_split():_parser     static _PyArg_Parser _parser
+Objects/clinic/bytearrayobject.c.h:bytearray_splitlines():_parser static _PyArg_Parser _parser
+Objects/clinic/bytearrayobject.c.h:bytearray_translate():_parser static _PyArg_Parser _parser
+Objects/clinic/bytesobject.c.h:bytes_decode():_parser            static _PyArg_Parser _parser
+Objects/clinic/bytesobject.c.h:bytes_hex():_parser               static _PyArg_Parser _parser
+Objects/clinic/bytesobject.c.h:bytes_rsplit():_parser            static _PyArg_Parser _parser
+Objects/clinic/bytesobject.c.h:bytes_split():_parser             static _PyArg_Parser _parser
+Objects/clinic/bytesobject.c.h:bytes_splitlines():_parser        static _PyArg_Parser _parser
+Objects/clinic/bytesobject.c.h:bytes_translate():_parser         static _PyArg_Parser _parser
+Objects/clinic/codeobject.c.h:code_replace():_parser             static _PyArg_Parser _parser
+Objects/clinic/complexobject.c.h:complex_new():_parser           static _PyArg_Parser _parser
+Objects/clinic/descrobject.c.h:mappingproxy_new():_parser        static _PyArg_Parser _parser
+Objects/clinic/descrobject.c.h:property_init():_parser           static _PyArg_Parser _parser
+Objects/clinic/enumobject.c.h:enum_new():_parser                 static _PyArg_Parser _parser
+Objects/clinic/funcobject.c.h:func_new():_parser                 static _PyArg_Parser _parser
+Objects/clinic/listobject.c.h:list_sort():_parser                static _PyArg_Parser _parser
+Objects/clinic/longobject.c.h:int_from_bytes():_parser           static _PyArg_Parser _parser
+Objects/clinic/longobject.c.h:int_to_bytes():_parser             static _PyArg_Parser _parser
+Objects/clinic/longobject.c.h:long_new():_parser                 static _PyArg_Parser _parser
+Objects/clinic/memoryobject.c.h:memoryview_hex():_parser         static _PyArg_Parser _parser
+Objects/clinic/moduleobject.c.h:module___init__():_parser        static _PyArg_Parser _parser
+Objects/clinic/odictobject.c.h:OrderedDict_fromkeys():_parser    static _PyArg_Parser _parser
+Objects/clinic/odictobject.c.h:OrderedDict_move_to_end():_parser static _PyArg_Parser _parser
+Objects/clinic/odictobject.c.h:OrderedDict_popitem():_parser     static _PyArg_Parser _parser
+Objects/clinic/odictobject.c.h:OrderedDict_setdefault():_parser  static _PyArg_Parser _parser
+Objects/clinic/structseq.c.h:structseq_new():_parser             static _PyArg_Parser _parser
+Objects/clinic/unicodeobject.c.h:unicode_encode():_parser        static _PyArg_Parser _parser
+Objects/clinic/unicodeobject.c.h:unicode_expandtabs():_parser    static _PyArg_Parser _parser
+Objects/clinic/unicodeobject.c.h:unicode_rsplit():_parser        static _PyArg_Parser _parser
+Objects/clinic/unicodeobject.c.h:unicode_split():_parser         static _PyArg_Parser _parser
+Objects/clinic/unicodeobject.c.h:unicode_splitlines():_parser    static _PyArg_Parser _parser
+Objects/stringlib/clinic/transmogrify.h.h:stringlib_expandtabs():_parser static _PyArg_Parser _parser
+Python/bltinmodule.c:builtin_print():_parser                     static struct _PyArg_Parser _parser
+Python/clinic/_warnings.c.h:warnings_warn():_parser              static _PyArg_Parser _parser
+Python/clinic/bltinmodule.c.h:builtin_compile():_parser          static _PyArg_Parser _parser
+Python/clinic/bltinmodule.c.h:builtin_round():_parser            static _PyArg_Parser _parser
+Python/clinic/bltinmodule.c.h:builtin_sum():_parser              static _PyArg_Parser _parser
+Python/clinic/import.c.h:_imp_source_hash():_parser              static _PyArg_Parser _parser
+Python/clinic/sysmodule.c.h:sys_addaudithook():_parser           static _PyArg_Parser _parser
+Python/clinic/sysmodule.c.h:sys_set_coroutine_origin_tracking_depth():_parser static _PyArg_Parser _parser
+Python/clinic/traceback.c.h:tb_new():_parser                     static _PyArg_Parser _parser
diff --git a/Tools/c-analyzer/c_globals/supported.py b/Tools/c-analyzer/c_globals/supported.py
index 4643e4ef6e88..d185daa2463b 100644
--- a/Tools/c-analyzer/c_globals/supported.py
+++ b/Tools/c-analyzer/c_globals/supported.py
@@ -253,25 +253,50 @@ def _is_vartype_okay(vartype, ignoredtypes=None):
     return None
 
 
+PYOBJECT_RE = re.compile(r'''
+        ^
+        (
+            # must start with "static "
+            static \s+
+            (
+                identifier
+            )
+            \b
+        ) |
+        (
+            # may start with "static "
+            ( static \s+ )?
+            (
+                .*
+                (
+                    PyObject |
+                    PyTypeObject |
+                    _? Py \w+ Object |
+                    _PyArg_Parser |
+                    _Py_Identifier |
+                    traceback_t |
+                    PyAsyncGenASend |
+                    _PyAsyncGenWrappedValue |
+                    PyContext |
+                    method_cache_entry
+                )
+                \b
+            ) |
+            (
+                (
+                    _Py_IDENTIFIER |
+                    _Py_static_string
+                )
+                [(]
+            )
+        )
+        ''', re.VERBOSE)
+
+
 def _is_object(vartype):
-    if re.match(r'.*\bPy\w*Object\b', vartype):
-        return True
-    if '_PyArg_Parser ' in vartype:
-        return True
-    if vartype.startswith(('_Py_IDENTIFIER(', 'static _Py_Identifier',
-                           '_Py_static_string(')):
-        return True
-    if 'traceback_t' in vartype:
-        return True
-    if 'PyAsyncGenASend' in vartype:
-        return True
-    if '_PyAsyncGenWrappedValue' in vartype:
-        return True
-    if 'PyContext' in vartype:
-        return True
-    if 'method_cache_entry' in vartype:
-        return True
-    if vartype.startswith('static identifier '):
+    if 'PyDictKeysObject' in vartype:
+        return False
+    if PYOBJECT_RE.match(vartype):
         return True
     if vartype.endswith((' _Py_FalseStruct', ' _Py_TrueStruct')):
         return True
diff --git a/Tools/c-analyzer/known.tsv b/Tools/c-analyzer/known.tsv
index ce2afcb9591e..db44080bc2e3 100644
--- a/Tools/c-analyzer/known.tsv
+++ b/Tools/c-analyzer/known.tsv
@@ -787,6 +787,8 @@ Objects/genobject.c	-	_PyAsyncGenAThrow_Type	variable	PyTypeObject _PyAsyncGenAT
 Objects/genobject.c	-	PyAsyncGen_Type	variable	PyTypeObject PyAsyncGen_Type
 Objects/genobject.c	-	_PyAsyncGenWrappedValue_Type	variable	PyTypeObject _PyAsyncGenWrappedValue_Type
 Objects/typeobject.c	-	PyBaseObject_Type	variable	PyTypeObject PyBaseObject_Type
+Modules/_blake2/blake2b_impl.c	-	PyBlake2_BLAKE2bType	variable	PyTypeObject PyBlake2_BLAKE2bType
+Modules/_blake2/blake2s_impl.c	-	PyBlake2_BLAKE2sType	variable	PyTypeObject PyBlake2_BLAKE2sType
 Objects/boolobject.c	-	PyBool_Type	variable	PyTypeObject PyBool_Type
 Modules/_io/bufferedio.c	-	PyBufferedIOBase_Type	variable	PyTypeObject PyBufferedIOBase_Type
 Modules/_io/bufferedio.c	-	PyBufferedRandom_Type	variable	PyTypeObject PyBufferedRandom_Type
@@ -972,6 +974,7 @@ Objects/exceptions.c	-	PyExc_Warning	variable	static PyTypeObject PyExc_Warning
 Objects/exceptions.c	-	_PyExc_ZeroDivisionError	variable	static PyTypeObject _PyExc_ZeroDivisionError
 Objects/exceptions.c	-	PyExc_ZeroDivisionError	variable	static PyTypeObject PyExc_ZeroDivisionError
 Objects/boolobject.c	-	_Py_FalseStruct	variable	static struct _longobject _Py_FalseStruct
+Objects/tupleobject.c	-	_Py_fast_tuple_allocs	variable	Py_ssize_t _Py_fast_tuple_allocs
 Objects/stringlib/unicode_format.h	-	PyFieldNameIter_Type	variable	static PyTypeObject PyFieldNameIter_Type
 Modules/_io/fileio.c	-	PyFileIO_Type	variable	PyTypeObject PyFileIO_Type
 Python/preconfig.c	-	Py_FileSystemDefaultEncodeErrors	variable	const char *Py_FileSystemDefaultEncodeErrors
@@ -1498,6 +1501,7 @@ Python/initconfig.c	-	Py_NoSiteFlag	variable	int Py_NoSiteFlag
 Objects/object.c	-	_Py_NotImplementedStruct	variable	PyObject _Py_NotImplementedStruct
 Objects/object.c	-	_PyNotImplemented_Type	variable	PyTypeObject _PyNotImplemented_Type
 Python/initconfig.c	-	Py_NoUserSiteDirectory	variable	int Py_NoUserSiteDirectory
+Objects/bytesobject.c	-	_Py_null_strings	variable	Py_ssize_t _Py_null_strings
 Objects/obmalloc.c	-	_PyObject	variable	static PyMemAllocatorEx _PyObject
 Objects/obmalloc.c	-	_PyObject_Arena	variable	static PyObjectArenaAllocator _PyObject_Arena
 Objects/odictobject.c	-	PyODictItems_Type	variable	PyTypeObject PyODictItems_Type
@@ -1506,6 +1510,7 @@ Objects/odictobject.c	-	PyODictKeys_Type	variable	PyTypeObject PyODictKeys_Type
 Objects/odictobject.c	-	PyODict_Type	variable	PyTypeObject PyODict_Type
 Objects/odictobject.c	-	PyODictValues_Type	variable	PyTypeObject PyODictValues_Type
 Python/fileutils.c	-	_Py_open_cloexec_works	variable	int _Py_open_cloexec_works
+Objects/bytesobject.c	-	_Py_one_strings	variable	Py_ssize_t _Py_one_strings
 Python/initconfig.c	-	Py_OptimizeFlag	variable	int Py_OptimizeFlag
 Parser/myreadline.c	-	PyOS_InputHook	variable	int (*PyOS_InputHook)(void)
 Python/pylifecycle.c	-	_PyOS_mystrnicmp_hack	variable	int (*_PyOS_mystrnicmp_hack)(const char *, const char *, Py_ssize_t)
@@ -1521,6 +1526,8 @@ Python/pathconfig.c	-	_Py_path_config	variable	_PyPathConfig _Py_path_config
 Objects/picklebufobject.c	-	PyPickleBuffer_Type	variable	PyTypeObject PyPickleBuffer_Type
 Objects/descrobject.c	-	PyProperty_Type	variable	PyTypeObject PyProperty_Type
 Python/initconfig.c	-	Py_QuietFlag	variable	int Py_QuietFlag
+Objects.longobject.c	-	_Py_quick_int_allocs	variable	Py_ssize_t _Py_quick_int_allocs
+Objects.longobject.c	-	_Py_quick_new_int_allocs	variable	Py_ssize_t _Py_quick_new_int_allocs
 Objects/rangeobject.c	-	PyRangeIter_Type	variable	PyTypeObject PyRangeIter_Type
 Objects/rangeobject.c	-	PyRange_Type	variable	PyTypeObject PyRange_Type
 Modules/_io/iobase.c	-	PyRawIOBase_Type	variable	PyTypeObject PyRawIOBase_Type
@@ -1551,6 +1558,7 @@ Objects/obmalloc.c	-	_Py_tracemalloc_config	variable	struct _PyTraceMalloc_Confi
 Objects/boolobject.c	-	_Py_TrueStruct	variable	static struct _longobject _Py_TrueStruct
 Objects/tupleobject.c	-	PyTupleIter_Type	variable	PyTypeObject PyTupleIter_Type
 Objects/tupleobject.c	-	PyTuple_Type	variable	PyTypeObject PyTuple_Type
+Objects/tupleobject.c	-	_Py_tuple_zero_allocs	variable	Py_ssize_t _Py_tuple_zero_allocs
 Objects/typeobject.c	-	PyType_Type	variable	PyTypeObject PyType_Type
 Python/initconfig.c	-	Py_UnbufferedStdioFlag	variable	int Py_UnbufferedStdioFlag
 Python/pylifecycle.c	-	_Py_UnhandledKeyboardInterrupt	variable	int _Py_UnhandledKeyboardInterrupt


From webhook-mailer at python.org  Thu Sep 12 12:04:37 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 16:04:37 -0000
Subject: [Python-checkins] closes bpo-37803: pdb: fix handling of options
 (--help / --version) (GH-15193)
Message-ID: <mailman.684.1568304279.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7da8c0aee2039685f3eff878610684f94f7f7873
commit: 7da8c0aee2039685f3eff878610684f94f7f7873
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T09:04:34-07:00
summary:

closes bpo-37803: pdb: fix handling of options (--help / --version) (GH-15193)


The "--" should not be included with long options passed to
getopt.getopt.

Fixes https://bugs.python.org/issue37803
(cherry picked from commit 855df7f273c3988c72f01e51ba57091887ec38b2)

Co-authored-by: Daniel Hahler <github at thequod.de>

files:
A Misc/NEWS.d/next/Tools-Demos/2019-09-12-16-15-55.bpo-37803.chEizy.rst
M Lib/pdb.py

diff --git a/Lib/pdb.py b/Lib/pdb.py
index 11d763994458..806bc6741f2b 100755
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -1657,7 +1657,7 @@ def help():
 def main():
     import getopt
 
-    opts, args = getopt.getopt(sys.argv[1:], 'mhc:', ['--help', '--command='])
+    opts, args = getopt.getopt(sys.argv[1:], 'mhc:', ['help', 'command='])
 
     if not args:
         print(_usage)
diff --git a/Misc/NEWS.d/next/Tools-Demos/2019-09-12-16-15-55.bpo-37803.chEizy.rst b/Misc/NEWS.d/next/Tools-Demos/2019-09-12-16-15-55.bpo-37803.chEizy.rst
new file mode 100644
index 000000000000..5c0eedb93709
--- /dev/null
+++ b/Misc/NEWS.d/next/Tools-Demos/2019-09-12-16-15-55.bpo-37803.chEizy.rst
@@ -0,0 +1 @@
+pdb's ``--help`` and ``--version`` long options now work.


From webhook-mailer at python.org  Thu Sep 12 12:05:56 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 16:05:56 -0000
Subject: [Python-checkins] closes bpo-37803: pdb: fix handling of options
 (--help / --version) (GH-15193)
Message-ID: <mailman.685.1568304356.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/21bfff9a5a766acfbdd899687850f563893a0f9e
commit: 21bfff9a5a766acfbdd899687850f563893a0f9e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T09:05:52-07:00
summary:

closes bpo-37803: pdb: fix handling of options (--help / --version) (GH-15193)


The "--" should not be included with long options passed to
getopt.getopt.

Fixes https://bugs.python.org/issue37803
(cherry picked from commit 855df7f273c3988c72f01e51ba57091887ec38b2)

Co-authored-by: Daniel Hahler <github at thequod.de>

files:
A Misc/NEWS.d/next/Tools-Demos/2019-09-12-16-15-55.bpo-37803.chEizy.rst
M Lib/pdb.py

diff --git a/Lib/pdb.py b/Lib/pdb.py
index 8c1c96163ed9..8639204891cb 100755
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -1660,7 +1660,7 @@ def help():
 def main():
     import getopt
 
-    opts, args = getopt.getopt(sys.argv[1:], 'mhc:', ['--help', '--command='])
+    opts, args = getopt.getopt(sys.argv[1:], 'mhc:', ['help', 'command='])
 
     if not args:
         print(_usage)
diff --git a/Misc/NEWS.d/next/Tools-Demos/2019-09-12-16-15-55.bpo-37803.chEizy.rst b/Misc/NEWS.d/next/Tools-Demos/2019-09-12-16-15-55.bpo-37803.chEizy.rst
new file mode 100644
index 000000000000..5c0eedb93709
--- /dev/null
+++ b/Misc/NEWS.d/next/Tools-Demos/2019-09-12-16-15-55.bpo-37803.chEizy.rst
@@ -0,0 +1 @@
+pdb's ``--help`` and ``--version`` long options now work.


From webhook-mailer at python.org  Thu Sep 12 12:19:10 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 12 Sep 2019 16:19:10 -0000
Subject: [Python-checkins] Emphasize the need to always call
 PySequence_Fast. (GH-11140)
Message-ID: <mailman.686.1568305151.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/57b7dbc46e71269d855e644d30826d33eedee2a1
commit: 57b7dbc46e71269d855e644d30826d33eedee2a1
branch: master
author: Matti Picus <matti.picus at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-12T17:19:06+01:00
summary:

Emphasize the need to always call PySequence_Fast. (GH-11140)

files:
M Doc/c-api/sequence.rst

diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst
index d11a2dde54dd..fe1c252ce9ba 100644
--- a/Doc/c-api/sequence.rst
+++ b/Doc/c-api/sequence.rst
@@ -130,10 +130,17 @@ Sequence Protocol
 
 .. c:function:: PyObject* PySequence_Fast(PyObject *o, const char *m)
 
-   Return the sequence or iterable *o* as a list, unless it is already a tuple or list, in
-   which case *o* is returned.  Use :c:func:`PySequence_Fast_GET_ITEM` to access
-   the members of the result.  Returns *NULL* on failure.  If the object is not
-   a sequence or iterable, raises :exc:`TypeError` with *m* as the message text.
+   Return the sequence or iterable *o* as an object usable by the other
+   ``PySequence_Fast*`` family of functions. If the object is not a sequence or
+   iterable, raises :exc:`TypeError` with *m* as the message text. Returns
+   *NULL* on failure.
+
+   The ``PySequence_Fast*`` functions are thus named because they assume
+   *o* is a :c:type:`PyTupleObject` or a :c:type:`PyListObject` and access
+   the data fields of *o* directly.
+
+   As a CPython implementation detail, if *o* is already a sequence or list, it
+   will be returned.
 
 
 .. c:function:: Py_ssize_t PySequence_Fast_GET_SIZE(PyObject *o)
@@ -141,8 +148,8 @@ Sequence Protocol
    Returns the length of *o*, assuming that *o* was returned by
    :c:func:`PySequence_Fast` and that *o* is not *NULL*.  The size can also be
    gotten by calling :c:func:`PySequence_Size` on *o*, but
-   :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a list
-   or tuple.
+   :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a
+   list or tuple.
 
 
 .. c:function:: PyObject* PySequence_Fast_GET_ITEM(PyObject *o, Py_ssize_t i)
@@ -163,7 +170,7 @@ Sequence Protocol
 
 .. c:function:: PyObject* PySequence_ITEM(PyObject *o, Py_ssize_t i)
 
-   Return the *i*\ th element of *o* or *NULL* on failure. Macro form of
+   Return the *i*\ th element of *o* or *NULL* on failure. Faster form of
    :c:func:`PySequence_GetItem` but without checking that
    :c:func:`PySequence_Check` on *o* is true and without adjustment for negative
    indices.


From Python-checkins at python.org  Thu Sep 12 03:29:35 2019
From: Python-checkins at python.org (Python-checkins at python.org)
Date: Thu, 12 Sep 2019 15:29:35 +0800 (CST)
Subject: [Python-checkins] Klientskie Bazy
 http://prodawez.tilda.ws/page7270311.html
Message-ID: <20190912072935.7E26B11E3D0C@localhost.localdomain>

Klientskie Bazy http://prodawez.tilda.ws/page7270311.html

From webhook-mailer at python.org  Thu Sep 12 12:25:58 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 16:25:58 -0000
Subject: [Python-checkins] Emphasize the need to always call
 PySequence_Fast. (GH-11140)
Message-ID: <mailman.687.1568305560.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6e0c066f890d66bf07c410bee6d90221a42745a8
commit: 6e0c066f890d66bf07c410bee6d90221a42745a8
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T09:25:55-07:00
summary:

Emphasize the need to always call PySequence_Fast. (GH-11140)

(cherry picked from commit 57b7dbc46e71269d855e644d30826d33eedee2a1)

Co-authored-by: Matti Picus <matti.picus at gmail.com>

files:
M Doc/c-api/sequence.rst

diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst
index 6d22f35e22b1..d653319f8064 100644
--- a/Doc/c-api/sequence.rst
+++ b/Doc/c-api/sequence.rst
@@ -130,10 +130,17 @@ Sequence Protocol
 
 .. c:function:: PyObject* PySequence_Fast(PyObject *o, const char *m)
 
-   Return the sequence or iterable *o* as a list, unless it is already a tuple or list, in
-   which case *o* is returned.  Use :c:func:`PySequence_Fast_GET_ITEM` to access
-   the members of the result.  Returns *NULL* on failure.  If the object is not
-   a sequence or iterable, raises :exc:`TypeError` with *m* as the message text.
+   Return the sequence or iterable *o* as an object usable by the other
+   ``PySequence_Fast*`` family of functions. If the object is not a sequence or
+   iterable, raises :exc:`TypeError` with *m* as the message text. Returns
+   *NULL* on failure.
+
+   The ``PySequence_Fast*`` functions are thus named because they assume
+   *o* is a :c:type:`PyTupleObject` or a :c:type:`PyListObject` and access
+   the data fields of *o* directly.
+
+   As a CPython implementation detail, if *o* is already a sequence or list, it
+   will be returned.
 
 
 .. c:function:: Py_ssize_t PySequence_Fast_GET_SIZE(PyObject *o)
@@ -141,8 +148,8 @@ Sequence Protocol
    Returns the length of *o*, assuming that *o* was returned by
    :c:func:`PySequence_Fast` and that *o* is not *NULL*.  The size can also be
    gotten by calling :c:func:`PySequence_Size` on *o*, but
-   :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a list
-   or tuple.
+   :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a
+   list or tuple.
 
 
 .. c:function:: PyObject* PySequence_Fast_GET_ITEM(PyObject *o, Py_ssize_t i)
@@ -163,7 +170,7 @@ Sequence Protocol
 
 .. c:function:: PyObject* PySequence_ITEM(PyObject *o, Py_ssize_t i)
 
-   Return the *i*\ th element of *o* or *NULL* on failure. Macro form of
+   Return the *i*\ th element of *o* or *NULL* on failure. Faster form of
    :c:func:`PySequence_GetItem` but without checking that
    :c:func:`PySequence_Check` on *o* is true and without adjustment for negative
    indices.


From webhook-mailer at python.org  Thu Sep 12 12:26:21 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 16:26:21 -0000
Subject: [Python-checkins] Emphasize the need to always call
 PySequence_Fast. (GH-11140)
Message-ID: <mailman.688.1568305582.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4145f627599b1894850df4bd6b72d5e86784a300
commit: 4145f627599b1894850df4bd6b72d5e86784a300
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T09:26:17-07:00
summary:

Emphasize the need to always call PySequence_Fast. (GH-11140)

(cherry picked from commit 57b7dbc46e71269d855e644d30826d33eedee2a1)

Co-authored-by: Matti Picus <matti.picus at gmail.com>

files:
M Doc/c-api/sequence.rst

diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst
index d11a2dde54dd..fe1c252ce9ba 100644
--- a/Doc/c-api/sequence.rst
+++ b/Doc/c-api/sequence.rst
@@ -130,10 +130,17 @@ Sequence Protocol
 
 .. c:function:: PyObject* PySequence_Fast(PyObject *o, const char *m)
 
-   Return the sequence or iterable *o* as a list, unless it is already a tuple or list, in
-   which case *o* is returned.  Use :c:func:`PySequence_Fast_GET_ITEM` to access
-   the members of the result.  Returns *NULL* on failure.  If the object is not
-   a sequence or iterable, raises :exc:`TypeError` with *m* as the message text.
+   Return the sequence or iterable *o* as an object usable by the other
+   ``PySequence_Fast*`` family of functions. If the object is not a sequence or
+   iterable, raises :exc:`TypeError` with *m* as the message text. Returns
+   *NULL* on failure.
+
+   The ``PySequence_Fast*`` functions are thus named because they assume
+   *o* is a :c:type:`PyTupleObject` or a :c:type:`PyListObject` and access
+   the data fields of *o* directly.
+
+   As a CPython implementation detail, if *o* is already a sequence or list, it
+   will be returned.
 
 
 .. c:function:: Py_ssize_t PySequence_Fast_GET_SIZE(PyObject *o)
@@ -141,8 +148,8 @@ Sequence Protocol
    Returns the length of *o*, assuming that *o* was returned by
    :c:func:`PySequence_Fast` and that *o* is not *NULL*.  The size can also be
    gotten by calling :c:func:`PySequence_Size` on *o*, but
-   :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a list
-   or tuple.
+   :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a
+   list or tuple.
 
 
 .. c:function:: PyObject* PySequence_Fast_GET_ITEM(PyObject *o, Py_ssize_t i)
@@ -163,7 +170,7 @@ Sequence Protocol
 
 .. c:function:: PyObject* PySequence_ITEM(PyObject *o, Py_ssize_t i)
 
-   Return the *i*\ th element of *o* or *NULL* on failure. Macro form of
+   Return the *i*\ th element of *o* or *NULL* on failure. Faster form of
    :c:func:`PySequence_GetItem` but without checking that
    :c:func:`PySequence_Check` on *o* is true and without adjustment for negative
    indices.


From webhook-mailer at python.org  Thu Sep 12 13:15:48 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Thu, 12 Sep 2019 17:15:48 -0000
Subject: [Python-checkins] bpo-36046: Add user and group parameters to
 subprocess (GH-11950)
Message-ID: <mailman.689.1568308549.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2b2ead74382513d0bb9ef34504e283a71e6a706f
commit: 2b2ead74382513d0bb9ef34504e283a71e6a706f
branch: master
author: Patrick McLean <47801044+patrick-mclean at users.noreply.github.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-12T18:15:44+01:00
summary:

bpo-36046: Add user and group parameters to subprocess (GH-11950)

* subprocess: Add user, group and extra_groups paremeters to subprocess.Popen

This adds a `user` parameter to the Popen constructor that will call
setreuid() in the child before calling exec(). This allows processes
running as root to safely drop privileges before running the subprocess
without having to use a preexec_fn.

This also adds a `group` parameter that will call setregid() in
the child process before calling exec().

Finally an `extra_groups` parameter was added that will call
setgroups() to set the supplimental groups.

files:
A Misc/NEWS.d/next/Library/2019-02-19-17-32-45.bpo-36046.fX9OPj.rst
M Doc/library/subprocess.rst
M Lib/multiprocessing/util.py
M Lib/subprocess.py
M Lib/test/test_capi.py
M Lib/test/test_subprocess.py
M Modules/_posixsubprocess.c

diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
index 954e0fec1182..1a98bb33d0c1 100644
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -339,8 +339,9 @@ functions.
                  stderr=None, preexec_fn=None, close_fds=True, shell=False, \
                  cwd=None, env=None, universal_newlines=None, \
                  startupinfo=None, creationflags=0, restore_signals=True, \
-                 start_new_session=False, pass_fds=(), *, \
-                 encoding=None, errors=None, text=None)
+                 start_new_session=False, pass_fds=(), *, group=None, \
+                 extra_groups=None, user=None, encoding=None, errors=None, \
+                 text=None)
 
    Execute a child program in a new process.  On POSIX, the class uses
    :meth:`os.execvp`-like behavior to execute the child program.  On Windows,
@@ -544,6 +545,33 @@ functions.
    .. versionchanged:: 3.2
       *start_new_session* was added.
 
+   If *group* is not ``None``, the setregid() system call will be made in the
+   child process prior to the execution of the subprocess. If the provided
+   value is a string, it will be looked up via :func:`grp.getgrnam()` and
+   the value in ``gr_gid`` will be used. If the value is an integer, it
+   will be passed verbatim. (POSIX only)
+
+   .. availability:: POSIX
+   .. versionadded:: 3.9
+
+   If *extra_groups* is not ``None``, the setgroups() system call will be
+   made in the child process prior to the execution of the subprocess.
+   Strings provided in *extra_groups* will be looked up via
+   :func:`grp.getgrnam()` and the values in ``gr_gid`` will be used.
+   Integer values will be passed verbatim. (POSIX only)
+
+   .. availability:: POSIX
+   .. versionadded:: 3.9
+
+   If *user* is not ``None``, the setreuid() system call will be made in the
+   child process prior to the execution of the subprocess. If the provided
+   value is a string, it will be looked up via :func:`pwd.getpwnam()` and
+   the value in ``pw_uid`` will be used. If the value is an integer, it will
+   be passed verbatim. (POSIX only)
+
+   .. availability:: POSIX
+   .. versionadded:: 3.9
+
    If *env* is not ``None``, it must be a mapping that defines the environment
    variables for the new process; these are used instead of the default
    behavior of inheriting the current process' environment.
diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py
index 32b51b04373f..207a2f70b254 100644
--- a/Lib/multiprocessing/util.py
+++ b/Lib/multiprocessing/util.py
@@ -429,7 +429,7 @@ def spawnv_passfds(path, args, passfds):
         return _posixsubprocess.fork_exec(
             args, [os.fsencode(path)], True, passfds, None, None,
             -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write,
-            False, False, None)
+            False, False, None, None, None, None)
     finally:
         os.close(errpipe_read)
         os.close(errpipe_write)
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 85b9ea078546..85e7969c0928 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -53,6 +53,14 @@
 import contextlib
 from time import monotonic as _time
 
+try:
+    import pwd
+except ImportError:
+    pwd = None
+try:
+    import grp
+except ImportError:
+    grp = None
 
 __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
            "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL",
@@ -719,6 +727,12 @@ class Popen(object):
 
       start_new_session (POSIX only)
 
+      group (POSIX only)
+
+      extra_groups (POSIX only)
+
+      user (POSIX only)
+
       pass_fds (POSIX only)
 
       encoding and errors: Text mode encoding and error handling to use for
@@ -735,7 +749,8 @@ def __init__(self, args, bufsize=-1, executable=None,
                  shell=False, cwd=None, env=None, universal_newlines=None,
                  startupinfo=None, creationflags=0,
                  restore_signals=True, start_new_session=False,
-                 pass_fds=(), *, encoding=None, errors=None, text=None):
+                 pass_fds=(), *, user=None, group=None, extra_groups=None,
+                 encoding=None, errors=None, text=None):
         """Create new Popen instance."""
         _cleanup()
         # Held while anything is calling waitpid before returncode has been
@@ -833,6 +848,78 @@ def __init__(self, args, bufsize=-1, executable=None,
             else:
                 line_buffering = False
 
+        gid = None
+        if group is not None:
+            if not hasattr(os, 'setregid'):
+                raise ValueError("The 'group' parameter is not supported on the "
+                                 "current platform")
+
+            elif isinstance(group, str):
+                if grp is None:
+                    raise ValueError("The group parameter cannot be a string "
+                                     "on systems without the grp module")
+
+                gid = grp.getgrnam(group).gr_gid
+            elif isinstance(group, int):
+                gid = group
+            else:
+                raise TypeError("Group must be a string or an integer, not {}"
+                                .format(type(group)))
+
+            if gid < 0:
+                raise ValueError(f"Group ID cannot be negative, got {gid}")
+
+        gids = None
+        if extra_groups is not None:
+            if not hasattr(os, 'setgroups'):
+                raise ValueError("The 'extra_groups' parameter is not "
+                                 "supported on the current platform")
+
+            elif isinstance(extra_groups, str):
+                raise ValueError("Groups must be a list, not a string")
+
+            gids = []
+            for extra_group in extra_groups:
+                if isinstance(extra_group, str):
+                    if grp is None:
+                        raise ValueError("Items in extra_groups cannot be "
+                                         "strings on systems without the "
+                                         "grp module")
+
+                    gids.append(grp.getgrnam(extra_group).gr_gid)
+                elif isinstance(extra_group, int):
+                    gids.append(extra_group)
+                else:
+                    raise TypeError("Items in extra_groups must be a string "
+                                    "or integer, not {}"
+                                    .format(type(extra_group)))
+
+            # make sure that the gids are all positive here so we can do less
+            # checking in the C code
+            for gid_check in gids:
+                if gid_check < 0:
+                    raise ValueError(f"Group ID cannot be negative, got {gid_check}")
+
+        uid = None
+        if user is not None:
+            if not hasattr(os, 'setreuid'):
+                raise ValueError("The 'user' parameter is not supported on "
+                                 "the current platform")
+
+            elif isinstance(user, str):
+                if pwd is None:
+                    raise ValueError("The user parameter cannot be a string "
+                                     "on systems without the pwd module")
+
+                uid = pwd.getpwnam(user).pw_uid
+            elif isinstance(user, int):
+                uid = user
+            else:
+                raise TypeError("User must be a string or an integer")
+
+            if uid < 0:
+                raise ValueError(f"User ID cannot be negative, got {uid}")
+
         try:
             if p2cwrite != -1:
                 self.stdin = io.open(p2cwrite, 'wb', bufsize)
@@ -857,7 +944,9 @@ def __init__(self, args, bufsize=-1, executable=None,
                                 p2cread, p2cwrite,
                                 c2pread, c2pwrite,
                                 errread, errwrite,
-                                restore_signals, start_new_session)
+                                restore_signals,
+                                gid, gids, uid,
+                                start_new_session)
         except:
             # Cleanup if the child failed starting.
             for f in filter(None, (self.stdin, self.stdout, self.stderr)):
@@ -1227,7 +1316,9 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
                            p2cread, p2cwrite,
                            c2pread, c2pwrite,
                            errread, errwrite,
-                           unused_restore_signals, unused_start_new_session):
+                           unused_restore_signals,
+                           unused_gid, unused_gids, unused_uid,
+                           unused_start_new_session):
             """Execute program (MS Windows version)"""
 
             assert not pass_fds, "pass_fds not supported on Windows."
@@ -1553,7 +1644,9 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
                            p2cread, p2cwrite,
                            c2pread, c2pwrite,
                            errread, errwrite,
-                           restore_signals, start_new_session):
+                           restore_signals,
+                           gid, gids, uid,
+                           start_new_session):
             """Execute program (POSIX version)"""
 
             if isinstance(args, (str, bytes)):
@@ -1641,7 +1734,9 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
                             p2cread, p2cwrite, c2pread, c2pwrite,
                             errread, errwrite,
                             errpipe_read, errpipe_write,
-                            restore_signals, start_new_session, preexec_fn)
+                            restore_signals, start_new_session,
+                            gid, gids, uid,
+                            preexec_fn)
                     self._child_created = True
                 finally:
                     # be sure the FD is closed no matter what
diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 4d6e2f21551a..a38406481a0e 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -96,7 +96,7 @@ class Z(object):
             def __len__(self):
                 return 1
         self.assertRaises(TypeError, _posixsubprocess.fork_exec,
-                          1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17)
+                          1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20)
         # Issue #15736: overflow in _PySequence_BytesToCharpArray()
         class Z(object):
             def __len__(self):
@@ -104,7 +104,7 @@ def __len__(self):
             def __getitem__(self, i):
                 return b'x'
         self.assertRaises(MemoryError, _posixsubprocess.fork_exec,
-                          1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17)
+                          1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20)
 
     @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.')
     def test_subprocess_fork_exec(self):
@@ -114,7 +114,7 @@ def __len__(self):
 
         # Issue #15738: crash in subprocess_fork_exec()
         self.assertRaises(TypeError, _posixsubprocess.fork_exec,
-                          Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17)
+                          Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20)
 
     @unittest.skipIf(MISSING_C_DOCSTRINGS,
                      "Signature information for builtins requires docstrings")
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index 91f525df4607..f107022d86a0 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -18,6 +18,7 @@
 import threading
 import gc
 import textwrap
+import json
 from test.support import FakePath
 
 try:
@@ -25,6 +26,14 @@
 except ImportError:
     _testcapi = None
 
+try:
+    import pwd
+except ImportError:
+    pwd = None
+try:
+    import grp
+except ImportError:
+    grp = None
 
 if support.PGO:
     raise unittest.SkipTest("test is not helpful for PGO")
@@ -1733,6 +1742,139 @@ def test_start_new_session(self):
             child_sid = int(output)
             self.assertNotEqual(parent_sid, child_sid)
 
+    @unittest.skipUnless(hasattr(os, 'setreuid'), 'no setreuid on platform')
+    def test_user(self):
+        # For code coverage of the user parameter.  We don't care if we get an
+        # EPERM error from it depending on the test execution environment, that
+        # still indicates that it was called.
+
+        uid = os.geteuid()
+        test_users = [65534 if uid != 65534 else 65533, uid]
+        name_uid = "nobody" if sys.platform != 'darwin' else "unknown"
+
+        if pwd is not None:
+            test_users.append(name_uid)
+
+        for user in test_users:
+            with self.subTest(user=user):
+                try:
+                    output = subprocess.check_output(
+                            [sys.executable, "-c",
+                             "import os; print(os.getuid())"],
+                            user=user)
+                except OSError as e:
+                    if e.errno != errno.EPERM:
+                        raise
+                else:
+                    if isinstance(user, str):
+                        user_uid = pwd.getpwnam(user).pw_uid
+                    else:
+                        user_uid = user
+                    child_user = int(output)
+                    self.assertEqual(child_user, user_uid)
+
+        with self.assertRaises(ValueError):
+            subprocess.check_call([sys.executable, "-c", "pass"], user=-1)
+
+        if pwd is None:
+            with self.assertRaises(ValueError):
+                subprocess.check_call([sys.executable, "-c", "pass"], user=name_uid)
+
+    @unittest.skipIf(hasattr(os, 'setreuid'), 'setreuid() available on platform')
+    def test_user_error(self):
+        with self.assertRaises(ValueError):
+            subprocess.check_call([sys.executable, "-c", "pass"], user=65535)
+
+    @unittest.skipUnless(hasattr(os, 'setregid'), 'no setregid() on platform')
+    def test_group(self):
+        gid = os.getegid()
+        group_list = [65534 if gid != 65534 else 65533]
+        name_group = "nogroup" if sys.platform != 'darwin' else "staff"
+
+        if grp is not None:
+            group_list.append(name_group)
+
+        for group in group_list + [gid]:
+            with self.subTest(group=group):
+                try:
+                    output = subprocess.check_output(
+                            [sys.executable, "-c",
+                             "import os; print(os.getgid())"],
+                            group=group)
+                except OSError as e:
+                    if e.errno != errno.EPERM:
+                        raise
+                else:
+                    if isinstance(group, str):
+                        group_gid = grp.getgrnam(group).gr_gid
+                    else:
+                        group_gid = group
+
+                    child_group = int(output)
+                    self.assertEqual(child_group, group_gid)
+
+        # make sure we bomb on negative values
+        with self.assertRaises(ValueError):
+            subprocess.check_call([sys.executable, "-c", "pass"], group=-1)
+
+        if grp is None:
+            with self.assertRaises(ValueError):
+                subprocess.check_call([sys.executable, "-c", "pass"], group=name_group)
+
+    @unittest.skipIf(hasattr(os, 'setregid'), 'setregid() available on platform')
+    def test_group_error(self):
+        with self.assertRaises(ValueError):
+            subprocess.check_call([sys.executable, "-c", "pass"], group=65535)
+
+    @unittest.skipUnless(hasattr(os, 'setgroups'), 'no setgroups() on platform')
+    def test_extra_groups(self):
+        gid = os.getegid()
+        group_list = [65534 if gid != 65534 else 65533]
+        name_group = "nogroup" if sys.platform != 'darwin' else "staff"
+        perm_error = False
+
+        if grp is not None:
+            group_list.append(name_group)
+
+        try:
+            output = subprocess.check_output(
+                    [sys.executable, "-c",
+                     "import os, sys, json; json.dump(os.getgroups(), sys.stdout)"],
+                    extra_groups=group_list)
+        except OSError as ex:
+            if ex.errno != errno.EPERM:
+                raise
+            perm_error = True
+
+        else:
+            parent_groups = os.getgroups()
+            child_groups = json.loads(output)
+
+            if grp is not None:
+                desired_gids = [grp.getgrnam(g).gr_gid if isinstance(g, str) else g
+                                for g in group_list]
+            else:
+                desired_gids = group_list
+
+            if perm_error:
+                self.assertEqual(set(child_groups), set(parent_groups))
+            else:
+                self.assertEqual(set(desired_gids), set(child_groups))
+
+        # make sure we bomb on negative values
+        with self.assertRaises(ValueError):
+            subprocess.check_call([sys.executable, "-c", "pass"], extra_groups=[-1])
+
+        if grp is None:
+            with self.assertRaises(ValueError):
+                subprocess.check_call([sys.executable, "-c", "pass"],
+                                      extra_groups=[name_group])
+
+    @unittest.skipIf(hasattr(os, 'setgroups'), 'setgroups() available on platform')
+    def test_extra_groups_error(self):
+        with self.assertRaises(ValueError):
+            subprocess.check_call([sys.executable, "-c", "pass"], extra_groups=[])
+
     def test_run_abort(self):
         # returncode handles signal termination
         with support.SuppressCrashReport():
@@ -2782,13 +2924,23 @@ def test_fork_exec(self):
                 ([b"arg"], [b"exe"], 123,  [b"env"]),
                 ([b"arg"], [b"exe"], None, 123),
             ):
-                with self.assertRaises(TypeError):
+                with self.assertRaises(TypeError) as err:
                     _posixsubprocess.fork_exec(
                         args, exe_list,
                         True, (), cwd, env_list,
                         -1, -1, -1, -1,
                         1, 2, 3, 4,
-                        True, True, func)
+                        True, True,
+                        False, [], 0,
+                        func)
+                # Attempt to prevent
+                # "TypeError: fork_exec() takes exactly N arguments (M given)"
+                # from passing the test.  More refactoring to have us start
+                # with a valid *args list, confirm a good call with that works
+                # before mutating it in various ways to ensure that bad calls
+                # with individual arg type errors raise a typeerror would be
+                # ideal.  Saving that for a future PR...
+                self.assertNotIn('takes exactly', str(err.exception))
         finally:
             if not gc_enabled:
                 gc.disable()
@@ -2827,7 +2979,9 @@ def __int__(self):
                         True, fds_to_keep, None, [b"env"],
                         -1, -1, -1, -1,
                         1, 2, 3, 4,
-                        True, True, None)
+                        True, True,
+                        None, None, None,
+                        None)
                 self.assertIn('fds_to_keep', str(c.exception))
         finally:
             if not gc_enabled:
@@ -3239,7 +3393,7 @@ def test_getoutput(self):
 
     def test__all__(self):
         """Ensure that __all__ is populated properly."""
-        intentionally_excluded = {"list2cmdline", "Handle"}
+        intentionally_excluded = {"list2cmdline", "Handle", "pwd", "grp"}
         exported = set(subprocess.__all__)
         possible_exports = set()
         import types
diff --git a/Misc/NEWS.d/next/Library/2019-02-19-17-32-45.bpo-36046.fX9OPj.rst b/Misc/NEWS.d/next/Library/2019-02-19-17-32-45.bpo-36046.fX9OPj.rst
new file mode 100644
index 000000000000..5e809d6a6032
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-02-19-17-32-45.bpo-36046.fX9OPj.rst
@@ -0,0 +1,2 @@
+Added ``user``, ``group`` and ``extra_groups`` parameters to the
+subprocess.Popen constructor. Patch by Patrick McLean.
diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c
index cbdeecfda1c2..25af00f70dec 100644
--- a/Modules/_posixsubprocess.c
+++ b/Modules/_posixsubprocess.c
@@ -20,6 +20,11 @@
 #ifdef HAVE_DIRENT_H
 #include <dirent.h>
 #endif
+#ifdef HAVE_GRP_H
+#include <grp.h>
+#endif /* HAVE_GRP_H */
+
+#include "posixmodule.h"
 
 #ifdef _Py_MEMORY_SANITIZER
 # include <sanitizer/msan_interface.h>
@@ -47,6 +52,12 @@
 # define FD_DIR "/proc/self/fd"
 #endif
 
+#ifdef NGROUPS_MAX
+#define MAX_GROUPS NGROUPS_MAX
+#else
+#define MAX_GROUPS 64
+#endif
+
 #define POSIX_CALL(call)   do { if ((call) == -1) goto error; } while (0)
 
 typedef struct {
@@ -415,6 +426,9 @@ child_exec(char *const exec_array[],
            int errpipe_read, int errpipe_write,
            int close_fds, int restore_signals,
            int call_setsid,
+           int call_setgid, gid_t gid,
+           int call_setgroups, size_t groups_size, const gid_t *groups,
+           int call_setuid, uid_t uid,
            PyObject *py_fds_to_keep,
            PyObject *preexec_fn,
            PyObject *preexec_fn_args_tuple)
@@ -492,6 +506,22 @@ child_exec(char *const exec_array[],
         POSIX_CALL(setsid());
 #endif
 
+#ifdef HAVE_SETGROUPS
+    if (call_setgroups)
+        POSIX_CALL(setgroups(groups_size, groups));
+#endif /* HAVE_SETGROUPS */
+
+#ifdef HAVE_SETREGID
+    if (call_setgid)
+        POSIX_CALL(setregid(gid, gid));
+#endif /* HAVE_SETREGID */
+
+#ifdef HAVE_SETREUID
+    if (call_setuid)
+        POSIX_CALL(setreuid(uid, uid));
+#endif /* HAVE_SETREUID */
+
+
     reached_preexec = 1;
     if (preexec_fn != Py_None && preexec_fn_args_tuple) {
         /* This is where the user has asked us to deadlock their program. */
@@ -571,26 +601,33 @@ subprocess_fork_exec(PyObject* self, PyObject *args)
     PyObject *env_list, *preexec_fn;
     PyObject *process_args, *converted_args = NULL, *fast_args = NULL;
     PyObject *preexec_fn_args_tuple = NULL;
+    PyObject *groups_list;
+    PyObject *uid_object, *gid_object;
     int p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite;
     int errpipe_read, errpipe_write, close_fds, restore_signals;
     int call_setsid;
+    int call_setgid = 0, call_setgroups = 0, call_setuid = 0;
+    uid_t uid;
+    gid_t gid, *groups = NULL;
     PyObject *cwd_obj, *cwd_obj2;
     const char *cwd;
     pid_t pid;
     int need_to_reenable_gc = 0;
     char *const *exec_array, *const *argv = NULL, *const *envp = NULL;
-    Py_ssize_t arg_num;
+    Py_ssize_t arg_num, num_groups = 0;
     int need_after_fork = 0;
     int saved_errno = 0;
 
     if (!PyArg_ParseTuple(
-            args, "OOpO!OOiiiiiiiiiiO:fork_exec",
+            args, "OOpO!OOiiiiiiiiiiOOOO:fork_exec",
             &process_args, &executable_list,
             &close_fds, &PyTuple_Type, &py_fds_to_keep,
             &cwd_obj, &env_list,
             &p2cread, &p2cwrite, &c2pread, &c2pwrite,
             &errread, &errwrite, &errpipe_read, &errpipe_write,
-            &restore_signals, &call_setsid, &preexec_fn))
+            &restore_signals, &call_setsid,
+            &gid_object, &groups_list, &uid_object,
+            &preexec_fn))
         return NULL;
 
     if ((preexec_fn != Py_None) &&
@@ -689,6 +726,90 @@ subprocess_fork_exec(PyObject* self, PyObject *args)
         cwd_obj2 = NULL;
     }
 
+    if (groups_list != Py_None) {
+#ifdef HAVE_SETGROUPS
+        Py_ssize_t i;
+        unsigned long gid;
+
+        if (!PyList_Check(groups_list)) {
+            PyErr_SetString(PyExc_TypeError,
+                    "setgroups argument must be a list");
+            goto cleanup;
+        }
+        num_groups = PySequence_Size(groups_list);
+
+        if (num_groups < 0)
+            goto cleanup;
+
+        if (num_groups > MAX_GROUPS) {
+            PyErr_SetString(PyExc_ValueError, "too many groups");
+            goto cleanup;
+        }
+
+        if ((groups = PyMem_RawMalloc(num_groups * sizeof(gid_t))) == NULL) {
+            PyErr_SetString(PyExc_MemoryError,
+                    "failed to allocate memory for group list");
+            goto cleanup;
+        }
+
+        for (i = 0; i < num_groups; i++) {
+            PyObject *elem;
+            elem = PySequence_GetItem(groups_list, i);
+            if (!elem)
+                goto cleanup;
+            if (!PyLong_Check(elem)) {
+                PyErr_SetString(PyExc_TypeError,
+                                "groups must be integers");
+                Py_DECREF(elem);
+                goto cleanup;
+            } else {
+                /* In posixmodule.c UnsignedLong is used as a fallback value
+                * if the value provided does not fit in a Long. Since we are
+                * already doing the bounds checking on the Python side, we
+                * can go directly to an UnsignedLong here. */
+                if (!_Py_Gid_Converter(elem, &gid)) {
+                    Py_DECREF(elem);
+                    PyErr_SetString(PyExc_ValueError, "invalid group id");
+                    goto cleanup;
+                }
+                groups[i] = gid;
+            }
+            Py_DECREF(elem);
+        }
+        call_setgroups = 1;
+
+#else /* HAVE_SETGROUPS */
+        PyErr_BadInternalCall();
+        goto cleanup;
+#endif /* HAVE_SETGROUPS */
+    }
+
+    if (gid_object != Py_None) {
+#ifdef HAVE_SETREGID
+        if (!_Py_Gid_Converter(gid_object, &gid))
+            goto cleanup;
+
+        call_setgid = 1;
+
+#else /* HAVE_SETREGID */
+        PyErr_BadInternalCall();
+        goto cleanup;
+#endif /* HAVE_SETREUID */
+    }
+
+    if (uid_object != Py_None) {
+#ifdef HAVE_SETREUID
+        if (!_Py_Uid_Converter(uid_object, &uid))
+            goto cleanup;
+
+        call_setuid = 1;
+
+#else /* HAVE_SETREUID */
+        PyErr_BadInternalCall();
+        goto cleanup;
+#endif /* HAVE_SETREUID */
+    }
+
     /* This must be the last thing done before fork() because we do not
      * want to call PyOS_BeforeFork() if there is any chance of another
      * error leading to the cleanup: code without calling fork(). */
@@ -721,6 +842,8 @@ subprocess_fork_exec(PyObject* self, PyObject *args)
                    p2cread, p2cwrite, c2pread, c2pwrite,
                    errread, errwrite, errpipe_read, errpipe_write,
                    close_fds, restore_signals, call_setsid,
+                   call_setgid, gid, call_setgroups, num_groups, groups,
+                   call_setuid, uid,
                    py_fds_to_keep, preexec_fn, preexec_fn_args_tuple);
         _exit(255);
         return NULL;  /* Dead code to avoid a potential compiler warning. */
@@ -765,6 +888,8 @@ subprocess_fork_exec(PyObject* self, PyObject *args)
         _Py_FreeCharPArray(argv);
     if (exec_array)
         _Py_FreeCharPArray(exec_array);
+
+    PyMem_RawFree(groups);
     Py_XDECREF(converted_args);
     Py_XDECREF(fast_args);
     Py_XDECREF(preexec_fn_args_tuple);
@@ -778,7 +903,10 @@ PyDoc_STRVAR(subprocess_fork_exec_doc,
 "fork_exec(args, executable_list, close_fds, cwd, env,\n\
           p2cread, p2cwrite, c2pread, c2pwrite,\n\
           errread, errwrite, errpipe_read, errpipe_write,\n\
-          restore_signals, call_setsid, preexec_fn)\n\
+          restore_signals, call_setsid,\n\
+          call_setgid, gid, groups_size, gids,\n\
+          call_setuid, uid,\n\
+          preexec_fn)\n\
 \n\
 Forks a child process, closes parent file descriptors as appropriate in the\n\
 child and dups the few that are needed before calling exec() in the child\n\


From webhook-mailer at python.org  Thu Sep 12 13:16:54 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Thu, 12 Sep 2019 17:16:54 -0000
Subject: [Python-checkins] bpo-38133: Allow py.exe launcher to locate
 installations from the Microsoft Store (GH-16025)
Message-ID: <mailman.690.1568308615.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ed93a8852d120c5a3606720edc723bf5aa6a1fc2
commit: ed93a8852d120c5a3606720edc723bf5aa6a1fc2
branch: master
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-12T18:16:50+01:00
summary:

bpo-38133: Allow py.exe launcher to locate installations from the Microsoft Store (GH-16025)

files:
A Misc/NEWS.d/next/Windows/2019-09-12-12-05-55.bpo-38133.yFeRGS.rst
M PC/launcher.c

diff --git a/Misc/NEWS.d/next/Windows/2019-09-12-12-05-55.bpo-38133.yFeRGS.rst b/Misc/NEWS.d/next/Windows/2019-09-12-12-05-55.bpo-38133.yFeRGS.rst
new file mode 100644
index 000000000000..3fbf01693d55
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-12-12-05-55.bpo-38133.yFeRGS.rst
@@ -0,0 +1,2 @@
+Allow py.exe launcher to locate installations from the Microsoft Store and
+improve display of active virtual environments.
diff --git a/PC/launcher.c b/PC/launcher.c
index d23637c54ca1..a260860f3c27 100644
--- a/PC/launcher.c
+++ b/PC/launcher.c
@@ -162,12 +162,13 @@ static wchar_t * get_env(wchar_t * key)
 #endif
 #endif
 
-#define MAX_VERSION_SIZE    4
+#define MAX_VERSION_SIZE    8
 
 typedef struct {
     wchar_t version[MAX_VERSION_SIZE]; /* m.n */
     int bits;   /* 32 or 64 */
     wchar_t executable[MAX_PATH];
+    wchar_t exe_display[MAX_PATH];
 } INSTALLED_PYTHON;
 
 /*
@@ -185,10 +186,18 @@ static size_t num_installed_pythons = 0;
  * The version name can be longer than MAX_VERSION_SIZE, but will be
  * truncated to just X.Y for comparisons.
  */
-#define IP_BASE_SIZE 40
+#define IP_BASE_SIZE 80
 #define IP_VERSION_SIZE 8
 #define IP_SIZE (IP_BASE_SIZE + IP_VERSION_SIZE)
 #define CORE_PATH L"SOFTWARE\\Python\\PythonCore"
+/*
+ * Installations from the Microsoft Store will set the same registry keys,
+ * but because of a limitation in Windows they cannot be enumerated normally
+ * (unless you have no other Python installations... which is probably false
+ * because that's the most likely way to get this launcher!)
+ * This key is under HKEY_LOCAL_MACHINE
+ */
+#define LOOKASIDE_PATH L"SOFTWARE\\Microsoft\\AppModel\\Lookaside\\user\\Software\\Python\\PythonCore"
 
 static wchar_t * location_checks[] = {
     L"\\",
@@ -201,7 +210,7 @@ static wchar_t * location_checks[] = {
 };
 
 static INSTALLED_PYTHON *
-find_existing_python(wchar_t * path)
+find_existing_python(const wchar_t * path)
 {
     INSTALLED_PYTHON * result = NULL;
     size_t i;
@@ -216,15 +225,32 @@ find_existing_python(wchar_t * path)
     return result;
 }
 
+static INSTALLED_PYTHON *
+find_existing_python2(int bits, const wchar_t * version)
+{
+    INSTALLED_PYTHON * result = NULL;
+    size_t i;
+    INSTALLED_PYTHON * ip;
+
+    for (i = 0, ip = installed_pythons; i < num_installed_pythons; i++, ip++) {
+        if (bits == ip->bits && _wcsicmp(version, ip->version) == 0) {
+            result = ip;
+            break;
+        }
+    }
+    return result;
+}
+
 static void
-locate_pythons_for_key(HKEY root, REGSAM flags)
+_locate_pythons_for_key(HKEY root, LPCWSTR subkey, REGSAM flags, int bits,
+                        int display_name_only)
 {
     HKEY core_root, ip_key;
-    LSTATUS status = RegOpenKeyExW(root, CORE_PATH, 0, flags, &core_root);
+    LSTATUS status = RegOpenKeyExW(root, subkey, 0, flags, &core_root);
     wchar_t message[MSGSIZE];
     DWORD i;
     size_t n;
-    BOOL ok;
+    BOOL ok, append_name;
     DWORD type, data_size, attrs;
     INSTALLED_PYTHON * ip, * pip;
     wchar_t ip_version[IP_VERSION_SIZE];
@@ -252,8 +278,16 @@ locate_pythons_for_key(HKEY root, REGSAM flags)
             else {
                 wcsncpy_s(ip->version, MAX_VERSION_SIZE, ip_version,
                           MAX_VERSION_SIZE-1);
+                /* Still treating version as "x.y" rather than sys.winver
+                 * When PEP 514 tags are properly used, we shouldn't need
+                 * to strip this off here.
+                 */
+                check = wcsrchr(ip->version, L'-');
+                if (check && !wcscmp(check, L"-32")) {
+                    *check = L'\0';
+                }
                 _snwprintf_s(ip_path, IP_SIZE, _TRUNCATE,
-                             L"%ls\\%ls\\InstallPath", CORE_PATH, ip_version);
+                             L"%ls\\%ls\\InstallPath", subkey, ip_version);
                 status = RegOpenKeyExW(root, ip_path, 0, flags, &ip_key);
                 if (status != ERROR_SUCCESS) {
                     winerror(status, message, MSGSIZE);
@@ -262,42 +296,57 @@ locate_pythons_for_key(HKEY root, REGSAM flags)
                     continue;
                 }
                 data_size = sizeof(ip->executable) - 1;
-                status = RegQueryValueExW(ip_key, NULL, NULL, &type,
+                append_name = FALSE;
+                status = RegQueryValueExW(ip_key, L"ExecutablePath", NULL, &type,
                                           (LPBYTE)ip->executable, &data_size);
+                if (status != ERROR_SUCCESS || type != REG_SZ || !data_size) {
+                    append_name = TRUE;
+                    data_size = sizeof(ip->executable) - 1;
+                    status = RegQueryValueExW(ip_key, NULL, NULL, &type,
+                                              (LPBYTE)ip->executable, &data_size);
+                    if (status != ERROR_SUCCESS) {
+                        winerror(status, message, MSGSIZE);
+                        debug(L"%ls\\%ls: %ls\n", key_name, ip_path, message);
+                        RegCloseKey(ip_key);
+                        continue;
+                    }
+                }
                 RegCloseKey(ip_key);
-                if (status != ERROR_SUCCESS) {
-                    winerror(status, message, MSGSIZE);
-                    debug(L"%ls\\%ls: %ls\n", key_name, ip_path, message);
+                if (type != REG_SZ) {
                     continue;
                 }
-                if (type == REG_SZ) {
-                    data_size = data_size / sizeof(wchar_t) - 1;  /* for NUL */
-                    if (ip->executable[data_size - 1] == L'\\')
-                        --data_size; /* reg value ended in a backslash */
-                    /* ip->executable is data_size long */
-                    for (checkp = location_checks; *checkp; ++checkp) {
-                        check = *checkp;
+
+                data_size = data_size / sizeof(wchar_t) - 1;  /* for NUL */
+                if (ip->executable[data_size - 1] == L'\\')
+                    --data_size; /* reg value ended in a backslash */
+                /* ip->executable is data_size long */
+                for (checkp = location_checks; *checkp; ++checkp) {
+                    check = *checkp;
+                    if (append_name) {
                         _snwprintf_s(&ip->executable[data_size],
                                      MAX_PATH - data_size,
                                      MAX_PATH - data_size,
                                      L"%ls%ls", check, PYTHON_EXECUTABLE);
-                        attrs = GetFileAttributesW(ip->executable);
-                        if (attrs == INVALID_FILE_ATTRIBUTES) {
-                            winerror(GetLastError(), message, MSGSIZE);
-                            debug(L"locate_pythons_for_key: %ls: %ls",
-                                  ip->executable, message);
-                        }
-                        else if (attrs & FILE_ATTRIBUTE_DIRECTORY) {
-                            debug(L"locate_pythons_for_key: '%ls' is a \
-directory\n",
-                                  ip->executable, attrs);
-                        }
-                        else if (find_existing_python(ip->executable)) {
-                            debug(L"locate_pythons_for_key: %ls: already \
-found\n", ip->executable);
-                        }
-                        else {
-                            /* check the executable type. */
+                    }
+                    attrs = GetFileAttributesW(ip->executable);
+                    if (attrs == INVALID_FILE_ATTRIBUTES) {
+                        winerror(GetLastError(), message, MSGSIZE);
+                        debug(L"locate_pythons_for_key: %ls: %ls",
+                              ip->executable, message);
+                    }
+                    else if (attrs & FILE_ATTRIBUTE_DIRECTORY) {
+                        debug(L"locate_pythons_for_key: '%ls' is a directory\n",
+                              ip->executable, attrs);
+                    }
+                    else if (find_existing_python(ip->executable)) {
+                        debug(L"locate_pythons_for_key: %ls: already found\n",
+                              ip->executable);
+                    }
+                    else {
+                        /* check the executable type. */
+                        if (bits) {
+                            ip->bits = bits;
+                        } else {
                             ok = GetBinaryTypeW(ip->executable, &attrs);
                             if (!ok) {
                                 debug(L"Failure getting binary type: %ls\n",
@@ -310,32 +359,48 @@ found\n", ip->executable);
                                     ip->bits = 32;
                                 else
                                     ip->bits = 0;
-                                if (ip->bits == 0) {
-                                    debug(L"locate_pythons_for_key: %ls: \
+                            }
+                        }
+                        if (ip->bits == 0) {
+                            debug(L"locate_pythons_for_key: %ls: \
 invalid binary type: %X\n",
-                                          ip->executable, attrs);
+                                  ip->executable, attrs);
+                        }
+                        else {
+                            if (display_name_only) {
+                                /* display just the executable name. This is
+                                 * primarily for the Store installs */
+                                const wchar_t *name = wcsrchr(ip->executable, L'\\');
+                                if (name) {
+                                    wcscpy_s(ip->exe_display, MAX_PATH, name+1);
                                 }
-                                else {
-                                    if (wcschr(ip->executable, L' ') != NULL) {
-                                        /* has spaces, so quote */
-                                        n = wcslen(ip->executable);
-                                        memmove(&ip->executable[1],
-                                                ip->executable, n * sizeof(wchar_t));
-                                        ip->executable[0] = L'\"';
-                                        ip->executable[n + 1] = L'\"';
-                                        ip->executable[n + 2] = L'\0';
-                                    }
-                                    debug(L"locate_pythons_for_key: %ls \
-is a %dbit executable\n",
-                                        ip->executable, ip->bits);
-                                    ++num_installed_pythons;
-                                    pip = ip++;
-                                    if (num_installed_pythons >=
-                                        MAX_INSTALLED_PYTHONS)
-                                        break;
-                                    /* Copy over the attributes for the next */
-                                    *ip = *pip;
+                            }
+                            if (wcschr(ip->executable, L' ') != NULL) {
+                                /* has spaces, so quote, and set original as
+                                 * the display name */
+                                if (!ip->exe_display[0]) {
+                                    wcscpy_s(ip->exe_display, MAX_PATH, ip->executable);
                                 }
+                                n = wcslen(ip->executable);
+                                memmove(&ip->executable[1],
+                                        ip->executable, n * sizeof(wchar_t));
+                                ip->executable[0] = L'\"';
+                                ip->executable[n + 1] = L'\"';
+                                ip->executable[n + 2] = L'\0';
+                            }
+                            debug(L"locate_pythons_for_key: %ls \
+is a %dbit executable\n",
+                                ip->executable, ip->bits);
+                            if (find_existing_python2(ip->bits, ip->version)) {
+                                debug(L"locate_pythons_for_key: %ls-%i: already \
+found\n", ip->version, ip->bits);
+                            }
+                            else {
+                                ++num_installed_pythons;
+                                pip = ip++;
+                                if (num_installed_pythons >=
+                                    MAX_INSTALLED_PYTHONS)
+                                    break;
                             }
                         }
                     }
@@ -359,9 +424,63 @@ compare_pythons(const void * p1, const void * p2)
     return result;
 }
 
+static void
+locate_pythons_for_key(HKEY root, REGSAM flags)
+{
+    _locate_pythons_for_key(root, CORE_PATH, flags, 0, FALSE);
+}
+
+static void
+locate_store_pythons()
+{
+#if defined(_M_X64)
+    /* 64bit process, so look in native registry */
+    _locate_pythons_for_key(HKEY_LOCAL_MACHINE, LOOKASIDE_PATH,
+                            KEY_READ, 64, TRUE);
+#else
+    /* 32bit process, so check that we're on 64bit OS */
+    BOOL f64 = FALSE;
+    if (IsWow64Process(GetCurrentProcess(), &f64) && f64) {
+        _locate_pythons_for_key(HKEY_LOCAL_MACHINE, LOOKASIDE_PATH,
+                                KEY_READ | KEY_WOW64_64KEY, 64, TRUE);
+    }
+#endif
+}
+
+static void
+locate_venv_python()
+{
+    static wchar_t venv_python[MAX_PATH];
+    INSTALLED_PYTHON * ip;
+    wchar_t *virtual_env = get_env(L"VIRTUAL_ENV");
+    DWORD attrs;
+
+    /* Check for VIRTUAL_ENV environment variable */
+    if (virtual_env == NULL || virtual_env[0] == L'\0') {
+        return;
+    }
+
+    /* Check for a python executable in the venv */
+    debug(L"Checking for Python executable in virtual env '%ls'\n", virtual_env);
+    _snwprintf_s(venv_python, MAX_PATH, _TRUNCATE,
+            L"%ls\\Scripts\\%ls", virtual_env, PYTHON_EXECUTABLE);
+    attrs = GetFileAttributesW(venv_python);
+    if (attrs == INVALID_FILE_ATTRIBUTES) {
+        debug(L"Python executable %ls missing from virtual env\n", venv_python);
+        return;
+    }
+
+    ip = &installed_pythons[num_installed_pythons++];
+    wcscpy_s(ip->executable, MAX_PATH, venv_python);
+    ip->bits = 0;
+    wcscpy_s(ip->version, MAX_VERSION_SIZE, L"venv");
+}
+
 static void
 locate_all_pythons()
 {
+    /* venv Python is highest priority */
+    locate_venv_python();
 #if defined(_M_X64)
     /* If we are a 64bit process, first hit the 32bit keys. */
     debug(L"locating Pythons in 32bit registry\n");
@@ -380,6 +499,8 @@ locate_all_pythons()
     debug(L"locating Pythons in native registry\n");
     locate_pythons_for_key(HKEY_CURRENT_USER, KEY_READ);
     locate_pythons_for_key(HKEY_LOCAL_MACHINE, KEY_READ);
+    /* Store-installed Python is lowest priority */
+    locate_store_pythons();
     qsort(installed_pythons, num_installed_pythons, sizeof(INSTALLED_PYTHON),
           compare_pythons);
 }
@@ -416,31 +537,6 @@ find_python_by_version(wchar_t const * wanted_ver)
 }
 
 
-static wchar_t *
-find_python_by_venv()
-{
-    static wchar_t venv_python[MAX_PATH];
-    wchar_t *virtual_env = get_env(L"VIRTUAL_ENV");
-    DWORD attrs;
-
-    /* Check for VIRTUAL_ENV environment variable */
-    if (virtual_env == NULL || virtual_env[0] == L'\0') {
-        return NULL;
-    }
-
-    /* Check for a python executable in the venv */
-    debug(L"Checking for Python executable in virtual env '%ls'\n", virtual_env);
-    _snwprintf_s(venv_python, MAX_PATH, _TRUNCATE,
-            L"%ls\\Scripts\\%ls", virtual_env, PYTHON_EXECUTABLE);
-    attrs = GetFileAttributesW(venv_python);
-    if (attrs == INVALID_FILE_ATTRIBUTES) {
-        debug(L"Python executable %ls missing from virtual env\n", venv_python);
-        return NULL;
-    }
-
-    return venv_python;
-}
-
 static wchar_t appdata_ini_path[MAX_PATH];
 static wchar_t launcher_ini_path[MAX_PATH];
 
@@ -523,9 +619,12 @@ locate_python(wchar_t * wanted_ver, BOOL from_shebang)
     }
     else {
         *last_char = L'\0'; /* look for an overall default */
-        configured_value = get_configured_value(config_key);
-        if (configured_value)
-            result = find_python_by_version(configured_value);
+        result = find_python_by_version(L"venv");
+        if (result == NULL) {
+            configured_value = get_configured_value(config_key);
+            if (configured_value)
+                result = find_python_by_version(configured_value);
+        }
         /* Not found a value yet - try by major version.
          * If we're looking for an interpreter specified in a shebang line,
          * we want to try Python 2 first, then Python 3 (for Unix and backward
@@ -1443,7 +1542,8 @@ show_python_list(wchar_t ** argv)
     INSTALLED_PYTHON * defpy = locate_python(L"", FALSE);
     size_t i = 0;
     wchar_t *p = argv[1];
-    wchar_t *fmt = L"\n -%ls-%d"; /* print VER-BITS */
+    wchar_t *ver_fmt = L"-%ls-%d";
+    wchar_t *fmt = L"\n %ls";
     wchar_t *defind = L" *"; /* Default indicator */
 
     /*
@@ -1452,8 +1552,8 @@ show_python_list(wchar_t ** argv)
     */
     fwprintf(stderr,
              L"Installed Pythons found by %s Launcher for Windows", argv[0]);
-    if (!_wcsicmp(p, L"-0p") || !_wcsicmp(p, L"--list-paths")) /* Show path? */
-        fmt = L"\n -%ls-%d\t%ls"; /* print VER-BITS path */
+    if (!_wcsicmp(p, L"-0p") || !_wcsicmp(p, L"--list-paths"))
+        fmt = L"\n %-15ls%ls"; /* include path */
 
     if (num_installed_pythons == 0) /* We have somehow got here without searching for pythons */
         locate_all_pythons(); /* Find them, Populates installed_pythons */
@@ -1463,9 +1563,22 @@ show_python_list(wchar_t ** argv)
     else
     {
         for (i = 0; i < num_installed_pythons; i++, ip++) {
-            fwprintf(stdout, fmt, ip->version, ip->bits, ip->executable);
+            wchar_t version[BUFSIZ];
+            if (wcscmp(ip->version, L"venv") == 0) {
+                wcscpy_s(version, BUFSIZ, L"(venv)");
+            }
+            else {
+                swprintf_s(version, BUFSIZ, ver_fmt, ip->version, ip->bits);
+            }
+
+            if (ip->exe_display[0]) {
+                fwprintf(stdout, fmt, version, ip->exe_display);
+            }
+            else {
+                fwprintf(stdout, fmt, version, ip->executable);
+            }
             /* If there is a default indicate it */
-            if ((defpy != NULL) && !_wcsicmp(ip->executable, defpy->executable))
+            if (defpy == ip)
                 fwprintf(stderr, defind);
         }
     }
@@ -1850,16 +1963,11 @@ installed, use -0 for available pythons", &p[1]);
             executable = NULL; /* Info call only */
         }
         else {
-            /* Look for an active virtualenv */
-            executable = find_python_by_venv();
-
-            /* If we didn't find one, look for the default Python */
-            if (executable == NULL) {
-                ip = locate_python(L"", FALSE);
-                if (ip == NULL)
-                    error(RC_NO_PYTHON, L"Can't find a default Python.");
-                executable = ip->executable;
-            }
+            /* look for the default Python */
+            ip = locate_python(L"", FALSE);
+            if (ip == NULL)
+                error(RC_NO_PYTHON, L"Can't find a default Python.");
+            executable = ip->executable;
         }
     }
     if (executable != NULL)


From webhook-mailer at python.org  Thu Sep 12 13:19:50 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 12 Sep 2019 17:19:50 -0000
Subject: [Python-checkins] Move Eric's TODO file into his directory.
 (GH-16072)
Message-ID: <mailman.691.1568308791.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a44f3dc21d8b59ceeffa7dd1d9a7c05ed6be3e04
commit: a44f3dc21d8b59ceeffa7dd1d9a7c05ed6be3e04
branch: master
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-12T18:19:46+01:00
summary:

Move Eric's TODO file into his directory. (GH-16072)

files:
A Tools/c-analyzer/TODO
D TODO

diff --git a/TODO b/Tools/c-analyzer/TODO
similarity index 100%
rename from TODO
rename to Tools/c-analyzer/TODO


From webhook-mailer at python.org  Thu Sep 12 13:36:18 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 12 Sep 2019 17:36:18 -0000
Subject: [Python-checkins] bpo-38133: Allow py.exe launcher to locate
 installations from the Microsoft Store (GH-16025)
Message-ID: <mailman.692.1568309782.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/664d56a52eb5b98ac24b2d0cd4a080ffd49fd93d
commit: 664d56a52eb5b98ac24b2d0cd4a080ffd49fd93d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-12T10:36:14-07:00
summary:

bpo-38133: Allow py.exe launcher to locate installations from the Microsoft Store (GH-16025)

(cherry picked from commit ed93a8852d120c5a3606720edc723bf5aa6a1fc2)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-12-12-05-55.bpo-38133.yFeRGS.rst
M PC/launcher.c

diff --git a/Misc/NEWS.d/next/Windows/2019-09-12-12-05-55.bpo-38133.yFeRGS.rst b/Misc/NEWS.d/next/Windows/2019-09-12-12-05-55.bpo-38133.yFeRGS.rst
new file mode 100644
index 000000000000..3fbf01693d55
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-12-12-05-55.bpo-38133.yFeRGS.rst
@@ -0,0 +1,2 @@
+Allow py.exe launcher to locate installations from the Microsoft Store and
+improve display of active virtual environments.
diff --git a/PC/launcher.c b/PC/launcher.c
index d23637c54ca1..a260860f3c27 100644
--- a/PC/launcher.c
+++ b/PC/launcher.c
@@ -162,12 +162,13 @@ static wchar_t * get_env(wchar_t * key)
 #endif
 #endif
 
-#define MAX_VERSION_SIZE    4
+#define MAX_VERSION_SIZE    8
 
 typedef struct {
     wchar_t version[MAX_VERSION_SIZE]; /* m.n */
     int bits;   /* 32 or 64 */
     wchar_t executable[MAX_PATH];
+    wchar_t exe_display[MAX_PATH];
 } INSTALLED_PYTHON;
 
 /*
@@ -185,10 +186,18 @@ static size_t num_installed_pythons = 0;
  * The version name can be longer than MAX_VERSION_SIZE, but will be
  * truncated to just X.Y for comparisons.
  */
-#define IP_BASE_SIZE 40
+#define IP_BASE_SIZE 80
 #define IP_VERSION_SIZE 8
 #define IP_SIZE (IP_BASE_SIZE + IP_VERSION_SIZE)
 #define CORE_PATH L"SOFTWARE\\Python\\PythonCore"
+/*
+ * Installations from the Microsoft Store will set the same registry keys,
+ * but because of a limitation in Windows they cannot be enumerated normally
+ * (unless you have no other Python installations... which is probably false
+ * because that's the most likely way to get this launcher!)
+ * This key is under HKEY_LOCAL_MACHINE
+ */
+#define LOOKASIDE_PATH L"SOFTWARE\\Microsoft\\AppModel\\Lookaside\\user\\Software\\Python\\PythonCore"
 
 static wchar_t * location_checks[] = {
     L"\\",
@@ -201,7 +210,7 @@ static wchar_t * location_checks[] = {
 };
 
 static INSTALLED_PYTHON *
-find_existing_python(wchar_t * path)
+find_existing_python(const wchar_t * path)
 {
     INSTALLED_PYTHON * result = NULL;
     size_t i;
@@ -216,15 +225,32 @@ find_existing_python(wchar_t * path)
     return result;
 }
 
+static INSTALLED_PYTHON *
+find_existing_python2(int bits, const wchar_t * version)
+{
+    INSTALLED_PYTHON * result = NULL;
+    size_t i;
+    INSTALLED_PYTHON * ip;
+
+    for (i = 0, ip = installed_pythons; i < num_installed_pythons; i++, ip++) {
+        if (bits == ip->bits && _wcsicmp(version, ip->version) == 0) {
+            result = ip;
+            break;
+        }
+    }
+    return result;
+}
+
 static void
-locate_pythons_for_key(HKEY root, REGSAM flags)
+_locate_pythons_for_key(HKEY root, LPCWSTR subkey, REGSAM flags, int bits,
+                        int display_name_only)
 {
     HKEY core_root, ip_key;
-    LSTATUS status = RegOpenKeyExW(root, CORE_PATH, 0, flags, &core_root);
+    LSTATUS status = RegOpenKeyExW(root, subkey, 0, flags, &core_root);
     wchar_t message[MSGSIZE];
     DWORD i;
     size_t n;
-    BOOL ok;
+    BOOL ok, append_name;
     DWORD type, data_size, attrs;
     INSTALLED_PYTHON * ip, * pip;
     wchar_t ip_version[IP_VERSION_SIZE];
@@ -252,8 +278,16 @@ locate_pythons_for_key(HKEY root, REGSAM flags)
             else {
                 wcsncpy_s(ip->version, MAX_VERSION_SIZE, ip_version,
                           MAX_VERSION_SIZE-1);
+                /* Still treating version as "x.y" rather than sys.winver
+                 * When PEP 514 tags are properly used, we shouldn't need
+                 * to strip this off here.
+                 */
+                check = wcsrchr(ip->version, L'-');
+                if (check && !wcscmp(check, L"-32")) {
+                    *check = L'\0';
+                }
                 _snwprintf_s(ip_path, IP_SIZE, _TRUNCATE,
-                             L"%ls\\%ls\\InstallPath", CORE_PATH, ip_version);
+                             L"%ls\\%ls\\InstallPath", subkey, ip_version);
                 status = RegOpenKeyExW(root, ip_path, 0, flags, &ip_key);
                 if (status != ERROR_SUCCESS) {
                     winerror(status, message, MSGSIZE);
@@ -262,42 +296,57 @@ locate_pythons_for_key(HKEY root, REGSAM flags)
                     continue;
                 }
                 data_size = sizeof(ip->executable) - 1;
-                status = RegQueryValueExW(ip_key, NULL, NULL, &type,
+                append_name = FALSE;
+                status = RegQueryValueExW(ip_key, L"ExecutablePath", NULL, &type,
                                           (LPBYTE)ip->executable, &data_size);
+                if (status != ERROR_SUCCESS || type != REG_SZ || !data_size) {
+                    append_name = TRUE;
+                    data_size = sizeof(ip->executable) - 1;
+                    status = RegQueryValueExW(ip_key, NULL, NULL, &type,
+                                              (LPBYTE)ip->executable, &data_size);
+                    if (status != ERROR_SUCCESS) {
+                        winerror(status, message, MSGSIZE);
+                        debug(L"%ls\\%ls: %ls\n", key_name, ip_path, message);
+                        RegCloseKey(ip_key);
+                        continue;
+                    }
+                }
                 RegCloseKey(ip_key);
-                if (status != ERROR_SUCCESS) {
-                    winerror(status, message, MSGSIZE);
-                    debug(L"%ls\\%ls: %ls\n", key_name, ip_path, message);
+                if (type != REG_SZ) {
                     continue;
                 }
-                if (type == REG_SZ) {
-                    data_size = data_size / sizeof(wchar_t) - 1;  /* for NUL */
-                    if (ip->executable[data_size - 1] == L'\\')
-                        --data_size; /* reg value ended in a backslash */
-                    /* ip->executable is data_size long */
-                    for (checkp = location_checks; *checkp; ++checkp) {
-                        check = *checkp;
+
+                data_size = data_size / sizeof(wchar_t) - 1;  /* for NUL */
+                if (ip->executable[data_size - 1] == L'\\')
+                    --data_size; /* reg value ended in a backslash */
+                /* ip->executable is data_size long */
+                for (checkp = location_checks; *checkp; ++checkp) {
+                    check = *checkp;
+                    if (append_name) {
                         _snwprintf_s(&ip->executable[data_size],
                                      MAX_PATH - data_size,
                                      MAX_PATH - data_size,
                                      L"%ls%ls", check, PYTHON_EXECUTABLE);
-                        attrs = GetFileAttributesW(ip->executable);
-                        if (attrs == INVALID_FILE_ATTRIBUTES) {
-                            winerror(GetLastError(), message, MSGSIZE);
-                            debug(L"locate_pythons_for_key: %ls: %ls",
-                                  ip->executable, message);
-                        }
-                        else if (attrs & FILE_ATTRIBUTE_DIRECTORY) {
-                            debug(L"locate_pythons_for_key: '%ls' is a \
-directory\n",
-                                  ip->executable, attrs);
-                        }
-                        else if (find_existing_python(ip->executable)) {
-                            debug(L"locate_pythons_for_key: %ls: already \
-found\n", ip->executable);
-                        }
-                        else {
-                            /* check the executable type. */
+                    }
+                    attrs = GetFileAttributesW(ip->executable);
+                    if (attrs == INVALID_FILE_ATTRIBUTES) {
+                        winerror(GetLastError(), message, MSGSIZE);
+                        debug(L"locate_pythons_for_key: %ls: %ls",
+                              ip->executable, message);
+                    }
+                    else if (attrs & FILE_ATTRIBUTE_DIRECTORY) {
+                        debug(L"locate_pythons_for_key: '%ls' is a directory\n",
+                              ip->executable, attrs);
+                    }
+                    else if (find_existing_python(ip->executable)) {
+                        debug(L"locate_pythons_for_key: %ls: already found\n",
+                              ip->executable);
+                    }
+                    else {
+                        /* check the executable type. */
+                        if (bits) {
+                            ip->bits = bits;
+                        } else {
                             ok = GetBinaryTypeW(ip->executable, &attrs);
                             if (!ok) {
                                 debug(L"Failure getting binary type: %ls\n",
@@ -310,32 +359,48 @@ found\n", ip->executable);
                                     ip->bits = 32;
                                 else
                                     ip->bits = 0;
-                                if (ip->bits == 0) {
-                                    debug(L"locate_pythons_for_key: %ls: \
+                            }
+                        }
+                        if (ip->bits == 0) {
+                            debug(L"locate_pythons_for_key: %ls: \
 invalid binary type: %X\n",
-                                          ip->executable, attrs);
+                                  ip->executable, attrs);
+                        }
+                        else {
+                            if (display_name_only) {
+                                /* display just the executable name. This is
+                                 * primarily for the Store installs */
+                                const wchar_t *name = wcsrchr(ip->executable, L'\\');
+                                if (name) {
+                                    wcscpy_s(ip->exe_display, MAX_PATH, name+1);
                                 }
-                                else {
-                                    if (wcschr(ip->executable, L' ') != NULL) {
-                                        /* has spaces, so quote */
-                                        n = wcslen(ip->executable);
-                                        memmove(&ip->executable[1],
-                                                ip->executable, n * sizeof(wchar_t));
-                                        ip->executable[0] = L'\"';
-                                        ip->executable[n + 1] = L'\"';
-                                        ip->executable[n + 2] = L'\0';
-                                    }
-                                    debug(L"locate_pythons_for_key: %ls \
-is a %dbit executable\n",
-                                        ip->executable, ip->bits);
-                                    ++num_installed_pythons;
-                                    pip = ip++;
-                                    if (num_installed_pythons >=
-                                        MAX_INSTALLED_PYTHONS)
-                                        break;
-                                    /* Copy over the attributes for the next */
-                                    *ip = *pip;
+                            }
+                            if (wcschr(ip->executable, L' ') != NULL) {
+                                /* has spaces, so quote, and set original as
+                                 * the display name */
+                                if (!ip->exe_display[0]) {
+                                    wcscpy_s(ip->exe_display, MAX_PATH, ip->executable);
                                 }
+                                n = wcslen(ip->executable);
+                                memmove(&ip->executable[1],
+                                        ip->executable, n * sizeof(wchar_t));
+                                ip->executable[0] = L'\"';
+                                ip->executable[n + 1] = L'\"';
+                                ip->executable[n + 2] = L'\0';
+                            }
+                            debug(L"locate_pythons_for_key: %ls \
+is a %dbit executable\n",
+                                ip->executable, ip->bits);
+                            if (find_existing_python2(ip->bits, ip->version)) {
+                                debug(L"locate_pythons_for_key: %ls-%i: already \
+found\n", ip->version, ip->bits);
+                            }
+                            else {
+                                ++num_installed_pythons;
+                                pip = ip++;
+                                if (num_installed_pythons >=
+                                    MAX_INSTALLED_PYTHONS)
+                                    break;
                             }
                         }
                     }
@@ -359,9 +424,63 @@ compare_pythons(const void * p1, const void * p2)
     return result;
 }
 
+static void
+locate_pythons_for_key(HKEY root, REGSAM flags)
+{
+    _locate_pythons_for_key(root, CORE_PATH, flags, 0, FALSE);
+}
+
+static void
+locate_store_pythons()
+{
+#if defined(_M_X64)
+    /* 64bit process, so look in native registry */
+    _locate_pythons_for_key(HKEY_LOCAL_MACHINE, LOOKASIDE_PATH,
+                            KEY_READ, 64, TRUE);
+#else
+    /* 32bit process, so check that we're on 64bit OS */
+    BOOL f64 = FALSE;
+    if (IsWow64Process(GetCurrentProcess(), &f64) && f64) {
+        _locate_pythons_for_key(HKEY_LOCAL_MACHINE, LOOKASIDE_PATH,
+                                KEY_READ | KEY_WOW64_64KEY, 64, TRUE);
+    }
+#endif
+}
+
+static void
+locate_venv_python()
+{
+    static wchar_t venv_python[MAX_PATH];
+    INSTALLED_PYTHON * ip;
+    wchar_t *virtual_env = get_env(L"VIRTUAL_ENV");
+    DWORD attrs;
+
+    /* Check for VIRTUAL_ENV environment variable */
+    if (virtual_env == NULL || virtual_env[0] == L'\0') {
+        return;
+    }
+
+    /* Check for a python executable in the venv */
+    debug(L"Checking for Python executable in virtual env '%ls'\n", virtual_env);
+    _snwprintf_s(venv_python, MAX_PATH, _TRUNCATE,
+            L"%ls\\Scripts\\%ls", virtual_env, PYTHON_EXECUTABLE);
+    attrs = GetFileAttributesW(venv_python);
+    if (attrs == INVALID_FILE_ATTRIBUTES) {
+        debug(L"Python executable %ls missing from virtual env\n", venv_python);
+        return;
+    }
+
+    ip = &installed_pythons[num_installed_pythons++];
+    wcscpy_s(ip->executable, MAX_PATH, venv_python);
+    ip->bits = 0;
+    wcscpy_s(ip->version, MAX_VERSION_SIZE, L"venv");
+}
+
 static void
 locate_all_pythons()
 {
+    /* venv Python is highest priority */
+    locate_venv_python();
 #if defined(_M_X64)
     /* If we are a 64bit process, first hit the 32bit keys. */
     debug(L"locating Pythons in 32bit registry\n");
@@ -380,6 +499,8 @@ locate_all_pythons()
     debug(L"locating Pythons in native registry\n");
     locate_pythons_for_key(HKEY_CURRENT_USER, KEY_READ);
     locate_pythons_for_key(HKEY_LOCAL_MACHINE, KEY_READ);
+    /* Store-installed Python is lowest priority */
+    locate_store_pythons();
     qsort(installed_pythons, num_installed_pythons, sizeof(INSTALLED_PYTHON),
           compare_pythons);
 }
@@ -416,31 +537,6 @@ find_python_by_version(wchar_t const * wanted_ver)
 }
 
 
-static wchar_t *
-find_python_by_venv()
-{
-    static wchar_t venv_python[MAX_PATH];
-    wchar_t *virtual_env = get_env(L"VIRTUAL_ENV");
-    DWORD attrs;
-
-    /* Check for VIRTUAL_ENV environment variable */
-    if (virtual_env == NULL || virtual_env[0] == L'\0') {
-        return NULL;
-    }
-
-    /* Check for a python executable in the venv */
-    debug(L"Checking for Python executable in virtual env '%ls'\n", virtual_env);
-    _snwprintf_s(venv_python, MAX_PATH, _TRUNCATE,
-            L"%ls\\Scripts\\%ls", virtual_env, PYTHON_EXECUTABLE);
-    attrs = GetFileAttributesW(venv_python);
-    if (attrs == INVALID_FILE_ATTRIBUTES) {
-        debug(L"Python executable %ls missing from virtual env\n", venv_python);
-        return NULL;
-    }
-
-    return venv_python;
-}
-
 static wchar_t appdata_ini_path[MAX_PATH];
 static wchar_t launcher_ini_path[MAX_PATH];
 
@@ -523,9 +619,12 @@ locate_python(wchar_t * wanted_ver, BOOL from_shebang)
     }
     else {
         *last_char = L'\0'; /* look for an overall default */
-        configured_value = get_configured_value(config_key);
-        if (configured_value)
-            result = find_python_by_version(configured_value);
+        result = find_python_by_version(L"venv");
+        if (result == NULL) {
+            configured_value = get_configured_value(config_key);
+            if (configured_value)
+                result = find_python_by_version(configured_value);
+        }
         /* Not found a value yet - try by major version.
          * If we're looking for an interpreter specified in a shebang line,
          * we want to try Python 2 first, then Python 3 (for Unix and backward
@@ -1443,7 +1542,8 @@ show_python_list(wchar_t ** argv)
     INSTALLED_PYTHON * defpy = locate_python(L"", FALSE);
     size_t i = 0;
     wchar_t *p = argv[1];
-    wchar_t *fmt = L"\n -%ls-%d"; /* print VER-BITS */
+    wchar_t *ver_fmt = L"-%ls-%d";
+    wchar_t *fmt = L"\n %ls";
     wchar_t *defind = L" *"; /* Default indicator */
 
     /*
@@ -1452,8 +1552,8 @@ show_python_list(wchar_t ** argv)
     */
     fwprintf(stderr,
              L"Installed Pythons found by %s Launcher for Windows", argv[0]);
-    if (!_wcsicmp(p, L"-0p") || !_wcsicmp(p, L"--list-paths")) /* Show path? */
-        fmt = L"\n -%ls-%d\t%ls"; /* print VER-BITS path */
+    if (!_wcsicmp(p, L"-0p") || !_wcsicmp(p, L"--list-paths"))
+        fmt = L"\n %-15ls%ls"; /* include path */
 
     if (num_installed_pythons == 0) /* We have somehow got here without searching for pythons */
         locate_all_pythons(); /* Find them, Populates installed_pythons */
@@ -1463,9 +1563,22 @@ show_python_list(wchar_t ** argv)
     else
     {
         for (i = 0; i < num_installed_pythons; i++, ip++) {
-            fwprintf(stdout, fmt, ip->version, ip->bits, ip->executable);
+            wchar_t version[BUFSIZ];
+            if (wcscmp(ip->version, L"venv") == 0) {
+                wcscpy_s(version, BUFSIZ, L"(venv)");
+            }
+            else {
+                swprintf_s(version, BUFSIZ, ver_fmt, ip->version, ip->bits);
+            }
+
+            if (ip->exe_display[0]) {
+                fwprintf(stdout, fmt, version, ip->exe_display);
+            }
+            else {
+                fwprintf(stdout, fmt, version, ip->executable);
+            }
             /* If there is a default indicate it */
-            if ((defpy != NULL) && !_wcsicmp(ip->executable, defpy->executable))
+            if (defpy == ip)
                 fwprintf(stderr, defind);
         }
     }
@@ -1850,16 +1963,11 @@ installed, use -0 for available pythons", &p[1]);
             executable = NULL; /* Info call only */
         }
         else {
-            /* Look for an active virtualenv */
-            executable = find_python_by_venv();
-
-            /* If we didn't find one, look for the default Python */
-            if (executable == NULL) {
-                ip = locate_python(L"", FALSE);
-                if (ip == NULL)
-                    error(RC_NO_PYTHON, L"Can't find a default Python.");
-                executable = ip->executable;
-            }
+            /* look for the default Python */
+            ip = locate_python(L"", FALSE);
+            if (ip == NULL)
+                error(RC_NO_PYTHON, L"Can't find a default Python.");
+            executable = ip->executable;
         }
     }
     if (executable != NULL)


From webhook-mailer at python.org  Thu Sep 12 14:12:26 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 12 Sep 2019 18:12:26 -0000
Subject: [Python-checkins] Cut disused recode_encoding logic in
 _PyBytes_DecodeEscape. (GH-16013)
Message-ID: <mailman.693.1568311946.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3a4f66707e824ef3a8384827590ebaa6ca463dc0
commit: 3a4f66707e824ef3a8384827590ebaa6ca463dc0
branch: master
author: Greg Price <gnprice at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-12T19:12:22+01:00
summary:

Cut disused recode_encoding logic in _PyBytes_DecodeEscape. (GH-16013)

All call sites pass NULL for `recode_encoding`, so this path is
completely untested.  That's been true since before Python 3.0.
It adds significant complexity to this logic, so it's best to
take it out.

All call sites now have a literal NULL, and that's been true since
commit 768921cf3 eliminated a conditional (`foo ? bar : NULL`) at
the call site in Python/ast.c where we're parsing a bytes literal.
But even before then, that condition `foo` had been a constant
since unadorned string literals started meaning Unicode, in commit
572dbf8f1 aka v3.0a1~1035 .

The `unicode` parameter is already unused, so mark it as unused too.
The code that acted on it was also taken out before Python 3.0, in
commit 8d30cc014 aka v3.0a1~1031 .

The function (PyBytes_DecodeEscape) is exposed in the API, but it's
never been documented.

files:
M Include/bytesobject.h
M Include/longobject.h
M Objects/bytesobject.c
M Python/ast.c

diff --git a/Include/bytesobject.h b/Include/bytesobject.h
index 3fde4a221fdb..fc9981e56d27 100644
--- a/Include/bytesobject.h
+++ b/Include/bytesobject.h
@@ -77,9 +77,7 @@ PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t,
 #ifndef Py_LIMITED_API
 /* Helper for PyBytes_DecodeEscape that detects invalid escape chars. */
 PyAPI_FUNC(PyObject *) _PyBytes_DecodeEscape(const char *, Py_ssize_t,
-                                             const char *, Py_ssize_t,
-                                             const char *,
-                                             const char **);
+                                             const char *, const char **);
 #endif
 
 /* Macro, trading safety for speed */
diff --git a/Include/longobject.h b/Include/longobject.h
index 1e7a58d994b8..87b4d017d323 100644
--- a/Include/longobject.h
+++ b/Include/longobject.h
@@ -74,7 +74,7 @@ PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *);
 #endif
 
 /* Used by Python/mystrtoul.c, _PyBytes_FromHex(),
-   _PyBytes_DecodeEscapeRecode(), etc. */
+   _PyBytes_DecodeEscape(), etc. */
 #ifndef Py_LIMITED_API
 PyAPI_DATA(unsigned char) _PyLong_DigitValue[256];
 #endif
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
index e1f5ee2f62f3..4b2a77b4b8c1 100644
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -1077,52 +1077,10 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
     return NULL;
 }
 
-/* Unescape a backslash-escaped string. If unicode is non-zero,
-   the string is a u-literal. If recode_encoding is non-zero,
-   the string is UTF-8 encoded and should be re-encoded in the
-   specified encoding.  */
-
-static char *
-_PyBytes_DecodeEscapeRecode(const char **s, const char *end,
-                            const char *errors, const char *recode_encoding,
-                            _PyBytesWriter *writer, char *p)
-{
-    PyObject *u, *w;
-    const char* t;
-
-    t = *s;
-    /* Decode non-ASCII bytes as UTF-8. */
-    while (t < end && (*t & 0x80))
-        t++;
-    u = PyUnicode_DecodeUTF8(*s, t - *s, errors);
-    if (u == NULL)
-        return NULL;
-
-    /* Recode them in target encoding. */
-    w = PyUnicode_AsEncodedString(u, recode_encoding, errors);
-    Py_DECREF(u);
-    if  (w == NULL)
-        return NULL;
-    assert(PyBytes_Check(w));
-
-    /* Append bytes to output buffer. */
-    writer->min_size--;   /* subtract 1 preallocated byte */
-    p = _PyBytesWriter_WriteBytes(writer, p,
-                                  PyBytes_AS_STRING(w),
-                                  PyBytes_GET_SIZE(w));
-    Py_DECREF(w);
-    if (p == NULL)
-        return NULL;
-
-    *s = t;
-    return p;
-}
-
+/* Unescape a backslash-escaped string. */
 PyObject *_PyBytes_DecodeEscape(const char *s,
                                 Py_ssize_t len,
                                 const char *errors,
-                                Py_ssize_t unicode,
-                                const char *recode_encoding,
                                 const char **first_invalid_escape)
 {
     int c;
@@ -1142,17 +1100,7 @@ PyObject *_PyBytes_DecodeEscape(const char *s,
     end = s + len;
     while (s < end) {
         if (*s != '\\') {
-            if (!(recode_encoding && (*s & 0x80))) {
-                *p++ = *s++;
-            }
-            else {
-                /* non-ASCII character and need to recode */
-                p = _PyBytes_DecodeEscapeRecode(&s, end,
-                                                errors, recode_encoding,
-                                                &writer, p);
-                if (p == NULL)
-                    goto failed;
-            }
+            *p++ = *s++;
             continue;
         }
 
@@ -1241,12 +1189,11 @@ PyObject *_PyBytes_DecodeEscape(const char *s,
 PyObject *PyBytes_DecodeEscape(const char *s,
                                 Py_ssize_t len,
                                 const char *errors,
-                                Py_ssize_t unicode,
-                                const char *recode_encoding)
+                                Py_ssize_t Py_UNUSED(unicode),
+                                const char *Py_UNUSED(recode_encoding))
 {
     const char* first_invalid_escape;
-    PyObject *result = _PyBytes_DecodeEscape(s, len, errors, unicode,
-                                             recode_encoding,
+    PyObject *result = _PyBytes_DecodeEscape(s, len, errors,
                                              &first_invalid_escape);
     if (result == NULL)
         return NULL;
diff --git a/Python/ast.c b/Python/ast.c
index e6f71671c18e..05147a49fc1d 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -4766,7 +4766,7 @@ decode_bytes_with_escapes(struct compiling *c, const node *n, const char *s,
                           size_t len)
 {
     const char *first_invalid_escape;
-    PyObject *result = _PyBytes_DecodeEscape(s, len, NULL, 0, NULL,
+    PyObject *result = _PyBytes_DecodeEscape(s, len, NULL,
                                              &first_invalid_escape);
     if (result == NULL)
         return NULL;


From webhook-mailer at python.org  Thu Sep 12 20:30:18 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Fri, 13 Sep 2019 00:30:18 -0000
Subject: [Python-checkins] bpo-9216: Add usedforsecurity to hashlib
 constructors (GH-16044)
Message-ID: <mailman.694.1568334619.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7cad53e6b084435a220e6604010f1fa5778bd0b1
commit: 7cad53e6b084435a220e6604010f1fa5778bd0b1
branch: master
author: Christian Heimes <christian at python.org>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-12T19:30:00-05:00
summary:

bpo-9216: Add usedforsecurity to hashlib constructors (GH-16044)

The usedforsecurity keyword only argument added to the hash constructors is useful for FIPS builds and similar restrictive environment with non-technical requirements that legacy algorithms be forbidden by their implementations without being explicitly annotated as not being used for any security related purposes.  Linux distros with FIPS support benefit from this being standard rather than making up their own way(s) to do it.

Contributed and Signed-off-by: Christian Heimes christian at python.org

files:
A Misc/NEWS.d/next/Library/2019-09-12-14-54-45.bpo-9216.W7QMpC.rst
M Doc/library/hashlib.rst
M Lib/test/test_hashlib.py
M Lib/uuid.py
M Modules/_blake2/blake2b_impl.c
M Modules/_blake2/blake2s_impl.c
M Modules/_blake2/clinic/blake2b_impl.c.h
M Modules/_blake2/clinic/blake2s_impl.c.h
M Modules/_hashopenssl.c
M Modules/_sha3/clinic/sha3module.c.h
M Modules/_sha3/sha3module.c
M Modules/clinic/_hashopenssl.c.h
M Modules/clinic/md5module.c.h
M Modules/clinic/sha1module.c.h
M Modules/clinic/sha256module.c.h
M Modules/clinic/sha512module.c.h
M Modules/md5module.c
M Modules/sha1module.c
M Modules/sha256module.c
M Modules/sha512module.c

diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst
index a16c7cd4d7cd..6eb3a7bb4cc3 100644
--- a/Doc/library/hashlib.rst
+++ b/Doc/library/hashlib.rst
@@ -67,7 +67,7 @@ Constructors for hash algorithms that are always present in this module are
 :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`,
 :func:`sha512`, :func:`blake2b`, and :func:`blake2s`.
 :func:`md5` is normally available as well, though it
-may be missing if you are using a rare "FIPS compliant" build of Python.
+may be missing or blocked if you are using a rare "FIPS compliant" build of Python.
 Additional algorithms may also be available depending upon the OpenSSL
 library that Python uses on your platform. On most platforms the
 :func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`,
@@ -80,6 +80,13 @@ library that Python uses on your platform. On most platforms the
 .. versionadded:: 3.6
    :func:`blake2b` and :func:`blake2s` were added.
 
+.. versionchanged:: 3.9
+   All hashlib constructors take a keyword-only argument *usedforsecurity*
+   with default value *True*. A false value allows the use of insecure and
+   blocked hashing algorithms in restricted environments. *False* indicates
+   that the hashing algorithm is not used in a security context, e.g. as a
+   non-cryptographic one-way compression function.
+
 For example, to obtain the digest of the byte string ``b'Nobody inspects the
 spammish repetition'``::
 
@@ -99,7 +106,7 @@ More condensed:
    >>> hashlib.sha224(b"Nobody inspects the spammish repetition").hexdigest()
    'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2'
 
-.. function:: new(name[, data])
+.. function:: new(name[, data], *, usedforsecurity=True)
 
    Is a generic constructor that takes the string *name* of the desired
    algorithm as its first parameter.  It also exists to allow access to the
@@ -308,11 +315,13 @@ New hash objects are created by calling constructor functions:
 
 .. function:: blake2b(data=b'', *, digest_size=64, key=b'', salt=b'', \
                 person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0,  \
-                node_depth=0, inner_size=0, last_node=False)
+                node_depth=0, inner_size=0, last_node=False, \
+                usedforsecurity=True)
 
 .. function:: blake2s(data=b'', *, digest_size=32, key=b'', salt=b'', \
                 person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0,  \
-                node_depth=0, inner_size=0, last_node=False)
+                node_depth=0, inner_size=0, last_node=False, \
+                usedforsecurity=True)
 
 
 These functions return the corresponding hash objects for calculating
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index b7b04a37f0d0..46088e52dc54 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -190,6 +190,15 @@ def test_algorithms_available(self):
         self.assertTrue(set(hashlib.algorithms_guaranteed).
                             issubset(hashlib.algorithms_available))
 
+    def test_usedforsecurity(self):
+        for cons in self.hash_constructors:
+            cons(usedforsecurity=True)
+            cons(usedforsecurity=False)
+            cons(b'', usedforsecurity=True)
+            cons(b'', usedforsecurity=False)
+        hashlib.new("sha256", usedforsecurity=True)
+        hashlib.new("sha256", usedforsecurity=False)
+
     def test_unknown_hash(self):
         self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam')
         self.assertRaises(TypeError, hashlib.new, 1)
diff --git a/Lib/uuid.py b/Lib/uuid.py
index 188e16ba14e3..5f3bc9e8de44 100644
--- a/Lib/uuid.py
+++ b/Lib/uuid.py
@@ -772,8 +772,11 @@ def uuid1(node=None, clock_seq=None):
 def uuid3(namespace, name):
     """Generate a UUID from the MD5 hash of a namespace UUID and a name."""
     from hashlib import md5
-    hash = md5(namespace.bytes + bytes(name, "utf-8")).digest()
-    return UUID(bytes=hash[:16], version=3)
+    digest = md5(
+        namespace.bytes + bytes(name, "utf-8"),
+        usedforsecurity=False
+    ).digest()
+    return UUID(bytes=digest[:16], version=3)
 
 def uuid4():
     """Generate a random UUID."""
diff --git a/Misc/NEWS.d/next/Library/2019-09-12-14-54-45.bpo-9216.W7QMpC.rst b/Misc/NEWS.d/next/Library/2019-09-12-14-54-45.bpo-9216.W7QMpC.rst
new file mode 100644
index 000000000000..a97ca4b8b4f2
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-12-14-54-45.bpo-9216.W7QMpC.rst
@@ -0,0 +1,2 @@
+hashlib constructors now support usedforsecurity flag to signal that a
+hashing algorithm is not used in a security context.
diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c
index edab31ea222a..88c103f56c4a 100644
--- a/Modules/_blake2/blake2b_impl.c
+++ b/Modules/_blake2/blake2b_impl.c
@@ -81,6 +81,7 @@ _blake2.blake2b.__new__ as py_blake2b_new
     node_depth: int = 0
     inner_size: int = 0
     last_node: bool = False
+    usedforsecurity: bool = True
 
 Return a new BLAKE2b hash object.
 [clinic start generated code]*/
@@ -90,8 +91,8 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size,
                     Py_buffer *key, Py_buffer *salt, Py_buffer *person,
                     int fanout, int depth, unsigned long leaf_size,
                     unsigned long long node_offset, int node_depth,
-                    int inner_size, int last_node)
-/*[clinic end generated code: output=65e732c66c2297a0 input=82be35a4e6a9daa2]*/
+                    int inner_size, int last_node, int usedforsecurity)
+/*[clinic end generated code: output=32bfd8f043c6896f input=b947312abff46977]*/
 {
     BLAKE2bObject *self = NULL;
     Py_buffer buf;
diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c
index ef2f7e1980ff..c7c7eafdee22 100644
--- a/Modules/_blake2/blake2s_impl.c
+++ b/Modules/_blake2/blake2s_impl.c
@@ -81,6 +81,7 @@ _blake2.blake2s.__new__ as py_blake2s_new
     node_depth: int = 0
     inner_size: int = 0
     last_node: bool = False
+    usedforsecurity: bool = True
 
 Return a new BLAKE2s hash object.
 [clinic start generated code]*/
@@ -90,8 +91,8 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size,
                     Py_buffer *key, Py_buffer *salt, Py_buffer *person,
                     int fanout, int depth, unsigned long leaf_size,
                     unsigned long long node_offset, int node_depth,
-                    int inner_size, int last_node)
-/*[clinic end generated code: output=b95806be0514dcf7 input=641c0509debf714d]*/
+                    int inner_size, int last_node, int usedforsecurity)
+/*[clinic end generated code: output=556181f73905c686 input=4dda87723f23abb0]*/
 {
     BLAKE2sObject *self = NULL;
     Py_buffer buf;
diff --git a/Modules/_blake2/clinic/blake2b_impl.c.h b/Modules/_blake2/clinic/blake2b_impl.c.h
index cd329c07c999..07258c31c9bf 100644
--- a/Modules/_blake2/clinic/blake2b_impl.c.h
+++ b/Modules/_blake2/clinic/blake2b_impl.c.h
@@ -5,7 +5,8 @@ preserve
 PyDoc_STRVAR(py_blake2b_new__doc__,
 "blake2b(data=b\'\', /, *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n"
 "        key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n"
-"        node_offset=0, node_depth=0, inner_size=0, last_node=False)\n"
+"        node_offset=0, node_depth=0, inner_size=0, last_node=False,\n"
+"        usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Return a new BLAKE2b hash object.");
@@ -15,15 +16,15 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size,
                     Py_buffer *key, Py_buffer *salt, Py_buffer *person,
                     int fanout, int depth, unsigned long leaf_size,
                     unsigned long long node_offset, int node_depth,
-                    int inner_size, int last_node);
+                    int inner_size, int last_node, int usedforsecurity);
 
 static PyObject *
 py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", NULL};
+    static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "blake2b", 0};
-    PyObject *argsbuf[12];
+    PyObject *argsbuf[13];
     PyObject * const *fastargs;
     Py_ssize_t nargs = PyTuple_GET_SIZE(args);
     Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0;
@@ -39,6 +40,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
     int node_depth = 0;
     int inner_size = 0;
     int last_node = 0;
+    int usedforsecurity = 1;
 
     fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf);
     if (!fastargs) {
@@ -175,12 +177,21 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
             goto skip_optional_kwonly;
         }
     }
-    last_node = PyObject_IsTrue(fastargs[11]);
-    if (last_node < 0) {
+    if (fastargs[11]) {
+        last_node = PyObject_IsTrue(fastargs[11]);
+        if (last_node < 0) {
+            goto exit;
+        }
+        if (!--noptargs) {
+            goto skip_optional_kwonly;
+        }
+    }
+    usedforsecurity = PyObject_IsTrue(fastargs[12]);
+    if (usedforsecurity < 0) {
         goto exit;
     }
 skip_optional_kwonly:
-    return_value = py_blake2b_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node);
+    return_value = py_blake2b_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity);
 
 exit:
     /* Cleanup for key */
@@ -261,4 +272,4 @@ _blake2_blake2b_hexdigest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored))
 {
     return _blake2_blake2b_hexdigest_impl(self);
 }
-/*[clinic end generated code: output=cbb625d7f60c288c input=a9049054013a1b77]*/
+/*[clinic end generated code: output=2d6d0fe9aa42a42a input=a9049054013a1b77]*/
diff --git a/Modules/_blake2/clinic/blake2s_impl.c.h b/Modules/_blake2/clinic/blake2s_impl.c.h
index 560bd68160b0..71c5706fb660 100644
--- a/Modules/_blake2/clinic/blake2s_impl.c.h
+++ b/Modules/_blake2/clinic/blake2s_impl.c.h
@@ -5,7 +5,8 @@ preserve
 PyDoc_STRVAR(py_blake2s_new__doc__,
 "blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n"
 "        key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n"
-"        node_offset=0, node_depth=0, inner_size=0, last_node=False)\n"
+"        node_offset=0, node_depth=0, inner_size=0, last_node=False,\n"
+"        usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Return a new BLAKE2s hash object.");
@@ -15,15 +16,15 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size,
                     Py_buffer *key, Py_buffer *salt, Py_buffer *person,
                     int fanout, int depth, unsigned long leaf_size,
                     unsigned long long node_offset, int node_depth,
-                    int inner_size, int last_node);
+                    int inner_size, int last_node, int usedforsecurity);
 
 static PyObject *
 py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", NULL};
+    static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "blake2s", 0};
-    PyObject *argsbuf[12];
+    PyObject *argsbuf[13];
     PyObject * const *fastargs;
     Py_ssize_t nargs = PyTuple_GET_SIZE(args);
     Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0;
@@ -39,6 +40,7 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
     int node_depth = 0;
     int inner_size = 0;
     int last_node = 0;
+    int usedforsecurity = 1;
 
     fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf);
     if (!fastargs) {
@@ -175,12 +177,21 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
             goto skip_optional_kwonly;
         }
     }
-    last_node = PyObject_IsTrue(fastargs[11]);
-    if (last_node < 0) {
+    if (fastargs[11]) {
+        last_node = PyObject_IsTrue(fastargs[11]);
+        if (last_node < 0) {
+            goto exit;
+        }
+        if (!--noptargs) {
+            goto skip_optional_kwonly;
+        }
+    }
+    usedforsecurity = PyObject_IsTrue(fastargs[12]);
+    if (usedforsecurity < 0) {
         goto exit;
     }
 skip_optional_kwonly:
-    return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node);
+    return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity);
 
 exit:
     /* Cleanup for key */
@@ -261,4 +272,4 @@ _blake2_blake2s_hexdigest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored))
 {
     return _blake2_blake2s_hexdigest_impl(self);
 }
-/*[clinic end generated code: output=39af5a74c8805b36 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=c80d8d06ce40a192 input=a9049054013a1b77]*/
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index c685279c7a7b..c65c698cbba1 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -405,7 +405,7 @@ static PyTypeObject EVPtype = {
 
 static PyObject *
 EVPnew(const EVP_MD *digest,
-       const unsigned char *cp, Py_ssize_t len)
+       const unsigned char *cp, Py_ssize_t len, int usedforsecurity)
 {
     int result = 0;
     EVPobject *self;
@@ -418,6 +418,13 @@ EVPnew(const EVP_MD *digest,
     if ((self = newEVPobject()) == NULL)
         return NULL;
 
+    if (!usedforsecurity) {
+#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
+        EVP_MD_CTX_set_flags(self->ctx, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
+#endif
+    }
+
+
     if (!EVP_DigestInit(self->ctx, digest)) {
         _setException(PyExc_ValueError);
         Py_DECREF(self);
@@ -449,6 +456,8 @@ _hashlib.new as EVP_new
 
     name as name_obj: object
     string as data_obj: object(py_default="b''") = NULL
+    *
+    usedforsecurity: bool = True
 
 Return a new hash object using the named algorithm.
 
@@ -459,8 +468,9 @@ The MD5 and SHA1 algorithms are always supported.
 [clinic start generated code]*/
 
 static PyObject *
-EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj)
-/*[clinic end generated code: output=9e7cf664e04b0226 input=1c46e40e0fec91f3]*/
+EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj,
+             int usedforsecurity)
+/*[clinic end generated code: output=ddd5053f92dffe90 input=e9ac115d80962ddf]*/
 {
     Py_buffer view = { 0 };
     PyObject *ret_obj;
@@ -477,7 +487,9 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj)
 
     digest = EVP_get_digestbyname(name);
 
-    ret_obj = EVPnew(digest, (unsigned char*)view.buf, view.len);
+    ret_obj = EVPnew(digest,
+                     (unsigned char*)view.buf, view.len,
+                     usedforsecurity);
 
     if (data_obj)
         PyBuffer_Release(&view);
@@ -485,7 +497,8 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj)
 }
 
 static PyObject*
-EVP_fast_new(PyObject *module, PyObject *data_obj, const EVP_MD *digest)
+EVP_fast_new(PyObject *module, PyObject *data_obj, const EVP_MD *digest,
+             int usedforsecurity)
 {
     Py_buffer view = { 0 };
     PyObject *ret_obj;
@@ -493,7 +506,9 @@ EVP_fast_new(PyObject *module, PyObject *data_obj, const EVP_MD *digest)
     if (data_obj)
         GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view);
 
-    ret_obj = EVPnew(digest, (unsigned char*)view.buf, view.len);
+    ret_obj = EVPnew(digest,
+                     (unsigned char*)view.buf, view.len,
+                     usedforsecurity);
 
     if (data_obj)
         PyBuffer_Release(&view);
@@ -505,16 +520,19 @@ EVP_fast_new(PyObject *module, PyObject *data_obj, const EVP_MD *digest)
 _hashlib.openssl_md5
 
     string as data_obj: object(py_default="b''") = NULL
+    *
+    usedforsecurity: bool = True
 
 Returns a md5 hash object; optionally initialized with a string
 
 [clinic start generated code]*/
 
 static PyObject *
-_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj)
-/*[clinic end generated code: output=6caae75b73e22c3f input=52010d3869e1b1a7]*/
+_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj,
+                          int usedforsecurity)
+/*[clinic end generated code: output=87b0186440a44f8c input=990e36d5e689b16e]*/
 {
-    return EVP_fast_new(module, data_obj, EVP_md5());
+    return EVP_fast_new(module, data_obj, EVP_md5(), usedforsecurity);
 }
 
 
@@ -522,16 +540,19 @@ _hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj)
 _hashlib.openssl_sha1
 
     string as data_obj: object(py_default="b''") = NULL
+    *
+    usedforsecurity: bool = True
 
 Returns a sha1 hash object; optionally initialized with a string
 
 [clinic start generated code]*/
 
 static PyObject *
-_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj)
-/*[clinic end generated code: output=07606d8f75153e61 input=16807d30e4aa8ae9]*/
+_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj,
+                           int usedforsecurity)
+/*[clinic end generated code: output=6813024cf690670d input=948f2f4b6deabc10]*/
 {
-    return EVP_fast_new(module, data_obj, EVP_sha1());
+    return EVP_fast_new(module, data_obj, EVP_sha1(), usedforsecurity);
 }
 
 
@@ -539,16 +560,19 @@ _hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj)
 _hashlib.openssl_sha224
 
     string as data_obj: object(py_default="b''") = NULL
+    *
+    usedforsecurity: bool = True
 
 Returns a sha224 hash object; optionally initialized with a string
 
 [clinic start generated code]*/
 
 static PyObject *
-_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj)
-/*[clinic end generated code: output=55e848761bcef0c9 input=5dbc2f1d84eb459b]*/
+_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj,
+                             int usedforsecurity)
+/*[clinic end generated code: output=a2dfe7cc4eb14ebb input=f9272821fadca505]*/
 {
-    return EVP_fast_new(module, data_obj, EVP_sha224());
+    return EVP_fast_new(module, data_obj, EVP_sha224(), usedforsecurity);
 }
 
 
@@ -556,16 +580,19 @@ _hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj)
 _hashlib.openssl_sha256
 
     string as data_obj: object(py_default="b''") = NULL
+    *
+    usedforsecurity: bool = True
 
 Returns a sha256 hash object; optionally initialized with a string
 
 [clinic start generated code]*/
 
 static PyObject *
-_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj)
-/*[clinic end generated code: output=05851d7cce34ac65 input=a68a5d21cda5a80f]*/
+_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj,
+                             int usedforsecurity)
+/*[clinic end generated code: output=1f874a34870f0a68 input=549fad9d2930d4c5]*/
 {
-    return EVP_fast_new(module, data_obj, EVP_sha256());
+    return EVP_fast_new(module, data_obj, EVP_sha256(), usedforsecurity);
 }
 
 
@@ -573,16 +600,19 @@ _hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj)
 _hashlib.openssl_sha384
 
     string as data_obj: object(py_default="b''") = NULL
+    *
+    usedforsecurity: bool = True
 
 Returns a sha384 hash object; optionally initialized with a string
 
 [clinic start generated code]*/
 
 static PyObject *
-_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj)
-/*[clinic end generated code: output=5101a4704a932c2f input=6bdfa006622b64ea]*/
+_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj,
+                             int usedforsecurity)
+/*[clinic end generated code: output=58529eff9ca457b2 input=48601a6e3bf14ad7]*/
 {
-    return EVP_fast_new(module, data_obj, EVP_sha384());
+    return EVP_fast_new(module, data_obj, EVP_sha384(), usedforsecurity);
 }
 
 
@@ -590,16 +620,19 @@ _hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj)
 _hashlib.openssl_sha512
 
     string as data_obj: object(py_default="b''") = NULL
+    *
+    usedforsecurity: bool = True
 
 Returns a sha512 hash object; optionally initialized with a string
 
 [clinic start generated code]*/
 
 static PyObject *
-_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj)
-/*[clinic end generated code: output=20c8e63ee560a5cb input=ece50182ad4b76a6]*/
+_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj,
+                             int usedforsecurity)
+/*[clinic end generated code: output=2c744c9e4a40d5f6 input=c5c46a2a817aa98f]*/
 {
-    return EVP_fast_new(module, data_obj, EVP_sha512());
+    return EVP_fast_new(module, data_obj, EVP_sha512(), usedforsecurity);
 }
 
 
diff --git a/Modules/_sha3/clinic/sha3module.c.h b/Modules/_sha3/clinic/sha3module.c.h
index 554442df0ec2..1c79c269391c 100644
--- a/Modules/_sha3/clinic/sha3module.c.h
+++ b/Modules/_sha3/clinic/sha3module.c.h
@@ -2,6 +2,52 @@
 preserve
 [clinic start generated code]*/
 
+PyDoc_STRVAR(py_sha3_new__doc__,
+"sha3_224(data=b\'\', /, *, usedforsecurity=True)\n"
+"--\n"
+"\n"
+"Return a new BLAKE2b hash object.");
+
+static PyObject *
+py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity);
+
+static PyObject *
+py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"", "usedforsecurity", NULL};
+    static _PyArg_Parser _parser = {NULL, _keywords, "sha3_224", 0};
+    PyObject *argsbuf[2];
+    PyObject * const *fastargs;
+    Py_ssize_t nargs = PyTuple_GET_SIZE(args);
+    Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0;
+    PyObject *data = NULL;
+    int usedforsecurity = 1;
+
+    fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf);
+    if (!fastargs) {
+        goto exit;
+    }
+    if (nargs < 1) {
+        goto skip_optional_posonly;
+    }
+    noptargs--;
+    data = fastargs[0];
+skip_optional_posonly:
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(fastargs[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = py_sha3_new_impl(type, data, usedforsecurity);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(_sha3_sha3_224_copy__doc__,
 "copy($self, /)\n"
 "--\n"
@@ -118,4 +164,4 @@ _sha3_shake_128_hexdigest(SHA3object *self, PyObject *arg)
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=5b3e99b9a96471e8 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=c8a97b34e80def62 input=a9049054013a1b77]*/
diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c
index c1fb6185e243..d4ca9a111da6 100644
--- a/Modules/_sha3/sha3module.c
+++ b/Modules/_sha3/sha3module.c
@@ -169,21 +169,24 @@ newSHA3object(PyTypeObject *type)
     return newobj;
 }
 
+/*[clinic input]
+ at classmethod
+_sha3.sha3_224.__new__ as py_sha3_new
+    data: object(c_default="NULL") = b''
+    /
+    *
+    usedforsecurity: bool = True
+
+Return a new BLAKE2b hash object.
+[clinic start generated code]*/
 
 static PyObject *
-py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
+py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity)
+/*[clinic end generated code: output=90409addc5d5e8b0 input=bcfcdf2e4368347a]*/
 {
     SHA3object *self = NULL;
     Py_buffer buf = {NULL, NULL};
     HashReturn res;
-    PyObject *data = NULL;
-
-    if (!_PyArg_NoKeywords(_PyType_Name(type), kwargs)) {
-        return NULL;
-    }
-    if (!PyArg_UnpackTuple(args, _PyType_Name(type), 0, 1, &data)) {
-        return NULL;
-    }
 
     self = newSHA3object(type);
     if (self == NULL) {
@@ -529,22 +532,22 @@ static PyGetSetDef SHA3_getseters[] = {
     }
 
 PyDoc_STRVAR(sha3_224__doc__,
-"sha3_224([data]) -> SHA3 object\n\
+"sha3_224([data], *, usedforsecurity=True) -> SHA3 object\n\
 \n\
 Return a new SHA3 hash object with a hashbit length of 28 bytes.");
 
 PyDoc_STRVAR(sha3_256__doc__,
-"sha3_256([data]) -> SHA3 object\n\
+"sha3_256([data], *, usedforsecurity=True) -> SHA3 object\n\
 \n\
 Return a new SHA3 hash object with a hashbit length of 32 bytes.");
 
 PyDoc_STRVAR(sha3_384__doc__,
-"sha3_384([data]) -> SHA3 object\n\
+"sha3_384([data], *, usedforsecurity=True) -> SHA3 object\n\
 \n\
 Return a new SHA3 hash object with a hashbit length of 48 bytes.");
 
 PyDoc_STRVAR(sha3_512__doc__,
-"sha3_512([data]) -> SHA3 object\n\
+"sha3_512([data], *, usedforsecurity=True) -> SHA3 object\n\
 \n\
 Return a new SHA3 hash object with a hashbit length of 64 bytes.");
 
@@ -555,22 +558,22 @@ SHA3_TYPE(SHA3_512type, "_sha3.sha3_512", sha3_512__doc__, SHA3_methods);
 
 #ifdef PY_WITH_KECCAK
 PyDoc_STRVAR(keccak_224__doc__,
-"keccak_224([data]) -> Keccak object\n\
+"keccak_224([data], *, usedforsecurity=True) -> Keccak object\n\
 \n\
 Return a new Keccak hash object with a hashbit length of 28 bytes.");
 
 PyDoc_STRVAR(keccak_256__doc__,
-"keccak_256([data]) -> Keccak object\n\
+"keccak_256([data], *, usedforsecurity=True) -> Keccak object\n\
 \n\
 Return a new Keccak hash object with a hashbit length of 32 bytes.");
 
 PyDoc_STRVAR(keccak_384__doc__,
-"keccak_384([data]) -> Keccak object\n\
+"keccak_384([data], *, usedforsecurity=True) -> Keccak object\n\
 \n\
 Return a new Keccak hash object with a hashbit length of 48 bytes.");
 
 PyDoc_STRVAR(keccak_512__doc__,
-"keccak_512([data]) -> Keccak object\n\
+"keccak_512([data], *, usedforsecurity=True) -> Keccak object\n\
 \n\
 Return a new Keccak hash object with a hashbit length of 64 bytes.");
 
@@ -672,12 +675,12 @@ static PyMethodDef SHAKE_methods[] = {
 };
 
 PyDoc_STRVAR(shake_128__doc__,
-"shake_128([data]) -> SHAKE object\n\
+"shake_128([data], *, usedforsecurity=True) -> SHAKE object\n\
 \n\
 Return a new SHAKE hash object.");
 
 PyDoc_STRVAR(shake_256__doc__,
-"shake_256([data]) -> SHAKE object\n\
+"shake_256([data], *, usedforsecurity=True) -> SHAKE object\n\
 \n\
 Return a new SHAKE hash object.");
 
diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h
index 9aaea47e8328..de53e7e196c2 100644
--- a/Modules/clinic/_hashopenssl.c.h
+++ b/Modules/clinic/_hashopenssl.c.h
@@ -66,7 +66,7 @@ PyDoc_STRVAR(EVP_update__doc__,
     {"update", (PyCFunction)EVP_update, METH_O, EVP_update__doc__},
 
 PyDoc_STRVAR(EVP_new__doc__,
-"new($module, /, name, string=b\'\')\n"
+"new($module, /, name, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Return a new hash object using the named algorithm.\n"
@@ -80,18 +80,20 @@ PyDoc_STRVAR(EVP_new__doc__,
     {"new", (PyCFunction)(void(*)(void))EVP_new, METH_FASTCALL|METH_KEYWORDS, EVP_new__doc__},
 
 static PyObject *
-EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj);
+EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj,
+             int usedforsecurity);
 
 static PyObject *
 EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"name", "string", NULL};
+    static const char * const _keywords[] = {"name", "string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "new", 0};
-    PyObject *argsbuf[2];
+    PyObject *argsbuf[3];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
     PyObject *name_obj;
     PyObject *data_obj = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf);
     if (!args) {
@@ -101,16 +103,29 @@ EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    data_obj = args[1];
+    if (args[1]) {
+        data_obj = args[1];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = EVP_new_impl(module, name_obj, data_obj);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[2]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = EVP_new_impl(module, name_obj, data_obj, usedforsecurity);
 
 exit:
     return return_value;
 }
 
 PyDoc_STRVAR(_hashlib_openssl_md5__doc__,
-"openssl_md5($module, /, string=b\'\')\n"
+"openssl_md5($module, /, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Returns a md5 hash object; optionally initialized with a string");
@@ -119,17 +134,19 @@ PyDoc_STRVAR(_hashlib_openssl_md5__doc__,
     {"openssl_md5", (PyCFunction)(void(*)(void))_hashlib_openssl_md5, METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_md5__doc__},
 
 static PyObject *
-_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj);
+_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj,
+                          int usedforsecurity);
 
 static PyObject *
 _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"string", NULL};
+    static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "openssl_md5", 0};
-    PyObject *argsbuf[1];
+    PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     PyObject *data_obj = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
     if (!args) {
@@ -138,16 +155,29 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    data_obj = args[0];
+    if (args[0]) {
+        data_obj = args[0];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = _hashlib_openssl_md5_impl(module, data_obj);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = _hashlib_openssl_md5_impl(module, data_obj, usedforsecurity);
 
 exit:
     return return_value;
 }
 
 PyDoc_STRVAR(_hashlib_openssl_sha1__doc__,
-"openssl_sha1($module, /, string=b\'\')\n"
+"openssl_sha1($module, /, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Returns a sha1 hash object; optionally initialized with a string");
@@ -156,17 +186,19 @@ PyDoc_STRVAR(_hashlib_openssl_sha1__doc__,
     {"openssl_sha1", (PyCFunction)(void(*)(void))_hashlib_openssl_sha1, METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha1__doc__},
 
 static PyObject *
-_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj);
+_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj,
+                           int usedforsecurity);
 
 static PyObject *
 _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"string", NULL};
+    static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "openssl_sha1", 0};
-    PyObject *argsbuf[1];
+    PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     PyObject *data_obj = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
     if (!args) {
@@ -175,16 +207,29 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    data_obj = args[0];
+    if (args[0]) {
+        data_obj = args[0];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = _hashlib_openssl_sha1_impl(module, data_obj);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = _hashlib_openssl_sha1_impl(module, data_obj, usedforsecurity);
 
 exit:
     return return_value;
 }
 
 PyDoc_STRVAR(_hashlib_openssl_sha224__doc__,
-"openssl_sha224($module, /, string=b\'\')\n"
+"openssl_sha224($module, /, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Returns a sha224 hash object; optionally initialized with a string");
@@ -193,17 +238,19 @@ PyDoc_STRVAR(_hashlib_openssl_sha224__doc__,
     {"openssl_sha224", (PyCFunction)(void(*)(void))_hashlib_openssl_sha224, METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha224__doc__},
 
 static PyObject *
-_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj);
+_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj,
+                             int usedforsecurity);
 
 static PyObject *
 _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"string", NULL};
+    static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "openssl_sha224", 0};
-    PyObject *argsbuf[1];
+    PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     PyObject *data_obj = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
     if (!args) {
@@ -212,16 +259,29 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    data_obj = args[0];
+    if (args[0]) {
+        data_obj = args[0];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = _hashlib_openssl_sha224_impl(module, data_obj);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = _hashlib_openssl_sha224_impl(module, data_obj, usedforsecurity);
 
 exit:
     return return_value;
 }
 
 PyDoc_STRVAR(_hashlib_openssl_sha256__doc__,
-"openssl_sha256($module, /, string=b\'\')\n"
+"openssl_sha256($module, /, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Returns a sha256 hash object; optionally initialized with a string");
@@ -230,17 +290,19 @@ PyDoc_STRVAR(_hashlib_openssl_sha256__doc__,
     {"openssl_sha256", (PyCFunction)(void(*)(void))_hashlib_openssl_sha256, METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha256__doc__},
 
 static PyObject *
-_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj);
+_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj,
+                             int usedforsecurity);
 
 static PyObject *
 _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"string", NULL};
+    static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "openssl_sha256", 0};
-    PyObject *argsbuf[1];
+    PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     PyObject *data_obj = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
     if (!args) {
@@ -249,16 +311,29 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    data_obj = args[0];
+    if (args[0]) {
+        data_obj = args[0];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = _hashlib_openssl_sha256_impl(module, data_obj);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = _hashlib_openssl_sha256_impl(module, data_obj, usedforsecurity);
 
 exit:
     return return_value;
 }
 
 PyDoc_STRVAR(_hashlib_openssl_sha384__doc__,
-"openssl_sha384($module, /, string=b\'\')\n"
+"openssl_sha384($module, /, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Returns a sha384 hash object; optionally initialized with a string");
@@ -267,17 +342,19 @@ PyDoc_STRVAR(_hashlib_openssl_sha384__doc__,
     {"openssl_sha384", (PyCFunction)(void(*)(void))_hashlib_openssl_sha384, METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha384__doc__},
 
 static PyObject *
-_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj);
+_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj,
+                             int usedforsecurity);
 
 static PyObject *
 _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"string", NULL};
+    static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "openssl_sha384", 0};
-    PyObject *argsbuf[1];
+    PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     PyObject *data_obj = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
     if (!args) {
@@ -286,16 +363,29 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    data_obj = args[0];
+    if (args[0]) {
+        data_obj = args[0];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = _hashlib_openssl_sha384_impl(module, data_obj);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = _hashlib_openssl_sha384_impl(module, data_obj, usedforsecurity);
 
 exit:
     return return_value;
 }
 
 PyDoc_STRVAR(_hashlib_openssl_sha512__doc__,
-"openssl_sha512($module, /, string=b\'\')\n"
+"openssl_sha512($module, /, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Returns a sha512 hash object; optionally initialized with a string");
@@ -304,17 +394,19 @@ PyDoc_STRVAR(_hashlib_openssl_sha512__doc__,
     {"openssl_sha512", (PyCFunction)(void(*)(void))_hashlib_openssl_sha512, METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha512__doc__},
 
 static PyObject *
-_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj);
+_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj,
+                             int usedforsecurity);
 
 static PyObject *
 _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"string", NULL};
+    static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "openssl_sha512", 0};
-    PyObject *argsbuf[1];
+    PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     PyObject *data_obj = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
     if (!args) {
@@ -323,9 +415,22 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    data_obj = args[0];
+    if (args[0]) {
+        data_obj = args[0];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = _hashlib_openssl_sha512_impl(module, data_obj);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = _hashlib_openssl_sha512_impl(module, data_obj, usedforsecurity);
 
 exit:
     return return_value;
@@ -623,4 +728,4 @@ _hashlib_hmac_digest(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
 #ifndef _HASHLIB_SCRYPT_METHODDEF
     #define _HASHLIB_SCRYPT_METHODDEF
 #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */
-/*[clinic end generated code: output=38c2637f67e9bb79 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=acb22ccddb7043c7 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/md5module.c.h b/Modules/clinic/md5module.c.h
index 12484cc0e3dd..c109f9efec6b 100644
--- a/Modules/clinic/md5module.c.h
+++ b/Modules/clinic/md5module.c.h
@@ -66,7 +66,7 @@ PyDoc_STRVAR(MD5Type_update__doc__,
     {"update", (PyCFunction)MD5Type_update, METH_O, MD5Type_update__doc__},
 
 PyDoc_STRVAR(_md5_md5__doc__,
-"md5($module, /, string=b\'\')\n"
+"md5($module, /, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Return a new MD5 hash object; optionally initialized with a string.");
@@ -75,17 +75,18 @@ PyDoc_STRVAR(_md5_md5__doc__,
     {"md5", (PyCFunction)(void(*)(void))_md5_md5, METH_FASTCALL|METH_KEYWORDS, _md5_md5__doc__},
 
 static PyObject *
-_md5_md5_impl(PyObject *module, PyObject *string);
+_md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity);
 
 static PyObject *
 _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"string", NULL};
+    static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "md5", 0};
-    PyObject *argsbuf[1];
+    PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     PyObject *string = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
     if (!args) {
@@ -94,11 +95,24 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    string = args[0];
+    if (args[0]) {
+        string = args[0];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = _md5_md5_impl(module, string);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = _md5_md5_impl(module, string, usedforsecurity);
 
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=53133f08cf9095fc input=a9049054013a1b77]*/
+/*[clinic end generated code: output=dbe3abc60086f3ef input=a9049054013a1b77]*/
diff --git a/Modules/clinic/sha1module.c.h b/Modules/clinic/sha1module.c.h
index 001c6af73782..fc37b1ab880f 100644
--- a/Modules/clinic/sha1module.c.h
+++ b/Modules/clinic/sha1module.c.h
@@ -66,7 +66,7 @@ PyDoc_STRVAR(SHA1Type_update__doc__,
     {"update", (PyCFunction)SHA1Type_update, METH_O, SHA1Type_update__doc__},
 
 PyDoc_STRVAR(_sha1_sha1__doc__,
-"sha1($module, /, string=b\'\')\n"
+"sha1($module, /, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Return a new SHA1 hash object; optionally initialized with a string.");
@@ -75,17 +75,18 @@ PyDoc_STRVAR(_sha1_sha1__doc__,
     {"sha1", (PyCFunction)(void(*)(void))_sha1_sha1, METH_FASTCALL|METH_KEYWORDS, _sha1_sha1__doc__},
 
 static PyObject *
-_sha1_sha1_impl(PyObject *module, PyObject *string);
+_sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity);
 
 static PyObject *
 _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"string", NULL};
+    static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "sha1", 0};
-    PyObject *argsbuf[1];
+    PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     PyObject *string = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
     if (!args) {
@@ -94,11 +95,24 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    string = args[0];
+    if (args[0]) {
+        string = args[0];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = _sha1_sha1_impl(module, string);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = _sha1_sha1_impl(module, string, usedforsecurity);
 
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=1ae7e73ec84a27d5 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=3ddd637ae17e14b3 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/sha256module.c.h b/Modules/clinic/sha256module.c.h
index 658abb15cf30..2a788ea98499 100644
--- a/Modules/clinic/sha256module.c.h
+++ b/Modules/clinic/sha256module.c.h
@@ -66,7 +66,7 @@ PyDoc_STRVAR(SHA256Type_update__doc__,
     {"update", (PyCFunction)SHA256Type_update, METH_O, SHA256Type_update__doc__},
 
 PyDoc_STRVAR(_sha256_sha256__doc__,
-"sha256($module, /, string=b\'\')\n"
+"sha256($module, /, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Return a new SHA-256 hash object; optionally initialized with a string.");
@@ -75,17 +75,18 @@ PyDoc_STRVAR(_sha256_sha256__doc__,
     {"sha256", (PyCFunction)(void(*)(void))_sha256_sha256, METH_FASTCALL|METH_KEYWORDS, _sha256_sha256__doc__},
 
 static PyObject *
-_sha256_sha256_impl(PyObject *module, PyObject *string);
+_sha256_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity);
 
 static PyObject *
 _sha256_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"string", NULL};
+    static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "sha256", 0};
-    PyObject *argsbuf[1];
+    PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     PyObject *string = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
     if (!args) {
@@ -94,16 +95,29 @@ _sha256_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    string = args[0];
+    if (args[0]) {
+        string = args[0];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = _sha256_sha256_impl(module, string);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = _sha256_sha256_impl(module, string, usedforsecurity);
 
 exit:
     return return_value;
 }
 
 PyDoc_STRVAR(_sha256_sha224__doc__,
-"sha224($module, /, string=b\'\')\n"
+"sha224($module, /, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Return a new SHA-224 hash object; optionally initialized with a string.");
@@ -112,17 +126,18 @@ PyDoc_STRVAR(_sha256_sha224__doc__,
     {"sha224", (PyCFunction)(void(*)(void))_sha256_sha224, METH_FASTCALL|METH_KEYWORDS, _sha256_sha224__doc__},
 
 static PyObject *
-_sha256_sha224_impl(PyObject *module, PyObject *string);
+_sha256_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity);
 
 static PyObject *
 _sha256_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"string", NULL};
+    static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "sha224", 0};
-    PyObject *argsbuf[1];
+    PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     PyObject *string = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
     if (!args) {
@@ -131,11 +146,24 @@ _sha256_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    string = args[0];
+    if (args[0]) {
+        string = args[0];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = _sha256_sha224_impl(module, string);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = _sha256_sha224_impl(module, string, usedforsecurity);
 
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=c54d0956ec88409d input=a9049054013a1b77]*/
+/*[clinic end generated code: output=c8cca8adbe72ec9a input=a9049054013a1b77]*/
diff --git a/Modules/clinic/sha512module.c.h b/Modules/clinic/sha512module.c.h
index 459a9341cfc5..b8185b62bb66 100644
--- a/Modules/clinic/sha512module.c.h
+++ b/Modules/clinic/sha512module.c.h
@@ -66,7 +66,7 @@ PyDoc_STRVAR(SHA512Type_update__doc__,
     {"update", (PyCFunction)SHA512Type_update, METH_O, SHA512Type_update__doc__},
 
 PyDoc_STRVAR(_sha512_sha512__doc__,
-"sha512($module, /, string=b\'\')\n"
+"sha512($module, /, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Return a new SHA-512 hash object; optionally initialized with a string.");
@@ -75,17 +75,18 @@ PyDoc_STRVAR(_sha512_sha512__doc__,
     {"sha512", (PyCFunction)(void(*)(void))_sha512_sha512, METH_FASTCALL|METH_KEYWORDS, _sha512_sha512__doc__},
 
 static PyObject *
-_sha512_sha512_impl(PyObject *module, PyObject *string);
+_sha512_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity);
 
 static PyObject *
 _sha512_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"string", NULL};
+    static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "sha512", 0};
-    PyObject *argsbuf[1];
+    PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     PyObject *string = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
     if (!args) {
@@ -94,16 +95,29 @@ _sha512_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    string = args[0];
+    if (args[0]) {
+        string = args[0];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = _sha512_sha512_impl(module, string);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = _sha512_sha512_impl(module, string, usedforsecurity);
 
 exit:
     return return_value;
 }
 
 PyDoc_STRVAR(_sha512_sha384__doc__,
-"sha384($module, /, string=b\'\')\n"
+"sha384($module, /, string=b\'\', *, usedforsecurity=True)\n"
 "--\n"
 "\n"
 "Return a new SHA-384 hash object; optionally initialized with a string.");
@@ -112,17 +126,18 @@ PyDoc_STRVAR(_sha512_sha384__doc__,
     {"sha384", (PyCFunction)(void(*)(void))_sha512_sha384, METH_FASTCALL|METH_KEYWORDS, _sha512_sha384__doc__},
 
 static PyObject *
-_sha512_sha384_impl(PyObject *module, PyObject *string);
+_sha512_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity);
 
 static PyObject *
 _sha512_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"string", NULL};
+    static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "sha384", 0};
-    PyObject *argsbuf[1];
+    PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     PyObject *string = NULL;
+    int usedforsecurity = 1;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
     if (!args) {
@@ -131,11 +146,24 @@ _sha512_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje
     if (!noptargs) {
         goto skip_optional_pos;
     }
-    string = args[0];
+    if (args[0]) {
+        string = args[0];
+        if (!--noptargs) {
+            goto skip_optional_pos;
+        }
+    }
 skip_optional_pos:
-    return_value = _sha512_sha384_impl(module, string);
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    usedforsecurity = PyObject_IsTrue(args[1]);
+    if (usedforsecurity < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = _sha512_sha384_impl(module, string, usedforsecurity);
 
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=580df4b667084a7e input=a9049054013a1b77]*/
+/*[clinic end generated code: output=bbfa72d8703c82b5 input=a9049054013a1b77]*/
diff --git a/Modules/md5module.c b/Modules/md5module.c
index b9a351a8c1cd..f2c2d32cbe7d 100644
--- a/Modules/md5module.c
+++ b/Modules/md5module.c
@@ -503,13 +503,15 @@ static PyTypeObject MD5type = {
 _md5.md5
 
     string: object(c_default="NULL") = b''
+    *
+    usedforsecurity: bool = True
 
 Return a new MD5 hash object; optionally initialized with a string.
 [clinic start generated code]*/
 
 static PyObject *
-_md5_md5_impl(PyObject *module, PyObject *string)
-/*[clinic end generated code: output=2cfd0f8c091b97e6 input=d12ef8f72d684f7b]*/
+_md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity)
+/*[clinic end generated code: output=587071f76254a4ac input=7a144a1905636985]*/
 {
     MD5object *new;
     Py_buffer buf;
diff --git a/Modules/sha1module.c b/Modules/sha1module.c
index ce2ad267e775..4d191c3c4888 100644
--- a/Modules/sha1module.c
+++ b/Modules/sha1module.c
@@ -480,13 +480,15 @@ static PyTypeObject SHA1type = {
 _sha1.sha1
 
     string: object(c_default="NULL") = b''
+    *
+    usedforsecurity: bool = True
 
 Return a new SHA1 hash object; optionally initialized with a string.
 [clinic start generated code]*/
 
 static PyObject *
-_sha1_sha1_impl(PyObject *module, PyObject *string)
-/*[clinic end generated code: output=e5982830d1dece51 input=27ea54281d995ec2]*/
+_sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity)
+/*[clinic end generated code: output=6f8b3af05126e18e input=bd54b68e2bf36a8a]*/
 {
     SHA1object *new;
     Py_buffer buf;
diff --git a/Modules/sha256module.c b/Modules/sha256module.c
index b8d6c4cf8006..245f4c04542a 100644
--- a/Modules/sha256module.c
+++ b/Modules/sha256module.c
@@ -601,13 +601,15 @@ static PyTypeObject SHA256type = {
 _sha256.sha256
 
     string: object(c_default="NULL") = b''
+    *
+    usedforsecurity: bool = True
 
 Return a new SHA-256 hash object; optionally initialized with a string.
 [clinic start generated code]*/
 
 static PyObject *
-_sha256_sha256_impl(PyObject *module, PyObject *string)
-/*[clinic end generated code: output=fa644436dcea5c31 input=09cce3fb855056b2]*/
+_sha256_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity)
+/*[clinic end generated code: output=a1de327e8e1185cf input=9be86301aeb14ea5]*/
 {
     SHAobject *new;
     Py_buffer buf;
@@ -641,13 +643,15 @@ _sha256_sha256_impl(PyObject *module, PyObject *string)
 _sha256.sha224
 
     string: object(c_default="NULL") = b''
+    *
+    usedforsecurity: bool = True
 
 Return a new SHA-224 hash object; optionally initialized with a string.
 [clinic start generated code]*/
 
 static PyObject *
-_sha256_sha224_impl(PyObject *module, PyObject *string)
-/*[clinic end generated code: output=21e3ba22c3404f93 input=27a04ba24c353a73]*/
+_sha256_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity)
+/*[clinic end generated code: output=08be6b36569bc69c input=9fcfb46e460860ac]*/
 {
     SHAobject *new;
     Py_buffer buf;
diff --git a/Modules/sha512module.c b/Modules/sha512module.c
index 98b97917f4ca..df4f9d2d7415 100644
--- a/Modules/sha512module.c
+++ b/Modules/sha512module.c
@@ -666,13 +666,15 @@ static PyTypeObject SHA512type = {
 _sha512.sha512
 
     string: object(c_default="NULL") = b''
+    *
+    usedforsecurity: bool = True
 
 Return a new SHA-512 hash object; optionally initialized with a string.
 [clinic start generated code]*/
 
 static PyObject *
-_sha512_sha512_impl(PyObject *module, PyObject *string)
-/*[clinic end generated code: output=8b865a2df73bd387 input=e69bad9ae9b6a308]*/
+_sha512_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity)
+/*[clinic end generated code: output=a8d9e5f9e6a0831c input=23b4daebc2ebb9c9]*/
 {
     SHAobject *new;
     Py_buffer buf;
@@ -706,13 +708,15 @@ _sha512_sha512_impl(PyObject *module, PyObject *string)
 _sha512.sha384
 
     string: object(c_default="NULL") = b''
+    *
+    usedforsecurity: bool = True
 
 Return a new SHA-384 hash object; optionally initialized with a string.
 [clinic start generated code]*/
 
 static PyObject *
-_sha512_sha384_impl(PyObject *module, PyObject *string)
-/*[clinic end generated code: output=ae4b2e26decf81e8 input=c9327788d4ea4545]*/
+_sha512_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity)
+/*[clinic end generated code: output=da7d594a08027ac3 input=59ef72f039a6b431]*/
 {
     SHAobject *new;
     Py_buffer buf;


From webhook-mailer at python.org  Fri Sep 13 05:25:36 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Fri, 13 Sep 2019 09:25:36 -0000
Subject: [Python-checkins] bpo-37785: Fix xgettext warning in argparse
 (GH-15161)
Message-ID: <mailman.695.1568366736.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/42671aea2db6cbc54369617da0fd3545048e0a45
commit: 42671aea2db6cbc54369617da0fd3545048e0a45
branch: master
author: Jakub Kul?k <Kulikjak at gmail.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-13T11:25:32+02:00
summary:

bpo-37785: Fix xgettext warning in argparse (GH-15161)

files:
A Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst
M Lib/argparse.py

diff --git a/Lib/argparse.py b/Lib/argparse.py
index 2e46e76a4997..370692bd1bf4 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -1214,8 +1214,9 @@ def __call__(self, string):
             return open(string, self._mode, self._bufsize, self._encoding,
                         self._errors)
         except OSError as e:
-            message = _("can't open '%s': %s")
-            raise ArgumentTypeError(message % (string, e))
+            args = {'filename': string, 'error': e}
+            message = _("can't open '%(filename)s': %(error)s")
+            raise ArgumentTypeError(message % args)
 
     def __repr__(self):
         args = self._mode, self._bufsize
diff --git a/Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst b/Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst
new file mode 100644
index 000000000000..2d3aa4f9a75a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst
@@ -0,0 +1 @@
+Fix xgettext warnings in :mod:`argparse`.


From webhook-mailer at python.org  Fri Sep 13 05:44:08 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 09:44:08 -0000
Subject: [Python-checkins] bpo-37785: Fix xgettext warning in argparse
 (GH-15161)
Message-ID: <mailman.696.1568367849.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b50eff65906f8e9b4597cb0128ea1729341346fc
commit: b50eff65906f8e9b4597cb0128ea1729341346fc
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T02:44:04-07:00
summary:

bpo-37785: Fix xgettext warning in argparse (GH-15161)

(cherry picked from commit 42671aea2db6cbc54369617da0fd3545048e0a45)

Co-authored-by: Jakub Kul?k <Kulikjak at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst
M Lib/argparse.py

diff --git a/Lib/argparse.py b/Lib/argparse.py
index 24af355b7441..35d77a88a247 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -1207,8 +1207,9 @@ def __call__(self, string):
             return open(string, self._mode, self._bufsize, self._encoding,
                         self._errors)
         except OSError as e:
-            message = _("can't open '%s': %s")
-            raise ArgumentTypeError(message % (string, e))
+            args = {'filename': string, 'error': e}
+            message = _("can't open '%(filename)s': %(error)s")
+            raise ArgumentTypeError(message % args)
 
     def __repr__(self):
         args = self._mode, self._bufsize
diff --git a/Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst b/Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst
new file mode 100644
index 000000000000..2d3aa4f9a75a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst
@@ -0,0 +1 @@
+Fix xgettext warnings in :mod:`argparse`.


From webhook-mailer at python.org  Fri Sep 13 05:45:31 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 09:45:31 -0000
Subject: [Python-checkins] bpo-37785: Fix xgettext warning in argparse
 (GH-15161)
Message-ID: <mailman.697.1568367932.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8750dfe09eaec24231e19962d17d171c323794ad
commit: 8750dfe09eaec24231e19962d17d171c323794ad
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T02:45:27-07:00
summary:

bpo-37785: Fix xgettext warning in argparse (GH-15161)

(cherry picked from commit 42671aea2db6cbc54369617da0fd3545048e0a45)

Co-authored-by: Jakub Kul?k <Kulikjak at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst
M Lib/argparse.py

diff --git a/Lib/argparse.py b/Lib/argparse.py
index 456ec8bb7fc8..e590225c8eb8 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -1214,8 +1214,9 @@ def __call__(self, string):
             return open(string, self._mode, self._bufsize, self._encoding,
                         self._errors)
         except OSError as e:
-            message = _("can't open '%s': %s")
-            raise ArgumentTypeError(message % (string, e))
+            args = {'filename': string, 'error': e}
+            message = _("can't open '%(filename)s': %(error)s")
+            raise ArgumentTypeError(message % args)
 
     def __repr__(self):
         args = self._mode, self._bufsize
diff --git a/Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst b/Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst
new file mode 100644
index 000000000000..2d3aa4f9a75a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst
@@ -0,0 +1 @@
+Fix xgettext warnings in :mod:`argparse`.


From webhook-mailer at python.org  Fri Sep 13 06:12:31 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Fri, 13 Sep 2019 10:12:31 -0000
Subject: [Python-checkins] bpo-38075: Port _randommodule.c to PEP-384
 (GH-15798)
Message-ID: <mailman.698.1568369553.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/04f0bbfbedf8d2bb69b012f853de6648b1a9f27f
commit: 04f0bbfbedf8d2bb69b012f853de6648b1a9f27f
branch: master
author: Dino Viehland <dinoviehland at fb.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-13T03:12:27-07:00
summary:

bpo-38075: Port _randommodule.c to PEP-384 (GH-15798)

- Migrate `Random_Type` to `PyType_FromSpec`
- To simulate an old use of `PyLong_Type.tp_as_number->nb_absolute`, I added
  code to the module init function to stash `int.__abs__` for later
  use. Ideally we'd use `PyType_GetSlot()` instead, but it doesn't currently
  work for static types in CPython, and implementing it just for this case
  doesn't seem worth it.
- Do exact check for long and dispatch to PyNumber_Absolute, use vector call when not exact.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-59-50.bpo-38075.N8OZKF.rst
M Modules/_randommodule.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-59-50.bpo-38075.N8OZKF.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-59-50.bpo-38075.N8OZKF.rst
new file mode 100644
index 000000000000..015951b96b67
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-09-15-59-50.bpo-38075.N8OZKF.rst	
@@ -0,0 +1 @@
+The random module is now PEP-384 compatible
\ No newline at end of file
diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c
index 4e9ac4073c77..8b0a0244bfad 100644
--- a/Modules/_randommodule.c
+++ b/Modules/_randommodule.c
@@ -79,15 +79,23 @@
 #define UPPER_MASK 0x80000000U  /* most significant w-r bits */
 #define LOWER_MASK 0x7fffffffU  /* least significant r bits */
 
+typedef struct {
+    PyObject *Random_Type;
+    PyObject *Long___abs__;
+} _randomstate;
+
+#define _randomstate(o) ((_randomstate *)PyModule_GetState(o))
+
+static struct PyModuleDef _randommodule;
+
+#define _randomstate_global _randomstate(PyState_FindModule(&_randommodule))
+
 typedef struct {
     PyObject_HEAD
     int index;
     uint32_t state[N];
 } RandomObject;
 
-static PyTypeObject Random_Type;
-
-#define RandomObject_Check(v)      (Py_TYPE(v) == &Random_Type)
 
 #include "clinic/_randommodule.c.h"
 
@@ -256,6 +264,7 @@ random_seed(RandomObject *self, PyObject *arg)
     uint32_t *key = NULL;
     size_t bits, keyused;
     int res;
+    PyObject *args[1];
 
     if (arg == NULL || arg == Py_None) {
        if (random_seed_urandom(self) < 0) {
@@ -272,10 +281,14 @@ random_seed(RandomObject *self, PyObject *arg)
      * So: if the arg is a PyLong, use its absolute value.
      * Otherwise use its hash value, cast to unsigned.
      */
-    if (PyLong_Check(arg)) {
+    if (PyLong_CheckExact(arg)) {
+        n = PyNumber_Absolute(arg);
+    } else if (PyLong_Check(arg)) {
         /* Calling int.__abs__() prevents calling arg.__abs__(), which might
            return an invalid value. See issue #31478. */
-        n = PyLong_Type.tp_as_number->nb_absolute(arg);
+        args[0] = arg;
+        n = _PyObject_Vectorcall(_randomstate_global->Long___abs__, args, 0,
+                                         NULL);
     }
     else {
         Py_hash_t hash = PyObject_Hash(arg);
@@ -500,10 +513,12 @@ random_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     RandomObject *self;
     PyObject *tmp;
 
-    if (type == &Random_Type && !_PyArg_NoKeywords("Random", kwds))
+    if (type == (PyTypeObject*)_randomstate_global->Random_Type &&
+        !_PyArg_NoKeywords("Random()", kwds)) {
         return NULL;
+    }
 
-    self = (RandomObject *)type->tp_alloc(type, 0);
+    self = (RandomObject *)PyType_GenericAlloc(type, 0);
     if (self == NULL)
         return NULL;
     tmp = random_seed(self, args);
@@ -527,64 +542,55 @@ static PyMethodDef random_methods[] = {
 PyDoc_STRVAR(random_doc,
 "Random() -> create a random number generator with its own internal state.");
 
-static PyTypeObject Random_Type = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    "_random.Random",                   /*tp_name*/
-    sizeof(RandomObject),               /*tp_basicsize*/
-    0,                                  /*tp_itemsize*/
-    /* methods */
-    0,                                  /*tp_dealloc*/
-    0,                                  /*tp_vectorcall_offset*/
-    0,                                  /*tp_getattr*/
-    0,                                  /*tp_setattr*/
-    0,                                  /*tp_as_async*/
-    0,                                  /*tp_repr*/
-    0,                                  /*tp_as_number*/
-    0,                                  /*tp_as_sequence*/
-    0,                                  /*tp_as_mapping*/
-    0,                                  /*tp_hash*/
-    0,                                  /*tp_call*/
-    0,                                  /*tp_str*/
-    PyObject_GenericGetAttr,            /*tp_getattro*/
-    0,                                  /*tp_setattro*/
-    0,                                  /*tp_as_buffer*/
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,           /*tp_flags*/
-    random_doc,                         /*tp_doc*/
-    0,                                  /*tp_traverse*/
-    0,                                  /*tp_clear*/
-    0,                                  /*tp_richcompare*/
-    0,                                  /*tp_weaklistoffset*/
-    0,                                  /*tp_iter*/
-    0,                                  /*tp_iternext*/
-    random_methods,                     /*tp_methods*/
-    0,                                  /*tp_members*/
-    0,                                  /*tp_getset*/
-    0,                                  /*tp_base*/
-    0,                                  /*tp_dict*/
-    0,                                  /*tp_descr_get*/
-    0,                                  /*tp_descr_set*/
-    0,                                  /*tp_dictoffset*/
-    0,                                  /*tp_init*/
-    0,                                  /*tp_alloc*/
-    random_new,                         /*tp_new*/
-    PyObject_Free,                      /*tp_free*/
-    0,                                  /*tp_is_gc*/
+static PyType_Slot Random_Type_slots[] = {
+    {Py_tp_doc, random_doc},
+    {Py_tp_methods, random_methods},
+    {Py_tp_new, random_new},
+    {Py_tp_free, PyObject_Free},
+    {0, 0},
+};
+
+static PyType_Spec Random_Type_spec = {
+    "_random.Random",
+    sizeof(RandomObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    Random_Type_slots
 };
 
 PyDoc_STRVAR(module_doc,
 "Module implements the Mersenne Twister random number generator.");
 
+static int
+_random_traverse(PyObject *module, visitproc visit, void *arg)
+{
+    Py_VISIT(_randomstate(module)->Random_Type);
+    return 0;
+}
+
+static int
+_random_clear(PyObject *module)
+{
+    Py_CLEAR(_randomstate(module)->Random_Type);
+    return 0;
+}
+
+static void
+_random_free(void *module)
+{
+    _random_clear((PyObject *)module);
+}
 
 static struct PyModuleDef _randommodule = {
     PyModuleDef_HEAD_INIT,
     "_random",
     module_doc,
-    -1,
-    NULL,
+    sizeof(_randomstate),
     NULL,
     NULL,
-    NULL,
-    NULL
+    _random_traverse,
+    _random_clear,
+    _random_free,
 };
 
 PyMODINIT_FUNC
@@ -592,12 +598,41 @@ PyInit__random(void)
 {
     PyObject *m;
 
-    if (PyType_Ready(&Random_Type) < 0)
+    PyObject *Random_Type = PyType_FromSpec(&Random_Type_spec);
+    if (Random_Type == NULL) {
         return NULL;
+    }
+
     m = PyModule_Create(&_randommodule);
-    if (m == NULL)
+    if (m == NULL) {
+        Py_DECREF(Random_Type);
         return NULL;
-    Py_INCREF(&Random_Type);
-    PyModule_AddObject(m, "Random", (PyObject *)&Random_Type);
+    }
+    _randomstate(m)->Random_Type = Random_Type;
+
+    Py_INCREF(Random_Type);
+    PyModule_AddObject(m, "Random", Random_Type);
+
+    /* Look up and save int.__abs__, which is needed in random_seed(). */
+    PyObject *longval = NULL, *longtype = NULL;
+    longval = PyLong_FromLong(0);
+    if (longval == NULL) goto fail;
+
+    longtype = PyObject_Type(longval);
+    if (longtype == NULL) goto fail;
+
+    PyObject *abs = PyObject_GetAttrString(longtype, "__abs__");
+    if (abs == NULL) goto fail;
+
+    Py_DECREF(longtype);
+    Py_DECREF(longval);
+    _randomstate(m)->Long___abs__ = abs;
+
     return m;
+
+fail:
+    Py_XDECREF(longtype);
+    Py_XDECREF(longval);
+    Py_DECREF(m);
+    return NULL;
 }


From webhook-mailer at python.org  Fri Sep 13 06:20:18 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Fri, 13 Sep 2019 10:20:18 -0000
Subject: [Python-checkins] Revert "bpo-37785: Fix xgettext warning in
 argparse (GH-15161)" (GH-16082)
Message-ID: <mailman.699.1568370018.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f14fcbf2995b43c8607a1448402633ac558a745f
commit: f14fcbf2995b43c8607a1448402633ac558a745f
branch: 3.7
author: Julien Palard <julien at palard.fr>
committer: GitHub <noreply at github.com>
date: 2019-09-13T12:20:14+02:00
summary:

Revert "bpo-37785: Fix xgettext warning in argparse (GH-15161)" (GH-16082)

This reverts commit b50eff65906f8e9b4597cb0128ea1729341346fc because
it's an incompatible change that would have broken the existing
translations.

files:
D Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst
M Lib/argparse.py

diff --git a/Lib/argparse.py b/Lib/argparse.py
index 35d77a88a247..24af355b7441 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -1207,9 +1207,8 @@ def __call__(self, string):
             return open(string, self._mode, self._bufsize, self._encoding,
                         self._errors)
         except OSError as e:
-            args = {'filename': string, 'error': e}
-            message = _("can't open '%(filename)s': %(error)s")
-            raise ArgumentTypeError(message % args)
+            message = _("can't open '%s': %s")
+            raise ArgumentTypeError(message % (string, e))
 
     def __repr__(self):
         args = self._mode, self._bufsize
diff --git a/Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst b/Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst
deleted file mode 100644
index 2d3aa4f9a75a..000000000000
--- a/Misc/NEWS.d/next/Library/2019-08-07-14-49-22.bpo-37785.y7OlT8.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix xgettext warnings in :mod:`argparse`.


From webhook-mailer at python.org  Fri Sep 13 06:52:42 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 10:52:42 -0000
Subject: [Python-checkins] bpo-36889: Document Stream class and add
 docstrings (GH-14488)
Message-ID: <mailman.700.1568371963.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d31b31516c71890e8735606aec1dbf2bfb8fd6be
commit: d31b31516c71890e8735606aec1dbf2bfb8fd6be
branch: master
author: Xtreak <tir.karthi at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-13T03:52:38-07:00
summary:

bpo-36889: Document Stream class and add docstrings (GH-14488)



* This just copies the docs from `StreamWriter` and `StreamReader`.
* Add docstring for asyncio functions.


https://bugs.python.org/issue36889



Automerge-Triggered-By: @asvetlov

files:
M Doc/library/asyncio-api-index.rst
M Doc/library/asyncio-stream.rst
M Lib/asyncio/streams.py

diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst
index d5b5659abc65..716cf09dc99e 100644
--- a/Doc/library/asyncio-api-index.rst
+++ b/Doc/library/asyncio-api-index.rst
@@ -132,23 +132,47 @@ High-level APIs to work with network IO.
     :widths: 50 50
     :class: full-width-table
 
+    * - ``await`` :func:`connect`
+      -  Establish a TCP connection to send and receive data.
+
     * - ``await`` :func:`open_connection`
-      -  Establish a TCP connection.
+      -  Establish a TCP connection. (Deprecated in favor of :func:`connect`)
+
+    * - ``await`` :func:`connect_unix`
+      -  Establish a Unix socket connection to send and receive data.
 
     * - ``await`` :func:`open_unix_connection`
-      -  Establish a Unix socket connection.
+      -  Establish a Unix socket connection. (Deprecated in favor of :func:`connect_unix`)
 
-    * - ``await`` :func:`start_server`
+    * - :class:`StreamServer`
       - Start a TCP server.
 
-    * - ``await`` :func:`start_unix_server`
+    * - ``await`` :func:`start_server`
+      - Start a TCP server. (Deprecated in favor of :class:`StreamServer`)
+
+    * - :class:`UnixStreamServer`
       - Start a Unix socket server.
 
+    * - ``await`` :func:`start_unix_server`
+      - Start a Unix socket server. (Deprecated in favor of :class:`UnixStreamServer`)
+
+    * - :func:`connect_read_pipe`
+      - Establish a connection to :term:`file-like object <file object>` *pipe*
+        to receive data.
+
+    * - :func:`connect_write_pipe`
+      - Establish a connection to :term:`file-like object <file object>` *pipe*
+        to send data.
+
+    * - :class:`Stream`
+      - Stream is a single object combining APIs of :class:`StreamReader` and
+        :class:`StreamWriter`.
+
     * - :class:`StreamReader`
-      - High-level async/await object to receive network data.
+      - High-level async/await object to receive network data. (Deprecated in favor of :class:`Stream`)
 
     * - :class:`StreamWriter`
-      - High-level async/await object to send network data.
+      - High-level async/await object to send network data. (Deprecated in favor of :class:`Stream`)
 
 
 .. rubric:: Examples
diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst
index dfe520de56bf..feebd227eb8c 100644
--- a/Doc/library/asyncio-stream.rst
+++ b/Doc/library/asyncio-stream.rst
@@ -46,7 +46,6 @@ and work with streams:
    Connect to TCP socket on *host* : *port* address and return a :class:`Stream`
    object of mode :attr:`StreamMode.READWRITE`.
 
-
    *limit* determines the buffer size limit used by the returned :class:`Stream`
    instance. By default the *limit* is set to 64 KiB.
 
@@ -367,13 +366,184 @@ Stream
 
    Represents a Stream object that provides APIs to read and write data
    to the IO stream . It includes the API provided by :class:`StreamReader`
-   and :class:`StreamWriter`.
+   and :class:`StreamWriter`. It can also be used as :term:`asynchronous iterator`
+   where :meth:`readline` is used. It raises :exc:`StopAsyncIteration` when
+   :meth:`readline` returns empty data.
 
    Do not instantiate *Stream* objects directly; use API like :func:`connect`
    and :class:`StreamServer` instead.
 
    .. versionadded:: 3.8
 
+   .. attribute:: mode
+
+      Returns the mode of the stream which is a :class:`StreamMode` value. It could
+      be one of the below:
+
+      * :attr:`StreamMode.READ` - Connection can receive data.
+      * :attr:`StreamMode.WRITE` - Connection can send data.
+      * :attr:`StreamMode.READWRITE` - Connection can send and receive data.
+
+   .. coroutinemethod:: abort()
+
+      Aborts the connection immediately, without waiting for the send buffer to drain.
+
+   .. method:: at_eof()
+
+      Return ``True`` if the buffer is empty.
+
+   .. method:: can_write_eof()
+
+      Return *True* if the underlying transport supports
+      the :meth:`write_eof` method, *False* otherwise.
+
+   .. method:: close()
+
+      The method closes the stream and the underlying socket.
+
+      It is possible to directly await on the `close()` method::
+
+         await stream.close()
+
+      The ``await`` pauses the current coroutine until the stream and the underlying
+      socket are closed (and SSL shutdown is performed for a secure connection).
+
+   .. coroutinemethod:: drain()
+
+      Wait until it is appropriate to resume writing to the stream.
+      Example::
+
+          stream.write(data)
+          await stream.drain()
+
+      This is a flow control method that interacts with the underlying
+      IO write buffer.  When the size of the buffer reaches
+      the high watermark, *drain()* blocks until the size of the
+      buffer is drained down to the low watermark and writing can
+      be resumed.  When there is nothing to wait for, the :meth:`drain`
+      returns immediately.
+
+      .. deprecated:: 3.8
+
+      It is recommended to directly await on the `write()` method instead::
+
+         await stream.write(data)
+
+   .. method:: get_extra_info(name, default=None)
+
+      Access optional transport information; see
+      :meth:`BaseTransport.get_extra_info` for details.
+
+   .. method:: is_closing()
+
+      Return ``True`` if the stream is closed or in the process of
+      being closed.
+
+   .. coroutinemethod:: read(n=-1)
+
+      Read up to *n* bytes.  If *n* is not provided, or set to ``-1``,
+      read until EOF and return all read bytes.
+
+      If EOF was received and the internal buffer is empty,
+      return an empty ``bytes`` object.
+
+   .. coroutinemethod:: readexactly(n)
+
+      Read exactly *n* bytes.
+
+      Raise an :exc:`IncompleteReadError` if EOF is reached before *n*
+      can be read.  Use the :attr:`IncompleteReadError.partial`
+      attribute to get the partially read data.
+
+   .. coroutinemethod:: readline()
+
+      Read one line, where "line" is a sequence of bytes
+      ending with ``\n``.
+
+      If EOF is received and ``\n`` was not found, the method
+      returns partially read data.
+
+      If EOF is received and the internal buffer is empty,
+      return an empty ``bytes`` object.
+
+   .. coroutinemethod:: readuntil(separator=b'\\n')
+
+      Read data from the stream until *separator* is found.
+
+      On success, the data and separator will be removed from the
+      internal buffer (consumed). Returned data will include the
+      separator at the end.
+
+      If the amount of data read exceeds the configured stream limit, a
+      :exc:`LimitOverrunError` exception is raised, and the data
+      is left in the internal buffer and can be read again.
+
+      If EOF is reached before the complete separator is found,
+      an :exc:`IncompleteReadError` exception is raised, and the internal
+      buffer is reset.  The :attr:`IncompleteReadError.partial` attribute
+      may contain a portion of the separator.
+
+   .. coroutinemethod:: sendfile(file, offset=0, count=None, *, fallback=True)
+
+      Sends a *file* over the stream using an optimized syscall if available.
+
+      For other parameters meaning please see :meth:`AbstractEventloop.sendfile`.
+
+   .. coroutinemethod:: start_tls(sslcontext, *, server_hostname=None, \
+                                  ssl_handshake_timeout=None)
+
+      Upgrades the existing transport-based connection to TLS.
+
+      For other parameters meaning please see :meth:`AbstractEventloop.start_tls`.
+
+   .. coroutinemethod:: wait_closed()
+
+      Wait until the stream is closed.
+
+      Should be called after :meth:`close` to wait until the underlying
+      connection is closed.
+
+   .. coroutinemethod:: write(data)
+
+      Write *data* to the underlying socket; wait until the data is sent, e.g.::
+
+         await stream.write(data)
+
+   .. method:: write(data)
+
+      The method attempts to write the *data* to the underlying socket immediately.
+      If that fails, the data is queued in an internal write buffer until it can be
+      sent. :meth:`drain` can be used to flush the underlying buffer once writing is
+      available::
+
+         stream.write(data)
+         await stream.drain()
+
+      .. deprecated:: 3.8
+
+      It is recommended to directly await on the `write()` method instead::
+
+          await stream.write(data)
+
+   .. method:: writelines(data)
+
+      The method writes a list (or any iterable) of bytes to the underlying socket
+      immediately.
+      If that fails, the data is queued in an internal write buffer until it can be
+      sent.
+
+      It is possible to directly await on the `writelines()` method::
+
+         await stream.writelines(lines)
+
+      The ``await`` pauses the current coroutine until the data is written to the
+      socket.
+
+   .. method:: write_eof()
+
+      Close the write end of the stream after the buffered write
+      data is flushed.
+
 
 StreamMode
 ==========
@@ -459,8 +629,7 @@ StreamReader
 
    .. method:: at_eof()
 
-      Return ``True`` if the buffer is empty and :meth:`feed_eof`
-      was called.
+      Return ``True`` if the buffer is empty.
 
 
 StreamWriter
@@ -504,7 +673,7 @@ StreamWriter
       If that fails, the data is queued in an internal write buffer until it can be
       sent.
 
-      Starting with Python 3.8, it is possible to directly await on the `write()`
+      Starting with Python 3.8, it is possible to directly await on the `writelines()`
       method::
 
          await stream.writelines(lines)
diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py
index 4943e8e83e1e..b709dc11477e 100644
--- a/Lib/asyncio/streams.py
+++ b/Lib/asyncio/streams.py
@@ -70,6 +70,13 @@ def connect(host=None, port=None, *,
             server_hostname=None,
             ssl_handshake_timeout=None,
             happy_eyeballs_delay=None, interleave=None):
+    """Connect to TCP socket on *host* : *port* address to send and receive data.
+
+    *limit* determines the buffer size limit used by the returned `Stream`
+    instance. By default the *limit* is set to 64 KiB.
+
+    The rest of the arguments are passed directly to `loop.create_connection()`.
+    """
     # Design note:
     # Don't use decorator approach but explicit non-async
     # function to fail fast and explicitly
@@ -108,6 +115,13 @@ def connect(host=None, port=None, *,
 
 
 def connect_read_pipe(pipe, *, limit=_DEFAULT_LIMIT):
+    """Establish a connection to a file-like object *pipe* to receive data.
+
+    Takes a file-like object *pipe* to return a Stream object of the mode
+    StreamMode.READ that has similar API of StreamReader. It can also be used
+    as an async context manager.
+    """
+
     # Design note:
     # Don't use decorator approach but explicit non-async
     # function to fail fast and explicitly
@@ -129,6 +143,13 @@ def connect_read_pipe(pipe, *, limit=_DEFAULT_LIMIT):
 
 
 def connect_write_pipe(pipe, *, limit=_DEFAULT_LIMIT):
+    """Establish a connection to a file-like object *pipe* to send data.
+
+    Takes a file-like object *pipe* to return a Stream object of the mode
+    StreamMode.WRITE that has similar API of StreamWriter. It can also be used
+    as an async context manager.
+    """
+
     # Design note:
     # Don't use decorator approach but explicit non-async
     # function to fail fast and explicitly


From webhook-mailer at python.org  Fri Sep 13 06:53:26 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 10:53:26 -0000
Subject: [Python-checkins] Fix the ImportWarning regarding __spec__ and
 __package__ being None (GH-16003)
Message-ID: <mailman.701.1568372008.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/52c99aeb9f454b26baca61c545202057c7259bcf
commit: 52c99aeb9f454b26baca61c545202057c7259bcf
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T03:53:22-07:00
summary:

Fix the ImportWarning regarding __spec__ and __package__ being None (GH-16003)

(cherry picked from commit 6e1a30b15e73ebc82e6790495fd54cc8971723ec)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
M Lib/test/test_builtin.py

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 0bd4772dbb6e..9baac3cf3b95 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -161,9 +161,10 @@ def test_import(self):
         self.assertRaises(ValueError, __import__, '')
         self.assertRaises(TypeError, __import__, 'sys', name='sys')
         # Relative import outside of a package with no __package__ or __spec__ (bpo-37409).
-        self.assertRaises(ImportError, __import__, '',
-                          {'__package__': None, '__spec__': None, '__name__': '__main__'},
-                          locals={}, fromlist=('foo',), level=1)
+        with self.assertWarns(ImportWarning):
+            self.assertRaises(ImportError, __import__, '',
+                              {'__package__': None, '__spec__': None, '__name__': '__main__'},
+                              locals={}, fromlist=('foo',), level=1)
         # embedded null character
         self.assertRaises(ModuleNotFoundError, __import__, 'string\x00')
 


From webhook-mailer at python.org  Fri Sep 13 06:54:37 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 10:54:37 -0000
Subject: [Python-checkins] Fix the ImportWarning regarding __spec__ and
 __package__ being None (GH-16003)
Message-ID: <mailman.702.1568372079.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d3f8a11c06b7daa8df6ebf90ac711460e72b7a77
commit: d3f8a11c06b7daa8df6ebf90ac711460e72b7a77
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T03:54:34-07:00
summary:

Fix the ImportWarning regarding __spec__ and __package__ being None (GH-16003)

(cherry picked from commit 6e1a30b15e73ebc82e6790495fd54cc8971723ec)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
M Lib/test/test_builtin.py

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 5216da459f67..c60560ea6d3a 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -155,9 +155,10 @@ def test_import(self):
         self.assertRaises(ValueError, __import__, '')
         self.assertRaises(TypeError, __import__, 'sys', name='sys')
         # Relative import outside of a package with no __package__ or __spec__ (bpo-37409).
-        self.assertRaises(ImportError, __import__, '',
-                          {'__package__': None, '__spec__': None, '__name__': '__main__'},
-                          locals={}, fromlist=('foo',), level=1)
+        with self.assertWarns(ImportWarning):
+            self.assertRaises(ImportError, __import__, '',
+                              {'__package__': None, '__spec__': None, '__name__': '__main__'},
+                              locals={}, fromlist=('foo',), level=1)
         # embedded null character
         self.assertRaises(ModuleNotFoundError, __import__, 'string\x00')
 


From webhook-mailer at python.org  Fri Sep 13 07:23:52 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 11:23:52 -0000
Subject: [Python-checkins] bpo-36889: Document Stream class and add
 docstrings (GH-14488)
Message-ID: <mailman.703.1568373833.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b9bfe143d151d184615fa3890f78874c5d4ed4c6
commit: b9bfe143d151d184615fa3890f78874c5d4ed4c6
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T04:23:43-07:00
summary:

bpo-36889: Document Stream class and add docstrings (GH-14488)


* This just copies the docs from `StreamWriter` and `StreamReader`.
* Add docstring for asyncio functions.

https://bugs.python.org/issue36889

Automerge-Triggered-By: @asvetlov
(cherry picked from commit d31b31516c71890e8735606aec1dbf2bfb8fd6be)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
M Doc/library/asyncio-api-index.rst
M Doc/library/asyncio-stream.rst
M Lib/asyncio/streams.py

diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst
index d5b5659abc65..716cf09dc99e 100644
--- a/Doc/library/asyncio-api-index.rst
+++ b/Doc/library/asyncio-api-index.rst
@@ -132,23 +132,47 @@ High-level APIs to work with network IO.
     :widths: 50 50
     :class: full-width-table
 
+    * - ``await`` :func:`connect`
+      -  Establish a TCP connection to send and receive data.
+
     * - ``await`` :func:`open_connection`
-      -  Establish a TCP connection.
+      -  Establish a TCP connection. (Deprecated in favor of :func:`connect`)
+
+    * - ``await`` :func:`connect_unix`
+      -  Establish a Unix socket connection to send and receive data.
 
     * - ``await`` :func:`open_unix_connection`
-      -  Establish a Unix socket connection.
+      -  Establish a Unix socket connection. (Deprecated in favor of :func:`connect_unix`)
 
-    * - ``await`` :func:`start_server`
+    * - :class:`StreamServer`
       - Start a TCP server.
 
-    * - ``await`` :func:`start_unix_server`
+    * - ``await`` :func:`start_server`
+      - Start a TCP server. (Deprecated in favor of :class:`StreamServer`)
+
+    * - :class:`UnixStreamServer`
       - Start a Unix socket server.
 
+    * - ``await`` :func:`start_unix_server`
+      - Start a Unix socket server. (Deprecated in favor of :class:`UnixStreamServer`)
+
+    * - :func:`connect_read_pipe`
+      - Establish a connection to :term:`file-like object <file object>` *pipe*
+        to receive data.
+
+    * - :func:`connect_write_pipe`
+      - Establish a connection to :term:`file-like object <file object>` *pipe*
+        to send data.
+
+    * - :class:`Stream`
+      - Stream is a single object combining APIs of :class:`StreamReader` and
+        :class:`StreamWriter`.
+
     * - :class:`StreamReader`
-      - High-level async/await object to receive network data.
+      - High-level async/await object to receive network data. (Deprecated in favor of :class:`Stream`)
 
     * - :class:`StreamWriter`
-      - High-level async/await object to send network data.
+      - High-level async/await object to send network data. (Deprecated in favor of :class:`Stream`)
 
 
 .. rubric:: Examples
diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst
index dfe520de56bf..feebd227eb8c 100644
--- a/Doc/library/asyncio-stream.rst
+++ b/Doc/library/asyncio-stream.rst
@@ -46,7 +46,6 @@ and work with streams:
    Connect to TCP socket on *host* : *port* address and return a :class:`Stream`
    object of mode :attr:`StreamMode.READWRITE`.
 
-
    *limit* determines the buffer size limit used by the returned :class:`Stream`
    instance. By default the *limit* is set to 64 KiB.
 
@@ -367,13 +366,184 @@ Stream
 
    Represents a Stream object that provides APIs to read and write data
    to the IO stream . It includes the API provided by :class:`StreamReader`
-   and :class:`StreamWriter`.
+   and :class:`StreamWriter`. It can also be used as :term:`asynchronous iterator`
+   where :meth:`readline` is used. It raises :exc:`StopAsyncIteration` when
+   :meth:`readline` returns empty data.
 
    Do not instantiate *Stream* objects directly; use API like :func:`connect`
    and :class:`StreamServer` instead.
 
    .. versionadded:: 3.8
 
+   .. attribute:: mode
+
+      Returns the mode of the stream which is a :class:`StreamMode` value. It could
+      be one of the below:
+
+      * :attr:`StreamMode.READ` - Connection can receive data.
+      * :attr:`StreamMode.WRITE` - Connection can send data.
+      * :attr:`StreamMode.READWRITE` - Connection can send and receive data.
+
+   .. coroutinemethod:: abort()
+
+      Aborts the connection immediately, without waiting for the send buffer to drain.
+
+   .. method:: at_eof()
+
+      Return ``True`` if the buffer is empty.
+
+   .. method:: can_write_eof()
+
+      Return *True* if the underlying transport supports
+      the :meth:`write_eof` method, *False* otherwise.
+
+   .. method:: close()
+
+      The method closes the stream and the underlying socket.
+
+      It is possible to directly await on the `close()` method::
+
+         await stream.close()
+
+      The ``await`` pauses the current coroutine until the stream and the underlying
+      socket are closed (and SSL shutdown is performed for a secure connection).
+
+   .. coroutinemethod:: drain()
+
+      Wait until it is appropriate to resume writing to the stream.
+      Example::
+
+          stream.write(data)
+          await stream.drain()
+
+      This is a flow control method that interacts with the underlying
+      IO write buffer.  When the size of the buffer reaches
+      the high watermark, *drain()* blocks until the size of the
+      buffer is drained down to the low watermark and writing can
+      be resumed.  When there is nothing to wait for, the :meth:`drain`
+      returns immediately.
+
+      .. deprecated:: 3.8
+
+      It is recommended to directly await on the `write()` method instead::
+
+         await stream.write(data)
+
+   .. method:: get_extra_info(name, default=None)
+
+      Access optional transport information; see
+      :meth:`BaseTransport.get_extra_info` for details.
+
+   .. method:: is_closing()
+
+      Return ``True`` if the stream is closed or in the process of
+      being closed.
+
+   .. coroutinemethod:: read(n=-1)
+
+      Read up to *n* bytes.  If *n* is not provided, or set to ``-1``,
+      read until EOF and return all read bytes.
+
+      If EOF was received and the internal buffer is empty,
+      return an empty ``bytes`` object.
+
+   .. coroutinemethod:: readexactly(n)
+
+      Read exactly *n* bytes.
+
+      Raise an :exc:`IncompleteReadError` if EOF is reached before *n*
+      can be read.  Use the :attr:`IncompleteReadError.partial`
+      attribute to get the partially read data.
+
+   .. coroutinemethod:: readline()
+
+      Read one line, where "line" is a sequence of bytes
+      ending with ``\n``.
+
+      If EOF is received and ``\n`` was not found, the method
+      returns partially read data.
+
+      If EOF is received and the internal buffer is empty,
+      return an empty ``bytes`` object.
+
+   .. coroutinemethod:: readuntil(separator=b'\\n')
+
+      Read data from the stream until *separator* is found.
+
+      On success, the data and separator will be removed from the
+      internal buffer (consumed). Returned data will include the
+      separator at the end.
+
+      If the amount of data read exceeds the configured stream limit, a
+      :exc:`LimitOverrunError` exception is raised, and the data
+      is left in the internal buffer and can be read again.
+
+      If EOF is reached before the complete separator is found,
+      an :exc:`IncompleteReadError` exception is raised, and the internal
+      buffer is reset.  The :attr:`IncompleteReadError.partial` attribute
+      may contain a portion of the separator.
+
+   .. coroutinemethod:: sendfile(file, offset=0, count=None, *, fallback=True)
+
+      Sends a *file* over the stream using an optimized syscall if available.
+
+      For other parameters meaning please see :meth:`AbstractEventloop.sendfile`.
+
+   .. coroutinemethod:: start_tls(sslcontext, *, server_hostname=None, \
+                                  ssl_handshake_timeout=None)
+
+      Upgrades the existing transport-based connection to TLS.
+
+      For other parameters meaning please see :meth:`AbstractEventloop.start_tls`.
+
+   .. coroutinemethod:: wait_closed()
+
+      Wait until the stream is closed.
+
+      Should be called after :meth:`close` to wait until the underlying
+      connection is closed.
+
+   .. coroutinemethod:: write(data)
+
+      Write *data* to the underlying socket; wait until the data is sent, e.g.::
+
+         await stream.write(data)
+
+   .. method:: write(data)
+
+      The method attempts to write the *data* to the underlying socket immediately.
+      If that fails, the data is queued in an internal write buffer until it can be
+      sent. :meth:`drain` can be used to flush the underlying buffer once writing is
+      available::
+
+         stream.write(data)
+         await stream.drain()
+
+      .. deprecated:: 3.8
+
+      It is recommended to directly await on the `write()` method instead::
+
+          await stream.write(data)
+
+   .. method:: writelines(data)
+
+      The method writes a list (or any iterable) of bytes to the underlying socket
+      immediately.
+      If that fails, the data is queued in an internal write buffer until it can be
+      sent.
+
+      It is possible to directly await on the `writelines()` method::
+
+         await stream.writelines(lines)
+
+      The ``await`` pauses the current coroutine until the data is written to the
+      socket.
+
+   .. method:: write_eof()
+
+      Close the write end of the stream after the buffered write
+      data is flushed.
+
 
 StreamMode
 ==========
@@ -459,8 +629,7 @@ StreamReader
 
    .. method:: at_eof()
 
-      Return ``True`` if the buffer is empty and :meth:`feed_eof`
-      was called.
+      Return ``True`` if the buffer is empty.
 
 
 StreamWriter
@@ -504,7 +673,7 @@ StreamWriter
       If that fails, the data is queued in an internal write buffer until it can be
       sent.
 
-      Starting with Python 3.8, it is possible to directly await on the `write()`
+      Starting with Python 3.8, it is possible to directly await on the `writelines()`
       method::
 
          await stream.writelines(lines)
diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py
index 01aa6ffd49e4..20c642a001d2 100644
--- a/Lib/asyncio/streams.py
+++ b/Lib/asyncio/streams.py
@@ -70,6 +70,13 @@ def connect(host=None, port=None, *,
             server_hostname=None,
             ssl_handshake_timeout=None,
             happy_eyeballs_delay=None, interleave=None):
+    """Connect to TCP socket on *host* : *port* address to send and receive data.
+
+    *limit* determines the buffer size limit used by the returned `Stream`
+    instance. By default the *limit* is set to 64 KiB.
+
+    The rest of the arguments are passed directly to `loop.create_connection()`.
+    """
     # Design note:
     # Don't use decorator approach but explicit non-async
     # function to fail fast and explicitly
@@ -108,6 +115,13 @@ def connect(host=None, port=None, *,
 
 
 def connect_read_pipe(pipe, *, limit=_DEFAULT_LIMIT):
+    """Establish a connection to a file-like object *pipe* to receive data.
+
+    Takes a file-like object *pipe* to return a Stream object of the mode
+    StreamMode.READ that has similar API of StreamReader. It can also be used
+    as an async context manager.
+    """
+
     # Design note:
     # Don't use decorator approach but explicit non-async
     # function to fail fast and explicitly
@@ -129,6 +143,13 @@ def connect_read_pipe(pipe, *, limit=_DEFAULT_LIMIT):
 
 
 def connect_write_pipe(pipe, *, limit=_DEFAULT_LIMIT):
+    """Establish a connection to a file-like object *pipe* to send data.
+
+    Takes a file-like object *pipe* to return a Stream object of the mode
+    StreamMode.WRITE that has similar API of StreamWriter. It can also be used
+    as an async context manager.
+    """
+
     # Design note:
     # Don't use decorator approach but explicit non-async
     # function to fail fast and explicitly


From webhook-mailer at python.org  Fri Sep 13 07:29:04 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 11:29:04 -0000
Subject: [Python-checkins] bpo-12144: Handle cookies with expires attribute
 in CookieJar.make_cookies (GH-13921)
Message-ID: <mailman.704.1568374145.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bb41147eab15a2958f4ad38261e5bf608f6ace1b
commit: bb41147eab15a2958f4ad38261e5bf608f6ace1b
branch: master
author: Xtreak <tir.karthi at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-13T04:29:00-07:00
summary:

bpo-12144: Handle cookies with expires attribute in CookieJar.make_cookies (GH-13921)



Handle time comparison for cookies with `expires` attribute when `CookieJar.make_cookies` is called.

Co-authored-by: Demian Brecht <demianbrecht at gmail.com>


https://bugs.python.org/issue12144



Automerge-Triggered-By: @asvetlov

files:
A Misc/NEWS.d/next/Library/2019-06-08-23-26-58.bpo-12144.Z7mz-q.rst
M Lib/http/cookiejar.py
M Lib/test/test_http_cookiejar.py

diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py
index db8238235718..adc7ed62425d 100644
--- a/Lib/http/cookiejar.py
+++ b/Lib/http/cookiejar.py
@@ -1593,6 +1593,7 @@ def make_cookies(self, response, request):
         headers = response.info()
         rfc2965_hdrs = headers.get_all("Set-Cookie2", [])
         ns_hdrs = headers.get_all("Set-Cookie", [])
+        self._policy._now = self._now = int(time.time())
 
         rfc2965 = self._policy.rfc2965
         netscape = self._policy.netscape
@@ -1672,8 +1673,6 @@ def extract_cookies(self, response, request):
         _debug("extract_cookies: %s", response.info())
         self._cookies_lock.acquire()
         try:
-            self._policy._now = self._now = int(time.time())
-
             for cookie in self.make_cookies(response, request):
                 if self._policy.set_ok(cookie, request):
                     _debug(" setting cookie: %s", cookie)
diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py
index 22bf41cf1d91..853a40044965 100644
--- a/Lib/test/test_http_cookiejar.py
+++ b/Lib/test/test_http_cookiejar.py
@@ -585,6 +585,15 @@ def test_expires(self):
         # if expires is in future, keep cookie...
         c = CookieJar()
         future = time2netscape(time.time()+3600)
+
+        with test.support.check_no_warnings(self):
+            headers = [f"Set-Cookie: FOO=BAR; path=/; expires={future}"]
+            req = urllib.request.Request("http://www.coyote.com/")
+            res = FakeResponse(headers, "http://www.coyote.com/")
+            cookies = c.make_cookies(res, req)
+            self.assertEqual(len(cookies), 1)
+            self.assertEqual(time2netscape(cookies[0].expires), future)
+
         interact_netscape(c, "http://www.acme.com/", 'spam="bar"; expires=%s' %
                           future)
         self.assertEqual(len(c), 1)
diff --git a/Misc/NEWS.d/next/Library/2019-06-08-23-26-58.bpo-12144.Z7mz-q.rst b/Misc/NEWS.d/next/Library/2019-06-08-23-26-58.bpo-12144.Z7mz-q.rst
new file mode 100644
index 000000000000..ee802f81477e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-08-23-26-58.bpo-12144.Z7mz-q.rst
@@ -0,0 +1,2 @@
+Ensure cookies with ``expires`` attribute are handled in
+:meth:`CookieJar.make_cookies`.


From webhook-mailer at python.org  Fri Sep 13 07:47:55 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 11:47:55 -0000
Subject: [Python-checkins] bpo-12144: Handle cookies with expires attribute
 in CookieJar.make_cookies (GH-13921)
Message-ID: <mailman.705.1568375276.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/44cb89a78a308b7a613bdd01539ec84be914d693
commit: 44cb89a78a308b7a613bdd01539ec84be914d693
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T04:47:51-07:00
summary:

bpo-12144: Handle cookies with expires attribute in CookieJar.make_cookies (GH-13921)


Handle time comparison for cookies with `expires` attribute when `CookieJar.make_cookies` is called.

Co-authored-by: Demian Brecht <demianbrecht at gmail.com>

https://bugs.python.org/issue12144

Automerge-Triggered-By: @asvetlov
(cherry picked from commit bb41147eab15a2958f4ad38261e5bf608f6ace1b)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-06-08-23-26-58.bpo-12144.Z7mz-q.rst
M Lib/http/cookiejar.py
M Lib/test/test_http_cookiejar.py

diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py
index db8238235718..adc7ed62425d 100644
--- a/Lib/http/cookiejar.py
+++ b/Lib/http/cookiejar.py
@@ -1593,6 +1593,7 @@ def make_cookies(self, response, request):
         headers = response.info()
         rfc2965_hdrs = headers.get_all("Set-Cookie2", [])
         ns_hdrs = headers.get_all("Set-Cookie", [])
+        self._policy._now = self._now = int(time.time())
 
         rfc2965 = self._policy.rfc2965
         netscape = self._policy.netscape
@@ -1672,8 +1673,6 @@ def extract_cookies(self, response, request):
         _debug("extract_cookies: %s", response.info())
         self._cookies_lock.acquire()
         try:
-            self._policy._now = self._now = int(time.time())
-
             for cookie in self.make_cookies(response, request):
                 if self._policy.set_ok(cookie, request):
                     _debug(" setting cookie: %s", cookie)
diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py
index 22bf41cf1d91..853a40044965 100644
--- a/Lib/test/test_http_cookiejar.py
+++ b/Lib/test/test_http_cookiejar.py
@@ -585,6 +585,15 @@ def test_expires(self):
         # if expires is in future, keep cookie...
         c = CookieJar()
         future = time2netscape(time.time()+3600)
+
+        with test.support.check_no_warnings(self):
+            headers = [f"Set-Cookie: FOO=BAR; path=/; expires={future}"]
+            req = urllib.request.Request("http://www.coyote.com/")
+            res = FakeResponse(headers, "http://www.coyote.com/")
+            cookies = c.make_cookies(res, req)
+            self.assertEqual(len(cookies), 1)
+            self.assertEqual(time2netscape(cookies[0].expires), future)
+
         interact_netscape(c, "http://www.acme.com/", 'spam="bar"; expires=%s' %
                           future)
         self.assertEqual(len(c), 1)
diff --git a/Misc/NEWS.d/next/Library/2019-06-08-23-26-58.bpo-12144.Z7mz-q.rst b/Misc/NEWS.d/next/Library/2019-06-08-23-26-58.bpo-12144.Z7mz-q.rst
new file mode 100644
index 000000000000..ee802f81477e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-08-23-26-58.bpo-12144.Z7mz-q.rst
@@ -0,0 +1,2 @@
+Ensure cookies with ``expires`` attribute are handled in
+:meth:`CookieJar.make_cookies`.


From webhook-mailer at python.org  Fri Sep 13 08:18:52 2019
From: webhook-mailer at python.org (Andrew Svetlov)
Date: Fri, 13 Sep 2019 12:18:52 -0000
Subject: [Python-checkins] bpo-38148: Add slots to asyncio transports
 (GH-16077)
Message-ID: <mailman.706.1568377133.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9eb35ab0d71a6bd680e84fa0f828cb634e72b681
commit: 9eb35ab0d71a6bd680e84fa0f828cb634e72b681
branch: master
author: Andrew Svetlov <andrew.svetlov at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T15:18:46+03:00
summary:

bpo-38148: Add slots to asyncio transports (GH-16077)

* bpo-38148: Add slots to asyncio transports

* Update Misc/NEWS.d/next/Library/2019-09-13-08-55-43.bpo-38148.Lnww6D.rst

Co-Authored-By: Kyle Stanley <aeros167 at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-13-08-55-43.bpo-38148.Lnww6D.rst
M Lib/asyncio/transports.py
M Lib/test/test_asyncio/test_transports.py

diff --git a/Lib/asyncio/transports.py b/Lib/asyncio/transports.py
index 47b37fa9b7f0..513b1c024a4d 100644
--- a/Lib/asyncio/transports.py
+++ b/Lib/asyncio/transports.py
@@ -9,6 +9,8 @@
 class BaseTransport:
     """Base class for transports."""
 
+    __slots__ = ('_extra',)
+
     def __init__(self, extra=None):
         if extra is None:
             extra = {}
@@ -44,6 +46,8 @@ def get_protocol(self):
 class ReadTransport(BaseTransport):
     """Interface for read-only transports."""
 
+    __slots__ = ()
+
     def is_reading(self):
         """Return True if the transport is receiving."""
         raise NotImplementedError
@@ -68,6 +72,8 @@ def resume_reading(self):
 class WriteTransport(BaseTransport):
     """Interface for write-only transports."""
 
+    __slots__ = ()
+
     def set_write_buffer_limits(self, high=None, low=None):
         """Set the high- and low-water limits for write flow control.
 
@@ -154,10 +160,14 @@ class Transport(ReadTransport, WriteTransport):
     except writelines(), which calls write() in a loop.
     """
 
+    __slots__ = ()
+
 
 class DatagramTransport(BaseTransport):
     """Interface for datagram (UDP) transports."""
 
+    __slots__ = ()
+
     def sendto(self, data, addr=None):
         """Send data to the transport.
 
@@ -180,6 +190,8 @@ def abort(self):
 
 class SubprocessTransport(BaseTransport):
 
+    __slots__ = ()
+
     def get_pid(self):
         """Get subprocess id."""
         raise NotImplementedError
@@ -247,6 +259,8 @@ class _FlowControlMixin(Transport):
     resume_writing() may be called.
     """
 
+    __slots__ = ('_loop', '_protocol_paused', '_high_water', '_low_water')
+
     def __init__(self, extra=None, loop=None):
         super().__init__(extra)
         assert loop is not None
diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py
index 7de9b3e81a34..df448557a7b7 100644
--- a/Lib/test/test_asyncio/test_transports.py
+++ b/Lib/test/test_asyncio/test_transports.py
@@ -22,14 +22,19 @@ def test_get_extra_info(self):
         self.assertIs(default, transport.get_extra_info('unknown', default))
 
     def test_writelines(self):
-        transport = asyncio.Transport()
-        transport.write = mock.Mock()
+        writer = mock.Mock()
+
+        class MyTransport(asyncio.Transport):
+            def write(self, data):
+                writer(data)
+
+        transport = MyTransport()
 
         transport.writelines([b'line1',
                               bytearray(b'line2'),
                               memoryview(b'line3')])
-        self.assertEqual(1, transport.write.call_count)
-        transport.write.assert_called_with(b'line1line2line3')
+        self.assertEqual(1, writer.call_count)
+        writer.assert_called_with(b'line1line2line3')
 
     def test_not_implemented(self):
         transport = asyncio.Transport()
diff --git a/Misc/NEWS.d/next/Library/2019-09-13-08-55-43.bpo-38148.Lnww6D.rst b/Misc/NEWS.d/next/Library/2019-09-13-08-55-43.bpo-38148.Lnww6D.rst
new file mode 100644
index 000000000000..88c0809b1c2a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-13-08-55-43.bpo-38148.Lnww6D.rst
@@ -0,0 +1 @@
+Add slots to :mod:`asyncio` transport classes, which can reduce memory usage.


From webhook-mailer at python.org  Fri Sep 13 08:22:16 2019
From: webhook-mailer at python.org (Andrew Svetlov)
Date: Fri, 13 Sep 2019 12:22:16 -0000
Subject: [Python-checkins] [3.7] bpo-12144: Handle cookies with expires
 attribute in CookieJar.make_cookies (GH-13921) (GH-16092)
Message-ID: <mailman.707.1568377337.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e7b7edf5ebaed14dc68c841a8a98260f1330ef9a
commit: e7b7edf5ebaed14dc68c841a8a98260f1330ef9a
branch: 3.7
author: Xtreak <tir.karthi at gmail.com>
committer: Andrew Svetlov <andrew.svetlov at gmail.com>
date: 2019-09-13T15:22:12+03:00
summary:

 [3.7] bpo-12144: Handle cookies with expires attribute in CookieJar.make_cookies (GH-13921) (GH-16092)

* [3.7] bpo-12144: Handle cookies with expires attribute in CookieJar.make_cookies (GH-13921)

Handle time comparison for cookies with `expires` attribute when `CookieJar.make_cookies` is called.

Co-authored-by: Demian Brecht <demianbrecht at gmail.com>

https://bugs.python.org/issue12144

Automerge-Triggered-By: @asvetlov
(cherry picked from commit bb41147)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

* Use warnings module instead of test.support.check_no_warnings

* [3.7] bpo-12144: Handle cookies with expires attribute in CookieJar.make_cookies (GH-13921)

Handle time comparison for cookies with `expires` attribute when `CookieJar.make_cookies` is called.

Co-authored-by: Demian Brecht <demianbrecht at gmail.com>

https://bugs.python.org/issue12144

Automerge-Triggered-By: @asvetlov.
(cherry picked from commit bb41147eab15a2958f4ad38261e5bf608f6ace1b)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-06-08-23-26-58.bpo-12144.Z7mz-q.rst
M Lib/http/cookiejar.py
M Lib/test/test_http_cookiejar.py

diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py
index d63544a5f52e..1bed893b1e26 100644
--- a/Lib/http/cookiejar.py
+++ b/Lib/http/cookiejar.py
@@ -1590,6 +1590,7 @@ def make_cookies(self, response, request):
         headers = response.info()
         rfc2965_hdrs = headers.get_all("Set-Cookie2", [])
         ns_hdrs = headers.get_all("Set-Cookie", [])
+        self._policy._now = self._now = int(time.time())
 
         rfc2965 = self._policy.rfc2965
         netscape = self._policy.netscape
@@ -1669,8 +1670,6 @@ def extract_cookies(self, response, request):
         _debug("extract_cookies: %s", response.info())
         self._cookies_lock.acquire()
         try:
-            self._policy._now = self._now = int(time.time())
-
             for cookie in self.make_cookies(response, request):
                 if self._policy.set_ok(cookie, request):
                     _debug(" setting cookie: %s", cookie)
diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py
index 16edf34a9925..6eabeeae999d 100644
--- a/Lib/test/test_http_cookiejar.py
+++ b/Lib/test/test_http_cookiejar.py
@@ -6,6 +6,7 @@
 import time
 import unittest
 import urllib.request
+import warnings
 
 from http.cookiejar import (time2isoz, http2time, iso2time, time2netscape,
      parse_ns_headers, join_header_words, split_header_words, Cookie,
@@ -560,6 +561,16 @@ def test_expires(self):
         # if expires is in future, keep cookie...
         c = CookieJar()
         future = time2netscape(time.time()+3600)
+
+        with warnings.catch_warnings(record=True) as warns:
+            headers = [f"Set-Cookie: FOO=BAR; path=/; expires={future}"]
+            req = urllib.request.Request("http://www.coyote.com/")
+            res = FakeResponse(headers, "http://www.coyote.com/")
+            cookies = c.make_cookies(res, req)
+            self.assertEqual(len(cookies), 1)
+            self.assertEqual(time2netscape(cookies[0].expires), future)
+            self.assertEqual(len(warns), 0)
+
         interact_netscape(c, "http://www.acme.com/", 'spam="bar"; expires=%s' %
                           future)
         self.assertEqual(len(c), 1)
diff --git a/Misc/NEWS.d/next/Library/2019-06-08-23-26-58.bpo-12144.Z7mz-q.rst b/Misc/NEWS.d/next/Library/2019-06-08-23-26-58.bpo-12144.Z7mz-q.rst
new file mode 100644
index 000000000000..ee802f81477e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-08-23-26-58.bpo-12144.Z7mz-q.rst
@@ -0,0 +1,2 @@
+Ensure cookies with ``expires`` attribute are handled in
+:meth:`CookieJar.make_cookies`.


From webhook-mailer at python.org  Fri Sep 13 09:07:41 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Fri, 13 Sep 2019 13:07:41 -0000
Subject: [Python-checkins] bpo-29986: Doc: Delete tip to raise TypeError
 from tp_richcompare. (GH-16095)
Message-ID: <mailman.708.1568380062.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/375a3e2bdbeb4dce69aba4b5bc90f55fe27e81b4
commit: 375a3e2bdbeb4dce69aba4b5bc90f55fe27e81b4
branch: master
author: Julien Palard <julien at palard.fr>
committer: GitHub <noreply at github.com>
date: 2019-09-13T15:07:37+02:00
summary:

bpo-29986: Doc: Delete tip to raise TypeError from tp_richcompare. (GH-16095)

files:
M Doc/c-api/typeobj.rst

diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index f8e30a0380b9..7c7a79129ccc 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -1311,12 +1311,6 @@ and :c:type:`PyType_Type` effectively act as defaults.)
    ``Py_NotImplemented``, if another error occurred it must return *NULL* and
    set an exception condition.
 
-   .. note::
-
-      If you want to implement a type for which only a limited set of
-      comparisons makes sense (e.g. ``==`` and ``!=``, but not ``<`` and
-      friends), directly raise :exc:`TypeError` in the rich comparison function.
-
    The following constants are defined to be used as the third argument for
    :c:member:`~PyTypeObject.tp_richcompare` and for :c:func:`PyObject_RichCompare`:
 


From webhook-mailer at python.org  Fri Sep 13 09:07:52 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 13:07:52 -0000
Subject: [Python-checkins] Doc: Improve consistency of os.path.normcase with
 other os.path functions (GH-14004)
Message-ID: <mailman.709.1568380075.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/60d5e2c5a57aed4b5a8a82ae6e95fa67552e9fc9
commit: 60d5e2c5a57aed4b5a8a82ae6e95fa67552e9fc9
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T06:07:48-07:00
summary:

Doc: Improve consistency of os.path.normcase with other os.path functions (GH-14004)

(cherry picked from commit 53f78ec9e181f544b1a4575f32d42e296e05e4b4)

Co-authored-by: Kexuan Sun <me at kianasun.com>

files:
M Doc/library/os.path.rst

diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst
index 640ceecbc78f..a5abacf02144 100644
--- a/Doc/library/os.path.rst
+++ b/Doc/library/os.path.rst
@@ -328,8 +328,6 @@ the :mod:`glob` module.)
    Normalize the case of a pathname.  On Windows, convert all characters in the
    pathname to lowercase, and also convert forward slashes to backward slashes.
    On other operating systems, return the path unchanged.
-   Raise a :exc:`TypeError` if the type of *path* is not ``str`` or ``bytes`` (directly
-   or indirectly through the :class:`os.PathLike` interface).
 
    .. versionchanged:: 3.6
       Accepts a :term:`path-like object`.


From webhook-mailer at python.org  Fri Sep 13 09:14:46 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 13:14:46 -0000
Subject: [Python-checkins] bpo-29986: Doc: Delete tip to raise TypeError
 from tp_richcompare. (GH-16095)
Message-ID: <mailman.710.1568380487.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4556b1d35c352c975f3cf066362cb6e24efe0668
commit: 4556b1d35c352c975f3cf066362cb6e24efe0668
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T06:14:42-07:00
summary:

bpo-29986: Doc: Delete tip to raise TypeError from tp_richcompare. (GH-16095)

(cherry picked from commit 375a3e2bdbeb4dce69aba4b5bc90f55fe27e81b4)

Co-authored-by: Julien Palard <julien at palard.fr>

files:
M Doc/c-api/typeobj.rst

diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index f8e30a0380b9..7c7a79129ccc 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -1311,12 +1311,6 @@ and :c:type:`PyType_Type` effectively act as defaults.)
    ``Py_NotImplemented``, if another error occurred it must return *NULL* and
    set an exception condition.
 
-   .. note::
-
-      If you want to implement a type for which only a limited set of
-      comparisons makes sense (e.g. ``==`` and ``!=``, but not ``<`` and
-      friends), directly raise :exc:`TypeError` in the rich comparison function.
-
    The following constants are defined to be used as the third argument for
    :c:member:`~PyTypeObject.tp_richcompare` and for :c:func:`PyObject_RichCompare`:
 


From webhook-mailer at python.org  Fri Sep 13 09:14:59 2019
From: webhook-mailer at python.org (Andrew Svetlov)
Date: Fri, 13 Sep 2019 13:14:59 -0000
Subject: [Python-checkins] [3.8] bpo-38148: Add slots to asyncio transports
 (GH-16077) (GH-16093)
Message-ID: <mailman.711.1568380500.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6638c9226066205a646e18da95b33e619d784b0b
commit: 6638c9226066205a646e18da95b33e619d784b0b
branch: 3.8
author: Andrew Svetlov <andrew.svetlov at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T16:14:55+03:00
summary:

[3.8] bpo-38148: Add slots to asyncio transports (GH-16077) (GH-16093)

* bpo-38148: Add slots to asyncio transports

* Update Misc/NEWS.d/next/Library/2019-09-13-08-55-43.bpo-38148.Lnww6D.rst

Co-Authored-By: Kyle Stanley <aeros167 at gmail.com>
(cherry picked from commit 9eb35ab0d71a6bd680e84fa0f828cb634e72b681)

Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-13-08-55-43.bpo-38148.Lnww6D.rst
M Lib/asyncio/transports.py
M Lib/test/test_asyncio/test_transports.py

diff --git a/Lib/asyncio/transports.py b/Lib/asyncio/transports.py
index 47b37fa9b7f0..513b1c024a4d 100644
--- a/Lib/asyncio/transports.py
+++ b/Lib/asyncio/transports.py
@@ -9,6 +9,8 @@
 class BaseTransport:
     """Base class for transports."""
 
+    __slots__ = ('_extra',)
+
     def __init__(self, extra=None):
         if extra is None:
             extra = {}
@@ -44,6 +46,8 @@ def get_protocol(self):
 class ReadTransport(BaseTransport):
     """Interface for read-only transports."""
 
+    __slots__ = ()
+
     def is_reading(self):
         """Return True if the transport is receiving."""
         raise NotImplementedError
@@ -68,6 +72,8 @@ def resume_reading(self):
 class WriteTransport(BaseTransport):
     """Interface for write-only transports."""
 
+    __slots__ = ()
+
     def set_write_buffer_limits(self, high=None, low=None):
         """Set the high- and low-water limits for write flow control.
 
@@ -154,10 +160,14 @@ class Transport(ReadTransport, WriteTransport):
     except writelines(), which calls write() in a loop.
     """
 
+    __slots__ = ()
+
 
 class DatagramTransport(BaseTransport):
     """Interface for datagram (UDP) transports."""
 
+    __slots__ = ()
+
     def sendto(self, data, addr=None):
         """Send data to the transport.
 
@@ -180,6 +190,8 @@ def abort(self):
 
 class SubprocessTransport(BaseTransport):
 
+    __slots__ = ()
+
     def get_pid(self):
         """Get subprocess id."""
         raise NotImplementedError
@@ -247,6 +259,8 @@ class _FlowControlMixin(Transport):
     resume_writing() may be called.
     """
 
+    __slots__ = ('_loop', '_protocol_paused', '_high_water', '_low_water')
+
     def __init__(self, extra=None, loop=None):
         super().__init__(extra)
         assert loop is not None
diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py
index 7de9b3e81a34..df448557a7b7 100644
--- a/Lib/test/test_asyncio/test_transports.py
+++ b/Lib/test/test_asyncio/test_transports.py
@@ -22,14 +22,19 @@ def test_get_extra_info(self):
         self.assertIs(default, transport.get_extra_info('unknown', default))
 
     def test_writelines(self):
-        transport = asyncio.Transport()
-        transport.write = mock.Mock()
+        writer = mock.Mock()
+
+        class MyTransport(asyncio.Transport):
+            def write(self, data):
+                writer(data)
+
+        transport = MyTransport()
 
         transport.writelines([b'line1',
                               bytearray(b'line2'),
                               memoryview(b'line3')])
-        self.assertEqual(1, transport.write.call_count)
-        transport.write.assert_called_with(b'line1line2line3')
+        self.assertEqual(1, writer.call_count)
+        writer.assert_called_with(b'line1line2line3')
 
     def test_not_implemented(self):
         transport = asyncio.Transport()
diff --git a/Misc/NEWS.d/next/Library/2019-09-13-08-55-43.bpo-38148.Lnww6D.rst b/Misc/NEWS.d/next/Library/2019-09-13-08-55-43.bpo-38148.Lnww6D.rst
new file mode 100644
index 000000000000..88c0809b1c2a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-13-08-55-43.bpo-38148.Lnww6D.rst
@@ -0,0 +1 @@
+Add slots to :mod:`asyncio` transport classes, which can reduce memory usage.


From webhook-mailer at python.org  Fri Sep 13 09:31:24 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Fri, 13 Sep 2019 13:31:24 -0000
Subject: [Python-checkins] bpo-38153: Normalize hashlib algorithm names
 (GH-16083)
Message-ID: <mailman.712.1568381485.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/995b5d38e7cc24cac3de8dfd516115f86b0bcf80
commit: 995b5d38e7cc24cac3de8dfd516115f86b0bcf80
branch: master
author: Christian Heimes <christian at python.org>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-13T14:31:19+01:00
summary:

bpo-38153: Normalize hashlib algorithm names (GH-16083)

Signed-off-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Library/2019-09-13-12-18-51.bpo-38153.nHAbuJ.rst
M Lib/hashlib.py
M Lib/test/test_hashlib.py
M Modules/_hashopenssl.c

diff --git a/Lib/hashlib.py b/Lib/hashlib.py
index 4e783a86a34e..56873b7278b6 100644
--- a/Lib/hashlib.py
+++ b/Lib/hashlib.py
@@ -70,37 +70,44 @@
 
 __builtin_constructor_cache = {}
 
+__block_openssl_constructor = {
+    'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
+    'shake_128', 'shake_256',
+    'blake2b', 'blake2s',
+}
+
 def __get_builtin_constructor(name):
     cache = __builtin_constructor_cache
     constructor = cache.get(name)
     if constructor is not None:
         return constructor
     try:
-        if name in ('SHA1', 'sha1'):
+        if name in {'SHA1', 'sha1'}:
             import _sha1
             cache['SHA1'] = cache['sha1'] = _sha1.sha1
-        elif name in ('MD5', 'md5'):
+        elif name in {'MD5', 'md5'}:
             import _md5
             cache['MD5'] = cache['md5'] = _md5.md5
-        elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'):
+        elif name in {'SHA256', 'sha256', 'SHA224', 'sha224'}:
             import _sha256
             cache['SHA224'] = cache['sha224'] = _sha256.sha224
             cache['SHA256'] = cache['sha256'] = _sha256.sha256
-        elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'):
+        elif name in {'SHA512', 'sha512', 'SHA384', 'sha384'}:
             import _sha512
             cache['SHA384'] = cache['sha384'] = _sha512.sha384
             cache['SHA512'] = cache['sha512'] = _sha512.sha512
-        elif name in ('blake2b', 'blake2s'):
+        elif name in {'blake2b', 'blake2s'}:
             import _blake2
             cache['blake2b'] = _blake2.blake2b
             cache['blake2s'] = _blake2.blake2s
-        elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
-                      'shake_128', 'shake_256'}:
+        elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512'}:
             import _sha3
             cache['sha3_224'] = _sha3.sha3_224
             cache['sha3_256'] = _sha3.sha3_256
             cache['sha3_384'] = _sha3.sha3_384
             cache['sha3_512'] = _sha3.sha3_512
+        elif name in {'shake_128', 'shake_256'}:
+            import _sha3
             cache['shake_128'] = _sha3.shake_128
             cache['shake_256'] = _sha3.shake_256
     except ImportError:
@@ -114,8 +121,8 @@ def __get_builtin_constructor(name):
 
 
 def __get_openssl_constructor(name):
-    if name in {'blake2b', 'blake2s'}:
-        # Prefer our blake2 implementation.
+    if name in __block_openssl_constructor:
+        # Prefer our blake2 and sha3 implementation.
         return __get_builtin_constructor(name)
     try:
         f = getattr(_hashlib, 'openssl_' + name)
@@ -140,8 +147,8 @@ def __hash_new(name, data=b'', **kwargs):
     """new(name, data=b'') - Return a new hashing object using the named algorithm;
     optionally initialized with data (which must be a bytes-like object).
     """
-    if name in {'blake2b', 'blake2s'}:
-        # Prefer our blake2 implementation.
+    if name in __block_openssl_constructor:
+        # Prefer our blake2 and sha3 implementation
         # OpenSSL 1.1.0 comes with a limited implementation of blake2b/s.
         # It does neither support keyed blake2 nor advanced features like
         # salt, personal, tree hashing or SSE.
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index 46088e52dc54..9204b44bf4f9 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -26,6 +26,11 @@
 c_hashlib = import_fresh_module('hashlib', fresh=['_hashlib'])
 py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib'])
 
+try:
+    from _hashlib import HASH
+except ImportError:
+    HASH = None
+
 try:
     import _blake2
 except ImportError:
@@ -386,6 +391,9 @@ def check_sha3(self, name, capacity, rate, suffix):
         constructors = self.constructors_to_test[name]
         for hash_object_constructor in constructors:
             m = hash_object_constructor()
+            if HASH is not None and isinstance(m, HASH):
+                # _hashopenssl's variant does not have extra SHA3 attributes
+                continue
             self.assertEqual(capacity + rate, 1600)
             self.assertEqual(m._capacity_bits, capacity)
             self.assertEqual(m._rate_bits, rate)
@@ -985,6 +993,10 @@ def test_scrypt(self):
                 hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1,
                                dklen=dklen)
 
+    def test_normalized_name(self):
+        self.assertNotIn("blake2b512", hashlib.algorithms_available)
+        self.assertNotIn("sha3-512", hashlib.algorithms_available)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Library/2019-09-13-12-18-51.bpo-38153.nHAbuJ.rst b/Misc/NEWS.d/next/Library/2019-09-13-12-18-51.bpo-38153.nHAbuJ.rst
new file mode 100644
index 000000000000..8a483c760adc
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-13-12-18-51.bpo-38153.nHAbuJ.rst
@@ -0,0 +1,3 @@
+Names of hashing algorithms frome OpenSSL are now normalized to follow
+Python's naming conventions. For example OpenSSL uses sha3-512 instead of
+sha3_512 or blake2b512 instead of blake2b. 
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index c65c698cbba1..29ebec77a4b3 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -34,6 +34,14 @@
 
 #define MUNCH_SIZE INT_MAX
 
+#if defined(NID_sha3_224) && defined(EVP_MD_FLAG_XOF)
+#define PY_OPENSSL_HAS_SHA3 1
+#endif
+
+#ifdef NID_blake2b512
+#define PY_OPENSSL_HAS_BLAKE2 1
+#endif
+
 typedef struct {
     PyObject_HEAD
     EVP_MD_CTX          *ctx;   /* OpenSSL message digest context */
@@ -82,6 +90,135 @@ _setException(PyObject *exc)
 }
 /* LCOV_EXCL_STOP */
 
+static PyObject*
+py_digest_name(const EVP_MD *md)
+{
+    int nid = EVP_MD_nid(md);
+    const char *name = NULL;
+
+    /* Hard-coded names for well-known hashing algorithms.
+     * OpenSSL uses slightly different names algorithms like SHA3.
+     */
+    switch (nid) {
+    case NID_md5:
+        name = "md5";
+        break;
+    case NID_sha1:
+        name = "sha1";
+        break;
+    case NID_sha224:
+        name ="sha224";
+        break;
+    case NID_sha256:
+        name ="sha256";
+        break;
+    case NID_sha384:
+        name ="sha384";
+        break;
+    case NID_sha512:
+        name ="sha512";
+        break;
+#ifdef NID_sha512_224
+    case NID_sha512_224:
+        name ="sha512_224";
+        break;
+    case NID_sha512_256:
+        name ="sha512_256";
+        break;
+#endif
+#ifdef PY_OPENSSL_HAS_SHA3
+    case NID_sha3_224:
+        name ="sha3_224";
+        break;
+    case NID_sha3_256:
+        name ="sha3_256";
+        break;
+    case NID_sha3_384:
+        name ="sha3_384";
+        break;
+    case NID_sha3_512:
+        name ="sha3_512";
+        break;
+    case NID_shake128:
+        name ="shake_128";
+        break;
+    case NID_shake256:
+        name ="shake_256";
+        break;
+#endif
+#ifdef PY_OPENSSL_HAS_BLAKE2
+    case NID_blake2s256:
+        name ="blake2s";
+        break;
+    case NID_blake2b512:
+        name ="blake2b";
+        break;
+#endif
+    default:
+        /* Ignore aliased names and only use long, lowercase name. The aliases
+         * pollute the list and OpenSSL appears to have its own definition of
+         * alias as the resulting list still contains duplicate and alternate
+         * names for several algorithms.
+         */
+        name = OBJ_nid2ln(nid);
+        if (name == NULL)
+            name = OBJ_nid2sn(nid);
+        break;
+    }
+
+    return PyUnicode_FromString(name);
+}
+
+static const EVP_MD*
+py_digest_by_name(const char *name)
+{
+    const EVP_MD *digest = EVP_get_digestbyname(name);
+
+    /* OpenSSL uses dash instead of underscore in names of some algorithms
+     * like SHA3 and SHAKE. Detect different spellings. */
+    if (digest == NULL) {
+#ifdef NID_sha512_224
+        if (!strcmp(name, "sha512_224") || !strcmp(name, "SHA512_224")) {
+            digest = EVP_sha512_224();
+        }
+        else if (!strcmp(name, "sha512_256") || !strcmp(name, "SHA512_256")) {
+            digest = EVP_sha512_256();
+        }
+#endif
+#ifdef PY_OPENSSL_HAS_SHA3
+        /* could be sha3_ or shake_, Python never defined upper case */
+        else if (!strcmp(name, "sha3_224")) {
+            digest = EVP_sha3_224();
+        }
+        else if (!strcmp(name, "sha3_256")) {
+            digest = EVP_sha3_256();
+        }
+        else if (!strcmp(name, "sha3_384")) {
+            digest = EVP_sha3_384();
+        }
+        else if (!strcmp(name, "sha3_512")) {
+            digest = EVP_sha3_512();
+        }
+        else if (!strcmp(name, "shake_128")) {
+            digest = EVP_shake128();
+        }
+        else if (!strcmp(name, "shake_256")) {
+            digest = EVP_shake256();
+        }
+#endif
+#ifdef PY_OPENSSL_HAS_BLAKE2
+        else if (!strcmp(name, "blake2s256")) {
+            digest = EVP_blake2s256();
+        }
+        else if (!strcmp(name, "blake2b512")) {
+            digest = EVP_blake2b512();
+        }
+#endif
+    }
+
+    return digest;
+}
+
 static EVPobject *
 newEVPobject(void)
 {
@@ -304,16 +441,7 @@ EVP_get_digest_size(EVPobject *self, void *closure)
 static PyObject *
 EVP_get_name(EVPobject *self, void *closure)
 {
-    const char *name = EVP_MD_name(EVP_MD_CTX_md(self->ctx));
-    PyObject *name_obj, *name_lower;
-
-    name_obj = PyUnicode_FromString(name);
-    if (!name_obj) {
-        return NULL;
-    }
-    name_lower = PyObject_CallMethod(name_obj, "lower", NULL);
-    Py_DECREF(name_obj);
-    return name_lower;
+    return py_digest_name(EVP_MD_CTX_md(self->ctx));
 }
 
 static PyGetSetDef EVP_getseters[] = {
@@ -337,7 +465,7 @@ static PyObject *
 EVP_repr(EVPobject *self)
 {
     PyObject *name_obj, *repr;
-    name_obj = EVP_get_name(self, NULL);
+    name_obj = py_digest_name(EVP_MD_CTX_md(self->ctx));
     if (!name_obj) {
         return NULL;
     }
@@ -403,6 +531,7 @@ static PyTypeObject EVPtype = {
     0,                  /* tp_dictoffset */
 };
 
+\
 static PyObject *
 EVPnew(const EVP_MD *digest,
        const unsigned char *cp, Py_ssize_t len, int usedforsecurity)
@@ -485,7 +614,7 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj,
     if (data_obj)
         GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view);
 
-    digest = EVP_get_digestbyname(name);
+    digest = py_digest_by_name(name);
 
     ret_obj = EVPnew(digest,
                      (unsigned char*)view.buf, view.len,
@@ -922,21 +1051,17 @@ typedef struct _internal_name_mapper_state {
 
 /* A callback function to pass to OpenSSL's OBJ_NAME_do_all(...) */
 static void
-_openssl_hash_name_mapper(const OBJ_NAME *openssl_obj_name, void *arg)
+_openssl_hash_name_mapper(const EVP_MD *md, const char *from,
+                          const char *to, void *arg)
 {
     _InternalNameMapperState *state = (_InternalNameMapperState *)arg;
     PyObject *py_name;
 
     assert(state != NULL);
-    if (openssl_obj_name == NULL)
-        return;
-    /* Ignore aliased names, they pollute the list and OpenSSL appears to
-     * have its own definition of alias as the resulting list still
-     * contains duplicate and alternate names for several algorithms.     */
-    if (openssl_obj_name->alias)
+    if (md == NULL)
         return;
 
-    py_name = PyUnicode_FromString(openssl_obj_name->name);
+    py_name = py_digest_name(md);
     if (py_name == NULL) {
         state->error = 1;
     } else {
@@ -958,7 +1083,7 @@ generate_hash_name_list(void)
         return NULL;
     state.error = 0;
 
-    OBJ_NAME_do_all(OBJ_NAME_TYPE_MD_METH, &_openssl_hash_name_mapper, &state);
+    EVP_MD_do_all(&_openssl_hash_name_mapper, &state);
 
     if (state.error) {
         Py_DECREF(state.set);


From webhook-mailer at python.org  Fri Sep 13 09:36:29 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Fri, 13 Sep 2019 13:36:29 -0000
Subject: [Python-checkins] Adjust code ownership so @gpshead don't get
 asyncio subprocess PRs. (GH-16100)
Message-ID: <mailman.713.1568381792.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0765411d514788383d49f6b749b950707b0411d9
commit: 0765411d514788383d49f6b749b950707b0411d9
branch: master
author: Gregory P. Smith <greg at krypto.org>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-13T14:36:26+01:00
summary:

Adjust code ownership so @gpshead don't get asyncio subprocess PRs. (GH-16100)

files:
M .github/CODEOWNERS

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 8c3970babcc5..440a2b93f912 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -66,7 +66,9 @@ Include/pytime.h              @pganssle @abalkin
 **/*sqlite*                   @berkerpeksag
 
 # subprocess
-**/*subprocess*               @gpshead
+/Lib/subprocess.py            @gpshead
+/Lib/test/test_subprocess.py  @gpshead
+/Modules/*subprocess*         @gpshead
 
 # Windows
 /PC/                          @python/windows-team


From webhook-mailer at python.org  Fri Sep 13 09:38:11 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Fri, 13 Sep 2019 13:38:11 -0000
Subject: [Python-checkins] closes bpo-36002: Use AC_PATH_TOOL to find
 llvm-profdata and llvm-ar. (GH-14998)
Message-ID: <mailman.714.1568381892.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0519d497b04b252f50dfff9101fb7f4b9e33f1c4
commit: 0519d497b04b252f50dfff9101fb7f4b9e33f1c4
branch: master
author: Doyle Rowland <doyle.rowland at reliaqual.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-13T14:38:07+01:00
summary:

closes bpo-36002: Use AC_PATH_TOOL to find llvm-profdata and llvm-ar. (GH-14998)

files:
A Misc/NEWS.d/next/Build/2019-09-13-14-12-36.bpo-36002.Bcl4oe.rst
M configure.ac

diff --git a/Misc/NEWS.d/next/Build/2019-09-13-14-12-36.bpo-36002.Bcl4oe.rst b/Misc/NEWS.d/next/Build/2019-09-13-14-12-36.bpo-36002.Bcl4oe.rst
new file mode 100644
index 000000000000..374e7b234cfe
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-09-13-14-12-36.bpo-36002.Bcl4oe.rst
@@ -0,0 +1,2 @@
+Locate ``llvm-profdata`` and ``llvm-ar`` binaries using ``AC_PATH_TOOL``
+rather than ``AC_PATH_TARGET_TOOL``.
diff --git a/configure.ac b/configure.ac
index 5ca775cced09..69e2a5ea556c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1338,7 +1338,7 @@ if test "$Py_LTO" = 'true' ; then
   case $CC in
     *clang*)
       AC_SUBST(LLVM_AR)
-      AC_PATH_TARGET_TOOL(LLVM_AR, llvm-ar, '', ${llvm_path})
+      AC_PATH_TOOL(LLVM_AR, llvm-ar, '', ${llvm_path})
       AC_SUBST(LLVM_AR_FOUND)
       if test -n "${LLVM_AR}" -a -x "${LLVM_AR}"
       then
@@ -1404,7 +1404,7 @@ AC_SUBST(LLVM_PROF_MERGER)
 AC_SUBST(LLVM_PROF_FILE)
 AC_SUBST(LLVM_PROF_ERR)
 AC_SUBST(LLVM_PROFDATA)
-AC_PATH_TARGET_TOOL(LLVM_PROFDATA, llvm-profdata, '', ${llvm_path})
+AC_PATH_TOOL(LLVM_PROFDATA, llvm-profdata, '', ${llvm_path})
 AC_SUBST(LLVM_PROF_FOUND)
 if test -n "${LLVM_PROFDATA}" -a -x "${LLVM_PROFDATA}"
 then


From webhook-mailer at python.org  Fri Sep 13 09:43:38 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 13:43:38 -0000
Subject: [Python-checkins] bpo-26468: Doc: improve the documentation of
 shutil.copy2 when it can fail. (GH-13765)
Message-ID: <mailman.715.1568382219.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c27bcc3b476bbda7958724376189816c4e9d19c4
commit: c27bcc3b476bbda7958724376189816c4e9d19c4
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T06:43:34-07:00
summary:

bpo-26468: Doc: improve the documentation of shutil.copy2 when it can fail. (GH-13765)

(cherry picked from commit 9585f46b97931d2640c3343dfe03aed15beb9fea)

Co-authored-by: Windson yang <wiwindson at outlook.com>

files:
M Doc/library/shutil.rst

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index 88c5f62dd4c9..174b7e875a38 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -187,7 +187,8 @@ Directory and files operations
    However, this functionality is not available on all platforms.
    On platforms where some or all of this functionality is
    unavailable, :func:`copy2` will preserve all the metadata
-   it can; :func:`copy2` never returns failure.
+   it can; :func:`copy2` never raises an exception because it
+   cannot preserve file metadata.
 
    :func:`copy2` uses :func:`copystat` to copy the file metadata.
    Please see :func:`copystat` for more information


From webhook-mailer at python.org  Fri Sep 13 09:43:39 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Fri, 13 Sep 2019 13:43:39 -0000
Subject: [Python-checkins] bpo-36046: Fix buildbot failures (GH-16091)
Message-ID: <mailman.716.1568382220.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/693aa80a434590ea7dcd35c000209e53d01b9425
commit: 693aa80a434590ea7dcd35c000209e53d01b9425
branch: master
author: Gregory P. Smith <greg at krypto.org>
committer: T. Wouters <thomas at python.org>
date: 2019-09-13T14:43:35+01:00
summary:

bpo-36046: Fix buildbot failures (GH-16091)

Varying user/group/permission check needs on platforms.

files:
M Lib/test/test_subprocess.py

diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index f107022d86a0..42f376cda5d2 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -1588,6 +1588,18 @@ def test_run_with_shell_timeout_and_capture_output(self):
                         f"{stacks}```")
 
 
+def _get_test_grp_name():
+    for name_group in ('staff', 'nogroup', 'grp'):
+        if grp:
+            try:
+                grp.getgrnam(name_group)
+            except KeyError:
+                continue
+            return name_group
+    else:
+        raise unittest.SkipTest('No identified group name to use for this test on this platform.')
+
+
 @unittest.skipIf(mswindows, "POSIX specific tests")
 class POSIXProcessTestCase(BaseTestCase):
 
@@ -1762,8 +1774,10 @@ def test_user(self):
                             [sys.executable, "-c",
                              "import os; print(os.getuid())"],
                             user=user)
+                except PermissionError:  # errno.EACCES
+                    pass
                 except OSError as e:
-                    if e.errno != errno.EPERM:
+                    if e.errno not in (errno.EACCES, errno.EPERM):
                         raise
                 else:
                     if isinstance(user, str):
@@ -1789,7 +1803,7 @@ def test_user_error(self):
     def test_group(self):
         gid = os.getegid()
         group_list = [65534 if gid != 65534 else 65533]
-        name_group = "nogroup" if sys.platform != 'darwin' else "staff"
+        name_group = _get_test_grp_name()
 
         if grp is not None:
             group_list.append(name_group)
@@ -1830,7 +1844,7 @@ def test_group_error(self):
     def test_extra_groups(self):
         gid = os.getegid()
         group_list = [65534 if gid != 65534 else 65533]
-        name_group = "nogroup" if sys.platform != 'darwin' else "staff"
+        name_group = _get_test_grp_name()
         perm_error = False
 
         if grp is not None:


From webhook-mailer at python.org  Fri Sep 13 09:55:15 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 13:55:15 -0000
Subject: [Python-checkins] closes bpo-36002: Use AC_PATH_TOOL to find
 llvm-profdata and llvm-ar. (GH-14998)
Message-ID: <mailman.717.1568382917.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d112ea66b003deb7a4f222e47a552ac79098f4df
commit: d112ea66b003deb7a4f222e47a552ac79098f4df
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T06:55:11-07:00
summary:

closes bpo-36002: Use AC_PATH_TOOL to find llvm-profdata and llvm-ar. (GH-14998)

(cherry picked from commit 0519d497b04b252f50dfff9101fb7f4b9e33f1c4)

Co-authored-by: Doyle Rowland <doyle.rowland at reliaqual.com>

files:
A Misc/NEWS.d/next/Build/2019-09-13-14-12-36.bpo-36002.Bcl4oe.rst
M configure.ac

diff --git a/Misc/NEWS.d/next/Build/2019-09-13-14-12-36.bpo-36002.Bcl4oe.rst b/Misc/NEWS.d/next/Build/2019-09-13-14-12-36.bpo-36002.Bcl4oe.rst
new file mode 100644
index 000000000000..374e7b234cfe
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-09-13-14-12-36.bpo-36002.Bcl4oe.rst
@@ -0,0 +1,2 @@
+Locate ``llvm-profdata`` and ``llvm-ar`` binaries using ``AC_PATH_TOOL``
+rather than ``AC_PATH_TARGET_TOOL``.
diff --git a/configure.ac b/configure.ac
index 378d1bd17571..93f3b0d5a6be 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1339,7 +1339,7 @@ if test "$Py_LTO" = 'true' ; then
   case $CC in
     *clang*)
       AC_SUBST(LLVM_AR)
-      AC_PATH_TARGET_TOOL(LLVM_AR, llvm-ar, '', ${llvm_path})
+      AC_PATH_TOOL(LLVM_AR, llvm-ar, '', ${llvm_path})
       AC_SUBST(LLVM_AR_FOUND)
       if test -n "${LLVM_AR}" -a -x "${LLVM_AR}"
       then
@@ -1405,7 +1405,7 @@ AC_SUBST(LLVM_PROF_MERGER)
 AC_SUBST(LLVM_PROF_FILE)
 AC_SUBST(LLVM_PROF_ERR)
 AC_SUBST(LLVM_PROFDATA)
-AC_PATH_TARGET_TOOL(LLVM_PROFDATA, llvm-profdata, '', ${llvm_path})
+AC_PATH_TOOL(LLVM_PROFDATA, llvm-profdata, '', ${llvm_path})
 AC_SUBST(LLVM_PROF_FOUND)
 if test -n "${LLVM_PROFDATA}" -a -x "${LLVM_PROFDATA}"
 then


From webhook-mailer at python.org  Fri Sep 13 09:57:49 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 13:57:49 -0000
Subject: [Python-checkins] closes bpo-36002: Use AC_PATH_TOOL to find
 llvm-profdata and llvm-ar. (GH-14998)
Message-ID: <mailman.718.1568383071.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bd2e7cc3af26fdd8bb86009048533e2000c0a490
commit: bd2e7cc3af26fdd8bb86009048533e2000c0a490
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T06:57:45-07:00
summary:

closes bpo-36002: Use AC_PATH_TOOL to find llvm-profdata and llvm-ar. (GH-14998)

(cherry picked from commit 0519d497b04b252f50dfff9101fb7f4b9e33f1c4)

Co-authored-by: Doyle Rowland <doyle.rowland at reliaqual.com>

files:
A Misc/NEWS.d/next/Build/2019-09-13-14-12-36.bpo-36002.Bcl4oe.rst
M configure.ac

diff --git a/Misc/NEWS.d/next/Build/2019-09-13-14-12-36.bpo-36002.Bcl4oe.rst b/Misc/NEWS.d/next/Build/2019-09-13-14-12-36.bpo-36002.Bcl4oe.rst
new file mode 100644
index 000000000000..374e7b234cfe
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-09-13-14-12-36.bpo-36002.Bcl4oe.rst
@@ -0,0 +1,2 @@
+Locate ``llvm-profdata`` and ``llvm-ar`` binaries using ``AC_PATH_TOOL``
+rather than ``AC_PATH_TARGET_TOOL``.
diff --git a/configure.ac b/configure.ac
index f7e0d80f02e3..7b3b0b8be48b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1338,7 +1338,7 @@ if test "$Py_LTO" = 'true' ; then
   case $CC in
     *clang*)
       AC_SUBST(LLVM_AR)
-      AC_PATH_TARGET_TOOL(LLVM_AR, llvm-ar, '', ${llvm_path})
+      AC_PATH_TOOL(LLVM_AR, llvm-ar, '', ${llvm_path})
       AC_SUBST(LLVM_AR_FOUND)
       if test -n "${LLVM_AR}" -a -x "${LLVM_AR}"
       then
@@ -1404,7 +1404,7 @@ AC_SUBST(LLVM_PROF_MERGER)
 AC_SUBST(LLVM_PROF_FILE)
 AC_SUBST(LLVM_PROF_ERR)
 AC_SUBST(LLVM_PROFDATA)
-AC_PATH_TARGET_TOOL(LLVM_PROFDATA, llvm-profdata, '', ${llvm_path})
+AC_PATH_TOOL(LLVM_PROFDATA, llvm-profdata, '', ${llvm_path})
 AC_SUBST(LLVM_PROF_FOUND)
 if test -n "${LLVM_PROFDATA}" -a -x "${LLVM_PROFDATA}"
 then


From webhook-mailer at python.org  Fri Sep 13 10:26:04 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 14:26:04 -0000
Subject: [Python-checkins] bpo-25068: urllib.request.ProxyHandler now
 lowercases the dict keys (GH-13489)
Message-ID: <mailman.719.1568384764.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/590ed09a5b422d59cc1f049c64ac30733545eef0
commit: 590ed09a5b422d59cc1f049c64ac30733545eef0
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T07:25:51-07:00
summary:

bpo-25068: urllib.request.ProxyHandler now lowercases the dict keys (GH-13489)

(cherry picked from commit b761e3aed1fbada4572a776f6a0d3c4be491d595)

Co-authored-by: Zackery Spytz <zspytz at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-05-22-04-52-35.bpo-25068.vR_rC-.rst
M Lib/test/test_urllib2.py
M Lib/urllib/request.py

diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
index 810ee5e6ea28..e7e80bc5dabf 100644
--- a/Lib/test/test_urllib2.py
+++ b/Lib/test/test_urllib2.py
@@ -1340,21 +1340,22 @@ def http_open(self, req):
                 self.assertTrue(request.startswith(expected), repr(request))
 
     def test_proxy(self):
-        o = OpenerDirector()
-        ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
-        o.add_handler(ph)
-        meth_spec = [
-            [("http_open", "return response")]
-            ]
-        handlers = add_ordered_mock_handlers(o, meth_spec)
-
-        req = Request("http://acme.example.com/")
-        self.assertEqual(req.host, "acme.example.com")
-        o.open(req)
-        self.assertEqual(req.host, "proxy.example.com:3128")
-
-        self.assertEqual([(handlers[0], "http_open")],
-                         [tup[0:2] for tup in o.calls])
+        u = "proxy.example.com:3128"
+        for d in dict(http=u), dict(HTTP=u):
+            o = OpenerDirector()
+            ph = urllib.request.ProxyHandler(d)
+            o.add_handler(ph)
+            meth_spec = [
+                [("http_open", "return response")]
+                ]
+            handlers = add_ordered_mock_handlers(o, meth_spec)
+
+            req = Request("http://acme.example.com/")
+            self.assertEqual(req.host, "acme.example.com")
+            o.open(req)
+            self.assertEqual(req.host, u)
+            self.assertEqual([(handlers[0], "http_open")],
+                             [tup[0:2] for tup in o.calls])
 
     def test_proxy_no_proxy(self):
         os.environ['no_proxy'] = 'python.org'
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
index f6ce9cb6d586..92c6a5d01505 100644
--- a/Lib/urllib/request.py
+++ b/Lib/urllib/request.py
@@ -800,6 +800,7 @@ def __init__(self, proxies=None):
         assert hasattr(proxies, 'keys'), "proxies must be a mapping"
         self.proxies = proxies
         for type, url in proxies.items():
+            type = type.lower()
             setattr(self, '%s_open' % type,
                     lambda r, proxy=url, type=type, meth=self.proxy_open:
                         meth(r, proxy, type))
diff --git a/Misc/NEWS.d/next/Library/2019-05-22-04-52-35.bpo-25068.vR_rC-.rst b/Misc/NEWS.d/next/Library/2019-05-22-04-52-35.bpo-25068.vR_rC-.rst
new file mode 100644
index 000000000000..89de83bce029
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-22-04-52-35.bpo-25068.vR_rC-.rst
@@ -0,0 +1,2 @@
+:class:`urllib.request.ProxyHandler` now lowercases the keys of the passed
+dictionary.


From webhook-mailer at python.org  Fri Sep 13 10:30:44 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Fri, 13 Sep 2019 14:30:44 -0000
Subject: [Python-checkins] Run autoreconf. (GH-16106)
Message-ID: <mailman.720.1568385045.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f3095b0b58ad9d59cf221bb6498f5aa673c0b25a
commit: f3095b0b58ad9d59cf221bb6498f5aa673c0b25a
branch: master
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-13T15:30:36+01:00
summary:

Run autoreconf. (GH-16106)

files:
M configure

diff --git a/configure b/configure
index 25f9de9fbbb8..bb8fbc26c74e 100755
--- a/configure
+++ b/configure
@@ -681,10 +681,6 @@ LLVM_PROF_MERGER
 PGO_PROF_USE_FLAG
 PGO_PROF_GEN_FLAG
 LLVM_AR_FOUND
-target_os
-target_vendor
-target_cpu
-target
 LLVM_AR
 PROFILE_TASK
 DEF_MAKE_RULE
@@ -1473,7 +1469,6 @@ _ACEOF
 System types:
   --build=BUILD     configure for building on BUILD [guessed]
   --host=HOST       cross-compile to build programs to run on HOST [BUILD]
-  --target=TARGET   configure for building compilers for TARGET [HOST]
 _ACEOF
 fi
 
@@ -6498,47 +6493,9 @@ if test "$Py_LTO" = 'true' ; then
   case $CC in
     *clang*)
 
-      { $as_echo "$as_me:${as_lineno-$LINENO}: checking target system type" >&5
-$as_echo_n "checking target system type... " >&6; }
-if ${ac_cv_target+:} false; then :
-  $as_echo_n "(cached) " >&6
-else
-  if test "x$target_alias" = x; then
-  ac_cv_target=$ac_cv_host
-else
-  ac_cv_target=`$SHELL "$ac_aux_dir/config.sub" $target_alias` ||
-    as_fn_error $? "$SHELL $ac_aux_dir/config.sub $target_alias failed" "$LINENO" 5
-fi
-
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_target" >&5
-$as_echo "$ac_cv_target" >&6; }
-case $ac_cv_target in
-*-*-*) ;;
-*) as_fn_error $? "invalid value of canonical target" "$LINENO" 5;;
-esac
-target=$ac_cv_target
-ac_save_IFS=$IFS; IFS='-'
-set x $ac_cv_target
-shift
-target_cpu=$1
-target_vendor=$2
-shift; shift
-# Remember, the first character of IFS is used to create $*,
-# except with old shells:
-target_os=$*
-IFS=$ac_save_IFS
-case $target_os in *\ *) target_os=`echo "$target_os" | sed 's/ /-/g'`;; esac
-
-
-# The aliases save the names the user supplied, while $host etc.
-# will get canonicalized.
-test -n "$target_alias" &&
-  test "$program_prefix$program_suffix$program_transform_name" = \
-    NONENONEs,x,x, &&
-  program_prefix=${target_alias}-
-# Extract the first word of "$target_alias-llvm-ar", so it can be a program name with args.
-set dummy $target_alias-llvm-ar; ac_word=$2
+      if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}llvm-ar", so it can be a program name with args.
+set dummy ${ac_tool_prefix}llvm-ar; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
 if ${ac_cv_path_LLVM_AR+:} false; then :
@@ -6577,10 +6534,10 @@ $as_echo "no" >&6; }
 fi
 
 
+fi
 if test -z "$ac_cv_path_LLVM_AR"; then
-  if test "$build" = "$target"; then
-    ac_pt_LLVM_AR=$LLVM_AR
-    # Extract the first word of "llvm-ar", so it can be a program name with args.
+  ac_pt_LLVM_AR=$LLVM_AR
+  # Extract the first word of "llvm-ar", so it can be a program name with args.
 set dummy llvm-ar; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
@@ -6607,7 +6564,6 @@ done
   done
 IFS=$as_save_IFS
 
-  test -z "$ac_cv_path_ac_pt_LLVM_AR" && ac_cv_path_ac_pt_LLVM_AR="''"
   ;;
 esac
 fi
@@ -6620,9 +6576,16 @@ else
 $as_echo "no" >&6; }
 fi
 
-    LLVM_AR=$ac_pt_LLVM_AR
-  else
+  if test "x$ac_pt_LLVM_AR" = x; then
     LLVM_AR="''"
+  else
+    case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+    LLVM_AR=$ac_pt_LLVM_AR
   fi
 else
   LLVM_AR="$ac_cv_path_LLVM_AR"
@@ -6694,8 +6657,9 @@ fi
 
 
 
-# Extract the first word of "$target_alias-llvm-profdata", so it can be a program name with args.
-set dummy $target_alias-llvm-profdata; ac_word=$2
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}llvm-profdata", so it can be a program name with args.
+set dummy ${ac_tool_prefix}llvm-profdata; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
 if ${ac_cv_path_LLVM_PROFDATA+:} false; then :
@@ -6734,10 +6698,10 @@ $as_echo "no" >&6; }
 fi
 
 
+fi
 if test -z "$ac_cv_path_LLVM_PROFDATA"; then
-  if test "$build" = "$target"; then
-    ac_pt_LLVM_PROFDATA=$LLVM_PROFDATA
-    # Extract the first word of "llvm-profdata", so it can be a program name with args.
+  ac_pt_LLVM_PROFDATA=$LLVM_PROFDATA
+  # Extract the first word of "llvm-profdata", so it can be a program name with args.
 set dummy llvm-profdata; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
@@ -6764,7 +6728,6 @@ done
   done
 IFS=$as_save_IFS
 
-  test -z "$ac_cv_path_ac_pt_LLVM_PROFDATA" && ac_cv_path_ac_pt_LLVM_PROFDATA="''"
   ;;
 esac
 fi
@@ -6777,9 +6740,16 @@ else
 $as_echo "no" >&6; }
 fi
 
-    LLVM_PROFDATA=$ac_pt_LLVM_PROFDATA
-  else
+  if test "x$ac_pt_LLVM_PROFDATA" = x; then
     LLVM_PROFDATA="''"
+  else
+    case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+    LLVM_PROFDATA=$ac_pt_LLVM_PROFDATA
   fi
 else
   LLVM_PROFDATA="$ac_cv_path_LLVM_PROFDATA"


From webhook-mailer at python.org  Fri Sep 13 10:50:48 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 14:50:48 -0000
Subject: [Python-checkins] Run autoreconf. (GH-16106)
Message-ID: <mailman.721.1568386249.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/97d7ba4f22ed09b4e9b1d5690c6c77f4046c96c5
commit: 97d7ba4f22ed09b4e9b1d5690c6c77f4046c96c5
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T07:50:43-07:00
summary:

Run autoreconf. (GH-16106)

(cherry picked from commit f3095b0b58ad9d59cf221bb6498f5aa673c0b25a)

Co-authored-by: Benjamin Peterson <benjamin at python.org>

files:
M configure

diff --git a/configure b/configure
index 4beaf7a9c5e8..379ac6408eab 100755
--- a/configure
+++ b/configure
@@ -681,10 +681,6 @@ LLVM_PROF_MERGER
 PGO_PROF_USE_FLAG
 PGO_PROF_GEN_FLAG
 LLVM_AR_FOUND
-target_os
-target_vendor
-target_cpu
-target
 LLVM_AR
 PROFILE_TASK
 DEF_MAKE_RULE
@@ -1473,7 +1469,6 @@ _ACEOF
 System types:
   --build=BUILD     configure for building on BUILD [guessed]
   --host=HOST       cross-compile to build programs to run on HOST [BUILD]
-  --target=TARGET   configure for building compilers for TARGET [HOST]
 _ACEOF
 fi
 
@@ -6498,47 +6493,9 @@ if test "$Py_LTO" = 'true' ; then
   case $CC in
     *clang*)
 
-      { $as_echo "$as_me:${as_lineno-$LINENO}: checking target system type" >&5
-$as_echo_n "checking target system type... " >&6; }
-if ${ac_cv_target+:} false; then :
-  $as_echo_n "(cached) " >&6
-else
-  if test "x$target_alias" = x; then
-  ac_cv_target=$ac_cv_host
-else
-  ac_cv_target=`$SHELL "$ac_aux_dir/config.sub" $target_alias` ||
-    as_fn_error $? "$SHELL $ac_aux_dir/config.sub $target_alias failed" "$LINENO" 5
-fi
-
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_target" >&5
-$as_echo "$ac_cv_target" >&6; }
-case $ac_cv_target in
-*-*-*) ;;
-*) as_fn_error $? "invalid value of canonical target" "$LINENO" 5;;
-esac
-target=$ac_cv_target
-ac_save_IFS=$IFS; IFS='-'
-set x $ac_cv_target
-shift
-target_cpu=$1
-target_vendor=$2
-shift; shift
-# Remember, the first character of IFS is used to create $*,
-# except with old shells:
-target_os=$*
-IFS=$ac_save_IFS
-case $target_os in *\ *) target_os=`echo "$target_os" | sed 's/ /-/g'`;; esac
-
-
-# The aliases save the names the user supplied, while $host etc.
-# will get canonicalized.
-test -n "$target_alias" &&
-  test "$program_prefix$program_suffix$program_transform_name" = \
-    NONENONEs,x,x, &&
-  program_prefix=${target_alias}-
-# Extract the first word of "$target_alias-llvm-ar", so it can be a program name with args.
-set dummy $target_alias-llvm-ar; ac_word=$2
+      if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}llvm-ar", so it can be a program name with args.
+set dummy ${ac_tool_prefix}llvm-ar; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
 if ${ac_cv_path_LLVM_AR+:} false; then :
@@ -6577,10 +6534,10 @@ $as_echo "no" >&6; }
 fi
 
 
+fi
 if test -z "$ac_cv_path_LLVM_AR"; then
-  if test "$build" = "$target"; then
-    ac_pt_LLVM_AR=$LLVM_AR
-    # Extract the first word of "llvm-ar", so it can be a program name with args.
+  ac_pt_LLVM_AR=$LLVM_AR
+  # Extract the first word of "llvm-ar", so it can be a program name with args.
 set dummy llvm-ar; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
@@ -6607,7 +6564,6 @@ done
   done
 IFS=$as_save_IFS
 
-  test -z "$ac_cv_path_ac_pt_LLVM_AR" && ac_cv_path_ac_pt_LLVM_AR="''"
   ;;
 esac
 fi
@@ -6620,9 +6576,16 @@ else
 $as_echo "no" >&6; }
 fi
 
-    LLVM_AR=$ac_pt_LLVM_AR
-  else
+  if test "x$ac_pt_LLVM_AR" = x; then
     LLVM_AR="''"
+  else
+    case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+    LLVM_AR=$ac_pt_LLVM_AR
   fi
 else
   LLVM_AR="$ac_cv_path_LLVM_AR"
@@ -6694,8 +6657,9 @@ fi
 
 
 
-# Extract the first word of "$target_alias-llvm-profdata", so it can be a program name with args.
-set dummy $target_alias-llvm-profdata; ac_word=$2
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}llvm-profdata", so it can be a program name with args.
+set dummy ${ac_tool_prefix}llvm-profdata; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
 if ${ac_cv_path_LLVM_PROFDATA+:} false; then :
@@ -6734,10 +6698,10 @@ $as_echo "no" >&6; }
 fi
 
 
+fi
 if test -z "$ac_cv_path_LLVM_PROFDATA"; then
-  if test "$build" = "$target"; then
-    ac_pt_LLVM_PROFDATA=$LLVM_PROFDATA
-    # Extract the first word of "llvm-profdata", so it can be a program name with args.
+  ac_pt_LLVM_PROFDATA=$LLVM_PROFDATA
+  # Extract the first word of "llvm-profdata", so it can be a program name with args.
 set dummy llvm-profdata; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
@@ -6764,7 +6728,6 @@ done
   done
 IFS=$as_save_IFS
 
-  test -z "$ac_cv_path_ac_pt_LLVM_PROFDATA" && ac_cv_path_ac_pt_LLVM_PROFDATA="''"
   ;;
 esac
 fi
@@ -6777,9 +6740,16 @@ else
 $as_echo "no" >&6; }
 fi
 
-    LLVM_PROFDATA=$ac_pt_LLVM_PROFDATA
-  else
+  if test "x$ac_pt_LLVM_PROFDATA" = x; then
     LLVM_PROFDATA="''"
+  else
+    case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+    LLVM_PROFDATA=$ac_pt_LLVM_PROFDATA
   fi
 else
   LLVM_PROFDATA="$ac_cv_path_LLVM_PROFDATA"


From webhook-mailer at python.org  Fri Sep 13 10:51:18 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 14:51:18 -0000
Subject: [Python-checkins] Run autoreconf. (GH-16106)
Message-ID: <mailman.722.1568386279.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0a5a1a1eb29bafcec696fb3132cbf6ab2991cb76
commit: 0a5a1a1eb29bafcec696fb3132cbf6ab2991cb76
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T07:51:15-07:00
summary:

Run autoreconf. (GH-16106)

(cherry picked from commit f3095b0b58ad9d59cf221bb6498f5aa673c0b25a)

Co-authored-by: Benjamin Peterson <benjamin at python.org>

files:
M configure

diff --git a/configure b/configure
index 2db11e6e8667..cfdbb6cfeda6 100755
--- a/configure
+++ b/configure
@@ -680,10 +680,6 @@ LLVM_PROF_MERGER
 PGO_PROF_USE_FLAG
 PGO_PROF_GEN_FLAG
 LLVM_AR_FOUND
-target_os
-target_vendor
-target_cpu
-target
 LLVM_AR
 DEF_MAKE_RULE
 DEF_MAKE_ALL_RULE
@@ -1459,7 +1455,6 @@ _ACEOF
 System types:
   --build=BUILD     configure for building on BUILD [guessed]
   --host=HOST       cross-compile to build programs to run on HOST [BUILD]
-  --target=TARGET   configure for building compilers for TARGET [HOST]
 _ACEOF
 fi
 
@@ -6486,47 +6481,9 @@ if test "$Py_LTO" = 'true' ; then
   case $CC in
     *clang*)
 
-      { $as_echo "$as_me:${as_lineno-$LINENO}: checking target system type" >&5
-$as_echo_n "checking target system type... " >&6; }
-if ${ac_cv_target+:} false; then :
-  $as_echo_n "(cached) " >&6
-else
-  if test "x$target_alias" = x; then
-  ac_cv_target=$ac_cv_host
-else
-  ac_cv_target=`$SHELL "$ac_aux_dir/config.sub" $target_alias` ||
-    as_fn_error $? "$SHELL $ac_aux_dir/config.sub $target_alias failed" "$LINENO" 5
-fi
-
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_target" >&5
-$as_echo "$ac_cv_target" >&6; }
-case $ac_cv_target in
-*-*-*) ;;
-*) as_fn_error $? "invalid value of canonical target" "$LINENO" 5;;
-esac
-target=$ac_cv_target
-ac_save_IFS=$IFS; IFS='-'
-set x $ac_cv_target
-shift
-target_cpu=$1
-target_vendor=$2
-shift; shift
-# Remember, the first character of IFS is used to create $*,
-# except with old shells:
-target_os=$*
-IFS=$ac_save_IFS
-case $target_os in *\ *) target_os=`echo "$target_os" | sed 's/ /-/g'`;; esac
-
-
-# The aliases save the names the user supplied, while $host etc.
-# will get canonicalized.
-test -n "$target_alias" &&
-  test "$program_prefix$program_suffix$program_transform_name" = \
-    NONENONEs,x,x, &&
-  program_prefix=${target_alias}-
-# Extract the first word of "$target_alias-llvm-ar", so it can be a program name with args.
-set dummy $target_alias-llvm-ar; ac_word=$2
+      if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}llvm-ar", so it can be a program name with args.
+set dummy ${ac_tool_prefix}llvm-ar; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
 if ${ac_cv_path_LLVM_AR+:} false; then :
@@ -6565,10 +6522,10 @@ $as_echo "no" >&6; }
 fi
 
 
+fi
 if test -z "$ac_cv_path_LLVM_AR"; then
-  if test "$build" = "$target"; then
-    ac_pt_LLVM_AR=$LLVM_AR
-    # Extract the first word of "llvm-ar", so it can be a program name with args.
+  ac_pt_LLVM_AR=$LLVM_AR
+  # Extract the first word of "llvm-ar", so it can be a program name with args.
 set dummy llvm-ar; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
@@ -6595,7 +6552,6 @@ done
   done
 IFS=$as_save_IFS
 
-  test -z "$ac_cv_path_ac_pt_LLVM_AR" && ac_cv_path_ac_pt_LLVM_AR="''"
   ;;
 esac
 fi
@@ -6608,9 +6564,16 @@ else
 $as_echo "no" >&6; }
 fi
 
-    LLVM_AR=$ac_pt_LLVM_AR
-  else
+  if test "x$ac_pt_LLVM_AR" = x; then
     LLVM_AR="''"
+  else
+    case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+    LLVM_AR=$ac_pt_LLVM_AR
   fi
 else
   LLVM_AR="$ac_cv_path_LLVM_AR"
@@ -6682,8 +6645,9 @@ fi
 
 
 
-# Extract the first word of "$target_alias-llvm-profdata", so it can be a program name with args.
-set dummy $target_alias-llvm-profdata; ac_word=$2
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}llvm-profdata", so it can be a program name with args.
+set dummy ${ac_tool_prefix}llvm-profdata; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
 if ${ac_cv_path_LLVM_PROFDATA+:} false; then :
@@ -6722,10 +6686,10 @@ $as_echo "no" >&6; }
 fi
 
 
+fi
 if test -z "$ac_cv_path_LLVM_PROFDATA"; then
-  if test "$build" = "$target"; then
-    ac_pt_LLVM_PROFDATA=$LLVM_PROFDATA
-    # Extract the first word of "llvm-profdata", so it can be a program name with args.
+  ac_pt_LLVM_PROFDATA=$LLVM_PROFDATA
+  # Extract the first word of "llvm-profdata", so it can be a program name with args.
 set dummy llvm-profdata; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
@@ -6752,7 +6716,6 @@ done
   done
 IFS=$as_save_IFS
 
-  test -z "$ac_cv_path_ac_pt_LLVM_PROFDATA" && ac_cv_path_ac_pt_LLVM_PROFDATA="''"
   ;;
 esac
 fi
@@ -6765,9 +6728,16 @@ else
 $as_echo "no" >&6; }
 fi
 
-    LLVM_PROFDATA=$ac_pt_LLVM_PROFDATA
-  else
+  if test "x$ac_pt_LLVM_PROFDATA" = x; then
     LLVM_PROFDATA="''"
+  else
+    case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+    LLVM_PROFDATA=$ac_pt_LLVM_PROFDATA
   fi
 else
   LLVM_PROFDATA="$ac_cv_path_LLVM_PROFDATA"


From webhook-mailer at python.org  Fri Sep 13 11:53:16 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Fri, 13 Sep 2019 15:53:16 -0000
Subject: [Python-checkins] bpo-37199: Replace the early returns added in
 c2cda63. (GH-14535)
Message-ID: <mailman.723.1568389997.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/81319a81b2a3f42fe1047c2e6f5fd958faab6cdb
commit: 81319a81b2a3f42fe1047c2e6f5fd958faab6cdb
branch: master
author: Zackery Spytz <zspytz at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-13T16:53:13+01:00
summary:

bpo-37199: Replace the early returns added in c2cda63. (GH-14535)

files:
M Lib/test/test_asyncio/test_base_events.py

diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
index 370e6a6dfa79..ccdd3bae7336 100644
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -91,28 +91,26 @@ def test_ipaddr_info(self):
         self.assertIsNone(
             base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, 0, 0))
 
-        if not support.IPV6_ENABLED:
-            return
+        if support.IPV6_ENABLED:
+            # IPv4 address with family IPv6.
+            self.assertIsNone(
+                base_events._ipaddr_info('1.2.3.4', 1, INET6, STREAM, TCP))
 
-        # IPv4 address with family IPv6.
-        self.assertIsNone(
-            base_events._ipaddr_info('1.2.3.4', 1, INET6, STREAM, TCP))
-
-        self.assertEqual(
-            (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)),
-            base_events._ipaddr_info('::3', 1, INET6, STREAM, TCP))
+            self.assertEqual(
+                (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)),
+                base_events._ipaddr_info('::3', 1, INET6, STREAM, TCP))
 
-        self.assertEqual(
-            (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)),
-            base_events._ipaddr_info('::3', 1, UNSPEC, STREAM, TCP))
+            self.assertEqual(
+                (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)),
+                base_events._ipaddr_info('::3', 1, UNSPEC, STREAM, TCP))
 
-        # IPv6 address with family IPv4.
-        self.assertIsNone(
-            base_events._ipaddr_info('::3', 1, INET, STREAM, TCP))
+            # IPv6 address with family IPv4.
+            self.assertIsNone(
+                base_events._ipaddr_info('::3', 1, INET, STREAM, TCP))
 
-        # IPv6 address with zone index.
-        self.assertIsNone(
-            base_events._ipaddr_info('::3%lo0', 1, INET6, STREAM, TCP))
+            # IPv6 address with zone index.
+            self.assertIsNone(
+                base_events._ipaddr_info('::3%lo0', 1, INET6, STREAM, TCP))
 
     def test_port_parameter_types(self):
         # Test obscure kinds of arguments for "port".
@@ -1284,25 +1282,24 @@ def _test_create_connection_ip_addr(self, m_socket, allow_inet_pton):
             t.close()
             test_utils.run_briefly(self.loop)  # allow transport to close
 
-        if not support.IPV6_ENABLED:
-            return
-
-        sock.family = socket.AF_INET6
-        coro = self.loop.create_connection(asyncio.Protocol, '::1', 80)
-        t, p = self.loop.run_until_complete(coro)
-        try:
-            # Without inet_pton we use getaddrinfo, which transforms ('::1', 80)
-            # to ('::1', 80, 0, 0). The last 0s are flow info, scope id.
-            [address] = sock.connect.call_args[0]
-            host, port = address[:2]
-            self.assertRegex(host, r'::(0\.)*1')
-            self.assertEqual(port, 80)
-            _, kwargs = m_socket.socket.call_args
-            self.assertEqual(kwargs['family'], m_socket.AF_INET6)
-            self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM)
-        finally:
-            t.close()
-            test_utils.run_briefly(self.loop)  # allow transport to close
+        if support.IPV6_ENABLED:
+            sock.family = socket.AF_INET6
+            coro = self.loop.create_connection(asyncio.Protocol, '::1', 80)
+            t, p = self.loop.run_until_complete(coro)
+            try:
+                # Without inet_pton we use getaddrinfo, which transforms
+                # ('::1', 80) to ('::1', 80, 0, 0). The last 0s are flow info,
+                # scope id.
+                [address] = sock.connect.call_args[0]
+                host, port = address[:2]
+                self.assertRegex(host, r'::(0\.)*1')
+                self.assertEqual(port, 80)
+                _, kwargs = m_socket.socket.call_args
+                self.assertEqual(kwargs['family'], m_socket.AF_INET6)
+                self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM)
+            finally:
+                t.close()
+                test_utils.run_briefly(self.loop)  # allow transport to close
 
     @unittest.skipUnless(support.IPV6_ENABLED, 'no IPv6 support')
     @unittest.skipIf(sys.platform.startswith('aix'),


From webhook-mailer at python.org  Fri Sep 13 11:54:36 2019
From: webhook-mailer at python.org (Paul Ganssle)
Date: Fri, 13 Sep 2019 15:54:36 -0000
Subject: [Python-checkins] bpo-37555: Update _CallList.__contains__ to
 respect ANY (#14700)
Message-ID: <mailman.724.1568390077.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d6a9d17d8b6c68073931dd8ffa213b4ac351a4ab
commit: d6a9d17d8b6c68073931dd8ffa213b4ac351a4ab
branch: master
author: Elizabeth Uselton <elizabeth.uselton at gmail.com>
committer: Paul Ganssle <paul at ganssle.io>
date: 2019-09-13T16:54:32+01:00
summary:

bpo-37555: Update _CallList.__contains__ to respect ANY (#14700)

* Flip equality to use mock calls' __eq__

* bpo-37555: Regression test demonstrating assert_has_calls not working with ANY and spec_set

Co-authored-by: Neal Finne <neal at nealfinne.com>

* Revert "Flip equality to use mock calls' __eq__"

This reverts commit 94ddf54c5a8aab7d00d9ab93e1cc5695c28d73e7.

* bpo-37555: Add regression tests for mock ANY ordering issues

Add regression tests for whether __eq__ is order agnostic on _Call and _CallList, which is useful for comparisons involving ANY, especially if the ANY comparison is to a class not defaulting __eq__ to NotImplemented.

Co-authored-by: Neal Finne <neal at nealfinne.com>

* bpo-37555: Fix _CallList and _Call order sensitivity

_Call and _CallList depend on ordering to correctly process that an object being compared to ANY with __eq__ should return True. This fix updates the comparison to check both a == b and b == a and return True if either condition is met, fixing situations from the tests in the previous two commits where assertEqual would not be commutative if checking _Call or _CallList objects. This seems like a reasonable fix considering that the Python data model specifies that if an object doesn't know how to compare itself to another object it should return NotImplemented, and that on getting NotImplemented from a == b, it should try b == a, implying that good behavior for __eq__ is commutative. This also flips the order of comparison in _CallList's __contains__ method, guaranteeing ANY will be on the left and have it's __eq__ called for equality checking, fixing the interaction between assert_has_calls and ANY.

Co-author: Neal Finne <neal at neal.finne.com>

* bpo-37555: Ensure _call_matcher returns _Call object

* Adding ACK and news entry

* bpo-37555: Replacing __eq__ with == to sidestep NotImplemented

bool(NotImplemented) returns True, so it's necessary to use ==
instead of __eq__ in this comparison.

* bpo-37555: cleaning up changes unnecessary to the final product

* bpo-37555: Fixed call on bound arguments to respect args and kwargs

* Revert "bpo-37555: Add regression tests for mock ANY ordering issues"

This reverts commit 49c5310ad493c4356dd3bc58c03653cd9466c4fa.

* Revert "bpo-37555: cleaning up changes unnecessary to the final product"

This reverts commit 18e964ba0126d8964d89842cb95534b63c2d326e.

* Revert "bpo-37555: Replacing __eq__ with == to sidestep NotImplemented"

This reverts commit f295eaca5bceac6636c0e2b10e6c7d9a8ee8296a.

* Revert "bpo-37555: Fix _CallList and _Call order sensitivity"

This reverts commit 874fb697b8376fcea130116e56189061f944fde6.

* Updated NEWS.d

* bpo-37555: Add tests checking every function using _call_matcher both with and without spec

* bpo-37555: Ensure all assert methods using _call_matcher are actually passing calls

* Remove AnyCompare and use call objects everywhere.

* Revert "Remove AnyCompare and use call objects everywhere."

This reverts commit 24973c0b32ce7d796a7f4eeaf259832222aae0f5.

* Check for exception in assert_any_await

files:
A Misc/NEWS.d/next/Library/2019-07-19-20-13-48.bpo-37555.S5am28.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py
M Lib/unittest/test/testmock/testhelpers.py
M Misc/ACKS

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index a98decc2c9f0..513cd5c924d4 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -852,7 +852,8 @@ def _call_matcher(self, _call):
             else:
                 name, args, kwargs = _call
             try:
-                return name, sig.bind(*args, **kwargs)
+                bound_call = sig.bind(*args, **kwargs)
+                return call(name, bound_call.args, bound_call.kwargs)
             except TypeError as e:
                 return e.with_traceback(None)
         else:
@@ -901,9 +902,9 @@ def assert_called_with(self, /, *args, **kwargs):
         def _error_message():
             msg = self._format_mock_failure_message(args, kwargs)
             return msg
-        expected = self._call_matcher((args, kwargs))
+        expected = self._call_matcher(_Call((args, kwargs), two=True))
         actual = self._call_matcher(self.call_args)
-        if expected != actual:
+        if actual != expected:
             cause = expected if isinstance(expected, Exception) else None
             raise AssertionError(_error_message()) from cause
 
@@ -963,10 +964,10 @@ def assert_any_call(self, /, *args, **kwargs):
         The assert passes if the mock has *ever* been called, unlike
         `assert_called_with` and `assert_called_once_with` that only pass if
         the call is the most recent one."""
-        expected = self._call_matcher((args, kwargs))
+        expected = self._call_matcher(_Call((args, kwargs), two=True))
+        cause = expected if isinstance(expected, Exception) else None
         actual = [self._call_matcher(c) for c in self.call_args_list]
-        if expected not in actual:
-            cause = expected if isinstance(expected, Exception) else None
+        if cause or expected not in _AnyComparer(actual):
             expected_string = self._format_mock_call_signature(args, kwargs)
             raise AssertionError(
                 '%s call not found' % expected_string
@@ -1019,6 +1020,22 @@ def _calls_repr(self, prefix="Calls"):
         return f"\n{prefix}: {safe_repr(self.mock_calls)}."
 
 
+class _AnyComparer(list):
+    """A list which checks if it contains a call which may have an
+    argument of ANY, flipping the components of item and self from
+    their traditional locations so that ANY is guaranteed to be on
+    the left."""
+    def __contains__(self, item):
+        for _call in self:
+            if len(item) != len(_call):
+                continue
+            if all([
+                expected == actual
+                for expected, actual in zip(item, _call)
+            ]):
+                return True
+        return False
+
 
 def _try_iter(obj):
     if obj is None:
@@ -2171,9 +2188,9 @@ def _error_message():
             msg = self._format_mock_failure_message(args, kwargs, action='await')
             return msg
 
-        expected = self._call_matcher((args, kwargs))
+        expected = self._call_matcher(_Call((args, kwargs), two=True))
         actual = self._call_matcher(self.await_args)
-        if expected != actual:
+        if actual != expected:
             cause = expected if isinstance(expected, Exception) else None
             raise AssertionError(_error_message()) from cause
 
@@ -2192,10 +2209,10 @@ def assert_any_await(self, /, *args, **kwargs):
         """
         Assert the mock has ever been awaited with the specified arguments.
         """
-        expected = self._call_matcher((args, kwargs))
+        expected = self._call_matcher(_Call((args, kwargs), two=True))
+        cause = expected if isinstance(expected, Exception) else None
         actual = [self._call_matcher(c) for c in self.await_args_list]
-        if expected not in actual:
-            cause = expected if isinstance(expected, Exception) else None
+        if cause or expected not in _AnyComparer(actual):
             expected_string = self._format_mock_call_signature(args, kwargs)
             raise AssertionError(
                 '%s await not found' % expected_string
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index 550234223cc3..87f3cfc8fa47 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -2,8 +2,8 @@
 import inspect
 import unittest
 
-from unittest.mock import (call, AsyncMock, patch, MagicMock, create_autospec,
-                           _AwaitEvent)
+from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock,
+                           create_autospec, _AwaitEvent)
 
 
 def tearDownModule():
@@ -192,6 +192,10 @@ def test_create_autospec(self):
         spec.assert_awaited_with(1, 2, c=3)
         spec.assert_awaited()
 
+        with self.assertRaises(AssertionError):
+            spec.assert_any_await(e=1)
+
+
     def test_patch_with_autospec(self):
 
         async def test_async():
@@ -607,6 +611,30 @@ def test_assert_has_awaits_no_order(self):
         asyncio.run(self._runnable_test('SomethingElse'))
         self.mock.assert_has_awaits(calls)
 
+    def test_awaits_asserts_with_any(self):
+        class Foo:
+            def __eq__(self, other): pass
+
+        asyncio.run(self._runnable_test(Foo(), 1))
+
+        self.mock.assert_has_awaits([call(ANY, 1)])
+        self.mock.assert_awaited_with(ANY, 1)
+        self.mock.assert_any_await(ANY, 1)
+
+    def test_awaits_asserts_with_spec_and_any(self):
+        class Foo:
+            def __eq__(self, other): pass
+
+        mock_with_spec = AsyncMock(spec=Foo)
+
+        async def _custom_mock_runnable_test(*args):
+            await mock_with_spec(*args)
+
+        asyncio.run(_custom_mock_runnable_test(Foo(), 1))
+        mock_with_spec.assert_has_awaits([call(ANY, 1)])
+        mock_with_spec.assert_awaited_with(ANY, 1)
+        mock_with_spec.assert_any_await(ANY, 1)
+
     def test_assert_has_awaits_ordered(self):
         calls = [call('NormalFoo'), call('baz')]
         with self.assertRaises(AssertionError):
diff --git a/Lib/unittest/test/testmock/testhelpers.py b/Lib/unittest/test/testmock/testhelpers.py
index f3c7acb98c21..9e7ec5d62d5d 100644
--- a/Lib/unittest/test/testmock/testhelpers.py
+++ b/Lib/unittest/test/testmock/testhelpers.py
@@ -64,7 +64,28 @@ def __ne__(self, other): pass
             self.assertEqual(expected, mock.mock_calls)
             self.assertEqual(mock.mock_calls, expected)
 
+    def test_any_no_spec(self):
+        # This is a regression test for bpo-37555
+        class Foo:
+            def __eq__(self, other): pass
+
+        mock = Mock()
+        mock(Foo(), 1)
+        mock.assert_has_calls([call(ANY, 1)])
+        mock.assert_called_with(ANY, 1)
+        mock.assert_any_call(ANY, 1)
+
+    def test_any_and_spec_set(self):
+        # This is a regression test for bpo-37555
+        class Foo:
+            def __eq__(self, other): pass
+
+        mock = Mock(spec=Foo)
 
+        mock(Foo(), 1)
+        mock.assert_has_calls([call(ANY, 1)])
+        mock.assert_called_with(ANY, 1)
+        mock.assert_any_call(ANY, 1)
 
 class CallTest(unittest.TestCase):
 
diff --git a/Misc/ACKS b/Misc/ACKS
index f321131b2ab4..71e61c3db386 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -503,6 +503,7 @@ Tomer Filiba
 Segev Finer
 Jeffrey Finkelstein
 Russell Finn
+Neal Finne
 Dan Finnie
 Nils Fischbeck
 Frederik Fix
@@ -1714,6 +1715,7 @@ Roger Upole
 Daniel Urban
 Michael Urman
 Hector Urtubia
+Elizabeth Uselton
 Lukas Vacek
 Ville Vainio
 Yann Vaginay
diff --git a/Misc/NEWS.d/next/Library/2019-07-19-20-13-48.bpo-37555.S5am28.rst b/Misc/NEWS.d/next/Library/2019-07-19-20-13-48.bpo-37555.S5am28.rst
new file mode 100644
index 000000000000..16d1d62d3096
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-07-19-20-13-48.bpo-37555.S5am28.rst
@@ -0,0 +1,2 @@
+Fix `NonCallableMock._call_matcher` returning tuple instead of `_Call` object
+when `self._spec_signature` exists. Patch by Elizabeth Uselton
\ No newline at end of file


From webhook-mailer at python.org  Fri Sep 13 12:01:24 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 16:01:24 -0000
Subject: [Python-checkins] bpo-37449: Move ensurepip off of pkgutil and to
 importlib.resources (GH-15109)
Message-ID: <mailman.725.1568390485.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2ccb50cf773881255c5669942a093eb68ad042f6
commit: 2ccb50cf773881255c5669942a093eb68ad042f6
branch: master
author: Joannah Nanjekye <33177550+nanjekyejoannah at users.noreply.github.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-13T09:01:20-07:00
summary:

bpo-37449: Move ensurepip off of pkgutil and to importlib.resources (GH-15109)



Move ensurepip off of pkgutil and to importlib.resources.


https://bugs.python.org/issue37449

files:
A Lib/ensurepip/_bundled/__init__.py
A Misc/NEWS.d/next/Library/2019-08-04-17-22-33.bpo-37449.ycbL2z.rst
M Lib/ensurepip/__init__.py

diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py
index fc0edec6e3de..386ed6c25c76 100644
--- a/Lib/ensurepip/__init__.py
+++ b/Lib/ensurepip/__init__.py
@@ -1,8 +1,11 @@
 import os
 import os.path
-import pkgutil
 import sys
 import tempfile
+from importlib import resources
+
+from . import _bundled
+
 
 
 __all__ = ["version", "bootstrap"]
@@ -96,9 +99,9 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
         additional_paths = []
         for project, version in _PROJECTS:
             wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
-            whl = pkgutil.get_data(
-                "ensurepip",
-                "_bundled/{}".format(wheel_name),
+            whl = resources.read_binary(
+                _bundled,
+                wheel_name,
             )
             with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
                 fp.write(whl)
diff --git a/Lib/ensurepip/_bundled/__init__.py b/Lib/ensurepip/_bundled/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/Misc/NEWS.d/next/Library/2019-08-04-17-22-33.bpo-37449.ycbL2z.rst b/Misc/NEWS.d/next/Library/2019-08-04-17-22-33.bpo-37449.ycbL2z.rst
new file mode 100644
index 000000000000..b34bfea9816f
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-04-17-22-33.bpo-37449.ycbL2z.rst
@@ -0,0 +1,2 @@
+`ensurepip` now uses `importlib.resources.read_binary()` to read data instead of `pkgutil.get_data()`.
+Patch by Joannah Nanjekye.
\ No newline at end of file


From webhook-mailer at python.org  Fri Sep 13 12:12:37 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 16:12:37 -0000
Subject: [Python-checkins] bpo-37199: Replace the early returns added in
 c2cda63. (GH-14535)
Message-ID: <mailman.726.1568391157.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/83c21fdc36070f9fa3eb3b68181630e2cc6480fe
commit: 83c21fdc36070f9fa3eb3b68181630e2cc6480fe
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T09:12:33-07:00
summary:

bpo-37199: Replace the early returns added in c2cda63. (GH-14535)

(cherry picked from commit 81319a81b2a3f42fe1047c2e6f5fd958faab6cdb)

Co-authored-by: Zackery Spytz <zspytz at gmail.com>

files:
M Lib/test/test_asyncio/test_base_events.py

diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
index 370e6a6dfa79..ccdd3bae7336 100644
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -91,28 +91,26 @@ def test_ipaddr_info(self):
         self.assertIsNone(
             base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, 0, 0))
 
-        if not support.IPV6_ENABLED:
-            return
+        if support.IPV6_ENABLED:
+            # IPv4 address with family IPv6.
+            self.assertIsNone(
+                base_events._ipaddr_info('1.2.3.4', 1, INET6, STREAM, TCP))
 
-        # IPv4 address with family IPv6.
-        self.assertIsNone(
-            base_events._ipaddr_info('1.2.3.4', 1, INET6, STREAM, TCP))
-
-        self.assertEqual(
-            (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)),
-            base_events._ipaddr_info('::3', 1, INET6, STREAM, TCP))
+            self.assertEqual(
+                (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)),
+                base_events._ipaddr_info('::3', 1, INET6, STREAM, TCP))
 
-        self.assertEqual(
-            (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)),
-            base_events._ipaddr_info('::3', 1, UNSPEC, STREAM, TCP))
+            self.assertEqual(
+                (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)),
+                base_events._ipaddr_info('::3', 1, UNSPEC, STREAM, TCP))
 
-        # IPv6 address with family IPv4.
-        self.assertIsNone(
-            base_events._ipaddr_info('::3', 1, INET, STREAM, TCP))
+            # IPv6 address with family IPv4.
+            self.assertIsNone(
+                base_events._ipaddr_info('::3', 1, INET, STREAM, TCP))
 
-        # IPv6 address with zone index.
-        self.assertIsNone(
-            base_events._ipaddr_info('::3%lo0', 1, INET6, STREAM, TCP))
+            # IPv6 address with zone index.
+            self.assertIsNone(
+                base_events._ipaddr_info('::3%lo0', 1, INET6, STREAM, TCP))
 
     def test_port_parameter_types(self):
         # Test obscure kinds of arguments for "port".
@@ -1284,25 +1282,24 @@ def _test_create_connection_ip_addr(self, m_socket, allow_inet_pton):
             t.close()
             test_utils.run_briefly(self.loop)  # allow transport to close
 
-        if not support.IPV6_ENABLED:
-            return
-
-        sock.family = socket.AF_INET6
-        coro = self.loop.create_connection(asyncio.Protocol, '::1', 80)
-        t, p = self.loop.run_until_complete(coro)
-        try:
-            # Without inet_pton we use getaddrinfo, which transforms ('::1', 80)
-            # to ('::1', 80, 0, 0). The last 0s are flow info, scope id.
-            [address] = sock.connect.call_args[0]
-            host, port = address[:2]
-            self.assertRegex(host, r'::(0\.)*1')
-            self.assertEqual(port, 80)
-            _, kwargs = m_socket.socket.call_args
-            self.assertEqual(kwargs['family'], m_socket.AF_INET6)
-            self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM)
-        finally:
-            t.close()
-            test_utils.run_briefly(self.loop)  # allow transport to close
+        if support.IPV6_ENABLED:
+            sock.family = socket.AF_INET6
+            coro = self.loop.create_connection(asyncio.Protocol, '::1', 80)
+            t, p = self.loop.run_until_complete(coro)
+            try:
+                # Without inet_pton we use getaddrinfo, which transforms
+                # ('::1', 80) to ('::1', 80, 0, 0). The last 0s are flow info,
+                # scope id.
+                [address] = sock.connect.call_args[0]
+                host, port = address[:2]
+                self.assertRegex(host, r'::(0\.)*1')
+                self.assertEqual(port, 80)
+                _, kwargs = m_socket.socket.call_args
+                self.assertEqual(kwargs['family'], m_socket.AF_INET6)
+                self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM)
+            finally:
+                t.close()
+                test_utils.run_briefly(self.loop)  # allow transport to close
 
     @unittest.skipUnless(support.IPV6_ENABLED, 'no IPv6 support')
     @unittest.skipIf(sys.platform.startswith('aix'),


From webhook-mailer at python.org  Fri Sep 13 12:13:55 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Fri, 13 Sep 2019 16:13:55 -0000
Subject: [Python-checkins] bpo-34706: Preserve subclassing in
 inspect.Signature.from_callable (GH-16108)
Message-ID: <mailman.727.1568391236.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5b9ff7a0dcb16d6f5c3cd4f1f52e0ca6a4bde586
commit: 5b9ff7a0dcb16d6f5c3cd4f1f52e0ca6a4bde586
branch: master
author: Gregory P. Smith <greg at krypto.org>
committer: GitHub <noreply at github.com>
date: 2019-09-13T17:13:51+01:00
summary:

bpo-34706: Preserve subclassing in inspect.Signature.from_callable (GH-16108)

https://bugs.python.org/issue34706

Specifically in the case of a class that does not override its
constructor signature inherited from object.

These are Buck Evan @bukzor's changes cherrypicked from GH-9344.

files:
A Misc/NEWS.d/next/Library/2019-09-13-14-54-33.bpo-34706.HWVpOY.rst
M Lib/inspect.py
M Lib/test/test_inspect.py

diff --git a/Lib/inspect.py b/Lib/inspect.py
index 0a57749ccdd4..c2a1ed4148e7 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -2367,7 +2367,7 @@ def _signature_from_callable(obj, *,
                 if (obj.__init__ is object.__init__ and
                     obj.__new__ is object.__new__):
                     # Return a signature of 'object' builtin.
-                    return signature(object)
+                    return sigcls.from_callable(object)
                 else:
                     raise ValueError(
                         'no signature found for builtin type {!r}'.format(obj))
diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py
index 6c9f52977807..7148dfa0bab9 100644
--- a/Lib/test/test_inspect.py
+++ b/Lib/test/test_inspect.py
@@ -3153,14 +3153,21 @@ def test_signature_from_callable_python_obj(self):
         class MySignature(inspect.Signature): pass
         def foo(a, *, b:1): pass
         foo_sig = MySignature.from_callable(foo)
-        self.assertTrue(isinstance(foo_sig, MySignature))
+        self.assertIsInstance(foo_sig, MySignature)
+
+    def test_signature_from_callable_class(self):
+        # A regression test for a class inheriting its signature from `object`.
+        class MySignature(inspect.Signature): pass
+        class foo: pass
+        foo_sig = MySignature.from_callable(foo)
+        self.assertIsInstance(foo_sig, MySignature)
 
     @unittest.skipIf(MISSING_C_DOCSTRINGS,
                      "Signature information for builtins requires docstrings")
     def test_signature_from_callable_builtin_obj(self):
         class MySignature(inspect.Signature): pass
         sig = MySignature.from_callable(_pickle.Pickler)
-        self.assertTrue(isinstance(sig, MySignature))
+        self.assertIsInstance(sig, MySignature)
 
     def test_signature_definition_order_preserved_on_kwonly(self):
         for fn in signatures_with_lexicographic_keyword_only_parameters():
diff --git a/Misc/NEWS.d/next/Library/2019-09-13-14-54-33.bpo-34706.HWVpOY.rst b/Misc/NEWS.d/next/Library/2019-09-13-14-54-33.bpo-34706.HWVpOY.rst
new file mode 100644
index 000000000000..1df3742abd27
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-13-14-54-33.bpo-34706.HWVpOY.rst
@@ -0,0 +1 @@
+Preserve subclassing in inspect.Signature.from_callable.


From webhook-mailer at python.org  Fri Sep 13 12:40:24 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 16:40:24 -0000
Subject: [Python-checkins] bpo-38092: Reduce overhead when using
 multiprocessing in a Windows virtual environment (GH-16098)
Message-ID: <mailman.728.1568392825.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f2b7556ef851ac85e7cbf189d1b29fdeb9539b88
commit: f2b7556ef851ac85e7cbf189d1b29fdeb9539b88
branch: master
author: Steve Dower <steve.dower at python.org>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-13T09:40:19-07:00
summary:

bpo-38092: Reduce overhead when using multiprocessing in a Windows virtual environment (GH-16098)



https://bugs.python.org/issue38092

files:
A Misc/NEWS.d/next/Windows/2019-09-13-14-11-42.bpo-38092.x31ehI.rst
M Lib/multiprocessing/popen_spawn_win32.py
M Lib/multiprocessing/spawn.py

diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py
index ea9c555da39a..9c4098d0fa4f 100644
--- a/Lib/multiprocessing/popen_spawn_win32.py
+++ b/Lib/multiprocessing/popen_spawn_win32.py
@@ -72,7 +72,7 @@ def __init__(self, process_obj):
             try:
                 hp, ht, pid, tid = _winapi.CreateProcess(
                     python_exe, cmd,
-                    env, None, False, 0, None, None, None)
+                    None, None, False, 0, env, None, None)
                 _winapi.CloseHandle(ht)
             except:
                 _winapi.CloseHandle(rhandle)
diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py
index 7cc129e26107..075f3455478b 100644
--- a/Lib/multiprocessing/spawn.py
+++ b/Lib/multiprocessing/spawn.py
@@ -36,7 +36,7 @@
 if WINSERVICE:
     _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
 else:
-    _python_exe = sys.executable
+    _python_exe = sys._base_executable
 
 def set_executable(exe):
     global _python_exe
diff --git a/Misc/NEWS.d/next/Windows/2019-09-13-14-11-42.bpo-38092.x31ehI.rst b/Misc/NEWS.d/next/Windows/2019-09-13-14-11-42.bpo-38092.x31ehI.rst
new file mode 100644
index 000000000000..48e83c8d1567
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-13-14-11-42.bpo-38092.x31ehI.rst
@@ -0,0 +1 @@
+Reduce overhead when using multiprocessing in a Windows virtual environment.


From webhook-mailer at python.org  Fri Sep 13 12:48:07 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 16:48:07 -0000
Subject: [Python-checkins] bpo-38150 Fix refleak in the finalizer of a
 _testcapimodule type (GH-16115)
Message-ID: <mailman.729.1568393288.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/a67ac2f2d9550e5a36d28f9b6eeacf6575dda2d5
commit: a67ac2f2d9550e5a36d28f9b6eeacf6575dda2d5
branch: master
author: Eddie Elizondo <eelizondo at fb.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-13T09:48:03-07:00
summary:

bpo-38150 Fix refleak in the finalizer of a _testcapimodule type (GH-16115)



The PyLong created in the finalizer was not being cleaned up


https://bugs.python.org/issue38150



Automerge-Triggered-By: @matrixise

files:
M Modules/_testcapimodule.c

diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index dee06de107a1..4cca784d0148 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -6304,7 +6304,7 @@ static void
 heapctypesubclasswithfinalizer_finalize(PyObject *self)
 {
     PyObject *error_type, *error_value, *error_traceback, *m;
-    PyObject *oldtype = NULL, *newtype = NULL;
+    PyObject *oldtype = NULL, *newtype = NULL, *refcnt = NULL;
 
     /* Save the current exception, if any. */
     PyErr_Fetch(&error_type, &error_value, &error_traceback);
@@ -6322,18 +6322,26 @@ heapctypesubclasswithfinalizer_finalize(PyObject *self)
     if (PyObject_SetAttrString(self, "__class__", newtype) < 0) {
         goto cleanup_finalize;
     }
-    if (PyObject_SetAttrString(
-        oldtype, "refcnt_in_del", PyLong_FromSsize_t(Py_REFCNT(oldtype))) < 0) {
+    refcnt = PyLong_FromSsize_t(Py_REFCNT(oldtype));
+    if (refcnt == NULL) {
         goto cleanup_finalize;
     }
-    if (PyObject_SetAttrString(
-        newtype, "refcnt_in_del", PyLong_FromSsize_t(Py_REFCNT(newtype))) < 0) {
+    if (PyObject_SetAttrString(oldtype, "refcnt_in_del", refcnt) < 0) {
+        goto cleanup_finalize;
+    }
+    Py_DECREF(refcnt);
+    refcnt = PyLong_FromSsize_t(Py_REFCNT(newtype));
+    if (refcnt == NULL) {
+        goto cleanup_finalize;
+    }
+    if (PyObject_SetAttrString(newtype, "refcnt_in_del", refcnt) < 0) {
         goto cleanup_finalize;
     }
 
 cleanup_finalize:
     Py_XDECREF(oldtype);
     Py_XDECREF(newtype);
+    Py_XDECREF(refcnt);
 
     /* Restore the saved exception. */
     PyErr_Restore(error_type, error_value, error_traceback);


From webhook-mailer at python.org  Fri Sep 13 12:59:15 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 16:59:15 -0000
Subject: [Python-checkins] bpo-38092: Reduce overhead when using
 multiprocessing in a Windows virtual environment (GH-16098)
Message-ID: <mailman.730.1568393955.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/436b429ade87b10879b3f944e99a515478e86e5e
commit: 436b429ade87b10879b3f944e99a515478e86e5e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T09:59:11-07:00
summary:

bpo-38092: Reduce overhead when using multiprocessing in a Windows virtual environment (GH-16098)


https://bugs.python.org/issue38092
(cherry picked from commit f2b7556ef851ac85e7cbf189d1b29fdeb9539b88)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-13-14-11-42.bpo-38092.x31ehI.rst
M Lib/multiprocessing/popen_spawn_win32.py
M Lib/multiprocessing/spawn.py

diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py
index ea9c555da39a..9c4098d0fa4f 100644
--- a/Lib/multiprocessing/popen_spawn_win32.py
+++ b/Lib/multiprocessing/popen_spawn_win32.py
@@ -72,7 +72,7 @@ def __init__(self, process_obj):
             try:
                 hp, ht, pid, tid = _winapi.CreateProcess(
                     python_exe, cmd,
-                    env, None, False, 0, None, None, None)
+                    None, None, False, 0, env, None, None)
                 _winapi.CloseHandle(ht)
             except:
                 _winapi.CloseHandle(rhandle)
diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py
index 7cc129e26107..075f3455478b 100644
--- a/Lib/multiprocessing/spawn.py
+++ b/Lib/multiprocessing/spawn.py
@@ -36,7 +36,7 @@
 if WINSERVICE:
     _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
 else:
-    _python_exe = sys.executable
+    _python_exe = sys._base_executable
 
 def set_executable(exe):
     global _python_exe
diff --git a/Misc/NEWS.d/next/Windows/2019-09-13-14-11-42.bpo-38092.x31ehI.rst b/Misc/NEWS.d/next/Windows/2019-09-13-14-11-42.bpo-38092.x31ehI.rst
new file mode 100644
index 000000000000..48e83c8d1567
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-13-14-11-42.bpo-38092.x31ehI.rst
@@ -0,0 +1 @@
+Reduce overhead when using multiprocessing in a Windows virtual environment.


From webhook-mailer at python.org  Fri Sep 13 13:26:54 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 17:26:54 -0000
Subject: [Python-checkins] bpo-32790: Add info about alt format using GH-
 for 'g' in chart (GH-6624)
Message-ID: <mailman.731.1568395615.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/77878cadc58aaca234482dffbb5fe89c74c026fa
commit: 77878cadc58aaca234482dffbb5fe89c74c026fa
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T10:26:51-07:00
summary:

bpo-32790: Add info about alt format using GH- for 'g' in chart (GH-6624)

(cherry picked from commit d44542f9a231bf725ecd82eb640a672c759a8227)

Co-authored-by: bchhabra2490 <bchhabra2490 at gmail.com>

files:
M Doc/library/string.rst

diff --git a/Doc/library/string.rst b/Doc/library/string.rst
index 46b2bfc82b73..bbbfed273ee4 100644
--- a/Doc/library/string.rst
+++ b/Doc/library/string.rst
@@ -508,7 +508,8 @@ The available presentation types for floating point and decimal values are:
    |         | with presentation type ``'e'`` and precision ``p-1``.    |
    |         | In both cases insignificant trailing zeros are removed   |
    |         | from the significand, and the decimal point is also      |
-   |         | removed if there are no remaining digits following it.   |
+   |         | removed if there are no remaining digits following it,   |
+   |         | unless the ``'#'`` option is used.                       |
    |         |                                                          |
    |         | Positive and negative infinity, positive and negative    |
    |         | zero, and nans, are formatted as ``inf``, ``-inf``,      |


From webhook-mailer at python.org  Fri Sep 13 13:28:02 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Fri, 13 Sep 2019 17:28:02 -0000
Subject: [Python-checkins] Fix typo in test_api.py. (GH-16119)
Message-ID: <mailman.732.1568395684.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0bc17ea2f5966f429b5b8d6b4ccb9c01f1f610d0
commit: 0bc17ea2f5966f429b5b8d6b4ccb9c01f1f610d0
branch: master
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-13T18:27:59+01:00
summary:

Fix typo in test_api.py. (GH-16119)

files:
M Lib/test/test_importlib/test_api.py

diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py
index edb745c2cd49..0fb1346f9eec 100644
--- a/Lib/test/test_importlib/test_api.py
+++ b/Lib/test/test_importlib/test_api.py
@@ -363,7 +363,7 @@ def test_reload_submodule(self):
 
     def test_module_missing_spec(self):
         #Test that reload() throws ModuleNotFounderror when reloading
-        # a module who's missing a spec. (bpo-29851)
+        # a module whose missing a spec. (bpo-29851)
         name = 'spam'
         with test_util.uncache(name):
             module = sys.modules[name] = types.ModuleType(name)


From webhook-mailer at python.org  Fri Sep 13 13:28:50 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 17:28:50 -0000
Subject: [Python-checkins] bpo-32790: Add info about alt format using GH-
 for 'g' in chart (GH-6624)
Message-ID: <mailman.733.1568395731.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e6b14c026fd9045a0d460b62dbcb512fca4c64ec
commit: e6b14c026fd9045a0d460b62dbcb512fca4c64ec
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T10:28:46-07:00
summary:

bpo-32790: Add info about alt format using GH- for 'g' in chart (GH-6624)

(cherry picked from commit d44542f9a231bf725ecd82eb640a672c759a8227)

Co-authored-by: bchhabra2490 <bchhabra2490 at gmail.com>

files:
M Doc/library/string.rst

diff --git a/Doc/library/string.rst b/Doc/library/string.rst
index af52af0482c4..e2983db1ac83 100644
--- a/Doc/library/string.rst
+++ b/Doc/library/string.rst
@@ -508,7 +508,8 @@ The available presentation types for floating point and decimal values are:
    |         | with presentation type ``'e'`` and precision ``p-1``.    |
    |         | In both cases insignificant trailing zeros are removed   |
    |         | from the significand, and the decimal point is also      |
-   |         | removed if there are no remaining digits following it.   |
+   |         | removed if there are no remaining digits following it,   |
+   |         | unless the ``'#'`` option is used.                       |
    |         |                                                          |
    |         | Positive and negative infinity, positive and negative    |
    |         | zero, and nans, are formatted as ``inf``, ``-inf``,      |


From webhook-mailer at python.org  Fri Sep 13 13:39:45 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 17:39:45 -0000
Subject: [Python-checkins] Doc: fcntl.lockf() is more powerful than written
 (GH-6750)
Message-ID: <mailman.734.1568396386.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/10873831ed95386d4834dcfb49f1c3d02f741eed
commit: 10873831ed95386d4834dcfb49f1c3d02f741eed
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T10:39:42-07:00
summary:

Doc: fcntl.lockf() is more powerful than written (GH-6750)

(cherry picked from commit 77cd0ceab2f6c1696fb1d31115c2f880b2e21934)

Co-authored-by: Eric O. LEBIGOT (EOL) <lebigot at users.noreply.github.com>

files:
M Doc/library/fcntl.rst

diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst
index 2db9674952d7..a7390150f791 100644
--- a/Doc/library/fcntl.rst
+++ b/Doc/library/fcntl.rst
@@ -120,7 +120,8 @@ The module defines the following functions:
 .. function:: lockf(fd, cmd, len=0, start=0, whence=0)
 
    This is essentially a wrapper around the :func:`~fcntl.fcntl` locking calls.
-   *fd* is the file descriptor of the file to lock or unlock, and *cmd*
+   *fd* is the file descriptor (file objects providing a :meth:`~io.IOBase.fileno`
+   method are accepted as well) of the file to lock or unlock, and *cmd*
    is one of the following values:
 
    * :const:`LOCK_UN` -- unlock
@@ -171,4 +172,3 @@ using the :func:`flock` call may be better.
       present in the :mod:`os` module (on BSD only), the :func:`os.open`
       function provides an alternative to the :func:`lockf` and :func:`flock`
       functions.
-


From webhook-mailer at python.org  Fri Sep 13 13:48:06 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 17:48:06 -0000
Subject: [Python-checkins] Fix typo in test_api.py. (GH-16119)
Message-ID: <mailman.735.1568396887.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3ac6520b830a6af47121e6d83d091db2dbef2eb3
commit: 3ac6520b830a6af47121e6d83d091db2dbef2eb3
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T10:48:02-07:00
summary:

Fix typo in test_api.py. (GH-16119)

(cherry picked from commit 0bc17ea2f5966f429b5b8d6b4ccb9c01f1f610d0)

Co-authored-by: Benjamin Peterson <benjamin at python.org>

files:
M Lib/test/test_importlib/test_api.py

diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py
index edb745c2cd49..0fb1346f9eec 100644
--- a/Lib/test/test_importlib/test_api.py
+++ b/Lib/test/test_importlib/test_api.py
@@ -363,7 +363,7 @@ def test_reload_submodule(self):
 
     def test_module_missing_spec(self):
         #Test that reload() throws ModuleNotFounderror when reloading
-        # a module who's missing a spec. (bpo-29851)
+        # a module whose missing a spec. (bpo-29851)
         name = 'spam'
         with test_util.uncache(name):
             module = sys.modules[name] = types.ModuleType(name)


From webhook-mailer at python.org  Fri Sep 13 13:49:13 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 17:49:13 -0000
Subject: [Python-checkins] Fix typo in test_api.py. (GH-16119)
Message-ID: <mailman.736.1568396954.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/cd85200f6d7a92fab04dd8b4745e6ff8d79f231e
commit: cd85200f6d7a92fab04dd8b4745e6ff8d79f231e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T10:49:09-07:00
summary:

Fix typo in test_api.py. (GH-16119)

(cherry picked from commit 0bc17ea2f5966f429b5b8d6b4ccb9c01f1f610d0)

Co-authored-by: Benjamin Peterson <benjamin at python.org>

files:
M Lib/test/test_importlib/test_api.py

diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py
index edb745c2cd49..0fb1346f9eec 100644
--- a/Lib/test/test_importlib/test_api.py
+++ b/Lib/test/test_importlib/test_api.py
@@ -363,7 +363,7 @@ def test_reload_submodule(self):
 
     def test_module_missing_spec(self):
         #Test that reload() throws ModuleNotFounderror when reloading
-        # a module who's missing a spec. (bpo-29851)
+        # a module whose missing a spec. (bpo-29851)
         name = 'spam'
         with test_util.uncache(name):
             module = sys.modules[name] = types.ModuleType(name)


From webhook-mailer at python.org  Fri Sep 13 15:41:01 2019
From: webhook-mailer at python.org (Ivan Levkivskyi)
Date: Fri, 13 Sep 2019 19:41:01 -0000
Subject: [Python-checkins] bpo-37953: Fix ForwardRef hash and equality
 checks (GH-15400)
Message-ID: <mailman.737.1568403662.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e082e7cbe4a934b86f7a07354d97d4e14a9dd46a
commit: e082e7cbe4a934b86f7a07354d97d4e14a9dd46a
branch: master
author: plokmijnuhby <39633434+plokmijnuhby at users.noreply.github.com>
committer: Ivan Levkivskyi <levkivskyi at gmail.com>
date: 2019-09-13T20:40:54+01:00
summary:

bpo-37953: Fix ForwardRef hash and equality checks (GH-15400)

Ideally if we stick a ForwardRef in a dictionary we would like to reliably be able to get it out again.

https://bugs.python.org/issue37953

files:
A Misc/NEWS.d/next/Library/2019-09-06-17-40-34.bpo-37953.db5FQq.rst
M Lib/test/test_typing.py
M Lib/typing.py

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index 3402996b2d2e..28b3d316d350 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -2361,6 +2361,65 @@ def test_forward_equality(self):
         self.assertEqual(fr, typing.ForwardRef('int'))
         self.assertNotEqual(List['int'], List[int])
 
+    def test_forward_equality_gth(self):
+        c1 = typing.ForwardRef('C')
+        c1_gth = typing.ForwardRef('C')
+        c2 = typing.ForwardRef('C')
+        c2_gth = typing.ForwardRef('C')
+
+        class C:
+            pass
+        def foo(a: c1_gth, b: c2_gth):
+            pass
+
+        self.assertEqual(get_type_hints(foo, globals(), locals()), {'a': C, 'b': C})
+        self.assertEqual(c1, c2)
+        self.assertEqual(c1, c1_gth)
+        self.assertEqual(c1_gth, c2_gth)
+        self.assertEqual(List[c1], List[c1_gth])
+        self.assertNotEqual(List[c1], List[C])
+        self.assertNotEqual(List[c1_gth], List[C])
+        self.assertEquals(Union[c1, c1_gth], Union[c1])
+        self.assertEquals(Union[c1, c1_gth, int], Union[c1, int])
+
+    def test_forward_equality_hash(self):
+        c1 = typing.ForwardRef('int')
+        c1_gth = typing.ForwardRef('int')
+        c2 = typing.ForwardRef('int')
+        c2_gth = typing.ForwardRef('int')
+
+        def foo(a: c1_gth, b: c2_gth):
+            pass
+        get_type_hints(foo, globals(), locals())
+
+        self.assertEqual(hash(c1), hash(c2))
+        self.assertEqual(hash(c1_gth), hash(c2_gth))
+        self.assertEqual(hash(c1), hash(c1_gth))
+
+    def test_forward_equality_namespace(self):
+        class A:
+            pass
+        def namespace1():
+            a = typing.ForwardRef('A')
+            def fun(x: a):
+                pass
+            get_type_hints(fun, globals(), locals())
+            return a
+
+        def namespace2():
+            a = typing.ForwardRef('A')
+
+            class A:
+                pass
+            def fun(x: a):
+                pass
+
+            get_type_hints(fun, globals(), locals())
+            return a
+
+        self.assertEqual(namespace1(), namespace1())
+        self.assertNotEqual(namespace1(), namespace2())
+
     def test_forward_repr(self):
         self.assertEqual(repr(List['int']), "typing.List[ForwardRef('int')]")
 
@@ -2380,6 +2439,63 @@ def foo(a: Tuple['T']):
         self.assertEqual(get_type_hints(foo, globals(), locals()),
                          {'a': Tuple[T]})
 
+    def test_forward_recursion_actually(self):
+        def namespace1():
+            a = typing.ForwardRef('A')
+            A = a
+            def fun(x: a): pass
+
+            ret = get_type_hints(fun, globals(), locals())
+            return a
+
+        def namespace2():
+            a = typing.ForwardRef('A')
+            A = a
+            def fun(x: a): pass
+
+            ret = get_type_hints(fun, globals(), locals())
+            return a
+
+        def cmp(o1, o2):
+            return o1 == o2
+
+        r1 = namespace1()
+        r2 = namespace2()
+        self.assertIsNot(r1, r2)
+        self.assertRaises(RecursionError, cmp, r1, r2)
+
+    def test_union_forward_recursion(self):
+        ValueList = List['Value']
+        Value = Union[str, ValueList]
+
+        class C:
+            foo: List[Value]
+        class D:
+            foo: Union[Value, ValueList]
+        class E:
+            foo: Union[List[Value], ValueList]
+        class F:
+            foo: Union[Value, List[Value], ValueList]
+
+        self.assertEqual(get_type_hints(C, globals(), locals()), get_type_hints(C, globals(), locals()))
+        self.assertEqual(get_type_hints(C, globals(), locals()),
+                         {'foo': List[Union[str, List[Union[str, List['Value']]]]]})
+        self.assertEqual(get_type_hints(D, globals(), locals()),
+                         {'foo': Union[str, List[Union[str, List['Value']]]]})
+        self.assertEqual(get_type_hints(E, globals(), locals()),
+                         {'foo': Union[
+                             List[Union[str, List[Union[str, List['Value']]]]],
+                             List[Union[str, List['Value']]]
+                         ]
+                          })
+        self.assertEqual(get_type_hints(F, globals(), locals()),
+                         {'foo': Union[
+                             str,
+                             List[Union[str, List['Value']]],
+                             List[Union[str, List[Union[str, List['Value']]]]]
+                         ]
+                          })
+
     def test_callable_forward(self):
 
         def foo(a: Callable[['T'], 'T']):
diff --git a/Lib/typing.py b/Lib/typing.py
index 7b07112c3abf..32011332127f 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -524,11 +524,13 @@ def _evaluate(self, globalns, localns):
     def __eq__(self, other):
         if not isinstance(other, ForwardRef):
             return NotImplemented
-        return (self.__forward_arg__ == other.__forward_arg__ and
-                self.__forward_value__ == other.__forward_value__)
+        if self.__forward_evaluated__ and other.__forward_evaluated__:
+            return (self.__forward_arg__ == other.__forward_arg__ and
+                    self.__forward_value__ == other.__forward_value__)
+        return self.__forward_arg__ == other.__forward_arg__
 
     def __hash__(self):
-        return hash((self.__forward_arg__, self.__forward_value__))
+        return hash(self.__forward_arg__)
 
     def __repr__(self):
         return f'ForwardRef({self.__forward_arg__!r})'
diff --git a/Misc/NEWS.d/next/Library/2019-09-06-17-40-34.bpo-37953.db5FQq.rst b/Misc/NEWS.d/next/Library/2019-09-06-17-40-34.bpo-37953.db5FQq.rst
new file mode 100644
index 000000000000..4eff4f7479aa
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-06-17-40-34.bpo-37953.db5FQq.rst
@@ -0,0 +1,2 @@
+In :mod:`typing`, improved the ``__hash__`` and ``__eq__`` methods for
+:class:`ForwardReferences`.


From webhook-mailer at python.org  Fri Sep 13 15:50:33 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Fri, 13 Sep 2019 19:50:33 -0000
Subject: [Python-checkins] bpo-38005: Fixed comparing and creating of
 InterpreterID and ChannelID. (GH-15652)
Message-ID: <mailman.738.1568404234.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bf169915ecdd42329726104278eb723a7dda2736
commit: bf169915ecdd42329726104278eb723a7dda2736
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T22:50:27+03:00
summary:

bpo-38005: Fixed comparing and creating of InterpreterID and ChannelID. (GH-15652)

* Fix a crash in comparing with float (and maybe other crashes).
* They are now never equal to strings and non-integer numbers.
* Comparison with a large number no longer raises OverflowError.
* Arbitrary exceptions no longer silenced in constructors and comparisons.
* TypeError raised in the constructor contains now the name of the type.
* Accept only ChannelID and int-like objects in channel functions.
* Accept only InterpreterId, int-like objects and str in the InterpreterId constructor.
* Accept int-like objects, not just int in interpreter related functions.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-02-20-00-31.bpo-38005.e7VsTA.rst
M Include/cpython/interpreteridobject.h
M Lib/test/test__xxsubinterpreters.py
M Modules/_xxsubinterpretersmodule.c
M Objects/interpreteridobject.c

diff --git a/Include/cpython/interpreteridobject.h b/Include/cpython/interpreteridobject.h
index cb72c2b0895a..67ec5873542d 100644
--- a/Include/cpython/interpreteridobject.h
+++ b/Include/cpython/interpreteridobject.h
@@ -14,8 +14,6 @@ PyAPI_FUNC(PyObject *) _PyInterpreterID_New(int64_t);
 PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *);
 PyAPI_FUNC(PyInterpreterState *) _PyInterpreterID_LookUp(PyObject *);
 
-PyAPI_FUNC(int64_t) _Py_CoerceID(PyObject *);
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py
index 78b2030a1f6d..f5474f4101d0 100644
--- a/Lib/test/test__xxsubinterpreters.py
+++ b/Lib/test/test__xxsubinterpreters.py
@@ -526,30 +526,23 @@ def test_with_int(self):
         self.assertEqual(int(id), 10)
 
     def test_coerce_id(self):
-        id = interpreters.InterpreterID('10', force=True)
-        self.assertEqual(int(id), 10)
-
-        id = interpreters.InterpreterID(10.0, force=True)
-        self.assertEqual(int(id), 10)
-
         class Int(str):
-            def __init__(self, value):
-                self._value = value
-            def __int__(self):
-                return self._value
+            def __index__(self):
+                return 10
 
-        id = interpreters.InterpreterID(Int(10), force=True)
-        self.assertEqual(int(id), 10)
+        for id in ('10', '1_0', Int()):
+            with self.subTest(id=id):
+                id = interpreters.InterpreterID(id, force=True)
+                self.assertEqual(int(id), 10)
 
     def test_bad_id(self):
-        for id in [-1, 'spam']:
-            with self.subTest(id):
-                with self.assertRaises(ValueError):
-                    interpreters.InterpreterID(id)
-        with self.assertRaises(OverflowError):
-            interpreters.InterpreterID(2**64)
-        with self.assertRaises(TypeError):
-            interpreters.InterpreterID(object())
+        self.assertRaises(TypeError, interpreters.InterpreterID, object())
+        self.assertRaises(TypeError, interpreters.InterpreterID, 10.0)
+        self.assertRaises(TypeError, interpreters.InterpreterID, b'10')
+        self.assertRaises(ValueError, interpreters.InterpreterID, -1)
+        self.assertRaises(ValueError, interpreters.InterpreterID, '-1')
+        self.assertRaises(ValueError, interpreters.InterpreterID, 'spam')
+        self.assertRaises(OverflowError, interpreters.InterpreterID, 2**64)
 
     def test_does_not_exist(self):
         id = interpreters.channel_create()
@@ -572,6 +565,14 @@ def test_equality(self):
         self.assertTrue(id1 == id1)
         self.assertTrue(id1 == id2)
         self.assertTrue(id1 == int(id1))
+        self.assertTrue(int(id1) == id1)
+        self.assertTrue(id1 == float(int(id1)))
+        self.assertTrue(float(int(id1)) == id1)
+        self.assertFalse(id1 == float(int(id1)) + 0.1)
+        self.assertFalse(id1 == str(int(id1)))
+        self.assertFalse(id1 == 2**1000)
+        self.assertFalse(id1 == float('inf'))
+        self.assertFalse(id1 == 'spam')
         self.assertFalse(id1 == id3)
 
         self.assertFalse(id1 != id1)
@@ -1105,30 +1106,20 @@ def test_with_kwargs(self):
         self.assertEqual(cid.end, 'both')
 
     def test_coerce_id(self):
-        cid = interpreters._channel_id('10', force=True)
-        self.assertEqual(int(cid), 10)
-
-        cid = interpreters._channel_id(10.0, force=True)
-        self.assertEqual(int(cid), 10)
-
         class Int(str):
-            def __init__(self, value):
-                self._value = value
-            def __int__(self):
-                return self._value
+            def __index__(self):
+                return 10
 
-        cid = interpreters._channel_id(Int(10), force=True)
+        cid = interpreters._channel_id(Int(), force=True)
         self.assertEqual(int(cid), 10)
 
     def test_bad_id(self):
-        for cid in [-1, 'spam']:
-            with self.subTest(cid):
-                with self.assertRaises(ValueError):
-                    interpreters._channel_id(cid)
-        with self.assertRaises(OverflowError):
-            interpreters._channel_id(2**64)
-        with self.assertRaises(TypeError):
-            interpreters._channel_id(object())
+        self.assertRaises(TypeError, interpreters._channel_id, object())
+        self.assertRaises(TypeError, interpreters._channel_id, 10.0)
+        self.assertRaises(TypeError, interpreters._channel_id, '10')
+        self.assertRaises(TypeError, interpreters._channel_id, b'10')
+        self.assertRaises(ValueError, interpreters._channel_id, -1)
+        self.assertRaises(OverflowError, interpreters._channel_id, 2**64)
 
     def test_bad_kwargs(self):
         with self.assertRaises(ValueError):
@@ -1164,6 +1155,14 @@ def test_equality(self):
         self.assertTrue(cid1 == cid1)
         self.assertTrue(cid1 == cid2)
         self.assertTrue(cid1 == int(cid1))
+        self.assertTrue(int(cid1) == cid1)
+        self.assertTrue(cid1 == float(int(cid1)))
+        self.assertTrue(float(int(cid1)) == cid1)
+        self.assertFalse(cid1 == float(int(cid1)) + 0.1)
+        self.assertFalse(cid1 == str(int(cid1)))
+        self.assertFalse(cid1 == 2**1000)
+        self.assertFalse(cid1 == float('inf'))
+        self.assertFalse(cid1 == 'spam')
         self.assertFalse(cid1 == cid3)
 
         self.assertFalse(cid1 != cid1)
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-02-20-00-31.bpo-38005.e7VsTA.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-02-20-00-31.bpo-38005.e7VsTA.rst
new file mode 100644
index 000000000000..706abf587b90
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-02-20-00-31.bpo-38005.e7VsTA.rst	
@@ -0,0 +1 @@
+Fixed comparing and creating of InterpreterID and ChannelID.
diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c
index 19d98fd96934..7842947e54ac 100644
--- a/Modules/_xxsubinterpretersmodule.c
+++ b/Modules/_xxsubinterpretersmodule.c
@@ -1405,6 +1405,34 @@ typedef struct channelid {
     _channels *channels;
 } channelid;
 
+static int
+channel_id_converter(PyObject *arg, void *ptr)
+{
+    int64_t cid;
+    if (PyObject_TypeCheck(arg, &ChannelIDtype)) {
+        cid = ((channelid *)arg)->id;
+    }
+    else if (PyIndex_Check(arg)) {
+        cid = PyLong_AsLongLong(arg);
+        if (cid == -1 && PyErr_Occurred()) {
+            return 0;
+        }
+        if (cid < 0) {
+            PyErr_Format(PyExc_ValueError,
+                        "channel ID must be a non-negative int, got %R", arg);
+            return 0;
+        }
+    }
+    else {
+        PyErr_Format(PyExc_TypeError,
+                     "channel ID must be an int, got %.100s",
+                     arg->ob_type->tp_name);
+        return 0;
+    }
+    *(int64_t *)ptr = cid;
+    return 1;
+}
+
 static channelid *
 newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels,
              int force, int resolve)
@@ -1437,28 +1465,16 @@ static PyObject *
 channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
 {
     static char *kwlist[] = {"id", "send", "recv", "force", "_resolve", NULL};
-    PyObject *id;
+    int64_t cid;
     int send = -1;
     int recv = -1;
     int force = 0;
     int resolve = 0;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O|$pppp:ChannelID.__new__", kwlist,
-                                     &id, &send, &recv, &force, &resolve))
+                                     "O&|$pppp:ChannelID.__new__", kwlist,
+                                     channel_id_converter, &cid, &send, &recv, &force, &resolve))
         return NULL;
 
-    // Coerce and check the ID.
-    int64_t cid;
-    if (PyObject_TypeCheck(id, &ChannelIDtype)) {
-        cid = ((channelid *)id)->id;
-    }
-    else {
-        cid = _Py_CoerceID(id);
-        if (cid < 0) {
-            return NULL;
-        }
-    }
-
     // Handle "send" and "recv".
     if (send == 0 && recv == 0) {
         PyErr_SetString(PyExc_ValueError,
@@ -1592,30 +1608,28 @@ channelid_richcompare(PyObject *self, PyObject *other, int op)
     int equal;
     if (PyObject_TypeCheck(other, &ChannelIDtype)) {
         channelid *othercid = (channelid *)other;
-        if (cid->end != othercid->end) {
-            equal = 0;
-        }
-        else {
-            equal = (cid->id == othercid->id);
-        }
+        equal = (cid->end == othercid->end) && (cid->id == othercid->id);
     }
-    else {
-        other = PyNumber_Long(other);
-        if (other == NULL) {
-            PyErr_Clear();
-            Py_RETURN_NOTIMPLEMENTED;
-        }
-        int64_t othercid = PyLong_AsLongLong(other);
-        Py_DECREF(other);
-        if (othercid == -1 && PyErr_Occurred() != NULL) {
+    else if (PyLong_Check(other)) {
+        /* Fast path */
+        int overflow;
+        long long othercid = PyLong_AsLongLongAndOverflow(other, &overflow);
+        if (othercid == -1 && PyErr_Occurred()) {
             return NULL;
         }
-        if (othercid < 0) {
-            equal = 0;
-        }
-        else {
-            equal = (cid->id == othercid);
+        equal = !overflow && (othercid >= 0) && (cid->id == othercid);
+    }
+    else if (PyNumber_Check(other)) {
+        PyObject *pyid = PyLong_FromLongLong(cid->id);
+        if (pyid == NULL) {
+            return NULL;
         }
+        PyObject *res = PyObject_RichCompare(pyid, other, op);
+        Py_DECREF(pyid);
+        return res;
+    }
+    else {
+        Py_RETURN_NOTIMPLEMENTED;
     }
 
     if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) {
@@ -1754,8 +1768,7 @@ static PyTypeObject ChannelIDtype = {
     0,                              /* tp_getattro */
     0,                              /* tp_setattro */
     0,                              /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
-        Py_TPFLAGS_LONG_SUBCLASS,   /* tp_flags */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
     channelid_doc,                  /* tp_doc */
     0,                              /* tp_traverse */
     0,                              /* tp_clear */
@@ -2017,10 +2030,6 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
                                      "O:destroy", kwlist, &id)) {
         return NULL;
     }
-    if (!PyLong_Check(id)) {
-        PyErr_SetString(PyExc_TypeError, "ID must be an int");
-        return NULL;
-    }
 
     // Look up the interpreter.
     PyInterpreterState *interp = _PyInterpreterID_LookUp(id);
@@ -2145,10 +2154,6 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
                                      &id, &code, &shared)) {
         return NULL;
     }
-    if (!PyLong_Check(id)) {
-        PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int");
-        return NULL;
-    }
 
     // Look up the interpreter.
     PyInterpreterState *interp = _PyInterpreterID_LookUp(id);
@@ -2216,10 +2221,6 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds)
                                      "O:is_running", kwlist, &id)) {
         return NULL;
     }
-    if (!PyLong_Check(id)) {
-        PyErr_SetString(PyExc_TypeError, "ID must be an int");
-        return NULL;
-    }
 
     PyInterpreterState *interp = _PyInterpreterID_LookUp(id);
     if (interp == NULL) {
@@ -2268,13 +2269,9 @@ static PyObject *
 channel_destroy(PyObject *self, PyObject *args, PyObject *kwds)
 {
     static char *kwlist[] = {"cid", NULL};
-    PyObject *id;
-    if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O:channel_destroy", kwlist, &id)) {
-        return NULL;
-    }
-    int64_t cid = _Py_CoerceID(id);
-    if (cid < 0) {
+    int64_t cid;
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:channel_destroy", kwlist,
+                                     channel_id_converter, &cid)) {
         return NULL;
     }
 
@@ -2331,14 +2328,10 @@ static PyObject *
 channel_send(PyObject *self, PyObject *args, PyObject *kwds)
 {
     static char *kwlist[] = {"cid", "obj", NULL};
-    PyObject *id;
+    int64_t cid;
     PyObject *obj;
-    if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "OO:channel_send", kwlist, &id, &obj)) {
-        return NULL;
-    }
-    int64_t cid = _Py_CoerceID(id);
-    if (cid < 0) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O:channel_send", kwlist,
+                                     channel_id_converter, &cid, &obj)) {
         return NULL;
     }
 
@@ -2357,13 +2350,9 @@ static PyObject *
 channel_recv(PyObject *self, PyObject *args, PyObject *kwds)
 {
     static char *kwlist[] = {"cid", NULL};
-    PyObject *id;
-    if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O:channel_recv", kwlist, &id)) {
-        return NULL;
-    }
-    int64_t cid = _Py_CoerceID(id);
-    if (cid < 0) {
+    int64_t cid;
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:channel_recv", kwlist,
+                                     channel_id_converter, &cid)) {
         return NULL;
     }
 
@@ -2379,17 +2368,13 @@ static PyObject *
 channel_close(PyObject *self, PyObject *args, PyObject *kwds)
 {
     static char *kwlist[] = {"cid", "send", "recv", "force", NULL};
-    PyObject *id;
+    int64_t cid;
     int send = 0;
     int recv = 0;
     int force = 0;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O|$ppp:channel_close", kwlist,
-                                     &id, &send, &recv, &force)) {
-        return NULL;
-    }
-    int64_t cid = _Py_CoerceID(id);
-    if (cid < 0) {
+                                     "O&|$ppp:channel_close", kwlist,
+                                     channel_id_converter, &cid, &send, &recv, &force)) {
         return NULL;
     }
 
@@ -2431,17 +2416,13 @@ channel_release(PyObject *self, PyObject *args, PyObject *kwds)
 {
     // Note that only the current interpreter is affected.
     static char *kwlist[] = {"cid", "send", "recv", "force", NULL};
-    PyObject *id;
+    int64_t cid;
     int send = 0;
     int recv = 0;
     int force = 0;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O|$ppp:channel_release", kwlist,
-                                     &id, &send, &recv, &force)) {
-        return NULL;
-    }
-    int64_t cid = _Py_CoerceID(id);
-    if (cid < 0) {
+                                     "O&|$ppp:channel_release", kwlist,
+                                     channel_id_converter, &cid, &send, &recv, &force)) {
         return NULL;
     }
     if (send == 0 && recv == 0) {
@@ -2538,7 +2519,6 @@ PyInit__xxsubinterpreters(void)
     }
 
     /* Initialize types */
-    ChannelIDtype.tp_base = &PyLong_Type;
     if (PyType_Ready(&ChannelIDtype) != 0) {
         return NULL;
     }
diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c
index 0a1dfa25795f..3edbb85e6ac3 100644
--- a/Objects/interpreteridobject.c
+++ b/Objects/interpreteridobject.c
@@ -5,38 +5,6 @@
 #include "interpreteridobject.h"
 
 
-int64_t
-_Py_CoerceID(PyObject *orig)
-{
-    PyObject *pyid = PyNumber_Long(orig);
-    if (pyid == NULL) {
-        if (PyErr_ExceptionMatches(PyExc_TypeError)) {
-            PyErr_Format(PyExc_TypeError,
-                         "'id' must be a non-negative int, got %R", orig);
-        }
-        else {
-            PyErr_Format(PyExc_ValueError,
-                         "'id' must be a non-negative int, got %R", orig);
-        }
-        return -1;
-    }
-    int64_t id = PyLong_AsLongLong(pyid);
-    Py_DECREF(pyid);
-    if (id == -1 && PyErr_Occurred() != NULL) {
-        if (!PyErr_ExceptionMatches(PyExc_OverflowError)) {
-            PyErr_Format(PyExc_ValueError,
-                         "'id' must be a non-negative int, got %R", orig);
-        }
-        return -1;
-    }
-    if (id < 0) {
-        PyErr_Format(PyExc_ValueError,
-                     "'id' must be a non-negative int, got %R", orig);
-        return -1;
-    }
-    return id;
-}
-
 typedef struct interpid {
     PyObject_HEAD
     int64_t id;
@@ -85,8 +53,31 @@ interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
         id = ((interpid *)idobj)->id;
     }
     else {
-        id = _Py_CoerceID(idobj);
+        PyObject *pyid;
+        if (PyIndex_Check(idobj)) {
+            pyid = idobj;
+            Py_INCREF(pyid);
+        }
+        else if (PyUnicode_Check(idobj)) {
+            pyid = PyNumber_Long(idobj);
+            if (pyid == NULL) {
+                return NULL;
+            }
+        }
+        else {
+            PyErr_Format(PyExc_TypeError,
+                         "interpreter ID must be an int, got %.100s",
+                         idobj->ob_type->tp_name);
+            return NULL;
+        }
+        id = PyLong_AsLongLong(pyid);
+        Py_DECREF(pyid);
+        if (id == -1 && PyErr_Occurred()) {
+            return NULL;
+        }
         if (id < 0) {
+            PyErr_Format(PyExc_ValueError,
+                         "interpreter ID must be a non-negative int, got %R", idobj);
             return NULL;
         }
     }
@@ -202,23 +193,26 @@ interpid_richcompare(PyObject *self, PyObject *other, int op)
         interpid *otherid = (interpid *)other;
         equal = (id->id == otherid->id);
     }
-    else {
-        other = PyNumber_Long(other);
-        if (other == NULL) {
-            PyErr_Clear();
-            Py_RETURN_NOTIMPLEMENTED;
-        }
-        int64_t otherid = PyLong_AsLongLong(other);
-        Py_DECREF(other);
-        if (otherid == -1 && PyErr_Occurred() != NULL) {
+    else if (PyLong_CheckExact(other)) {
+        /* Fast path */
+        int overflow;
+        long long otherid = PyLong_AsLongLongAndOverflow(other, &overflow);
+        if (otherid == -1 && PyErr_Occurred()) {
             return NULL;
         }
-        if (otherid < 0) {
-            equal = 0;
-        }
-        else {
-            equal = (id->id == otherid);
+        equal = !overflow && (otherid >= 0) && (id->id == otherid);
+    }
+    else if (PyNumber_Check(other)) {
+        PyObject *pyid = PyLong_FromLongLong(id->id);
+        if (pyid == NULL) {
+            return NULL;
         }
+        PyObject *res = PyObject_RichCompare(pyid, other, op);
+        Py_DECREF(pyid);
+        return res;
+    }
+    else {
+        Py_RETURN_NOTIMPLEMENTED;
     }
 
     if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) {
@@ -250,8 +244,7 @@ PyTypeObject _PyInterpreterID_Type = {
     0,                              /* tp_getattro */
     0,                              /* tp_setattro */
     0,                              /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
-        Py_TPFLAGS_LONG_SUBCLASS,   /* tp_flags */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
     interpid_doc,                   /* tp_doc */
     0,                              /* tp_traverse */
     0,                              /* tp_clear */
@@ -262,7 +255,7 @@ PyTypeObject _PyInterpreterID_Type = {
     0,                              /* tp_methods */
     0,                              /* tp_members */
     0,                              /* tp_getset */
-    &PyLong_Type,                   /* tp_base */
+    0,                              /* tp_base */
     0,                              /* tp_dict */
     0,                              /* tp_descr_get */
     0,                              /* tp_descr_set */
@@ -297,12 +290,17 @@ _PyInterpreterID_LookUp(PyObject *requested_id)
     if (PyObject_TypeCheck(requested_id, &_PyInterpreterID_Type)) {
         id = ((interpid *)requested_id)->id;
     }
-    else {
+    else if (PyIndex_Check(requested_id)) {
         id = PyLong_AsLongLong(requested_id);
         if (id == -1 && PyErr_Occurred() != NULL) {
             return NULL;
         }
         assert(id <= INT64_MAX);
     }
+    else {
+        PyErr_Format(PyExc_TypeError, "interpreter ID must be an int, got %.100s",
+                     requested_id->ob_type->tp_name);
+        return NULL;
+    }
     return _PyInterpreterState_LookUpID(id);
 }


From webhook-mailer at python.org  Fri Sep 13 16:00:43 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 13 Sep 2019 20:00:43 -0000
Subject: [Python-checkins] bpo-37953: Fix ForwardRef hash and equality
 checks (GH-15400)
Message-ID: <mailman.739.1568404844.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e91edfed4214dbae17b8906b5dc7778769aac620
commit: e91edfed4214dbae17b8906b5dc7778769aac620
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T13:00:39-07:00
summary:

bpo-37953: Fix ForwardRef hash and equality checks (GH-15400)


Ideally if we stick a ForwardRef in a dictionary we would like to reliably be able to get it out again.

https://bugs.python.org/issue37953
(cherry picked from commit e082e7cbe4a934b86f7a07354d97d4e14a9dd46a)

Co-authored-by: plokmijnuhby <39633434+plokmijnuhby at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Library/2019-09-06-17-40-34.bpo-37953.db5FQq.rst
M Lib/test/test_typing.py
M Lib/typing.py

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index 3402996b2d2e..28b3d316d350 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -2361,6 +2361,65 @@ def test_forward_equality(self):
         self.assertEqual(fr, typing.ForwardRef('int'))
         self.assertNotEqual(List['int'], List[int])
 
+    def test_forward_equality_gth(self):
+        c1 = typing.ForwardRef('C')
+        c1_gth = typing.ForwardRef('C')
+        c2 = typing.ForwardRef('C')
+        c2_gth = typing.ForwardRef('C')
+
+        class C:
+            pass
+        def foo(a: c1_gth, b: c2_gth):
+            pass
+
+        self.assertEqual(get_type_hints(foo, globals(), locals()), {'a': C, 'b': C})
+        self.assertEqual(c1, c2)
+        self.assertEqual(c1, c1_gth)
+        self.assertEqual(c1_gth, c2_gth)
+        self.assertEqual(List[c1], List[c1_gth])
+        self.assertNotEqual(List[c1], List[C])
+        self.assertNotEqual(List[c1_gth], List[C])
+        self.assertEquals(Union[c1, c1_gth], Union[c1])
+        self.assertEquals(Union[c1, c1_gth, int], Union[c1, int])
+
+    def test_forward_equality_hash(self):
+        c1 = typing.ForwardRef('int')
+        c1_gth = typing.ForwardRef('int')
+        c2 = typing.ForwardRef('int')
+        c2_gth = typing.ForwardRef('int')
+
+        def foo(a: c1_gth, b: c2_gth):
+            pass
+        get_type_hints(foo, globals(), locals())
+
+        self.assertEqual(hash(c1), hash(c2))
+        self.assertEqual(hash(c1_gth), hash(c2_gth))
+        self.assertEqual(hash(c1), hash(c1_gth))
+
+    def test_forward_equality_namespace(self):
+        class A:
+            pass
+        def namespace1():
+            a = typing.ForwardRef('A')
+            def fun(x: a):
+                pass
+            get_type_hints(fun, globals(), locals())
+            return a
+
+        def namespace2():
+            a = typing.ForwardRef('A')
+
+            class A:
+                pass
+            def fun(x: a):
+                pass
+
+            get_type_hints(fun, globals(), locals())
+            return a
+
+        self.assertEqual(namespace1(), namespace1())
+        self.assertNotEqual(namespace1(), namespace2())
+
     def test_forward_repr(self):
         self.assertEqual(repr(List['int']), "typing.List[ForwardRef('int')]")
 
@@ -2380,6 +2439,63 @@ def foo(a: Tuple['T']):
         self.assertEqual(get_type_hints(foo, globals(), locals()),
                          {'a': Tuple[T]})
 
+    def test_forward_recursion_actually(self):
+        def namespace1():
+            a = typing.ForwardRef('A')
+            A = a
+            def fun(x: a): pass
+
+            ret = get_type_hints(fun, globals(), locals())
+            return a
+
+        def namespace2():
+            a = typing.ForwardRef('A')
+            A = a
+            def fun(x: a): pass
+
+            ret = get_type_hints(fun, globals(), locals())
+            return a
+
+        def cmp(o1, o2):
+            return o1 == o2
+
+        r1 = namespace1()
+        r2 = namespace2()
+        self.assertIsNot(r1, r2)
+        self.assertRaises(RecursionError, cmp, r1, r2)
+
+    def test_union_forward_recursion(self):
+        ValueList = List['Value']
+        Value = Union[str, ValueList]
+
+        class C:
+            foo: List[Value]
+        class D:
+            foo: Union[Value, ValueList]
+        class E:
+            foo: Union[List[Value], ValueList]
+        class F:
+            foo: Union[Value, List[Value], ValueList]
+
+        self.assertEqual(get_type_hints(C, globals(), locals()), get_type_hints(C, globals(), locals()))
+        self.assertEqual(get_type_hints(C, globals(), locals()),
+                         {'foo': List[Union[str, List[Union[str, List['Value']]]]]})
+        self.assertEqual(get_type_hints(D, globals(), locals()),
+                         {'foo': Union[str, List[Union[str, List['Value']]]]})
+        self.assertEqual(get_type_hints(E, globals(), locals()),
+                         {'foo': Union[
+                             List[Union[str, List[Union[str, List['Value']]]]],
+                             List[Union[str, List['Value']]]
+                         ]
+                          })
+        self.assertEqual(get_type_hints(F, globals(), locals()),
+                         {'foo': Union[
+                             str,
+                             List[Union[str, List['Value']]],
+                             List[Union[str, List[Union[str, List['Value']]]]]
+                         ]
+                          })
+
     def test_callable_forward(self):
 
         def foo(a: Callable[['T'], 'T']):
diff --git a/Lib/typing.py b/Lib/typing.py
index 7b07112c3abf..32011332127f 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -524,11 +524,13 @@ def _evaluate(self, globalns, localns):
     def __eq__(self, other):
         if not isinstance(other, ForwardRef):
             return NotImplemented
-        return (self.__forward_arg__ == other.__forward_arg__ and
-                self.__forward_value__ == other.__forward_value__)
+        if self.__forward_evaluated__ and other.__forward_evaluated__:
+            return (self.__forward_arg__ == other.__forward_arg__ and
+                    self.__forward_value__ == other.__forward_value__)
+        return self.__forward_arg__ == other.__forward_arg__
 
     def __hash__(self):
-        return hash((self.__forward_arg__, self.__forward_value__))
+        return hash(self.__forward_arg__)
 
     def __repr__(self):
         return f'ForwardRef({self.__forward_arg__!r})'
diff --git a/Misc/NEWS.d/next/Library/2019-09-06-17-40-34.bpo-37953.db5FQq.rst b/Misc/NEWS.d/next/Library/2019-09-06-17-40-34.bpo-37953.db5FQq.rst
new file mode 100644
index 000000000000..4eff4f7479aa
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-06-17-40-34.bpo-37953.db5FQq.rst
@@ -0,0 +1,2 @@
+In :mod:`typing`, improved the ``__hash__`` and ``__eq__`` methods for
+:class:`ForwardReferences`.


From webhook-mailer at python.org  Fri Sep 13 20:35:49 2019
From: webhook-mailer at python.org (Nick Coghlan)
Date: Sat, 14 Sep 2019 00:35:49 -0000
Subject: [Python-checkins] bpo-33095: Add reference to isolated mode in -m
 and script option (GH-7764)
Message-ID: <mailman.740.1568421350.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/bdd6945d4dbd1fe6a7fcff95f7d6908db7d791a1
commit: bdd6945d4dbd1fe6a7fcff95f7d6908db7d791a1
branch: master
author: Xtreak <tir.karthi at gmail.com>
committer: Nick Coghlan <ncoghlan at gmail.com>
date: 2019-09-14T10:35:44+10:00
summary:

bpo-33095: Add reference to isolated mode in -m and script option (GH-7764)

Attempt to make isolated mode easier to discover via additional inline documentation.

Co-Authored-By: Julien Palard <julien at palard.fr>

files:
M Doc/using/cmdline.rst

diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
index d04c3b61dd5b..457a7fcb6d9e 100644
--- a/Doc/using/cmdline.rst
+++ b/Doc/using/cmdline.rst
@@ -101,6 +101,11 @@ source.
    first element will be set to ``"-m"``). As with the :option:`-c` option,
    the current directory will be added to the start of :data:`sys.path`.
 
+   :option:`-I` option can  be used to run the script in isolated mode where
+   :data:`sys.path` contains neither the current directory nor the user's
+   site-packages directory. All :envvar:`PYTHON*` environment variables are
+   ignored, too.
+
    Many standard library modules contain code that is invoked on their execution
    as a script.  An example is the :mod:`timeit` module::
 
@@ -121,6 +126,7 @@ source.
    .. versionchanged:: 3.4
       namespace packages are also supported
 
+.. _cmdarg-dash:
 
 .. describe:: -
 
@@ -133,6 +139,8 @@ source.
 
    .. audit-event:: cpython.run_stdin "" ""
 
+.. _cmdarg-script:
+
 .. describe:: <script>
 
    Execute the Python code contained in *script*, which must be a filesystem
@@ -151,6 +159,11 @@ source.
    added to the start of :data:`sys.path` and the ``__main__.py`` file in
    that location is executed as the :mod:`__main__` module.
 
+   :option:`-I` option can  be used to run the script in isolated mode where
+   :data:`sys.path` contains neither the script's directory nor the user's
+   site-packages directory. All :envvar:`PYTHON*` environment variables are
+   ignored, too.
+
    .. audit-event:: cpython.run_file filename
 
    .. seealso::


From webhook-mailer at python.org  Sat Sep 14 02:23:24 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 14 Sep 2019 06:23:24 -0000
Subject: [Python-checkins] Doc: Fix link to window.getch in curses
 documentation (GH-16132)
Message-ID: <mailman.741.1568442204.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4fac58165b27186482599b99b55347d99f2b44c5
commit: 4fac58165b27186482599b99b55347d99f2b44c5
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-13T23:23:20-07:00
summary:

Doc: Fix link to window.getch in curses documentation (GH-16132)

(cherry picked from commit a26ace19bddea2d7a999a6de8286b3f27b132f35)

Co-authored-by: Anthony Sottile <asottile at umich.edu>

files:
M Doc/library/curses.rst

diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst
index 2a2ee2be84af..3e4856e14d34 100644
--- a/Doc/library/curses.rst
+++ b/Doc/library/curses.rst
@@ -1276,7 +1276,7 @@ The :mod:`curses` module defines the following data members:
 
 .. data:: ERR
 
-   Some curses routines  that  return  an integer, such as  :func:`getch`, return
+   Some curses routines  that  return  an integer, such as :meth:`~window.getch`, return
    :const:`ERR` upon failure.
 
 


From webhook-mailer at python.org  Sat Sep 14 03:43:02 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 14 Sep 2019 07:43:02 -0000
Subject: [Python-checkins] bpo-37953: Fix deprecation warnings in
 test_typing (GH-16133)
Message-ID: <mailman.742.1568446983.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d057b896f97e6d7447b9bf9246770c41cf205299
commit: d057b896f97e6d7447b9bf9246770c41cf205299
branch: master
author: Zackery Spytz <zspytz at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-14T00:42:56-07:00
summary:

bpo-37953: Fix deprecation warnings in test_typing (GH-16133)



self.assertEquals() is deprecated.

```
./python -We -m test test_typing
Run tests sequentially
0:00:00 load avg: 0.23 [1/1] test_typing
test test_typing failed -- Traceback (most recent call last):
  File "/home/lubuntu2/cpython/Lib/test/test_typing.py", line 2382, in test_forward_equality_gth
    self.assertEquals(Union[c1, c1_gth], Union[c1])
  File "/home/lubuntu2/cpython/Lib/unittest/case.py", line 1390, in deprecated_func
    warnings.warn(
DeprecationWarning: Please use assertEqual instead.

test_typing failed

== Tests result: FAILURE ==

1 test failed:
    test_typing

Total duration: 140 ms
Tests result: FAILURE
```



https://bugs.python.org/issue37953

files:
M Lib/test/test_typing.py

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index 28b3d316d350..59b1b4e6d548 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -2379,8 +2379,8 @@ def foo(a: c1_gth, b: c2_gth):
         self.assertEqual(List[c1], List[c1_gth])
         self.assertNotEqual(List[c1], List[C])
         self.assertNotEqual(List[c1_gth], List[C])
-        self.assertEquals(Union[c1, c1_gth], Union[c1])
-        self.assertEquals(Union[c1, c1_gth, int], Union[c1, int])
+        self.assertEqual(Union[c1, c1_gth], Union[c1])
+        self.assertEqual(Union[c1, c1_gth, int], Union[c1, int])
 
     def test_forward_equality_hash(self):
         c1 = typing.ForwardRef('int')


From webhook-mailer at python.org  Sat Sep 14 04:02:23 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 14 Sep 2019 08:02:23 -0000
Subject: [Python-checkins] bpo-37953: Fix deprecation warnings in
 test_typing (GH-16133)
Message-ID: <mailman.743.1568448144.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/66da347ef0034ad9bddc7fad112025c886249f0d
commit: 66da347ef0034ad9bddc7fad112025c886249f0d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-14T01:02:19-07:00
summary:

bpo-37953: Fix deprecation warnings in test_typing (GH-16133)


self.assertEquals() is deprecated.

```
./python -We -m test test_typing
Run tests sequentially
0:00:00 load avg: 0.23 [1/1] test_typing
test test_typing failed -- Traceback (most recent call last):
  File "/home/lubuntu2/cpython/Lib/test/test_typing.py", line 2382, in test_forward_equality_gth
    self.assertEquals(Union[c1, c1_gth], Union[c1])
  File "/home/lubuntu2/cpython/Lib/unittest/case.py", line 1390, in deprecated_func
    warnings.warn(
DeprecationWarning: Please use assertEqual instead.

test_typing failed

== Tests result: FAILURE ==

1 test failed:
    test_typing

Total duration: 140 ms
Tests result: FAILURE
```

https://bugs.python.org/issue37953
(cherry picked from commit d057b896f97e6d7447b9bf9246770c41cf205299)

Co-authored-by: Zackery Spytz <zspytz at gmail.com>

files:
M Lib/test/test_typing.py

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index 28b3d316d350..59b1b4e6d548 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -2379,8 +2379,8 @@ def foo(a: c1_gth, b: c2_gth):
         self.assertEqual(List[c1], List[c1_gth])
         self.assertNotEqual(List[c1], List[C])
         self.assertNotEqual(List[c1_gth], List[C])
-        self.assertEquals(Union[c1, c1_gth], Union[c1])
-        self.assertEquals(Union[c1, c1_gth, int], Union[c1, int])
+        self.assertEqual(Union[c1, c1_gth], Union[c1])
+        self.assertEqual(Union[c1, c1_gth, int], Union[c1, int])
 
     def test_forward_equality_hash(self):
         c1 = typing.ForwardRef('int')


From webhook-mailer at python.org  Sat Sep 14 05:24:10 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sat, 14 Sep 2019 09:24:10 -0000
Subject: [Python-checkins] bpo-37206: Unrepresentable default values no
 longer represented as None. (GH-13933)
Message-ID: <mailman.744.1568453052.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/279f44678c8b84a183f9eeb85e0b086228154497
commit: 279f44678c8b84a183f9eeb85e0b086228154497
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-14T12:24:05+03:00
summary:

bpo-37206: Unrepresentable default values no longer represented as None. (GH-13933)

In ArgumentClinic, value "NULL" should now be used only for unrepresentable default values
(like in the optional third parameter of getattr). "None" should be used if None is accepted
as argument and passing None has the same effect as not passing the argument at all.

files:
A Misc/NEWS.d/next/Library/2019-06-09-22-25-03.bpo-37206.2WBg4q.rst
M Lib/test/clinic.test
M Lib/test/test_inspect.py
M Modules/_codecsmodule.c
M Modules/_cursesmodule.c
M Modules/_elementtree.c
M Modules/_hashopenssl.c
M Modules/_io/_iomodule.c
M Modules/_io/clinic/fileio.c.h
M Modules/_io/fileio.c
M Modules/_io/textio.c
M Modules/_pickle.c
M Modules/_ssl.c
M Modules/_tkinter.c
M Modules/cjkcodecs/multibytecodec.c
M Modules/clinic/_asynciomodule.c.h
M Modules/clinic/_codecsmodule.c.h
M Modules/clinic/_elementtree.c.h
M Modules/clinic/_pickle.c.h
M Modules/clinic/_ssl.c.h
M Modules/clinic/_tkinter.c.h
M Modules/clinic/binascii.c.h
M Modules/clinic/cmathmodule.c.h
M Modules/clinic/posixmodule.c.h
M Modules/clinic/pyexpat.c.h
M Modules/clinic/unicodedata.c.h
M Modules/cmathmodule.c
M Modules/posixmodule.c
M Modules/pyexpat.c
M Objects/bytesobject.c
M Objects/clinic/bytearrayobject.c.h
M Objects/clinic/bytesobject.c.h
M Objects/clinic/dictobject.c.h
M Objects/clinic/floatobject.c.h
M Objects/clinic/memoryobject.c.h
M Objects/clinic/unicodeobject.c.h
M Objects/dictobject.c
M Objects/floatobject.c
M Objects/structseq.c
M Objects/unicodeobject.c
M Python/bltinmodule.c
M Python/clinic/bltinmodule.c.h
M Python/clinic/context.c.h
M Python/clinic/import.c.h
M Python/clinic/sysmodule.c.h
M Python/sysmodule.c
M Tools/clinic/clinic.py

diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test
index f9c55e2a577f..0d84d5ef5359 100644
--- a/Lib/test/clinic.test
+++ b/Lib/test/clinic.test
@@ -92,7 +92,7 @@ test_objects_converter
 [clinic start generated code]*/
 
 PyDoc_STRVAR(test_objects_converter__doc__,
-"test_objects_converter($module, a, b=None, /)\n"
+"test_objects_converter($module, a, b=<unrepresentable>, /)\n"
 "--\n"
 "\n");
 
@@ -126,7 +126,7 @@ exit:
 
 static PyObject *
 test_objects_converter_impl(PyObject *module, PyObject *a, PyObject *b)
-/*[clinic end generated code: output=58009c0e42b4834e input=4cbb3d9edd2a36f3]*/
+/*[clinic end generated code: output=0e461f38d3b2bd08 input=4cbb3d9edd2a36f3]*/
 
 
 /*[clinic input]
@@ -1718,8 +1718,8 @@ test_str_converter
 [clinic start generated code]*/
 
 PyDoc_STRVAR(test_str_converter__doc__,
-"test_str_converter($module, a=None, b=\'ab\', c=\'cd\', d=\'cef\', e=\'gh\',\n"
-"                   f=\'ij\', g=\'kl\', h=\'mn\', /)\n"
+"test_str_converter($module, a=<unrepresentable>, b=\'ab\', c=\'cd\',\n"
+"                   d=\'cef\', e=\'gh\', f=\'ij\', g=\'kl\', h=\'mn\', /)\n"
 "--\n"
 "\n");
 
@@ -1765,7 +1765,7 @@ test_str_converter_impl(PyObject *module, const char *a, const char *b,
                         const char *f, Py_ssize_clean_t f_length,
                         const char *g, Py_ssize_clean_t g_length,
                         const char *h, Py_ssize_clean_t h_length)
-/*[clinic end generated code: output=8415d82c56154307 input=8afe9da8185cd38c]*/
+/*[clinic end generated code: output=ad868ad94a488e32 input=8afe9da8185cd38c]*/
 
 
 /*[clinic input]
diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py
index 7148dfa0bab9..9d28d5ad570d 100644
--- a/Lib/test/test_inspect.py
+++ b/Lib/test/test_inspect.py
@@ -2269,8 +2269,8 @@ def p(name): return signature.parameters[name].default
         test_callable(d.dump)
 
         # static method
-        test_callable(str.maketrans)
-        test_callable('abc'.maketrans)
+        test_callable(bytes.maketrans)
+        test_callable(b'abc'.maketrans)
 
         # class method
         test_callable(dict.fromkeys)
diff --git a/Misc/NEWS.d/next/Library/2019-06-09-22-25-03.bpo-37206.2WBg4q.rst b/Misc/NEWS.d/next/Library/2019-06-09-22-25-03.bpo-37206.2WBg4q.rst
new file mode 100644
index 000000000000..2c6c6821e639
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-09-22-25-03.bpo-37206.2WBg4q.rst
@@ -0,0 +1,2 @@
+Default values which cannot be represented as Python objects no longer
+improperly represented as ``None`` in function signatures.
diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c
index 90b3e37d1641..a8ffb699557a 100644
--- a/Modules/_codecsmodule.c
+++ b/Modules/_codecsmodule.c
@@ -171,14 +171,14 @@ PyObject *codec_tuple(PyObject *decoded,
 /*[clinic input]
 _codecs.escape_decode
     data: Py_buffer(accept={str, buffer})
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_escape_decode_impl(PyObject *module, Py_buffer *data,
                            const char *errors)
-/*[clinic end generated code: output=505200ba8056979a input=0018edfd99db714d]*/
+/*[clinic end generated code: output=505200ba8056979a input=77298a561c90bd82]*/
 {
     PyObject *decoded = PyBytes_DecodeEscape(data->buf, data->len,
                                              errors, 0, NULL);
@@ -188,14 +188,14 @@ _codecs_escape_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.escape_encode
     data: object(subclass_of='&PyBytes_Type')
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_escape_encode_impl(PyObject *module, PyObject *data,
                            const char *errors)
-/*[clinic end generated code: output=4af1d477834bab34 input=da9ded00992f32f2]*/
+/*[clinic end generated code: output=4af1d477834bab34 input=8f4b144799a94245]*/
 {
     Py_ssize_t size;
     Py_ssize_t newsize;
@@ -252,7 +252,7 @@ _codecs_escape_encode_impl(PyObject *module, PyObject *data,
 /*[clinic input]
 _codecs.utf_7_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -260,7 +260,7 @@ _codecs.utf_7_decode
 static PyObject *
 _codecs_utf_7_decode_impl(PyObject *module, Py_buffer *data,
                           const char *errors, int final)
-/*[clinic end generated code: output=0cd3a944a32a4089 input=2d94a5a1f170c8ae]*/
+/*[clinic end generated code: output=0cd3a944a32a4089 input=22c395d357815d26]*/
 {
     Py_ssize_t consumed = data->len;
     PyObject *decoded = PyUnicode_DecodeUTF7Stateful(data->buf, data->len,
@@ -272,7 +272,7 @@ _codecs_utf_7_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_8_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -280,7 +280,7 @@ _codecs.utf_8_decode
 static PyObject *
 _codecs_utf_8_decode_impl(PyObject *module, Py_buffer *data,
                           const char *errors, int final)
-/*[clinic end generated code: output=10f74dec8d9bb8bf input=1ea6c21492e8bcbe]*/
+/*[clinic end generated code: output=10f74dec8d9bb8bf input=f611b3867352ba59]*/
 {
     Py_ssize_t consumed = data->len;
     PyObject *decoded = PyUnicode_DecodeUTF8Stateful(data->buf, data->len,
@@ -292,7 +292,7 @@ _codecs_utf_8_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_16_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -300,7 +300,7 @@ _codecs.utf_16_decode
 static PyObject *
 _codecs_utf_16_decode_impl(PyObject *module, Py_buffer *data,
                            const char *errors, int final)
-/*[clinic end generated code: output=783b442abcbcc2d0 input=2ba128c28ea0bb40]*/
+/*[clinic end generated code: output=783b442abcbcc2d0 input=191d360bd7309180]*/
 {
     int byteorder = 0;
     /* This is overwritten unless final is true. */
@@ -314,7 +314,7 @@ _codecs_utf_16_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_16_le_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -322,7 +322,7 @@ _codecs.utf_16_le_decode
 static PyObject *
 _codecs_utf_16_le_decode_impl(PyObject *module, Py_buffer *data,
                               const char *errors, int final)
-/*[clinic end generated code: output=899b9e6364379dcd input=43aeb8b0461cace5]*/
+/*[clinic end generated code: output=899b9e6364379dcd input=c6904fdc27fb4724]*/
 {
     int byteorder = -1;
     /* This is overwritten unless final is true. */
@@ -336,7 +336,7 @@ _codecs_utf_16_le_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_16_be_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -344,7 +344,7 @@ _codecs.utf_16_be_decode
 static PyObject *
 _codecs_utf_16_be_decode_impl(PyObject *module, Py_buffer *data,
                               const char *errors, int final)
-/*[clinic end generated code: output=49f6465ea07669c8 input=339e554c804f34b2]*/
+/*[clinic end generated code: output=49f6465ea07669c8 input=e49012400974649b]*/
 {
     int byteorder = 1;
     /* This is overwritten unless final is true. */
@@ -365,7 +365,7 @@ _codecs_utf_16_be_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_16_ex_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     byteorder: int = 0
     final: bool(accept={int}) = False
     /
@@ -374,7 +374,7 @@ _codecs.utf_16_ex_decode
 static PyObject *
 _codecs_utf_16_ex_decode_impl(PyObject *module, Py_buffer *data,
                               const char *errors, int byteorder, int final)
-/*[clinic end generated code: output=0f385f251ecc1988 input=3201aeddb9636889]*/
+/*[clinic end generated code: output=0f385f251ecc1988 input=5a9c19f2e6b6cf0e]*/
 {
     /* This is overwritten unless final is true. */
     Py_ssize_t consumed = data->len;
@@ -390,7 +390,7 @@ _codecs_utf_16_ex_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_32_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -398,7 +398,7 @@ _codecs.utf_32_decode
 static PyObject *
 _codecs_utf_32_decode_impl(PyObject *module, Py_buffer *data,
                            const char *errors, int final)
-/*[clinic end generated code: output=2fc961807f7b145f input=155a5c673a4e2514]*/
+/*[clinic end generated code: output=2fc961807f7b145f input=fd7193965627eb58]*/
 {
     int byteorder = 0;
     /* This is overwritten unless final is true. */
@@ -412,7 +412,7 @@ _codecs_utf_32_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_32_le_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -420,7 +420,7 @@ _codecs.utf_32_le_decode
 static PyObject *
 _codecs_utf_32_le_decode_impl(PyObject *module, Py_buffer *data,
                               const char *errors, int final)
-/*[clinic end generated code: output=ec8f46b67a94f3e6 input=7baf061069e92d3b]*/
+/*[clinic end generated code: output=ec8f46b67a94f3e6 input=9078ec70acfe7613]*/
 {
     int byteorder = -1;
     /* This is overwritten unless final is true. */
@@ -434,7 +434,7 @@ _codecs_utf_32_le_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_32_be_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -442,7 +442,7 @@ _codecs.utf_32_be_decode
 static PyObject *
 _codecs_utf_32_be_decode_impl(PyObject *module, Py_buffer *data,
                               const char *errors, int final)
-/*[clinic end generated code: output=ff82bae862c92c4e input=b182026300dae595]*/
+/*[clinic end generated code: output=ff82bae862c92c4e input=f1ae1bbbb86648ff]*/
 {
     int byteorder = 1;
     /* This is overwritten unless final is true. */
@@ -463,7 +463,7 @@ _codecs_utf_32_be_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_32_ex_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     byteorder: int = 0
     final: bool(accept={int}) = False
     /
@@ -472,7 +472,7 @@ _codecs.utf_32_ex_decode
 static PyObject *
 _codecs_utf_32_ex_decode_impl(PyObject *module, Py_buffer *data,
                               const char *errors, int byteorder, int final)
-/*[clinic end generated code: output=6bfb177dceaf4848 input=7b9c2cb819fb237a]*/
+/*[clinic end generated code: output=6bfb177dceaf4848 input=e46a73bc859d0bd0]*/
 {
     Py_ssize_t consumed = data->len;
     PyObject *decoded = PyUnicode_DecodeUTF32Stateful(data->buf, data->len,
@@ -486,14 +486,14 @@ _codecs_utf_32_ex_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.unicode_escape_decode
     data: Py_buffer(accept={str, buffer})
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
                                    const char *errors)
-/*[clinic end generated code: output=3ca3c917176b82ab input=49fd27d06813a7f5]*/
+/*[clinic end generated code: output=3ca3c917176b82ab input=8328081a3a569bd6]*/
 {
     PyObject *decoded = PyUnicode_DecodeUnicodeEscape(data->buf, data->len,
                                                       errors);
@@ -503,14 +503,14 @@ _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.raw_unicode_escape_decode
     data: Py_buffer(accept={str, buffer})
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_raw_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
                                        const char *errors)
-/*[clinic end generated code: output=c98eeb56028070a6 input=770903a211434ebc]*/
+/*[clinic end generated code: output=c98eeb56028070a6 input=d2f5159ce3b3392f]*/
 {
     PyObject *decoded = PyUnicode_DecodeRawUnicodeEscape(data->buf, data->len,
                                                          errors);
@@ -520,14 +520,14 @@ _codecs_raw_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.latin_1_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_latin_1_decode_impl(PyObject *module, Py_buffer *data,
                             const char *errors)
-/*[clinic end generated code: output=07f3dfa3f72c7d8f input=5cad0f1759c618ec]*/
+/*[clinic end generated code: output=07f3dfa3f72c7d8f input=76ca58fd6dcd08c7]*/
 {
     PyObject *decoded = PyUnicode_DecodeLatin1(data->buf, data->len, errors);
     return codec_tuple(decoded, data->len);
@@ -536,14 +536,14 @@ _codecs_latin_1_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.ascii_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_ascii_decode_impl(PyObject *module, Py_buffer *data,
                           const char *errors)
-/*[clinic end generated code: output=2627d72058d42429 input=ad1106f64037bd16]*/
+/*[clinic end generated code: output=2627d72058d42429 input=e428a267a04b4481]*/
 {
     PyObject *decoded = PyUnicode_DecodeASCII(data->buf, data->len, errors);
     return codec_tuple(decoded, data->len);
@@ -552,15 +552,15 @@ _codecs_ascii_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.charmap_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
-    mapping: object = NULL
+    errors: str(accept={str, NoneType}) = None
+    mapping: object = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_charmap_decode_impl(PyObject *module, Py_buffer *data,
                             const char *errors, PyObject *mapping)
-/*[clinic end generated code: output=2c335b09778cf895 input=19712ca35c5a80e2]*/
+/*[clinic end generated code: output=2c335b09778cf895 input=15b69df43458eb40]*/
 {
     PyObject *decoded;
 
@@ -576,7 +576,7 @@ _codecs_charmap_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.mbcs_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -584,7 +584,7 @@ _codecs.mbcs_decode
 static PyObject *
 _codecs_mbcs_decode_impl(PyObject *module, Py_buffer *data,
                          const char *errors, int final)
-/*[clinic end generated code: output=39b65b8598938c4b input=b5f2fe568f311297]*/
+/*[clinic end generated code: output=39b65b8598938c4b input=1c1d50f08fa53789]*/
 {
     Py_ssize_t consumed = data->len;
     PyObject *decoded = PyUnicode_DecodeMBCSStateful(data->buf, data->len,
@@ -595,7 +595,7 @@ _codecs_mbcs_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.oem_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -603,7 +603,7 @@ _codecs.oem_decode
 static PyObject *
 _codecs_oem_decode_impl(PyObject *module, Py_buffer *data,
                         const char *errors, int final)
-/*[clinic end generated code: output=da1617612f3fcad8 input=278709bcfd374a9c]*/
+/*[clinic end generated code: output=da1617612f3fcad8 input=81b67cba811022e5]*/
 {
     Py_ssize_t consumed = data->len;
     PyObject *decoded = PyUnicode_DecodeCodePageStateful(CP_OEMCP,
@@ -615,7 +615,7 @@ _codecs_oem_decode_impl(PyObject *module, Py_buffer *data,
 _codecs.code_page_decode
     codepage: int
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -623,7 +623,7 @@ _codecs.code_page_decode
 static PyObject *
 _codecs_code_page_decode_impl(PyObject *module, int codepage,
                               Py_buffer *data, const char *errors, int final)
-/*[clinic end generated code: output=53008ea967da3fff input=51f6169021c68dd5]*/
+/*[clinic end generated code: output=53008ea967da3fff input=c5f58d036cb63575]*/
 {
     Py_ssize_t consumed = data->len;
     PyObject *decoded = PyUnicode_DecodeCodePageStateful(codepage,
@@ -640,14 +640,14 @@ _codecs_code_page_decode_impl(PyObject *module, int codepage,
 /*[clinic input]
 _codecs.readbuffer_encode
     data: Py_buffer(accept={str, buffer})
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_readbuffer_encode_impl(PyObject *module, Py_buffer *data,
                                const char *errors)
-/*[clinic end generated code: output=c645ea7cdb3d6e86 input=b7c322b89d4ab923]*/
+/*[clinic end generated code: output=c645ea7cdb3d6e86 input=aa10cfdf252455c5]*/
 {
     PyObject *result = PyBytes_FromStringAndSize(data->buf, data->len);
     return codec_tuple(result, data->len);
@@ -656,14 +656,14 @@ _codecs_readbuffer_encode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_7_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_utf_7_encode_impl(PyObject *module, PyObject *str,
                           const char *errors)
-/*[clinic end generated code: output=0feda21ffc921bc8 input=d1a47579e79cbe15]*/
+/*[clinic end generated code: output=0feda21ffc921bc8 input=2546dbbb3fa53114]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF7(str, 0, 0, errors),
                        PyUnicode_GET_LENGTH(str));
@@ -672,14 +672,14 @@ _codecs_utf_7_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_8_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_utf_8_encode_impl(PyObject *module, PyObject *str,
                           const char *errors)
-/*[clinic end generated code: output=02bf47332b9c796c input=42e3ba73c4392eef]*/
+/*[clinic end generated code: output=02bf47332b9c796c input=a3e71ae01c3f93f3]*/
 {
     return codec_tuple(_PyUnicode_AsUTF8String(str, errors),
                        PyUnicode_GET_LENGTH(str));
@@ -695,7 +695,7 @@ _codecs_utf_8_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_16_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     byteorder: int = 0
     /
 [clinic start generated code]*/
@@ -703,7 +703,7 @@ _codecs.utf_16_encode
 static PyObject *
 _codecs_utf_16_encode_impl(PyObject *module, PyObject *str,
                            const char *errors, int byteorder)
-/*[clinic end generated code: output=c654e13efa2e64e4 input=ff46416b04edb944]*/
+/*[clinic end generated code: output=c654e13efa2e64e4 input=68cdc2eb8338555d]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF16(str, errors, byteorder),
                        PyUnicode_GET_LENGTH(str));
@@ -712,14 +712,14 @@ _codecs_utf_16_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_16_le_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_utf_16_le_encode_impl(PyObject *module, PyObject *str,
                               const char *errors)
-/*[clinic end generated code: output=431b01e55f2d4995 input=cb385455ea8f2fe0]*/
+/*[clinic end generated code: output=431b01e55f2d4995 input=83d042706eed6798]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF16(str, errors, -1),
                        PyUnicode_GET_LENGTH(str));
@@ -728,14 +728,14 @@ _codecs_utf_16_le_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_16_be_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_utf_16_be_encode_impl(PyObject *module, PyObject *str,
                               const char *errors)
-/*[clinic end generated code: output=96886a6fd54dcae3 input=9119997066bdaefd]*/
+/*[clinic end generated code: output=96886a6fd54dcae3 input=6f1e9e623b03071b]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF16(str, errors, +1),
                        PyUnicode_GET_LENGTH(str));
@@ -751,7 +751,7 @@ _codecs_utf_16_be_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_32_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     byteorder: int = 0
     /
 [clinic start generated code]*/
@@ -759,7 +759,7 @@ _codecs.utf_32_encode
 static PyObject *
 _codecs_utf_32_encode_impl(PyObject *module, PyObject *str,
                            const char *errors, int byteorder)
-/*[clinic end generated code: output=5c760da0c09a8b83 input=c5e77da82fbe5c2a]*/
+/*[clinic end generated code: output=5c760da0c09a8b83 input=8ec4c64d983bc52b]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF32(str, errors, byteorder),
                        PyUnicode_GET_LENGTH(str));
@@ -768,14 +768,14 @@ _codecs_utf_32_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_32_le_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_utf_32_le_encode_impl(PyObject *module, PyObject *str,
                               const char *errors)
-/*[clinic end generated code: output=b65cd176de8e36d6 input=9993b25fe0877848]*/
+/*[clinic end generated code: output=b65cd176de8e36d6 input=f0918d41de3eb1b1]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF32(str, errors, -1),
                        PyUnicode_GET_LENGTH(str));
@@ -784,14 +784,14 @@ _codecs_utf_32_le_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_32_be_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_utf_32_be_encode_impl(PyObject *module, PyObject *str,
                               const char *errors)
-/*[clinic end generated code: output=1d9e71a9358709e9 input=d3e0ccaa02920431]*/
+/*[clinic end generated code: output=1d9e71a9358709e9 input=967a99a95748b557]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF32(str, errors, +1),
                        PyUnicode_GET_LENGTH(str));
@@ -800,14 +800,14 @@ _codecs_utf_32_be_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.unicode_escape_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_unicode_escape_encode_impl(PyObject *module, PyObject *str,
                                    const char *errors)
-/*[clinic end generated code: output=66271b30bc4f7a3c input=65d9eefca65b455a]*/
+/*[clinic end generated code: output=66271b30bc4f7a3c input=8c4de07597054e33]*/
 {
     return codec_tuple(PyUnicode_AsUnicodeEscapeString(str),
                        PyUnicode_GET_LENGTH(str));
@@ -816,14 +816,14 @@ _codecs_unicode_escape_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.raw_unicode_escape_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_raw_unicode_escape_encode_impl(PyObject *module, PyObject *str,
                                        const char *errors)
-/*[clinic end generated code: output=a66a806ed01c830a input=5aa33e4a133391ab]*/
+/*[clinic end generated code: output=a66a806ed01c830a input=4aa6f280d78e4574]*/
 {
     return codec_tuple(PyUnicode_AsRawUnicodeEscapeString(str),
                        PyUnicode_GET_LENGTH(str));
@@ -832,14 +832,14 @@ _codecs_raw_unicode_escape_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.latin_1_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_latin_1_encode_impl(PyObject *module, PyObject *str,
                             const char *errors)
-/*[clinic end generated code: output=2c28c83a27884e08 input=30b11c9e49a65150]*/
+/*[clinic end generated code: output=2c28c83a27884e08 input=ec3ef74bf85c5c5d]*/
 {
     return codec_tuple(_PyUnicode_AsLatin1String(str, errors),
                        PyUnicode_GET_LENGTH(str));
@@ -848,14 +848,14 @@ _codecs_latin_1_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.ascii_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_ascii_encode_impl(PyObject *module, PyObject *str,
                           const char *errors)
-/*[clinic end generated code: output=b5e035182d33befc input=843a1d268e6dfa8e]*/
+/*[clinic end generated code: output=b5e035182d33befc input=93e6e602838bd3de]*/
 {
     return codec_tuple(_PyUnicode_AsASCIIString(str, errors),
                        PyUnicode_GET_LENGTH(str));
@@ -864,15 +864,15 @@ _codecs_ascii_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.charmap_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
-    mapping: object = NULL
+    errors: str(accept={str, NoneType}) = None
+    mapping: object = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_charmap_encode_impl(PyObject *module, PyObject *str,
                             const char *errors, PyObject *mapping)
-/*[clinic end generated code: output=047476f48495a9e9 input=0752cde07a6d6d00]*/
+/*[clinic end generated code: output=047476f48495a9e9 input=2a98feae73dadce8]*/
 {
     if (mapping == Py_None)
         mapping = NULL;
@@ -899,13 +899,13 @@ _codecs_charmap_build_impl(PyObject *module, PyObject *map)
 /*[clinic input]
 _codecs.mbcs_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_mbcs_encode_impl(PyObject *module, PyObject *str, const char *errors)
-/*[clinic end generated code: output=76e2e170c966c080 input=de471e0815947553]*/
+/*[clinic end generated code: output=76e2e170c966c080 input=2e932fc289ea5a5b]*/
 {
     return codec_tuple(PyUnicode_EncodeCodePage(CP_ACP, str, errors),
                        PyUnicode_GET_LENGTH(str));
@@ -914,13 +914,13 @@ _codecs_mbcs_encode_impl(PyObject *module, PyObject *str, const char *errors)
 /*[clinic input]
 _codecs.oem_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_oem_encode_impl(PyObject *module, PyObject *str, const char *errors)
-/*[clinic end generated code: output=65d5982c737de649 input=3fc5f0028aad3cda]*/
+/*[clinic end generated code: output=65d5982c737de649 input=9eac86dc21eb14f2]*/
 {
     return codec_tuple(PyUnicode_EncodeCodePage(CP_OEMCP, str, errors),
         PyUnicode_GET_LENGTH(str));
@@ -930,14 +930,14 @@ _codecs_oem_encode_impl(PyObject *module, PyObject *str, const char *errors)
 _codecs.code_page_encode
     code_page: int
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_code_page_encode_impl(PyObject *module, int code_page, PyObject *str,
                               const char *errors)
-/*[clinic end generated code: output=45673f6085657a9e input=786421ae617d680b]*/
+/*[clinic end generated code: output=45673f6085657a9e input=7d18a33bc8cd0f94]*/
 {
     return codec_tuple(PyUnicode_EncodeCodePage(code_page, str, errors),
                        PyUnicode_GET_LENGTH(str));
diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c
index c66f7943b695..98fd1f33b62f 100644
--- a/Modules/_cursesmodule.c
+++ b/Modules/_cursesmodule.c
@@ -3202,7 +3202,7 @@ _curses_initscr_impl(PyObject *module)
 /*[clinic input]
 _curses.setupterm
 
-    term: str(accept={str, NoneType}) = NULL
+    term: str(accept={str, NoneType}) = None
         Terminal name.
         If omitted, the value of the TERM environment variable will be used.
     fd: int = -1
@@ -3214,7 +3214,7 @@ Initialize the terminal.
 
 static PyObject *
 _curses_setupterm_impl(PyObject *module, const char *term, int fd)
-/*[clinic end generated code: output=4584e587350f2848 input=8ac5f78ec6268be3]*/
+/*[clinic end generated code: output=4584e587350f2848 input=4511472766af0c12]*/
 {
     int err;
 
diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
index b88e8a1d6e9e..9050144b70f3 100644
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -2408,10 +2408,10 @@ treebuilder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 /*[clinic input]
 _elementtree.TreeBuilder.__init__
 
-    element_factory: object = NULL
+    element_factory: object = None
     *
-    comment_factory: object = NULL
-    pi_factory: object = NULL
+    comment_factory: object = None
+    pi_factory: object = None
     insert_comments: bool = False
     insert_pis: bool = False
 
@@ -2423,16 +2423,16 @@ _elementtree_TreeBuilder___init___impl(TreeBuilderObject *self,
                                        PyObject *comment_factory,
                                        PyObject *pi_factory,
                                        int insert_comments, int insert_pis)
-/*[clinic end generated code: output=8571d4dcadfdf952 input=1f967b5c245e0a71]*/
+/*[clinic end generated code: output=8571d4dcadfdf952 input=ae98a94df20b5cc3]*/
 {
-    if (element_factory && element_factory != Py_None) {
+    if (element_factory != Py_None) {
         Py_INCREF(element_factory);
         Py_XSETREF(self->element_factory, element_factory);
     } else {
         Py_CLEAR(self->element_factory);
     }
 
-    if (!comment_factory || comment_factory == Py_None) {
+    if (comment_factory == Py_None) {
         elementtreestate *st = ET_STATE_GLOBAL;
         comment_factory = st->comment_factory;
     }
@@ -2445,7 +2445,7 @@ _elementtree_TreeBuilder___init___impl(TreeBuilderObject *self,
         self->insert_comments = 0;
     }
 
-    if (!pi_factory || pi_factory == Py_None) {
+    if (pi_factory == Py_None) {
         elementtreestate *st = ET_STATE_GLOBAL;
         pi_factory = st->pi_factory;
     }
@@ -3651,14 +3651,14 @@ _elementtree.XMLParser.__init__
 
     *
     target: object = NULL
-    encoding: str(accept={str, NoneType}) = NULL
+    encoding: str(accept={str, NoneType}) = None
 
 [clinic start generated code]*/
 
 static int
 _elementtree_XMLParser___init___impl(XMLParserObject *self, PyObject *target,
                                      const char *encoding)
-/*[clinic end generated code: output=3ae45ec6cdf344e4 input=96288fcba916cfce]*/
+/*[clinic end generated code: output=3ae45ec6cdf344e4 input=53e35a829ae043e8]*/
 {
     self->entity = PyDict_New();
     if (!self->entity)
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 29ebec77a4b3..aa91be2f2e3c 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -584,7 +584,7 @@ EVPnew(const EVP_MD *digest,
 _hashlib.new as EVP_new
 
     name as name_obj: object
-    string as data_obj: object(py_default="b''") = NULL
+    string as data_obj: object(c_default="NULL") = b''
     *
     usedforsecurity: bool = True
 
@@ -599,7 +599,7 @@ The MD5 and SHA1 algorithms are always supported.
 static PyObject *
 EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj,
              int usedforsecurity)
-/*[clinic end generated code: output=ddd5053f92dffe90 input=e9ac115d80962ddf]*/
+/*[clinic end generated code: output=ddd5053f92dffe90 input=c24554d0337be1b0]*/
 {
     Py_buffer view = { 0 };
     PyObject *ret_obj;
diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c
index 162b288852b3..b4cc89400402 100644
--- a/Modules/_io/_iomodule.c
+++ b/Modules/_io/_iomodule.c
@@ -103,9 +103,9 @@ _io.open
     file: object
     mode: str = "r"
     buffering: int = -1
-    encoding: str(accept={str, NoneType}) = NULL
-    errors: str(accept={str, NoneType}) = NULL
-    newline: str(accept={str, NoneType}) = NULL
+    encoding: str(accept={str, NoneType}) = None
+    errors: str(accept={str, NoneType}) = None
+    newline: str(accept={str, NoneType}) = None
     closefd: bool(accept={int}) = True
     opener: object = None
 
@@ -233,7 +233,7 @@ static PyObject *
 _io_open_impl(PyObject *module, PyObject *file, const char *mode,
               int buffering, const char *encoding, const char *errors,
               const char *newline, int closefd, PyObject *opener)
-/*[clinic end generated code: output=aefafc4ce2b46dc0 input=03da2940c8a65871]*/
+/*[clinic end generated code: output=aefafc4ce2b46dc0 input=7295902222e6b311]*/
 {
     unsigned i;
 
diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h
index 64addde57f3e..53e7067cf7a7 100644
--- a/Modules/_io/clinic/fileio.c.h
+++ b/Modules/_io/clinic/fileio.c.h
@@ -408,7 +408,7 @@ static PyObject *
 _io_FileIO_truncate(fileio *self, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
-    PyObject *posobj = NULL;
+    PyObject *posobj = Py_None;
 
     if (!_PyArg_CheckPositional("truncate", nargs, 0, 1)) {
         goto exit;
@@ -447,4 +447,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored))
 #ifndef _IO_FILEIO_TRUNCATE_METHODDEF
     #define _IO_FILEIO_TRUNCATE_METHODDEF
 #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */
-/*[clinic end generated code: output=a7e9cca3613660fb input=a9049054013a1b77]*/
+/*[clinic end generated code: output=e7682d0a3264d284 input=a9049054013a1b77]*/
diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c
index e4cbbfa9bb8c..fd32b1921260 100644
--- a/Modules/_io/fileio.c
+++ b/Modules/_io/fileio.c
@@ -979,7 +979,7 @@ _io_FileIO_tell_impl(fileio *self)
 #ifdef HAVE_FTRUNCATE
 /*[clinic input]
 _io.FileIO.truncate
-    size as posobj: object = NULL
+    size as posobj: object = None
     /
 
 Truncate the file to at most size bytes and return the truncated size.
@@ -990,7 +990,7 @@ The current file position is changed to the value of size.
 
 static PyObject *
 _io_FileIO_truncate_impl(fileio *self, PyObject *posobj)
-/*[clinic end generated code: output=e49ca7a916c176fa input=9026af44686b7318]*/
+/*[clinic end generated code: output=e49ca7a916c176fa input=b0ac133939823875]*/
 {
     Py_off_t pos;
     int ret;
@@ -1002,7 +1002,7 @@ _io_FileIO_truncate_impl(fileio *self, PyObject *posobj)
     if (!self->writable)
         return err_mode("writing");
 
-    if (posobj == Py_None || posobj == NULL) {
+    if (posobj == Py_None) {
         /* Get the current position. */
         posobj = portable_lseek(self, NULL, 1);
         if (posobj == NULL)
diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c
index 1db8d94ed12a..5ebeb024e579 100644
--- a/Modules/_io/textio.c
+++ b/Modules/_io/textio.c
@@ -1031,9 +1031,9 @@ io_check_errors(PyObject *errors)
 /*[clinic input]
 _io.TextIOWrapper.__init__
     buffer: object
-    encoding: str(accept={str, NoneType}) = NULL
+    encoding: str(accept={str, NoneType}) = None
     errors: object = None
-    newline: str(accept={str, NoneType}) = NULL
+    newline: str(accept={str, NoneType}) = None
     line_buffering: bool(accept={int}) = False
     write_through: bool(accept={int}) = False
 
@@ -1072,7 +1072,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer,
                                 const char *encoding, PyObject *errors,
                                 const char *newline, int line_buffering,
                                 int write_through)
-/*[clinic end generated code: output=72267c0c01032ed2 input=1c5dd5d78bfcc675]*/
+/*[clinic end generated code: output=72267c0c01032ed2 input=77d8696d1a1f460b]*/
 {
     PyObject *raw, *codec_info = NULL;
     _PyIO_State *state = NULL;
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index 9e82d145cc66..7370be593883 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -1142,7 +1142,7 @@ _Pickler_SetProtocol(PicklerObject *self, PyObject *protocol, int fix_imports)
 {
     long proto;
 
-    if (protocol == NULL || protocol == Py_None) {
+    if (protocol == Py_None) {
         proto = DEFAULT_PROTOCOL;
     }
     else {
@@ -4638,9 +4638,9 @@ Pickler_clear(PicklerObject *self)
 _pickle.Pickler.__init__
 
   file: object
-  protocol: object = NULL
+  protocol: object = None
   fix_imports: bool = True
-  buffer_callback: object = NULL
+  buffer_callback: object = None
 
 This takes a binary file for writing a pickle data stream.
 
@@ -4678,7 +4678,7 @@ static int
 _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file,
                               PyObject *protocol, int fix_imports,
                               PyObject *buffer_callback)
-/*[clinic end generated code: output=0abedc50590d259b input=9a43a1c50ab91652]*/
+/*[clinic end generated code: output=0abedc50590d259b input=bb886e00443a7811]*/
 {
     _Py_IDENTIFIER(persistent_id);
     _Py_IDENTIFIER(dispatch_table);
@@ -7200,7 +7200,7 @@ _pickle.Unpickler.__init__
   fix_imports: bool = True
   encoding: str = 'ASCII'
   errors: str = 'strict'
-  buffers: object = NULL
+  buffers: object(c_default="NULL") = ()
 
 This takes a binary file for reading a pickle data stream.
 
@@ -7228,7 +7228,7 @@ static int
 _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file,
                                 int fix_imports, const char *encoding,
                                 const char *errors, PyObject *buffers)
-/*[clinic end generated code: output=09f0192649ea3f85 input=da4b62d9edb68700]*/
+/*[clinic end generated code: output=09f0192649ea3f85 input=ca4c1faea9553121]*/
 {
     _Py_IDENTIFIER(persistent_load);
 
@@ -7620,10 +7620,10 @@ _pickle.dump
 
   obj: object
   file: object
-  protocol: object = NULL
+  protocol: object = None
   *
   fix_imports: bool = True
-  buffer_callback: object = NULL
+  buffer_callback: object = None
 
 Write a pickled representation of obj to the open file object file.
 
@@ -7658,7 +7658,7 @@ static PyObject *
 _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file,
                   PyObject *protocol, int fix_imports,
                   PyObject *buffer_callback)
-/*[clinic end generated code: output=706186dba996490c input=2f035f02cc0f9547]*/
+/*[clinic end generated code: output=706186dba996490c input=cfdcaf573ed6e46c]*/
 {
     PicklerObject *pickler = _Pickler_New();
 
@@ -7693,10 +7693,10 @@ _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file,
 _pickle.dumps
 
   obj: object
-  protocol: object = NULL
+  protocol: object = None
   *
   fix_imports: bool = True
-  buffer_callback: object = NULL
+  buffer_callback: object = None
 
 Return the pickled representation of the object as a bytes object.
 
@@ -7722,7 +7722,7 @@ into *file* as part of the pickle stream.  It is an error if
 static PyObject *
 _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol,
                    int fix_imports, PyObject *buffer_callback)
-/*[clinic end generated code: output=fbab0093a5580fdf input=001f167df711b9f1]*/
+/*[clinic end generated code: output=fbab0093a5580fdf input=9f334d535ff7194f]*/
 {
     PyObject *result;
     PicklerObject *pickler = _Pickler_New();
@@ -7757,7 +7757,7 @@ _pickle.load
   fix_imports: bool = True
   encoding: str = 'ASCII'
   errors: str = 'strict'
-  buffers: object = NULL
+  buffers: object(c_default="NULL") = ()
 
 Read and return an object from the pickle data stored in a file.
 
@@ -7788,7 +7788,7 @@ static PyObject *
 _pickle_load_impl(PyObject *module, PyObject *file, int fix_imports,
                   const char *encoding, const char *errors,
                   PyObject *buffers)
-/*[clinic end generated code: output=250452d141c23e76 input=29fae982fe778156]*/
+/*[clinic end generated code: output=250452d141c23e76 input=46c7c31c92f4f371]*/
 {
     PyObject *result;
     UnpicklerObject *unpickler = _Unpickler_New();
@@ -7825,7 +7825,7 @@ _pickle.loads
   fix_imports: bool = True
   encoding: str = 'ASCII'
   errors: str = 'strict'
-  buffers: object = NULL
+  buffers: object(c_default="NULL") = ()
 
 Read and return an object from the given pickle data.
 
@@ -7847,7 +7847,7 @@ static PyObject *
 _pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports,
                    const char *encoding, const char *errors,
                    PyObject *buffers)
-/*[clinic end generated code: output=82ac1e6b588e6d02 input=c6004393f8276867]*/
+/*[clinic end generated code: output=82ac1e6b588e6d02 input=9c2ab6a0960185ea]*/
 {
     PyObject *result;
     UnpicklerObject *unpickler = _Unpickler_New();
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index 64060c64c4df..6f1f9c881530 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -3883,15 +3883,15 @@ _password_callback(char *buf, int size, int rwflag, void *userdata)
 /*[clinic input]
 _ssl._SSLContext.load_cert_chain
     certfile: object
-    keyfile: object = NULL
-    password: object = NULL
+    keyfile: object = None
+    password: object = None
 
 [clinic start generated code]*/
 
 static PyObject *
 _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile,
                                       PyObject *keyfile, PyObject *password)
-/*[clinic end generated code: output=9480bc1c380e2095 input=7cf9ac673cbee6fc]*/
+/*[clinic end generated code: output=9480bc1c380e2095 input=30bc7e967ea01a58]*/
 {
     PyObject *certfile_bytes = NULL, *keyfile_bytes = NULL;
     pem_password_cb *orig_passwd_cb = SSL_CTX_get_default_passwd_cb(self->ctx);
@@ -3917,7 +3917,7 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile,
         }
         goto error;
     }
-    if (password && password != Py_None) {
+    if (password != Py_None) {
         if (PyCallable_Check(password)) {
             pw_info.callable = password;
         } else if (!_pwinfo_set(&pw_info, password,
@@ -4075,9 +4075,9 @@ _add_ca_certs(PySSLContext *self, void *data, Py_ssize_t len,
 
 /*[clinic input]
 _ssl._SSLContext.load_verify_locations
-    cafile: object = NULL
-    capath: object = NULL
-    cadata: object = NULL
+    cafile: object = None
+    capath: object = None
+    cadata: object = None
 
 [clinic start generated code]*/
 
@@ -4086,7 +4086,7 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self,
                                             PyObject *cafile,
                                             PyObject *capath,
                                             PyObject *cadata)
-/*[clinic end generated code: output=454c7e41230ca551 input=997f1fb3a784ef88]*/
+/*[clinic end generated code: output=454c7e41230ca551 input=42ecfe258233e194]*/
 {
     PyObject *cafile_bytes = NULL, *capath_bytes = NULL;
     const char *cafile_buf = NULL, *capath_buf = NULL;
diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c
index 036f9b68bb23..94d5b649e042 100644
--- a/Modules/_tkinter.c
+++ b/Modules/_tkinter.c
@@ -3121,8 +3121,8 @@ _tkinter__flatten(PyObject *module, PyObject *item)
 /*[clinic input]
 _tkinter.create
 
-    screenName: str(accept={str, NoneType}) = NULL
-    baseName: str = NULL
+    screenName: str(accept={str, NoneType}) = None
+    baseName: str = ""
     className: str = "Tk"
     interactive: bool(accept={int}) = False
     wantobjects: bool(accept={int}) = False
@@ -3130,7 +3130,7 @@ _tkinter.create
         if false, then Tk_Init() doesn't get called
     sync: bool(accept={int}) = False
         if true, then pass -sync to wish
-    use: str(accept={str, NoneType}) = NULL
+    use: str(accept={str, NoneType}) = None
         if not None, then pass -use to wish
     /
 
@@ -3141,7 +3141,7 @@ _tkinter_create_impl(PyObject *module, const char *screenName,
                      const char *baseName, const char *className,
                      int interactive, int wantobjects, int wantTk, int sync,
                      const char *use)
-/*[clinic end generated code: output=e3315607648e6bb4 input=431907c134c80085]*/
+/*[clinic end generated code: output=e3315607648e6bb4 input=da9b17ee7358d862]*/
 {
     /* XXX baseName is not used anymore;
      * try getting rid of it. */
diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c
index 4de678757b38..56d007607e21 100644
--- a/Modules/cjkcodecs/multibytecodec.c
+++ b/Modules/cjkcodecs/multibytecodec.c
@@ -538,7 +538,7 @@ multibytecodec_encode(MultibyteCodec *codec,
 _multibytecodec.MultibyteCodec.encode
 
   input: object
-  errors: str(accept={str, NoneType}) = NULL
+  errors: str(accept={str, NoneType}) = None
 
 Return an encoded string version of `input'.
 
@@ -552,7 +552,7 @@ static PyObject *
 _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self,
                                            PyObject *input,
                                            const char *errors)
-/*[clinic end generated code: output=7b26652045ba56a9 input=05f6ced3c8dd0582]*/
+/*[clinic end generated code: output=7b26652045ba56a9 input=606d0e128a577bae]*/
 {
     MultibyteCodec_State state;
     PyObject *errorcb, *r, *ucvt;
@@ -607,7 +607,7 @@ _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self,
 _multibytecodec.MultibyteCodec.decode
 
   input: Py_buffer
-  errors: str(accept={str, NoneType}) = NULL
+  errors: str(accept={str, NoneType}) = None
 
 Decodes 'input'.
 
@@ -621,7 +621,7 @@ static PyObject *
 _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self,
                                            Py_buffer *input,
                                            const char *errors)
-/*[clinic end generated code: output=ff419f65bad6cc77 input=a7d45f87f75e5e02]*/
+/*[clinic end generated code: output=ff419f65bad6cc77 input=e0c78fc7ab190def]*/
 {
     MultibyteCodec_State state;
     MultibyteDecodeBuffer buf;
diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h
index b9daee6aae96..17eb77334d0a 100644
--- a/Modules/clinic/_asynciomodule.c.h
+++ b/Modules/clinic/_asynciomodule.c.h
@@ -119,7 +119,7 @@ PyDoc_STRVAR(_asyncio_Future_set_exception__doc__,
     {"set_exception", (PyCFunction)_asyncio_Future_set_exception, METH_O, _asyncio_Future_set_exception__doc__},
 
 PyDoc_STRVAR(_asyncio_Future_add_done_callback__doc__,
-"add_done_callback($self, fn, /, *, context=None)\n"
+"add_done_callback($self, fn, /, *, context=<unrepresentable>)\n"
 "--\n"
 "\n"
 "Add a callback to be run when the future becomes done.\n"
@@ -832,4 +832,4 @@ _asyncio__leave_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=51c50219f6a0863a input=a9049054013a1b77]*/
+/*[clinic end generated code: output=585ba1f8de5b4103 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h
index be1d2f70cf1b..772c8ca538da 100644
--- a/Modules/clinic/_codecsmodule.c.h
+++ b/Modules/clinic/_codecsmodule.c.h
@@ -1434,7 +1434,7 @@ _codecs_charmap_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs
     PyObject *return_value = NULL;
     Py_buffer data = {NULL, NULL};
     const char *errors = NULL;
-    PyObject *mapping = NULL;
+    PyObject *mapping = Py_None;
 
     if (!_PyArg_CheckPositional("charmap_decode", nargs, 1, 3)) {
         goto exit;
@@ -2542,7 +2542,7 @@ _codecs_charmap_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs
     PyObject *return_value = NULL;
     PyObject *str;
     const char *errors = NULL;
-    PyObject *mapping = NULL;
+    PyObject *mapping = Py_None;
 
     if (!_PyArg_CheckPositional("charmap_encode", nargs, 1, 3)) {
         goto exit;
@@ -2922,4 +2922,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg)
 #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF
     #define _CODECS_CODE_PAGE_ENCODE_METHODDEF
 #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */
-/*[clinic end generated code: output=59726a305e4ec24a input=a9049054013a1b77]*/
+/*[clinic end generated code: output=51b42d170889524c input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h
index a58ad737f854..a184b0ffb787 100644
--- a/Modules/clinic/_elementtree.c.h
+++ b/Modules/clinic/_elementtree.c.h
@@ -597,9 +597,9 @@ _elementtree_TreeBuilder___init__(PyObject *self, PyObject *args, PyObject *kwar
     PyObject * const *fastargs;
     Py_ssize_t nargs = PyTuple_GET_SIZE(args);
     Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0;
-    PyObject *element_factory = NULL;
-    PyObject *comment_factory = NULL;
-    PyObject *pi_factory = NULL;
+    PyObject *element_factory = Py_None;
+    PyObject *comment_factory = Py_None;
+    PyObject *pi_factory = Py_None;
     int insert_comments = 0;
     int insert_pis = 0;
 
@@ -916,4 +916,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args,
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=f5dbf9b4a095d310 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=bee26d0735a3fddc input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h
index b92daa72105a..9da3f1195a3b 100644
--- a/Modules/clinic/_pickle.c.h
+++ b/Modules/clinic/_pickle.c.h
@@ -112,9 +112,9 @@ _pickle_Pickler___init__(PyObject *self, PyObject *args, PyObject *kwargs)
     Py_ssize_t nargs = PyTuple_GET_SIZE(args);
     Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1;
     PyObject *file;
-    PyObject *protocol = NULL;
+    PyObject *protocol = Py_None;
     int fix_imports = 1;
-    PyObject *buffer_callback = NULL;
+    PyObject *buffer_callback = Py_None;
 
     fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 4, 0, argsbuf);
     if (!fastargs) {
@@ -292,7 +292,7 @@ _pickle_Unpickler___sizeof__(UnpicklerObject *self, PyObject *Py_UNUSED(ignored)
 
 PyDoc_STRVAR(_pickle_Unpickler___init____doc__,
 "Unpickler(file, *, fix_imports=True, encoding=\'ASCII\', errors=\'strict\',\n"
-"          buffers=None)\n"
+"          buffers=())\n"
 "--\n"
 "\n"
 "This takes a binary file for reading a pickle data stream.\n"
@@ -502,9 +502,9 @@ _pickle_dump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2;
     PyObject *obj;
     PyObject *file;
-    PyObject *protocol = NULL;
+    PyObject *protocol = Py_None;
     int fix_imports = 1;
-    PyObject *buffer_callback = NULL;
+    PyObject *buffer_callback = Py_None;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf);
     if (!args) {
@@ -582,9 +582,9 @@ _pickle_dumps(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
     PyObject *argsbuf[4];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
     PyObject *obj;
-    PyObject *protocol = NULL;
+    PyObject *protocol = Py_None;
     int fix_imports = 1;
-    PyObject *buffer_callback = NULL;
+    PyObject *buffer_callback = Py_None;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf);
     if (!args) {
@@ -623,7 +623,7 @@ _pickle_dumps(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
 
 PyDoc_STRVAR(_pickle_load__doc__,
 "load($module, /, file, *, fix_imports=True, encoding=\'ASCII\',\n"
-"     errors=\'strict\', buffers=None)\n"
+"     errors=\'strict\', buffers=())\n"
 "--\n"
 "\n"
 "Read and return an object from the pickle data stored in a file.\n"
@@ -735,7 +735,7 @@ _pickle_load(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject
 
 PyDoc_STRVAR(_pickle_loads__doc__,
 "loads($module, /, data, *, fix_imports=True, encoding=\'ASCII\',\n"
-"      errors=\'strict\', buffers=None)\n"
+"      errors=\'strict\', buffers=())\n"
 "--\n"
 "\n"
 "Read and return an object from the given pickle data.\n"
@@ -835,4 +835,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=17f6a76ebd94325d input=a9049054013a1b77]*/
+/*[clinic end generated code: output=de075ec48d4ee0e1 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h
index 25769a225af9..ce8669ae212e 100644
--- a/Modules/clinic/_ssl.c.h
+++ b/Modules/clinic/_ssl.c.h
@@ -571,8 +571,8 @@ _ssl__SSLContext_load_cert_chain(PySSLContext *self, PyObject *const *args, Py_s
     PyObject *argsbuf[3];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
     PyObject *certfile;
-    PyObject *keyfile = NULL;
-    PyObject *password = NULL;
+    PyObject *keyfile = Py_None;
+    PyObject *password = Py_None;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf);
     if (!args) {
@@ -618,9 +618,9 @@ _ssl__SSLContext_load_verify_locations(PySSLContext *self, PyObject *const *args
     static _PyArg_Parser _parser = {NULL, _keywords, "load_verify_locations", 0};
     PyObject *argsbuf[3];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
-    PyObject *cafile = NULL;
-    PyObject *capath = NULL;
-    PyObject *cadata = NULL;
+    PyObject *cafile = Py_None;
+    PyObject *capath = Py_None;
+    PyObject *cadata = Py_None;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf);
     if (!args) {
@@ -1482,4 +1482,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje
 #ifndef _SSL_ENUM_CRLS_METHODDEF
     #define _SSL_ENUM_CRLS_METHODDEF
 #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */
-/*[clinic end generated code: output=aa4947067c3fef2d input=a9049054013a1b77]*/
+/*[clinic end generated code: output=a4aeb3f92a091c64 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h
index 4870e9b4aad7..73c3faeaf962 100644
--- a/Modules/clinic/_tkinter.c.h
+++ b/Modules/clinic/_tkinter.c.h
@@ -674,7 +674,7 @@ PyDoc_STRVAR(_tkinter__flatten__doc__,
     {"_flatten", (PyCFunction)_tkinter__flatten, METH_O, _tkinter__flatten__doc__},
 
 PyDoc_STRVAR(_tkinter_create__doc__,
-"create($module, screenName=None, baseName=None, className=\'Tk\',\n"
+"create($module, screenName=None, baseName=\'\', className=\'Tk\',\n"
 "       interactive=False, wantobjects=False, wantTk=True, sync=False,\n"
 "       use=None, /)\n"
 "--\n"
@@ -702,7 +702,7 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
     const char *screenName = NULL;
-    const char *baseName = NULL;
+    const char *baseName = "";
     const char *className = "Tk";
     int interactive = 0;
     int wantobjects = 0;
@@ -912,4 +912,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored))
 #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF
     #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF
 #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */
-/*[clinic end generated code: output=da0115c470aac1c0 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=492b8b833fe54bc9 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h
index e18d407ba9e8..82942f08a686 100644
--- a/Modules/clinic/binascii.c.h
+++ b/Modules/clinic/binascii.c.h
@@ -432,7 +432,7 @@ binascii_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 }
 
 PyDoc_STRVAR(binascii_b2a_hex__doc__,
-"b2a_hex($module, /, data, sep=None, bytes_per_sep=1)\n"
+"b2a_hex($module, /, data, sep=<unrepresentable>, bytes_per_sep=1)\n"
 "--\n"
 "\n"
 "Hexadecimal representation of binary data.\n"
@@ -515,7 +515,7 @@ binascii_b2a_hex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb
 }
 
 PyDoc_STRVAR(binascii_hexlify__doc__,
-"hexlify($module, /, data, sep=None, bytes_per_sep=1)\n"
+"hexlify($module, /, data, sep=<unrepresentable>, bytes_per_sep=1)\n"
 "--\n"
 "\n"
 "Hexadecimal representation of binary data.\n"
@@ -801,4 +801,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj
 
     return return_value;
 }
-/*[clinic end generated code: output=e13bd02ac40496f0 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=ec26d03c2007eaac input=a9049054013a1b77]*/
diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h
index 33509872069f..81a8437c3a7d 100644
--- a/Modules/clinic/cmathmodule.c.h
+++ b/Modules/clinic/cmathmodule.c.h
@@ -648,10 +648,10 @@ cmath_tanh(PyObject *module, PyObject *arg)
 }
 
 PyDoc_STRVAR(cmath_log__doc__,
-"log($module, x, y_obj=None, /)\n"
+"log($module, z, base=<unrepresentable>, /)\n"
 "--\n"
 "\n"
-"The logarithm of z to the given base.\n"
+"log(z[, base]) -> the logarithm of z to the given base.\n"
 "\n"
 "If the base not specified, returns the natural logarithm (base e) of z.");
 
@@ -968,4 +968,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=3ab228947d1709cc input=a9049054013a1b77]*/
+/*[clinic end generated code: output=3edc4484b10ae752 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h
index cb4ef5978dbf..b21d17477ee6 100644
--- a/Modules/clinic/posixmodule.c.h
+++ b/Modules/clinic/posixmodule.c.h
@@ -1971,8 +1971,8 @@ os_uname(PyObject *module, PyObject *Py_UNUSED(ignored))
 #endif /* defined(HAVE_UNAME) */
 
 PyDoc_STRVAR(os_utime__doc__,
-"utime($module, /, path, times=None, *, ns=None, dir_fd=None,\n"
-"      follow_symlinks=True)\n"
+"utime($module, /, path, times=None, *, ns=<unrepresentable>,\n"
+"      dir_fd=None, follow_symlinks=True)\n"
 "--\n"
 "\n"
 "Set the access and modified time of path.\n"
@@ -2015,7 +2015,7 @@ os_utime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw
     PyObject *argsbuf[5];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
     path_t path = PATH_T_INITIALIZE("utime", "path", 0, PATH_UTIME_HAVE_FD);
-    PyObject *times = NULL;
+    PyObject *times = Py_None;
     PyObject *ns = NULL;
     int dir_fd = DEFAULT_DIR_FD;
     int follow_symlinks = 1;
@@ -2208,8 +2208,8 @@ os_execve(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k
 
 PyDoc_STRVAR(os_posix_spawn__doc__,
 "posix_spawn($module, path, argv, env, /, *, file_actions=(),\n"
-"            setpgroup=None, resetids=False, setsid=False,\n"
-"            setsigmask=(), setsigdef=(), scheduler=None)\n"
+"            setpgroup=<unrepresentable>, resetids=False, setsid=False,\n"
+"            setsigmask=(), setsigdef=(), scheduler=<unrepresentable>)\n"
 "--\n"
 "\n"
 "Execute the program specified by path in a new process.\n"
@@ -2345,8 +2345,8 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje
 
 PyDoc_STRVAR(os_posix_spawnp__doc__,
 "posix_spawnp($module, path, argv, env, /, *, file_actions=(),\n"
-"             setpgroup=None, resetids=False, setsid=False,\n"
-"             setsigmask=(), setsigdef=(), scheduler=None)\n"
+"             setpgroup=<unrepresentable>, resetids=False, setsid=False,\n"
+"             setsigmask=(), setsigdef=(), scheduler=<unrepresentable>)\n"
 "--\n"
 "\n"
 "Execute the program specified by path in a new process.\n"
@@ -2598,8 +2598,9 @@ os_spawnve(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 #if defined(HAVE_FORK)
 
 PyDoc_STRVAR(os_register_at_fork__doc__,
-"register_at_fork($module, /, *, before=None, after_in_child=None,\n"
-"                 after_in_parent=None)\n"
+"register_at_fork($module, /, *, before=<unrepresentable>,\n"
+"                 after_in_child=<unrepresentable>,\n"
+"                 after_in_parent=<unrepresentable>)\n"
 "--\n"
 "\n"
 "Register callables to be called when forking a new process.\n"
@@ -6854,11 +6855,9 @@ os_abort(PyObject *module, PyObject *Py_UNUSED(ignored))
 #if defined(MS_WINDOWS)
 
 PyDoc_STRVAR(os_startfile__doc__,
-"startfile($module, /, filepath, operation=None)\n"
+"startfile($module, /, filepath, operation=<unrepresentable>)\n"
 "--\n"
 "\n"
-"startfile(filepath [, operation])\n"
-"\n"
 "Start a file with its associated application.\n"
 "\n"
 "When \"operation\" is not specified or \"open\", this acts like\n"
@@ -8724,4 +8723,4 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar
 #ifndef OS__REMOVE_DLL_DIRECTORY_METHODDEF
     #define OS__REMOVE_DLL_DIRECTORY_METHODDEF
 #endif /* !defined(OS__REMOVE_DLL_DIRECTORY_METHODDEF) */
-/*[clinic end generated code: output=3c5cb675b0d09145 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=113d9fbdd18a62f5 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h
index 41ab536bce23..ee5907ca7e44 100644
--- a/Modules/clinic/pyexpat.c.h
+++ b/Modules/clinic/pyexpat.c.h
@@ -133,7 +133,8 @@ pyexpat_xmlparser_GetInputContext(xmlparseobject *self, PyObject *Py_UNUSED(igno
 }
 
 PyDoc_STRVAR(pyexpat_xmlparser_ExternalEntityParserCreate__doc__,
-"ExternalEntityParserCreate($self, context, encoding=None, /)\n"
+"ExternalEntityParserCreate($self, context, encoding=<unrepresentable>,\n"
+"                           /)\n"
 "--\n"
 "\n"
 "Create a parser for parsing an external entity based on the information passed to the ExternalEntityRefHandler.");
@@ -280,7 +281,7 @@ pyexpat_xmlparser_UseForeignDTD(xmlparseobject *self, PyObject *const *args, Py_
 
 PyDoc_STRVAR(pyexpat_ParserCreate__doc__,
 "ParserCreate($module, /, encoding=None, namespace_separator=None,\n"
-"             intern=None)\n"
+"             intern=<unrepresentable>)\n"
 "--\n"
 "\n"
 "Return a new XML parser object.");
@@ -401,4 +402,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg)
 #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF
     #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF
 #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */
-/*[clinic end generated code: output=133a4105d508ebec input=a9049054013a1b77]*/
+/*[clinic end generated code: output=68ce25024280af41 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/unicodedata.c.h b/Modules/clinic/unicodedata.c.h
index 77e4f22593b0..4251db2edc7b 100644
--- a/Modules/clinic/unicodedata.c.h
+++ b/Modules/clinic/unicodedata.c.h
@@ -3,7 +3,7 @@ preserve
 [clinic start generated code]*/
 
 PyDoc_STRVAR(unicodedata_UCD_decimal__doc__,
-"decimal($self, chr, default=None, /)\n"
+"decimal($self, chr, default=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Converts a Unicode character into its equivalent decimal value.\n"
@@ -53,7 +53,7 @@ unicodedata_UCD_decimal(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 }
 
 PyDoc_STRVAR(unicodedata_UCD_digit__doc__,
-"digit($self, chr, default=None, /)\n"
+"digit($self, chr, default=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Converts a Unicode character into its equivalent digit value.\n"
@@ -102,7 +102,7 @@ unicodedata_UCD_digit(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 }
 
 PyDoc_STRVAR(unicodedata_UCD_numeric__doc__,
-"numeric($self, chr, default=None, /)\n"
+"numeric($self, chr, default=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Converts a Unicode character into its equivalent numeric value.\n"
@@ -481,7 +481,7 @@ unicodedata_UCD_normalize(PyObject *self, PyObject *const *args, Py_ssize_t narg
 }
 
 PyDoc_STRVAR(unicodedata_UCD_name__doc__,
-"name($self, chr, default=None, /)\n"
+"name($self, chr, default=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Returns the name assigned to the character chr as a string.\n"
@@ -559,4 +559,4 @@ unicodedata_UCD_lookup(PyObject *self, PyObject *arg)
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=3238f76814fb48d1 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=10c23477dbe8a202 input=a9049054013a1b77]*/
diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c
index b421f04fa605..02c09bbea48a 100644
--- a/Modules/cmathmodule.c
+++ b/Modules/cmathmodule.c
@@ -944,18 +944,18 @@ cmath_tanh_impl(PyObject *module, Py_complex z)
 /*[clinic input]
 cmath.log
 
-    x: Py_complex
-    y_obj: object = NULL
+    z as x: Py_complex
+    base as y_obj: object = NULL
     /
 
-The logarithm of z to the given base.
+log(z[, base]) -> the logarithm of z to the given base.
 
 If the base not specified, returns the natural logarithm (base e) of z.
 [clinic start generated code]*/
 
 static PyObject *
 cmath_log_impl(PyObject *module, Py_complex x, PyObject *y_obj)
-/*[clinic end generated code: output=4effdb7d258e0d94 input=ee0e823a7c6e68ea]*/
+/*[clinic end generated code: output=4effdb7d258e0d94 input=230ed3a71ecd000a]*/
 {
     Py_complex y;
 
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index e71d4b28bd1e..c60d0c5e39ff 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -4736,7 +4736,7 @@ split_py_long_to_s_and_ns(PyObject *py_long, time_t *s, long *ns)
 os.utime
 
     path: path_t(allow_fd='PATH_UTIME_HAVE_FD')
-    times: object = NULL
+    times: object = None
     *
     ns: object = NULL
     dir_fd: dir_fd(requires='futimensat') = None
@@ -4773,7 +4773,7 @@ dir_fd and follow_symlinks may not be available on your platform.
 static PyObject *
 os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns,
               int dir_fd, int follow_symlinks)
-/*[clinic end generated code: output=cfcac69d027b82cf input=081cdc54ca685385]*/
+/*[clinic end generated code: output=cfcac69d027b82cf input=2fbd62a2f228f8f4]*/
 {
 #ifdef MS_WINDOWS
     HANDLE hFile;
@@ -4786,14 +4786,14 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns,
 
     memset(&utime, 0, sizeof(utime_t));
 
-    if (times && (times != Py_None) && ns) {
+    if (times != Py_None && ns) {
         PyErr_SetString(PyExc_ValueError,
                      "utime: you may specify either 'times'"
                      " or 'ns' but not both");
         return NULL;
     }
 
-    if (times && (times != Py_None)) {
+    if (times != Py_None) {
         time_t a_sec, m_sec;
         long a_nsec, m_nsec;
         if (!PyTuple_CheckExact(times) || (PyTuple_Size(times) != 2)) {
@@ -11579,8 +11579,6 @@ os.startfile
     filepath: path_t
     operation: Py_UNICODE = NULL
 
-startfile(filepath [, operation])
-
 Start a file with its associated application.
 
 When "operation" is not specified or "open", this acts like
@@ -11602,7 +11600,7 @@ the underlying Win32 ShellExecute function doesn't work if it is.
 static PyObject *
 os_startfile_impl(PyObject *module, path_t *filepath,
                   const Py_UNICODE *operation)
-/*[clinic end generated code: output=66dc311c94d50797 input=63950bf2986380d0]*/
+/*[clinic end generated code: output=66dc311c94d50797 input=c940888a5390f039]*/
 {
     HINSTANCE rc;
 
diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c
index 5d284cddcfc7..d9123354bbed 100644
--- a/Modules/pyexpat.c
+++ b/Modules/pyexpat.c
@@ -1503,8 +1503,8 @@ static PyTypeObject Xmlparsetype = {
 /*[clinic input]
 pyexpat.ParserCreate
 
-    encoding: str(accept={str, NoneType}) = NULL
-    namespace_separator: str(accept={str, NoneType}) = NULL
+    encoding: str(accept={str, NoneType}) = None
+    namespace_separator: str(accept={str, NoneType}) = None
     intern: object = NULL
 
 Return a new XML parser object.
@@ -1513,7 +1513,7 @@ Return a new XML parser object.
 static PyObject *
 pyexpat_ParserCreate_impl(PyObject *module, const char *encoding,
                           const char *namespace_separator, PyObject *intern)
-/*[clinic end generated code: output=295c0cf01ab1146c input=23d29704acad385d]*/
+/*[clinic end generated code: output=295c0cf01ab1146c input=e8da8e8d7122cb5d]*/
 {
     PyObject *result;
     int intern_decref = 0;
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
index 4b2a77b4b8c1..7df93440a14c 100644
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -1948,7 +1948,7 @@ do_strip(PyBytesObject *self, int striptype)
 Py_LOCAL_INLINE(PyObject *)
 do_argstrip(PyBytesObject *self, int striptype, PyObject *bytes)
 {
-    if (bytes != NULL && bytes != Py_None) {
+    if (bytes != Py_None) {
         return do_xstrip(self, striptype, bytes);
     }
     return do_strip(self, striptype);
diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h
index ad8ed7eba7e8..05577077a5f8 100644
--- a/Objects/clinic/bytearrayobject.c.h
+++ b/Objects/clinic/bytearrayobject.c.h
@@ -868,7 +868,7 @@ bytearray_fromhex(PyTypeObject *type, PyObject *arg)
 }
 
 PyDoc_STRVAR(bytearray_hex__doc__,
-"hex($self, /, sep=None, bytes_per_sep=1)\n"
+"hex($self, /, sep=<unrepresentable>, bytes_per_sep=1)\n"
 "--\n"
 "\n"
 "Create a str of hexadecimal numbers from a bytearray object.\n"
@@ -1011,4 +1011,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored))
 {
     return bytearray_sizeof_impl(self);
 }
-/*[clinic end generated code: output=09df354d3374dfb2 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=508dce79cf2dffcc input=a9049054013a1b77]*/
diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h
index 7e2ce1af285c..22024ab155c8 100644
--- a/Objects/clinic/bytesobject.c.h
+++ b/Objects/clinic/bytesobject.c.h
@@ -688,7 +688,7 @@ bytes_fromhex(PyTypeObject *type, PyObject *arg)
 }
 
 PyDoc_STRVAR(bytes_hex__doc__,
-"hex($self, /, sep=None, bytes_per_sep=1)\n"
+"hex($self, /, sep=<unrepresentable>, bytes_per_sep=1)\n"
 "--\n"
 "\n"
 "Create a str of hexadecimal numbers from a bytes object.\n"
@@ -755,4 +755,4 @@ bytes_hex(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=dbed3c3ff6faa382 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=ca60dfccf8d51e88 input=a9049054013a1b77]*/
diff --git a/Objects/clinic/dictobject.c.h b/Objects/clinic/dictobject.c.h
index b87244d87348..7395e3bcebba 100644
--- a/Objects/clinic/dictobject.c.h
+++ b/Objects/clinic/dictobject.c.h
@@ -117,10 +117,10 @@ dict_setdefault(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs)
 }
 
 PyDoc_STRVAR(dict_pop__doc__,
-"pop($self, key, default=None, /)\n"
+"pop($self, key, default=<unrepresentable>, /)\n"
 "--\n"
 "\n"
-"Remove specified key and return the corresponding value.\n"
+"D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\n"
 "\n"
 "If key is not found, default is returned if given, otherwise KeyError is raised");
 
@@ -190,4 +190,4 @@ dict___reversed__(PyDictObject *self, PyObject *Py_UNUSED(ignored))
 {
     return dict___reversed___impl(self);
 }
-/*[clinic end generated code: output=0fd5cafc61a51d3c input=a9049054013a1b77]*/
+/*[clinic end generated code: output=4d98145508da8fa3 input=a9049054013a1b77]*/
diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h
index 6515d118e506..b684ba0ef27a 100644
--- a/Objects/clinic/floatobject.c.h
+++ b/Objects/clinic/floatobject.c.h
@@ -56,7 +56,7 @@ static PyObject *
 float___round__(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
-    PyObject *o_ndigits = NULL;
+    PyObject *o_ndigits = Py_None;
 
     if (!_PyArg_CheckPositional("__round__", nargs, 0, 1)) {
         goto exit;
@@ -351,4 +351,4 @@ float___format__(PyObject *self, PyObject *arg)
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=cc8098eb73f1a64c input=a9049054013a1b77]*/
+/*[clinic end generated code: output=1676433b9f04fbc9 input=a9049054013a1b77]*/
diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h
index 64fce10acb5c..75ac2011261f 100644
--- a/Objects/clinic/memoryobject.c.h
+++ b/Objects/clinic/memoryobject.c.h
@@ -3,7 +3,7 @@ preserve
 [clinic start generated code]*/
 
 PyDoc_STRVAR(memoryview_hex__doc__,
-"hex($self, /, sep=None, bytes_per_sep=1)\n"
+"hex($self, /, sep=<unrepresentable>, bytes_per_sep=1)\n"
 "--\n"
 "\n"
 "Return the data in the buffer as a str of hexadecimal numbers.\n"
@@ -71,4 +71,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=5e44e2bcf01057b5 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=ee265a73f68b0077 input=a9049054013a1b77]*/
diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h
index cb350874dad1..c7d6edb5fd48 100644
--- a/Objects/clinic/unicodeobject.c.h
+++ b/Objects/clinic/unicodeobject.c.h
@@ -630,7 +630,7 @@ static PyObject *
 unicode_lstrip(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
-    PyObject *chars = NULL;
+    PyObject *chars = Py_None;
 
     if (!_PyArg_CheckPositional("lstrip", nargs, 0, 1)) {
         goto exit;
@@ -664,7 +664,7 @@ static PyObject *
 unicode_rstrip(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
-    PyObject *chars = NULL;
+    PyObject *chars = Py_None;
 
     if (!_PyArg_CheckPositional("rstrip", nargs, 0, 1)) {
         goto exit;
@@ -1045,7 +1045,7 @@ unicode_swapcase(PyObject *self, PyObject *Py_UNUSED(ignored))
 }
 
 PyDoc_STRVAR(unicode_maketrans__doc__,
-"maketrans(x, y=None, z=None, /)\n"
+"maketrans(x, y=<unrepresentable>, z=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Return a translation table usable for str.translate().\n"
@@ -1232,4 +1232,4 @@ unicode_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored))
 {
     return unicode_sizeof_impl(self);
 }
-/*[clinic end generated code: output=d9a6ee45ddd0ccfd input=a9049054013a1b77]*/
+/*[clinic end generated code: output=5e15747f78f18329 input=a9049054013a1b77]*/
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 5c0640d1dfc8..2f2410583512 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -2993,14 +2993,14 @@ dict.pop
     default: object = NULL
     /
 
-Remove specified key and return the corresponding value.
+D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
 
 If key is not found, default is returned if given, otherwise KeyError is raised
 [clinic start generated code]*/
 
 static PyObject *
 dict_pop_impl(PyDictObject *self, PyObject *key, PyObject *default_value)
-/*[clinic end generated code: output=3abb47b89f24c21c input=016f6a000e4e633b]*/
+/*[clinic end generated code: output=3abb47b89f24c21c input=eeebec7812190348]*/
 {
     return _PyDict_Pop((PyObject*)self, key, default_value);
 }
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
index 20b7f5120bd4..8c08866d7372 100644
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -1042,7 +1042,7 @@ double_round(double x, int ndigits) {
 /*[clinic input]
 float.__round__
 
-    ndigits as o_ndigits: object = NULL
+    ndigits as o_ndigits: object = None
     /
 
 Return the Integral closest to x, rounding half toward even.
@@ -1052,13 +1052,13 @@ When an argument is passed, work like built-in round(x, ndigits).
 
 static PyObject *
 float___round___impl(PyObject *self, PyObject *o_ndigits)
-/*[clinic end generated code: output=374c36aaa0f13980 input=1ca2316b510293b8]*/
+/*[clinic end generated code: output=374c36aaa0f13980 input=fc0fe25924fbc9ed]*/
 {
     double x, rounded;
     Py_ssize_t ndigits;
 
     x = PyFloat_AsDouble(self);
-    if (o_ndigits == NULL || o_ndigits == Py_None) {
+    if (o_ndigits == Py_None) {
         /* single-argument round or with None ndigits:
          * round to nearest integer */
         rounded = round(x);
diff --git a/Objects/structseq.c b/Objects/structseq.c
index 320bf080d7e3..c158afccb97f 100644
--- a/Objects/structseq.c
+++ b/Objects/structseq.c
@@ -107,12 +107,12 @@ class structseq "PyStructSequence *" "NULL"
 @classmethod
 structseq.__new__ as structseq_new
     sequence as arg: object
-    dict: object = NULL
+    dict: object(c_default="NULL") = {}
 [clinic start generated code]*/
 
 static PyObject *
 structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict)
-/*[clinic end generated code: output=baa082e788b171da input=9b44810243907377]*/
+/*[clinic end generated code: output=baa082e788b171da input=90532511101aa3fb]*/
 {
     PyObject *ob;
     PyStructSequence *res = NULL;
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index d4b2c93a8452..070f795aa3cb 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -12522,7 +12522,7 @@ do_strip(PyObject *self, int striptype)
 static PyObject *
 do_argstrip(PyObject *self, int striptype, PyObject *sep)
 {
-    if (sep != NULL && sep != Py_None) {
+    if (sep != Py_None) {
         if (PyUnicode_Check(sep))
             return _PyUnicode_XStrip(self, striptype, sep);
         else {
@@ -12559,7 +12559,7 @@ unicode_strip_impl(PyObject *self, PyObject *chars)
 /*[clinic input]
 str.lstrip as unicode_lstrip
 
-    chars: object = NULL
+    chars: object = None
     /
 
 Return a copy of the string with leading whitespace removed.
@@ -12569,7 +12569,7 @@ If chars is given and not None, remove characters in chars instead.
 
 static PyObject *
 unicode_lstrip_impl(PyObject *self, PyObject *chars)
-/*[clinic end generated code: output=3b43683251f79ca7 input=9e56f3c45f5ff4c3]*/
+/*[clinic end generated code: output=3b43683251f79ca7 input=529f9f3834448671]*/
 {
     return do_argstrip(self, LEFTSTRIP, chars);
 }
@@ -12578,7 +12578,7 @@ unicode_lstrip_impl(PyObject *self, PyObject *chars)
 /*[clinic input]
 str.rstrip as unicode_rstrip
 
-    chars: object = NULL
+    chars: object = None
     /
 
 Return a copy of the string with trailing whitespace removed.
@@ -12588,7 +12588,7 @@ If chars is given and not None, remove characters in chars instead.
 
 static PyObject *
 unicode_rstrip_impl(PyObject *self, PyObject *chars)
-/*[clinic end generated code: output=4a59230017cc3b7a input=ac89d0219cb411ee]*/
+/*[clinic end generated code: output=4a59230017cc3b7a input=62566c627916557f]*/
 {
     return do_argstrip(self, RIGHTSTRIP, chars);
 }
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index d069f2fd2610..a40eb421f5de 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -2140,7 +2140,7 @@ builtin_repr(PyObject *module, PyObject *obj)
 round as builtin_round
 
     number: object
-    ndigits: object = NULL
+    ndigits: object = None
 
 Round a number to a given precision in decimal digits.
 
@@ -2150,7 +2150,7 @@ the return value has the same type as the number.  ndigits may be negative.
 
 static PyObject *
 builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits)
-/*[clinic end generated code: output=ff0d9dd176c02ede input=854bc3a217530c3d]*/
+/*[clinic end generated code: output=ff0d9dd176c02ede input=275678471d7aca15]*/
 {
     PyObject *round, *result;
 
@@ -2168,7 +2168,7 @@ builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits)
         return NULL;
     }
 
-    if (ndigits == NULL || ndigits == Py_None)
+    if (ndigits == Py_None)
         result = _PyObject_CallNoArg(round);
     else
         result = _PyObject_CallOneArg(round, ndigits);
diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h
index 49608ccf3df9..b936b0cf3721 100644
--- a/Python/clinic/bltinmodule.c.h
+++ b/Python/clinic/bltinmodule.c.h
@@ -719,7 +719,7 @@ builtin_round(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
     PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
     PyObject *number;
-    PyObject *ndigits = NULL;
+    PyObject *ndigits = Py_None;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf);
     if (!args) {
@@ -849,4 +849,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=1927f3c9abd00c35 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=4e118c2cd2cd98f3 input=a9049054013a1b77]*/
diff --git a/Python/clinic/context.c.h b/Python/clinic/context.c.h
index eedfc88654b6..2ac8bf7c0b8c 100644
--- a/Python/clinic/context.c.h
+++ b/Python/clinic/context.c.h
@@ -115,7 +115,7 @@ _contextvars_Context_copy(PyContext *self, PyObject *Py_UNUSED(ignored))
 }
 
 PyDoc_STRVAR(_contextvars_ContextVar_get__doc__,
-"get($self, default=None, /)\n"
+"get($self, default=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Return a value for the context variable for the current context.\n"
@@ -177,4 +177,4 @@ PyDoc_STRVAR(_contextvars_ContextVar_reset__doc__,
 
 #define _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF    \
     {"reset", (PyCFunction)_contextvars_ContextVar_reset, METH_O, _contextvars_ContextVar_reset__doc__},
-/*[clinic end generated code: output=a86b66e1516c25d4 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=f2e42f34e358e179 input=a9049054013a1b77]*/
diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h
index 743d00222914..e4867f34d4ef 100644
--- a/Python/clinic/import.c.h
+++ b/Python/clinic/import.c.h
@@ -300,7 +300,7 @@ _imp_is_frozen(PyObject *module, PyObject *arg)
 #if defined(HAVE_DYNAMIC_LOADING)
 
 PyDoc_STRVAR(_imp_create_dynamic__doc__,
-"create_dynamic($module, spec, file=None, /)\n"
+"create_dynamic($module, spec, file=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Create an extension module.");
@@ -454,4 +454,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb
 #ifndef _IMP_EXEC_DYNAMIC_METHODDEF
     #define _IMP_EXEC_DYNAMIC_METHODDEF
 #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */
-/*[clinic end generated code: output=ff06f7cf4b73eb76 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=3dc495e9c64d944e input=a9049054013a1b77]*/
diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h
index f9ab28dd055c..daca0e64110b 100644
--- a/Python/clinic/sysmodule.c.h
+++ b/Python/clinic/sysmodule.c.h
@@ -135,7 +135,7 @@ static PyObject *
 sys_exit(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
-    PyObject *status = NULL;
+    PyObject *status = Py_None;
 
     if (!_PyArg_CheckPositional("exit", nargs, 0, 1)) {
         goto exit;
@@ -995,4 +995,4 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored))
 #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF
     #define SYS_GETANDROIDAPILEVEL_METHODDEF
 #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */
-/*[clinic end generated code: output=8b250245a1265eef input=a9049054013a1b77]*/
+/*[clinic end generated code: output=decd687b7631de4b input=a9049054013a1b77]*/
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 476e154b1194..9cc7b4b27c15 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -749,7 +749,7 @@ sys_unraisablehook(PyObject *module, PyObject *unraisable)
 /*[clinic input]
 sys.exit
 
-    status: object = NULL
+    status: object = None
     /
 
 Exit the interpreter by raising SystemExit(status).
@@ -762,7 +762,7 @@ exit status will be one (i.e., failure).
 
 static PyObject *
 sys_exit_impl(PyObject *module, PyObject *status)
-/*[clinic end generated code: output=13870986c1ab2ec0 input=a737351f86685e9c]*/
+/*[clinic end generated code: output=13870986c1ab2ec0 input=b86ca9497baa94f2]*/
 {
     /* Raise SystemExit so callers may catch it or clean up. */
     PyThreadState *tstate = _PyThreadState_GET();
diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py
index cec7c5392db3..c5edc7c9336f 100755
--- a/Tools/clinic/clinic.py
+++ b/Tools/clinic/clinic.py
@@ -43,9 +43,6 @@
 
 version = '1'
 
-_empty = inspect._empty
-_void = inspect._void
-
 NoneType = type(None)
 
 class Unspecified:
@@ -2175,7 +2172,7 @@ class Function:
     def __init__(self, parameters=None, *, name,
                  module, cls=None, c_basename=None,
                  full_name=None,
-                 return_converter, return_annotation=_empty,
+                 return_converter, return_annotation=inspect.Signature.empty,
                  docstring=None, kind=CALLABLE, coexist=False,
                  docstring_only=False):
         self.parameters = parameters or collections.OrderedDict()
@@ -2253,8 +2250,8 @@ class Parameter:
     Mutable duck type of inspect.Parameter.
     """
 
-    def __init__(self, name, kind, *, default=_empty,
-                 function, converter, annotation=_empty,
+    def __init__(self, name, kind, *, default=inspect.Parameter.empty,
+                 function, converter, annotation=inspect.Parameter.empty,
                  docstring=None, group=0):
         self.name = name
         self.kind = kind
@@ -3231,6 +3228,8 @@ def converter_init(self, *, accept={str}, encoding=None, zeroes=False):
             # sorry, clinic can't support preallocated buffers
             # for es# and et#
             self.c_default = "NULL"
+        if NoneType in accept and self.c_default == "Py_None":
+            self.c_default = "NULL"
 
     def cleanup(self):
         if self.encoding:
@@ -4433,7 +4432,7 @@ def bad_node(self, node):
                 # mild hack: explicitly support NULL as a default value
                 if isinstance(expr, ast.Name) and expr.id == 'NULL':
                     value = NULL
-                    py_default = 'None'
+                    py_default = '<unrepresentable>'
                     c_default = "NULL"
                 elif (isinstance(expr, ast.BinOp) or
                     (isinstance(expr, ast.UnaryOp) and


From webhook-mailer at python.org  Sat Sep 14 06:31:54 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sat, 14 Sep 2019 10:31:54 -0000
Subject: [Python-checkins] [3.8] bpo-37206: Unrepresentable default values
 no longer represented as None. (GH-13933) (GH-16141)
Message-ID: <mailman.745.1568457116.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d322abbb83eb751045246a70f39d040d13a6108b
commit: d322abbb83eb751045246a70f39d040d13a6108b
branch: 3.8
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-14T13:31:50+03:00
summary:

[3.8] bpo-37206: Unrepresentable default values no longer represented as None. (GH-13933) (GH-16141)

In ArgumentClinic, value "NULL" should now be used only for unrepresentable default values
(like in the optional third parameter of getattr). "None" should be used if None is accepted
as argument and passing None has the same effect as not passing the argument at all.
(cherry picked from commit 279f44678c8b84a183f9eeb85e0b086228154497)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-06-09-22-25-03.bpo-37206.2WBg4q.rst
M Lib/test/clinic.test
M Lib/test/test_inspect.py
M Modules/_codecsmodule.c
M Modules/_cursesmodule.c
M Modules/_elementtree.c
M Modules/_hashopenssl.c
M Modules/_io/_iomodule.c
M Modules/_io/clinic/fileio.c.h
M Modules/_io/fileio.c
M Modules/_io/textio.c
M Modules/_pickle.c
M Modules/_ssl.c
M Modules/_tkinter.c
M Modules/cjkcodecs/multibytecodec.c
M Modules/clinic/_asynciomodule.c.h
M Modules/clinic/_codecsmodule.c.h
M Modules/clinic/_elementtree.c.h
M Modules/clinic/_pickle.c.h
M Modules/clinic/_ssl.c.h
M Modules/clinic/_tkinter.c.h
M Modules/clinic/binascii.c.h
M Modules/clinic/cmathmodule.c.h
M Modules/clinic/posixmodule.c.h
M Modules/clinic/pyexpat.c.h
M Modules/clinic/unicodedata.c.h
M Modules/cmathmodule.c
M Modules/posixmodule.c
M Modules/pyexpat.c
M Objects/bytesobject.c
M Objects/clinic/bytearrayobject.c.h
M Objects/clinic/bytesobject.c.h
M Objects/clinic/floatobject.c.h
M Objects/clinic/memoryobject.c.h
M Objects/clinic/unicodeobject.c.h
M Objects/floatobject.c
M Objects/structseq.c
M Objects/unicodeobject.c
M Python/bltinmodule.c
M Python/clinic/bltinmodule.c.h
M Python/clinic/context.c.h
M Python/clinic/import.c.h
M Python/clinic/sysmodule.c.h
M Python/sysmodule.c
M Tools/clinic/clinic.py

diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test
index f9c55e2a577f..0d84d5ef5359 100644
--- a/Lib/test/clinic.test
+++ b/Lib/test/clinic.test
@@ -92,7 +92,7 @@ test_objects_converter
 [clinic start generated code]*/
 
 PyDoc_STRVAR(test_objects_converter__doc__,
-"test_objects_converter($module, a, b=None, /)\n"
+"test_objects_converter($module, a, b=<unrepresentable>, /)\n"
 "--\n"
 "\n");
 
@@ -126,7 +126,7 @@ exit:
 
 static PyObject *
 test_objects_converter_impl(PyObject *module, PyObject *a, PyObject *b)
-/*[clinic end generated code: output=58009c0e42b4834e input=4cbb3d9edd2a36f3]*/
+/*[clinic end generated code: output=0e461f38d3b2bd08 input=4cbb3d9edd2a36f3]*/
 
 
 /*[clinic input]
@@ -1718,8 +1718,8 @@ test_str_converter
 [clinic start generated code]*/
 
 PyDoc_STRVAR(test_str_converter__doc__,
-"test_str_converter($module, a=None, b=\'ab\', c=\'cd\', d=\'cef\', e=\'gh\',\n"
-"                   f=\'ij\', g=\'kl\', h=\'mn\', /)\n"
+"test_str_converter($module, a=<unrepresentable>, b=\'ab\', c=\'cd\',\n"
+"                   d=\'cef\', e=\'gh\', f=\'ij\', g=\'kl\', h=\'mn\', /)\n"
 "--\n"
 "\n");
 
@@ -1765,7 +1765,7 @@ test_str_converter_impl(PyObject *module, const char *a, const char *b,
                         const char *f, Py_ssize_clean_t f_length,
                         const char *g, Py_ssize_clean_t g_length,
                         const char *h, Py_ssize_clean_t h_length)
-/*[clinic end generated code: output=8415d82c56154307 input=8afe9da8185cd38c]*/
+/*[clinic end generated code: output=ad868ad94a488e32 input=8afe9da8185cd38c]*/
 
 
 /*[clinic input]
diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py
index db9ac36f7198..1c516546f768 100644
--- a/Lib/test/test_inspect.py
+++ b/Lib/test/test_inspect.py
@@ -2273,8 +2273,8 @@ def p(name): return signature.parameters[name].default
         test_callable(d.dump)
 
         # static method
-        test_callable(str.maketrans)
-        test_callable('abc'.maketrans)
+        test_callable(bytes.maketrans)
+        test_callable(b'abc'.maketrans)
 
         # class method
         test_callable(dict.fromkeys)
diff --git a/Misc/NEWS.d/next/Library/2019-06-09-22-25-03.bpo-37206.2WBg4q.rst b/Misc/NEWS.d/next/Library/2019-06-09-22-25-03.bpo-37206.2WBg4q.rst
new file mode 100644
index 000000000000..2c6c6821e639
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-09-22-25-03.bpo-37206.2WBg4q.rst
@@ -0,0 +1,2 @@
+Default values which cannot be represented as Python objects no longer
+improperly represented as ``None`` in function signatures.
diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c
index 90b3e37d1641..a8ffb699557a 100644
--- a/Modules/_codecsmodule.c
+++ b/Modules/_codecsmodule.c
@@ -171,14 +171,14 @@ PyObject *codec_tuple(PyObject *decoded,
 /*[clinic input]
 _codecs.escape_decode
     data: Py_buffer(accept={str, buffer})
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_escape_decode_impl(PyObject *module, Py_buffer *data,
                            const char *errors)
-/*[clinic end generated code: output=505200ba8056979a input=0018edfd99db714d]*/
+/*[clinic end generated code: output=505200ba8056979a input=77298a561c90bd82]*/
 {
     PyObject *decoded = PyBytes_DecodeEscape(data->buf, data->len,
                                              errors, 0, NULL);
@@ -188,14 +188,14 @@ _codecs_escape_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.escape_encode
     data: object(subclass_of='&PyBytes_Type')
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_escape_encode_impl(PyObject *module, PyObject *data,
                            const char *errors)
-/*[clinic end generated code: output=4af1d477834bab34 input=da9ded00992f32f2]*/
+/*[clinic end generated code: output=4af1d477834bab34 input=8f4b144799a94245]*/
 {
     Py_ssize_t size;
     Py_ssize_t newsize;
@@ -252,7 +252,7 @@ _codecs_escape_encode_impl(PyObject *module, PyObject *data,
 /*[clinic input]
 _codecs.utf_7_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -260,7 +260,7 @@ _codecs.utf_7_decode
 static PyObject *
 _codecs_utf_7_decode_impl(PyObject *module, Py_buffer *data,
                           const char *errors, int final)
-/*[clinic end generated code: output=0cd3a944a32a4089 input=2d94a5a1f170c8ae]*/
+/*[clinic end generated code: output=0cd3a944a32a4089 input=22c395d357815d26]*/
 {
     Py_ssize_t consumed = data->len;
     PyObject *decoded = PyUnicode_DecodeUTF7Stateful(data->buf, data->len,
@@ -272,7 +272,7 @@ _codecs_utf_7_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_8_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -280,7 +280,7 @@ _codecs.utf_8_decode
 static PyObject *
 _codecs_utf_8_decode_impl(PyObject *module, Py_buffer *data,
                           const char *errors, int final)
-/*[clinic end generated code: output=10f74dec8d9bb8bf input=1ea6c21492e8bcbe]*/
+/*[clinic end generated code: output=10f74dec8d9bb8bf input=f611b3867352ba59]*/
 {
     Py_ssize_t consumed = data->len;
     PyObject *decoded = PyUnicode_DecodeUTF8Stateful(data->buf, data->len,
@@ -292,7 +292,7 @@ _codecs_utf_8_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_16_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -300,7 +300,7 @@ _codecs.utf_16_decode
 static PyObject *
 _codecs_utf_16_decode_impl(PyObject *module, Py_buffer *data,
                            const char *errors, int final)
-/*[clinic end generated code: output=783b442abcbcc2d0 input=2ba128c28ea0bb40]*/
+/*[clinic end generated code: output=783b442abcbcc2d0 input=191d360bd7309180]*/
 {
     int byteorder = 0;
     /* This is overwritten unless final is true. */
@@ -314,7 +314,7 @@ _codecs_utf_16_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_16_le_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -322,7 +322,7 @@ _codecs.utf_16_le_decode
 static PyObject *
 _codecs_utf_16_le_decode_impl(PyObject *module, Py_buffer *data,
                               const char *errors, int final)
-/*[clinic end generated code: output=899b9e6364379dcd input=43aeb8b0461cace5]*/
+/*[clinic end generated code: output=899b9e6364379dcd input=c6904fdc27fb4724]*/
 {
     int byteorder = -1;
     /* This is overwritten unless final is true. */
@@ -336,7 +336,7 @@ _codecs_utf_16_le_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_16_be_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -344,7 +344,7 @@ _codecs.utf_16_be_decode
 static PyObject *
 _codecs_utf_16_be_decode_impl(PyObject *module, Py_buffer *data,
                               const char *errors, int final)
-/*[clinic end generated code: output=49f6465ea07669c8 input=339e554c804f34b2]*/
+/*[clinic end generated code: output=49f6465ea07669c8 input=e49012400974649b]*/
 {
     int byteorder = 1;
     /* This is overwritten unless final is true. */
@@ -365,7 +365,7 @@ _codecs_utf_16_be_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_16_ex_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     byteorder: int = 0
     final: bool(accept={int}) = False
     /
@@ -374,7 +374,7 @@ _codecs.utf_16_ex_decode
 static PyObject *
 _codecs_utf_16_ex_decode_impl(PyObject *module, Py_buffer *data,
                               const char *errors, int byteorder, int final)
-/*[clinic end generated code: output=0f385f251ecc1988 input=3201aeddb9636889]*/
+/*[clinic end generated code: output=0f385f251ecc1988 input=5a9c19f2e6b6cf0e]*/
 {
     /* This is overwritten unless final is true. */
     Py_ssize_t consumed = data->len;
@@ -390,7 +390,7 @@ _codecs_utf_16_ex_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_32_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -398,7 +398,7 @@ _codecs.utf_32_decode
 static PyObject *
 _codecs_utf_32_decode_impl(PyObject *module, Py_buffer *data,
                            const char *errors, int final)
-/*[clinic end generated code: output=2fc961807f7b145f input=155a5c673a4e2514]*/
+/*[clinic end generated code: output=2fc961807f7b145f input=fd7193965627eb58]*/
 {
     int byteorder = 0;
     /* This is overwritten unless final is true. */
@@ -412,7 +412,7 @@ _codecs_utf_32_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_32_le_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -420,7 +420,7 @@ _codecs.utf_32_le_decode
 static PyObject *
 _codecs_utf_32_le_decode_impl(PyObject *module, Py_buffer *data,
                               const char *errors, int final)
-/*[clinic end generated code: output=ec8f46b67a94f3e6 input=7baf061069e92d3b]*/
+/*[clinic end generated code: output=ec8f46b67a94f3e6 input=9078ec70acfe7613]*/
 {
     int byteorder = -1;
     /* This is overwritten unless final is true. */
@@ -434,7 +434,7 @@ _codecs_utf_32_le_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_32_be_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -442,7 +442,7 @@ _codecs.utf_32_be_decode
 static PyObject *
 _codecs_utf_32_be_decode_impl(PyObject *module, Py_buffer *data,
                               const char *errors, int final)
-/*[clinic end generated code: output=ff82bae862c92c4e input=b182026300dae595]*/
+/*[clinic end generated code: output=ff82bae862c92c4e input=f1ae1bbbb86648ff]*/
 {
     int byteorder = 1;
     /* This is overwritten unless final is true. */
@@ -463,7 +463,7 @@ _codecs_utf_32_be_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_32_ex_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     byteorder: int = 0
     final: bool(accept={int}) = False
     /
@@ -472,7 +472,7 @@ _codecs.utf_32_ex_decode
 static PyObject *
 _codecs_utf_32_ex_decode_impl(PyObject *module, Py_buffer *data,
                               const char *errors, int byteorder, int final)
-/*[clinic end generated code: output=6bfb177dceaf4848 input=7b9c2cb819fb237a]*/
+/*[clinic end generated code: output=6bfb177dceaf4848 input=e46a73bc859d0bd0]*/
 {
     Py_ssize_t consumed = data->len;
     PyObject *decoded = PyUnicode_DecodeUTF32Stateful(data->buf, data->len,
@@ -486,14 +486,14 @@ _codecs_utf_32_ex_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.unicode_escape_decode
     data: Py_buffer(accept={str, buffer})
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
                                    const char *errors)
-/*[clinic end generated code: output=3ca3c917176b82ab input=49fd27d06813a7f5]*/
+/*[clinic end generated code: output=3ca3c917176b82ab input=8328081a3a569bd6]*/
 {
     PyObject *decoded = PyUnicode_DecodeUnicodeEscape(data->buf, data->len,
                                                       errors);
@@ -503,14 +503,14 @@ _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.raw_unicode_escape_decode
     data: Py_buffer(accept={str, buffer})
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_raw_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
                                        const char *errors)
-/*[clinic end generated code: output=c98eeb56028070a6 input=770903a211434ebc]*/
+/*[clinic end generated code: output=c98eeb56028070a6 input=d2f5159ce3b3392f]*/
 {
     PyObject *decoded = PyUnicode_DecodeRawUnicodeEscape(data->buf, data->len,
                                                          errors);
@@ -520,14 +520,14 @@ _codecs_raw_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.latin_1_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_latin_1_decode_impl(PyObject *module, Py_buffer *data,
                             const char *errors)
-/*[clinic end generated code: output=07f3dfa3f72c7d8f input=5cad0f1759c618ec]*/
+/*[clinic end generated code: output=07f3dfa3f72c7d8f input=76ca58fd6dcd08c7]*/
 {
     PyObject *decoded = PyUnicode_DecodeLatin1(data->buf, data->len, errors);
     return codec_tuple(decoded, data->len);
@@ -536,14 +536,14 @@ _codecs_latin_1_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.ascii_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_ascii_decode_impl(PyObject *module, Py_buffer *data,
                           const char *errors)
-/*[clinic end generated code: output=2627d72058d42429 input=ad1106f64037bd16]*/
+/*[clinic end generated code: output=2627d72058d42429 input=e428a267a04b4481]*/
 {
     PyObject *decoded = PyUnicode_DecodeASCII(data->buf, data->len, errors);
     return codec_tuple(decoded, data->len);
@@ -552,15 +552,15 @@ _codecs_ascii_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.charmap_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
-    mapping: object = NULL
+    errors: str(accept={str, NoneType}) = None
+    mapping: object = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_charmap_decode_impl(PyObject *module, Py_buffer *data,
                             const char *errors, PyObject *mapping)
-/*[clinic end generated code: output=2c335b09778cf895 input=19712ca35c5a80e2]*/
+/*[clinic end generated code: output=2c335b09778cf895 input=15b69df43458eb40]*/
 {
     PyObject *decoded;
 
@@ -576,7 +576,7 @@ _codecs_charmap_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.mbcs_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -584,7 +584,7 @@ _codecs.mbcs_decode
 static PyObject *
 _codecs_mbcs_decode_impl(PyObject *module, Py_buffer *data,
                          const char *errors, int final)
-/*[clinic end generated code: output=39b65b8598938c4b input=b5f2fe568f311297]*/
+/*[clinic end generated code: output=39b65b8598938c4b input=1c1d50f08fa53789]*/
 {
     Py_ssize_t consumed = data->len;
     PyObject *decoded = PyUnicode_DecodeMBCSStateful(data->buf, data->len,
@@ -595,7 +595,7 @@ _codecs_mbcs_decode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.oem_decode
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -603,7 +603,7 @@ _codecs.oem_decode
 static PyObject *
 _codecs_oem_decode_impl(PyObject *module, Py_buffer *data,
                         const char *errors, int final)
-/*[clinic end generated code: output=da1617612f3fcad8 input=278709bcfd374a9c]*/
+/*[clinic end generated code: output=da1617612f3fcad8 input=81b67cba811022e5]*/
 {
     Py_ssize_t consumed = data->len;
     PyObject *decoded = PyUnicode_DecodeCodePageStateful(CP_OEMCP,
@@ -615,7 +615,7 @@ _codecs_oem_decode_impl(PyObject *module, Py_buffer *data,
 _codecs.code_page_decode
     codepage: int
     data: Py_buffer
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     final: bool(accept={int}) = False
     /
 [clinic start generated code]*/
@@ -623,7 +623,7 @@ _codecs.code_page_decode
 static PyObject *
 _codecs_code_page_decode_impl(PyObject *module, int codepage,
                               Py_buffer *data, const char *errors, int final)
-/*[clinic end generated code: output=53008ea967da3fff input=51f6169021c68dd5]*/
+/*[clinic end generated code: output=53008ea967da3fff input=c5f58d036cb63575]*/
 {
     Py_ssize_t consumed = data->len;
     PyObject *decoded = PyUnicode_DecodeCodePageStateful(codepage,
@@ -640,14 +640,14 @@ _codecs_code_page_decode_impl(PyObject *module, int codepage,
 /*[clinic input]
 _codecs.readbuffer_encode
     data: Py_buffer(accept={str, buffer})
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_readbuffer_encode_impl(PyObject *module, Py_buffer *data,
                                const char *errors)
-/*[clinic end generated code: output=c645ea7cdb3d6e86 input=b7c322b89d4ab923]*/
+/*[clinic end generated code: output=c645ea7cdb3d6e86 input=aa10cfdf252455c5]*/
 {
     PyObject *result = PyBytes_FromStringAndSize(data->buf, data->len);
     return codec_tuple(result, data->len);
@@ -656,14 +656,14 @@ _codecs_readbuffer_encode_impl(PyObject *module, Py_buffer *data,
 /*[clinic input]
 _codecs.utf_7_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_utf_7_encode_impl(PyObject *module, PyObject *str,
                           const char *errors)
-/*[clinic end generated code: output=0feda21ffc921bc8 input=d1a47579e79cbe15]*/
+/*[clinic end generated code: output=0feda21ffc921bc8 input=2546dbbb3fa53114]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF7(str, 0, 0, errors),
                        PyUnicode_GET_LENGTH(str));
@@ -672,14 +672,14 @@ _codecs_utf_7_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_8_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_utf_8_encode_impl(PyObject *module, PyObject *str,
                           const char *errors)
-/*[clinic end generated code: output=02bf47332b9c796c input=42e3ba73c4392eef]*/
+/*[clinic end generated code: output=02bf47332b9c796c input=a3e71ae01c3f93f3]*/
 {
     return codec_tuple(_PyUnicode_AsUTF8String(str, errors),
                        PyUnicode_GET_LENGTH(str));
@@ -695,7 +695,7 @@ _codecs_utf_8_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_16_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     byteorder: int = 0
     /
 [clinic start generated code]*/
@@ -703,7 +703,7 @@ _codecs.utf_16_encode
 static PyObject *
 _codecs_utf_16_encode_impl(PyObject *module, PyObject *str,
                            const char *errors, int byteorder)
-/*[clinic end generated code: output=c654e13efa2e64e4 input=ff46416b04edb944]*/
+/*[clinic end generated code: output=c654e13efa2e64e4 input=68cdc2eb8338555d]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF16(str, errors, byteorder),
                        PyUnicode_GET_LENGTH(str));
@@ -712,14 +712,14 @@ _codecs_utf_16_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_16_le_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_utf_16_le_encode_impl(PyObject *module, PyObject *str,
                               const char *errors)
-/*[clinic end generated code: output=431b01e55f2d4995 input=cb385455ea8f2fe0]*/
+/*[clinic end generated code: output=431b01e55f2d4995 input=83d042706eed6798]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF16(str, errors, -1),
                        PyUnicode_GET_LENGTH(str));
@@ -728,14 +728,14 @@ _codecs_utf_16_le_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_16_be_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_utf_16_be_encode_impl(PyObject *module, PyObject *str,
                               const char *errors)
-/*[clinic end generated code: output=96886a6fd54dcae3 input=9119997066bdaefd]*/
+/*[clinic end generated code: output=96886a6fd54dcae3 input=6f1e9e623b03071b]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF16(str, errors, +1),
                        PyUnicode_GET_LENGTH(str));
@@ -751,7 +751,7 @@ _codecs_utf_16_be_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_32_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     byteorder: int = 0
     /
 [clinic start generated code]*/
@@ -759,7 +759,7 @@ _codecs.utf_32_encode
 static PyObject *
 _codecs_utf_32_encode_impl(PyObject *module, PyObject *str,
                            const char *errors, int byteorder)
-/*[clinic end generated code: output=5c760da0c09a8b83 input=c5e77da82fbe5c2a]*/
+/*[clinic end generated code: output=5c760da0c09a8b83 input=8ec4c64d983bc52b]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF32(str, errors, byteorder),
                        PyUnicode_GET_LENGTH(str));
@@ -768,14 +768,14 @@ _codecs_utf_32_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_32_le_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_utf_32_le_encode_impl(PyObject *module, PyObject *str,
                               const char *errors)
-/*[clinic end generated code: output=b65cd176de8e36d6 input=9993b25fe0877848]*/
+/*[clinic end generated code: output=b65cd176de8e36d6 input=f0918d41de3eb1b1]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF32(str, errors, -1),
                        PyUnicode_GET_LENGTH(str));
@@ -784,14 +784,14 @@ _codecs_utf_32_le_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.utf_32_be_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_utf_32_be_encode_impl(PyObject *module, PyObject *str,
                               const char *errors)
-/*[clinic end generated code: output=1d9e71a9358709e9 input=d3e0ccaa02920431]*/
+/*[clinic end generated code: output=1d9e71a9358709e9 input=967a99a95748b557]*/
 {
     return codec_tuple(_PyUnicode_EncodeUTF32(str, errors, +1),
                        PyUnicode_GET_LENGTH(str));
@@ -800,14 +800,14 @@ _codecs_utf_32_be_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.unicode_escape_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_unicode_escape_encode_impl(PyObject *module, PyObject *str,
                                    const char *errors)
-/*[clinic end generated code: output=66271b30bc4f7a3c input=65d9eefca65b455a]*/
+/*[clinic end generated code: output=66271b30bc4f7a3c input=8c4de07597054e33]*/
 {
     return codec_tuple(PyUnicode_AsUnicodeEscapeString(str),
                        PyUnicode_GET_LENGTH(str));
@@ -816,14 +816,14 @@ _codecs_unicode_escape_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.raw_unicode_escape_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_raw_unicode_escape_encode_impl(PyObject *module, PyObject *str,
                                        const char *errors)
-/*[clinic end generated code: output=a66a806ed01c830a input=5aa33e4a133391ab]*/
+/*[clinic end generated code: output=a66a806ed01c830a input=4aa6f280d78e4574]*/
 {
     return codec_tuple(PyUnicode_AsRawUnicodeEscapeString(str),
                        PyUnicode_GET_LENGTH(str));
@@ -832,14 +832,14 @@ _codecs_raw_unicode_escape_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.latin_1_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_latin_1_encode_impl(PyObject *module, PyObject *str,
                             const char *errors)
-/*[clinic end generated code: output=2c28c83a27884e08 input=30b11c9e49a65150]*/
+/*[clinic end generated code: output=2c28c83a27884e08 input=ec3ef74bf85c5c5d]*/
 {
     return codec_tuple(_PyUnicode_AsLatin1String(str, errors),
                        PyUnicode_GET_LENGTH(str));
@@ -848,14 +848,14 @@ _codecs_latin_1_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.ascii_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_ascii_encode_impl(PyObject *module, PyObject *str,
                           const char *errors)
-/*[clinic end generated code: output=b5e035182d33befc input=843a1d268e6dfa8e]*/
+/*[clinic end generated code: output=b5e035182d33befc input=93e6e602838bd3de]*/
 {
     return codec_tuple(_PyUnicode_AsASCIIString(str, errors),
                        PyUnicode_GET_LENGTH(str));
@@ -864,15 +864,15 @@ _codecs_ascii_encode_impl(PyObject *module, PyObject *str,
 /*[clinic input]
 _codecs.charmap_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
-    mapping: object = NULL
+    errors: str(accept={str, NoneType}) = None
+    mapping: object = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_charmap_encode_impl(PyObject *module, PyObject *str,
                             const char *errors, PyObject *mapping)
-/*[clinic end generated code: output=047476f48495a9e9 input=0752cde07a6d6d00]*/
+/*[clinic end generated code: output=047476f48495a9e9 input=2a98feae73dadce8]*/
 {
     if (mapping == Py_None)
         mapping = NULL;
@@ -899,13 +899,13 @@ _codecs_charmap_build_impl(PyObject *module, PyObject *map)
 /*[clinic input]
 _codecs.mbcs_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_mbcs_encode_impl(PyObject *module, PyObject *str, const char *errors)
-/*[clinic end generated code: output=76e2e170c966c080 input=de471e0815947553]*/
+/*[clinic end generated code: output=76e2e170c966c080 input=2e932fc289ea5a5b]*/
 {
     return codec_tuple(PyUnicode_EncodeCodePage(CP_ACP, str, errors),
                        PyUnicode_GET_LENGTH(str));
@@ -914,13 +914,13 @@ _codecs_mbcs_encode_impl(PyObject *module, PyObject *str, const char *errors)
 /*[clinic input]
 _codecs.oem_encode
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_oem_encode_impl(PyObject *module, PyObject *str, const char *errors)
-/*[clinic end generated code: output=65d5982c737de649 input=3fc5f0028aad3cda]*/
+/*[clinic end generated code: output=65d5982c737de649 input=9eac86dc21eb14f2]*/
 {
     return codec_tuple(PyUnicode_EncodeCodePage(CP_OEMCP, str, errors),
         PyUnicode_GET_LENGTH(str));
@@ -930,14 +930,14 @@ _codecs_oem_encode_impl(PyObject *module, PyObject *str, const char *errors)
 _codecs.code_page_encode
     code_page: int
     str: unicode
-    errors: str(accept={str, NoneType}) = NULL
+    errors: str(accept={str, NoneType}) = None
     /
 [clinic start generated code]*/
 
 static PyObject *
 _codecs_code_page_encode_impl(PyObject *module, int code_page, PyObject *str,
                               const char *errors)
-/*[clinic end generated code: output=45673f6085657a9e input=786421ae617d680b]*/
+/*[clinic end generated code: output=45673f6085657a9e input=7d18a33bc8cd0f94]*/
 {
     return codec_tuple(PyUnicode_EncodeCodePage(code_page, str, errors),
                        PyUnicode_GET_LENGTH(str));
diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c
index 819e1ee41f1c..b2b1117fb093 100644
--- a/Modules/_cursesmodule.c
+++ b/Modules/_cursesmodule.c
@@ -3202,7 +3202,7 @@ _curses_initscr_impl(PyObject *module)
 /*[clinic input]
 _curses.setupterm
 
-    term: str(accept={str, NoneType}) = NULL
+    term: str(accept={str, NoneType}) = None
         Terminal name.
         If omitted, the value of the TERM environment variable will be used.
     fd: int = -1
@@ -3214,7 +3214,7 @@ Initialize the terminal.
 
 static PyObject *
 _curses_setupterm_impl(PyObject *module, const char *term, int fd)
-/*[clinic end generated code: output=4584e587350f2848 input=8ac5f78ec6268be3]*/
+/*[clinic end generated code: output=4584e587350f2848 input=4511472766af0c12]*/
 {
     int err;
 
diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
index 75c94bcf3a09..c3f30c9339cd 100644
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -2465,10 +2465,10 @@ treebuilder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 /*[clinic input]
 _elementtree.TreeBuilder.__init__
 
-    element_factory: object = NULL
+    element_factory: object = None
     *
-    comment_factory: object = NULL
-    pi_factory: object = NULL
+    comment_factory: object = None
+    pi_factory: object = None
     insert_comments: bool = False
     insert_pis: bool = False
 
@@ -2480,16 +2480,16 @@ _elementtree_TreeBuilder___init___impl(TreeBuilderObject *self,
                                        PyObject *comment_factory,
                                        PyObject *pi_factory,
                                        int insert_comments, int insert_pis)
-/*[clinic end generated code: output=8571d4dcadfdf952 input=1f967b5c245e0a71]*/
+/*[clinic end generated code: output=8571d4dcadfdf952 input=ae98a94df20b5cc3]*/
 {
-    if (element_factory && element_factory != Py_None) {
+    if (element_factory != Py_None) {
         Py_INCREF(element_factory);
         Py_XSETREF(self->element_factory, element_factory);
     } else {
         Py_CLEAR(self->element_factory);
     }
 
-    if (!comment_factory || comment_factory == Py_None) {
+    if (comment_factory == Py_None) {
         elementtreestate *st = ET_STATE_GLOBAL;
         comment_factory = st->comment_factory;
     }
@@ -2502,7 +2502,7 @@ _elementtree_TreeBuilder___init___impl(TreeBuilderObject *self,
         self->insert_comments = 0;
     }
 
-    if (!pi_factory || pi_factory == Py_None) {
+    if (pi_factory == Py_None) {
         elementtreestate *st = ET_STATE_GLOBAL;
         pi_factory = st->pi_factory;
     }
@@ -3708,14 +3708,14 @@ _elementtree.XMLParser.__init__
 
     *
     target: object = NULL
-    encoding: str(accept={str, NoneType}) = NULL
+    encoding: str(accept={str, NoneType}) = None
 
 [clinic start generated code]*/
 
 static int
 _elementtree_XMLParser___init___impl(XMLParserObject *self, PyObject *target,
                                      const char *encoding)
-/*[clinic end generated code: output=3ae45ec6cdf344e4 input=96288fcba916cfce]*/
+/*[clinic end generated code: output=3ae45ec6cdf344e4 input=53e35a829ae043e8]*/
 {
     self->entity = PyDict_New();
     if (!self->entity)
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index c685279c7a7b..88bf8da03b64 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -448,7 +448,7 @@ EVPnew(const EVP_MD *digest,
 _hashlib.new as EVP_new
 
     name as name_obj: object
-    string as data_obj: object(py_default="b''") = NULL
+    string as data_obj: object(c_default="NULL") = b''
 
 Return a new hash object using the named algorithm.
 
@@ -460,7 +460,7 @@ The MD5 and SHA1 algorithms are always supported.
 
 static PyObject *
 EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj)
-/*[clinic end generated code: output=9e7cf664e04b0226 input=1c46e40e0fec91f3]*/
+/*[clinic end generated code: output=9e7cf664e04b0226 input=7eb79bf30058bd02]*/
 {
     Py_buffer view = { 0 };
     PyObject *ret_obj;
diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c
index 5c2f019e840b..49ed2cb00de5 100644
--- a/Modules/_io/_iomodule.c
+++ b/Modules/_io/_iomodule.c
@@ -103,9 +103,9 @@ _io.open
     file: object
     mode: str = "r"
     buffering: int = -1
-    encoding: str(accept={str, NoneType}) = NULL
-    errors: str(accept={str, NoneType}) = NULL
-    newline: str(accept={str, NoneType}) = NULL
+    encoding: str(accept={str, NoneType}) = None
+    errors: str(accept={str, NoneType}) = None
+    newline: str(accept={str, NoneType}) = None
     closefd: bool(accept={int}) = True
     opener: object = None
 
@@ -233,7 +233,7 @@ static PyObject *
 _io_open_impl(PyObject *module, PyObject *file, const char *mode,
               int buffering, const char *encoding, const char *errors,
               const char *newline, int closefd, PyObject *opener)
-/*[clinic end generated code: output=aefafc4ce2b46dc0 input=03da2940c8a65871]*/
+/*[clinic end generated code: output=aefafc4ce2b46dc0 input=7295902222e6b311]*/
 {
     unsigned i;
 
diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h
index 64addde57f3e..53e7067cf7a7 100644
--- a/Modules/_io/clinic/fileio.c.h
+++ b/Modules/_io/clinic/fileio.c.h
@@ -408,7 +408,7 @@ static PyObject *
 _io_FileIO_truncate(fileio *self, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
-    PyObject *posobj = NULL;
+    PyObject *posobj = Py_None;
 
     if (!_PyArg_CheckPositional("truncate", nargs, 0, 1)) {
         goto exit;
@@ -447,4 +447,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored))
 #ifndef _IO_FILEIO_TRUNCATE_METHODDEF
     #define _IO_FILEIO_TRUNCATE_METHODDEF
 #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */
-/*[clinic end generated code: output=a7e9cca3613660fb input=a9049054013a1b77]*/
+/*[clinic end generated code: output=e7682d0a3264d284 input=a9049054013a1b77]*/
diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c
index 7f784a34c30d..9166c607e737 100644
--- a/Modules/_io/fileio.c
+++ b/Modules/_io/fileio.c
@@ -979,7 +979,7 @@ _io_FileIO_tell_impl(fileio *self)
 #ifdef HAVE_FTRUNCATE
 /*[clinic input]
 _io.FileIO.truncate
-    size as posobj: object = NULL
+    size as posobj: object = None
     /
 
 Truncate the file to at most size bytes and return the truncated size.
@@ -990,7 +990,7 @@ The current file position is changed to the value of size.
 
 static PyObject *
 _io_FileIO_truncate_impl(fileio *self, PyObject *posobj)
-/*[clinic end generated code: output=e49ca7a916c176fa input=9026af44686b7318]*/
+/*[clinic end generated code: output=e49ca7a916c176fa input=b0ac133939823875]*/
 {
     Py_off_t pos;
     int ret;
@@ -1002,7 +1002,7 @@ _io_FileIO_truncate_impl(fileio *self, PyObject *posobj)
     if (!self->writable)
         return err_mode("writing");
 
-    if (posobj == Py_None || posobj == NULL) {
+    if (posobj == Py_None) {
         /* Get the current position. */
         posobj = portable_lseek(self, NULL, 1);
         if (posobj == NULL)
diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c
index 4fe1b295f885..2f4a5245681f 100644
--- a/Modules/_io/textio.c
+++ b/Modules/_io/textio.c
@@ -991,9 +991,9 @@ _textiowrapper_fix_encoder_state(textio *self)
 /*[clinic input]
 _io.TextIOWrapper.__init__
     buffer: object
-    encoding: str(accept={str, NoneType}) = NULL
+    encoding: str(accept={str, NoneType}) = None
     errors: object = None
-    newline: str(accept={str, NoneType}) = NULL
+    newline: str(accept={str, NoneType}) = None
     line_buffering: bool(accept={int}) = False
     write_through: bool(accept={int}) = False
 
@@ -1032,7 +1032,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer,
                                 const char *encoding, PyObject *errors,
                                 const char *newline, int line_buffering,
                                 int write_through)
-/*[clinic end generated code: output=72267c0c01032ed2 input=1c5dd5d78bfcc675]*/
+/*[clinic end generated code: output=72267c0c01032ed2 input=77d8696d1a1f460b]*/
 {
     PyObject *raw, *codec_info = NULL;
     _PyIO_State *state = NULL;
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index fa2ef7d9dcd0..8ee7d8ee2a79 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -1142,7 +1142,7 @@ _Pickler_SetProtocol(PicklerObject *self, PyObject *protocol, int fix_imports)
 {
     long proto;
 
-    if (protocol == NULL || protocol == Py_None) {
+    if (protocol == Py_None) {
         proto = DEFAULT_PROTOCOL;
     }
     else {
@@ -4640,9 +4640,9 @@ Pickler_clear(PicklerObject *self)
 _pickle.Pickler.__init__
 
   file: object
-  protocol: object = NULL
+  protocol: object = None
   fix_imports: bool = True
-  buffer_callback: object = NULL
+  buffer_callback: object = None
 
 This takes a binary file for writing a pickle data stream.
 
@@ -4680,7 +4680,7 @@ static int
 _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file,
                               PyObject *protocol, int fix_imports,
                               PyObject *buffer_callback)
-/*[clinic end generated code: output=0abedc50590d259b input=9a43a1c50ab91652]*/
+/*[clinic end generated code: output=0abedc50590d259b input=bb886e00443a7811]*/
 {
     _Py_IDENTIFIER(persistent_id);
     _Py_IDENTIFIER(dispatch_table);
@@ -7202,7 +7202,7 @@ _pickle.Unpickler.__init__
   fix_imports: bool = True
   encoding: str = 'ASCII'
   errors: str = 'strict'
-  buffers: object = NULL
+  buffers: object(c_default="NULL") = ()
 
 This takes a binary file for reading a pickle data stream.
 
@@ -7230,7 +7230,7 @@ static int
 _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file,
                                 int fix_imports, const char *encoding,
                                 const char *errors, PyObject *buffers)
-/*[clinic end generated code: output=09f0192649ea3f85 input=da4b62d9edb68700]*/
+/*[clinic end generated code: output=09f0192649ea3f85 input=ca4c1faea9553121]*/
 {
     _Py_IDENTIFIER(persistent_load);
 
@@ -7622,10 +7622,10 @@ _pickle.dump
 
   obj: object
   file: object
-  protocol: object = NULL
+  protocol: object = None
   *
   fix_imports: bool = True
-  buffer_callback: object = NULL
+  buffer_callback: object = None
 
 Write a pickled representation of obj to the open file object file.
 
@@ -7660,7 +7660,7 @@ static PyObject *
 _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file,
                   PyObject *protocol, int fix_imports,
                   PyObject *buffer_callback)
-/*[clinic end generated code: output=706186dba996490c input=2f035f02cc0f9547]*/
+/*[clinic end generated code: output=706186dba996490c input=cfdcaf573ed6e46c]*/
 {
     PicklerObject *pickler = _Pickler_New();
 
@@ -7695,10 +7695,10 @@ _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file,
 _pickle.dumps
 
   obj: object
-  protocol: object = NULL
+  protocol: object = None
   *
   fix_imports: bool = True
-  buffer_callback: object = NULL
+  buffer_callback: object = None
 
 Return the pickled representation of the object as a bytes object.
 
@@ -7724,7 +7724,7 @@ into *file* as part of the pickle stream.  It is an error if
 static PyObject *
 _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol,
                    int fix_imports, PyObject *buffer_callback)
-/*[clinic end generated code: output=fbab0093a5580fdf input=001f167df711b9f1]*/
+/*[clinic end generated code: output=fbab0093a5580fdf input=9f334d535ff7194f]*/
 {
     PyObject *result;
     PicklerObject *pickler = _Pickler_New();
@@ -7759,7 +7759,7 @@ _pickle.load
   fix_imports: bool = True
   encoding: str = 'ASCII'
   errors: str = 'strict'
-  buffers: object = NULL
+  buffers: object(c_default="NULL") = ()
 
 Read and return an object from the pickle data stored in a file.
 
@@ -7790,7 +7790,7 @@ static PyObject *
 _pickle_load_impl(PyObject *module, PyObject *file, int fix_imports,
                   const char *encoding, const char *errors,
                   PyObject *buffers)
-/*[clinic end generated code: output=250452d141c23e76 input=29fae982fe778156]*/
+/*[clinic end generated code: output=250452d141c23e76 input=46c7c31c92f4f371]*/
 {
     PyObject *result;
     UnpicklerObject *unpickler = _Unpickler_New();
@@ -7827,7 +7827,7 @@ _pickle.loads
   fix_imports: bool = True
   encoding: str = 'ASCII'
   errors: str = 'strict'
-  buffers: object = NULL
+  buffers: object(c_default="NULL") = ()
 
 Read and return an object from the given pickle data.
 
@@ -7849,7 +7849,7 @@ static PyObject *
 _pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports,
                    const char *encoding, const char *errors,
                    PyObject *buffers)
-/*[clinic end generated code: output=82ac1e6b588e6d02 input=c6004393f8276867]*/
+/*[clinic end generated code: output=82ac1e6b588e6d02 input=9c2ab6a0960185ea]*/
 {
     PyObject *result;
     UnpicklerObject *unpickler = _Unpickler_New();
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index 64060c64c4df..6f1f9c881530 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -3883,15 +3883,15 @@ _password_callback(char *buf, int size, int rwflag, void *userdata)
 /*[clinic input]
 _ssl._SSLContext.load_cert_chain
     certfile: object
-    keyfile: object = NULL
-    password: object = NULL
+    keyfile: object = None
+    password: object = None
 
 [clinic start generated code]*/
 
 static PyObject *
 _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile,
                                       PyObject *keyfile, PyObject *password)
-/*[clinic end generated code: output=9480bc1c380e2095 input=7cf9ac673cbee6fc]*/
+/*[clinic end generated code: output=9480bc1c380e2095 input=30bc7e967ea01a58]*/
 {
     PyObject *certfile_bytes = NULL, *keyfile_bytes = NULL;
     pem_password_cb *orig_passwd_cb = SSL_CTX_get_default_passwd_cb(self->ctx);
@@ -3917,7 +3917,7 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile,
         }
         goto error;
     }
-    if (password && password != Py_None) {
+    if (password != Py_None) {
         if (PyCallable_Check(password)) {
             pw_info.callable = password;
         } else if (!_pwinfo_set(&pw_info, password,
@@ -4075,9 +4075,9 @@ _add_ca_certs(PySSLContext *self, void *data, Py_ssize_t len,
 
 /*[clinic input]
 _ssl._SSLContext.load_verify_locations
-    cafile: object = NULL
-    capath: object = NULL
-    cadata: object = NULL
+    cafile: object = None
+    capath: object = None
+    cadata: object = None
 
 [clinic start generated code]*/
 
@@ -4086,7 +4086,7 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self,
                                             PyObject *cafile,
                                             PyObject *capath,
                                             PyObject *cadata)
-/*[clinic end generated code: output=454c7e41230ca551 input=997f1fb3a784ef88]*/
+/*[clinic end generated code: output=454c7e41230ca551 input=42ecfe258233e194]*/
 {
     PyObject *cafile_bytes = NULL, *capath_bytes = NULL;
     const char *cafile_buf = NULL, *capath_buf = NULL;
diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c
index 613a95b08972..02924d480f25 100644
--- a/Modules/_tkinter.c
+++ b/Modules/_tkinter.c
@@ -3121,8 +3121,8 @@ _tkinter__flatten(PyObject *module, PyObject *item)
 /*[clinic input]
 _tkinter.create
 
-    screenName: str(accept={str, NoneType}) = NULL
-    baseName: str = NULL
+    screenName: str(accept={str, NoneType}) = None
+    baseName: str = ""
     className: str = "Tk"
     interactive: bool(accept={int}) = False
     wantobjects: bool(accept={int}) = False
@@ -3130,7 +3130,7 @@ _tkinter.create
         if false, then Tk_Init() doesn't get called
     sync: bool(accept={int}) = False
         if true, then pass -sync to wish
-    use: str(accept={str, NoneType}) = NULL
+    use: str(accept={str, NoneType}) = None
         if not None, then pass -use to wish
     /
 
@@ -3141,7 +3141,7 @@ _tkinter_create_impl(PyObject *module, const char *screenName,
                      const char *baseName, const char *className,
                      int interactive, int wantobjects, int wantTk, int sync,
                      const char *use)
-/*[clinic end generated code: output=e3315607648e6bb4 input=431907c134c80085]*/
+/*[clinic end generated code: output=e3315607648e6bb4 input=da9b17ee7358d862]*/
 {
     /* XXX baseName is not used anymore;
      * try getting rid of it. */
diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c
index c01a0e5dfadf..4a751f7ca555 100644
--- a/Modules/cjkcodecs/multibytecodec.c
+++ b/Modules/cjkcodecs/multibytecodec.c
@@ -548,7 +548,7 @@ multibytecodec_encode(MultibyteCodec *codec,
 _multibytecodec.MultibyteCodec.encode
 
   input: object
-  errors: str(accept={str, NoneType}) = NULL
+  errors: str(accept={str, NoneType}) = None
 
 Return an encoded string version of `input'.
 
@@ -562,7 +562,7 @@ static PyObject *
 _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self,
                                            PyObject *input,
                                            const char *errors)
-/*[clinic end generated code: output=7b26652045ba56a9 input=05f6ced3c8dd0582]*/
+/*[clinic end generated code: output=7b26652045ba56a9 input=606d0e128a577bae]*/
 {
     MultibyteCodec_State state;
     PyObject *errorcb, *r, *ucvt;
@@ -617,7 +617,7 @@ _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self,
 _multibytecodec.MultibyteCodec.decode
 
   input: Py_buffer
-  errors: str(accept={str, NoneType}) = NULL
+  errors: str(accept={str, NoneType}) = None
 
 Decodes 'input'.
 
@@ -631,7 +631,7 @@ static PyObject *
 _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self,
                                            Py_buffer *input,
                                            const char *errors)
-/*[clinic end generated code: output=ff419f65bad6cc77 input=a7d45f87f75e5e02]*/
+/*[clinic end generated code: output=ff419f65bad6cc77 input=e0c78fc7ab190def]*/
 {
     MultibyteCodec_State state;
     MultibyteDecodeBuffer buf;
diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h
index b9daee6aae96..17eb77334d0a 100644
--- a/Modules/clinic/_asynciomodule.c.h
+++ b/Modules/clinic/_asynciomodule.c.h
@@ -119,7 +119,7 @@ PyDoc_STRVAR(_asyncio_Future_set_exception__doc__,
     {"set_exception", (PyCFunction)_asyncio_Future_set_exception, METH_O, _asyncio_Future_set_exception__doc__},
 
 PyDoc_STRVAR(_asyncio_Future_add_done_callback__doc__,
-"add_done_callback($self, fn, /, *, context=None)\n"
+"add_done_callback($self, fn, /, *, context=<unrepresentable>)\n"
 "--\n"
 "\n"
 "Add a callback to be run when the future becomes done.\n"
@@ -832,4 +832,4 @@ _asyncio__leave_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=51c50219f6a0863a input=a9049054013a1b77]*/
+/*[clinic end generated code: output=585ba1f8de5b4103 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h
index be1d2f70cf1b..772c8ca538da 100644
--- a/Modules/clinic/_codecsmodule.c.h
+++ b/Modules/clinic/_codecsmodule.c.h
@@ -1434,7 +1434,7 @@ _codecs_charmap_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs
     PyObject *return_value = NULL;
     Py_buffer data = {NULL, NULL};
     const char *errors = NULL;
-    PyObject *mapping = NULL;
+    PyObject *mapping = Py_None;
 
     if (!_PyArg_CheckPositional("charmap_decode", nargs, 1, 3)) {
         goto exit;
@@ -2542,7 +2542,7 @@ _codecs_charmap_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs
     PyObject *return_value = NULL;
     PyObject *str;
     const char *errors = NULL;
-    PyObject *mapping = NULL;
+    PyObject *mapping = Py_None;
 
     if (!_PyArg_CheckPositional("charmap_encode", nargs, 1, 3)) {
         goto exit;
@@ -2922,4 +2922,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg)
 #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF
     #define _CODECS_CODE_PAGE_ENCODE_METHODDEF
 #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */
-/*[clinic end generated code: output=59726a305e4ec24a input=a9049054013a1b77]*/
+/*[clinic end generated code: output=51b42d170889524c input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h
index 324e549d486a..0bc4bb5d15ce 100644
--- a/Modules/clinic/_elementtree.c.h
+++ b/Modules/clinic/_elementtree.c.h
@@ -650,9 +650,9 @@ _elementtree_TreeBuilder___init__(PyObject *self, PyObject *args, PyObject *kwar
     PyObject * const *fastargs;
     Py_ssize_t nargs = PyTuple_GET_SIZE(args);
     Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0;
-    PyObject *element_factory = NULL;
-    PyObject *comment_factory = NULL;
-    PyObject *pi_factory = NULL;
+    PyObject *element_factory = Py_None;
+    PyObject *comment_factory = Py_None;
+    PyObject *pi_factory = Py_None;
     int insert_comments = 0;
     int insert_pis = 0;
 
@@ -969,4 +969,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args,
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=50e0b1954c5f9e0f input=a9049054013a1b77]*/
+/*[clinic end generated code: output=1443ed7bb9f9e03e input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h
index b92daa72105a..9da3f1195a3b 100644
--- a/Modules/clinic/_pickle.c.h
+++ b/Modules/clinic/_pickle.c.h
@@ -112,9 +112,9 @@ _pickle_Pickler___init__(PyObject *self, PyObject *args, PyObject *kwargs)
     Py_ssize_t nargs = PyTuple_GET_SIZE(args);
     Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1;
     PyObject *file;
-    PyObject *protocol = NULL;
+    PyObject *protocol = Py_None;
     int fix_imports = 1;
-    PyObject *buffer_callback = NULL;
+    PyObject *buffer_callback = Py_None;
 
     fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 4, 0, argsbuf);
     if (!fastargs) {
@@ -292,7 +292,7 @@ _pickle_Unpickler___sizeof__(UnpicklerObject *self, PyObject *Py_UNUSED(ignored)
 
 PyDoc_STRVAR(_pickle_Unpickler___init____doc__,
 "Unpickler(file, *, fix_imports=True, encoding=\'ASCII\', errors=\'strict\',\n"
-"          buffers=None)\n"
+"          buffers=())\n"
 "--\n"
 "\n"
 "This takes a binary file for reading a pickle data stream.\n"
@@ -502,9 +502,9 @@ _pickle_dump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2;
     PyObject *obj;
     PyObject *file;
-    PyObject *protocol = NULL;
+    PyObject *protocol = Py_None;
     int fix_imports = 1;
-    PyObject *buffer_callback = NULL;
+    PyObject *buffer_callback = Py_None;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf);
     if (!args) {
@@ -582,9 +582,9 @@ _pickle_dumps(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
     PyObject *argsbuf[4];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
     PyObject *obj;
-    PyObject *protocol = NULL;
+    PyObject *protocol = Py_None;
     int fix_imports = 1;
-    PyObject *buffer_callback = NULL;
+    PyObject *buffer_callback = Py_None;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf);
     if (!args) {
@@ -623,7 +623,7 @@ _pickle_dumps(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
 
 PyDoc_STRVAR(_pickle_load__doc__,
 "load($module, /, file, *, fix_imports=True, encoding=\'ASCII\',\n"
-"     errors=\'strict\', buffers=None)\n"
+"     errors=\'strict\', buffers=())\n"
 "--\n"
 "\n"
 "Read and return an object from the pickle data stored in a file.\n"
@@ -735,7 +735,7 @@ _pickle_load(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject
 
 PyDoc_STRVAR(_pickle_loads__doc__,
 "loads($module, /, data, *, fix_imports=True, encoding=\'ASCII\',\n"
-"      errors=\'strict\', buffers=None)\n"
+"      errors=\'strict\', buffers=())\n"
 "--\n"
 "\n"
 "Read and return an object from the given pickle data.\n"
@@ -835,4 +835,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=17f6a76ebd94325d input=a9049054013a1b77]*/
+/*[clinic end generated code: output=de075ec48d4ee0e1 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h
index 25769a225af9..ce8669ae212e 100644
--- a/Modules/clinic/_ssl.c.h
+++ b/Modules/clinic/_ssl.c.h
@@ -571,8 +571,8 @@ _ssl__SSLContext_load_cert_chain(PySSLContext *self, PyObject *const *args, Py_s
     PyObject *argsbuf[3];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
     PyObject *certfile;
-    PyObject *keyfile = NULL;
-    PyObject *password = NULL;
+    PyObject *keyfile = Py_None;
+    PyObject *password = Py_None;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf);
     if (!args) {
@@ -618,9 +618,9 @@ _ssl__SSLContext_load_verify_locations(PySSLContext *self, PyObject *const *args
     static _PyArg_Parser _parser = {NULL, _keywords, "load_verify_locations", 0};
     PyObject *argsbuf[3];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
-    PyObject *cafile = NULL;
-    PyObject *capath = NULL;
-    PyObject *cadata = NULL;
+    PyObject *cafile = Py_None;
+    PyObject *capath = Py_None;
+    PyObject *cadata = Py_None;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf);
     if (!args) {
@@ -1482,4 +1482,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje
 #ifndef _SSL_ENUM_CRLS_METHODDEF
     #define _SSL_ENUM_CRLS_METHODDEF
 #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */
-/*[clinic end generated code: output=aa4947067c3fef2d input=a9049054013a1b77]*/
+/*[clinic end generated code: output=a4aeb3f92a091c64 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h
index 4870e9b4aad7..73c3faeaf962 100644
--- a/Modules/clinic/_tkinter.c.h
+++ b/Modules/clinic/_tkinter.c.h
@@ -674,7 +674,7 @@ PyDoc_STRVAR(_tkinter__flatten__doc__,
     {"_flatten", (PyCFunction)_tkinter__flatten, METH_O, _tkinter__flatten__doc__},
 
 PyDoc_STRVAR(_tkinter_create__doc__,
-"create($module, screenName=None, baseName=None, className=\'Tk\',\n"
+"create($module, screenName=None, baseName=\'\', className=\'Tk\',\n"
 "       interactive=False, wantobjects=False, wantTk=True, sync=False,\n"
 "       use=None, /)\n"
 "--\n"
@@ -702,7 +702,7 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
     const char *screenName = NULL;
-    const char *baseName = NULL;
+    const char *baseName = "";
     const char *className = "Tk";
     int interactive = 0;
     int wantobjects = 0;
@@ -912,4 +912,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored))
 #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF
     #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF
 #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */
-/*[clinic end generated code: output=da0115c470aac1c0 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=492b8b833fe54bc9 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h
index e18d407ba9e8..82942f08a686 100644
--- a/Modules/clinic/binascii.c.h
+++ b/Modules/clinic/binascii.c.h
@@ -432,7 +432,7 @@ binascii_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 }
 
 PyDoc_STRVAR(binascii_b2a_hex__doc__,
-"b2a_hex($module, /, data, sep=None, bytes_per_sep=1)\n"
+"b2a_hex($module, /, data, sep=<unrepresentable>, bytes_per_sep=1)\n"
 "--\n"
 "\n"
 "Hexadecimal representation of binary data.\n"
@@ -515,7 +515,7 @@ binascii_b2a_hex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb
 }
 
 PyDoc_STRVAR(binascii_hexlify__doc__,
-"hexlify($module, /, data, sep=None, bytes_per_sep=1)\n"
+"hexlify($module, /, data, sep=<unrepresentable>, bytes_per_sep=1)\n"
 "--\n"
 "\n"
 "Hexadecimal representation of binary data.\n"
@@ -801,4 +801,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj
 
     return return_value;
 }
-/*[clinic end generated code: output=e13bd02ac40496f0 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=ec26d03c2007eaac input=a9049054013a1b77]*/
diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h
index 33509872069f..81a8437c3a7d 100644
--- a/Modules/clinic/cmathmodule.c.h
+++ b/Modules/clinic/cmathmodule.c.h
@@ -648,10 +648,10 @@ cmath_tanh(PyObject *module, PyObject *arg)
 }
 
 PyDoc_STRVAR(cmath_log__doc__,
-"log($module, x, y_obj=None, /)\n"
+"log($module, z, base=<unrepresentable>, /)\n"
 "--\n"
 "\n"
-"The logarithm of z to the given base.\n"
+"log(z[, base]) -> the logarithm of z to the given base.\n"
 "\n"
 "If the base not specified, returns the natural logarithm (base e) of z.");
 
@@ -968,4 +968,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=3ab228947d1709cc input=a9049054013a1b77]*/
+/*[clinic end generated code: output=3edc4484b10ae752 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h
index 1a5b03cb5be1..c0d1d4df7957 100644
--- a/Modules/clinic/posixmodule.c.h
+++ b/Modules/clinic/posixmodule.c.h
@@ -1971,8 +1971,8 @@ os_uname(PyObject *module, PyObject *Py_UNUSED(ignored))
 #endif /* defined(HAVE_UNAME) */
 
 PyDoc_STRVAR(os_utime__doc__,
-"utime($module, /, path, times=None, *, ns=None, dir_fd=None,\n"
-"      follow_symlinks=True)\n"
+"utime($module, /, path, times=None, *, ns=<unrepresentable>,\n"
+"      dir_fd=None, follow_symlinks=True)\n"
 "--\n"
 "\n"
 "Set the access and modified time of path.\n"
@@ -2015,7 +2015,7 @@ os_utime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw
     PyObject *argsbuf[5];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
     path_t path = PATH_T_INITIALIZE("utime", "path", 0, PATH_UTIME_HAVE_FD);
-    PyObject *times = NULL;
+    PyObject *times = Py_None;
     PyObject *ns = NULL;
     int dir_fd = DEFAULT_DIR_FD;
     int follow_symlinks = 1;
@@ -2208,8 +2208,8 @@ os_execve(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k
 
 PyDoc_STRVAR(os_posix_spawn__doc__,
 "posix_spawn($module, path, argv, env, /, *, file_actions=(),\n"
-"            setpgroup=None, resetids=False, setsid=False,\n"
-"            setsigmask=(), setsigdef=(), scheduler=None)\n"
+"            setpgroup=<unrepresentable>, resetids=False, setsid=False,\n"
+"            setsigmask=(), setsigdef=(), scheduler=<unrepresentable>)\n"
 "--\n"
 "\n"
 "Execute the program specified by path in a new process.\n"
@@ -2345,8 +2345,8 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje
 
 PyDoc_STRVAR(os_posix_spawnp__doc__,
 "posix_spawnp($module, path, argv, env, /, *, file_actions=(),\n"
-"             setpgroup=None, resetids=False, setsid=False,\n"
-"             setsigmask=(), setsigdef=(), scheduler=None)\n"
+"             setpgroup=<unrepresentable>, resetids=False, setsid=False,\n"
+"             setsigmask=(), setsigdef=(), scheduler=<unrepresentable>)\n"
 "--\n"
 "\n"
 "Execute the program specified by path in a new process.\n"
@@ -2598,8 +2598,9 @@ os_spawnve(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 #if defined(HAVE_FORK)
 
 PyDoc_STRVAR(os_register_at_fork__doc__,
-"register_at_fork($module, /, *, before=None, after_in_child=None,\n"
-"                 after_in_parent=None)\n"
+"register_at_fork($module, /, *, before=<unrepresentable>,\n"
+"                 after_in_child=<unrepresentable>,\n"
+"                 after_in_parent=<unrepresentable>)\n"
 "--\n"
 "\n"
 "Register callables to be called when forking a new process.\n"
@@ -6854,11 +6855,9 @@ os_abort(PyObject *module, PyObject *Py_UNUSED(ignored))
 #if defined(MS_WINDOWS)
 
 PyDoc_STRVAR(os_startfile__doc__,
-"startfile($module, /, filepath, operation=None)\n"
+"startfile($module, /, filepath, operation=<unrepresentable>)\n"
 "--\n"
 "\n"
-"startfile(filepath [, operation])\n"
-"\n"
 "Start a file with its associated application.\n"
 "\n"
 "When \"operation\" is not specified or \"open\", this acts like\n"
@@ -8724,4 +8723,4 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar
 #ifndef OS__REMOVE_DLL_DIRECTORY_METHODDEF
     #define OS__REMOVE_DLL_DIRECTORY_METHODDEF
 #endif /* !defined(OS__REMOVE_DLL_DIRECTORY_METHODDEF) */
-/*[clinic end generated code: output=ba6bac1702f55dbb input=a9049054013a1b77]*/
+/*[clinic end generated code: output=1ded1fbc8fd37b27 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h
index 41ab536bce23..ee5907ca7e44 100644
--- a/Modules/clinic/pyexpat.c.h
+++ b/Modules/clinic/pyexpat.c.h
@@ -133,7 +133,8 @@ pyexpat_xmlparser_GetInputContext(xmlparseobject *self, PyObject *Py_UNUSED(igno
 }
 
 PyDoc_STRVAR(pyexpat_xmlparser_ExternalEntityParserCreate__doc__,
-"ExternalEntityParserCreate($self, context, encoding=None, /)\n"
+"ExternalEntityParserCreate($self, context, encoding=<unrepresentable>,\n"
+"                           /)\n"
 "--\n"
 "\n"
 "Create a parser for parsing an external entity based on the information passed to the ExternalEntityRefHandler.");
@@ -280,7 +281,7 @@ pyexpat_xmlparser_UseForeignDTD(xmlparseobject *self, PyObject *const *args, Py_
 
 PyDoc_STRVAR(pyexpat_ParserCreate__doc__,
 "ParserCreate($module, /, encoding=None, namespace_separator=None,\n"
-"             intern=None)\n"
+"             intern=<unrepresentable>)\n"
 "--\n"
 "\n"
 "Return a new XML parser object.");
@@ -401,4 +402,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg)
 #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF
     #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF
 #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */
-/*[clinic end generated code: output=133a4105d508ebec input=a9049054013a1b77]*/
+/*[clinic end generated code: output=68ce25024280af41 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/unicodedata.c.h b/Modules/clinic/unicodedata.c.h
index 77e4f22593b0..4251db2edc7b 100644
--- a/Modules/clinic/unicodedata.c.h
+++ b/Modules/clinic/unicodedata.c.h
@@ -3,7 +3,7 @@ preserve
 [clinic start generated code]*/
 
 PyDoc_STRVAR(unicodedata_UCD_decimal__doc__,
-"decimal($self, chr, default=None, /)\n"
+"decimal($self, chr, default=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Converts a Unicode character into its equivalent decimal value.\n"
@@ -53,7 +53,7 @@ unicodedata_UCD_decimal(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 }
 
 PyDoc_STRVAR(unicodedata_UCD_digit__doc__,
-"digit($self, chr, default=None, /)\n"
+"digit($self, chr, default=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Converts a Unicode character into its equivalent digit value.\n"
@@ -102,7 +102,7 @@ unicodedata_UCD_digit(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 }
 
 PyDoc_STRVAR(unicodedata_UCD_numeric__doc__,
-"numeric($self, chr, default=None, /)\n"
+"numeric($self, chr, default=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Converts a Unicode character into its equivalent numeric value.\n"
@@ -481,7 +481,7 @@ unicodedata_UCD_normalize(PyObject *self, PyObject *const *args, Py_ssize_t narg
 }
 
 PyDoc_STRVAR(unicodedata_UCD_name__doc__,
-"name($self, chr, default=None, /)\n"
+"name($self, chr, default=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Returns the name assigned to the character chr as a string.\n"
@@ -559,4 +559,4 @@ unicodedata_UCD_lookup(PyObject *self, PyObject *arg)
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=3238f76814fb48d1 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=10c23477dbe8a202 input=a9049054013a1b77]*/
diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c
index b421f04fa605..02c09bbea48a 100644
--- a/Modules/cmathmodule.c
+++ b/Modules/cmathmodule.c
@@ -944,18 +944,18 @@ cmath_tanh_impl(PyObject *module, Py_complex z)
 /*[clinic input]
 cmath.log
 
-    x: Py_complex
-    y_obj: object = NULL
+    z as x: Py_complex
+    base as y_obj: object = NULL
     /
 
-The logarithm of z to the given base.
+log(z[, base]) -> the logarithm of z to the given base.
 
 If the base not specified, returns the natural logarithm (base e) of z.
 [clinic start generated code]*/
 
 static PyObject *
 cmath_log_impl(PyObject *module, Py_complex x, PyObject *y_obj)
-/*[clinic end generated code: output=4effdb7d258e0d94 input=ee0e823a7c6e68ea]*/
+/*[clinic end generated code: output=4effdb7d258e0d94 input=230ed3a71ecd000a]*/
 {
     Py_complex y;
 
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index a066eb1e66d4..b09204d63392 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -4739,7 +4739,7 @@ split_py_long_to_s_and_ns(PyObject *py_long, time_t *s, long *ns)
 os.utime
 
     path: path_t(allow_fd='PATH_UTIME_HAVE_FD')
-    times: object = NULL
+    times: object = None
     *
     ns: object = NULL
     dir_fd: dir_fd(requires='futimensat') = None
@@ -4776,7 +4776,7 @@ dir_fd and follow_symlinks may not be available on your platform.
 static PyObject *
 os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns,
               int dir_fd, int follow_symlinks)
-/*[clinic end generated code: output=cfcac69d027b82cf input=081cdc54ca685385]*/
+/*[clinic end generated code: output=cfcac69d027b82cf input=2fbd62a2f228f8f4]*/
 {
 #ifdef MS_WINDOWS
     HANDLE hFile;
@@ -4789,14 +4789,14 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns,
 
     memset(&utime, 0, sizeof(utime_t));
 
-    if (times && (times != Py_None) && ns) {
+    if (times != Py_None && ns) {
         PyErr_SetString(PyExc_ValueError,
                      "utime: you may specify either 'times'"
                      " or 'ns' but not both");
         return NULL;
     }
 
-    if (times && (times != Py_None)) {
+    if (times != Py_None) {
         time_t a_sec, m_sec;
         long a_nsec, m_nsec;
         if (!PyTuple_CheckExact(times) || (PyTuple_Size(times) != 2)) {
@@ -11576,8 +11576,6 @@ os.startfile
     filepath: path_t
     operation: Py_UNICODE = NULL
 
-startfile(filepath [, operation])
-
 Start a file with its associated application.
 
 When "operation" is not specified or "open", this acts like
@@ -11599,7 +11597,7 @@ the underlying Win32 ShellExecute function doesn't work if it is.
 static PyObject *
 os_startfile_impl(PyObject *module, path_t *filepath,
                   const Py_UNICODE *operation)
-/*[clinic end generated code: output=66dc311c94d50797 input=63950bf2986380d0]*/
+/*[clinic end generated code: output=66dc311c94d50797 input=c940888a5390f039]*/
 {
     HINSTANCE rc;
 
diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c
index bb5e25cc585d..df1a69b653b3 100644
--- a/Modules/pyexpat.c
+++ b/Modules/pyexpat.c
@@ -1503,8 +1503,8 @@ static PyTypeObject Xmlparsetype = {
 /*[clinic input]
 pyexpat.ParserCreate
 
-    encoding: str(accept={str, NoneType}) = NULL
-    namespace_separator: str(accept={str, NoneType}) = NULL
+    encoding: str(accept={str, NoneType}) = None
+    namespace_separator: str(accept={str, NoneType}) = None
     intern: object = NULL
 
 Return a new XML parser object.
@@ -1513,7 +1513,7 @@ Return a new XML parser object.
 static PyObject *
 pyexpat_ParserCreate_impl(PyObject *module, const char *encoding,
                           const char *namespace_separator, PyObject *intern)
-/*[clinic end generated code: output=295c0cf01ab1146c input=23d29704acad385d]*/
+/*[clinic end generated code: output=295c0cf01ab1146c input=e8da8e8d7122cb5d]*/
 {
     PyObject *result;
     int intern_decref = 0;
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
index 9358e5effd56..3b69fecde27b 100644
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -2004,7 +2004,7 @@ do_strip(PyBytesObject *self, int striptype)
 Py_LOCAL_INLINE(PyObject *)
 do_argstrip(PyBytesObject *self, int striptype, PyObject *bytes)
 {
-    if (bytes != NULL && bytes != Py_None) {
+    if (bytes != Py_None) {
         return do_xstrip(self, striptype, bytes);
     }
     return do_strip(self, striptype);
diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h
index ad8ed7eba7e8..05577077a5f8 100644
--- a/Objects/clinic/bytearrayobject.c.h
+++ b/Objects/clinic/bytearrayobject.c.h
@@ -868,7 +868,7 @@ bytearray_fromhex(PyTypeObject *type, PyObject *arg)
 }
 
 PyDoc_STRVAR(bytearray_hex__doc__,
-"hex($self, /, sep=None, bytes_per_sep=1)\n"
+"hex($self, /, sep=<unrepresentable>, bytes_per_sep=1)\n"
 "--\n"
 "\n"
 "Create a str of hexadecimal numbers from a bytearray object.\n"
@@ -1011,4 +1011,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored))
 {
     return bytearray_sizeof_impl(self);
 }
-/*[clinic end generated code: output=09df354d3374dfb2 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=508dce79cf2dffcc input=a9049054013a1b77]*/
diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h
index 7e2ce1af285c..22024ab155c8 100644
--- a/Objects/clinic/bytesobject.c.h
+++ b/Objects/clinic/bytesobject.c.h
@@ -688,7 +688,7 @@ bytes_fromhex(PyTypeObject *type, PyObject *arg)
 }
 
 PyDoc_STRVAR(bytes_hex__doc__,
-"hex($self, /, sep=None, bytes_per_sep=1)\n"
+"hex($self, /, sep=<unrepresentable>, bytes_per_sep=1)\n"
 "--\n"
 "\n"
 "Create a str of hexadecimal numbers from a bytes object.\n"
@@ -755,4 +755,4 @@ bytes_hex(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=dbed3c3ff6faa382 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=ca60dfccf8d51e88 input=a9049054013a1b77]*/
diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h
index 6515d118e506..b684ba0ef27a 100644
--- a/Objects/clinic/floatobject.c.h
+++ b/Objects/clinic/floatobject.c.h
@@ -56,7 +56,7 @@ static PyObject *
 float___round__(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
-    PyObject *o_ndigits = NULL;
+    PyObject *o_ndigits = Py_None;
 
     if (!_PyArg_CheckPositional("__round__", nargs, 0, 1)) {
         goto exit;
@@ -351,4 +351,4 @@ float___format__(PyObject *self, PyObject *arg)
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=cc8098eb73f1a64c input=a9049054013a1b77]*/
+/*[clinic end generated code: output=1676433b9f04fbc9 input=a9049054013a1b77]*/
diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h
index 64fce10acb5c..75ac2011261f 100644
--- a/Objects/clinic/memoryobject.c.h
+++ b/Objects/clinic/memoryobject.c.h
@@ -3,7 +3,7 @@ preserve
 [clinic start generated code]*/
 
 PyDoc_STRVAR(memoryview_hex__doc__,
-"hex($self, /, sep=None, bytes_per_sep=1)\n"
+"hex($self, /, sep=<unrepresentable>, bytes_per_sep=1)\n"
 "--\n"
 "\n"
 "Return the data in the buffer as a str of hexadecimal numbers.\n"
@@ -71,4 +71,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=5e44e2bcf01057b5 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=ee265a73f68b0077 input=a9049054013a1b77]*/
diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h
index cb350874dad1..c7d6edb5fd48 100644
--- a/Objects/clinic/unicodeobject.c.h
+++ b/Objects/clinic/unicodeobject.c.h
@@ -630,7 +630,7 @@ static PyObject *
 unicode_lstrip(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
-    PyObject *chars = NULL;
+    PyObject *chars = Py_None;
 
     if (!_PyArg_CheckPositional("lstrip", nargs, 0, 1)) {
         goto exit;
@@ -664,7 +664,7 @@ static PyObject *
 unicode_rstrip(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
-    PyObject *chars = NULL;
+    PyObject *chars = Py_None;
 
     if (!_PyArg_CheckPositional("rstrip", nargs, 0, 1)) {
         goto exit;
@@ -1045,7 +1045,7 @@ unicode_swapcase(PyObject *self, PyObject *Py_UNUSED(ignored))
 }
 
 PyDoc_STRVAR(unicode_maketrans__doc__,
-"maketrans(x, y=None, z=None, /)\n"
+"maketrans(x, y=<unrepresentable>, z=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Return a translation table usable for str.translate().\n"
@@ -1232,4 +1232,4 @@ unicode_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored))
 {
     return unicode_sizeof_impl(self);
 }
-/*[clinic end generated code: output=d9a6ee45ddd0ccfd input=a9049054013a1b77]*/
+/*[clinic end generated code: output=5e15747f78f18329 input=a9049054013a1b77]*/
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
index d909e415c127..1cb8ff795fb8 100644
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -1042,7 +1042,7 @@ double_round(double x, int ndigits) {
 /*[clinic input]
 float.__round__
 
-    ndigits as o_ndigits: object = NULL
+    ndigits as o_ndigits: object = None
     /
 
 Return the Integral closest to x, rounding half toward even.
@@ -1052,13 +1052,13 @@ When an argument is passed, work like built-in round(x, ndigits).
 
 static PyObject *
 float___round___impl(PyObject *self, PyObject *o_ndigits)
-/*[clinic end generated code: output=374c36aaa0f13980 input=1ca2316b510293b8]*/
+/*[clinic end generated code: output=374c36aaa0f13980 input=fc0fe25924fbc9ed]*/
 {
     double x, rounded;
     Py_ssize_t ndigits;
 
     x = PyFloat_AsDouble(self);
-    if (o_ndigits == NULL || o_ndigits == Py_None) {
+    if (o_ndigits == Py_None) {
         /* single-argument round or with None ndigits:
          * round to nearest integer */
         rounded = round(x);
diff --git a/Objects/structseq.c b/Objects/structseq.c
index 320bf080d7e3..c158afccb97f 100644
--- a/Objects/structseq.c
+++ b/Objects/structseq.c
@@ -107,12 +107,12 @@ class structseq "PyStructSequence *" "NULL"
 @classmethod
 structseq.__new__ as structseq_new
     sequence as arg: object
-    dict: object = NULL
+    dict: object(c_default="NULL") = {}
 [clinic start generated code]*/
 
 static PyObject *
 structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict)
-/*[clinic end generated code: output=baa082e788b171da input=9b44810243907377]*/
+/*[clinic end generated code: output=baa082e788b171da input=90532511101aa3fb]*/
 {
     PyObject *ob;
     PyStructSequence *res = NULL;
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 8521a4c9d7de..2afc28d8435e 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -12455,7 +12455,7 @@ do_strip(PyObject *self, int striptype)
 static PyObject *
 do_argstrip(PyObject *self, int striptype, PyObject *sep)
 {
-    if (sep != NULL && sep != Py_None) {
+    if (sep != Py_None) {
         if (PyUnicode_Check(sep))
             return _PyUnicode_XStrip(self, striptype, sep);
         else {
@@ -12492,7 +12492,7 @@ unicode_strip_impl(PyObject *self, PyObject *chars)
 /*[clinic input]
 str.lstrip as unicode_lstrip
 
-    chars: object = NULL
+    chars: object = None
     /
 
 Return a copy of the string with leading whitespace removed.
@@ -12502,7 +12502,7 @@ If chars is given and not None, remove characters in chars instead.
 
 static PyObject *
 unicode_lstrip_impl(PyObject *self, PyObject *chars)
-/*[clinic end generated code: output=3b43683251f79ca7 input=9e56f3c45f5ff4c3]*/
+/*[clinic end generated code: output=3b43683251f79ca7 input=529f9f3834448671]*/
 {
     return do_argstrip(self, LEFTSTRIP, chars);
 }
@@ -12511,7 +12511,7 @@ unicode_lstrip_impl(PyObject *self, PyObject *chars)
 /*[clinic input]
 str.rstrip as unicode_rstrip
 
-    chars: object = NULL
+    chars: object = None
     /
 
 Return a copy of the string with trailing whitespace removed.
@@ -12521,7 +12521,7 @@ If chars is given and not None, remove characters in chars instead.
 
 static PyObject *
 unicode_rstrip_impl(PyObject *self, PyObject *chars)
-/*[clinic end generated code: output=4a59230017cc3b7a input=ac89d0219cb411ee]*/
+/*[clinic end generated code: output=4a59230017cc3b7a input=62566c627916557f]*/
 {
     return do_argstrip(self, RIGHTSTRIP, chars);
 }
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 8f9e81329c50..c8d34c940f42 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -2147,7 +2147,7 @@ builtin_repr(PyObject *module, PyObject *obj)
 round as builtin_round
 
     number: object
-    ndigits: object = NULL
+    ndigits: object = None
 
 Round a number to a given precision in decimal digits.
 
@@ -2157,7 +2157,7 @@ the return value has the same type as the number.  ndigits may be negative.
 
 static PyObject *
 builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits)
-/*[clinic end generated code: output=ff0d9dd176c02ede input=854bc3a217530c3d]*/
+/*[clinic end generated code: output=ff0d9dd176c02ede input=275678471d7aca15]*/
 {
     PyObject *round, *result;
 
@@ -2175,7 +2175,7 @@ builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits)
         return NULL;
     }
 
-    if (ndigits == NULL || ndigits == Py_None)
+    if (ndigits == Py_None)
         result = _PyObject_CallNoArg(round);
     else
         result = PyObject_CallFunctionObjArgs(round, ndigits, NULL);
diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h
index 49608ccf3df9..b936b0cf3721 100644
--- a/Python/clinic/bltinmodule.c.h
+++ b/Python/clinic/bltinmodule.c.h
@@ -719,7 +719,7 @@ builtin_round(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
     PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
     PyObject *number;
-    PyObject *ndigits = NULL;
+    PyObject *ndigits = Py_None;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf);
     if (!args) {
@@ -849,4 +849,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=1927f3c9abd00c35 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=4e118c2cd2cd98f3 input=a9049054013a1b77]*/
diff --git a/Python/clinic/context.c.h b/Python/clinic/context.c.h
index eedfc88654b6..2ac8bf7c0b8c 100644
--- a/Python/clinic/context.c.h
+++ b/Python/clinic/context.c.h
@@ -115,7 +115,7 @@ _contextvars_Context_copy(PyContext *self, PyObject *Py_UNUSED(ignored))
 }
 
 PyDoc_STRVAR(_contextvars_ContextVar_get__doc__,
-"get($self, default=None, /)\n"
+"get($self, default=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Return a value for the context variable for the current context.\n"
@@ -177,4 +177,4 @@ PyDoc_STRVAR(_contextvars_ContextVar_reset__doc__,
 
 #define _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF    \
     {"reset", (PyCFunction)_contextvars_ContextVar_reset, METH_O, _contextvars_ContextVar_reset__doc__},
-/*[clinic end generated code: output=a86b66e1516c25d4 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=f2e42f34e358e179 input=a9049054013a1b77]*/
diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h
index 743d00222914..e4867f34d4ef 100644
--- a/Python/clinic/import.c.h
+++ b/Python/clinic/import.c.h
@@ -300,7 +300,7 @@ _imp_is_frozen(PyObject *module, PyObject *arg)
 #if defined(HAVE_DYNAMIC_LOADING)
 
 PyDoc_STRVAR(_imp_create_dynamic__doc__,
-"create_dynamic($module, spec, file=None, /)\n"
+"create_dynamic($module, spec, file=<unrepresentable>, /)\n"
 "--\n"
 "\n"
 "Create an extension module.");
@@ -454,4 +454,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb
 #ifndef _IMP_EXEC_DYNAMIC_METHODDEF
     #define _IMP_EXEC_DYNAMIC_METHODDEF
 #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */
-/*[clinic end generated code: output=ff06f7cf4b73eb76 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=3dc495e9c64d944e input=a9049054013a1b77]*/
diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h
index 93bc3320e337..d2d150392620 100644
--- a/Python/clinic/sysmodule.c.h
+++ b/Python/clinic/sysmodule.c.h
@@ -135,7 +135,7 @@ static PyObject *
 sys_exit(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
-    PyObject *status = NULL;
+    PyObject *status = Py_None;
 
     if (!_PyArg_CheckPositional("exit", nargs, 0, 1)) {
         goto exit;
@@ -1088,4 +1088,4 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored))
 #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF
     #define SYS_GETANDROIDAPILEVEL_METHODDEF
 #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */
-/*[clinic end generated code: output=092edc868de055a6 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=273f9cec8bfcab91 input=a9049054013a1b77]*/
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 57f7a9a5a750..5175158a3b00 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -705,7 +705,7 @@ sys_unraisablehook(PyObject *module, PyObject *unraisable)
 /*[clinic input]
 sys.exit
 
-    status: object = NULL
+    status: object = None
     /
 
 Exit the interpreter by raising SystemExit(status).
@@ -718,7 +718,7 @@ exit status will be one (i.e., failure).
 
 static PyObject *
 sys_exit_impl(PyObject *module, PyObject *status)
-/*[clinic end generated code: output=13870986c1ab2ec0 input=a737351f86685e9c]*/
+/*[clinic end generated code: output=13870986c1ab2ec0 input=b86ca9497baa94f2]*/
 {
     /* Raise SystemExit so callers may catch it or clean up. */
     PyErr_SetObject(PyExc_SystemExit, status);
diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py
index cec7c5392db3..c5edc7c9336f 100755
--- a/Tools/clinic/clinic.py
+++ b/Tools/clinic/clinic.py
@@ -43,9 +43,6 @@
 
 version = '1'
 
-_empty = inspect._empty
-_void = inspect._void
-
 NoneType = type(None)
 
 class Unspecified:
@@ -2175,7 +2172,7 @@ class Function:
     def __init__(self, parameters=None, *, name,
                  module, cls=None, c_basename=None,
                  full_name=None,
-                 return_converter, return_annotation=_empty,
+                 return_converter, return_annotation=inspect.Signature.empty,
                  docstring=None, kind=CALLABLE, coexist=False,
                  docstring_only=False):
         self.parameters = parameters or collections.OrderedDict()
@@ -2253,8 +2250,8 @@ class Parameter:
     Mutable duck type of inspect.Parameter.
     """
 
-    def __init__(self, name, kind, *, default=_empty,
-                 function, converter, annotation=_empty,
+    def __init__(self, name, kind, *, default=inspect.Parameter.empty,
+                 function, converter, annotation=inspect.Parameter.empty,
                  docstring=None, group=0):
         self.name = name
         self.kind = kind
@@ -3231,6 +3228,8 @@ def converter_init(self, *, accept={str}, encoding=None, zeroes=False):
             # sorry, clinic can't support preallocated buffers
             # for es# and et#
             self.c_default = "NULL"
+        if NoneType in accept and self.c_default == "Py_None":
+            self.c_default = "NULL"
 
     def cleanup(self):
         if self.encoding:
@@ -4433,7 +4432,7 @@ def bad_node(self, node):
                 # mild hack: explicitly support NULL as a default value
                 if isinstance(expr, ast.Name) and expr.id == 'NULL':
                     value = NULL
-                    py_default = 'None'
+                    py_default = '<unrepresentable>'
                     c_default = "NULL"
                 elif (isinstance(expr, ast.BinOp) or
                     (isinstance(expr, ast.UnaryOp) and


From webhook-mailer at python.org  Sat Sep 14 09:38:21 2019
From: webhook-mailer at python.org (Dino Viehland)
Date: Sat, 14 Sep 2019 13:38:21 -0000
Subject: [Python-checkins] Fix leaks in Python-ast.c (#16127)
Message-ID: <mailman.746.1568468302.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0247e80f3c529900689425676342cb70ea31a13d
commit: 0247e80f3c529900689425676342cb70ea31a13d
branch: master
author: Eddie Elizondo <eelizondo at fb.com>
committer: Dino Viehland <dinoviehland at gmail.com>
date: 2019-09-14T14:38:17+01:00
summary:

Fix leaks in Python-ast.c (#16127)

files:
M Parser/asdl_c.py
M Python/Python-ast.c

diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 6b31ffd8315d..0b72f9ee236c 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -638,9 +638,13 @@ def visitModule(self, mod):
 ast_dealloc(AST_object *self)
 {
     /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyTypeObject *tp = Py_TYPE(self);
     PyObject_GC_UnTrack(self);
     Py_CLEAR(self->dict);
-    Py_TYPE(self)->tp_free(self);
+    freefunc free_func = PyType_GetSlot(tp, Py_tp_free);
+    assert(free_func != NULL);
+    free_func(self);
+    Py_DECREF(tp);
 }
 
 static int
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 5190a90f13bf..7a38e98c435a 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -1130,9 +1130,13 @@ static void
 ast_dealloc(AST_object *self)
 {
     /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyTypeObject *tp = Py_TYPE(self);
     PyObject_GC_UnTrack(self);
     Py_CLEAR(self->dict);
-    Py_TYPE(self)->tp_free(self);
+    freefunc free_func = PyType_GetSlot(tp, Py_tp_free);
+    assert(free_func != NULL);
+    free_func(self);
+    Py_DECREF(tp);
 }
 
 static int


From webhook-mailer at python.org  Sat Sep 14 10:20:31 2019
From: webhook-mailer at python.org (Dino Viehland)
Date: Sat, 14 Sep 2019 14:20:31 -0000
Subject: [Python-checkins] bpo-38116: Convert select module to PEP-384
 (#15971)
Message-ID: <mailman.747.1568470832.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f919054e539a5c1afde1b31c9fd7a8f5b2313311
commit: f919054e539a5c1afde1b31c9fd7a8f5b2313311
branch: master
author: Dino Viehland <dinoviehland at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-14T15:20:27+01:00
summary:

bpo-38116: Convert select module to PEP-384 (#15971)

* Convert select module to PEP-384

Summary: Do the necessary versions to be Pyro-compatible, including migrating `PyType_Ready` to `PyType_FromSpec` and moving static data into a new `_selectstate` struct.

* ?? Added by blurb_it.

* Fixup Mac OS/X build

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-11-14-49-42.bpo-38116.KDwmwt.rst
M Modules/clinic/selectmodule.c.h
M Modules/selectmodule.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-11-14-49-42.bpo-38116.KDwmwt.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-11-14-49-42.bpo-38116.KDwmwt.rst
new file mode 100644
index 000000000000..d6d630b64b05
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-11-14-49-42.bpo-38116.KDwmwt.rst	
@@ -0,0 +1 @@
+The select module is now PEP-384 compliant and no longer has static state
\ No newline at end of file
diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h
index 9015816f80c1..670af6a3d8de 100644
--- a/Modules/clinic/selectmodule.c.h
+++ b/Modules/clinic/selectmodule.c.h
@@ -959,11 +959,11 @@ select_kqueue(PyTypeObject *type, PyObject *args, PyObject *kwargs)
 {
     PyObject *return_value = NULL;
 
-    if ((type == &kqueue_queue_Type) &&
+    if ((type == _selectstate_global->kqueue_queue_Type) &&
         !_PyArg_NoPositional("kqueue", args)) {
         goto exit;
     }
-    if ((type == &kqueue_queue_Type) &&
+    if ((type == _selectstate_global->kqueue_queue_Type) &&
         !_PyArg_NoKeywords("kqueue", kwargs)) {
         goto exit;
     }
@@ -1215,4 +1215,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize
 #ifndef SELECT_KQUEUE_CONTROL_METHODDEF
     #define SELECT_KQUEUE_CONTROL_METHODDEF
 #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */
-/*[clinic end generated code: output=03041f3d09b04a3d input=a9049054013a1b77]*/
+/*[clinic end generated code: output=26bb05e5fba2bfd1 input=a9049054013a1b77]*/
diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c
index 2e06a897132a..98da7d5027f4 100644
--- a/Modules/selectmodule.c
+++ b/Modules/selectmodule.c
@@ -58,14 +58,28 @@ extern void bzero(void *, int);
 #  define SOCKET int
 #endif
 
+typedef struct {
+    PyObject *close;
+    PyTypeObject *poll_Type;
+    PyTypeObject *devpoll_Type;
+    PyTypeObject *pyEpoll_Type;
+    PyTypeObject *kqueue_event_Type;
+    PyTypeObject *kqueue_queue_Type;
+} _selectstate;
+
+static struct PyModuleDef selectmodule;
+
+#define _selectstate(o) ((_selectstate *)PyModule_GetState(o))
+#define _selectstate_global _selectstate(PyState_FindModule(&selectmodule))
+
 /*[clinic input]
 module select
 class select.poll "pollObject *" "&poll_Type"
 class select.devpoll "devpollObject *" "&devpoll_Type"
 class select.epoll "pyEpoll_Object *" "&pyEpoll_Type"
-class select.kqueue "kqueue_queue_Object *" "&kqueue_queue_Type"
+class select.kqueue "kqueue_queue_Object *" "_selectstate_global->kqueue_queue_Type"
 [clinic start generated code]*/
-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ded80abdad2b7552]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=41071028e0ede093]*/
 
 static int
 fildes_converter(PyObject *o, void *p)
@@ -400,8 +414,6 @@ typedef struct {
     int poll_running;
 } pollObject;
 
-static PyTypeObject poll_Type;
-
 /* Update the malloc'ed array of pollfds to match the dictionary
    contained within a pollObject.  Return 1 on success, 0 on an error.
 */
@@ -709,7 +721,7 @@ static pollObject *
 newPollObject(void)
 {
     pollObject *self;
-    self = PyObject_New(pollObject, &poll_Type);
+    self = PyObject_New(pollObject, _selectstate_global->poll_Type);
     if (self == NULL)
         return NULL;
     /* ufd_uptodate is a Boolean, denoting whether the
@@ -725,13 +737,22 @@ newPollObject(void)
     return self;
 }
 
+static PyObject *
+poll_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
+{
+    PyErr_Format(PyExc_TypeError, "Cannot create '%.200s' instances", _PyType_Name(type));
+    return NULL;
+}
+
 static void
 poll_dealloc(pollObject *self)
 {
+    PyObject* type = (PyObject *)Py_TYPE(self);
     if (self->ufds != NULL)
         PyMem_DEL(self->ufds);
     Py_XDECREF(self->dict);
     PyObject_Del(self);
+    Py_DECREF(type);
 }
 
 
@@ -744,8 +765,6 @@ typedef struct {
     struct pollfd *fds;
 } devpollObject;
 
-static PyTypeObject devpoll_Type;
-
 static PyObject *
 devpoll_err_closed(void)
 {
@@ -1091,7 +1110,7 @@ newDevPollObject(void)
         return NULL;
     }
 
-    self = PyObject_New(devpollObject, &devpoll_Type);
+    self = PyObject_New(devpollObject, _selectstate_global->devpoll_Type);
     if (self == NULL) {
         close(fd_devpoll);
         PyMem_DEL(fds);
@@ -1105,14 +1124,39 @@ newDevPollObject(void)
     return self;
 }
 
+static PyObject *
+devpoll_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
+{
+    PyErr_Format(PyExc_TypeError, "Cannot create '%.200s' instances", _PyType_Name(type));
+    return NULL;
+}
+
 static void
 devpoll_dealloc(devpollObject *self)
 {
+    PyObject *type = (PyObject *)Py_TYPE(self);
     (void)devpoll_internal_close(self);
     PyMem_DEL(self->fds);
     PyObject_Del(self);
+    Py_DECREF(type);
 }
 
+static PyType_Slot devpoll_Type_slots[] = {
+    {Py_tp_dealloc, devpoll_dealloc},
+    {Py_tp_getset, devpoll_getsetlist},
+    {Py_tp_methods, devpoll_methods},
+    {Py_tp_new, devpoll_new},
+    {0, 0},
+};
+
+static PyType_Spec devpoll_Type_spec = {
+    "select.devpoll",
+    sizeof(devpollObject),
+    0,
+    Py_TPFLAGS_DEFAULT,
+    devpoll_Type_slots
+};
+
 #endif  /* HAVE_SYS_DEVPOLL_H */
 
 
@@ -1201,8 +1245,7 @@ typedef struct {
     SOCKET epfd;                        /* epoll control file descriptor */
 } pyEpoll_Object;
 
-static PyTypeObject pyEpoll_Type;
-#define pyepoll_CHECK(op) (PyObject_TypeCheck((op), &pyEpoll_Type))
+#define pyepoll_CHECK(op) (PyObject_TypeCheck((op), _selectstate_global->pyEpoll_Type))
 
 static PyObject *
 pyepoll_err_closed(void)
@@ -1230,9 +1273,10 @@ static PyObject *
 newPyEpoll_Object(PyTypeObject *type, int sizehint, SOCKET fd)
 {
     pyEpoll_Object *self;
-
-    assert(type != NULL && type->tp_alloc != NULL);
-    self = (pyEpoll_Object *) type->tp_alloc(type, 0);
+    assert(type != NULL);
+    allocfunc epoll_alloc = PyType_GetSlot(type, Py_tp_alloc);
+    assert(epoll_alloc != NULL);
+    self = (pyEpoll_Object *) epoll_alloc(type, 0);
     if (self == NULL)
         return NULL;
 
@@ -1307,8 +1351,11 @@ select_epoll_impl(PyTypeObject *type, int sizehint, int flags)
 static void
 pyepoll_dealloc(pyEpoll_Object *self)
 {
+    PyTypeObject* type = Py_TYPE(self);
     (void)pyepoll_internal_close(self);
-    Py_TYPE(self)->tp_free(self);
+    freefunc epoll_free = PyType_GetSlot(type, Py_tp_free);
+    epoll_free((PyObject *)self);
+    Py_DECREF((PyObject *)type);
 }
 
 /*[clinic input]
@@ -1632,9 +1679,7 @@ select_epoll___exit___impl(pyEpoll_Object *self, PyObject *exc_type,
                            PyObject *exc_value, PyObject *exc_tb)
 /*[clinic end generated code: output=c480f38ce361748e input=7ae81a5a4c1a98d8]*/
 {
-    _Py_IDENTIFIER(close);
-
-    return _PyObject_CallMethodIdNoArgs((PyObject *)self, &PyId_close);
+    return PyObject_CallMethodObjArgs((PyObject *)self, _selectstate_global->close, NULL);
 }
 
 static PyGetSetDef pyepoll_getsetlist[] = {
@@ -1643,6 +1688,15 @@ static PyGetSetDef pyepoll_getsetlist[] = {
     {0},
 };
 
+PyDoc_STRVAR(pyepoll_doc,
+"select.epoll(sizehint=-1, flags=0)\n\
+\n\
+Returns an epolling object\n\
+\n\
+sizehint must be a positive integer or -1 for the default size. The\n\
+sizehint is used to optimize internal data structures. It doesn't limit\n\
+the maximum number of monitored events.");
+
 #endif /* HAVE_EPOLL */
 
 #ifdef HAVE_KQUEUE
@@ -1699,18 +1753,14 @@ typedef struct {
     struct kevent e;
 } kqueue_event_Object;
 
-static PyTypeObject kqueue_event_Type;
-
-#define kqueue_event_Check(op) (PyObject_TypeCheck((op), &kqueue_event_Type))
+#define kqueue_event_Check(op) (PyObject_TypeCheck((op), _selectstate_global->kqueue_event_Type))
 
 typedef struct {
     PyObject_HEAD
     SOCKET kqfd;                /* kqueue control fd */
 } kqueue_queue_Object;
 
-static PyTypeObject kqueue_queue_Type;
-
-#define kqueue_queue_Check(op) (PyObject_TypeCheck((op), &kqueue_queue_Type))
+#define kqueue_queue_Check(op) (PyObject_TypeCheck((op), _selectstate_global->kqueue_queue_Type))
 
 #if (SIZEOF_UINTPTR_T != SIZEOF_VOID_P)
 #   error uintptr_t does not match void *!
@@ -1870,6 +1920,24 @@ kqueue_event_richcompare(kqueue_event_Object *s, kqueue_event_Object *o,
     Py_RETURN_RICHCOMPARE(result, 0, op);
 }
 
+static PyType_Slot kqueue_event_Type_slots[] = {
+    {Py_tp_doc, (void*)kqueue_event_doc},
+    {Py_tp_init, kqueue_event_init},
+    {Py_tp_members, kqueue_event_members},
+    {Py_tp_new, PyType_GenericNew},
+    {Py_tp_repr, kqueue_event_repr},
+    {Py_tp_richcompare, kqueue_event_richcompare},
+    {0, 0},
+};
+
+static PyType_Spec kqueue_event_Type_spec = {
+    "select.kevent",
+    sizeof(kqueue_event_Object),
+    0,
+    Py_TPFLAGS_DEFAULT,
+    kqueue_event_Type_slots
+};
+
 static PyObject *
 kqueue_queue_err_closed(void)
 {
@@ -1896,8 +1964,10 @@ static PyObject *
 newKqueue_Object(PyTypeObject *type, SOCKET fd)
 {
     kqueue_queue_Object *self;
-    assert(type != NULL && type->tp_alloc != NULL);
-    self = (kqueue_queue_Object *) type->tp_alloc(type, 0);
+    assert(type != NULL);
+    allocfunc queue_alloc = PyType_GetSlot(type, Py_tp_alloc);
+    assert(queue_alloc != NULL);
+    self = (kqueue_queue_Object *) queue_alloc(type, 0);
     if (self == NULL) {
         return NULL;
     }
@@ -1954,8 +2024,11 @@ select_kqueue_impl(PyTypeObject *type)
 static void
 kqueue_queue_dealloc(kqueue_queue_Object *self)
 {
+    PyTypeObject* type = Py_TYPE(self);
     kqueue_queue_internal_close(self);
-    Py_TYPE(self)->tp_free(self);
+    freefunc kqueue_free = PyType_GetSlot(type, Py_tp_free);
+    kqueue_free((PyObject *)self);
+    Py_DECREF((PyObject *)type);
 }
 
 /*[clinic input]
@@ -2072,7 +2145,7 @@ select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist,
             PyErr_Format(PyExc_TypeError,
                 "timeout argument must be a number "
                 "or None, got %.200s",
-                Py_TYPE(otimeout)->tp_name);
+                _PyType_Name(Py_TYPE(otimeout)));
             return NULL;
         }
 
@@ -2168,7 +2241,7 @@ select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist,
     for (i = 0; i < gotevents; i++) {
         kqueue_event_Object *ch;
 
-        ch = PyObject_New(kqueue_event_Object, &kqueue_event_Type);
+        ch = PyObject_New(kqueue_event_Object, _selectstate_global->kqueue_event_Type);
         if (ch == NULL) {
             goto error;
         }
@@ -2210,38 +2283,20 @@ static PyMethodDef poll_methods[] = {
     {NULL, NULL}           /* sentinel */
 };
 
-static PyTypeObject poll_Type = {
-    /* The ob_type field must be initialized in the module init function
-     * to be portable to Windows without using C++. */
-    PyVarObject_HEAD_INIT(NULL, 0)
-    "select.poll",              /*tp_name*/
-    sizeof(pollObject),         /*tp_basicsize*/
-    0,                          /*tp_itemsize*/
-    /* methods */
-    (destructor)poll_dealloc, /*tp_dealloc*/
-    0,                          /*tp_vectorcall_offset*/
-    0,                          /*tp_getattr*/
-    0,                      /*tp_setattr*/
-    0,                          /*tp_as_async*/
-    0,                          /*tp_repr*/
-    0,                          /*tp_as_number*/
-    0,                          /*tp_as_sequence*/
-    0,                          /*tp_as_mapping*/
-    0,                          /*tp_hash*/
-    0,                          /*tp_call*/
-    0,                          /*tp_str*/
-    0,                          /*tp_getattro*/
-    0,                          /*tp_setattro*/
-    0,                          /*tp_as_buffer*/
-    Py_TPFLAGS_DEFAULT,         /*tp_flags*/
-    0,                          /*tp_doc*/
-    0,                          /*tp_traverse*/
-    0,                          /*tp_clear*/
-    0,                          /*tp_richcompare*/
-    0,                          /*tp_weaklistoffset*/
-    0,                          /*tp_iter*/
-    0,                          /*tp_iternext*/
-    poll_methods,               /*tp_methods*/
+
+static PyType_Slot poll_Type_slots[] = {
+    {Py_tp_dealloc, poll_dealloc},
+    {Py_tp_methods, poll_methods},
+    {Py_tp_new, poll_new},
+    {0, 0},
+};
+
+static PyType_Spec poll_Type_spec = {
+    "select.poll",
+    sizeof(pollObject),
+    0,
+    Py_TPFLAGS_DEFAULT,
+    poll_Type_slots
 };
 
 #ifdef HAVE_SYS_DEVPOLL_H
@@ -2256,42 +2311,6 @@ static PyMethodDef devpoll_methods[] = {
     {NULL,              NULL}           /* sentinel */
 };
 
-static PyTypeObject devpoll_Type = {
-    /* The ob_type field must be initialized in the module init function
-     * to be portable to Windows without using C++. */
-    PyVarObject_HEAD_INIT(NULL, 0)
-    "select.devpoll",           /*tp_name*/
-    sizeof(devpollObject),      /*tp_basicsize*/
-    0,                          /*tp_itemsize*/
-    /* methods */
-    (destructor)devpoll_dealloc, /*tp_dealloc*/
-    0,                          /*tp_vectorcall_offset*/
-    0,                          /*tp_getattr*/
-    0,                          /*tp_setattr*/
-    0,                          /*tp_as_async*/
-    0,                          /*tp_repr*/
-    0,                          /*tp_as_number*/
-    0,                          /*tp_as_sequence*/
-    0,                          /*tp_as_mapping*/
-    0,                          /*tp_hash*/
-    0,                          /*tp_call*/
-    0,                          /*tp_str*/
-    0,                          /*tp_getattro*/
-    0,                          /*tp_setattro*/
-    0,                          /*tp_as_buffer*/
-    Py_TPFLAGS_DEFAULT,         /*tp_flags*/
-    0,                          /*tp_doc*/
-    0,                          /*tp_traverse*/
-    0,                          /*tp_clear*/
-    0,                          /*tp_richcompare*/
-    0,                          /*tp_weaklistoffset*/
-    0,                          /*tp_iter*/
-    0,                          /*tp_iternext*/
-    devpoll_methods,            /*tp_methods*/
-    0,                          /* tp_members */
-    devpoll_getsetlist,         /* tp_getset */
-};
-
 #endif  /* HAVE_SYS_DEVPOLL_H */
 
 #endif /* HAVE_POLL */
@@ -2311,94 +2330,28 @@ static PyMethodDef pyepoll_methods[] = {
     {NULL,      NULL},
 };
 
-static PyTypeObject pyEpoll_Type = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    "select.epoll",                                     /* tp_name */
-    sizeof(pyEpoll_Object),                             /* tp_basicsize */
-    0,                                                  /* tp_itemsize */
-    (destructor)pyepoll_dealloc,                        /* tp_dealloc */
-    0,                                                  /* tp_vectorcall_offset */
-    0,                                                  /* tp_getattr */
-    0,                                                  /* tp_setattr */
-    0,                                                  /* tp_as_async */
-    0,                                                  /* tp_repr */
-    0,                                                  /* tp_as_number */
-    0,                                                  /* tp_as_sequence */
-    0,                                                  /* tp_as_mapping */
-    0,                                                  /* tp_hash */
-    0,                                                  /* tp_call */
-    0,                                                  /* tp_str */
-    PyObject_GenericGetAttr,                            /* tp_getattro */
-    0,                                                  /* tp_setattro */
-    0,                                                  /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,                                 /* tp_flags */
-    select_epoll__doc__,                                /* tp_doc */
-    0,                                                  /* tp_traverse */
-    0,                                                  /* tp_clear */
-    0,                                                  /* tp_richcompare */
-    0,                                                  /* tp_weaklistoffset */
-    0,                                                  /* tp_iter */
-    0,                                                  /* tp_iternext */
-    pyepoll_methods,                                    /* tp_methods */
-    0,                                                  /* tp_members */
-    pyepoll_getsetlist,                                 /* tp_getset */
-    0,                                                  /* tp_base */
-    0,                                                  /* tp_dict */
-    0,                                                  /* tp_descr_get */
-    0,                                                  /* tp_descr_set */
-    0,                                                  /* tp_dictoffset */
-    0,                                                  /* tp_init */
-    0,                                                  /* tp_alloc */
-    select_epoll,                                       /* tp_new */
-    0,                                                  /* tp_free */
+static PyType_Slot pyEpoll_Type_slots[] = {
+    {Py_tp_dealloc, pyepoll_dealloc},
+    {Py_tp_doc, (void*)pyepoll_doc},
+    {Py_tp_getattro, PyObject_GenericGetAttr},
+    {Py_tp_getset, pyepoll_getsetlist},
+    {Py_tp_methods, pyepoll_methods},
+    {Py_tp_new, select_epoll},
+    {0, 0},
+};
+
+static PyType_Spec pyEpoll_Type_spec = {
+    "select.epoll",
+    sizeof(pyEpoll_Object),
+    0,
+    Py_TPFLAGS_DEFAULT,
+    pyEpoll_Type_slots
 };
 
 #endif /* HAVE_EPOLL */
 
 #ifdef HAVE_KQUEUE
 
-static PyTypeObject kqueue_event_Type = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    "select.kevent",                                    /* tp_name */
-    sizeof(kqueue_event_Object),                        /* tp_basicsize */
-    0,                                                  /* tp_itemsize */
-    0,                                                  /* tp_dealloc */
-    0,                                                  /* tp_vectorcall_offset */
-    0,                                                  /* tp_getattr */
-    0,                                                  /* tp_setattr */
-    0,                                                  /* tp_as_async */
-    (reprfunc)kqueue_event_repr,                        /* tp_repr */
-    0,                                                  /* tp_as_number */
-    0,                                                  /* tp_as_sequence */
-    0,                                                  /* tp_as_mapping */
-    0,                                                  /* tp_hash */
-    0,                                                  /* tp_call */
-    0,                                                  /* tp_str */
-    0,                                                  /* tp_getattro */
-    0,                                                  /* tp_setattro */
-    0,                                                  /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,                                 /* tp_flags */
-    kqueue_event_doc,                                   /* tp_doc */
-    0,                                                  /* tp_traverse */
-    0,                                                  /* tp_clear */
-    (richcmpfunc)kqueue_event_richcompare,              /* tp_richcompare */
-    0,                                                  /* tp_weaklistoffset */
-    0,                                                  /* tp_iter */
-    0,                                                  /* tp_iternext */
-    0,                                                  /* tp_methods */
-    kqueue_event_members,                               /* tp_members */
-    0,                                                  /* tp_getset */
-    0,                                                  /* tp_base */
-    0,                                                  /* tp_dict */
-    0,                                                  /* tp_descr_get */
-    0,                                                  /* tp_descr_set */
-    0,                                                  /* tp_dictoffset */
-    (initproc)kqueue_event_init,                        /* tp_init */
-    0,                                                  /* tp_alloc */
-    0,                                                  /* tp_new */
-    0,                                                  /* tp_free */
-};
-
 static PyMethodDef kqueue_queue_methods[] = {
     SELECT_KQUEUE_FROMFD_METHODDEF
     SELECT_KQUEUE_CLOSE_METHODDEF
@@ -2407,46 +2360,21 @@ static PyMethodDef kqueue_queue_methods[] = {
     {NULL,      NULL},
 };
 
-static PyTypeObject kqueue_queue_Type = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    "select.kqueue",                                    /* tp_name */
-    sizeof(kqueue_queue_Object),                        /* tp_basicsize */
-    0,                                                  /* tp_itemsize */
-    (destructor)kqueue_queue_dealloc,                   /* tp_dealloc */
-    0,                                                  /* tp_vectorcall_offset */
-    0,                                                  /* tp_getattr */
-    0,                                                  /* tp_setattr */
-    0,                                                  /* tp_as_async */
-    0,                                                  /* tp_repr */
-    0,                                                  /* tp_as_number */
-    0,                                                  /* tp_as_sequence */
-    0,                                                  /* tp_as_mapping */
-    0,                                                  /* tp_hash */
-    0,                                                  /* tp_call */
-    0,                                                  /* tp_str */
-    0,                                                  /* tp_getattro */
-    0,                                                  /* tp_setattro */
-    0,                                                  /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,                                 /* tp_flags */
-    select_kqueue__doc__,                               /* tp_doc */
-    0,                                                  /* tp_traverse */
-    0,                                                  /* tp_clear */
-    0,                                                  /* tp_richcompare */
-    0,                                                  /* tp_weaklistoffset */
-    0,                                                  /* tp_iter */
-    0,                                                  /* tp_iternext */
-    kqueue_queue_methods,                               /* tp_methods */
-    0,                                                  /* tp_members */
-    kqueue_queue_getsetlist,                            /* tp_getset */
-    0,                                                  /* tp_base */
-    0,                                                  /* tp_dict */
-    0,                                                  /* tp_descr_get */
-    0,                                                  /* tp_descr_set */
-    0,                                                  /* tp_dictoffset */
-    0,                                                  /* tp_init */
-    0,                                                  /* tp_alloc */
-    select_kqueue,                                      /* tp_new */
-    0,                                                  /* tp_free */
+static PyType_Slot kqueue_queue_Type_slots[] = {
+    {Py_tp_dealloc, kqueue_queue_dealloc},
+    {Py_tp_doc, select_kqueue__doc__},
+    {Py_tp_getset, kqueue_queue_getsetlist},
+    {Py_tp_methods, kqueue_queue_methods},
+    {Py_tp_new, select_kqueue},
+    {0, 0},
+};
+
+static PyType_Spec kqueue_queue_Type_spec = {
+    "select.kqueue",
+    sizeof(kqueue_queue_Object),
+    0,
+    Py_TPFLAGS_DEFAULT,
+    kqueue_queue_Type_slots
 };
 
 #endif /* HAVE_KQUEUE */
@@ -2472,21 +2400,49 @@ PyDoc_STRVAR(module_doc,
 On Windows, only sockets are supported; on Unix, all file descriptors.");
 
 
+
+static int
+_select_traverse(PyObject *module, visitproc visit, void *arg)
+{
+    Py_VISIT(_selectstate(module)->close);
+    Py_VISIT(_selectstate(module)->poll_Type);
+    Py_VISIT(_selectstate(module)->devpoll_Type);
+    Py_VISIT(_selectstate(module)->pyEpoll_Type);
+    Py_VISIT(_selectstate(module)->kqueue_event_Type);
+    Py_VISIT(_selectstate(module)->kqueue_queue_Type);
+    return 0;
+}
+
+static int
+_select_clear(PyObject *module)
+{
+    Py_CLEAR(_selectstate(module)->close);
+    Py_CLEAR(_selectstate(module)->poll_Type);
+    Py_CLEAR(_selectstate(module)->devpoll_Type);
+    Py_CLEAR(_selectstate(module)->pyEpoll_Type);
+    Py_CLEAR(_selectstate(module)->kqueue_event_Type);
+    Py_CLEAR(_selectstate(module)->kqueue_queue_Type);
+    return 0;
+}
+
+static void
+_select_free(void *module)
+{
+    _select_clear((PyObject *)module);
+}
+
 static struct PyModuleDef selectmodule = {
     PyModuleDef_HEAD_INIT,
     "select",
     module_doc,
-    -1,
+    sizeof(_selectstate),
     select_methods,
     NULL,
-    NULL,
-    NULL,
-    NULL
+    _select_traverse,
+    _select_clear,
+    _select_free,
 };
 
-
-
-
 PyMODINIT_FUNC
 PyInit_select(void)
 {
@@ -2495,6 +2451,8 @@ PyInit_select(void)
     if (m == NULL)
         return NULL;
 
+    _selectstate(m)->close = PyUnicode_InternFromString("close");
+
     Py_INCREF(PyExc_OSError);
     PyModule_AddObject(m, "error", PyExc_OSError);
 
@@ -2516,8 +2474,12 @@ PyInit_select(void)
 #else
     {
 #endif
-        if (PyType_Ready(&poll_Type) < 0)
+        PyObject *poll_Type = PyType_FromSpec(&poll_Type_spec);
+        if (poll_Type == NULL)
             return NULL;
+        _selectstate(m)->poll_Type = (PyTypeObject *)poll_Type;
+        Py_INCREF(poll_Type);
+
         PyModule_AddIntMacro(m, POLLIN);
         PyModule_AddIntMacro(m, POLLPRI);
         PyModule_AddIntMacro(m, POLLOUT);
@@ -2548,17 +2510,20 @@ PyInit_select(void)
 #endif /* HAVE_POLL */
 
 #ifdef HAVE_SYS_DEVPOLL_H
-    if (PyType_Ready(&devpoll_Type) < 0)
+    PyObject *devpoll_Type = PyType_FromSpec(&devpoll_Type_spec);
+    if (devpoll_Type == NULL)
         return NULL;
+    _selectstate(m)->devpoll_Type = (PyTypeObject *)devpoll_Type;
+    Py_INCREF(devpoll_Type);
 #endif
 
 #ifdef HAVE_EPOLL
-    Py_TYPE(&pyEpoll_Type) = &PyType_Type;
-    if (PyType_Ready(&pyEpoll_Type) < 0)
+    PyObject *pyEpoll_Type = PyType_FromSpec(&pyEpoll_Type_spec);
+    if (pyEpoll_Type == NULL)
         return NULL;
-
-    Py_INCREF(&pyEpoll_Type);
-    PyModule_AddObject(m, "epoll", (PyObject *) &pyEpoll_Type);
+    _selectstate(m)->pyEpoll_Type = (PyTypeObject *)pyEpoll_Type;
+    Py_INCREF(pyEpoll_Type);
+    PyModule_AddObject(m, "epoll", (PyObject *)_selectstate(m)->pyEpoll_Type);
 
     PyModule_AddIntMacro(m, EPOLLIN);
     PyModule_AddIntMacro(m, EPOLLOUT);
@@ -2600,19 +2565,19 @@ PyInit_select(void)
 #endif /* HAVE_EPOLL */
 
 #ifdef HAVE_KQUEUE
-    kqueue_event_Type.tp_new = PyType_GenericNew;
-    Py_TYPE(&kqueue_event_Type) = &PyType_Type;
-    if(PyType_Ready(&kqueue_event_Type) < 0)
+    PyObject *kqueue_event_Type = PyType_FromSpec(&kqueue_event_Type_spec);
+    if (kqueue_event_Type == NULL)
         return NULL;
+    _selectstate(m)->kqueue_event_Type = (PyTypeObject *)kqueue_event_Type;
+    Py_INCREF(_selectstate(m)->kqueue_event_Type);
+    PyModule_AddObject(m, "kevent", kqueue_event_Type);
 
-    Py_INCREF(&kqueue_event_Type);
-    PyModule_AddObject(m, "kevent", (PyObject *)&kqueue_event_Type);
-
-    Py_TYPE(&kqueue_queue_Type) = &PyType_Type;
-    if(PyType_Ready(&kqueue_queue_Type) < 0)
+    PyObject *kqueue_queue_Type = PyType_FromSpec(&kqueue_queue_Type_spec);
+    if (kqueue_queue_Type == NULL)
         return NULL;
-    Py_INCREF(&kqueue_queue_Type);
-    PyModule_AddObject(m, "kqueue", (PyObject *)&kqueue_queue_Type);
+    _selectstate(m)->kqueue_queue_Type = (PyTypeObject *)kqueue_queue_Type;
+    Py_INCREF(_selectstate(m)->kqueue_queue_Type);
+    PyModule_AddObject(m, "kqueue", kqueue_queue_Type);
 
     /* event filters */
     PyModule_AddIntConstant(m, "KQ_FILTER_READ", EVFILT_READ);


From webhook-mailer at python.org  Sat Sep 14 11:29:59 2019
From: webhook-mailer at python.org (Christian Heimes)
Date: Sat, 14 Sep 2019 15:29:59 -0000
Subject: [Python-checkins] bpo-38153: detect shake independently from sha3
 (GH-16143)
Message-ID: <mailman.748.1568475000.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/eb2b0c694aef6122fdf95015abb24e0d095b6401
commit: eb2b0c694aef6122fdf95015abb24e0d095b6401
branch: master
author: Christian Heimes <christian at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-14T17:29:54+02:00
summary:

bpo-38153: detect shake independently from sha3 (GH-16143)

XOF digests (SHAKE) are not available in OpenSSL 1.1.0 but SHA3 fixed-length digests are.

Signed-off-by: Christian Heimes <christian at python.org>

files:
M Modules/_hashopenssl.c

diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index aa91be2f2e3c..b86cbd09743e 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -34,10 +34,14 @@
 
 #define MUNCH_SIZE INT_MAX
 
-#if defined(NID_sha3_224) && defined(EVP_MD_FLAG_XOF)
+#ifdef NID_sha3_224
 #define PY_OPENSSL_HAS_SHA3 1
 #endif
 
+#if defined(EVP_MD_FLAG_XOF) && defined(NID_shake128)
+#define PY_OPENSSL_HAS_SHAKE 1
+#endif
+
 #ifdef NID_blake2b512
 #define PY_OPENSSL_HAS_BLAKE2 1
 #endif
@@ -139,6 +143,8 @@ py_digest_name(const EVP_MD *md)
     case NID_sha3_512:
         name ="sha3_512";
         break;
+#endif
+#ifdef PY_OPENSSL_HAS_SHAKE
     case NID_shake128:
         name ="shake_128";
         break;
@@ -177,8 +183,9 @@ py_digest_by_name(const char *name)
     /* OpenSSL uses dash instead of underscore in names of some algorithms
      * like SHA3 and SHAKE. Detect different spellings. */
     if (digest == NULL) {
+        if (0) {}
 #ifdef NID_sha512_224
-        if (!strcmp(name, "sha512_224") || !strcmp(name, "SHA512_224")) {
+        else if (!strcmp(name, "sha512_224") || !strcmp(name, "SHA512_224")) {
             digest = EVP_sha512_224();
         }
         else if (!strcmp(name, "sha512_256") || !strcmp(name, "SHA512_256")) {
@@ -199,6 +206,8 @@ py_digest_by_name(const char *name)
         else if (!strcmp(name, "sha3_512")) {
             digest = EVP_sha3_512();
         }
+#endif
+#ifdef PY_OPENSSL_HAS_SHAKE
         else if (!strcmp(name, "shake_128")) {
             digest = EVP_shake128();
         }


From webhook-mailer at python.org  Sat Sep 14 12:36:23 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sat, 14 Sep 2019 16:36:23 -0000
Subject: [Python-checkins] [3.8] bpo-38005: Fixed comparing and creating of
 InterpreterID and ChannelID. (GH-15652) (GH-16145)
Message-ID: <mailman.749.1568478985.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f37a9831027ecfe948697cdb5e35b417805d94e5
commit: f37a9831027ecfe948697cdb5e35b417805d94e5
branch: 3.8
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-14T19:36:19+03:00
summary:

[3.8] bpo-38005: Fixed comparing and creating of InterpreterID and ChannelID. (GH-15652) (GH-16145)

* Fix a crash in comparing with float (and maybe other crashes).
* They are now never equal to strings and non-integer numbers.
* Comparison with a large number no longer raises OverflowError.
* Arbitrary exceptions no longer silenced in constructors and comparisons.
* TypeError raised in the constructor contains now the name of the type.
* Accept only ChannelID and int-like objects in channel functions.
* Accept only InterpreterId, int-like objects and str in the InterpreterId constructor.
* Accept int-like objects, not just int in interpreter related functions.
(cherry picked from commit bf169915ecdd42329726104278eb723a7dda2736)

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-02-20-00-31.bpo-38005.e7VsTA.rst
M Include/cpython/interpreteridobject.h
M Lib/test/test__xxsubinterpreters.py
M Modules/_xxsubinterpretersmodule.c
M Objects/interpreteridobject.c

diff --git a/Include/cpython/interpreteridobject.h b/Include/cpython/interpreteridobject.h
index cb72c2b0895a..67ec5873542d 100644
--- a/Include/cpython/interpreteridobject.h
+++ b/Include/cpython/interpreteridobject.h
@@ -14,8 +14,6 @@ PyAPI_FUNC(PyObject *) _PyInterpreterID_New(int64_t);
 PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *);
 PyAPI_FUNC(PyInterpreterState *) _PyInterpreterID_LookUp(PyObject *);
 
-PyAPI_FUNC(int64_t) _Py_CoerceID(PyObject *);
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py
index 78b2030a1f6d..f5474f4101d0 100644
--- a/Lib/test/test__xxsubinterpreters.py
+++ b/Lib/test/test__xxsubinterpreters.py
@@ -526,30 +526,23 @@ def test_with_int(self):
         self.assertEqual(int(id), 10)
 
     def test_coerce_id(self):
-        id = interpreters.InterpreterID('10', force=True)
-        self.assertEqual(int(id), 10)
-
-        id = interpreters.InterpreterID(10.0, force=True)
-        self.assertEqual(int(id), 10)
-
         class Int(str):
-            def __init__(self, value):
-                self._value = value
-            def __int__(self):
-                return self._value
+            def __index__(self):
+                return 10
 
-        id = interpreters.InterpreterID(Int(10), force=True)
-        self.assertEqual(int(id), 10)
+        for id in ('10', '1_0', Int()):
+            with self.subTest(id=id):
+                id = interpreters.InterpreterID(id, force=True)
+                self.assertEqual(int(id), 10)
 
     def test_bad_id(self):
-        for id in [-1, 'spam']:
-            with self.subTest(id):
-                with self.assertRaises(ValueError):
-                    interpreters.InterpreterID(id)
-        with self.assertRaises(OverflowError):
-            interpreters.InterpreterID(2**64)
-        with self.assertRaises(TypeError):
-            interpreters.InterpreterID(object())
+        self.assertRaises(TypeError, interpreters.InterpreterID, object())
+        self.assertRaises(TypeError, interpreters.InterpreterID, 10.0)
+        self.assertRaises(TypeError, interpreters.InterpreterID, b'10')
+        self.assertRaises(ValueError, interpreters.InterpreterID, -1)
+        self.assertRaises(ValueError, interpreters.InterpreterID, '-1')
+        self.assertRaises(ValueError, interpreters.InterpreterID, 'spam')
+        self.assertRaises(OverflowError, interpreters.InterpreterID, 2**64)
 
     def test_does_not_exist(self):
         id = interpreters.channel_create()
@@ -572,6 +565,14 @@ def test_equality(self):
         self.assertTrue(id1 == id1)
         self.assertTrue(id1 == id2)
         self.assertTrue(id1 == int(id1))
+        self.assertTrue(int(id1) == id1)
+        self.assertTrue(id1 == float(int(id1)))
+        self.assertTrue(float(int(id1)) == id1)
+        self.assertFalse(id1 == float(int(id1)) + 0.1)
+        self.assertFalse(id1 == str(int(id1)))
+        self.assertFalse(id1 == 2**1000)
+        self.assertFalse(id1 == float('inf'))
+        self.assertFalse(id1 == 'spam')
         self.assertFalse(id1 == id3)
 
         self.assertFalse(id1 != id1)
@@ -1105,30 +1106,20 @@ def test_with_kwargs(self):
         self.assertEqual(cid.end, 'both')
 
     def test_coerce_id(self):
-        cid = interpreters._channel_id('10', force=True)
-        self.assertEqual(int(cid), 10)
-
-        cid = interpreters._channel_id(10.0, force=True)
-        self.assertEqual(int(cid), 10)
-
         class Int(str):
-            def __init__(self, value):
-                self._value = value
-            def __int__(self):
-                return self._value
+            def __index__(self):
+                return 10
 
-        cid = interpreters._channel_id(Int(10), force=True)
+        cid = interpreters._channel_id(Int(), force=True)
         self.assertEqual(int(cid), 10)
 
     def test_bad_id(self):
-        for cid in [-1, 'spam']:
-            with self.subTest(cid):
-                with self.assertRaises(ValueError):
-                    interpreters._channel_id(cid)
-        with self.assertRaises(OverflowError):
-            interpreters._channel_id(2**64)
-        with self.assertRaises(TypeError):
-            interpreters._channel_id(object())
+        self.assertRaises(TypeError, interpreters._channel_id, object())
+        self.assertRaises(TypeError, interpreters._channel_id, 10.0)
+        self.assertRaises(TypeError, interpreters._channel_id, '10')
+        self.assertRaises(TypeError, interpreters._channel_id, b'10')
+        self.assertRaises(ValueError, interpreters._channel_id, -1)
+        self.assertRaises(OverflowError, interpreters._channel_id, 2**64)
 
     def test_bad_kwargs(self):
         with self.assertRaises(ValueError):
@@ -1164,6 +1155,14 @@ def test_equality(self):
         self.assertTrue(cid1 == cid1)
         self.assertTrue(cid1 == cid2)
         self.assertTrue(cid1 == int(cid1))
+        self.assertTrue(int(cid1) == cid1)
+        self.assertTrue(cid1 == float(int(cid1)))
+        self.assertTrue(float(int(cid1)) == cid1)
+        self.assertFalse(cid1 == float(int(cid1)) + 0.1)
+        self.assertFalse(cid1 == str(int(cid1)))
+        self.assertFalse(cid1 == 2**1000)
+        self.assertFalse(cid1 == float('inf'))
+        self.assertFalse(cid1 == 'spam')
         self.assertFalse(cid1 == cid3)
 
         self.assertFalse(cid1 != cid1)
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-02-20-00-31.bpo-38005.e7VsTA.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-02-20-00-31.bpo-38005.e7VsTA.rst
new file mode 100644
index 000000000000..706abf587b90
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-02-20-00-31.bpo-38005.e7VsTA.rst	
@@ -0,0 +1 @@
+Fixed comparing and creating of InterpreterID and ChannelID.
diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c
index 19d98fd96934..7842947e54ac 100644
--- a/Modules/_xxsubinterpretersmodule.c
+++ b/Modules/_xxsubinterpretersmodule.c
@@ -1405,6 +1405,34 @@ typedef struct channelid {
     _channels *channels;
 } channelid;
 
+static int
+channel_id_converter(PyObject *arg, void *ptr)
+{
+    int64_t cid;
+    if (PyObject_TypeCheck(arg, &ChannelIDtype)) {
+        cid = ((channelid *)arg)->id;
+    }
+    else if (PyIndex_Check(arg)) {
+        cid = PyLong_AsLongLong(arg);
+        if (cid == -1 && PyErr_Occurred()) {
+            return 0;
+        }
+        if (cid < 0) {
+            PyErr_Format(PyExc_ValueError,
+                        "channel ID must be a non-negative int, got %R", arg);
+            return 0;
+        }
+    }
+    else {
+        PyErr_Format(PyExc_TypeError,
+                     "channel ID must be an int, got %.100s",
+                     arg->ob_type->tp_name);
+        return 0;
+    }
+    *(int64_t *)ptr = cid;
+    return 1;
+}
+
 static channelid *
 newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels,
              int force, int resolve)
@@ -1437,28 +1465,16 @@ static PyObject *
 channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
 {
     static char *kwlist[] = {"id", "send", "recv", "force", "_resolve", NULL};
-    PyObject *id;
+    int64_t cid;
     int send = -1;
     int recv = -1;
     int force = 0;
     int resolve = 0;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O|$pppp:ChannelID.__new__", kwlist,
-                                     &id, &send, &recv, &force, &resolve))
+                                     "O&|$pppp:ChannelID.__new__", kwlist,
+                                     channel_id_converter, &cid, &send, &recv, &force, &resolve))
         return NULL;
 
-    // Coerce and check the ID.
-    int64_t cid;
-    if (PyObject_TypeCheck(id, &ChannelIDtype)) {
-        cid = ((channelid *)id)->id;
-    }
-    else {
-        cid = _Py_CoerceID(id);
-        if (cid < 0) {
-            return NULL;
-        }
-    }
-
     // Handle "send" and "recv".
     if (send == 0 && recv == 0) {
         PyErr_SetString(PyExc_ValueError,
@@ -1592,30 +1608,28 @@ channelid_richcompare(PyObject *self, PyObject *other, int op)
     int equal;
     if (PyObject_TypeCheck(other, &ChannelIDtype)) {
         channelid *othercid = (channelid *)other;
-        if (cid->end != othercid->end) {
-            equal = 0;
-        }
-        else {
-            equal = (cid->id == othercid->id);
-        }
+        equal = (cid->end == othercid->end) && (cid->id == othercid->id);
     }
-    else {
-        other = PyNumber_Long(other);
-        if (other == NULL) {
-            PyErr_Clear();
-            Py_RETURN_NOTIMPLEMENTED;
-        }
-        int64_t othercid = PyLong_AsLongLong(other);
-        Py_DECREF(other);
-        if (othercid == -1 && PyErr_Occurred() != NULL) {
+    else if (PyLong_Check(other)) {
+        /* Fast path */
+        int overflow;
+        long long othercid = PyLong_AsLongLongAndOverflow(other, &overflow);
+        if (othercid == -1 && PyErr_Occurred()) {
             return NULL;
         }
-        if (othercid < 0) {
-            equal = 0;
-        }
-        else {
-            equal = (cid->id == othercid);
+        equal = !overflow && (othercid >= 0) && (cid->id == othercid);
+    }
+    else if (PyNumber_Check(other)) {
+        PyObject *pyid = PyLong_FromLongLong(cid->id);
+        if (pyid == NULL) {
+            return NULL;
         }
+        PyObject *res = PyObject_RichCompare(pyid, other, op);
+        Py_DECREF(pyid);
+        return res;
+    }
+    else {
+        Py_RETURN_NOTIMPLEMENTED;
     }
 
     if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) {
@@ -1754,8 +1768,7 @@ static PyTypeObject ChannelIDtype = {
     0,                              /* tp_getattro */
     0,                              /* tp_setattro */
     0,                              /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
-        Py_TPFLAGS_LONG_SUBCLASS,   /* tp_flags */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
     channelid_doc,                  /* tp_doc */
     0,                              /* tp_traverse */
     0,                              /* tp_clear */
@@ -2017,10 +2030,6 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
                                      "O:destroy", kwlist, &id)) {
         return NULL;
     }
-    if (!PyLong_Check(id)) {
-        PyErr_SetString(PyExc_TypeError, "ID must be an int");
-        return NULL;
-    }
 
     // Look up the interpreter.
     PyInterpreterState *interp = _PyInterpreterID_LookUp(id);
@@ -2145,10 +2154,6 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
                                      &id, &code, &shared)) {
         return NULL;
     }
-    if (!PyLong_Check(id)) {
-        PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int");
-        return NULL;
-    }
 
     // Look up the interpreter.
     PyInterpreterState *interp = _PyInterpreterID_LookUp(id);
@@ -2216,10 +2221,6 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds)
                                      "O:is_running", kwlist, &id)) {
         return NULL;
     }
-    if (!PyLong_Check(id)) {
-        PyErr_SetString(PyExc_TypeError, "ID must be an int");
-        return NULL;
-    }
 
     PyInterpreterState *interp = _PyInterpreterID_LookUp(id);
     if (interp == NULL) {
@@ -2268,13 +2269,9 @@ static PyObject *
 channel_destroy(PyObject *self, PyObject *args, PyObject *kwds)
 {
     static char *kwlist[] = {"cid", NULL};
-    PyObject *id;
-    if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O:channel_destroy", kwlist, &id)) {
-        return NULL;
-    }
-    int64_t cid = _Py_CoerceID(id);
-    if (cid < 0) {
+    int64_t cid;
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:channel_destroy", kwlist,
+                                     channel_id_converter, &cid)) {
         return NULL;
     }
 
@@ -2331,14 +2328,10 @@ static PyObject *
 channel_send(PyObject *self, PyObject *args, PyObject *kwds)
 {
     static char *kwlist[] = {"cid", "obj", NULL};
-    PyObject *id;
+    int64_t cid;
     PyObject *obj;
-    if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "OO:channel_send", kwlist, &id, &obj)) {
-        return NULL;
-    }
-    int64_t cid = _Py_CoerceID(id);
-    if (cid < 0) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O:channel_send", kwlist,
+                                     channel_id_converter, &cid, &obj)) {
         return NULL;
     }
 
@@ -2357,13 +2350,9 @@ static PyObject *
 channel_recv(PyObject *self, PyObject *args, PyObject *kwds)
 {
     static char *kwlist[] = {"cid", NULL};
-    PyObject *id;
-    if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O:channel_recv", kwlist, &id)) {
-        return NULL;
-    }
-    int64_t cid = _Py_CoerceID(id);
-    if (cid < 0) {
+    int64_t cid;
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:channel_recv", kwlist,
+                                     channel_id_converter, &cid)) {
         return NULL;
     }
 
@@ -2379,17 +2368,13 @@ static PyObject *
 channel_close(PyObject *self, PyObject *args, PyObject *kwds)
 {
     static char *kwlist[] = {"cid", "send", "recv", "force", NULL};
-    PyObject *id;
+    int64_t cid;
     int send = 0;
     int recv = 0;
     int force = 0;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O|$ppp:channel_close", kwlist,
-                                     &id, &send, &recv, &force)) {
-        return NULL;
-    }
-    int64_t cid = _Py_CoerceID(id);
-    if (cid < 0) {
+                                     "O&|$ppp:channel_close", kwlist,
+                                     channel_id_converter, &cid, &send, &recv, &force)) {
         return NULL;
     }
 
@@ -2431,17 +2416,13 @@ channel_release(PyObject *self, PyObject *args, PyObject *kwds)
 {
     // Note that only the current interpreter is affected.
     static char *kwlist[] = {"cid", "send", "recv", "force", NULL};
-    PyObject *id;
+    int64_t cid;
     int send = 0;
     int recv = 0;
     int force = 0;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O|$ppp:channel_release", kwlist,
-                                     &id, &send, &recv, &force)) {
-        return NULL;
-    }
-    int64_t cid = _Py_CoerceID(id);
-    if (cid < 0) {
+                                     "O&|$ppp:channel_release", kwlist,
+                                     channel_id_converter, &cid, &send, &recv, &force)) {
         return NULL;
     }
     if (send == 0 && recv == 0) {
@@ -2538,7 +2519,6 @@ PyInit__xxsubinterpreters(void)
     }
 
     /* Initialize types */
-    ChannelIDtype.tp_base = &PyLong_Type;
     if (PyType_Ready(&ChannelIDtype) != 0) {
         return NULL;
     }
diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c
index 0a1dfa25795f..3edbb85e6ac3 100644
--- a/Objects/interpreteridobject.c
+++ b/Objects/interpreteridobject.c
@@ -5,38 +5,6 @@
 #include "interpreteridobject.h"
 
 
-int64_t
-_Py_CoerceID(PyObject *orig)
-{
-    PyObject *pyid = PyNumber_Long(orig);
-    if (pyid == NULL) {
-        if (PyErr_ExceptionMatches(PyExc_TypeError)) {
-            PyErr_Format(PyExc_TypeError,
-                         "'id' must be a non-negative int, got %R", orig);
-        }
-        else {
-            PyErr_Format(PyExc_ValueError,
-                         "'id' must be a non-negative int, got %R", orig);
-        }
-        return -1;
-    }
-    int64_t id = PyLong_AsLongLong(pyid);
-    Py_DECREF(pyid);
-    if (id == -1 && PyErr_Occurred() != NULL) {
-        if (!PyErr_ExceptionMatches(PyExc_OverflowError)) {
-            PyErr_Format(PyExc_ValueError,
-                         "'id' must be a non-negative int, got %R", orig);
-        }
-        return -1;
-    }
-    if (id < 0) {
-        PyErr_Format(PyExc_ValueError,
-                     "'id' must be a non-negative int, got %R", orig);
-        return -1;
-    }
-    return id;
-}
-
 typedef struct interpid {
     PyObject_HEAD
     int64_t id;
@@ -85,8 +53,31 @@ interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
         id = ((interpid *)idobj)->id;
     }
     else {
-        id = _Py_CoerceID(idobj);
+        PyObject *pyid;
+        if (PyIndex_Check(idobj)) {
+            pyid = idobj;
+            Py_INCREF(pyid);
+        }
+        else if (PyUnicode_Check(idobj)) {
+            pyid = PyNumber_Long(idobj);
+            if (pyid == NULL) {
+                return NULL;
+            }
+        }
+        else {
+            PyErr_Format(PyExc_TypeError,
+                         "interpreter ID must be an int, got %.100s",
+                         idobj->ob_type->tp_name);
+            return NULL;
+        }
+        id = PyLong_AsLongLong(pyid);
+        Py_DECREF(pyid);
+        if (id == -1 && PyErr_Occurred()) {
+            return NULL;
+        }
         if (id < 0) {
+            PyErr_Format(PyExc_ValueError,
+                         "interpreter ID must be a non-negative int, got %R", idobj);
             return NULL;
         }
     }
@@ -202,23 +193,26 @@ interpid_richcompare(PyObject *self, PyObject *other, int op)
         interpid *otherid = (interpid *)other;
         equal = (id->id == otherid->id);
     }
-    else {
-        other = PyNumber_Long(other);
-        if (other == NULL) {
-            PyErr_Clear();
-            Py_RETURN_NOTIMPLEMENTED;
-        }
-        int64_t otherid = PyLong_AsLongLong(other);
-        Py_DECREF(other);
-        if (otherid == -1 && PyErr_Occurred() != NULL) {
+    else if (PyLong_CheckExact(other)) {
+        /* Fast path */
+        int overflow;
+        long long otherid = PyLong_AsLongLongAndOverflow(other, &overflow);
+        if (otherid == -1 && PyErr_Occurred()) {
             return NULL;
         }
-        if (otherid < 0) {
-            equal = 0;
-        }
-        else {
-            equal = (id->id == otherid);
+        equal = !overflow && (otherid >= 0) && (id->id == otherid);
+    }
+    else if (PyNumber_Check(other)) {
+        PyObject *pyid = PyLong_FromLongLong(id->id);
+        if (pyid == NULL) {
+            return NULL;
         }
+        PyObject *res = PyObject_RichCompare(pyid, other, op);
+        Py_DECREF(pyid);
+        return res;
+    }
+    else {
+        Py_RETURN_NOTIMPLEMENTED;
     }
 
     if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) {
@@ -250,8 +244,7 @@ PyTypeObject _PyInterpreterID_Type = {
     0,                              /* tp_getattro */
     0,                              /* tp_setattro */
     0,                              /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
-        Py_TPFLAGS_LONG_SUBCLASS,   /* tp_flags */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
     interpid_doc,                   /* tp_doc */
     0,                              /* tp_traverse */
     0,                              /* tp_clear */
@@ -262,7 +255,7 @@ PyTypeObject _PyInterpreterID_Type = {
     0,                              /* tp_methods */
     0,                              /* tp_members */
     0,                              /* tp_getset */
-    &PyLong_Type,                   /* tp_base */
+    0,                              /* tp_base */
     0,                              /* tp_dict */
     0,                              /* tp_descr_get */
     0,                              /* tp_descr_set */
@@ -297,12 +290,17 @@ _PyInterpreterID_LookUp(PyObject *requested_id)
     if (PyObject_TypeCheck(requested_id, &_PyInterpreterID_Type)) {
         id = ((interpid *)requested_id)->id;
     }
-    else {
+    else if (PyIndex_Check(requested_id)) {
         id = PyLong_AsLongLong(requested_id);
         if (id == -1 && PyErr_Occurred() != NULL) {
             return NULL;
         }
         assert(id <= INT64_MAX);
     }
+    else {
+        PyErr_Format(PyExc_TypeError, "interpreter ID must be an int, got %.100s",
+                     requested_id->ob_type->tp_name);
+        return NULL;
+    }
     return _PyInterpreterState_LookUpID(id);
 }


From webhook-mailer at python.org  Sat Sep 14 13:26:41 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 14 Sep 2019 17:26:41 -0000
Subject: [Python-checkins] [2.7] bpo-34155: Dont parse domains containing @
 (GH-13079) (GH-16006)
Message-ID: <mailman.750.1568482002.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4cbcd2f8c4e12b912e4d21fd892eedf7a3813d8e
commit: 4cbcd2f8c4e12b912e4d21fd892eedf7a3813d8e
branch: 2.7
author: Roberto C. S?nchez <roberto at connexer.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-14T10:26:38-07:00
summary:

[2.7] bpo-34155: Dont parse domains containing @ (GH-13079) (GH-16006)



This change skips parsing of email addresses where domains include a "@" character, which can be maliciously used since the local part is returned as a complete address. 

(cherry picked from commit 8cb65d1381b027f0b09ee36bfed7f35bb4dec9a9)

Excludes changes to Lib/email/_header_value_parser.py, which did not
exist in 2.7.

Co-authored-by: jpic <jpic at users.noreply.github.com>


https://bugs.python.org/issue34155

files:
A Misc/NEWS.d/next/Security/2019-05-04-13-33-37.bpo-34155.MJll68.rst
M Lib/email/_parseaddr.py
M Lib/email/test/test_email.py

diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py
index 690db2c22d34..dc49d2e45a5e 100644
--- a/Lib/email/_parseaddr.py
+++ b/Lib/email/_parseaddr.py
@@ -336,7 +336,12 @@ def getaddrspec(self):
         aslist.append('@')
         self.pos += 1
         self.gotonext()
-        return EMPTYSTRING.join(aslist) + self.getdomain()
+        domain = self.getdomain()
+        if not domain:
+            # Invalid domain, return an empty address instead of returning a
+            # local part to denote failed parsing.
+            return EMPTYSTRING
+        return EMPTYSTRING.join(aslist) + domain
 
     def getdomain(self):
         """Get the complete domain name from an address."""
@@ -351,6 +356,10 @@ def getdomain(self):
             elif self.field[self.pos] == '.':
                 self.pos += 1
                 sdlist.append('.')
+            elif self.field[self.pos] == '@':
+                # bpo-34155: Don't parse domains with two `@` like
+                # `a at malicious.org@important.com`.
+                return EMPTYSTRING
             elif self.field[self.pos] in self.atomends:
                 break
             else:
diff --git a/Lib/email/test/test_email.py b/Lib/email/test/test_email.py
index 4b4dee3d3464..2efe44ac5a73 100644
--- a/Lib/email/test/test_email.py
+++ b/Lib/email/test/test_email.py
@@ -2306,6 +2306,20 @@ def test_parseaddr_empty(self):
         self.assertEqual(Utils.parseaddr('<>'), ('', ''))
         self.assertEqual(Utils.formataddr(Utils.parseaddr('<>')), '')
 
+    def test_parseaddr_multiple_domains(self):
+        self.assertEqual(
+            Utils.parseaddr('a at b@c'),
+            ('', '')
+        )
+        self.assertEqual(
+            Utils.parseaddr('a at b.c@c'),
+            ('', '')
+        )
+        self.assertEqual(
+            Utils.parseaddr('a at 172.17.0.1@c'),
+            ('', '')
+        )
+
     def test_noquote_dump(self):
         self.assertEqual(
             Utils.formataddr(('A Silly Person', 'person at dom.ain')),
diff --git a/Misc/NEWS.d/next/Security/2019-05-04-13-33-37.bpo-34155.MJll68.rst b/Misc/NEWS.d/next/Security/2019-05-04-13-33-37.bpo-34155.MJll68.rst
new file mode 100644
index 000000000000..50292e29ed1d
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-05-04-13-33-37.bpo-34155.MJll68.rst
@@ -0,0 +1 @@
+Fix parsing of invalid email addresses with more than one ``@`` (e.g. a at b@c.com.) to not return the part before 2nd ``@`` as valid email address. Patch by maxking & jpic.


From webhook-mailer at python.org  Sat Sep 14 16:29:39 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 14 Sep 2019 20:29:39 -0000
Subject: [Python-checkins] bpo-37635: Update arg name for seek() in IO
 tutorial (GH-16147)
Message-ID: <mailman.751.1568492980.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ff603f6c3d3dc0e9ea8c1c51ce907c4821f42c54
commit: ff603f6c3d3dc0e9ea8c1c51ce907c4821f42c54
branch: master
author: Kyle Stanley <aeros167 at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-14T13:29:23-07:00
summary:

bpo-37635: Update arg name for seek() in IO tutorial (GH-16147)



Typically, the second positional argument for ``seek()`` is *whence*. That is the POSIX standard name (http://man7.org/linux/man-pages/man3/lseek.3p.html) and the name listed in the documentation for ``io`` module (https://docs.python.org/3/library/io.html#io.IOBase.seek). 

The tutorial for IO is the only location where the second positional argument for ``seek()`` is referred to as *from_what*. I suspect this was created at an early point in Python's history, and was never updated (as this section predates the GitHub repository):

```
$ git grep "from_what"
Doc/tutorial/inputoutput.rst:To change the file object's position, use ``f.seek(offset, from_what)``.  The position is computed
Doc/tutorial/inputoutput.rst:the *from_what* argument.  A *from_what* value of 0 measures from the beginning
Doc/tutorial/inputoutput.rst:the reference point.  *from_what* can be omitted and defaults to 0, using the
```

For consistency, I am suggesting that the tutorial be updated to use the same argument name as the IO documentation and POSIX standard for ``seek()``, particularly since this is the only location where *from_what* is being used.

Note: In the POSIX standard, *whence* is technically the third positional argument, but the first argument *fildes* (file descriptor) is implicit in Python.


https://bugs.python.org/issue37635

files:
M Doc/tutorial/inputoutput.rst

diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst
index 9fe1276ec9d1..a404f4be19f1 100644
--- a/Doc/tutorial/inputoutput.rst
+++ b/Doc/tutorial/inputoutput.rst
@@ -412,11 +412,11 @@ or a bytes object (in binary mode) -- before writing them::
 represented as number of bytes from the beginning of the file when in binary mode and
 an opaque number when in text mode.
 
-To change the file object's position, use ``f.seek(offset, from_what)``.  The position is computed
+To change the file object's position, use ``f.seek(offset, whence)``.  The position is computed
 from adding *offset* to a reference point; the reference point is selected by
-the *from_what* argument.  A *from_what* value of 0 measures from the beginning
+the *whence* argument.  A *whence* value of 0 measures from the beginning
 of the file, 1 uses the current file position, and 2 uses the end of the file as
-the reference point.  *from_what* can be omitted and defaults to 0, using the
+the reference point.  *whence* can be omitted and defaults to 0, using the
 beginning of the file as the reference point. ::
 
    >>> f = open('workfile', 'rb+')


From webhook-mailer at python.org  Sat Sep 14 16:47:35 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 14 Sep 2019 20:47:35 -0000
Subject: [Python-checkins] bpo-37635: Update arg name for seek() in IO
 tutorial (GH-16147)
Message-ID: <mailman.752.1568494057.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4a71df88cdba77c409ee70146dd6445b19267df4
commit: 4a71df88cdba77c409ee70146dd6445b19267df4
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-14T13:47:31-07:00
summary:

bpo-37635: Update arg name for seek() in IO tutorial (GH-16147)


Typically, the second positional argument for ``seek()`` is *whence*. That is the POSIX standard name (http://man7.org/linux/man-pages/man3/lseek.3p.html) and the name listed in the documentation for ``io`` module (https://docs.python.org/3/library/io.htmlGH-io.IOBase.seek).

The tutorial for IO is the only location where the second positional argument for ``seek()`` is referred to as *from_what*. I suspect this was created at an early point in Python's history, and was never updated (as this section predates the GitHub repository):

```
$ git grep "from_what"
Doc/tutorial/inputoutput.rst:To change the file object's position, use ``f.seek(offset, from_what)``.  The position is computed
Doc/tutorial/inputoutput.rst:the *from_what* argument.  A *from_what* value of 0 measures from the beginning
Doc/tutorial/inputoutput.rst:the reference point.  *from_what* can be omitted and defaults to 0, using the
```

For consistency, I am suggesting that the tutorial be updated to use the same argument name as the IO documentation and POSIX standard for ``seek()``, particularly since this is the only location where *from_what* is being used.

Note: In the POSIX standard, *whence* is technically the third positional argument, but the first argument *fildes* (file descriptor) is implicit in Python.

https://bugs.python.org/issue37635
(cherry picked from commit ff603f6c3d3dc0e9ea8c1c51ce907c4821f42c54)

Co-authored-by: Kyle Stanley <aeros167 at gmail.com>

files:
M Doc/tutorial/inputoutput.rst

diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst
index 9fe1276ec9d1..a404f4be19f1 100644
--- a/Doc/tutorial/inputoutput.rst
+++ b/Doc/tutorial/inputoutput.rst
@@ -412,11 +412,11 @@ or a bytes object (in binary mode) -- before writing them::
 represented as number of bytes from the beginning of the file when in binary mode and
 an opaque number when in text mode.
 
-To change the file object's position, use ``f.seek(offset, from_what)``.  The position is computed
+To change the file object's position, use ``f.seek(offset, whence)``.  The position is computed
 from adding *offset* to a reference point; the reference point is selected by
-the *from_what* argument.  A *from_what* value of 0 measures from the beginning
+the *whence* argument.  A *whence* value of 0 measures from the beginning
 of the file, 1 uses the current file position, and 2 uses the end of the file as
-the reference point.  *from_what* can be omitted and defaults to 0, using the
+the reference point.  *whence* can be omitted and defaults to 0, using the
 beginning of the file as the reference point. ::
 
    >>> f = open('workfile', 'rb+')


From webhook-mailer at python.org  Sat Sep 14 16:47:42 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 14 Sep 2019 20:47:42 -0000
Subject: [Python-checkins] bpo-37635: Update arg name for seek() in IO
 tutorial (GH-16147)
Message-ID: <mailman.753.1568494064.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b9f932f9e2a170a8d39b3c17f5fabb0967839d85
commit: b9f932f9e2a170a8d39b3c17f5fabb0967839d85
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-14T13:47:39-07:00
summary:

bpo-37635: Update arg name for seek() in IO tutorial (GH-16147)


Typically, the second positional argument for ``seek()`` is *whence*. That is the POSIX standard name (http://man7.org/linux/man-pages/man3/lseek.3p.html) and the name listed in the documentation for ``io`` module (https://docs.python.org/3/library/io.htmlGH-io.IOBase.seek).

The tutorial for IO is the only location where the second positional argument for ``seek()`` is referred to as *from_what*. I suspect this was created at an early point in Python's history, and was never updated (as this section predates the GitHub repository):

```
$ git grep "from_what"
Doc/tutorial/inputoutput.rst:To change the file object's position, use ``f.seek(offset, from_what)``.  The position is computed
Doc/tutorial/inputoutput.rst:the *from_what* argument.  A *from_what* value of 0 measures from the beginning
Doc/tutorial/inputoutput.rst:the reference point.  *from_what* can be omitted and defaults to 0, using the
```

For consistency, I am suggesting that the tutorial be updated to use the same argument name as the IO documentation and POSIX standard for ``seek()``, particularly since this is the only location where *from_what* is being used.

Note: In the POSIX standard, *whence* is technically the third positional argument, but the first argument *fildes* (file descriptor) is implicit in Python.

https://bugs.python.org/issue37635
(cherry picked from commit ff603f6c3d3dc0e9ea8c1c51ce907c4821f42c54)

Co-authored-by: Kyle Stanley <aeros167 at gmail.com>

files:
M Doc/tutorial/inputoutput.rst

diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst
index 1dc01e82102d..74f381d4a0da 100644
--- a/Doc/tutorial/inputoutput.rst
+++ b/Doc/tutorial/inputoutput.rst
@@ -410,11 +410,11 @@ or a bytes object (in binary mode) -- before writing them::
 represented as number of bytes from the beginning of the file when in binary mode and
 an opaque number when in text mode.
 
-To change the file object's position, use ``f.seek(offset, from_what)``.  The position is computed
+To change the file object's position, use ``f.seek(offset, whence)``.  The position is computed
 from adding *offset* to a reference point; the reference point is selected by
-the *from_what* argument.  A *from_what* value of 0 measures from the beginning
+the *whence* argument.  A *whence* value of 0 measures from the beginning
 of the file, 1 uses the current file position, and 2 uses the end of the file as
-the reference point.  *from_what* can be omitted and defaults to 0, using the
+the reference point.  *whence* can be omitted and defaults to 0, using the
 beginning of the file as the reference point. ::
 
    >>> f = open('workfile', 'rb+')


From webhook-mailer at python.org  Sun Sep 15 02:50:10 2019
From: webhook-mailer at python.org (Stefan Behnel)
Date: Sun, 15 Sep 2019 06:50:10 -0000
Subject: [Python-checkins] bpo38158: Removing nonexistant member "doc" from
 PyType_Spec documentation (GH-16142)
Message-ID: <mailman.754.1568530211.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8b31a11a698cb5aa9b439b349c8de4e388846f73
commit: 8b31a11a698cb5aa9b439b349c8de4e388846f73
branch: master
author: t k <tahia.khan at utoronto.ca>
committer: Stefan Behnel <stefan_ml at behnel.de>
date: 2019-09-15T07:50:05+01:00
summary:

bpo38158: Removing nonexistant member "doc" from PyType_Spec documentation (GH-16142)

files:
M Doc/c-api/type.rst

diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index 8f8367ab77c8..6416951ac5a9 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -142,10 +142,6 @@ The following functions and structs are used to create
 
       Name of the type, used to set :c:member:`PyTypeObject.tp_name`.
 
-   .. c:member:: const char* PyType_Spec.doc
-
-      Type docstring, used to set :c:member:`PyTypeObject.tp_doc`.
-
    .. c:member:: int PyType_Spec.basicsize
    .. c:member:: int PyType_Spec.itemsize
 


From webhook-mailer at python.org  Sun Sep 15 03:00:48 2019
From: webhook-mailer at python.org (Stefan Behnel)
Date: Sun, 15 Sep 2019 07:00:48 -0000
Subject: [Python-checkins] bpo-38158: Removing nonexistant member "doc" from
 PyType_Spec documentation (GH-16142) (GH-16154)
Message-ID: <mailman.755.1568530849.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b65be6cd3d9b6102227d27f4f35385f999a7dd7d
commit: b65be6cd3d9b6102227d27f4f35385f999a7dd7d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Stefan Behnel <stefan_ml at behnel.de>
date: 2019-09-15T08:00:44+01:00
summary:

bpo-38158: Removing nonexistant member "doc" from PyType_Spec documentation (GH-16142) (GH-16154)

(cherry picked from commit 8b31a11a698cb5aa9b439b349c8de4e388846f73)

Co-authored-by: t k <tahia.khan at utoronto.ca>

files:
M Doc/c-api/type.rst

diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index 8f8367ab77c8..6416951ac5a9 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -142,10 +142,6 @@ The following functions and structs are used to create
 
       Name of the type, used to set :c:member:`PyTypeObject.tp_name`.
 
-   .. c:member:: const char* PyType_Spec.doc
-
-      Type docstring, used to set :c:member:`PyTypeObject.tp_doc`.
-
    .. c:member:: int PyType_Spec.basicsize
    .. c:member:: int PyType_Spec.itemsize
 


From webhook-mailer at python.org  Sun Sep 15 10:51:53 2019
From: webhook-mailer at python.org (Dino Viehland)
Date: Sun, 15 Sep 2019 14:51:53 -0000
Subject: [Python-checkins] Fix missing dec ref (#16158)
Message-ID: <mailman.756.1568559114.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/09dc2c672f937cbe53300cb680fca1f9c78ff976
commit: 09dc2c672f937cbe53300cb680fca1f9c78ff976
branch: master
author: Dino Viehland <dinoviehland at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-15T15:51:44+01:00
summary:

Fix missing dec ref (#16158)

files:
M Modules/_randommodule.c

diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c
index 8b0a0244bfad..1ea2bf28abcc 100644
--- a/Modules/_randommodule.c
+++ b/Modules/_randommodule.c
@@ -572,6 +572,7 @@ static int
 _random_clear(PyObject *module)
 {
     Py_CLEAR(_randomstate(module)->Random_Type);
+    Py_CLEAR(_randomstate(module)->Long___abs__);
     return 0;
 }
 


From webhook-mailer at python.org  Sun Sep 15 12:36:42 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Sun, 15 Sep 2019 16:36:42 -0000
Subject: [Python-checkins] bpo-37798: Prevent undefined behavior in direct
 calls to the C helper function. (#16149)
Message-ID: <mailman.757.1568565403.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6e27a0d77520bf2c4412e367496212510f81b983
commit: 6e27a0d77520bf2c4412e367496212510f81b983
branch: master
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-15T09:36:38-07:00
summary:

bpo-37798: Prevent undefined behavior in direct calls to the C helper function. (#16149)

files:
M Modules/_statisticsmodule.c

diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c
index 16a992485237..a646e96d0165 100644
--- a/Modules/_statisticsmodule.c
+++ b/Modules/_statisticsmodule.c
@@ -32,8 +32,11 @@ _statistics__normal_dist_inv_cdf_impl(PyObject *module, double p, double mu,
 /*[clinic end generated code: output=02fd19ddaab36602 input=24715a74be15296a]*/
 {
     double q, num, den, r, x;
+    if (p <= 0.0 || p >= 1.0 || sigma <= 0.0) {
+        goto error;
+    }
+
     q = p - 0.5;
-    // Algorithm AS 241: The Percentage Points of the Normal Distribution
     if(fabs(q) <= 0.425) {
         r = 0.180625 - q * q;
         // Hash sum-55.8831928806149014439
@@ -53,10 +56,16 @@ _statistics__normal_dist_inv_cdf_impl(PyObject *module, double p, double mu,
                      6.8718700749205790830e+2) * r +
                      4.2313330701600911252e+1) * r +
                      1.0);
+        if (den == 0.0) {
+            goto error;
+        }
         x = num / den;
         return mu + (x * sigma);
     }
     r = (q <= 0.0) ? p : (1.0 - p);
+    if (r <= 0.0 || r >= 1.0) {
+        goto error;
+    }
     r = sqrt(-log(r));
     if (r <= 5.0) {
         r = r - 1.6;
@@ -97,11 +106,18 @@ _statistics__normal_dist_inv_cdf_impl(PyObject *module, double p, double mu,
                      5.99832206555887937690e-1) * r +
                      1.0);
     }
+    if (den == 0.0) {
+        goto error;
+    }
     x = num / den;
     if (q < 0.0) {
         x = -x;
     }
     return mu + (x * sigma);
+
+  error:
+    PyErr_SetString(PyExc_ValueError, "inv_cdf undefined for these parameters");
+    return -1.0;
 }
 
 


From webhook-mailer at python.org  Sun Sep 15 13:04:04 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Sun, 15 Sep 2019 17:04:04 -0000
Subject: [Python-checkins] bpo-37798: Prevent undefined behavior in direct
 calls to the C helper function. (GH-16149) (GH-16160)
Message-ID: <mailman.758.1568567045.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d6fdfc82dd307161ca2222ae938b7a6c85215bc1
commit: d6fdfc82dd307161ca2222ae938b7a6c85215bc1
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-15T10:04:00-07:00
summary:

bpo-37798: Prevent undefined behavior in direct calls to the C helper function. (GH-16149) (GH-16160)

(cherry picked from commit 6e27a0d77520bf2c4412e367496212510f81b983)

Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com>

files:
M Modules/_statisticsmodule.c

diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c
index 16a992485237..a646e96d0165 100644
--- a/Modules/_statisticsmodule.c
+++ b/Modules/_statisticsmodule.c
@@ -32,8 +32,11 @@ _statistics__normal_dist_inv_cdf_impl(PyObject *module, double p, double mu,
 /*[clinic end generated code: output=02fd19ddaab36602 input=24715a74be15296a]*/
 {
     double q, num, den, r, x;
+    if (p <= 0.0 || p >= 1.0 || sigma <= 0.0) {
+        goto error;
+    }
+
     q = p - 0.5;
-    // Algorithm AS 241: The Percentage Points of the Normal Distribution
     if(fabs(q) <= 0.425) {
         r = 0.180625 - q * q;
         // Hash sum-55.8831928806149014439
@@ -53,10 +56,16 @@ _statistics__normal_dist_inv_cdf_impl(PyObject *module, double p, double mu,
                      6.8718700749205790830e+2) * r +
                      4.2313330701600911252e+1) * r +
                      1.0);
+        if (den == 0.0) {
+            goto error;
+        }
         x = num / den;
         return mu + (x * sigma);
     }
     r = (q <= 0.0) ? p : (1.0 - p);
+    if (r <= 0.0 || r >= 1.0) {
+        goto error;
+    }
     r = sqrt(-log(r));
     if (r <= 5.0) {
         r = r - 1.6;
@@ -97,11 +106,18 @@ _statistics__normal_dist_inv_cdf_impl(PyObject *module, double p, double mu,
                      5.99832206555887937690e-1) * r +
                      1.0);
     }
+    if (den == 0.0) {
+        goto error;
+    }
     x = num / den;
     if (q < 0.0) {
         x = -x;
     }
     return mu + (x * sigma);
+
+  error:
+    PyErr_SetString(PyExc_ValueError, "inv_cdf undefined for these parameters");
+    return -1.0;
 }
 
 


From webhook-mailer at python.org  Sun Sep 15 13:06:05 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sun, 15 Sep 2019 17:06:05 -0000
Subject: [Python-checkins] bpo-38178: Don't explicitly pass "loop" to
 EchoClientProtocol. (GH-16159)
Message-ID: <mailman.759.1568567166.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c717c73fa33a2f3591442059eaf6e7a673e2c725
commit: c717c73fa33a2f3591442059eaf6e7a673e2c725
branch: master
author: Hrvoje Nik?i? <hniksic at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-15T10:06:02-07:00
summary:

bpo-38178: Don't explicitly pass "loop" to EchoClientProtocol. (GH-16159)



https://bugs.python.org/issue38178

files:
M Doc/library/asyncio-protocol.rst

diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index 3e5a4dd8b891..ca37581c288d 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -767,9 +767,8 @@ data, and waits until the connection is closed::
 
 
     class EchoClientProtocol(asyncio.Protocol):
-        def __init__(self, message, on_con_lost, loop):
+        def __init__(self, message, on_con_lost):
             self.message = message
-            self.loop = loop
             self.on_con_lost = on_con_lost
 
         def connection_made(self, transport):
@@ -869,11 +868,10 @@ method, sends data and closes the transport when it receives the answer::
 
 
     class EchoClientProtocol:
-        def __init__(self, message, loop):
+        def __init__(self, message, on_con_lost):
             self.message = message
-            self.loop = loop
             self.transport = None
-            self.on_con_lost = loop.create_future()
+            self.on_con_lost = on_con_lost
 
         def connection_made(self, transport):
             self.transport = transport
@@ -899,13 +897,15 @@ method, sends data and closes the transport when it receives the answer::
         # low-level APIs.
         loop = asyncio.get_running_loop()
 
+        on_con_lost = loop.create_future()
         message = "Hello World!"
+
         transport, protocol = await loop.create_datagram_endpoint(
-            lambda: EchoClientProtocol(message, loop),
+            lambda: EchoClientProtocol(message, on_con_lost),
             remote_addr=('127.0.0.1', 9999))
 
         try:
-            await protocol.on_con_lost
+            await on_con_lost
         finally:
             transport.close()
 


From webhook-mailer at python.org  Sun Sep 15 13:12:16 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sun, 15 Sep 2019 17:12:16 -0000
Subject: [Python-checkins] bpo-38178: Don't explicitly pass "loop" to
 EchoClientProtocol. (GH-16159)
Message-ID: <mailman.760.1568567538.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/701c4886bb0e19ad5ed8c8f556f65ab8597841e7
commit: 701c4886bb0e19ad5ed8c8f556f65ab8597841e7
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-15T10:12:13-07:00
summary:

bpo-38178: Don't explicitly pass "loop" to EchoClientProtocol. (GH-16159)


https://bugs.python.org/issue38178
(cherry picked from commit c717c73fa33a2f3591442059eaf6e7a673e2c725)

Co-authored-by: Hrvoje Nik?i? <hniksic at gmail.com>

files:
M Doc/library/asyncio-protocol.rst

diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index d8d7947c030a..d421905f0f32 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -767,9 +767,8 @@ data, and waits until the connection is closed::
 
 
     class EchoClientProtocol(asyncio.Protocol):
-        def __init__(self, message, on_con_lost, loop):
+        def __init__(self, message, on_con_lost):
             self.message = message
-            self.loop = loop
             self.on_con_lost = on_con_lost
 
         def connection_made(self, transport):
@@ -869,11 +868,10 @@ method, sends data and closes the transport when it receives the answer::
 
 
     class EchoClientProtocol:
-        def __init__(self, message, loop):
+        def __init__(self, message, on_con_lost):
             self.message = message
-            self.loop = loop
             self.transport = None
-            self.on_con_lost = loop.create_future()
+            self.on_con_lost = on_con_lost
 
         def connection_made(self, transport):
             self.transport = transport
@@ -899,13 +897,15 @@ method, sends data and closes the transport when it receives the answer::
         # low-level APIs.
         loop = asyncio.get_running_loop()
 
+        on_con_lost = loop.create_future()
         message = "Hello World!"
+
         transport, protocol = await loop.create_datagram_endpoint(
-            lambda: EchoClientProtocol(message, loop),
+            lambda: EchoClientProtocol(message, on_con_lost),
             remote_addr=('127.0.0.1', 9999))
 
         try:
-            await protocol.on_con_lost
+            await on_con_lost
         finally:
             transport.close()
 


From webhook-mailer at python.org  Sun Sep 15 13:13:38 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sun, 15 Sep 2019 17:13:38 -0000
Subject: [Python-checkins] bpo-38178: Don't explicitly pass "loop" to
 EchoClientProtocol. (GH-16159)
Message-ID: <mailman.761.1568567618.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1cd6e926d30552056457820a682c1c361ee0f3ff
commit: 1cd6e926d30552056457820a682c1c361ee0f3ff
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-15T10:13:34-07:00
summary:

bpo-38178: Don't explicitly pass "loop" to EchoClientProtocol. (GH-16159)


https://bugs.python.org/issue38178
(cherry picked from commit c717c73fa33a2f3591442059eaf6e7a673e2c725)

Co-authored-by: Hrvoje Nik?i? <hniksic at gmail.com>

files:
M Doc/library/asyncio-protocol.rst

diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index 3e5a4dd8b891..ca37581c288d 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -767,9 +767,8 @@ data, and waits until the connection is closed::
 
 
     class EchoClientProtocol(asyncio.Protocol):
-        def __init__(self, message, on_con_lost, loop):
+        def __init__(self, message, on_con_lost):
             self.message = message
-            self.loop = loop
             self.on_con_lost = on_con_lost
 
         def connection_made(self, transport):
@@ -869,11 +868,10 @@ method, sends data and closes the transport when it receives the answer::
 
 
     class EchoClientProtocol:
-        def __init__(self, message, loop):
+        def __init__(self, message, on_con_lost):
             self.message = message
-            self.loop = loop
             self.transport = None
-            self.on_con_lost = loop.create_future()
+            self.on_con_lost = on_con_lost
 
         def connection_made(self, transport):
             self.transport = transport
@@ -899,13 +897,15 @@ method, sends data and closes the transport when it receives the answer::
         # low-level APIs.
         loop = asyncio.get_running_loop()
 
+        on_con_lost = loop.create_future()
         message = "Hello World!"
+
         transport, protocol = await loop.create_datagram_endpoint(
-            lambda: EchoClientProtocol(message, loop),
+            lambda: EchoClientProtocol(message, on_con_lost),
             remote_addr=('127.0.0.1', 9999))
 
         try:
-            await protocol.on_con_lost
+            await on_con_lost
         finally:
             transport.close()
 


From webhook-mailer at python.org  Sun Sep 15 22:38:36 2019
From: webhook-mailer at python.org (Ned Deily)
Date: Mon, 16 Sep 2019 02:38:36 -0000
Subject: [Python-checkins] bpo-38117: Updated OpenSSL to 1.1.1d in macOS
 installer. (GH-16170)
Message-ID: <mailman.762.1568601518.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/24d1597e430498ebe2d3d18fba2cacb3957b494d
commit: 24d1597e430498ebe2d3d18fba2cacb3957b494d
branch: master
author: Ned Deily <nad at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-16T03:38:27+01:00
summary:

bpo-38117: Updated OpenSSL to 1.1.1d in macOS installer. (GH-16170)

files:
A Misc/NEWS.d/next/macOS/2019-09-15-21-29-13.bpo-38117.ZLsoAZ.rst
M Mac/BuildScript/build-installer.py

diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py
index 88e34322200f..0dd53647dd33 100755
--- a/Mac/BuildScript/build-installer.py
+++ b/Mac/BuildScript/build-installer.py
@@ -215,9 +215,9 @@ def library_recipes():
 
     result.extend([
           dict(
-              name="OpenSSL 1.1.1c",
-              url="https://www.openssl.org/source/openssl-1.1.1c.tar.gz",
-              checksum='15e21da6efe8aa0e0768ffd8cd37a5f6',
+              name="OpenSSL 1.1.1d",
+              url="https://www.openssl.org/source/openssl-1.1.1d.tar.gz",
+              checksum='3be209000dbc7e1b95bcdf47980a3baa',
               buildrecipe=build_universal_openssl,
               configure=None,
               install=None,
diff --git a/Misc/NEWS.d/next/macOS/2019-09-15-21-29-13.bpo-38117.ZLsoAZ.rst b/Misc/NEWS.d/next/macOS/2019-09-15-21-29-13.bpo-38117.ZLsoAZ.rst
new file mode 100644
index 000000000000..644f147c9553
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2019-09-15-21-29-13.bpo-38117.ZLsoAZ.rst
@@ -0,0 +1 @@
+Updated OpenSSL to 1.1.1d in macOS installer.


From webhook-mailer at python.org  Sun Sep 15 22:56:08 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 02:56:08 -0000
Subject: [Python-checkins] bpo-38117: Updated OpenSSL to 1.1.1d in macOS
 installer. (GH-16170)
Message-ID: <mailman.763.1568602568.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/26c44b5144a51f452dd0693b0278ec69d6a954a1
commit: 26c44b5144a51f452dd0693b0278ec69d6a954a1
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-15T19:56:04-07:00
summary:

bpo-38117: Updated OpenSSL to 1.1.1d in macOS installer. (GH-16170)

(cherry picked from commit 24d1597e430498ebe2d3d18fba2cacb3957b494d)

Co-authored-by: Ned Deily <nad at python.org>

files:
A Misc/NEWS.d/next/macOS/2019-09-15-21-29-13.bpo-38117.ZLsoAZ.rst
M Mac/BuildScript/build-installer.py

diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py
index 024ec910ab0a..6c6c5d2048d1 100755
--- a/Mac/BuildScript/build-installer.py
+++ b/Mac/BuildScript/build-installer.py
@@ -215,9 +215,9 @@ def library_recipes():
 
     result.extend([
           dict(
-              name="OpenSSL 1.1.1c",
-              url="https://www.openssl.org/source/openssl-1.1.1c.tar.gz",
-              checksum='15e21da6efe8aa0e0768ffd8cd37a5f6',
+              name="OpenSSL 1.1.1d",
+              url="https://www.openssl.org/source/openssl-1.1.1d.tar.gz",
+              checksum='3be209000dbc7e1b95bcdf47980a3baa',
               buildrecipe=build_universal_openssl,
               configure=None,
               install=None,
diff --git a/Misc/NEWS.d/next/macOS/2019-09-15-21-29-13.bpo-38117.ZLsoAZ.rst b/Misc/NEWS.d/next/macOS/2019-09-15-21-29-13.bpo-38117.ZLsoAZ.rst
new file mode 100644
index 000000000000..644f147c9553
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2019-09-15-21-29-13.bpo-38117.ZLsoAZ.rst
@@ -0,0 +1 @@
+Updated OpenSSL to 1.1.1d in macOS installer.


From webhook-mailer at python.org  Sun Sep 15 22:57:32 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 02:57:32 -0000
Subject: [Python-checkins] bpo-38117: Updated OpenSSL to 1.1.1d in macOS
 installer. (GH-16170)
Message-ID: <mailman.764.1568602653.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/346b7c928b82d9da8f6654391e9cfe55625dcfac
commit: 346b7c928b82d9da8f6654391e9cfe55625dcfac
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-15T19:57:28-07:00
summary:

bpo-38117: Updated OpenSSL to 1.1.1d in macOS installer. (GH-16170)

(cherry picked from commit 24d1597e430498ebe2d3d18fba2cacb3957b494d)

Co-authored-by: Ned Deily <nad at python.org>

files:
A Misc/NEWS.d/next/macOS/2019-09-15-21-29-13.bpo-38117.ZLsoAZ.rst
M Mac/BuildScript/build-installer.py

diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py
index 88e34322200f..0dd53647dd33 100755
--- a/Mac/BuildScript/build-installer.py
+++ b/Mac/BuildScript/build-installer.py
@@ -215,9 +215,9 @@ def library_recipes():
 
     result.extend([
           dict(
-              name="OpenSSL 1.1.1c",
-              url="https://www.openssl.org/source/openssl-1.1.1c.tar.gz",
-              checksum='15e21da6efe8aa0e0768ffd8cd37a5f6',
+              name="OpenSSL 1.1.1d",
+              url="https://www.openssl.org/source/openssl-1.1.1d.tar.gz",
+              checksum='3be209000dbc7e1b95bcdf47980a3baa',
               buildrecipe=build_universal_openssl,
               configure=None,
               install=None,
diff --git a/Misc/NEWS.d/next/macOS/2019-09-15-21-29-13.bpo-38117.ZLsoAZ.rst b/Misc/NEWS.d/next/macOS/2019-09-15-21-29-13.bpo-38117.ZLsoAZ.rst
new file mode 100644
index 000000000000..644f147c9553
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2019-09-15-21-29-13.bpo-38117.ZLsoAZ.rst
@@ -0,0 +1 @@
+Updated OpenSSL to 1.1.1d in macOS installer.


From webhook-mailer at python.org  Sun Sep 15 23:36:05 2019
From: webhook-mailer at python.org (Ned Deily)
Date: Mon, 16 Sep 2019 03:36:05 -0000
Subject: [Python-checkins] bpo-38117: Updated OpenSSL to 1.0.2t in macOS
 installer for 2.7.x. (GH-16171)
Message-ID: <mailman.765.1568604967.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8dd358caf0d3f300ee64799812a020fe0de53637
commit: 8dd358caf0d3f300ee64799812a020fe0de53637
branch: 2.7
author: Ned Deily <nad at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-16T04:35:55+01:00
summary:

bpo-38117: Updated OpenSSL to 1.0.2t in macOS installer for 2.7.x. (GH-16171)

files:
A Misc/NEWS.d/next/macOS/2019-09-15-21-45-15.bpo-38117._U9F2r.rst
M Mac/BuildScript/build-installer.py

diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py
index e24d28e97e9e..3edd947f0df1 100755
--- a/Mac/BuildScript/build-installer.py
+++ b/Mac/BuildScript/build-installer.py
@@ -213,9 +213,9 @@ def library_recipes():
 
     result.extend([
           dict(
-              name="OpenSSL 1.0.2s",
-              url="https://www.openssl.org/source/openssl-1.0.2s.tar.gz",
-              checksum='98ec4e085962689b91d25e1dcdfc14a2',
+              name="OpenSSL 1.0.2t",
+              url="https://www.openssl.org/source/openssl-1.0.2t.tar.gz",
+              checksum='ef66581b80f06eae42f5268bc0b50c6d',
               buildrecipe=build_universal_openssl,
               configure=None,
               install=None,
diff --git a/Misc/NEWS.d/next/macOS/2019-09-15-21-45-15.bpo-38117._U9F2r.rst b/Misc/NEWS.d/next/macOS/2019-09-15-21-45-15.bpo-38117._U9F2r.rst
new file mode 100644
index 000000000000..dd9840da084d
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2019-09-15-21-45-15.bpo-38117._U9F2r.rst
@@ -0,0 +1 @@
+Updated OpenSSL to 1.0.2t in macOS installer for 2.7.x.


From webhook-mailer at python.org  Mon Sep 16 01:57:20 2019
From: webhook-mailer at python.org (Xiang Zhang)
Date: Mon, 16 Sep 2019 05:57:20 -0000
Subject: [Python-checkins] Fix a possbile refleak in setint() of
 mmapmodule.c (GH-16136)
Message-ID: <mailman.766.1568613441.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/56a45142e70a1ccf3233d43cb60c47255252e89a
commit: 56a45142e70a1ccf3233d43cb60c47255252e89a
branch: master
author: Hai Shi <shihai1992 at gmail.com>
committer: Xiang Zhang <angwerzx at 126.com>
date: 2019-09-16T13:56:57+08:00
summary:

Fix a possbile refleak in setint() of mmapmodule.c (GH-16136)

files:
M Modules/mmapmodule.c

diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c
index 51ab3f054f24..0c641636a1a8 100644
--- a/Modules/mmapmodule.c
+++ b/Modules/mmapmodule.c
@@ -1468,7 +1468,8 @@ static void
 setint(PyObject *d, const char *name, long value)
 {
     PyObject *o = PyLong_FromLong(value);
-    if (o && PyDict_SetItemString(d, name, o) == 0) {
+    if (o) {
+        PyDict_SetItemString(d, name, o);
         Py_DECREF(o);
     }
 }


From webhook-mailer at python.org  Mon Sep 16 02:27:06 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 06:27:06 -0000
Subject: [Python-checkins] [3.8] bpo-38168: Fix a possbile refleak in
 setint() of mmapmodule.c (GH-16136) (GH-16174)
Message-ID: <mailman.767.1568615226.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/322309efe658122cac340adf4995ce40fa0c2e74
commit: 322309efe658122cac340adf4995ce40fa0c2e74
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-15T23:26:57-07:00
summary:

[3.8] bpo-38168: Fix a possbile refleak in setint() of mmapmodule.c (GH-16136) (GH-16174)



(cherry picked from commit 56a45142e70a1ccf3233d43cb60c47255252e89a)


Co-authored-by: Hai Shi <shihai1992 at gmail.com>

https://bugs.python.org/issue38168



Automerge-Triggered-By: @zhangyangyu

files:
M Modules/mmapmodule.c

diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c
index 9e3414f94e31..18758861a62e 100644
--- a/Modules/mmapmodule.c
+++ b/Modules/mmapmodule.c
@@ -1468,7 +1468,8 @@ static void
 setint(PyObject *d, const char *name, long value)
 {
     PyObject *o = PyLong_FromLong(value);
-    if (o && PyDict_SetItemString(d, name, o) == 0) {
+    if (o) {
+        PyDict_SetItemString(d, name, o);
         Py_DECREF(o);
     }
 }


From webhook-mailer at python.org  Mon Sep 16 02:27:42 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 06:27:42 -0000
Subject: [Python-checkins] [3.7] bpo-38168: Fix a possbile refleak in
 setint() of mmapmodule.c (GH-16136) (GH-16175)
Message-ID: <mailman.768.1568615264.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e857fe624d985616f3e1a6e095a5db732e6b3efe
commit: e857fe624d985616f3e1a6e095a5db732e6b3efe
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-15T23:27:38-07:00
summary:

[3.7] bpo-38168: Fix a possbile refleak in setint() of mmapmodule.c (GH-16136) (GH-16175)



(cherry picked from commit 56a45142e70a1ccf3233d43cb60c47255252e89a)


Co-authored-by: Hai Shi <shihai1992 at gmail.com>

Automerge-Triggered-By: @zhangyangyu

files:
M Modules/mmapmodule.c

diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c
index 223afacf3140..37a5c3dade8e 100644
--- a/Modules/mmapmodule.c
+++ b/Modules/mmapmodule.c
@@ -1392,7 +1392,8 @@ static void
 setint(PyObject *d, const char *name, long value)
 {
     PyObject *o = PyLong_FromLong(value);
-    if (o && PyDict_SetItemString(d, name, o) == 0) {
+    if (o) {
+        PyDict_SetItemString(d, name, o);
         Py_DECREF(o);
     }
 }


From webhook-mailer at python.org  Mon Sep 16 03:05:04 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Mon, 16 Sep 2019 07:05:04 -0000
Subject: [Python-checkins] Doc: remove duplicate word in controlflow
 tutorial (GH-16163)
Message-ID: <mailman.769.1568617505.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b7af4e75657e6478920d00260c48a1a3020002fc
commit: b7af4e75657e6478920d00260c48a1a3020002fc
branch: master
author: Adorilson Bezerra <adorilson at gmail.com>
committer: Julien Palard <julien at palard.fr>
date: 2019-09-16T09:04:58+02:00
summary:

Doc: remove duplicate word in controlflow tutorial (GH-16163)

files:
M Doc/tutorial/controlflow.rst

diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst
index b7e003c55018..3730a215d748 100644
--- a/Doc/tutorial/controlflow.rst
+++ b/Doc/tutorial/controlflow.rst
@@ -695,7 +695,7 @@ As guidance:
 * Use keyword-only when names have meaning and the function definition is
   more understandable by being explicit with names or you want to prevent
   users relying on the position of the argument being passed.
-* For an API, use positional-only to prevent prevent breaking API changes
+* For an API, use positional-only to prevent breaking API changes
   if the parameter's name is modified in the future.
 
 .. _tut-arbitraryargs:


From webhook-mailer at python.org  Mon Sep 16 03:07:36 2019
From: webhook-mailer at python.org (Xiang Zhang)
Date: Mon, 16 Sep 2019 07:07:36 -0000
Subject: [Python-checkins] [2.7] bpo-38168: Fix a possbile refleak in
 setint() of mmapmodule.c (GH-16136) (GH-16176)
Message-ID: <mailman.770.1568617656.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/68d8c122972d7a49627b983af4ccbfad9f5ade17
commit: 68d8c122972d7a49627b983af4ccbfad9f5ade17
branch: 2.7
author: Xiang Zhang <angwerzx at 126.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T15:07:32+08:00
summary:

[2.7] bpo-38168: Fix a possbile refleak in setint() of mmapmodule.c (GH-16136) (GH-16176)

(cherry picked from commit 56a4514)

Co-authored-by: Hai Shi shihai1992 at gmail.com

https://bugs.python.org/issue38168

files:
M Modules/mmapmodule.c

diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c
index 02b31ca438a9..57cc40c53fd6 100644
--- a/Modules/mmapmodule.c
+++ b/Modules/mmapmodule.c
@@ -1436,7 +1436,8 @@ static void
 setint(PyObject *d, const char *name, long value)
 {
     PyObject *o = PyInt_FromLong(value);
-    if (o && PyDict_SetItemString(d, name, o) == 0) {
+    if (o) {
+        PyDict_SetItemString(d, name, o);
         Py_DECREF(o);
     }
 }


From webhook-mailer at python.org  Mon Sep 16 03:13:18 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 07:13:18 -0000
Subject: [Python-checkins] Doc: remove duplicate word in controlflow
 tutorial (GH-16163)
Message-ID: <mailman.771.1568618000.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/817227ebd522f2051004525fc39f31a9edd47292
commit: 817227ebd522f2051004525fc39f31a9edd47292
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T00:13:14-07:00
summary:

Doc: remove duplicate word in controlflow tutorial (GH-16163)

(cherry picked from commit b7af4e75657e6478920d00260c48a1a3020002fc)

Co-authored-by: Adorilson Bezerra <adorilson at gmail.com>

files:
M Doc/tutorial/controlflow.rst

diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst
index b7e003c55018..3730a215d748 100644
--- a/Doc/tutorial/controlflow.rst
+++ b/Doc/tutorial/controlflow.rst
@@ -695,7 +695,7 @@ As guidance:
 * Use keyword-only when names have meaning and the function definition is
   more understandable by being explicit with names or you want to prevent
   users relying on the position of the argument being passed.
-* For an API, use positional-only to prevent prevent breaking API changes
+* For an API, use positional-only to prevent breaking API changes
   if the parameter's name is modified in the future.
 
 .. _tut-arbitraryargs:


From webhook-mailer at python.org  Mon Sep 16 08:07:49 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 16 Sep 2019 12:07:49 -0000
Subject: [Python-checkins] bpo-38117: Updates bundled OpenSSL to 1.0.2t
 (GH-16178)
Message-ID: <mailman.772.1568635670.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d8903416d223d01e23dfa8bffe48b1428d90d202
commit: d8903416d223d01e23dfa8bffe48b1428d90d202
branch: 2.7
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-16T13:07:40+01:00
summary:

bpo-38117: Updates bundled OpenSSL to 1.0.2t (GH-16178)

files:
A Misc/NEWS.d/next/Windows/2019-09-16-12-47-12.bpo-38117.JaiB0X.rst
M PCbuild/get_external.py
M PCbuild/get_externals.bat
M PCbuild/python.props

diff --git a/Misc/NEWS.d/next/Windows/2019-09-16-12-47-12.bpo-38117.JaiB0X.rst b/Misc/NEWS.d/next/Windows/2019-09-16-12-47-12.bpo-38117.JaiB0X.rst
new file mode 100644
index 000000000000..35e1cabb9b4a
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-16-12-47-12.bpo-38117.JaiB0X.rst
@@ -0,0 +1 @@
+Updates bundled OpenSSL to 1.0.2t
diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py
index a682d3849f14..10dacdd984dc 100644
--- a/PCbuild/get_external.py
+++ b/PCbuild/get_external.py
@@ -1,12 +1,16 @@
 #!/usr/bin/env python3
 
+# We do not need to import built modules from PCbuild (duh), so
+# filter those out here to save Steve from debugging it yet again.
+import sys
+sys.path[:] = [p for p in sys.path if p and 'PCbuild' not in p]
+
 import argparse
 import os
 import pathlib
 import zipfile
 from urllib.request import urlretrieve
 
-
 def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose):
     repo = f'cpython-{"bin" if binary else "source"}-deps'
     url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip'
diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat
index eba19492d94b..9f1b09e9b659 100644
--- a/PCbuild/get_externals.bat
+++ b/PCbuild/get_externals.bat
@@ -47,7 +47,7 @@ rem files in both this dir and PC\VS9.0
 set libraries=
 set libraries=%libraries%                                    bzip2-1.0.6
 if NOT "%IncludeBsddb%"=="false" set libraries=%libraries%   bsddb-4.7.25.0
-if NOT "%IncludeSSL%"=="false" set libraries=%libraries%     openssl-1.0.2s
+if NOT "%IncludeSSL%"=="false" set libraries=%libraries%     openssl-1.0.2t
 set libraries=%libraries%                                    sqlite-3.28.0.0
 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-8.5.19.0
 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.5.19.0
diff --git a/PCbuild/python.props b/PCbuild/python.props
index 355bb3f05239..f0b531e50eb4 100644
--- a/PCbuild/python.props
+++ b/PCbuild/python.props
@@ -35,7 +35,7 @@
     <sqlite3Dir>$(ExternalsDir)sqlite-3.28.0.0\</sqlite3Dir>
     <bz2Dir>$(ExternalsDir)bzip2-1.0.6\</bz2Dir>
     <bsddbDir>$(ExternalsDir)bsddb-4.7.25.0</bsddbDir>
-    <opensslDir>$(ExternalsDir)openssl-1.0.2s\</opensslDir>
+    <opensslDir>$(ExternalsDir)openssl-1.0.2t\</opensslDir>
     <opensslIncludeDir>$(opensslDir)include32</opensslIncludeDir>
     <opensslIncludeDir Condition="'$(ArchName)' == 'amd64'">$(opensslDir)include64</opensslIncludeDir>
     <nasmDir>$(ExternalsDir)\nasm-2.11.06\</nasmDir>


From webhook-mailer at python.org  Mon Sep 16 08:08:59 2019
From: webhook-mailer at python.org (Christian Heimes)
Date: Mon, 16 Sep 2019 12:08:59 -0000
Subject: [Python-checkins] [3.8] bpo-38153: Normalize hashlib algorithm
 names (GH-16083) (GH-16144)
Message-ID: <mailman.773.1568635741.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/e8d7fa2db8a6c641019b06943852492f24ac3e69
commit: e8d7fa2db8a6c641019b06943852492f24ac3e69
branch: 3.8
author: Christian Heimes <christian at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-16T14:08:55+02:00
summary:

[3.8] bpo-38153: Normalize hashlib algorithm names (GH-16083) (GH-16144)

Signed-off-by: Christian Heimes <christian at python.org>
(cherry picked from commit 995b5d38e7cc24cac3de8dfd516115f86b0bcf80)

Co-authored-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Library/2019-09-13-12-18-51.bpo-38153.nHAbuJ.rst
M Lib/hashlib.py
M Lib/test/test_hashlib.py
M Modules/_hashopenssl.c

diff --git a/Lib/hashlib.py b/Lib/hashlib.py
index 4e783a86a34e..56873b7278b6 100644
--- a/Lib/hashlib.py
+++ b/Lib/hashlib.py
@@ -70,37 +70,44 @@
 
 __builtin_constructor_cache = {}
 
+__block_openssl_constructor = {
+    'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
+    'shake_128', 'shake_256',
+    'blake2b', 'blake2s',
+}
+
 def __get_builtin_constructor(name):
     cache = __builtin_constructor_cache
     constructor = cache.get(name)
     if constructor is not None:
         return constructor
     try:
-        if name in ('SHA1', 'sha1'):
+        if name in {'SHA1', 'sha1'}:
             import _sha1
             cache['SHA1'] = cache['sha1'] = _sha1.sha1
-        elif name in ('MD5', 'md5'):
+        elif name in {'MD5', 'md5'}:
             import _md5
             cache['MD5'] = cache['md5'] = _md5.md5
-        elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'):
+        elif name in {'SHA256', 'sha256', 'SHA224', 'sha224'}:
             import _sha256
             cache['SHA224'] = cache['sha224'] = _sha256.sha224
             cache['SHA256'] = cache['sha256'] = _sha256.sha256
-        elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'):
+        elif name in {'SHA512', 'sha512', 'SHA384', 'sha384'}:
             import _sha512
             cache['SHA384'] = cache['sha384'] = _sha512.sha384
             cache['SHA512'] = cache['sha512'] = _sha512.sha512
-        elif name in ('blake2b', 'blake2s'):
+        elif name in {'blake2b', 'blake2s'}:
             import _blake2
             cache['blake2b'] = _blake2.blake2b
             cache['blake2s'] = _blake2.blake2s
-        elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
-                      'shake_128', 'shake_256'}:
+        elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512'}:
             import _sha3
             cache['sha3_224'] = _sha3.sha3_224
             cache['sha3_256'] = _sha3.sha3_256
             cache['sha3_384'] = _sha3.sha3_384
             cache['sha3_512'] = _sha3.sha3_512
+        elif name in {'shake_128', 'shake_256'}:
+            import _sha3
             cache['shake_128'] = _sha3.shake_128
             cache['shake_256'] = _sha3.shake_256
     except ImportError:
@@ -114,8 +121,8 @@ def __get_builtin_constructor(name):
 
 
 def __get_openssl_constructor(name):
-    if name in {'blake2b', 'blake2s'}:
-        # Prefer our blake2 implementation.
+    if name in __block_openssl_constructor:
+        # Prefer our blake2 and sha3 implementation.
         return __get_builtin_constructor(name)
     try:
         f = getattr(_hashlib, 'openssl_' + name)
@@ -140,8 +147,8 @@ def __hash_new(name, data=b'', **kwargs):
     """new(name, data=b'') - Return a new hashing object using the named algorithm;
     optionally initialized with data (which must be a bytes-like object).
     """
-    if name in {'blake2b', 'blake2s'}:
-        # Prefer our blake2 implementation.
+    if name in __block_openssl_constructor:
+        # Prefer our blake2 and sha3 implementation
         # OpenSSL 1.1.0 comes with a limited implementation of blake2b/s.
         # It does neither support keyed blake2 nor advanced features like
         # salt, personal, tree hashing or SSE.
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index b7b04a37f0d0..e7cc6ccdc602 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -26,6 +26,11 @@
 c_hashlib = import_fresh_module('hashlib', fresh=['_hashlib'])
 py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib'])
 
+try:
+    from _hashlib import HASH
+except ImportError:
+    HASH = None
+
 try:
     import _blake2
 except ImportError:
@@ -377,6 +382,9 @@ def check_sha3(self, name, capacity, rate, suffix):
         constructors = self.constructors_to_test[name]
         for hash_object_constructor in constructors:
             m = hash_object_constructor()
+            if HASH is not None and isinstance(m, HASH):
+                # _hashopenssl's variant does not have extra SHA3 attributes
+                continue
             self.assertEqual(capacity + rate, 1600)
             self.assertEqual(m._capacity_bits, capacity)
             self.assertEqual(m._rate_bits, rate)
@@ -976,6 +984,10 @@ def test_scrypt(self):
                 hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1,
                                dklen=dklen)
 
+    def test_normalized_name(self):
+        self.assertNotIn("blake2b512", hashlib.algorithms_available)
+        self.assertNotIn("sha3-512", hashlib.algorithms_available)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Library/2019-09-13-12-18-51.bpo-38153.nHAbuJ.rst b/Misc/NEWS.d/next/Library/2019-09-13-12-18-51.bpo-38153.nHAbuJ.rst
new file mode 100644
index 000000000000..8a483c760adc
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-13-12-18-51.bpo-38153.nHAbuJ.rst
@@ -0,0 +1,3 @@
+Names of hashing algorithms frome OpenSSL are now normalized to follow
+Python's naming conventions. For example OpenSSL uses sha3-512 instead of
+sha3_512 or blake2b512 instead of blake2b. 
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 88bf8da03b64..11a26ce8c0bb 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -34,6 +34,14 @@
 
 #define MUNCH_SIZE INT_MAX
 
+#if defined(NID_sha3_224) && defined(EVP_MD_FLAG_XOF)
+#define PY_OPENSSL_HAS_SHA3 1
+#endif
+
+#ifdef NID_blake2b512
+#define PY_OPENSSL_HAS_BLAKE2 1
+#endif
+
 typedef struct {
     PyObject_HEAD
     EVP_MD_CTX          *ctx;   /* OpenSSL message digest context */
@@ -82,6 +90,135 @@ _setException(PyObject *exc)
 }
 /* LCOV_EXCL_STOP */
 
+static PyObject*
+py_digest_name(const EVP_MD *md)
+{
+    int nid = EVP_MD_nid(md);
+    const char *name = NULL;
+
+    /* Hard-coded names for well-known hashing algorithms.
+     * OpenSSL uses slightly different names algorithms like SHA3.
+     */
+    switch (nid) {
+    case NID_md5:
+        name = "md5";
+        break;
+    case NID_sha1:
+        name = "sha1";
+        break;
+    case NID_sha224:
+        name ="sha224";
+        break;
+    case NID_sha256:
+        name ="sha256";
+        break;
+    case NID_sha384:
+        name ="sha384";
+        break;
+    case NID_sha512:
+        name ="sha512";
+        break;
+#ifdef NID_sha512_224
+    case NID_sha512_224:
+        name ="sha512_224";
+        break;
+    case NID_sha512_256:
+        name ="sha512_256";
+        break;
+#endif
+#ifdef PY_OPENSSL_HAS_SHA3
+    case NID_sha3_224:
+        name ="sha3_224";
+        break;
+    case NID_sha3_256:
+        name ="sha3_256";
+        break;
+    case NID_sha3_384:
+        name ="sha3_384";
+        break;
+    case NID_sha3_512:
+        name ="sha3_512";
+        break;
+    case NID_shake128:
+        name ="shake_128";
+        break;
+    case NID_shake256:
+        name ="shake_256";
+        break;
+#endif
+#ifdef PY_OPENSSL_HAS_BLAKE2
+    case NID_blake2s256:
+        name ="blake2s";
+        break;
+    case NID_blake2b512:
+        name ="blake2b";
+        break;
+#endif
+    default:
+        /* Ignore aliased names and only use long, lowercase name. The aliases
+         * pollute the list and OpenSSL appears to have its own definition of
+         * alias as the resulting list still contains duplicate and alternate
+         * names for several algorithms.
+         */
+        name = OBJ_nid2ln(nid);
+        if (name == NULL)
+            name = OBJ_nid2sn(nid);
+        break;
+    }
+
+    return PyUnicode_FromString(name);
+}
+
+static const EVP_MD*
+py_digest_by_name(const char *name)
+{
+    const EVP_MD *digest = EVP_get_digestbyname(name);
+
+    /* OpenSSL uses dash instead of underscore in names of some algorithms
+     * like SHA3 and SHAKE. Detect different spellings. */
+    if (digest == NULL) {
+#ifdef NID_sha512_224
+        if (!strcmp(name, "sha512_224") || !strcmp(name, "SHA512_224")) {
+            digest = EVP_sha512_224();
+        }
+        else if (!strcmp(name, "sha512_256") || !strcmp(name, "SHA512_256")) {
+            digest = EVP_sha512_256();
+        }
+#endif
+#ifdef PY_OPENSSL_HAS_SHA3
+        /* could be sha3_ or shake_, Python never defined upper case */
+        else if (!strcmp(name, "sha3_224")) {
+            digest = EVP_sha3_224();
+        }
+        else if (!strcmp(name, "sha3_256")) {
+            digest = EVP_sha3_256();
+        }
+        else if (!strcmp(name, "sha3_384")) {
+            digest = EVP_sha3_384();
+        }
+        else if (!strcmp(name, "sha3_512")) {
+            digest = EVP_sha3_512();
+        }
+        else if (!strcmp(name, "shake_128")) {
+            digest = EVP_shake128();
+        }
+        else if (!strcmp(name, "shake_256")) {
+            digest = EVP_shake256();
+        }
+#endif
+#ifdef PY_OPENSSL_HAS_BLAKE2
+        else if (!strcmp(name, "blake2s256")) {
+            digest = EVP_blake2s256();
+        }
+        else if (!strcmp(name, "blake2b512")) {
+            digest = EVP_blake2b512();
+        }
+#endif
+    }
+
+    return digest;
+}
+
 static EVPobject *
 newEVPobject(void)
 {
@@ -304,16 +441,7 @@ EVP_get_digest_size(EVPobject *self, void *closure)
 static PyObject *
 EVP_get_name(EVPobject *self, void *closure)
 {
-    const char *name = EVP_MD_name(EVP_MD_CTX_md(self->ctx));
-    PyObject *name_obj, *name_lower;
-
-    name_obj = PyUnicode_FromString(name);
-    if (!name_obj) {
-        return NULL;
-    }
-    name_lower = PyObject_CallMethod(name_obj, "lower", NULL);
-    Py_DECREF(name_obj);
-    return name_lower;
+    return py_digest_name(EVP_MD_CTX_md(self->ctx));
 }
 
 static PyGetSetDef EVP_getseters[] = {
@@ -337,7 +465,7 @@ static PyObject *
 EVP_repr(EVPobject *self)
 {
     PyObject *name_obj, *repr;
-    name_obj = EVP_get_name(self, NULL);
+    name_obj = py_digest_name(EVP_MD_CTX_md(self->ctx));
     if (!name_obj) {
         return NULL;
     }
@@ -403,6 +531,7 @@ static PyTypeObject EVPtype = {
     0,                  /* tp_dictoffset */
 };
 
+\
 static PyObject *
 EVPnew(const EVP_MD *digest,
        const unsigned char *cp, Py_ssize_t len)
@@ -475,7 +604,7 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj)
     if (data_obj)
         GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view);
 
-    digest = EVP_get_digestbyname(name);
+    digest = py_digest_by_name(name);
 
     ret_obj = EVPnew(digest, (unsigned char*)view.buf, view.len);
 
@@ -889,21 +1018,17 @@ typedef struct _internal_name_mapper_state {
 
 /* A callback function to pass to OpenSSL's OBJ_NAME_do_all(...) */
 static void
-_openssl_hash_name_mapper(const OBJ_NAME *openssl_obj_name, void *arg)
+_openssl_hash_name_mapper(const EVP_MD *md, const char *from,
+                          const char *to, void *arg)
 {
     _InternalNameMapperState *state = (_InternalNameMapperState *)arg;
     PyObject *py_name;
 
     assert(state != NULL);
-    if (openssl_obj_name == NULL)
-        return;
-    /* Ignore aliased names, they pollute the list and OpenSSL appears to
-     * have its own definition of alias as the resulting list still
-     * contains duplicate and alternate names for several algorithms.     */
-    if (openssl_obj_name->alias)
+    if (md == NULL)
         return;
 
-    py_name = PyUnicode_FromString(openssl_obj_name->name);
+    py_name = py_digest_name(md);
     if (py_name == NULL) {
         state->error = 1;
     } else {
@@ -925,7 +1050,7 @@ generate_hash_name_list(void)
         return NULL;
     state.error = 0;
 
-    OBJ_NAME_do_all(OBJ_NAME_TYPE_MD_METH, &_openssl_hash_name_mapper, &state);
+    EVP_MD_do_all(&_openssl_hash_name_mapper, &state);
 
     if (state.error) {
         Py_DECREF(state.set);


From webhook-mailer at python.org  Mon Sep 16 08:21:07 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 12:21:07 -0000
Subject: [Python-checkins] bpo-33095: Add reference to isolated mode in -m
 and script option (GH-7764)
Message-ID: <mailman.774.1568636467.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/07186c3959a61c3d73b5ccae431cdd32afe70e5e
commit: 07186c3959a61c3d73b5ccae431cdd32afe70e5e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T05:21:03-07:00
summary:

bpo-33095: Add reference to isolated mode in -m and script option (GH-7764)


Attempt to make isolated mode easier to discover via additional inline documentation.

Co-Authored-By: Julien Palard <julien at palard.fr>
(cherry picked from commit bdd6945d4dbd1fe6a7fcff95f7d6908db7d791a1)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
M Doc/using/cmdline.rst

diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
index 50143f2bd020..69169d7b14f9 100644
--- a/Doc/using/cmdline.rst
+++ b/Doc/using/cmdline.rst
@@ -101,6 +101,11 @@ source.
    first element will be set to ``"-m"``). As with the :option:`-c` option,
    the current directory will be added to the start of :data:`sys.path`.
 
+   :option:`-I` option can  be used to run the script in isolated mode where
+   :data:`sys.path` contains neither the current directory nor the user's
+   site-packages directory. All :envvar:`PYTHON*` environment variables are
+   ignored, too.
+
    Many standard library modules contain code that is invoked on their execution
    as a script.  An example is the :mod:`timeit` module::
 
@@ -121,6 +126,7 @@ source.
    .. versionchanged:: 3.4
       namespace packages are also supported
 
+.. _cmdarg-dash:
 
 .. describe:: -
 
@@ -133,6 +139,8 @@ source.
 
    .. audit-event:: cpython.run_stdin "" ""
 
+.. _cmdarg-script:
+
 .. describe:: <script>
 
    Execute the Python code contained in *script*, which must be a filesystem
@@ -151,6 +159,11 @@ source.
    added to the start of :data:`sys.path` and the ``__main__.py`` file in
    that location is executed as the :mod:`__main__` module.
 
+   :option:`-I` option can  be used to run the script in isolated mode where
+   :data:`sys.path` contains neither the script's directory nor the user's
+   site-packages directory. All :envvar:`PYTHON*` environment variables are
+   ignored, too.
+
    .. audit-event:: cpython.run_file filename
 
    .. seealso::


From webhook-mailer at python.org  Mon Sep 16 08:28:36 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 12:28:36 -0000
Subject: [Python-checkins] bpo-38153: detect shake independently from sha3
 (GH-16143)
Message-ID: <mailman.775.1568636918.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0067fc287a86cdd8d71dc2d402d596950ff88fca
commit: 0067fc287a86cdd8d71dc2d402d596950ff88fca
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T05:28:32-07:00
summary:

bpo-38153: detect shake independently from sha3 (GH-16143)


XOF digests (SHAKE) are not available in OpenSSL 1.1.0 but SHA3 fixed-length digests are.

Signed-off-by: Christian Heimes <christian at python.org>
(cherry picked from commit eb2b0c694aef6122fdf95015abb24e0d095b6401)

Co-authored-by: Christian Heimes <christian at python.org>

files:
M Modules/_hashopenssl.c

diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 11a26ce8c0bb..0e29fe1d3301 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -34,10 +34,14 @@
 
 #define MUNCH_SIZE INT_MAX
 
-#if defined(NID_sha3_224) && defined(EVP_MD_FLAG_XOF)
+#ifdef NID_sha3_224
 #define PY_OPENSSL_HAS_SHA3 1
 #endif
 
+#if defined(EVP_MD_FLAG_XOF) && defined(NID_shake128)
+#define PY_OPENSSL_HAS_SHAKE 1
+#endif
+
 #ifdef NID_blake2b512
 #define PY_OPENSSL_HAS_BLAKE2 1
 #endif
@@ -139,6 +143,8 @@ py_digest_name(const EVP_MD *md)
     case NID_sha3_512:
         name ="sha3_512";
         break;
+#endif
+#ifdef PY_OPENSSL_HAS_SHAKE
     case NID_shake128:
         name ="shake_128";
         break;
@@ -177,8 +183,9 @@ py_digest_by_name(const char *name)
     /* OpenSSL uses dash instead of underscore in names of some algorithms
      * like SHA3 and SHAKE. Detect different spellings. */
     if (digest == NULL) {
+        if (0) {}
 #ifdef NID_sha512_224
-        if (!strcmp(name, "sha512_224") || !strcmp(name, "SHA512_224")) {
+        else if (!strcmp(name, "sha512_224") || !strcmp(name, "SHA512_224")) {
             digest = EVP_sha512_224();
         }
         else if (!strcmp(name, "sha512_256") || !strcmp(name, "SHA512_256")) {
@@ -199,6 +206,8 @@ py_digest_by_name(const char *name)
         else if (!strcmp(name, "sha3_512")) {
             digest = EVP_sha3_512();
         }
+#endif
+#ifdef PY_OPENSSL_HAS_SHAKE
         else if (!strcmp(name, "shake_128")) {
             digest = EVP_shake128();
         }


From webhook-mailer at python.org  Mon Sep 16 08:30:36 2019
From: webhook-mailer at python.org (Julien Palard)
Date: Mon, 16 Sep 2019 12:30:36 -0000
Subject: [Python-checkins] [3.7] bpo-33095: Add reference to isolated mode
 in -m and script option (GH-7764) (GH-16181)
Message-ID: <mailman.776.1568637038.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/210dc3bb37ec57a8b8b16ffe0881181138a6e973
commit: 210dc3bb37ec57a8b8b16ffe0881181138a6e973
branch: 3.7
author: Julien Palard <julien at palard.fr>
committer: GitHub <noreply at github.com>
date: 2019-09-16T14:30:33+02:00
summary:

[3.7] bpo-33095: Add reference to isolated mode in -m and script option (GH-7764) (GH-16181)

Attempt to make isolated mode easier to discover via additional inline documentation.

Co-Authored-By: Julien Palard <julien at palard.fr>.
(cherry picked from commit bdd6945d4dbd1fe6a7fcff95f7d6908db7d791a1)

Co-authored-by: Xtreak <tir.karthi at gmail.com>

files:
M Doc/using/cmdline.rst

diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
index dca89eceeda9..6a60bc4954c6 100644
--- a/Doc/using/cmdline.rst
+++ b/Doc/using/cmdline.rst
@@ -100,6 +100,11 @@ source.
    first element will be set to ``"-m"``). As with the :option:`-c` option,
    the current directory will be added to the start of :data:`sys.path`.
 
+   :option:`-I` option can  be used to run the script in isolated mode where
+   :data:`sys.path` contains neither the current directory nor the user's
+   site-packages directory. All :envvar:`PYTHON*` environment variables are
+   ignored, too.
+
    Many standard library modules contain code that is invoked on their execution
    as a script.  An example is the :mod:`timeit` module::
 
@@ -119,6 +124,7 @@ source.
    .. versionchanged:: 3.4
       namespace packages are also supported
 
+.. _cmdarg-dash:
 
 .. describe:: -
 
@@ -130,6 +136,8 @@ source.
    :data:`sys.path`.
 
 
+.. _cmdarg-script:
+
 .. describe:: <script>
 
    Execute the Python code contained in *script*, which must be a filesystem
@@ -148,6 +156,11 @@ source.
    added to the start of :data:`sys.path` and the ``__main__.py`` file in
    that location is executed as the :mod:`__main__` module.
 
+   :option:`-I` option can  be used to run the script in isolated mode where
+   :data:`sys.path` contains neither the script's directory nor the user's
+   site-packages directory. All :envvar:`PYTHON*` environment variables are
+   ignored, too.
+
    .. seealso::
       :func:`runpy.run_path`
          Equivalent functionality directly available to Python code


From webhook-mailer at python.org  Mon Sep 16 10:21:20 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 16 Sep 2019 14:21:20 -0000
Subject: [Python-checkins] bpo-38117: Update bundled Windows OpenSSL to
 1.1.1d (GH-16184)
Message-ID: <mailman.777.1568643681.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3ab73f6bbf4ff66e7ace3c4eab25461b5c7d71d4
commit: 3ab73f6bbf4ff66e7ace3c4eab25461b5c7d71d4
branch: master
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-16T15:21:16+01:00
summary:

bpo-38117: Update bundled Windows OpenSSL to 1.1.1d (GH-16184)

files:
A Misc/NEWS.d/next/Windows/2019-09-16-14-07-11.bpo-38117.hJVf0C.rst
M PCbuild/get_externals.bat
M PCbuild/python.props

diff --git a/Misc/NEWS.d/next/Windows/2019-09-16-14-07-11.bpo-38117.hJVf0C.rst b/Misc/NEWS.d/next/Windows/2019-09-16-14-07-11.bpo-38117.hJVf0C.rst
new file mode 100644
index 000000000000..4fd8a66d279e
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-16-14-07-11.bpo-38117.hJVf0C.rst
@@ -0,0 +1 @@
+Update bundled OpenSSL to 1.1.1d
diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat
index 04c71acd469f..100149e83c16 100644
--- a/PCbuild/get_externals.bat
+++ b/PCbuild/get_externals.bat
@@ -53,7 +53,7 @@ echo.Fetching external libraries...
 set libraries=
 set libraries=%libraries%                                       bzip2-1.0.6
 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries%  libffi-3.3.0-rc0-r1
-if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries%     openssl-1.1.1c
+if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries%     openssl-1.1.1d
 set libraries=%libraries%                                       sqlite-3.28.0.0
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.9.0
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.9.0
@@ -77,7 +77,7 @@ echo.Fetching external binaries...
 
 set binaries=
 if NOT "%IncludeLibffi%"=="false"  set binaries=%binaries% libffi
-if NOT "%IncludeSSL%"=="false"     set binaries=%binaries% openssl-bin-1.1.1c
+if NOT "%IncludeSSL%"=="false"     set binaries=%binaries% openssl-bin-1.1.1d
 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.9.0
 if NOT "%IncludeSSLSrc%"=="false"  set binaries=%binaries% nasm-2.11.06
 
diff --git a/PCbuild/python.props b/PCbuild/python.props
index ba68ce334b68..8583138feb6b 100644
--- a/PCbuild/python.props
+++ b/PCbuild/python.props
@@ -62,8 +62,8 @@
     <libffiDir>$(ExternalsDir)libffi\</libffiDir>
     <libffiOutDir>$(ExternalsDir)libffi\$(ArchName)\</libffiOutDir>
     <libffiIncludeDir>$(libffiOutDir)include</libffiIncludeDir>
-    <opensslDir>$(ExternalsDir)openssl-1.1.1c\</opensslDir>
-    <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.1c\$(ArchName)\</opensslOutDir>
+    <opensslDir>$(ExternalsDir)openssl-1.1.1d\</opensslDir>
+    <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.1d\$(ArchName)\</opensslOutDir>
     <opensslIncludeDir>$(opensslOutDir)include</opensslIncludeDir>
     <nasmDir>$(ExternalsDir)\nasm-2.11.06\</nasmDir>
     <zlibDir>$(ExternalsDir)\zlib-1.2.11\</zlibDir>


From webhook-mailer at python.org  Mon Sep 16 10:25:15 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 16 Sep 2019 14:25:15 -0000
Subject: [Python-checkins] bpo-38081: Add more non-fatal error codes for
 ntpath.realpath (GH-16156)
Message-ID: <mailman.778.1568643917.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/89b8933bb537179f81003928786c5cc6183af591
commit: 89b8933bb537179f81003928786c5cc6183af591
branch: master
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-16T15:25:11+01:00
summary:

bpo-38081: Add more non-fatal error codes for ntpath.realpath (GH-16156)

files:
M Lib/ntpath.py

diff --git a/Lib/ntpath.py b/Lib/ntpath.py
index 1b5e16f95f16..01f1f423c906 100644
--- a/Lib/ntpath.py
+++ b/Lib/ntpath.py
@@ -530,15 +530,28 @@ def _readlink_deep(path, seen=None):
         if seen is None:
             seen = set()
 
+        # These error codes indicate that we should stop reading links and
+        # return the path we currently have.
+        # 1: ERROR_INVALID_FUNCTION
+        # 2: ERROR_FILE_NOT_FOUND
+        # 3: ERROR_DIRECTORY_NOT_FOUND
+        # 5: ERROR_ACCESS_DENIED
+        # 21: ERROR_NOT_READY (implies drive with no media)
+        # 32: ERROR_SHARING_VIOLATION (probably an NTFS paging file)
+        # 50: ERROR_NOT_SUPPORTED (implies no support for reparse points)
+        # 67: ERROR_BAD_NET_NAME (implies remote server unavailable)
+        # 87: ERROR_INVALID_PARAMETER
+        # 4390: ERROR_NOT_A_REPARSE_POINT
+        # 4392: ERROR_INVALID_REPARSE_DATA
+        # 4393: ERROR_REPARSE_TAG_INVALID
+        allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 67, 87, 4390, 4392, 4393
+
         while normcase(path) not in seen:
             seen.add(normcase(path))
             try:
                 path = _nt_readlink(path)
             except OSError as ex:
-                # Stop on incorrect function (1), file (2) or
-                # directory (3) not found, or paths that are
-                # not reparse points (4390)
-                if ex.winerror in (1, 2, 3, 4390):
+                if ex.winerror in allowed_winerror:
                     break
                 raise
             except ValueError:
@@ -554,9 +567,20 @@ def _getfinalpathname_nonstrict(path):
         except OSError:
             pass
 
-        # Allow file (2) or directory (3) not found, incorrect parameter (87),
-        # invalid syntax (123), and symlinks that cannot be followed (1921)
-        allowed_winerror = 2, 3, 87, 123, 1921
+        # These error codes indicate that we should stop resolving the path
+        # and return the value we currently have.
+        # 1: ERROR_INVALID_FUNCTION
+        # 2: ERROR_FILE_NOT_FOUND
+        # 3: ERROR_DIRECTORY_NOT_FOUND
+        # 5: ERROR_ACCESS_DENIED
+        # 21: ERROR_NOT_READY (implies drive with no media)
+        # 32: ERROR_SHARING_VIOLATION (probably an NTFS paging file)
+        # 50: ERROR_NOT_SUPPORTED
+        # 67: ERROR_BAD_NET_NAME (implies remote server unavailable)
+        # 87: ERROR_INVALID_PARAMETER
+        # 123: ERROR_INVALID_NAME
+        # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink)
+        allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 67, 87, 123, 1921
 
         # Non-strict algorithm is to find as much of the target directory
         # as we can and join the rest.
@@ -571,6 +595,9 @@ def _getfinalpathname_nonstrict(path):
                 if ex.winerror not in allowed_winerror:
                     raise
                 path, name = split(path)
+                # TODO (bpo-38186): Request the real file name from the directory
+                # entry using FindFirstFileW. For now, we will return the path
+                # as best we have it
                 if path and not name:
                     return abspath(path + tail)
                 tail = join(name, tail) if tail else name


From webhook-mailer at python.org  Mon Sep 16 10:40:24 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 14:40:24 -0000
Subject: [Python-checkins] bpo-38117: Update bundled Windows OpenSSL to
 1.1.1d (GH-16184)
Message-ID: <mailman.779.1568644825.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/26729ce74a47c3d16f219aee0a712011622aeeb5
commit: 26729ce74a47c3d16f219aee0a712011622aeeb5
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T07:40:19-07:00
summary:

bpo-38117: Update bundled Windows OpenSSL to 1.1.1d (GH-16184)

(cherry picked from commit 3ab73f6bbf4ff66e7ace3c4eab25461b5c7d71d4)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
A Misc/NEWS.d/next/Windows/2019-09-16-14-07-11.bpo-38117.hJVf0C.rst
M PCbuild/get_externals.bat
M PCbuild/python.props

diff --git a/Misc/NEWS.d/next/Windows/2019-09-16-14-07-11.bpo-38117.hJVf0C.rst b/Misc/NEWS.d/next/Windows/2019-09-16-14-07-11.bpo-38117.hJVf0C.rst
new file mode 100644
index 000000000000..4fd8a66d279e
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-16-14-07-11.bpo-38117.hJVf0C.rst
@@ -0,0 +1 @@
+Update bundled OpenSSL to 1.1.1d
diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat
index 04c71acd469f..100149e83c16 100644
--- a/PCbuild/get_externals.bat
+++ b/PCbuild/get_externals.bat
@@ -53,7 +53,7 @@ echo.Fetching external libraries...
 set libraries=
 set libraries=%libraries%                                       bzip2-1.0.6
 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries%  libffi-3.3.0-rc0-r1
-if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries%     openssl-1.1.1c
+if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries%     openssl-1.1.1d
 set libraries=%libraries%                                       sqlite-3.28.0.0
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.9.0
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.9.0
@@ -77,7 +77,7 @@ echo.Fetching external binaries...
 
 set binaries=
 if NOT "%IncludeLibffi%"=="false"  set binaries=%binaries% libffi
-if NOT "%IncludeSSL%"=="false"     set binaries=%binaries% openssl-bin-1.1.1c
+if NOT "%IncludeSSL%"=="false"     set binaries=%binaries% openssl-bin-1.1.1d
 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.9.0
 if NOT "%IncludeSSLSrc%"=="false"  set binaries=%binaries% nasm-2.11.06
 
diff --git a/PCbuild/python.props b/PCbuild/python.props
index ba68ce334b68..8583138feb6b 100644
--- a/PCbuild/python.props
+++ b/PCbuild/python.props
@@ -62,8 +62,8 @@
     <libffiDir>$(ExternalsDir)libffi\</libffiDir>
     <libffiOutDir>$(ExternalsDir)libffi\$(ArchName)\</libffiOutDir>
     <libffiIncludeDir>$(libffiOutDir)include</libffiIncludeDir>
-    <opensslDir>$(ExternalsDir)openssl-1.1.1c\</opensslDir>
-    <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.1c\$(ArchName)\</opensslOutDir>
+    <opensslDir>$(ExternalsDir)openssl-1.1.1d\</opensslDir>
+    <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.1d\$(ArchName)\</opensslOutDir>
     <opensslIncludeDir>$(opensslOutDir)include</opensslIncludeDir>
     <nasmDir>$(ExternalsDir)\nasm-2.11.06\</nasmDir>
     <zlibDir>$(ExternalsDir)\zlib-1.2.11\</zlibDir>


From webhook-mailer at python.org  Mon Sep 16 10:43:41 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 14:43:41 -0000
Subject: [Python-checkins] bpo-38081: Add more non-fatal error codes for
 ntpath.realpath (GH-16156)
Message-ID: <mailman.780.1568645021.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4924d558478c9bd7f7ee7cd9c00c72c0f281f1a5
commit: 4924d558478c9bd7f7ee7cd9c00c72c0f281f1a5
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T07:43:36-07:00
summary:

bpo-38081: Add more non-fatal error codes for ntpath.realpath (GH-16156)

(cherry picked from commit 89b8933bb537179f81003928786c5cc6183af591)

Co-authored-by: Steve Dower <steve.dower at python.org>

files:
M Lib/ntpath.py

diff --git a/Lib/ntpath.py b/Lib/ntpath.py
index 1b5e16f95f16..01f1f423c906 100644
--- a/Lib/ntpath.py
+++ b/Lib/ntpath.py
@@ -530,15 +530,28 @@ def _readlink_deep(path, seen=None):
         if seen is None:
             seen = set()
 
+        # These error codes indicate that we should stop reading links and
+        # return the path we currently have.
+        # 1: ERROR_INVALID_FUNCTION
+        # 2: ERROR_FILE_NOT_FOUND
+        # 3: ERROR_DIRECTORY_NOT_FOUND
+        # 5: ERROR_ACCESS_DENIED
+        # 21: ERROR_NOT_READY (implies drive with no media)
+        # 32: ERROR_SHARING_VIOLATION (probably an NTFS paging file)
+        # 50: ERROR_NOT_SUPPORTED (implies no support for reparse points)
+        # 67: ERROR_BAD_NET_NAME (implies remote server unavailable)
+        # 87: ERROR_INVALID_PARAMETER
+        # 4390: ERROR_NOT_A_REPARSE_POINT
+        # 4392: ERROR_INVALID_REPARSE_DATA
+        # 4393: ERROR_REPARSE_TAG_INVALID
+        allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 67, 87, 4390, 4392, 4393
+
         while normcase(path) not in seen:
             seen.add(normcase(path))
             try:
                 path = _nt_readlink(path)
             except OSError as ex:
-                # Stop on incorrect function (1), file (2) or
-                # directory (3) not found, or paths that are
-                # not reparse points (4390)
-                if ex.winerror in (1, 2, 3, 4390):
+                if ex.winerror in allowed_winerror:
                     break
                 raise
             except ValueError:
@@ -554,9 +567,20 @@ def _getfinalpathname_nonstrict(path):
         except OSError:
             pass
 
-        # Allow file (2) or directory (3) not found, incorrect parameter (87),
-        # invalid syntax (123), and symlinks that cannot be followed (1921)
-        allowed_winerror = 2, 3, 87, 123, 1921
+        # These error codes indicate that we should stop resolving the path
+        # and return the value we currently have.
+        # 1: ERROR_INVALID_FUNCTION
+        # 2: ERROR_FILE_NOT_FOUND
+        # 3: ERROR_DIRECTORY_NOT_FOUND
+        # 5: ERROR_ACCESS_DENIED
+        # 21: ERROR_NOT_READY (implies drive with no media)
+        # 32: ERROR_SHARING_VIOLATION (probably an NTFS paging file)
+        # 50: ERROR_NOT_SUPPORTED
+        # 67: ERROR_BAD_NET_NAME (implies remote server unavailable)
+        # 87: ERROR_INVALID_PARAMETER
+        # 123: ERROR_INVALID_NAME
+        # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink)
+        allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 67, 87, 123, 1921
 
         # Non-strict algorithm is to find as much of the target directory
         # as we can and join the rest.
@@ -571,6 +595,9 @@ def _getfinalpathname_nonstrict(path):
                 if ex.winerror not in allowed_winerror:
                     raise
                 path, name = split(path)
+                # TODO (bpo-38186): Request the real file name from the directory
+                # entry using FindFirstFileW. For now, we will return the path
+                # as best we have it
                 if path and not name:
                     return abspath(path + tail)
                 tail = join(name, tail) if tail else name


From webhook-mailer at python.org  Mon Sep 16 10:55:12 2019
From: webhook-mailer at python.org (Steve Dower)
Date: Mon, 16 Sep 2019 14:55:12 -0000
Subject: [Python-checkins] bpo-38117: Update bundled Windows OpenSSL to
 1.1.1d (GH-16184)
Message-ID: <mailman.781.1568645713.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/76d8fb771659a5574604bf78eb2ee090f5c337cc
commit: 76d8fb771659a5574604bf78eb2ee090f5c337cc
branch: 3.7
author: Steve Dower <steve.dower at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-16T15:55:07+01:00
summary:

bpo-38117: Update bundled Windows OpenSSL to 1.1.1d (GH-16184)

files:
A Misc/NEWS.d/next/Windows/2019-09-16-14-07-11.bpo-38117.hJVf0C.rst
M PCbuild/get_externals.bat
M PCbuild/python.props

diff --git a/Misc/NEWS.d/next/Windows/2019-09-16-14-07-11.bpo-38117.hJVf0C.rst b/Misc/NEWS.d/next/Windows/2019-09-16-14-07-11.bpo-38117.hJVf0C.rst
new file mode 100644
index 000000000000..4fd8a66d279e
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-16-14-07-11.bpo-38117.hJVf0C.rst
@@ -0,0 +1 @@
+Update bundled OpenSSL to 1.1.1d
diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat
index 3bdad7bfec0e..83596989e4f1 100644
--- a/PCbuild/get_externals.bat
+++ b/PCbuild/get_externals.bat
@@ -49,7 +49,7 @@ echo.Fetching external libraries...
 
 set libraries=
 set libraries=%libraries%                                       bzip2-1.0.6
-if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries%     openssl-1.1.1c
+if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries%     openssl-1.1.1d
 set libraries=%libraries%                                       sqlite-3.28.0.0
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.9.0
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.9.0
@@ -72,7 +72,7 @@ for %%e in (%libraries%) do (
 echo.Fetching external binaries...
 
 set binaries=
-if NOT "%IncludeSSL%"=="false"     set binaries=%binaries% openssl-bin-1.1.1c
+if NOT "%IncludeSSL%"=="false"     set binaries=%binaries% openssl-bin-1.1.1d
 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.9.0
 if NOT "%IncludeSSLSrc%"=="false"  set binaries=%binaries% nasm-2.11.06
 
diff --git a/PCbuild/python.props b/PCbuild/python.props
index c26d482c6397..6de150be02a5 100644
--- a/PCbuild/python.props
+++ b/PCbuild/python.props
@@ -49,8 +49,8 @@
     <sqlite3Dir>$(ExternalsDir)sqlite-3.28.0.0\</sqlite3Dir>
     <bz2Dir>$(ExternalsDir)bzip2-1.0.6\</bz2Dir>
     <lzmaDir>$(ExternalsDir)xz-5.2.2\</lzmaDir>
-    <opensslDir>$(ExternalsDir)openssl-1.1.1c\</opensslDir>
-    <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.1c\$(ArchName)\</opensslOutDir>
+    <opensslDir>$(ExternalsDir)openssl-1.1.1d\</opensslDir>
+    <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.1d\$(ArchName)\</opensslOutDir>
     <opensslIncludeDir>$(opensslOutDir)include</opensslIncludeDir>
     <nasmDir>$(ExternalsDir)\nasm-2.11.06\</nasmDir>
     <zlibDir>$(ExternalsDir)\zlib-1.2.11\</zlibDir>


From webhook-mailer at python.org  Mon Sep 16 12:52:50 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 16:52:50 -0000
Subject: [Python-checkins] bpo-38100: Fix spelling error in unittest.mock
 code (GH-16168)
Message-ID: <mailman.782.1568652772.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/cc8edfbd9c66a40e2d405e49b75227a8d2a4dd55
commit: cc8edfbd9c66a40e2d405e49b75227a8d2a4dd55
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T09:52:45-07:00
summary:

bpo-38100: Fix spelling error in unittest.mock code (GH-16168)

(cherry picked from commit a9187c31185fe7ea47271839898416400cc3d976)

Co-authored-by: marcoramirezmx <55331462+marcoramirezmx at users.noreply.github.com>

files:
M Lib/unittest/mock.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index cbc4d76b8513..497aa6f9b9ba 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -1974,9 +1974,9 @@ def _set_return_value(mock, method, name):
         method.return_value = fixed
         return
 
-    return_calulator = _calculate_return_value.get(name)
-    if return_calulator is not None:
-        return_value = return_calulator(mock)
+    return_calculator = _calculate_return_value.get(name)
+    if return_calculator is not None:
+        return_value = return_calculator(mock)
         method.return_value = return_value
         return
 


From webhook-mailer at python.org  Mon Sep 16 13:15:22 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Mon, 16 Sep 2019 17:15:22 -0000
Subject: [Python-checkins] bpo-38175: Fix a memory leak in comparison of
 sqlite3.Row objects. (GH-16155)
Message-ID: <mailman.783.1568654123.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8debfa50407107ff2329d01081cdc12d359f1d12
commit: 8debfa50407107ff2329d01081cdc12d359f1d12
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T20:15:18+03:00
summary:

bpo-38175: Fix a memory leak in comparison of sqlite3.Row objects. (GH-16155)

files:
A Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst
M Lib/sqlite3/test/factory.py
M Modules/_sqlite/row.c

diff --git a/Lib/sqlite3/test/factory.py b/Lib/sqlite3/test/factory.py
index ced8445536e3..f103211ed9f1 100644
--- a/Lib/sqlite3/test/factory.py
+++ b/Lib/sqlite3/test/factory.py
@@ -169,19 +169,33 @@ def CheckSqliteRowHashCmp(self):
         row_1 = self.con.execute("select 1 as a, 2 as b").fetchone()
         row_2 = self.con.execute("select 1 as a, 2 as b").fetchone()
         row_3 = self.con.execute("select 1 as a, 3 as b").fetchone()
+        row_4 = self.con.execute("select 1 as b, 2 as a").fetchone()
+        row_5 = self.con.execute("select 2 as b, 1 as a").fetchone()
 
-        self.assertEqual(row_1, row_1)
-        self.assertEqual(row_1, row_2)
-        self.assertTrue(row_2 != row_3)
+        self.assertTrue(row_1 == row_1)
+        self.assertTrue(row_1 == row_2)
+        self.assertFalse(row_1 == row_3)
+        self.assertFalse(row_1 == row_4)
+        self.assertFalse(row_1 == row_5)
+        self.assertFalse(row_1 == object())
 
         self.assertFalse(row_1 != row_1)
         self.assertFalse(row_1 != row_2)
-        self.assertFalse(row_2 == row_3)
+        self.assertTrue(row_1 != row_3)
+        self.assertTrue(row_1 != row_4)
+        self.assertTrue(row_1 != row_5)
+        self.assertTrue(row_1 != object())
+
+        with self.assertRaises(TypeError):
+            row_1 > row_2
+        with self.assertRaises(TypeError):
+            row_1 < row_2
+        with self.assertRaises(TypeError):
+            row_1 >= row_2
+        with self.assertRaises(TypeError):
+            row_1 <= row_2
 
-        self.assertEqual(row_1, row_2)
         self.assertEqual(hash(row_1), hash(row_2))
-        self.assertNotEqual(row_1, row_3)
-        self.assertNotEqual(hash(row_1), hash(row_3))
 
     def CheckSqliteRowAsSequence(self):
         """ Checks if the row object can act like a sequence """
diff --git a/Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst b/Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst
new file mode 100644
index 000000000000..6d9f280bbd05
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst
@@ -0,0 +1 @@
+Fix a memory leak in comparison of :class:`sqlite3.Row` objects.
diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c
index 5c2f40082402..758518a8ff80 100644
--- a/Modules/_sqlite/row.c
+++ b/Modules/_sqlite/row.c
@@ -192,14 +192,16 @@ static PyObject* pysqlite_row_richcompare(pysqlite_Row *self, PyObject *_other,
     if (opid != Py_EQ && opid != Py_NE)
         Py_RETURN_NOTIMPLEMENTED;
 
-    if (PyType_IsSubtype(Py_TYPE(_other), &pysqlite_RowType)) {
+    if (PyObject_TypeCheck(_other, &pysqlite_RowType)) {
         pysqlite_Row *other = (pysqlite_Row *)_other;
-        PyObject *res = PyObject_RichCompare(self->description, other->description, opid);
-        if ((opid == Py_EQ && res == Py_True)
-            || (opid == Py_NE && res == Py_False)) {
-            Py_DECREF(res);
+        int eq = PyObject_RichCompareBool(self->description, other->description, Py_EQ);
+        if (eq < 0) {
+            return NULL;
+        }
+        if (eq) {
             return PyObject_RichCompare(self->data, other->data, opid);
         }
+        return PyBool_FromLong(opid != Py_EQ);
     }
     Py_RETURN_NOTIMPLEMENTED;
 }


From webhook-mailer at python.org  Mon Sep 16 14:16:41 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 18:16:41 -0000
Subject: [Python-checkins] [3.7] bpo-38100: Fix spelling error in
 unittest.mock code (GH-16195)
Message-ID: <mailman.784.1568657802.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7aae4f004e69ebec011098f85ab232e2778a0ca4
commit: 7aae4f004e69ebec011098f85ab232e2778a0ca4
branch: 3.7
author: St?phane Wirtel <stephane at wirtel.be>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-16T11:16:34-07:00
summary:

[3.7] bpo-38100: Fix spelling error in unittest.mock code (GH-16195)



https://bugs.python.org/issue38100



Automerge-Triggered-By: @matrixise

files:
M Lib/unittest/mock.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 1ec998ccd1a7..b1ab8631a422 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -1876,10 +1876,10 @@ def _set_return_value(mock, method, name):
         method.return_value = fixed
         return
 
-    return_calulator = _calculate_return_value.get(name)
-    if return_calulator is not None:
+    return_calculator = _calculate_return_value.get(name)
+    if return_calculator is not None:
         try:
-            return_value = return_calulator(mock)
+            return_value = return_calculator(mock)
         except AttributeError:
             # XXXX why do we return AttributeError here?
             #      set it as a side_effect instead?


From webhook-mailer at python.org  Mon Sep 16 14:55:09 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 18:55:09 -0000
Subject: [Python-checkins] Doc: Fix Wikipedia link for functools.lru_cache
 (GH-16183)
Message-ID: <mailman.785.1568660110.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/917c622b537d770a8f513607165a5cce134b1dab
commit: 917c622b537d770a8f513607165a5cce134b1dab
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T11:55:04-07:00
summary:

Doc: Fix Wikipedia link for functools.lru_cache (GH-16183)

(cherry picked from commit 336b3064d8981bc7f76c5cc6f6a0527df69771d6)

Co-authored-by: amist <amistern0 at gmail.com>

files:
M Lib/functools.py

diff --git a/Lib/functools.py b/Lib/functools.py
index a674a685df65..b41dea79083f 100644
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -499,7 +499,7 @@ def lru_cache(maxsize=128, typed=False):
     with f.cache_info().  Clear the cache and statistics with f.cache_clear().
     Access the underlying function with f.__wrapped__.
 
-    See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
+    See:  http://en.wikipedia.org/wiki/Cache_replacement_policies#Least_recently_used_(LRU)
 
     """
 


From webhook-mailer at python.org  Mon Sep 16 14:55:47 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 18:55:47 -0000
Subject: [Python-checkins] Doc: Fix Wikipedia link for functools.lru_cache
 (GH-16183)
Message-ID: <mailman.786.1568660149.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4651a7a496110525a682ae91843ff0d5fb44a435
commit: 4651a7a496110525a682ae91843ff0d5fb44a435
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T11:55:43-07:00
summary:

Doc: Fix Wikipedia link for functools.lru_cache (GH-16183)

(cherry picked from commit 336b3064d8981bc7f76c5cc6f6a0527df69771d6)

Co-authored-by: amist <amistern0 at gmail.com>

files:
M Lib/functools.py

diff --git a/Lib/functools.py b/Lib/functools.py
index 1daa1d177591..badaa826af9e 100644
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -471,7 +471,7 @@ def lru_cache(maxsize=128, typed=False):
     with f.cache_info().  Clear the cache and statistics with f.cache_clear().
     Access the underlying function with f.__wrapped__.
 
-    See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
+    See:  http://en.wikipedia.org/wiki/Cache_replacement_policies#Least_recently_used_(LRU)
 
     """
 


From webhook-mailer at python.org  Mon Sep 16 15:10:10 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 19:10:10 -0000
Subject: [Python-checkins] bpo-33936: Don't call obsolete init methods with
 OpenSSL 1.1.0+ (GH-16140)
Message-ID: <mailman.787.1568661011.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/724f1a57231f9287c37255adf0e4364d12cf693d
commit: 724f1a57231f9287c37255adf0e4364d12cf693d
branch: master
author: Christian Heimes <christian at python.org>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-16T12:10:05-07:00
summary:

bpo-33936: Don't call obsolete init methods with OpenSSL 1.1.0+ (GH-16140)



``OPENSSL_VERSION_1_1`` was never defined in ``_hashopenssl.c``. 


https://bugs.python.org/issue33936

files:
A Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst
M Modules/_hashopenssl.c

diff --git a/Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst b/Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst
new file mode 100644
index 000000000000..7bc7fed483a2
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst
@@ -0,0 +1,2 @@
+_hashlib no longer calls obsolete OpenSSL initialization function with
+OpenSSL 1.1.0+.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index b86cbd09743e..d963c7a57221 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -1138,7 +1138,7 @@ PyInit__hashlib(void)
 {
     PyObject *m, *openssl_md_meth_names;
 
-#ifndef OPENSSL_VERSION_1_1
+#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
     /* Load all digest algorithms and initialize cpuid */
     OPENSSL_add_all_algorithms_noconf();
     ERR_load_crypto_strings();


From webhook-mailer at python.org  Mon Sep 16 15:48:07 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 19:48:07 -0000
Subject: [Python-checkins] bpo-33936: Don't call obsolete init methods with
 OpenSSL 1.1.0+ (GH-16140)
Message-ID: <mailman.788.1568663288.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1ecc75ad1cdf458c2aa7192844e83c4665a5f9b5
commit: 1ecc75ad1cdf458c2aa7192844e83c4665a5f9b5
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T12:48:03-07:00
summary:

bpo-33936: Don't call obsolete init methods with OpenSSL 1.1.0+ (GH-16140)


``OPENSSL_VERSION_1_1`` was never defined in ``_hashopenssl.c``.

https://bugs.python.org/issue33936
(cherry picked from commit 724f1a57231f9287c37255adf0e4364d12cf693d)

Co-authored-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst
M Modules/_hashopenssl.c

diff --git a/Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst b/Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst
new file mode 100644
index 000000000000..7bc7fed483a2
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst
@@ -0,0 +1,2 @@
+_hashlib no longer calls obsolete OpenSSL initialization function with
+OpenSSL 1.1.0+.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 0e29fe1d3301..84906a4da4f5 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -1105,7 +1105,7 @@ PyInit__hashlib(void)
 {
     PyObject *m, *openssl_md_meth_names;
 
-#ifndef OPENSSL_VERSION_1_1
+#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
     /* Load all digest algorithms and initialize cpuid */
     OPENSSL_add_all_algorithms_noconf();
     ERR_load_crypto_strings();


From webhook-mailer at python.org  Mon Sep 16 15:48:25 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 19:48:25 -0000
Subject: [Python-checkins] bpo-33936: Don't call obsolete init methods with
 OpenSSL 1.1.0+ (GH-16140)
Message-ID: <mailman.789.1568663306.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5d55d52b61f56522e616ac1423437d4e3f641ce6
commit: 5d55d52b61f56522e616ac1423437d4e3f641ce6
branch: 2.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T12:48:21-07:00
summary:

bpo-33936: Don't call obsolete init methods with OpenSSL 1.1.0+ (GH-16140)


``OPENSSL_VERSION_1_1`` was never defined in ``_hashopenssl.c``.

https://bugs.python.org/issue33936
(cherry picked from commit 724f1a57231f9287c37255adf0e4364d12cf693d)

Co-authored-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst
M Modules/_hashopenssl.c

diff --git a/Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst b/Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst
new file mode 100644
index 000000000000..7bc7fed483a2
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst
@@ -0,0 +1,2 @@
+_hashlib no longer calls obsolete OpenSSL initialization function with
+OpenSSL 1.1.0+.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 78445ebabdd3..5df08e54b247 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -901,7 +901,7 @@ init_hashlib(void)
 {
     PyObject *m, *openssl_md_meth_names;
 
-#ifndef OPENSSL_VERSION_1_1
+#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
     /* Load all digest algorithms and initialize cpuid */
     OPENSSL_add_all_algorithms_noconf();
     ERR_load_crypto_strings();


From webhook-mailer at python.org  Mon Sep 16 15:49:11 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 19:49:11 -0000
Subject: [Python-checkins] [3.7] bpo-33936: Don't call obsolete init methods
 with OpenSSL 1.1.0+ (GH-16140) (GH-16200)
Message-ID: <mailman.790.1568663352.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c55f69552cdab26362c1ff4416fe1c3e9b27ccbe
commit: c55f69552cdab26362c1ff4416fe1c3e9b27ccbe
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T12:49:07-07:00
summary:

[3.7] bpo-33936: Don't call obsolete init methods with OpenSSL 1.1.0+ (GH-16140) (GH-16200)



``OPENSSL_VERSION_1_1`` was never defined in ``_hashopenssl.c``. 


https://bugs.python.org/issue33936
(cherry picked from commit 724f1a57231f9287c37255adf0e4364d12cf693d)


Co-authored-by: Christian Heimes <christian at python.org>


https://bugs.python.org/issue33936



Automerge-Triggered-By: @tiran

files:
A Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst
M Modules/_hashopenssl.c

diff --git a/Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst b/Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst
new file mode 100644
index 000000000000..7bc7fed483a2
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-14-10-34-00.bpo-33936.8wCI_n.rst
@@ -0,0 +1,2 @@
+_hashlib no longer calls obsolete OpenSSL initialization function with
+OpenSSL 1.1.0+.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index b69f16c61a50..b13ade604961 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -1073,7 +1073,7 @@ PyInit__hashlib(void)
 {
     PyObject *m, *openssl_md_meth_names;
 
-#ifndef OPENSSL_VERSION_1_1
+#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
     /* Load all digest algorithms and initialize cpuid */
     OPENSSL_add_all_algorithms_noconf();
     ERR_load_crypto_strings();


From webhook-mailer at python.org  Mon Sep 16 18:04:13 2019
From: webhook-mailer at python.org (Ivan Levkivskyi)
Date: Mon, 16 Sep 2019 22:04:13 -0000
Subject: [Python-checkins] bpo-28556: Update the opening note in typing docs
 (GH-16204)
Message-ID: <mailman.791.1568671454.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/81528ba2e81c39f4d6bca5b785e818c7d08b8501
commit: 81528ba2e81c39f4d6bca5b785e818c7d08b8501
branch: master
author: Ivan Levkivskyi <levkivskyi at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T23:04:06+01:00
summary:

bpo-28556: Update the opening note in typing docs (GH-16204)

This PR replaces the old note mentioning that `typing` is a provisional module with a new one mentioning types are not enforced at runtime. I am not sure if there was any official announcement about making `typing` non-provisional, but _de-facto_ no new features were added during Python 3.7, and no backwards incompatible changes were made except for few small things that were considered bugs.

files:
M Doc/library/typing.rst

diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst
index 12efde131657..f4d1b2404328 100644
--- a/Doc/library/typing.rst
+++ b/Doc/library/typing.rst
@@ -10,10 +10,9 @@
 
 .. note::
 
-   The typing module has been included in the standard library on a
-   :term:`provisional basis <provisional api>`. New features might
-   be added and API may change even between minor releases if deemed
-   necessary by the core developers.
+   The Python runtime does not enforce function and variable type annotations.
+   They can be used by third party tools such as type checkers, IDEs, linters,
+   etc.
 
 --------------
 


From webhook-mailer at python.org  Mon Sep 16 18:10:15 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 22:10:15 -0000
Subject: [Python-checkins] bpo-28556: Update the opening note in typing docs
 (GH-16204)
Message-ID: <mailman.792.1568671816.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/733193d050f317a7965faac6c3718e6e8316a8a0
commit: 733193d050f317a7965faac6c3718e6e8316a8a0
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T15:10:11-07:00
summary:

bpo-28556: Update the opening note in typing docs (GH-16204)


This PR replaces the old note mentioning that `typing` is a provisional module with a new one mentioning types are not enforced at runtime. I am not sure if there was any official announcement about making `typing` non-provisional, but _de-facto_ no new features were added during Python 3.7, and no backwards incompatible changes were made except for few small things that were considered bugs.
(cherry picked from commit 81528ba2e81c39f4d6bca5b785e818c7d08b8501)

Co-authored-by: Ivan Levkivskyi <levkivskyi at gmail.com>

files:
M Doc/library/typing.rst

diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst
index c0b048cb2bd3..627c1f77c157 100644
--- a/Doc/library/typing.rst
+++ b/Doc/library/typing.rst
@@ -10,10 +10,9 @@
 
 .. note::
 
-   The typing module has been included in the standard library on a
-   :term:`provisional basis <provisional api>`. New features might
-   be added and API may change even between minor releases if deemed
-   necessary by the core developers.
+   The Python runtime does not enforce function and variable type annotations.
+   They can be used by third party tools such as type checkers, IDEs, linters,
+   etc.
 
 --------------
 


From webhook-mailer at python.org  Mon Sep 16 18:11:14 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 16 Sep 2019 22:11:14 -0000
Subject: [Python-checkins] bpo-28556: Update the opening note in typing docs
 (GH-16204)
Message-ID: <mailman.793.1568671875.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7076764992cd29e0e1f8b0ac2b92403e1a698aa6
commit: 7076764992cd29e0e1f8b0ac2b92403e1a698aa6
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T15:11:10-07:00
summary:

bpo-28556: Update the opening note in typing docs (GH-16204)


This PR replaces the old note mentioning that `typing` is a provisional module with a new one mentioning types are not enforced at runtime. I am not sure if there was any official announcement about making `typing` non-provisional, but _de-facto_ no new features were added during Python 3.7, and no backwards incompatible changes were made except for few small things that were considered bugs.
(cherry picked from commit 81528ba2e81c39f4d6bca5b785e818c7d08b8501)

Co-authored-by: Ivan Levkivskyi <levkivskyi at gmail.com>

files:
M Doc/library/typing.rst

diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst
index 12efde131657..f4d1b2404328 100644
--- a/Doc/library/typing.rst
+++ b/Doc/library/typing.rst
@@ -10,10 +10,9 @@
 
 .. note::
 
-   The typing module has been included in the standard library on a
-   :term:`provisional basis <provisional api>`. New features might
-   be added and API may change even between minor releases if deemed
-   necessary by the core developers.
+   The Python runtime does not enforce function and variable type annotations.
+   They can be used by third party tools such as type checkers, IDEs, linters,
+   etc.
 
 --------------
 


From webhook-mailer at python.org  Mon Sep 16 19:04:25 2019
From: webhook-mailer at python.org (Terry Jan Reedy)
Date: Mon, 16 Sep 2019 23:04:25 -0000
Subject: [Python-checkins] bpo-38183: Test_idle ignores user config
 directory GH-16198)
Message-ID: <mailman.794.1568675065.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0048afc16a7e7301d5c565237db271505e5fbed9
commit: 0048afc16a7e7301d5c565237db271505e5fbed9
branch: master
author: Terry Jan Reedy <tjreedy at udel.edu>
committer: GitHub <noreply at github.com>
date: 2019-09-16T19:04:21-04:00
summary:

bpo-38183: Test_idle ignores user config directory GH-16198)

It no longer tries to create or access .idlerc or any files within.
Users must run IDLE to discover problems with saving settings.

files:
A Misc/NEWS.d/next/IDLE/2019-09-16-15-04-29.bpo-38183.eudCN1.rst
M Lib/idlelib/config.py
M Lib/idlelib/editor.py
M Lib/idlelib/idle_test/test_config.py
M Lib/idlelib/pyshell.py

diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py
index 683b000a488b..12e6f9fae78b 100644
--- a/Lib/idlelib/config.py
+++ b/Lib/idlelib/config.py
@@ -130,7 +130,7 @@ def Save(self):
         to disk. Otherwise, remove the file from disk if it exists.
         """
         fname = self.file
-        if fname:
+        if fname and fname[0] != '#':
             if not self.IsEmpty():
                 try:
                     cfgFile = open(fname, 'w')
@@ -166,12 +166,12 @@ def __init__(self, _utest=False):
     def CreateConfigHandlers(self):
         "Populate default and user config parser dictionaries."
         idledir = os.path.dirname(__file__)
-        self.userdir = userdir = self.GetUserCfgDir()
+        self.userdir = userdir = '' if idlelib.testing else self.GetUserCfgDir()
         for cfg_type in self.config_types:
             self.defaultCfg[cfg_type] = IdleConfParser(
                 os.path.join(idledir, f'config-{cfg_type}.def'))
             self.userCfg[cfg_type] = IdleUserConfParser(
-                os.path.join(userdir, f'config-{cfg_type}.cfg'))
+                os.path.join(userdir or '#', f'config-{cfg_type}.cfg'))
 
     def GetUserCfgDir(self):
         """Return a filesystem directory for storing user config files.
@@ -182,12 +182,13 @@ def GetUserCfgDir(self):
         userDir = os.path.expanduser('~')
         if userDir != '~': # expanduser() found user home dir
             if not os.path.exists(userDir):
-                warn = ('\n Warning: os.path.expanduser("~") points to\n ' +
-                        userDir + ',\n but the path does not exist.')
-                try:
-                    print(warn, file=sys.stderr)
-                except OSError:
-                    pass
+                if not idlelib.testing:
+                    warn = ('\n Warning: os.path.expanduser("~") points to\n ' +
+                            userDir + ',\n but the path does not exist.')
+                    try:
+                        print(warn, file=sys.stderr)
+                    except OSError:
+                        pass
                 userDir = '~'
         if userDir == "~": # still no path to home!
             # traditionally IDLE has defaulted to os.getcwd(), is this adequate?
@@ -197,10 +198,13 @@ def GetUserCfgDir(self):
             try:
                 os.mkdir(userDir)
             except OSError:
-                warn = ('\n Warning: unable to create user config directory\n' +
-                        userDir + '\n Check path and permissions.\n Exiting!\n')
                 if not idlelib.testing:
-                    print(warn, file=sys.stderr)
+                    warn = ('\n Warning: unable to create user config directory\n' +
+                            userDir + '\n Check path and permissions.\n Exiting!\n')
+                    try:
+                        print(warn, file=sys.stderr)
+                    except OSError:
+                        pass
                 raise SystemExit
         # TODO continue without userDIr instead of exit
         return userDir
diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py
index 5cbf704ab27b..838cb04b8008 100644
--- a/Lib/idlelib/editor.py
+++ b/Lib/idlelib/editor.py
@@ -116,7 +116,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None):
             self.tkinter_vars = {}  # keys: Tkinter event names
                                     # values: Tkinter variable instances
             self.top.instance_dict = {}
-        self.recent_files_path = os.path.join(
+        self.recent_files_path = idleConf.userdir and os.path.join(
                 idleConf.userdir, 'recent-files.lst')
 
         self.prompt_last_line = ''  # Override in PyShell
@@ -924,9 +924,11 @@ def display_extra_help(helpfile=helpfile):
 
     def update_recent_files_list(self, new_file=None):
         "Load and update the recent files list and menus"
+        # TODO: move to iomenu.
         rf_list = []
-        if os.path.exists(self.recent_files_path):
-            with open(self.recent_files_path, 'r',
+        file_path = self.recent_files_path
+        if file_path and os.path.exists(file_path):
+            with open(file_path, 'r',
                       encoding='utf_8', errors='replace') as rf_list_file:
                 rf_list = rf_list_file.readlines()
         if new_file:
@@ -942,19 +944,19 @@ def update_recent_files_list(self, new_file=None):
         rf_list = [path for path in rf_list if path not in bad_paths]
         ulchars = "1234567890ABCDEFGHIJK"
         rf_list = rf_list[0:len(ulchars)]
-        try:
-            with open(self.recent_files_path, 'w',
-                        encoding='utf_8', errors='replace') as rf_file:
-                rf_file.writelines(rf_list)
-        except OSError as err:
-            if not getattr(self.root, "recentfilelist_error_displayed", False):
-                self.root.recentfilelist_error_displayed = True
-                tkMessageBox.showwarning(title='IDLE Warning',
-                    message="Cannot update File menu Recent Files list. "
-                            "Your operating system says:\n%s\n"
-                            "Select OK and IDLE will continue without updating."
-                        % self._filename_to_unicode(str(err)),
-                    parent=self.text)
+        if file_path:
+            try:
+                with open(file_path, 'w',
+                          encoding='utf_8', errors='replace') as rf_file:
+                    rf_file.writelines(rf_list)
+            except OSError as err:
+                if not getattr(self.root, "recentfiles_message", False):
+                    self.root.recentfiles_message = True
+                    tkMessageBox.showwarning(title='IDLE Warning',
+                        message="Cannot save Recent Files list to disk.\n"
+                                f"  {err}\n"
+                                "Select OK to continue.",
+                        parent=self.text)
         # for each edit window instance, construct the recent files menu
         for instance in self.top.instance_dict:
             menu = instance.recent_files_menu
diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py
index 492f2f648924..697fda527968 100644
--- a/Lib/idlelib/idle_test/test_config.py
+++ b/Lib/idlelib/idle_test/test_config.py
@@ -220,7 +220,7 @@ def mock_config(self):
 
     @unittest.skipIf(sys.platform.startswith('win'), 'this is test for unix system')
     def test_get_user_cfg_dir_unix(self):
-        "Test to get user config directory under unix"
+        # Test to get user config directory under unix.
         conf = self.new_config(_utest=True)
 
         # Check normal way should success
@@ -243,7 +243,7 @@ def test_get_user_cfg_dir_unix(self):
 
     @unittest.skipIf(not sys.platform.startswith('win'), 'this is test for Windows system')
     def test_get_user_cfg_dir_windows(self):
-        "Test to get user config directory under Windows"
+        # Test to get user config directory under Windows.
         conf = self.new_config(_utest=True)
 
         # Check normal way should success
@@ -284,12 +284,12 @@ def test_create_config_handlers(self):
             self.assertIsInstance(user_parser, config.IdleUserConfParser)
 
         # Check config path are correct
-        for config_type, parser in conf.defaultCfg.items():
+        for cfg_type, parser in conf.defaultCfg.items():
             self.assertEqual(parser.file,
-                             os.path.join(idle_dir, 'config-%s.def' % config_type))
-        for config_type, parser in conf.userCfg.items():
+                             os.path.join(idle_dir, f'config-{cfg_type}.def'))
+        for cfg_type, parser in conf.userCfg.items():
             self.assertEqual(parser.file,
-                             os.path.join(conf.userdir, 'config-%s.cfg' % config_type))
+                             os.path.join(conf.userdir or '#', f'config-{cfg_type}.cfg'))
 
     def test_load_cfg_files(self):
         conf = self.new_config(_utest=True)
@@ -373,7 +373,7 @@ def test_get_highlight(self):
                                                        'background': '#171717'})
 
     def test_get_theme_dict(self):
-        "XXX: NOT YET DONE"
+        # TODO: finish.
         conf = self.mock_config()
 
         # These two should be the same
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index 08716a9733b7..2e4dfad40122 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -133,6 +133,7 @@ def __init__(self, *args):
         self.text.bind("<<clear-breakpoint-here>>", self.clear_breakpoint_here)
         self.text.bind("<<open-python-shell>>", self.flist.open_shell)
 
+        #TODO: don't read/write this from/to .idlerc when testing
         self.breakpointPath = os.path.join(
                 idleConf.userdir, 'breakpoints.lst')
         # whenever a file is changed, restore breakpoints
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-16-15-04-29.bpo-38183.eudCN1.rst b/Misc/NEWS.d/next/IDLE/2019-09-16-15-04-29.bpo-38183.eudCN1.rst
new file mode 100644
index 000000000000..e545f49bc836
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-16-15-04-29.bpo-38183.eudCN1.rst
@@ -0,0 +1,3 @@
+To avoid problems, test_idle ignores the user config directory.
+It no longer tries to create or access .idlerc or any files within.
+Users must run IDLE to discover problems with saving settings.


From webhook-mailer at python.org  Mon Sep 16 23:33:08 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 03:33:08 -0000
Subject: [Python-checkins] bpo-38183: Test_idle ignores user config
 directory GH-16198)
Message-ID: <mailman.795.1568691189.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/ad845becf29afe0507a46a4b885a9670b5d7639d
commit: ad845becf29afe0507a46a4b885a9670b5d7639d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T20:32:55-07:00
summary:

bpo-38183: Test_idle ignores user config directory GH-16198)


It no longer tries to create or access .idlerc or any files within.
Users must run IDLE to discover problems with saving settings.
(cherry picked from commit 0048afc16a7e7301d5c565237db271505e5fbed9)

Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu>

files:
A Misc/NEWS.d/next/IDLE/2019-09-16-15-04-29.bpo-38183.eudCN1.rst
M Lib/idlelib/config.py
M Lib/idlelib/editor.py
M Lib/idlelib/idle_test/test_config.py
M Lib/idlelib/pyshell.py

diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py
index 683b000a488b..12e6f9fae78b 100644
--- a/Lib/idlelib/config.py
+++ b/Lib/idlelib/config.py
@@ -130,7 +130,7 @@ def Save(self):
         to disk. Otherwise, remove the file from disk if it exists.
         """
         fname = self.file
-        if fname:
+        if fname and fname[0] != '#':
             if not self.IsEmpty():
                 try:
                     cfgFile = open(fname, 'w')
@@ -166,12 +166,12 @@ def __init__(self, _utest=False):
     def CreateConfigHandlers(self):
         "Populate default and user config parser dictionaries."
         idledir = os.path.dirname(__file__)
-        self.userdir = userdir = self.GetUserCfgDir()
+        self.userdir = userdir = '' if idlelib.testing else self.GetUserCfgDir()
         for cfg_type in self.config_types:
             self.defaultCfg[cfg_type] = IdleConfParser(
                 os.path.join(idledir, f'config-{cfg_type}.def'))
             self.userCfg[cfg_type] = IdleUserConfParser(
-                os.path.join(userdir, f'config-{cfg_type}.cfg'))
+                os.path.join(userdir or '#', f'config-{cfg_type}.cfg'))
 
     def GetUserCfgDir(self):
         """Return a filesystem directory for storing user config files.
@@ -182,12 +182,13 @@ def GetUserCfgDir(self):
         userDir = os.path.expanduser('~')
         if userDir != '~': # expanduser() found user home dir
             if not os.path.exists(userDir):
-                warn = ('\n Warning: os.path.expanduser("~") points to\n ' +
-                        userDir + ',\n but the path does not exist.')
-                try:
-                    print(warn, file=sys.stderr)
-                except OSError:
-                    pass
+                if not idlelib.testing:
+                    warn = ('\n Warning: os.path.expanduser("~") points to\n ' +
+                            userDir + ',\n but the path does not exist.')
+                    try:
+                        print(warn, file=sys.stderr)
+                    except OSError:
+                        pass
                 userDir = '~'
         if userDir == "~": # still no path to home!
             # traditionally IDLE has defaulted to os.getcwd(), is this adequate?
@@ -197,10 +198,13 @@ def GetUserCfgDir(self):
             try:
                 os.mkdir(userDir)
             except OSError:
-                warn = ('\n Warning: unable to create user config directory\n' +
-                        userDir + '\n Check path and permissions.\n Exiting!\n')
                 if not idlelib.testing:
-                    print(warn, file=sys.stderr)
+                    warn = ('\n Warning: unable to create user config directory\n' +
+                            userDir + '\n Check path and permissions.\n Exiting!\n')
+                    try:
+                        print(warn, file=sys.stderr)
+                    except OSError:
+                        pass
                 raise SystemExit
         # TODO continue without userDIr instead of exit
         return userDir
diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py
index 5cbf704ab27b..838cb04b8008 100644
--- a/Lib/idlelib/editor.py
+++ b/Lib/idlelib/editor.py
@@ -116,7 +116,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None):
             self.tkinter_vars = {}  # keys: Tkinter event names
                                     # values: Tkinter variable instances
             self.top.instance_dict = {}
-        self.recent_files_path = os.path.join(
+        self.recent_files_path = idleConf.userdir and os.path.join(
                 idleConf.userdir, 'recent-files.lst')
 
         self.prompt_last_line = ''  # Override in PyShell
@@ -924,9 +924,11 @@ def display_extra_help(helpfile=helpfile):
 
     def update_recent_files_list(self, new_file=None):
         "Load and update the recent files list and menus"
+        # TODO: move to iomenu.
         rf_list = []
-        if os.path.exists(self.recent_files_path):
-            with open(self.recent_files_path, 'r',
+        file_path = self.recent_files_path
+        if file_path and os.path.exists(file_path):
+            with open(file_path, 'r',
                       encoding='utf_8', errors='replace') as rf_list_file:
                 rf_list = rf_list_file.readlines()
         if new_file:
@@ -942,19 +944,19 @@ def update_recent_files_list(self, new_file=None):
         rf_list = [path for path in rf_list if path not in bad_paths]
         ulchars = "1234567890ABCDEFGHIJK"
         rf_list = rf_list[0:len(ulchars)]
-        try:
-            with open(self.recent_files_path, 'w',
-                        encoding='utf_8', errors='replace') as rf_file:
-                rf_file.writelines(rf_list)
-        except OSError as err:
-            if not getattr(self.root, "recentfilelist_error_displayed", False):
-                self.root.recentfilelist_error_displayed = True
-                tkMessageBox.showwarning(title='IDLE Warning',
-                    message="Cannot update File menu Recent Files list. "
-                            "Your operating system says:\n%s\n"
-                            "Select OK and IDLE will continue without updating."
-                        % self._filename_to_unicode(str(err)),
-                    parent=self.text)
+        if file_path:
+            try:
+                with open(file_path, 'w',
+                          encoding='utf_8', errors='replace') as rf_file:
+                    rf_file.writelines(rf_list)
+            except OSError as err:
+                if not getattr(self.root, "recentfiles_message", False):
+                    self.root.recentfiles_message = True
+                    tkMessageBox.showwarning(title='IDLE Warning',
+                        message="Cannot save Recent Files list to disk.\n"
+                                f"  {err}\n"
+                                "Select OK to continue.",
+                        parent=self.text)
         # for each edit window instance, construct the recent files menu
         for instance in self.top.instance_dict:
             menu = instance.recent_files_menu
diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py
index 492f2f648924..697fda527968 100644
--- a/Lib/idlelib/idle_test/test_config.py
+++ b/Lib/idlelib/idle_test/test_config.py
@@ -220,7 +220,7 @@ def mock_config(self):
 
     @unittest.skipIf(sys.platform.startswith('win'), 'this is test for unix system')
     def test_get_user_cfg_dir_unix(self):
-        "Test to get user config directory under unix"
+        # Test to get user config directory under unix.
         conf = self.new_config(_utest=True)
 
         # Check normal way should success
@@ -243,7 +243,7 @@ def test_get_user_cfg_dir_unix(self):
 
     @unittest.skipIf(not sys.platform.startswith('win'), 'this is test for Windows system')
     def test_get_user_cfg_dir_windows(self):
-        "Test to get user config directory under Windows"
+        # Test to get user config directory under Windows.
         conf = self.new_config(_utest=True)
 
         # Check normal way should success
@@ -284,12 +284,12 @@ def test_create_config_handlers(self):
             self.assertIsInstance(user_parser, config.IdleUserConfParser)
 
         # Check config path are correct
-        for config_type, parser in conf.defaultCfg.items():
+        for cfg_type, parser in conf.defaultCfg.items():
             self.assertEqual(parser.file,
-                             os.path.join(idle_dir, 'config-%s.def' % config_type))
-        for config_type, parser in conf.userCfg.items():
+                             os.path.join(idle_dir, f'config-{cfg_type}.def'))
+        for cfg_type, parser in conf.userCfg.items():
             self.assertEqual(parser.file,
-                             os.path.join(conf.userdir, 'config-%s.cfg' % config_type))
+                             os.path.join(conf.userdir or '#', f'config-{cfg_type}.cfg'))
 
     def test_load_cfg_files(self):
         conf = self.new_config(_utest=True)
@@ -373,7 +373,7 @@ def test_get_highlight(self):
                                                        'background': '#171717'})
 
     def test_get_theme_dict(self):
-        "XXX: NOT YET DONE"
+        # TODO: finish.
         conf = self.mock_config()
 
         # These two should be the same
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index 08716a9733b7..2e4dfad40122 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -133,6 +133,7 @@ def __init__(self, *args):
         self.text.bind("<<clear-breakpoint-here>>", self.clear_breakpoint_here)
         self.text.bind("<<open-python-shell>>", self.flist.open_shell)
 
+        #TODO: don't read/write this from/to .idlerc when testing
         self.breakpointPath = os.path.join(
                 idleConf.userdir, 'breakpoints.lst')
         # whenever a file is changed, restore breakpoints
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-16-15-04-29.bpo-38183.eudCN1.rst b/Misc/NEWS.d/next/IDLE/2019-09-16-15-04-29.bpo-38183.eudCN1.rst
new file mode 100644
index 000000000000..e545f49bc836
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-16-15-04-29.bpo-38183.eudCN1.rst
@@ -0,0 +1,3 @@
+To avoid problems, test_idle ignores the user config directory.
+It no longer tries to create or access .idlerc or any files within.
+Users must run IDLE to discover problems with saving settings.


From webhook-mailer at python.org  Mon Sep 16 23:33:11 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 03:33:11 -0000
Subject: [Python-checkins] bpo-38183: Test_idle ignores user config
 directory GH-16198)
Message-ID: <mailman.796.1568691192.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/523497cbdc8292bfc12938aaaca0ad02a7708c25
commit: 523497cbdc8292bfc12938aaaca0ad02a7708c25
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T20:33:06-07:00
summary:

bpo-38183: Test_idle ignores user config directory GH-16198)


It no longer tries to create or access .idlerc or any files within.
Users must run IDLE to discover problems with saving settings.
(cherry picked from commit 0048afc16a7e7301d5c565237db271505e5fbed9)

Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu>

files:
A Misc/NEWS.d/next/IDLE/2019-09-16-15-04-29.bpo-38183.eudCN1.rst
M Lib/idlelib/config.py
M Lib/idlelib/editor.py
M Lib/idlelib/idle_test/test_config.py
M Lib/idlelib/pyshell.py

diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py
index 683b000a488b..12e6f9fae78b 100644
--- a/Lib/idlelib/config.py
+++ b/Lib/idlelib/config.py
@@ -130,7 +130,7 @@ def Save(self):
         to disk. Otherwise, remove the file from disk if it exists.
         """
         fname = self.file
-        if fname:
+        if fname and fname[0] != '#':
             if not self.IsEmpty():
                 try:
                     cfgFile = open(fname, 'w')
@@ -166,12 +166,12 @@ def __init__(self, _utest=False):
     def CreateConfigHandlers(self):
         "Populate default and user config parser dictionaries."
         idledir = os.path.dirname(__file__)
-        self.userdir = userdir = self.GetUserCfgDir()
+        self.userdir = userdir = '' if idlelib.testing else self.GetUserCfgDir()
         for cfg_type in self.config_types:
             self.defaultCfg[cfg_type] = IdleConfParser(
                 os.path.join(idledir, f'config-{cfg_type}.def'))
             self.userCfg[cfg_type] = IdleUserConfParser(
-                os.path.join(userdir, f'config-{cfg_type}.cfg'))
+                os.path.join(userdir or '#', f'config-{cfg_type}.cfg'))
 
     def GetUserCfgDir(self):
         """Return a filesystem directory for storing user config files.
@@ -182,12 +182,13 @@ def GetUserCfgDir(self):
         userDir = os.path.expanduser('~')
         if userDir != '~': # expanduser() found user home dir
             if not os.path.exists(userDir):
-                warn = ('\n Warning: os.path.expanduser("~") points to\n ' +
-                        userDir + ',\n but the path does not exist.')
-                try:
-                    print(warn, file=sys.stderr)
-                except OSError:
-                    pass
+                if not idlelib.testing:
+                    warn = ('\n Warning: os.path.expanduser("~") points to\n ' +
+                            userDir + ',\n but the path does not exist.')
+                    try:
+                        print(warn, file=sys.stderr)
+                    except OSError:
+                        pass
                 userDir = '~'
         if userDir == "~": # still no path to home!
             # traditionally IDLE has defaulted to os.getcwd(), is this adequate?
@@ -197,10 +198,13 @@ def GetUserCfgDir(self):
             try:
                 os.mkdir(userDir)
             except OSError:
-                warn = ('\n Warning: unable to create user config directory\n' +
-                        userDir + '\n Check path and permissions.\n Exiting!\n')
                 if not idlelib.testing:
-                    print(warn, file=sys.stderr)
+                    warn = ('\n Warning: unable to create user config directory\n' +
+                            userDir + '\n Check path and permissions.\n Exiting!\n')
+                    try:
+                        print(warn, file=sys.stderr)
+                    except OSError:
+                        pass
                 raise SystemExit
         # TODO continue without userDIr instead of exit
         return userDir
diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py
index 5cbf704ab27b..838cb04b8008 100644
--- a/Lib/idlelib/editor.py
+++ b/Lib/idlelib/editor.py
@@ -116,7 +116,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None):
             self.tkinter_vars = {}  # keys: Tkinter event names
                                     # values: Tkinter variable instances
             self.top.instance_dict = {}
-        self.recent_files_path = os.path.join(
+        self.recent_files_path = idleConf.userdir and os.path.join(
                 idleConf.userdir, 'recent-files.lst')
 
         self.prompt_last_line = ''  # Override in PyShell
@@ -924,9 +924,11 @@ def display_extra_help(helpfile=helpfile):
 
     def update_recent_files_list(self, new_file=None):
         "Load and update the recent files list and menus"
+        # TODO: move to iomenu.
         rf_list = []
-        if os.path.exists(self.recent_files_path):
-            with open(self.recent_files_path, 'r',
+        file_path = self.recent_files_path
+        if file_path and os.path.exists(file_path):
+            with open(file_path, 'r',
                       encoding='utf_8', errors='replace') as rf_list_file:
                 rf_list = rf_list_file.readlines()
         if new_file:
@@ -942,19 +944,19 @@ def update_recent_files_list(self, new_file=None):
         rf_list = [path for path in rf_list if path not in bad_paths]
         ulchars = "1234567890ABCDEFGHIJK"
         rf_list = rf_list[0:len(ulchars)]
-        try:
-            with open(self.recent_files_path, 'w',
-                        encoding='utf_8', errors='replace') as rf_file:
-                rf_file.writelines(rf_list)
-        except OSError as err:
-            if not getattr(self.root, "recentfilelist_error_displayed", False):
-                self.root.recentfilelist_error_displayed = True
-                tkMessageBox.showwarning(title='IDLE Warning',
-                    message="Cannot update File menu Recent Files list. "
-                            "Your operating system says:\n%s\n"
-                            "Select OK and IDLE will continue without updating."
-                        % self._filename_to_unicode(str(err)),
-                    parent=self.text)
+        if file_path:
+            try:
+                with open(file_path, 'w',
+                          encoding='utf_8', errors='replace') as rf_file:
+                    rf_file.writelines(rf_list)
+            except OSError as err:
+                if not getattr(self.root, "recentfiles_message", False):
+                    self.root.recentfiles_message = True
+                    tkMessageBox.showwarning(title='IDLE Warning',
+                        message="Cannot save Recent Files list to disk.\n"
+                                f"  {err}\n"
+                                "Select OK to continue.",
+                        parent=self.text)
         # for each edit window instance, construct the recent files menu
         for instance in self.top.instance_dict:
             menu = instance.recent_files_menu
diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py
index 492f2f648924..697fda527968 100644
--- a/Lib/idlelib/idle_test/test_config.py
+++ b/Lib/idlelib/idle_test/test_config.py
@@ -220,7 +220,7 @@ def mock_config(self):
 
     @unittest.skipIf(sys.platform.startswith('win'), 'this is test for unix system')
     def test_get_user_cfg_dir_unix(self):
-        "Test to get user config directory under unix"
+        # Test to get user config directory under unix.
         conf = self.new_config(_utest=True)
 
         # Check normal way should success
@@ -243,7 +243,7 @@ def test_get_user_cfg_dir_unix(self):
 
     @unittest.skipIf(not sys.platform.startswith('win'), 'this is test for Windows system')
     def test_get_user_cfg_dir_windows(self):
-        "Test to get user config directory under Windows"
+        # Test to get user config directory under Windows.
         conf = self.new_config(_utest=True)
 
         # Check normal way should success
@@ -284,12 +284,12 @@ def test_create_config_handlers(self):
             self.assertIsInstance(user_parser, config.IdleUserConfParser)
 
         # Check config path are correct
-        for config_type, parser in conf.defaultCfg.items():
+        for cfg_type, parser in conf.defaultCfg.items():
             self.assertEqual(parser.file,
-                             os.path.join(idle_dir, 'config-%s.def' % config_type))
-        for config_type, parser in conf.userCfg.items():
+                             os.path.join(idle_dir, f'config-{cfg_type}.def'))
+        for cfg_type, parser in conf.userCfg.items():
             self.assertEqual(parser.file,
-                             os.path.join(conf.userdir, 'config-%s.cfg' % config_type))
+                             os.path.join(conf.userdir or '#', f'config-{cfg_type}.cfg'))
 
     def test_load_cfg_files(self):
         conf = self.new_config(_utest=True)
@@ -373,7 +373,7 @@ def test_get_highlight(self):
                                                        'background': '#171717'})
 
     def test_get_theme_dict(self):
-        "XXX: NOT YET DONE"
+        # TODO: finish.
         conf = self.mock_config()
 
         # These two should be the same
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index 08716a9733b7..2e4dfad40122 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -133,6 +133,7 @@ def __init__(self, *args):
         self.text.bind("<<clear-breakpoint-here>>", self.clear_breakpoint_here)
         self.text.bind("<<open-python-shell>>", self.flist.open_shell)
 
+        #TODO: don't read/write this from/to .idlerc when testing
         self.breakpointPath = os.path.join(
                 idleConf.userdir, 'breakpoints.lst')
         # whenever a file is changed, restore breakpoints
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-16-15-04-29.bpo-38183.eudCN1.rst b/Misc/NEWS.d/next/IDLE/2019-09-16-15-04-29.bpo-38183.eudCN1.rst
new file mode 100644
index 000000000000..e545f49bc836
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-16-15-04-29.bpo-38183.eudCN1.rst
@@ -0,0 +1,3 @@
+To avoid problems, test_idle ignores the user config directory.
+It no longer tries to create or access .idlerc or any files within.
+Users must run IDLE to discover problems with saving settings.


From webhook-mailer at python.org  Tue Sep 17 02:05:08 2019
From: webhook-mailer at python.org (Terry Jan Reedy)
Date: Tue, 17 Sep 2019 06:05:08 -0000
Subject: [Python-checkins] bpo-35379: When exiting IDLE,
 catch any AttributeError. (GH-16212)
Message-ID: <mailman.797.1568700308.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/dfd34a9cd58e8150c324190f746de919e140abe8
commit: dfd34a9cd58e8150c324190f746de919e140abe8
branch: master
author: Terry Jan Reedy <tjreedy at udel.edu>
committer: GitHub <noreply at github.com>
date: 2019-09-17T02:05:04-04:00
summary:

bpo-35379: When exiting IDLE, catch any AttributeError. (GH-16212)

One happens when EditorWindow.close is called twice.
Printing a traceback, when IDLE is run from a terminal,
is useless and annoying.

files:
A Misc/NEWS.d/next/IDLE/2019-09-17-01-28-56.bpo-35379.yAECDr.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/editor.py

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 3ccc5c07fb00..9d26bda42594 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -2,6 +2,13 @@ What's New in IDLE 3.8.0 (since 3.7.0)
 Released on 2019-10-20?
 ======================================
 
+bpo-35379: When exiting IDLE, catch any AttributeError.  One happens
+when EditorWindow.close is called twice.  Printing a traceback, when
+IDLE is run from a terminal, is useless and annoying.
+
+bpo-38183: To avoid test issues, test_idle ignores the user config
+directory.  It no longer tries to create or access .idlerc or any files
+within.  Users must run IDLE to discover problems with saving settings.
 
 bpo-38077: IDLE no longer adds 'argv' to the user namespace when
 initializing it.  This bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4.
diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py
index 838cb04b8008..b969f8c493b2 100644
--- a/Lib/idlelib/editor.py
+++ b/Lib/idlelib/editor.py
@@ -1061,10 +1061,13 @@ def maybesave(self):
             return self.io.maybesave()
 
     def close(self):
-        reply = self.maybesave()
-        if str(reply) != "cancel":
-            self._close()
-        return reply
+        try:
+            reply = self.maybesave()
+            if str(reply) != "cancel":
+                self._close()
+            return reply
+        except AttributeError:  # bpo-35379: close called twice
+            pass
 
     def _close(self):
         if self.io.filename:
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-17-01-28-56.bpo-35379.yAECDr.rst b/Misc/NEWS.d/next/IDLE/2019-09-17-01-28-56.bpo-35379.yAECDr.rst
new file mode 100644
index 000000000000..98d41674b65d
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-17-01-28-56.bpo-35379.yAECDr.rst
@@ -0,0 +1,3 @@
+When exiting IDLE, catch any AttributeError.  One happens when
+EditorWindow.close is called twice.  Printing a traceback, when IDLE is run
+from a terminal, is useless and annoying.


From webhook-mailer at python.org  Tue Sep 17 02:14:24 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 06:14:24 -0000
Subject: [Python-checkins] bpo-38175: Fix a memory leak in comparison of
 sqlite3.Row objects. (GH-16155)
Message-ID: <mailman.798.1568700866.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f04299d978793be33828dc4d5f46e9b54c81a0a2
commit: f04299d978793be33828dc4d5f46e9b54c81a0a2
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T23:14:20-07:00
summary:

bpo-38175: Fix a memory leak in comparison of sqlite3.Row objects. (GH-16155)

(cherry picked from commit 8debfa50407107ff2329d01081cdc12d359f1d12)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst
M Lib/sqlite3/test/factory.py
M Modules/_sqlite/row.c

diff --git a/Lib/sqlite3/test/factory.py b/Lib/sqlite3/test/factory.py
index ced8445536e3..f103211ed9f1 100644
--- a/Lib/sqlite3/test/factory.py
+++ b/Lib/sqlite3/test/factory.py
@@ -169,19 +169,33 @@ def CheckSqliteRowHashCmp(self):
         row_1 = self.con.execute("select 1 as a, 2 as b").fetchone()
         row_2 = self.con.execute("select 1 as a, 2 as b").fetchone()
         row_3 = self.con.execute("select 1 as a, 3 as b").fetchone()
+        row_4 = self.con.execute("select 1 as b, 2 as a").fetchone()
+        row_5 = self.con.execute("select 2 as b, 1 as a").fetchone()
 
-        self.assertEqual(row_1, row_1)
-        self.assertEqual(row_1, row_2)
-        self.assertTrue(row_2 != row_3)
+        self.assertTrue(row_1 == row_1)
+        self.assertTrue(row_1 == row_2)
+        self.assertFalse(row_1 == row_3)
+        self.assertFalse(row_1 == row_4)
+        self.assertFalse(row_1 == row_5)
+        self.assertFalse(row_1 == object())
 
         self.assertFalse(row_1 != row_1)
         self.assertFalse(row_1 != row_2)
-        self.assertFalse(row_2 == row_3)
+        self.assertTrue(row_1 != row_3)
+        self.assertTrue(row_1 != row_4)
+        self.assertTrue(row_1 != row_5)
+        self.assertTrue(row_1 != object())
+
+        with self.assertRaises(TypeError):
+            row_1 > row_2
+        with self.assertRaises(TypeError):
+            row_1 < row_2
+        with self.assertRaises(TypeError):
+            row_1 >= row_2
+        with self.assertRaises(TypeError):
+            row_1 <= row_2
 
-        self.assertEqual(row_1, row_2)
         self.assertEqual(hash(row_1), hash(row_2))
-        self.assertNotEqual(row_1, row_3)
-        self.assertNotEqual(hash(row_1), hash(row_3))
 
     def CheckSqliteRowAsSequence(self):
         """ Checks if the row object can act like a sequence """
diff --git a/Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst b/Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst
new file mode 100644
index 000000000000..6d9f280bbd05
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst
@@ -0,0 +1 @@
+Fix a memory leak in comparison of :class:`sqlite3.Row` objects.
diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c
index 5c2f40082402..758518a8ff80 100644
--- a/Modules/_sqlite/row.c
+++ b/Modules/_sqlite/row.c
@@ -192,14 +192,16 @@ static PyObject* pysqlite_row_richcompare(pysqlite_Row *self, PyObject *_other,
     if (opid != Py_EQ && opid != Py_NE)
         Py_RETURN_NOTIMPLEMENTED;
 
-    if (PyType_IsSubtype(Py_TYPE(_other), &pysqlite_RowType)) {
+    if (PyObject_TypeCheck(_other, &pysqlite_RowType)) {
         pysqlite_Row *other = (pysqlite_Row *)_other;
-        PyObject *res = PyObject_RichCompare(self->description, other->description, opid);
-        if ((opid == Py_EQ && res == Py_True)
-            || (opid == Py_NE && res == Py_False)) {
-            Py_DECREF(res);
+        int eq = PyObject_RichCompareBool(self->description, other->description, Py_EQ);
+        if (eq < 0) {
+            return NULL;
+        }
+        if (eq) {
             return PyObject_RichCompare(self->data, other->data, opid);
         }
+        return PyBool_FromLong(opid != Py_EQ);
     }
     Py_RETURN_NOTIMPLEMENTED;
 }


From webhook-mailer at python.org  Tue Sep 17 02:14:33 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 06:14:33 -0000
Subject: [Python-checkins] bpo-38175: Fix a memory leak in comparison of
 sqlite3.Row objects. (GH-16155)
Message-ID: <mailman.799.1568700875.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/4ac1be28281d8961e71b13623c3671245c125760
commit: 4ac1be28281d8961e71b13623c3671245c125760
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T23:14:30-07:00
summary:

bpo-38175: Fix a memory leak in comparison of sqlite3.Row objects. (GH-16155)

(cherry picked from commit 8debfa50407107ff2329d01081cdc12d359f1d12)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst
M Lib/sqlite3/test/factory.py
M Modules/_sqlite/row.c

diff --git a/Lib/sqlite3/test/factory.py b/Lib/sqlite3/test/factory.py
index ced8445536e3..f103211ed9f1 100644
--- a/Lib/sqlite3/test/factory.py
+++ b/Lib/sqlite3/test/factory.py
@@ -169,19 +169,33 @@ def CheckSqliteRowHashCmp(self):
         row_1 = self.con.execute("select 1 as a, 2 as b").fetchone()
         row_2 = self.con.execute("select 1 as a, 2 as b").fetchone()
         row_3 = self.con.execute("select 1 as a, 3 as b").fetchone()
+        row_4 = self.con.execute("select 1 as b, 2 as a").fetchone()
+        row_5 = self.con.execute("select 2 as b, 1 as a").fetchone()
 
-        self.assertEqual(row_1, row_1)
-        self.assertEqual(row_1, row_2)
-        self.assertTrue(row_2 != row_3)
+        self.assertTrue(row_1 == row_1)
+        self.assertTrue(row_1 == row_2)
+        self.assertFalse(row_1 == row_3)
+        self.assertFalse(row_1 == row_4)
+        self.assertFalse(row_1 == row_5)
+        self.assertFalse(row_1 == object())
 
         self.assertFalse(row_1 != row_1)
         self.assertFalse(row_1 != row_2)
-        self.assertFalse(row_2 == row_3)
+        self.assertTrue(row_1 != row_3)
+        self.assertTrue(row_1 != row_4)
+        self.assertTrue(row_1 != row_5)
+        self.assertTrue(row_1 != object())
+
+        with self.assertRaises(TypeError):
+            row_1 > row_2
+        with self.assertRaises(TypeError):
+            row_1 < row_2
+        with self.assertRaises(TypeError):
+            row_1 >= row_2
+        with self.assertRaises(TypeError):
+            row_1 <= row_2
 
-        self.assertEqual(row_1, row_2)
         self.assertEqual(hash(row_1), hash(row_2))
-        self.assertNotEqual(row_1, row_3)
-        self.assertNotEqual(hash(row_1), hash(row_3))
 
     def CheckSqliteRowAsSequence(self):
         """ Checks if the row object can act like a sequence """
diff --git a/Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst b/Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst
new file mode 100644
index 000000000000..6d9f280bbd05
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst
@@ -0,0 +1 @@
+Fix a memory leak in comparison of :class:`sqlite3.Row` objects.
diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c
index 122bee05f570..0edc69fc8cac 100644
--- a/Modules/_sqlite/row.c
+++ b/Modules/_sqlite/row.c
@@ -197,14 +197,16 @@ static PyObject* pysqlite_row_richcompare(pysqlite_Row *self, PyObject *_other,
     if (opid != Py_EQ && opid != Py_NE)
         Py_RETURN_NOTIMPLEMENTED;
 
-    if (PyType_IsSubtype(Py_TYPE(_other), &pysqlite_RowType)) {
+    if (PyObject_TypeCheck(_other, &pysqlite_RowType)) {
         pysqlite_Row *other = (pysqlite_Row *)_other;
-        PyObject *res = PyObject_RichCompare(self->description, other->description, opid);
-        if ((opid == Py_EQ && res == Py_True)
-            || (opid == Py_NE && res == Py_False)) {
-            Py_DECREF(res);
+        int eq = PyObject_RichCompareBool(self->description, other->description, Py_EQ);
+        if (eq < 0) {
+            return NULL;
+        }
+        if (eq) {
             return PyObject_RichCompare(self->data, other->data, opid);
         }
+        return PyBool_FromLong(opid != Py_EQ);
     }
     Py_RETURN_NOTIMPLEMENTED;
 }


From webhook-mailer at python.org  Tue Sep 17 02:21:00 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Tue, 17 Sep 2019 06:21:00 -0000
Subject: [Python-checkins] bpo-38185: Fixed case-insensitive string
 comparison in sqlite3.Row indexing. (GH-16190)
Message-ID: <mailman.800.1568701261.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f669581a9527afb0d2325f9845a86715c0ba365d
commit: f669581a9527afb0d2325f9845a86715c0ba365d
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T09:20:56+03:00
summary:

bpo-38185: Fixed case-insensitive string comparison in sqlite3.Row indexing. (GH-16190)

files:
A Misc/NEWS.d/next/Library/2019-09-16-19-12-57.bpo-38185.zYWppY.rst
M Lib/sqlite3/test/factory.py
M Modules/_sqlite/row.c

diff --git a/Lib/sqlite3/test/factory.py b/Lib/sqlite3/test/factory.py
index f103211ed9f1..95dd24bdfadc 100644
--- a/Lib/sqlite3/test/factory.py
+++ b/Lib/sqlite3/test/factory.py
@@ -98,16 +98,14 @@ def CheckCustomFactory(self):
 
     def CheckSqliteRowIndex(self):
         self.con.row_factory = sqlite.Row
-        row = self.con.execute("select 1 as a, 2 as b").fetchone()
+        row = self.con.execute("select 1 as a_1, 2 as b").fetchone()
         self.assertIsInstance(row, sqlite.Row)
 
-        col1, col2 = row["a"], row["b"]
-        self.assertEqual(col1, 1, "by name: wrong result for column 'a'")
-        self.assertEqual(col2, 2, "by name: wrong result for column 'a'")
+        self.assertEqual(row["a_1"], 1, "by name: wrong result for column 'a_1'")
+        self.assertEqual(row["b"], 2, "by name: wrong result for column 'b'")
 
-        col1, col2 = row["A"], row["B"]
-        self.assertEqual(col1, 1, "by name: wrong result for column 'A'")
-        self.assertEqual(col2, 2, "by name: wrong result for column 'B'")
+        self.assertEqual(row["A_1"], 1, "by name: wrong result for column 'A_1'")
+        self.assertEqual(row["B"], 2, "by name: wrong result for column 'B'")
 
         self.assertEqual(row[0], 1, "by index: wrong result for column 0")
         self.assertEqual(row[1], 2, "by index: wrong result for column 1")
@@ -116,6 +114,10 @@ def CheckSqliteRowIndex(self):
 
         with self.assertRaises(IndexError):
             row['c']
+        with self.assertRaises(IndexError):
+            row['a_\x11']
+        with self.assertRaises(IndexError):
+            row['a\x7f1']
         with self.assertRaises(IndexError):
             row[2]
         with self.assertRaises(IndexError):
@@ -123,6 +125,15 @@ def CheckSqliteRowIndex(self):
         with self.assertRaises(IndexError):
             row[2**1000]
 
+    def CheckSqliteRowIndexUnicode(self):
+        self.con.row_factory = sqlite.Row
+        row = self.con.execute("select 1 as \xff").fetchone()
+        self.assertEqual(row["\xff"], 1)
+        with self.assertRaises(IndexError):
+            row['\u0178']
+        with self.assertRaises(IndexError):
+            row['\xdf']
+
     def CheckSqliteRowSlice(self):
         # A sqlite.Row can be sliced like a list.
         self.con.row_factory = sqlite.Row
diff --git a/Misc/NEWS.d/next/Library/2019-09-16-19-12-57.bpo-38185.zYWppY.rst b/Misc/NEWS.d/next/Library/2019-09-16-19-12-57.bpo-38185.zYWppY.rst
new file mode 100644
index 000000000000..2260db68e8cf
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-16-19-12-57.bpo-38185.zYWppY.rst
@@ -0,0 +1 @@
+Fixed case-insensitive string comparison in :class:`sqlite3.Row` indexing.
diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c
index 758518a8ff80..4b47108278a0 100644
--- a/Modules/_sqlite/row.c
+++ b/Modules/_sqlite/row.c
@@ -76,16 +76,38 @@ PyObject* pysqlite_row_item(pysqlite_Row* self, Py_ssize_t idx)
    return item;
 }
 
+static int
+equal_ignore_case(PyObject *left, PyObject *right)
+{
+    int eq = PyObject_RichCompareBool(left, right, Py_EQ);
+    if (eq) { /* equal or error */
+        return eq;
+    }
+    if (!PyUnicode_Check(left) || !PyUnicode_Check(right)) {
+        return 0;
+    }
+    if (!PyUnicode_IS_ASCII(left) || !PyUnicode_IS_ASCII(right)) {
+        return 0;
+    }
+
+    Py_ssize_t len = PyUnicode_GET_LENGTH(left);
+    if (PyUnicode_GET_LENGTH(right) != len) {
+        return 0;
+    }
+    const Py_UCS1 *p1 = PyUnicode_1BYTE_DATA(left);
+    const Py_UCS1 *p2 = PyUnicode_1BYTE_DATA(right);
+    for (; len; len--, p1++, p2++) {
+        if (Py_TOLOWER(*p1) != Py_TOLOWER(*p2)) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
 PyObject* pysqlite_row_subscript(pysqlite_Row* self, PyObject* idx)
 {
     Py_ssize_t _idx;
-    const char *key;
     Py_ssize_t nitems, i;
-    const char *compare_key;
-
-    const char *p1;
-    const char *p2;
-
     PyObject* item;
 
     if (PyLong_Check(idx)) {
@@ -98,44 +120,22 @@ PyObject* pysqlite_row_subscript(pysqlite_Row* self, PyObject* idx)
         Py_XINCREF(item);
         return item;
     } else if (PyUnicode_Check(idx)) {
-        key = PyUnicode_AsUTF8(idx);
-        if (key == NULL)
-            return NULL;
-
         nitems = PyTuple_Size(self->description);
 
         for (i = 0; i < nitems; i++) {
             PyObject *obj;
             obj = PyTuple_GET_ITEM(self->description, i);
             obj = PyTuple_GET_ITEM(obj, 0);
-            compare_key = PyUnicode_AsUTF8(obj);
-            if (!compare_key) {
+            int eq = equal_ignore_case(idx, obj);
+            if (eq < 0) {
                 return NULL;
             }
-
-            p1 = key;
-            p2 = compare_key;
-
-            while (1) {
-                if ((*p1 == (char)0) || (*p2 == (char)0)) {
-                    break;
-                }
-
-                if ((*p1 | 0x20) != (*p2 | 0x20)) {
-                    break;
-                }
-
-                p1++;
-                p2++;
-            }
-
-            if ((*p1 == (char)0) && (*p2 == (char)0)) {
+            if (eq) {
                 /* found item */
                 item = PyTuple_GetItem(self->data, i);
                 Py_INCREF(item);
                 return item;
             }
-
         }
 
         PyErr_SetString(PyExc_IndexError, "No item with that key");


From webhook-mailer at python.org  Tue Sep 17 02:23:17 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 06:23:17 -0000
Subject: [Python-checkins] bpo-35379: When exiting IDLE,
 catch any AttributeError. (GH-16212)
Message-ID: <mailman.801.1568701400.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/73ccc3322f7ad0d016478ef20237bccd0a314f0a
commit: 73ccc3322f7ad0d016478ef20237bccd0a314f0a
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T23:23:12-07:00
summary:

bpo-35379: When exiting IDLE, catch any AttributeError. (GH-16212)


One happens when EditorWindow.close is called twice.
Printing a traceback, when IDLE is run from a terminal,
is useless and annoying.
(cherry picked from commit dfd34a9cd58e8150c324190f746de919e140abe8)

Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu>

files:
A Misc/NEWS.d/next/IDLE/2019-09-17-01-28-56.bpo-35379.yAECDr.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/editor.py

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 3ccc5c07fb00..9d26bda42594 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -2,6 +2,13 @@ What's New in IDLE 3.8.0 (since 3.7.0)
 Released on 2019-10-20?
 ======================================
 
+bpo-35379: When exiting IDLE, catch any AttributeError.  One happens
+when EditorWindow.close is called twice.  Printing a traceback, when
+IDLE is run from a terminal, is useless and annoying.
+
+bpo-38183: To avoid test issues, test_idle ignores the user config
+directory.  It no longer tries to create or access .idlerc or any files
+within.  Users must run IDLE to discover problems with saving settings.
 
 bpo-38077: IDLE no longer adds 'argv' to the user namespace when
 initializing it.  This bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4.
diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py
index 838cb04b8008..b969f8c493b2 100644
--- a/Lib/idlelib/editor.py
+++ b/Lib/idlelib/editor.py
@@ -1061,10 +1061,13 @@ def maybesave(self):
             return self.io.maybesave()
 
     def close(self):
-        reply = self.maybesave()
-        if str(reply) != "cancel":
-            self._close()
-        return reply
+        try:
+            reply = self.maybesave()
+            if str(reply) != "cancel":
+                self._close()
+            return reply
+        except AttributeError:  # bpo-35379: close called twice
+            pass
 
     def _close(self):
         if self.io.filename:
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-17-01-28-56.bpo-35379.yAECDr.rst b/Misc/NEWS.d/next/IDLE/2019-09-17-01-28-56.bpo-35379.yAECDr.rst
new file mode 100644
index 000000000000..98d41674b65d
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-17-01-28-56.bpo-35379.yAECDr.rst
@@ -0,0 +1,3 @@
+When exiting IDLE, catch any AttributeError.  One happens when
+EditorWindow.close is called twice.  Printing a traceback, when IDLE is run
+from a terminal, is useless and annoying.


From webhook-mailer at python.org  Tue Sep 17 02:24:06 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 06:24:06 -0000
Subject: [Python-checkins] bpo-35379: When exiting IDLE,
 catch any AttributeError. (GH-16212)
Message-ID: <mailman.802.1568701447.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3d916a7b1ee29896bece4a6a37d8084fa3c1abf6
commit: 3d916a7b1ee29896bece4a6a37d8084fa3c1abf6
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T23:24:02-07:00
summary:

bpo-35379: When exiting IDLE, catch any AttributeError. (GH-16212)


One happens when EditorWindow.close is called twice.
Printing a traceback, when IDLE is run from a terminal,
is useless and annoying.
(cherry picked from commit dfd34a9cd58e8150c324190f746de919e140abe8)

Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu>

files:
A Misc/NEWS.d/next/IDLE/2019-09-17-01-28-56.bpo-35379.yAECDr.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/editor.py

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index da2512913f0e..32fda8c75bb8 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -2,6 +2,13 @@ What's New in IDLE 3.7.5
 Released on 2019-09-30?
 ======================================
 
+bpo-35379: When exiting IDLE, catch any AttributeError.  One happens
+when EditorWindow.close is called twice.  Printing a traceback, when
+IDLE is run from a terminal, is useless and annoying.
+
+bpo-38183: To avoid test issues, test_idle ignores the user config
+directory.  It no longer tries to create or access .idlerc or any files
+within.  Users must run IDLE to discover problems with saving settings.
 
 bpo-38077: IDLE no longer adds 'argv' to the user namespace when
 initializing it.  This bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4.
diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py
index 838cb04b8008..b969f8c493b2 100644
--- a/Lib/idlelib/editor.py
+++ b/Lib/idlelib/editor.py
@@ -1061,10 +1061,13 @@ def maybesave(self):
             return self.io.maybesave()
 
     def close(self):
-        reply = self.maybesave()
-        if str(reply) != "cancel":
-            self._close()
-        return reply
+        try:
+            reply = self.maybesave()
+            if str(reply) != "cancel":
+                self._close()
+            return reply
+        except AttributeError:  # bpo-35379: close called twice
+            pass
 
     def _close(self):
         if self.io.filename:
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-17-01-28-56.bpo-35379.yAECDr.rst b/Misc/NEWS.d/next/IDLE/2019-09-17-01-28-56.bpo-35379.yAECDr.rst
new file mode 100644
index 000000000000..98d41674b65d
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-17-01-28-56.bpo-35379.yAECDr.rst
@@ -0,0 +1,3 @@
+When exiting IDLE, catch any AttributeError.  One happens when
+EditorWindow.close is called twice.  Printing a traceback, when IDLE is run
+from a terminal, is useless and annoying.


From webhook-mailer at python.org  Tue Sep 17 02:39:15 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 06:39:15 -0000
Subject: [Python-checkins] bpo-38185: Fixed case-insensitive string
 comparison in sqlite3.Row indexing. (GH-16190)
Message-ID: <mailman.803.1568702356.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d8d653c2d0d267eb1e44f6cdb916945029fec3b1
commit: d8d653c2d0d267eb1e44f6cdb916945029fec3b1
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T23:39:11-07:00
summary:

bpo-38185: Fixed case-insensitive string comparison in sqlite3.Row indexing. (GH-16190)

(cherry picked from commit f669581a9527afb0d2325f9845a86715c0ba365d)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-16-19-12-57.bpo-38185.zYWppY.rst
M Lib/sqlite3/test/factory.py
M Modules/_sqlite/row.c

diff --git a/Lib/sqlite3/test/factory.py b/Lib/sqlite3/test/factory.py
index f103211ed9f1..95dd24bdfadc 100644
--- a/Lib/sqlite3/test/factory.py
+++ b/Lib/sqlite3/test/factory.py
@@ -98,16 +98,14 @@ def CheckCustomFactory(self):
 
     def CheckSqliteRowIndex(self):
         self.con.row_factory = sqlite.Row
-        row = self.con.execute("select 1 as a, 2 as b").fetchone()
+        row = self.con.execute("select 1 as a_1, 2 as b").fetchone()
         self.assertIsInstance(row, sqlite.Row)
 
-        col1, col2 = row["a"], row["b"]
-        self.assertEqual(col1, 1, "by name: wrong result for column 'a'")
-        self.assertEqual(col2, 2, "by name: wrong result for column 'a'")
+        self.assertEqual(row["a_1"], 1, "by name: wrong result for column 'a_1'")
+        self.assertEqual(row["b"], 2, "by name: wrong result for column 'b'")
 
-        col1, col2 = row["A"], row["B"]
-        self.assertEqual(col1, 1, "by name: wrong result for column 'A'")
-        self.assertEqual(col2, 2, "by name: wrong result for column 'B'")
+        self.assertEqual(row["A_1"], 1, "by name: wrong result for column 'A_1'")
+        self.assertEqual(row["B"], 2, "by name: wrong result for column 'B'")
 
         self.assertEqual(row[0], 1, "by index: wrong result for column 0")
         self.assertEqual(row[1], 2, "by index: wrong result for column 1")
@@ -116,6 +114,10 @@ def CheckSqliteRowIndex(self):
 
         with self.assertRaises(IndexError):
             row['c']
+        with self.assertRaises(IndexError):
+            row['a_\x11']
+        with self.assertRaises(IndexError):
+            row['a\x7f1']
         with self.assertRaises(IndexError):
             row[2]
         with self.assertRaises(IndexError):
@@ -123,6 +125,15 @@ def CheckSqliteRowIndex(self):
         with self.assertRaises(IndexError):
             row[2**1000]
 
+    def CheckSqliteRowIndexUnicode(self):
+        self.con.row_factory = sqlite.Row
+        row = self.con.execute("select 1 as \xff").fetchone()
+        self.assertEqual(row["\xff"], 1)
+        with self.assertRaises(IndexError):
+            row['\u0178']
+        with self.assertRaises(IndexError):
+            row['\xdf']
+
     def CheckSqliteRowSlice(self):
         # A sqlite.Row can be sliced like a list.
         self.con.row_factory = sqlite.Row
diff --git a/Misc/NEWS.d/next/Library/2019-09-16-19-12-57.bpo-38185.zYWppY.rst b/Misc/NEWS.d/next/Library/2019-09-16-19-12-57.bpo-38185.zYWppY.rst
new file mode 100644
index 000000000000..2260db68e8cf
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-16-19-12-57.bpo-38185.zYWppY.rst
@@ -0,0 +1 @@
+Fixed case-insensitive string comparison in :class:`sqlite3.Row` indexing.
diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c
index 758518a8ff80..4b47108278a0 100644
--- a/Modules/_sqlite/row.c
+++ b/Modules/_sqlite/row.c
@@ -76,16 +76,38 @@ PyObject* pysqlite_row_item(pysqlite_Row* self, Py_ssize_t idx)
    return item;
 }
 
+static int
+equal_ignore_case(PyObject *left, PyObject *right)
+{
+    int eq = PyObject_RichCompareBool(left, right, Py_EQ);
+    if (eq) { /* equal or error */
+        return eq;
+    }
+    if (!PyUnicode_Check(left) || !PyUnicode_Check(right)) {
+        return 0;
+    }
+    if (!PyUnicode_IS_ASCII(left) || !PyUnicode_IS_ASCII(right)) {
+        return 0;
+    }
+
+    Py_ssize_t len = PyUnicode_GET_LENGTH(left);
+    if (PyUnicode_GET_LENGTH(right) != len) {
+        return 0;
+    }
+    const Py_UCS1 *p1 = PyUnicode_1BYTE_DATA(left);
+    const Py_UCS1 *p2 = PyUnicode_1BYTE_DATA(right);
+    for (; len; len--, p1++, p2++) {
+        if (Py_TOLOWER(*p1) != Py_TOLOWER(*p2)) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
 PyObject* pysqlite_row_subscript(pysqlite_Row* self, PyObject* idx)
 {
     Py_ssize_t _idx;
-    const char *key;
     Py_ssize_t nitems, i;
-    const char *compare_key;
-
-    const char *p1;
-    const char *p2;
-
     PyObject* item;
 
     if (PyLong_Check(idx)) {
@@ -98,44 +120,22 @@ PyObject* pysqlite_row_subscript(pysqlite_Row* self, PyObject* idx)
         Py_XINCREF(item);
         return item;
     } else if (PyUnicode_Check(idx)) {
-        key = PyUnicode_AsUTF8(idx);
-        if (key == NULL)
-            return NULL;
-
         nitems = PyTuple_Size(self->description);
 
         for (i = 0; i < nitems; i++) {
             PyObject *obj;
             obj = PyTuple_GET_ITEM(self->description, i);
             obj = PyTuple_GET_ITEM(obj, 0);
-            compare_key = PyUnicode_AsUTF8(obj);
-            if (!compare_key) {
+            int eq = equal_ignore_case(idx, obj);
+            if (eq < 0) {
                 return NULL;
             }
-
-            p1 = key;
-            p2 = compare_key;
-
-            while (1) {
-                if ((*p1 == (char)0) || (*p2 == (char)0)) {
-                    break;
-                }
-
-                if ((*p1 | 0x20) != (*p2 | 0x20)) {
-                    break;
-                }
-
-                p1++;
-                p2++;
-            }
-
-            if ((*p1 == (char)0) && (*p2 == (char)0)) {
+            if (eq) {
                 /* found item */
                 item = PyTuple_GetItem(self->data, i);
                 Py_INCREF(item);
                 return item;
             }
-
         }
 
         PyErr_SetString(PyExc_IndexError, "No item with that key");


From webhook-mailer at python.org  Tue Sep 17 02:39:53 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 06:39:53 -0000
Subject: [Python-checkins] bpo-38185: Fixed case-insensitive string
 comparison in sqlite3.Row indexing. (GH-16190)
Message-ID: <mailman.804.1568702394.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/80cb055aa45f37951b8e3097fe4367d443d455ae
commit: 80cb055aa45f37951b8e3097fe4367d443d455ae
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-16T23:39:49-07:00
summary:

bpo-38185: Fixed case-insensitive string comparison in sqlite3.Row indexing. (GH-16190)

(cherry picked from commit f669581a9527afb0d2325f9845a86715c0ba365d)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-16-19-12-57.bpo-38185.zYWppY.rst
M Lib/sqlite3/test/factory.py
M Modules/_sqlite/row.c

diff --git a/Lib/sqlite3/test/factory.py b/Lib/sqlite3/test/factory.py
index f103211ed9f1..95dd24bdfadc 100644
--- a/Lib/sqlite3/test/factory.py
+++ b/Lib/sqlite3/test/factory.py
@@ -98,16 +98,14 @@ def CheckCustomFactory(self):
 
     def CheckSqliteRowIndex(self):
         self.con.row_factory = sqlite.Row
-        row = self.con.execute("select 1 as a, 2 as b").fetchone()
+        row = self.con.execute("select 1 as a_1, 2 as b").fetchone()
         self.assertIsInstance(row, sqlite.Row)
 
-        col1, col2 = row["a"], row["b"]
-        self.assertEqual(col1, 1, "by name: wrong result for column 'a'")
-        self.assertEqual(col2, 2, "by name: wrong result for column 'a'")
+        self.assertEqual(row["a_1"], 1, "by name: wrong result for column 'a_1'")
+        self.assertEqual(row["b"], 2, "by name: wrong result for column 'b'")
 
-        col1, col2 = row["A"], row["B"]
-        self.assertEqual(col1, 1, "by name: wrong result for column 'A'")
-        self.assertEqual(col2, 2, "by name: wrong result for column 'B'")
+        self.assertEqual(row["A_1"], 1, "by name: wrong result for column 'A_1'")
+        self.assertEqual(row["B"], 2, "by name: wrong result for column 'B'")
 
         self.assertEqual(row[0], 1, "by index: wrong result for column 0")
         self.assertEqual(row[1], 2, "by index: wrong result for column 1")
@@ -116,6 +114,10 @@ def CheckSqliteRowIndex(self):
 
         with self.assertRaises(IndexError):
             row['c']
+        with self.assertRaises(IndexError):
+            row['a_\x11']
+        with self.assertRaises(IndexError):
+            row['a\x7f1']
         with self.assertRaises(IndexError):
             row[2]
         with self.assertRaises(IndexError):
@@ -123,6 +125,15 @@ def CheckSqliteRowIndex(self):
         with self.assertRaises(IndexError):
             row[2**1000]
 
+    def CheckSqliteRowIndexUnicode(self):
+        self.con.row_factory = sqlite.Row
+        row = self.con.execute("select 1 as \xff").fetchone()
+        self.assertEqual(row["\xff"], 1)
+        with self.assertRaises(IndexError):
+            row['\u0178']
+        with self.assertRaises(IndexError):
+            row['\xdf']
+
     def CheckSqliteRowSlice(self):
         # A sqlite.Row can be sliced like a list.
         self.con.row_factory = sqlite.Row
diff --git a/Misc/NEWS.d/next/Library/2019-09-16-19-12-57.bpo-38185.zYWppY.rst b/Misc/NEWS.d/next/Library/2019-09-16-19-12-57.bpo-38185.zYWppY.rst
new file mode 100644
index 000000000000..2260db68e8cf
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-16-19-12-57.bpo-38185.zYWppY.rst
@@ -0,0 +1 @@
+Fixed case-insensitive string comparison in :class:`sqlite3.Row` indexing.
diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c
index 0edc69fc8cac..7c85ae22f943 100644
--- a/Modules/_sqlite/row.c
+++ b/Modules/_sqlite/row.c
@@ -76,16 +76,38 @@ PyObject* pysqlite_row_item(pysqlite_Row* self, Py_ssize_t idx)
    return item;
 }
 
+static int
+equal_ignore_case(PyObject *left, PyObject *right)
+{
+    int eq = PyObject_RichCompareBool(left, right, Py_EQ);
+    if (eq) { /* equal or error */
+        return eq;
+    }
+    if (!PyUnicode_Check(left) || !PyUnicode_Check(right)) {
+        return 0;
+    }
+    if (!PyUnicode_IS_ASCII(left) || !PyUnicode_IS_ASCII(right)) {
+        return 0;
+    }
+
+    Py_ssize_t len = PyUnicode_GET_LENGTH(left);
+    if (PyUnicode_GET_LENGTH(right) != len) {
+        return 0;
+    }
+    const Py_UCS1 *p1 = PyUnicode_1BYTE_DATA(left);
+    const Py_UCS1 *p2 = PyUnicode_1BYTE_DATA(right);
+    for (; len; len--, p1++, p2++) {
+        if (Py_TOLOWER(*p1) != Py_TOLOWER(*p2)) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
 PyObject* pysqlite_row_subscript(pysqlite_Row* self, PyObject* idx)
 {
     Py_ssize_t _idx;
-    const char *key;
     Py_ssize_t nitems, i;
-    const char *compare_key;
-
-    const char *p1;
-    const char *p2;
-
     PyObject* item;
 
     if (PyLong_Check(idx)) {
@@ -98,44 +120,22 @@ PyObject* pysqlite_row_subscript(pysqlite_Row* self, PyObject* idx)
         Py_XINCREF(item);
         return item;
     } else if (PyUnicode_Check(idx)) {
-        key = PyUnicode_AsUTF8(idx);
-        if (key == NULL)
-            return NULL;
-
         nitems = PyTuple_Size(self->description);
 
         for (i = 0; i < nitems; i++) {
             PyObject *obj;
             obj = PyTuple_GET_ITEM(self->description, i);
             obj = PyTuple_GET_ITEM(obj, 0);
-            compare_key = PyUnicode_AsUTF8(obj);
-            if (!compare_key) {
+            int eq = equal_ignore_case(idx, obj);
+            if (eq < 0) {
                 return NULL;
             }
-
-            p1 = key;
-            p2 = compare_key;
-
-            while (1) {
-                if ((*p1 == (char)0) || (*p2 == (char)0)) {
-                    break;
-                }
-
-                if ((*p1 | 0x20) != (*p2 | 0x20)) {
-                    break;
-                }
-
-                p1++;
-                p2++;
-            }
-
-            if ((*p1 == (char)0) && (*p2 == (char)0)) {
+            if (eq) {
                 /* found item */
                 item = PyTuple_GetItem(self->data, i);
                 Py_INCREF(item);
                 return item;
             }
-
         }
 
         PyErr_SetString(PyExc_IndexError, "No item with that key");


From webhook-mailer at python.org  Tue Sep 17 02:56:31 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Tue, 17 Sep 2019 06:56:31 -0000
Subject: [Python-checkins] [2.7] bpo-38175: Fix a memory leak in comparison
 of sqlite3.Row objects. (GH-16155). (GH-16215)
Message-ID: <mailman.805.1568703392.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/be257bcad12b1d72a2c2a8199d2291ae22bccf78
commit: be257bcad12b1d72a2c2a8199d2291ae22bccf78
branch: 2.7
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T09:56:27+03:00
summary:

[2.7] bpo-38175: Fix a memory leak in comparison of sqlite3.Row objects. (GH-16155). (GH-16215)

(cherry picked from commit 8debfa50407107ff2329d01081cdc12d359f1d12)

files:
A Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst
M Lib/sqlite3/test/factory.py
M Modules/_sqlite/row.c

diff --git a/Lib/sqlite3/test/factory.py b/Lib/sqlite3/test/factory.py
index b9e9cd7fd7f4..b8e0f645a00e 100644
--- a/Lib/sqlite3/test/factory.py
+++ b/Lib/sqlite3/test/factory.py
@@ -159,19 +159,24 @@ def CheckSqliteRowHashCmp(self):
         row_1 = self.con.execute("select 1 as a, 2 as b").fetchone()
         row_2 = self.con.execute("select 1 as a, 2 as b").fetchone()
         row_3 = self.con.execute("select 1 as a, 3 as b").fetchone()
+        row_4 = self.con.execute("select 1 as b, 2 as a").fetchone()
+        row_5 = self.con.execute("select 2 as b, 1 as a").fetchone()
 
-        self.assertEqual(row_1, row_1)
-        self.assertEqual(row_1, row_2)
-        self.assertTrue(row_2 != row_3)
+        self.assertTrue(row_1 == row_1)
+        self.assertTrue(row_1 == row_2)
+        self.assertFalse(row_1 == row_3)
+        self.assertFalse(row_1 == row_4)
+        self.assertFalse(row_1 == row_5)
+        self.assertFalse(row_1 == object())
 
         self.assertFalse(row_1 != row_1)
         self.assertFalse(row_1 != row_2)
-        self.assertFalse(row_2 == row_3)
+        self.assertTrue(row_1 != row_3)
+        self.assertTrue(row_1 != row_4)
+        self.assertTrue(row_1 != row_5)
+        self.assertTrue(row_1 != object())
 
-        self.assertEqual(row_1, row_2)
         self.assertEqual(hash(row_1), hash(row_2))
-        self.assertNotEqual(row_1, row_3)
-        self.assertNotEqual(hash(row_1), hash(row_3))
 
     def CheckSqliteRowAsSequence(self):
         """ Checks if the row object can act like a sequence """
diff --git a/Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst b/Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst
new file mode 100644
index 000000000000..6d9f280bbd05
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-15-10-30-33.bpo-38175.61XlUv.rst
@@ -0,0 +1 @@
+Fix a memory leak in comparison of :class:`sqlite3.Row` objects.
diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c
index 9ebe7b7b81fb..c0da329c13eb 100644
--- a/Modules/_sqlite/row.c
+++ b/Modules/_sqlite/row.c
@@ -199,14 +199,16 @@ static PyObject* pysqlite_row_richcompare(pysqlite_Row *self, PyObject *_other,
         Py_INCREF(Py_NotImplemented);
         return Py_NotImplemented;
     }
-    if (PyType_IsSubtype(Py_TYPE(_other), &pysqlite_RowType)) {
+    if (PyObject_TypeCheck(_other, &pysqlite_RowType)) {
         pysqlite_Row *other = (pysqlite_Row *)_other;
-        PyObject *res = PyObject_RichCompare(self->description, other->description, opid);
-        if ((opid == Py_EQ && res == Py_True)
-            || (opid == Py_NE && res == Py_False)) {
-            Py_DECREF(res);
+        int eq = PyObject_RichCompareBool(self->description, other->description, Py_EQ);
+        if (eq < 0) {
+            return NULL;
+        }
+        if (eq) {
             return PyObject_RichCompare(self->data, other->data, opid);
         }
+        return PyBool_FromLong(opid != Py_EQ);
     }
     Py_INCREF(Py_NotImplemented);
     return Py_NotImplemented;


From webhook-mailer at python.org  Tue Sep 17 03:16:47 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 07:16:47 -0000
Subject: [Python-checkins] bpo-38192: Fix remaining passing of "loop" in the
 protocol examples (GH-16202)
Message-ID: <mailman.806.1568704608.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5d359cc62e0244e1fd8d17146a4135079d6843bf
commit: 5d359cc62e0244e1fd8d17146a4135079d6843bf
branch: master
author: Hrvoje Nik?i? <hniksic at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-17T00:16:43-07:00
summary:

bpo-38192: Fix remaining passing of "loop" in the protocol examples (GH-16202)



See https://bugs.python.org/issue38192 .



https://bugs.python.org/issue38192

files:
M Doc/library/asyncio-protocol.rst

diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index ca37581c288d..cb0317ea20c1 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -792,7 +792,7 @@ data, and waits until the connection is closed::
         message = 'Hello World!'
 
         transport, protocol = await loop.create_connection(
-            lambda: EchoClientProtocol(message, on_con_lost, loop),
+            lambda: EchoClientProtocol(message, on_con_lost),
             '127.0.0.1', 8888)
 
         # Wait until the protocol signals that the connection
@@ -870,8 +870,8 @@ method, sends data and closes the transport when it receives the answer::
     class EchoClientProtocol:
         def __init__(self, message, on_con_lost):
             self.message = message
-            self.transport = None
             self.on_con_lost = on_con_lost
+            self.transport = None
 
         def connection_made(self, transport):
             self.transport = transport
@@ -927,9 +927,9 @@ Wait until a socket receives data using the
 
     class MyProtocol(asyncio.Protocol):
 
-        def __init__(self, loop):
+        def __init__(self, on_con_lost):
             self.transport = None
-            self.on_con_lost = loop.create_future()
+            self.on_con_lost = on_con_lost
 
         def connection_made(self, transport):
             self.transport = transport
@@ -950,13 +950,14 @@ Wait until a socket receives data using the
         # Get a reference to the event loop as we plan to use
         # low-level APIs.
         loop = asyncio.get_running_loop()
+        on_con_lost = loop.create_future()
 
         # Create a pair of connected sockets
         rsock, wsock = socket.socketpair()
 
         # Register the socket to wait for data.
         transport, protocol = await loop.create_connection(
-            lambda: MyProtocol(loop), sock=rsock)
+            lambda: MyProtocol(on_con_lost), sock=rsock)
 
         # Simulate the reception of data from the network.
         loop.call_soon(wsock.send, 'abc'.encode())


From webhook-mailer at python.org  Tue Sep 17 03:22:52 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 07:22:52 -0000
Subject: [Python-checkins] bpo-38192: Fix remaining passing of "loop" in the
 protocol examples (GH-16202)
Message-ID: <mailman.807.1568704973.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/facbd316b409f3775da2a74d0845938b4c7b88f0
commit: facbd316b409f3775da2a74d0845938b4c7b88f0
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T00:22:48-07:00
summary:

bpo-38192: Fix remaining passing of "loop" in the protocol examples (GH-16202)


See https://bugs.python.org/issue38192 .

https://bugs.python.org/issue38192
(cherry picked from commit 5d359cc62e0244e1fd8d17146a4135079d6843bf)

Co-authored-by: Hrvoje Nik?i? <hniksic at gmail.com>

files:
M Doc/library/asyncio-protocol.rst

diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index d421905f0f32..1a7794e21261 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -792,7 +792,7 @@ data, and waits until the connection is closed::
         message = 'Hello World!'
 
         transport, protocol = await loop.create_connection(
-            lambda: EchoClientProtocol(message, on_con_lost, loop),
+            lambda: EchoClientProtocol(message, on_con_lost),
             '127.0.0.1', 8888)
 
         # Wait until the protocol signals that the connection
@@ -870,8 +870,8 @@ method, sends data and closes the transport when it receives the answer::
     class EchoClientProtocol:
         def __init__(self, message, on_con_lost):
             self.message = message
-            self.transport = None
             self.on_con_lost = on_con_lost
+            self.transport = None
 
         def connection_made(self, transport):
             self.transport = transport
@@ -927,9 +927,9 @@ Wait until a socket receives data using the
 
     class MyProtocol(asyncio.Protocol):
 
-        def __init__(self, loop):
+        def __init__(self, on_con_lost):
             self.transport = None
-            self.on_con_lost = loop.create_future()
+            self.on_con_lost = on_con_lost
 
         def connection_made(self, transport):
             self.transport = transport
@@ -950,13 +950,14 @@ Wait until a socket receives data using the
         # Get a reference to the event loop as we plan to use
         # low-level APIs.
         loop = asyncio.get_running_loop()
+        on_con_lost = loop.create_future()
 
         # Create a pair of connected sockets
         rsock, wsock = socket.socketpair()
 
         # Register the socket to wait for data.
         transport, protocol = await loop.create_connection(
-            lambda: MyProtocol(loop), sock=rsock)
+            lambda: MyProtocol(on_con_lost), sock=rsock)
 
         # Simulate the reception of data from the network.
         loop.call_soon(wsock.send, 'abc'.encode())


From webhook-mailer at python.org  Tue Sep 17 03:24:35 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 07:24:35 -0000
Subject: [Python-checkins] bpo-38192: Fix remaining passing of "loop" in the
 protocol examples (GH-16202)
Message-ID: <mailman.808.1568705076.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1ba74719e3a213a2c473b92f3a79f8399e65f80a
commit: 1ba74719e3a213a2c473b92f3a79f8399e65f80a
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T00:24:31-07:00
summary:

bpo-38192: Fix remaining passing of "loop" in the protocol examples (GH-16202)


See https://bugs.python.org/issue38192 .

https://bugs.python.org/issue38192
(cherry picked from commit 5d359cc62e0244e1fd8d17146a4135079d6843bf)

Co-authored-by: Hrvoje Nik?i? <hniksic at gmail.com>

files:
M Doc/library/asyncio-protocol.rst

diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index ca37581c288d..cb0317ea20c1 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -792,7 +792,7 @@ data, and waits until the connection is closed::
         message = 'Hello World!'
 
         transport, protocol = await loop.create_connection(
-            lambda: EchoClientProtocol(message, on_con_lost, loop),
+            lambda: EchoClientProtocol(message, on_con_lost),
             '127.0.0.1', 8888)
 
         # Wait until the protocol signals that the connection
@@ -870,8 +870,8 @@ method, sends data and closes the transport when it receives the answer::
     class EchoClientProtocol:
         def __init__(self, message, on_con_lost):
             self.message = message
-            self.transport = None
             self.on_con_lost = on_con_lost
+            self.transport = None
 
         def connection_made(self, transport):
             self.transport = transport
@@ -927,9 +927,9 @@ Wait until a socket receives data using the
 
     class MyProtocol(asyncio.Protocol):
 
-        def __init__(self, loop):
+        def __init__(self, on_con_lost):
             self.transport = None
-            self.on_con_lost = loop.create_future()
+            self.on_con_lost = on_con_lost
 
         def connection_made(self, transport):
             self.transport = transport
@@ -950,13 +950,14 @@ Wait until a socket receives data using the
         # Get a reference to the event loop as we plan to use
         # low-level APIs.
         loop = asyncio.get_running_loop()
+        on_con_lost = loop.create_future()
 
         # Create a pair of connected sockets
         rsock, wsock = socket.socketpair()
 
         # Register the socket to wait for data.
         transport, protocol = await loop.create_connection(
-            lambda: MyProtocol(loop), sock=rsock)
+            lambda: MyProtocol(on_con_lost), sock=rsock)
 
         # Simulate the reception of data from the network.
         loop.call_soon(wsock.send, 'abc'.encode())


From webhook-mailer at python.org  Tue Sep 17 04:08:31 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 17 Sep 2019 08:08:31 -0000
Subject: [Python-checkins] bpo-37531: regrtest main process uses shorter
 timeout (GH-16220)
Message-ID: <mailman.809.1568707711.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/46b0b81220a23bc4aee5ba3ba67e8cf1b5df7960
commit: 46b0b81220a23bc4aee5ba3ba67e8cf1b5df7960
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T10:08:19+02:00
summary:

bpo-37531: regrtest main process uses shorter timeout (GH-16220)

When using multiprocesss (-jN), the main process now uses a timeout
of 60 seconds instead of the double of the --timeout value. The
buildbot server stops a job which does not produce any output in 1200
seconds.

files:
M Lib/test/libregrtest/runtest_mp.py

diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py
index c5ac6e090376..9cb5be6bb8ae 100644
--- a/Lib/test/libregrtest/runtest_mp.py
+++ b/Lib/test/libregrtest/runtest_mp.py
@@ -20,6 +20,7 @@
 
 # Display the running tests if nothing happened last N seconds
 PROGRESS_UPDATE = 30.0   # seconds
+assert PROGRESS_UPDATE >= PROGRESS_MIN_TIME
 
 # Time to wait until a worker completes: should be immediate
 JOIN_TIMEOUT = 30.0   # seconds
@@ -305,10 +306,8 @@ def __init__(self, regrtest):
         self.pending = MultiprocessIterator(self.regrtest.tests)
         if self.ns.timeout is not None:
             self.worker_timeout = self.ns.timeout * 1.5
-            self.main_timeout = self.ns.timeout * 2.0
         else:
             self.worker_timeout = None
-            self.main_timeout = None
         self.workers = None
 
     def start_workers(self):
@@ -345,12 +344,13 @@ def _get_result(self):
             except queue.Empty:
                 return None
 
+        use_faulthandler = (self.ns.timeout is not None)
+        timeout = PROGRESS_UPDATE
         while True:
-            if self.main_timeout is not None:
-                faulthandler.dump_traceback_later(self.main_timeout, exit=True)
+            if use_faulthandler:
+                faulthandler.dump_traceback_later(timeout * 2.0, exit=True)
 
             # wait for a thread
-            timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME)
             try:
                 return self.output.get(timeout=timeout)
             except queue.Empty:
@@ -415,7 +415,7 @@ def run_tests(self):
             print()
             self.regrtest.interrupted = True
         finally:
-            if self.main_timeout is not None:
+            if self.ns.timeout is not None:
                 faulthandler.cancel_dump_traceback_later()
 
         # a test failed (and --failfast is set) or all tests completed


From webhook-mailer at python.org  Tue Sep 17 06:49:12 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 10:49:12 -0000
Subject: [Python-checkins] Doc: Fix grammar/spelling in
 ssl.VERIFY_CRL_CHECK_LEAF docs (GH-16221)
Message-ID: <mailman.810.1568717353.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6422392f34871b90ab9c77043f049b35baf4faf5
commit: 6422392f34871b90ab9c77043f049b35baf4faf5
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T03:49:08-07:00
summary:

Doc: Fix grammar/spelling in ssl.VERIFY_CRL_CHECK_LEAF docs (GH-16221)

(cherry picked from commit 219fb9d65ef7e5363eccc9dde0988bb085db1c86)

Co-authored-by: J?rn Heissler <joernheissler at users.noreply.github.com>

files:
M Doc/library/ssl.rst

diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index b22136810d28..3730d74e05d6 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -605,9 +605,9 @@ Constants
 .. data:: VERIFY_CRL_CHECK_LEAF
 
    Possible value for :attr:`SSLContext.verify_flags`. In this mode, only the
-   peer cert is check but non of the intermediate CA certificates. The mode
+   peer cert is checked but none of the intermediate CA certificates. The mode
    requires a valid CRL that is signed by the peer cert's issuer (its direct
-   ancestor CA). If no proper has been loaded
+   ancestor CA). If no proper CRL has has been loaded with
    :attr:`SSLContext.load_verify_locations`, validation will fail.
 
    .. versionadded:: 3.4


From webhook-mailer at python.org  Tue Sep 17 06:49:24 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 10:49:24 -0000
Subject: [Python-checkins] Doc: Fix grammar/spelling in
 ssl.VERIFY_CRL_CHECK_LEAF docs (GH-16221)
Message-ID: <mailman.811.1568717365.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/728bea60e5ff2bea36f8f9a3cafe6b96e452c211
commit: 728bea60e5ff2bea36f8f9a3cafe6b96e452c211
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T03:49:20-07:00
summary:

Doc: Fix grammar/spelling in ssl.VERIFY_CRL_CHECK_LEAF docs (GH-16221)

(cherry picked from commit 219fb9d65ef7e5363eccc9dde0988bb085db1c86)

Co-authored-by: J?rn Heissler <joernheissler at users.noreply.github.com>

files:
M Doc/library/ssl.rst

diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index 9b0ba8ed1b8b..633e68ae273f 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -613,9 +613,9 @@ Constants
 .. data:: VERIFY_CRL_CHECK_LEAF
 
    Possible value for :attr:`SSLContext.verify_flags`. In this mode, only the
-   peer cert is check but non of the intermediate CA certificates. The mode
+   peer cert is checked but none of the intermediate CA certificates. The mode
    requires a valid CRL that is signed by the peer cert's issuer (its direct
-   ancestor CA). If no proper has been loaded
+   ancestor CA). If no proper CRL has has been loaded with
    :attr:`SSLContext.load_verify_locations`, validation will fail.
 
    .. versionadded:: 3.4


From webhook-mailer at python.org  Tue Sep 17 07:16:12 2019
From: webhook-mailer at python.org (Pablo Galindo)
Date: Tue, 17 Sep 2019 11:16:12 -0000
Subject: [Python-checkins] bpo-37828: Fix default mock_name in
 unittest.mock.assert_called error (GH-16166)
Message-ID: <mailman.812.1568718973.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5f5f11faf9de0d8dcbe1a8a4eb35d2a4232d6eaa
commit: 5f5f11faf9de0d8dcbe1a8a4eb35d2a4232d6eaa
branch: master
author: Abraham Toriz Cruz <awonderfulcode at gmail.com>
committer: Pablo Galindo <Pablogsal at gmail.com>
date: 2019-09-17T12:16:08+01:00
summary:

bpo-37828: Fix default mock_name in unittest.mock.assert_called error (GH-16166)

In the format string for assert_called the evaluation order is incorrect and hence for mock's without name, 'None' is printed whereas it should be 'mock' like for other messages. The error message is ("Expected '%s' to have been called." % self._mock_name or 'mock').

files:
A Misc/NEWS.d/next/Library/2019-09-15-21-31-18.bpo-37828.gLLDX7.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testmock.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 74d32af9bf99..4cf8e60ccc15 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -868,7 +868,7 @@ def assert_called(self):
         """
         if self.call_count == 0:
             msg = ("Expected '%s' to have been called." %
-                   self._mock_name or 'mock')
+                   (self._mock_name or 'mock'))
             raise AssertionError(msg)
 
     def assert_called_once(self):
diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py
index 581afaaeb815..2bafa8266b63 100644
--- a/Lib/unittest/test/testmock/testmock.py
+++ b/Lib/unittest/test/testmock/testmock.py
@@ -396,6 +396,14 @@ def _check(mock):
         _check(mock)
 
 
+    def test_assert_called_exception_message(self):
+        msg = "Expected '{0}' to have been called"
+        with self.assertRaisesRegex(AssertionError, msg.format('mock')):
+            Mock().assert_called()
+        with self.assertRaisesRegex(AssertionError, msg.format('test_name')):
+            Mock(name="test_name").assert_called()
+
+
     def test_assert_called_once_with(self):
         mock = Mock()
         mock()
diff --git a/Misc/NEWS.d/next/Library/2019-09-15-21-31-18.bpo-37828.gLLDX7.rst b/Misc/NEWS.d/next/Library/2019-09-15-21-31-18.bpo-37828.gLLDX7.rst
new file mode 100644
index 000000000000..c364009b2408
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-15-21-31-18.bpo-37828.gLLDX7.rst
@@ -0,0 +1,2 @@
+Fix default mock name in :meth:`unittest.mock.Mock.assert_called` exceptions.
+Patch by Abraham Toriz Cruz.


From webhook-mailer at python.org  Tue Sep 17 07:36:03 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 11:36:03 -0000
Subject: [Python-checkins] bpo-37828: Fix default mock_name in
 unittest.mock.assert_called error (GH-16166)
Message-ID: <mailman.813.1568720164.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/f668d2b775da4bcd07e142c4bc5ebd88165fadf4
commit: f668d2b775da4bcd07e142c4bc5ebd88165fadf4
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T04:35:56-07:00
summary:

bpo-37828: Fix default mock_name in unittest.mock.assert_called error (GH-16166)


In the format string for assert_called the evaluation order is incorrect and hence for mock's without name, 'None' is printed whereas it should be 'mock' like for other messages. The error message is ("Expected '%s' to have been called." % self._mock_name or 'mock').
(cherry picked from commit 5f5f11faf9de0d8dcbe1a8a4eb35d2a4232d6eaa)

Co-authored-by: Abraham Toriz Cruz <awonderfulcode at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-15-21-31-18.bpo-37828.gLLDX7.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testmock.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 497aa6f9b9ba..9fd5c3ce9d0a 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -873,7 +873,7 @@ def assert_called(self):
         """
         if self.call_count == 0:
             msg = ("Expected '%s' to have been called." %
-                   self._mock_name or 'mock')
+                   (self._mock_name or 'mock'))
             raise AssertionError(msg)
 
     def assert_called_once(self):
diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py
index 413ee6895101..2847d650cddc 100644
--- a/Lib/unittest/test/testmock/testmock.py
+++ b/Lib/unittest/test/testmock/testmock.py
@@ -388,6 +388,14 @@ def _check(mock):
         _check(mock)
 
 
+    def test_assert_called_exception_message(self):
+        msg = "Expected '{0}' to have been called"
+        with self.assertRaisesRegex(AssertionError, msg.format('mock')):
+            Mock().assert_called()
+        with self.assertRaisesRegex(AssertionError, msg.format('test_name')):
+            Mock(name="test_name").assert_called()
+
+
     def test_assert_called_once_with(self):
         mock = Mock()
         mock()
diff --git a/Misc/NEWS.d/next/Library/2019-09-15-21-31-18.bpo-37828.gLLDX7.rst b/Misc/NEWS.d/next/Library/2019-09-15-21-31-18.bpo-37828.gLLDX7.rst
new file mode 100644
index 000000000000..c364009b2408
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-15-21-31-18.bpo-37828.gLLDX7.rst
@@ -0,0 +1,2 @@
+Fix default mock name in :meth:`unittest.mock.Mock.assert_called` exceptions.
+Patch by Abraham Toriz Cruz.


From webhook-mailer at python.org  Tue Sep 17 08:34:17 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 17 Sep 2019 12:34:17 -0000
Subject: [Python-checkins] bpo-37531: regrtest main process uses shorter
 timeout (GH-16220) (GH-16224)
Message-ID: <mailman.814.1568723659.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5f1590d5e679f4dd0b611ef54ae512f137e78f1b
commit: 5f1590d5e679f4dd0b611ef54ae512f137e78f1b
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-17T14:34:13+02:00
summary:

bpo-37531: regrtest main process uses shorter timeout (GH-16220) (GH-16224)

When using multiprocesss (-jN), the main process now uses a timeout
of 60 seconds instead of the double of the --timeout value. The
buildbot server stops a job which does not produce any output in 1200
seconds.
(cherry picked from commit 46b0b81220a23bc4aee5ba3ba67e8cf1b5df7960)

Co-authored-by: Victor Stinner <vstinner at redhat.com>

files:
M Lib/test/libregrtest/runtest_mp.py

diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py
index c22479b7976f..482bb80726dc 100644
--- a/Lib/test/libregrtest/runtest_mp.py
+++ b/Lib/test/libregrtest/runtest_mp.py
@@ -20,6 +20,7 @@
 
 # Display the running tests if nothing happened last N seconds
 PROGRESS_UPDATE = 30.0   # seconds
+assert PROGRESS_UPDATE >= PROGRESS_MIN_TIME
 
 # Time to wait until a worker completes: should be immediate
 JOIN_TIMEOUT = 30.0   # seconds
@@ -305,10 +306,8 @@ def __init__(self, regrtest):
         self.pending = MultiprocessIterator(self.regrtest.tests)
         if self.ns.timeout is not None:
             self.worker_timeout = self.ns.timeout * 1.5
-            self.main_timeout = self.ns.timeout * 2.0
         else:
             self.worker_timeout = None
-            self.main_timeout = None
         self.workers = None
 
     def start_workers(self):
@@ -345,12 +344,13 @@ def _get_result(self):
             except queue.Empty:
                 return None
 
+        use_faulthandler = (self.ns.timeout is not None)
+        timeout = PROGRESS_UPDATE
         while True:
-            if self.main_timeout is not None:
-                faulthandler.dump_traceback_later(self.main_timeout, exit=True)
+            if use_faulthandler:
+                faulthandler.dump_traceback_later(timeout * 2.0, exit=True)
 
             # wait for a thread
-            timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME)
             try:
                 return self.output.get(timeout=timeout)
             except queue.Empty:
@@ -415,7 +415,7 @@ def run_tests(self):
             print()
             self.regrtest.interrupted = True
         finally:
-            if self.main_timeout is not None:
+            if self.ns.timeout is not None:
                 faulthandler.cancel_dump_traceback_later()
 
         # a test failed (and --failfast is set) or all tests completed


From webhook-mailer at python.org  Tue Sep 17 08:34:19 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 17 Sep 2019 12:34:19 -0000
Subject: [Python-checkins] bpo-37531: regrtest main process uses shorter
 timeout (GH-16220) (GH-16223)
Message-ID: <mailman.815.1568723660.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6591b4bbb1c0b9c26b99e4b2dba1e5cc8546732d
commit: 6591b4bbb1c0b9c26b99e4b2dba1e5cc8546732d
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-17T14:34:03+02:00
summary:

bpo-37531: regrtest main process uses shorter timeout (GH-16220) (GH-16223)

When using multiprocesss (-jN), the main process now uses a timeout
of 60 seconds instead of the double of the --timeout value. The
buildbot server stops a job which does not produce any output in 1200
seconds.
(cherry picked from commit 46b0b81220a23bc4aee5ba3ba67e8cf1b5df7960)

Co-authored-by: Victor Stinner <vstinner at redhat.com>

files:
M Lib/test/libregrtest/runtest_mp.py

diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py
index c22479b7976f..482bb80726dc 100644
--- a/Lib/test/libregrtest/runtest_mp.py
+++ b/Lib/test/libregrtest/runtest_mp.py
@@ -20,6 +20,7 @@
 
 # Display the running tests if nothing happened last N seconds
 PROGRESS_UPDATE = 30.0   # seconds
+assert PROGRESS_UPDATE >= PROGRESS_MIN_TIME
 
 # Time to wait until a worker completes: should be immediate
 JOIN_TIMEOUT = 30.0   # seconds
@@ -305,10 +306,8 @@ def __init__(self, regrtest):
         self.pending = MultiprocessIterator(self.regrtest.tests)
         if self.ns.timeout is not None:
             self.worker_timeout = self.ns.timeout * 1.5
-            self.main_timeout = self.ns.timeout * 2.0
         else:
             self.worker_timeout = None
-            self.main_timeout = None
         self.workers = None
 
     def start_workers(self):
@@ -345,12 +344,13 @@ def _get_result(self):
             except queue.Empty:
                 return None
 
+        use_faulthandler = (self.ns.timeout is not None)
+        timeout = PROGRESS_UPDATE
         while True:
-            if self.main_timeout is not None:
-                faulthandler.dump_traceback_later(self.main_timeout, exit=True)
+            if use_faulthandler:
+                faulthandler.dump_traceback_later(timeout * 2.0, exit=True)
 
             # wait for a thread
-            timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME)
             try:
                 return self.output.get(timeout=timeout)
             except queue.Empty:
@@ -415,7 +415,7 @@ def run_tests(self):
             print()
             self.regrtest.interrupted = True
         finally:
-            if self.main_timeout is not None:
+            if self.ns.timeout is not None:
                 faulthandler.cancel_dump_traceback_later()
 
         # a test failed (and --failfast is set) or all tests completed


From webhook-mailer at python.org  Tue Sep 17 08:59:54 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 12:59:54 -0000
Subject: [Python-checkins] bpo-38013: make async_generator_athrow object
 tolerant to throwing exceptions (GH-16070)
Message-ID: <mailman.816.1568725195.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c275312a6284bd319ea33c9abd7e15c230eca43f
commit: c275312a6284bd319ea33c9abd7e15c230eca43f
branch: master
author: Andrew Svetlov <andrew.svetlov at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-17T05:59:49-07:00
summary:

bpo-38013: make async_generator_athrow object tolerant to throwing exceptions (GH-16070)



Even when the helper is not started yet.

This behavior follows conventional generator one.
There is no reason for `async_generator_athrow` to handle `gen.throw()` differently.



https://bugs.python.org/issue38013

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-12-19-50-01.bpo-38013.I7btD0.rst
M Lib/test/test_asyncgen.py
M Objects/genobject.c

diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py
index 71b0968c7944..3a8d5fd4007f 100644
--- a/Lib/test/test_asyncgen.py
+++ b/Lib/test/test_asyncgen.py
@@ -1125,6 +1125,28 @@ def make_arange(n):
         res = self.loop.run_until_complete(run())
         self.assertEqual(res, [i * 2 for i in range(1, 10)])
 
+    def test_asyncgen_nonstarted_hooks_are_cancellable(self):
+        # See https://bugs.python.org/issue38013
+        messages = []
+
+        def exception_handler(loop, context):
+            messages.append(context)
+
+        async def async_iterate():
+            yield 1
+            yield 2
+
+        async def main():
+            loop = asyncio.get_running_loop()
+            loop.set_exception_handler(exception_handler)
+
+            async for i in async_iterate():
+                break
+
+        asyncio.run(main())
+
+        self.assertEqual([], messages)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-12-19-50-01.bpo-38013.I7btD0.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-12-19-50-01.bpo-38013.I7btD0.rst
new file mode 100644
index 000000000000..a61aa48ff1ef
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-12-19-50-01.bpo-38013.I7btD0.rst	
@@ -0,0 +1,3 @@
+Allow to call ``async_generator_athrow().throw(...)`` even for non-started
+async generator helper. It fixes annoying warning at the end of
+:func:`asyncio.run` call.
diff --git a/Objects/genobject.c b/Objects/genobject.c
index 9f490b4e2e48..82e6e55868f6 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -1884,11 +1884,6 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *args)
 {
     PyObject *retval;
 
-    if (o->agt_state == AWAITABLE_STATE_INIT) {
-        PyErr_SetString(PyExc_RuntimeError, NON_INIT_CORO_MSG);
-        return NULL;
-    }
-
     if (o->agt_state == AWAITABLE_STATE_CLOSED) {
         PyErr_SetNone(PyExc_StopIteration);
         return NULL;


From webhook-mailer at python.org  Tue Sep 17 09:19:33 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 13:19:33 -0000
Subject: [Python-checkins] bpo-38013: make async_generator_athrow object
 tolerant to throwing exceptions (GH-16070)
Message-ID: <mailman.817.1568726373.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/fc022f04b41a79cacdff380435c30c8042c82b99
commit: fc022f04b41a79cacdff380435c30c8042c82b99
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T06:19:25-07:00
summary:

bpo-38013: make async_generator_athrow object tolerant to throwing exceptions (GH-16070)


Even when the helper is not started yet.

This behavior follows conventional generator one.
There is no reason for `async_generator_athrow` to handle `gen.throw()` differently.

https://bugs.python.org/issue38013
(cherry picked from commit c275312a6284bd319ea33c9abd7e15c230eca43f)

Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-12-19-50-01.bpo-38013.I7btD0.rst
M Lib/test/test_asyncgen.py
M Objects/genobject.c

diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py
index 5a36423dc97a..94c2633f330a 100644
--- a/Lib/test/test_asyncgen.py
+++ b/Lib/test/test_asyncgen.py
@@ -1068,6 +1068,28 @@ def make_arange(n):
         res = self.loop.run_until_complete(run())
         self.assertEqual(res, [i * 2 for i in range(1, 10)])
 
+    def test_asyncgen_nonstarted_hooks_are_cancellable(self):
+        # See https://bugs.python.org/issue38013
+        messages = []
+
+        def exception_handler(loop, context):
+            messages.append(context)
+
+        async def async_iterate():
+            yield 1
+            yield 2
+
+        async def main():
+            loop = asyncio.get_running_loop()
+            loop.set_exception_handler(exception_handler)
+
+            async for i in async_iterate():
+                break
+
+        asyncio.run(main())
+
+        self.assertEqual([], messages)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-12-19-50-01.bpo-38013.I7btD0.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-12-19-50-01.bpo-38013.I7btD0.rst
new file mode 100644
index 000000000000..a61aa48ff1ef
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-12-19-50-01.bpo-38013.I7btD0.rst	
@@ -0,0 +1,3 @@
+Allow to call ``async_generator_athrow().throw(...)`` even for non-started
+async generator helper. It fixes annoying warning at the end of
+:func:`asyncio.run` call.
diff --git a/Objects/genobject.c b/Objects/genobject.c
index b0e877072c78..8a978b1a1fe0 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -1905,11 +1905,6 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *args)
 {
     PyObject *retval;
 
-    if (o->agt_state == AWAITABLE_STATE_INIT) {
-        PyErr_SetString(PyExc_RuntimeError, NON_INIT_CORO_MSG);
-        return NULL;
-    }
-
     if (o->agt_state == AWAITABLE_STATE_CLOSED) {
         PyErr_SetNone(PyExc_StopIteration);
         return NULL;


From webhook-mailer at python.org  Tue Sep 17 09:20:11 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 13:20:11 -0000
Subject: [Python-checkins] bpo-38013: make async_generator_athrow object
 tolerant to throwing exceptions (GH-16070)
Message-ID: <mailman.818.1568726412.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3c1786f18b1542e71454f37e3f3ca1ef3eec0e5f
commit: 3c1786f18b1542e71454f37e3f3ca1ef3eec0e5f
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T06:20:06-07:00
summary:

bpo-38013: make async_generator_athrow object tolerant to throwing exceptions (GH-16070)


Even when the helper is not started yet.

This behavior follows conventional generator one.
There is no reason for `async_generator_athrow` to handle `gen.throw()` differently.

https://bugs.python.org/issue38013
(cherry picked from commit c275312a6284bd319ea33c9abd7e15c230eca43f)

Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-12-19-50-01.bpo-38013.I7btD0.rst
M Lib/test/test_asyncgen.py
M Objects/genobject.c

diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py
index 71b0968c7944..3a8d5fd4007f 100644
--- a/Lib/test/test_asyncgen.py
+++ b/Lib/test/test_asyncgen.py
@@ -1125,6 +1125,28 @@ def make_arange(n):
         res = self.loop.run_until_complete(run())
         self.assertEqual(res, [i * 2 for i in range(1, 10)])
 
+    def test_asyncgen_nonstarted_hooks_are_cancellable(self):
+        # See https://bugs.python.org/issue38013
+        messages = []
+
+        def exception_handler(loop, context):
+            messages.append(context)
+
+        async def async_iterate():
+            yield 1
+            yield 2
+
+        async def main():
+            loop = asyncio.get_running_loop()
+            loop.set_exception_handler(exception_handler)
+
+            async for i in async_iterate():
+                break
+
+        asyncio.run(main())
+
+        self.assertEqual([], messages)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-12-19-50-01.bpo-38013.I7btD0.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-12-19-50-01.bpo-38013.I7btD0.rst
new file mode 100644
index 000000000000..a61aa48ff1ef
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-12-19-50-01.bpo-38013.I7btD0.rst	
@@ -0,0 +1,3 @@
+Allow to call ``async_generator_athrow().throw(...)`` even for non-started
+async generator helper. It fixes annoying warning at the end of
+:func:`asyncio.run` call.
diff --git a/Objects/genobject.c b/Objects/genobject.c
index 2d9a2860a3d2..f1e9fec78a46 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -1900,11 +1900,6 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *args)
 {
     PyObject *retval;
 
-    if (o->agt_state == AWAITABLE_STATE_INIT) {
-        PyErr_SetString(PyExc_RuntimeError, NON_INIT_CORO_MSG);
-        return NULL;
-    }
-
     if (o->agt_state == AWAITABLE_STATE_CLOSED) {
         PyErr_SetNone(PyExc_StopIteration);
         return NULL;


From webhook-mailer at python.org  Tue Sep 17 12:05:07 2019
From: webhook-mailer at python.org (Pablo Galindo)
Date: Tue, 17 Sep 2019 16:05:07 -0000
Subject: [Python-checkins] bpo-38187: Fix reference leak in test_tools
 (GH-16233)
Message-ID: <mailman.819.1568736308.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/6fbc924696b4b5097c273c06ca2d82662940e184
commit: 6fbc924696b4b5097c273c06ca2d82662940e184
branch: master
author: Pablo Galindo <Pablogsal at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T17:04:46+01:00
summary:

bpo-38187: Fix reference leak in test_tools (GH-16233)

files:
M Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py
M Tools/c-analyzer/c_parser/info.py

diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py
index 215023da577b..93100e0438cd 100644
--- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py
@@ -15,6 +15,9 @@ class FromFileTests(unittest.TestCase):
 
     _return_read_tsv = ()
 
+    def tearDown(self):
+        Variable._isglobal.instances.clear()
+
     @property
     def calls(self):
         try:
diff --git a/Tools/c-analyzer/c_parser/info.py b/Tools/c-analyzer/c_parser/info.py
index 9ab697978638..d7368b48cde3 100644
--- a/Tools/c-analyzer/c_parser/info.py
+++ b/Tools/c-analyzer/c_parser/info.py
@@ -22,6 +22,9 @@ class Variable(_NTBase,
     __slots__ = ()
     _isglobal = util.Slot()
 
+    def __del__(self):
+        del self._isglobal
+
     @classonly
     def from_parts(cls, filename, funcname, name, vartype, isglobal=False):
         id = info.ID(filename, funcname, name)


From webhook-mailer at python.org  Tue Sep 17 12:58:02 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 16:58:02 -0000
Subject: [Python-checkins] bpo-37904: Edition on python tutorial - section 4
 (GH-16169)
Message-ID: <mailman.820.1568739483.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b57481318e3e3cbacd398b898f9849ec8f2d7eec
commit: b57481318e3e3cbacd398b898f9849ec8f2d7eec
branch: master
author: Diego Alberto Barriga Mart?nez <diegobarriga at protonmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-17T09:57:55-07:00
summary:

bpo-37904: Edition on python tutorial - section 4 (GH-16169)



A little change on first paragraph of python tutorial to be more clearly



https://bugs.python.org/issue37904



Automerge-Triggered-By: @ericvsmith

files:
M Doc/tutorial/controlflow.rst

diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst
index 3730a215d748..7dfd33af2588 100644
--- a/Doc/tutorial/controlflow.rst
+++ b/Doc/tutorial/controlflow.rst
@@ -4,8 +4,8 @@
 More Control Flow Tools
 ***********************
 
-Besides the :keyword:`while` statement just introduced, Python knows the usual
-control flow statements known from other languages, with some twists.
+Besides the :keyword:`while` statement just introduced, Python uses the usual
+flow control statements known from other languages, with some twists.
 
 
 .. _tut-if:


From webhook-mailer at python.org  Tue Sep 17 14:22:05 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Tue, 17 Sep 2019 18:22:05 -0000
Subject: [Python-checkins] bpo-38191: Accept arbitrary keyword names in
 NamedTuple() and TypedDict(). (GH-16222)
Message-ID: <mailman.821.1568744526.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2bf31ccab3d17f3f35b42dca97f99576dfe2fc7d
commit: 2bf31ccab3d17f3f35b42dca97f99576dfe2fc7d
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T21:22:00+03:00
summary:

bpo-38191: Accept arbitrary keyword names in NamedTuple() and TypedDict(). (GH-16222)

This includes such names as "cls", "self", "typename", "_typename",
"fields" and "_fields".
Passing positional arguments by keyword is deprecated.

files:
A Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst
M Lib/test/test_typing.py
M Lib/typing.py

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index 59b1b4e6d548..5914f314db0a 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -3563,6 +3563,36 @@ def test_namedtuple_keyword_usage(self):
         with self.assertRaises(TypeError):
             NamedTuple('Name', x=1, y='a')
 
+    def test_namedtuple_special_keyword_names(self):
+        NT = NamedTuple("NT", cls=type, self=object, typename=str, fields=list)
+        self.assertEqual(NT.__name__, 'NT')
+        self.assertEqual(NT._fields, ('cls', 'self', 'typename', 'fields'))
+        a = NT(cls=str, self=42, typename='foo', fields=[('bar', tuple)])
+        self.assertEqual(a.cls, str)
+        self.assertEqual(a.self, 42)
+        self.assertEqual(a.typename, 'foo')
+        self.assertEqual(a.fields, [('bar', tuple)])
+
+    def test_namedtuple_errors(self):
+        with self.assertRaises(TypeError):
+            NamedTuple.__new__()
+        with self.assertRaises(TypeError):
+            NamedTuple()
+        with self.assertRaises(TypeError):
+            NamedTuple('Emp', [('name', str)], None)
+        with self.assertRaises(ValueError):
+            NamedTuple('Emp', [('_name', str)])
+
+        with self.assertWarns(DeprecationWarning):
+            Emp = NamedTuple(typename='Emp', name=str, id=int)
+        self.assertEqual(Emp.__name__, 'Emp')
+        self.assertEqual(Emp._fields, ('name', 'id'))
+
+        with self.assertWarns(DeprecationWarning):
+            Emp = NamedTuple('Emp', fields=[('name', str), ('id', int)])
+        self.assertEqual(Emp.__name__, 'Emp')
+        self.assertEqual(Emp._fields, ('name', 'id'))
+
     def test_pickle(self):
         global Emp  # pickle wants to reference the class by name
         Emp = NamedTuple('Emp', [('name', str), ('id', int)])
@@ -3604,6 +3634,36 @@ def test_basics_keywords_syntax(self):
         self.assertEqual(Emp.__annotations__, {'name': str, 'id': int})
         self.assertEqual(Emp.__total__, True)
 
+    def test_typeddict_special_keyword_names(self):
+        TD = TypedDict("TD", cls=type, self=object, typename=str, _typename=int, fields=list, _fields=dict)
+        self.assertEqual(TD.__name__, 'TD')
+        self.assertEqual(TD.__annotations__, {'cls': type, 'self': object, 'typename': str, '_typename': int, 'fields': list, '_fields': dict})
+        a = TD(cls=str, self=42, typename='foo', _typename=53, fields=[('bar', tuple)], _fields={'baz', set})
+        self.assertEqual(a['cls'], str)
+        self.assertEqual(a['self'], 42)
+        self.assertEqual(a['typename'], 'foo')
+        self.assertEqual(a['_typename'], 53)
+        self.assertEqual(a['fields'], [('bar', tuple)])
+        self.assertEqual(a['_fields'], {'baz', set})
+
+    def test_typeddict_create_errors(self):
+        with self.assertRaises(TypeError):
+            TypedDict.__new__()
+        with self.assertRaises(TypeError):
+            TypedDict()
+        with self.assertRaises(TypeError):
+            TypedDict('Emp', [('name', str)], None)
+
+        with self.assertWarns(DeprecationWarning):
+            Emp = TypedDict(_typename='Emp', name=str, id=int)
+        self.assertEqual(Emp.__name__, 'Emp')
+        self.assertEqual(Emp.__annotations__, {'name': str, 'id': int})
+
+        with self.assertWarns(DeprecationWarning):
+            Emp = TypedDict('Emp', _fields={'name': str, 'id': int})
+        self.assertEqual(Emp.__name__, 'Emp')
+        self.assertEqual(Emp.__annotations__, {'name': str, 'id': int})
+
     def test_typeddict_errors(self):
         Emp = TypedDict('Emp', {'name': str, 'id': int})
         self.assertEqual(TypedDict.__module__, 'typing')
diff --git a/Lib/typing.py b/Lib/typing.py
index 32011332127f..43486a7c8ba8 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -1653,35 +1653,96 @@ class Employee(NamedTuple):
     """
     _root = True
 
-    def __new__(self, typename, fields=None, **kwargs):
+    def __new__(*args, **kwargs):
+        if not args:
+            raise TypeError('NamedTuple.__new__(): not enough arguments')
+        cls, *args = args  # allow the "cls" keyword be passed
+        if args:
+            typename, *args = args # allow the "typename" keyword be passed
+        elif 'typename' in kwargs:
+            typename = kwargs.pop('typename')
+            import warnings
+            warnings.warn("Passing 'typename' as keyword argument is deprecated",
+                          DeprecationWarning, stacklevel=2)
+        else:
+            raise TypeError("NamedTuple.__new__() missing 1 required positional "
+                            "argument: 'typename'")
+        if args:
+            try:
+                fields, = args # allow the "fields" keyword be passed
+            except ValueError:
+                raise TypeError(f'NamedTuple.__new__() takes from 2 to 3 '
+                                f'positional arguments but {len(args) + 2} '
+                                f'were given') from None
+        elif 'fields' in kwargs and len(kwargs) == 1:
+            fields = kwargs.pop('fields')
+            import warnings
+            warnings.warn("Passing 'fields' as keyword argument is deprecated",
+                          DeprecationWarning, stacklevel=2)
+        else:
+            fields = None
+
         if fields is None:
             fields = kwargs.items()
         elif kwargs:
             raise TypeError("Either list of fields or keywords"
                             " can be provided to NamedTuple, not both")
         return _make_nmtuple(typename, fields)
+    __new__.__text_signature__ = '($cls, typename, fields=None, /, **kwargs)'
 
 
-def _dict_new(cls, *args, **kwargs):
+def _dict_new(*args, **kwargs):
+    if not args:
+        raise TypeError('TypedDict.__new__(): not enough arguments')
+    cls, *args = args  # allow the "cls" keyword be passed
     return dict(*args, **kwargs)
-
-
-def _typeddict_new(cls, _typename, _fields=None, **kwargs):
-    total = kwargs.pop('total', True)
-    if _fields is None:
-        _fields = kwargs
+_dict_new.__text_signature__ = '($cls, _typename, _fields=None, /, **kwargs)'
+
+
+def _typeddict_new(*args, total=True, **kwargs):
+    if not args:
+        raise TypeError('TypedDict.__new__(): not enough arguments')
+    cls, *args = args  # allow the "cls" keyword be passed
+    if args:
+        typename, *args = args # allow the "_typename" keyword be passed
+    elif '_typename' in kwargs:
+        typename = kwargs.pop('_typename')
+        import warnings
+        warnings.warn("Passing '_typename' as keyword argument is deprecated",
+                      DeprecationWarning, stacklevel=2)
+    else:
+        raise TypeError("TypedDict.__new__() missing 1 required positional "
+                        "argument: '_typename'")
+    if args:
+        try:
+            fields, = args # allow the "_fields" keyword be passed
+        except ValueError:
+            raise TypeError(f'TypedDict.__new__() takes from 2 to 3 '
+                            f'positional arguments but {len(args) + 2} '
+                            f'were given') from None
+    elif '_fields' in kwargs and len(kwargs) == 1:
+        fields = kwargs.pop('_fields')
+        import warnings
+        warnings.warn("Passing '_fields' as keyword argument is deprecated",
+                      DeprecationWarning, stacklevel=2)
+    else:
+        fields = None
+
+    if fields is None:
+        fields = kwargs
     elif kwargs:
         raise TypeError("TypedDict takes either a dict or keyword arguments,"
                         " but not both")
 
-    ns = {'__annotations__': dict(_fields), '__total__': total}
+    ns = {'__annotations__': dict(fields), '__total__': total}
     try:
         # Setting correct module is necessary to make typed dict classes pickleable.
         ns['__module__'] = sys._getframe(1).f_globals.get('__name__', '__main__')
     except (AttributeError, ValueError):
         pass
 
-    return _TypedDictMeta(_typename, (), ns)
+    return _TypedDictMeta(typename, (), ns)
+_typeddict_new.__text_signature__ = '($cls, _typename, _fields=None, /, *, total=True, **kwargs)'
 
 
 def _check_fails(cls, other):
diff --git a/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst b/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst
new file mode 100644
index 000000000000..1a6de60e46c1
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst
@@ -0,0 +1,4 @@
+Constructors of :class:`~typing.NamedTuple` and :class:`~typing.TypedDict`
+types now accept arbitrary keyword argument names, including "cls", "self",
+"typename", "_typename", "fields" and "_fields".  Passing positional
+arguments by keyword is deprecated.


From webhook-mailer at python.org  Tue Sep 17 14:41:59 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 17 Sep 2019 18:41:59 -0000
Subject: [Python-checkins] bpo-38191: Accept arbitrary keyword names in
 NamedTuple() and TypedDict(). (GH-16222)
Message-ID: <mailman.822.1568745720.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/54ba5f19d4a654768c785468d736ed0bd05947f5
commit: 54ba5f19d4a654768c785468d736ed0bd05947f5
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T11:41:55-07:00
summary:

bpo-38191: Accept arbitrary keyword names in NamedTuple() and TypedDict(). (GH-16222)


This includes such names as "cls", "self", "typename", "_typename",
"fields" and "_fields".
Passing positional arguments by keyword is deprecated.
(cherry picked from commit 2bf31ccab3d17f3f35b42dca97f99576dfe2fc7d)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst
M Lib/test/test_typing.py
M Lib/typing.py

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index 59b1b4e6d548..5914f314db0a 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -3563,6 +3563,36 @@ def test_namedtuple_keyword_usage(self):
         with self.assertRaises(TypeError):
             NamedTuple('Name', x=1, y='a')
 
+    def test_namedtuple_special_keyword_names(self):
+        NT = NamedTuple("NT", cls=type, self=object, typename=str, fields=list)
+        self.assertEqual(NT.__name__, 'NT')
+        self.assertEqual(NT._fields, ('cls', 'self', 'typename', 'fields'))
+        a = NT(cls=str, self=42, typename='foo', fields=[('bar', tuple)])
+        self.assertEqual(a.cls, str)
+        self.assertEqual(a.self, 42)
+        self.assertEqual(a.typename, 'foo')
+        self.assertEqual(a.fields, [('bar', tuple)])
+
+    def test_namedtuple_errors(self):
+        with self.assertRaises(TypeError):
+            NamedTuple.__new__()
+        with self.assertRaises(TypeError):
+            NamedTuple()
+        with self.assertRaises(TypeError):
+            NamedTuple('Emp', [('name', str)], None)
+        with self.assertRaises(ValueError):
+            NamedTuple('Emp', [('_name', str)])
+
+        with self.assertWarns(DeprecationWarning):
+            Emp = NamedTuple(typename='Emp', name=str, id=int)
+        self.assertEqual(Emp.__name__, 'Emp')
+        self.assertEqual(Emp._fields, ('name', 'id'))
+
+        with self.assertWarns(DeprecationWarning):
+            Emp = NamedTuple('Emp', fields=[('name', str), ('id', int)])
+        self.assertEqual(Emp.__name__, 'Emp')
+        self.assertEqual(Emp._fields, ('name', 'id'))
+
     def test_pickle(self):
         global Emp  # pickle wants to reference the class by name
         Emp = NamedTuple('Emp', [('name', str), ('id', int)])
@@ -3604,6 +3634,36 @@ def test_basics_keywords_syntax(self):
         self.assertEqual(Emp.__annotations__, {'name': str, 'id': int})
         self.assertEqual(Emp.__total__, True)
 
+    def test_typeddict_special_keyword_names(self):
+        TD = TypedDict("TD", cls=type, self=object, typename=str, _typename=int, fields=list, _fields=dict)
+        self.assertEqual(TD.__name__, 'TD')
+        self.assertEqual(TD.__annotations__, {'cls': type, 'self': object, 'typename': str, '_typename': int, 'fields': list, '_fields': dict})
+        a = TD(cls=str, self=42, typename='foo', _typename=53, fields=[('bar', tuple)], _fields={'baz', set})
+        self.assertEqual(a['cls'], str)
+        self.assertEqual(a['self'], 42)
+        self.assertEqual(a['typename'], 'foo')
+        self.assertEqual(a['_typename'], 53)
+        self.assertEqual(a['fields'], [('bar', tuple)])
+        self.assertEqual(a['_fields'], {'baz', set})
+
+    def test_typeddict_create_errors(self):
+        with self.assertRaises(TypeError):
+            TypedDict.__new__()
+        with self.assertRaises(TypeError):
+            TypedDict()
+        with self.assertRaises(TypeError):
+            TypedDict('Emp', [('name', str)], None)
+
+        with self.assertWarns(DeprecationWarning):
+            Emp = TypedDict(_typename='Emp', name=str, id=int)
+        self.assertEqual(Emp.__name__, 'Emp')
+        self.assertEqual(Emp.__annotations__, {'name': str, 'id': int})
+
+        with self.assertWarns(DeprecationWarning):
+            Emp = TypedDict('Emp', _fields={'name': str, 'id': int})
+        self.assertEqual(Emp.__name__, 'Emp')
+        self.assertEqual(Emp.__annotations__, {'name': str, 'id': int})
+
     def test_typeddict_errors(self):
         Emp = TypedDict('Emp', {'name': str, 'id': int})
         self.assertEqual(TypedDict.__module__, 'typing')
diff --git a/Lib/typing.py b/Lib/typing.py
index 32011332127f..43486a7c8ba8 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -1653,35 +1653,96 @@ class Employee(NamedTuple):
     """
     _root = True
 
-    def __new__(self, typename, fields=None, **kwargs):
+    def __new__(*args, **kwargs):
+        if not args:
+            raise TypeError('NamedTuple.__new__(): not enough arguments')
+        cls, *args = args  # allow the "cls" keyword be passed
+        if args:
+            typename, *args = args # allow the "typename" keyword be passed
+        elif 'typename' in kwargs:
+            typename = kwargs.pop('typename')
+            import warnings
+            warnings.warn("Passing 'typename' as keyword argument is deprecated",
+                          DeprecationWarning, stacklevel=2)
+        else:
+            raise TypeError("NamedTuple.__new__() missing 1 required positional "
+                            "argument: 'typename'")
+        if args:
+            try:
+                fields, = args # allow the "fields" keyword be passed
+            except ValueError:
+                raise TypeError(f'NamedTuple.__new__() takes from 2 to 3 '
+                                f'positional arguments but {len(args) + 2} '
+                                f'were given') from None
+        elif 'fields' in kwargs and len(kwargs) == 1:
+            fields = kwargs.pop('fields')
+            import warnings
+            warnings.warn("Passing 'fields' as keyword argument is deprecated",
+                          DeprecationWarning, stacklevel=2)
+        else:
+            fields = None
+
         if fields is None:
             fields = kwargs.items()
         elif kwargs:
             raise TypeError("Either list of fields or keywords"
                             " can be provided to NamedTuple, not both")
         return _make_nmtuple(typename, fields)
+    __new__.__text_signature__ = '($cls, typename, fields=None, /, **kwargs)'
 
 
-def _dict_new(cls, *args, **kwargs):
+def _dict_new(*args, **kwargs):
+    if not args:
+        raise TypeError('TypedDict.__new__(): not enough arguments')
+    cls, *args = args  # allow the "cls" keyword be passed
     return dict(*args, **kwargs)
-
-
-def _typeddict_new(cls, _typename, _fields=None, **kwargs):
-    total = kwargs.pop('total', True)
-    if _fields is None:
-        _fields = kwargs
+_dict_new.__text_signature__ = '($cls, _typename, _fields=None, /, **kwargs)'
+
+
+def _typeddict_new(*args, total=True, **kwargs):
+    if not args:
+        raise TypeError('TypedDict.__new__(): not enough arguments')
+    cls, *args = args  # allow the "cls" keyword be passed
+    if args:
+        typename, *args = args # allow the "_typename" keyword be passed
+    elif '_typename' in kwargs:
+        typename = kwargs.pop('_typename')
+        import warnings
+        warnings.warn("Passing '_typename' as keyword argument is deprecated",
+                      DeprecationWarning, stacklevel=2)
+    else:
+        raise TypeError("TypedDict.__new__() missing 1 required positional "
+                        "argument: '_typename'")
+    if args:
+        try:
+            fields, = args # allow the "_fields" keyword be passed
+        except ValueError:
+            raise TypeError(f'TypedDict.__new__() takes from 2 to 3 '
+                            f'positional arguments but {len(args) + 2} '
+                            f'were given') from None
+    elif '_fields' in kwargs and len(kwargs) == 1:
+        fields = kwargs.pop('_fields')
+        import warnings
+        warnings.warn("Passing '_fields' as keyword argument is deprecated",
+                      DeprecationWarning, stacklevel=2)
+    else:
+        fields = None
+
+    if fields is None:
+        fields = kwargs
     elif kwargs:
         raise TypeError("TypedDict takes either a dict or keyword arguments,"
                         " but not both")
 
-    ns = {'__annotations__': dict(_fields), '__total__': total}
+    ns = {'__annotations__': dict(fields), '__total__': total}
     try:
         # Setting correct module is necessary to make typed dict classes pickleable.
         ns['__module__'] = sys._getframe(1).f_globals.get('__name__', '__main__')
     except (AttributeError, ValueError):
         pass
 
-    return _TypedDictMeta(_typename, (), ns)
+    return _TypedDictMeta(typename, (), ns)
+_typeddict_new.__text_signature__ = '($cls, _typename, _fields=None, /, *, total=True, **kwargs)'
 
 
 def _check_fails(cls, other):
diff --git a/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst b/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst
new file mode 100644
index 000000000000..1a6de60e46c1
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst
@@ -0,0 +1,4 @@
+Constructors of :class:`~typing.NamedTuple` and :class:`~typing.TypedDict`
+types now accept arbitrary keyword argument names, including "cls", "self",
+"typename", "_typename", "fields" and "_fields".  Passing positional
+arguments by keyword is deprecated.


From webhook-mailer at python.org  Tue Sep 17 15:09:16 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Tue, 17 Sep 2019 19:09:16 -0000
Subject: [Python-checkins] [3.7] bpo-38191: Accept arbitrary keyword names
 in NamedTuple(). (GH-16222) (GH-16239)
Message-ID: <mailman.823.1568747357.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/69b3718b183a698202bd67488639bffd64a488bc
commit: 69b3718b183a698202bd67488639bffd64a488bc
branch: 3.7
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T22:09:09+03:00
summary:

[3.7] bpo-38191: Accept arbitrary keyword names in NamedTuple(). (GH-16222) (GH-16239)

This includes such names as "cls", "self", "typename" and "fields".
(cherry picked from commit 2bf31ccab3d17f3f35b42dca97f99576dfe2fc7d)

files:
A Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst
M Lib/test/test_typing.py
M Lib/typing.py

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index b396283a02ee..4871fb7c4ddc 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -2536,6 +2536,34 @@ def test_namedtuple_keyword_usage(self):
         with self.assertRaises(TypeError):
             NamedTuple('Name', x=1, y='a')
 
+    def test_namedtuple_special_keyword_names(self):
+        NT = NamedTuple("NT", cls=type, self=object, typename=str, fields=list)
+        self.assertEqual(NT.__name__, 'NT')
+        self.assertEqual(NT._fields, ('cls', 'self', 'typename', 'fields'))
+        a = NT(cls=str, self=42, typename='foo', fields=[('bar', tuple)])
+        self.assertEqual(a.cls, str)
+        self.assertEqual(a.self, 42)
+        self.assertEqual(a.typename, 'foo')
+        self.assertEqual(a.fields, [('bar', tuple)])
+
+    def test_namedtuple_errors(self):
+        with self.assertRaises(TypeError):
+            NamedTuple.__new__()
+        with self.assertRaises(TypeError):
+            NamedTuple()
+        with self.assertRaises(TypeError):
+            NamedTuple('Emp', [('name', str)], None)
+        with self.assertRaises(ValueError):
+            NamedTuple('Emp', [('_name', str)])
+
+        Emp = NamedTuple(typename='Emp', name=str, id=int)
+        self.assertEqual(Emp.__name__, 'Emp')
+        self.assertEqual(Emp._fields, ('name', 'id'))
+
+        Emp = NamedTuple('Emp', fields=[('name', str), ('id', int)])
+        self.assertEqual(Emp.__name__, 'Emp')
+        self.assertEqual(Emp._fields, ('name', 'id'))
+
     def test_pickle(self):
         global Emp  # pickle wants to reference the class by name
         Emp = NamedTuple('Emp', [('name', str), ('id', int)])
diff --git a/Lib/typing.py b/Lib/typing.py
index 9851cb4c7ebd..021ef9552e65 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -1405,13 +1405,36 @@ class Employee(NamedTuple):
     """
     _root = True
 
-    def __new__(self, typename, fields=None, **kwargs):
+    def __new__(*args, **kwargs):
+        if not args:
+            raise TypeError('NamedTuple.__new__(): not enough arguments')
+        cls, *args = args  # allow the "cls" keyword be passed
+        if args:
+            typename, *args = args # allow the "typename" keyword be passed
+        elif 'typename' in kwargs:
+            typename = kwargs.pop('typename')
+        else:
+            raise TypeError("NamedTuple.__new__() missing 1 required positional "
+                            "argument: 'typename'")
+        if args:
+            try:
+                fields, = args # allow the "fields" keyword be passed
+            except ValueError:
+                raise TypeError(f'NamedTuple.__new__() takes from 2 to 3 '
+                                f'positional arguments but {len(args) + 2} '
+                                f'were given') from None
+        elif 'fields' in kwargs and len(kwargs) == 1:
+            fields = kwargs.pop('fields')
+        else:
+            fields = None
+
         if fields is None:
             fields = kwargs.items()
         elif kwargs:
             raise TypeError("Either list of fields or keywords"
                             " can be provided to NamedTuple, not both")
         return _make_nmtuple(typename, fields)
+    __new__.__text_signature__ = '($cls, typename, fields=None, /, **kwargs)'
 
 
 def NewType(name, tp):
diff --git a/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst b/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst
new file mode 100644
index 000000000000..15e8fbb41ade
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst
@@ -0,0 +1,2 @@
+Constructor of :class:`~typing.NamedTuple` type now accepts arbitrary keyword
+argument names, including "cls", "self", "typename" and "fields".


From webhook-mailer at python.org  Tue Sep 17 15:42:00 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Tue, 17 Sep 2019 19:42:00 -0000
Subject: [Python-checkins] bpo-38191: Turn warnings into errors in
 NamedTuple() and TypedDict(). (GH-16238)
Message-ID: <mailman.824.1568749321.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8fc5839a9def34c13b6025c291434ba5fb5d6442
commit: 8fc5839a9def34c13b6025c291434ba5fb5d6442
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T22:41:55+03:00
summary:

bpo-38191: Turn warnings into errors in NamedTuple() and TypedDict(). (GH-16238)

files:
M Lib/test/test_typing.py
M Lib/typing.py
M Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index 5914f314db0a..97e6d88ebf70 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -3582,16 +3582,10 @@ def test_namedtuple_errors(self):
             NamedTuple('Emp', [('name', str)], None)
         with self.assertRaises(ValueError):
             NamedTuple('Emp', [('_name', str)])
-
-        with self.assertWarns(DeprecationWarning):
-            Emp = NamedTuple(typename='Emp', name=str, id=int)
-        self.assertEqual(Emp.__name__, 'Emp')
-        self.assertEqual(Emp._fields, ('name', 'id'))
-
-        with self.assertWarns(DeprecationWarning):
-            Emp = NamedTuple('Emp', fields=[('name', str), ('id', int)])
-        self.assertEqual(Emp.__name__, 'Emp')
-        self.assertEqual(Emp._fields, ('name', 'id'))
+        with self.assertRaises(TypeError):
+            NamedTuple(typename='Emp', name=str, id=int)
+        with self.assertRaises(TypeError):
+            NamedTuple('Emp', fields=[('name', str), ('id', int)])
 
     def test_pickle(self):
         global Emp  # pickle wants to reference the class by name
@@ -3654,15 +3648,10 @@ def test_typeddict_create_errors(self):
         with self.assertRaises(TypeError):
             TypedDict('Emp', [('name', str)], None)
 
-        with self.assertWarns(DeprecationWarning):
-            Emp = TypedDict(_typename='Emp', name=str, id=int)
-        self.assertEqual(Emp.__name__, 'Emp')
-        self.assertEqual(Emp.__annotations__, {'name': str, 'id': int})
-
-        with self.assertWarns(DeprecationWarning):
-            Emp = TypedDict('Emp', _fields={'name': str, 'id': int})
-        self.assertEqual(Emp.__name__, 'Emp')
-        self.assertEqual(Emp.__annotations__, {'name': str, 'id': int})
+        with self.assertRaises(TypeError):
+            TypedDict(_typename='Emp', name=str, id=int)
+        with self.assertRaises(TypeError):
+            TypedDict('Emp', _fields={'name': str, 'id': int})
 
     def test_typeddict_errors(self):
         Emp = TypedDict('Emp', {'name': str, 'id': int})
diff --git a/Lib/typing.py b/Lib/typing.py
index 43486a7c8ba8..b1ac33e00e70 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -1653,81 +1653,20 @@ class Employee(NamedTuple):
     """
     _root = True
 
-    def __new__(*args, **kwargs):
-        if not args:
-            raise TypeError('NamedTuple.__new__(): not enough arguments')
-        cls, *args = args  # allow the "cls" keyword be passed
-        if args:
-            typename, *args = args # allow the "typename" keyword be passed
-        elif 'typename' in kwargs:
-            typename = kwargs.pop('typename')
-            import warnings
-            warnings.warn("Passing 'typename' as keyword argument is deprecated",
-                          DeprecationWarning, stacklevel=2)
-        else:
-            raise TypeError("NamedTuple.__new__() missing 1 required positional "
-                            "argument: 'typename'")
-        if args:
-            try:
-                fields, = args # allow the "fields" keyword be passed
-            except ValueError:
-                raise TypeError(f'NamedTuple.__new__() takes from 2 to 3 '
-                                f'positional arguments but {len(args) + 2} '
-                                f'were given') from None
-        elif 'fields' in kwargs and len(kwargs) == 1:
-            fields = kwargs.pop('fields')
-            import warnings
-            warnings.warn("Passing 'fields' as keyword argument is deprecated",
-                          DeprecationWarning, stacklevel=2)
-        else:
-            fields = None
-
+    def __new__(cls, typename, fields=None, /, **kwargs):
         if fields is None:
             fields = kwargs.items()
         elif kwargs:
             raise TypeError("Either list of fields or keywords"
                             " can be provided to NamedTuple, not both")
         return _make_nmtuple(typename, fields)
-    __new__.__text_signature__ = '($cls, typename, fields=None, /, **kwargs)'
 
 
-def _dict_new(*args, **kwargs):
-    if not args:
-        raise TypeError('TypedDict.__new__(): not enough arguments')
-    cls, *args = args  # allow the "cls" keyword be passed
+def _dict_new(cls, /, *args, **kwargs):
     return dict(*args, **kwargs)
-_dict_new.__text_signature__ = '($cls, _typename, _fields=None, /, **kwargs)'
-
-
-def _typeddict_new(*args, total=True, **kwargs):
-    if not args:
-        raise TypeError('TypedDict.__new__(): not enough arguments')
-    cls, *args = args  # allow the "cls" keyword be passed
-    if args:
-        typename, *args = args # allow the "_typename" keyword be passed
-    elif '_typename' in kwargs:
-        typename = kwargs.pop('_typename')
-        import warnings
-        warnings.warn("Passing '_typename' as keyword argument is deprecated",
-                      DeprecationWarning, stacklevel=2)
-    else:
-        raise TypeError("TypedDict.__new__() missing 1 required positional "
-                        "argument: '_typename'")
-    if args:
-        try:
-            fields, = args # allow the "_fields" keyword be passed
-        except ValueError:
-            raise TypeError(f'TypedDict.__new__() takes from 2 to 3 '
-                            f'positional arguments but {len(args) + 2} '
-                            f'were given') from None
-    elif '_fields' in kwargs and len(kwargs) == 1:
-        fields = kwargs.pop('_fields')
-        import warnings
-        warnings.warn("Passing '_fields' as keyword argument is deprecated",
-                      DeprecationWarning, stacklevel=2)
-    else:
-        fields = None
 
+
+def _typeddict_new(cls, typename, fields=None, /, *, total=True, **kwargs):
     if fields is None:
         fields = kwargs
     elif kwargs:
@@ -1742,7 +1681,6 @@ def _typeddict_new(*args, total=True, **kwargs):
         pass
 
     return _TypedDictMeta(typename, (), ns)
-_typeddict_new.__text_signature__ = '($cls, _typename, _fields=None, /, *, total=True, **kwargs)'
 
 
 def _check_fails(cls, other):
diff --git a/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst b/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst
index 1a6de60e46c1..05c7daaeafbe 100644
--- a/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst
+++ b/Misc/NEWS.d/next/Library/2019-09-17-12-28-27.bpo-38191.1TU0HV.rst
@@ -1,4 +1,3 @@
 Constructors of :class:`~typing.NamedTuple` and :class:`~typing.TypedDict`
 types now accept arbitrary keyword argument names, including "cls", "self",
-"typename", "_typename", "fields" and "_fields".  Passing positional
-arguments by keyword is deprecated.
+"typename", "_typename", "fields" and "_fields".


From webhook-mailer at python.org  Tue Sep 17 17:36:19 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 17 Sep 2019 21:36:19 -0000
Subject: [Python-checkins] bpo-38070: _Py_DumpTraceback() writes <no Python
 frame> (GH-16244)
Message-ID: <mailman.825.1568756181.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8fa3e1740b3f03ea65ddb68411c2238c5f98eec2
commit: 8fa3e1740b3f03ea65ddb68411c2238c5f98eec2
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T23:36:16+02:00
summary:

bpo-38070: _Py_DumpTraceback() writes <no Python frame> (GH-16244)

When a Python thread has no frame, _Py_DumpTraceback() and
_Py_DumpTracebackThreads() now write "<no Python frame>", rather than
writing nothing.

files:
M Python/traceback.c

diff --git a/Python/traceback.c b/Python/traceback.c
index f396f1ad983c..8aaee12031d2 100644
--- a/Python/traceback.c
+++ b/Python/traceback.c
@@ -797,12 +797,15 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header)
     PyFrameObject *frame;
     unsigned int depth;
 
-    if (write_header)
+    if (write_header) {
         PUTS(fd, "Stack (most recent call first):\n");
+    }
 
     frame = _PyThreadState_GetFrame(tstate);
-    if (frame == NULL)
+    if (frame == NULL) {
+        PUTS(fd, "<no Python frame>\n");
         return;
+    }
 
     depth = 0;
     while (frame != NULL) {
@@ -870,9 +873,9 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp,
            Python thread state of the current thread.
 
            PyThreadState_Get() doesn't give the state of the thread that caused
-           the fault if the thread released the GIL, and so this function
-           cannot be used. Read the thread specific storage (TSS) instead: call
-           PyGILState_GetThisThreadState(). */
+           the fault if the thread released the GIL, and so
+           _PyThreadState_GET() cannot be used. Read the thread specific
+           storage (TSS) instead: call PyGILState_GetThisThreadState(). */
         current_tstate = PyGILState_GetThisThreadState();
     }
 


From webhook-mailer at python.org  Tue Sep 17 17:36:32 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 17 Sep 2019 21:36:32 -0000
Subject: [Python-checkins] bpo-38070: Enhance _PyObject_Dump() (GH-16243)
Message-ID: <mailman.826.1568756194.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b39afb78768418d9405c4b528c80fa968ccc974d
commit: b39afb78768418d9405c4b528c80fa968ccc974d
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T23:36:28+02:00
summary:

bpo-38070: Enhance _PyObject_Dump() (GH-16243)

_PyObject_Dump() now dumps the object address for freed objects and
objects with ob_type=NULL.

files:
M Objects/object.c

diff --git a/Objects/object.c b/Objects/object.c
index 7f2c23a9ff8c..43ed4c59a71b 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -464,7 +464,7 @@ void
 _PyObject_Dump(PyObject* op)
 {
     if (op == NULL) {
-        fprintf(stderr, "<NULL object>\n");
+        fprintf(stderr, "<object at NULL>\n");
         fflush(stderr);
         return;
     }
@@ -472,7 +472,7 @@ _PyObject_Dump(PyObject* op)
     if (_PyObject_IsFreed(op)) {
         /* It seems like the object memory has been freed:
            don't access it to prevent a segmentation fault. */
-        fprintf(stderr, "<Freed object>\n");
+        fprintf(stderr, "<object at %p is freed>\n", op);
         return;
     }
 
@@ -2160,18 +2160,19 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg,
     fflush(stderr);
 
     if (obj == NULL) {
-        fprintf(stderr, "<NULL object>\n");
+        fprintf(stderr, "<object at NULL>\n");
     }
     else if (_PyObject_IsFreed(obj)) {
         /* It seems like the object memory has been freed:
            don't access it to prevent a segmentation fault. */
-        fprintf(stderr, "<object: freed>\n");
+        fprintf(stderr, "<object at %p is freed>\n", obj);
     }
     else if (Py_TYPE(obj) == NULL) {
-        fprintf(stderr, "<object: ob_type=NULL>\n");
+        fprintf(stderr, "<object at %p: ob_type=NULL>\n", obj);
     }
     else if (_PyObject_IsFreed((PyObject *)Py_TYPE(obj))) {
-        fprintf(stderr, "<object: freed type %p>\n", (void *)Py_TYPE(obj));
+        fprintf(stderr, "<object at %p: type at %p is freed>\n",
+                obj, (void *)Py_TYPE(obj));
     }
     else {
         /* Display the traceback where the object has been allocated.


From webhook-mailer at python.org  Tue Sep 17 17:59:56 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 17 Sep 2019 21:59:56 -0000
Subject: [Python-checkins] bpo-38070: Add _PyRuntimeState.preinitializing
 (GH-16245)
Message-ID: <mailman.827.1568757597.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d3b904144e86e2442961de6a7dccecbe133d5c6d
commit: d3b904144e86e2442961de6a7dccecbe133d5c6d
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T23:59:51+02:00
summary:

bpo-38070: Add _PyRuntimeState.preinitializing (GH-16245)

Add _PyRuntimeState.preinitializing field: set to 1 while
Py_PreInitialize() is running.

_PyRuntimeState: rename also pre_initialized field to preinitialized.

files:
M Include/internal/pycore_pystate.h
M Python/pylifecycle.c

diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h
index 0f92f67b9779..8e5a022b4a09 100644
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -190,8 +190,11 @@ struct _gilstate_runtime_state {
 /* Full Python runtime state */
 
 typedef struct pyruntimestate {
-    /* Is Python pre-initialized? Set to 1 by Py_PreInitialize() */
-    int pre_initialized;
+    /* Is running Py_PreInitialize()? */
+    int preinitializing;
+
+    /* Is Python preinitialized? Set to 1 by Py_PreInitialize() */
+    int preinitialized;
 
     /* Is Python core initialized? Set to 1 by _Py_InitializeCore() */
     int core_initialized;
@@ -199,6 +202,8 @@ typedef struct pyruntimestate {
     /* Is Python fully initialized? Set to 1 by Py_Initialize() */
     int initialized;
 
+    /* Set by Py_FinalizeEx(). Only reset to NULL if Py_Initialize()
+       is called again. */
     PyThreadState *finalizing;
 
     struct pyinterpreters {
@@ -241,7 +246,7 @@ typedef struct pyruntimestate {
 } _PyRuntimeState;
 
 #define _PyRuntimeState_INIT \
-    {.pre_initialized = 0, .core_initialized = 0, .initialized = 0}
+    {.preinitialized = 0, .core_initialized = 0, .initialized = 0}
 /* Note: _PyRuntimeState_INIT sets other fields to 0/NULL */
 
 PyAPI_DATA(_PyRuntimeState) _PyRuntime;
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index ea89b3a626bd..1bc7f608a13a 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -725,11 +725,15 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args)
     }
     _PyRuntimeState *runtime = &_PyRuntime;
 
-    if (runtime->pre_initialized) {
+    if (runtime->preinitialized) {
         /* If it's already configured: ignored the new configuration */
         return _PyStatus_OK();
     }
 
+    /* Note: preinitialized remains 1 on error, it is only set to 0
+       at exit on success. */
+    runtime->preinitializing = 1;
+
     PyPreConfig config;
     _PyPreConfig_InitFromPreConfig(&config, src_config);
 
@@ -743,7 +747,8 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args)
         return status;
     }
 
-    runtime->pre_initialized = 1;
+    runtime->preinitializing = 0;
+    runtime->preinitialized = 1;
     return _PyStatus_OK();
 }
 
@@ -783,7 +788,7 @@ _Py_PreInitializeFromConfig(const PyConfig *config,
     }
     _PyRuntimeState *runtime = &_PyRuntime;
 
-    if (runtime->pre_initialized) {
+    if (runtime->preinitialized) {
         /* Already initialized: do nothing */
         return _PyStatus_OK();
     }


From webhook-mailer at python.org  Tue Sep 17 19:35:37 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 17 Sep 2019 23:35:37 -0000
Subject: [Python-checkins] bpo-38070: Py_FatalError() logs runtime state
 (GH-16246)
Message-ID: <mailman.828.1568763338.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1ce16fb0977283ae42a9f8917bbca5f44aa69324
commit: 1ce16fb0977283ae42a9f8917bbca5f44aa69324
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-18T01:35:33+02:00
summary:

bpo-38070: Py_FatalError() logs runtime state (GH-16246)

files:
M Lib/test/test_capi.py
M Lib/test/test_faulthandler.py
M Python/pylifecycle.c

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index a38406481a0e..b1d045c53080 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -198,6 +198,7 @@ def test_return_null_without_error(self):
             self.assertRegex(err.replace(b'\r', b''),
                              br'Fatal Python error: a function returned NULL '
                                 br'without setting an error\n'
+                             br'Python runtime state: initialized\n'
                              br'SystemError: <built-in function '
                                  br'return_null_without_error> returned NULL '
                                  br'without setting an error\n'
@@ -225,6 +226,7 @@ def test_return_result_with_error(self):
             self.assertRegex(err.replace(b'\r', b''),
                              br'Fatal Python error: a function returned a '
                                 br'result with an error set\n'
+                             br'Python runtime state: initialized\n'
                              br'ValueError\n'
                              br'\n'
                              br'The above exception was the direct cause '
diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py
index b1aa8c32284b..24487446f25b 100644
--- a/Lib/test/test_faulthandler.py
+++ b/Lib/test/test_faulthandler.py
@@ -90,7 +90,8 @@ def get_output(self, code, filename=None, fd=None):
 
     def check_error(self, code, line_number, fatal_error, *,
                     filename=None, all_threads=True, other_regex=None,
-                    fd=None, know_current_thread=True):
+                    fd=None, know_current_thread=True,
+                    py_fatal_error=False):
         """
         Check that the fault handler for fatal errors is enabled and check the
         traceback from the child process output.
@@ -110,10 +111,12 @@ def check_error(self, code, line_number, fatal_error, *,
             {header} \(most recent call first\):
               File "<string>", line {lineno} in <module>
             """
-        regex = dedent(regex.format(
+        if py_fatal_error:
+            fatal_error += "\nPython runtime state: initialized"
+        regex = dedent(regex).format(
             lineno=line_number,
             fatal_error=fatal_error,
-            header=header)).strip()
+            header=header).strip()
         if other_regex:
             regex += '|' + other_regex
         output, exitcode = self.get_output(code, filename=filename, fd=fd)
@@ -170,7 +173,8 @@ def test_fatal_error_c_thread(self):
             """,
             3,
             'in new thread',
-            know_current_thread=False)
+            know_current_thread=False,
+            py_fatal_error=True)
 
     def test_sigabrt(self):
         self.check_fatal_error("""
@@ -226,7 +230,8 @@ def test_fatal_error(self):
             faulthandler._fatal_error(b'xyz')
             """,
             2,
-            'xyz')
+            'xyz',
+            py_fatal_error=True)
 
     def test_fatal_error_without_gil(self):
         self.check_fatal_error("""
@@ -234,7 +239,8 @@ def test_fatal_error_without_gil(self):
             faulthandler._fatal_error(b'xyz', True)
             """,
             2,
-            'xyz')
+            'xyz',
+            py_fatal_error=True)
 
     @unittest.skipIf(sys.platform.startswith('openbsd'),
                      "Issue #12868: sigaltstack() doesn't work on "
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 1bc7f608a13a..f9ff8f418078 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1975,13 +1975,14 @@ init_sys_streams(PyThreadState *tstate)
 
 
 static void
-_Py_FatalError_DumpTracebacks(int fd)
+_Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp,
+                              PyThreadState *tstate)
 {
     fputc('\n', stderr);
     fflush(stderr);
 
     /* display the current Python stack */
-    _Py_DumpTracebackThreads(fd, NULL, NULL);
+    _Py_DumpTracebackThreads(fd, interp, tstate);
 }
 
 /* Print the current exception (if an exception is set) with its traceback,
@@ -2079,10 +2080,39 @@ fatal_output_debug(const char *msg)
 }
 #endif
 
+
+static void
+fatal_error_dump_runtime(FILE *stream, _PyRuntimeState *runtime)
+{
+    fprintf(stream, "Python runtime state: ");
+    if (runtime->finalizing) {
+        fprintf(stream, "finalizing (tstate=%p)", runtime->finalizing);
+    }
+    else if (runtime->initialized) {
+        fprintf(stream, "initialized");
+    }
+    else if (runtime->core_initialized) {
+        fprintf(stream, "core initialized");
+    }
+    else if (runtime->preinitialized) {
+        fprintf(stream, "preinitialized");
+    }
+    else if (runtime->preinitializing) {
+        fprintf(stream, "preinitializing");
+    }
+    else {
+        fprintf(stream, "unknown");
+    }
+    fprintf(stream, "\n");
+    fflush(stream);
+}
+
+
 static void _Py_NO_RETURN
 fatal_error(const char *prefix, const char *msg, int status)
 {
-    const int fd = fileno(stderr);
+    FILE *stream = stderr;
+    const int fd = fileno(stream);
     static int reentrant = 0;
 
     if (reentrant) {
@@ -2092,45 +2122,48 @@ fatal_error(const char *prefix, const char *msg, int status)
     }
     reentrant = 1;
 
-    fprintf(stderr, "Fatal Python error: ");
+    fprintf(stream, "Fatal Python error: ");
     if (prefix) {
-        fputs(prefix, stderr);
-        fputs(": ", stderr);
+        fputs(prefix, stream);
+        fputs(": ", stream);
     }
     if (msg) {
-        fputs(msg, stderr);
+        fputs(msg, stream);
     }
     else {
-        fprintf(stderr, "<message not set>");
+        fprintf(stream, "<message not set>");
     }
-    fputs("\n", stderr);
-    fflush(stderr); /* it helps in Windows debug build */
+    fputs("\n", stream);
+    fflush(stream); /* it helps in Windows debug build */
 
-    /* Check if the current thread has a Python thread state
-       and holds the GIL */
-    PyThreadState *tss_tstate = PyGILState_GetThisThreadState();
-    if (tss_tstate != NULL) {
-        PyThreadState *tstate = _PyThreadState_GET();
-        if (tss_tstate != tstate) {
-            /* The Python thread does not hold the GIL */
-            tss_tstate = NULL;
-        }
-    }
-    else {
-        /* Py_FatalError() has been called from a C thread
-           which has no Python thread state. */
+    _PyRuntimeState *runtime = &_PyRuntime;
+    fatal_error_dump_runtime(stream, runtime);
+
+    PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
+    PyInterpreterState *interp = NULL;
+    if (tstate != NULL) {
+        interp = tstate->interp;
     }
-    int has_tstate_and_gil = (tss_tstate != NULL);
 
+    /* Check if the current thread has a Python thread state
+       and holds the GIL.
+
+       tss_tstate is NULL if Py_FatalError() is called from a C thread which
+       has no Python thread state.
+
+       tss_tstate != tstate if the current Python thread does not hold the GIL.
+       */
+    PyThreadState *tss_tstate = PyGILState_GetThisThreadState();
+    int has_tstate_and_gil = (tss_tstate != NULL && tss_tstate == tstate);
     if (has_tstate_and_gil) {
         /* If an exception is set, print the exception with its traceback */
         if (!_Py_FatalError_PrintExc(fd)) {
             /* No exception is set, or an exception is set without traceback */
-            _Py_FatalError_DumpTracebacks(fd);
+            _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate);
         }
     }
     else {
-        _Py_FatalError_DumpTracebacks(fd);
+        _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate);
     }
 
     /* The main purpose of faulthandler is to display the traceback.


From webhook-mailer at python.org  Tue Sep 17 19:58:33 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 17 Sep 2019 23:58:33 -0000
Subject: [Python-checkins] bpo-37531: Skip
 test_regrtest.test_multiprocessing_timeout() on Windows (GH-16247)
Message-ID: <mailman.829.1568764714.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b9877cd2cc47b6f3512c171814c4f630286279b9
commit: b9877cd2cc47b6f3512c171814c4f630286279b9
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-18T01:58:29+02:00
summary:

bpo-37531: Skip test_regrtest.test_multiprocessing_timeout() on Windows (GH-16247)

It is a known and tracked bug: disable the test until it's fixed.

files:
M Lib/test/test_regrtest.py

diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 931f125eb8f9..ef2ee90d2a0d 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -1154,6 +1154,8 @@ def test_garbage(self):
                                   env_changed=[testname],
                                   fail_env_changed=True)
 
+    @unittest.skipIf(sys.platform == 'win32',
+                     'bpo-37531, bpo-38207: test hangs randomly on Windows')
     def test_multiprocessing_timeout(self):
         code = textwrap.dedent(r"""
             import time


From webhook-mailer at python.org  Tue Sep 17 23:45:13 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Wed, 18 Sep 2019 03:45:13 -0000
Subject: [Python-checkins] bpo-36546: No longer a need to make "data"
 positional only (GH-16252)
Message-ID: <mailman.830.1568778314.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/272d0d017aef585acf84bb0af99a90a2a8582b2c
commit: 272d0d017aef585acf84bb0af99a90a2a8582b2c
branch: master
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-17T20:45:05-07:00
summary:

bpo-36546: No longer a need to make "data" positional only (GH-16252)

files:
M Lib/statistics.py

diff --git a/Lib/statistics.py b/Lib/statistics.py
index 70c48d605d19..d81596e5d8ab 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -615,7 +615,7 @@ def multimode(data):
 # position is that fewer options make for easier choices and that
 # external packages can be used for anything more advanced.
 
-def quantiles(data, /, *, n=4, method='exclusive'):
+def quantiles(data, *, n=4, method='exclusive'):
     """Divide *data* into *n* continuous intervals with equal probability.
 
     Returns a list of (n - 1) cut points separating the intervals.


From webhook-mailer at python.org  Wed Sep 18 00:06:57 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Wed, 18 Sep 2019 04:06:57 -0000
Subject: [Python-checkins] bpo-36546: No longer a need to make "data"
 positional only (GH-16252) (GH-16253)
Message-ID: <mailman.831.1568779619.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/31af1cce9d799475ba8c3dec3e239b4a75ce268f
commit: 31af1cce9d799475ba8c3dec3e239b4a75ce268f
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-17T21:06:53-07:00
summary:

bpo-36546: No longer a need to make "data" positional only (GH-16252) (GH-16253)

(cherry picked from commit 272d0d017aef585acf84bb0af99a90a2a8582b2c)

Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com>

files:
M Lib/statistics.py

diff --git a/Lib/statistics.py b/Lib/statistics.py
index 70c48d605d19..d81596e5d8ab 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -615,7 +615,7 @@ def multimode(data):
 # position is that fewer options make for easier choices and that
 # external packages can be used for anything more advanced.
 
-def quantiles(data, /, *, n=4, method='exclusive'):
+def quantiles(data, *, n=4, method='exclusive'):
     """Divide *data* into *n* continuous intervals with equal probability.
 
     Returns a list of (n - 1) cut points separating the intervals.


From webhook-mailer at python.org  Wed Sep 18 02:08:05 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Wed, 18 Sep 2019 06:08:05 -0000
Subject: [Python-checkins] bpo-38191: Use positional-only parameters in
 TypedDict(). (GH-16240)
Message-ID: <mailman.832.1568786886.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/2adcd79bf90415147691a51a4cb046d5920f6ce7
commit: 2adcd79bf90415147691a51a4cb046d5920f6ce7
branch: 3.8
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-18T09:08:01+03:00
summary:

bpo-38191: Use positional-only parameters in TypedDict(). (GH-16240)

files:
M Lib/test/test_typing.py
M Lib/typing.py

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index 5914f314db0a..3013f8ec41e4 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -3653,16 +3653,10 @@ def test_typeddict_create_errors(self):
             TypedDict()
         with self.assertRaises(TypeError):
             TypedDict('Emp', [('name', str)], None)
-
-        with self.assertWarns(DeprecationWarning):
-            Emp = TypedDict(_typename='Emp', name=str, id=int)
-        self.assertEqual(Emp.__name__, 'Emp')
-        self.assertEqual(Emp.__annotations__, {'name': str, 'id': int})
-
-        with self.assertWarns(DeprecationWarning):
-            Emp = TypedDict('Emp', _fields={'name': str, 'id': int})
-        self.assertEqual(Emp.__name__, 'Emp')
-        self.assertEqual(Emp.__annotations__, {'name': str, 'id': int})
+        with self.assertRaises(TypeError):
+            TypedDict(_typename='Emp', name=str, id=int)
+        with self.assertRaises(TypeError):
+            TypedDict('Emp', _fields={'name': str, 'id': int})
 
     def test_typeddict_errors(self):
         Emp = TypedDict('Emp', {'name': str, 'id': int})
diff --git a/Lib/typing.py b/Lib/typing.py
index 43486a7c8ba8..048db3175e61 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -1691,43 +1691,11 @@ def __new__(*args, **kwargs):
     __new__.__text_signature__ = '($cls, typename, fields=None, /, **kwargs)'
 
 
-def _dict_new(*args, **kwargs):
-    if not args:
-        raise TypeError('TypedDict.__new__(): not enough arguments')
-    cls, *args = args  # allow the "cls" keyword be passed
+def _dict_new(cls, /, *args, **kwargs):
     return dict(*args, **kwargs)
-_dict_new.__text_signature__ = '($cls, _typename, _fields=None, /, **kwargs)'
-
-
-def _typeddict_new(*args, total=True, **kwargs):
-    if not args:
-        raise TypeError('TypedDict.__new__(): not enough arguments')
-    cls, *args = args  # allow the "cls" keyword be passed
-    if args:
-        typename, *args = args # allow the "_typename" keyword be passed
-    elif '_typename' in kwargs:
-        typename = kwargs.pop('_typename')
-        import warnings
-        warnings.warn("Passing '_typename' as keyword argument is deprecated",
-                      DeprecationWarning, stacklevel=2)
-    else:
-        raise TypeError("TypedDict.__new__() missing 1 required positional "
-                        "argument: '_typename'")
-    if args:
-        try:
-            fields, = args # allow the "_fields" keyword be passed
-        except ValueError:
-            raise TypeError(f'TypedDict.__new__() takes from 2 to 3 '
-                            f'positional arguments but {len(args) + 2} '
-                            f'were given') from None
-    elif '_fields' in kwargs and len(kwargs) == 1:
-        fields = kwargs.pop('_fields')
-        import warnings
-        warnings.warn("Passing '_fields' as keyword argument is deprecated",
-                      DeprecationWarning, stacklevel=2)
-    else:
-        fields = None
 
+
+def _typeddict_new(cls, typename, fields=None, /, *, total=True, **kwargs):
     if fields is None:
         fields = kwargs
     elif kwargs:
@@ -1742,7 +1710,6 @@ def _typeddict_new(*args, total=True, **kwargs):
         pass
 
     return _TypedDictMeta(typename, (), ns)
-_typeddict_new.__text_signature__ = '($cls, _typename, _fields=None, /, *, total=True, **kwargs)'
 
 
 def _check_fails(cls, other):


From webhook-mailer at python.org  Wed Sep 18 02:29:33 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 18 Sep 2019 06:29:33 -0000
Subject: [Python-checkins] bpo-38203: regrtest: put a 2 min timeout on
 Python exit (GH-16250)
Message-ID: <mailman.833.1568788174.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/da57599af51c602b015b6880123fef6eccdabcf5
commit: da57599af51c602b015b6880123fef6eccdabcf5
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-18T08:29:25+02:00
summary:

bpo-38203: regrtest: put a 2 min timeout on Python exit (GH-16250)

files:
M Lib/test/libregrtest/main.py

diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 3cfbb4017dfb..47c60126ea65 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -22,6 +22,12 @@
 from test import support
 
 
+# bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()).
+# Used to protect against threading._shutdown() hang.
+# Must be smaller than buildbot "1200 seconds without output" limit.
+EXIT_TIMEOUT = 120.0
+
+
 class Regrtest:
     """Execute a test suite.
 
@@ -616,16 +622,24 @@ def main(self, tests=None, **kwargs):
 
         test_cwd = self.create_temp_dir()
 
-        # Run the tests in a context manager that temporarily changes the CWD
-        # to a temporary and writable directory. If it's not possible to
-        # create or change the CWD, the original CWD will be used.
-        # The original CWD is available from support.SAVEDCWD.
-        with support.temp_cwd(test_cwd, quiet=True):
-            # When using multiprocessing, worker processes will use test_cwd
-            # as their parent temporary directory. So when the main process
-            # exit, it removes also subdirectories of worker processes.
-            self.ns.tempdir = test_cwd
-            self._main(tests, kwargs)
+        try:
+            # Run the tests in a context manager that temporarily changes the CWD
+            # to a temporary and writable directory. If it's not possible to
+            # create or change the CWD, the original CWD will be used.
+            # The original CWD is available from support.SAVEDCWD.
+            with support.temp_cwd(test_cwd, quiet=True):
+                # When using multiprocessing, worker processes will use test_cwd
+                # as their parent temporary directory. So when the main process
+                # exit, it removes also subdirectories of worker processes.
+                self.ns.tempdir = test_cwd
+
+                self._main(tests, kwargs)
+        except SystemExit as exc:
+            # bpo-38203: Python can hang at exit in Py_Finalize(), especially
+            # on threading._shutdown() call: put a timeout
+            faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True)
+
+            sys.exit(exc.code)
 
     def getloadavg(self):
         if self.win_load_tracker is not None:


From webhook-mailer at python.org  Wed Sep 18 06:36:30 2019
From: webhook-mailer at python.org (Eric V. Smith)
Date: Wed, 18 Sep 2019 10:36:30 -0000
Subject: [Python-checkins] bpo-37904: Edition on python tutorial - section 4
 (GH-16169) (GH-16234)
Message-ID: <mailman.834.1568802990.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/7a2f68776a77c782c0abf40dc9e3fb687787e730
commit: 7a2f68776a77c782c0abf40dc9e3fb687787e730
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Eric V. Smith <ericvsmith at users.noreply.github.com>
date: 2019-09-18T06:36:15-04:00
summary:

bpo-37904: Edition on python tutorial - section 4 (GH-16169) (GH-16234)

A little change on first paragraph of python tutorial to be more clearly

https://bugs.python.org/issue37904

Automerge-Triggered-By: @ericvsmith
(cherry picked from commit b57481318e3e3cbacd398b898f9849ec8f2d7eec)

Co-authored-by: Diego Alberto Barriga Mart?nez <diegobarriga at protonmail.com>

files:
M Doc/tutorial/controlflow.rst

diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst
index 3730a215d748..7dfd33af2588 100644
--- a/Doc/tutorial/controlflow.rst
+++ b/Doc/tutorial/controlflow.rst
@@ -4,8 +4,8 @@
 More Control Flow Tools
 ***********************
 
-Besides the :keyword:`while` statement just introduced, Python knows the usual
-control flow statements known from other languages, with some twists.
+Besides the :keyword:`while` statement just introduced, Python uses the usual
+flow control statements known from other languages, with some twists.
 
 
 .. _tut-if:


From webhook-mailer at python.org  Wed Sep 18 06:36:39 2019
From: webhook-mailer at python.org (Eric V. Smith)
Date: Wed, 18 Sep 2019 10:36:39 -0000
Subject: [Python-checkins] bpo-37904: Edition on python tutorial - section 4
 (GH-16169) (GH-16235)
Message-ID: <mailman.835.1568803000.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/8b907a8875d1b22d8f060dee9430cc82d471e86b
commit: 8b907a8875d1b22d8f060dee9430cc82d471e86b
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Eric V. Smith <ericvsmith at users.noreply.github.com>
date: 2019-09-18T06:36:35-04:00
summary:

bpo-37904: Edition on python tutorial - section 4 (GH-16169) (GH-16235)

A little change on first paragraph of python tutorial to be more clearly

https://bugs.python.org/issue37904

Automerge-Triggered-By: @ericvsmith
(cherry picked from commit b57481318e3e3cbacd398b898f9849ec8f2d7eec)

Co-authored-by: Diego Alberto Barriga Mart?nez <diegobarriga at protonmail.com>

files:
M Doc/tutorial/controlflow.rst

diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst
index 482a34399c7b..b7638dfffa4f 100644
--- a/Doc/tutorial/controlflow.rst
+++ b/Doc/tutorial/controlflow.rst
@@ -4,8 +4,8 @@
 More Control Flow Tools
 ***********************
 
-Besides the :keyword:`while` statement just introduced, Python knows the usual
-control flow statements known from other languages, with some twists.
+Besides the :keyword:`while` statement just introduced, Python uses the usual
+flow control statements known from other languages, with some twists.
 
 
 .. _tut-if:


From webhook-mailer at python.org  Wed Sep 18 06:37:01 2019
From: webhook-mailer at python.org (Eric V. Smith)
Date: Wed, 18 Sep 2019 10:37:01 -0000
Subject: [Python-checkins] bpo-37904: Edition on python tutorial - section 4
 (GH-16169) (GH-16236)
Message-ID: <mailman.836.1568803022.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/c47c8ba2969c9faf1c036b3dc4b0a5cf0d3aa0cc
commit: c47c8ba2969c9faf1c036b3dc4b0a5cf0d3aa0cc
branch: 2.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Eric V. Smith <ericvsmith at users.noreply.github.com>
date: 2019-09-18T06:36:57-04:00
summary:

bpo-37904: Edition on python tutorial - section 4 (GH-16169) (GH-16236)

A little change on first paragraph of python tutorial to be more clearly

https://bugs.python.org/issue37904

Automerge-Triggered-By: @ericvsmith
(cherry picked from commit b57481318e3e3cbacd398b898f9849ec8f2d7eec)

Co-authored-by: Diego Alberto Barriga Mart?nez <diegobarriga at protonmail.com>

files:
M Doc/tutorial/controlflow.rst

diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst
index c1c3d28fd077..2e1c1222ce3c 100644
--- a/Doc/tutorial/controlflow.rst
+++ b/Doc/tutorial/controlflow.rst
@@ -4,8 +4,8 @@
 More Control Flow Tools
 ***********************
 
-Besides the :keyword:`while` statement just introduced, Python knows the usual
-control flow statements known from other languages, with some twists.
+Besides the :keyword:`while` statement just introduced, Python uses the usual
+flow control statements known from other languages, with some twists.
 
 
 .. _tut-if:


From webhook-mailer at python.org  Wed Sep 18 08:10:34 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 18 Sep 2019 12:10:34 -0000
Subject: [Python-checkins] [3.8] bpo-38070: Py_FatalError() logs runtime
 state (GH-16258)
Message-ID: <mailman.837.1568808636.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/47bbab9f76735acc1991e541d12fd18be6b13b16
commit: 47bbab9f76735acc1991e541d12fd18be6b13b16
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-18T14:10:16+02:00
summary:

[3.8] bpo-38070: Py_FatalError() logs runtime state (GH-16258)

* bpo-38070: _Py_DumpTraceback() writes <no Python frame> (GH-16244)

When a Python thread has no frame, _Py_DumpTraceback() and
_Py_DumpTracebackThreads() now write "<no Python frame>", rather than
writing nothing.

(cherry picked from commit 8fa3e1740b3f03ea65ddb68411c2238c5f98eec2)

* bpo-38070: Enhance _PyObject_Dump() (GH-16243)

_PyObject_Dump() now dumps the object address for freed objects and
objects with ob_type=NULL.

(cherry picked from commit b39afb78768418d9405c4b528c80fa968ccc974d)

* bpo-38070: Add _PyRuntimeState.preinitializing (GH-16245)

Add _PyRuntimeState.preinitializing field: set to 1 while
Py_PreInitialize() is running.

_PyRuntimeState: rename also pre_initialized field to preinitialized.

(cherry picked from commit d3b904144e86e2442961de6a7dccecbe133d5c6d)

* bpo-38070: Py_FatalError() logs runtime state (GH-16246)


(cherry picked from commit 1ce16fb0977283ae42a9f8917bbca5f44aa69324)

files:
M Include/internal/pycore_pystate.h
M Lib/test/test_capi.py
M Lib/test/test_faulthandler.py
M Objects/object.c
M Python/pylifecycle.c
M Python/traceback.c

diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h
index 3ab4009770c9..f90e7e1ab78e 100644
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -193,8 +193,11 @@ struct _gilstate_runtime_state {
 /* Full Python runtime state */
 
 typedef struct pyruntimestate {
-    /* Is Python pre-initialized? Set to 1 by Py_PreInitialize() */
-    int pre_initialized;
+    /* Is running Py_PreInitialize()? */
+    int preinitializing;
+
+    /* Is Python preinitialized? Set to 1 by Py_PreInitialize() */
+    int preinitialized;
 
     /* Is Python core initialized? Set to 1 by _Py_InitializeCore() */
     int core_initialized;
@@ -202,6 +205,8 @@ typedef struct pyruntimestate {
     /* Is Python fully initialized? Set to 1 by Py_Initialize() */
     int initialized;
 
+    /* Set by Py_FinalizeEx(). Only reset to NULL if Py_Initialize()
+       is called again. */
     PyThreadState *finalizing;
 
     struct pyinterpreters {
@@ -244,7 +249,7 @@ typedef struct pyruntimestate {
 } _PyRuntimeState;
 
 #define _PyRuntimeState_INIT \
-    {.pre_initialized = 0, .core_initialized = 0, .initialized = 0}
+    {.preinitialized = 0, .core_initialized = 0, .initialized = 0}
 /* Note: _PyRuntimeState_INIT sets other fields to 0/NULL */
 
 PyAPI_DATA(_PyRuntimeState) _PyRuntime;
diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 4d6e2f21551a..ff7acac43b61 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -198,6 +198,7 @@ def test_return_null_without_error(self):
             self.assertRegex(err.replace(b'\r', b''),
                              br'Fatal Python error: a function returned NULL '
                                 br'without setting an error\n'
+                             br'Python runtime state: initialized\n'
                              br'SystemError: <built-in function '
                                  br'return_null_without_error> returned NULL '
                                  br'without setting an error\n'
@@ -225,6 +226,7 @@ def test_return_result_with_error(self):
             self.assertRegex(err.replace(b'\r', b''),
                              br'Fatal Python error: a function returned a '
                                 br'result with an error set\n'
+                             br'Python runtime state: initialized\n'
                              br'ValueError\n'
                              br'\n'
                              br'The above exception was the direct cause '
diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py
index 1cf20db1c7ff..d87510393482 100644
--- a/Lib/test/test_faulthandler.py
+++ b/Lib/test/test_faulthandler.py
@@ -90,7 +90,8 @@ def get_output(self, code, filename=None, fd=None):
 
     def check_error(self, code, line_number, fatal_error, *,
                     filename=None, all_threads=True, other_regex=None,
-                    fd=None, know_current_thread=True):
+                    fd=None, know_current_thread=True,
+                    py_fatal_error=False):
         """
         Check that the fault handler for fatal errors is enabled and check the
         traceback from the child process output.
@@ -110,10 +111,12 @@ def check_error(self, code, line_number, fatal_error, *,
             {header} \(most recent call first\):
               File "<string>", line {lineno} in <module>
             """
-        regex = dedent(regex.format(
+        if py_fatal_error:
+            fatal_error += "\nPython runtime state: initialized"
+        regex = dedent(regex).format(
             lineno=line_number,
             fatal_error=fatal_error,
-            header=header)).strip()
+            header=header).strip()
         if other_regex:
             regex += '|' + other_regex
         output, exitcode = self.get_output(code, filename=filename, fd=fd)
@@ -170,7 +173,8 @@ def test_fatal_error_c_thread(self):
             """,
             3,
             'in new thread',
-            know_current_thread=False)
+            know_current_thread=False,
+            py_fatal_error=True)
 
     def test_sigabrt(self):
         self.check_fatal_error("""
@@ -226,7 +230,8 @@ def test_fatal_error(self):
             faulthandler._fatal_error(b'xyz')
             """,
             2,
-            'xyz')
+            'xyz',
+            py_fatal_error=True)
 
     def test_fatal_error_without_gil(self):
         self.check_fatal_error("""
@@ -234,7 +239,8 @@ def test_fatal_error_without_gil(self):
             faulthandler._fatal_error(b'xyz', True)
             """,
             2,
-            'xyz')
+            'xyz',
+            py_fatal_error=True)
 
     @unittest.skipIf(sys.platform.startswith('openbsd'),
                      "Issue #12868: sigaltstack() doesn't work on "
diff --git a/Objects/object.c b/Objects/object.c
index 585a9748c846..df2531371fa5 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -464,7 +464,7 @@ void
 _PyObject_Dump(PyObject* op)
 {
     if (op == NULL) {
-        fprintf(stderr, "<NULL object>\n");
+        fprintf(stderr, "<object at NULL>\n");
         fflush(stderr);
         return;
     }
@@ -472,7 +472,7 @@ _PyObject_Dump(PyObject* op)
     if (_PyObject_IsFreed(op)) {
         /* It seems like the object memory has been freed:
            don't access it to prevent a segmentation fault. */
-        fprintf(stderr, "<Freed object>\n");
+        fprintf(stderr, "<object at %p is freed>\n", op);
         return;
     }
 
@@ -2162,18 +2162,19 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg,
     fflush(stderr);
 
     if (obj == NULL) {
-        fprintf(stderr, "<NULL object>\n");
+        fprintf(stderr, "<object at NULL>\n");
     }
     else if (_PyObject_IsFreed(obj)) {
         /* It seems like the object memory has been freed:
            don't access it to prevent a segmentation fault. */
-        fprintf(stderr, "<object: freed>\n");
+        fprintf(stderr, "<object at %p is freed>\n", obj);
     }
     else if (Py_TYPE(obj) == NULL) {
-        fprintf(stderr, "<object: ob_type=NULL>\n");
+        fprintf(stderr, "<object at %p: ob_type=NULL>\n", obj);
     }
     else if (_PyObject_IsFreed((PyObject *)Py_TYPE(obj))) {
-        fprintf(stderr, "<object: freed type %p>\n", (void *)Py_TYPE(obj));
+        fprintf(stderr, "<object at %p: type at %p is freed>\n",
+                obj, (void *)Py_TYPE(obj));
     }
     else {
         /* Display the traceback where the object has been allocated.
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 751c4d6d1d63..97bb99256d89 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -719,11 +719,15 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args)
     }
     _PyRuntimeState *runtime = &_PyRuntime;
 
-    if (runtime->pre_initialized) {
+    if (runtime->preinitialized) {
         /* If it's already configured: ignored the new configuration */
         return _PyStatus_OK();
     }
 
+    /* Note: preinitialized remains 1 on error, it is only set to 0
+       at exit on success. */
+    runtime->preinitializing = 1;
+
     PyPreConfig config;
     _PyPreConfig_InitFromPreConfig(&config, src_config);
 
@@ -737,7 +741,8 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args)
         return status;
     }
 
-    runtime->pre_initialized = 1;
+    runtime->preinitializing = 0;
+    runtime->preinitialized = 1;
     return _PyStatus_OK();
 }
 
@@ -777,7 +782,7 @@ _Py_PreInitializeFromConfig(const PyConfig *config,
     }
     _PyRuntimeState *runtime = &_PyRuntime;
 
-    if (runtime->pre_initialized) {
+    if (runtime->preinitialized) {
         /* Already initialized: do nothing */
         return _PyStatus_OK();
     }
@@ -1961,13 +1966,14 @@ init_sys_streams(PyInterpreterState *interp)
 
 
 static void
-_Py_FatalError_DumpTracebacks(int fd)
+_Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp,
+                              PyThreadState *tstate)
 {
     fputc('\n', stderr);
     fflush(stderr);
 
     /* display the current Python stack */
-    _Py_DumpTracebackThreads(fd, NULL, NULL);
+    _Py_DumpTracebackThreads(fd, interp, tstate);
 }
 
 /* Print the current exception (if an exception is set) with its traceback,
@@ -2062,10 +2068,39 @@ fatal_output_debug(const char *msg)
 }
 #endif
 
+
+static void
+fatal_error_dump_runtime(FILE *stream, _PyRuntimeState *runtime)
+{
+    fprintf(stream, "Python runtime state: ");
+    if (runtime->finalizing) {
+        fprintf(stream, "finalizing (tstate=%p)", runtime->finalizing);
+    }
+    else if (runtime->initialized) {
+        fprintf(stream, "initialized");
+    }
+    else if (runtime->core_initialized) {
+        fprintf(stream, "core initialized");
+    }
+    else if (runtime->preinitialized) {
+        fprintf(stream, "preinitialized");
+    }
+    else if (runtime->preinitializing) {
+        fprintf(stream, "preinitializing");
+    }
+    else {
+        fprintf(stream, "unknown");
+    }
+    fprintf(stream, "\n");
+    fflush(stream);
+}
+
+
 static void _Py_NO_RETURN
 fatal_error(const char *prefix, const char *msg, int status)
 {
-    const int fd = fileno(stderr);
+    FILE *stream = stderr;
+    const int fd = fileno(stream);
     static int reentrant = 0;
 
     if (reentrant) {
@@ -2075,45 +2110,48 @@ fatal_error(const char *prefix, const char *msg, int status)
     }
     reentrant = 1;
 
-    fprintf(stderr, "Fatal Python error: ");
+    fprintf(stream, "Fatal Python error: ");
     if (prefix) {
-        fputs(prefix, stderr);
-        fputs(": ", stderr);
+        fputs(prefix, stream);
+        fputs(": ", stream);
     }
     if (msg) {
-        fputs(msg, stderr);
+        fputs(msg, stream);
     }
     else {
-        fprintf(stderr, "<message not set>");
+        fprintf(stream, "<message not set>");
     }
-    fputs("\n", stderr);
-    fflush(stderr); /* it helps in Windows debug build */
+    fputs("\n", stream);
+    fflush(stream); /* it helps in Windows debug build */
 
-    /* Check if the current thread has a Python thread state
-       and holds the GIL */
-    PyThreadState *tss_tstate = PyGILState_GetThisThreadState();
-    if (tss_tstate != NULL) {
-        PyThreadState *tstate = _PyThreadState_GET();
-        if (tss_tstate != tstate) {
-            /* The Python thread does not hold the GIL */
-            tss_tstate = NULL;
-        }
-    }
-    else {
-        /* Py_FatalError() has been called from a C thread
-           which has no Python thread state. */
+    _PyRuntimeState *runtime = &_PyRuntime;
+    fatal_error_dump_runtime(stream, runtime);
+
+    PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
+    PyInterpreterState *interp = NULL;
+    if (tstate != NULL) {
+        interp = tstate->interp;
     }
-    int has_tstate_and_gil = (tss_tstate != NULL);
 
+    /* Check if the current thread has a Python thread state
+       and holds the GIL.
+
+       tss_tstate is NULL if Py_FatalError() is called from a C thread which
+       has no Python thread state.
+
+       tss_tstate != tstate if the current Python thread does not hold the GIL.
+       */
+    PyThreadState *tss_tstate = PyGILState_GetThisThreadState();
+    int has_tstate_and_gil = (tss_tstate != NULL && tss_tstate == tstate);
     if (has_tstate_and_gil) {
         /* If an exception is set, print the exception with its traceback */
         if (!_Py_FatalError_PrintExc(fd)) {
             /* No exception is set, or an exception is set without traceback */
-            _Py_FatalError_DumpTracebacks(fd);
+            _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate);
         }
     }
     else {
-        _Py_FatalError_DumpTracebacks(fd);
+        _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate);
     }
 
     /* The main purpose of faulthandler is to display the traceback.
diff --git a/Python/traceback.c b/Python/traceback.c
index 0463eb6d8c98..8e2f15e85d6b 100644
--- a/Python/traceback.c
+++ b/Python/traceback.c
@@ -797,12 +797,15 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header)
     PyFrameObject *frame;
     unsigned int depth;
 
-    if (write_header)
+    if (write_header) {
         PUTS(fd, "Stack (most recent call first):\n");
+    }
 
     frame = _PyThreadState_GetFrame(tstate);
-    if (frame == NULL)
+    if (frame == NULL) {
+        PUTS(fd, "<no Python frame>\n");
         return;
+    }
 
     depth = 0;
     while (frame != NULL) {
@@ -870,9 +873,9 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp,
            Python thread state of the current thread.
 
            PyThreadState_Get() doesn't give the state of the thread that caused
-           the fault if the thread released the GIL, and so this function
-           cannot be used. Read the thread specific storage (TSS) instead: call
-           PyGILState_GetThisThreadState(). */
+           the fault if the thread released the GIL, and so
+           _PyThreadState_GET() cannot be used. Read the thread specific
+           storage (TSS) instead: call PyGILState_GetThisThreadState(). */
         current_tstate = PyGILState_GetThisThreadState();
     }
 


From webhook-mailer at python.org  Wed Sep 18 08:15:15 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 18 Sep 2019 12:15:15 -0000
Subject: [Python-checkins] bpo-38203: faulthandler.dump_traceback_later() is
 always available (GH-16249)
Message-ID: <mailman.838.1568808917.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/0a963fbc9c4cf4569f1eadaa2aa1229bb0081256
commit: 0a963fbc9c4cf4569f1eadaa2aa1229bb0081256
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-18T14:15:10+02:00
summary:

bpo-38203: faulthandler.dump_traceback_later() is always available (GH-16249)

dump_traceback_later() and cancel_dump_traceback_later() functions of
the faulthandler module are always available since Python 3.7.

files:
M Doc/library/faulthandler.rst
M Lib/test/eintrdata/eintr_tester.py
M Lib/test/libregrtest/main.py
M Lib/test/test_faulthandler.py
M Lib/test/test_regrtest.py
M Modules/faulthandler.c

diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst
index 94ebd87639c5..b588dfa18db2 100644
--- a/Doc/library/faulthandler.rst
+++ b/Doc/library/faulthandler.rst
@@ -100,8 +100,10 @@ Dumping the tracebacks after a timeout
    :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file
    descriptors <faulthandler-fd>`.
 
-   This function is implemented using a watchdog thread and therefore is not
-   available if Python is compiled with threads disabled.
+   This function is implemented using a watchdog thread.
+
+   .. versionchanged:: 3.7
+      This function is now always available.
 
    .. versionchanged:: 3.5
       Added support for passing file descriptor to this function.
diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py
index 404934ce97a2..4e0e30596f5a 100644
--- a/Lib/test/eintrdata/eintr_tester.py
+++ b/Lib/test/eintrdata/eintr_tester.py
@@ -57,9 +57,8 @@ def setUp(self):
 
         # Use faulthandler as watchdog to debug when a test hangs
         # (timeout of 10 minutes)
-        if hasattr(faulthandler, 'dump_traceback_later'):
-            faulthandler.dump_traceback_later(10 * 60, exit=True,
-                                              file=sys.__stderr__)
+        faulthandler.dump_traceback_later(10 * 60, exit=True,
+                                          file=sys.__stderr__)
 
     @staticmethod
     def stop_alarm():
@@ -68,8 +67,7 @@ def stop_alarm():
     def tearDown(self):
         self.stop_alarm()
         signal.signal(signal.SIGALRM, self.orig_handler)
-        if hasattr(faulthandler, 'cancel_dump_traceback_later'):
-            faulthandler.cancel_dump_traceback_later()
+        faulthandler.cancel_dump_traceback_later()
 
     def subprocess(self, *args, **kw):
         cmd_args = (sys.executable, '-c') + args
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 47c60126ea65..299700686928 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -164,11 +164,6 @@ def display_progress(self, test_index, text):
     def parse_args(self, kwargs):
         ns = _parse_args(sys.argv[1:], **kwargs)
 
-        if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'):
-            print("Warning: The timeout option requires "
-                  "faulthandler.dump_traceback_later", file=sys.stderr)
-            ns.timeout = None
-
         if ns.xmlpath:
             support.junit_xml_list = self.testsuite_xml = []
 
diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py
index 24487446f25b..a4427a537d1f 100644
--- a/Lib/test/test_faulthandler.py
+++ b/Lib/test/test_faulthandler.py
@@ -535,8 +535,6 @@ def test_dump_traceback_threads_file(self):
         with temporary_filename() as filename:
             self.check_dump_traceback_threads(filename)
 
-    @unittest.skipIf(not hasattr(faulthandler, 'dump_traceback_later'),
-                     'need faulthandler.dump_traceback_later()')
     def check_dump_traceback_later(self, repeat=False, cancel=False, loops=1,
                                    *, filename=None, fd=None):
         """
@@ -744,9 +742,8 @@ def test_stderr_None(self):
             faulthandler.enable()
         with self.check_stderr_none():
             faulthandler.dump_traceback()
-        if hasattr(faulthandler, 'dump_traceback_later'):
-            with self.check_stderr_none():
-                faulthandler.dump_traceback_later(1e-3)
+        with self.check_stderr_none():
+            faulthandler.dump_traceback_later(1e-3)
         if hasattr(faulthandler, "register"):
             with self.check_stderr_none():
                 faulthandler.register(signal.SIGUSR1)
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index ef2ee90d2a0d..50911995ad1e 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -54,8 +54,6 @@ def test_help(self):
                     libregrtest._parse_args([opt])
                 self.assertIn('Run Python regression tests.', out.getvalue())
 
-    @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'),
-                         "faulthandler.dump_traceback_later() required")
     def test_timeout(self):
         ns = libregrtest._parse_args(['--timeout', '4.2'])
         self.assertEqual(ns.timeout, 4.2)
@@ -572,8 +570,7 @@ def setUp(self):
         self.python_args = ['-Wd', '-E', '-bb']
         self.regrtest_args = ['-uall', '-rwW',
                               '--testdir=%s' % self.tmptestdir]
-        if hasattr(faulthandler, 'dump_traceback_later'):
-            self.regrtest_args.extend(('--timeout', '3600', '-j4'))
+        self.regrtest_args.extend(('--timeout', '3600', '-j4'))
         if sys.platform == 'win32':
             self.regrtest_args.append('-n')
 
diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c
index 011ab5fa28f2..129a104dba3f 100644
--- a/Modules/faulthandler.c
+++ b/Modules/faulthandler.c
@@ -19,8 +19,6 @@
 /* Allocate at maximum 100 MiB of the stack to raise the stack overflow */
 #define STACK_OVERFLOW_MAX_SIZE (100 * 1024 * 1024)
 
-#define FAULTHANDLER_LATER
-
 #ifndef MS_WINDOWS
    /* register() is useless on Windows, because only SIGSEGV, SIGABRT and
       SIGILL can be handled by the process, and these signals can only be used
@@ -60,7 +58,6 @@ static struct {
 #endif
 } fatal_error = {0, NULL, -1, 0};
 
-#ifdef FAULTHANDLER_LATER
 static struct {
     PyObject *file;
     int fd;
@@ -77,7 +74,6 @@ static struct {
     /* released by child thread when joined */
     PyThread_type_lock running;
 } thread;
-#endif
 
 #ifdef FAULTHANDLER_USER
 typedef struct {
@@ -589,8 +585,6 @@ faulthandler_is_enabled(PyObject *self, PyObject *Py_UNUSED(ignored))
     return PyBool_FromLong(fatal_error.enabled);
 }
 
-#ifdef FAULTHANDLER_LATER
-
 static void
 faulthandler_thread(void *unused)
 {
@@ -790,7 +784,6 @@ faulthandler_cancel_dump_traceback_later_py(PyObject *self,
     cancel_dump_traceback_later();
     Py_RETURN_NONE;
 }
-#endif  /* FAULTHANDLER_LATER */
 
 
 #ifdef FAULTHANDLER_USER
@@ -1230,9 +1223,7 @@ faulthandler_stack_overflow(PyObject *self, PyObject *Py_UNUSED(ignored))
 static int
 faulthandler_traverse(PyObject *module, visitproc visit, void *arg)
 {
-#ifdef FAULTHANDLER_LATER
     Py_VISIT(thread.file);
-#endif
 #ifdef FAULTHANDLER_USER
     if (user_signals != NULL) {
         for (size_t signum=0; signum < NSIG; signum++)
@@ -1273,7 +1264,6 @@ static PyMethodDef module_methods[] = {
      PyDoc_STR("dump_traceback(file=sys.stderr, all_threads=True): "
                "dump the traceback of the current thread, or of all threads "
                "if all_threads is True, into file")},
-#ifdef FAULTHANDLER_LATER
     {"dump_traceback_later",
      (PyCFunction)(void(*)(void))faulthandler_dump_traceback_later, METH_VARARGS|METH_KEYWORDS,
      PyDoc_STR("dump_traceback_later(timeout, repeat=False, file=sys.stderrn, exit=False):\n"
@@ -1284,8 +1274,6 @@ static PyMethodDef module_methods[] = {
      faulthandler_cancel_dump_traceback_later_py, METH_NOARGS,
      PyDoc_STR("cancel_dump_traceback_later():\ncancel the previous call "
                "to dump_traceback_later().")},
-#endif
-
 #ifdef FAULTHANDLER_USER
     {"register",
      (PyCFunction)(void(*)(void))faulthandler_register_py, METH_VARARGS|METH_KEYWORDS,
@@ -1298,7 +1286,6 @@ static PyMethodDef module_methods[] = {
      PyDoc_STR("unregister(signum): unregister the handler of the signal "
                 "'signum' registered by register()")},
 #endif
-
     {"_read_null", faulthandler_read_null, METH_NOARGS,
      PyDoc_STR("_read_null(): read from NULL, raise "
                "a SIGSEGV or SIGBUS signal depending on the platform")},
@@ -1399,9 +1386,7 @@ _PyFaulthandler_Init(int enable)
     stack.ss_size = SIGSTKSZ * 2;
 #endif
 
-#ifdef FAULTHANDLER_LATER
     memset(&thread, 0, sizeof(thread));
-#endif
 
     if (enable) {
         if (faulthandler_init_enable() < 0) {
@@ -1413,7 +1398,6 @@ _PyFaulthandler_Init(int enable)
 
 void _PyFaulthandler_Fini(void)
 {
-#ifdef FAULTHANDLER_LATER
     /* later */
     if (thread.cancel_event) {
         cancel_dump_traceback_later();
@@ -1425,7 +1409,6 @@ void _PyFaulthandler_Fini(void)
         PyThread_free_lock(thread.running);
         thread.running = NULL;
     }
-#endif
 
 #ifdef FAULTHANDLER_USER
     /* user */


From webhook-mailer at python.org  Wed Sep 18 08:37:09 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 18 Sep 2019 12:37:09 -0000
Subject: [Python-checkins] bpo-38203: faulthandler.dump_traceback_later() is
 always available (GH-16260)
Message-ID: <mailman.839.1568810229.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/064e1e384120635330493abf300b1113eadd904c
commit: 064e1e384120635330493abf300b1113eadd904c
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-18T14:37:02+02:00
summary:

bpo-38203: faulthandler.dump_traceback_later() is always available (GH-16260)

dump_traceback_later() and cancel_dump_traceback_later() functions of
the faulthandler module are always available since Python 3.7.

files:
M Doc/library/faulthandler.rst

diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst
index 94ebd87639c5..b588dfa18db2 100644
--- a/Doc/library/faulthandler.rst
+++ b/Doc/library/faulthandler.rst
@@ -100,8 +100,10 @@ Dumping the tracebacks after a timeout
    :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file
    descriptors <faulthandler-fd>`.
 
-   This function is implemented using a watchdog thread and therefore is not
-   available if Python is compiled with threads disabled.
+   This function is implemented using a watchdog thread.
+
+   .. versionchanged:: 3.7
+      This function is now always available.
 
    .. versionchanged:: 3.5
       Added support for passing file descriptor to this function.


From webhook-mailer at python.org  Wed Sep 18 08:39:24 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 18 Sep 2019 12:39:24 -0000
Subject: [Python-checkins] Fix _PyTraceMalloc_Fini() definition (GH-16259)
Message-ID: <mailman.840.1568810365.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/d299b8b47d9dbea8fec2c2abcb6c32ad32f5f4d1
commit: d299b8b47d9dbea8fec2c2abcb6c32ad32f5f4d1
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-18T14:39:20+02:00
summary:

Fix _PyTraceMalloc_Fini() definition (GH-16259)

The function return type is void, not int.

files:
M Include/internal/pycore_pylifecycle.h

diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index cf844cc063f8..cdf5c09a4262 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -84,7 +84,7 @@ extern void _PyUnicode_Fini(void);
 extern void _PyLong_Fini(void);
 extern void _PyFaulthandler_Fini(void);
 extern void _PyHash_Fini(void);
-extern int _PyTraceMalloc_Fini(void);
+extern void _PyTraceMalloc_Fini(void);
 extern void _PyWarnings_Fini(PyInterpreterState *interp);
 
 extern void _PyGILState_Init(


From webhook-mailer at python.org  Wed Sep 18 08:43:21 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 18 Sep 2019 12:43:21 -0000
Subject: [Python-checkins] bpo-38203: faulthandler.dump_traceback_later() is
 always available (GH-16260)
Message-ID: <mailman.841.1568810602.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/1a3a48ddaf41dff3c0fcedcfe7ec0940158c568d
commit: 1a3a48ddaf41dff3c0fcedcfe7ec0940158c568d
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-18T05:43:17-07:00
summary:

bpo-38203: faulthandler.dump_traceback_later() is always available (GH-16260)


dump_traceback_later() and cancel_dump_traceback_later() functions of
the faulthandler module are always available since Python 3.7.
(cherry picked from commit 064e1e384120635330493abf300b1113eadd904c)

Co-authored-by: Victor Stinner <vstinner at redhat.com>

files:
M Doc/library/faulthandler.rst

diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst
index 94ebd87639c5..b588dfa18db2 100644
--- a/Doc/library/faulthandler.rst
+++ b/Doc/library/faulthandler.rst
@@ -100,8 +100,10 @@ Dumping the tracebacks after a timeout
    :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file
    descriptors <faulthandler-fd>`.
 
-   This function is implemented using a watchdog thread and therefore is not
-   available if Python is compiled with threads disabled.
+   This function is implemented using a watchdog thread.
+
+   .. versionchanged:: 3.7
+      This function is now always available.
 
    .. versionchanged:: 3.5
       Added support for passing file descriptor to this function.


From webhook-mailer at python.org  Wed Sep 18 11:10:29 2019
From: webhook-mailer at python.org (Inada Naoki)
Date: Wed, 18 Sep 2019 15:10:29 -0000
Subject: [Python-checkins] bpo-35696: Simplify long_compare() (GH-16146)
Message-ID: <mailman.842.1568819429.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/42acb7b8d29d078bc97b0cfd7c4911b2266b26b9
commit: 42acb7b8d29d078bc97b0cfd7c4911b2266b26b9
branch: master
author: HongWeipeng <961365124 at qq.com>
committer: Inada Naoki <songofacandy at gmail.com>
date: 2019-09-19T00:10:15+09:00
summary:

bpo-35696: Simplify long_compare() (GH-16146)

files:
M Objects/longobject.c

diff --git a/Objects/longobject.c b/Objects/longobject.c
index 5da6951b2589..f2f6f9540892 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -3028,33 +3028,32 @@ PyLong_AsDouble(PyObject *v)
 
 /* Methods */
 
-static int
+/* if a < b, return a negative number
+   if a == b, return 0
+   if a > b, return a positive number */
+
+static Py_ssize_t
 long_compare(PyLongObject *a, PyLongObject *b)
 {
-    Py_ssize_t sign;
-
-    if (Py_SIZE(a) != Py_SIZE(b)) {
-        sign = Py_SIZE(a) - Py_SIZE(b);
-    }
-    else {
+    Py_ssize_t sign = Py_SIZE(a) - Py_SIZE(b);
+    if (sign == 0) {
         Py_ssize_t i = Py_ABS(Py_SIZE(a));
-        while (--i >= 0 && a->ob_digit[i] == b->ob_digit[i])
-            ;
-        if (i < 0)
-            sign = 0;
-        else {
-            sign = (sdigit)a->ob_digit[i] - (sdigit)b->ob_digit[i];
-            if (Py_SIZE(a) < 0)
-                sign = -sign;
+        sdigit diff = 0;
+        while (--i >= 0) {
+            diff = (sdigit) a->ob_digit[i] - (sdigit) b->ob_digit[i];
+            if (diff) {
+                break;
+            }
         }
+        sign = Py_SIZE(a) < 0 ? -diff : diff;
     }
-    return sign < 0 ? -1 : sign > 0 ? 1 : 0;
+    return sign;
 }
 
 static PyObject *
 long_richcompare(PyObject *self, PyObject *other, int op)
 {
-    int result;
+    Py_ssize_t result;
     CHECK_BINOP(self, other);
     if (self == other)
         result = 0;
@@ -5210,7 +5209,8 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b)
 {
     PyLongObject *quo = NULL, *rem = NULL;
     PyObject *twice_rem, *result, *temp;
-    int cmp, quo_is_odd, quo_is_neg;
+    int quo_is_odd, quo_is_neg;
+    Py_ssize_t cmp;
 
     /* Equivalent Python code:
 


From webhook-mailer at python.org  Thu Sep 19 01:57:36 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 19 Sep 2019 05:57:36 -0000
Subject: [Python-checkins] Doc: Clarify dict equality irrespective of
 ordering. (GH-16266)
Message-ID: <mailman.843.1568872657.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/50ecedc10309373809cc45595a1027076e808607
commit: 50ecedc10309373809cc45595a1027076e808607
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-18T22:57:32-07:00
summary:

Doc: Clarify dict equality irrespective of ordering. (GH-16266)

(cherry picked from commit 3171d67a6aaf7fe88685b3a80644f0284686ef63)

Co-authored-by: toonarmycaptain <toonarmycaptain at hotmail.com>

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 7a8faba00fd9..57b72c5b6562 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -4280,7 +4280,7 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098:
          False
 
    Dictionaries compare equal if and only if they have the same ``(key,
-   value)`` pairs. Order comparisons ('<', '<=', '>=', '>') raise
+   value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise
    :exc:`TypeError`.
 
    Dictionaries preserve insertion order.  Note that updating a key does not


From webhook-mailer at python.org  Thu Sep 19 01:58:45 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 19 Sep 2019 05:58:45 -0000
Subject: [Python-checkins] Doc: Clarify dict equality irrespective of
 ordering. (GH-16266)
Message-ID: <mailman.844.1568872726.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/16eb6232c0ac86e2a0072b98f5add46ab7cb08c0
commit: 16eb6232c0ac86e2a0072b98f5add46ab7cb08c0
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-18T22:58:42-07:00
summary:

Doc: Clarify dict equality irrespective of ordering. (GH-16266)

(cherry picked from commit 3171d67a6aaf7fe88685b3a80644f0284686ef63)

Co-authored-by: toonarmycaptain <toonarmycaptain at hotmail.com>

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 06b0c00ed10a..9bc4cbe4f7bc 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -4356,7 +4356,7 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098:
          False
 
    Dictionaries compare equal if and only if they have the same ``(key,
-   value)`` pairs. Order comparisons ('<', '<=', '>=', '>') raise
+   value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise
    :exc:`TypeError`.
 
    Dictionaries preserve insertion order.  Note that updating a key does not


From webhook-mailer at python.org  Thu Sep 19 08:47:29 2019
From: webhook-mailer at python.org (Andrew Svetlov)
Date: Thu, 19 Sep 2019 12:47:29 -0000
Subject: [Python-checkins] bpo-34037: Fix test_asyncio failure and add
 loop.shutdown_default_executor() (GH-15735)
Message-ID: <mailman.845.1568897251.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/9fdc64cf1266b6d5bf0503847b5c38e5edc53a14
commit: 9fdc64cf1266b6d5bf0503847b5c38e5edc53a14
branch: master
author: Kyle Stanley <aeros167 at gmail.com>
committer: Andrew Svetlov <andrew.svetlov at gmail.com>
date: 2019-09-19T15:47:22+03:00
summary:

bpo-34037: Fix test_asyncio failure and add loop.shutdown_default_executor() (GH-15735)

files:
A Misc/NEWS.d/next/Library/2019-09-11-21-38-41.bpo-34037.LIAS_3.rst
M Doc/library/asyncio-eventloop.rst
M Doc/library/asyncio-task.rst
M Lib/asyncio/base_events.py
M Lib/asyncio/events.py
M Lib/asyncio/runners.py

diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index 8f7974be66ea..2fd4cf30bd45 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -167,6 +167,18 @@ Running and stopping the loop
 
    .. versionadded:: 3.6
 
+.. coroutinemethod:: loop.shutdown_default_executor()
+
+   Schedule the closure of the default executor and wait for it to join all of
+   the threads in the :class:`ThreadPoolExecutor`. After calling this method, a
+   :exc:`RuntimeError` will be raised if :meth:`loop.run_in_executor` is called
+   while using the default executor.
+
+   Note that there is no need to call this function when
+   :func:`asyncio.run` is used.
+
+   .. versionadded:: 3.9
+
 
 Scheduling callbacks
 ^^^^^^^^^^^^^^^^^^^^
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index 57e0e07ad3bb..d9320422db3b 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -213,8 +213,8 @@ Running an asyncio Program
 .. function:: run(coro, \*, debug=False)
 
     This function runs the passed coroutine, taking care of
-    managing the asyncio event loop and *finalizing asynchronous
-    generators*.
+    managing the asyncio event loop, *finalizing asynchronous
+    generators*, and closing the threadpool.
 
     This function cannot be called when another asyncio event loop is
     running in the same thread.
@@ -229,6 +229,8 @@ Running an asyncio Program
        **Important:** this function has been added to asyncio in
        Python 3.7 on a :term:`provisional basis <provisional api>`.
 
+    .. versionchanged:: 3.9
+       Updated to use :meth:`loop.shutdown_default_executor`.
 
 Creating Tasks
 ==============
diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py
index 14b80bdda9c0..031071281b38 100644
--- a/Lib/asyncio/base_events.py
+++ b/Lib/asyncio/base_events.py
@@ -406,6 +406,8 @@ def __init__(self):
         self._asyncgens = weakref.WeakSet()
         # Set to True when `loop.shutdown_asyncgens` is called.
         self._asyncgens_shutdown_called = False
+        # Set to True when `loop.shutdown_default_executor` is called.
+        self._executor_shutdown_called = False
 
     def __repr__(self):
         return (
@@ -503,6 +505,10 @@ def _check_closed(self):
         if self._closed:
             raise RuntimeError('Event loop is closed')
 
+    def _check_default_executor(self):
+        if self._executor_shutdown_called:
+            raise RuntimeError('Executor shutdown has been called')
+
     def _asyncgen_finalizer_hook(self, agen):
         self._asyncgens.discard(agen)
         if not self.is_closed():
@@ -543,6 +549,26 @@ def _asyncgen_firstiter_hook(self, agen):
                     'asyncgen': agen
                 })
 
+    async def shutdown_default_executor(self):
+        """Schedule the shutdown of the default executor."""
+        self._executor_shutdown_called = True
+        if self._default_executor is None:
+            return
+        future = self.create_future()
+        thread = threading.Thread(target=self._do_shutdown, args=(future,))
+        thread.start()
+        try:
+            await future
+        finally:
+            thread.join()
+
+    def _do_shutdown(self, future):
+        try:
+            self._default_executor.shutdown(wait=True)
+            self.call_soon_threadsafe(future.set_result, None)
+        except Exception as ex:
+            self.call_soon_threadsafe(future.set_exception, ex)
+
     def run_forever(self):
         """Run until stop() is called."""
         self._check_closed()
@@ -632,6 +658,7 @@ def close(self):
         self._closed = True
         self._ready.clear()
         self._scheduled.clear()
+        self._executor_shutdown_called = True
         executor = self._default_executor
         if executor is not None:
             self._default_executor = None
@@ -768,6 +795,8 @@ def run_in_executor(self, executor, func, *args):
             self._check_callback(func, 'run_in_executor')
         if executor is None:
             executor = self._default_executor
+            # Only check when the default executor is being used
+            self._check_default_executor()
             if executor is None:
                 executor = concurrent.futures.ThreadPoolExecutor()
                 self._default_executor = executor
diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py
index 5fb546429cbe..2f06c4ae795d 100644
--- a/Lib/asyncio/events.py
+++ b/Lib/asyncio/events.py
@@ -249,6 +249,10 @@ def close(self):
         """Shutdown all active asynchronous generators."""
         raise NotImplementedError
 
+    async def shutdown_default_executor(self):
+        """Schedule the shutdown of the default executor."""
+        raise NotImplementedError
+
     # Methods scheduling callbacks.  All these return Handles.
 
     def _timer_handle_cancelled(self, handle):
diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py
index 5fbab03dd001..6c87747e770b 100644
--- a/Lib/asyncio/runners.py
+++ b/Lib/asyncio/runners.py
@@ -45,6 +45,7 @@ def run(main, *, debug=False):
         try:
             _cancel_all_tasks(loop)
             loop.run_until_complete(loop.shutdown_asyncgens())
+            loop.run_until_complete(loop.shutdown_default_executor())
         finally:
             events.set_event_loop(None)
             loop.close()
diff --git a/Misc/NEWS.d/next/Library/2019-09-11-21-38-41.bpo-34037.LIAS_3.rst b/Misc/NEWS.d/next/Library/2019-09-11-21-38-41.bpo-34037.LIAS_3.rst
new file mode 100644
index 000000000000..75345160a2fa
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-11-21-38-41.bpo-34037.LIAS_3.rst
@@ -0,0 +1,4 @@
+For :mod:`asyncio`, add a new coroutine :meth:`loop.shutdown_default_executor`.
+The new coroutine provides an API to schedule an executor shutdown that waits
+on the threadpool to finish closing. Also, :func:`asyncio.run` has been updated
+to utilize the new coroutine. Patch by Kyle Stanley.


From webhook-mailer at python.org  Thu Sep 19 09:34:45 2019
From: webhook-mailer at python.org (Paul Ganssle)
Date: Thu, 19 Sep 2019 13:34:45 -0000
Subject: [Python-checkins] bpo-38155: Add __all__ to datetime module
 (GH-16203)
Message-ID: <mailman.846.1568900085.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/96b1c59c71534db3f0f3799cd84e2006923a5098
commit: 96b1c59c71534db3f0f3799cd84e2006923a5098
branch: master
author: t k <tahia.khan at gmail.com>
committer: Paul Ganssle <paul at ganssle.io>
date: 2019-09-19T14:34:41+01:00
summary:

bpo-38155: Add __all__ to datetime module (GH-16203)

https://bugs.python.org/issue38155

files:
A Misc/NEWS.d/next/Library/2019-09-16-21-47-48.bpo-38155.d92lRc.rst
M Lib/datetime.py
M Lib/test/datetimetester.py

diff --git a/Lib/datetime.py b/Lib/datetime.py
index 0adf1dd67dfb..67555191d02c 100644
--- a/Lib/datetime.py
+++ b/Lib/datetime.py
@@ -4,6 +4,10 @@
 time zone and DST data sources.
 """
 
+__all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo",
+           "MINYEAR", "MAXYEAR")
+
+
 import time as _time
 import math as _math
 import sys
diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index 32977b12ebae..42e2cecaeb72 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -62,6 +62,12 @@ def test_constants(self):
         self.assertEqual(datetime.MINYEAR, 1)
         self.assertEqual(datetime.MAXYEAR, 9999)
 
+    def test_all(self):
+        """Test that __all__ only points to valid attributes."""
+        all_attrs = dir(datetime_module)
+        for attr in datetime_module.__all__:
+            self.assertIn(attr, all_attrs)
+
     def test_name_cleanup(self):
         if '_Pure' in self.__class__.__name__:
             self.skipTest('Only run for Fast C implementation')
diff --git a/Misc/NEWS.d/next/Library/2019-09-16-21-47-48.bpo-38155.d92lRc.rst b/Misc/NEWS.d/next/Library/2019-09-16-21-47-48.bpo-38155.d92lRc.rst
new file mode 100644
index 000000000000..14b6e2da67f9
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-16-21-47-48.bpo-38155.d92lRc.rst
@@ -0,0 +1 @@
+Add ``__all__`` to  :mod:`datetime`. Patch by Tahia Khan.


From webhook-mailer at python.org  Thu Sep 19 10:38:24 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 19 Sep 2019 14:38:24 -0000
Subject: [Python-checkins] bpo-37531: Skip
 test_regrtest.test_multiprocessing_timeout() on all platforms (GH-16282)
Message-ID: <mailman.847.1568903905.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/b2dd2dd6e9212ba6b03885b998736a180ef283d6
commit: b2dd2dd6e9212ba6b03885b998736a180ef283d6
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-19T16:38:13+02:00
summary:

bpo-37531: Skip test_regrtest.test_multiprocessing_timeout() on all platforms (GH-16282)

files:
M Lib/test/test_regrtest.py

diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 50911995ad1e..fa37ecde3762 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -1151,8 +1151,7 @@ def test_garbage(self):
                                   env_changed=[testname],
                                   fail_env_changed=True)
 
-    @unittest.skipIf(sys.platform == 'win32',
-                     'bpo-37531, bpo-38207: test hangs randomly on Windows')
+    @unittest.skipIf(True, 'bpo-37531, bpo-38207: test hangs randomly')
     def test_multiprocessing_timeout(self):
         code = textwrap.dedent(r"""
             import time


From webhook-mailer at python.org  Thu Sep 19 10:45:10 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 19 Sep 2019 14:45:10 -0000
Subject: [Python-checkins] bpo-34037: test_asyncio uses
 shutdown_default_executor() (GH-16284)
Message-ID: <mailman.848.1568904311.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/079931d12223ec98cbf53185b90db48efa61f93f
commit: 079931d12223ec98cbf53185b90db48efa61f93f
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-19T16:45:06+02:00
summary:

bpo-34037: test_asyncio uses shutdown_default_executor() (GH-16284)

files:
M Lib/test/test_asyncio/test_base_events.py
M Lib/test/test_asyncio/test_tasks.py
M Lib/test/test_asyncio/utils.py

diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
index ccdd3bae7336..66191d9d1c1a 100644
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -216,6 +216,9 @@ def submit(self, fn, *args, **kwargs):
                 raise NotImplementedError(
                     'cannot submit into a dummy executor')
 
+        self.loop._process_events = mock.Mock()
+        self.loop._write_to_self = mock.Mock()
+
         executor = DummyExecutor()
         self.loop.set_default_executor(executor)
         self.assertIs(executor, self.loop._default_executor)
@@ -226,6 +229,9 @@ def test_set_default_executor_deprecation_warnings(self):
         with self.assertWarns(DeprecationWarning):
             self.loop.set_default_executor(executor)
 
+        # Avoid cleaning up the executor mock
+        self.loop._default_executor = None
+
     def test_call_soon(self):
         def cb():
             pass
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
index 6e832eab6d58..576714faa888 100644
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -3232,6 +3232,8 @@ def task_factory(loop, coro):
         self.loop.set_exception_handler(callback)
 
         # Set corrupted task factory
+        self.addCleanup(self.loop.set_task_factory,
+                        self.loop.get_task_factory())
         self.loop.set_task_factory(task_factory)
 
         # Run event loop
diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py
index 5b4bb123a9ec..ea608f4a10c7 100644
--- a/Lib/test/test_asyncio/utils.py
+++ b/Lib/test/test_asyncio/utils.py
@@ -509,9 +509,11 @@ def get_function_source(func):
 class TestCase(unittest.TestCase):
     @staticmethod
     def close_loop(loop):
-        executor = loop._default_executor
-        if executor is not None:
-            executor.shutdown(wait=True)
+        if loop._default_executor is not None:
+            if not loop.is_closed():
+                loop.run_until_complete(loop.shutdown_default_executor())
+            else:
+                loop._default_executor.shutdown(wait=True)
         loop.close()
         policy = support.maybe_get_event_loop_policy()
         if policy is not None:


From webhook-mailer at python.org  Thu Sep 19 12:05:12 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 19 Sep 2019 16:05:12 -0000
Subject: [Python-checkins] bpo-37531: sync regrtest with master branch
 (GH-16285)
Message-ID: <mailman.849.1568909114.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/fb7746d5d10ec4a34198da672018ba15f5667079
commit: fb7746d5d10ec4a34198da672018ba15f5667079
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-19T18:05:09+02:00
summary:

bpo-37531: sync regrtest with master branch (GH-16285)

files:
M Lib/test/libregrtest/main.py
M Lib/test/libregrtest/runtest_mp.py

diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 3cfbb4017dfb..299700686928 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -22,6 +22,12 @@
 from test import support
 
 
+# bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()).
+# Used to protect against threading._shutdown() hang.
+# Must be smaller than buildbot "1200 seconds without output" limit.
+EXIT_TIMEOUT = 120.0
+
+
 class Regrtest:
     """Execute a test suite.
 
@@ -158,11 +164,6 @@ def display_progress(self, test_index, text):
     def parse_args(self, kwargs):
         ns = _parse_args(sys.argv[1:], **kwargs)
 
-        if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'):
-            print("Warning: The timeout option requires "
-                  "faulthandler.dump_traceback_later", file=sys.stderr)
-            ns.timeout = None
-
         if ns.xmlpath:
             support.junit_xml_list = self.testsuite_xml = []
 
@@ -616,16 +617,24 @@ def main(self, tests=None, **kwargs):
 
         test_cwd = self.create_temp_dir()
 
-        # Run the tests in a context manager that temporarily changes the CWD
-        # to a temporary and writable directory. If it's not possible to
-        # create or change the CWD, the original CWD will be used.
-        # The original CWD is available from support.SAVEDCWD.
-        with support.temp_cwd(test_cwd, quiet=True):
-            # When using multiprocessing, worker processes will use test_cwd
-            # as their parent temporary directory. So when the main process
-            # exit, it removes also subdirectories of worker processes.
-            self.ns.tempdir = test_cwd
-            self._main(tests, kwargs)
+        try:
+            # Run the tests in a context manager that temporarily changes the CWD
+            # to a temporary and writable directory. If it's not possible to
+            # create or change the CWD, the original CWD will be used.
+            # The original CWD is available from support.SAVEDCWD.
+            with support.temp_cwd(test_cwd, quiet=True):
+                # When using multiprocessing, worker processes will use test_cwd
+                # as their parent temporary directory. So when the main process
+                # exit, it removes also subdirectories of worker processes.
+                self.ns.tempdir = test_cwd
+
+                self._main(tests, kwargs)
+        except SystemExit as exc:
+            # bpo-38203: Python can hang at exit in Py_Finalize(), especially
+            # on threading._shutdown() call: put a timeout
+            faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True)
+
+            sys.exit(exc.code)
 
     def getloadavg(self):
         if self.win_load_tracker is not None:
diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py
index 482bb80726dc..9cb5be6bb8ae 100644
--- a/Lib/test/libregrtest/runtest_mp.py
+++ b/Lib/test/libregrtest/runtest_mp.py
@@ -185,14 +185,14 @@ def mp_result_error(self, test_name, error_type, stdout='', stderr='',
     def _timedout(self, test_name):
         self._kill()
 
-        stdout = sterr = ''
+        stdout = stderr = ''
         popen = self._popen
         try:
             stdout, stderr = popen.communicate(timeout=JOIN_TIMEOUT)
         except (subprocess.TimeoutExpired, OSError) as exc:
             print("WARNING: Failed to read worker process %s output "
                   "(timeout=%.1f sec): %r"
-                  % (popen.pid, exc, timeout),
+                  % (popen.pid, JOIN_TIMEOUT, exc),
                   file=sys.stderr, flush=True)
 
         self._close_wait()


From webhook-mailer at python.org  Thu Sep 19 12:29:09 2019
From: webhook-mailer at python.org (Dino Viehland)
Date: Thu, 19 Sep 2019 16:29:09 -0000
Subject: [Python-checkins] bpo-38140: Make dict and weakref offsets opaque
 for C heap types (#16076)
Message-ID: <mailman.850.1568910550.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/3368f3c6ae4140a0883e19350e672fd09c9db616
commit: 3368f3c6ae4140a0883e19350e672fd09c9db616
branch: master
author: Eddie Elizondo <eelizondo at fb.com>
committer: Dino Viehland <dinoviehland at gmail.com>
date: 2019-09-19T17:29:05+01:00
summary:

bpo-38140: Make dict and weakref offsets opaque for C heap types (#16076)

* Make dict and weakref offsets opaque for C heap types

* Add news

files:
A Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst
M Doc/c-api/type.rst
M Lib/test/test_capi.py
M Modules/_struct.c
M Modules/_testcapimodule.c
M Objects/typeobject.c
M Parser/asdl_c.py
M Python/Python-ast.c

diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index 6416951ac5a9..ad3e022d42ca 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -187,9 +187,8 @@ The following functions and structs are used to create
       * :c:member:`~PyTypeObject.tp_cache`
       * :c:member:`~PyTypeObject.tp_subclasses`
       * :c:member:`~PyTypeObject.tp_weaklist`
+      * :c:member:`~PyTypeObject.tp_vectorcall`
       * :c:member:`~PyTypeObject.tp_print`
-      * :c:member:`~PyTypeObject.tp_weaklistoffset`
-      * :c:member:`~PyTypeObject.tp_dictoffset`
       * :c:member:`~PyBufferProcs.bf_getbuffer`
       * :c:member:`~PyBufferProcs.bf_releasebuffer`
 
diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index b1d045c53080..79059bc7c964 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -12,6 +12,7 @@
 import threading
 import time
 import unittest
+import weakref
 from test import support
 from test.support import MISSING_C_DOCSTRINGS
 from test.support.script_helper import assert_python_failure, assert_python_ok
@@ -437,6 +438,32 @@ def __del__(self):
         # Test that subtype_dealloc decref the newly assigned __class__ only once
         self.assertEqual(new_type_refcnt, sys.getrefcount(A))
 
+    def test_heaptype_with_dict(self):
+        inst = _testcapi.HeapCTypeWithDict()
+        inst.foo = 42
+        self.assertEqual(inst.foo, 42)
+        self.assertEqual(inst.dictobj, inst.__dict__)
+        self.assertEqual(inst.dictobj, {"foo": 42})
+
+        inst = _testcapi.HeapCTypeWithDict()
+        self.assertEqual({}, inst.__dict__)
+
+    def test_heaptype_with_negative_dict(self):
+        inst = _testcapi.HeapCTypeWithNegativeDict()
+        inst.foo = 42
+        self.assertEqual(inst.foo, 42)
+        self.assertEqual(inst.dictobj, inst.__dict__)
+        self.assertEqual(inst.dictobj, {"foo": 42})
+
+        inst = _testcapi.HeapCTypeWithNegativeDict()
+        self.assertEqual({}, inst.__dict__)
+
+    def test_heaptype_with_weakref(self):
+        inst = _testcapi.HeapCTypeWithWeakref()
+        ref = weakref.ref(inst)
+        self.assertEqual(ref(), inst)
+        self.assertEqual(inst.weakreflist, ref)
+
     def test_c_subclass_of_heap_ctype_with_tpdealloc_decrefs_once(self):
         subclass_instance = _testcapi.HeapCTypeSubclass()
         type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass)
diff --git a/Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst b/Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst
new file mode 100644
index 000000000000..7df1d25632b3
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst	
@@ -0,0 +1,2 @@
+Make dict and weakref offsets opaque for C heap types by passing the offsets
+through PyMemberDef
diff --git a/Modules/_struct.c b/Modules/_struct.c
index c387ae2ffc19..cc536b46a623 100644
--- a/Modules/_struct.c
+++ b/Modules/_struct.c
@@ -2024,7 +2024,7 @@ static struct PyMethodDef s_methods[] = {
 };
 
 static PyMemberDef s_members[] = {
-    {"__weaklistoffset__", T_NONE, offsetof(PyStructObject, weakreflist), READONLY},
+    {"__weaklistoffset__", T_PYSSIZET, offsetof(PyStructObject, weakreflist), READONLY},
     {NULL}  /* sentinel */
 };
 
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 4cca784d0148..30a786cd3455 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -6363,6 +6363,106 @@ static PyType_Spec HeapCTypeSubclassWithFinalizer_spec = {
     HeapCTypeSubclassWithFinalizer_slots
 };
 
+typedef struct {
+    PyObject_HEAD
+    PyObject *dict;
+} HeapCTypeWithDictObject;
+
+static void
+heapctypewithdict_dealloc(HeapCTypeWithDictObject* self)
+{
+
+    PyTypeObject *tp = Py_TYPE(self);
+    Py_XDECREF(self->dict);
+    PyObject_DEL(self);
+    Py_DECREF(tp);
+}
+
+static PyGetSetDef heapctypewithdict_getsetlist[] = {
+    {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict},
+    {NULL} /* Sentinel */
+};
+
+static struct PyMemberDef heapctypewithdict_members[] = {
+    {"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)},
+    {"__dictoffset__", T_PYSSIZET, offsetof(HeapCTypeWithDictObject, dict), READONLY},
+    {NULL} /* Sentinel */
+};
+
+static PyType_Slot HeapCTypeWithDict_slots[] = {
+    {Py_tp_members, heapctypewithdict_members},
+    {Py_tp_getset, heapctypewithdict_getsetlist},
+    {Py_tp_dealloc, heapctypewithdict_dealloc},
+    {0, 0},
+};
+
+static PyType_Spec HeapCTypeWithDict_spec = {
+    "_testcapi.HeapCTypeWithDict",
+    sizeof(HeapCTypeWithDictObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    HeapCTypeWithDict_slots
+};
+
+static struct PyMemberDef heapctypewithnegativedict_members[] = {
+    {"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)},
+    {"__dictoffset__", T_PYSSIZET, -sizeof(void*), READONLY},
+    {NULL} /* Sentinel */
+};
+
+static PyType_Slot HeapCTypeWithNegativeDict_slots[] = {
+    {Py_tp_members, heapctypewithnegativedict_members},
+    {Py_tp_getset, heapctypewithdict_getsetlist},
+    {Py_tp_dealloc, heapctypewithdict_dealloc},
+    {0, 0},
+};
+
+static PyType_Spec HeapCTypeWithNegativeDict_spec = {
+    "_testcapi.HeapCTypeWithNegativeDict",
+    sizeof(HeapCTypeWithDictObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    HeapCTypeWithNegativeDict_slots
+};
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *weakreflist;
+} HeapCTypeWithWeakrefObject;
+
+static struct PyMemberDef heapctypewithweakref_members[] = {
+    {"weakreflist", T_OBJECT, offsetof(HeapCTypeWithWeakrefObject, weakreflist)},
+    {"__weaklistoffset__", T_PYSSIZET,
+      offsetof(HeapCTypeWithWeakrefObject, weakreflist), READONLY},
+    {NULL} /* Sentinel */
+};
+
+static void
+heapctypewithweakref_dealloc(HeapCTypeWithWeakrefObject* self)
+{
+
+    PyTypeObject *tp = Py_TYPE(self);
+    if (self->weakreflist != NULL)
+        PyObject_ClearWeakRefs((PyObject *) self);
+    Py_XDECREF(self->weakreflist);
+    PyObject_DEL(self);
+    Py_DECREF(tp);
+}
+
+static PyType_Slot HeapCTypeWithWeakref_slots[] = {
+    {Py_tp_members, heapctypewithweakref_members},
+    {Py_tp_dealloc, heapctypewithweakref_dealloc},
+    {0, 0},
+};
+
+static PyType_Spec HeapCTypeWithWeakref_spec = {
+    "_testcapi.HeapCTypeWithWeakref",
+    sizeof(HeapCTypeWithWeakrefObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    HeapCTypeWithWeakref_slots
+};
+
 static PyMethodDef meth_instance_methods[] = {
     {"meth_varargs", meth_varargs, METH_VARARGS},
     {"meth_varargs_keywords", (PyCFunction)(void(*)(void))meth_varargs_keywords, METH_VARARGS|METH_KEYWORDS},
@@ -6596,6 +6696,24 @@ PyInit__testcapi(void)
     Py_DECREF(subclass_bases);
     PyModule_AddObject(m, "HeapCTypeSubclass", HeapCTypeSubclass);
 
+    PyObject *HeapCTypeWithDict = PyType_FromSpec(&HeapCTypeWithDict_spec);
+    if (HeapCTypeWithDict == NULL) {
+        return NULL;
+    }
+    PyModule_AddObject(m, "HeapCTypeWithDict", HeapCTypeWithDict);
+
+    PyObject *HeapCTypeWithNegativeDict = PyType_FromSpec(&HeapCTypeWithNegativeDict_spec);
+    if (HeapCTypeWithNegativeDict == NULL) {
+        return NULL;
+    }
+    PyModule_AddObject(m, "HeapCTypeWithNegativeDict", HeapCTypeWithNegativeDict);
+
+    PyObject *HeapCTypeWithWeakref = PyType_FromSpec(&HeapCTypeWithWeakref_spec);
+    if (HeapCTypeWithWeakref == NULL) {
+        return NULL;
+    }
+    PyModule_AddObject(m, "HeapCTypeWithWeakref", HeapCTypeWithWeakref);
+
     PyObject *subclass_with_finalizer_bases = PyTuple_Pack(1, HeapCTypeSubclass);
     if (subclass_with_finalizer_bases == NULL) {
         return NULL;
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index dfdac9e2e4f5..94a4da22d679 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -2853,15 +2853,27 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
     PyTypeObject *type, *base;
 
     PyType_Slot *slot;
-    Py_ssize_t nmembers;
+    Py_ssize_t nmembers, weaklistoffset, dictoffset;
     char *s, *res_start;
 
-    nmembers = 0;
+    nmembers = weaklistoffset = dictoffset = 0;
     for (slot = spec->slots; slot->slot; slot++) {
         if (slot->slot == Py_tp_members) {
             nmembers = 0;
             for (memb = slot->pfunc; memb->name != NULL; memb++) {
                 nmembers++;
+                if (strcmp(memb->name, "__weaklistoffset__") == 0) {
+                    // The PyMemberDef must be a Py_ssize_t and readonly
+                    assert(memb->type == T_PYSSIZET);
+                    assert(memb->flags == READONLY);
+                    weaklistoffset = memb->offset;
+                }
+                if (strcmp(memb->name, "__dictoffset__") == 0) {
+                    // The PyMemberDef must be a Py_ssize_t and readonly
+                    assert(memb->type == T_PYSSIZET);
+                    assert(memb->flags == READONLY);
+                    dictoffset = memb->offset;
+                }
             }
         }
     }
@@ -2990,6 +3002,17 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
         res->ht_cached_keys = _PyDict_NewKeysForClass();
     }
 
+    if (weaklistoffset) {
+        type->tp_weaklistoffset = weaklistoffset;
+        if (PyDict_DelItemString((PyObject *)type->tp_dict, "__weaklistoffset__") < 0)
+            goto fail;
+    }
+    if (dictoffset) {
+        type->tp_dictoffset = dictoffset;
+        if (PyDict_DelItemString((PyObject *)type->tp_dict, "__dictoffset__") < 0)
+            goto fail;
+    }
+
     /* Set type.__module__ */
     s = strrchr(spec->name, '.');
     if (s != NULL) {
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 0b72f9ee236c..3a9d4e6b4b92 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -724,6 +724,11 @@ def visitModule(self, mod):
     return Py_BuildValue("O()", Py_TYPE(self));
 }
 
+static PyMemberDef ast_type_members[] = {
+    {"__dictoffset__", T_PYSSIZET, offsetof(AST_object, dict), READONLY},
+    {NULL}  /* Sentinel */
+};
+
 static PyMethodDef ast_type_methods[] = {
     {"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
     {NULL}
@@ -740,6 +745,7 @@ def visitModule(self, mod):
     {Py_tp_setattro, PyObject_GenericSetAttr},
     {Py_tp_traverse, ast_traverse},
     {Py_tp_clear, ast_clear},
+    {Py_tp_members, ast_type_members},
     {Py_tp_methods, ast_type_methods},
     {Py_tp_getset, ast_type_getsets},
     {Py_tp_init, ast_type_init},
@@ -930,7 +936,6 @@ def visitModule(self, mod):
         self.emit("if (init_identifiers() < 0) return 0;", 1)
         self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1)
         self.emit("if (!state->AST_type) return 0;", 1)
-        self.emit("((PyTypeObject*)state->AST_type)->tp_dictoffset = offsetof(AST_object, dict);", 1)
         self.emit("if (add_ast_fields() < 0) return 0;", 1)
         for dfn in mod.dfns:
             self.visit(dfn)
@@ -1372,6 +1377,7 @@ def main(srcfile, dump_module=False):
             f.write('\n')
             f.write('#include "Python.h"\n')
             f.write('#include "%s-ast.h"\n' % mod.name)
+            f.write('#include "structmember.h"\n')
             f.write('\n')
 
             generate_module_def(f, mod)
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 7a38e98c435a..558d9ebea5c7 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -4,6 +4,7 @@
 
 #include "Python.h"
 #include "Python-ast.h"
+#include "structmember.h"
 
 typedef struct {
     int initialized;
@@ -1216,6 +1217,11 @@ ast_type_reduce(PyObject *self, PyObject *unused)
     return Py_BuildValue("O()", Py_TYPE(self));
 }
 
+static PyMemberDef ast_type_members[] = {
+    {"__dictoffset__", T_PYSSIZET, offsetof(AST_object, dict), READONLY},
+    {NULL}  /* Sentinel */
+};
+
 static PyMethodDef ast_type_methods[] = {
     {"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
     {NULL}
@@ -1232,6 +1238,7 @@ static PyType_Slot AST_type_slots[] = {
     {Py_tp_setattro, PyObject_GenericSetAttr},
     {Py_tp_traverse, ast_traverse},
     {Py_tp_clear, ast_clear},
+    {Py_tp_members, ast_type_members},
     {Py_tp_methods, ast_type_methods},
     {Py_tp_getset, ast_type_getsets},
     {Py_tp_init, ast_type_init},
@@ -1421,8 +1428,6 @@ static int init_types(void)
     if (init_identifiers() < 0) return 0;
     state->AST_type = PyType_FromSpec(&AST_type_spec);
     if (!state->AST_type) return 0;
-    ((PyTypeObject*)state->AST_type)->tp_dictoffset = offsetof(AST_object,
-     dict);
     if (add_ast_fields() < 0) return 0;
     state->mod_type = make_type("mod", state->AST_type, NULL, 0);
     if (!state->mod_type) return 0;


From webhook-mailer at python.org  Thu Sep 19 12:40:51 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 19 Sep 2019 16:40:51 -0000
Subject: [Python-checkins] bpo-37531: sync regrtest with master branch
 (GH-16285) (GH-16289)
Message-ID: <mailman.851.1568911252.30340.python-checkins@python.org>

https://github.com/python/cpython/commit/5e1400a6bcbb3350a6665176980a2b8343075c63
commit: 5e1400a6bcbb3350a6665176980a2b8343075c63
branch: 3.7
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-19T18:40:46+02:00
summary:

bpo-37531: sync regrtest with master branch (GH-16285) (GH-16289)

(cherry picked from commit fb7746d5d10ec4a34198da672018ba15f5667079)

files:
M Lib/test/libregrtest/main.py
M Lib/test/libregrtest/runtest_mp.py

diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 7d6a62863599..a59a03d4f251 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -21,6 +21,12 @@
 from test import support
 
 
+# bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()).
+# Used to protect against threading._shutdown() hang.
+# Must be smaller than buildbot "1200 seconds without output" limit.
+EXIT_TIMEOUT = 120.0
+
+
 class Regrtest:
     """Execute a test suite.
 
@@ -157,11 +163,6 @@ def display_progress(self, test_index, text):
     def parse_args(self, kwargs):
         ns = _parse_args(sys.argv[1:], **kwargs)
 
-        if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'):
-            print("Warning: The timeout option requires "
-                  "faulthandler.dump_traceback_later", file=sys.stderr)
-            ns.timeout = None
-
         if ns.xmlpath:
             support.junit_xml_list = self.testsuite_xml = []
 
@@ -611,16 +612,24 @@ def main(self, tests=None, **kwargs):
 
         test_cwd = self.create_temp_dir()
 
-        # Run the tests in a context manager that temporarily changes the CWD
-        # to a temporary and writable directory. If it's not possible to
-        # create or change the CWD, the original CWD will be used.
-        # The original CWD is available from support.SAVEDCWD.
-        with support.temp_cwd(test_cwd, quiet=True):
-            # When using multiprocessing, worker processes will use test_cwd
-            # as their parent temporary directory. So when the main process
-            # exit, it removes also subdirectories of worker processes.
-            self.ns.tempdir = test_cwd
-            self._main(tests, kwargs)
+        try:
+            # Run the tests in a context manager that temporarily changes the CWD
+            # to a temporary and writable directory. If it's not possible to
+            # create or change the CWD, the original CWD will be used.
+            # The original CWD is available from support.SAVEDCWD.
+            with support.temp_cwd(test_cwd, quiet=True):
+                # When using multiprocessing, worker processes will use test_cwd
+                # as their parent temporary directory. So when the main process
+                # exit, it removes also subdirectories of worker processes.
+                self.ns.tempdir = test_cwd
+
+                self._main(tests, kwargs)
+        except SystemExit as exc:
+            # bpo-38203: Python can hang at exit in Py_Finalize(), especially
+            # on threading._shutdown() call: put a timeout
+            faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True)
+
+            sys.exit(exc.code)
 
     def getloadavg(self):
         if self.win_load_tracker is not None:
diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py
index 482bb80726dc..9cb5be6bb8ae 100644
--- a/Lib/test/libregrtest/runtest_mp.py
+++ b/Lib/test/libregrtest/runtest_mp.py
@@ -185,14 +185,14 @@ def mp_result_error(self, test_name, error_type, stdout='', stderr='',
     def _timedout(self, test_name):
         self._kill()
 
-        stdout = sterr = ''
+        stdout = stderr = ''
         popen = self._popen
         try:
             stdout, stderr = popen.communicate(timeout=JOIN_TIMEOUT)
         except (subprocess.TimeoutExpired, OSError) as exc:
             print("WARNING: Failed to read worker process %s output "
                   "(timeout=%.1f sec): %r"
-                  % (popen.pid, exc, timeout),
+                  % (popen.pid, JOIN_TIMEOUT, exc),
                   file=sys.stderr, flush=True)
 
         self._close_wait()


From webhook-mailer at python.org  Thu Sep 19 23:26:00 2019
From: webhook-mailer at python.org (Abhilash Raj)
Date: Fri, 20 Sep 2019 03:26:00 -0000
Subject: [Python-checkins] bpo-34002: Minor efficiency and clarity
 improvements in email package. (GH-7999)
Message-ID: <mailman.0.1568949961.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/2702638eabe5f7b25f36d295f0ad78cb8d4eda05
commit: 2702638eabe5f7b25f36d295f0ad78cb8d4eda05
branch: master
author: Michael Selik <mike at selik.org>
committer: Abhilash Raj <maxking at users.noreply.github.com>
date: 2019-09-19T20:25:55-07:00
summary:

bpo-34002: Minor efficiency and clarity improvements in email package. (GH-7999)

* Check intersection of two sets explicitly

Comparing ``len(a) > ``len(a - b)`` is essentially looking for an
intersection between the two sets. If set ``b`` does not intersect ``a``
then ``len(a - b)`` will be equal to ``len(a)``. This logic is more
clearly expressed as ``a & b``.

* Change while/pop to a for-loop

Copying the list, then repeatedly popping the first element was
unnecessarily slow. I also cleaned up a couple other inefficiencies.
There's no need to unpack a tuple, then re-pack and append it. The list
can be created with the first element instead of empty. Secondly, the
``endswith`` method returns a bool, so there's no need for an if-
statement to set ``encoding`` to True or False.

* Use set.intersection to check for intersections

``a.intersection(b)`` method is more clear of purpose than ``not
a.isdisjoint(b)`` and avoids an unnecessary set construction that ``a &
set(b)`` performs.

* Use not isdisjoint instead of intersection

While it reads slightly worse, the isdisjoint method will stop when it
finds a counterexample and returns a bool, rather than looping over the
entire iterable and constructing a new set.

files:
A Misc/NEWS.d/next/Library/2019-09-19-19-58-33.bpo-34002.KBnaVX.rst
M Lib/email/headerregistry.py
M Lib/email/utils.py

diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py
index dcc960b2cdc1..cc1d19129181 100644
--- a/Lib/email/headerregistry.py
+++ b/Lib/email/headerregistry.py
@@ -69,11 +69,9 @@ def addr_spec(self):
         """The addr_spec (username at domain) portion of the address, quoted
         according to RFC 5322 rules, but with no Content Transfer Encoding.
         """
-        nameset = set(self.username)
-        if len(nameset) > len(nameset-parser.DOT_ATOM_ENDS):
-            lp = parser.quote_string(self.username)
-        else:
-            lp = self.username
+        lp = self.username
+        if not parser.DOT_ATOM_ENDS.isdisjoint(lp):
+            lp = parser.quote_string(lp)
         if self.domain:
             return lp + '@' + self.domain
         if not lp:
@@ -86,11 +84,9 @@ def __repr__(self):
                         self.display_name, self.username, self.domain)
 
     def __str__(self):
-        nameset = set(self.display_name)
-        if len(nameset) > len(nameset-parser.SPECIALS):
-            disp = parser.quote_string(self.display_name)
-        else:
-            disp = self.display_name
+        disp = self.display_name
+        if not parser.SPECIALS.isdisjoint(disp):
+            disp = parser.quote_string(disp)
         if disp:
             addr_spec = '' if self.addr_spec=='<>' else self.addr_spec
             return "{} <{}>".format(disp, addr_spec)
@@ -141,10 +137,8 @@ def __str__(self):
         if self.display_name is None and len(self.addresses)==1:
             return str(self.addresses[0])
         disp = self.display_name
-        if disp is not None:
-            nameset = set(disp)
-            if len(nameset) > len(nameset-parser.SPECIALS):
-                disp = parser.quote_string(disp)
+        if disp is not None and not parser.SPECIALS.isdisjoint(disp):
+            disp = parser.quote_string(disp)
         adrstr = ", ".join(str(x) for x in self.addresses)
         adrstr = ' ' + adrstr if adrstr else adrstr
         return "{}:{};".format(disp, adrstr)
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
index 858f620e25bf..b137ce3973a4 100644
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -259,21 +259,13 @@ def decode_params(params):
 
     params is a sequence of 2-tuples containing (param name, string value).
     """
-    # Copy params so we don't mess with the original
-    params = params[:]
-    new_params = []
+    new_params = [params[0]]
     # Map parameter's name to a list of continuations.  The values are a
     # 3-tuple of the continuation number, the string value, and a flag
     # specifying whether a particular segment is %-encoded.
     rfc2231_params = {}
-    name, value = params.pop(0)
-    new_params.append((name, value))
-    while params:
-        name, value = params.pop(0)
-        if name.endswith('*'):
-            encoded = True
-        else:
-            encoded = False
+    for name, value in params[1:]:
+        encoded = name.endswith('*')
         value = unquote(value)
         mo = rfc2231_continuation.match(name)
         if mo:
diff --git a/Misc/NEWS.d/next/Library/2019-09-19-19-58-33.bpo-34002.KBnaVX.rst b/Misc/NEWS.d/next/Library/2019-09-19-19-58-33.bpo-34002.KBnaVX.rst
new file mode 100644
index 000000000000..5d4c231c74c6
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-19-19-58-33.bpo-34002.KBnaVX.rst
@@ -0,0 +1,2 @@
+Improve efficiency in parts of email package by changing while-pop to a for
+loop, using isdisjoint instead of set intersections.


From webhook-mailer at python.org  Fri Sep 20 00:04:22 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Fri, 20 Sep 2019 04:04:22 -0000
Subject: [Python-checkins] bpo-38093: Correctly returns AsyncMock for async
 subclasses. (GH-15947)
Message-ID: <mailman.1.1568952263.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/8b03f943c37e07fb2394acdcfacd066647f9b1fd
commit: 8b03f943c37e07fb2394acdcfacd066647f9b1fd
branch: master
author: Lisa Roach <lisaroach14 at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-19T21:04:18-07:00
summary:

bpo-38093: Correctly returns AsyncMock for async subclasses. (GH-15947)

files:
A Misc/NEWS.d/next/Library/2019-09-11-14-45-30.bpo-38093.yQ6k7y.rst
M Doc/library/unittest.mock-examples.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py
M Lib/unittest/test/testmock/testmagicmethods.py

diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst
index cf6b671b5bee..e650bb1e23e0 100644
--- a/Doc/library/unittest.mock-examples.rst
+++ b/Doc/library/unittest.mock-examples.rst
@@ -14,7 +14,7 @@
 
     import asyncio
     import unittest
-    from unittest.mock import Mock, MagicMock, patch, call, sentinel
+    from unittest.mock import Mock, MagicMock, AsyncMock, patch, call, sentinel
 
     class SomeClass:
         attribute = 'this is a doctest'
@@ -280,14 +280,16 @@ function returns is what the call returns:
 Mocking asynchronous iterators
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Since Python 3.8, ``MagicMock`` has support to mock :ref:`async-iterators`
-through ``__aiter__``. The :attr:`~Mock.return_value` attribute of ``__aiter__``
-can be used to set the return values to be used for iteration.
+Since Python 3.8, ``AsyncMock`` and ``MagicMock`` have support to mock
+:ref:`async-iterators` through ``__aiter__``. The :attr:`~Mock.return_value`
+attribute of ``__aiter__`` can be used to set the return values to be used for
+iteration.
 
-    >>> mock = MagicMock()
+    >>> mock = MagicMock()  # AsyncMock also works here
     >>> mock.__aiter__.return_value = [1, 2, 3]
     >>> async def main():
     ...     return [i async for i in mock]
+    ...
     >>> asyncio.run(main())
     [1, 2, 3]
 
@@ -295,24 +297,25 @@ can be used to set the return values to be used for iteration.
 Mocking asynchronous context manager
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Since Python 3.8, ``MagicMock`` has support to mock
-:ref:`async-context-managers` through ``__aenter__`` and ``__aexit__``. The
-return value of ``__aenter__`` is an :class:`AsyncMock`.
+Since Python 3.8, ``AsyncMock`` and ``MagicMock`` have support to mock
+:ref:`async-context-managers` through ``__aenter__`` and ``__aexit__``.
+By default, ``__aenter__`` and ``__aexit__`` are ``AsyncMock`` instances that
+return an async function.
 
     >>> class AsyncContextManager:
-    ...
     ...     async def __aenter__(self):
     ...         return self
-    ...
-    ...     async def __aexit__(self):
+    ...     async def __aexit__(self, exc_type, exc, tb):
     ...         pass
-    >>> mock_instance = MagicMock(AsyncContextManager())
+    ...
+    >>> mock_instance = MagicMock(AsyncContextManager())  # AsyncMock also works here
     >>> async def main():
     ...     async with mock_instance as result:
     ...         pass
+    ...
     >>> asyncio.run(main())
-    >>> mock_instance.__aenter__.assert_called_once()
-    >>> mock_instance.__aexit__.assert_called_once()
+    >>> mock_instance.__aenter__.assert_awaited_once()
+    >>> mock_instance.__aexit__.assert_awaited_once()
 
 
 Creating a Mock from an Existing Object
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 4cf8e60ccc15..0a16e26f1d8a 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -983,9 +983,13 @@ def _get_child_mock(self, /, **kw):
         _type = type(self)
         if issubclass(_type, MagicMock) and _new_name in _async_method_magics:
             klass = AsyncMock
-        if issubclass(_type, AsyncMockMixin):
+        elif _new_name  in _sync_async_magics:
+            # Special case these ones b/c users will assume they are async,
+            # but they are actually sync (ie. __aiter__)
             klass = MagicMock
-        if not issubclass(_type, CallableMixin):
+        elif issubclass(_type, AsyncMockMixin):
+            klass = AsyncMock
+        elif not issubclass(_type, CallableMixin):
             if issubclass(_type, NonCallableMagicMock):
                 klass = MagicMock
             elif issubclass(_type, NonCallableMock) :
@@ -1881,7 +1885,7 @@ def _patch_stopall():
     '__reduce__', '__reduce_ex__', '__getinitargs__', '__getnewargs__',
     '__getstate__', '__setstate__', '__getformat__', '__setformat__',
     '__repr__', '__dir__', '__subclasses__', '__format__',
-    '__getnewargs_ex__', '__aenter__', '__aexit__', '__anext__', '__aiter__',
+    '__getnewargs_ex__',
 }
 
 
@@ -1900,10 +1904,12 @@ def method(self, /, *args, **kw):
 
 # Magic methods used for async `with` statements
 _async_method_magics = {"__aenter__", "__aexit__", "__anext__"}
-# `__aiter__` is a plain function but used with async calls
-_async_magics = _async_method_magics | {"__aiter__"}
+# Magic methods that are only used with async calls but are synchronous functions themselves
+_sync_async_magics = {"__aiter__"}
+_async_magics = _async_method_magics | _sync_async_magics
 
-_all_magics = _magics | _non_defaults
+_all_sync_magics = _magics | _non_defaults
+_all_magics = _all_sync_magics | _async_magics
 
 _unsupported_magics = {
     '__getattr__', '__setattr__',
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index 87f3cfc8fa47..0d68c9757980 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -382,35 +382,88 @@ def test_add_side_effect_iterable(self):
 class AsyncContextManagerTest(unittest.TestCase):
 
     class WithAsyncContextManager:
-
         async def __aenter__(self, *args, **kwargs):
             return self
 
         async def __aexit__(self, *args, **kwargs):
             pass
 
-    def test_magic_methods_are_async_mocks(self):
-        mock = MagicMock(self.WithAsyncContextManager())
-        self.assertIsInstance(mock.__aenter__, AsyncMock)
-        self.assertIsInstance(mock.__aexit__, AsyncMock)
+    class WithSyncContextManager:
+        def __enter__(self, *args, **kwargs):
+            return self
+
+        def __exit__(self, *args, **kwargs):
+            pass
+
+    class ProductionCode:
+        # Example real-world(ish) code
+        def __init__(self):
+            self.session = None
+
+        async def main(self):
+            async with self.session.post('https://python.org') as response:
+                val = await response.json()
+                return val
+
+    def test_async_magic_methods_are_async_mocks_with_magicmock(self):
+        cm_mock = MagicMock(self.WithAsyncContextManager())
+        self.assertIsInstance(cm_mock.__aenter__, AsyncMock)
+        self.assertIsInstance(cm_mock.__aexit__, AsyncMock)
+
+    def test_magicmock_has_async_magic_methods(self):
+        cm = MagicMock(name='magic_cm')
+        self.assertTrue(hasattr(cm, "__aenter__"))
+        self.assertTrue(hasattr(cm, "__aexit__"))
+
+    def test_magic_methods_are_async_functions(self):
+        cm = MagicMock(name='magic_cm')
+        self.assertIsInstance(cm.__aenter__, AsyncMock)
+        self.assertIsInstance(cm.__aexit__, AsyncMock)
+        # AsyncMocks are also coroutine functions
+        self.assertTrue(asyncio.iscoroutinefunction(cm.__aenter__))
+        self.assertTrue(asyncio.iscoroutinefunction(cm.__aexit__))
+
+    def test_set_return_value_of_aenter(self):
+        def inner_test(mock_type):
+            pc = self.ProductionCode()
+            pc.session = MagicMock(name='sessionmock')
+            cm = mock_type(name='magic_cm')
+            response = AsyncMock(name='response')
+            response.json = AsyncMock(return_value={'json': 123})
+            cm.__aenter__.return_value = response
+            pc.session.post.return_value = cm
+            result = asyncio.run(pc.main())
+            self.assertEqual(result, {'json': 123})
+
+        for mock_type in [AsyncMock, MagicMock]:
+            with self.subTest(f"test set return value of aenter with {mock_type}"):
+                inner_test(mock_type)
 
     def test_mock_supports_async_context_manager(self):
-        called = False
-        instance = self.WithAsyncContextManager()
-        mock_instance = MagicMock(instance)
+        def inner_test(mock_type):
+            called = False
+            cm = self.WithAsyncContextManager()
+            cm_mock = mock_type(cm)
+
+            async def use_context_manager():
+                nonlocal called
+                async with cm_mock as result:
+                    called = True
+                return result
 
-        async def use_context_manager():
-            nonlocal called
-            async with mock_instance as result:
-                called = True
-            return result
+            cm_result = asyncio.run(use_context_manager())
+            self.assertTrue(called)
+            self.assertTrue(cm_mock.__aenter__.called)
+            self.assertTrue(cm_mock.__aexit__.called)
+            cm_mock.__aenter__.assert_awaited()
+            cm_mock.__aexit__.assert_awaited()
+            # We mock __aenter__ so it does not return self
+            self.assertIsNot(cm_mock, cm_result)
+
+        for mock_type in [AsyncMock, MagicMock]:
+            with self.subTest(f"test context manager magics with {mock_type}"):
+                inner_test(mock_type)
 
-        result = asyncio.run(use_context_manager())
-        self.assertTrue(called)
-        self.assertTrue(mock_instance.__aenter__.called)
-        self.assertTrue(mock_instance.__aexit__.called)
-        self.assertIsNot(mock_instance, result)
-        self.assertIsInstance(result, AsyncMock)
 
     def test_mock_customize_async_context_manager(self):
         instance = self.WithAsyncContextManager()
@@ -478,27 +531,30 @@ def __aiter__(self):
 
             raise StopAsyncIteration
 
-    def test_mock_aiter_and_anext(self):
-        instance = self.WithAsyncIterator()
-        mock_instance = MagicMock(instance)
-
-        self.assertEqual(asyncio.iscoroutine(instance.__aiter__),
-                         asyncio.iscoroutine(mock_instance.__aiter__))
-        self.assertEqual(asyncio.iscoroutine(instance.__anext__),
-                         asyncio.iscoroutine(mock_instance.__anext__))
-
-        iterator = instance.__aiter__()
-        if asyncio.iscoroutine(iterator):
-            iterator = asyncio.run(iterator)
-
-        mock_iterator = mock_instance.__aiter__()
-        if asyncio.iscoroutine(mock_iterator):
-            mock_iterator = asyncio.run(mock_iterator)
+    def test_aiter_set_return_value(self):
+        mock_iter = AsyncMock(name="tester")
+        mock_iter.__aiter__.return_value = [1, 2, 3]
+        async def main():
+            return [i async for i in mock_iter]
+        result = asyncio.run(main())
+        self.assertEqual(result, [1, 2, 3])
+
+    def test_mock_aiter_and_anext_asyncmock(self):
+        def inner_test(mock_type):
+            instance = self.WithAsyncIterator()
+            mock_instance = mock_type(instance)
+            # Check that the mock and the real thing bahave the same
+            # __aiter__ is not actually async, so not a coroutinefunction
+            self.assertFalse(asyncio.iscoroutinefunction(instance.__aiter__))
+            self.assertFalse(asyncio.iscoroutinefunction(mock_instance.__aiter__))
+            # __anext__ is async
+            self.assertTrue(asyncio.iscoroutinefunction(instance.__anext__))
+            self.assertTrue(asyncio.iscoroutinefunction(mock_instance.__anext__))
+
+        for mock_type in [AsyncMock, MagicMock]:
+            with self.subTest(f"test aiter and anext corourtine with {mock_type}"):
+                inner_test(mock_type)
 
-        self.assertEqual(asyncio.iscoroutine(iterator.__aiter__),
-                         asyncio.iscoroutine(mock_iterator.__aiter__))
-        self.assertEqual(asyncio.iscoroutine(iterator.__anext__),
-                         asyncio.iscoroutine(mock_iterator.__anext__))
 
     def test_mock_async_for(self):
         async def iterate(iterator):
@@ -509,19 +565,30 @@ def test_mock_async_for(self):
             return accumulator
 
         expected = ["FOO", "BAR", "BAZ"]
-        with self.subTest("iterate through default value"):
-            mock_instance = MagicMock(self.WithAsyncIterator())
-            self.assertEqual([], asyncio.run(iterate(mock_instance)))
+        def test_default(mock_type):
+            mock_instance = mock_type(self.WithAsyncIterator())
+            self.assertEqual(asyncio.run(iterate(mock_instance)), [])
+
 
-        with self.subTest("iterate through set return_value"):
-            mock_instance = MagicMock(self.WithAsyncIterator())
+        def test_set_return_value(mock_type):
+            mock_instance = mock_type(self.WithAsyncIterator())
             mock_instance.__aiter__.return_value = expected[:]
-            self.assertEqual(expected, asyncio.run(iterate(mock_instance)))
+            self.assertEqual(asyncio.run(iterate(mock_instance)), expected)
 
-        with self.subTest("iterate through set return_value iterator"):
-            mock_instance = MagicMock(self.WithAsyncIterator())
+        def test_set_return_value_iter(mock_type):
+            mock_instance = mock_type(self.WithAsyncIterator())
             mock_instance.__aiter__.return_value = iter(expected[:])
-            self.assertEqual(expected, asyncio.run(iterate(mock_instance)))
+            self.assertEqual(asyncio.run(iterate(mock_instance)), expected)
+
+        for mock_type in [AsyncMock, MagicMock]:
+            with self.subTest(f"default value with {mock_type}"):
+                test_default(mock_type)
+
+            with self.subTest(f"set return_value with {mock_type}"):
+                test_set_return_value(mock_type)
+
+            with self.subTest(f"set return_value iterator with {mock_type}"):
+                test_set_return_value_iter(mock_type)
 
 
 class AsyncMockAssert(unittest.TestCase):
diff --git a/Lib/unittest/test/testmock/testmagicmethods.py b/Lib/unittest/test/testmock/testmagicmethods.py
index 130a3397ba0d..57f85e951e20 100644
--- a/Lib/unittest/test/testmock/testmagicmethods.py
+++ b/Lib/unittest/test/testmock/testmagicmethods.py
@@ -1,8 +1,9 @@
+import asyncio
 import math
 import unittest
 import os
 import sys
-from unittest.mock import Mock, MagicMock, _magics
+from unittest.mock import AsyncMock, Mock, MagicMock, _magics
 
 
 
@@ -271,6 +272,34 @@ def test_magic_mock_equality(self):
         self.assertEqual(mock != mock, False)
 
 
+    # This should be fixed with issue38163
+    @unittest.expectedFailure
+    def test_asyncmock_defaults(self):
+        mock = AsyncMock()
+        self.assertEqual(int(mock), 1)
+        self.assertEqual(complex(mock), 1j)
+        self.assertEqual(float(mock), 1.0)
+        self.assertNotIn(object(), mock)
+        self.assertEqual(len(mock), 0)
+        self.assertEqual(list(mock), [])
+        self.assertEqual(hash(mock), object.__hash__(mock))
+        self.assertEqual(str(mock), object.__str__(mock))
+        self.assertTrue(bool(mock))
+        self.assertEqual(round(mock), mock.__round__())
+        self.assertEqual(math.trunc(mock), mock.__trunc__())
+        self.assertEqual(math.floor(mock), mock.__floor__())
+        self.assertEqual(math.ceil(mock), mock.__ceil__())
+        self.assertTrue(asyncio.iscoroutinefunction(mock.__aexit__))
+        self.assertTrue(asyncio.iscoroutinefunction(mock.__aenter__))
+        self.assertIsInstance(mock.__aenter__, AsyncMock)
+        self.assertIsInstance(mock.__aexit__, AsyncMock)
+
+        # in Python 3 oct and hex use __index__
+        # so these tests are for __index__ in py3k
+        self.assertEqual(oct(mock), '0o1')
+        self.assertEqual(hex(mock), '0x1')
+        # how to test __sizeof__ ?
+
     def test_magicmock_defaults(self):
         mock = MagicMock()
         self.assertEqual(int(mock), 1)
@@ -286,6 +315,10 @@ def test_magicmock_defaults(self):
         self.assertEqual(math.trunc(mock), mock.__trunc__())
         self.assertEqual(math.floor(mock), mock.__floor__())
         self.assertEqual(math.ceil(mock), mock.__ceil__())
+        self.assertTrue(asyncio.iscoroutinefunction(mock.__aexit__))
+        self.assertTrue(asyncio.iscoroutinefunction(mock.__aenter__))
+        self.assertIsInstance(mock.__aenter__, AsyncMock)
+        self.assertIsInstance(mock.__aexit__, AsyncMock)
 
         # in Python 3 oct and hex use __index__
         # so these tests are for __index__ in py3k
diff --git a/Misc/NEWS.d/next/Library/2019-09-11-14-45-30.bpo-38093.yQ6k7y.rst b/Misc/NEWS.d/next/Library/2019-09-11-14-45-30.bpo-38093.yQ6k7y.rst
new file mode 100644
index 000000000000..24a53013cec7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-11-14-45-30.bpo-38093.yQ6k7y.rst
@@ -0,0 +1,2 @@
+Fixes AsyncMock so it doesn't crash when used with AsyncContextManagers
+or AsyncIterators.


From webhook-mailer at python.org  Fri Sep 20 09:55:39 2019
From: webhook-mailer at python.org (Zachary Ware)
Date: Fri, 20 Sep 2019 13:55:39 -0000
Subject: [Python-checkins] bpo-37353: Updated parser note about source code
 compatibility(GH-14277)
Message-ID: <mailman.2.1568987740.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/062cfe3b11c61d03ccc2915e360f9b0d80e23642
commit: 062cfe3b11c61d03ccc2915e360f9b0d80e23642
branch: master
author: Prateek Nayak <45075669+Kriyszig at users.noreply.github.com>
committer: Zachary Ware <zachary.ware at gmail.com>
date: 2019-09-20T08:55:26-05:00
summary:

bpo-37353: Updated parser note about source code compatibility(GH-14277)

files:
M Doc/library/parser.rst

diff --git a/Doc/library/parser.rst b/Doc/library/parser.rst
index c55cd653b371..e4d6d1d0f5a3 100644
--- a/Doc/library/parser.rst
+++ b/Doc/library/parser.rst
@@ -51,7 +51,8 @@ Python version to another as source text will always allow correct parse trees
 to be created in the target version, with the only restriction being that
 migrating to an older version of the interpreter will not support more recent
 language constructs.  The parse trees are not typically compatible from one
-version to another, whereas source code has always been forward-compatible.
+version to another, though source code has usually been forward-compatible within
+a major release series.
 
 Each element of the sequences returned by :func:`st2list` or :func:`st2tuple`
 has a simple form.  Sequences representing non-terminal elements in the grammar


From webhook-mailer at python.org  Fri Sep 20 10:02:05 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 20 Sep 2019 14:02:05 -0000
Subject: [Python-checkins] bpo-37353: Updated parser note about source code
 compatibility(GH-14277)
Message-ID: <mailman.3.1568988126.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/2552e27b752f449b1d7046b384b14db0a69f46ad
commit: 2552e27b752f449b1d7046b384b14db0a69f46ad
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-20T07:02:00-07:00
summary:

bpo-37353: Updated parser note about source code compatibility(GH-14277)

(cherry picked from commit 062cfe3b11c61d03ccc2915e360f9b0d80e23642)

Co-authored-by: Prateek Nayak <45075669+Kriyszig at users.noreply.github.com>

files:
M Doc/library/parser.rst

diff --git a/Doc/library/parser.rst b/Doc/library/parser.rst
index a302681eca05..1d2da300709c 100644
--- a/Doc/library/parser.rst
+++ b/Doc/library/parser.rst
@@ -51,7 +51,8 @@ Python version to another as source text will always allow correct parse trees
 to be created in the target version, with the only restriction being that
 migrating to an older version of the interpreter will not support more recent
 language constructs.  The parse trees are not typically compatible from one
-version to another, whereas source code has always been forward-compatible.
+version to another, though source code has usually been forward-compatible within
+a major release series.
 
 Each element of the sequences returned by :func:`st2list` or :func:`st2tuple`
 has a simple form.  Sequences representing non-terminal elements in the grammar


From webhook-mailer at python.org  Fri Sep 20 10:03:25 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 20 Sep 2019 14:03:25 -0000
Subject: [Python-checkins] bpo-37353: Updated parser note about source code
 compatibility(GH-14277)
Message-ID: <mailman.4.1568988206.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/0ab6b01820afef44beea8158ba08b6469bf264a0
commit: 0ab6b01820afef44beea8158ba08b6469bf264a0
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-20T07:03:21-07:00
summary:

bpo-37353: Updated parser note about source code compatibility(GH-14277)

(cherry picked from commit 062cfe3b11c61d03ccc2915e360f9b0d80e23642)

Co-authored-by: Prateek Nayak <45075669+Kriyszig at users.noreply.github.com>

files:
M Doc/library/parser.rst

diff --git a/Doc/library/parser.rst b/Doc/library/parser.rst
index a302681eca05..1d2da300709c 100644
--- a/Doc/library/parser.rst
+++ b/Doc/library/parser.rst
@@ -51,7 +51,8 @@ Python version to another as source text will always allow correct parse trees
 to be created in the target version, with the only restriction being that
 migrating to an older version of the interpreter will not support more recent
 language constructs.  The parse trees are not typically compatible from one
-version to another, whereas source code has always been forward-compatible.
+version to another, though source code has usually been forward-compatible within
+a major release series.
 
 Each element of the sequences returned by :func:`st2list` or :func:`st2tuple`
 has a simple form.  Sequences representing non-terminal elements in the grammar


From webhook-mailer at python.org  Fri Sep 20 11:19:15 2019
From: webhook-mailer at python.org (Nick Coghlan)
Date: Fri, 20 Sep 2019 15:19:15 -0000
Subject: [Python-checkins] bpo-37937: Mention frame.f_trace in sys.settrace
 docs (GH-15439)
Message-ID: <mailman.5.1568992756.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/9c2682efc69568e1b42a0c1759489d6f2e3b30ea
commit: 9c2682efc69568e1b42a0c1759489d6f2e3b30ea
branch: master
author: Ram Rachum <ram at rachum.com>
committer: Nick Coghlan <ncoghlan at gmail.com>
date: 2019-09-21T01:19:02+10:00
summary:

bpo-37937: Mention frame.f_trace in sys.settrace docs (GH-15439)

Mention frame.f_trace in sys.settrace docs, as well as the fact you still
need to call `sys.settrace` to enable the tracing machinery before setting
`frame.f_trace` will have any effect.

files:
A Misc/NEWS.d/next/Documentation/2019-08-24-12-59-06.bpo-37937.F7fHbt.rst
M Doc/library/sys.rst

diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index ca43dc994cb7..c6c643b38de4 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -1301,6 +1301,17 @@ always available.
    Note that as an exception is propagated down the chain of callers, an
    ``'exception'`` event is generated at each level.
 
+   For more fine-grained usage, it's possible to set a trace function by
+   assigning ``frame.f_trace = tracefunc`` explicitly, rather than relying on
+   it being set indirectly via the return value from an already installed
+   trace function. This is also required for activating the trace function on
+   the current frame, which :func:`settrace` doesn't do. Note that in order
+   for this to work, a global tracing function must have been installed
+   with :func:`settrace` in order to enable the runtime tracing machinery,
+   but it doesn't need to be the same tracing function (e.g. it could be a
+   low overhead tracing function that simply returns ``None`` to disable
+   itself immediately on each frame).
+
    For more information on code and frame objects, refer to :ref:`types`.
 
    .. audit-event:: sys.settrace "" sys.settrace
diff --git a/Misc/NEWS.d/next/Documentation/2019-08-24-12-59-06.bpo-37937.F7fHbt.rst b/Misc/NEWS.d/next/Documentation/2019-08-24-12-59-06.bpo-37937.F7fHbt.rst
new file mode 100644
index 000000000000..26cc16a74d1c
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-08-24-12-59-06.bpo-37937.F7fHbt.rst
@@ -0,0 +1 @@
+Mention ``frame.f_trace`` in :func:`sys.settrace` docs.


From webhook-mailer at python.org  Fri Sep 20 11:25:16 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 20 Sep 2019 15:25:16 -0000
Subject: [Python-checkins] bpo-37937: Mention frame.f_trace in sys.settrace
 docs (GH-15439)
Message-ID: <mailman.6.1568993116.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/fd4f28d894ee55d7ee1625c726c1859fcd1e2249
commit: fd4f28d894ee55d7ee1625c726c1859fcd1e2249
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-20T08:25:12-07:00
summary:

bpo-37937: Mention frame.f_trace in sys.settrace docs (GH-15439)


Mention frame.f_trace in sys.settrace docs, as well as the fact you still
need to call `sys.settrace` to enable the tracing machinery before setting
`frame.f_trace` will have any effect.
(cherry picked from commit 9c2682efc69568e1b42a0c1759489d6f2e3b30ea)

Co-authored-by: Ram Rachum <ram at rachum.com>

files:
A Misc/NEWS.d/next/Documentation/2019-08-24-12-59-06.bpo-37937.F7fHbt.rst
M Doc/library/sys.rst

diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index ae946e830f6b..822e22e334bb 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -1233,6 +1233,17 @@ always available.
    Note that as an exception is propagated down the chain of callers, an
    ``'exception'`` event is generated at each level.
 
+   For more fine-grained usage, it's possible to set a trace function by
+   assigning ``frame.f_trace = tracefunc`` explicitly, rather than relying on
+   it being set indirectly via the return value from an already installed
+   trace function. This is also required for activating the trace function on
+   the current frame, which :func:`settrace` doesn't do. Note that in order
+   for this to work, a global tracing function must have been installed
+   with :func:`settrace` in order to enable the runtime tracing machinery,
+   but it doesn't need to be the same tracing function (e.g. it could be a
+   low overhead tracing function that simply returns ``None`` to disable
+   itself immediately on each frame).
+
    For more information on code and frame objects, refer to :ref:`types`.
 
    .. impl-detail::
diff --git a/Misc/NEWS.d/next/Documentation/2019-08-24-12-59-06.bpo-37937.F7fHbt.rst b/Misc/NEWS.d/next/Documentation/2019-08-24-12-59-06.bpo-37937.F7fHbt.rst
new file mode 100644
index 000000000000..26cc16a74d1c
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-08-24-12-59-06.bpo-37937.F7fHbt.rst
@@ -0,0 +1 @@
+Mention ``frame.f_trace`` in :func:`sys.settrace` docs.


From webhook-mailer at python.org  Fri Sep 20 11:26:31 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 20 Sep 2019 15:26:31 -0000
Subject: [Python-checkins] bpo-37937: Mention frame.f_trace in sys.settrace
 docs (GH-15439)
Message-ID: <mailman.7.1568993192.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/c410390e89071c4a4c2a437a227c14274add4116
commit: c410390e89071c4a4c2a437a227c14274add4116
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-20T08:26:27-07:00
summary:

bpo-37937: Mention frame.f_trace in sys.settrace docs (GH-15439)


Mention frame.f_trace in sys.settrace docs, as well as the fact you still
need to call `sys.settrace` to enable the tracing machinery before setting
`frame.f_trace` will have any effect.
(cherry picked from commit 9c2682efc69568e1b42a0c1759489d6f2e3b30ea)

Co-authored-by: Ram Rachum <ram at rachum.com>

files:
A Misc/NEWS.d/next/Documentation/2019-08-24-12-59-06.bpo-37937.F7fHbt.rst
M Doc/library/sys.rst

diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 975c14c88824..a698ecb64d85 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -1324,6 +1324,17 @@ always available.
    Note that as an exception is propagated down the chain of callers, an
    ``'exception'`` event is generated at each level.
 
+   For more fine-grained usage, it's possible to set a trace function by
+   assigning ``frame.f_trace = tracefunc`` explicitly, rather than relying on
+   it being set indirectly via the return value from an already installed
+   trace function. This is also required for activating the trace function on
+   the current frame, which :func:`settrace` doesn't do. Note that in order
+   for this to work, a global tracing function must have been installed
+   with :func:`settrace` in order to enable the runtime tracing machinery,
+   but it doesn't need to be the same tracing function (e.g. it could be a
+   low overhead tracing function that simply returns ``None`` to disable
+   itself immediately on each frame).
+
    For more information on code and frame objects, refer to :ref:`types`.
 
    .. audit-event:: sys.settrace "" sys.settrace
diff --git a/Misc/NEWS.d/next/Documentation/2019-08-24-12-59-06.bpo-37937.F7fHbt.rst b/Misc/NEWS.d/next/Documentation/2019-08-24-12-59-06.bpo-37937.F7fHbt.rst
new file mode 100644
index 000000000000..26cc16a74d1c
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-08-24-12-59-06.bpo-37937.F7fHbt.rst
@@ -0,0 +1 @@
+Mention ``frame.f_trace`` in :func:`sys.settrace` docs.


From webhook-mailer at python.org  Fri Sep 20 13:01:13 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 20 Sep 2019 17:01:13 -0000
Subject: [Python-checkins] bpo-6559: Update _posixsubprocess.fork_exec doc
 (GH-16283)
Message-ID: <mailman.8.1568998874.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/77abf23c67c1a465a8899666c69f6bcd6930e003
commit: 77abf23c67c1a465a8899666c69f6bcd6930e003
branch: master
author: Orivej Desh <orivej at gmx.fr>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-20T10:01:09-07:00
summary:

bpo-6559: Update _posixsubprocess.fork_exec doc (GH-16283)



It did not list the argument added in d4cc7bf993eda4149a05ed25f2f33e18e43fd7c1.


https://bugs.python.org/issue6559



Automerge-Triggered-By: @gpshead

files:
M Modules/_posixsubprocess.c

diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c
index 25af00f70dec..66db93e4500d 100644
--- a/Modules/_posixsubprocess.c
+++ b/Modules/_posixsubprocess.c
@@ -900,19 +900,22 @@ subprocess_fork_exec(PyObject* self, PyObject *args)
 
 
 PyDoc_STRVAR(subprocess_fork_exec_doc,
-"fork_exec(args, executable_list, close_fds, cwd, env,\n\
+"fork_exec(args, executable_list, close_fds, pass_fds, cwd, env,\n\
           p2cread, p2cwrite, c2pread, c2pwrite,\n\
           errread, errwrite, errpipe_read, errpipe_write,\n\
           restore_signals, call_setsid,\n\
-          call_setgid, gid, groups_size, gids,\n\
-          call_setuid, uid,\n\
+          gid, groups_list, uid,\n\
           preexec_fn)\n\
 \n\
 Forks a child process, closes parent file descriptors as appropriate in the\n\
 child and dups the few that are needed before calling exec() in the child\n\
 process.\n\
 \n\
-The preexec_fn, if supplied, will be called immediately before exec.\n\
+If close_fds is true, close file descriptors 3 and higher, except those listed\n\
+in the sorted tuple pass_fds.\n\
+\n\
+The preexec_fn, if supplied, will be called immediately before closing file\n\
+descriptors and exec.\n\
 WARNING: preexec_fn is NOT SAFE if your application uses threads.\n\
          It may trigger infrequent, difficult to debug deadlocks.\n\
 \n\


From webhook-mailer at python.org  Fri Sep 20 17:08:03 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Fri, 20 Sep 2019 21:08:03 -0000
Subject: [Python-checkins] bpo-33694: Remove test_asyncio
 ProactorDatagramTransportTests (GH-16288)
Message-ID: <mailman.9.1569013684.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/bc2256ea177a653bcab15b06b5f5725b10c1fff3
commit: bc2256ea177a653bcab15b06b5f5725b10c1fff3
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-20T23:07:59+02:00
summary:

bpo-33694: Remove test_asyncio ProactorDatagramTransportTests (GH-16288)

ProactorDatagramTransportTests tests are too close to the
implementation.

files:
M Lib/test/test_asyncio/test_proactor_events.py

diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py
index c838c5ce85ae..007039a7cdf5 100644
--- a/Lib/test/test_asyncio/test_proactor_events.py
+++ b/Lib/test/test_asyncio/test_proactor_events.py
@@ -472,259 +472,6 @@ def test_dont_pause_writing(self):
         self.assertFalse(self.protocol.pause_writing.called)
 
 
- at unittest.skip('FIXME: bpo-33694: these tests are too close '
-               'to the implementation and should be refactored or removed')
-class ProactorSocketTransportBufferedProtoTests(test_utils.TestCase):
-
-    def setUp(self):
-        super().setUp()
-        self.loop = self.new_test_loop()
-        self.addCleanup(self.loop.close)
-        self.proactor = mock.Mock()
-        self.loop._proactor = self.proactor
-
-        self.protocol = test_utils.make_test_protocol(asyncio.BufferedProtocol)
-        self.buf = bytearray(1)
-        self.protocol.get_buffer.side_effect = lambda hint: self.buf
-
-        self.sock = mock.Mock(socket.socket)
-
-    def socket_transport(self, waiter=None):
-        transport = _ProactorSocketTransport(self.loop, self.sock,
-                                             self.protocol, waiter=waiter)
-        self.addCleanup(close_transport, transport)
-        return transport
-
-    def test_ctor(self):
-        fut = self.loop.create_future()
-        tr = self.socket_transport(waiter=fut)
-        test_utils.run_briefly(self.loop)
-        self.assertIsNone(fut.result())
-        self.protocol.connection_made(tr)
-        self.proactor.recv_into.assert_called_with(self.sock, self.buf)
-
-    def test_loop_reading(self):
-        tr = self.socket_transport()
-        tr._loop_reading()
-        self.loop._proactor.recv_into.assert_called_with(self.sock, self.buf)
-        self.assertTrue(self.protocol.get_buffer.called)
-        self.assertFalse(self.protocol.buffer_updated.called)
-        self.assertFalse(self.protocol.eof_received.called)
-
-    def test_get_buffer_error(self):
-        transport = self.socket_transport()
-        transport._fatal_error = mock.Mock()
-
-        self.loop.call_exception_handler = mock.Mock()
-        self.protocol.get_buffer.side_effect = LookupError()
-
-        transport._loop_reading()
-
-        self.assertTrue(transport._fatal_error.called)
-        self.assertTrue(self.protocol.get_buffer.called)
-        self.assertFalse(self.protocol.buffer_updated.called)
-
-    def test_get_buffer_zerosized(self):
-        transport = self.socket_transport()
-        transport._fatal_error = mock.Mock()
-
-        self.loop.call_exception_handler = mock.Mock()
-        self.protocol.get_buffer.side_effect = lambda hint: bytearray(0)
-
-        transport._loop_reading()
-
-        self.assertTrue(transport._fatal_error.called)
-        self.assertTrue(self.protocol.get_buffer.called)
-        self.assertFalse(self.protocol.buffer_updated.called)
-
-    def test_proto_type_switch(self):
-        self.protocol = test_utils.make_test_protocol(asyncio.Protocol)
-        tr = self.socket_transport()
-
-        res = self.loop.create_future()
-        res.set_result(b'data')
-
-        tr = self.socket_transport()
-        tr._read_fut = res
-        tr._loop_reading(res)
-        self.loop._proactor.recv.assert_called_with(self.sock, 32768)
-        self.protocol.data_received.assert_called_with(b'data')
-
-        # switch protocol to a BufferedProtocol
-
-        buf_proto = test_utils.make_test_protocol(asyncio.BufferedProtocol)
-        buf = bytearray(4)
-        buf_proto.get_buffer.side_effect = lambda hint: buf
-
-        tr.set_protocol(buf_proto)
-        test_utils.run_briefly(self.loop)
-        res = self.loop.create_future()
-        res.set_result(4)
-
-        tr._read_fut = res
-        tr._loop_reading(res)
-        self.loop._proactor.recv_into.assert_called_with(self.sock, buf)
-        buf_proto.buffer_updated.assert_called_with(4)
-
-    @unittest.skip('FIXME: bpo-33694: this test is too close to the '
-                   'implementation and should be refactored or removed')
-    def test_proto_buf_switch(self):
-        tr = self.socket_transport()
-        test_utils.run_briefly(self.loop)
-        self.protocol.get_buffer.assert_called_with(-1)
-
-        # switch protocol to *another* BufferedProtocol
-
-        buf_proto = test_utils.make_test_protocol(asyncio.BufferedProtocol)
-        buf = bytearray(4)
-        buf_proto.get_buffer.side_effect = lambda hint: buf
-        tr._read_fut.done.side_effect = lambda: False
-        tr.set_protocol(buf_proto)
-        self.assertFalse(buf_proto.get_buffer.called)
-        test_utils.run_briefly(self.loop)
-        buf_proto.get_buffer.assert_called_with(-1)
-
-    def test_buffer_updated_error(self):
-        transport = self.socket_transport()
-        transport._fatal_error = mock.Mock()
-
-        self.loop.call_exception_handler = mock.Mock()
-        self.protocol.buffer_updated.side_effect = LookupError()
-
-        res = self.loop.create_future()
-        res.set_result(10)
-        transport._read_fut = res
-        transport._loop_reading(res)
-
-        self.assertTrue(transport._fatal_error.called)
-        self.assertFalse(self.protocol.get_buffer.called)
-        self.assertTrue(self.protocol.buffer_updated.called)
-
-    def test_loop_eof_received_error(self):
-        res = self.loop.create_future()
-        res.set_result(0)
-
-        self.protocol.eof_received.side_effect = LookupError()
-
-        tr = self.socket_transport()
-        tr._fatal_error = mock.Mock()
-
-        tr.close = mock.Mock()
-        tr._read_fut = res
-        tr._loop_reading(res)
-        self.assertFalse(self.loop._proactor.recv_into.called)
-        self.assertTrue(self.protocol.eof_received.called)
-        self.assertTrue(tr._fatal_error.called)
-
-    def test_loop_reading_data(self):
-        res = self.loop.create_future()
-        res.set_result(4)
-
-        tr = self.socket_transport()
-        tr._read_fut = res
-        tr._loop_reading(res)
-        self.loop._proactor.recv_into.assert_called_with(self.sock, self.buf)
-        self.protocol.buffer_updated.assert_called_with(4)
-
-    def test_loop_reading_no_data(self):
-        res = self.loop.create_future()
-        res.set_result(0)
-
-        tr = self.socket_transport()
-        self.assertRaises(AssertionError, tr._loop_reading, res)
-
-        tr.close = mock.Mock()
-        tr._read_fut = res
-        tr._loop_reading(res)
-        self.assertFalse(self.loop._proactor.recv_into.called)
-        self.assertTrue(self.protocol.eof_received.called)
-        self.assertTrue(tr.close.called)
-
-    def test_loop_reading_aborted(self):
-        err = self.loop._proactor.recv_into.side_effect = \
-            ConnectionAbortedError()
-
-        tr = self.socket_transport()
-        tr._fatal_error = mock.Mock()
-        tr._loop_reading()
-        tr._fatal_error.assert_called_with(
-            err, 'Fatal read error on pipe transport')
-
-    def test_loop_reading_aborted_closing(self):
-        self.loop._proactor.recv.side_effect = ConnectionAbortedError()
-
-        tr = self.socket_transport()
-        tr._closing = True
-        tr._fatal_error = mock.Mock()
-        tr._loop_reading()
-        self.assertFalse(tr._fatal_error.called)
-
-    def test_loop_reading_aborted_is_fatal(self):
-        self.loop._proactor.recv_into.side_effect = ConnectionAbortedError()
-        tr = self.socket_transport()
-        tr._closing = False
-        tr._fatal_error = mock.Mock()
-        tr._loop_reading()
-        self.assertTrue(tr._fatal_error.called)
-
-    def test_loop_reading_conn_reset_lost(self):
-        err = self.loop._proactor.recv_into.side_effect = ConnectionResetError()
-
-        tr = self.socket_transport()
-        tr._closing = False
-        tr._fatal_error = mock.Mock()
-        tr._force_close = mock.Mock()
-        tr._loop_reading()
-        self.assertFalse(tr._fatal_error.called)
-        tr._force_close.assert_called_with(err)
-
-    def test_loop_reading_exception(self):
-        err = self.loop._proactor.recv_into.side_effect = OSError()
-
-        tr = self.socket_transport()
-        tr._fatal_error = mock.Mock()
-        tr._loop_reading()
-        tr._fatal_error.assert_called_with(
-            err, 'Fatal read error on pipe transport')
-
-    def test_pause_resume_reading(self):
-        tr = self.socket_transport()
-        futures = []
-        for msg in [10, 20, 30, 40, 0]:
-            f = self.loop.create_future()
-            f.set_result(msg)
-            futures.append(f)
-
-        self.loop._proactor.recv_into.side_effect = futures
-        self.loop._run_once()
-        self.assertFalse(tr._paused)
-        self.assertTrue(tr.is_reading())
-        self.loop._run_once()
-        self.protocol.buffer_updated.assert_called_with(10)
-        self.loop._run_once()
-        self.protocol.buffer_updated.assert_called_with(20)
-
-        tr.pause_reading()
-        tr.pause_reading()
-        self.assertTrue(tr._paused)
-        self.assertFalse(tr.is_reading())
-        for i in range(10):
-            self.loop._run_once()
-        self.protocol.buffer_updated.assert_called_with(20)
-
-        tr.resume_reading()
-        tr.resume_reading()
-        self.assertFalse(tr._paused)
-        self.assertTrue(tr.is_reading())
-        self.loop._run_once()
-        self.protocol.buffer_updated.assert_called_with(30)
-        self.loop._run_once()
-        self.protocol.buffer_updated.assert_called_with(40)
-        tr.close()
-
-        self.assertFalse(tr.is_reading())
-
-
 class ProactorDatagramTransportTests(test_utils.TestCase):
 
     def setUp(self):


From webhook-mailer at python.org  Fri Sep 20 17:10:20 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Fri, 20 Sep 2019 21:10:20 -0000
Subject: [Python-checkins] bpo-38205: Py_UNREACHABLE() calls Py_FatalError()
 (GH-16290)
Message-ID: <mailman.10.1569013820.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/b1542583bee204130934c2b90684041e29378250
commit: b1542583bee204130934c2b90684041e29378250
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-20T23:10:16+02:00
summary:

bpo-38205: Py_UNREACHABLE() calls Py_FatalError() (GH-16290)

files:
A Misc/NEWS.d/next/C API/2019-09-19-18-26-29.bpo-38205.Db1OJL.rst
M Include/pymacro.h

diff --git a/Include/pymacro.h b/Include/pymacro.h
index ae09063e10d8..c080fb164e35 100644
--- a/Include/pymacro.h
+++ b/Include/pymacro.h
@@ -101,31 +101,26 @@
 #endif
 
 #if defined(RANDALL_WAS_HERE)
-#define Py_UNREACHABLE() do { \
-        fputs( \
-            "ERROR:\n\n" \
-            "If you're seeing this, the code is in what I thought was\n" \
-            "an unreachable state.\n\n" \
-            "I could give you advice for what to do, but honestly, why\n" \
-            "should you trust me?  I clearly screwed this up.  I'm writing\n" \
-            "a message that should never appear, yet I know it will\n" \
-            "probably appear someday.\n\n" \
-            "On a deep level, I know I'm not up to this task.\n" \
-            "I'm so sorry.\n\n" \
-            "https://xkcd.com/2200\n", stderr); \
-        abort(); \
-    } while(0)
+#define Py_UNREACHABLE() \
+    Py_FatalError( \
+        "If you're seeing this, the code is in what I thought was\n" \
+        "an unreachable state.\n\n" \
+        "I could give you advice for what to do, but honestly, why\n" \
+        "should you trust me?  I clearly screwed this up.  I'm writing\n" \
+        "a message that should never appear, yet I know it will\n" \
+        "probably appear someday.\n\n" \
+        "On a deep level, I know I'm not up to this task.\n" \
+        "I'm so sorry.\n" \
+        "https://xkcd.com/2200")
 #elif defined(Py_DEBUG)
-#define Py_UNREACHABLE() do { \
-        fputs( \
-            "ERROR:\n\n" \
-            "We've reached an unreachable state. Anything is possible.\n" \
-            "The limits were in our heads all along. Follow your dreams.\n\n" \
-            "https://xkcd.com/2200\n", stderr); \
-        abort(); \
-    } while(0)
+#define Py_UNREACHABLE() \
+    Py_FatalError( \
+        "We've reached an unreachable state. Anything is possible.\n" \
+        "The limits were in our heads all along. Follow your dreams.\n" \
+        "https://xkcd.com/2200")
 #else
-#define Py_UNREACHABLE() abort()
+#define Py_UNREACHABLE() \
+    Py_FatalError("Unreachable C code path reached")
 #endif
 
 #endif /* Py_PYMACRO_H */
diff --git a/Misc/NEWS.d/next/C API/2019-09-19-18-26-29.bpo-38205.Db1OJL.rst b/Misc/NEWS.d/next/C API/2019-09-19-18-26-29.bpo-38205.Db1OJL.rst
new file mode 100644
index 000000000000..1818e54d461a
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-09-19-18-26-29.bpo-38205.Db1OJL.rst	
@@ -0,0 +1 @@
+The :c:func:`Py_UNREACHABLE` macro now calls :c:func:`Py_FatalError`.


From webhook-mailer at python.org  Fri Sep 20 17:27:32 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 20 Sep 2019 21:27:32 -0000
Subject: [Python-checkins] bpo-33694: Remove test_asyncio
 ProactorDatagramTransportTests (GH-16288)
Message-ID: <mailman.11.1569014890.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/b50edac42fd39d58ba51eb9cee2d3645a2e28ca4
commit: b50edac42fd39d58ba51eb9cee2d3645a2e28ca4
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-20T14:27:28-07:00
summary:

bpo-33694: Remove test_asyncio ProactorDatagramTransportTests (GH-16288)


ProactorDatagramTransportTests tests are too close to the
implementation.
(cherry picked from commit bc2256ea177a653bcab15b06b5f5725b10c1fff3)

Co-authored-by: Victor Stinner <vstinner at redhat.com>

files:
M Lib/test/test_asyncio/test_proactor_events.py

diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py
index c838c5ce85ae..007039a7cdf5 100644
--- a/Lib/test/test_asyncio/test_proactor_events.py
+++ b/Lib/test/test_asyncio/test_proactor_events.py
@@ -472,259 +472,6 @@ def test_dont_pause_writing(self):
         self.assertFalse(self.protocol.pause_writing.called)
 
 
- at unittest.skip('FIXME: bpo-33694: these tests are too close '
-               'to the implementation and should be refactored or removed')
-class ProactorSocketTransportBufferedProtoTests(test_utils.TestCase):
-
-    def setUp(self):
-        super().setUp()
-        self.loop = self.new_test_loop()
-        self.addCleanup(self.loop.close)
-        self.proactor = mock.Mock()
-        self.loop._proactor = self.proactor
-
-        self.protocol = test_utils.make_test_protocol(asyncio.BufferedProtocol)
-        self.buf = bytearray(1)
-        self.protocol.get_buffer.side_effect = lambda hint: self.buf
-
-        self.sock = mock.Mock(socket.socket)
-
-    def socket_transport(self, waiter=None):
-        transport = _ProactorSocketTransport(self.loop, self.sock,
-                                             self.protocol, waiter=waiter)
-        self.addCleanup(close_transport, transport)
-        return transport
-
-    def test_ctor(self):
-        fut = self.loop.create_future()
-        tr = self.socket_transport(waiter=fut)
-        test_utils.run_briefly(self.loop)
-        self.assertIsNone(fut.result())
-        self.protocol.connection_made(tr)
-        self.proactor.recv_into.assert_called_with(self.sock, self.buf)
-
-    def test_loop_reading(self):
-        tr = self.socket_transport()
-        tr._loop_reading()
-        self.loop._proactor.recv_into.assert_called_with(self.sock, self.buf)
-        self.assertTrue(self.protocol.get_buffer.called)
-        self.assertFalse(self.protocol.buffer_updated.called)
-        self.assertFalse(self.protocol.eof_received.called)
-
-    def test_get_buffer_error(self):
-        transport = self.socket_transport()
-        transport._fatal_error = mock.Mock()
-
-        self.loop.call_exception_handler = mock.Mock()
-        self.protocol.get_buffer.side_effect = LookupError()
-
-        transport._loop_reading()
-
-        self.assertTrue(transport._fatal_error.called)
-        self.assertTrue(self.protocol.get_buffer.called)
-        self.assertFalse(self.protocol.buffer_updated.called)
-
-    def test_get_buffer_zerosized(self):
-        transport = self.socket_transport()
-        transport._fatal_error = mock.Mock()
-
-        self.loop.call_exception_handler = mock.Mock()
-        self.protocol.get_buffer.side_effect = lambda hint: bytearray(0)
-
-        transport._loop_reading()
-
-        self.assertTrue(transport._fatal_error.called)
-        self.assertTrue(self.protocol.get_buffer.called)
-        self.assertFalse(self.protocol.buffer_updated.called)
-
-    def test_proto_type_switch(self):
-        self.protocol = test_utils.make_test_protocol(asyncio.Protocol)
-        tr = self.socket_transport()
-
-        res = self.loop.create_future()
-        res.set_result(b'data')
-
-        tr = self.socket_transport()
-        tr._read_fut = res
-        tr._loop_reading(res)
-        self.loop._proactor.recv.assert_called_with(self.sock, 32768)
-        self.protocol.data_received.assert_called_with(b'data')
-
-        # switch protocol to a BufferedProtocol
-
-        buf_proto = test_utils.make_test_protocol(asyncio.BufferedProtocol)
-        buf = bytearray(4)
-        buf_proto.get_buffer.side_effect = lambda hint: buf
-
-        tr.set_protocol(buf_proto)
-        test_utils.run_briefly(self.loop)
-        res = self.loop.create_future()
-        res.set_result(4)
-
-        tr._read_fut = res
-        tr._loop_reading(res)
-        self.loop._proactor.recv_into.assert_called_with(self.sock, buf)
-        buf_proto.buffer_updated.assert_called_with(4)
-
-    @unittest.skip('FIXME: bpo-33694: this test is too close to the '
-                   'implementation and should be refactored or removed')
-    def test_proto_buf_switch(self):
-        tr = self.socket_transport()
-        test_utils.run_briefly(self.loop)
-        self.protocol.get_buffer.assert_called_with(-1)
-
-        # switch protocol to *another* BufferedProtocol
-
-        buf_proto = test_utils.make_test_protocol(asyncio.BufferedProtocol)
-        buf = bytearray(4)
-        buf_proto.get_buffer.side_effect = lambda hint: buf
-        tr._read_fut.done.side_effect = lambda: False
-        tr.set_protocol(buf_proto)
-        self.assertFalse(buf_proto.get_buffer.called)
-        test_utils.run_briefly(self.loop)
-        buf_proto.get_buffer.assert_called_with(-1)
-
-    def test_buffer_updated_error(self):
-        transport = self.socket_transport()
-        transport._fatal_error = mock.Mock()
-
-        self.loop.call_exception_handler = mock.Mock()
-        self.protocol.buffer_updated.side_effect = LookupError()
-
-        res = self.loop.create_future()
-        res.set_result(10)
-        transport._read_fut = res
-        transport._loop_reading(res)
-
-        self.assertTrue(transport._fatal_error.called)
-        self.assertFalse(self.protocol.get_buffer.called)
-        self.assertTrue(self.protocol.buffer_updated.called)
-
-    def test_loop_eof_received_error(self):
-        res = self.loop.create_future()
-        res.set_result(0)
-
-        self.protocol.eof_received.side_effect = LookupError()
-
-        tr = self.socket_transport()
-        tr._fatal_error = mock.Mock()
-
-        tr.close = mock.Mock()
-        tr._read_fut = res
-        tr._loop_reading(res)
-        self.assertFalse(self.loop._proactor.recv_into.called)
-        self.assertTrue(self.protocol.eof_received.called)
-        self.assertTrue(tr._fatal_error.called)
-
-    def test_loop_reading_data(self):
-        res = self.loop.create_future()
-        res.set_result(4)
-
-        tr = self.socket_transport()
-        tr._read_fut = res
-        tr._loop_reading(res)
-        self.loop._proactor.recv_into.assert_called_with(self.sock, self.buf)
-        self.protocol.buffer_updated.assert_called_with(4)
-
-    def test_loop_reading_no_data(self):
-        res = self.loop.create_future()
-        res.set_result(0)
-
-        tr = self.socket_transport()
-        self.assertRaises(AssertionError, tr._loop_reading, res)
-
-        tr.close = mock.Mock()
-        tr._read_fut = res
-        tr._loop_reading(res)
-        self.assertFalse(self.loop._proactor.recv_into.called)
-        self.assertTrue(self.protocol.eof_received.called)
-        self.assertTrue(tr.close.called)
-
-    def test_loop_reading_aborted(self):
-        err = self.loop._proactor.recv_into.side_effect = \
-            ConnectionAbortedError()
-
-        tr = self.socket_transport()
-        tr._fatal_error = mock.Mock()
-        tr._loop_reading()
-        tr._fatal_error.assert_called_with(
-            err, 'Fatal read error on pipe transport')
-
-    def test_loop_reading_aborted_closing(self):
-        self.loop._proactor.recv.side_effect = ConnectionAbortedError()
-
-        tr = self.socket_transport()
-        tr._closing = True
-        tr._fatal_error = mock.Mock()
-        tr._loop_reading()
-        self.assertFalse(tr._fatal_error.called)
-
-    def test_loop_reading_aborted_is_fatal(self):
-        self.loop._proactor.recv_into.side_effect = ConnectionAbortedError()
-        tr = self.socket_transport()
-        tr._closing = False
-        tr._fatal_error = mock.Mock()
-        tr._loop_reading()
-        self.assertTrue(tr._fatal_error.called)
-
-    def test_loop_reading_conn_reset_lost(self):
-        err = self.loop._proactor.recv_into.side_effect = ConnectionResetError()
-
-        tr = self.socket_transport()
-        tr._closing = False
-        tr._fatal_error = mock.Mock()
-        tr._force_close = mock.Mock()
-        tr._loop_reading()
-        self.assertFalse(tr._fatal_error.called)
-        tr._force_close.assert_called_with(err)
-
-    def test_loop_reading_exception(self):
-        err = self.loop._proactor.recv_into.side_effect = OSError()
-
-        tr = self.socket_transport()
-        tr._fatal_error = mock.Mock()
-        tr._loop_reading()
-        tr._fatal_error.assert_called_with(
-            err, 'Fatal read error on pipe transport')
-
-    def test_pause_resume_reading(self):
-        tr = self.socket_transport()
-        futures = []
-        for msg in [10, 20, 30, 40, 0]:
-            f = self.loop.create_future()
-            f.set_result(msg)
-            futures.append(f)
-
-        self.loop._proactor.recv_into.side_effect = futures
-        self.loop._run_once()
-        self.assertFalse(tr._paused)
-        self.assertTrue(tr.is_reading())
-        self.loop._run_once()
-        self.protocol.buffer_updated.assert_called_with(10)
-        self.loop._run_once()
-        self.protocol.buffer_updated.assert_called_with(20)
-
-        tr.pause_reading()
-        tr.pause_reading()
-        self.assertTrue(tr._paused)
-        self.assertFalse(tr.is_reading())
-        for i in range(10):
-            self.loop._run_once()
-        self.protocol.buffer_updated.assert_called_with(20)
-
-        tr.resume_reading()
-        tr.resume_reading()
-        self.assertFalse(tr._paused)
-        self.assertTrue(tr.is_reading())
-        self.loop._run_once()
-        self.protocol.buffer_updated.assert_called_with(30)
-        self.loop._run_once()
-        self.protocol.buffer_updated.assert_called_with(40)
-        tr.close()
-
-        self.assertFalse(tr.is_reading())
-
-
 class ProactorDatagramTransportTests(test_utils.TestCase):
 
     def setUp(self):


From webhook-mailer at python.org  Fri Sep 20 17:36:36 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Fri, 20 Sep 2019 21:36:36 -0000
Subject: [Python-checkins] bpo-38205: Py_UNREACHABLE() calls Py_FatalError()
 (GH-16290) (GH-16306)
Message-ID: <mailman.12.1569015397.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/245d439320e0921459502883fe809213e3b93e35
commit: 245d439320e0921459502883fe809213e3b93e35
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-20T23:36:32+02:00
summary:

bpo-38205: Py_UNREACHABLE() calls Py_FatalError() (GH-16290) (GH-16306)

(cherry picked from commit b1542583bee204130934c2b90684041e29378250)

files:
A Misc/NEWS.d/next/C API/2019-09-19-18-26-29.bpo-38205.Db1OJL.rst
M Include/pymacro.h

diff --git a/Include/pymacro.h b/Include/pymacro.h
index 1890619099a3..495c2c25c908 100644
--- a/Include/pymacro.h
+++ b/Include/pymacro.h
@@ -100,6 +100,7 @@
 #  define Py_UNUSED(name) _unused_ ## name
 #endif
 
-#define Py_UNREACHABLE() abort()
+#define Py_UNREACHABLE() \
+    Py_FatalError("Unreachable C code path reached")
 
 #endif /* Py_PYMACRO_H */
diff --git a/Misc/NEWS.d/next/C API/2019-09-19-18-26-29.bpo-38205.Db1OJL.rst b/Misc/NEWS.d/next/C API/2019-09-19-18-26-29.bpo-38205.Db1OJL.rst
new file mode 100644
index 000000000000..1818e54d461a
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-09-19-18-26-29.bpo-38205.Db1OJL.rst	
@@ -0,0 +1 @@
+The :c:func:`Py_UNREACHABLE` macro now calls :c:func:`Py_FatalError`.


From webhook-mailer at python.org  Fri Sep 20 19:03:00 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Fri, 20 Sep 2019 23:03:00 -0000
Subject: [Python-checkins] bpo-38234: Remove _PyPathConfig.dll_path
 (GH-16307)
Message-ID: <mailman.13.1569020581.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/c422167749f92d4170203e996a2c619c818335ea
commit: c422167749f92d4170203e996a2c619c818335ea
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-21T01:02:56+02:00
summary:

bpo-38234: Remove _PyPathConfig.dll_path (GH-16307)

The DLL path is not computed from any user configuration and cannot
be configured by PyConfig. Instead, add a new _Py_dll_path global variable.

Remove _PyConfig_SetPathConfig(): replaced with _PyPathConfig_Init().

Py_Initialize() now longer sets the "global path configuration",
but only initialize _Py_dll_path.

files:
M Include/internal/pycore_initconfig.h
M Include/internal/pycore_pathconfig.h
M PC/getpathp.c
M Python/pathconfig.c
M Python/pylifecycle.c

diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h
index c0b3d957827a..a7c1994eca16 100644
--- a/Include/internal/pycore_initconfig.h
+++ b/Include/internal/pycore_initconfig.h
@@ -149,8 +149,6 @@ extern PyStatus _PyConfig_Copy(
     PyConfig *config,
     const PyConfig *config2);
 extern PyStatus _PyConfig_InitPathConfig(PyConfig *config);
-extern PyStatus _PyConfig_SetPathConfig(
-    const PyConfig *config);
 extern void _PyConfig_Write(const PyConfig *config,
     _PyRuntimeState *runtime);
 extern PyStatus _PyConfig_SetPyArgv(
diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h
index 9e0ba0b01a80..a2c488cdc2eb 100644
--- a/Include/internal/pycore_pathconfig.h
+++ b/Include/internal/pycore_pathconfig.h
@@ -13,10 +13,7 @@ typedef struct _PyPathConfig {
     wchar_t *program_full_path;
     wchar_t *prefix;
     wchar_t *exec_prefix;
-#ifdef MS_WINDOWS
-    wchar_t *dll_path;
-#endif
-    /* Set by Py_SetPath(), or computed by _PyPathConfig_Init() */
+    /* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */
     wchar_t *module_search_path;
     /* Python program name */
     wchar_t *program_name;
@@ -38,6 +35,9 @@ typedef struct _PyPathConfig {
 /* Note: _PyPathConfig_INIT sets other fields to 0/NULL */
 
 PyAPI_DATA(_PyPathConfig) _Py_path_config;
+#ifdef MS_WINDOWS
+PyAPI_DATA(wchar_t*) _Py_dll_path;
+#endif
 
 extern void _PyPathConfig_ClearGlobal(void);
 extern PyStatus _PyPathConfig_SetGlobal(
@@ -59,6 +59,8 @@ extern int _Py_FindEnvConfigValue(
 extern wchar_t* _Py_GetDLLPath(void);
 #endif
 
+extern PyStatus _PyPathConfig_Init(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/PC/getpathp.c b/PC/getpathp.c
index 7465d314b043..01455a660be6 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -128,6 +128,8 @@ typedef struct {
 
     wchar_t argv0_path[MAXPATHLEN+1];
     wchar_t zip_path[MAXPATHLEN+1];
+
+    wchar_t *dll_path;
 } PyCalculatePath;
 
 
@@ -666,28 +668,36 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
 }
 
 
-static void
+static PyStatus
 calculate_init(PyCalculatePath *calculate,
                const PyConfig *config)
 {
     calculate->home = config->home;
     calculate->path_env = _wgetenv(L"PATH");
+
+    calculate->dll_path = _Py_GetDLLPath();
+    if (calculate->dll_path == NULL) {
+        return _PyStatus_NO_MEMORY();
+    }
+
+    return _PyStatus_OK();
 }
 
 
 static int
-get_pth_filename(wchar_t *spbuffer, _PyPathConfig *pathconfig)
+get_pth_filename(PyCalculatePath *calculate, wchar_t *filename,
+                 const _PyPathConfig *pathconfig)
 {
-    if (pathconfig->dll_path[0]) {
-        if (!change_ext(spbuffer, pathconfig->dll_path, L"._pth") &&
-            exists(spbuffer))
+    if (calculate->dll_path[0]) {
+        if (!change_ext(filename, calculate->dll_path, L"._pth") &&
+            exists(filename))
         {
             return 1;
         }
     }
     if (pathconfig->program_full_path[0]) {
-        if (!change_ext(spbuffer, pathconfig->program_full_path, L"._pth") &&
-            exists(spbuffer))
+        if (!change_ext(filename, pathconfig->program_full_path, L"._pth") &&
+            exists(filename))
         {
             return 1;
         }
@@ -697,15 +707,16 @@ get_pth_filename(wchar_t *spbuffer, _PyPathConfig *pathconfig)
 
 
 static int
-calculate_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix)
+calculate_pth_file(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                   wchar_t *prefix)
 {
-    wchar_t spbuffer[MAXPATHLEN+1];
+    wchar_t filename[MAXPATHLEN+1];
 
-    if (!get_pth_filename(spbuffer, pathconfig)) {
+    if (!get_pth_filename(calculate, filename, pathconfig)) {
         return 0;
     }
 
-    return read_pth_file(pathconfig, prefix, spbuffer);
+    return read_pth_file(pathconfig, prefix, filename);
 }
 
 
@@ -966,13 +977,6 @@ calculate_path_impl(const PyConfig *config,
 {
     PyStatus status;
 
-    assert(pathconfig->dll_path == NULL);
-
-    pathconfig->dll_path = _Py_GetDLLPath();
-    if (pathconfig->dll_path == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
-
     status = get_program_full_path(config, calculate, pathconfig);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
@@ -986,7 +990,7 @@ calculate_path_impl(const PyConfig *config,
     memset(prefix, 0, sizeof(prefix));
 
     /* Search for a sys.path file */
-    if (calculate_pth_file(pathconfig, prefix)) {
+    if (calculate_pth_file(calculate, pathconfig, prefix)) {
         goto done;
     }
 
@@ -994,7 +998,7 @@ calculate_path_impl(const PyConfig *config,
 
     /* Calculate zip archive path from DLL or exe path */
     change_ext(calculate->zip_path,
-               pathconfig->dll_path[0] ? pathconfig->dll_path : pathconfig->program_full_path,
+               calculate->dll_path[0] ? calculate->dll_path : pathconfig->program_full_path,
                L".zip");
 
     calculate_home_prefix(calculate, prefix);
@@ -1023,23 +1027,23 @@ calculate_free(PyCalculatePath *calculate)
 {
     PyMem_RawFree(calculate->machine_path);
     PyMem_RawFree(calculate->user_path);
+    PyMem_RawFree(calculate->dll_path);
 }
 
 
 PyStatus
 _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
 {
+    PyStatus status;
     PyCalculatePath calculate;
     memset(&calculate, 0, sizeof(calculate));
 
-    calculate_init(&calculate, config);
-
-    PyStatus status = calculate_path_impl(config, &calculate, pathconfig);
+    status = calculate_init(&calculate, config);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
     }
 
-    status = _PyStatus_OK();
+    status = calculate_path_impl(config, &calculate, pathconfig);
 
 done:
     calculate_free(&calculate);
@@ -1067,7 +1071,12 @@ _Py_CheckPython3(void)
 
     /* If there is a python3.dll next to the python3y.dll,
        assume this is a build tree; use that DLL */
-    wcscpy(py3path, _Py_path_config.dll_path);
+    if (_Py_dll_path != NULL) {
+        wcscpy(py3path, _Py_dll_path);
+    }
+    else {
+        wcscpy(py3path, L"");
+    }
     s = wcsrchr(py3path, L'\\');
     if (!s) {
         s = py3path;
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index ee220611357e..ccab83274251 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -15,6 +15,9 @@ extern "C" {
 
 
 _PyPathConfig _Py_path_config = _PyPathConfig_INIT;
+#ifdef MS_WINDOWS
+wchar_t *_Py_dll_path = NULL;
+#endif
 
 
 static int
@@ -51,9 +54,6 @@ pathconfig_clear(_PyPathConfig *config)
     CLEAR(config->prefix);
     CLEAR(config->program_full_path);
     CLEAR(config->exec_prefix);
-#ifdef MS_WINDOWS
-    CLEAR(config->dll_path);
-#endif
     CLEAR(config->module_search_path);
     CLEAR(config->home);
     CLEAR(config->program_name);
@@ -114,47 +114,6 @@ pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config)
 }
 
 
-PyStatus
-_PyPathConfig_SetGlobal(const _PyPathConfig *config)
-{
-    PyStatus status;
-    _PyPathConfig new_config = _PyPathConfig_INIT;
-
-    PyMemAllocatorEx old_alloc;
-    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-
-#define COPY_ATTR(ATTR) \
-    do { \
-        if (copy_wstr(&new_config.ATTR, config->ATTR) < 0) { \
-            pathconfig_clear(&new_config); \
-            status = _PyStatus_NO_MEMORY(); \
-            goto done; \
-        } \
-    } while (0)
-
-    COPY_ATTR(program_full_path);
-    COPY_ATTR(prefix);
-    COPY_ATTR(exec_prefix);
-#ifdef MS_WINDOWS
-    COPY_ATTR(dll_path);
-#endif
-    COPY_ATTR(module_search_path);
-    COPY_ATTR(program_name);
-    COPY_ATTR(home);
-    COPY_ATTR(base_executable);
-
-    pathconfig_clear(&_Py_path_config);
-    /* Steal new_config strings; don't clear new_config */
-    _Py_path_config = new_config;
-
-    status = _PyStatus_OK();
-
-done:
-    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-    return status;
-}
-
-
 void
 _PyPathConfig_ClearGlobal(void)
 {
@@ -162,6 +121,10 @@ _PyPathConfig_ClearGlobal(void)
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
     pathconfig_clear(&_Py_path_config);
+#ifdef MS_WINDOWS
+    PyMem_RawFree(_Py_dll_path);
+    _Py_dll_path = NULL;
+#endif
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 }
@@ -200,12 +163,36 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep)
 
 /* Set the global path configuration from config. */
 PyStatus
-_PyConfig_SetPathConfig(const PyConfig *config)
+_PyPathConfig_Init(void)
 {
+#ifdef MS_WINDOWS
+    if (_Py_dll_path == NULL) {
+        /* Already set: nothing to do */
+        return _PyStatus_OK();
+    }
+
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
+    _Py_dll_path = _Py_GetDLLPath();
+
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    if (_Py_dll_path == NULL) {
+        return _PyStatus_NO_MEMORY();
+    }
+#endif
+    return _PyStatus_OK();
+}
+
+
+static PyStatus
+pathconfig_global_init_from_config(const PyConfig *config)
+{
     PyStatus status;
+    PyMemAllocatorEx old_alloc;
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
     _PyPathConfig pathconfig = _PyPathConfig_INIT;
 
     pathconfig.module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM);
@@ -222,12 +209,6 @@ _PyConfig_SetPathConfig(const PyConfig *config)
     if (copy_wstr(&pathconfig.exec_prefix, config->exec_prefix) < 0) {
         goto no_memory;
     }
-#ifdef MS_WINDOWS
-    pathconfig.dll_path = _Py_GetDLLPath();
-    if (pathconfig.dll_path == NULL) {
-        goto no_memory;
-    }
-#endif
     if (copy_wstr(&pathconfig.program_name, config->program_name) < 0) {
         goto no_memory;
     }
@@ -238,19 +219,18 @@ _PyConfig_SetPathConfig(const PyConfig *config)
         goto no_memory;
     }
 
-    status = _PyPathConfig_SetGlobal(&pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto done;
-    }
+    pathconfig_clear(&_Py_path_config);
+    /* Steal new_config strings; don't clear new_config */
+    _Py_path_config = pathconfig;
 
     status = _PyStatus_OK();
     goto done;
 
 no_memory:
+    pathconfig_clear(&pathconfig);
     status = _PyStatus_NO_MEMORY();
 
 done:
-    pathconfig_clear(&pathconfig);
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
     return status;
 }
@@ -402,12 +382,17 @@ _PyConfig_InitPathConfig(PyConfig *config)
 static void
 pathconfig_global_init(void)
 {
+    /* Initialize _Py_dll_path if needed */
+    PyStatus status = _PyPathConfig_Init();
+    if (_PyStatus_EXCEPTION(status)) {
+        Py_ExitStatusException(status);
+    }
+
     if (_Py_path_config.module_search_path != NULL) {
         /* Already initialized */
         return;
     }
 
-    PyStatus status;
     PyConfig config;
     _PyConfig_InitCompatConfig(&config);
 
@@ -416,7 +401,7 @@ pathconfig_global_init(void)
         goto error;
     }
 
-    status = _PyConfig_SetPathConfig(&config);
+    status = pathconfig_global_init_from_config(&config);
     if (_PyStatus_EXCEPTION(status)) {
         goto error;
     }
@@ -450,10 +435,6 @@ Py_SetPath(const wchar_t *path)
     alloc_error |= (new_config.prefix == NULL);
     new_config.exec_prefix = _PyMem_RawWcsdup(L"");
     alloc_error |= (new_config.exec_prefix == NULL);
-#ifdef MS_WINDOWS
-    new_config.dll_path = _Py_GetDLLPath();
-    alloc_error |= (new_config.dll_path == NULL);
-#endif
     new_config.module_search_path = _PyMem_RawWcsdup(path);
     alloc_error |= (new_config.module_search_path == NULL);
 
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index f9ff8f418078..e5b6283d8db0 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -475,7 +475,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime,
     config = &interp->config;
 
     if (config->_install_importlib) {
-        status = _PyConfig_SetPathConfig(config);
+        status = _PyPathConfig_Init();
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
@@ -646,7 +646,7 @@ pycore_init_import_warnings(PyThreadState *tstate, PyObject *sysmod)
     }
 
     if (config->_install_importlib) {
-        status = _PyConfig_SetPathConfig(config);
+        status = _PyPathConfig_Init();
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }


From webhook-mailer at python.org  Fri Sep 20 19:46:26 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 20 Sep 2019 23:46:26 -0000
Subject: [Python-checkins] Doc: Remove provisional note for asyncio.run()
 (GH-16310)
Message-ID: <mailman.14.1569023187.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/77af2290e594479002aa7d8f914d1f38b14a4854
commit: 77af2290e594479002aa7d8f914d1f38b14a4854
branch: master
author: Kyle Stanley <aeros167 at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-20T16:46:21-07:00
summary:

Doc: Remove provisional note for asyncio.run() (GH-16310)



Based on a comment from @asvetlov https://github.com/python/cpython/pull/15735#discussion_r323619076, this removes the provisional note for ``asyncio.run()`` in the documentation.

Automerge-Triggered-By: @1st1

files:
M Doc/library/asyncio-task.rst

diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index d9320422db3b..1fb882935895 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -226,8 +226,6 @@ Running an asyncio Program
     programs, and should ideally only be called once.
 
     .. versionadded:: 3.7
-       **Important:** this function has been added to asyncio in
-       Python 3.7 on a :term:`provisional basis <provisional api>`.
 
     .. versionchanged:: 3.9
        Updated to use :meth:`loop.shutdown_default_executor`.


From webhook-mailer at python.org  Fri Sep 20 19:50:20 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Fri, 20 Sep 2019 23:50:20 -0000
Subject: [Python-checkins] bpo-38234: Fix PyConfig_Read() when Py_SetPath()
 was called (GH-16298)
Message-ID: <mailman.15.1569023421.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/e267793aa4101b2771ed0e66aaff5743d23f59af
commit: e267793aa4101b2771ed0e66aaff5743d23f59af
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-21T01:50:16+02:00
summary:

bpo-38234: Fix PyConfig_Read() when Py_SetPath() was called (GH-16298)

* If Py_SetPath() has been called, _PyConfig_InitPathConfig() now
  uses its value.
* Py_Initialize() now longer copies path configuration from PyConfig
  to the global path configuration (_Py_path_config).

files:
A Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst
M Modules/getpath.c
M PC/getpathp.c
M Python/pathconfig.c

diff --git a/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst b/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst
new file mode 100644
index 000000000000..6310f19f004e
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst	
@@ -0,0 +1,2 @@
+Python ignored path passed to :c:func:`Py_SetPath`, fix Python
+initialization to use the specified path.
diff --git a/Modules/getpath.c b/Modules/getpath.c
index 36f9860ea131..3946623862cf 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -1213,10 +1213,12 @@ calculate_path_impl(const PyConfig *config,
                 "Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]\n");
     }
 
-    status = calculate_module_search_path(config, calculate,
-                                       prefix, exec_prefix, pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+    if (pathconfig->module_search_path == NULL) {
+        status = calculate_module_search_path(config, calculate,
+                                              prefix, exec_prefix, pathconfig);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
     }
 
     status = calculate_reduce_prefix(calculate, prefix, Py_ARRAY_LENGTH(prefix));
diff --git a/PC/getpathp.c b/PC/getpathp.c
index 01455a660be6..0ee53080bf31 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -1003,9 +1003,12 @@ calculate_path_impl(const PyConfig *config,
 
     calculate_home_prefix(calculate, prefix);
 
-    status = calculate_module_search_path(config, calculate, pathconfig, prefix);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+    if (pathconfig->module_search_path == NULL) {
+        status = calculate_module_search_path(config, calculate,
+                                              pathconfig, prefix);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
     }
 
 done:
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index ccab83274251..2d00f4b6b125 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -74,6 +74,13 @@ pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config)
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
+    if (copy_wstr(&new_config.module_search_path,
+                  _Py_path_config.module_search_path) < 0)
+    {
+        status = _PyStatus_NO_MEMORY();
+        goto error;
+    }
+
     /* Calculate program_full_path, prefix, exec_prefix,
        dll_path (Windows), and module_search_path */
     status = _PyPathConfig_Calculate(&new_config, config);


From webhook-mailer at python.org  Fri Sep 20 20:13:18 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Sat, 21 Sep 2019 00:13:18 -0000
Subject: [Python-checkins] [3.8] bpo-38234: Fix PyConfig_Read() when
 Py_SetPath() was called (GH-16298) (GH-16313)
Message-ID: <mailman.16.1569024798.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/9f3dcf802eefeb5ab821ce3c7204ab46557d53d7
commit: 9f3dcf802eefeb5ab821ce3c7204ab46557d53d7
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-21T02:13:14+02:00
summary:

[3.8] bpo-38234: Fix PyConfig_Read() when Py_SetPath() was called (GH-16298) (GH-16313)

* bpo-38234: Remove _PyPathConfig.dll_path (GH-16307)

The DLL path is not computed from any user configuration and cannot
be configured by PyConfig. Instead, add a new _Py_dll_path global variable.

Remove _PyConfig_SetPathConfig(): replaced with _PyPathConfig_Init().

Py_Initialize() now longer sets the "global path configuration",
but only initialize _Py_dll_path.

(cherry picked from commit c422167749f92d4170203e996a2c619c818335ea)

* bpo-38234: Fix PyConfig_Read() when Py_SetPath() was called (GH-16298)

* If Py_SetPath() has been called, _PyConfig_InitPathConfig() now
  uses its value.
* Py_Initialize() now longer copies path configuration from PyConfig
  to the global path configuration (_Py_path_config).

(cherry picked from commit e267793aa4101b2771ed0e66aaff5743d23f59af)

files:
A Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst
M Include/internal/pycore_initconfig.h
M Include/internal/pycore_pathconfig.h
M Modules/getpath.c
M PC/getpathp.c
M Python/pathconfig.c
M Python/pylifecycle.c

diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h
index c0b3d957827a..a7c1994eca16 100644
--- a/Include/internal/pycore_initconfig.h
+++ b/Include/internal/pycore_initconfig.h
@@ -149,8 +149,6 @@ extern PyStatus _PyConfig_Copy(
     PyConfig *config,
     const PyConfig *config2);
 extern PyStatus _PyConfig_InitPathConfig(PyConfig *config);
-extern PyStatus _PyConfig_SetPathConfig(
-    const PyConfig *config);
 extern void _PyConfig_Write(const PyConfig *config,
     _PyRuntimeState *runtime);
 extern PyStatus _PyConfig_SetPyArgv(
diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h
index 9e0ba0b01a80..a2c488cdc2eb 100644
--- a/Include/internal/pycore_pathconfig.h
+++ b/Include/internal/pycore_pathconfig.h
@@ -13,10 +13,7 @@ typedef struct _PyPathConfig {
     wchar_t *program_full_path;
     wchar_t *prefix;
     wchar_t *exec_prefix;
-#ifdef MS_WINDOWS
-    wchar_t *dll_path;
-#endif
-    /* Set by Py_SetPath(), or computed by _PyPathConfig_Init() */
+    /* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */
     wchar_t *module_search_path;
     /* Python program name */
     wchar_t *program_name;
@@ -38,6 +35,9 @@ typedef struct _PyPathConfig {
 /* Note: _PyPathConfig_INIT sets other fields to 0/NULL */
 
 PyAPI_DATA(_PyPathConfig) _Py_path_config;
+#ifdef MS_WINDOWS
+PyAPI_DATA(wchar_t*) _Py_dll_path;
+#endif
 
 extern void _PyPathConfig_ClearGlobal(void);
 extern PyStatus _PyPathConfig_SetGlobal(
@@ -59,6 +59,8 @@ extern int _Py_FindEnvConfigValue(
 extern wchar_t* _Py_GetDLLPath(void);
 #endif
 
+extern PyStatus _PyPathConfig_Init(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst b/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst
new file mode 100644
index 000000000000..6310f19f004e
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst	
@@ -0,0 +1,2 @@
+Python ignored path passed to :c:func:`Py_SetPath`, fix Python
+initialization to use the specified path.
diff --git a/Modules/getpath.c b/Modules/getpath.c
index dc0574851c3b..5afb5b152638 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -1213,10 +1213,12 @@ calculate_path_impl(const PyConfig *config,
                 "Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]\n");
     }
 
-    status = calculate_module_search_path(config, calculate,
-                                       prefix, exec_prefix, pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+    if (pathconfig->module_search_path == NULL) {
+        status = calculate_module_search_path(config, calculate,
+                                              prefix, exec_prefix, pathconfig);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
     }
 
     status = calculate_reduce_prefix(calculate, prefix, Py_ARRAY_LENGTH(prefix));
diff --git a/PC/getpathp.c b/PC/getpathp.c
index 7465d314b043..0ee53080bf31 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -128,6 +128,8 @@ typedef struct {
 
     wchar_t argv0_path[MAXPATHLEN+1];
     wchar_t zip_path[MAXPATHLEN+1];
+
+    wchar_t *dll_path;
 } PyCalculatePath;
 
 
@@ -666,28 +668,36 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
 }
 
 
-static void
+static PyStatus
 calculate_init(PyCalculatePath *calculate,
                const PyConfig *config)
 {
     calculate->home = config->home;
     calculate->path_env = _wgetenv(L"PATH");
+
+    calculate->dll_path = _Py_GetDLLPath();
+    if (calculate->dll_path == NULL) {
+        return _PyStatus_NO_MEMORY();
+    }
+
+    return _PyStatus_OK();
 }
 
 
 static int
-get_pth_filename(wchar_t *spbuffer, _PyPathConfig *pathconfig)
+get_pth_filename(PyCalculatePath *calculate, wchar_t *filename,
+                 const _PyPathConfig *pathconfig)
 {
-    if (pathconfig->dll_path[0]) {
-        if (!change_ext(spbuffer, pathconfig->dll_path, L"._pth") &&
-            exists(spbuffer))
+    if (calculate->dll_path[0]) {
+        if (!change_ext(filename, calculate->dll_path, L"._pth") &&
+            exists(filename))
         {
             return 1;
         }
     }
     if (pathconfig->program_full_path[0]) {
-        if (!change_ext(spbuffer, pathconfig->program_full_path, L"._pth") &&
-            exists(spbuffer))
+        if (!change_ext(filename, pathconfig->program_full_path, L"._pth") &&
+            exists(filename))
         {
             return 1;
         }
@@ -697,15 +707,16 @@ get_pth_filename(wchar_t *spbuffer, _PyPathConfig *pathconfig)
 
 
 static int
-calculate_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix)
+calculate_pth_file(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                   wchar_t *prefix)
 {
-    wchar_t spbuffer[MAXPATHLEN+1];
+    wchar_t filename[MAXPATHLEN+1];
 
-    if (!get_pth_filename(spbuffer, pathconfig)) {
+    if (!get_pth_filename(calculate, filename, pathconfig)) {
         return 0;
     }
 
-    return read_pth_file(pathconfig, prefix, spbuffer);
+    return read_pth_file(pathconfig, prefix, filename);
 }
 
 
@@ -966,13 +977,6 @@ calculate_path_impl(const PyConfig *config,
 {
     PyStatus status;
 
-    assert(pathconfig->dll_path == NULL);
-
-    pathconfig->dll_path = _Py_GetDLLPath();
-    if (pathconfig->dll_path == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
-
     status = get_program_full_path(config, calculate, pathconfig);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
@@ -986,7 +990,7 @@ calculate_path_impl(const PyConfig *config,
     memset(prefix, 0, sizeof(prefix));
 
     /* Search for a sys.path file */
-    if (calculate_pth_file(pathconfig, prefix)) {
+    if (calculate_pth_file(calculate, pathconfig, prefix)) {
         goto done;
     }
 
@@ -994,14 +998,17 @@ calculate_path_impl(const PyConfig *config,
 
     /* Calculate zip archive path from DLL or exe path */
     change_ext(calculate->zip_path,
-               pathconfig->dll_path[0] ? pathconfig->dll_path : pathconfig->program_full_path,
+               calculate->dll_path[0] ? calculate->dll_path : pathconfig->program_full_path,
                L".zip");
 
     calculate_home_prefix(calculate, prefix);
 
-    status = calculate_module_search_path(config, calculate, pathconfig, prefix);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+    if (pathconfig->module_search_path == NULL) {
+        status = calculate_module_search_path(config, calculate,
+                                              pathconfig, prefix);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
     }
 
 done:
@@ -1023,23 +1030,23 @@ calculate_free(PyCalculatePath *calculate)
 {
     PyMem_RawFree(calculate->machine_path);
     PyMem_RawFree(calculate->user_path);
+    PyMem_RawFree(calculate->dll_path);
 }
 
 
 PyStatus
 _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
 {
+    PyStatus status;
     PyCalculatePath calculate;
     memset(&calculate, 0, sizeof(calculate));
 
-    calculate_init(&calculate, config);
-
-    PyStatus status = calculate_path_impl(config, &calculate, pathconfig);
+    status = calculate_init(&calculate, config);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
     }
 
-    status = _PyStatus_OK();
+    status = calculate_path_impl(config, &calculate, pathconfig);
 
 done:
     calculate_free(&calculate);
@@ -1067,7 +1074,12 @@ _Py_CheckPython3(void)
 
     /* If there is a python3.dll next to the python3y.dll,
        assume this is a build tree; use that DLL */
-    wcscpy(py3path, _Py_path_config.dll_path);
+    if (_Py_dll_path != NULL) {
+        wcscpy(py3path, _Py_dll_path);
+    }
+    else {
+        wcscpy(py3path, L"");
+    }
     s = wcsrchr(py3path, L'\\');
     if (!s) {
         s = py3path;
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index 79ec4af00d83..1934f2e1c4b8 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -15,6 +15,9 @@ extern "C" {
 
 
 _PyPathConfig _Py_path_config = _PyPathConfig_INIT;
+#ifdef MS_WINDOWS
+wchar_t *_Py_dll_path = NULL;
+#endif
 
 
 static int
@@ -51,9 +54,6 @@ pathconfig_clear(_PyPathConfig *config)
     CLEAR(config->prefix);
     CLEAR(config->program_full_path);
     CLEAR(config->exec_prefix);
-#ifdef MS_WINDOWS
-    CLEAR(config->dll_path);
-#endif
     CLEAR(config->module_search_path);
     CLEAR(config->home);
     CLEAR(config->program_name);
@@ -74,6 +74,13 @@ pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config)
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
+    if (copy_wstr(&new_config.module_search_path,
+                  _Py_path_config.module_search_path) < 0)
+    {
+        status = _PyStatus_NO_MEMORY();
+        goto error;
+    }
+
     /* Calculate program_full_path, prefix, exec_prefix,
        dll_path (Windows), and module_search_path */
     status = _PyPathConfig_Calculate(&new_config, config);
@@ -114,47 +121,6 @@ pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config)
 }
 
 
-PyStatus
-_PyPathConfig_SetGlobal(const _PyPathConfig *config)
-{
-    PyStatus status;
-    _PyPathConfig new_config = _PyPathConfig_INIT;
-
-    PyMemAllocatorEx old_alloc;
-    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-
-#define COPY_ATTR(ATTR) \
-    do { \
-        if (copy_wstr(&new_config.ATTR, config->ATTR) < 0) { \
-            pathconfig_clear(&new_config); \
-            status = _PyStatus_NO_MEMORY(); \
-            goto done; \
-        } \
-    } while (0)
-
-    COPY_ATTR(program_full_path);
-    COPY_ATTR(prefix);
-    COPY_ATTR(exec_prefix);
-#ifdef MS_WINDOWS
-    COPY_ATTR(dll_path);
-#endif
-    COPY_ATTR(module_search_path);
-    COPY_ATTR(program_name);
-    COPY_ATTR(home);
-    COPY_ATTR(base_executable);
-
-    pathconfig_clear(&_Py_path_config);
-    /* Steal new_config strings; don't clear new_config */
-    _Py_path_config = new_config;
-
-    status = _PyStatus_OK();
-
-done:
-    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-    return status;
-}
-
-
 void
 _PyPathConfig_ClearGlobal(void)
 {
@@ -162,6 +128,10 @@ _PyPathConfig_ClearGlobal(void)
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
     pathconfig_clear(&_Py_path_config);
+#ifdef MS_WINDOWS
+    PyMem_RawFree(_Py_dll_path);
+    _Py_dll_path = NULL;
+#endif
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 }
@@ -200,12 +170,36 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep)
 
 /* Set the global path configuration from config. */
 PyStatus
-_PyConfig_SetPathConfig(const PyConfig *config)
+_PyPathConfig_Init(void)
 {
+#ifdef MS_WINDOWS
+    if (_Py_dll_path == NULL) {
+        /* Already set: nothing to do */
+        return _PyStatus_OK();
+    }
+
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
+    _Py_dll_path = _Py_GetDLLPath();
+
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    if (_Py_dll_path == NULL) {
+        return _PyStatus_NO_MEMORY();
+    }
+#endif
+    return _PyStatus_OK();
+}
+
+
+static PyStatus
+pathconfig_global_init_from_config(const PyConfig *config)
+{
     PyStatus status;
+    PyMemAllocatorEx old_alloc;
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
     _PyPathConfig pathconfig = _PyPathConfig_INIT;
 
     pathconfig.module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM);
@@ -222,12 +216,6 @@ _PyConfig_SetPathConfig(const PyConfig *config)
     if (copy_wstr(&pathconfig.exec_prefix, config->exec_prefix) < 0) {
         goto no_memory;
     }
-#ifdef MS_WINDOWS
-    pathconfig.dll_path = _Py_GetDLLPath();
-    if (pathconfig.dll_path == NULL) {
-        goto no_memory;
-    }
-#endif
     if (copy_wstr(&pathconfig.program_name, config->program_name) < 0) {
         goto no_memory;
     }
@@ -238,19 +226,18 @@ _PyConfig_SetPathConfig(const PyConfig *config)
         goto no_memory;
     }
 
-    status = _PyPathConfig_SetGlobal(&pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto done;
-    }
+    pathconfig_clear(&_Py_path_config);
+    /* Steal new_config strings; don't clear new_config */
+    _Py_path_config = pathconfig;
 
     status = _PyStatus_OK();
     goto done;
 
 no_memory:
+    pathconfig_clear(&pathconfig);
     status = _PyStatus_NO_MEMORY();
 
 done:
-    pathconfig_clear(&pathconfig);
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
     return status;
 }
@@ -402,12 +389,17 @@ _PyConfig_InitPathConfig(PyConfig *config)
 static void
 pathconfig_global_init(void)
 {
+    /* Initialize _Py_dll_path if needed */
+    PyStatus status = _PyPathConfig_Init();
+    if (_PyStatus_EXCEPTION(status)) {
+        Py_ExitStatusException(status);
+    }
+
     if (_Py_path_config.module_search_path != NULL) {
         /* Already initialized */
         return;
     }
 
-    PyStatus status;
     PyConfig config;
     _PyConfig_InitCompatConfig(&config);
 
@@ -416,7 +408,7 @@ pathconfig_global_init(void)
         goto error;
     }
 
-    status = _PyConfig_SetPathConfig(&config);
+    status = pathconfig_global_init_from_config(&config);
     if (_PyStatus_EXCEPTION(status)) {
         goto error;
     }
@@ -450,10 +442,6 @@ Py_SetPath(const wchar_t *path)
     alloc_error |= (new_config.prefix == NULL);
     new_config.exec_prefix = _PyMem_RawWcsdup(L"");
     alloc_error |= (new_config.exec_prefix == NULL);
-#ifdef MS_WINDOWS
-    new_config.dll_path = _Py_GetDLLPath();
-    alloc_error |= (new_config.dll_path == NULL);
-#endif
     new_config.module_search_path = _PyMem_RawWcsdup(path);
     alloc_error |= (new_config.module_search_path == NULL);
 
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 97bb99256d89..a9a6589d7a43 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -472,7 +472,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime,
     config = &interp->config;
 
     if (config->_install_importlib) {
-        status = _PyConfig_SetPathConfig(config);
+        status = _PyPathConfig_Init();
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
@@ -623,6 +623,8 @@ pycore_init_builtins(PyInterpreterState *interp)
 static PyStatus
 pycore_init_import_warnings(PyInterpreterState *interp, PyObject *sysmod)
 {
+    const PyConfig *config = &interp->config;
+
     PyStatus status = _PyImport_Init(interp);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
@@ -638,15 +640,15 @@ pycore_init_import_warnings(PyInterpreterState *interp, PyObject *sysmod)
         return _PyStatus_ERR("can't initialize warnings");
     }
 
-    if (interp->config._install_importlib) {
-        status = _PyConfig_SetPathConfig(&interp->config);
+    if (config->_install_importlib) {
+        status = _PyPathConfig_Init();
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
     }
 
     /* This call sets up builtin and frozen import support */
-    if (interp->config._install_importlib) {
+    if (config->_install_importlib) {
         status = init_importlib(interp, sysmod);
         if (_PyStatus_EXCEPTION(status)) {
             return status;


From webhook-mailer at python.org  Sat Sep 21 00:28:53 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 21 Sep 2019 04:28:53 -0000
Subject: [Python-checkins] bpo-38237: Make pow's arguments have more
 descriptive names and be keyword passable (GH-16302)
Message-ID: <mailman.17.1569040134.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/87d6cd3604e5c83c06339276228139f5e040b0e7
commit: 87d6cd3604e5c83c06339276228139f5e040b0e7
branch: master
author: Ammar Askar <ammar at ammaraskar.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-20T21:28:49-07:00
summary:

bpo-38237: Make pow's arguments have more descriptive names and be keyword passable (GH-16302)



Edit: `math.pow` changes removed on Mark's request.


https://bugs.python.org/issue38237



Automerge-Triggered-By: @rhettinger

files:
A Misc/NEWS.d/next/Library/2019-09-20-14-27-17.bpo-38237.xRUZbx.rst
M Doc/faq/programming.rst
M Doc/library/functions.rst
M Lib/test/test_builtin.py
M Python/bltinmodule.c
M Python/clinic/bltinmodule.c.h

diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst
index 2ff7236df1dc..9d45765abaac 100644
--- a/Doc/faq/programming.rst
+++ b/Doc/faq/programming.rst
@@ -779,26 +779,23 @@ A slash in the argument list of a function denotes that the parameters prior to
 it are positional-only.  Positional-only parameters are the ones without an
 externally-usable name.  Upon calling a function that accepts positional-only
 parameters, arguments are mapped to parameters based solely on their position.
-For example, :func:`pow` is a function that accepts positional-only parameters.
-Its documentation looks like this::
+For example, :func:`divmod` is a function that accepts positional-only
+parameters. Its documentation looks like this::
 
-   >>> help(pow)
-   Help on built-in function pow in module builtins:
+   >>> help(divmod)
+   Help on built-in function divmod in module builtins:
 
-   pow(x, y, z=None, /)
-      Equivalent to x**y (with two arguments) or x**y % z (with three arguments)
+   divmod(x, y, /)
+       Return the tuple (x//y, x%y).  Invariant: div*y + mod == x.
 
-      Some types, such as ints, are able to use a more efficient algorithm when
-      invoked using the three argument form.
+The slash at the end of the parameter list means that both parameters are
+positional-only. Thus, calling :func:`divmod` with keyword arguments would lead
+to an error::
 
-The slash at the end of the parameter list means that all three parameters are
-positional-only. Thus, calling :func:`pow` with keyword arguments would lead to
-an error::
-
-   >>> pow(x=3, y=4)
+   >>> divmod(x=3, y=4)
    Traceback (most recent call last):
      File "<stdin>", line 1, in <module>
-   TypeError: pow() takes no keyword arguments
+   TypeError: divmod() takes no keyword arguments
 
 
 Numbers and strings
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index ea2777f75367..4d3b2f593c0f 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -1274,11 +1274,12 @@ are always available.  They are listed here in alphabetical order.
    returns ``8364``.  This is the inverse of :func:`chr`.
 
 
-.. function:: pow(x, y[, z])
+.. function:: pow(base, exp[, mod])
 
-   Return *x* to the power *y*; if *z* is present, return *x* to the power *y*,
-   modulo *z* (computed more efficiently than ``pow(x, y) % z``). The two-argument
-   form ``pow(x, y)`` is equivalent to using the power operator: ``x**y``.
+   Return *base* to the power *exp*; if *mod* is present, return *base* to the
+   power *exp*, modulo *mod* (computed more efficiently than
+   ``pow(base, exp) % mod``). The two-argument form ``pow(base, exp)`` is
+   equivalent to using the power operator: ``base**exp``.
 
    The arguments must have numeric types.  With mixed operand types, the
    coercion rules for binary arithmetic operators apply.  For :class:`int`
@@ -1287,14 +1288,15 @@ are always available.  They are listed here in alphabetical order.
    converted to float and a float result is delivered.  For example, ``10**2``
    returns ``100``, but ``10**-2`` returns ``0.01``.
 
-   For :class:`int` operands *x* and *y*, if *z* is present, *z* must also be
-   of integer type and *z* must be nonzero. If *z* is present and *y* is
-   negative, *x* must be relatively prime to *z*. In that case, ``pow(inv_x,
-   -y, z)`` is returned, where *inv_x* is an inverse to *x* modulo *z*.
+   For :class:`int` operands *base* and *exp*, if *mod* is present, *mod* must
+   also be of integer type and *mod* must be nonzero. If *mod* is present and
+   *exp* is negative, *base* must be relatively prime to *mod*. In that case,
+   ``pow(inv_base, -exp, mod)`` is returned, where *inv_base* is an inverse to
+   *base* modulo *mod*.
 
    Here's an example of computing an inverse for ``38`` modulo ``97``::
 
-      >>> pow(38, -1, 97)
+      >>> pow(38, -1, mod=97)
       23
       >>> 23 * 38 % 97 == 1
       True
@@ -1304,6 +1306,10 @@ are always available.  They are listed here in alphabetical order.
       the second argument to be negative, permitting computation of modular
       inverses.
 
+   .. versionchanged:: 3.9
+      Allow keyword arguments.  Formerly, only positional arguments were
+      supported.
+
 
 .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False)
 
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index e1d2fffb36ea..abccf3222744 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -19,6 +19,7 @@
 import unittest
 import warnings
 from contextlib import ExitStack
+from functools import partial
 from inspect import CO_COROUTINE
 from itertools import product
 from textwrap import dedent
@@ -1206,6 +1207,18 @@ def test_pow(self):
 
         self.assertRaises(TypeError, pow)
 
+        # Test passing in arguments as keywords.
+        self.assertEqual(pow(0, exp=0), 1)
+        self.assertEqual(pow(base=2, exp=4), 16)
+        self.assertEqual(pow(base=5, exp=2, mod=14), 11)
+        twopow = partial(pow, base=2)
+        self.assertEqual(twopow(exp=5), 32)
+        fifth_power = partial(pow, exp=5)
+        self.assertEqual(fifth_power(2), 32)
+        mod10 = partial(pow, mod=10)
+        self.assertEqual(mod10(2, 6), 4)
+        self.assertEqual(mod10(exp=6, base=2), 4)
+
     def test_input(self):
         self.write_testfile()
         fp = open(TESTFN, 'r')
diff --git a/Misc/NEWS.d/next/Library/2019-09-20-14-27-17.bpo-38237.xRUZbx.rst b/Misc/NEWS.d/next/Library/2019-09-20-14-27-17.bpo-38237.xRUZbx.rst
new file mode 100644
index 000000000000..07e1bf712db3
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-20-14-27-17.bpo-38237.xRUZbx.rst
@@ -0,0 +1,2 @@
+The arguments for the builtin pow function are more descriptive. They can now
+also be passed in as keywords.
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index a40eb421f5de..31e7ad7658b4 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -1796,22 +1796,22 @@ builtin_ord(PyObject *module, PyObject *c)
 /*[clinic input]
 pow as builtin_pow
 
-    x: object
-    y: object
-    z: object = None
-    /
+    base: object
+    exp: object
+    mod: object = None
 
-Equivalent to x**y (with two arguments) or x**y % z (with three arguments)
+Equivalent to base**exp (with two arguments) or base**exp % mod (with three arguments)
 
 Some types, such as ints, are able to use a more efficient algorithm when
 invoked using the three argument form.
 [clinic start generated code]*/
 
 static PyObject *
-builtin_pow_impl(PyObject *module, PyObject *x, PyObject *y, PyObject *z)
-/*[clinic end generated code: output=50a14d5d130d404b input=653d57d38d41fc07]*/
+builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp,
+                 PyObject *mod)
+/*[clinic end generated code: output=3ca1538221bbf15f input=bd72d0a0ec8e5eb5]*/
 {
-    return PyNumber_Power(x, y, z);
+    return PyNumber_Power(base, exp, mod);
 }
 
 
diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h
index b936b0cf3721..f6c5b7c1c8bd 100644
--- a/Python/clinic/bltinmodule.c.h
+++ b/Python/clinic/bltinmodule.c.h
@@ -608,39 +608,45 @@ PyDoc_STRVAR(builtin_ord__doc__,
     {"ord", (PyCFunction)builtin_ord, METH_O, builtin_ord__doc__},
 
 PyDoc_STRVAR(builtin_pow__doc__,
-"pow($module, x, y, z=None, /)\n"
+"pow($module, /, base, exp, mod=None)\n"
 "--\n"
 "\n"
-"Equivalent to x**y (with two arguments) or x**y % z (with three arguments)\n"
+"Equivalent to base**exp (with two arguments) or base**exp % mod (with three arguments)\n"
 "\n"
 "Some types, such as ints, are able to use a more efficient algorithm when\n"
 "invoked using the three argument form.");
 
 #define BUILTIN_POW_METHODDEF    \
-    {"pow", (PyCFunction)(void(*)(void))builtin_pow, METH_FASTCALL, builtin_pow__doc__},
+    {"pow", (PyCFunction)(void(*)(void))builtin_pow, METH_FASTCALL|METH_KEYWORDS, builtin_pow__doc__},
 
 static PyObject *
-builtin_pow_impl(PyObject *module, PyObject *x, PyObject *y, PyObject *z);
+builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp,
+                 PyObject *mod);
 
 static PyObject *
-builtin_pow(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
+builtin_pow(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    PyObject *x;
-    PyObject *y;
-    PyObject *z = Py_None;
-
-    if (!_PyArg_CheckPositional("pow", nargs, 2, 3)) {
+    static const char * const _keywords[] = {"base", "exp", "mod", NULL};
+    static _PyArg_Parser _parser = {NULL, _keywords, "pow", 0};
+    PyObject *argsbuf[3];
+    Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2;
+    PyObject *base;
+    PyObject *exp;
+    PyObject *mod = Py_None;
+
+    args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf);
+    if (!args) {
         goto exit;
     }
-    x = args[0];
-    y = args[1];
-    if (nargs < 3) {
-        goto skip_optional;
+    base = args[0];
+    exp = args[1];
+    if (!noptargs) {
+        goto skip_optional_pos;
     }
-    z = args[2];
-skip_optional:
-    return_value = builtin_pow_impl(module, x, y, z);
+    mod = args[2];
+skip_optional_pos:
+    return_value = builtin_pow_impl(module, base, exp, mod);
 
 exit:
     return return_value;
@@ -849,4 +855,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=4e118c2cd2cd98f3 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=1e2a6185e05ecd11 input=a9049054013a1b77]*/


From webhook-mailer at python.org  Sat Sep 21 00:46:56 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Sat, 21 Sep 2019 04:46:56 -0000
Subject: [Python-checkins] Minor code and comment cleanup (GH-16315)
Message-ID: <mailman.18.1569041217.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/7ce4bfa8cfcc78d03e164f2de64a2caad1d919af
commit: 7ce4bfa8cfcc78d03e164f2de64a2caad1d919af
branch: master
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-20T21:46:52-07:00
summary:

Minor code and comment cleanup (GH-16315)

files:
M Lib/statistics.py

diff --git a/Lib/statistics.py b/Lib/statistics.py
index d81596e5d8ab..0d747b3d6c05 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -555,8 +555,9 @@ def mode(data):
 
     """
     data = iter(data)
+    pairs = Counter(data).most_common(1)
     try:
-        return Counter(data).most_common(1)[0][0]
+        return pairs[0][0]
     except IndexError:
         raise StatisticsError('no mode for empty data') from None
 
@@ -602,6 +603,7 @@ def multimode(data):
 # mean=0.300.  Only the latter (which corresponds with R6) gives the
 # desired cut point with 30% of the population falling below that
 # value, making it comparable to a result from an inv_cdf() function.
+# The R6 exclusive method is also idempotent.
 
 # For describing population data where the end points are known to
 # be included in the data, the R7 inclusive method is a reasonable


From webhook-mailer at python.org  Sat Sep 21 01:18:14 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Sat, 21 Sep 2019 05:18:14 -0000
Subject: [Python-checkins] Minor code and comment cleanup (GH-16315)
 (GH-16316)
Message-ID: <mailman.19.1569043095.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/8fe4755aa3a3ef21d1cc90c5e393c7b13b89f9f9
commit: 8fe4755aa3a3ef21d1cc90c5e393c7b13b89f9f9
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-20T22:18:10-07:00
summary:

Minor code and comment cleanup (GH-16315) (GH-16316)

(cherry picked from commit 7ce4bfa8cfcc78d03e164f2de64a2caad1d919af)

Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com>

files:
M Lib/statistics.py

diff --git a/Lib/statistics.py b/Lib/statistics.py
index d81596e5d8ab..0d747b3d6c05 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -555,8 +555,9 @@ def mode(data):
 
     """
     data = iter(data)
+    pairs = Counter(data).most_common(1)
     try:
-        return Counter(data).most_common(1)[0][0]
+        return pairs[0][0]
     except IndexError:
         raise StatisticsError('no mode for empty data') from None
 
@@ -602,6 +603,7 @@ def multimode(data):
 # mean=0.300.  Only the latter (which corresponds with R6) gives the
 # desired cut point with 30% of the population falling below that
 # value, making it comparable to a result from an inv_cdf() function.
+# The R6 exclusive method is also idempotent.
 
 # For describing population data where the end points are known to
 # be included in the data, the R7 inclusive method is a reasonable


From webhook-mailer at python.org  Sat Sep 21 01:40:46 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 21 Sep 2019 05:40:46 -0000
Subject: [Python-checkins] Doc: Fix spelling errors of 'initial' in enum.py
 (GH-16314)
Message-ID: <mailman.20.1569044447.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/8d4c52ffb47a4d3590758220f297ba58108e1ef3
commit: 8d4c52ffb47a4d3590758220f297ba58108e1ef3
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-20T22:40:42-07:00
summary:

Doc: Fix spelling errors of 'initial' in enum.py (GH-16314)

(cherry picked from commit bb16fb2cb8dca1fa1d67621c3175793eda7dbdc2)

Co-authored-by: HongWeipeng <hongweichen8888 at sina.com>

files:
M Lib/enum.py

diff --git a/Lib/enum.py b/Lib/enum.py
index 5e97a9e8d810..d0b03b4aa56c 100644
--- a/Lib/enum.py
+++ b/Lib/enum.py
@@ -682,7 +682,7 @@ def _generate_next_value_(name, start, count, last_values):
         Generate the next value when not given.
 
         name: the name of the member
-        start: the initital start value or None
+        start: the initial start value or None
         count: the number of existing members
         last_value: the last value assigned or None
         """


From webhook-mailer at python.org  Sat Sep 21 01:41:38 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 21 Sep 2019 05:41:38 -0000
Subject: [Python-checkins] Doc: Fix spelling errors of 'initial' in enum.py
 (GH-16314)
Message-ID: <mailman.21.1569044500.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/f4e0ceb211e65dc080b64477380dfae923eb1306
commit: f4e0ceb211e65dc080b64477380dfae923eb1306
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-20T22:41:35-07:00
summary:

Doc: Fix spelling errors of 'initial' in enum.py (GH-16314)

(cherry picked from commit bb16fb2cb8dca1fa1d67621c3175793eda7dbdc2)

Co-authored-by: HongWeipeng <hongweichen8888 at sina.com>

files:
M Lib/enum.py

diff --git a/Lib/enum.py b/Lib/enum.py
index 6ef17c7f6dc8..108d389d948d 100644
--- a/Lib/enum.py
+++ b/Lib/enum.py
@@ -677,7 +677,7 @@ def _generate_next_value_(name, start, count, last_values):
         Generate the next value when not given.
 
         name: the name of the member
-        start: the initital start value or None
+        start: the initial start value or None
         count: the number of existing members
         last_value: the last value assigned or None
         """


From webhook-mailer at python.org  Sat Sep 21 04:22:38 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Sat, 21 Sep 2019 08:22:38 -0000
Subject: [Python-checkins] bpo-38237: Let pow() support keyword arguments
 (GH-16302) (GH-16320)
Message-ID: <mailman.22.1569054159.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/37bc93552375cb1bc616927b5c1905bae3c0e99d
commit: 37bc93552375cb1bc616927b5c1905bae3c0e99d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-21T01:22:29-07:00
summary:

bpo-38237: Let pow() support keyword arguments (GH-16302) (GH-16320)

Backported with release manager approval

files:
A Misc/NEWS.d/next/Library/2019-09-20-14-27-17.bpo-38237.xRUZbx.rst
M Doc/faq/programming.rst
M Doc/library/functions.rst
M Lib/test/test_builtin.py
M Python/bltinmodule.c
M Python/clinic/bltinmodule.c.h

diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst
index 2ff7236df1dc..9d45765abaac 100644
--- a/Doc/faq/programming.rst
+++ b/Doc/faq/programming.rst
@@ -779,26 +779,23 @@ A slash in the argument list of a function denotes that the parameters prior to
 it are positional-only.  Positional-only parameters are the ones without an
 externally-usable name.  Upon calling a function that accepts positional-only
 parameters, arguments are mapped to parameters based solely on their position.
-For example, :func:`pow` is a function that accepts positional-only parameters.
-Its documentation looks like this::
+For example, :func:`divmod` is a function that accepts positional-only
+parameters. Its documentation looks like this::
 
-   >>> help(pow)
-   Help on built-in function pow in module builtins:
+   >>> help(divmod)
+   Help on built-in function divmod in module builtins:
 
-   pow(x, y, z=None, /)
-      Equivalent to x**y (with two arguments) or x**y % z (with three arguments)
+   divmod(x, y, /)
+       Return the tuple (x//y, x%y).  Invariant: div*y + mod == x.
 
-      Some types, such as ints, are able to use a more efficient algorithm when
-      invoked using the three argument form.
+The slash at the end of the parameter list means that both parameters are
+positional-only. Thus, calling :func:`divmod` with keyword arguments would lead
+to an error::
 
-The slash at the end of the parameter list means that all three parameters are
-positional-only. Thus, calling :func:`pow` with keyword arguments would lead to
-an error::
-
-   >>> pow(x=3, y=4)
+   >>> divmod(x=3, y=4)
    Traceback (most recent call last):
      File "<stdin>", line 1, in <module>
-   TypeError: pow() takes no keyword arguments
+   TypeError: divmod() takes no keyword arguments
 
 
 Numbers and strings
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 2f3ef4f7fc76..947a0e54aed0 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -1272,11 +1272,12 @@ are always available.  They are listed here in alphabetical order.
    returns ``8364``.  This is the inverse of :func:`chr`.
 
 
-.. function:: pow(x, y[, z])
+.. function:: pow(base, exp[, mod])
 
-   Return *x* to the power *y*; if *z* is present, return *x* to the power *y*,
-   modulo *z* (computed more efficiently than ``pow(x, y) % z``). The two-argument
-   form ``pow(x, y)`` is equivalent to using the power operator: ``x**y``.
+   Return *base* to the power *exp*; if *mod* is present, return *base* to the
+   power *exp*, modulo *mod* (computed more efficiently than
+   ``pow(base, exp) % mod``). The two-argument form ``pow(base, exp)`` is
+   equivalent to using the power operator: ``base**exp``.
 
    The arguments must have numeric types.  With mixed operand types, the
    coercion rules for binary arithmetic operators apply.  For :class:`int`
@@ -1285,14 +1286,15 @@ are always available.  They are listed here in alphabetical order.
    converted to float and a float result is delivered.  For example, ``10**2``
    returns ``100``, but ``10**-2`` returns ``0.01``.
 
-   For :class:`int` operands *x* and *y*, if *z* is present, *z* must also be
-   of integer type and *z* must be nonzero. If *z* is present and *y* is
-   negative, *x* must be relatively prime to *z*. In that case, ``pow(inv_x,
-   -y, z)`` is returned, where *inv_x* is an inverse to *x* modulo *z*.
+   For :class:`int` operands *base* and *exp*, if *mod* is present, *mod* must
+   also be of integer type and *mod* must be nonzero. If *mod* is present and
+   *exp* is negative, *base* must be relatively prime to *mod*. In that case,
+   ``pow(inv_base, -exp, mod)`` is returned, where *inv_base* is an inverse to
+   *base* modulo *mod*.
 
    Here's an example of computing an inverse for ``38`` modulo ``97``::
 
-      >>> pow(38, -1, 97)
+      >>> pow(38, -1, mod=97)
       23
       >>> 23 * 38 % 97 == 1
       True
@@ -1302,6 +1304,10 @@ are always available.  They are listed here in alphabetical order.
       the second argument to be negative, permitting computation of modular
       inverses.
 
+   .. versionchanged:: 3.9
+      Allow keyword arguments.  Formerly, only positional arguments were
+      supported.
+
 
 .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False)
 
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 9baac3cf3b95..cc20551b0dc2 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -19,6 +19,7 @@
 import unittest
 import warnings
 from contextlib import ExitStack
+from functools import partial
 from inspect import CO_COROUTINE
 from itertools import product
 from textwrap import dedent
@@ -1206,6 +1207,18 @@ def test_pow(self):
 
         self.assertRaises(TypeError, pow)
 
+        # Test passing in arguments as keywords.
+        self.assertEqual(pow(0, exp=0), 1)
+        self.assertEqual(pow(base=2, exp=4), 16)
+        self.assertEqual(pow(base=5, exp=2, mod=14), 11)
+        twopow = partial(pow, base=2)
+        self.assertEqual(twopow(exp=5), 32)
+        fifth_power = partial(pow, exp=5)
+        self.assertEqual(fifth_power(2), 32)
+        mod10 = partial(pow, mod=10)
+        self.assertEqual(mod10(2, 6), 4)
+        self.assertEqual(mod10(exp=6, base=2), 4)
+
     def test_input(self):
         self.write_testfile()
         fp = open(TESTFN, 'r')
diff --git a/Misc/NEWS.d/next/Library/2019-09-20-14-27-17.bpo-38237.xRUZbx.rst b/Misc/NEWS.d/next/Library/2019-09-20-14-27-17.bpo-38237.xRUZbx.rst
new file mode 100644
index 000000000000..07e1bf712db3
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-20-14-27-17.bpo-38237.xRUZbx.rst
@@ -0,0 +1,2 @@
+The arguments for the builtin pow function are more descriptive. They can now
+also be passed in as keywords.
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index c8d34c940f42..0d3f7edb1d90 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -1797,22 +1797,22 @@ builtin_ord(PyObject *module, PyObject *c)
 /*[clinic input]
 pow as builtin_pow
 
-    x: object
-    y: object
-    z: object = None
-    /
+    base: object
+    exp: object
+    mod: object = None
 
-Equivalent to x**y (with two arguments) or x**y % z (with three arguments)
+Equivalent to base**exp (with two arguments) or base**exp % mod (with three arguments)
 
 Some types, such as ints, are able to use a more efficient algorithm when
 invoked using the three argument form.
 [clinic start generated code]*/
 
 static PyObject *
-builtin_pow_impl(PyObject *module, PyObject *x, PyObject *y, PyObject *z)
-/*[clinic end generated code: output=50a14d5d130d404b input=653d57d38d41fc07]*/
+builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp,
+                 PyObject *mod)
+/*[clinic end generated code: output=3ca1538221bbf15f input=bd72d0a0ec8e5eb5]*/
 {
-    return PyNumber_Power(x, y, z);
+    return PyNumber_Power(base, exp, mod);
 }
 
 
diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h
index b936b0cf3721..f6c5b7c1c8bd 100644
--- a/Python/clinic/bltinmodule.c.h
+++ b/Python/clinic/bltinmodule.c.h
@@ -608,39 +608,45 @@ PyDoc_STRVAR(builtin_ord__doc__,
     {"ord", (PyCFunction)builtin_ord, METH_O, builtin_ord__doc__},
 
 PyDoc_STRVAR(builtin_pow__doc__,
-"pow($module, x, y, z=None, /)\n"
+"pow($module, /, base, exp, mod=None)\n"
 "--\n"
 "\n"
-"Equivalent to x**y (with two arguments) or x**y % z (with three arguments)\n"
+"Equivalent to base**exp (with two arguments) or base**exp % mod (with three arguments)\n"
 "\n"
 "Some types, such as ints, are able to use a more efficient algorithm when\n"
 "invoked using the three argument form.");
 
 #define BUILTIN_POW_METHODDEF    \
-    {"pow", (PyCFunction)(void(*)(void))builtin_pow, METH_FASTCALL, builtin_pow__doc__},
+    {"pow", (PyCFunction)(void(*)(void))builtin_pow, METH_FASTCALL|METH_KEYWORDS, builtin_pow__doc__},
 
 static PyObject *
-builtin_pow_impl(PyObject *module, PyObject *x, PyObject *y, PyObject *z);
+builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp,
+                 PyObject *mod);
 
 static PyObject *
-builtin_pow(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
+builtin_pow(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    PyObject *x;
-    PyObject *y;
-    PyObject *z = Py_None;
-
-    if (!_PyArg_CheckPositional("pow", nargs, 2, 3)) {
+    static const char * const _keywords[] = {"base", "exp", "mod", NULL};
+    static _PyArg_Parser _parser = {NULL, _keywords, "pow", 0};
+    PyObject *argsbuf[3];
+    Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2;
+    PyObject *base;
+    PyObject *exp;
+    PyObject *mod = Py_None;
+
+    args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf);
+    if (!args) {
         goto exit;
     }
-    x = args[0];
-    y = args[1];
-    if (nargs < 3) {
-        goto skip_optional;
+    base = args[0];
+    exp = args[1];
+    if (!noptargs) {
+        goto skip_optional_pos;
     }
-    z = args[2];
-skip_optional:
-    return_value = builtin_pow_impl(module, x, y, z);
+    mod = args[2];
+skip_optional_pos:
+    return_value = builtin_pow_impl(module, base, exp, mod);
 
 exit:
     return return_value;
@@ -849,4 +855,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=4e118c2cd2cd98f3 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=1e2a6185e05ecd11 input=a9049054013a1b77]*/


From webhook-mailer at python.org  Sat Sep 21 15:57:50 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Sat, 21 Sep 2019 19:57:50 -0000
Subject: [Python-checkins] Shorter docstring (GH-16322)
Message-ID: <mailman.23.1569095871.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/b104ecbbafc14f9ca0c8371963c45dca893f6b75
commit: b104ecbbafc14f9ca0c8371963c45dca893f6b75
branch: master
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-21T12:57:44-07:00
summary:

Shorter docstring (GH-16322)

files:
M Python/bltinmodule.c
M Python/clinic/bltinmodule.c.h

diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 31e7ad7658b4..728ba5bdc99b 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -1800,7 +1800,7 @@ pow as builtin_pow
     exp: object
     mod: object = None
 
-Equivalent to base**exp (with two arguments) or base**exp % mod (with three arguments)
+Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments
 
 Some types, such as ints, are able to use a more efficient algorithm when
 invoked using the three argument form.
@@ -1809,7 +1809,7 @@ invoked using the three argument form.
 static PyObject *
 builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp,
                  PyObject *mod)
-/*[clinic end generated code: output=3ca1538221bbf15f input=bd72d0a0ec8e5eb5]*/
+/*[clinic end generated code: output=3ca1538221bbf15f input=435dbd48a12efb23]*/
 {
     return PyNumber_Power(base, exp, mod);
 }
diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h
index f6c5b7c1c8bd..d15af1f7f377 100644
--- a/Python/clinic/bltinmodule.c.h
+++ b/Python/clinic/bltinmodule.c.h
@@ -611,7 +611,7 @@ PyDoc_STRVAR(builtin_pow__doc__,
 "pow($module, /, base, exp, mod=None)\n"
 "--\n"
 "\n"
-"Equivalent to base**exp (with two arguments) or base**exp % mod (with three arguments)\n"
+"Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments\n"
 "\n"
 "Some types, such as ints, are able to use a more efficient algorithm when\n"
 "invoked using the three argument form.");
@@ -855,4 +855,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=1e2a6185e05ecd11 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=29686a89b739d600 input=a9049054013a1b77]*/


From webhook-mailer at python.org  Sat Sep 21 16:32:12 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Sat, 21 Sep 2019 20:32:12 -0000
Subject: [Python-checkins] bpo-38237: Shorter docstring (GH-16322) (GH-16323)
Message-ID: <mailman.24.1569097933.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/24231ca75c721c8167a7394deb300727ccdcba51
commit: 24231ca75c721c8167a7394deb300727ccdcba51
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-21T13:32:07-07:00
summary:

bpo-38237: Shorter docstring (GH-16322) (GH-16323)

(cherry picked from commit b104ecbbafc14f9ca0c8371963c45dca893f6b75)

Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com>

files:
M Python/bltinmodule.c
M Python/clinic/bltinmodule.c.h

diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 0d3f7edb1d90..90a6b9fc6e24 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -1801,7 +1801,7 @@ pow as builtin_pow
     exp: object
     mod: object = None
 
-Equivalent to base**exp (with two arguments) or base**exp % mod (with three arguments)
+Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments
 
 Some types, such as ints, are able to use a more efficient algorithm when
 invoked using the three argument form.
@@ -1810,7 +1810,7 @@ invoked using the three argument form.
 static PyObject *
 builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp,
                  PyObject *mod)
-/*[clinic end generated code: output=3ca1538221bbf15f input=bd72d0a0ec8e5eb5]*/
+/*[clinic end generated code: output=3ca1538221bbf15f input=435dbd48a12efb23]*/
 {
     return PyNumber_Power(base, exp, mod);
 }
diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h
index f6c5b7c1c8bd..d15af1f7f377 100644
--- a/Python/clinic/bltinmodule.c.h
+++ b/Python/clinic/bltinmodule.c.h
@@ -611,7 +611,7 @@ PyDoc_STRVAR(builtin_pow__doc__,
 "pow($module, /, base, exp, mod=None)\n"
 "--\n"
 "\n"
-"Equivalent to base**exp (with two arguments) or base**exp % mod (with three arguments)\n"
+"Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments\n"
 "\n"
 "Some types, such as ints, are able to use a more efficient algorithm when\n"
 "invoked using the three argument form.");
@@ -855,4 +855,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=1e2a6185e05ecd11 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=29686a89b739d600 input=a9049054013a1b77]*/


From webhook-mailer at python.org  Sat Sep 21 22:51:55 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Sun, 22 Sep 2019 02:51:55 -0000
Subject: [Python-checkins] Updated incorrect level-setting code to use
 setLevel(). (GH-16325)
Message-ID: <mailman.25.1569120715.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/1d094af716e8ce5e5710e1dfbce7832ba333be55
commit: 1d094af716e8ce5e5710e1dfbce7832ba333be55
branch: master
author: Vinay Sajip <vinay_sajip at yahoo.co.uk>
committer: GitHub <noreply at github.com>
date: 2019-09-22T03:51:51+01:00
summary:

Updated incorrect level-setting code to use setLevel(). (GH-16325)

files:
M Lib/logging/config.py
M Lib/test/test_logging.py

diff --git a/Lib/logging/config.py b/Lib/logging/config.py
index cfd93116eedd..9dd35e11aab3 100644
--- a/Lib/logging/config.py
+++ b/Lib/logging/config.py
@@ -1,4 +1,4 @@
-# Copyright 2001-2016 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved.
 #
 # Permission to use, copy, modify, and distribute this software and its
 # documentation for any purpose and without fee is hereby granted,
@@ -19,7 +19,7 @@
 is based on PEP 282 and comments thereto in comp.lang.python, and influenced
 by Apache's log4j system.
 
-Copyright (C) 2001-2016 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved.
 
 To use, simply 'import logging' and log away!
 """
@@ -173,9 +173,10 @@ def _handle_existing_loggers(existing, child_loggers, disable_existing):
     for log in existing:
         logger = root.manager.loggerDict[log]
         if log in child_loggers:
-            logger.level = logging.NOTSET
-            logger.handlers = []
-            logger.propagate = True
+            if not isinstance(logger, logging.PlaceHolder):
+                logger.setLevel(logging.NOTSET)
+                logger.handlers = []
+                logger.propagate = True
         else:
             logger.disabled = disable_existing
 
diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
index dca744c59092..74ccf48b7ebe 100644
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -4315,7 +4315,7 @@ def cleanup(self):
         logging._handlers.clear()
         logging._handlers.update(self.saved_handlers)
         logging._handlerList[:] = self.saved_handler_list
-        logging.root.level = self.original_logging_level
+        logging.root.setLevel(self.original_logging_level)
 
     def test_no_kwargs(self):
         logging.basicConfig()


From webhook-mailer at python.org  Sat Sep 21 23:26:40 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Sun, 22 Sep 2019 03:26:40 -0000
Subject: [Python-checkins] Updated incorrect level-setting code to use
 setLevel(). (GH-16325) (GH-16327)
Message-ID: <mailman.26.1569122801.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/fe9089a08b6dd6dd1ba8b238afc2fc4dfaac689e
commit: fe9089a08b6dd6dd1ba8b238afc2fc4dfaac689e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Vinay Sajip <vinay_sajip at yahoo.co.uk>
date: 2019-09-22T04:26:37+01:00
summary:

Updated incorrect level-setting code to use setLevel(). (GH-16325) (GH-16327)

(cherry picked from commit 1d094af716e8ce5e5710e1dfbce7832ba333be55)

files:
M Lib/logging/config.py
M Lib/test/test_logging.py

diff --git a/Lib/logging/config.py b/Lib/logging/config.py
index cfd93116eedd..9dd35e11aab3 100644
--- a/Lib/logging/config.py
+++ b/Lib/logging/config.py
@@ -1,4 +1,4 @@
-# Copyright 2001-2016 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved.
 #
 # Permission to use, copy, modify, and distribute this software and its
 # documentation for any purpose and without fee is hereby granted,
@@ -19,7 +19,7 @@
 is based on PEP 282 and comments thereto in comp.lang.python, and influenced
 by Apache's log4j system.
 
-Copyright (C) 2001-2016 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved.
 
 To use, simply 'import logging' and log away!
 """
@@ -173,9 +173,10 @@ def _handle_existing_loggers(existing, child_loggers, disable_existing):
     for log in existing:
         logger = root.manager.loggerDict[log]
         if log in child_loggers:
-            logger.level = logging.NOTSET
-            logger.handlers = []
-            logger.propagate = True
+            if not isinstance(logger, logging.PlaceHolder):
+                logger.setLevel(logging.NOTSET)
+                logger.handlers = []
+                logger.propagate = True
         else:
             logger.disabled = disable_existing
 
diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
index 7399bb362f6e..37655a5ccc40 100644
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -4315,7 +4315,7 @@ def cleanup(self):
         logging._handlers.clear()
         logging._handlers.update(self.saved_handlers)
         logging._handlerList[:] = self.saved_handler_list
-        logging.root.level = self.original_logging_level
+        logging.root.setLevel(self.original_logging_level)
 
     def test_no_kwargs(self):
         logging.basicConfig()


From webhook-mailer at python.org  Sat Sep 21 23:27:04 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Sun, 22 Sep 2019 03:27:04 -0000
Subject: [Python-checkins] Updated incorrect level-setting code to use
 setLevel(). (GH-16325) (GH-16326)
Message-ID: <mailman.27.1569122825.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/6641a109fba8489acf2762b2ac2e8c2c20c3a640
commit: 6641a109fba8489acf2762b2ac2e8c2c20c3a640
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Vinay Sajip <vinay_sajip at yahoo.co.uk>
date: 2019-09-22T04:27:00+01:00
summary:

Updated incorrect level-setting code to use setLevel(). (GH-16325) (GH-16326)

(cherry picked from commit 1d094af716e8ce5e5710e1dfbce7832ba333be55)

files:
M Lib/logging/config.py
M Lib/test/test_logging.py

diff --git a/Lib/logging/config.py b/Lib/logging/config.py
index fa1a398aee2a..f58be9308935 100644
--- a/Lib/logging/config.py
+++ b/Lib/logging/config.py
@@ -1,4 +1,4 @@
-# Copyright 2001-2016 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved.
 #
 # Permission to use, copy, modify, and distribute this software and its
 # documentation for any purpose and without fee is hereby granted,
@@ -19,7 +19,7 @@
 is based on PEP 282 and comments thereto in comp.lang.python, and influenced
 by Apache's log4j system.
 
-Copyright (C) 2001-2016 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved.
 
 To use, simply 'import logging' and log away!
 """
@@ -173,9 +173,10 @@ def _handle_existing_loggers(existing, child_loggers, disable_existing):
     for log in existing:
         logger = root.manager.loggerDict[log]
         if log in child_loggers:
-            logger.level = logging.NOTSET
-            logger.handlers = []
-            logger.propagate = True
+            if not isinstance(logger, logging.PlaceHolder):
+                logger.setLevel(logging.NOTSET)
+                logger.handlers = []
+                logger.propagate = True
         else:
             logger.disabled = disable_existing
 
diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
index 13393cd8b372..546cea98c1ea 100644
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -4020,7 +4020,7 @@ def cleanup(self):
         logging._handlers.clear()
         logging._handlers.update(self.saved_handlers)
         logging._handlerList[:] = self.saved_handler_list
-        logging.root.level = self.original_logging_level
+        logging.root.setLevel(self.original_logging_level)
 
     def test_no_kwargs(self):
         logging.basicConfig()


From webhook-mailer at python.org  Sun Sep 22 06:32:51 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Sun, 22 Sep 2019 10:32:51 -0000
Subject: [Python-checkins] bpo-38209: Simplify dataclasses.InitVar by using
 __class_getitem__(). (GH-16255)
Message-ID: <mailman.28.1569148372.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/b4d0b39a9b4cd203bcc5b236dc96456e9658119a
commit: b4d0b39a9b4cd203bcc5b236dc96456e9658119a
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-22T13:32:41+03:00
summary:

bpo-38209: Simplify dataclasses.InitVar by using __class_getitem__(). (GH-16255)

files:
M Lib/dataclasses.py

diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py
index 9020c905d117..9135b07c9f25 100644
--- a/Lib/dataclasses.py
+++ b/Lib/dataclasses.py
@@ -199,11 +199,7 @@ def __repr__(self):
 # https://bugs.python.org/issue33453 for details.
 _MODULE_IDENTIFIER_RE = re.compile(r'^(?:\s*(\w+)\s*\.)?\s*(\w+)')
 
-class _InitVarMeta(type):
-    def __getitem__(self, params):
-        return InitVar(params)
-
-class InitVar(metaclass=_InitVarMeta):
+class InitVar:
     __slots__ = ('type', )
 
     def __init__(self, type):
@@ -212,6 +208,9 @@ def __init__(self, type):
     def __repr__(self):
         return f'dataclasses.InitVar[{self.type.__name__}]'
 
+    def __class_getitem__(cls, type):
+        return InitVar(type)
+
 
 # Instances of Field are only ever created from within this module,
 # and only from the field() function, although Field instances are


From webhook-mailer at python.org  Mon Sep 23 08:46:04 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 23 Sep 2019 12:46:04 -0000
Subject: [Python-checkins] bpo-38236: Dump path config at first import error
 (GH-16300)
Message-ID: <mailman.29.1569242765.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/fcdb027234566c4d506d6d753c7d5638490fb088
commit: fcdb027234566c4d506d6d753c7d5638490fb088
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-23T14:45:47+02:00
summary:

bpo-38236: Dump path config at first import error (GH-16300)

Python now dumps path configuration if it fails to import the Python
codecs of the filesystem and stdio encodings.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-20-19-06-23.bpo-38236.eQ0Tmj.rst
M Doc/c-api/init_config.rst
M Include/internal/pycore_pathconfig.h
M Objects/unicodeobject.c
M PC/getpathp.c
M Python/initconfig.c

diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index c198df3fb1d0..bc24fa081317 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -425,6 +425,11 @@ PyConfig
 
       :data:`sys.base_exec_prefix`.
 
+   .. c:member:: wchar_t* base_executable
+
+      :data:`sys._base_executable`: ``__PYVENV_LAUNCHER__`` environment
+      variable value, or copy of :c:member:`PyConfig.executable`.
+
    .. c:member:: wchar_t* base_prefix
 
       :data:`sys.base_prefix`.
@@ -862,11 +867,13 @@ Path Configuration
 * Path configuration input fields:
 
   * :c:member:`PyConfig.home`
-  * :c:member:`PyConfig.pythonpath_env`
   * :c:member:`PyConfig.pathconfig_warnings`
+  * :c:member:`PyConfig.program_name`
+  * :c:member:`PyConfig.pythonpath_env`
 
 * Path configuration output fields:
 
+  * :c:member:`PyConfig.base_executable`
   * :c:member:`PyConfig.exec_prefix`
   * :c:member:`PyConfig.executable`
   * :c:member:`PyConfig.prefix`
@@ -918,6 +925,9 @@ The following configuration files are used by the path configuration:
 * ``python._pth`` (Windows only)
 * ``pybuilddir.txt`` (Unix only)
 
+The ``__PYVENV_LAUNCHER__`` environment variable is used to set
+:c:member:`PyConfig.base_executable`
+
 
 Py_RunMain()
 ------------
diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h
index a2c488cdc2eb..61b3790fe1f4 100644
--- a/Include/internal/pycore_pathconfig.h
+++ b/Include/internal/pycore_pathconfig.h
@@ -60,6 +60,7 @@ extern wchar_t* _Py_GetDLLPath(void);
 #endif
 
 extern PyStatus _PyPathConfig_Init(void);
+extern void _Py_DumpPathConfig(PyThreadState *tstate);
 
 #ifdef __cplusplus
 }
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-20-19-06-23.bpo-38236.eQ0Tmj.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-20-19-06-23.bpo-38236.eQ0Tmj.rst
new file mode 100644
index 000000000000..5c7729296cee
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-20-19-06-23.bpo-38236.eQ0Tmj.rst	
@@ -0,0 +1,2 @@
+Python now dumps path configuration if it fails to import the Python codecs
+of the filesystem and stdio encodings.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 070f795aa3cb..55b251918173 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -15804,13 +15804,13 @@ config_get_codec_name(wchar_t **config_encoding)
 
 
 static PyStatus
-init_stdio_encoding(PyInterpreterState *interp)
+init_stdio_encoding(PyThreadState *tstate)
 {
     /* Update the stdio encoding to the normalized Python codec name. */
-    PyConfig *config = &interp->config;
+    PyConfig *config = &tstate->interp->config;
     if (config_get_codec_name(&config->stdio_encoding) < 0) {
         return _PyStatus_ERR("failed to get the Python codec name "
-                            "of the stdio encoding");
+                             "of the stdio encoding");
     }
     return _PyStatus_OK();
 }
@@ -15864,15 +15864,18 @@ init_fs_codec(PyInterpreterState *interp)
 
 
 static PyStatus
-init_fs_encoding(PyInterpreterState *interp)
+init_fs_encoding(PyThreadState *tstate)
 {
+    PyInterpreterState *interp = tstate->interp;
+
     /* Update the filesystem encoding to the normalized Python codec name.
        For example, replace "ANSI_X3.4-1968" (locale encoding) with "ascii"
        (Python codec name). */
     PyConfig *config = &interp->config;
     if (config_get_codec_name(&config->filesystem_encoding) < 0) {
+        _Py_DumpPathConfig(tstate);
         return _PyStatus_ERR("failed to get the Python codec "
-                            "of the filesystem encoding");
+                             "of the filesystem encoding");
     }
 
     if (init_fs_codec(interp) < 0) {
@@ -15885,14 +15888,12 @@ init_fs_encoding(PyInterpreterState *interp)
 PyStatus
 _PyUnicode_InitEncodings(PyThreadState *tstate)
 {
-    PyInterpreterState *interp = tstate->interp;
-
-    PyStatus status = init_fs_encoding(interp);
+    PyStatus status = init_fs_encoding(tstate);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
-    return init_stdio_encoding(interp);
+    return init_stdio_encoding(tstate);
 }
 
 
diff --git a/PC/getpathp.c b/PC/getpathp.c
index 0ee53080bf31..888bef1146fd 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -532,8 +532,7 @@ _Py_GetDLLPath(void)
 
 
 static PyStatus
-get_program_full_path(const PyConfig *config,
-                      PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+get_program_full_path(_PyPathConfig *pathconfig)
 {
     const wchar_t *pyvenv_launcher;
     wchar_t program_full_path[MAXPATHLEN+1];
@@ -977,7 +976,7 @@ calculate_path_impl(const PyConfig *config,
 {
     PyStatus status;
 
-    status = get_program_full_path(config, calculate, pathconfig);
+    status = get_program_full_path(pathconfig);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
diff --git a/Python/initconfig.c b/Python/initconfig.c
index e67fea7b109c..cda4c863736c 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -1,12 +1,13 @@
 #include "Python.h"
 #include "osdefs.h"       /* DELIM */
-#include "pycore_initconfig.h"
 #include "pycore_fileutils.h"
 #include "pycore_getopt.h"
+#include "pycore_initconfig.h"
+#include "pycore_pathconfig.h"
+#include "pycore_pyerrors.h"
 #include "pycore_pylifecycle.h"
 #include "pycore_pymem.h"
 #include "pycore_pystate.h"   /* _PyRuntime */
-#include "pycore_pathconfig.h"
 #include <locale.h>       /* setlocale() */
 #ifdef HAVE_LANGINFO_H
 #  include <langinfo.h>   /* nl_langinfo(CODESET) */
@@ -2539,3 +2540,97 @@ _Py_GetConfigsAsDict(void)
     Py_XDECREF(dict);
     return NULL;
 }
+
+
+static void
+init_dump_ascii_wstr(const wchar_t *str)
+{
+    if (str == NULL) {
+        PySys_WriteStderr("(not set)");
+    }
+
+    PySys_WriteStderr("'");
+    for (; *str != L'\0'; str++) {
+        wchar_t ch = *str;
+        if (ch == L'\'') {
+            PySys_WriteStderr("\\'");
+        } else if (0x20 <= ch && ch < 0x7f) {
+            PySys_WriteStderr("%lc", ch);
+        }
+        else if (ch <= 0xff) {
+            PySys_WriteStderr("\\x%02x", ch);
+        }
+#if SIZEOF_WCHAR_T > 2
+        else if (ch > 0xffff) {
+            PySys_WriteStderr("\\U%08x", ch);
+        }
+#endif
+        else {
+            PySys_WriteStderr("\\u%04x", ch);
+        }
+    }
+    PySys_WriteStderr("'");
+}
+
+
+/* Dump the Python path configuration into sys.stderr */
+void
+_Py_DumpPathConfig(PyThreadState *tstate)
+{
+    PyObject *exc_type, *exc_value, *exc_tb;
+    _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb);
+
+    PySys_WriteStderr("Python path configuration:\n");
+
+#define DUMP_CONFIG(NAME, FIELD) \
+        do { \
+            PySys_WriteStderr("  " NAME " = "); \
+            init_dump_ascii_wstr(config->FIELD); \
+            PySys_WriteStderr("\n"); \
+        } while (0)
+
+    PyConfig *config = &tstate->interp->config;
+    DUMP_CONFIG("PYTHONHOME", home);
+    DUMP_CONFIG("PYTHONPATH", pythonpath_env);
+    DUMP_CONFIG("program name", program_name);
+    PySys_WriteStderr("  isolated = %i\n", config->isolated);
+    PySys_WriteStderr("  environment = %i\n", config->use_environment);
+    PySys_WriteStderr("  user site = %i\n", config->user_site_directory);
+    PySys_WriteStderr("  import site = %i\n", config->site_import);
+#undef DUMP_CONFIG
+
+#define DUMP_SYS(NAME) \
+        do { \
+            obj = PySys_GetObject(#NAME); \
+            PySys_FormatStderr("  sys.%s = ", #NAME); \
+            if (obj != NULL) { \
+                PySys_FormatStderr("%A", obj); \
+            } \
+            else { \
+                PySys_WriteStderr("(not set)"); \
+            } \
+            PySys_FormatStderr("\n"); \
+        } while (0)
+
+    PyObject *obj;
+    DUMP_SYS(_base_executable);
+    DUMP_SYS(base_prefix);
+    DUMP_SYS(base_exec_prefix);
+    DUMP_SYS(executable);
+    DUMP_SYS(prefix);
+    DUMP_SYS(exec_prefix);
+#undef DUMP_SYS
+
+    PyObject *sys_path = PySys_GetObject("path");  /* borrowed reference */
+    if (sys_path != NULL && PyList_Check(sys_path)) {
+        PySys_WriteStderr("  sys.path = [\n");
+        Py_ssize_t len = PyList_GET_SIZE(sys_path);
+        for (Py_ssize_t i=0; i < len; i++) {
+            PyObject *path = PyList_GET_ITEM(sys_path, i);
+            PySys_FormatStderr("    %A,\n", path);
+        }
+        PySys_WriteStderr("  ]\n");
+    }
+
+    _PyErr_Restore(tstate, exc_type, exc_value, exc_tb);
+}


From webhook-mailer at python.org  Mon Sep 23 09:36:00 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 23 Sep 2019 13:36:00 -0000
Subject: [Python-checkins] bpo-38236: Fix init_dump_ascii_wstr() (GH-16333)
Message-ID: <mailman.30.1569245761.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/88e6447451fb5525e83e802c66c3e51b4a45bf86
commit: 88e6447451fb5525e83e802c66c3e51b4a45bf86
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-23T15:35:46+02:00
summary:

bpo-38236: Fix init_dump_ascii_wstr() (GH-16333)

Add missing "return;" (to not dereference NULL pointer).

files:
M Python/initconfig.c

diff --git a/Python/initconfig.c b/Python/initconfig.c
index cda4c863736c..9f04e3d183ab 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -2547,6 +2547,7 @@ init_dump_ascii_wstr(const wchar_t *str)
 {
     if (str == NULL) {
         PySys_WriteStderr("(not set)");
+        return;
     }
 
     PySys_WriteStderr("'");


From webhook-mailer at python.org  Mon Sep 23 09:59:09 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 23 Sep 2019 13:59:09 -0000
Subject: [Python-checkins] bpo-38236: Dump path config at first import error
 (GH-16300) (GH-16332)
Message-ID: <mailman.31.1569247149.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/c5c642565e260477ae2fb29d0c86a91e19702ae3
commit: c5c642565e260477ae2fb29d0c86a91e19702ae3
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-23T15:59:00+02:00
summary:

bpo-38236: Dump path config at first import error (GH-16300) (GH-16332)

Python now dumps path configuration if it fails to import the Python
codecs of the filesystem and stdio encodings.

(cherry picked from commit fcdb027234566c4d506d6d753c7d5638490fb088)

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-20-19-06-23.bpo-38236.eQ0Tmj.rst
M Doc/c-api/init_config.rst
M Include/internal/pycore_pathconfig.h
M Include/internal/pycore_pylifecycle.h
M Objects/unicodeobject.c
M PC/getpathp.c
M Python/initconfig.c
M Python/pylifecycle.c

diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index c198df3fb1d0..bc24fa081317 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -425,6 +425,11 @@ PyConfig
 
       :data:`sys.base_exec_prefix`.
 
+   .. c:member:: wchar_t* base_executable
+
+      :data:`sys._base_executable`: ``__PYVENV_LAUNCHER__`` environment
+      variable value, or copy of :c:member:`PyConfig.executable`.
+
    .. c:member:: wchar_t* base_prefix
 
       :data:`sys.base_prefix`.
@@ -862,11 +867,13 @@ Path Configuration
 * Path configuration input fields:
 
   * :c:member:`PyConfig.home`
-  * :c:member:`PyConfig.pythonpath_env`
   * :c:member:`PyConfig.pathconfig_warnings`
+  * :c:member:`PyConfig.program_name`
+  * :c:member:`PyConfig.pythonpath_env`
 
 * Path configuration output fields:
 
+  * :c:member:`PyConfig.base_executable`
   * :c:member:`PyConfig.exec_prefix`
   * :c:member:`PyConfig.executable`
   * :c:member:`PyConfig.prefix`
@@ -918,6 +925,9 @@ The following configuration files are used by the path configuration:
 * ``python._pth`` (Windows only)
 * ``pybuilddir.txt`` (Unix only)
 
+The ``__PYVENV_LAUNCHER__`` environment variable is used to set
+:c:member:`PyConfig.base_executable`
+
 
 Py_RunMain()
 ------------
diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h
index a2c488cdc2eb..61b3790fe1f4 100644
--- a/Include/internal/pycore_pathconfig.h
+++ b/Include/internal/pycore_pathconfig.h
@@ -60,6 +60,7 @@ extern wchar_t* _Py_GetDLLPath(void);
 #endif
 
 extern PyStatus _PyPathConfig_Init(void);
+extern void _Py_DumpPathConfig(PyThreadState *tstate);
 
 #ifdef __cplusplus
 }
diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index 852280e84027..cb568d28fb99 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -19,7 +19,7 @@ extern int _Py_SetFileSystemEncoding(
     const char *encoding,
     const char *errors);
 extern void _Py_ClearFileSystemEncoding(void);
-extern PyStatus _PyUnicode_InitEncodings(PyInterpreterState *interp);
+extern PyStatus _PyUnicode_InitEncodings(PyThreadState *tstate);
 #ifdef MS_WINDOWS
 extern int _PyUnicode_EnableLegacyWindowsFSEncoding(void);
 #endif
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-20-19-06-23.bpo-38236.eQ0Tmj.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-20-19-06-23.bpo-38236.eQ0Tmj.rst
new file mode 100644
index 000000000000..5c7729296cee
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-20-19-06-23.bpo-38236.eQ0Tmj.rst	
@@ -0,0 +1,2 @@
+Python now dumps path configuration if it fails to import the Python codecs
+of the filesystem and stdio encodings.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 2afc28d8435e..1466e8a6e72b 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -15727,13 +15727,13 @@ config_get_codec_name(wchar_t **config_encoding)
 
 
 static PyStatus
-init_stdio_encoding(PyInterpreterState *interp)
+init_stdio_encoding(PyThreadState *tstate)
 {
     /* Update the stdio encoding to the normalized Python codec name. */
-    PyConfig *config = &interp->config;
+    PyConfig *config = &tstate->interp->config;
     if (config_get_codec_name(&config->stdio_encoding) < 0) {
         return _PyStatus_ERR("failed to get the Python codec name "
-                            "of the stdio encoding");
+                             "of the stdio encoding");
     }
     return _PyStatus_OK();
 }
@@ -15787,15 +15787,18 @@ init_fs_codec(PyInterpreterState *interp)
 
 
 static PyStatus
-init_fs_encoding(PyInterpreterState *interp)
+init_fs_encoding(PyThreadState *tstate)
 {
+    PyInterpreterState *interp = tstate->interp;
+
     /* Update the filesystem encoding to the normalized Python codec name.
        For example, replace "ANSI_X3.4-1968" (locale encoding) with "ascii"
        (Python codec name). */
     PyConfig *config = &interp->config;
     if (config_get_codec_name(&config->filesystem_encoding) < 0) {
+        _Py_DumpPathConfig(tstate);
         return _PyStatus_ERR("failed to get the Python codec "
-                            "of the filesystem encoding");
+                             "of the filesystem encoding");
     }
 
     if (init_fs_codec(interp) < 0) {
@@ -15806,14 +15809,14 @@ init_fs_encoding(PyInterpreterState *interp)
 
 
 PyStatus
-_PyUnicode_InitEncodings(PyInterpreterState *interp)
+_PyUnicode_InitEncodings(PyThreadState *tstate)
 {
-    PyStatus status = init_fs_encoding(interp);
+    PyStatus status = init_fs_encoding(tstate);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
-    return init_stdio_encoding(interp);
+    return init_stdio_encoding(tstate);
 }
 
 
diff --git a/PC/getpathp.c b/PC/getpathp.c
index 0ee53080bf31..888bef1146fd 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -532,8 +532,7 @@ _Py_GetDLLPath(void)
 
 
 static PyStatus
-get_program_full_path(const PyConfig *config,
-                      PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+get_program_full_path(_PyPathConfig *pathconfig)
 {
     const wchar_t *pyvenv_launcher;
     wchar_t program_full_path[MAXPATHLEN+1];
@@ -977,7 +976,7 @@ calculate_path_impl(const PyConfig *config,
 {
     PyStatus status;
 
-    status = get_program_full_path(config, calculate, pathconfig);
+    status = get_program_full_path(pathconfig);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
diff --git a/Python/initconfig.c b/Python/initconfig.c
index 49659045f7ab..a527271452ea 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -1,12 +1,13 @@
 #include "Python.h"
 #include "osdefs.h"       /* DELIM */
-#include "pycore_initconfig.h"
 #include "pycore_fileutils.h"
 #include "pycore_getopt.h"
+#include "pycore_initconfig.h"
+#include "pycore_pathconfig.h"
+#include "pycore_pyerrors.h"
 #include "pycore_pylifecycle.h"
 #include "pycore_pymem.h"
 #include "pycore_pystate.h"   /* _PyRuntime */
-#include "pycore_pathconfig.h"
 #include <locale.h>       /* setlocale() */
 #ifdef HAVE_LANGINFO_H
 #  include <langinfo.h>   /* nl_langinfo(CODESET) */
@@ -2492,3 +2493,98 @@ _Py_GetConfigsAsDict(void)
     Py_XDECREF(dict);
     return NULL;
 }
+
+
+static void
+init_dump_ascii_wstr(const wchar_t *str)
+{
+    if (str == NULL) {
+        PySys_WriteStderr("(not set)");
+        return;
+    }
+
+    PySys_WriteStderr("'");
+    for (; *str != L'\0'; str++) {
+        wchar_t ch = *str;
+        if (ch == L'\'') {
+            PySys_WriteStderr("\\'");
+        } else if (0x20 <= ch && ch < 0x7f) {
+            PySys_WriteStderr("%lc", ch);
+        }
+        else if (ch <= 0xff) {
+            PySys_WriteStderr("\\x%02x", ch);
+        }
+#if SIZEOF_WCHAR_T > 2
+        else if (ch > 0xffff) {
+            PySys_WriteStderr("\\U%08x", ch);
+        }
+#endif
+        else {
+            PySys_WriteStderr("\\u%04x", ch);
+        }
+    }
+    PySys_WriteStderr("'");
+}
+
+
+/* Dump the Python path configuration into sys.stderr */
+void
+_Py_DumpPathConfig(PyThreadState *tstate)
+{
+    PyObject *exc_type, *exc_value, *exc_tb;
+    _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb);
+
+    PySys_WriteStderr("Python path configuration:\n");
+
+#define DUMP_CONFIG(NAME, FIELD) \
+        do { \
+            PySys_WriteStderr("  " NAME " = "); \
+            init_dump_ascii_wstr(config->FIELD); \
+            PySys_WriteStderr("\n"); \
+        } while (0)
+
+    PyConfig *config = &tstate->interp->config;
+    DUMP_CONFIG("PYTHONHOME", home);
+    DUMP_CONFIG("PYTHONPATH", pythonpath_env);
+    DUMP_CONFIG("program name", program_name);
+    PySys_WriteStderr("  isolated = %i\n", config->isolated);
+    PySys_WriteStderr("  environment = %i\n", config->use_environment);
+    PySys_WriteStderr("  user site = %i\n", config->user_site_directory);
+    PySys_WriteStderr("  import site = %i\n", config->site_import);
+#undef DUMP_CONFIG
+
+#define DUMP_SYS(NAME) \
+        do { \
+            obj = PySys_GetObject(#NAME); \
+            PySys_FormatStderr("  sys.%s = ", #NAME); \
+            if (obj != NULL) { \
+                PySys_FormatStderr("%A", obj); \
+            } \
+            else { \
+                PySys_WriteStderr("(not set)"); \
+            } \
+            PySys_FormatStderr("\n"); \
+        } while (0)
+
+    PyObject *obj;
+    DUMP_SYS(_base_executable);
+    DUMP_SYS(base_prefix);
+    DUMP_SYS(base_exec_prefix);
+    DUMP_SYS(executable);
+    DUMP_SYS(prefix);
+    DUMP_SYS(exec_prefix);
+#undef DUMP_SYS
+
+    PyObject *sys_path = PySys_GetObject("path");  /* borrowed reference */
+    if (sys_path != NULL && PyList_Check(sys_path)) {
+        PySys_WriteStderr("  sys.path = [\n");
+        Py_ssize_t len = PyList_GET_SIZE(sys_path);
+        for (Py_ssize_t i=0; i < len; i++) {
+            PyObject *path = PyList_GET_ITEM(sys_path, i);
+            PySys_FormatStderr("    %A,\n", path);
+        }
+        PySys_WriteStderr("  ]\n");
+    }
+
+    _PyErr_Restore(tstate, exc_type, exc_value, exc_tb);
+}
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index a9a6589d7a43..4d7873fd5224 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -941,7 +941,8 @@ pyinit_main(_PyRuntimeState *runtime, PyInterpreterState *interp)
         return status;
     }
 
-    status = _PyUnicode_InitEncodings(interp);
+    PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
+    status = _PyUnicode_InitEncodings(tstate);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -1508,7 +1509,7 @@ new_interpreter(PyThreadState **tstate_p)
             return status;
         }
 
-        status = _PyUnicode_InitEncodings(interp);
+        status = _PyUnicode_InitEncodings(tstate);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }


From webhook-mailer at python.org  Mon Sep 23 12:47:41 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 23 Sep 2019 16:47:41 -0000
Subject: [Python-checkins] bpo-38234: Fix _PyConfig_InitPathConfig()
 (GH-16335)
Message-ID: <mailman.32.1569257262.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/9c42f8cda552694f3b47d6388d4ae84d61731872
commit: 9c42f8cda552694f3b47d6388d4ae84d61731872
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-23T18:47:29+02:00
summary:

bpo-38234: Fix _PyConfig_InitPathConfig() (GH-16335)

* _PyConfig_InitPathConfig() now starts by copying the global path
  configuration, and then override values set in PyConfig.
* _PyPathConfig_Calculate() implementations no longer override
  _PyPathConfig fields which are already computed. For example,
  if _PyPathConfig.prefix is not NULL, leave it unchanged.
* If Py_SetPath() has been called, _PyConfig_InitPathConfig() doesn't
  call _PyPathConfig_Calculate() anymore.
* _PyPathConfig_Calculate() no longer uses PyConfig,
  except to initialize PyCalculatePath structure.
* pathconfig_calculate(): remove useless temporary
  "_PyPathConfig new_config" variable.
* calculate_module_search_path(): remove hack to workaround memory
  allocation failure, call Py_FatalError() instead.
* Fix get_program_full_path(): handle memory allocation failure.

files:
M Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst
M Modules/getpath.c
M PC/getpathp.c
M Python/pathconfig.c

diff --git a/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst b/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst
index 6310f19f004e..fe658ad75d89 100644
--- a/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst	
+++ b/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst	
@@ -1,2 +1,3 @@
-Python ignored path passed to :c:func:`Py_SetPath`, fix Python
-initialization to use the specified path.
+Python ignored arguments passed to :c:func:`Py_SetPath`,
+:c:func:`Py_SetPythonHome` and :c:func:`Py_SetProgramName`: fix Python
+initialization to use specified arguments.
diff --git a/Modules/getpath.c b/Modules/getpath.c
index 3946623862cf..270355e251e1 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -133,6 +133,9 @@ typedef struct {
 
     int prefix_found;         /* found platform independent libraries? */
     int exec_prefix_found;    /* found the platform dependent libraries? */
+
+    int warnings;
+    const wchar_t *pythonpath_env;
 } PyCalculatePath;
 
 static const wchar_t delimiter[2] = {DELIM, '\0'};
@@ -365,17 +368,16 @@ add_exe_suffix(wchar_t *progpath, size_t progpathlen)
    bytes long.
 */
 static PyStatus
-search_for_prefix(const PyConfig *config, PyCalculatePath *calculate,
-                  wchar_t *prefix, size_t prefix_len,
-                  int *found)
+search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                  wchar_t *prefix, size_t prefix_len, int *found)
 {
     PyStatus status;
     size_t n;
     wchar_t *vpath;
 
     /* If PYTHONHOME is set, we believe it unconditionally */
-    if (config->home) {
-        if (safe_wcscpy(prefix, config->home, prefix_len) < 0) {
+    if (pathconfig->home) {
+        if (safe_wcscpy(prefix, pathconfig->home, prefix_len) < 0) {
             return PATHLEN_ERR();
         }
         wchar_t *delim = wcschr(prefix, DELIM);
@@ -482,19 +484,19 @@ search_for_prefix(const PyConfig *config, PyCalculatePath *calculate,
 
 
 static PyStatus
-calculate_prefix(const PyConfig *config,
-                 PyCalculatePath *calculate, wchar_t *prefix, size_t prefix_len)
+calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                 wchar_t *prefix, size_t prefix_len)
 {
     PyStatus status;
 
-    status = search_for_prefix(config, calculate, prefix, prefix_len,
-                            &calculate->prefix_found);
+    status = search_for_prefix(calculate, pathconfig, prefix, prefix_len,
+                               &calculate->prefix_found);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     if (!calculate->prefix_found) {
-        if (config->pathconfig_warnings) {
+        if (calculate->warnings) {
             fprintf(stderr,
                 "Could not find platform independent libraries <prefix>\n");
         }
@@ -544,8 +546,7 @@ calculate_reduce_prefix(PyCalculatePath *calculate,
    MAXPATHLEN bytes long.
 */
 static PyStatus
-search_for_exec_prefix(const PyConfig *config,
-                       PyCalculatePath *calculate,
+search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
                        wchar_t *exec_prefix, size_t exec_prefix_len,
                        int *found)
 {
@@ -553,15 +554,15 @@ search_for_exec_prefix(const PyConfig *config,
     size_t n;
 
     /* If PYTHONHOME is set, we believe it unconditionally */
-    if (config->home) {
-        wchar_t *delim = wcschr(config->home, DELIM);
+    if (pathconfig->home) {
+        wchar_t *delim = wcschr(pathconfig->home, DELIM);
         if (delim) {
             if (safe_wcscpy(exec_prefix, delim+1, exec_prefix_len) < 0) {
                 return PATHLEN_ERR();
             }
         }
         else {
-            if (safe_wcscpy(exec_prefix, config->home, exec_prefix_len) < 0) {
+            if (safe_wcscpy(exec_prefix, pathconfig->home, exec_prefix_len) < 0) {
                 return PATHLEN_ERR();
             }
         }
@@ -668,21 +669,20 @@ search_for_exec_prefix(const PyConfig *config,
 
 
 static PyStatus
-calculate_exec_prefix(const PyConfig *config,
-                      PyCalculatePath *calculate,
+calculate_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
                       wchar_t *exec_prefix, size_t exec_prefix_len)
 {
     PyStatus status;
 
-    status = search_for_exec_prefix(config, calculate,
-                                 exec_prefix, exec_prefix_len,
-                                 &calculate->exec_prefix_found);
+    status = search_for_exec_prefix(calculate, pathconfig,
+                                    exec_prefix, exec_prefix_len,
+                                    &calculate->exec_prefix_found);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     if (!calculate->exec_prefix_found) {
-        if (config->pathconfig_warnings) {
+        if (calculate->warnings) {
             fprintf(stderr,
                 "Could not find platform dependent libraries <exec_prefix>\n");
         }
@@ -721,8 +721,7 @@ calculate_reduce_exec_prefix(PyCalculatePath *calculate,
 
 
 static PyStatus
-calculate_program_full_path(const PyConfig *config,
-                            PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+calculate_program_full_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
 {
     PyStatus status;
     wchar_t program_full_path[MAXPATHLEN + 1];
@@ -743,8 +742,8 @@ calculate_program_full_path(const PyConfig *config,
      * other way to find a directory to start the search from.  If
      * $PATH isn't exported, you lose.
      */
-    if (wcschr(config->program_name, SEP)) {
-        if (safe_wcscpy(program_full_path, config->program_name,
+    if (wcschr(pathconfig->program_name, SEP)) {
+        if (safe_wcscpy(program_full_path, pathconfig->program_name,
                         program_full_path_len) < 0) {
             return PATHLEN_ERR();
         }
@@ -795,8 +794,8 @@ calculate_program_full_path(const PyConfig *config,
                 }
             }
 
-            status = joinpath(program_full_path, config->program_name,
-                            program_full_path_len);
+            status = joinpath(program_full_path, pathconfig->program_name,
+                              program_full_path_len);
             if (_PyStatus_EXCEPTION(status)) {
                 return status;
             }
@@ -1030,15 +1029,14 @@ calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix)
 
 
 static PyStatus
-calculate_module_search_path(const PyConfig *config,
-                             PyCalculatePath *calculate,
-                             const wchar_t *prefix, const wchar_t *exec_prefix,
-                             _PyPathConfig *pathconfig)
+calculate_module_search_path(PyCalculatePath *calculate,
+                             _PyPathConfig *pathconfig,
+                             const wchar_t *prefix, const wchar_t *exec_prefix)
 {
     /* Calculate size of return buffer */
     size_t bufsz = 0;
-    if (config->pythonpath_env != NULL) {
-        bufsz += wcslen(config->pythonpath_env) + 1;
+    if (calculate->pythonpath_env != NULL) {
+        bufsz += wcslen(calculate->pythonpath_env) + 1;
     }
 
     wchar_t *defpath = calculate->pythonpath;
@@ -1072,8 +1070,8 @@ calculate_module_search_path(const PyConfig *config,
     buf[0] = '\0';
 
     /* Run-time value of $PYTHONPATH goes first */
-    if (config->pythonpath_env) {
-        wcscpy(buf, config->pythonpath_env);
+    if (calculate->pythonpath_env) {
+        wcscpy(buf, calculate->pythonpath_env);
         wcscat(buf, delimiter);
     }
 
@@ -1149,6 +1147,10 @@ calculate_init(PyCalculatePath *calculate,
     if (!calculate->lib_python) {
         return DECODE_LOCALE_ERR("EXEC_PREFIX define", len);
     }
+
+    calculate->warnings = config->pathconfig_warnings;
+    calculate->pythonpath_env = config->pythonpath_env;
+
     return _PyStatus_OK();
 }
 
@@ -1165,14 +1167,15 @@ calculate_free(PyCalculatePath *calculate)
 
 
 static PyStatus
-calculate_path_impl(const PyConfig *config,
-                    PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
 {
     PyStatus status;
 
-    status = calculate_program_full_path(config, calculate, pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+    if (pathconfig->program_full_path == NULL) {
+        status = calculate_program_full_path(calculate, pathconfig);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
     }
 
     status = calculate_argv0_path(calculate, pathconfig->program_full_path);
@@ -1187,8 +1190,8 @@ calculate_path_impl(const PyConfig *config,
 
     wchar_t prefix[MAXPATHLEN+1];
     memset(prefix, 0, sizeof(prefix));
-    status = calculate_prefix(config, calculate,
-                           prefix, Py_ARRAY_LENGTH(prefix));
+    status = calculate_prefix(calculate, pathconfig,
+                              prefix, Py_ARRAY_LENGTH(prefix));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -1200,52 +1203,65 @@ calculate_path_impl(const PyConfig *config,
 
     wchar_t exec_prefix[MAXPATHLEN+1];
     memset(exec_prefix, 0, sizeof(exec_prefix));
-    status = calculate_exec_prefix(config, calculate,
-                                exec_prefix, Py_ARRAY_LENGTH(exec_prefix));
+    status = calculate_exec_prefix(calculate, pathconfig,
+                                   exec_prefix, Py_ARRAY_LENGTH(exec_prefix));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     if ((!calculate->prefix_found || !calculate->exec_prefix_found) &&
-        config->pathconfig_warnings)
+        calculate->warnings)
     {
         fprintf(stderr,
                 "Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]\n");
     }
 
     if (pathconfig->module_search_path == NULL) {
-        status = calculate_module_search_path(config, calculate,
-                                              prefix, exec_prefix, pathconfig);
+        status = calculate_module_search_path(calculate, pathconfig,
+                                              prefix, exec_prefix);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
     }
 
-    status = calculate_reduce_prefix(calculate, prefix, Py_ARRAY_LENGTH(prefix));
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-
-    pathconfig->prefix = _PyMem_RawWcsdup(prefix);
     if (pathconfig->prefix == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
+        status = calculate_reduce_prefix(calculate, prefix, Py_ARRAY_LENGTH(prefix));
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
 
-    status = calculate_reduce_exec_prefix(calculate,
-                                       exec_prefix, Py_ARRAY_LENGTH(exec_prefix));
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+        pathconfig->prefix = _PyMem_RawWcsdup(prefix);
+        if (pathconfig->prefix == NULL) {
+            return _PyStatus_NO_MEMORY();
+        }
     }
 
-    pathconfig->exec_prefix = _PyMem_RawWcsdup(exec_prefix);
     if (pathconfig->exec_prefix == NULL) {
-        return _PyStatus_NO_MEMORY();
+        status = calculate_reduce_exec_prefix(calculate,
+                                              exec_prefix,
+                                              Py_ARRAY_LENGTH(exec_prefix));
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
+
+        pathconfig->exec_prefix = _PyMem_RawWcsdup(exec_prefix);
+        if (pathconfig->exec_prefix == NULL) {
+            return _PyStatus_NO_MEMORY();
+        }
     }
 
     return _PyStatus_OK();
 }
 
 
+/* Calculate 'pathconfig' attributes:
+
+   - program_full_path
+   - module_search_path
+   - prefix
+   - exec_prefix
+
+   If an attribute is already set (non NULL), it is left unchanged. */
 PyStatus
 _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
 {
@@ -1258,7 +1274,7 @@ _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
         goto done;
     }
 
-    status = calculate_path_impl(config, &calculate, pathconfig);
+    status = calculate_path(&calculate, pathconfig);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
     }
diff --git a/PC/getpathp.c b/PC/getpathp.c
index 888bef1146fd..5670fa2ee4e1 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -130,6 +130,8 @@ typedef struct {
     wchar_t zip_path[MAXPATHLEN+1];
 
     wchar_t *dll_path;
+
+    const wchar_t *pythonpath_env;
 } PyCalculatePath;
 
 
@@ -322,7 +324,7 @@ gotlandmark(wchar_t *prefix, const wchar_t *landmark)
 
 
 /* assumes argv0_path is MAXPATHLEN+1 bytes long, already \0 term'd.
-   assumption provided by only caller, calculate_path_impl() */
+   assumption provided by only caller, calculate_path() */
 static int
 search_for_prefix(wchar_t *prefix, const wchar_t *argv0_path, const wchar_t *landmark)
 {
@@ -534,6 +536,7 @@ _Py_GetDLLPath(void)
 static PyStatus
 get_program_full_path(_PyPathConfig *pathconfig)
 {
+    PyStatus status;
     const wchar_t *pyvenv_launcher;
     wchar_t program_full_path[MAXPATHLEN+1];
     memset(program_full_path, 0, sizeof(program_full_path));
@@ -548,12 +551,18 @@ get_program_full_path(_PyPathConfig *pathconfig)
     pyvenv_launcher = _wgetenv(L"__PYVENV_LAUNCHER__");
     if (pyvenv_launcher && pyvenv_launcher[0]) {
         /* If overridden, preserve the original full path */
-        pathconfig->base_executable = PyMem_RawMalloc(
-            sizeof(wchar_t) * (MAXPATHLEN + 1));
-        PyStatus status = canonicalize(pathconfig->base_executable,
-                                       program_full_path);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
+        if (pathconfig->base_executable == NULL) {
+            pathconfig->base_executable = PyMem_RawMalloc(
+                sizeof(wchar_t) * (MAXPATHLEN + 1));
+            if (pathconfig->base_executable == NULL) {
+                return _PyStatus_NO_MEMORY();
+            }
+
+            status = canonicalize(pathconfig->base_executable,
+                                  program_full_path);
+            if (_PyStatus_EXCEPTION(status)) {
+                return status;
+            }
         }
 
         wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher);
@@ -562,11 +571,20 @@ get_program_full_path(_PyPathConfig *pathconfig)
         _wputenv_s(L"__PYVENV_LAUNCHER__", L"");
     }
 
-    pathconfig->program_full_path = PyMem_RawMalloc(
-        sizeof(wchar_t) * (MAXPATHLEN + 1));
+    if (pathconfig->program_full_path == NULL) {
+        pathconfig->program_full_path = PyMem_RawMalloc(
+            sizeof(wchar_t) * (MAXPATHLEN + 1));
+        if (pathconfig->program_full_path == NULL) {
+            return _PyStatus_NO_MEMORY();
+        }
 
-    return canonicalize(pathconfig->program_full_path,
-                        program_full_path);
+        status = canonicalize(pathconfig->program_full_path,
+                              program_full_path);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
+    }
+    return _PyStatus_OK();
 }
 
 
@@ -657,7 +675,13 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
     }
 
     fclose(sp_file);
-    pathconfig->module_search_path = buf;
+    if (pathconfig->module_search_path == NULL) {
+        pathconfig->module_search_path = _PyMem_RawWcsdup(buf);
+        if (pathconfig->module_search_path == NULL) {
+            Py_FatalError("out of memory");
+        }
+    }
+    PyMem_RawFree(buf);
     return 1;
 
 error:
@@ -668,8 +692,7 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
 
 
 static PyStatus
-calculate_init(PyCalculatePath *calculate,
-               const PyConfig *config)
+calculate_init(PyCalculatePath *calculate, const PyConfig *config)
 {
     calculate->home = config->home;
     calculate->path_env = _wgetenv(L"PATH");
@@ -679,6 +702,8 @@ calculate_init(PyCalculatePath *calculate,
         return _PyStatus_NO_MEMORY();
     }
 
+    calculate->pythonpath_env = config->pythonpath_env;
+
     return _PyStatus_OK();
 }
 
@@ -785,8 +810,8 @@ calculate_home_prefix(PyCalculatePath *calculate, wchar_t *prefix)
 
 
 static PyStatus
-calculate_module_search_path(const PyConfig *config,
-                             PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+calculate_module_search_path(PyCalculatePath *calculate,
+                             _PyPathConfig *pathconfig,
                              wchar_t *prefix)
 {
     int skiphome = calculate->home==NULL ? 0 : 1;
@@ -796,7 +821,7 @@ calculate_module_search_path(const PyConfig *config,
 #endif
     /* We only use the default relative PYTHONPATH if we haven't
        anything better to use! */
-    int skipdefault = (config->pythonpath_env != NULL ||
+    int skipdefault = (calculate->pythonpath_env != NULL ||
                        calculate->home != NULL ||
                        calculate->machine_path != NULL ||
                        calculate->user_path != NULL);
@@ -835,30 +860,20 @@ calculate_module_search_path(const PyConfig *config,
         bufsz += wcslen(calculate->machine_path) + 1;
     }
     bufsz += wcslen(calculate->zip_path) + 1;
-    if (config->pythonpath_env != NULL) {
-        bufsz += wcslen(config->pythonpath_env) + 1;
+    if (calculate->pythonpath_env != NULL) {
+        bufsz += wcslen(calculate->pythonpath_env) + 1;
     }
 
     wchar_t *buf, *start_buf;
     buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t));
     if (buf == NULL) {
-        /* We can't exit, so print a warning and limp along */
-        fprintf(stderr, "Can't malloc dynamic PYTHONPATH.\n");
-        if (config->pythonpath_env) {
-            fprintf(stderr, "Using environment $PYTHONPATH.\n");
-            pathconfig->module_search_path = config->pythonpath_env;
-        }
-        else {
-            fprintf(stderr, "Using default static path.\n");
-            pathconfig->module_search_path = PYTHONPATH;
-        }
-        return _PyStatus_OK();
+        Py_FatalError("Can't malloc dynamic PYTHONPATH");
     }
     start_buf = buf;
 
-    if (config->pythonpath_env) {
+    if (calculate->pythonpath_env) {
         if (wcscpy_s(buf, bufsz - (buf - start_buf),
-                     config->pythonpath_env)) {
+                     calculate->pythonpath_env)) {
             return INIT_ERR_BUFFER_OVERFLOW();
         }
         buf = wcschr(buf, L'\0');
@@ -971,8 +986,7 @@ calculate_module_search_path(const PyConfig *config,
 
 
 static PyStatus
-calculate_path_impl(const PyConfig *config,
-                    PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
 {
     PyStatus status;
 
@@ -1003,21 +1017,24 @@ calculate_path_impl(const PyConfig *config,
     calculate_home_prefix(calculate, prefix);
 
     if (pathconfig->module_search_path == NULL) {
-        status = calculate_module_search_path(config, calculate,
-                                              pathconfig, prefix);
+        status = calculate_module_search_path(calculate, pathconfig, prefix);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
     }
 
 done:
-    pathconfig->prefix = _PyMem_RawWcsdup(prefix);
     if (pathconfig->prefix == NULL) {
-        return _PyStatus_NO_MEMORY();
+        pathconfig->prefix = _PyMem_RawWcsdup(prefix);
+        if (pathconfig->prefix == NULL) {
+            return _PyStatus_NO_MEMORY();
+        }
     }
-    pathconfig->exec_prefix = _PyMem_RawWcsdup(prefix);
     if (pathconfig->exec_prefix == NULL) {
-        return _PyStatus_NO_MEMORY();
+        pathconfig->exec_prefix = _PyMem_RawWcsdup(prefix);
+        if (pathconfig->exec_prefix == NULL) {
+            return _PyStatus_NO_MEMORY();
+        }
     }
 
     return _PyStatus_OK();
@@ -1033,6 +1050,17 @@ calculate_free(PyCalculatePath *calculate)
 }
 
 
+/* Calculate 'pathconfig' attributes:
+
+   - base_executable
+   - program_full_path
+   - module_search_path
+   - prefix
+   - exec_prefix
+   - isolated
+   - site_import
+
+   If an attribute is already set (non NULL), it is left unchanged. */
 PyStatus
 _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
 {
@@ -1045,7 +1073,7 @@ _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
         goto done;
     }
 
-    status = calculate_path_impl(config, &calculate, pathconfig);
+    status = calculate_path(&calculate, pathconfig);
 
 done:
     calculate_free(&calculate);
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index 2d00f4b6b125..7677a155797e 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -51,12 +51,12 @@ pathconfig_clear(_PyPathConfig *config)
         ATTR = NULL; \
     } while (0)
 
-    CLEAR(config->prefix);
     CLEAR(config->program_full_path);
+    CLEAR(config->prefix);
     CLEAR(config->exec_prefix);
     CLEAR(config->module_search_path);
-    CLEAR(config->home);
     CLEAR(config->program_name);
+    CLEAR(config->home);
     CLEAR(config->base_executable);
 #undef CLEAR
 
@@ -64,60 +64,31 @@ pathconfig_clear(_PyPathConfig *config)
 }
 
 
-/* Calculate the path configuration: initialize pathconfig from config */
 static PyStatus
-pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config)
+pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2)
 {
-    PyStatus status;
-    _PyPathConfig new_config = _PyPathConfig_INIT;
-
-    PyMemAllocatorEx old_alloc;
-    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+    pathconfig_clear(config);
 
-    if (copy_wstr(&new_config.module_search_path,
-                  _Py_path_config.module_search_path) < 0)
-    {
-        status = _PyStatus_NO_MEMORY();
-        goto error;
-    }
-
-    /* Calculate program_full_path, prefix, exec_prefix,
-       dll_path (Windows), and module_search_path */
-    status = _PyPathConfig_Calculate(&new_config, config);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto error;
-    }
-
-    /* Copy home and program_name from config */
-    if (copy_wstr(&new_config.home, config->home) < 0) {
-        status = _PyStatus_NO_MEMORY();
-        goto error;
-    }
-    if (copy_wstr(&new_config.program_name, config->program_name) < 0) {
-        status = _PyStatus_NO_MEMORY();
-        goto error;
-    }
-    if (config->base_executable) {
-        PyMem_RawFree(new_config.base_executable);
-        if (copy_wstr(&new_config.base_executable,
-                      config->base_executable) < 0) {
-            status = _PyStatus_NO_MEMORY();
-            goto error;
-        }
-    }
-
-    pathconfig_clear(pathconfig);
-    *pathconfig = new_config;
+#define COPY_ATTR(ATTR) \
+    do { \
+        if (copy_wstr(&config->ATTR, config2->ATTR) < 0) { \
+            return _PyStatus_NO_MEMORY(); \
+        } \
+    } while (0)
 
-    status = _PyStatus_OK();
-    goto done;
+    COPY_ATTR(program_full_path);
+    COPY_ATTR(prefix);
+    COPY_ATTR(exec_prefix);
+    COPY_ATTR(module_search_path);
+    COPY_ATTR(program_name);
+    COPY_ATTR(home);
+    config->isolated = config2->isolated;
+    config->site_import = config2->site_import;
+    COPY_ATTR(base_executable);
 
-error:
-    pathconfig_clear(&new_config);
+#undef COPY_ATTR
 
-done:
-    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-    return status;
+    return _PyStatus_OK();
 }
 
 
@@ -168,7 +139,7 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep)
 }
 
 
-/* Set the global path configuration from config. */
+/* Initialize _Py_dll_path on Windows. Do nothing on other platforms. */
 PyStatus
 _PyPathConfig_Init(void)
 {
@@ -194,47 +165,39 @@ _PyPathConfig_Init(void)
 
 
 static PyStatus
-pathconfig_global_init_from_config(const PyConfig *config)
+pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
 {
     PyStatus status;
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    _PyPathConfig pathconfig = _PyPathConfig_INIT;
-
-    pathconfig.module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM);
-    if (pathconfig.module_search_path == NULL) {
-        goto no_memory;
+    if (config->module_search_paths_set) {
+        pathconfig->module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM);
+        if (pathconfig->module_search_path == NULL) {
+            goto no_memory;
+        }
     }
 
-    if (copy_wstr(&pathconfig.program_full_path, config->executable) < 0) {
-        goto no_memory;
-    }
-    if (copy_wstr(&pathconfig.prefix, config->prefix) < 0) {
-        goto no_memory;
-    }
-    if (copy_wstr(&pathconfig.exec_prefix, config->exec_prefix) < 0) {
-        goto no_memory;
-    }
-    if (copy_wstr(&pathconfig.program_name, config->program_name) < 0) {
-        goto no_memory;
-    }
-    if (copy_wstr(&pathconfig.home, config->home) < 0) {
-        goto no_memory;
-    }
-    if (copy_wstr(&pathconfig.base_executable, config->base_executable) < 0) {
-        goto no_memory;
-    }
+#define COPY_CONFIG(PATH_ATTR, CONFIG_ATTR) \
+        if (config->CONFIG_ATTR) { \
+            if (copy_wstr(&pathconfig->PATH_ATTR, config->CONFIG_ATTR) < 0) { \
+                goto no_memory; \
+            } \
+        }
 
-    pathconfig_clear(&_Py_path_config);
-    /* Steal new_config strings; don't clear new_config */
-    _Py_path_config = pathconfig;
+    COPY_CONFIG(base_executable, base_executable);
+    COPY_CONFIG(program_full_path, executable);
+    COPY_CONFIG(prefix, prefix);
+    COPY_CONFIG(exec_prefix, exec_prefix);
+    COPY_CONFIG(program_name, program_name);
+    COPY_CONFIG(home, home);
+
+#undef COPY_CONFIG
 
     status = _PyStatus_OK();
     goto done;
 
 no_memory:
-    pathconfig_clear(&pathconfig);
     status = _PyStatus_NO_MEMORY();
 
 done:
@@ -283,6 +246,61 @@ config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig)
 }
 
 
+/* Calculate the path configuration:
+
+   - exec_prefix
+   - module_search_path
+   - prefix
+   - program_full_path
+
+   On Windows, more fields are calculated:
+
+   - base_executable
+   - isolated
+   - site_import
+
+   On other platforms, isolated and site_import are left unchanged, and
+   _PyConfig_InitPathConfig() copies executable to base_executable (if it's not
+   set).
+
+   Priority, highest to lowest:
+
+   - PyConfig
+   - _Py_path_config: set by Py_SetPath(), Py_SetPythonHome()
+     and Py_SetProgramName()
+   - _PyPathConfig_Calculate()
+*/
+static PyStatus
+pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config)
+{
+    PyStatus status;
+
+    PyMemAllocatorEx old_alloc;
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    status = pathconfig_copy(pathconfig, &_Py_path_config);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto done;
+    }
+
+    status = pathconfig_set_from_config(pathconfig, config);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto done;
+    }
+
+    if (_Py_path_config.module_search_path == NULL) {
+        status = _PyPathConfig_Calculate(pathconfig, config);
+    }
+    else {
+        /* Py_SetPath() has been called: avoid _PyPathConfig_Calculate() */
+    }
+
+done:
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+    return status;
+}
+
+
 static PyStatus
 config_calculate_pathconfig(PyConfig *config)
 {
@@ -291,42 +309,28 @@ config_calculate_pathconfig(PyConfig *config)
 
     status = pathconfig_calculate(&pathconfig, config);
     if (_PyStatus_EXCEPTION(status)) {
-        goto error;
+        goto done;
     }
 
     if (!config->module_search_paths_set) {
         status = config_init_module_search_paths(config, &pathconfig);
         if (_PyStatus_EXCEPTION(status)) {
-            goto error;
-        }
-    }
-
-    if (config->executable == NULL) {
-        if (copy_wstr(&config->executable,
-                      pathconfig.program_full_path) < 0) {
-            goto no_memory;
-        }
-    }
-
-    if (config->prefix == NULL) {
-        if (copy_wstr(&config->prefix, pathconfig.prefix) < 0) {
-            goto no_memory;
+            goto done;
         }
     }
 
-    if (config->exec_prefix == NULL) {
-        if (copy_wstr(&config->exec_prefix,
-                      pathconfig.exec_prefix) < 0) {
-            goto no_memory;
+#define COPY_ATTR(PATH_ATTR, CONFIG_ATTR) \
+        if (config->CONFIG_ATTR == NULL) { \
+            if (copy_wstr(&config->CONFIG_ATTR, pathconfig.PATH_ATTR) < 0) { \
+                goto no_memory; \
+            } \
         }
-    }
 
-    if (config->base_executable == NULL) {
-        if (copy_wstr(&config->base_executable,
-                      pathconfig.base_executable) < 0) {
-            goto no_memory;
-        }
-    }
+    COPY_ATTR(program_full_path, executable);
+    COPY_ATTR(prefix, prefix);
+    COPY_ATTR(exec_prefix, exec_prefix);
+    COPY_ATTR(base_executable, base_executable);
+#undef COPY_ATTR
 
     if (pathconfig.isolated != -1) {
         config->isolated = pathconfig.isolated;
@@ -335,13 +339,13 @@ config_calculate_pathconfig(PyConfig *config)
         config->site_import = pathconfig.site_import;
     }
 
-    pathconfig_clear(&pathconfig);
-    return _PyStatus_OK();
+    status = _PyStatus_OK();
+    goto done;
 
 no_memory:
     status = _PyStatus_NO_MEMORY();
 
-error:
+done:
     pathconfig_clear(&pathconfig);
     return status;
 }
@@ -386,39 +390,55 @@ _PyConfig_InitPathConfig(PyConfig *config)
 }
 
 
-static void
-pathconfig_global_init(void)
+static PyStatus
+pathconfig_global_read(_PyPathConfig *pathconfig)
 {
-    /* Initialize _Py_dll_path if needed */
-    PyStatus status = _PyPathConfig_Init();
-    if (_PyStatus_EXCEPTION(status)) {
-        Py_ExitStatusException(status);
-    }
-
-    if (_Py_path_config.module_search_path != NULL) {
-        /* Already initialized */
-        return;
-    }
-
+    PyStatus status;
     PyConfig config;
     _PyConfig_InitCompatConfig(&config);
 
+    /* Call _PyConfig_InitPathConfig() */
     status = PyConfig_Read(&config);
     if (_PyStatus_EXCEPTION(status)) {
-        goto error;
+        goto done;
     }
 
-    status = pathconfig_global_init_from_config(&config);
+    status = pathconfig_set_from_config(pathconfig, &config);
+
+done:
+    PyConfig_Clear(&config);
+    return status;
+}
+
+
+static void
+pathconfig_global_init(void)
+{
+    PyStatus status;
+
+    /* Initialize _Py_dll_path if needed */
+    status = _PyPathConfig_Init();
     if (_PyStatus_EXCEPTION(status)) {
-        goto error;
+        Py_ExitStatusException(status);
     }
 
-    PyConfig_Clear(&config);
-    return;
+    if (_Py_path_config.module_search_path == NULL) {
+        status = pathconfig_global_read(&_Py_path_config);
+        if (_PyStatus_EXCEPTION(status)) {
+            Py_ExitStatusException(status);
+        }
+    }
+    else {
+        /* Global configuration already initialized */
+    }
 
-error:
-    PyConfig_Clear(&config);
-    Py_ExitStatusException(status);
+    assert(_Py_path_config.program_full_path != NULL);
+    assert(_Py_path_config.prefix != NULL);
+    assert(_Py_path_config.exec_prefix != NULL);
+    assert(_Py_path_config.module_search_path != NULL);
+    assert(_Py_path_config.program_name != NULL);
+    /* home can be NULL */
+    assert(_Py_path_config.base_executable != NULL);
 }
 
 
@@ -435,30 +455,27 @@ Py_SetPath(const wchar_t *path)
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    _PyPathConfig new_config;
-    new_config.program_full_path = _PyMem_RawWcsdup(Py_GetProgramName());
-    int alloc_error = (new_config.program_full_path == NULL);
-    new_config.prefix = _PyMem_RawWcsdup(L"");
-    alloc_error |= (new_config.prefix == NULL);
-    new_config.exec_prefix = _PyMem_RawWcsdup(L"");
-    alloc_error |= (new_config.exec_prefix == NULL);
-    new_config.module_search_path = _PyMem_RawWcsdup(path);
-    alloc_error |= (new_config.module_search_path == NULL);
-
-    /* steal the home and program_name values (to leave them unchanged) */
-    new_config.home = _Py_path_config.home;
-    _Py_path_config.home = NULL;
-    new_config.program_name = _Py_path_config.program_name;
-    _Py_path_config.program_name = NULL;
-    new_config.base_executable = _Py_path_config.base_executable;
-    _Py_path_config.base_executable = NULL;
+    /* Getting the program name calls pathconfig_global_init() */
+    wchar_t *program_name = _PyMem_RawWcsdup(Py_GetProgramName());
 
-    pathconfig_clear(&_Py_path_config);
-    _Py_path_config = new_config;
+    PyMem_RawFree(_Py_path_config.program_full_path);
+    PyMem_RawFree(_Py_path_config.prefix);
+    PyMem_RawFree(_Py_path_config.exec_prefix);
+    PyMem_RawFree(_Py_path_config.module_search_path);
+
+    /* Copy program_name to program_full_path  */
+    _Py_path_config.program_full_path = program_name;
+    _Py_path_config.prefix = _PyMem_RawWcsdup(L"");
+    _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L"");
+    _Py_path_config.module_search_path = _PyMem_RawWcsdup(path);
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    if (alloc_error) {
+    if (_Py_path_config.program_full_path == NULL
+        || _Py_path_config.prefix == NULL
+        || _Py_path_config.exec_prefix == NULL
+        || _Py_path_config.module_search_path == NULL)
+    {
         Py_FatalError("Py_SetPath() failed: out of memory");
     }
 }


From webhook-mailer at python.org  Mon Sep 23 13:50:33 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 23 Sep 2019 17:50:33 -0000
Subject: [Python-checkins] bpo-38234: Fix _PyConfig_InitPathConfig()
 (GH-16335) (GH-16336)
Message-ID: <mailman.33.1569261035.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/3f5409a3f13c59baa34656bccefdc3728e46c9ef
commit: 3f5409a3f13c59baa34656bccefdc3728e46c9ef
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-23T19:50:27+02:00
summary:

bpo-38234: Fix _PyConfig_InitPathConfig() (GH-16335) (GH-16336)

* _PyConfig_InitPathConfig() now starts by copying the global path
  configuration, and then override values set in PyConfig.
* _PyPathConfig_Calculate() implementations no longer override
  _PyPathConfig fields which are already computed. For example,
  if _PyPathConfig.prefix is not NULL, leave it unchanged.
* If Py_SetPath() has been called, _PyConfig_InitPathConfig() doesn't
  call _PyPathConfig_Calculate() anymore.
* _PyPathConfig_Calculate() no longer uses PyConfig,
  except to initialize PyCalculatePath structure.
* pathconfig_calculate(): remove useless temporary
  "_PyPathConfig new_config" variable.
* calculate_module_search_path(): remove hack to workaround memory
  allocation failure, call Py_FatalError() instead.
* Fix get_program_full_path(): handle memory allocation failure.

(cherry picked from commit 9c42f8cda552694f3b47d6388d4ae84d61731872)

files:
M Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst
M Modules/getpath.c
M PC/getpathp.c
M Python/pathconfig.c

diff --git a/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst b/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst
index 6310f19f004e..fe658ad75d89 100644
--- a/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst	
+++ b/Misc/NEWS.d/next/C API/2019-09-20-17-22-41.bpo-38234.ZbquVK.rst	
@@ -1,2 +1,3 @@
-Python ignored path passed to :c:func:`Py_SetPath`, fix Python
-initialization to use the specified path.
+Python ignored arguments passed to :c:func:`Py_SetPath`,
+:c:func:`Py_SetPythonHome` and :c:func:`Py_SetProgramName`: fix Python
+initialization to use specified arguments.
diff --git a/Modules/getpath.c b/Modules/getpath.c
index 5afb5b152638..f60fe3c015bb 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -133,6 +133,9 @@ typedef struct {
 
     int prefix_found;         /* found platform independent libraries? */
     int exec_prefix_found;    /* found the platform dependent libraries? */
+
+    int warnings;
+    const wchar_t *pythonpath_env;
 } PyCalculatePath;
 
 static const wchar_t delimiter[2] = {DELIM, '\0'};
@@ -365,17 +368,16 @@ add_exe_suffix(wchar_t *progpath, size_t progpathlen)
    bytes long.
 */
 static PyStatus
-search_for_prefix(const PyConfig *config, PyCalculatePath *calculate,
-                  wchar_t *prefix, size_t prefix_len,
-                  int *found)
+search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                  wchar_t *prefix, size_t prefix_len, int *found)
 {
     PyStatus status;
     size_t n;
     wchar_t *vpath;
 
     /* If PYTHONHOME is set, we believe it unconditionally */
-    if (config->home) {
-        if (safe_wcscpy(prefix, config->home, prefix_len) < 0) {
+    if (pathconfig->home) {
+        if (safe_wcscpy(prefix, pathconfig->home, prefix_len) < 0) {
             return PATHLEN_ERR();
         }
         wchar_t *delim = wcschr(prefix, DELIM);
@@ -482,19 +484,19 @@ search_for_prefix(const PyConfig *config, PyCalculatePath *calculate,
 
 
 static PyStatus
-calculate_prefix(const PyConfig *config,
-                 PyCalculatePath *calculate, wchar_t *prefix, size_t prefix_len)
+calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                 wchar_t *prefix, size_t prefix_len)
 {
     PyStatus status;
 
-    status = search_for_prefix(config, calculate, prefix, prefix_len,
-                            &calculate->prefix_found);
+    status = search_for_prefix(calculate, pathconfig, prefix, prefix_len,
+                               &calculate->prefix_found);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     if (!calculate->prefix_found) {
-        if (config->pathconfig_warnings) {
+        if (calculate->warnings) {
             fprintf(stderr,
                 "Could not find platform independent libraries <prefix>\n");
         }
@@ -544,8 +546,7 @@ calculate_reduce_prefix(PyCalculatePath *calculate,
    MAXPATHLEN bytes long.
 */
 static PyStatus
-search_for_exec_prefix(const PyConfig *config,
-                       PyCalculatePath *calculate,
+search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
                        wchar_t *exec_prefix, size_t exec_prefix_len,
                        int *found)
 {
@@ -553,15 +554,15 @@ search_for_exec_prefix(const PyConfig *config,
     size_t n;
 
     /* If PYTHONHOME is set, we believe it unconditionally */
-    if (config->home) {
-        wchar_t *delim = wcschr(config->home, DELIM);
+    if (pathconfig->home) {
+        wchar_t *delim = wcschr(pathconfig->home, DELIM);
         if (delim) {
             if (safe_wcscpy(exec_prefix, delim+1, exec_prefix_len) < 0) {
                 return PATHLEN_ERR();
             }
         }
         else {
-            if (safe_wcscpy(exec_prefix, config->home, exec_prefix_len) < 0) {
+            if (safe_wcscpy(exec_prefix, pathconfig->home, exec_prefix_len) < 0) {
                 return PATHLEN_ERR();
             }
         }
@@ -668,21 +669,20 @@ search_for_exec_prefix(const PyConfig *config,
 
 
 static PyStatus
-calculate_exec_prefix(const PyConfig *config,
-                      PyCalculatePath *calculate,
+calculate_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
                       wchar_t *exec_prefix, size_t exec_prefix_len)
 {
     PyStatus status;
 
-    status = search_for_exec_prefix(config, calculate,
-                                 exec_prefix, exec_prefix_len,
-                                 &calculate->exec_prefix_found);
+    status = search_for_exec_prefix(calculate, pathconfig,
+                                    exec_prefix, exec_prefix_len,
+                                    &calculate->exec_prefix_found);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     if (!calculate->exec_prefix_found) {
-        if (config->pathconfig_warnings) {
+        if (calculate->warnings) {
             fprintf(stderr,
                 "Could not find platform dependent libraries <exec_prefix>\n");
         }
@@ -721,8 +721,7 @@ calculate_reduce_exec_prefix(PyCalculatePath *calculate,
 
 
 static PyStatus
-calculate_program_full_path(const PyConfig *config,
-                            PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+calculate_program_full_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
 {
     PyStatus status;
     wchar_t program_full_path[MAXPATHLEN + 1];
@@ -743,8 +742,8 @@ calculate_program_full_path(const PyConfig *config,
      * other way to find a directory to start the search from.  If
      * $PATH isn't exported, you lose.
      */
-    if (wcschr(config->program_name, SEP)) {
-        if (safe_wcscpy(program_full_path, config->program_name,
+    if (wcschr(pathconfig->program_name, SEP)) {
+        if (safe_wcscpy(program_full_path, pathconfig->program_name,
                         program_full_path_len) < 0) {
             return PATHLEN_ERR();
         }
@@ -795,8 +794,8 @@ calculate_program_full_path(const PyConfig *config,
                 }
             }
 
-            status = joinpath(program_full_path, config->program_name,
-                            program_full_path_len);
+            status = joinpath(program_full_path, pathconfig->program_name,
+                              program_full_path_len);
             if (_PyStatus_EXCEPTION(status)) {
                 return status;
             }
@@ -1030,15 +1029,14 @@ calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix)
 
 
 static PyStatus
-calculate_module_search_path(const PyConfig *config,
-                             PyCalculatePath *calculate,
-                             const wchar_t *prefix, const wchar_t *exec_prefix,
-                             _PyPathConfig *pathconfig)
+calculate_module_search_path(PyCalculatePath *calculate,
+                             _PyPathConfig *pathconfig,
+                             const wchar_t *prefix, const wchar_t *exec_prefix)
 {
     /* Calculate size of return buffer */
     size_t bufsz = 0;
-    if (config->pythonpath_env != NULL) {
-        bufsz += wcslen(config->pythonpath_env) + 1;
+    if (calculate->pythonpath_env != NULL) {
+        bufsz += wcslen(calculate->pythonpath_env) + 1;
     }
 
     wchar_t *defpath = calculate->pythonpath;
@@ -1072,8 +1070,8 @@ calculate_module_search_path(const PyConfig *config,
     buf[0] = '\0';
 
     /* Run-time value of $PYTHONPATH goes first */
-    if (config->pythonpath_env) {
-        wcscpy(buf, config->pythonpath_env);
+    if (calculate->pythonpath_env) {
+        wcscpy(buf, calculate->pythonpath_env);
         wcscat(buf, delimiter);
     }
 
@@ -1149,6 +1147,10 @@ calculate_init(PyCalculatePath *calculate,
     if (!calculate->lib_python) {
         return DECODE_LOCALE_ERR("EXEC_PREFIX define", len);
     }
+
+    calculate->warnings = config->pathconfig_warnings;
+    calculate->pythonpath_env = config->pythonpath_env;
+
     return _PyStatus_OK();
 }
 
@@ -1165,14 +1167,15 @@ calculate_free(PyCalculatePath *calculate)
 
 
 static PyStatus
-calculate_path_impl(const PyConfig *config,
-                    PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
 {
     PyStatus status;
 
-    status = calculate_program_full_path(config, calculate, pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+    if (pathconfig->program_full_path == NULL) {
+        status = calculate_program_full_path(calculate, pathconfig);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
     }
 
     status = calculate_argv0_path(calculate, pathconfig->program_full_path);
@@ -1187,8 +1190,8 @@ calculate_path_impl(const PyConfig *config,
 
     wchar_t prefix[MAXPATHLEN+1];
     memset(prefix, 0, sizeof(prefix));
-    status = calculate_prefix(config, calculate,
-                           prefix, Py_ARRAY_LENGTH(prefix));
+    status = calculate_prefix(calculate, pathconfig,
+                              prefix, Py_ARRAY_LENGTH(prefix));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -1200,52 +1203,65 @@ calculate_path_impl(const PyConfig *config,
 
     wchar_t exec_prefix[MAXPATHLEN+1];
     memset(exec_prefix, 0, sizeof(exec_prefix));
-    status = calculate_exec_prefix(config, calculate,
-                                exec_prefix, Py_ARRAY_LENGTH(exec_prefix));
+    status = calculate_exec_prefix(calculate, pathconfig,
+                                   exec_prefix, Py_ARRAY_LENGTH(exec_prefix));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     if ((!calculate->prefix_found || !calculate->exec_prefix_found) &&
-        config->pathconfig_warnings)
+        calculate->warnings)
     {
         fprintf(stderr,
                 "Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]\n");
     }
 
     if (pathconfig->module_search_path == NULL) {
-        status = calculate_module_search_path(config, calculate,
-                                              prefix, exec_prefix, pathconfig);
+        status = calculate_module_search_path(calculate, pathconfig,
+                                              prefix, exec_prefix);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
     }
 
-    status = calculate_reduce_prefix(calculate, prefix, Py_ARRAY_LENGTH(prefix));
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-
-    pathconfig->prefix = _PyMem_RawWcsdup(prefix);
     if (pathconfig->prefix == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
+        status = calculate_reduce_prefix(calculate, prefix, Py_ARRAY_LENGTH(prefix));
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
 
-    status = calculate_reduce_exec_prefix(calculate,
-                                       exec_prefix, Py_ARRAY_LENGTH(exec_prefix));
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+        pathconfig->prefix = _PyMem_RawWcsdup(prefix);
+        if (pathconfig->prefix == NULL) {
+            return _PyStatus_NO_MEMORY();
+        }
     }
 
-    pathconfig->exec_prefix = _PyMem_RawWcsdup(exec_prefix);
     if (pathconfig->exec_prefix == NULL) {
-        return _PyStatus_NO_MEMORY();
+        status = calculate_reduce_exec_prefix(calculate,
+                                              exec_prefix,
+                                              Py_ARRAY_LENGTH(exec_prefix));
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
+
+        pathconfig->exec_prefix = _PyMem_RawWcsdup(exec_prefix);
+        if (pathconfig->exec_prefix == NULL) {
+            return _PyStatus_NO_MEMORY();
+        }
     }
 
     return _PyStatus_OK();
 }
 
 
+/* Calculate 'pathconfig' attributes:
+
+   - program_full_path
+   - module_search_path
+   - prefix
+   - exec_prefix
+
+   If an attribute is already set (non NULL), it is left unchanged. */
 PyStatus
 _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
 {
@@ -1258,7 +1274,7 @@ _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
         goto done;
     }
 
-    status = calculate_path_impl(config, &calculate, pathconfig);
+    status = calculate_path(&calculate, pathconfig);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
     }
diff --git a/PC/getpathp.c b/PC/getpathp.c
index 888bef1146fd..5670fa2ee4e1 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -130,6 +130,8 @@ typedef struct {
     wchar_t zip_path[MAXPATHLEN+1];
 
     wchar_t *dll_path;
+
+    const wchar_t *pythonpath_env;
 } PyCalculatePath;
 
 
@@ -322,7 +324,7 @@ gotlandmark(wchar_t *prefix, const wchar_t *landmark)
 
 
 /* assumes argv0_path is MAXPATHLEN+1 bytes long, already \0 term'd.
-   assumption provided by only caller, calculate_path_impl() */
+   assumption provided by only caller, calculate_path() */
 static int
 search_for_prefix(wchar_t *prefix, const wchar_t *argv0_path, const wchar_t *landmark)
 {
@@ -534,6 +536,7 @@ _Py_GetDLLPath(void)
 static PyStatus
 get_program_full_path(_PyPathConfig *pathconfig)
 {
+    PyStatus status;
     const wchar_t *pyvenv_launcher;
     wchar_t program_full_path[MAXPATHLEN+1];
     memset(program_full_path, 0, sizeof(program_full_path));
@@ -548,12 +551,18 @@ get_program_full_path(_PyPathConfig *pathconfig)
     pyvenv_launcher = _wgetenv(L"__PYVENV_LAUNCHER__");
     if (pyvenv_launcher && pyvenv_launcher[0]) {
         /* If overridden, preserve the original full path */
-        pathconfig->base_executable = PyMem_RawMalloc(
-            sizeof(wchar_t) * (MAXPATHLEN + 1));
-        PyStatus status = canonicalize(pathconfig->base_executable,
-                                       program_full_path);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
+        if (pathconfig->base_executable == NULL) {
+            pathconfig->base_executable = PyMem_RawMalloc(
+                sizeof(wchar_t) * (MAXPATHLEN + 1));
+            if (pathconfig->base_executable == NULL) {
+                return _PyStatus_NO_MEMORY();
+            }
+
+            status = canonicalize(pathconfig->base_executable,
+                                  program_full_path);
+            if (_PyStatus_EXCEPTION(status)) {
+                return status;
+            }
         }
 
         wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher);
@@ -562,11 +571,20 @@ get_program_full_path(_PyPathConfig *pathconfig)
         _wputenv_s(L"__PYVENV_LAUNCHER__", L"");
     }
 
-    pathconfig->program_full_path = PyMem_RawMalloc(
-        sizeof(wchar_t) * (MAXPATHLEN + 1));
+    if (pathconfig->program_full_path == NULL) {
+        pathconfig->program_full_path = PyMem_RawMalloc(
+            sizeof(wchar_t) * (MAXPATHLEN + 1));
+        if (pathconfig->program_full_path == NULL) {
+            return _PyStatus_NO_MEMORY();
+        }
 
-    return canonicalize(pathconfig->program_full_path,
-                        program_full_path);
+        status = canonicalize(pathconfig->program_full_path,
+                              program_full_path);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
+    }
+    return _PyStatus_OK();
 }
 
 
@@ -657,7 +675,13 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
     }
 
     fclose(sp_file);
-    pathconfig->module_search_path = buf;
+    if (pathconfig->module_search_path == NULL) {
+        pathconfig->module_search_path = _PyMem_RawWcsdup(buf);
+        if (pathconfig->module_search_path == NULL) {
+            Py_FatalError("out of memory");
+        }
+    }
+    PyMem_RawFree(buf);
     return 1;
 
 error:
@@ -668,8 +692,7 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
 
 
 static PyStatus
-calculate_init(PyCalculatePath *calculate,
-               const PyConfig *config)
+calculate_init(PyCalculatePath *calculate, const PyConfig *config)
 {
     calculate->home = config->home;
     calculate->path_env = _wgetenv(L"PATH");
@@ -679,6 +702,8 @@ calculate_init(PyCalculatePath *calculate,
         return _PyStatus_NO_MEMORY();
     }
 
+    calculate->pythonpath_env = config->pythonpath_env;
+
     return _PyStatus_OK();
 }
 
@@ -785,8 +810,8 @@ calculate_home_prefix(PyCalculatePath *calculate, wchar_t *prefix)
 
 
 static PyStatus
-calculate_module_search_path(const PyConfig *config,
-                             PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+calculate_module_search_path(PyCalculatePath *calculate,
+                             _PyPathConfig *pathconfig,
                              wchar_t *prefix)
 {
     int skiphome = calculate->home==NULL ? 0 : 1;
@@ -796,7 +821,7 @@ calculate_module_search_path(const PyConfig *config,
 #endif
     /* We only use the default relative PYTHONPATH if we haven't
        anything better to use! */
-    int skipdefault = (config->pythonpath_env != NULL ||
+    int skipdefault = (calculate->pythonpath_env != NULL ||
                        calculate->home != NULL ||
                        calculate->machine_path != NULL ||
                        calculate->user_path != NULL);
@@ -835,30 +860,20 @@ calculate_module_search_path(const PyConfig *config,
         bufsz += wcslen(calculate->machine_path) + 1;
     }
     bufsz += wcslen(calculate->zip_path) + 1;
-    if (config->pythonpath_env != NULL) {
-        bufsz += wcslen(config->pythonpath_env) + 1;
+    if (calculate->pythonpath_env != NULL) {
+        bufsz += wcslen(calculate->pythonpath_env) + 1;
     }
 
     wchar_t *buf, *start_buf;
     buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t));
     if (buf == NULL) {
-        /* We can't exit, so print a warning and limp along */
-        fprintf(stderr, "Can't malloc dynamic PYTHONPATH.\n");
-        if (config->pythonpath_env) {
-            fprintf(stderr, "Using environment $PYTHONPATH.\n");
-            pathconfig->module_search_path = config->pythonpath_env;
-        }
-        else {
-            fprintf(stderr, "Using default static path.\n");
-            pathconfig->module_search_path = PYTHONPATH;
-        }
-        return _PyStatus_OK();
+        Py_FatalError("Can't malloc dynamic PYTHONPATH");
     }
     start_buf = buf;
 
-    if (config->pythonpath_env) {
+    if (calculate->pythonpath_env) {
         if (wcscpy_s(buf, bufsz - (buf - start_buf),
-                     config->pythonpath_env)) {
+                     calculate->pythonpath_env)) {
             return INIT_ERR_BUFFER_OVERFLOW();
         }
         buf = wcschr(buf, L'\0');
@@ -971,8 +986,7 @@ calculate_module_search_path(const PyConfig *config,
 
 
 static PyStatus
-calculate_path_impl(const PyConfig *config,
-                    PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
 {
     PyStatus status;
 
@@ -1003,21 +1017,24 @@ calculate_path_impl(const PyConfig *config,
     calculate_home_prefix(calculate, prefix);
 
     if (pathconfig->module_search_path == NULL) {
-        status = calculate_module_search_path(config, calculate,
-                                              pathconfig, prefix);
+        status = calculate_module_search_path(calculate, pathconfig, prefix);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
     }
 
 done:
-    pathconfig->prefix = _PyMem_RawWcsdup(prefix);
     if (pathconfig->prefix == NULL) {
-        return _PyStatus_NO_MEMORY();
+        pathconfig->prefix = _PyMem_RawWcsdup(prefix);
+        if (pathconfig->prefix == NULL) {
+            return _PyStatus_NO_MEMORY();
+        }
     }
-    pathconfig->exec_prefix = _PyMem_RawWcsdup(prefix);
     if (pathconfig->exec_prefix == NULL) {
-        return _PyStatus_NO_MEMORY();
+        pathconfig->exec_prefix = _PyMem_RawWcsdup(prefix);
+        if (pathconfig->exec_prefix == NULL) {
+            return _PyStatus_NO_MEMORY();
+        }
     }
 
     return _PyStatus_OK();
@@ -1033,6 +1050,17 @@ calculate_free(PyCalculatePath *calculate)
 }
 
 
+/* Calculate 'pathconfig' attributes:
+
+   - base_executable
+   - program_full_path
+   - module_search_path
+   - prefix
+   - exec_prefix
+   - isolated
+   - site_import
+
+   If an attribute is already set (non NULL), it is left unchanged. */
 PyStatus
 _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
 {
@@ -1045,7 +1073,7 @@ _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
         goto done;
     }
 
-    status = calculate_path_impl(config, &calculate, pathconfig);
+    status = calculate_path(&calculate, pathconfig);
 
 done:
     calculate_free(&calculate);
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index 1934f2e1c4b8..09533100b463 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -51,12 +51,12 @@ pathconfig_clear(_PyPathConfig *config)
         ATTR = NULL; \
     } while (0)
 
-    CLEAR(config->prefix);
     CLEAR(config->program_full_path);
+    CLEAR(config->prefix);
     CLEAR(config->exec_prefix);
     CLEAR(config->module_search_path);
-    CLEAR(config->home);
     CLEAR(config->program_name);
+    CLEAR(config->home);
     CLEAR(config->base_executable);
 #undef CLEAR
 
@@ -64,60 +64,31 @@ pathconfig_clear(_PyPathConfig *config)
 }
 
 
-/* Calculate the path configuration: initialize pathconfig from config */
 static PyStatus
-pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config)
+pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2)
 {
-    PyStatus status;
-    _PyPathConfig new_config = _PyPathConfig_INIT;
-
-    PyMemAllocatorEx old_alloc;
-    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+    pathconfig_clear(config);
 
-    if (copy_wstr(&new_config.module_search_path,
-                  _Py_path_config.module_search_path) < 0)
-    {
-        status = _PyStatus_NO_MEMORY();
-        goto error;
-    }
-
-    /* Calculate program_full_path, prefix, exec_prefix,
-       dll_path (Windows), and module_search_path */
-    status = _PyPathConfig_Calculate(&new_config, config);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto error;
-    }
-
-    /* Copy home and program_name from config */
-    if (copy_wstr(&new_config.home, config->home) < 0) {
-        status = _PyStatus_NO_MEMORY();
-        goto error;
-    }
-    if (copy_wstr(&new_config.program_name, config->program_name) < 0) {
-        status = _PyStatus_NO_MEMORY();
-        goto error;
-    }
-    if (config->base_executable) {
-        PyMem_RawFree(new_config.base_executable);
-        if (copy_wstr(&new_config.base_executable,
-                      config->base_executable) < 0) {
-            status = _PyStatus_NO_MEMORY();
-            goto error;
-        }
-    }
-
-    pathconfig_clear(pathconfig);
-    *pathconfig = new_config;
+#define COPY_ATTR(ATTR) \
+    do { \
+        if (copy_wstr(&config->ATTR, config2->ATTR) < 0) { \
+            return _PyStatus_NO_MEMORY(); \
+        } \
+    } while (0)
 
-    status = _PyStatus_OK();
-    goto done;
+    COPY_ATTR(program_full_path);
+    COPY_ATTR(prefix);
+    COPY_ATTR(exec_prefix);
+    COPY_ATTR(module_search_path);
+    COPY_ATTR(program_name);
+    COPY_ATTR(home);
+    config->isolated = config2->isolated;
+    config->site_import = config2->site_import;
+    COPY_ATTR(base_executable);
 
-error:
-    pathconfig_clear(&new_config);
+#undef COPY_ATTR
 
-done:
-    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-    return status;
+    return _PyStatus_OK();
 }
 
 
@@ -168,7 +139,7 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep)
 }
 
 
-/* Set the global path configuration from config. */
+/* Initialize _Py_dll_path on Windows. Do nothing on other platforms. */
 PyStatus
 _PyPathConfig_Init(void)
 {
@@ -194,47 +165,39 @@ _PyPathConfig_Init(void)
 
 
 static PyStatus
-pathconfig_global_init_from_config(const PyConfig *config)
+pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
 {
     PyStatus status;
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    _PyPathConfig pathconfig = _PyPathConfig_INIT;
-
-    pathconfig.module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM);
-    if (pathconfig.module_search_path == NULL) {
-        goto no_memory;
+    if (config->module_search_paths_set) {
+        pathconfig->module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM);
+        if (pathconfig->module_search_path == NULL) {
+            goto no_memory;
+        }
     }
 
-    if (copy_wstr(&pathconfig.program_full_path, config->executable) < 0) {
-        goto no_memory;
-    }
-    if (copy_wstr(&pathconfig.prefix, config->prefix) < 0) {
-        goto no_memory;
-    }
-    if (copy_wstr(&pathconfig.exec_prefix, config->exec_prefix) < 0) {
-        goto no_memory;
-    }
-    if (copy_wstr(&pathconfig.program_name, config->program_name) < 0) {
-        goto no_memory;
-    }
-    if (copy_wstr(&pathconfig.home, config->home) < 0) {
-        goto no_memory;
-    }
-    if (copy_wstr(&pathconfig.base_executable, config->base_executable) < 0) {
-        goto no_memory;
-    }
+#define COPY_CONFIG(PATH_ATTR, CONFIG_ATTR) \
+        if (config->CONFIG_ATTR) { \
+            if (copy_wstr(&pathconfig->PATH_ATTR, config->CONFIG_ATTR) < 0) { \
+                goto no_memory; \
+            } \
+        }
 
-    pathconfig_clear(&_Py_path_config);
-    /* Steal new_config strings; don't clear new_config */
-    _Py_path_config = pathconfig;
+    COPY_CONFIG(base_executable, base_executable);
+    COPY_CONFIG(program_full_path, executable);
+    COPY_CONFIG(prefix, prefix);
+    COPY_CONFIG(exec_prefix, exec_prefix);
+    COPY_CONFIG(program_name, program_name);
+    COPY_CONFIG(home, home);
+
+#undef COPY_CONFIG
 
     status = _PyStatus_OK();
     goto done;
 
 no_memory:
-    pathconfig_clear(&pathconfig);
     status = _PyStatus_NO_MEMORY();
 
 done:
@@ -283,6 +246,61 @@ config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig)
 }
 
 
+/* Calculate the path configuration:
+
+   - exec_prefix
+   - module_search_path
+   - prefix
+   - program_full_path
+
+   On Windows, more fields are calculated:
+
+   - base_executable
+   - isolated
+   - site_import
+
+   On other platforms, isolated and site_import are left unchanged, and
+   _PyConfig_InitPathConfig() copies executable to base_executable (if it's not
+   set).
+
+   Priority, highest to lowest:
+
+   - PyConfig
+   - _Py_path_config: set by Py_SetPath(), Py_SetPythonHome()
+     and Py_SetProgramName()
+   - _PyPathConfig_Calculate()
+*/
+static PyStatus
+pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config)
+{
+    PyStatus status;
+
+    PyMemAllocatorEx old_alloc;
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    status = pathconfig_copy(pathconfig, &_Py_path_config);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto done;
+    }
+
+    status = pathconfig_set_from_config(pathconfig, config);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto done;
+    }
+
+    if (_Py_path_config.module_search_path == NULL) {
+        status = _PyPathConfig_Calculate(pathconfig, config);
+    }
+    else {
+        /* Py_SetPath() has been called: avoid _PyPathConfig_Calculate() */
+    }
+
+done:
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+    return status;
+}
+
+
 static PyStatus
 config_calculate_pathconfig(PyConfig *config)
 {
@@ -291,42 +309,28 @@ config_calculate_pathconfig(PyConfig *config)
 
     status = pathconfig_calculate(&pathconfig, config);
     if (_PyStatus_EXCEPTION(status)) {
-        goto error;
+        goto done;
     }
 
     if (!config->module_search_paths_set) {
         status = config_init_module_search_paths(config, &pathconfig);
         if (_PyStatus_EXCEPTION(status)) {
-            goto error;
-        }
-    }
-
-    if (config->executable == NULL) {
-        if (copy_wstr(&config->executable,
-                      pathconfig.program_full_path) < 0) {
-            goto no_memory;
-        }
-    }
-
-    if (config->prefix == NULL) {
-        if (copy_wstr(&config->prefix, pathconfig.prefix) < 0) {
-            goto no_memory;
+            goto done;
         }
     }
 
-    if (config->exec_prefix == NULL) {
-        if (copy_wstr(&config->exec_prefix,
-                      pathconfig.exec_prefix) < 0) {
-            goto no_memory;
+#define COPY_ATTR(PATH_ATTR, CONFIG_ATTR) \
+        if (config->CONFIG_ATTR == NULL) { \
+            if (copy_wstr(&config->CONFIG_ATTR, pathconfig.PATH_ATTR) < 0) { \
+                goto no_memory; \
+            } \
         }
-    }
 
-    if (config->base_executable == NULL) {
-        if (copy_wstr(&config->base_executable,
-                      pathconfig.base_executable) < 0) {
-            goto no_memory;
-        }
-    }
+    COPY_ATTR(program_full_path, executable);
+    COPY_ATTR(prefix, prefix);
+    COPY_ATTR(exec_prefix, exec_prefix);
+    COPY_ATTR(base_executable, base_executable);
+#undef COPY_ATTR
 
     if (pathconfig.isolated != -1) {
         config->isolated = pathconfig.isolated;
@@ -335,13 +339,13 @@ config_calculate_pathconfig(PyConfig *config)
         config->site_import = pathconfig.site_import;
     }
 
-    pathconfig_clear(&pathconfig);
-    return _PyStatus_OK();
+    status = _PyStatus_OK();
+    goto done;
 
 no_memory:
     status = _PyStatus_NO_MEMORY();
 
-error:
+done:
     pathconfig_clear(&pathconfig);
     return status;
 }
@@ -386,39 +390,55 @@ _PyConfig_InitPathConfig(PyConfig *config)
 }
 
 
-static void
-pathconfig_global_init(void)
+static PyStatus
+pathconfig_global_read(_PyPathConfig *pathconfig)
 {
-    /* Initialize _Py_dll_path if needed */
-    PyStatus status = _PyPathConfig_Init();
-    if (_PyStatus_EXCEPTION(status)) {
-        Py_ExitStatusException(status);
-    }
-
-    if (_Py_path_config.module_search_path != NULL) {
-        /* Already initialized */
-        return;
-    }
-
+    PyStatus status;
     PyConfig config;
     _PyConfig_InitCompatConfig(&config);
 
+    /* Call _PyConfig_InitPathConfig() */
     status = PyConfig_Read(&config);
     if (_PyStatus_EXCEPTION(status)) {
-        goto error;
+        goto done;
     }
 
-    status = pathconfig_global_init_from_config(&config);
+    status = pathconfig_set_from_config(pathconfig, &config);
+
+done:
+    PyConfig_Clear(&config);
+    return status;
+}
+
+
+static void
+pathconfig_global_init(void)
+{
+    PyStatus status;
+
+    /* Initialize _Py_dll_path if needed */
+    status = _PyPathConfig_Init();
     if (_PyStatus_EXCEPTION(status)) {
-        goto error;
+        Py_ExitStatusException(status);
     }
 
-    PyConfig_Clear(&config);
-    return;
+    if (_Py_path_config.module_search_path == NULL) {
+        status = pathconfig_global_read(&_Py_path_config);
+        if (_PyStatus_EXCEPTION(status)) {
+            Py_ExitStatusException(status);
+        }
+    }
+    else {
+        /* Global configuration already initialized */
+    }
 
-error:
-    PyConfig_Clear(&config);
-    Py_ExitStatusException(status);
+    assert(_Py_path_config.program_full_path != NULL);
+    assert(_Py_path_config.prefix != NULL);
+    assert(_Py_path_config.exec_prefix != NULL);
+    assert(_Py_path_config.module_search_path != NULL);
+    assert(_Py_path_config.program_name != NULL);
+    /* home can be NULL */
+    assert(_Py_path_config.base_executable != NULL);
 }
 
 
@@ -435,30 +455,27 @@ Py_SetPath(const wchar_t *path)
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    _PyPathConfig new_config;
-    new_config.program_full_path = _PyMem_RawWcsdup(Py_GetProgramName());
-    int alloc_error = (new_config.program_full_path == NULL);
-    new_config.prefix = _PyMem_RawWcsdup(L"");
-    alloc_error |= (new_config.prefix == NULL);
-    new_config.exec_prefix = _PyMem_RawWcsdup(L"");
-    alloc_error |= (new_config.exec_prefix == NULL);
-    new_config.module_search_path = _PyMem_RawWcsdup(path);
-    alloc_error |= (new_config.module_search_path == NULL);
-
-    /* steal the home and program_name values (to leave them unchanged) */
-    new_config.home = _Py_path_config.home;
-    _Py_path_config.home = NULL;
-    new_config.program_name = _Py_path_config.program_name;
-    _Py_path_config.program_name = NULL;
-    new_config.base_executable = _Py_path_config.base_executable;
-    _Py_path_config.base_executable = NULL;
+    /* Getting the program name calls pathconfig_global_init() */
+    wchar_t *program_name = _PyMem_RawWcsdup(Py_GetProgramName());
 
-    pathconfig_clear(&_Py_path_config);
-    _Py_path_config = new_config;
+    PyMem_RawFree(_Py_path_config.program_full_path);
+    PyMem_RawFree(_Py_path_config.prefix);
+    PyMem_RawFree(_Py_path_config.exec_prefix);
+    PyMem_RawFree(_Py_path_config.module_search_path);
+
+    /* Copy program_name to program_full_path  */
+    _Py_path_config.program_full_path = program_name;
+    _Py_path_config.prefix = _PyMem_RawWcsdup(L"");
+    _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L"");
+    _Py_path_config.module_search_path = _PyMem_RawWcsdup(path);
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    if (alloc_error) {
+    if (_Py_path_config.program_full_path == NULL
+        || _Py_path_config.prefix == NULL
+        || _Py_path_config.exec_prefix == NULL
+        || _Py_path_config.module_search_path == NULL)
+    {
         Py_FatalError("Py_SetPath() failed: out of memory");
     }
 }


From webhook-mailer at python.org  Mon Sep 23 18:55:53 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 23 Sep 2019 22:55:53 -0000
Subject: [Python-checkins] bpo-38234: read_pth_file() now returns PyStatus
 (GH-16338)
Message-ID: <mailman.34.1569279354.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/85ce0a7178801b538160cbb5cf9ef50a713c45bf
commit: 85ce0a7178801b538160cbb5cf9ef50a713c45bf
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-24T00:55:48+02:00
summary:

bpo-38234: read_pth_file() now returns PyStatus (GH-16338)

Refactor path configuration code:

* read_pth_file() now returns PyStatus to report errors, rather than
  calling Py_FatalError().
* Move argv0_path and zip_path buffers out of PyCalculatePath
  structures.
* On Windows, _PyPathConfig.home is now preferred over PyConfig.home.

files:
M Modules/getpath.c
M PC/getpathp.c

diff --git a/Modules/getpath.c b/Modules/getpath.c
index 270355e251e1..24e16b41b406 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -123,13 +123,11 @@ extern "C" {
 typedef struct {
     wchar_t *path_env;                 /* PATH environment variable */
 
-    wchar_t *pythonpath;               /* PYTHONPATH define */
-    wchar_t *prefix;                   /* PREFIX define */
-    wchar_t *exec_prefix;              /* EXEC_PREFIX define */
+    wchar_t *pythonpath;               /* PYTHONPATH macro */
+    wchar_t *prefix;                   /* PREFIX macro */
+    wchar_t *exec_prefix;              /* EXEC_PREFIX macro */
 
     wchar_t *lib_python;               /* "lib/pythonX.Y" */
-    wchar_t argv0_path[MAXPATHLEN+1];
-    wchar_t zip_path[MAXPATHLEN+1];    /* ".../lib/pythonXY.zip" */
 
     int prefix_found;         /* found platform independent libraries? */
     int exec_prefix_found;    /* found the platform dependent libraries? */
@@ -369,6 +367,7 @@ add_exe_suffix(wchar_t *progpath, size_t progpathlen)
 */
 static PyStatus
 search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                  const wchar_t *argv0_path,
                   wchar_t *prefix, size_t prefix_len, int *found)
 {
     PyStatus status;
@@ -397,7 +396,7 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
     }
 
     /* Check to see if argv[0] is in the build directory */
-    if (safe_wcscpy(prefix, calculate->argv0_path, prefix_len) < 0) {
+    if (safe_wcscpy(prefix, argv0_path, prefix_len) < 0) {
         return PATHLEN_ERR();
     }
     status = joinpath(prefix, L"Modules/Setup.local", prefix_len);
@@ -409,7 +408,7 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
         /* Check VPATH to see if argv0_path is in the build directory. */
         vpath = Py_DecodeLocale(VPATH, NULL);
         if (vpath != NULL) {
-            if (safe_wcscpy(prefix, calculate->argv0_path, prefix_len) < 0) {
+            if (safe_wcscpy(prefix, argv0_path, prefix_len) < 0) {
                 return PATHLEN_ERR();
             }
             status = joinpath(prefix, vpath, prefix_len);
@@ -435,7 +434,7 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
     }
 
     /* Search from argv0_path, until root is found */
-    status = copy_absolute(prefix, calculate->argv0_path, prefix_len);
+    status = copy_absolute(prefix, argv0_path, prefix_len);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -485,11 +484,13 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
 static PyStatus
 calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                 const wchar_t *argv0_path,
                  wchar_t *prefix, size_t prefix_len)
 {
     PyStatus status;
 
-    status = search_for_prefix(calculate, pathconfig, prefix, prefix_len,
+    status = search_for_prefix(calculate, pathconfig, argv0_path,
+                               prefix, prefix_len,
                                &calculate->prefix_found);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
@@ -516,8 +517,8 @@ calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
 
 static PyStatus
-calculate_reduce_prefix(PyCalculatePath *calculate,
-                        wchar_t *prefix, size_t prefix_len)
+calculate_set_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                     wchar_t *prefix)
 {
     /* Reduce prefix and exec_prefix to their essence,
      * e.g. /usr/local/lib/python1.5 is reduced to /usr/local.
@@ -532,11 +533,14 @@ calculate_reduce_prefix(PyCalculatePath *calculate,
         if (!prefix[0]) {
             wcscpy(prefix, separator);
         }
+        pathconfig->prefix = _PyMem_RawWcsdup(prefix);
     }
     else {
-        if (safe_wcscpy(prefix, calculate->prefix, prefix_len) < 0) {
-            return PATHLEN_ERR();
-        }
+        pathconfig->prefix = _PyMem_RawWcsdup(calculate->prefix);
+    }
+
+    if (pathconfig->prefix == NULL) {
+        return _PyStatus_NO_MEMORY();
     }
     return _PyStatus_OK();
 }
@@ -547,6 +551,7 @@ calculate_reduce_prefix(PyCalculatePath *calculate,
 */
 static PyStatus
 search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                       const wchar_t *argv0_path,
                        wchar_t *exec_prefix, size_t exec_prefix_len,
                        int *found)
 {
@@ -581,7 +586,7 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
     /* Check to see if argv[0] is in the build directory. "pybuilddir.txt"
        is written by setup.py and contains the relative path to the location
        of shared library modules. */
-    if (safe_wcscpy(exec_prefix, calculate->argv0_path, exec_prefix_len) < 0) {
+    if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
         return PATHLEN_ERR();
     }
     status = joinpath(exec_prefix, L"pybuilddir.txt", exec_prefix_len);
@@ -607,7 +612,7 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
                 return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len);
             }
 
-            if (safe_wcscpy(exec_prefix, calculate->argv0_path, exec_prefix_len) < 0) {
+            if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
                 return PATHLEN_ERR();
             }
             status = joinpath(exec_prefix, pybuilddir, exec_prefix_len);
@@ -622,7 +627,7 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
     }
 
     /* Search from argv0_path, until root is found */
-    status = copy_absolute(exec_prefix, calculate->argv0_path, exec_prefix_len);
+    status = copy_absolute(exec_prefix, argv0_path, exec_prefix_len);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -670,11 +675,12 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
 static PyStatus
 calculate_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                      const wchar_t *argv0_path,
                       wchar_t *exec_prefix, size_t exec_prefix_len)
 {
     PyStatus status;
 
-    status = search_for_exec_prefix(calculate, pathconfig,
+    status = search_for_exec_prefix(calculate, pathconfig, argv0_path,
                                     exec_prefix, exec_prefix_len,
                                     &calculate->exec_prefix_found);
     if (_PyStatus_EXCEPTION(status)) {
@@ -700,8 +706,9 @@ calculate_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
 
 static PyStatus
-calculate_reduce_exec_prefix(PyCalculatePath *calculate,
-                             wchar_t *exec_prefix, size_t exec_prefix_len)
+calculate_set_exec_prefix(PyCalculatePath *calculate,
+                          _PyPathConfig *pathconfig,
+                          wchar_t *exec_prefix)
 {
     if (calculate->exec_prefix_found > 0) {
         reduce(exec_prefix);
@@ -710,12 +717,17 @@ calculate_reduce_exec_prefix(PyCalculatePath *calculate,
         if (!exec_prefix[0]) {
             wcscpy(exec_prefix, separator);
         }
+
+        pathconfig->exec_prefix = _PyMem_RawWcsdup(exec_prefix);
     }
     else {
-        if (safe_wcscpy(exec_prefix, calculate->exec_prefix, exec_prefix_len) < 0) {
-            return PATHLEN_ERR();
-        }
+        pathconfig->exec_prefix = _PyMem_RawWcsdup(calculate->exec_prefix);
+    }
+
+    if (pathconfig->exec_prefix == NULL) {
+        return _PyStatus_NO_MEMORY();
     }
+
     return _PyStatus_OK();
 }
 
@@ -843,10 +855,10 @@ calculate_program_full_path(PyCalculatePath *calculate, _PyPathConfig *pathconfi
 
 
 static PyStatus
-calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_path)
+calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_path,
+                     wchar_t *argv0_path, size_t argv0_path_len)
 {
-    const size_t argv0_path_len = Py_ARRAY_LENGTH(calculate->argv0_path);
-    if (safe_wcscpy(calculate->argv0_path, program_full_path, argv0_path_len) < 0) {
+    if (safe_wcscpy(argv0_path, program_full_path, argv0_path_len) < 0) {
         return PATHLEN_ERR();
     }
 
@@ -877,32 +889,31 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_pat
             return DECODE_LOCALE_ERR("framework location", len);
         }
 
-        if (safe_wcscpy(calculate->argv0_path, wbuf, argv0_path_len) < 0) {
+        if (safe_wcscpy(argv0_path, wbuf, argv0_path_len) < 0) {
             return PATHLEN_ERR();
         }
-        reduce(calculate->argv0_path);
-        status = joinpath(calculate->argv0_path, calculate->lib_python, argv0_path_len);
+        reduce(argv0_path);
+        status = joinpath(argv0_path, calculate->lib_python, argv0_path_len);
         if (_PyStatus_EXCEPTION(status)) {
             PyMem_RawFree(wbuf);
             return status;
         }
-        status = joinpath(calculate->argv0_path, LANDMARK, argv0_path_len);
+        status = joinpath(argv0_path, LANDMARK, argv0_path_len);
         if (_PyStatus_EXCEPTION(status)) {
             PyMem_RawFree(wbuf);
             return status;
         }
-        if (!ismodule(calculate->argv0_path,
-                      Py_ARRAY_LENGTH(calculate->argv0_path))) {
+        if (!ismodule(argv0_path, Py_ARRAY_LENGTH(argv0_path))) {
             /* We are in the build directory so use the name of the
                executable - we know that the absolute path is passed */
-            if (safe_wcscpy(calculate->argv0_path, program_full_path,
+            if (safe_wcscpy(argv0_path, program_full_path,
                             argv0_path_len) < 0) {
                 return PATHLEN_ERR();
             }
         }
         else {
             /* Use the location of the library as the program_full_path */
-            if (safe_wcscpy(calculate->argv0_path, wbuf, argv0_path_len) < 0) {
+            if (safe_wcscpy(argv0_path, wbuf, argv0_path_len) < 0) {
                 return PATHLEN_ERR();
             }
         }
@@ -918,24 +929,24 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_pat
         if (_Py_isabs(tmpbuffer)) {
             /* tmpbuffer should never be longer than MAXPATHLEN,
                but extra check does not hurt */
-            if (safe_wcscpy(calculate->argv0_path, tmpbuffer, argv0_path_len) < 0) {
+            if (safe_wcscpy(argv0_path, tmpbuffer, argv0_path_len) < 0) {
                 return PATHLEN_ERR();
             }
         }
         else {
             /* Interpret relative to program_full_path */
             PyStatus status;
-            reduce(calculate->argv0_path);
-            status = joinpath(calculate->argv0_path, tmpbuffer, argv0_path_len);
+            reduce(argv0_path);
+            status = joinpath(argv0_path, tmpbuffer, argv0_path_len);
             if (_PyStatus_EXCEPTION(status)) {
                 return status;
             }
         }
-        linklen = _Py_wreadlink(calculate->argv0_path, tmpbuffer, buflen);
+        linklen = _Py_wreadlink(argv0_path, tmpbuffer, buflen);
     }
 #endif /* HAVE_READLINK */
 
-    reduce(calculate->argv0_path);
+    reduce(argv0_path);
     /* At this point, argv0_path is guaranteed to be less than
        MAXPATHLEN bytes long. */
     return _PyStatus_OK();
@@ -947,7 +958,8 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_pat
    If found, open it for use when searching for prefixes.
 */
 static PyStatus
-calculate_read_pyenv(PyCalculatePath *calculate)
+calculate_read_pyenv(PyCalculatePath *calculate,
+                     wchar_t *argv0_path, size_t argv0_path_len)
 {
     PyStatus status;
     wchar_t tmpbuffer[MAXPATHLEN+1];
@@ -955,7 +967,7 @@ calculate_read_pyenv(PyCalculatePath *calculate)
     wchar_t *env_cfg = L"pyvenv.cfg";
     FILE *env_file;
 
-    if (safe_wcscpy(tmpbuffer, calculate->argv0_path, buflen) < 0) {
+    if (safe_wcscpy(tmpbuffer, argv0_path, buflen) < 0) {
         return PATHLEN_ERR();
     }
 
@@ -986,8 +998,7 @@ calculate_read_pyenv(PyCalculatePath *calculate)
 
     /* Look for a 'home' variable and set argv0_path to it, if found */
     if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, buflen)) {
-        if (safe_wcscpy(calculate->argv0_path, tmpbuffer,
-                        Py_ARRAY_LENGTH(calculate->argv0_path)) < 0) {
+        if (safe_wcscpy(argv0_path, tmpbuffer, argv0_path_len) < 0) {
             return PATHLEN_ERR();
         }
     }
@@ -997,33 +1008,33 @@ calculate_read_pyenv(PyCalculatePath *calculate)
 
 
 static PyStatus
-calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix)
+calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix,
+                   wchar_t *zip_path, size_t zip_path_len)
 {
     PyStatus status;
-    const size_t zip_path_len = Py_ARRAY_LENGTH(calculate->zip_path);
-    if (safe_wcscpy(calculate->zip_path, prefix, zip_path_len) < 0) {
+    if (safe_wcscpy(zip_path, prefix, zip_path_len) < 0) {
         return PATHLEN_ERR();
     }
 
     if (calculate->prefix_found > 0) {
         /* Use the reduced prefix returned by Py_GetPrefix() */
-        reduce(calculate->zip_path);
-        reduce(calculate->zip_path);
+        reduce(zip_path);
+        reduce(zip_path);
     }
     else {
-        if (safe_wcscpy(calculate->zip_path, calculate->prefix, zip_path_len) < 0) {
+        if (safe_wcscpy(zip_path, calculate->prefix, zip_path_len) < 0) {
             return PATHLEN_ERR();
         }
     }
-    status = joinpath(calculate->zip_path, L"lib/python00.zip", zip_path_len);
+    status = joinpath(zip_path, L"lib/python00.zip", zip_path_len);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     /* Replace "00" with version */
-    size_t bufsz = wcslen(calculate->zip_path);
-    calculate->zip_path[bufsz - 6] = VERSION[0];
-    calculate->zip_path[bufsz - 5] = VERSION[2];
+    size_t bufsz = wcslen(zip_path);
+    zip_path[bufsz - 6] = VERSION[0];
+    zip_path[bufsz - 5] = VERSION[2];
     return _PyStatus_OK();
 }
 
@@ -1031,7 +1042,9 @@ calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix)
 static PyStatus
 calculate_module_search_path(PyCalculatePath *calculate,
                              _PyPathConfig *pathconfig,
-                             const wchar_t *prefix, const wchar_t *exec_prefix)
+                             const wchar_t *prefix,
+                             const wchar_t *exec_prefix,
+                             const wchar_t *zip_path)
 {
     /* Calculate size of return buffer */
     size_t bufsz = 0;
@@ -1059,7 +1072,7 @@ calculate_module_search_path(PyCalculatePath *calculate,
         defpath = delim + 1;
     }
 
-    bufsz += wcslen(calculate->zip_path) + 1;
+    bufsz += wcslen(zip_path) + 1;
     bufsz += wcslen(exec_prefix) + 1;
 
     /* Allocate the buffer */
@@ -1076,7 +1089,7 @@ calculate_module_search_path(PyCalculatePath *calculate,
     }
 
     /* Next is the default zip path */
-    wcscat(buf, calculate->zip_path);
+    wcscat(buf, zip_path);
     wcscat(buf, delimiter);
 
     /* Next goes merge of compile-time $PYTHONPATH with
@@ -1119,8 +1132,7 @@ calculate_module_search_path(PyCalculatePath *calculate,
 
 
 static PyStatus
-calculate_init(PyCalculatePath *calculate,
-               const PyConfig *config)
+calculate_init(PyCalculatePath *calculate, const PyConfig *config)
 {
     size_t len;
     const char *path = getenv("PATH");
@@ -1135,6 +1147,7 @@ calculate_init(PyCalculatePath *calculate,
     if (!calculate->pythonpath) {
         return DECODE_LOCALE_ERR("PYTHONPATH define", len);
     }
+
     calculate->prefix = Py_DecodeLocale(PREFIX, &len);
     if (!calculate->prefix) {
         return DECODE_LOCALE_ERR("PREFIX define", len);
@@ -1178,12 +1191,17 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
         }
     }
 
-    status = calculate_argv0_path(calculate, pathconfig->program_full_path);
+    wchar_t argv0_path[MAXPATHLEN+1];
+    memset(argv0_path, 0, sizeof(argv0_path));
+
+    status = calculate_argv0_path(calculate, pathconfig->program_full_path,
+                                  argv0_path, Py_ARRAY_LENGTH(argv0_path));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
-    status = calculate_read_pyenv(calculate);
+    status = calculate_read_pyenv(calculate,
+                                  argv0_path, Py_ARRAY_LENGTH(argv0_path));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -1191,19 +1209,24 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
     wchar_t prefix[MAXPATHLEN+1];
     memset(prefix, 0, sizeof(prefix));
     status = calculate_prefix(calculate, pathconfig,
+                              argv0_path,
                               prefix, Py_ARRAY_LENGTH(prefix));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
-    status = calculate_zip_path(calculate, prefix);
+    wchar_t zip_path[MAXPATHLEN+1];    /* ".../lib/pythonXY.zip" */
+    memset(zip_path, 0, sizeof(zip_path));
+
+    status = calculate_zip_path(calculate, prefix,
+                                zip_path, Py_ARRAY_LENGTH(zip_path));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     wchar_t exec_prefix[MAXPATHLEN+1];
     memset(exec_prefix, 0, sizeof(exec_prefix));
-    status = calculate_exec_prefix(calculate, pathconfig,
+    status = calculate_exec_prefix(calculate, pathconfig, argv0_path,
                                    exec_prefix, Py_ARRAY_LENGTH(exec_prefix));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
@@ -1218,50 +1241,60 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
 
     if (pathconfig->module_search_path == NULL) {
         status = calculate_module_search_path(calculate, pathconfig,
-                                              prefix, exec_prefix);
+                                              prefix, exec_prefix, zip_path);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
     }
 
     if (pathconfig->prefix == NULL) {
-        status = calculate_reduce_prefix(calculate, prefix, Py_ARRAY_LENGTH(prefix));
+        status = calculate_set_prefix(calculate, pathconfig, prefix);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
-
-        pathconfig->prefix = _PyMem_RawWcsdup(prefix);
-        if (pathconfig->prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
     }
 
     if (pathconfig->exec_prefix == NULL) {
-        status = calculate_reduce_exec_prefix(calculate,
-                                              exec_prefix,
-                                              Py_ARRAY_LENGTH(exec_prefix));
+        status = calculate_set_exec_prefix(calculate, pathconfig, exec_prefix);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
-
-        pathconfig->exec_prefix = _PyMem_RawWcsdup(exec_prefix);
-        if (pathconfig->exec_prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
     }
 
     return _PyStatus_OK();
 }
 
 
-/* Calculate 'pathconfig' attributes:
+/* Calculate the Python path configuration.
+
+   Inputs:
+
+   - PATH environment variable
+   - Macros: PYTHONPATH, PREFIX, EXEC_PREFIX, VERSION (ex: "3.9").
+     PREFIX and EXEC_PREFIX are generated by the configure script.
+     PYTHONPATH macro is the default search path.
+   - pybuilddir.txt file
+   - pyvenv.cfg configuration file
+   - PyConfig fields ('config' function argument):
+
+     - pathconfig_warnings
+     - pythonpath_env (PYTHONPATH environment variable)
+
+   - _PyPathConfig fields ('pathconfig' function argument):
+
+     - program_name: see config_init_program_name()
+     - home: Py_SetPythonHome() or PYTHONHOME environment variable
+
+   - current working directory: see copy_absolute()
+
+   Outputs, 'pathconfig' fields:
 
    - program_full_path
    - module_search_path
    - prefix
    - exec_prefix
 
-   If an attribute is already set (non NULL), it is left unchanged. */
+   If a field is already set (non NULL), it is left unchanged. */
 PyStatus
 _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
 {
diff --git a/PC/getpathp.c b/PC/getpathp.c
index 5670fa2ee4e1..c4c0636dddeb 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -115,20 +115,21 @@
  */
 
 #ifndef LANDMARK
-#define LANDMARK L"lib\\os.py"
+#  define LANDMARK L"lib\\os.py"
 #endif
 
+#define INIT_ERR_BUFFER_OVERFLOW() _PyStatus_ERR("buffer overflow")
+
+
 typedef struct {
     const wchar_t *path_env;           /* PATH environment variable */
     const wchar_t *home;               /* PYTHONHOME environment variable */
 
-    /* Registry key "Software\Python\PythonCore\PythonPath" */
+    /* Registry key "Software\Python\PythonCore\X.Y\PythonPath"
+       where X.Y is the Python version (major.minor) */
     wchar_t *machine_path;   /* from HKEY_LOCAL_MACHINE */
     wchar_t *user_path;      /* from HKEY_CURRENT_USER */
 
-    wchar_t argv0_path[MAXPATHLEN+1];
-    wchar_t zip_path[MAXPATHLEN+1];
-
     wchar_t *dll_path;
 
     const wchar_t *pythonpath_env;
@@ -276,7 +277,10 @@ typedef HRESULT(__stdcall *PPathCchCanonicalizeEx) (PWSTR pszPathOut, size_t cch
     PCWSTR pszPathIn, unsigned long dwFlags);
 static PPathCchCanonicalizeEx _PathCchCanonicalizeEx;
 
-static PyStatus canonicalize(wchar_t *buffer, const wchar_t *path)
+/* Call PathCchCanonicalizeEx(path): remove navigation elements such as "."
+   and ".." to produce a direct, well-formed path. */
+static PyStatus
+canonicalize(wchar_t *buffer, const wchar_t *path)
 {
     if (buffer == NULL) {
         return _PyStatus_NO_MEMORY();
@@ -295,12 +299,12 @@ static PyStatus canonicalize(wchar_t *buffer, const wchar_t *path)
 
     if (_PathCchCanonicalizeEx) {
         if (FAILED(_PathCchCanonicalizeEx(buffer, MAXPATHLEN + 1, path, 0))) {
-            return _PyStatus_ERR("buffer overflow in getpathp.c's canonicalize()");
+            return INIT_ERR_BUFFER_OVERFLOW();
         }
     }
     else {
         if (!PathCanonicalizeW(buffer, path)) {
-            return _PyStatus_ERR("buffer overflow in getpathp.c's canonicalize()");
+            return INIT_ERR_BUFFER_OVERFLOW();
         }
     }
     return _PyStatus_OK();
@@ -588,12 +592,18 @@ get_program_full_path(_PyPathConfig *pathconfig)
 }
 
 
-static int
-read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
+static PyStatus
+read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path,
+              int *found)
 {
-    FILE *sp_file = _Py_wfopen(path, L"r");
+    PyStatus status;
+    wchar_t *buf = NULL;
+    wchar_t *wline = NULL;
+    FILE *sp_file;
+
+    sp_file = _Py_wfopen(path, L"r");
     if (sp_file == NULL) {
-        return 0;
+        return _PyStatus_OK();
     }
 
     wcscpy_s(prefix, MAXPATHLEN+1, path);
@@ -604,15 +614,16 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
     size_t bufsiz = MAXPATHLEN;
     size_t prefixlen = wcslen(prefix);
 
-    wchar_t *buf = (wchar_t*)PyMem_RawMalloc(bufsiz * sizeof(wchar_t));
+    buf = (wchar_t*)PyMem_RawMalloc(bufsiz * sizeof(wchar_t));
     if (buf == NULL) {
-        goto error;
+        status = _PyStatus_NO_MEMORY();
+        goto done;
     }
     buf[0] = '\0';
 
     while (!feof(sp_file)) {
         char line[MAXPATHLEN + 1];
-        char *p = fgets(line, MAXPATHLEN + 1, sp_file);
+        char *p = fgets(line, Py_ARRAY_LENGTH(line), sp_file);
         if (!p) {
             break;
         }
@@ -631,13 +642,16 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
             continue;
         }
         else if (strncmp(line, "import ", 7) == 0) {
-            Py_FatalError("only 'import site' is supported in ._pth file");
+            status = _PyStatus_ERR("only 'import site' is supported "
+                                   "in ._pth file");
+            goto done;
         }
 
         DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, NULL, 0);
         wchar_t *wline = (wchar_t*)PyMem_RawMalloc((wn + 1) * sizeof(wchar_t));
         if (wline == NULL) {
-            goto error;
+            status = _PyStatus_NO_MEMORY();
+            goto done;
         }
         wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, wline, wn + 1);
         wline[wn] = '\0';
@@ -648,8 +662,8 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
             wchar_t *tmp = (wchar_t*)PyMem_RawRealloc(buf, (bufsiz + 1) *
                                                             sizeof(wchar_t));
             if (tmp == NULL) {
-                PyMem_RawFree(wline);
-                goto error;
+                status = _PyStatus_NO_MEMORY();
+                goto done;
             }
             buf = tmp;
         }
@@ -663,48 +677,39 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
         _Py_BEGIN_SUPPRESS_IPH
         result = wcscat_s(buf, bufsiz, prefix);
         _Py_END_SUPPRESS_IPH
+
         if (result == EINVAL) {
-            Py_FatalError("invalid argument during ._pth processing");
+            status = _PyStatus_ERR("invalid argument during ._pth processing");
+            goto done;
         } else if (result == ERANGE) {
-            Py_FatalError("buffer overflow during ._pth processing");
+            status = _PyStatus_ERR("buffer overflow during ._pth processing");
+            goto done;
         }
+
         wchar_t *b = &buf[usedsiz];
         join(b, wline);
 
         PyMem_RawFree(wline);
+        wline = NULL;
     }
 
-    fclose(sp_file);
     if (pathconfig->module_search_path == NULL) {
         pathconfig->module_search_path = _PyMem_RawWcsdup(buf);
         if (pathconfig->module_search_path == NULL) {
-            Py_FatalError("out of memory");
+            status = _PyStatus_NO_MEMORY();
+            goto done;
         }
     }
-    PyMem_RawFree(buf);
-    return 1;
 
-error:
+    *found = 1;
+    status = _PyStatus_OK();
+    goto done;
+
+done:
     PyMem_RawFree(buf);
+    PyMem_RawFree(wline);
     fclose(sp_file);
-    return 0;
-}
-
-
-static PyStatus
-calculate_init(PyCalculatePath *calculate, const PyConfig *config)
-{
-    calculate->home = config->home;
-    calculate->path_env = _wgetenv(L"PATH");
-
-    calculate->dll_path = _Py_GetDLLPath();
-    if (calculate->dll_path == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
-
-    calculate->pythonpath_env = config->pythonpath_env;
-
-    return _PyStatus_OK();
+    return status;
 }
 
 
@@ -730,17 +735,17 @@ get_pth_filename(PyCalculatePath *calculate, wchar_t *filename,
 }
 
 
-static int
+static PyStatus
 calculate_pth_file(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
-                   wchar_t *prefix)
+                   wchar_t *prefix, int *found)
 {
     wchar_t filename[MAXPATHLEN+1];
 
     if (!get_pth_filename(calculate, filename, pathconfig)) {
-        return 0;
+        return _PyStatus_OK();
     }
 
-    return read_pth_file(pathconfig, prefix, filename);
+    return read_pth_file(pathconfig, prefix, filename, found);
 }
 
 
@@ -749,12 +754,13 @@ calculate_pth_file(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
    If found, open it for use when searching for prefixes.
 */
 static void
-calculate_pyvenv_file(PyCalculatePath *calculate)
+calculate_pyvenv_file(PyCalculatePath *calculate,
+                      wchar_t *argv0_path, size_t argv0_path_len)
 {
     wchar_t envbuffer[MAXPATHLEN+1];
     const wchar_t *env_cfg = L"pyvenv.cfg";
 
-    wcscpy_s(envbuffer, MAXPATHLEN+1, calculate->argv0_path);
+    wcscpy_s(envbuffer, MAXPATHLEN+1, argv0_path);
     join(envbuffer, env_cfg);
 
     FILE *env_file = _Py_wfopen(envbuffer, L"r");
@@ -778,25 +784,25 @@ calculate_pyvenv_file(PyCalculatePath *calculate)
     /* Look for a 'home' variable and set argv0_path to it, if found */
     wchar_t tmpbuffer[MAXPATHLEN+1];
     if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, MAXPATHLEN)) {
-        wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, tmpbuffer);
+        wcscpy_s(argv0_path, argv0_path_len, tmpbuffer);
     }
     fclose(env_file);
 }
 
 
-#define INIT_ERR_BUFFER_OVERFLOW() _PyStatus_ERR("buffer overflow")
-
-
 static void
-calculate_home_prefix(PyCalculatePath *calculate, wchar_t *prefix)
+calculate_home_prefix(PyCalculatePath *calculate,
+                      const wchar_t *argv0_path,
+                      const wchar_t *zip_path,
+                      wchar_t *prefix)
 {
     if (calculate->home == NULL || *calculate->home == '\0') {
-        if (calculate->zip_path[0] && exists(calculate->zip_path)) {
-            wcscpy_s(prefix, MAXPATHLEN+1, calculate->zip_path);
+        if (zip_path[0] && exists(zip_path)) {
+            wcscpy_s(prefix, MAXPATHLEN+1, zip_path);
             reduce(prefix);
             calculate->home = prefix;
         }
-        else if (search_for_prefix(prefix, calculate->argv0_path, LANDMARK)) {
+        else if (search_for_prefix(prefix, argv0_path, LANDMARK)) {
             calculate->home = prefix;
         }
         else {
@@ -812,7 +818,9 @@ calculate_home_prefix(PyCalculatePath *calculate, wchar_t *prefix)
 static PyStatus
 calculate_module_search_path(PyCalculatePath *calculate,
                              _PyPathConfig *pathconfig,
-                             wchar_t *prefix)
+                             const wchar_t *argv0_path,
+                             wchar_t *prefix,
+                             const wchar_t *zip_path)
 {
     int skiphome = calculate->home==NULL ? 0 : 1;
 #ifdef Py_ENABLE_SHARED
@@ -852,14 +860,14 @@ calculate_module_search_path(PyCalculatePath *calculate,
         bufsz *= wcslen(calculate->home);
     }
     bufsz += wcslen(PYTHONPATH) + 1;
-    bufsz += wcslen(calculate->argv0_path) + 1;
+    bufsz += wcslen(argv0_path) + 1;
     if (calculate->user_path) {
         bufsz += wcslen(calculate->user_path) + 1;
     }
     if (calculate->machine_path) {
         bufsz += wcslen(calculate->machine_path) + 1;
     }
-    bufsz += wcslen(calculate->zip_path) + 1;
+    bufsz += wcslen(zip_path) + 1;
     if (calculate->pythonpath_env != NULL) {
         bufsz += wcslen(calculate->pythonpath_env) + 1;
     }
@@ -867,7 +875,7 @@ calculate_module_search_path(PyCalculatePath *calculate,
     wchar_t *buf, *start_buf;
     buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t));
     if (buf == NULL) {
-        Py_FatalError("Can't malloc dynamic PYTHONPATH");
+        return _PyStatus_NO_MEMORY();
     }
     start_buf = buf;
 
@@ -879,8 +887,8 @@ calculate_module_search_path(PyCalculatePath *calculate,
         buf = wcschr(buf, L'\0');
         *buf++ = DELIM;
     }
-    if (calculate->zip_path[0]) {
-        if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->zip_path)) {
+    if (zip_path[0]) {
+        if (wcscpy_s(buf, bufsz - (buf - start_buf), zip_path)) {
             return INIT_ERR_BUFFER_OVERFLOW();
         }
         buf = wcschr(buf, L'\0');
@@ -937,8 +945,8 @@ calculate_module_search_path(PyCalculatePath *calculate,
             p = q+1;
         }
     }
-    if (calculate->argv0_path) {
-        wcscpy(buf, calculate->argv0_path);
+    if (argv0_path) {
+        wcscpy(buf, argv0_path);
         buf = wcschr(buf, L'\0');
         *buf++ = DELIM;
     }
@@ -996,28 +1004,40 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
     }
 
     /* program_full_path guaranteed \0 terminated in MAXPATH+1 bytes. */
-    wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, pathconfig->program_full_path);
-    reduce(calculate->argv0_path);
+    wchar_t argv0_path[MAXPATHLEN+1];
+    memset(argv0_path, 0, sizeof(argv0_path));
+
+    wcscpy_s(argv0_path, MAXPATHLEN+1, pathconfig->program_full_path);
+    reduce(argv0_path);
 
     wchar_t prefix[MAXPATHLEN+1];
     memset(prefix, 0, sizeof(prefix));
 
     /* Search for a sys.path file */
-    if (calculate_pth_file(calculate, pathconfig, prefix)) {
+    int pth_found = 0;
+    status = calculate_pth_file(calculate, pathconfig, prefix, &pth_found);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+    if (pth_found) {
         goto done;
     }
 
-    calculate_pyvenv_file(calculate);
+    calculate_pyvenv_file(calculate, argv0_path, Py_ARRAY_LENGTH(argv0_path));
 
     /* Calculate zip archive path from DLL or exe path */
-    change_ext(calculate->zip_path,
+    wchar_t zip_path[MAXPATHLEN+1];
+    memset(zip_path, 0, sizeof(zip_path));
+
+    change_ext(zip_path,
                calculate->dll_path[0] ? calculate->dll_path : pathconfig->program_full_path,
                L".zip");
 
-    calculate_home_prefix(calculate, prefix);
+    calculate_home_prefix(calculate, argv0_path, zip_path, prefix);
 
     if (pathconfig->module_search_path == NULL) {
-        status = calculate_module_search_path(calculate, pathconfig, prefix);
+        status = calculate_module_search_path(calculate, pathconfig,
+                                              argv0_path, prefix, zip_path);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
@@ -1041,6 +1061,24 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
 }
 
 
+static PyStatus
+calculate_init(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+               const PyConfig *config)
+{
+    calculate->home = pathconfig->home;
+    calculate->path_env = _wgetenv(L"PATH");
+
+    calculate->dll_path = _Py_GetDLLPath();
+    if (calculate->dll_path == NULL) {
+        return _PyStatus_NO_MEMORY();
+    }
+
+    calculate->pythonpath_env = config->pythonpath_env;
+
+    return _PyStatus_OK();
+}
+
+
 static void
 calculate_free(PyCalculatePath *calculate)
 {
@@ -1050,7 +1088,24 @@ calculate_free(PyCalculatePath *calculate)
 }
 
 
-/* Calculate 'pathconfig' attributes:
+/* Calculate the Python path configuration.
+
+   Inputs:
+
+   - PyConfig.pythonpath_env: PYTHONPATH environment variable
+   - _PyPathConfig.home: Py_SetPythonHome() or PYTHONHOME environment variable
+   - DLL path: _Py_GetDLLPath()
+   - PATH environment variable
+   - __PYVENV_LAUNCHER__ environment variable
+   - GetModuleFileNameW(NULL): fully qualified path of the executable file of
+     the current process
+   - .pth configuration file
+   - pyvenv.cfg configuration file
+   - Registry key "Software\Python\PythonCore\X.Y\PythonPath"
+     of HKEY_LOCAL_MACHINE and HKEY_CURRENT_USER where X.Y is the Python
+     version (major.minor).
+
+   Outputs, 'pathconfig' fields:
 
    - base_executable
    - program_full_path
@@ -1060,7 +1115,7 @@ calculate_free(PyCalculatePath *calculate)
    - isolated
    - site_import
 
-   If an attribute is already set (non NULL), it is left unchanged. */
+   If a field is already set (non NULL), it is left unchanged. */
 PyStatus
 _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
 {
@@ -1068,7 +1123,7 @@ _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
     PyCalculatePath calculate;
     memset(&calculate, 0, sizeof(calculate));
 
-    status = calculate_init(&calculate, config);
+    status = calculate_init(&calculate, pathconfig, config);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
     }


From webhook-mailer at python.org  Mon Sep 23 19:16:57 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Mon, 23 Sep 2019 23:16:57 -0000
Subject: [Python-checkins] bpo-38234: read_pth_file() now returns PyStatus
 (GH-16338)
Message-ID: <mailman.35.1569280618.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/7f7cd899e3d773b803b7af5b0c19eeff83dd69fe
commit: 7f7cd899e3d773b803b7af5b0c19eeff83dd69fe
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-23T16:16:53-07:00
summary:

bpo-38234: read_pth_file() now returns PyStatus (GH-16338)


Refactor path configuration code:

* read_pth_file() now returns PyStatus to report errors, rather than
  calling Py_FatalError().
* Move argv0_path and zip_path buffers out of PyCalculatePath
  structures.
* On Windows, _PyPathConfig.home is now preferred over PyConfig.home.
(cherry picked from commit 85ce0a7178801b538160cbb5cf9ef50a713c45bf)

Co-authored-by: Victor Stinner <vstinner at redhat.com>

files:
M Modules/getpath.c
M PC/getpathp.c

diff --git a/Modules/getpath.c b/Modules/getpath.c
index f60fe3c015bb..de32c3d51702 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -123,13 +123,11 @@ extern "C" {
 typedef struct {
     wchar_t *path_env;                 /* PATH environment variable */
 
-    wchar_t *pythonpath;               /* PYTHONPATH define */
-    wchar_t *prefix;                   /* PREFIX define */
-    wchar_t *exec_prefix;              /* EXEC_PREFIX define */
+    wchar_t *pythonpath;               /* PYTHONPATH macro */
+    wchar_t *prefix;                   /* PREFIX macro */
+    wchar_t *exec_prefix;              /* EXEC_PREFIX macro */
 
     wchar_t *lib_python;               /* "lib/pythonX.Y" */
-    wchar_t argv0_path[MAXPATHLEN+1];
-    wchar_t zip_path[MAXPATHLEN+1];    /* ".../lib/pythonXY.zip" */
 
     int prefix_found;         /* found platform independent libraries? */
     int exec_prefix_found;    /* found the platform dependent libraries? */
@@ -369,6 +367,7 @@ add_exe_suffix(wchar_t *progpath, size_t progpathlen)
 */
 static PyStatus
 search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                  const wchar_t *argv0_path,
                   wchar_t *prefix, size_t prefix_len, int *found)
 {
     PyStatus status;
@@ -397,7 +396,7 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
     }
 
     /* Check to see if argv[0] is in the build directory */
-    if (safe_wcscpy(prefix, calculate->argv0_path, prefix_len) < 0) {
+    if (safe_wcscpy(prefix, argv0_path, prefix_len) < 0) {
         return PATHLEN_ERR();
     }
     status = joinpath(prefix, L"Modules/Setup.local", prefix_len);
@@ -409,7 +408,7 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
         /* Check VPATH to see if argv0_path is in the build directory. */
         vpath = Py_DecodeLocale(VPATH, NULL);
         if (vpath != NULL) {
-            if (safe_wcscpy(prefix, calculate->argv0_path, prefix_len) < 0) {
+            if (safe_wcscpy(prefix, argv0_path, prefix_len) < 0) {
                 return PATHLEN_ERR();
             }
             status = joinpath(prefix, vpath, prefix_len);
@@ -435,7 +434,7 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
     }
 
     /* Search from argv0_path, until root is found */
-    status = copy_absolute(prefix, calculate->argv0_path, prefix_len);
+    status = copy_absolute(prefix, argv0_path, prefix_len);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -485,11 +484,13 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
 static PyStatus
 calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                 const wchar_t *argv0_path,
                  wchar_t *prefix, size_t prefix_len)
 {
     PyStatus status;
 
-    status = search_for_prefix(calculate, pathconfig, prefix, prefix_len,
+    status = search_for_prefix(calculate, pathconfig, argv0_path,
+                               prefix, prefix_len,
                                &calculate->prefix_found);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
@@ -516,8 +517,8 @@ calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
 
 static PyStatus
-calculate_reduce_prefix(PyCalculatePath *calculate,
-                        wchar_t *prefix, size_t prefix_len)
+calculate_set_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                     wchar_t *prefix)
 {
     /* Reduce prefix and exec_prefix to their essence,
      * e.g. /usr/local/lib/python1.5 is reduced to /usr/local.
@@ -532,11 +533,14 @@ calculate_reduce_prefix(PyCalculatePath *calculate,
         if (!prefix[0]) {
             wcscpy(prefix, separator);
         }
+        pathconfig->prefix = _PyMem_RawWcsdup(prefix);
     }
     else {
-        if (safe_wcscpy(prefix, calculate->prefix, prefix_len) < 0) {
-            return PATHLEN_ERR();
-        }
+        pathconfig->prefix = _PyMem_RawWcsdup(calculate->prefix);
+    }
+
+    if (pathconfig->prefix == NULL) {
+        return _PyStatus_NO_MEMORY();
     }
     return _PyStatus_OK();
 }
@@ -547,6 +551,7 @@ calculate_reduce_prefix(PyCalculatePath *calculate,
 */
 static PyStatus
 search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                       const wchar_t *argv0_path,
                        wchar_t *exec_prefix, size_t exec_prefix_len,
                        int *found)
 {
@@ -581,7 +586,7 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
     /* Check to see if argv[0] is in the build directory. "pybuilddir.txt"
        is written by setup.py and contains the relative path to the location
        of shared library modules. */
-    if (safe_wcscpy(exec_prefix, calculate->argv0_path, exec_prefix_len) < 0) {
+    if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
         return PATHLEN_ERR();
     }
     status = joinpath(exec_prefix, L"pybuilddir.txt", exec_prefix_len);
@@ -607,7 +612,7 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
                 return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len);
             }
 
-            if (safe_wcscpy(exec_prefix, calculate->argv0_path, exec_prefix_len) < 0) {
+            if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
                 return PATHLEN_ERR();
             }
             status = joinpath(exec_prefix, pybuilddir, exec_prefix_len);
@@ -622,7 +627,7 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
     }
 
     /* Search from argv0_path, until root is found */
-    status = copy_absolute(exec_prefix, calculate->argv0_path, exec_prefix_len);
+    status = copy_absolute(exec_prefix, argv0_path, exec_prefix_len);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -670,11 +675,12 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
 static PyStatus
 calculate_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+                      const wchar_t *argv0_path,
                       wchar_t *exec_prefix, size_t exec_prefix_len)
 {
     PyStatus status;
 
-    status = search_for_exec_prefix(calculate, pathconfig,
+    status = search_for_exec_prefix(calculate, pathconfig, argv0_path,
                                     exec_prefix, exec_prefix_len,
                                     &calculate->exec_prefix_found);
     if (_PyStatus_EXCEPTION(status)) {
@@ -700,8 +706,9 @@ calculate_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
 
 static PyStatus
-calculate_reduce_exec_prefix(PyCalculatePath *calculate,
-                             wchar_t *exec_prefix, size_t exec_prefix_len)
+calculate_set_exec_prefix(PyCalculatePath *calculate,
+                          _PyPathConfig *pathconfig,
+                          wchar_t *exec_prefix)
 {
     if (calculate->exec_prefix_found > 0) {
         reduce(exec_prefix);
@@ -710,12 +717,17 @@ calculate_reduce_exec_prefix(PyCalculatePath *calculate,
         if (!exec_prefix[0]) {
             wcscpy(exec_prefix, separator);
         }
+
+        pathconfig->exec_prefix = _PyMem_RawWcsdup(exec_prefix);
     }
     else {
-        if (safe_wcscpy(exec_prefix, calculate->exec_prefix, exec_prefix_len) < 0) {
-            return PATHLEN_ERR();
-        }
+        pathconfig->exec_prefix = _PyMem_RawWcsdup(calculate->exec_prefix);
+    }
+
+    if (pathconfig->exec_prefix == NULL) {
+        return _PyStatus_NO_MEMORY();
     }
+
     return _PyStatus_OK();
 }
 
@@ -843,10 +855,10 @@ calculate_program_full_path(PyCalculatePath *calculate, _PyPathConfig *pathconfi
 
 
 static PyStatus
-calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_path)
+calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_path,
+                     wchar_t *argv0_path, size_t argv0_path_len)
 {
-    const size_t argv0_path_len = Py_ARRAY_LENGTH(calculate->argv0_path);
-    if (safe_wcscpy(calculate->argv0_path, program_full_path, argv0_path_len) < 0) {
+    if (safe_wcscpy(argv0_path, program_full_path, argv0_path_len) < 0) {
         return PATHLEN_ERR();
     }
 
@@ -877,32 +889,31 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_pat
             return DECODE_LOCALE_ERR("framework location", len);
         }
 
-        if (safe_wcscpy(calculate->argv0_path, wbuf, argv0_path_len) < 0) {
+        if (safe_wcscpy(argv0_path, wbuf, argv0_path_len) < 0) {
             return PATHLEN_ERR();
         }
-        reduce(calculate->argv0_path);
-        status = joinpath(calculate->argv0_path, calculate->lib_python, argv0_path_len);
+        reduce(argv0_path);
+        status = joinpath(argv0_path, calculate->lib_python, argv0_path_len);
         if (_PyStatus_EXCEPTION(status)) {
             PyMem_RawFree(wbuf);
             return status;
         }
-        status = joinpath(calculate->argv0_path, LANDMARK, argv0_path_len);
+        status = joinpath(argv0_path, LANDMARK, argv0_path_len);
         if (_PyStatus_EXCEPTION(status)) {
             PyMem_RawFree(wbuf);
             return status;
         }
-        if (!ismodule(calculate->argv0_path,
-                      Py_ARRAY_LENGTH(calculate->argv0_path))) {
+        if (!ismodule(argv0_path, Py_ARRAY_LENGTH(argv0_path))) {
             /* We are in the build directory so use the name of the
                executable - we know that the absolute path is passed */
-            if (safe_wcscpy(calculate->argv0_path, program_full_path,
+            if (safe_wcscpy(argv0_path, program_full_path,
                             argv0_path_len) < 0) {
                 return PATHLEN_ERR();
             }
         }
         else {
             /* Use the location of the library as the program_full_path */
-            if (safe_wcscpy(calculate->argv0_path, wbuf, argv0_path_len) < 0) {
+            if (safe_wcscpy(argv0_path, wbuf, argv0_path_len) < 0) {
                 return PATHLEN_ERR();
             }
         }
@@ -918,24 +929,24 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_pat
         if (tmpbuffer[0] == SEP) {
             /* tmpbuffer should never be longer than MAXPATHLEN,
                but extra check does not hurt */
-            if (safe_wcscpy(calculate->argv0_path, tmpbuffer, argv0_path_len) < 0) {
+            if (safe_wcscpy(argv0_path, tmpbuffer, argv0_path_len) < 0) {
                 return PATHLEN_ERR();
             }
         }
         else {
             /* Interpret relative to program_full_path */
             PyStatus status;
-            reduce(calculate->argv0_path);
-            status = joinpath(calculate->argv0_path, tmpbuffer, argv0_path_len);
+            reduce(argv0_path);
+            status = joinpath(argv0_path, tmpbuffer, argv0_path_len);
             if (_PyStatus_EXCEPTION(status)) {
                 return status;
             }
         }
-        linklen = _Py_wreadlink(calculate->argv0_path, tmpbuffer, buflen);
+        linklen = _Py_wreadlink(argv0_path, tmpbuffer, buflen);
     }
 #endif /* HAVE_READLINK */
 
-    reduce(calculate->argv0_path);
+    reduce(argv0_path);
     /* At this point, argv0_path is guaranteed to be less than
        MAXPATHLEN bytes long. */
     return _PyStatus_OK();
@@ -947,7 +958,8 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_pat
    If found, open it for use when searching for prefixes.
 */
 static PyStatus
-calculate_read_pyenv(PyCalculatePath *calculate)
+calculate_read_pyenv(PyCalculatePath *calculate,
+                     wchar_t *argv0_path, size_t argv0_path_len)
 {
     PyStatus status;
     wchar_t tmpbuffer[MAXPATHLEN+1];
@@ -955,7 +967,7 @@ calculate_read_pyenv(PyCalculatePath *calculate)
     wchar_t *env_cfg = L"pyvenv.cfg";
     FILE *env_file;
 
-    if (safe_wcscpy(tmpbuffer, calculate->argv0_path, buflen) < 0) {
+    if (safe_wcscpy(tmpbuffer, argv0_path, buflen) < 0) {
         return PATHLEN_ERR();
     }
 
@@ -986,8 +998,7 @@ calculate_read_pyenv(PyCalculatePath *calculate)
 
     /* Look for a 'home' variable and set argv0_path to it, if found */
     if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, buflen)) {
-        if (safe_wcscpy(calculate->argv0_path, tmpbuffer,
-                        Py_ARRAY_LENGTH(calculate->argv0_path)) < 0) {
+        if (safe_wcscpy(argv0_path, tmpbuffer, argv0_path_len) < 0) {
             return PATHLEN_ERR();
         }
     }
@@ -997,33 +1008,33 @@ calculate_read_pyenv(PyCalculatePath *calculate)
 
 
 static PyStatus
-calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix)
+calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix,
+                   wchar_t *zip_path, size_t zip_path_len)
 {
     PyStatus status;
-    const size_t zip_path_len = Py_ARRAY_LENGTH(calculate->zip_path);
-    if (safe_wcscpy(calculate->zip_path, prefix, zip_path_len) < 0) {
+    if (safe_wcscpy(zip_path, prefix, zip_path_len) < 0) {
         return PATHLEN_ERR();
     }
 
     if (calculate->prefix_found > 0) {
         /* Use the reduced prefix returned by Py_GetPrefix() */
-        reduce(calculate->zip_path);
-        reduce(calculate->zip_path);
+        reduce(zip_path);
+        reduce(zip_path);
     }
     else {
-        if (safe_wcscpy(calculate->zip_path, calculate->prefix, zip_path_len) < 0) {
+        if (safe_wcscpy(zip_path, calculate->prefix, zip_path_len) < 0) {
             return PATHLEN_ERR();
         }
     }
-    status = joinpath(calculate->zip_path, L"lib/python00.zip", zip_path_len);
+    status = joinpath(zip_path, L"lib/python00.zip", zip_path_len);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     /* Replace "00" with version */
-    size_t bufsz = wcslen(calculate->zip_path);
-    calculate->zip_path[bufsz - 6] = VERSION[0];
-    calculate->zip_path[bufsz - 5] = VERSION[2];
+    size_t bufsz = wcslen(zip_path);
+    zip_path[bufsz - 6] = VERSION[0];
+    zip_path[bufsz - 5] = VERSION[2];
     return _PyStatus_OK();
 }
 
@@ -1031,7 +1042,9 @@ calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix)
 static PyStatus
 calculate_module_search_path(PyCalculatePath *calculate,
                              _PyPathConfig *pathconfig,
-                             const wchar_t *prefix, const wchar_t *exec_prefix)
+                             const wchar_t *prefix,
+                             const wchar_t *exec_prefix,
+                             const wchar_t *zip_path)
 {
     /* Calculate size of return buffer */
     size_t bufsz = 0;
@@ -1059,7 +1072,7 @@ calculate_module_search_path(PyCalculatePath *calculate,
         defpath = delim + 1;
     }
 
-    bufsz += wcslen(calculate->zip_path) + 1;
+    bufsz += wcslen(zip_path) + 1;
     bufsz += wcslen(exec_prefix) + 1;
 
     /* Allocate the buffer */
@@ -1076,7 +1089,7 @@ calculate_module_search_path(PyCalculatePath *calculate,
     }
 
     /* Next is the default zip path */
-    wcscat(buf, calculate->zip_path);
+    wcscat(buf, zip_path);
     wcscat(buf, delimiter);
 
     /* Next goes merge of compile-time $PYTHONPATH with
@@ -1119,8 +1132,7 @@ calculate_module_search_path(PyCalculatePath *calculate,
 
 
 static PyStatus
-calculate_init(PyCalculatePath *calculate,
-               const PyConfig *config)
+calculate_init(PyCalculatePath *calculate, const PyConfig *config)
 {
     size_t len;
     const char *path = getenv("PATH");
@@ -1135,6 +1147,7 @@ calculate_init(PyCalculatePath *calculate,
     if (!calculate->pythonpath) {
         return DECODE_LOCALE_ERR("PYTHONPATH define", len);
     }
+
     calculate->prefix = Py_DecodeLocale(PREFIX, &len);
     if (!calculate->prefix) {
         return DECODE_LOCALE_ERR("PREFIX define", len);
@@ -1178,12 +1191,17 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
         }
     }
 
-    status = calculate_argv0_path(calculate, pathconfig->program_full_path);
+    wchar_t argv0_path[MAXPATHLEN+1];
+    memset(argv0_path, 0, sizeof(argv0_path));
+
+    status = calculate_argv0_path(calculate, pathconfig->program_full_path,
+                                  argv0_path, Py_ARRAY_LENGTH(argv0_path));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
-    status = calculate_read_pyenv(calculate);
+    status = calculate_read_pyenv(calculate,
+                                  argv0_path, Py_ARRAY_LENGTH(argv0_path));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -1191,19 +1209,24 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
     wchar_t prefix[MAXPATHLEN+1];
     memset(prefix, 0, sizeof(prefix));
     status = calculate_prefix(calculate, pathconfig,
+                              argv0_path,
                               prefix, Py_ARRAY_LENGTH(prefix));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
-    status = calculate_zip_path(calculate, prefix);
+    wchar_t zip_path[MAXPATHLEN+1];    /* ".../lib/pythonXY.zip" */
+    memset(zip_path, 0, sizeof(zip_path));
+
+    status = calculate_zip_path(calculate, prefix,
+                                zip_path, Py_ARRAY_LENGTH(zip_path));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     wchar_t exec_prefix[MAXPATHLEN+1];
     memset(exec_prefix, 0, sizeof(exec_prefix));
-    status = calculate_exec_prefix(calculate, pathconfig,
+    status = calculate_exec_prefix(calculate, pathconfig, argv0_path,
                                    exec_prefix, Py_ARRAY_LENGTH(exec_prefix));
     if (_PyStatus_EXCEPTION(status)) {
         return status;
@@ -1218,50 +1241,60 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
 
     if (pathconfig->module_search_path == NULL) {
         status = calculate_module_search_path(calculate, pathconfig,
-                                              prefix, exec_prefix);
+                                              prefix, exec_prefix, zip_path);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
     }
 
     if (pathconfig->prefix == NULL) {
-        status = calculate_reduce_prefix(calculate, prefix, Py_ARRAY_LENGTH(prefix));
+        status = calculate_set_prefix(calculate, pathconfig, prefix);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
-
-        pathconfig->prefix = _PyMem_RawWcsdup(prefix);
-        if (pathconfig->prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
     }
 
     if (pathconfig->exec_prefix == NULL) {
-        status = calculate_reduce_exec_prefix(calculate,
-                                              exec_prefix,
-                                              Py_ARRAY_LENGTH(exec_prefix));
+        status = calculate_set_exec_prefix(calculate, pathconfig, exec_prefix);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
-
-        pathconfig->exec_prefix = _PyMem_RawWcsdup(exec_prefix);
-        if (pathconfig->exec_prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
     }
 
     return _PyStatus_OK();
 }
 
 
-/* Calculate 'pathconfig' attributes:
+/* Calculate the Python path configuration.
+
+   Inputs:
+
+   - PATH environment variable
+   - Macros: PYTHONPATH, PREFIX, EXEC_PREFIX, VERSION (ex: "3.9").
+     PREFIX and EXEC_PREFIX are generated by the configure script.
+     PYTHONPATH macro is the default search path.
+   - pybuilddir.txt file
+   - pyvenv.cfg configuration file
+   - PyConfig fields ('config' function argument):
+
+     - pathconfig_warnings
+     - pythonpath_env (PYTHONPATH environment variable)
+
+   - _PyPathConfig fields ('pathconfig' function argument):
+
+     - program_name: see config_init_program_name()
+     - home: Py_SetPythonHome() or PYTHONHOME environment variable
+
+   - current working directory: see copy_absolute()
+
+   Outputs, 'pathconfig' fields:
 
    - program_full_path
    - module_search_path
    - prefix
    - exec_prefix
 
-   If an attribute is already set (non NULL), it is left unchanged. */
+   If a field is already set (non NULL), it is left unchanged. */
 PyStatus
 _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
 {
diff --git a/PC/getpathp.c b/PC/getpathp.c
index 5670fa2ee4e1..c4c0636dddeb 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -115,20 +115,21 @@
  */
 
 #ifndef LANDMARK
-#define LANDMARK L"lib\\os.py"
+#  define LANDMARK L"lib\\os.py"
 #endif
 
+#define INIT_ERR_BUFFER_OVERFLOW() _PyStatus_ERR("buffer overflow")
+
+
 typedef struct {
     const wchar_t *path_env;           /* PATH environment variable */
     const wchar_t *home;               /* PYTHONHOME environment variable */
 
-    /* Registry key "Software\Python\PythonCore\PythonPath" */
+    /* Registry key "Software\Python\PythonCore\X.Y\PythonPath"
+       where X.Y is the Python version (major.minor) */
     wchar_t *machine_path;   /* from HKEY_LOCAL_MACHINE */
     wchar_t *user_path;      /* from HKEY_CURRENT_USER */
 
-    wchar_t argv0_path[MAXPATHLEN+1];
-    wchar_t zip_path[MAXPATHLEN+1];
-
     wchar_t *dll_path;
 
     const wchar_t *pythonpath_env;
@@ -276,7 +277,10 @@ typedef HRESULT(__stdcall *PPathCchCanonicalizeEx) (PWSTR pszPathOut, size_t cch
     PCWSTR pszPathIn, unsigned long dwFlags);
 static PPathCchCanonicalizeEx _PathCchCanonicalizeEx;
 
-static PyStatus canonicalize(wchar_t *buffer, const wchar_t *path)
+/* Call PathCchCanonicalizeEx(path): remove navigation elements such as "."
+   and ".." to produce a direct, well-formed path. */
+static PyStatus
+canonicalize(wchar_t *buffer, const wchar_t *path)
 {
     if (buffer == NULL) {
         return _PyStatus_NO_MEMORY();
@@ -295,12 +299,12 @@ static PyStatus canonicalize(wchar_t *buffer, const wchar_t *path)
 
     if (_PathCchCanonicalizeEx) {
         if (FAILED(_PathCchCanonicalizeEx(buffer, MAXPATHLEN + 1, path, 0))) {
-            return _PyStatus_ERR("buffer overflow in getpathp.c's canonicalize()");
+            return INIT_ERR_BUFFER_OVERFLOW();
         }
     }
     else {
         if (!PathCanonicalizeW(buffer, path)) {
-            return _PyStatus_ERR("buffer overflow in getpathp.c's canonicalize()");
+            return INIT_ERR_BUFFER_OVERFLOW();
         }
     }
     return _PyStatus_OK();
@@ -588,12 +592,18 @@ get_program_full_path(_PyPathConfig *pathconfig)
 }
 
 
-static int
-read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
+static PyStatus
+read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path,
+              int *found)
 {
-    FILE *sp_file = _Py_wfopen(path, L"r");
+    PyStatus status;
+    wchar_t *buf = NULL;
+    wchar_t *wline = NULL;
+    FILE *sp_file;
+
+    sp_file = _Py_wfopen(path, L"r");
     if (sp_file == NULL) {
-        return 0;
+        return _PyStatus_OK();
     }
 
     wcscpy_s(prefix, MAXPATHLEN+1, path);
@@ -604,15 +614,16 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
     size_t bufsiz = MAXPATHLEN;
     size_t prefixlen = wcslen(prefix);
 
-    wchar_t *buf = (wchar_t*)PyMem_RawMalloc(bufsiz * sizeof(wchar_t));
+    buf = (wchar_t*)PyMem_RawMalloc(bufsiz * sizeof(wchar_t));
     if (buf == NULL) {
-        goto error;
+        status = _PyStatus_NO_MEMORY();
+        goto done;
     }
     buf[0] = '\0';
 
     while (!feof(sp_file)) {
         char line[MAXPATHLEN + 1];
-        char *p = fgets(line, MAXPATHLEN + 1, sp_file);
+        char *p = fgets(line, Py_ARRAY_LENGTH(line), sp_file);
         if (!p) {
             break;
         }
@@ -631,13 +642,16 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
             continue;
         }
         else if (strncmp(line, "import ", 7) == 0) {
-            Py_FatalError("only 'import site' is supported in ._pth file");
+            status = _PyStatus_ERR("only 'import site' is supported "
+                                   "in ._pth file");
+            goto done;
         }
 
         DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, NULL, 0);
         wchar_t *wline = (wchar_t*)PyMem_RawMalloc((wn + 1) * sizeof(wchar_t));
         if (wline == NULL) {
-            goto error;
+            status = _PyStatus_NO_MEMORY();
+            goto done;
         }
         wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, wline, wn + 1);
         wline[wn] = '\0';
@@ -648,8 +662,8 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
             wchar_t *tmp = (wchar_t*)PyMem_RawRealloc(buf, (bufsiz + 1) *
                                                             sizeof(wchar_t));
             if (tmp == NULL) {
-                PyMem_RawFree(wline);
-                goto error;
+                status = _PyStatus_NO_MEMORY();
+                goto done;
             }
             buf = tmp;
         }
@@ -663,48 +677,39 @@ read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path)
         _Py_BEGIN_SUPPRESS_IPH
         result = wcscat_s(buf, bufsiz, prefix);
         _Py_END_SUPPRESS_IPH
+
         if (result == EINVAL) {
-            Py_FatalError("invalid argument during ._pth processing");
+            status = _PyStatus_ERR("invalid argument during ._pth processing");
+            goto done;
         } else if (result == ERANGE) {
-            Py_FatalError("buffer overflow during ._pth processing");
+            status = _PyStatus_ERR("buffer overflow during ._pth processing");
+            goto done;
         }
+
         wchar_t *b = &buf[usedsiz];
         join(b, wline);
 
         PyMem_RawFree(wline);
+        wline = NULL;
     }
 
-    fclose(sp_file);
     if (pathconfig->module_search_path == NULL) {
         pathconfig->module_search_path = _PyMem_RawWcsdup(buf);
         if (pathconfig->module_search_path == NULL) {
-            Py_FatalError("out of memory");
+            status = _PyStatus_NO_MEMORY();
+            goto done;
         }
     }
-    PyMem_RawFree(buf);
-    return 1;
 
-error:
+    *found = 1;
+    status = _PyStatus_OK();
+    goto done;
+
+done:
     PyMem_RawFree(buf);
+    PyMem_RawFree(wline);
     fclose(sp_file);
-    return 0;
-}
-
-
-static PyStatus
-calculate_init(PyCalculatePath *calculate, const PyConfig *config)
-{
-    calculate->home = config->home;
-    calculate->path_env = _wgetenv(L"PATH");
-
-    calculate->dll_path = _Py_GetDLLPath();
-    if (calculate->dll_path == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
-
-    calculate->pythonpath_env = config->pythonpath_env;
-
-    return _PyStatus_OK();
+    return status;
 }
 
 
@@ -730,17 +735,17 @@ get_pth_filename(PyCalculatePath *calculate, wchar_t *filename,
 }
 
 
-static int
+static PyStatus
 calculate_pth_file(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
-                   wchar_t *prefix)
+                   wchar_t *prefix, int *found)
 {
     wchar_t filename[MAXPATHLEN+1];
 
     if (!get_pth_filename(calculate, filename, pathconfig)) {
-        return 0;
+        return _PyStatus_OK();
     }
 
-    return read_pth_file(pathconfig, prefix, filename);
+    return read_pth_file(pathconfig, prefix, filename, found);
 }
 
 
@@ -749,12 +754,13 @@ calculate_pth_file(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
    If found, open it for use when searching for prefixes.
 */
 static void
-calculate_pyvenv_file(PyCalculatePath *calculate)
+calculate_pyvenv_file(PyCalculatePath *calculate,
+                      wchar_t *argv0_path, size_t argv0_path_len)
 {
     wchar_t envbuffer[MAXPATHLEN+1];
     const wchar_t *env_cfg = L"pyvenv.cfg";
 
-    wcscpy_s(envbuffer, MAXPATHLEN+1, calculate->argv0_path);
+    wcscpy_s(envbuffer, MAXPATHLEN+1, argv0_path);
     join(envbuffer, env_cfg);
 
     FILE *env_file = _Py_wfopen(envbuffer, L"r");
@@ -778,25 +784,25 @@ calculate_pyvenv_file(PyCalculatePath *calculate)
     /* Look for a 'home' variable and set argv0_path to it, if found */
     wchar_t tmpbuffer[MAXPATHLEN+1];
     if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, MAXPATHLEN)) {
-        wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, tmpbuffer);
+        wcscpy_s(argv0_path, argv0_path_len, tmpbuffer);
     }
     fclose(env_file);
 }
 
 
-#define INIT_ERR_BUFFER_OVERFLOW() _PyStatus_ERR("buffer overflow")
-
-
 static void
-calculate_home_prefix(PyCalculatePath *calculate, wchar_t *prefix)
+calculate_home_prefix(PyCalculatePath *calculate,
+                      const wchar_t *argv0_path,
+                      const wchar_t *zip_path,
+                      wchar_t *prefix)
 {
     if (calculate->home == NULL || *calculate->home == '\0') {
-        if (calculate->zip_path[0] && exists(calculate->zip_path)) {
-            wcscpy_s(prefix, MAXPATHLEN+1, calculate->zip_path);
+        if (zip_path[0] && exists(zip_path)) {
+            wcscpy_s(prefix, MAXPATHLEN+1, zip_path);
             reduce(prefix);
             calculate->home = prefix;
         }
-        else if (search_for_prefix(prefix, calculate->argv0_path, LANDMARK)) {
+        else if (search_for_prefix(prefix, argv0_path, LANDMARK)) {
             calculate->home = prefix;
         }
         else {
@@ -812,7 +818,9 @@ calculate_home_prefix(PyCalculatePath *calculate, wchar_t *prefix)
 static PyStatus
 calculate_module_search_path(PyCalculatePath *calculate,
                              _PyPathConfig *pathconfig,
-                             wchar_t *prefix)
+                             const wchar_t *argv0_path,
+                             wchar_t *prefix,
+                             const wchar_t *zip_path)
 {
     int skiphome = calculate->home==NULL ? 0 : 1;
 #ifdef Py_ENABLE_SHARED
@@ -852,14 +860,14 @@ calculate_module_search_path(PyCalculatePath *calculate,
         bufsz *= wcslen(calculate->home);
     }
     bufsz += wcslen(PYTHONPATH) + 1;
-    bufsz += wcslen(calculate->argv0_path) + 1;
+    bufsz += wcslen(argv0_path) + 1;
     if (calculate->user_path) {
         bufsz += wcslen(calculate->user_path) + 1;
     }
     if (calculate->machine_path) {
         bufsz += wcslen(calculate->machine_path) + 1;
     }
-    bufsz += wcslen(calculate->zip_path) + 1;
+    bufsz += wcslen(zip_path) + 1;
     if (calculate->pythonpath_env != NULL) {
         bufsz += wcslen(calculate->pythonpath_env) + 1;
     }
@@ -867,7 +875,7 @@ calculate_module_search_path(PyCalculatePath *calculate,
     wchar_t *buf, *start_buf;
     buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t));
     if (buf == NULL) {
-        Py_FatalError("Can't malloc dynamic PYTHONPATH");
+        return _PyStatus_NO_MEMORY();
     }
     start_buf = buf;
 
@@ -879,8 +887,8 @@ calculate_module_search_path(PyCalculatePath *calculate,
         buf = wcschr(buf, L'\0');
         *buf++ = DELIM;
     }
-    if (calculate->zip_path[0]) {
-        if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->zip_path)) {
+    if (zip_path[0]) {
+        if (wcscpy_s(buf, bufsz - (buf - start_buf), zip_path)) {
             return INIT_ERR_BUFFER_OVERFLOW();
         }
         buf = wcschr(buf, L'\0');
@@ -937,8 +945,8 @@ calculate_module_search_path(PyCalculatePath *calculate,
             p = q+1;
         }
     }
-    if (calculate->argv0_path) {
-        wcscpy(buf, calculate->argv0_path);
+    if (argv0_path) {
+        wcscpy(buf, argv0_path);
         buf = wcschr(buf, L'\0');
         *buf++ = DELIM;
     }
@@ -996,28 +1004,40 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
     }
 
     /* program_full_path guaranteed \0 terminated in MAXPATH+1 bytes. */
-    wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, pathconfig->program_full_path);
-    reduce(calculate->argv0_path);
+    wchar_t argv0_path[MAXPATHLEN+1];
+    memset(argv0_path, 0, sizeof(argv0_path));
+
+    wcscpy_s(argv0_path, MAXPATHLEN+1, pathconfig->program_full_path);
+    reduce(argv0_path);
 
     wchar_t prefix[MAXPATHLEN+1];
     memset(prefix, 0, sizeof(prefix));
 
     /* Search for a sys.path file */
-    if (calculate_pth_file(calculate, pathconfig, prefix)) {
+    int pth_found = 0;
+    status = calculate_pth_file(calculate, pathconfig, prefix, &pth_found);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+    if (pth_found) {
         goto done;
     }
 
-    calculate_pyvenv_file(calculate);
+    calculate_pyvenv_file(calculate, argv0_path, Py_ARRAY_LENGTH(argv0_path));
 
     /* Calculate zip archive path from DLL or exe path */
-    change_ext(calculate->zip_path,
+    wchar_t zip_path[MAXPATHLEN+1];
+    memset(zip_path, 0, sizeof(zip_path));
+
+    change_ext(zip_path,
                calculate->dll_path[0] ? calculate->dll_path : pathconfig->program_full_path,
                L".zip");
 
-    calculate_home_prefix(calculate, prefix);
+    calculate_home_prefix(calculate, argv0_path, zip_path, prefix);
 
     if (pathconfig->module_search_path == NULL) {
-        status = calculate_module_search_path(calculate, pathconfig, prefix);
+        status = calculate_module_search_path(calculate, pathconfig,
+                                              argv0_path, prefix, zip_path);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
@@ -1041,6 +1061,24 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
 }
 
 
+static PyStatus
+calculate_init(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
+               const PyConfig *config)
+{
+    calculate->home = pathconfig->home;
+    calculate->path_env = _wgetenv(L"PATH");
+
+    calculate->dll_path = _Py_GetDLLPath();
+    if (calculate->dll_path == NULL) {
+        return _PyStatus_NO_MEMORY();
+    }
+
+    calculate->pythonpath_env = config->pythonpath_env;
+
+    return _PyStatus_OK();
+}
+
+
 static void
 calculate_free(PyCalculatePath *calculate)
 {
@@ -1050,7 +1088,24 @@ calculate_free(PyCalculatePath *calculate)
 }
 
 
-/* Calculate 'pathconfig' attributes:
+/* Calculate the Python path configuration.
+
+   Inputs:
+
+   - PyConfig.pythonpath_env: PYTHONPATH environment variable
+   - _PyPathConfig.home: Py_SetPythonHome() or PYTHONHOME environment variable
+   - DLL path: _Py_GetDLLPath()
+   - PATH environment variable
+   - __PYVENV_LAUNCHER__ environment variable
+   - GetModuleFileNameW(NULL): fully qualified path of the executable file of
+     the current process
+   - .pth configuration file
+   - pyvenv.cfg configuration file
+   - Registry key "Software\Python\PythonCore\X.Y\PythonPath"
+     of HKEY_LOCAL_MACHINE and HKEY_CURRENT_USER where X.Y is the Python
+     version (major.minor).
+
+   Outputs, 'pathconfig' fields:
 
    - base_executable
    - program_full_path
@@ -1060,7 +1115,7 @@ calculate_free(PyCalculatePath *calculate)
    - isolated
    - site_import
 
-   If an attribute is already set (non NULL), it is left unchanged. */
+   If a field is already set (non NULL), it is left unchanged. */
 PyStatus
 _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
 {
@@ -1068,7 +1123,7 @@ _PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
     PyCalculatePath calculate;
     memset(&calculate, 0, sizeof(calculate));
 
-    status = calculate_init(&calculate, config);
+    status = calculate_init(&calculate, pathconfig, config);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
     }


From webhook-mailer at python.org  Mon Sep 23 22:21:10 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Tue, 24 Sep 2019 02:21:10 -0000
Subject: [Python-checkins] closes bpo-38253: Fix typo of
 Py_SET_ERANGE_IF_OVERFLOW in pyport.h. (GH-16230)
Message-ID: <mailman.36.1569291671.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/4346bad3321699d49a45e3446270b57726ab5c8f
commit: 4346bad3321699d49a45e3446270b57726ab5c8f
branch: master
author: Hai Shi <shihai1992 at gmail.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-23T19:20:47-07:00
summary:

closes bpo-38253: Fix typo of Py_SET_ERANGE_IF_OVERFLOW in pyport.h. (GH-16230)

files:
M Include/pyport.h

diff --git a/Include/pyport.h b/Include/pyport.h
index 19415001c5b0..51967d186672 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -338,7 +338,7 @@ extern "C" {
         } \
     } while(0)
 
-/* Py_SET_ERANGE_ON_OVERFLOW(x)
+/* Py_SET_ERANGE_IF_OVERFLOW(x)
  * An alias of Py_SET_ERRNO_ON_MATH_ERROR for backward-compatibility.
  */
 #define Py_SET_ERANGE_IF_OVERFLOW(X) Py_SET_ERRNO_ON_MATH_ERROR(X)


From webhook-mailer at python.org  Mon Sep 23 22:31:30 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 24 Sep 2019 02:31:30 -0000
Subject: [Python-checkins] closes bpo-38253: Fix typo of
 Py_SET_ERANGE_IF_OVERFLOW in pyport.h. (GH-16230)
Message-ID: <mailman.37.1569292292.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/7c65adf688dfadb113eec64374d8d5ccb4fc892b
commit: 7c65adf688dfadb113eec64374d8d5ccb4fc892b
branch: 2.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-23T19:31:25-07:00
summary:

closes bpo-38253: Fix typo of Py_SET_ERANGE_IF_OVERFLOW in pyport.h. (GH-16230)

(cherry picked from commit 4346bad3321699d49a45e3446270b57726ab5c8f)

Co-authored-by: Hai Shi <shihai1992 at gmail.com>

files:
M Include/pyport.h

diff --git a/Include/pyport.h b/Include/pyport.h
index 0c78a1e5b668..35562a6a89e6 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -481,7 +481,7 @@ extern "C" {
         } \
     } while(0)
 
-/* Py_SET_ERANGE_ON_OVERFLOW(x)
+/* Py_SET_ERANGE_IF_OVERFLOW(x)
  * An alias of Py_SET_ERRNO_ON_MATH_ERROR for backward-compatibility.
  */
 #define Py_SET_ERANGE_IF_OVERFLOW(X) Py_SET_ERRNO_ON_MATH_ERROR(X)


From webhook-mailer at python.org  Mon Sep 23 22:39:41 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 24 Sep 2019 02:39:41 -0000
Subject: [Python-checkins] closes bpo-38253: Fix typo of
 Py_SET_ERANGE_IF_OVERFLOW in pyport.h. (GH-16230)
Message-ID: <mailman.38.1569292782.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/912b5a986c4fe0f4d758d7536ce818b1ab672437
commit: 912b5a986c4fe0f4d758d7536ce818b1ab672437
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-23T19:39:36-07:00
summary:

closes bpo-38253: Fix typo of Py_SET_ERANGE_IF_OVERFLOW in pyport.h. (GH-16230)

(cherry picked from commit 4346bad3321699d49a45e3446270b57726ab5c8f)

Co-authored-by: Hai Shi <shihai1992 at gmail.com>

files:
M Include/pyport.h

diff --git a/Include/pyport.h b/Include/pyport.h
index c1f4c7fbb52c..27e51b42c617 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -324,7 +324,7 @@ extern "C" {
         } \
     } while(0)
 
-/* Py_SET_ERANGE_ON_OVERFLOW(x)
+/* Py_SET_ERANGE_IF_OVERFLOW(x)
  * An alias of Py_SET_ERRNO_ON_MATH_ERROR for backward-compatibility.
  */
 #define Py_SET_ERANGE_IF_OVERFLOW(X) Py_SET_ERRNO_ON_MATH_ERROR(X)


From webhook-mailer at python.org  Mon Sep 23 22:41:10 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 24 Sep 2019 02:41:10 -0000
Subject: [Python-checkins] closes bpo-38253: Fix typo of
 Py_SET_ERANGE_IF_OVERFLOW in pyport.h. (GH-16230)
Message-ID: <mailman.39.1569292871.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/a7c73a0094c336e5f10d9364f244be70683409ff
commit: a7c73a0094c336e5f10d9364f244be70683409ff
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-23T19:41:06-07:00
summary:

closes bpo-38253: Fix typo of Py_SET_ERANGE_IF_OVERFLOW in pyport.h. (GH-16230)

(cherry picked from commit 4346bad3321699d49a45e3446270b57726ab5c8f)

Co-authored-by: Hai Shi <shihai1992 at gmail.com>

files:
M Include/pyport.h

diff --git a/Include/pyport.h b/Include/pyport.h
index 32d98c59a7c1..71f5794ded72 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -345,7 +345,7 @@ extern "C" {
         } \
     } while(0)
 
-/* Py_SET_ERANGE_ON_OVERFLOW(x)
+/* Py_SET_ERANGE_IF_OVERFLOW(x)
  * An alias of Py_SET_ERRNO_ON_MATH_ERROR for backward-compatibility.
  */
 #define Py_SET_ERANGE_IF_OVERFLOW(X) Py_SET_ERRNO_ON_MATH_ERROR(X)


From webhook-mailer at python.org  Mon Sep 23 23:34:17 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Tue, 24 Sep 2019 03:34:17 -0000
Subject: [Python-checkins] closes bpo-16637: libpython: construct integer
 object directly from gdbvalue (GH-15232)
Message-ID: <mailman.40.1569296058.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/6f53d34fb0f944a8c0ee530334c353559ac40f72
commit: 6f53d34fb0f944a8c0ee530334c353559ac40f72
branch: master
author: Marc Hartmayer <marc1006 at users.noreply.github.com>
committer: Benjamin Peterson <benjamin at python.org>
date: 2019-09-23T20:34:12-07:00
summary:

closes bpo-16637: libpython: construct integer object directly from gdbvalue (GH-15232)

This fixes the exception '`ValueError: invalid literal for int() with base 10`
if `str(gdbval)` returns a hexadecimal value (e.g. '0xa0'). This is the case if
the output-radix is set to 16 in gdb. See
https://sourceware.org/gdb/onlinedocs/gdb/Numbers.html for more information.

files:
M Tools/gdb/libpython.py

diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py
index e28513db261b..2ab7d3b11cad 100755
--- a/Tools/gdb/libpython.py
+++ b/Tools/gdb/libpython.py
@@ -1392,7 +1392,7 @@ def write_repr(self, out, visited):
 
 
 def int_from_int(gdbval):
-    return int(str(gdbval))
+    return int(gdbval)
 
 
 def stringify(val):


From webhook-mailer at python.org  Mon Sep 23 23:49:44 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Tue, 24 Sep 2019 03:49:44 -0000
Subject: [Python-checkins] bpo-38136: Updates await_count and call_count to
 be different things (GH-16192)
Message-ID: <mailman.41.1569296985.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/ef048517755db1f0d211fb6dfc655a8b412cc96f
commit: ef048517755db1f0d211fb6dfc655a8b412cc96f
branch: master
author: Lisa Roach <lisaroach14 at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-23T20:49:40-07:00
summary:

bpo-38136: Updates await_count and call_count to be different things (GH-16192)

files:
A Misc/NEWS.d/next/Library/2019-09-16-09-54-42.bpo-38136.MdI-Zb.rst
M Doc/library/unittest.mock.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py
M Lib/unittest/test/testmock/testmock.py

diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index b446ddb3598f..300f28c8e2ce 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -514,21 +514,6 @@ the *new_callable* argument to :func:`patch`.
             >>> mock.call_count
             2
 
-        For :class:`AsyncMock` the :attr:`call_count` is only iterated if the function
-        has been awaited:
-
-            >>> mock = AsyncMock()
-            >>> mock()  # doctest: +SKIP
-            <coroutine object AsyncMockMixin._mock_call at ...>
-            >>> mock.call_count
-            0
-            >>> async def main():
-            ...     await mock()
-            ...
-            >>> asyncio.run(main())
-            >>> mock.call_count
-            1
-
     .. attribute:: return_value
 
         Set this to configure the value returned by calling the mock:
@@ -907,19 +892,22 @@ object::
 
   .. method:: assert_awaited()
 
-      Assert that the mock was awaited at least once.
+      Assert that the mock was awaited at least once. Note that this is separate
+      from the object having been called, the ``await`` keyword must be used:
 
           >>> mock = AsyncMock()
-          >>> async def main():
-          ...     await mock()
+          >>> async def main(coroutine_mock):
+          ...     await coroutine_mock
           ...
-          >>> asyncio.run(main())
+          >>> coroutine_mock = mock()
+          >>> mock.called
+          True
           >>> mock.assert_awaited()
-          >>> mock_2 = AsyncMock()
-          >>> mock_2.assert_awaited()
           Traceback (most recent call last):
           ...
           AssertionError: Expected mock to have been awaited.
+          >>> asyncio.run(main(coroutine_mock))
+          >>> mock.assert_awaited()
 
   .. method:: assert_awaited_once()
 
@@ -1004,14 +992,15 @@ object::
         ...     await mock(*args, **kwargs)
         ...
         >>> calls = [call("foo"), call("bar")]
-        >>> mock.assert_has_calls(calls)
+        >>> mock.assert_has_awaits(calls)
         Traceback (most recent call last):
         ...
-        AssertionError: Calls not found.
+        AssertionError: Awaits not found.
         Expected: [call('foo'), call('bar')]
+        Actual: []
         >>> asyncio.run(main('foo'))
         >>> asyncio.run(main('bar'))
-        >>> mock.assert_has_calls(calls)
+        >>> mock.assert_has_awaits(calls)
 
   .. method:: assert_not_awaited()
 
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 0a16e26f1d8a..22d63a45884a 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -1076,14 +1076,20 @@ def __call__(self, /, *args, **kwargs):
         # can't use self in-case a function / method we are mocking uses self
         # in the signature
         self._mock_check_sig(*args, **kwargs)
+        self._increment_mock_call(*args, **kwargs)
         return self._mock_call(*args, **kwargs)
 
 
     def _mock_call(self, /, *args, **kwargs):
+        return self._execute_mock_call(*args, **kwargs)
+
+    def _increment_mock_call(self, /, *args, **kwargs):
         self.called = True
         self.call_count += 1
 
         # handle call_args
+        # needs to be set here so assertions on call arguments pass before
+        # execution in the case of awaited calls
         _call = _Call((args, kwargs), two=True)
         self.call_args = _call
         self.call_args_list.append(_call)
@@ -1123,6 +1129,10 @@ def _mock_call(self, /, *args, **kwargs):
             # follow the parental chain:
             _new_parent = _new_parent._mock_new_parent
 
+    def _execute_mock_call(self, /, *args, **kwargs):
+        # seperate from _increment_mock_call so that awaited functions are
+        # executed seperately from their call
+
         effect = self.side_effect
         if effect is not None:
             if _is_exception(effect):
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index 0d68c9757980..aca1cd0bbecc 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -3,7 +3,7 @@
 import unittest
 
 from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock,
-                           create_autospec, _AwaitEvent)
+                           create_autospec, _AwaitEvent, sentinel, _CallList)
 
 
 def tearDownModule():
@@ -595,11 +595,173 @@ class AsyncMockAssert(unittest.TestCase):
     def setUp(self):
         self.mock = AsyncMock()
 
-    async def _runnable_test(self, *args):
-        if not args:
-            await self.mock()
-        else:
-            await self.mock(*args)
+    async def _runnable_test(self, *args, **kwargs):
+        await self.mock(*args, **kwargs)
+
+    async def _await_coroutine(self, coroutine):
+        return await coroutine
+
+    def test_assert_called_but_not_awaited(self):
+        mock = AsyncMock(AsyncClass)
+        with self.assertWarns(RuntimeWarning):
+            # Will raise a warning because never awaited
+            mock.async_method()
+        self.assertTrue(asyncio.iscoroutinefunction(mock.async_method))
+        mock.async_method.assert_called()
+        mock.async_method.assert_called_once()
+        mock.async_method.assert_called_once_with()
+        with self.assertRaises(AssertionError):
+            mock.assert_awaited()
+        with self.assertRaises(AssertionError):
+            mock.async_method.assert_awaited()
+
+    def test_assert_called_then_awaited(self):
+        mock = AsyncMock(AsyncClass)
+        mock_coroutine = mock.async_method()
+        mock.async_method.assert_called()
+        mock.async_method.assert_called_once()
+        mock.async_method.assert_called_once_with()
+        with self.assertRaises(AssertionError):
+            mock.async_method.assert_awaited()
+
+        asyncio.run(self._await_coroutine(mock_coroutine))
+        # Assert we haven't re-called the function
+        mock.async_method.assert_called_once()
+        mock.async_method.assert_awaited()
+        mock.async_method.assert_awaited_once()
+        mock.async_method.assert_awaited_once_with()
+
+    def test_assert_called_and_awaited_at_same_time(self):
+        with self.assertRaises(AssertionError):
+            self.mock.assert_awaited()
+
+        with self.assertRaises(AssertionError):
+            self.mock.assert_called()
+
+        asyncio.run(self._runnable_test())
+        self.mock.assert_called_once()
+        self.mock.assert_awaited_once()
+
+    def test_assert_called_twice_and_awaited_once(self):
+        mock = AsyncMock(AsyncClass)
+        coroutine = mock.async_method()
+        with self.assertWarns(RuntimeWarning):
+            # The first call will be awaited so no warning there
+            # But this call will never get awaited, so it will warn here
+            mock.async_method()
+        with self.assertRaises(AssertionError):
+            mock.async_method.assert_awaited()
+        mock.async_method.assert_called()
+        asyncio.run(self._await_coroutine(coroutine))
+        mock.async_method.assert_awaited()
+        mock.async_method.assert_awaited_once()
+
+    def test_assert_called_once_and_awaited_twice(self):
+        mock = AsyncMock(AsyncClass)
+        coroutine = mock.async_method()
+        mock.async_method.assert_called_once()
+        asyncio.run(self._await_coroutine(coroutine))
+        with self.assertRaises(RuntimeError):
+            # Cannot reuse already awaited coroutine
+            asyncio.run(self._await_coroutine(coroutine))
+        mock.async_method.assert_awaited()
+
+    def test_assert_awaited_but_not_called(self):
+        with self.assertRaises(AssertionError):
+            self.mock.assert_awaited()
+        with self.assertRaises(AssertionError):
+            self.mock.assert_called()
+        with self.assertRaises(TypeError):
+            # You cannot await an AsyncMock, it must be a coroutine
+            asyncio.run(self._await_coroutine(self.mock))
+
+        with self.assertRaises(AssertionError):
+            self.mock.assert_awaited()
+        with self.assertRaises(AssertionError):
+            self.mock.assert_called()
+
+    def test_assert_has_calls_not_awaits(self):
+        kalls = [call('foo')]
+        with self.assertWarns(RuntimeWarning):
+            # Will raise a warning because never awaited
+            self.mock('foo')
+        self.mock.assert_has_calls(kalls)
+        with self.assertRaises(AssertionError):
+            self.mock.assert_has_awaits(kalls)
+
+    def test_assert_has_mock_calls_on_async_mock_no_spec(self):
+        with self.assertWarns(RuntimeWarning):
+            # Will raise a warning because never awaited
+            self.mock()
+        kalls_empty = [('', (), {})]
+        self.assertEqual(self.mock.mock_calls, kalls_empty)
+
+        with self.assertWarns(RuntimeWarning):
+            # Will raise a warning because never awaited
+            self.mock('foo')
+            self.mock('baz')
+        mock_kalls = ([call(), call('foo'), call('baz')])
+        self.assertEqual(self.mock.mock_calls, mock_kalls)
+
+    def test_assert_has_mock_calls_on_async_mock_with_spec(self):
+        a_class_mock = AsyncMock(AsyncClass)
+        with self.assertWarns(RuntimeWarning):
+            # Will raise a warning because never awaited
+            a_class_mock.async_method()
+        kalls_empty = [('', (), {})]
+        self.assertEqual(a_class_mock.async_method.mock_calls, kalls_empty)
+        self.assertEqual(a_class_mock.mock_calls, [call.async_method()])
+
+        with self.assertWarns(RuntimeWarning):
+            # Will raise a warning because never awaited
+            a_class_mock.async_method(1, 2, 3, a=4, b=5)
+        method_kalls = [call(), call(1, 2, 3, a=4, b=5)]
+        mock_kalls = [call.async_method(), call.async_method(1, 2, 3, a=4, b=5)]
+        self.assertEqual(a_class_mock.async_method.mock_calls, method_kalls)
+        self.assertEqual(a_class_mock.mock_calls, mock_kalls)
+
+    def test_async_method_calls_recorded(self):
+        with self.assertWarns(RuntimeWarning):
+            # Will raise warnings because never awaited
+            self.mock.something(3, fish=None)
+            self.mock.something_else.something(6, cake=sentinel.Cake)
+
+        self.assertEqual(self.mock.method_calls, [
+            ("something", (3,), {'fish': None}),
+            ("something_else.something", (6,), {'cake': sentinel.Cake})
+        ],
+            "method calls not recorded correctly")
+        self.assertEqual(self.mock.something_else.method_calls,
+                         [("something", (6,), {'cake': sentinel.Cake})],
+                         "method calls not recorded correctly")
+
+    def test_async_arg_lists(self):
+        def assert_attrs(mock):
+            names = ('call_args_list', 'method_calls', 'mock_calls')
+            for name in names:
+                attr = getattr(mock, name)
+                self.assertIsInstance(attr, _CallList)
+                self.assertIsInstance(attr, list)
+                self.assertEqual(attr, [])
+
+        assert_attrs(self.mock)
+        with self.assertWarns(RuntimeWarning):
+            # Will raise warnings because never awaited
+            self.mock()
+            self.mock(1, 2)
+            self.mock(a=3)
+
+        self.mock.reset_mock()
+        assert_attrs(self.mock)
+
+        a_mock = AsyncMock(AsyncClass)
+        with self.assertWarns(RuntimeWarning):
+            # Will raise warnings because never awaited
+            a_mock.async_method()
+            a_mock.async_method(1, a=3)
+
+        a_mock.reset_mock()
+        assert_attrs(a_mock)
 
     def test_assert_awaited(self):
         with self.assertRaises(AssertionError):
@@ -645,20 +807,20 @@ def test_assert_awaited_once_with(self):
 
     def test_assert_any_wait(self):
         with self.assertRaises(AssertionError):
-            self.mock.assert_any_await('NormalFoo')
+            self.mock.assert_any_await('foo')
 
-        asyncio.run(self._runnable_test('foo'))
+        asyncio.run(self._runnable_test('baz'))
         with self.assertRaises(AssertionError):
-            self.mock.assert_any_await('NormalFoo')
+            self.mock.assert_any_await('foo')
 
-        asyncio.run(self._runnable_test('NormalFoo'))
-        self.mock.assert_any_await('NormalFoo')
+        asyncio.run(self._runnable_test('foo'))
+        self.mock.assert_any_await('foo')
 
         asyncio.run(self._runnable_test('SomethingElse'))
-        self.mock.assert_any_await('NormalFoo')
+        self.mock.assert_any_await('foo')
 
     def test_assert_has_awaits_no_order(self):
-        calls = [call('NormalFoo'), call('baz')]
+        calls = [call('foo'), call('baz')]
 
         with self.assertRaises(AssertionError) as cm:
             self.mock.assert_has_awaits(calls)
@@ -668,7 +830,7 @@ def test_assert_has_awaits_no_order(self):
         with self.assertRaises(AssertionError):
             self.mock.assert_has_awaits(calls)
 
-        asyncio.run(self._runnable_test('NormalFoo'))
+        asyncio.run(self._runnable_test('foo'))
         with self.assertRaises(AssertionError):
             self.mock.assert_has_awaits(calls)
 
@@ -703,7 +865,7 @@ def __eq__(self, other): pass
         mock_with_spec.assert_any_await(ANY, 1)
 
     def test_assert_has_awaits_ordered(self):
-        calls = [call('NormalFoo'), call('baz')]
+        calls = [call('foo'), call('baz')]
         with self.assertRaises(AssertionError):
             self.mock.assert_has_awaits(calls, any_order=True)
 
@@ -711,11 +873,11 @@ def test_assert_has_awaits_ordered(self):
         with self.assertRaises(AssertionError):
             self.mock.assert_has_awaits(calls, any_order=True)
 
-        asyncio.run(self._runnable_test('foo'))
+        asyncio.run(self._runnable_test('bamf'))
         with self.assertRaises(AssertionError):
             self.mock.assert_has_awaits(calls, any_order=True)
 
-        asyncio.run(self._runnable_test('NormalFoo'))
+        asyncio.run(self._runnable_test('foo'))
         self.mock.assert_has_awaits(calls, any_order=True)
 
         asyncio.run(self._runnable_test('qux'))
diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py
index 2bafa8266b63..ad67f98f87cc 100644
--- a/Lib/unittest/test/testmock/testmock.py
+++ b/Lib/unittest/test/testmock/testmock.py
@@ -850,6 +850,7 @@ def test():
     def test_setting_call(self):
         mock = Mock()
         def __call__(self, a):
+            self._increment_mock_call(a)
             return self._mock_call(a)
 
         type(mock).__call__ = __call__
@@ -2025,7 +2026,7 @@ def trace(frame, event, arg):  # pragma: no cover
             )
 
             mocks = [
-                Mock, MagicMock, NonCallableMock, NonCallableMagicMock
+                Mock, MagicMock, NonCallableMock, NonCallableMagicMock, AsyncMock
             ]
 
             for mock in mocks:
diff --git a/Misc/NEWS.d/next/Library/2019-09-16-09-54-42.bpo-38136.MdI-Zb.rst b/Misc/NEWS.d/next/Library/2019-09-16-09-54-42.bpo-38136.MdI-Zb.rst
new file mode 100644
index 000000000000..78cad245e67d
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-16-09-54-42.bpo-38136.MdI-Zb.rst
@@ -0,0 +1,3 @@
+Changes AsyncMock call count and await count to be two different counters.
+Now await count only counts when a coroutine has been awaited, not when it
+has been called, and vice-versa. Update the documentation around this.


From webhook-mailer at python.org  Mon Sep 23 23:52:54 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 24 Sep 2019 03:52:54 -0000
Subject: [Python-checkins] closes bpo-16637: libpython: construct integer
 object directly from gdbvalue (GH-15232)
Message-ID: <mailman.42.1569297175.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/5b94eb8d14f5a5b20dc2a999d6ca6219214b458c
commit: 5b94eb8d14f5a5b20dc2a999d6ca6219214b458c
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-23T20:52:50-07:00
summary:

closes bpo-16637: libpython: construct integer object directly from gdbvalue (GH-15232)


This fixes the exception '`ValueError: invalid literal for int() with base 10`
if `str(gdbval)` returns a hexadecimal value (e.g. '0xa0'). This is the case if
the output-radix is set to 16 in gdb. See
https://sourceware.org/gdb/onlinedocs/gdb/Numbers.html for more information.
(cherry picked from commit 6f53d34fb0f944a8c0ee530334c353559ac40f72)

Co-authored-by: Marc Hartmayer <marc1006 at users.noreply.github.com>

files:
M Tools/gdb/libpython.py

diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py
index e40f79b5c3c7..93f997c21e77 100755
--- a/Tools/gdb/libpython.py
+++ b/Tools/gdb/libpython.py
@@ -1392,7 +1392,7 @@ def write_repr(self, out, visited):
 
 
 def int_from_int(gdbval):
-    return int(str(gdbval))
+    return int(gdbval)
 
 
 def stringify(val):


From webhook-mailer at python.org  Tue Sep 24 06:47:53 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 24 Sep 2019 10:47:53 -0000
Subject: [Python-checkins] bpo-38212: Increase MP
 test_queue_feeder_donot_stop_onexc() timeout (GH-16348)
Message-ID: <mailman.43.1569322074.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/99799c722065d0524f3ab0bc455e1938bb8dc60f
commit: 99799c722065d0524f3ab0bc455e1938bb8dc60f
branch: master
author: Victor Stinner <vstinner at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-24T12:47:49+02:00
summary:

bpo-38212: Increase MP test_queue_feeder_donot_stop_onexc() timeout (GH-16348)

Multiprocessing tests: increase test_queue_feeder_donot_stop_onexc()
timeout from 1 to 60 seconds.

files:
A Misc/NEWS.d/next/Tests/2019-09-24-12-24-05.bpo-38212.IWbhWz.rst
M Lib/test/_test_multiprocessing.py

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index c717d0aad287..b720a483e9be 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -1128,8 +1128,7 @@ def __reduce__(self):
             q = self.Queue()
             q.put(NotSerializable())
             q.put(True)
-            # bpo-30595: use a timeout of 1 second for slow buildbots
-            self.assertTrue(q.get(timeout=1.0))
+            self.assertTrue(q.get(timeout=TIMEOUT))
             close_queue(q)
 
         with test.support.captured_stderr():
diff --git a/Misc/NEWS.d/next/Tests/2019-09-24-12-24-05.bpo-38212.IWbhWz.rst b/Misc/NEWS.d/next/Tests/2019-09-24-12-24-05.bpo-38212.IWbhWz.rst
new file mode 100644
index 000000000000..83a812d46998
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-24-12-24-05.bpo-38212.IWbhWz.rst
@@ -0,0 +1,2 @@
+Multiprocessing tests: increase test_queue_feeder_donot_stop_onexc() timeout
+from 1 to 60 seconds.


From webhook-mailer at python.org  Tue Sep 24 07:06:19 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 24 Sep 2019 11:06:19 -0000
Subject: [Python-checkins] bpo-38212: Increase MP
 test_queue_feeder_donot_stop_onexc() timeout (GH-16348)
Message-ID: <mailman.44.1569323180.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/c13a4d6a94b5fab52422fcba612b3d1790aa3422
commit: c13a4d6a94b5fab52422fcba612b3d1790aa3422
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-24T04:06:05-07:00
summary:

bpo-38212: Increase MP test_queue_feeder_donot_stop_onexc() timeout (GH-16348)


Multiprocessing tests: increase test_queue_feeder_donot_stop_onexc()
timeout from 1 to 60 seconds.
(cherry picked from commit 99799c722065d0524f3ab0bc455e1938bb8dc60f)

Co-authored-by: Victor Stinner <vstinner at python.org>

files:
A Misc/NEWS.d/next/Tests/2019-09-24-12-24-05.bpo-38212.IWbhWz.rst
M Lib/test/_test_multiprocessing.py

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index fb65e5e1aba5..affe10248350 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -1057,8 +1057,7 @@ def __reduce__(self):
             q = self.Queue()
             q.put(NotSerializable())
             q.put(True)
-            # bpo-30595: use a timeout of 1 second for slow buildbots
-            self.assertTrue(q.get(timeout=1.0))
+            self.assertTrue(q.get(timeout=TIMEOUT))
             close_queue(q)
 
         with test.support.captured_stderr():
diff --git a/Misc/NEWS.d/next/Tests/2019-09-24-12-24-05.bpo-38212.IWbhWz.rst b/Misc/NEWS.d/next/Tests/2019-09-24-12-24-05.bpo-38212.IWbhWz.rst
new file mode 100644
index 000000000000..83a812d46998
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-24-12-24-05.bpo-38212.IWbhWz.rst
@@ -0,0 +1,2 @@
+Multiprocessing tests: increase test_queue_feeder_donot_stop_onexc() timeout
+from 1 to 60 seconds.


From webhook-mailer at python.org  Tue Sep 24 07:07:32 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 24 Sep 2019 11:07:32 -0000
Subject: [Python-checkins] bpo-38212: Increase MP
 test_queue_feeder_donot_stop_onexc() timeout (GH-16348)
Message-ID: <mailman.45.1569323253.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/fea9ca1b0c6279f483a73f2040b068dd0e0d3acd
commit: fea9ca1b0c6279f483a73f2040b068dd0e0d3acd
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-24T04:07:28-07:00
summary:

bpo-38212: Increase MP test_queue_feeder_donot_stop_onexc() timeout (GH-16348)


Multiprocessing tests: increase test_queue_feeder_donot_stop_onexc()
timeout from 1 to 60 seconds.
(cherry picked from commit 99799c722065d0524f3ab0bc455e1938bb8dc60f)

Co-authored-by: Victor Stinner <vstinner at python.org>

files:
A Misc/NEWS.d/next/Tests/2019-09-24-12-24-05.bpo-38212.IWbhWz.rst
M Lib/test/_test_multiprocessing.py

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 2fe0def2bcd2..0de350c5c6f2 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -1128,8 +1128,7 @@ def __reduce__(self):
             q = self.Queue()
             q.put(NotSerializable())
             q.put(True)
-            # bpo-30595: use a timeout of 1 second for slow buildbots
-            self.assertTrue(q.get(timeout=1.0))
+            self.assertTrue(q.get(timeout=TIMEOUT))
             close_queue(q)
 
         with test.support.captured_stderr():
diff --git a/Misc/NEWS.d/next/Tests/2019-09-24-12-24-05.bpo-38212.IWbhWz.rst b/Misc/NEWS.d/next/Tests/2019-09-24-12-24-05.bpo-38212.IWbhWz.rst
new file mode 100644
index 000000000000..83a812d46998
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-24-12-24-05.bpo-38212.IWbhWz.rst
@@ -0,0 +1,2 @@
+Multiprocessing tests: increase test_queue_feeder_donot_stop_onexc() timeout
+from 1 to 60 seconds.


From webhook-mailer at python.org  Tue Sep 24 08:19:55 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 24 Sep 2019 12:19:55 -0000
Subject: [Python-checkins] bpo-37123: multiprocessing test_mymanager()
 accepts SIGTERM (GH-16349)
Message-ID: <mailman.46.1569327597.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/b0e1ae5f5430433766e023c1a6936aeba0f2b84e
commit: b0e1ae5f5430433766e023c1a6936aeba0f2b84e
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-24T14:19:48+02:00
summary:

bpo-37123: multiprocessing test_mymanager() accepts SIGTERM (GH-16349)

Multiprocessing test test_mymanager() now also expects -SIGTERM, not
only exitcode 0.

bpo-30356: BaseManager._finalize_manager() sends SIGTERM to the
manager process if it takes longer than 1 second to stop, which
happens on slow buildbots.

files:
A Misc/NEWS.d/next/Tests/2019-09-24-12-30-55.bpo-37123.IoutBn.rst
M Lib/test/_test_multiprocessing.py

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index b720a483e9be..bfaa02b0fcc8 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -2799,16 +2799,17 @@ def test_mymanager(self):
         self.common(manager)
         manager.shutdown()
 
-        # If the manager process exited cleanly then the exitcode
-        # will be zero.  Otherwise (after a short timeout)
-        # terminate() is used, resulting in an exitcode of -SIGTERM.
-        self.assertEqual(manager._process.exitcode, 0)
+        # bpo-30356: BaseManager._finalize_manager() sends SIGTERM
+        # to the manager process if it takes longer than 1 second to stop,
+        # which happens on slow buildbots.
+        self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM))
 
     def test_mymanager_context(self):
         with MyManager() as manager:
             self.common(manager)
         # bpo-30356: BaseManager._finalize_manager() sends SIGTERM
-        # to the manager process if it takes longer than 1 second to stop.
+        # to the manager process if it takes longer than 1 second to stop,
+        # which happens on slow buildbots.
         self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM))
 
     def test_mymanager_context_prestarted(self):
diff --git a/Misc/NEWS.d/next/Tests/2019-09-24-12-30-55.bpo-37123.IoutBn.rst b/Misc/NEWS.d/next/Tests/2019-09-24-12-30-55.bpo-37123.IoutBn.rst
new file mode 100644
index 000000000000..200a5c324d6d
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-24-12-30-55.bpo-37123.IoutBn.rst
@@ -0,0 +1,4 @@
+Multiprocessing test test_mymanager() now also expects -SIGTERM, not only
+exitcode 0. BaseManager._finalize_manager() sends SIGTERM to the manager
+process if it takes longer than 1 second to stop, which happens on slow
+buildbots.


From webhook-mailer at python.org  Tue Sep 24 08:38:45 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 24 Sep 2019 12:38:45 -0000
Subject: [Python-checkins] bpo-37123: multiprocessing test_mymanager()
 accepts SIGTERM (GH-16349)
Message-ID: <mailman.47.1569328726.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/46f6c566a888fb637913ef71fed28568180eafb4
commit: 46f6c566a888fb637913ef71fed28568180eafb4
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-24T05:38:38-07:00
summary:

bpo-37123: multiprocessing test_mymanager() accepts SIGTERM (GH-16349)


Multiprocessing test test_mymanager() now also expects -SIGTERM, not
only exitcode 0.

bpo-30356: BaseManager._finalize_manager() sends SIGTERM to the
manager process if it takes longer than 1 second to stop, which
happens on slow buildbots.
(cherry picked from commit b0e1ae5f5430433766e023c1a6936aeba0f2b84e)

Co-authored-by: Victor Stinner <vstinner at redhat.com>

files:
A Misc/NEWS.d/next/Tests/2019-09-24-12-30-55.bpo-37123.IoutBn.rst
M Lib/test/_test_multiprocessing.py

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index affe10248350..18947170cbcd 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -2689,16 +2689,17 @@ def test_mymanager(self):
         self.common(manager)
         manager.shutdown()
 
-        # If the manager process exited cleanly then the exitcode
-        # will be zero.  Otherwise (after a short timeout)
-        # terminate() is used, resulting in an exitcode of -SIGTERM.
-        self.assertEqual(manager._process.exitcode, 0)
+        # bpo-30356: BaseManager._finalize_manager() sends SIGTERM
+        # to the manager process if it takes longer than 1 second to stop,
+        # which happens on slow buildbots.
+        self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM))
 
     def test_mymanager_context(self):
         with MyManager() as manager:
             self.common(manager)
         # bpo-30356: BaseManager._finalize_manager() sends SIGTERM
-        # to the manager process if it takes longer than 1 second to stop.
+        # to the manager process if it takes longer than 1 second to stop,
+        # which happens on slow buildbots.
         self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM))
 
     def test_mymanager_context_prestarted(self):
diff --git a/Misc/NEWS.d/next/Tests/2019-09-24-12-30-55.bpo-37123.IoutBn.rst b/Misc/NEWS.d/next/Tests/2019-09-24-12-30-55.bpo-37123.IoutBn.rst
new file mode 100644
index 000000000000..200a5c324d6d
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-24-12-30-55.bpo-37123.IoutBn.rst
@@ -0,0 +1,4 @@
+Multiprocessing test test_mymanager() now also expects -SIGTERM, not only
+exitcode 0. BaseManager._finalize_manager() sends SIGTERM to the manager
+process if it takes longer than 1 second to stop, which happens on slow
+buildbots.


From webhook-mailer at python.org  Tue Sep 24 08:40:02 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 24 Sep 2019 12:40:02 -0000
Subject: [Python-checkins] bpo-37123: multiprocessing test_mymanager()
 accepts SIGTERM (GH-16349)
Message-ID: <mailman.48.1569328803.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/081641fe520382d48ffead158ab528180f72017d
commit: 081641fe520382d48ffead158ab528180f72017d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-24T05:39:47-07:00
summary:

bpo-37123: multiprocessing test_mymanager() accepts SIGTERM (GH-16349)


Multiprocessing test test_mymanager() now also expects -SIGTERM, not
only exitcode 0.

bpo-30356: BaseManager._finalize_manager() sends SIGTERM to the
manager process if it takes longer than 1 second to stop, which
happens on slow buildbots.
(cherry picked from commit b0e1ae5f5430433766e023c1a6936aeba0f2b84e)

Co-authored-by: Victor Stinner <vstinner at redhat.com>

files:
A Misc/NEWS.d/next/Tests/2019-09-24-12-30-55.bpo-37123.IoutBn.rst
M Lib/test/_test_multiprocessing.py

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 0de350c5c6f2..1474624ac5d5 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -2799,16 +2799,17 @@ def test_mymanager(self):
         self.common(manager)
         manager.shutdown()
 
-        # If the manager process exited cleanly then the exitcode
-        # will be zero.  Otherwise (after a short timeout)
-        # terminate() is used, resulting in an exitcode of -SIGTERM.
-        self.assertEqual(manager._process.exitcode, 0)
+        # bpo-30356: BaseManager._finalize_manager() sends SIGTERM
+        # to the manager process if it takes longer than 1 second to stop,
+        # which happens on slow buildbots.
+        self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM))
 
     def test_mymanager_context(self):
         with MyManager() as manager:
             self.common(manager)
         # bpo-30356: BaseManager._finalize_manager() sends SIGTERM
-        # to the manager process if it takes longer than 1 second to stop.
+        # to the manager process if it takes longer than 1 second to stop,
+        # which happens on slow buildbots.
         self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM))
 
     def test_mymanager_context_prestarted(self):
diff --git a/Misc/NEWS.d/next/Tests/2019-09-24-12-30-55.bpo-37123.IoutBn.rst b/Misc/NEWS.d/next/Tests/2019-09-24-12-30-55.bpo-37123.IoutBn.rst
new file mode 100644
index 000000000000..200a5c324d6d
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-24-12-30-55.bpo-37123.IoutBn.rst
@@ -0,0 +1,4 @@
+Multiprocessing test test_mymanager() now also expects -SIGTERM, not only
+exitcode 0. BaseManager._finalize_manager() sends SIGTERM to the manager
+process if it takes longer than 1 second to stop, which happens on slow
+buildbots.


From webhook-mailer at python.org  Tue Sep 24 11:44:31 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 24 Sep 2019 15:44:31 -0000
Subject: [Python-checkins] bpo-38234: Py_SetPath() uses the program full
 path (GH-16357)
Message-ID: <mailman.49.1569339872.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/1ce152a42eaa917d7763bce93f1e1ca72530d7ca
commit: 1ce152a42eaa917d7763bce93f1e1ca72530d7ca
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-24T17:44:15+02:00
summary:

bpo-38234: Py_SetPath() uses the program full path (GH-16357)

Py_SetPath() now sets sys.executable to the program full path
(Py_GetProgramFullPath()), rather than to the program name
(Py_GetProgramName()).

Fix also memory leaks in pathconfig_set_from_config().

files:
A Misc/NEWS.d/next/C API/2019-09-24-17-09-48.bpo-38234.d0bhEA.rst
M Doc/c-api/init.rst
M Doc/whatsnew/3.8.rst
M Python/pathconfig.c

diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index 0f8ff3b0dde8..0b7a84d03153 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -472,8 +472,8 @@ Process-wide parameters
    dependent delimiter character, which is ``':'`` on Unix and Mac OS X, ``';'``
    on Windows.
 
-   This also causes :data:`sys.executable` to be set only to the raw program
-   name (see :c:func:`Py_SetProgramName`) and for :data:`sys.prefix` and
+   This also causes :data:`sys.executable` to be set to the program
+   full path (see :c:func:`Py_GetProgramFullPath`) and for :data:`sys.prefix` and
    :data:`sys.exec_prefix` to be empty.  It is up to the caller to modify these
    if required after calling :c:func:`Py_Initialize`.
 
@@ -483,6 +483,10 @@ Process-wide parameters
    The path argument is copied internally, so the caller may free it after the
    call completes.
 
+   .. versionchanged:: 3.8
+      The program full path is now used for :data:`sys.executable`, instead
+      of the program name.
+
 
 .. c:function:: const char* Py_GetVersion()
 
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index c2455f487b14..0995cb3b9119 100755
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -1347,6 +1347,11 @@ Build and C API Changes
   parameter for indicating the number of positional-only arguments.
   (Contributed by Pablo Galindo in :issue:`37221`.)
 
+* :c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full
+  path (:c:func:`Py_GetProgramFullPath`) rather than to the program name
+  (:c:func:`Py_GetProgramName`).
+  (Contributed by Victor Stinner in :issue:`38234`.)
+
 
 Deprecated
 ==========
diff --git a/Misc/NEWS.d/next/C API/2019-09-24-17-09-48.bpo-38234.d0bhEA.rst b/Misc/NEWS.d/next/C API/2019-09-24-17-09-48.bpo-38234.d0bhEA.rst
new file mode 100644
index 000000000000..ba4cc312e692
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-09-24-17-09-48.bpo-38234.d0bhEA.rst	
@@ -0,0 +1,3 @@
+:c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full
+path (:c:func:`Py_GetProgramFullPath`) rather than to the program name
+(:c:func:`Py_GetProgramName`).
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index 7677a155797e..8f76fa50c971 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -23,6 +23,7 @@ wchar_t *_Py_dll_path = NULL;
 static int
 copy_wstr(wchar_t **dst, const wchar_t *src)
 {
+    assert(*dst == NULL);
     if (src != NULL) {
         *dst = _PyMem_RawWcsdup(src);
         if (*dst == NULL) {
@@ -172,6 +173,7 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
     if (config->module_search_paths_set) {
+        PyMem_RawFree(pathconfig->module_search_path);
         pathconfig->module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM);
         if (pathconfig->module_search_path == NULL) {
             goto no_memory;
@@ -180,6 +182,8 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
 
 #define COPY_CONFIG(PATH_ATTR, CONFIG_ATTR) \
         if (config->CONFIG_ATTR) { \
+            PyMem_RawFree(pathconfig->PATH_ATTR); \
+            pathconfig->PATH_ATTR = NULL; \
             if (copy_wstr(&pathconfig->PATH_ATTR, config->CONFIG_ATTR) < 0) { \
                 goto no_memory; \
             } \
@@ -455,16 +459,15 @@ Py_SetPath(const wchar_t *path)
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    /* Getting the program name calls pathconfig_global_init() */
-    wchar_t *program_name = _PyMem_RawWcsdup(Py_GetProgramName());
+    /* Getting the program full path calls pathconfig_global_init() */
+    wchar_t *program_full_path = _PyMem_RawWcsdup(Py_GetProgramFullPath());
 
     PyMem_RawFree(_Py_path_config.program_full_path);
     PyMem_RawFree(_Py_path_config.prefix);
     PyMem_RawFree(_Py_path_config.exec_prefix);
     PyMem_RawFree(_Py_path_config.module_search_path);
 
-    /* Copy program_name to program_full_path  */
-    _Py_path_config.program_full_path = program_name;
+    _Py_path_config.program_full_path = program_full_path;
     _Py_path_config.prefix = _PyMem_RawWcsdup(L"");
     _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L"");
     _Py_path_config.module_search_path = _PyMem_RawWcsdup(path);


From webhook-mailer at python.org  Tue Sep 24 12:21:09 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Tue, 24 Sep 2019 16:21:09 -0000
Subject: [Python-checkins] bpo-38234: Add tests for Python init path config
 (GH-16358)
Message-ID: <mailman.50.1569342069.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/bb6bf7d342b4503a6227fd209fac934905b6a1aa
commit: bb6bf7d342b4503a6227fd209fac934905b6a1aa
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-24T18:21:02+02:00
summary:

bpo-38234: Add tests for Python init path config (GH-16358)

files:
M Lib/test/test_embed.py
M Modules/getpath.c
M Programs/_testembed.c

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 2a5ace0bad76..7808a1a2cf6a 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -470,7 +470,8 @@ def main_xoptions(self, xoptions_list):
                 xoptions[opt] = True
         return xoptions
 
-    def _get_expected_config(self, env):
+    def _get_expected_config_impl(self):
+        env = remove_python_envvars()
         code = textwrap.dedent('''
             import json
             import sys
@@ -499,13 +500,19 @@ def _get_expected_config(self, env):
         except json.JSONDecodeError:
             self.fail(f"fail to decode stdout: {stdout!r}")
 
+    def _get_expected_config(self):
+        cls = InitConfigTests
+        if cls.EXPECTED_CONFIG is None:
+            cls.EXPECTED_CONFIG = self._get_expected_config_impl()
+
+        # get a copy
+        return {key: dict(value)
+                for key, value in cls.EXPECTED_CONFIG.items()}
+
     def get_expected_config(self, expected_preconfig, expected, env, api,
                             modify_path_cb=None):
         cls = self.__class__
-        if cls.EXPECTED_CONFIG is None:
-            cls.EXPECTED_CONFIG = self._get_expected_config(env)
-        configs = {key: dict(value)
-                   for key, value in self.EXPECTED_CONFIG.items()}
+        configs = self._get_expected_config()
 
         pre_config = configs['pre_config']
         for key, value in expected_preconfig.items():
@@ -553,9 +560,10 @@ def get_expected_config(self, expected_preconfig, expected, env, api,
             if value is self.GET_DEFAULT_CONFIG:
                 expected[key] = config[key]
 
-        prepend_path = expected['pythonpath_env']
-        if prepend_path is not None:
-            expected['module_search_paths'] = [prepend_path, *expected['module_search_paths']]
+        pythonpath_env = expected['pythonpath_env']
+        if pythonpath_env is not None:
+            paths = pythonpath_env.split(os.path.pathsep)
+            expected['module_search_paths'] = [*paths, *expected['module_search_paths']]
         if modify_path_cb is not None:
             expected['module_search_paths'] = expected['module_search_paths'].copy()
             modify_path_cb(expected['module_search_paths'])
@@ -604,8 +612,11 @@ def check_global_config(self, configs):
 
     def check_all_configs(self, testname, expected_config=None,
                      expected_preconfig=None, modify_path_cb=None, stderr=None,
-                     *, api):
-        env = remove_python_envvars()
+                     *, api, env=None, ignore_stderr=False):
+        new_env = remove_python_envvars()
+        if env is not None:
+            new_env.update(env)
+        env = new_env
 
         if api == API_ISOLATED:
             default_preconfig = self.PRE_CONFIG_ISOLATED
@@ -634,7 +645,7 @@ def check_all_configs(self, testname, expected_config=None,
         out, err = self.run_embedded_interpreter(testname, env=env)
         if stderr is None and not expected_config['verbose']:
             stderr = ""
-        if stderr is not None:
+        if stderr is not None and not ignore_stderr:
             self.assertEqual(err.rstrip(), stderr)
         try:
             configs = json.loads(out)
@@ -966,6 +977,62 @@ def test_init_dont_parse_argv(self):
         self.check_all_configs("test_init_dont_parse_argv", config, pre_config,
                                api=API_PYTHON)
 
+    def test_init_setpath(self):
+        # Test Py_SetProgramName() + Py_SetPath()
+        config = self._get_expected_config()
+        paths = config['config']['module_search_paths']
+
+        config = {
+            'module_search_paths': paths,
+            'prefix': '',
+            'base_prefix': '',
+            'exec_prefix': '',
+            'base_exec_prefix': '',
+        }
+        env = {'TESTPATH': os.path.pathsep.join(paths)}
+        self.check_all_configs("test_init_setpath", config,
+                               api=API_COMPAT, env=env,
+                               ignore_stderr=True)
+
+    def test_init_setpythonhome(self):
+        # Test Py_SetPythonHome(home) + PYTHONPATH env var
+        # + Py_SetProgramName()
+        config = self._get_expected_config()
+        paths = config['config']['module_search_paths']
+        paths_str = os.path.pathsep.join(paths)
+
+        for path in paths:
+            if not os.path.isdir(path):
+                continue
+            if os.path.exists(os.path.join(path, 'os.py')):
+                home = os.path.dirname(path)
+                break
+        else:
+            self.fail(f"Unable to find home in {paths!r}")
+
+        prefix = exec_prefix = home
+        ver = sys.version_info
+        if MS_WINDOWS:
+            expected_paths = paths
+        else:
+            expected_paths = [
+                os.path.join(prefix, 'lib', f'python{ver.major}{ver.minor}.zip'),
+                os.path.join(home, 'lib', f'python{ver.major}.{ver.minor}'),
+                os.path.join(home, 'lib', f'python{ver.major}.{ver.minor}/lib-dynload')]
+
+        config = {
+            'home': home,
+            'module_search_paths': expected_paths,
+            'prefix': prefix,
+            'base_prefix': prefix,
+            'exec_prefix': exec_prefix,
+            'base_exec_prefix': exec_prefix,
+            'pythonpath_env': paths_str,
+        }
+        env = {'TESTHOME': home, 'TESTPATH': paths_str}
+        self.check_all_configs("test_init_setpythonhome", config,
+                               api=API_COMPAT, env=env)
+
 
 class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
     def test_open_code_hook(self):
diff --git a/Modules/getpath.c b/Modules/getpath.c
index 24e16b41b406..a4607ef2d567 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -1012,12 +1012,12 @@ calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix,
                    wchar_t *zip_path, size_t zip_path_len)
 {
     PyStatus status;
-    if (safe_wcscpy(zip_path, prefix, zip_path_len) < 0) {
-        return PATHLEN_ERR();
-    }
 
     if (calculate->prefix_found > 0) {
         /* Use the reduced prefix returned by Py_GetPrefix() */
+        if (safe_wcscpy(zip_path, prefix, zip_path_len) < 0) {
+            return PATHLEN_ERR();
+        }
         reduce(zip_path);
         reduce(zip_path);
     }
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index c3ccc0ec325b..ed07606398d8 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -20,11 +20,12 @@
  * Executed via 'EmbeddingTests' in Lib/test/test_capi.py
  *********************************************************/
 
+/* Use path starting with "./" avoids a search along the PATH */
+#define PROGRAM_NAME L"./_testembed"
+
 static void _testembed_Py_Initialize(void)
 {
-    /* HACK: the "./" at front avoids a search along the PATH in
-       Modules/getpath.c */
-    Py_SetProgramName(L"./_testembed");
+    Py_SetProgramName(PROGRAM_NAME);
     Py_Initialize();
 }
 
@@ -363,8 +364,7 @@ config_set_wide_string_list(PyConfig *config, PyWideStringList *list,
 
 static void config_set_program_name(PyConfig *config)
 {
-    /* Use path starting with "./" avoids a search along the PATH */
-    const wchar_t *program_name = L"./_testembed";
+    const wchar_t *program_name = PROGRAM_NAME;
     config_set_string(config, &config->program_name, program_name);
 }
 
@@ -1263,7 +1263,7 @@ static int _audit_hook_run(const char *eventName, PyObject *args, void *userData
 static int test_audit_run_command(void)
 {
     AuditRunCommandTest test = {"cpython.run_command"};
-    wchar_t *argv[] = {L"./_testembed", L"-c", L"pass"};
+    wchar_t *argv[] = {PROGRAM_NAME, L"-c", L"pass"};
 
     Py_IgnoreEnvironmentFlag = 0;
     PySys_AddAuditHook(_audit_hook_run, (void*)&test);
@@ -1274,7 +1274,7 @@ static int test_audit_run_command(void)
 static int test_audit_run_file(void)
 {
     AuditRunCommandTest test = {"cpython.run_file"};
-    wchar_t *argv[] = {L"./_testembed", L"filename.py"};
+    wchar_t *argv[] = {PROGRAM_NAME, L"filename.py"};
 
     Py_IgnoreEnvironmentFlag = 0;
     PySys_AddAuditHook(_audit_hook_run, (void*)&test);
@@ -1312,21 +1312,21 @@ static int run_audit_run_test(int argc, wchar_t **argv, void *test)
 static int test_audit_run_interactivehook(void)
 {
     AuditRunCommandTest test = {"cpython.run_interactivehook", 10};
-    wchar_t *argv[] = {L"./_testembed"};
+    wchar_t *argv[] = {PROGRAM_NAME};
     return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test);
 }
 
 static int test_audit_run_startup(void)
 {
     AuditRunCommandTest test = {"cpython.run_startup", 10};
-    wchar_t *argv[] = {L"./_testembed"};
+    wchar_t *argv[] = {PROGRAM_NAME};
     return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test);
 }
 
 static int test_audit_run_stdin(void)
 {
     AuditRunCommandTest test = {"cpython.run_stdin"};
-    wchar_t *argv[] = {L"./_testembed"};
+    wchar_t *argv[] = {PROGRAM_NAME};
     return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test);
 }
 
@@ -1423,6 +1423,88 @@ static int test_init_sys_add(void)
 }
 
 
+static int test_init_setpath(void)
+{
+    Py_SetProgramName(PROGRAM_NAME);
+
+    char *env = getenv("TESTPATH");
+    if (!env) {
+        fprintf(stderr, "missing TESTPATH env var\n");
+        return 1;
+    }
+    wchar_t *path = Py_DecodeLocale(env, NULL);
+    if (path == NULL) {
+        fprintf(stderr, "failed to decode TESTPATH\n");
+        return 1;
+    }
+    Py_SetPath(path);
+    PyMem_RawFree(path);
+    putenv("TESTPATH=");
+
+    Py_Initialize();
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
+static int mysetenv(const char *name, const char *value)
+{
+    size_t len = strlen(name) + 1 + strlen(value) + 1;
+    char *env = PyMem_RawMalloc(len);
+    if (env == NULL) {
+        fprintf(stderr, "out of memory\n");
+        return -1;
+    }
+    strcpy(env, name);
+    strcat(env, "=");
+    strcat(env, value);
+
+    putenv(env);
+
+    /* Don't call PyMem_RawFree(env), but leak env memory block:
+       putenv() does not copy the string. */
+
+    return 0;
+}
+
+
+static int test_init_setpythonhome(void)
+{
+    char *env = getenv("TESTHOME");
+    if (!env) {
+        fprintf(stderr, "missing TESTHOME env var\n");
+        return 1;
+    }
+    wchar_t *home = Py_DecodeLocale(env, NULL);
+    if (home == NULL) {
+        fprintf(stderr, "failed to decode TESTHOME\n");
+        return 1;
+    }
+    Py_SetPythonHome(home);
+    PyMem_RawFree(home);
+    putenv("TESTHOME=");
+
+    char *path = getenv("TESTPATH");
+    if (!path) {
+        fprintf(stderr, "missing TESTPATH env var\n");
+        return 1;
+    }
+
+    if (mysetenv("PYTHONPATH", path) < 0) {
+        return 1;
+    }
+    putenv("TESTPATH=");
+
+    Py_SetProgramName(PROGRAM_NAME);
+
+    Py_Initialize();
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
 static void configure_init_main(PyConfig *config)
 {
     wchar_t* argv[] = {
@@ -1559,7 +1641,10 @@ static struct TestCase TestCases[] = {
     {"test_init_run_main", test_init_run_main},
     {"test_init_main", test_init_main},
     {"test_init_sys_add", test_init_sys_add},
+    {"test_init_setpath", test_init_setpath},
+    {"test_init_setpythonhome", test_init_setpythonhome},
     {"test_run_main", test_run_main},
+
     {"test_open_code_hook", test_open_code_hook},
     {"test_audit", test_audit},
     {"test_audit_subinterpreter", test_audit_subinterpreter},


From webhook-mailer at python.org  Tue Sep 24 15:08:45 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 24 Sep 2019 19:08:45 -0000
Subject: [Python-checkins] bpo-36871: Handle spec errors in assert_has_calls
 (GH-16005)
Message-ID: <mailman.51.1569352125.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/b5a7a4f0c20717a4c92c371583b5521b83f40f32
commit: b5a7a4f0c20717a4c92c371583b5521b83f40f32
branch: master
author: Samuel Freilich <sfreilich at google.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-24T12:08:31-07:00
summary:

bpo-36871: Handle spec errors in assert_has_calls (GH-16005)



The fix in PR 13261 handled the underlying issue about the spec for specific methods not being applied correctly, but it didn't fix the issue that was causing the misleading error message.

The code currently grabs a list of responses from _call_matcher (which may include exceptions). But it doesn't reach inside the list when checking if the result is an exception. This results in a misleading error message when one of the provided calls does not match the spec.


https://bugs.python.org/issue36871



Automerge-Triggered-By: @gpshead

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-24-18-45-46.bpo-36871.p47knk.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py
M Lib/unittest/test/testmock/testmock.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 22d63a45884a..7bd11c8e7034 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -926,13 +926,21 @@ def assert_has_calls(self, calls, any_order=False):
         If `any_order` is True then the calls can be in any order, but
         they must all appear in `mock_calls`."""
         expected = [self._call_matcher(c) for c in calls]
-        cause = expected if isinstance(expected, Exception) else None
+        cause = next((e for e in expected if isinstance(e, Exception)), None)
         all_calls = _CallList(self._call_matcher(c) for c in self.mock_calls)
         if not any_order:
             if expected not in all_calls:
+                if cause is None:
+                    problem = 'Calls not found.'
+                else:
+                    problem = ('Error processing expected calls.\n'
+                               'Errors: {}').format(
+                                   [e if isinstance(e, Exception) else None
+                                    for e in expected])
                 raise AssertionError(
-                    'Calls not found.\nExpected: %r%s'
-                    % (_CallList(calls), self._calls_repr(prefix="Actual"))
+                    f'{problem}\n'
+                    f'Expected: {_CallList(calls)}\n'
+                    f'Actual: {self._calls_repr(prefix="Actual")}'
                 ) from cause
             return
 
@@ -2244,12 +2252,20 @@ def assert_has_awaits(self, calls, any_order=False):
         they must all appear in :attr:`await_args_list`.
         """
         expected = [self._call_matcher(c) for c in calls]
-        cause = expected if isinstance(expected, Exception) else None
+        cause = next((e for e in expected if isinstance(e, Exception)), None)
         all_awaits = _CallList(self._call_matcher(c) for c in self.await_args_list)
         if not any_order:
             if expected not in all_awaits:
+                if cause is None:
+                    problem = 'Awaits not found.'
+                else:
+                    problem = ('Error processing expected awaits.\n'
+                               'Errors: {}').format(
+                                   [e if isinstance(e, Exception) else None
+                                    for e in expected])
                 raise AssertionError(
-                    f'Awaits not found.\nExpected: {_CallList(calls)}\n'
+                    f'{problem}\n'
+                    f'Expected: {_CallList(calls)}\n'
                     f'Actual: {self.await_args_list}'
                 ) from cause
             return
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index aca1cd0bbecc..f951526dbeb8 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -1,5 +1,6 @@
 import asyncio
 import inspect
+import re
 import unittest
 
 from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock,
@@ -889,3 +890,23 @@ def test_assert_not_awaited(self):
         asyncio.run(self._runnable_test())
         with self.assertRaises(AssertionError):
             self.mock.assert_not_awaited()
+
+    def test_assert_has_awaits_not_matching_spec_error(self):
+        async def f(): pass
+
+        mock = AsyncMock(spec=f)
+
+        with self.assertRaisesRegex(
+                AssertionError,
+                re.escape('Awaits not found.\nExpected:')) as cm:
+            mock.assert_has_awaits([call()])
+        self.assertIsNone(cm.exception.__cause__)
+
+        with self.assertRaisesRegex(
+                AssertionError,
+                re.escape('Error processing expected awaits.\n'
+                          "Errors: [None, TypeError('too many positional "
+                          "arguments')]\n"
+                          'Expected:')) as cm:
+            mock.assert_has_awaits([call(), call('wrong')])
+        self.assertIsInstance(cm.exception.__cause__, TypeError)
diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py
index ad67f98f87cc..88807d71ccb0 100644
--- a/Lib/unittest/test/testmock/testmock.py
+++ b/Lib/unittest/test/testmock/testmock.py
@@ -1435,6 +1435,25 @@ def f(a, b, c, d=None): pass
             mock.assert_has_calls(calls[:-1])
         mock.assert_has_calls(calls[:-1], any_order=True)
 
+    def test_assert_has_calls_not_matching_spec_error(self):
+        def f(): pass
+
+        mock = Mock(spec=f)
+
+        with self.assertRaisesRegex(
+                AssertionError,
+                re.escape('Calls not found.\nExpected:')) as cm:
+            mock.assert_has_calls([call()])
+        self.assertIsNone(cm.exception.__cause__)
+
+        with self.assertRaisesRegex(
+                AssertionError,
+                re.escape('Error processing expected calls.\n'
+                          "Errors: [None, TypeError('too many positional "
+                          "arguments')]\n"
+                          'Expected:')) as cm:
+            mock.assert_has_calls([call(), call('wrong')])
+        self.assertIsInstance(cm.exception.__cause__, TypeError)
 
     def test_assert_any_call(self):
         mock = Mock()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-24-18-45-46.bpo-36871.p47knk.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-24-18-45-46.bpo-36871.p47knk.rst
new file mode 100644
index 000000000000..6b7b19a0d57e
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-24-18-45-46.bpo-36871.p47knk.rst	
@@ -0,0 +1,3 @@
+Improve error handling for the assert_has_calls and assert_has_awaits methods of
+mocks. Fixed a bug where any errors encountered while binding the expected calls
+to the mock's spec were silently swallowed, leading to misleading error output.


From webhook-mailer at python.org  Tue Sep 24 18:04:33 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Tue, 24 Sep 2019 22:04:33 -0000
Subject: [Python-checkins] bpo-36871: Avoid duplicated 'Actual:' in
 assertion message (GH-16361)
Message-ID: <mailman.52.1569362674.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/2180f6b058effbf49ec819f7cedbe76ddd4b700c
commit: 2180f6b058effbf49ec819f7cedbe76ddd4b700c
branch: master
author: Samuel Freilich <sfreilich at google.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-24T15:04:29-07:00
summary:

bpo-36871: Avoid duplicated 'Actual:' in assertion message (GH-16361)

Fixes an issue caught after merge of PR 16005.

Tightened test assertions to check the entire assertion message.

files:
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py
M Lib/unittest/test/testmock/testmock.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 7bd11c8e7034..0cd7af65b940 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -939,8 +939,8 @@ def assert_has_calls(self, calls, any_order=False):
                                     for e in expected])
                 raise AssertionError(
                     f'{problem}\n'
-                    f'Expected: {_CallList(calls)}\n'
-                    f'Actual: {self._calls_repr(prefix="Actual")}'
+                    f'Expected: {_CallList(calls)}'
+                    f'{self._calls_repr(prefix="Actual").rstrip(".")}'
                 ) from cause
             return
 
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index f951526dbeb8..fde1e4a690b2 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -892,21 +892,28 @@ def test_assert_not_awaited(self):
             self.mock.assert_not_awaited()
 
     def test_assert_has_awaits_not_matching_spec_error(self):
-        async def f(): pass
+        async def f(x=None): pass
 
-        mock = AsyncMock(spec=f)
+        self.mock = AsyncMock(spec=f)
+        asyncio.run(self._runnable_test(1))
 
         with self.assertRaisesRegex(
                 AssertionError,
-                re.escape('Awaits not found.\nExpected:')) as cm:
-            mock.assert_has_awaits([call()])
+                '^{}$'.format(
+                    re.escape('Awaits not found.\n'
+                              'Expected: [call()]\n'
+                              'Actual: [call(1)]'))) as cm:
+            self.mock.assert_has_awaits([call()])
         self.assertIsNone(cm.exception.__cause__)
 
         with self.assertRaisesRegex(
                 AssertionError,
-                re.escape('Error processing expected awaits.\n'
-                          "Errors: [None, TypeError('too many positional "
-                          "arguments')]\n"
-                          'Expected:')) as cm:
-            mock.assert_has_awaits([call(), call('wrong')])
+                '^{}$'.format(
+                    re.escape(
+                        'Error processing expected awaits.\n'
+                        "Errors: [None, TypeError('too many positional "
+                        "arguments')]\n"
+                        'Expected: [call(), call(1, 2)]\n'
+                        'Actual: [call(1)]'))) as cm:
+            self.mock.assert_has_awaits([call(), call(1, 2)])
         self.assertIsInstance(cm.exception.__cause__, TypeError)
diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py
index 88807d71ccb0..01bc47944465 100644
--- a/Lib/unittest/test/testmock/testmock.py
+++ b/Lib/unittest/test/testmock/testmock.py
@@ -1436,23 +1436,30 @@ def f(a, b, c, d=None): pass
         mock.assert_has_calls(calls[:-1], any_order=True)
 
     def test_assert_has_calls_not_matching_spec_error(self):
-        def f(): pass
+        def f(x=None): pass
 
         mock = Mock(spec=f)
+        mock(1)
 
         with self.assertRaisesRegex(
                 AssertionError,
-                re.escape('Calls not found.\nExpected:')) as cm:
+                '^{}$'.format(
+                    re.escape('Calls not found.\n'
+                              'Expected: [call()]\n'
+                              'Actual: [call(1)]'))) as cm:
             mock.assert_has_calls([call()])
         self.assertIsNone(cm.exception.__cause__)
 
+
         with self.assertRaisesRegex(
                 AssertionError,
-                re.escape('Error processing expected calls.\n'
-                          "Errors: [None, TypeError('too many positional "
-                          "arguments')]\n"
-                          'Expected:')) as cm:
-            mock.assert_has_calls([call(), call('wrong')])
+                '^{}$'.format(
+                    re.escape(
+                        'Error processing expected calls.\n'
+                        "Errors: [None, TypeError('too many positional arguments')]\n"
+                        "Expected: [call(), call(1, 2)]\n"
+                        'Actual: [call(1)]'))) as cm:
+            mock.assert_has_calls([call(), call(1, 2)])
         self.assertIsInstance(cm.exception.__cause__, TypeError)
 
     def test_assert_any_call(self):


From webhook-mailer at python.org  Tue Sep 24 20:10:39 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 25 Sep 2019 00:10:39 -0000
Subject: [Python-checkins] bpo-38234: test_embed: test pyvenv.cfg and
 pybuilddir.txt (GH-16366)
Message-ID: <mailman.53.1569370240.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/52ad33abbfb6637d74932617c7013bae0ccf6e32
commit: 52ad33abbfb6637d74932617c7013bae0ccf6e32
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T02:10:35+02:00
summary:

bpo-38234: test_embed: test pyvenv.cfg and pybuilddir.txt (GH-16366)

Add test_init_pybuilddir() and test_init_pyvenv_cfg() to test_embed
to test pyvenv.cfg and pybuilddir.txt configuration files.

Fix sysconfig._generate_posix_vars(): pybuilddir.txt uses UTF-8
encoding, not ASCII.

files:
M Lib/sysconfig.py
M Lib/test/test_embed.py

diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py
index e76e6927cb1f..b9e2fafbc084 100644
--- a/Lib/sysconfig.py
+++ b/Lib/sysconfig.py
@@ -412,7 +412,7 @@ def _generate_posix_vars():
         pprint.pprint(vars, stream=f)
 
     # Create file used for sys.path fixup -- see Modules/getpath.c
-    with open('pybuilddir.txt', 'w', encoding='ascii') as f:
+    with open('pybuilddir.txt', 'w', encoding='utf8') as f:
         f.write(pybuilddir)
 
 def _init_posix(vars):
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 7808a1a2cf6a..e7a10b6defea 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -3,11 +3,14 @@
 import unittest
 
 from collections import namedtuple
+import contextlib
 import json
 import os
 import re
+import shutil
 import subprocess
 import sys
+import tempfile
 import textwrap
 
 
@@ -25,6 +28,12 @@
 API_ISOLATED = 3
 
 
+def debug_build(program):
+    program = os.path.basename(program)
+    name = os.path.splitext(program)[0]
+    return name.endswith("_d")
+
+
 def remove_python_envvars():
     env = dict(os.environ)
     # Remove PYTHON* environment variables to get deterministic environment
@@ -40,7 +49,7 @@ def setUp(self):
         basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
         exename = "_testembed"
         if MS_WINDOWS:
-            ext = ("_d" if "_d" in sys.executable else "") + ".exe"
+            ext = ("_d" if debug_build(sys.executable) else "") + ".exe"
             exename += ext
             exepath = os.path.dirname(sys.executable)
         else:
@@ -58,7 +67,8 @@ def tearDown(self):
         os.chdir(self.oldcwd)
 
     def run_embedded_interpreter(self, *args, env=None,
-                                 timeout=None, returncode=0, input=None):
+                                 timeout=None, returncode=0, input=None,
+                                 cwd=None):
         """Runs a test in the embedded interpreter"""
         cmd = [self.test_exe]
         cmd.extend(args)
@@ -72,7 +82,8 @@ def run_embedded_interpreter(self, *args, env=None,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE,
                              universal_newlines=True,
-                             env=env)
+                             env=env,
+                             cwd=cwd)
         try:
             (out, err) = p.communicate(input=input, timeout=timeout)
         except:
@@ -460,6 +471,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
     EXPECTED_CONFIG = None
 
+    @classmethod
+    def tearDownClass(cls):
+        # clear cache
+        cls.EXPECTED_CONFIG = None
+
     def main_xoptions(self, xoptions_list):
         xoptions = {}
         for opt in xoptions_list:
@@ -490,11 +506,12 @@ def _get_expected_config_impl(self):
         args = [sys.executable, '-S', '-c', code]
         proc = subprocess.run(args, env=env,
                               stdout=subprocess.PIPE,
-                              stderr=subprocess.STDOUT)
+                              stderr=subprocess.PIPE)
         if proc.returncode:
             raise Exception(f"failed to get the default config: "
                             f"stdout={proc.stdout!r} stderr={proc.stderr!r}")
         stdout = proc.stdout.decode('utf-8')
+        # ignore stderr
         try:
             return json.loads(stdout)
         except json.JSONDecodeError:
@@ -506,8 +523,15 @@ def _get_expected_config(self):
             cls.EXPECTED_CONFIG = self._get_expected_config_impl()
 
         # get a copy
-        return {key: dict(value)
-                for key, value in cls.EXPECTED_CONFIG.items()}
+        configs = {}
+        for config_key, config_value in cls.EXPECTED_CONFIG.items():
+            config = {}
+            for key, value in config_value.items():
+                if isinstance(value, list):
+                    value = value.copy()
+                config[key] = value
+            configs[config_key] = config
+        return configs
 
     def get_expected_config(self, expected_preconfig, expected, env, api,
                             modify_path_cb=None):
@@ -612,7 +636,7 @@ def check_global_config(self, configs):
 
     def check_all_configs(self, testname, expected_config=None,
                      expected_preconfig=None, modify_path_cb=None, stderr=None,
-                     *, api, env=None, ignore_stderr=False):
+                     *, api, env=None, ignore_stderr=False, cwd=None):
         new_env = remove_python_envvars()
         if env is not None:
             new_env.update(env)
@@ -642,7 +666,8 @@ def check_all_configs(self, testname, expected_config=None,
                                  expected_config, env,
                                  api, modify_path_cb)
 
-        out, err = self.run_embedded_interpreter(testname, env=env)
+        out, err = self.run_embedded_interpreter(testname,
+                                                 env=env, cwd=cwd)
         if stderr is None and not expected_config['verbose']:
             stderr = ""
         if stderr is not None and not ignore_stderr:
@@ -994,6 +1019,48 @@ def test_init_setpath(self):
                                api=API_COMPAT, env=env,
                                ignore_stderr=True)
 
+    def module_search_paths(self, prefix=None, exec_prefix=None):
+        config = self._get_expected_config()
+        if prefix is None:
+            prefix = config['config']['prefix']
+        if exec_prefix is None:
+            exec_prefix = config['config']['prefix']
+        if MS_WINDOWS:
+            return config['config']['module_search_paths']
+        else:
+            ver = sys.version_info
+            return [
+                os.path.join(prefix, 'lib',
+                             f'python{ver.major}{ver.minor}.zip'),
+                os.path.join(prefix, 'lib',
+                             f'python{ver.major}.{ver.minor}'),
+                os.path.join(exec_prefix, 'lib',
+                             f'python{ver.major}.{ver.minor}', 'lib-dynload'),
+            ]
+
+    @contextlib.contextmanager
+    def tmpdir_with_python(self):
+        # Temporary directory with a copy of the Python program
+        with tempfile.TemporaryDirectory() as tmpdir:
+            if MS_WINDOWS:
+                # Copy pythonXY.dll (or pythonXY_d.dll)
+                ver = sys.version_info
+                dll = f'python{ver.major}{ver.minor}'
+                if debug_build(sys.executable):
+                    dll += '_d'
+                dll += '.dll'
+                dll = os.path.join(os.path.dirname(self.test_exe), dll)
+                dll_copy = os.path.join(tmpdir, os.path.basename(dll))
+                shutil.copyfile(dll, dll_copy)
+
+            # Copy Python program
+            exec_copy = os.path.join(tmpdir, os.path.basename(self.test_exe))
+            shutil.copyfile(self.test_exe, exec_copy)
+            shutil.copystat(self.test_exe, exec_copy)
+            self.test_exe = exec_copy
+
+            yield tmpdir
+
     def test_init_setpythonhome(self):
         # Test Py_SetPythonHome(home) + PYTHONPATH env var
         # + Py_SetProgramName()
@@ -1012,13 +1079,7 @@ def test_init_setpythonhome(self):
 
         prefix = exec_prefix = home
         ver = sys.version_info
-        if MS_WINDOWS:
-            expected_paths = paths
-        else:
-            expected_paths = [
-                os.path.join(prefix, 'lib', f'python{ver.major}{ver.minor}.zip'),
-                os.path.join(home, 'lib', f'python{ver.major}.{ver.minor}'),
-                os.path.join(home, 'lib', f'python{ver.major}.{ver.minor}/lib-dynload')]
+        expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)
 
         config = {
             'home': home,
@@ -1033,6 +1094,95 @@ def test_init_setpythonhome(self):
         self.check_all_configs("test_init_setpythonhome", config,
                                api=API_COMPAT, env=env)
 
+    def copy_paths_by_env(self, config):
+        all_configs = self._get_expected_config()
+        paths = all_configs['config']['module_search_paths']
+        paths_str = os.path.pathsep.join(paths)
+        config['pythonpath_env'] = paths_str
+        env = {'PYTHONPATH': paths_str}
+        return env
+
+    @unittest.skipIf(MS_WINDOWS, 'Windows does not use pybuilddir.txt')
+    def test_init_pybuilddir(self):
+        # Test path configuration with pybuilddir.txt configuration file
+
+        with self.tmpdir_with_python() as tmpdir:
+            # pybuilddir.txt is a sub-directory relative to the current
+            # directory (tmpdir)
+            subdir = 'libdir'
+            libdir = os.path.join(tmpdir, subdir)
+            os.mkdir(libdir)
+
+            filename = os.path.join(tmpdir, 'pybuilddir.txt')
+            with open(filename, "w", encoding="utf8") as fp:
+                fp.write(subdir)
+
+            module_search_paths = self.module_search_paths()
+            module_search_paths[-1] = libdir
+
+            executable = self.test_exe
+            config = {
+                'base_executable': executable,
+                'executable': executable,
+                'module_search_paths': module_search_paths,
+            }
+            env = self.copy_paths_by_env(config)
+            self.check_all_configs("test_init_compat_config", config,
+                                   api=API_COMPAT, env=env,
+                                   ignore_stderr=True, cwd=tmpdir)
+
+    def test_init_pyvenv_cfg(self):
+        # Test path configuration with pyvenv.cfg configuration file
+
+        with self.tmpdir_with_python() as tmpdir, \
+             tempfile.TemporaryDirectory() as pyvenv_home:
+            ver = sys.version_info
+
+            if not MS_WINDOWS:
+                lib_dynload = os.path.join(pyvenv_home,
+                                           'lib',
+                                           f'python{ver.major}.{ver.minor}',
+                                           'lib-dynload')
+                os.makedirs(lib_dynload)
+            else:
+                lib_dynload = os.path.join(pyvenv_home, 'lib')
+                os.makedirs(lib_dynload)
+                # getpathp.c uses Lib\os.py as the LANDMARK
+                shutil.copyfile(os.__file__, os.path.join(lib_dynload, 'os.py'))
+
+            filename = os.path.join(tmpdir, 'pyvenv.cfg')
+            with open(filename, "w", encoding="utf8") as fp:
+                print("home = %s" % pyvenv_home, file=fp)
+                print("include-system-site-packages = false", file=fp)
+
+            paths = self.module_search_paths()
+            if not MS_WINDOWS:
+                paths[-1] = lib_dynload
+            else:
+                for index, path in enumerate(paths):
+                    if index == 0:
+                        paths[index] = os.path.join(tmpdir, os.path.basename(path))
+                    else:
+                        paths[index] = os.path.join(pyvenv_home, os.path.basename(path))
+                paths[-1] = pyvenv_home
+
+            executable = self.test_exe
+            exec_prefix = pyvenv_home
+            config = {
+                'base_exec_prefix': exec_prefix,
+                'exec_prefix': exec_prefix,
+                'base_executable': executable,
+                'executable': executable,
+                'module_search_paths': paths,
+            }
+            if MS_WINDOWS:
+                config['base_prefix'] = pyvenv_home
+                config['prefix'] = pyvenv_home
+            env = self.copy_paths_by_env(config)
+            self.check_all_configs("test_init_compat_config", config,
+                                   api=API_COMPAT, env=env,
+                                   ignore_stderr=True, cwd=tmpdir)
+
 
 class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
     def test_open_code_hook(self):


From webhook-mailer at python.org  Tue Sep 24 20:29:21 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 25 Sep 2019 00:29:21 -0000
Subject: [Python-checkins] [3.8] bpo-36871: Handle spec errors in
 assert_has_calls (GH-16005) (GH-16364)
Message-ID: <mailman.54.1569371361.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/1a17a054f6314ce29cd2632c28aeed317a615360
commit: 1a17a054f6314ce29cd2632c28aeed317a615360
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-24T17:29:17-07:00
summary:

[3.8] bpo-36871: Handle spec errors in assert_has_calls (GH-16005) (GH-16364)



The fix in PR 13261 handled the underlying issue about the spec for specific methods not being applied correctly, but it didn't fix the issue that was causing the misleading error message.

The code currently grabs a list of responses from _call_matcher (which may include exceptions). But it doesn't reach inside the list when checking if the result is an exception. This results in a misleading error message when one of the provided calls does not match the spec.


https://bugs.python.org/issue36871



Automerge-Triggered-By: @gpshead
(cherry picked from commit b5a7a4f0c20717a4c92c371583b5521b83f40f32)


Co-authored-by: Samuel Freilich <sfreilich at google.com>


https://bugs.python.org/issue36871



Automerge-Triggered-By: @gpshead

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-24-18-45-46.bpo-36871.p47knk.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py
M Lib/unittest/test/testmock/testmock.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 0c7545bd2b7f..30f4663e5ef4 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -931,13 +931,21 @@ def assert_has_calls(self, calls, any_order=False):
         If `any_order` is True then the calls can be in any order, but
         they must all appear in `mock_calls`."""
         expected = [self._call_matcher(c) for c in calls]
-        cause = expected if isinstance(expected, Exception) else None
+        cause = next((e for e in expected if isinstance(e, Exception)), None)
         all_calls = _CallList(self._call_matcher(c) for c in self.mock_calls)
         if not any_order:
             if expected not in all_calls:
+                if cause is None:
+                    problem = 'Calls not found.'
+                else:
+                    problem = ('Error processing expected calls.\n'
+                               'Errors: {}').format(
+                                   [e if isinstance(e, Exception) else None
+                                    for e in expected])
                 raise AssertionError(
-                    'Calls not found.\nExpected: %r%s'
-                    % (_CallList(calls), self._calls_repr(prefix="Actual"))
+                    f'{problem}\n'
+                    f'Expected: {_CallList(calls)}'
+                    f'{self._calls_repr(prefix="Actual").rstrip(".")}'
                 ) from cause
             return
 
@@ -2220,12 +2228,20 @@ def assert_has_awaits(self, calls, any_order=False):
         they must all appear in :attr:`await_args_list`.
         """
         expected = [self._call_matcher(c) for c in calls]
-        cause = expected if isinstance(expected, Exception) else None
+        cause = next((e for e in expected if isinstance(e, Exception)), None)
         all_awaits = _CallList(self._call_matcher(c) for c in self.await_args_list)
         if not any_order:
             if expected not in all_awaits:
+                if cause is None:
+                    problem = 'Awaits not found.'
+                else:
+                    problem = ('Error processing expected awaits.\n'
+                               'Errors: {}').format(
+                                   [e if isinstance(e, Exception) else None
+                                    for e in expected])
                 raise AssertionError(
-                    f'Awaits not found.\nExpected: {_CallList(calls)}\n'
+                    f'{problem}\n'
+                    f'Expected: {_CallList(calls)}\n'
                     f'Actual: {self.await_args_list}'
                 ) from cause
             return
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index 9b50b6d86bbd..af53210d97b3 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -1,5 +1,6 @@
 import asyncio
 import inspect
+import re
 import unittest
 
 from unittest.mock import (call, AsyncMock, patch, MagicMock, create_autospec,
@@ -698,3 +699,30 @@ def test_assert_not_awaited(self):
         asyncio.run(self._runnable_test())
         with self.assertRaises(AssertionError):
             self.mock.assert_not_awaited()
+
+    def test_assert_has_awaits_not_matching_spec_error(self):
+        async def f(x=None): pass
+
+        self.mock = AsyncMock(spec=f)
+        asyncio.run(self._runnable_test(1))
+
+        with self.assertRaisesRegex(
+                AssertionError,
+                '^{}$'.format(
+                    re.escape('Awaits not found.\n'
+                              'Expected: [call()]\n'
+                              'Actual: [call(1)]'))) as cm:
+            self.mock.assert_has_awaits([call()])
+        self.assertIsNone(cm.exception.__cause__)
+
+        with self.assertRaisesRegex(
+                AssertionError,
+                '^{}$'.format(
+                    re.escape(
+                        'Error processing expected awaits.\n'
+                        "Errors: [None, TypeError('too many positional "
+                        "arguments')]\n"
+                        'Expected: [call(), call(1, 2)]\n'
+                        'Actual: [call(1)]'))) as cm:
+            self.mock.assert_has_awaits([call(), call(1, 2)])
+        self.assertIsInstance(cm.exception.__cause__, TypeError)
diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py
index 2847d650cddc..817c54822630 100644
--- a/Lib/unittest/test/testmock/testmock.py
+++ b/Lib/unittest/test/testmock/testmock.py
@@ -1426,6 +1426,32 @@ def f(a, b, c, d=None): pass
             mock.assert_has_calls(calls[:-1])
         mock.assert_has_calls(calls[:-1], any_order=True)
 
+    def test_assert_has_calls_not_matching_spec_error(self):
+        def f(x=None): pass
+
+        mock = Mock(spec=f)
+        mock(1)
+
+        with self.assertRaisesRegex(
+                AssertionError,
+                '^{}$'.format(
+                    re.escape('Calls not found.\n'
+                              'Expected: [call()]\n'
+                              'Actual: [call(1)]'))) as cm:
+            mock.assert_has_calls([call()])
+        self.assertIsNone(cm.exception.__cause__)
+
+
+        with self.assertRaisesRegex(
+                AssertionError,
+                '^{}$'.format(
+                    re.escape(
+                        'Error processing expected calls.\n'
+                        "Errors: [None, TypeError('too many positional arguments')]\n"
+                        "Expected: [call(), call(1, 2)]\n"
+                        'Actual: [call(1)]'))) as cm:
+            mock.assert_has_calls([call(), call(1, 2)])
+        self.assertIsInstance(cm.exception.__cause__, TypeError)
 
     def test_assert_any_call(self):
         mock = Mock()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-24-18-45-46.bpo-36871.p47knk.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-24-18-45-46.bpo-36871.p47knk.rst
new file mode 100644
index 000000000000..6b7b19a0d57e
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-24-18-45-46.bpo-36871.p47knk.rst	
@@ -0,0 +1,3 @@
+Improve error handling for the assert_has_calls and assert_has_awaits methods of
+mocks. Fixed a bug where any errors encountered while binding the expected calls
+to the mock's spec were silently swallowed, leading to misleading error output.


From webhook-mailer at python.org  Tue Sep 24 20:54:29 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 25 Sep 2019 00:54:29 -0000
Subject: [Python-checkins] bpo-38234: Cleanup getpath.c (GH-16367)
Message-ID: <mailman.55.1569372871.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/221fd84703c545408bbb4a6e0b58459651331f5c
commit: 221fd84703c545408bbb4a6e0b58459651331f5c
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T02:54:25+02:00
summary:

bpo-38234: Cleanup getpath.c (GH-16367)

* search_for_prefix() directly calls reduce() if found is greater
  than 0.
* Add calculate_pybuilddir() subfunction.
* search_for_prefix(): add path string buffer for readability.
* Fix some error handling code paths: release resources on error.
* calculate_read_pyenv(): rename tmpbuffer to filename.
* test.pythoninfo now also logs windows.dll_path

files:
M Lib/test/pythoninfo.py
M Modules/getpath.c
M PC/getpathp.c

diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py
index 28a902355e8a..d2fa6c593659 100644
--- a/Lib/test/pythoninfo.py
+++ b/Lib/test/pythoninfo.py
@@ -673,6 +673,13 @@ def collect_windows(info_add):
         res = bool(RtlAreLongPathsEnabled())
     info_add('windows.RtlAreLongPathsEnabled', res)
 
+    try:
+        import _winapi
+        dll_path = _winapi.GetModuleFileName(sys.dllhandle)
+        info_add('windows.dll_path', dll_path)
+    except (ImportError, AttributeError):
+        pass
+
 
 def collect_info(info):
     error = False
diff --git a/Modules/getpath.c b/Modules/getpath.c
index a4607ef2d567..4246d4ca5ac0 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -370,12 +370,15 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
                   const wchar_t *argv0_path,
                   wchar_t *prefix, size_t prefix_len, int *found)
 {
+    wchar_t path[MAXPATHLEN+1];
+    memset(path, 0, sizeof(path));
+    size_t path_len = Py_ARRAY_LENGTH(path);
+
     PyStatus status;
-    size_t n;
-    wchar_t *vpath;
 
     /* If PYTHONHOME is set, we believe it unconditionally */
     if (pathconfig->home) {
+        /* Path: <home> / <lib_python> */
         if (safe_wcscpy(prefix, pathconfig->home, prefix_len) < 0) {
             return PATHLEN_ERR();
         }
@@ -387,27 +390,25 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
-        status = joinpath(prefix, LANDMARK, prefix_len);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
         *found = 1;
         return _PyStatus_OK();
     }
 
     /* Check to see if argv[0] is in the build directory */
-    if (safe_wcscpy(prefix, argv0_path, prefix_len) < 0) {
+    if (safe_wcscpy(path, argv0_path, path_len) < 0) {
         return PATHLEN_ERR();
     }
-    status = joinpath(prefix, L"Modules/Setup.local", prefix_len);
+    status = joinpath(path, L"Modules/Setup.local", path_len);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
-    if (isfile(prefix)) {
-        /* Check VPATH to see if argv0_path is in the build directory. */
-        vpath = Py_DecodeLocale(VPATH, NULL);
+    if (isfile(path)) {
+        /* Check VPATH to see if argv0_path is in the build directory.
+           VPATH can be empty. */
+        wchar_t *vpath = Py_DecodeLocale(VPATH, NULL);
         if (vpath != NULL) {
+            /* Path: <argv0_path> / <vpath> / Lib / LANDMARK */
             if (safe_wcscpy(prefix, argv0_path, prefix_len) < 0) {
                 return PATHLEN_ERR();
             }
@@ -428,6 +429,7 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
             if (ismodule(prefix, prefix_len)) {
                 *found = -1;
+                reduce(prefix);
                 return _PyStatus_OK();
             }
         }
@@ -440,7 +442,8 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
     }
 
     do {
-        n = wcslen(prefix);
+        /* Path: <argv0_path or substring> / <lib_python> / LANDMARK */
+        size_t n = wcslen(prefix);
         status = joinpath(prefix, calculate->lib_python, prefix_len);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
@@ -452,13 +455,15 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
         if (ismodule(prefix, prefix_len)) {
             *found = 1;
+            reduce(prefix);
             return _PyStatus_OK();
         }
         prefix[n] = L'\0';
         reduce(prefix);
     } while (prefix[0]);
 
-    /* Look at configure's PREFIX */
+    /* Look at configure's PREFIX.
+       Path: <PREFIX macro> / <lib_python> / LANDMARK */
     if (safe_wcscpy(prefix, calculate->prefix, prefix_len) < 0) {
         return PATHLEN_ERR();
     }
@@ -473,6 +478,7 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
     if (ismodule(prefix, prefix_len)) {
         *found = 1;
+        reduce(prefix);
         return _PyStatus_OK();
     }
 
@@ -509,9 +515,6 @@ calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
             return status;
         }
     }
-    else {
-        reduce(prefix);
-    }
     return _PyStatus_OK();
 }
 
@@ -546,6 +549,67 @@ calculate_set_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 }
 
 
+static PyStatus
+calculate_pybuilddir(const wchar_t *argv0_path,
+                     wchar_t *exec_prefix, size_t exec_prefix_len,
+                     int *found)
+{
+    PyStatus status;
+
+    wchar_t filename[MAXPATHLEN+1];
+    memset(filename, 0, sizeof(filename));
+    size_t filename_len = Py_ARRAY_LENGTH(filename);
+
+    /* Check to see if argv[0] is in the build directory. "pybuilddir.txt"
+       is written by setup.py and contains the relative path to the location
+       of shared library modules.
+
+       Filename: <argv0_path> / "pybuilddir.txt" */
+    if (safe_wcscpy(filename, argv0_path, filename_len) < 0) {
+        return PATHLEN_ERR();
+    }
+    status = joinpath(filename, L"pybuilddir.txt", filename_len);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
+    if (!isfile(filename)) {
+        return _PyStatus_OK();
+    }
+
+    FILE *fp = _Py_wfopen(filename, L"rb");
+    if (fp == NULL) {
+        errno = 0;
+        return _PyStatus_OK();
+    }
+
+    char buf[MAXPATHLEN + 1];
+    size_t n = fread(buf, 1, Py_ARRAY_LENGTH(buf) - 1, fp);
+    buf[n] = '\0';
+    fclose(fp);
+
+    size_t dec_len;
+    wchar_t *pybuilddir = _Py_DecodeUTF8_surrogateescape(buf, n, &dec_len);
+    if (!pybuilddir) {
+        return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len);
+    }
+
+    /* Path: <argv0_path> / <pybuilddir content> */
+    if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
+        PyMem_RawFree(pybuilddir);
+        return PATHLEN_ERR();
+    }
+    status = joinpath(exec_prefix, pybuilddir, exec_prefix_len);
+    PyMem_RawFree(pybuilddir);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
+    *found = -1;
+    return _PyStatus_OK();
+}
+
+
 /* search_for_exec_prefix requires that argv0_path be no more than
    MAXPATHLEN bytes long.
 */
@@ -556,10 +620,10 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
                        int *found)
 {
     PyStatus status;
-    size_t n;
 
     /* If PYTHONHOME is set, we believe it unconditionally */
     if (pathconfig->home) {
+        /* Path: <home> / <lib_python> / "lib-dynload" */
         wchar_t *delim = wcschr(pathconfig->home, DELIM);
         if (delim) {
             if (safe_wcscpy(exec_prefix, delim+1, exec_prefix_len) < 0) {
@@ -583,47 +647,15 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
         return _PyStatus_OK();
     }
 
-    /* Check to see if argv[0] is in the build directory. "pybuilddir.txt"
-       is written by setup.py and contains the relative path to the location
-       of shared library modules. */
-    if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
-        return PATHLEN_ERR();
-    }
-    status = joinpath(exec_prefix, L"pybuilddir.txt", exec_prefix_len);
+    /* Check for pybuilddir.txt */
+    assert(*found == 0);
+    status = calculate_pybuilddir(argv0_path, exec_prefix, exec_prefix_len,
+                                  found);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
-
-    if (isfile(exec_prefix)) {
-        FILE *f = _Py_wfopen(exec_prefix, L"rb");
-        if (f == NULL) {
-            errno = 0;
-        }
-        else {
-            char buf[MAXPATHLEN + 1];
-            n = fread(buf, 1, Py_ARRAY_LENGTH(buf) - 1, f);
-            buf[n] = '\0';
-            fclose(f);
-
-            wchar_t *pybuilddir;
-            size_t dec_len;
-            pybuilddir = _Py_DecodeUTF8_surrogateescape(buf, n, &dec_len);
-            if (!pybuilddir) {
-                return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len);
-            }
-
-            if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
-                return PATHLEN_ERR();
-            }
-            status = joinpath(exec_prefix, pybuilddir, exec_prefix_len);
-            PyMem_RawFree(pybuilddir );
-            if (_PyStatus_EXCEPTION(status)) {
-                return status;
-            }
-
-            *found = -1;
-            return _PyStatus_OK();
-        }
+    if (*found) {
+        return _PyStatus_OK();
     }
 
     /* Search from argv0_path, until root is found */
@@ -633,7 +665,8 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
     }
 
     do {
-        n = wcslen(exec_prefix);
+        /* Path: <argv0_path or substring> / <lib_python> / "lib-dynload" */
+        size_t n = wcslen(exec_prefix);
         status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
@@ -650,7 +683,9 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
         reduce(exec_prefix);
     } while (exec_prefix[0]);
 
-    /* Look at configure's EXEC_PREFIX */
+    /* Look at configure's EXEC_PREFIX.
+
+       Path: <EXEC_PREFIX macro> / <lib_python> / "lib-dynload" */
     if (safe_wcscpy(exec_prefix, calculate->exec_prefix, exec_prefix_len) < 0) {
         return PATHLEN_ERR();
     }
@@ -962,43 +997,49 @@ calculate_read_pyenv(PyCalculatePath *calculate,
                      wchar_t *argv0_path, size_t argv0_path_len)
 {
     PyStatus status;
-    wchar_t tmpbuffer[MAXPATHLEN+1];
-    const size_t buflen = Py_ARRAY_LENGTH(tmpbuffer);
-    wchar_t *env_cfg = L"pyvenv.cfg";
+    const wchar_t *env_cfg = L"pyvenv.cfg";
     FILE *env_file;
 
-    if (safe_wcscpy(tmpbuffer, argv0_path, buflen) < 0) {
+    wchar_t filename[MAXPATHLEN+1];
+    const size_t filename_len = Py_ARRAY_LENGTH(filename);
+    memset(filename, 0, sizeof(filename));
+
+    /* Filename: <argv0_path_len> / "pyvenv.cfg" */
+    if (safe_wcscpy(filename, argv0_path, filename_len) < 0) {
         return PATHLEN_ERR();
     }
 
-    status = joinpath(tmpbuffer, env_cfg, buflen);
+    status = joinpath(filename, env_cfg, filename_len);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
-    env_file = _Py_wfopen(tmpbuffer, L"r");
+    env_file = _Py_wfopen(filename, L"r");
     if (env_file == NULL) {
         errno = 0;
 
-        reduce(tmpbuffer);
-        reduce(tmpbuffer);
-        status = joinpath(tmpbuffer, env_cfg, buflen);
+        /* Filename: <basename(basename(argv0_path_len))> / "pyvenv.cfg" */
+        reduce(filename);
+        reduce(filename);
+        status = joinpath(filename, env_cfg, filename_len);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
 
-        env_file = _Py_wfopen(tmpbuffer, L"r");
+        env_file = _Py_wfopen(filename, L"r");
         if (env_file == NULL) {
             errno = 0;
+            return _PyStatus_OK();
         }
     }
 
-    if (env_file == NULL) {
-        return _PyStatus_OK();
-    }
-
     /* Look for a 'home' variable and set argv0_path to it, if found */
-    if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, buflen)) {
-        if (safe_wcscpy(argv0_path, tmpbuffer, argv0_path_len) < 0) {
+    wchar_t home[MAXPATHLEN+1];
+    memset(home, 0, sizeof(home));
+
+    if (_Py_FindEnvConfigValue(env_file, L"home",
+                               home, Py_ARRAY_LENGTH(home))) {
+        if (safe_wcscpy(argv0_path, home, argv0_path_len) < 0) {
+            fclose(env_file);
             return PATHLEN_ERR();
         }
     }
@@ -1200,6 +1241,8 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
         return status;
     }
 
+    /* If a pyvenv.cfg configure file is found,
+       argv0_path is overriden with its 'home' variable. */
     status = calculate_read_pyenv(calculate,
                                   argv0_path, Py_ARRAY_LENGTH(argv0_path));
     if (_PyStatus_EXCEPTION(status)) {
diff --git a/PC/getpathp.c b/PC/getpathp.c
index c4c0636dddeb..0eb75b8daf31 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -757,34 +757,34 @@ static void
 calculate_pyvenv_file(PyCalculatePath *calculate,
                       wchar_t *argv0_path, size_t argv0_path_len)
 {
-    wchar_t envbuffer[MAXPATHLEN+1];
+    wchar_t filename[MAXPATHLEN+1];
     const wchar_t *env_cfg = L"pyvenv.cfg";
 
-    wcscpy_s(envbuffer, MAXPATHLEN+1, argv0_path);
-    join(envbuffer, env_cfg);
+    /* Filename: <argv0_path_len> / "pyvenv.cfg" */
+    wcscpy_s(filename, MAXPATHLEN+1, argv0_path);
+    join(filename, env_cfg);
 
-    FILE *env_file = _Py_wfopen(envbuffer, L"r");
+    FILE *env_file = _Py_wfopen(filename, L"r");
     if (env_file == NULL) {
         errno = 0;
 
-        reduce(envbuffer);
-        reduce(envbuffer);
-        join(envbuffer, env_cfg);
+        /* Filename: <basename(basename(argv0_path_len))> / "pyvenv.cfg" */
+        reduce(filename);
+        reduce(filename);
+        join(filename, env_cfg);
 
-        env_file = _Py_wfopen(envbuffer, L"r");
+        env_file = _Py_wfopen(filename, L"r");
         if (env_file == NULL) {
             errno = 0;
+            return;
         }
     }
 
-    if (env_file == NULL) {
-        return;
-    }
-
     /* Look for a 'home' variable and set argv0_path to it, if found */
-    wchar_t tmpbuffer[MAXPATHLEN+1];
-    if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, MAXPATHLEN)) {
-        wcscpy_s(argv0_path, argv0_path_len, tmpbuffer);
+    wchar_t home[MAXPATHLEN+1];
+    if (_Py_FindEnvConfigValue(env_file, L"home",
+                               home, Py_ARRAY_LENGTH(home))) {
+        wcscpy_s(argv0_path, argv0_path_len, home);
     }
     fclose(env_file);
 }


From webhook-mailer at python.org  Tue Sep 24 23:38:48 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Wed, 25 Sep 2019 03:38:48 -0000
Subject: [Python-checkins] =?utf-8?q?bpo-22273=3A_Update_ctypes_to_correc?=
 =?utf-8?q?tly_handle_arrays_in_small_structur=E2=80=A6_=28GH-15839=29?=
Message-ID: <mailman.56.1569382730.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/12f209eccb1587e8c98057d9c5f865c21f4a16c1
commit: 12f209eccb1587e8c98057d9c5f865c21f4a16c1
branch: master
author: Vinay Sajip <vinay_sajip at yahoo.co.uk>
committer: GitHub <noreply at github.com>
date: 2019-09-25T04:38:44+01:00
summary:

bpo-22273: Update ctypes to correctly handle arrays in small structur? (GH-15839)

files:
M Lib/ctypes/test/test_structures.py
M Modules/_ctypes/_ctypes_test.c
M Modules/_ctypes/stgdict.c

diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py
index fda104563d29..11c194b0b73e 100644
--- a/Lib/ctypes/test/test_structures.py
+++ b/Lib/ctypes/test/test_structures.py
@@ -477,6 +477,47 @@ class X(Structure):
         self.assertEqual(s.first, got.first)
         self.assertEqual(s.second, got.second)
 
+    def test_array_in_struct(self):
+        # See bpo-22273
+
+        # These should mirror the structures in Modules/_ctypes/_ctypes_test.c
+        class Test2(Structure):
+            _fields_ = [
+                ('data', c_ubyte * 16),
+            ]
+
+        class Test3(Structure):
+            _fields_ = [
+                ('data', c_double * 2),
+            ]
+
+        s = Test2()
+        expected = 0
+        for i in range(16):
+            s.data[i] = i
+            expected += i
+        dll = CDLL(_ctypes_test.__file__)
+        func = dll._testfunc_array_in_struct1
+        func.restype = c_int
+        func.argtypes = (Test2,)
+        result = func(s)
+        self.assertEqual(result, expected)
+        # check the passed-in struct hasn't changed
+        for i in range(16):
+            self.assertEqual(s.data[i], i)
+
+        s = Test3()
+        s.data[0] = 3.14159
+        s.data[1] = 2.71828
+        expected = 3.14159 + 2.71828
+        func = dll._testfunc_array_in_struct2
+        func.restype = c_double
+        func.argtypes = (Test3,)
+        result = func(s)
+        self.assertEqual(result, expected)
+        # check the passed-in struct hasn't changed
+        self.assertEqual(s.data[0], 3.14159)
+        self.assertEqual(s.data[1], 2.71828)
 
 class PointerMemberTestCase(unittest.TestCase):
 
diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c
index 4789d6bdda8c..10f6591d24cd 100644
--- a/Modules/_ctypes/_ctypes_test.c
+++ b/Modules/_ctypes/_ctypes_test.c
@@ -74,6 +74,45 @@ _testfunc_reg_struct_update_value(TestReg in)
     ((volatile TestReg *)&in)->second = 0x0badf00d;
 }
 
+/*
+ * See bpo-22273. Structs containing arrays should work on Linux 64-bit.
+ */
+
+typedef struct {
+    unsigned char data[16];
+} Test2;
+
+EXPORT(int)
+_testfunc_array_in_struct1(Test2 in)
+{
+    int result = 0;
+
+    for (unsigned i = 0; i < 16; i++)
+        result += in.data[i];
+    /* As the structure is passed by value, changes to it shouldn't be
+     * reflected in the caller.
+     */
+    memset(in.data, 0, sizeof(in.data));
+    return result;
+}
+
+typedef struct {
+    double data[2];
+} Test3;
+
+EXPORT(double)
+_testfunc_array_in_struct2(Test3 in)
+{
+    double result = 0;
+
+    for (unsigned i = 0; i < 2; i++)
+        result += in.data[i];
+    /* As the structure is passed by value, changes to it shouldn't be
+     * reflected in the caller.
+     */
+    memset(in.data, 0, sizeof(in.data));
+    return result;
+}
 
 EXPORT(void)testfunc_array(int values[4])
 {
diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c
index 8709cc5404a2..62dceceb4f4e 100644
--- a/Modules/_ctypes/stgdict.c
+++ b/Modules/_ctypes/stgdict.c
@@ -350,6 +350,9 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     int pack;
     Py_ssize_t ffi_ofs;
     int big_endian;
+#if defined(X86_64)
+    int arrays_seen = 0;
+#endif
 
     /* HACK Alert: I cannot be bothered to fix ctypes.com, so there has to
        be a way to use the old, broken semantics: _fields_ are not extended
@@ -501,6 +504,10 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
             Py_XDECREF(pair);
             return -1;
         }
+#if defined(X86_64)
+        if (PyCArrayTypeObject_Check(desc))
+            arrays_seen = 1;
+#endif
         dict = PyType_stgdict(desc);
         if (dict == NULL) {
             Py_DECREF(pair);
@@ -641,6 +648,106 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     stgdict->align = total_align;
     stgdict->length = len;      /* ADD ffi_ofs? */
 
+#if defined(X86_64)
+
+#define MAX_ELEMENTS 16
+
+    if (arrays_seen && (size <= 16)) {
+        /*
+         * See bpo-22273. Arrays are normally treated as pointers, which is
+         * fine when an array name is being passed as parameter, but not when
+         * passing structures by value that contain arrays. On 64-bit Linux,
+         * small structures passed by value are passed in registers, and in
+         * order to do this, libffi needs to know the true type of the array
+         * members of structs. Treating them as pointers breaks things.
+         *
+         * By small structures, we mean ones that are 16 bytes or less. In that
+         * case, there can't be more than 16 elements after unrolling arrays,
+         * as we (will) disallow bitfields. So we can collect the true ffi_type
+         * values in a fixed-size local array on the stack and, if any arrays
+         * were seen, replace the ffi_type_pointer.elements with a more
+         * accurate set, to allow libffi to marshal them into registers
+         * correctly. It means one more loop over the fields, but if we got
+         * here, the structure is small, so there aren't too many of those.
+         */
+        ffi_type *actual_types[MAX_ELEMENTS + 1];
+        int actual_type_index = 0;
+
+        memset(actual_types, 0, sizeof(actual_types));
+        for (i = 0; i < len; ++i) {
+            PyObject *name, *desc;
+            PyObject *pair = PySequence_GetItem(fields, i);
+            StgDictObject *dict;
+            int bitsize = 0;
+
+            if (pair == NULL) {
+                return -1;
+            }
+            if (!PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) {
+                PyErr_SetString(PyExc_TypeError,
+                                "'_fields_' must be a sequence of (name, C type) pairs");
+                Py_XDECREF(pair);
+                return -1;
+            }
+            dict = PyType_stgdict(desc);
+            if (dict == NULL) {
+                Py_DECREF(pair);
+                PyErr_Format(PyExc_TypeError,
+                             "second item in _fields_ tuple (index %zd) must be a C type",
+                             i);
+                return -1;
+            }
+            if (!PyCArrayTypeObject_Check(desc)) {
+                /* Not an array. Just copy over the element ffi_type. */
+                actual_types[actual_type_index++] = &dict->ffi_type_pointer;
+                assert(actual_type_index <= MAX_ELEMENTS);
+            }
+            else {
+                int length = dict->length;
+                StgDictObject *edict;
+
+                edict = PyType_stgdict(dict->proto);
+                if (edict == NULL) {
+                    Py_DECREF(pair);
+                    PyErr_Format(PyExc_TypeError,
+                                 "second item in _fields_ tuple (index %zd) must be a C type",
+                                 i);
+                    return -1;
+                }
+                /* Copy over the element's type, length times. */
+                while (length > 0) {
+                    actual_types[actual_type_index++] = &edict->ffi_type_pointer;
+                    assert(actual_type_index <= MAX_ELEMENTS);
+                    length--;
+                }
+            }
+            Py_DECREF(pair);
+        }
+
+        actual_types[actual_type_index++] = NULL;
+        /*
+         * Replace the old elements with the new, taking into account
+         * base class elements where necessary.
+         */
+        assert(stgdict->ffi_type_pointer.elements);
+        PyMem_Free(stgdict->ffi_type_pointer.elements);
+        stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *,
+                                                       ffi_ofs + actual_type_index);
+        if (stgdict->ffi_type_pointer.elements == NULL) {
+            PyErr_NoMemory();
+            return -1;
+        }
+        if (ffi_ofs) {
+            memcpy(stgdict->ffi_type_pointer.elements,
+                   basedict->ffi_type_pointer.elements,
+                   ffi_ofs * sizeof(ffi_type *));
+
+        }
+        memcpy(&stgdict->ffi_type_pointer.elements[ffi_ofs], actual_types,
+               actual_type_index * sizeof(ffi_type *));
+    }
+#endif
+
     /* We did check that this flag was NOT set above, it must not
        have been set until now. */
     if (stgdict->flags & DICTFLAG_FINAL) {


From webhook-mailer at python.org  Wed Sep 25 00:10:24 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Wed, 25 Sep 2019 04:10:24 -0000
Subject: [Python-checkins] =?utf-8?q?bpo-22273=3A_Update_ctypes_to_correc?=
 =?utf-8?q?tly_handle_arrays_in_small_structur=E2=80=A6_=28GH-15839=29_=28?=
 =?utf-8?q?GH-16370=29?=
Message-ID: <mailman.57.1569384625.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/ce62dcc460cf88f663c34c4a0948c1ee1dc53f4d
commit: ce62dcc460cf88f663c34c4a0948c1ee1dc53f4d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Vinay Sajip <vinay_sajip at yahoo.co.uk>
date: 2019-09-25T05:10:20+01:00
summary:

bpo-22273: Update ctypes to correctly handle arrays in small structur? (GH-15839) (GH-16370)

(cherry picked from commit 12f209eccb1587e8c98057d9c5f865c21f4a16c1)

files:
M Lib/ctypes/test/test_structures.py
M Modules/_ctypes/_ctypes_test.c
M Modules/_ctypes/stgdict.c

diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py
index fda104563d29..11c194b0b73e 100644
--- a/Lib/ctypes/test/test_structures.py
+++ b/Lib/ctypes/test/test_structures.py
@@ -477,6 +477,47 @@ class X(Structure):
         self.assertEqual(s.first, got.first)
         self.assertEqual(s.second, got.second)
 
+    def test_array_in_struct(self):
+        # See bpo-22273
+
+        # These should mirror the structures in Modules/_ctypes/_ctypes_test.c
+        class Test2(Structure):
+            _fields_ = [
+                ('data', c_ubyte * 16),
+            ]
+
+        class Test3(Structure):
+            _fields_ = [
+                ('data', c_double * 2),
+            ]
+
+        s = Test2()
+        expected = 0
+        for i in range(16):
+            s.data[i] = i
+            expected += i
+        dll = CDLL(_ctypes_test.__file__)
+        func = dll._testfunc_array_in_struct1
+        func.restype = c_int
+        func.argtypes = (Test2,)
+        result = func(s)
+        self.assertEqual(result, expected)
+        # check the passed-in struct hasn't changed
+        for i in range(16):
+            self.assertEqual(s.data[i], i)
+
+        s = Test3()
+        s.data[0] = 3.14159
+        s.data[1] = 2.71828
+        expected = 3.14159 + 2.71828
+        func = dll._testfunc_array_in_struct2
+        func.restype = c_double
+        func.argtypes = (Test3,)
+        result = func(s)
+        self.assertEqual(result, expected)
+        # check the passed-in struct hasn't changed
+        self.assertEqual(s.data[0], 3.14159)
+        self.assertEqual(s.data[1], 2.71828)
 
 class PointerMemberTestCase(unittest.TestCase):
 
diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c
index bae4976a08d3..a7a8976d2d88 100644
--- a/Modules/_ctypes/_ctypes_test.c
+++ b/Modules/_ctypes/_ctypes_test.c
@@ -74,6 +74,45 @@ _testfunc_reg_struct_update_value(TestReg in)
     ((volatile TestReg *)&in)->second = 0x0badf00d;
 }
 
+/*
+ * See bpo-22273. Structs containing arrays should work on Linux 64-bit.
+ */
+
+typedef struct {
+    unsigned char data[16];
+} Test2;
+
+EXPORT(int)
+_testfunc_array_in_struct1(Test2 in)
+{
+    int result = 0;
+
+    for (unsigned i = 0; i < 16; i++)
+        result += in.data[i];
+    /* As the structure is passed by value, changes to it shouldn't be
+     * reflected in the caller.
+     */
+    memset(in.data, 0, sizeof(in.data));
+    return result;
+}
+
+typedef struct {
+    double data[2];
+} Test3;
+
+EXPORT(double)
+_testfunc_array_in_struct2(Test3 in)
+{
+    double result = 0;
+
+    for (unsigned i = 0; i < 2; i++)
+        result += in.data[i];
+    /* As the structure is passed by value, changes to it shouldn't be
+     * reflected in the caller.
+     */
+    memset(in.data, 0, sizeof(in.data));
+    return result;
+}
 
 EXPORT(void)testfunc_array(int values[4])
 {
diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c
index 8709cc5404a2..62dceceb4f4e 100644
--- a/Modules/_ctypes/stgdict.c
+++ b/Modules/_ctypes/stgdict.c
@@ -350,6 +350,9 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     int pack;
     Py_ssize_t ffi_ofs;
     int big_endian;
+#if defined(X86_64)
+    int arrays_seen = 0;
+#endif
 
     /* HACK Alert: I cannot be bothered to fix ctypes.com, so there has to
        be a way to use the old, broken semantics: _fields_ are not extended
@@ -501,6 +504,10 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
             Py_XDECREF(pair);
             return -1;
         }
+#if defined(X86_64)
+        if (PyCArrayTypeObject_Check(desc))
+            arrays_seen = 1;
+#endif
         dict = PyType_stgdict(desc);
         if (dict == NULL) {
             Py_DECREF(pair);
@@ -641,6 +648,106 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     stgdict->align = total_align;
     stgdict->length = len;      /* ADD ffi_ofs? */
 
+#if defined(X86_64)
+
+#define MAX_ELEMENTS 16
+
+    if (arrays_seen && (size <= 16)) {
+        /*
+         * See bpo-22273. Arrays are normally treated as pointers, which is
+         * fine when an array name is being passed as parameter, but not when
+         * passing structures by value that contain arrays. On 64-bit Linux,
+         * small structures passed by value are passed in registers, and in
+         * order to do this, libffi needs to know the true type of the array
+         * members of structs. Treating them as pointers breaks things.
+         *
+         * By small structures, we mean ones that are 16 bytes or less. In that
+         * case, there can't be more than 16 elements after unrolling arrays,
+         * as we (will) disallow bitfields. So we can collect the true ffi_type
+         * values in a fixed-size local array on the stack and, if any arrays
+         * were seen, replace the ffi_type_pointer.elements with a more
+         * accurate set, to allow libffi to marshal them into registers
+         * correctly. It means one more loop over the fields, but if we got
+         * here, the structure is small, so there aren't too many of those.
+         */
+        ffi_type *actual_types[MAX_ELEMENTS + 1];
+        int actual_type_index = 0;
+
+        memset(actual_types, 0, sizeof(actual_types));
+        for (i = 0; i < len; ++i) {
+            PyObject *name, *desc;
+            PyObject *pair = PySequence_GetItem(fields, i);
+            StgDictObject *dict;
+            int bitsize = 0;
+
+            if (pair == NULL) {
+                return -1;
+            }
+            if (!PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) {
+                PyErr_SetString(PyExc_TypeError,
+                                "'_fields_' must be a sequence of (name, C type) pairs");
+                Py_XDECREF(pair);
+                return -1;
+            }
+            dict = PyType_stgdict(desc);
+            if (dict == NULL) {
+                Py_DECREF(pair);
+                PyErr_Format(PyExc_TypeError,
+                             "second item in _fields_ tuple (index %zd) must be a C type",
+                             i);
+                return -1;
+            }
+            if (!PyCArrayTypeObject_Check(desc)) {
+                /* Not an array. Just copy over the element ffi_type. */
+                actual_types[actual_type_index++] = &dict->ffi_type_pointer;
+                assert(actual_type_index <= MAX_ELEMENTS);
+            }
+            else {
+                int length = dict->length;
+                StgDictObject *edict;
+
+                edict = PyType_stgdict(dict->proto);
+                if (edict == NULL) {
+                    Py_DECREF(pair);
+                    PyErr_Format(PyExc_TypeError,
+                                 "second item in _fields_ tuple (index %zd) must be a C type",
+                                 i);
+                    return -1;
+                }
+                /* Copy over the element's type, length times. */
+                while (length > 0) {
+                    actual_types[actual_type_index++] = &edict->ffi_type_pointer;
+                    assert(actual_type_index <= MAX_ELEMENTS);
+                    length--;
+                }
+            }
+            Py_DECREF(pair);
+        }
+
+        actual_types[actual_type_index++] = NULL;
+        /*
+         * Replace the old elements with the new, taking into account
+         * base class elements where necessary.
+         */
+        assert(stgdict->ffi_type_pointer.elements);
+        PyMem_Free(stgdict->ffi_type_pointer.elements);
+        stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *,
+                                                       ffi_ofs + actual_type_index);
+        if (stgdict->ffi_type_pointer.elements == NULL) {
+            PyErr_NoMemory();
+            return -1;
+        }
+        if (ffi_ofs) {
+            memcpy(stgdict->ffi_type_pointer.elements,
+                   basedict->ffi_type_pointer.elements,
+                   ffi_ofs * sizeof(ffi_type *));
+
+        }
+        memcpy(&stgdict->ffi_type_pointer.elements[ffi_ofs], actual_types,
+               actual_type_index * sizeof(ffi_type *));
+    }
+#endif
+
     /* We did check that this flag was NOT set above, it must not
        have been set until now. */
     if (stgdict->flags & DICTFLAG_FINAL) {


From webhook-mailer at python.org  Wed Sep 25 00:10:47 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Wed, 25 Sep 2019 04:10:47 -0000
Subject: [Python-checkins] =?utf-8?q?bpo-22273=3A_Update_ctypes_to_correc?=
 =?utf-8?q?tly_handle_arrays_in_small_structur=E2=80=A6_=28GH-15839=29_=28?=
 =?utf-8?q?GH-16369=29?=
Message-ID: <mailman.58.1569384648.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/16c0f6df62a39f9f7712b1c0577de4eefcb4c1bf
commit: 16c0f6df62a39f9f7712b1c0577de4eefcb4c1bf
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Vinay Sajip <vinay_sajip at yahoo.co.uk>
date: 2019-09-25T05:10:43+01:00
summary:

bpo-22273: Update ctypes to correctly handle arrays in small structur? (GH-15839) (GH-16369)

(cherry picked from commit 12f209eccb1587e8c98057d9c5f865c21f4a16c1)

files:
M Lib/ctypes/test/test_structures.py
M Modules/_ctypes/_ctypes_test.c
M Modules/_ctypes/stgdict.c

diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py
index d1ea43bc7e3b..20d0fa72ce43 100644
--- a/Lib/ctypes/test/test_structures.py
+++ b/Lib/ctypes/test/test_structures.py
@@ -438,6 +438,47 @@ class X(Structure):
         self.assertEqual(s.first, got.first)
         self.assertEqual(s.second, got.second)
 
+    def test_array_in_struct(self):
+        # See bpo-22273
+
+        # These should mirror the structures in Modules/_ctypes/_ctypes_test.c
+        class Test2(Structure):
+            _fields_ = [
+                ('data', c_ubyte * 16),
+            ]
+
+        class Test3(Structure):
+            _fields_ = [
+                ('data', c_double * 2),
+            ]
+
+        s = Test2()
+        expected = 0
+        for i in range(16):
+            s.data[i] = i
+            expected += i
+        dll = CDLL(_ctypes_test.__file__)
+        func = dll._testfunc_array_in_struct1
+        func.restype = c_int
+        func.argtypes = (Test2,)
+        result = func(s)
+        self.assertEqual(result, expected)
+        # check the passed-in struct hasn't changed
+        for i in range(16):
+            self.assertEqual(s.data[i], i)
+
+        s = Test3()
+        s.data[0] = 3.14159
+        s.data[1] = 2.71828
+        expected = 3.14159 + 2.71828
+        func = dll._testfunc_array_in_struct2
+        func.restype = c_double
+        func.argtypes = (Test3,)
+        result = func(s)
+        self.assertEqual(result, expected)
+        # check the passed-in struct hasn't changed
+        self.assertEqual(s.data[0], 3.14159)
+        self.assertEqual(s.data[1], 2.71828)
 
 class PointerMemberTestCase(unittest.TestCase):
 
diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c
index 0152945ca1ad..0b3348b7fe00 100644
--- a/Modules/_ctypes/_ctypes_test.c
+++ b/Modules/_ctypes/_ctypes_test.c
@@ -74,6 +74,45 @@ _testfunc_reg_struct_update_value(TestReg in)
     ((volatile TestReg *)&in)->second = 0x0badf00d;
 }
 
+/*
+ * See bpo-22273. Structs containing arrays should work on Linux 64-bit.
+ */
+
+typedef struct {
+    unsigned char data[16];
+} Test2;
+
+EXPORT(int)
+_testfunc_array_in_struct1(Test2 in)
+{
+    int result = 0;
+
+    for (unsigned i = 0; i < 16; i++)
+        result += in.data[i];
+    /* As the structure is passed by value, changes to it shouldn't be
+     * reflected in the caller.
+     */
+    memset(in.data, 0, sizeof(in.data));
+    return result;
+}
+
+typedef struct {
+    double data[2];
+} Test3;
+
+EXPORT(double)
+_testfunc_array_in_struct2(Test3 in)
+{
+    double result = 0;
+
+    for (unsigned i = 0; i < 2; i++)
+        result += in.data[i];
+    /* As the structure is passed by value, changes to it shouldn't be
+     * reflected in the caller.
+     */
+    memset(in.data, 0, sizeof(in.data));
+    return result;
+}
 
 EXPORT(void)testfunc_array(int values[4])
 {
diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c
index 25656ff87875..e584abc96f1a 100644
--- a/Modules/_ctypes/stgdict.c
+++ b/Modules/_ctypes/stgdict.c
@@ -344,6 +344,9 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     int pack = 0;
     Py_ssize_t ffi_ofs;
     int big_endian;
+#if defined(X86_64)
+    int arrays_seen = 0;
+#endif
 
     /* HACK Alert: I cannot be bothered to fix ctypes.com, so there has to
        be a way to use the old, broken sematics: _fields_ are not extended
@@ -468,6 +471,10 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
             Py_XDECREF(pair);
             return -1;
         }
+#if defined(X86_64)
+        if (PyCArrayTypeObject_Check(desc))
+            arrays_seen = 1;
+#endif
         dict = PyType_stgdict(desc);
         if (dict == NULL) {
             Py_DECREF(pair);
@@ -608,6 +615,106 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     stgdict->align = total_align;
     stgdict->length = len;      /* ADD ffi_ofs? */
 
+#if defined(X86_64)
+
+#define MAX_ELEMENTS 16
+
+    if (arrays_seen && (size <= 16)) {
+        /*
+         * See bpo-22273. Arrays are normally treated as pointers, which is
+         * fine when an array name is being passed as parameter, but not when
+         * passing structures by value that contain arrays. On 64-bit Linux,
+         * small structures passed by value are passed in registers, and in
+         * order to do this, libffi needs to know the true type of the array
+         * members of structs. Treating them as pointers breaks things.
+         *
+         * By small structures, we mean ones that are 16 bytes or less. In that
+         * case, there can't be more than 16 elements after unrolling arrays,
+         * as we (will) disallow bitfields. So we can collect the true ffi_type
+         * values in a fixed-size local array on the stack and, if any arrays
+         * were seen, replace the ffi_type_pointer.elements with a more
+         * accurate set, to allow libffi to marshal them into registers
+         * correctly. It means one more loop over the fields, but if we got
+         * here, the structure is small, so there aren't too many of those.
+         */
+        ffi_type *actual_types[MAX_ELEMENTS + 1];
+        int actual_type_index = 0;
+
+        memset(actual_types, 0, sizeof(actual_types));
+        for (i = 0; i < len; ++i) {
+            PyObject *name, *desc;
+            PyObject *pair = PySequence_GetItem(fields, i);
+            StgDictObject *dict;
+            int bitsize = 0;
+
+            if (pair == NULL) {
+                return -1;
+            }
+            if (!PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) {
+                PyErr_SetString(PyExc_TypeError,
+                                "'_fields_' must be a sequence of (name, C type) pairs");
+                Py_XDECREF(pair);
+                return -1;
+            }
+            dict = PyType_stgdict(desc);
+            if (dict == NULL) {
+                Py_DECREF(pair);
+                PyErr_Format(PyExc_TypeError,
+                             "second item in _fields_ tuple (index %zd) must be a C type",
+                             i);
+                return -1;
+            }
+            if (!PyCArrayTypeObject_Check(desc)) {
+                /* Not an array. Just copy over the element ffi_type. */
+                actual_types[actual_type_index++] = &dict->ffi_type_pointer;
+                assert(actual_type_index <= MAX_ELEMENTS);
+            }
+            else {
+                int length = dict->length;
+                StgDictObject *edict;
+
+                edict = PyType_stgdict(dict->proto);
+                if (edict == NULL) {
+                    Py_DECREF(pair);
+                    PyErr_Format(PyExc_TypeError,
+                                 "second item in _fields_ tuple (index %zd) must be a C type",
+                                 i);
+                    return -1;
+                }
+                /* Copy over the element's type, length times. */
+                while (length > 0) {
+                    actual_types[actual_type_index++] = &edict->ffi_type_pointer;
+                    assert(actual_type_index <= MAX_ELEMENTS);
+                    length--;
+                }
+            }
+            Py_DECREF(pair);
+        }
+
+        actual_types[actual_type_index++] = NULL;
+        /*
+         * Replace the old elements with the new, taking into account
+         * base class elements where necessary.
+         */
+        assert(stgdict->ffi_type_pointer.elements);
+        PyMem_Free(stgdict->ffi_type_pointer.elements);
+        stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *,
+                                                       ffi_ofs + actual_type_index);
+        if (stgdict->ffi_type_pointer.elements == NULL) {
+            PyErr_NoMemory();
+            return -1;
+        }
+        if (ffi_ofs) {
+            memcpy(stgdict->ffi_type_pointer.elements,
+                   basedict->ffi_type_pointer.elements,
+                   ffi_ofs * sizeof(ffi_type *));
+
+        }
+        memcpy(&stgdict->ffi_type_pointer.elements[ffi_ofs], actual_types,
+               actual_type_index * sizeof(ffi_type *));
+    }
+#endif
+
     /* We did check that this flag was NOT set above, it must not
        have been set until now. */
     if (stgdict->flags & DICTFLAG_FINAL) {


From webhook-mailer at python.org  Wed Sep 25 01:29:23 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Wed, 25 Sep 2019 05:29:23 -0000
Subject: [Python-checkins] [3.7] bpo-36871: Handle spec errors in
 assert_has_calls (GH-16364) (GH-16374)
Message-ID: <mailman.59.1569389364.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/4042e1afd252858de53e2b79d946a51a0182d1ba
commit: 4042e1afd252858de53e2b79d946a51a0182d1ba
branch: 3.7
author: Gregory P. Smith <greg at krypto.org>
committer: GitHub <noreply at github.com>
date: 2019-09-24T22:29:17-07:00
summary:

[3.7] bpo-36871: Handle spec errors in assert_has_calls (GH-16364) (GH-16374)

Handle spec errors in assert_has_calls (GH-16005) (GH-16364)

The fix in PR 13261 handled the underlying issue about the spec for specific methods not being applied correctly, but it didn't fix the issue that was causing the misleading error message.

The code currently grabs a list of responses from _call_matcher (which may include exceptions). But it doesn't reach inside the list when checking if the result is an exception. This results in a misleading error message when one of the provided calls does not match the spec.

https://bugs.python.org/issue36871

Co-authored-by: Samuel Freilich <sfreilich at google.com>

(cherry picked from commit 1a17a054f6314ce29cd2632c28aeed317a615360)

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-24-18-45-46.bpo-36871.p47knk.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testmock.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index b1ab8631a422..2b9e7f14a756 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -895,13 +895,20 @@ def assert_has_calls(self, calls, any_order=False):
         If `any_order` is True then the calls can be in any order, but
         they must all appear in `mock_calls`."""
         expected = [self._call_matcher(c) for c in calls]
-        cause = expected if isinstance(expected, Exception) else None
+        cause = next((e for e in expected if isinstance(e, Exception)), None)
         all_calls = _CallList(self._call_matcher(c) for c in self.mock_calls)
         if not any_order:
             if expected not in all_calls:
+                if cause is None:
+                    problem = 'Calls not found.'
+                else:
+                    problem = ('Error processing expected calls.\n'
+                               'Errors: {}').format(
+                                   [e if isinstance(e, Exception) else None
+                                    for e in expected])
                 raise AssertionError(
-                    'Calls not found.\nExpected: %r\n'
-                    'Actual: %r' % (_CallList(calls), self.mock_calls)
+                    '%s\nExpected: %r\nActual: %r' % (
+                        problem, _CallList(calls), self.mock_calls)
                 ) from cause
             return
 
diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py
index 4e0f4694b771..3046d4cfe639 100644
--- a/Lib/unittest/test/testmock/testmock.py
+++ b/Lib/unittest/test/testmock/testmock.py
@@ -1,4 +1,5 @@
 import copy
+import re
 import sys
 import tempfile
 
@@ -1394,6 +1395,32 @@ def f(a, b, c, d=None):
             mock.assert_has_calls(calls[:-1])
         mock.assert_has_calls(calls[:-1], any_order=True)
 
+    def test_assert_has_calls_not_matching_spec_error(self):
+        def f(x=None): pass
+
+        mock = Mock(spec=f)
+        mock(1)
+
+        with self.assertRaisesRegex(
+                AssertionError,
+                '^{}$'.format(
+                    re.escape('Calls not found.\n'
+                              'Expected: [call()]\n'
+                              'Actual: [call(1)]'))) as cm:
+            mock.assert_has_calls([call()])
+        self.assertIsNone(cm.exception.__cause__)
+
+
+        with self.assertRaisesRegex(
+                AssertionError,
+                '^{}$'.format(
+                    re.escape(
+                        'Error processing expected calls.\n'
+                        "Errors: [None, TypeError('too many positional arguments')]\n"
+                        "Expected: [call(), call(1, 2)]\n"
+                        'Actual: [call(1)]'))) as cm:
+            mock.assert_has_calls([call(), call(1, 2)])
+        self.assertIsInstance(cm.exception.__cause__, TypeError)
 
     def test_assert_any_call(self):
         mock = Mock()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-24-18-45-46.bpo-36871.p47knk.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-24-18-45-46.bpo-36871.p47knk.rst
new file mode 100644
index 000000000000..56ee0f43f5d4
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-24-18-45-46.bpo-36871.p47knk.rst	
@@ -0,0 +1,3 @@
+Improve error handling for the assert_has_calls method of mocks.
+Fixed a bug where any errors encountered while binding the expected calls
+to the mock's spec were silently swallowed, leading to misleading error output.


From webhook-mailer at python.org  Wed Sep 25 01:47:08 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Wed, 25 Sep 2019 05:47:08 -0000
Subject: [Python-checkins] bpo-38265: Update os.pread to accept the length
 type as Py_ssize_t. (GH-16359)
Message-ID: <mailman.60.1569390428.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/ad7736faf5b82a24374c601a72599adf29951080
commit: ad7736faf5b82a24374c601a72599adf29951080
branch: master
author: Dong-hee Na <donghee.na92 at gmail.com>
committer: Serhiy Storchaka <storchaka at gmail.com>
date: 2019-09-25T08:47:04+03:00
summary:

bpo-38265: Update os.pread to accept the length type as Py_ssize_t. (GH-16359)

files:
A Misc/NEWS.d/next/Library/2019-09-25-05-16-19.bpo-38265.X6-gsT.rst
M Modules/clinic/posixmodule.c.h
M Modules/posixmodule.c

diff --git a/Misc/NEWS.d/next/Library/2019-09-25-05-16-19.bpo-38265.X6-gsT.rst b/Misc/NEWS.d/next/Library/2019-09-25-05-16-19.bpo-38265.X6-gsT.rst
new file mode 100644
index 000000000000..bd69a26149d1
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-25-05-16-19.bpo-38265.X6-gsT.rst
@@ -0,0 +1,2 @@
+Update the *length* parameter of :func:`os.pread` to accept
+:c:type:`Py_ssize_t` instead of :c:type:`int`.
diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h
index b21d17477ee6..00b05f7989a9 100644
--- a/Modules/clinic/posixmodule.c.h
+++ b/Modules/clinic/posixmodule.c.h
@@ -4799,14 +4799,14 @@ PyDoc_STRVAR(os_pread__doc__,
     {"pread", (PyCFunction)(void(*)(void))os_pread, METH_FASTCALL, os_pread__doc__},
 
 static PyObject *
-os_pread_impl(PyObject *module, int fd, int length, Py_off_t offset);
+os_pread_impl(PyObject *module, int fd, Py_ssize_t length, Py_off_t offset);
 
 static PyObject *
 os_pread(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
     int fd;
-    int length;
+    Py_ssize_t length;
     Py_off_t offset;
 
     if (!_PyArg_CheckPositional("pread", nargs, 3, 3)) {
@@ -4826,9 +4826,17 @@ os_pread(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
                         "integer argument expected, got float" );
         goto exit;
     }
-    length = _PyLong_AsInt(args[1]);
-    if (length == -1 && PyErr_Occurred()) {
-        goto exit;
+    {
+        Py_ssize_t ival = -1;
+        PyObject *iobj = PyNumber_Index(args[1]);
+        if (iobj != NULL) {
+            ival = PyLong_AsSsize_t(iobj);
+            Py_DECREF(iobj);
+        }
+        if (ival == -1 && PyErr_Occurred()) {
+            goto exit;
+        }
+        length = ival;
     }
     if (!Py_off_t_converter(args[2], &offset)) {
         goto exit;
@@ -8723,4 +8731,4 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar
 #ifndef OS__REMOVE_DLL_DIRECTORY_METHODDEF
     #define OS__REMOVE_DLL_DIRECTORY_METHODDEF
 #endif /* !defined(OS__REMOVE_DLL_DIRECTORY_METHODDEF) */
-/*[clinic end generated code: output=113d9fbdd18a62f5 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=799c75140d84ace5 input=a9049054013a1b77]*/
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index c60d0c5e39ff..5664027b2d3a 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -8827,11 +8827,10 @@ os_readv_impl(PyObject *module, int fd, PyObject *buffers)
 
 #ifdef HAVE_PREAD
 /*[clinic input]
-# TODO length should be size_t!  but Python doesn't support parsing size_t yet.
 os.pread
 
     fd: int
-    length: int
+    length: Py_ssize_t
     offset: Py_off_t
     /
 
@@ -8842,8 +8841,8 @@ the beginning of the file.  The file offset remains unchanged.
 [clinic start generated code]*/
 
 static PyObject *
-os_pread_impl(PyObject *module, int fd, int length, Py_off_t offset)
-/*[clinic end generated code: output=435b29ee32b54a78 input=084948dcbaa35d4c]*/
+os_pread_impl(PyObject *module, int fd, Py_ssize_t length, Py_off_t offset)
+/*[clinic end generated code: output=3f875c1eef82e32f input=85cb4a5589627144]*/
 {
     Py_ssize_t n;
     int async_err = 0;


From webhook-mailer at python.org  Wed Sep 25 02:47:04 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Wed, 25 Sep 2019 06:47:04 -0000
Subject: [Python-checkins] bpo-38219: Optimize dict creating and updating by
 a dict. (GH-16268)
Message-ID: <mailman.61.1569394025.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/f163aeaa8c15b806b622c9cb10bc1d2a6e034e24
commit: f163aeaa8c15b806b622c9cb10bc1d2a6e034e24
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T09:47:00+03:00
summary:

bpo-38219: Optimize dict creating and updating by a dict. (GH-16268)

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-22-13-56-18.bpo-38219.rFl7JD.rst
M Objects/dictobject.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-22-13-56-18.bpo-38219.rFl7JD.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-22-13-56-18.bpo-38219.rFl7JD.rst
new file mode 100644
index 000000000000..8d621cfb826a
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-22-13-56-18.bpo-38219.rFl7JD.rst	
@@ -0,0 +1,2 @@
+Optimized the :class:`dict` constructor and the :meth:`~dict.update` method
+for the case when the argument is a dict.
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 2f2410583512..2f86946b985f 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -2317,17 +2317,22 @@ dict_update_common(PyObject *self, PyObject *args, PyObject *kwds,
         result = -1;
     }
     else if (arg != NULL) {
-        _Py_IDENTIFIER(keys);
-        PyObject *func;
-        if (_PyObject_LookupAttrId(arg, &PyId_keys, &func) < 0) {
-            result = -1;
-        }
-        else if (func != NULL) {
-            Py_DECREF(func);
+        if (PyDict_CheckExact(arg)) {
             result = PyDict_Merge(self, arg, 1);
         }
         else {
-            result = PyDict_MergeFromSeq2(self, arg, 1);
+            _Py_IDENTIFIER(keys);
+            PyObject *func;
+            if (_PyObject_LookupAttrId(arg, &PyId_keys, &func) < 0) {
+                result = -1;
+            }
+            else if (func != NULL) {
+                Py_DECREF(func);
+                result = PyDict_Merge(self, arg, 1);
+            }
+            else {
+                result = PyDict_MergeFromSeq2(self, arg, 1);
+            }
         }
     }
 


From webhook-mailer at python.org  Wed Sep 25 02:58:36 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Wed, 25 Sep 2019 06:58:36 -0000
Subject: [Python-checkins] bpo-22273: Disabled tests while investigating
 buildbot failures on ARM7L/PPC64. (GH-16377)
Message-ID: <mailman.62.1569394717.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/57dc7d5ae8ebfb6da1ea2b25e61260ecb9c79faf
commit: 57dc7d5ae8ebfb6da1ea2b25e61260ecb9c79faf
branch: master
author: Vinay Sajip <vinay_sajip at yahoo.co.uk>
committer: GitHub <noreply at github.com>
date: 2019-09-25T07:58:32+01:00
summary:

bpo-22273: Disabled tests while investigating buildbot failures on ARM7L/PPC64. (GH-16377)

files:
M Lib/ctypes/test/test_structures.py
M Modules/_ctypes/stgdict.c

diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py
index 11c194b0b73e..67d98c9899c3 100644
--- a/Lib/ctypes/test/test_structures.py
+++ b/Lib/ctypes/test/test_structures.py
@@ -1,3 +1,4 @@
+import platform
 import unittest
 from ctypes import *
 from ctypes.test import need_symbol
@@ -5,6 +6,8 @@
 import _ctypes_test
 from test import support
 
+MACHINE = platform.machine()
+
 class SubclassesTest(unittest.TestCase):
     def test_subclass(self):
         class X(Structure):
@@ -477,6 +480,8 @@ class X(Structure):
         self.assertEqual(s.first, got.first)
         self.assertEqual(s.second, got.second)
 
+    @unittest.skipIf(MACHINE in ('armv7l', 'ppc64'),
+                     'Test temporarily disabled on this architecture')
     def test_array_in_struct(self):
         # See bpo-22273
 
diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c
index 62dceceb4f4e..9c42a4f68e51 100644
--- a/Modules/_ctypes/stgdict.c
+++ b/Modules/_ctypes/stgdict.c
@@ -652,7 +652,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
 
 #define MAX_ELEMENTS 16
 
-    if (arrays_seen && (size <= 16)) {
+    if (arrays_seen && (size <= MAX_ELEMENTS)) {
         /*
          * See bpo-22273. Arrays are normally treated as pointers, which is
          * fine when an array name is being passed as parameter, but not when


From webhook-mailer at python.org  Wed Sep 25 04:53:53 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 25 Sep 2019 08:53:53 -0000
Subject: [Python-checkins] bpo-38260: Add Docs on asyncio.run (GH-16337)
Message-ID: <mailman.63.1569401634.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/17deb16883fa574a86e42551cc37f044182347ad
commit: 17deb16883fa574a86e42551cc37f044182347ad
branch: master
author: Emmanuel Arias <emmanuelarias30 at gmail.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-25T01:53:49-07:00
summary:

bpo-38260: Add Docs on asyncio.run (GH-16337)



Add docs about return and raise exception on asyncio.run





https://bugs.python.org/issue38260



Automerge-Triggered-By: @asvetlov

files:
M Doc/library/asyncio-task.rst
M Lib/asyncio/runners.py

diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index 1fb882935895..9f55a3534a5e 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -225,6 +225,18 @@ Running an asyncio Program
     the end.  It should be used as a main entry point for asyncio
     programs, and should ideally only be called once.
 
+    Return a result of *coro* execution, or raise a :exc:`RuntimeError`
+    if ``asyncio.run()`` is called from a running event loop, or a
+    :exc:`ValueError` if *coro* is not a courutine.
+
+    Example::
+
+        async def main():
+            await asyncio.sleep(1)
+            print('hello')
+
+        asyncio.run(main())
+
     .. versionadded:: 3.7
 
     .. versionchanged:: 3.9
diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py
index 6c87747e770b..904102bf82be 100644
--- a/Lib/asyncio/runners.py
+++ b/Lib/asyncio/runners.py
@@ -21,6 +21,10 @@ def run(main, *, debug=False):
     It should be used as a main entry point for asyncio programs, and should
     ideally only be called once.
 
+    Return a result of *coro* execution, or raise a RuntimeError
+    if `asyncio.run()`is called from a running event loop, or a ValueError
+    if `main` is not a courutine.
+
     Example:
 
         async def main():


From webhook-mailer at python.org  Wed Sep 25 05:13:05 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 25 Sep 2019 09:13:05 -0000
Subject: [Python-checkins] bpo-38260: Add Docs on asyncio.run (GH-16337)
Message-ID: <mailman.64.1569402787.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/d3666945d78508970002653c4d46221a4306f55b
commit: d3666945d78508970002653c4d46221a4306f55b
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T02:13:00-07:00
summary:

bpo-38260: Add Docs on asyncio.run (GH-16337)


Add docs about return and raise exception on asyncio.run

https://bugs.python.org/issue38260

Automerge-Triggered-By: @asvetlov
(cherry picked from commit 17deb16883fa574a86e42551cc37f044182347ad)

Co-authored-by: Emmanuel Arias <emmanuelarias30 at gmail.com>

files:
M Doc/library/asyncio-task.rst
M Lib/asyncio/runners.py

diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index 29ccafee908b..a9f6f9cfc4b6 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -225,6 +225,18 @@ Running an asyncio Program
     the end.  It should be used as a main entry point for asyncio
     programs, and should ideally only be called once.
 
+    Return a result of *coro* execution, or raise a :exc:`RuntimeError`
+    if ``asyncio.run()`` is called from a running event loop, or a
+    :exc:`ValueError` if *coro* is not a courutine.
+
+    Example::
+
+        async def main():
+            await asyncio.sleep(1)
+            print('hello')
+
+        asyncio.run(main())
+
     .. versionadded:: 3.7
        **Important:** this function has been added to asyncio in
        Python 3.7 on a :term:`provisional basis <provisional api>`.
diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py
index 5fbab03dd001..df686386b8fd 100644
--- a/Lib/asyncio/runners.py
+++ b/Lib/asyncio/runners.py
@@ -21,6 +21,10 @@ def run(main, *, debug=False):
     It should be used as a main entry point for asyncio programs, and should
     ideally only be called once.
 
+    Return a result of *coro* execution, or raise a RuntimeError
+    if `asyncio.run()`is called from a running event loop, or a ValueError
+    if `main` is not a courutine.
+
     Example:
 
         async def main():


From webhook-mailer at python.org  Wed Sep 25 05:13:05 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 25 Sep 2019 09:13:05 -0000
Subject: [Python-checkins] bpo-38260: Add Docs on asyncio.run (GH-16337)
Message-ID: <mailman.65.1569402788.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/4633355a063c7bf068bdd72dfdd32f4c22cf382f
commit: 4633355a063c7bf068bdd72dfdd32f4c22cf382f
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T02:12:47-07:00
summary:

bpo-38260: Add Docs on asyncio.run (GH-16337)


Add docs about return and raise exception on asyncio.run

https://bugs.python.org/issue38260

Automerge-Triggered-By: @asvetlov
(cherry picked from commit 17deb16883fa574a86e42551cc37f044182347ad)

Co-authored-by: Emmanuel Arias <emmanuelarias30 at gmail.com>

files:
M Doc/library/asyncio-task.rst
M Lib/asyncio/runners.py

diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index 7efec5b6c1df..321b4f457659 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -225,6 +225,18 @@ Running an asyncio Program
     the end.  It should be used as a main entry point for asyncio
     programs, and should ideally only be called once.
 
+    Return a result of *coro* execution, or raise a :exc:`RuntimeError`
+    if ``asyncio.run()`` is called from a running event loop, or a
+    :exc:`ValueError` if *coro* is not a courutine.
+
+    Example::
+
+        async def main():
+            await asyncio.sleep(1)
+            print('hello')
+
+        asyncio.run(main())
+
     .. versionadded:: 3.7
 
 
diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py
index 5fbab03dd001..df686386b8fd 100644
--- a/Lib/asyncio/runners.py
+++ b/Lib/asyncio/runners.py
@@ -21,6 +21,10 @@ def run(main, *, debug=False):
     It should be used as a main entry point for asyncio programs, and should
     ideally only be called once.
 
+    Return a result of *coro* execution, or raise a RuntimeError
+    if `asyncio.run()`is called from a running event loop, or a ValueError
+    if `main` is not a courutine.
+
     Example:
 
         async def main():


From webhook-mailer at python.org  Wed Sep 25 06:12:05 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Wed, 25 Sep 2019 10:12:05 -0000
Subject: [Python-checkins] Changed conditions for ctypes array-in-struct
 handling. (GH-16381)
Message-ID: <mailman.66.1569406326.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/c64af8fad3c4f5751af624647fbb0ce023f525dc
commit: c64af8fad3c4f5751af624647fbb0ce023f525dc
branch: master
author: Vinay Sajip <vinay_sajip at yahoo.co.uk>
committer: GitHub <noreply at github.com>
date: 2019-09-25T11:11:57+01:00
summary:

Changed conditions for ctypes array-in-struct handling. (GH-16381)

files:
M Lib/ctypes/test/test_structures.py
M Modules/_ctypes/stgdict.c

diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py
index 67d98c9899c3..661efc1a0f85 100644
--- a/Lib/ctypes/test/test_structures.py
+++ b/Lib/ctypes/test/test_structures.py
@@ -480,7 +480,7 @@ class X(Structure):
         self.assertEqual(s.first, got.first)
         self.assertEqual(s.second, got.second)
 
-    @unittest.skipIf(MACHINE in ('armv7l', 'ppc64'),
+    @unittest.skipIf(MACHINE.startswith(('arm', 'ppc')),
                      'Test temporarily disabled on this architecture')
     def test_array_in_struct(self):
         # See bpo-22273
diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c
index 9c42a4f68e51..947e9c25592f 100644
--- a/Modules/_ctypes/stgdict.c
+++ b/Modules/_ctypes/stgdict.c
@@ -350,9 +350,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     int pack;
     Py_ssize_t ffi_ofs;
     int big_endian;
-#if defined(X86_64)
     int arrays_seen = 0;
-#endif
 
     /* HACK Alert: I cannot be bothered to fix ctypes.com, so there has to
        be a way to use the old, broken semantics: _fields_ are not extended
@@ -504,10 +502,8 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
             Py_XDECREF(pair);
             return -1;
         }
-#if defined(X86_64)
         if (PyCArrayTypeObject_Check(desc))
             arrays_seen = 1;
-#endif
         dict = PyType_stgdict(desc);
         if (dict == NULL) {
             Py_DECREF(pair);
@@ -648,8 +644,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     stgdict->align = total_align;
     stgdict->length = len;      /* ADD ffi_ofs? */
 
-#if defined(X86_64)
-
 #define MAX_ELEMENTS 16
 
     if (arrays_seen && (size <= MAX_ELEMENTS)) {
@@ -669,6 +663,10 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
          * accurate set, to allow libffi to marshal them into registers
          * correctly. It means one more loop over the fields, but if we got
          * here, the structure is small, so there aren't too many of those.
+         *
+         * Although the passing in registers is specific to 64-bit Linux, the
+         * array-in-struct vs. pointer problem is general. But we restrict the
+         * type transformation to small structs nonetheless.
          */
         ffi_type *actual_types[MAX_ELEMENTS + 1];
         int actual_type_index = 0;
@@ -746,7 +744,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
         memcpy(&stgdict->ffi_type_pointer.elements[ffi_ofs], actual_types,
                actual_type_index * sizeof(ffi_type *));
     }
-#endif
 
     /* We did check that this flag was NOT set above, it must not
        have been set until now. */


From webhook-mailer at python.org  Wed Sep 25 06:32:14 2019
From: webhook-mailer at python.org (Carol Willing)
Date: Wed, 25 Sep 2019 10:32:14 -0000
Subject: [Python-checkins] bpo-38248: Fix inconsistent immediate
 asyncio.Task cancellation (GH-16330)
Message-ID: <mailman.67.1569407535.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/edad4d89e357c92f70c0324b937845d652b20afd
commit: edad4d89e357c92f70c0324b937845d652b20afd
branch: master
author: Yury Selivanov <yury at edgedb.com>
committer: Carol Willing <carolcode at willingconsulting.com>
date: 2019-09-25T03:32:08-07:00
summary:

bpo-38248: Fix inconsistent immediate asyncio.Task cancellation (GH-16330)

files:
A Misc/NEWS.d/next/Library/2019-09-22-13-05-36.bpo-38248.Yo3N_1.rst
M Lib/asyncio/tasks.py
M Lib/test/test_asyncio/test_tasks.py
M Modules/_asynciomodule.c

diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
index a0cb884eacab..38d982716d46 100644
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -284,7 +284,7 @@ def __step(self, exc=None):
             if self._must_cancel:
                 # Task is cancelled right before coro stops.
                 self._must_cancel = False
-                super().set_exception(exceptions.CancelledError())
+                super().cancel()
             else:
                 super().set_result(exc.value)
         except exceptions.CancelledError:
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
index 576714faa888..dde84b84b103 100644
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -604,9 +604,11 @@ def test_cancel_current_task(self):
             return 12
 
         t = self.new_task(loop, task())
+        self.assertFalse(t.cancelled())
         self.assertRaises(
             asyncio.CancelledError, loop.run_until_complete, t)
         self.assertTrue(t.done())
+        self.assertTrue(t.cancelled())
         self.assertFalse(t._must_cancel)  # White-box test.
         self.assertFalse(t.cancel())
 
@@ -621,9 +623,11 @@ def test_cancel_at_end(self):
             return 12
 
         t = self.new_task(loop, task())
+        self.assertFalse(t.cancelled())
         self.assertRaises(
             asyncio.CancelledError, loop.run_until_complete, t)
         self.assertTrue(t.done())
+        self.assertTrue(t.cancelled())
         self.assertFalse(t._must_cancel)  # White-box test.
         self.assertFalse(t.cancel())
 
diff --git a/Misc/NEWS.d/next/Library/2019-09-22-13-05-36.bpo-38248.Yo3N_1.rst b/Misc/NEWS.d/next/Library/2019-09-22-13-05-36.bpo-38248.Yo3N_1.rst
new file mode 100644
index 000000000000..fc92209af387
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-22-13-05-36.bpo-38248.Yo3N_1.rst
@@ -0,0 +1 @@
+asyncio: Fix inconsistent immediate Task cancellation
diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c
index 4d503a418a2e..8ee0d7a7e14c 100644
--- a/Modules/_asynciomodule.c
+++ b/Modules/_asynciomodule.c
@@ -2628,18 +2628,19 @@ task_step_impl(TaskObj *task, PyObject *exc)
         if (_PyGen_FetchStopIterationValue(&o) == 0) {
             /* The error is StopIteration and that means that
                the underlying coroutine has resolved */
+
+            PyObject *res;
             if (task->task_must_cancel) {
                 // Task is cancelled right before coro stops.
-                Py_DECREF(o);
                 task->task_must_cancel = 0;
-                et = asyncio_CancelledError;
-                Py_INCREF(et);
-                ev = NULL;
-                tb = NULL;
-                goto set_exception;
+                res = future_cancel((FutureObj*)task);
+            }
+            else {
+                res = future_set_result((FutureObj*)task, o);
             }
-            PyObject *res = future_set_result((FutureObj*)task, o);
+
             Py_DECREF(o);
+
             if (res == NULL) {
                 return NULL;
             }


From webhook-mailer at python.org  Wed Sep 25 07:06:22 2019
From: webhook-mailer at python.org (Petr Viktorin)
Date: Wed, 25 Sep 2019 11:06:22 -0000
Subject: [Python-checkins] bpo-38140: Document offsets in PyMemberDef
 (GH-16354)
Message-ID: <mailman.68.1569409583.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/468f8a6ee0d83e36795bbab6f20037b0e380f493
commit: 468f8a6ee0d83e36795bbab6f20037b0e380f493
branch: master
author: Petr Viktorin <encukou at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T13:06:16+02:00
summary:

bpo-38140: Document offsets in PyMemberDef (GH-16354)

bpo-38140: Document offsets in PyMemberDef

Co-Authored-By: Ammar Askar <ammar_askar at hotmail.com>

files:
M Doc/c-api/structures.rst
M Doc/c-api/type.rst

diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst
index af797778ce63..b0a193876c53 100644
--- a/Doc/c-api/structures.rst
+++ b/Doc/c-api/structures.rst
@@ -346,6 +346,19 @@ Accessing attributes of extension types
    Only :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX`
    members can be deleted.  (They are set to *NULL*).
 
+   .. _pymemberdef-offsets:
+
+   Heap allocated types (created using :c:func:`PyType_FromSpec` or similar),
+   ``PyMemberDef`` may contain defintitions for the special members
+   ``__dictoffset__`` and ``__weaklistoffset__``, corresponding to
+   :c:member:`~PyTypeObject.tp_dictoffset` and
+   :c:member:`~PyTypeObject.tp_weaklistoffset` in type objects.
+   These must be defined with ``T_PYSSIZET`` and ``READONLY``, for example::
+
+      static PyMemberDef spam_type_members[] = {
+          {"__dictoffset__", T_PYSSIZET, offsetof(Spam_object, dict), READONLY},
+          {NULL}  /* Sentinel */
+      };
 
 .. c:type:: PyGetSetDef
 
diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index ad3e022d42ca..36309dd5254a 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -188,7 +188,10 @@ The following functions and structs are used to create
       * :c:member:`~PyTypeObject.tp_subclasses`
       * :c:member:`~PyTypeObject.tp_weaklist`
       * :c:member:`~PyTypeObject.tp_vectorcall`
-      * :c:member:`~PyTypeObject.tp_print`
+      * :c:member:`~PyTypeObject.tp_weaklistoffset`
+        (see :ref:`PyMemberDef <pymemberdef-offsets>`)
+      * :c:member:`~PyTypeObject.tp_dictoffset`
+        (see :ref:`PyMemberDef <pymemberdef-offsets>`)
       * :c:member:`~PyBufferProcs.bf_getbuffer`
       * :c:member:`~PyBufferProcs.bf_releasebuffer`
 


From webhook-mailer at python.org  Wed Sep 25 07:49:05 2019
From: webhook-mailer at python.org (Carol Willing)
Date: Wed, 25 Sep 2019 11:49:05 -0000
Subject: [Python-checkins] bpo-38248: Fix inconsistent immediate
 asyncio.Task cancellation (GH-16330) (GH-16383)
Message-ID: <mailman.69.1569412146.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/16cec136b75daf438080a5b6685d2679dfa406af
commit: 16cec136b75daf438080a5b6685d2679dfa406af
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Carol Willing <carolcode at willingconsulting.com>
date: 2019-09-25T04:48:52-07:00
summary:

bpo-38248: Fix inconsistent immediate asyncio.Task cancellation (GH-16330) (GH-16383)

(cherry picked from commit edad4d89e357c92f70c0324b937845d652b20afd)

Co-authored-by: Yury Selivanov <yury at edgedb.com>

files:
A Misc/NEWS.d/next/Library/2019-09-22-13-05-36.bpo-38248.Yo3N_1.rst
M Lib/asyncio/tasks.py
M Lib/test/test_asyncio/test_tasks.py
M Modules/_asynciomodule.c

diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
index a0cb884eacab..38d982716d46 100644
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -284,7 +284,7 @@ def __step(self, exc=None):
             if self._must_cancel:
                 # Task is cancelled right before coro stops.
                 self._must_cancel = False
-                super().set_exception(exceptions.CancelledError())
+                super().cancel()
             else:
                 super().set_result(exc.value)
         except exceptions.CancelledError:
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
index 6e832eab6d58..c21b06938fe3 100644
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -604,9 +604,11 @@ def test_cancel_current_task(self):
             return 12
 
         t = self.new_task(loop, task())
+        self.assertFalse(t.cancelled())
         self.assertRaises(
             asyncio.CancelledError, loop.run_until_complete, t)
         self.assertTrue(t.done())
+        self.assertTrue(t.cancelled())
         self.assertFalse(t._must_cancel)  # White-box test.
         self.assertFalse(t.cancel())
 
@@ -621,9 +623,11 @@ def test_cancel_at_end(self):
             return 12
 
         t = self.new_task(loop, task())
+        self.assertFalse(t.cancelled())
         self.assertRaises(
             asyncio.CancelledError, loop.run_until_complete, t)
         self.assertTrue(t.done())
+        self.assertTrue(t.cancelled())
         self.assertFalse(t._must_cancel)  # White-box test.
         self.assertFalse(t.cancel())
 
diff --git a/Misc/NEWS.d/next/Library/2019-09-22-13-05-36.bpo-38248.Yo3N_1.rst b/Misc/NEWS.d/next/Library/2019-09-22-13-05-36.bpo-38248.Yo3N_1.rst
new file mode 100644
index 000000000000..fc92209af387
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-22-13-05-36.bpo-38248.Yo3N_1.rst
@@ -0,0 +1 @@
+asyncio: Fix inconsistent immediate Task cancellation
diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c
index 281161b68611..cea3affe99ec 100644
--- a/Modules/_asynciomodule.c
+++ b/Modules/_asynciomodule.c
@@ -2635,18 +2635,19 @@ task_step_impl(TaskObj *task, PyObject *exc)
         if (_PyGen_FetchStopIterationValue(&o) == 0) {
             /* The error is StopIteration and that means that
                the underlying coroutine has resolved */
+
+            PyObject *res;
             if (task->task_must_cancel) {
                 // Task is cancelled right before coro stops.
-                Py_DECREF(o);
                 task->task_must_cancel = 0;
-                et = asyncio_CancelledError;
-                Py_INCREF(et);
-                ev = NULL;
-                tb = NULL;
-                goto set_exception;
+                res = future_cancel((FutureObj*)task);
+            }
+            else {
+                res = future_set_result((FutureObj*)task, o);
             }
-            PyObject *res = future_set_result((FutureObj*)task, o);
+
             Py_DECREF(o);
+
             if (res == NULL) {
                 return NULL;
             }


From webhook-mailer at python.org  Wed Sep 25 08:26:44 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 25 Sep 2019 12:26:44 -0000
Subject: [Python-checkins] bpo-37064: Add option -a to pathfix.py tool
 (GH-15717)
Message-ID: <mailman.70.1569414405.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/1dc1acbd73f05f14c974b7ce1041787d7abef31e
commit: 1dc1acbd73f05f14c974b7ce1041787d7abef31e
branch: master
author: PatrikKopkan <kopkanpatrik at gmail.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-25T14:26:28+02:00
summary:

bpo-37064: Add option -a to pathfix.py tool (GH-15717)

Add option -a to Tools/Scripts/pathfix.py script: add flags.

files:
M Lib/test/test_tools/test_pathfix.py
M Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
M Tools/scripts/pathfix.py

diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py
index 0c1aea721170..a87992491344 100644
--- a/Lib/test/test_tools/test_pathfix.py
+++ b/Lib/test/test_tools/test_pathfix.py
@@ -17,51 +17,90 @@ def setUp(self):
         self.temp_file = support.TESTFN
         self.addCleanup(support.unlink, support.TESTFN)
 
-    def pathfix(self, shebang, pathfix_flags):
+    def pathfix(self, shebang, pathfix_flags, exitcode=0, stdout='', stderr=''):
         with open(self.temp_file, 'w', encoding='utf8') as f:
             f.write(f'{shebang}\n' + 'print("Hello world")\n')
 
         proc = subprocess.run(
             [sys.executable, self.script,
              *pathfix_flags, '-n', self.temp_file],
-            capture_output=True)
-        self.assertEqual(proc.returncode, 0, proc)
+            capture_output=True, text=1)
+
+        if stdout == '' and proc.returncode == 0:
+            stdout = f'{self.temp_file}: updating\n'
+        self.assertEqual(proc.returncode, exitcode, proc)
+        self.assertEqual(proc.stdout, stdout, proc)
+        self.assertEqual(proc.stderr, stderr, proc)
 
         with open(self.temp_file, 'r', encoding='utf8') as f:
             output = f.read()
 
         lines = output.split('\n')
         self.assertEqual(lines[1:], ['print("Hello world")', ''])
-        shebang = lines[0]
-        return shebang
+        new_shebang = lines[0]
+
+        if proc.returncode != 0:
+            self.assertEqual(shebang, new_shebang)
+
+        return new_shebang
 
     def test_pathfix(self):
         self.assertEqual(
             self.pathfix(
                 '#! /usr/bin/env python',
-                ['-i', '/usr/bin/python3',]),
-            '#! /usr/bin/python3',
-        )
+                ['-i', '/usr/bin/python3']),
+            '#! /usr/bin/python3')
         self.assertEqual(
             self.pathfix(
                 '#! /usr/bin/env python -R',
-                ['-i', '/usr/bin/python3', ]),
-            '#! /usr/bin/python3',
-        )
+                ['-i', '/usr/bin/python3']),
+            '#! /usr/bin/python3')
 
     def test_pathfix_keeping_flags(self):
         self.assertEqual(
             self.pathfix(
                 '#! /usr/bin/env python -R',
-                ['-i', '/usr/bin/python3', '-k',]),
-            '#! /usr/bin/python3 -R',
-        )
+                ['-i', '/usr/bin/python3', '-k']),
+            '#! /usr/bin/python3 -R')
         self.assertEqual(
             self.pathfix(
                 '#! /usr/bin/env python',
-                ['-i', '/usr/bin/python3', '-k',]),
-            '#! /usr/bin/python3',
-        )
+                ['-i', '/usr/bin/python3', '-k']),
+            '#! /usr/bin/python3')
+
+    def test_pathfix_adding_flag(self):
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python',
+                ['-i', '/usr/bin/python3', '-a', 's']),
+            '#! /usr/bin/python3 -s')
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python -S',
+                ['-i', '/usr/bin/python3', '-a', 's']),
+            '#! /usr/bin/python3 -s')
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python -V',
+                ['-i', '/usr/bin/python3', '-a', 'v', '-k']),
+            '#! /usr/bin/python3 -vV')
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python',
+                ['-i', '/usr/bin/python3', '-a', 'Rs']),
+            '#! /usr/bin/python3 -Rs')
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python -W default',
+                ['-i', '/usr/bin/python3', '-a', 's', '-k']),
+            '#! /usr/bin/python3 -sW default')
+
+    def test_pathfix_adding_errors(self):
+        self.pathfix(
+            '#! /usr/bin/env python -E',
+            ['-i', '/usr/bin/python3', '-a', 'W default', '-k'],
+            exitcode=2,
+            stderr="-a option doesn't support whitespaces")
 
 
 if __name__ == '__main__':
diff --git a/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst b/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
index 02599077c9d5..d1210e29532b 100644
--- a/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
+++ b/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
@@ -1 +1,2 @@
-Add flag -k to pathscript.py script: preserve shebang flags.
\ No newline at end of file
+Add option -k to pathscript.py script: preserve shebang flags.
+Add option -a to pathscript.py script: add flags.
diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py
index eee684e3502d..237a3d96e9fa 100755
--- a/Tools/scripts/pathfix.py
+++ b/Tools/scripts/pathfix.py
@@ -14,6 +14,7 @@
 # Sometimes you may find shebangs with flags such as `#! /usr/bin/env python -si`.
 # Normally, pathfix overwrites the entire line, including the flags.
 # To change interpreter and keep flags from the original shebang line, use -k.
+# If you want to keep flags and add to them one single literal flag, use option -a.
 
 
 # Undoubtedly you can do this using find and sed or perl, but this is
@@ -39,6 +40,7 @@
 preserve_timestamps = False
 create_backup = True
 keep_flags = False
+add_flags = b''
 
 
 def main():
@@ -46,11 +48,12 @@ def main():
     global preserve_timestamps
     global create_backup
     global keep_flags
+    global add_flags
 
-    usage = ('usage: %s -i /interpreter -p -n -k file-or-directory ...\n' %
+    usage = ('usage: %s -i /interpreter -p -n -k -a file-or-directory ...\n' %
              sys.argv[0])
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'i:kpn')
+        opts, args = getopt.getopt(sys.argv[1:], 'i:a:kpn')
     except getopt.error as msg:
         err(str(msg) + '\n')
         err(usage)
@@ -64,6 +67,11 @@ def main():
             create_backup = False
         if o == '-k':
             keep_flags = True
+        if o == '-a':
+            add_flags = a.encode()
+            if b' ' in add_flags:
+                err("-a option doesn't support whitespaces")
+                sys.exit(2)
     if not new_interpreter or not new_interpreter.startswith(b'/') or \
            not args:
         err('-i option or file-or-directory missing\n')
@@ -188,15 +196,32 @@ def parse_shebang(shebangline):
     return shebangline[start:]
 
 
+def populate_flags(shebangline):
+    old_flags = b''
+    if keep_flags:
+        old_flags = parse_shebang(shebangline)
+        if old_flags:
+            old_flags = old_flags[2:]
+    if not (old_flags or add_flags):
+        return b''
+    # On Linux, the entire string following the interpreter name
+    # is passed as a single argument to the interpreter.
+    # e.g. "#! /usr/bin/python3 -W Error -s" runs "/usr/bin/python3 "-W Error -s"
+    # so shebang should have single '-' where flags are given and
+    # flag might need argument for that reasons adding new flags is
+    # between '-' and original flags
+    # e.g. #! /usr/bin/python3 -sW Error
+    return b' -' + add_flags + old_flags
+
+
 def fixline(line):
     if not line.startswith(b'#!'):
         return line
 
     if b"python" not in line:
         return line
-    flags = b''
-    if keep_flags:
-        flags = parse_shebang(line)
+
+    flags = populate_flags(line)
     return b'#! ' + new_interpreter + flags + b'\n'
 
 


From webhook-mailer at python.org  Wed Sep 25 09:22:44 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 25 Sep 2019 13:22:44 -0000
Subject: [Python-checkins] bpo-37064: Add -k and -a options to pathfix.py
 tool (GH-16387)
Message-ID: <mailman.71.1569417765.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/c71c54c62600fd721baed3c96709e3d6e9c33817
commit: c71c54c62600fd721baed3c96709e3d6e9c33817
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T15:22:40+02:00
summary:

bpo-37064: Add -k and -a options to pathfix.py tool (GH-16387)

* bpo-37064: Add option -k to Tools/scripts/pathfix.py (GH-15548)

Add flag -k to pathscript.py script: preserve shebang flags.

(cherry picked from commit 50254ac4c179cb412e90682098c97db786143929)

* bpo-37064: Add option -a to pathfix.py tool (GH-15717)

Add option -a to Tools/Scripts/pathfix.py script: add flags.

(cherry picked from commit 1dc1acbd73f05f14c974b7ce1041787d7abef31e)

files:
A Lib/test/test_tools/test_pathfix.py
A Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
M Tools/scripts/pathfix.py

diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py
new file mode 100644
index 000000000000..ce3d536c1293
--- /dev/null
+++ b/Lib/test/test_tools/test_pathfix.py
@@ -0,0 +1,103 @@
+import os
+import subprocess
+import sys
+import unittest
+from test import support
+from test.test_tools import import_tool, scriptsdir
+
+
+class TestPathfixFunctional(unittest.TestCase):
+    script = os.path.join(scriptsdir, 'pathfix.py')
+
+    def setUp(self):
+        self.temp_file = support.TESTFN
+        self.addCleanup(support.unlink, support.TESTFN)
+
+    def pathfix(self, shebang, pathfix_flags, exitcode=0, stdout='', stderr=''):
+        with open(self.temp_file, 'w', encoding='utf8') as f:
+            f.write(f'{shebang}\n' + 'print("Hello world")\n')
+
+        proc = subprocess.run(
+            [sys.executable, self.script,
+             *pathfix_flags, '-n', self.temp_file],
+            capture_output=True, text=1)
+
+        if stdout == '' and proc.returncode == 0:
+            stdout = f'{self.temp_file}: updating\n'
+        self.assertEqual(proc.returncode, exitcode, proc)
+        self.assertEqual(proc.stdout, stdout, proc)
+        self.assertEqual(proc.stderr, stderr, proc)
+
+        with open(self.temp_file, 'r', encoding='utf8') as f:
+            output = f.read()
+
+        lines = output.split('\n')
+        self.assertEqual(lines[1:], ['print("Hello world")', ''])
+        new_shebang = lines[0]
+
+        if proc.returncode != 0:
+            self.assertEqual(shebang, new_shebang)
+
+        return new_shebang
+
+    def test_pathfix(self):
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python',
+                ['-i', '/usr/bin/python3']),
+            '#! /usr/bin/python3')
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python -R',
+                ['-i', '/usr/bin/python3']),
+            '#! /usr/bin/python3')
+
+    def test_pathfix_keeping_flags(self):
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python -R',
+                ['-i', '/usr/bin/python3', '-k']),
+            '#! /usr/bin/python3 -R')
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python',
+                ['-i', '/usr/bin/python3', '-k']),
+            '#! /usr/bin/python3')
+
+    def test_pathfix_adding_flag(self):
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python',
+                ['-i', '/usr/bin/python3', '-a', 's']),
+            '#! /usr/bin/python3 -s')
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python -S',
+                ['-i', '/usr/bin/python3', '-a', 's']),
+            '#! /usr/bin/python3 -s')
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python -V',
+                ['-i', '/usr/bin/python3', '-a', 'v', '-k']),
+            '#! /usr/bin/python3 -vV')
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python',
+                ['-i', '/usr/bin/python3', '-a', 'Rs']),
+            '#! /usr/bin/python3 -Rs')
+        self.assertEqual(
+            self.pathfix(
+                '#! /usr/bin/env python -W default',
+                ['-i', '/usr/bin/python3', '-a', 's', '-k']),
+            '#! /usr/bin/python3 -sW default')
+
+    def test_pathfix_adding_errors(self):
+        self.pathfix(
+            '#! /usr/bin/env python -E',
+            ['-i', '/usr/bin/python3', '-a', 'W default', '-k'],
+            exitcode=2,
+            stderr="-a option doesn't support whitespaces")
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst b/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
new file mode 100644
index 000000000000..d1210e29532b
--- /dev/null
+++ b/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
@@ -0,0 +1,2 @@
+Add option -k to pathscript.py script: preserve shebang flags.
+Add option -a to pathscript.py script: add flags.
diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py
index 1a0cf1c9e69c..237a3d96e9fa 100755
--- a/Tools/scripts/pathfix.py
+++ b/Tools/scripts/pathfix.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Change the #! line occurring in Python scripts.  The new interpreter
+# Change the #! line (shebang) occurring in Python scripts.  The new interpreter
 # pathname must be given with a -i option.
 #
 # Command line arguments are files or directories to be processed.
@@ -10,7 +10,13 @@
 # arguments).
 # The original file is kept as a back-up (with a "~" attached to its name),
 # -n flag can be used to disable this.
-#
+
+# Sometimes you may find shebangs with flags such as `#! /usr/bin/env python -si`.
+# Normally, pathfix overwrites the entire line, including the flags.
+# To change interpreter and keep flags from the original shebang line, use -k.
+# If you want to keep flags and add to them one single literal flag, use option -a.
+
+
 # Undoubtedly you can do this using find and sed or perl, but this is
 # a nice example of Python code that recurses down a directory tree
 # and uses regular expressions.  Also note several subtleties like
@@ -33,16 +39,21 @@
 new_interpreter = None
 preserve_timestamps = False
 create_backup = True
+keep_flags = False
+add_flags = b''
 
 
 def main():
     global new_interpreter
     global preserve_timestamps
     global create_backup
-    usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' %
+    global keep_flags
+    global add_flags
+
+    usage = ('usage: %s -i /interpreter -p -n -k -a file-or-directory ...\n' %
              sys.argv[0])
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'i:pn')
+        opts, args = getopt.getopt(sys.argv[1:], 'i:a:kpn')
     except getopt.error as msg:
         err(str(msg) + '\n')
         err(usage)
@@ -54,6 +65,13 @@ def main():
             preserve_timestamps = True
         if o == '-n':
             create_backup = False
+        if o == '-k':
+            keep_flags = True
+        if o == '-a':
+            add_flags = a.encode()
+            if b' ' in add_flags:
+                err("-a option doesn't support whitespaces")
+                sys.exit(2)
     if not new_interpreter or not new_interpreter.startswith(b'/') or \
            not args:
         err('-i option or file-or-directory missing\n')
@@ -70,10 +88,14 @@ def main():
             if fix(arg): bad = 1
     sys.exit(bad)
 
+
 ispythonprog = re.compile(r'^[a-zA-Z0-9_]+\.py$')
+
+
 def ispython(name):
     return bool(ispythonprog.match(name))
 
+
 def recursedown(dirname):
     dbg('recursedown(%r)\n' % (dirname,))
     bad = 0
@@ -96,6 +118,7 @@ def recursedown(dirname):
         if recursedown(fullname): bad = 1
     return bad
 
+
 def fix(filename):
 ##  dbg('fix(%r)\n' % (filename,))
     try:
@@ -164,12 +187,43 @@ def fix(filename):
     # Return success
     return 0
 
+
+def parse_shebang(shebangline):
+    shebangline = shebangline.rstrip(b'\n')
+    start = shebangline.find(b' -')
+    if start == -1:
+        return b''
+    return shebangline[start:]
+
+
+def populate_flags(shebangline):
+    old_flags = b''
+    if keep_flags:
+        old_flags = parse_shebang(shebangline)
+        if old_flags:
+            old_flags = old_flags[2:]
+    if not (old_flags or add_flags):
+        return b''
+    # On Linux, the entire string following the interpreter name
+    # is passed as a single argument to the interpreter.
+    # e.g. "#! /usr/bin/python3 -W Error -s" runs "/usr/bin/python3 "-W Error -s"
+    # so shebang should have single '-' where flags are given and
+    # flag might need argument for that reasons adding new flags is
+    # between '-' and original flags
+    # e.g. #! /usr/bin/python3 -sW Error
+    return b' -' + add_flags + old_flags
+
+
 def fixline(line):
     if not line.startswith(b'#!'):
         return line
+
     if b"python" not in line:
         return line
-    return b'#! ' + new_interpreter + b'\n'
+
+    flags = populate_flags(line)
+    return b'#! ' + new_interpreter + flags + b'\n'
+
 
 if __name__ == '__main__':
     main()


From webhook-mailer at python.org  Wed Sep 25 09:52:54 2019
From: webhook-mailer at python.org (T. Wouters)
Date: Wed, 25 Sep 2019 13:52:54 -0000
Subject: [Python-checkins] bpo-36046: posix_spawn() doesn't support uid/gid
 (GH-16384)
Message-ID: <mailman.72.1569419575.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/faca8553425c231d867dcabf6a69a9dd21118b6c
commit: faca8553425c231d867dcabf6a69a9dd21118b6c
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: T. Wouters <thomas at python.org>
date: 2019-09-25T15:52:49+02:00
summary:

bpo-36046: posix_spawn() doesn't support uid/gid (GH-16384)

* subprocess.Popen now longer uses posix_spawn() if uid, gid or gids are set.
* test_subprocess: add "nobody" and "nfsnobody" group names for test_group().
* test_subprocess: test_user() and test_group() are now also tested with close_fds=False.

files:
M Lib/subprocess.py
M Lib/test/test_subprocess.py

diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 85e7969c0928..1016874c4a7f 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -1681,7 +1681,10 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
                     and (p2cread == -1 or p2cread > 2)
                     and (c2pwrite == -1 or c2pwrite > 2)
                     and (errwrite == -1 or errwrite > 2)
-                    and not start_new_session):
+                    and not start_new_session
+                    and gid is None
+                    and gids is None
+                    and uid is None):
                 self._posix_spawn(args, executable, env, restore_signals,
                                   p2cread, p2cwrite,
                                   c2pread, c2pwrite,
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index 42f376cda5d2..22516402da0e 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -1589,7 +1589,7 @@ def test_run_with_shell_timeout_and_capture_output(self):
 
 
 def _get_test_grp_name():
-    for name_group in ('staff', 'nogroup', 'grp'):
+    for name_group in ('staff', 'nogroup', 'grp', 'nobody', 'nfsnobody'):
         if grp:
             try:
                 grp.getgrnam(name_group)
@@ -1768,24 +1768,27 @@ def test_user(self):
             test_users.append(name_uid)
 
         for user in test_users:
-            with self.subTest(user=user):
-                try:
-                    output = subprocess.check_output(
-                            [sys.executable, "-c",
-                             "import os; print(os.getuid())"],
-                            user=user)
-                except PermissionError:  # errno.EACCES
-                    pass
-                except OSError as e:
-                    if e.errno not in (errno.EACCES, errno.EPERM):
-                        raise
-                else:
-                    if isinstance(user, str):
-                        user_uid = pwd.getpwnam(user).pw_uid
+            # posix_spawn() may be used with close_fds=False
+            for close_fds in (False, True):
+                with self.subTest(user=user, close_fds=close_fds):
+                    try:
+                        output = subprocess.check_output(
+                                [sys.executable, "-c",
+                                 "import os; print(os.getuid())"],
+                                user=user,
+                                close_fds=close_fds)
+                    except PermissionError:  # (EACCES, EPERM)
+                        pass
+                    except OSError as e:
+                        if e.errno not in (errno.EACCES, errno.EPERM):
+                            raise
                     else:
-                        user_uid = user
-                    child_user = int(output)
-                    self.assertEqual(child_user, user_uid)
+                        if isinstance(user, str):
+                            user_uid = pwd.getpwnam(user).pw_uid
+                        else:
+                            user_uid = user
+                        child_user = int(output)
+                        self.assertEqual(child_user, user_uid)
 
         with self.assertRaises(ValueError):
             subprocess.check_call([sys.executable, "-c", "pass"], user=-1)
@@ -1809,23 +1812,25 @@ def test_group(self):
             group_list.append(name_group)
 
         for group in group_list + [gid]:
-            with self.subTest(group=group):
-                try:
-                    output = subprocess.check_output(
-                            [sys.executable, "-c",
-                             "import os; print(os.getgid())"],
-                            group=group)
-                except OSError as e:
-                    if e.errno != errno.EPERM:
-                        raise
-                else:
-                    if isinstance(group, str):
-                        group_gid = grp.getgrnam(group).gr_gid
+            # posix_spawn() may be used with close_fds=False
+            for close_fds in (False, True):
+                with self.subTest(group=group, close_fds=close_fds):
+                    try:
+                        output = subprocess.check_output(
+                                [sys.executable, "-c",
+                                 "import os; print(os.getgid())"],
+                                group=group,
+                                close_fds=close_fds)
+                    except PermissionError:  # (EACCES, EPERM)
+                        pass
                     else:
-                        group_gid = group
+                        if isinstance(group, str):
+                            group_gid = grp.getgrnam(group).gr_gid
+                        else:
+                            group_gid = group
 
-                    child_group = int(output)
-                    self.assertEqual(child_group, group_gid)
+                        child_group = int(output)
+                        self.assertEqual(child_group, group_gid)
 
         # make sure we bomb on negative values
         with self.assertRaises(ValueError):


From webhook-mailer at python.org  Wed Sep 25 10:06:06 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Wed, 25 Sep 2019 14:06:06 -0000
Subject: [Python-checkins] bpo-22273: Re-enabled ctypes test on ARM
 machines. (GH-16388)
Message-ID: <mailman.73.1569420367.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/417089e88bd4ea146b9497e06e8edeb58a59cd65
commit: 417089e88bd4ea146b9497e06e8edeb58a59cd65
branch: master
author: Vinay Sajip <vinay_sajip at yahoo.co.uk>
committer: GitHub <noreply at github.com>
date: 2019-09-25T15:05:55+01:00
summary:

bpo-22273: Re-enabled ctypes test on ARM machines. (GH-16388)

files:
M Lib/ctypes/test/test_structures.py

diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py
index 661efc1a0f85..fedcae24dfd3 100644
--- a/Lib/ctypes/test/test_structures.py
+++ b/Lib/ctypes/test/test_structures.py
@@ -480,7 +480,7 @@ class X(Structure):
         self.assertEqual(s.first, got.first)
         self.assertEqual(s.second, got.second)
 
-    @unittest.skipIf(MACHINE.startswith(('arm', 'ppc')),
+    @unittest.skipIf(MACHINE.startswith('ppc'),
                      'Test temporarily disabled on this architecture')
     def test_array_in_struct(self):
         # See bpo-22273


From webhook-mailer at python.org  Wed Sep 25 10:12:36 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 25 Sep 2019 14:12:36 -0000
Subject: [Python-checkins] bpo-37064: Skip test_tools.test_pathfix if
 installed (GH-15705) (GH-16389)
Message-ID: <mailman.74.1569420758.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/2e566bf35e96f0d6ca6fe291f179e832d1c32186
commit: 2e566bf35e96f0d6ca6fe291f179e832d1c32186
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T16:12:32+02:00
summary:

bpo-37064: Skip test_tools.test_pathfix if installed (GH-15705) (GH-16389)

If Python is installed, skip test_tools.test_pathfix test because
Tools/scripts/pathfix.py script is not installed.

(cherry picked from commit 3f43ceff186da09978d0aff257bb18b8ac7611f7)

files:
M Lib/test/test_tools/test_pathfix.py

diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py
index ce3d536c1293..a87992491344 100644
--- a/Lib/test/test_tools/test_pathfix.py
+++ b/Lib/test/test_tools/test_pathfix.py
@@ -3,7 +3,11 @@
 import sys
 import unittest
 from test import support
-from test.test_tools import import_tool, scriptsdir
+from test.test_tools import import_tool, scriptsdir, skip_if_missing
+
+
+# need Tools/script/ directory: skip if run on Python installed on the system
+skip_if_missing()
 
 
 class TestPathfixFunctional(unittest.TestCase):


From webhook-mailer at python.org  Wed Sep 25 10:14:14 2019
From: webhook-mailer at python.org (Ned Deily)
Date: Wed, 25 Sep 2019 14:14:14 -0000
Subject: [Python-checkins] bpo-36210: correct logic in setup.py for optional
 extensions for AIX (GH-12202) (GH-16376)
Message-ID: <mailman.75.1569420855.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/e4be8c984d035da30f9ed051aa6f3f762f0e9060
commit: e4be8c984d035da30f9ed051aa6f3f762f0e9060
branch: 3.8
author: Michael Felt <aixtools at users.noreply.github.com>
committer: Ned Deily <nad at python.org>
date: 2019-09-25T10:14:09-04:00
summary:

bpo-36210: correct logic in setup.py for optional extensions for AIX (GH-12202) (GH-16376)

files:
A Misc/NEWS.d/next/Build/2019-09-24-22-47-47.bpo-36210.EmL9X1.rst
M setup.py

diff --git a/Misc/NEWS.d/next/Build/2019-09-24-22-47-47.bpo-36210.EmL9X1.rst b/Misc/NEWS.d/next/Build/2019-09-24-22-47-47.bpo-36210.EmL9X1.rst
new file mode 100644
index 000000000000..aa9a56fe57ae
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-09-24-22-47-47.bpo-36210.EmL9X1.rst
@@ -0,0 +1,9 @@
+Update optional extension module detection for AIX.
+ossaudiodev and spwd are not applicable for AIX, and
+are no longer reported as missing.
+3rd-party packaging of ncurses (with ASIS support)
+conflicts with officially supported AIX curses library,
+so configure AIX to use libcurses.a. However, skip
+trying to build _curses_panel.
+
+patch by M Felt
diff --git a/setup.py b/setup.py
index be2ac0b01d00..20d7f35652fe 100644
--- a/setup.py
+++ b/setup.py
@@ -43,6 +43,7 @@ def get_platform():
 MS_WINDOWS = (HOST_PLATFORM == 'win32')
 CYGWIN = (HOST_PLATFORM == 'cygwin')
 MACOS = (HOST_PLATFORM == 'darwin')
+AIX = (HOST_PLATFORM.startswith('aix'))
 VXWORKS = ('vxworks' in HOST_PLATFORM)
 
 
@@ -807,7 +808,9 @@ def detect_simple_extensions(self):
         if (self.config_h_vars.get('HAVE_GETSPNAM', False) or
                 self.config_h_vars.get('HAVE_GETSPENT', False)):
             self.add(Extension('spwd', ['spwdmodule.c']))
-        else:
+        # AIX has shadow passwords, but access is not via getspent(), etc.
+        # module support is not expected so it not 'missing'
+        elif not AIX:
             self.missing.append('spwd')
 
         # select(2); not on ancient System V
@@ -911,6 +914,10 @@ def detect_readline_curses(self):
             curses_library = readline_termcap_library
         elif self.compiler.find_library_file(self.lib_dirs, 'ncursesw'):
             curses_library = 'ncursesw'
+        # Issue 36210: OSS provided ncurses does not link on AIX
+        # Use IBM supplied 'curses' for successful build of _curses
+        elif AIX and self.compiler.find_library_file(self.lib_dirs, 'curses'):
+            curses_library = 'curses'
         elif self.compiler.find_library_file(self.lib_dirs, 'ncurses'):
             curses_library = 'ncurses'
         elif self.compiler.find_library_file(self.lib_dirs, 'curses'):
@@ -1006,13 +1013,15 @@ def detect_readline_curses(self):
             self.missing.append('_curses')
 
         # If the curses module is enabled, check for the panel module
-        if (curses_enabled and
-            self.compiler.find_library_file(self.lib_dirs, panel_library)):
+        # _curses_panel needs some form of ncurses
+        skip_curses_panel = True if AIX else False
+        if (curses_enabled and not skip_curses_panel and
+                self.compiler.find_library_file(self.lib_dirs, panel_library)):
             self.add(Extension('_curses_panel', ['_curses_panel.c'],
                                include_dirs=curses_includes,
                                define_macros=curses_defines,
                                libraries=[panel_library, *curses_libs]))
-        else:
+        elif not skip_curses_panel:
             self.missing.append('_curses_panel')
 
     def detect_crypt(self):
@@ -1465,7 +1474,7 @@ def detect_platform_specific_exts(self):
         # Platform-specific libraries
         if HOST_PLATFORM.startswith(('linux', 'freebsd', 'gnukfreebsd')):
             self.add(Extension('ossaudiodev', ['ossaudiodev.c']))
-        else:
+        elif not AIX:
             self.missing.append('ossaudiodev')
 
         if MACOS:


From webhook-mailer at python.org  Wed Sep 25 10:30:24 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 25 Sep 2019 14:30:24 -0000
Subject: [Python-checkins] bpo-38270: Check for hash digest algorithms and
 avoid MD5 (GH-16382)
Message-ID: <mailman.76.1569421825.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/c64a1a61e6fc542cada40eb069a239317e1af36e
commit: c64a1a61e6fc542cada40eb069a239317e1af36e
branch: master
author: Christian Heimes <christian at python.org>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-25T07:30:20-07:00
summary:

bpo-38270: Check for hash digest algorithms and avoid MD5 (GH-16382)



Make it easier to run and test Python on systems with restrict crypto policies:

* add requires_hashdigest to test.support to check if a hash digest algorithm is available and working
* avoid MD5 in test_hmac
* replace MD5 with SHA256 in test_tarfile
* mark network tests that require MD5 for MD5-based digest auth or CRAM-MD5


https://bugs.python.org/issue38270

files:
A Misc/NEWS.d/next/Tests/2019-09-25-12-18-31.bpo-38270._x-9uH.rst
M Lib/test/support/__init__.py
M Lib/test/test_hmac.py
M Lib/test/test_imaplib.py
M Lib/test/test_poplib.py
M Lib/test/test_smtplib.py
M Lib/test/test_tarfile.py
M Lib/test/test_urllib2_localnet.py

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 46d646fe5b76..e401090c2146 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -12,6 +12,7 @@
 import functools
 import gc
 import glob
+import hashlib
 import importlib
 import importlib.util
 import locale
@@ -648,6 +649,29 @@ def wrapper(*args, **kw):
     return decorator
 
 
+def requires_hashdigest(digestname):
+    """Decorator raising SkipTest if a hashing algorithm is not available
+
+    The hashing algorithm could be missing or blocked by a strict crypto
+    policy.
+
+    ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS
+    ValueError: unsupported hash type md4
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                hashlib.new(digestname)
+            except ValueError:
+                raise unittest.SkipTest(
+                    f"hash digest '{digestname}' is not available."
+                )
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
+
+
 HOST = "localhost"
 HOSTv4 = "127.0.0.1"
 HOSTv6 = "::1"
diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py
index f2eb6d716d52..2c09de8b26fa 100644
--- a/Lib/test/test_hmac.py
+++ b/Lib/test/test_hmac.py
@@ -6,6 +6,8 @@
 import unittest.mock
 import warnings
 
+from test.support import requires_hashdigest
+
 
 def ignore_warning(func):
     @functools.wraps(func)
@@ -19,6 +21,7 @@ def wrapper(*args, **kwargs):
 
 class TestVectorsTestCase(unittest.TestCase):
 
+    @requires_hashdigest('md5')
     def test_md5_vectors(self):
         # Test the HMAC module against test vectors from the RFC.
 
@@ -76,6 +79,7 @@ def md5test(key, data, digest):
                  b"and Larger Than One Block-Size Data"),
                 "6f630fad67cda0ee1fb1f562db3aa53e")
 
+    @requires_hashdigest('sha1')
     def test_sha_vectors(self):
         def shatest(key, data, digest):
             h = hmac.HMAC(key, data, digestmod=hashlib.sha1)
@@ -268,23 +272,28 @@ def hmactest(key, data, hexdigests):
                                    '134676fb6de0446065c97440fa8c6a58',
                  })
 
+    @requires_hashdigest('sha224')
     def test_sha224_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64)
 
+    @requires_hashdigest('sha256')
     def test_sha256_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64)
 
+    @requires_hashdigest('sha384')
     def test_sha384_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128)
 
+    @requires_hashdigest('sha512')
     def test_sha512_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128)
 
+    @requires_hashdigest('sha256')
     def test_legacy_block_size_warnings(self):
         class MockCrazyHash(object):
             """Ain't no block_size attribute here."""
             def __init__(self, *args):
-                self._x = hashlib.sha1(*args)
+                self._x = hashlib.sha256(*args)
                 self.digest_size = self._x.digest_size
             def update(self, v):
                 self._x.update(v)
@@ -308,65 +317,78 @@ def test_with_digestmod_no_default(self):
             data = b"Hi There"
             hmac.HMAC(key, data, digestmod=None)
 
+
 class ConstructorTestCase(unittest.TestCase):
 
+    expected = (
+        "6c845b47f52b3b47f6590c502db7825aad757bf4fadc8fa972f7cd2e76a5bdeb"
+    )
+
+    @requires_hashdigest('sha256')
     def test_normal(self):
         # Standard constructor call.
-        failed = 0
         try:
-            h = hmac.HMAC(b"key", digestmod='md5')
+            hmac.HMAC(b"key", digestmod='sha256')
         except Exception:
             self.fail("Standard constructor call raised exception.")
 
+    @requires_hashdigest('sha256')
     def test_with_str_key(self):
         # Pass a key of type str, which is an error, because it expects a key
         # of type bytes
         with self.assertRaises(TypeError):
-            h = hmac.HMAC("key", digestmod='md5')
+            h = hmac.HMAC("key", digestmod='sha256')
 
+    @requires_hashdigest('sha256')
     def test_dot_new_with_str_key(self):
         # Pass a key of type str, which is an error, because it expects a key
         # of type bytes
         with self.assertRaises(TypeError):
-            h = hmac.new("key", digestmod='md5')
+            h = hmac.new("key", digestmod='sha256')
 
+    @requires_hashdigest('sha256')
     def test_withtext(self):
         # Constructor call with text.
         try:
-            h = hmac.HMAC(b"key", b"hash this!", digestmod='md5')
+            h = hmac.HMAC(b"key", b"hash this!", digestmod='sha256')
         except Exception:
             self.fail("Constructor call with text argument raised exception.")
-        self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+        self.assertEqual(h.hexdigest(), self.expected)
 
+    @requires_hashdigest('sha256')
     def test_with_bytearray(self):
         try:
             h = hmac.HMAC(bytearray(b"key"), bytearray(b"hash this!"),
-                          digestmod="md5")
+                          digestmod="sha256")
         except Exception:
             self.fail("Constructor call with bytearray arguments raised exception.")
-        self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+            self.assertEqual(h.hexdigest(), self.expected)
 
+    @requires_hashdigest('sha256')
     def test_with_memoryview_msg(self):
         try:
-            h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="md5")
+            h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="sha256")
         except Exception:
             self.fail("Constructor call with memoryview msg raised exception.")
-        self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+            self.assertEqual(h.hexdigest(), self.expected)
 
+    @requires_hashdigest('sha256')
     def test_withmodule(self):
         # Constructor call with text and digest module.
         try:
-            h = hmac.HMAC(b"key", b"", hashlib.sha1)
+            h = hmac.HMAC(b"key", b"", hashlib.sha256)
         except Exception:
-            self.fail("Constructor call with hashlib.sha1 raised exception.")
+            self.fail("Constructor call with hashlib.sha256 raised exception.")
+
 
 class SanityTestCase(unittest.TestCase):
 
+    @requires_hashdigest('sha256')
     def test_exercise_all_methods(self):
         # Exercising all methods once.
         # This must not raise any exceptions
         try:
-            h = hmac.HMAC(b"my secret key", digestmod="md5")
+            h = hmac.HMAC(b"my secret key", digestmod="sha256")
             h.update(b"compute the hash of this text!")
             dig = h.digest()
             dig = h.hexdigest()
@@ -374,11 +396,13 @@ def test_exercise_all_methods(self):
         except Exception:
             self.fail("Exception raised during normal usage of HMAC class.")
 
+
 class CopyTestCase(unittest.TestCase):
 
+    @requires_hashdigest('sha256')
     def test_attributes(self):
         # Testing if attributes are of same type.
-        h1 = hmac.HMAC(b"key", digestmod="md5")
+        h1 = hmac.HMAC(b"key", digestmod="sha256")
         h2 = h1.copy()
         self.assertTrue(h1.digest_cons == h2.digest_cons,
             "digest constructors don't match.")
@@ -387,9 +411,10 @@ def test_attributes(self):
         self.assertEqual(type(h1.outer), type(h2.outer),
             "Types of outer don't match.")
 
+    @requires_hashdigest('sha256')
     def test_realcopy(self):
         # Testing if the copy method created a real copy.
-        h1 = hmac.HMAC(b"key", digestmod="md5")
+        h1 = hmac.HMAC(b"key", digestmod="sha256")
         h2 = h1.copy()
         # Using id() in case somebody has overridden __eq__/__ne__.
         self.assertTrue(id(h1) != id(h2), "No real copy of the HMAC instance.")
@@ -398,9 +423,10 @@ def test_realcopy(self):
         self.assertTrue(id(h1.outer) != id(h2.outer),
             "No real copy of the attribute 'outer'.")
 
+    @requires_hashdigest('sha256')
     def test_equality(self):
         # Testing if the copy has the same digests.
-        h1 = hmac.HMAC(b"key", digestmod="md5")
+        h1 = hmac.HMAC(b"key", digestmod="sha256")
         h1.update(b"some random text")
         h2 = h1.copy()
         self.assertEqual(h1.digest(), h2.digest(),
diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py
index 8ab532af3f0d..03cffbe39c6c 100644
--- a/Lib/test/test_imaplib.py
+++ b/Lib/test/test_imaplib.py
@@ -10,7 +10,8 @@
 import socket
 
 from test.support import (reap_threads, verbose, transient_internet,
-                          run_with_tz, run_with_locale, cpython_only)
+                          run_with_tz, run_with_locale, cpython_only,
+                          requires_hashdigest)
 import unittest
 from unittest import mock
 from datetime import datetime, timezone, timedelta
@@ -370,6 +371,7 @@ def cmd_AUTHENTICATE(self, tag, args):
         self.assertEqual(code, 'OK')
         self.assertEqual(server.response, b'ZmFrZQ==\r\n')  # b64 encoded 'fake'
 
+    @requires_hashdigest('md5')
     def test_login_cram_md5_bytes(self):
         class AuthHandler(SimpleIMAPHandler):
             capabilities = 'LOGINDISABLED AUTH=CRAM-MD5'
@@ -387,6 +389,7 @@ def cmd_AUTHENTICATE(self, tag, args):
         ret, _ = client.login_cram_md5("tim", b"tanstaaftanstaaf")
         self.assertEqual(ret, "OK")
 
+    @requires_hashdigest('md5')
     def test_login_cram_md5_plain_text(self):
         class AuthHandler(SimpleIMAPHandler):
             capabilities = 'LOGINDISABLED AUTH=CRAM-MD5'
@@ -797,6 +800,7 @@ def cmd_AUTHENTICATE(self, tag, args):
                              b'ZmFrZQ==\r\n')  # b64 encoded 'fake'
 
     @reap_threads
+    @requires_hashdigest('md5')
     def test_login_cram_md5(self):
 
         class AuthHandler(SimpleIMAPHandler):
diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py
index 20d4eeac12d4..7b1d854d1c08 100644
--- a/Lib/test/test_poplib.py
+++ b/Lib/test/test_poplib.py
@@ -309,9 +309,11 @@ def test_noop(self):
     def test_rpop(self):
         self.assertOK(self.client.rpop('foo'))
 
+    @test_support.requires_hashdigest('md5')
     def test_apop_normal(self):
         self.assertOK(self.client.apop('foo', 'dummypassword'))
 
+    @test_support.requires_hashdigest('md5')
     def test_apop_REDOS(self):
         # Replace welcome with very long evil welcome.
         # NB The upper bound on welcome length is currently 2048.
diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py
index f1332e9ef782..d0c9862edea9 100644
--- a/Lib/test/test_smtplib.py
+++ b/Lib/test/test_smtplib.py
@@ -4,6 +4,7 @@
 from email.message import EmailMessage
 from email.base64mime import body_encode as encode_base64
 import email.utils
+import hashlib
 import hmac
 import socket
 import smtpd
@@ -21,6 +22,7 @@
 from test import support, mock_socket
 from test.support import HOST
 from test.support import threading_setup, threading_cleanup, join_thread
+from test.support import requires_hashdigest
 from unittest.mock import Mock
 
 
@@ -1009,6 +1011,7 @@ def testAUTH_LOGIN(self):
         self.assertEqual(resp, (235, b'Authentication Succeeded'))
         smtp.close()
 
+    @requires_hashdigest('md5')
     def testAUTH_CRAM_MD5(self):
         self.serv.add_feature("AUTH CRAM-MD5")
         smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15)
@@ -1025,7 +1028,13 @@ def testAUTH_multiple(self):
         smtp.close()
 
     def test_auth_function(self):
-        supported = {'CRAM-MD5', 'PLAIN', 'LOGIN'}
+        supported = {'PLAIN', 'LOGIN'}
+        try:
+            hashlib.md5()
+        except ValueError:
+            pass
+        else:
+            supported.add('CRAM-MD5')
         for mechanism in supported:
             self.serv.add_feature("AUTH {}".format(mechanism))
         for mechanism in supported:
diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py
index 7e32cbccd6c5..15324a4e4881 100644
--- a/Lib/test/test_tarfile.py
+++ b/Lib/test/test_tarfile.py
@@ -1,7 +1,7 @@
 import sys
 import os
 import io
-from hashlib import md5
+from hashlib import sha256
 from contextlib import contextmanager
 from random import Random
 import pathlib
@@ -11,7 +11,7 @@
 import tarfile
 
 from test import support
-from test.support import script_helper
+from test.support import script_helper, requires_hashdigest
 
 # Check for our compression modules.
 try:
@@ -27,8 +27,8 @@
 except ImportError:
     lzma = None
 
-def md5sum(data):
-    return md5(data).hexdigest()
+def sha256sum(data):
+    return sha256(data).hexdigest()
 
 TEMPDIR = os.path.abspath(support.TESTFN) + "-tardir"
 tarextdir = TEMPDIR + '-extract-test'
@@ -39,8 +39,12 @@ def md5sum(data):
 tmpname = os.path.join(TEMPDIR, "tmp.tar")
 dotlessname = os.path.join(TEMPDIR, "testtar")
 
-md5_regtype = "65f477c818ad9e15f7feab0c6d37742f"
-md5_sparse = "a54fbc4ca4f4399a90e1b27164012fc6"
+sha256_regtype = (
+    "e09e4bc8b3c9d9177e77256353b36c159f5f040531bbd4b024a8f9b9196c71ce"
+)
+sha256_sparse = (
+    "4f05a776071146756345ceee937b33fc5644f5a96b9780d1c7d6a32cdf164d7b"
+)
 
 
 class TarTest:
@@ -95,7 +99,7 @@ def test_fileobj_regular_file(self):
             data = fobj.read()
             self.assertEqual(len(data), tarinfo.size,
                     "regular file extraction failed")
-            self.assertEqual(md5sum(data), md5_regtype,
+            self.assertEqual(sha256sum(data), sha256_regtype,
                     "regular file extraction failed")
 
     def test_fileobj_readlines(self):
@@ -178,7 +182,7 @@ def test_fileobj_text(self):
         with self.tar.extractfile("ustar/regtype") as fobj:
             fobj = io.TextIOWrapper(fobj)
             data = fobj.read().encode("iso8859-1")
-            self.assertEqual(md5sum(data), md5_regtype)
+            self.assertEqual(sha256sum(data), sha256_regtype)
             try:
                 fobj.seek(100)
             except AttributeError:
@@ -543,13 +547,13 @@ def test_extract_hardlink(self):
             self.addCleanup(support.unlink, os.path.join(TEMPDIR, "ustar/lnktype"))
             with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f:
                 data = f.read()
-            self.assertEqual(md5sum(data), md5_regtype)
+            self.assertEqual(sha256sum(data), sha256_regtype)
 
             tar.extract("ustar/symtype", TEMPDIR)
             self.addCleanup(support.unlink, os.path.join(TEMPDIR, "ustar/symtype"))
             with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f:
                 data = f.read()
-            self.assertEqual(md5sum(data), md5_regtype)
+            self.assertEqual(sha256sum(data), sha256_regtype)
 
     def test_extractall(self):
         # Test if extractall() correctly restores directory permissions
@@ -684,7 +688,7 @@ def test_fileobj_regular_file(self):
             data = fobj.read()
         self.assertEqual(len(data), tarinfo.size,
                 "regular file extraction failed")
-        self.assertEqual(md5sum(data), md5_regtype,
+        self.assertEqual(sha256sum(data), sha256_regtype,
                 "regular file extraction failed")
 
     def test_provoke_stream_error(self):
@@ -796,8 +800,8 @@ class MemberReadTest(ReadTest, unittest.TestCase):
     def _test_member(self, tarinfo, chksum=None, **kwargs):
         if chksum is not None:
             with self.tar.extractfile(tarinfo) as f:
-                self.assertEqual(md5sum(f.read()), chksum,
-                        "wrong md5sum for %s" % tarinfo.name)
+                self.assertEqual(sha256sum(f.read()), chksum,
+                        "wrong sha256sum for %s" % tarinfo.name)
 
         kwargs["mtime"] = 0o7606136617
         kwargs["uid"] = 1000
@@ -812,11 +816,11 @@ def _test_member(self, tarinfo, chksum=None, **kwargs):
 
     def test_find_regtype(self):
         tarinfo = self.tar.getmember("ustar/regtype")
-        self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
 
     def test_find_conttype(self):
         tarinfo = self.tar.getmember("ustar/conttype")
-        self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
 
     def test_find_dirtype(self):
         tarinfo = self.tar.getmember("ustar/dirtype")
@@ -848,28 +852,28 @@ def test_find_fifotype(self):
 
     def test_find_sparse(self):
         tarinfo = self.tar.getmember("ustar/sparse")
-        self._test_member(tarinfo, size=86016, chksum=md5_sparse)
+        self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
 
     def test_find_gnusparse(self):
         tarinfo = self.tar.getmember("gnu/sparse")
-        self._test_member(tarinfo, size=86016, chksum=md5_sparse)
+        self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
 
     def test_find_gnusparse_00(self):
         tarinfo = self.tar.getmember("gnu/sparse-0.0")
-        self._test_member(tarinfo, size=86016, chksum=md5_sparse)
+        self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
 
     def test_find_gnusparse_01(self):
         tarinfo = self.tar.getmember("gnu/sparse-0.1")
-        self._test_member(tarinfo, size=86016, chksum=md5_sparse)
+        self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
 
     def test_find_gnusparse_10(self):
         tarinfo = self.tar.getmember("gnu/sparse-1.0")
-        self._test_member(tarinfo, size=86016, chksum=md5_sparse)
+        self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
 
     def test_find_umlauts(self):
         tarinfo = self.tar.getmember("ustar/umlauts-"
                                      "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
-        self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
 
     def test_find_ustar_longname(self):
         name = "ustar/" + "12345/" * 39 + "1234567/longname"
@@ -877,7 +881,7 @@ def test_find_ustar_longname(self):
 
     def test_find_regtype_oldv7(self):
         tarinfo = self.tar.getmember("misc/regtype-old-v7")
-        self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
 
     def test_find_pax_umlauts(self):
         self.tar.close()
@@ -885,7 +889,7 @@ def test_find_pax_umlauts(self):
                                 encoding="iso8859-1")
         tarinfo = self.tar.getmember("pax/umlauts-"
                                      "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
-        self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
 
 
 class LongnameTest:
@@ -947,8 +951,8 @@ def _test_sparse_file(self, name):
         filename = os.path.join(TEMPDIR, name)
         with open(filename, "rb") as fobj:
             data = fobj.read()
-        self.assertEqual(md5sum(data), md5_sparse,
-                "wrong md5sum for %s" % name)
+        self.assertEqual(sha256sum(data), sha256_sparse,
+                "wrong sha256sum for %s" % name)
 
         if self._fs_supports_holes():
             s = os.stat(filename)
@@ -2443,7 +2447,7 @@ def _test_link_extraction(self, name):
         self.tar.extract(name, TEMPDIR)
         with open(os.path.join(TEMPDIR, name), "rb") as f:
             data = f.read()
-        self.assertEqual(md5sum(data), md5_regtype)
+        self.assertEqual(sha256sum(data), sha256_regtype)
 
     # See issues #1578269, #8879, and #17689 for some history on these skips
     @unittest.skipIf(hasattr(os.path, "islink"),
diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py
index 6b9b130fb6e9..8cfb214c9af9 100644
--- a/Lib/test/test_urllib2_localnet.py
+++ b/Lib/test/test_urllib2_localnet.py
@@ -322,6 +322,7 @@ class ProxyAuthTests(unittest.TestCase):
     PASSWD = "test123"
     REALM = "TestRealm"
 
+    @support.requires_hashdigest("md5")
     def setUp(self):
         super(ProxyAuthTests, self).setUp()
         # Ignore proxy bypass settings in the environment.
diff --git a/Misc/NEWS.d/next/Tests/2019-09-25-12-18-31.bpo-38270._x-9uH.rst b/Misc/NEWS.d/next/Tests/2019-09-25-12-18-31.bpo-38270._x-9uH.rst
new file mode 100644
index 000000000000..efb1b98b36d0
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-25-12-18-31.bpo-38270._x-9uH.rst
@@ -0,0 +1,4 @@
+test.support now has a helper function to check for availibility of a
+hash digest function. Several tests are refactored avoid MD5 and use
+SHA256 instead. Other tests are marked to use MD5 and skipped when MD5 is
+disabled.


From webhook-mailer at python.org  Wed Sep 25 10:30:40 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Wed, 25 Sep 2019 14:30:40 -0000
Subject: [Python-checkins] bpo-38234: Fix test_embed pathconfig tests
 (GH-16390)
Message-ID: <mailman.77.1569421841.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/00508a7407d7d300b487532e2271534b20e378a7
commit: 00508a7407d7d300b487532e2271534b20e378a7
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T16:30:36+02:00
summary:

bpo-38234: Fix test_embed pathconfig tests (GH-16390)

bpo-38234: On macOS and FreeBSD, the temporary directory can be
symbolic link. For example, /tmp can be a symbolic link to /var/tmp.
Call realpath() to resolve all symbolic links.

files:
M Lib/test/test_embed.py

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index e7a10b6defea..ed90fc0cbed0 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -1042,6 +1042,11 @@ def module_search_paths(self, prefix=None, exec_prefix=None):
     def tmpdir_with_python(self):
         # Temporary directory with a copy of the Python program
         with tempfile.TemporaryDirectory() as tmpdir:
+            # bpo-38234: On macOS and FreeBSD, the temporary directory
+            # can be symbolic link. For example, /tmp can be a symbolic link
+            # to /var/tmp. Call realpath() to resolve all symbolic links.
+            tmpdir = os.path.realpath(tmpdir)
+
             if MS_WINDOWS:
                 # Copy pythonXY.dll (or pythonXY_d.dll)
                 ver = sys.version_info


From webhook-mailer at python.org  Wed Sep 25 11:13:38 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Wed, 25 Sep 2019 15:13:38 -0000
Subject: [Python-checkins] bpo-38255: super() can search attributes as well
 as methods (GH-16368)
Message-ID: <mailman.78.1569424421.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/15ccc4fac09b866d61b069c6c136aabfe4bac09c
commit: 15ccc4fac09b866d61b069c6c136aabfe4bac09c
branch: master
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T08:13:19-07:00
summary:

bpo-38255: super() can search attributes as well as methods (GH-16368)

Improvement suggested by G?ry Ogam.

files:
M Doc/library/functions.rst

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 4d3b2f593c0f..28d9c7b99e6f 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -1637,6 +1637,10 @@ are always available.  They are listed here in alphabetical order.
               super().method(arg)    # This does the same thing as:
                                      # super(C, self).method(arg)
 
+   In addition to method lookups, :func:`super` also works for attribute
+   lookups.  One possible use case for this is calling :term:`descriptor`\s
+   in a parent or sibling class.
+
    Note that :func:`super` is implemented as part of the binding process for
    explicit dotted attribute lookups such as ``super().__getitem__(name)``.
    It does so by implementing its own :meth:`__getattribute__` method for searching


From webhook-mailer at python.org  Wed Sep 25 11:24:17 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Wed, 25 Sep 2019 15:24:17 -0000
Subject: [Python-checkins] bpo-38255: super() can search attributes as well
 as methods (GH-16368) (GH-16391)
Message-ID: <mailman.79.1569425058.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/7444a5a402bbac4268b95958a9578a9e3dae33e0
commit: 7444a5a402bbac4268b95958a9578a9e3dae33e0
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-25T08:24:12-07:00
summary:

bpo-38255: super() can search attributes as well as methods (GH-16368) (GH-16391)

Improvement suggested by G?ry Ogam.
(cherry picked from commit 15ccc4fac09b866d61b069c6c136aabfe4bac09c)

Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com>

files:
M Doc/library/functions.rst

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 947a0e54aed0..0236ff03da07 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -1635,6 +1635,10 @@ are always available.  They are listed here in alphabetical order.
               super().method(arg)    # This does the same thing as:
                                      # super(C, self).method(arg)
 
+   In addition to method lookups, :func:`super` also works for attribute
+   lookups.  One possible use case for this is calling :term:`descriptor`\s
+   in a parent or sibling class.
+
    Note that :func:`super` is implemented as part of the binding process for
    explicit dotted attribute lookups such as ``super().__getitem__(name)``.
    It does so by implementing its own :meth:`__getattribute__` method for searching


From webhook-mailer at python.org  Wed Sep 25 11:24:28 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Wed, 25 Sep 2019 15:24:28 -0000
Subject: [Python-checkins] bpo-38255: super() can search attributes as well
 as methods (GH-16368) (GH-16392)
Message-ID: <mailman.80.1569425069.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/3b5e9241142d6b7c22059115c0e68a8912753818
commit: 3b5e9241142d6b7c22059115c0e68a8912753818
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-25T08:24:24-07:00
summary:

bpo-38255: super() can search attributes as well as methods (GH-16368) (GH-16392)

Improvement suggested by G?ry Ogam.
(cherry picked from commit 15ccc4fac09b866d61b069c6c136aabfe4bac09c)

Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com>

files:
M Doc/library/functions.rst

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 313dbffdcde9..06ba323ca30c 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -1532,6 +1532,10 @@ are always available.  They are listed here in alphabetical order.
               super().method(arg)    # This does the same thing as:
                                      # super(C, self).method(arg)
 
+   In addition to method lookups, :func:`super` also works for attribute
+   lookups.  One possible use case for this is calling :term:`descriptor`\s
+   in a parent or sibling class.
+
    Note that :func:`super` is implemented as part of the binding process for
    explicit dotted attribute lookups such as ``super().__getitem__(name)``.
    It does so by implementing its own :meth:`__getattribute__` method for searching


From webhook-mailer at python.org  Wed Sep 25 11:36:01 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Wed, 25 Sep 2019 15:36:01 -0000
Subject: [Python-checkins] bpo-38005: Remove support of string argument in
 InterpreterID(). (GH-16227)
Message-ID: <mailman.81.1569425762.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/543a3951a1c96bae0ea839eacec71d3b1a563a10
commit: 543a3951a1c96bae0ea839eacec71d3b1a563a10
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T18:35:57+03:00
summary:

bpo-38005: Remove support of string argument in InterpreterID(). (GH-16227)

Make negative interpreter id to raise ValueError instead of RuntimeError.

files:
M Lib/test/test__xxsubinterpreters.py
M Objects/interpreteridobject.c

diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py
index f5474f4101d0..207b5db5d8fb 100644
--- a/Lib/test/test__xxsubinterpreters.py
+++ b/Lib/test/test__xxsubinterpreters.py
@@ -514,7 +514,7 @@ def test_does_not_exist(self):
             interpreters.is_running(1_000_000)
 
     def test_bad_id(self):
-        with self.assertRaises(RuntimeError):
+        with self.assertRaises(ValueError):
             interpreters.is_running(-1)
 
 
@@ -530,18 +530,15 @@ class Int(str):
             def __index__(self):
                 return 10
 
-        for id in ('10', '1_0', Int()):
-            with self.subTest(id=id):
-                id = interpreters.InterpreterID(id, force=True)
-                self.assertEqual(int(id), 10)
+        id = interpreters.InterpreterID(Int(), force=True)
+        self.assertEqual(int(id), 10)
 
     def test_bad_id(self):
         self.assertRaises(TypeError, interpreters.InterpreterID, object())
         self.assertRaises(TypeError, interpreters.InterpreterID, 10.0)
+        self.assertRaises(TypeError, interpreters.InterpreterID, '10')
         self.assertRaises(TypeError, interpreters.InterpreterID, b'10')
         self.assertRaises(ValueError, interpreters.InterpreterID, -1)
-        self.assertRaises(ValueError, interpreters.InterpreterID, '-1')
-        self.assertRaises(ValueError, interpreters.InterpreterID, 'spam')
         self.assertRaises(OverflowError, interpreters.InterpreterID, 2**64)
 
     def test_does_not_exist(self):
@@ -720,7 +717,7 @@ def test_does_not_exist(self):
             interpreters.destroy(1_000_000)
 
     def test_bad_id(self):
-        with self.assertRaises(RuntimeError):
+        with self.assertRaises(ValueError):
             interpreters.destroy(-1)
 
     def test_from_current(self):
@@ -863,7 +860,7 @@ def test_does_not_exist(self):
             interpreters.run_string(id, 'print("spam")')
 
     def test_error_id(self):
-        with self.assertRaises(RuntimeError):
+        with self.assertRaises(ValueError):
             interpreters.run_string(-1, 'print("spam")')
 
     def test_bad_id(self):
diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c
index 3edbb85e6ac3..94f5dd709bbd 100644
--- a/Objects/interpreteridobject.c
+++ b/Objects/interpreteridobject.c
@@ -35,52 +35,45 @@ newinterpid(PyTypeObject *cls, int64_t id, int force)
     return self;
 }
 
-static PyObject *
-interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
+static int
+interp_id_converter(PyObject *arg, void *ptr)
 {
-    static char *kwlist[] = {"id", "force", NULL};
-    PyObject *idobj;
-    int force = 0;
-    if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O|$p:InterpreterID.__init__", kwlist,
-                                     &idobj, &force)) {
-        return NULL;
-    }
-
-    // Coerce and check the ID.
     int64_t id;
-    if (PyObject_TypeCheck(idobj, &_PyInterpreterID_Type)) {
-        id = ((interpid *)idobj)->id;
+    if (PyObject_TypeCheck(arg, &_PyInterpreterID_Type)) {
+        id = ((interpid *)arg)->id;
     }
-    else {
-        PyObject *pyid;
-        if (PyIndex_Check(idobj)) {
-            pyid = idobj;
-            Py_INCREF(pyid);
-        }
-        else if (PyUnicode_Check(idobj)) {
-            pyid = PyNumber_Long(idobj);
-            if (pyid == NULL) {
-                return NULL;
-            }
-        }
-        else {
-            PyErr_Format(PyExc_TypeError,
-                         "interpreter ID must be an int, got %.100s",
-                         idobj->ob_type->tp_name);
-            return NULL;
-        }
-        id = PyLong_AsLongLong(pyid);
-        Py_DECREF(pyid);
+    else if (PyIndex_Check(arg)) {
+        id = PyLong_AsLongLong(arg);
         if (id == -1 && PyErr_Occurred()) {
-            return NULL;
+            return 0;
         }
         if (id < 0) {
             PyErr_Format(PyExc_ValueError,
-                         "interpreter ID must be a non-negative int, got %R", idobj);
-            return NULL;
+                         "interpreter ID must be a non-negative int, got %R", arg);
+            return 0;
         }
     }
+    else {
+        PyErr_Format(PyExc_TypeError,
+                     "interpreter ID must be an int, got %.100s",
+                     arg->ob_type->tp_name);
+        return 0;
+    }
+    *(int64_t *)ptr = id;
+    return 1;
+}
+
+static PyObject *
+interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
+{
+    static char *kwlist[] = {"id", "force", NULL};
+    int64_t id;
+    int force = 0;
+    if (!PyArg_ParseTupleAndKeywords(args, kwds,
+                                     "O&|$p:InterpreterID.__init__", kwlist,
+                                     interp_id_converter, &id, &force)) {
+        return NULL;
+    }
 
     return (PyObject *)newinterpid(cls, id, force);
 }
@@ -287,19 +280,7 @@ PyInterpreterState *
 _PyInterpreterID_LookUp(PyObject *requested_id)
 {
     int64_t id;
-    if (PyObject_TypeCheck(requested_id, &_PyInterpreterID_Type)) {
-        id = ((interpid *)requested_id)->id;
-    }
-    else if (PyIndex_Check(requested_id)) {
-        id = PyLong_AsLongLong(requested_id);
-        if (id == -1 && PyErr_Occurred() != NULL) {
-            return NULL;
-        }
-        assert(id <= INT64_MAX);
-    }
-    else {
-        PyErr_Format(PyExc_TypeError, "interpreter ID must be an int, got %.100s",
-                     requested_id->ob_type->tp_name);
+    if (!interp_id_converter(requested_id, &id)) {
         return NULL;
     }
     return _PyInterpreterState_LookUpID(id);


From webhook-mailer at python.org  Wed Sep 25 11:50:45 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 25 Sep 2019 15:50:45 -0000
Subject: [Python-checkins] [3.8] bpo-38270: Check for hash digest algorithms
 and avoid MD5 (GH-16382) (GH-16393)
Message-ID: <mailman.82.1569426647.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/66cd041df8dfadd001ae298292e16f0271c0d139
commit: 66cd041df8dfadd001ae298292e16f0271c0d139
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T08:50:31-07:00
summary:

[3.8] bpo-38270: Check for hash digest algorithms and avoid MD5 (GH-16382) (GH-16393)



Make it easier to run and test Python on systems with restrict crypto policies:

* add requires_hashdigest to test.support to check if a hash digest algorithm is available and working
* avoid MD5 in test_hmac
* replace MD5 with SHA256 in test_tarfile
* mark network tests that require MD5 for MD5-based digest auth or CRAM-MD5


https://bugs.python.org/issue38270
(cherry picked from commit c64a1a61e6fc542cada40eb069a239317e1af36e)


Co-authored-by: Christian Heimes <christian at python.org>


https://bugs.python.org/issue38270



Automerge-Triggered-By: @tiran

files:
A Misc/NEWS.d/next/Tests/2019-09-25-12-18-31.bpo-38270._x-9uH.rst
M Lib/test/support/__init__.py
M Lib/test/test_hmac.py
M Lib/test/test_imaplib.py
M Lib/test/test_poplib.py
M Lib/test/test_smtplib.py
M Lib/test/test_tarfile.py
M Lib/test/test_urllib2_localnet.py

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index e4e0481d336f..ad69ec94fb82 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -12,6 +12,7 @@
 import functools
 import gc
 import glob
+import hashlib
 import importlib
 import importlib.util
 import locale
@@ -646,6 +647,29 @@ def wrapper(*args, **kw):
     return decorator
 
 
+def requires_hashdigest(digestname):
+    """Decorator raising SkipTest if a hashing algorithm is not available
+
+    The hashing algorithm could be missing or blocked by a strict crypto
+    policy.
+
+    ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS
+    ValueError: unsupported hash type md4
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                hashlib.new(digestname)
+            except ValueError:
+                raise unittest.SkipTest(
+                    f"hash digest '{digestname}' is not available."
+                )
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
+
+
 HOST = "localhost"
 HOSTv4 = "127.0.0.1"
 HOSTv6 = "::1"
diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py
index f2eb6d716d52..2c09de8b26fa 100644
--- a/Lib/test/test_hmac.py
+++ b/Lib/test/test_hmac.py
@@ -6,6 +6,8 @@
 import unittest.mock
 import warnings
 
+from test.support import requires_hashdigest
+
 
 def ignore_warning(func):
     @functools.wraps(func)
@@ -19,6 +21,7 @@ def wrapper(*args, **kwargs):
 
 class TestVectorsTestCase(unittest.TestCase):
 
+    @requires_hashdigest('md5')
     def test_md5_vectors(self):
         # Test the HMAC module against test vectors from the RFC.
 
@@ -76,6 +79,7 @@ def md5test(key, data, digest):
                  b"and Larger Than One Block-Size Data"),
                 "6f630fad67cda0ee1fb1f562db3aa53e")
 
+    @requires_hashdigest('sha1')
     def test_sha_vectors(self):
         def shatest(key, data, digest):
             h = hmac.HMAC(key, data, digestmod=hashlib.sha1)
@@ -268,23 +272,28 @@ def hmactest(key, data, hexdigests):
                                    '134676fb6de0446065c97440fa8c6a58',
                  })
 
+    @requires_hashdigest('sha224')
     def test_sha224_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64)
 
+    @requires_hashdigest('sha256')
     def test_sha256_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64)
 
+    @requires_hashdigest('sha384')
     def test_sha384_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128)
 
+    @requires_hashdigest('sha512')
     def test_sha512_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128)
 
+    @requires_hashdigest('sha256')
     def test_legacy_block_size_warnings(self):
         class MockCrazyHash(object):
             """Ain't no block_size attribute here."""
             def __init__(self, *args):
-                self._x = hashlib.sha1(*args)
+                self._x = hashlib.sha256(*args)
                 self.digest_size = self._x.digest_size
             def update(self, v):
                 self._x.update(v)
@@ -308,65 +317,78 @@ def test_with_digestmod_no_default(self):
             data = b"Hi There"
             hmac.HMAC(key, data, digestmod=None)
 
+
 class ConstructorTestCase(unittest.TestCase):
 
+    expected = (
+        "6c845b47f52b3b47f6590c502db7825aad757bf4fadc8fa972f7cd2e76a5bdeb"
+    )
+
+    @requires_hashdigest('sha256')
     def test_normal(self):
         # Standard constructor call.
-        failed = 0
         try:
-            h = hmac.HMAC(b"key", digestmod='md5')
+            hmac.HMAC(b"key", digestmod='sha256')
         except Exception:
             self.fail("Standard constructor call raised exception.")
 
+    @requires_hashdigest('sha256')
     def test_with_str_key(self):
         # Pass a key of type str, which is an error, because it expects a key
         # of type bytes
         with self.assertRaises(TypeError):
-            h = hmac.HMAC("key", digestmod='md5')
+            h = hmac.HMAC("key", digestmod='sha256')
 
+    @requires_hashdigest('sha256')
     def test_dot_new_with_str_key(self):
         # Pass a key of type str, which is an error, because it expects a key
         # of type bytes
         with self.assertRaises(TypeError):
-            h = hmac.new("key", digestmod='md5')
+            h = hmac.new("key", digestmod='sha256')
 
+    @requires_hashdigest('sha256')
     def test_withtext(self):
         # Constructor call with text.
         try:
-            h = hmac.HMAC(b"key", b"hash this!", digestmod='md5')
+            h = hmac.HMAC(b"key", b"hash this!", digestmod='sha256')
         except Exception:
             self.fail("Constructor call with text argument raised exception.")
-        self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+        self.assertEqual(h.hexdigest(), self.expected)
 
+    @requires_hashdigest('sha256')
     def test_with_bytearray(self):
         try:
             h = hmac.HMAC(bytearray(b"key"), bytearray(b"hash this!"),
-                          digestmod="md5")
+                          digestmod="sha256")
         except Exception:
             self.fail("Constructor call with bytearray arguments raised exception.")
-        self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+            self.assertEqual(h.hexdigest(), self.expected)
 
+    @requires_hashdigest('sha256')
     def test_with_memoryview_msg(self):
         try:
-            h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="md5")
+            h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="sha256")
         except Exception:
             self.fail("Constructor call with memoryview msg raised exception.")
-        self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+            self.assertEqual(h.hexdigest(), self.expected)
 
+    @requires_hashdigest('sha256')
     def test_withmodule(self):
         # Constructor call with text and digest module.
         try:
-            h = hmac.HMAC(b"key", b"", hashlib.sha1)
+            h = hmac.HMAC(b"key", b"", hashlib.sha256)
         except Exception:
-            self.fail("Constructor call with hashlib.sha1 raised exception.")
+            self.fail("Constructor call with hashlib.sha256 raised exception.")
+
 
 class SanityTestCase(unittest.TestCase):
 
+    @requires_hashdigest('sha256')
     def test_exercise_all_methods(self):
         # Exercising all methods once.
         # This must not raise any exceptions
         try:
-            h = hmac.HMAC(b"my secret key", digestmod="md5")
+            h = hmac.HMAC(b"my secret key", digestmod="sha256")
             h.update(b"compute the hash of this text!")
             dig = h.digest()
             dig = h.hexdigest()
@@ -374,11 +396,13 @@ def test_exercise_all_methods(self):
         except Exception:
             self.fail("Exception raised during normal usage of HMAC class.")
 
+
 class CopyTestCase(unittest.TestCase):
 
+    @requires_hashdigest('sha256')
     def test_attributes(self):
         # Testing if attributes are of same type.
-        h1 = hmac.HMAC(b"key", digestmod="md5")
+        h1 = hmac.HMAC(b"key", digestmod="sha256")
         h2 = h1.copy()
         self.assertTrue(h1.digest_cons == h2.digest_cons,
             "digest constructors don't match.")
@@ -387,9 +411,10 @@ def test_attributes(self):
         self.assertEqual(type(h1.outer), type(h2.outer),
             "Types of outer don't match.")
 
+    @requires_hashdigest('sha256')
     def test_realcopy(self):
         # Testing if the copy method created a real copy.
-        h1 = hmac.HMAC(b"key", digestmod="md5")
+        h1 = hmac.HMAC(b"key", digestmod="sha256")
         h2 = h1.copy()
         # Using id() in case somebody has overridden __eq__/__ne__.
         self.assertTrue(id(h1) != id(h2), "No real copy of the HMAC instance.")
@@ -398,9 +423,10 @@ def test_realcopy(self):
         self.assertTrue(id(h1.outer) != id(h2.outer),
             "No real copy of the attribute 'outer'.")
 
+    @requires_hashdigest('sha256')
     def test_equality(self):
         # Testing if the copy has the same digests.
-        h1 = hmac.HMAC(b"key", digestmod="md5")
+        h1 = hmac.HMAC(b"key", digestmod="sha256")
         h1.update(b"some random text")
         h2 = h1.copy()
         self.assertEqual(h1.digest(), h2.digest(),
diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py
index 8ab532af3f0d..03cffbe39c6c 100644
--- a/Lib/test/test_imaplib.py
+++ b/Lib/test/test_imaplib.py
@@ -10,7 +10,8 @@
 import socket
 
 from test.support import (reap_threads, verbose, transient_internet,
-                          run_with_tz, run_with_locale, cpython_only)
+                          run_with_tz, run_with_locale, cpython_only,
+                          requires_hashdigest)
 import unittest
 from unittest import mock
 from datetime import datetime, timezone, timedelta
@@ -370,6 +371,7 @@ def cmd_AUTHENTICATE(self, tag, args):
         self.assertEqual(code, 'OK')
         self.assertEqual(server.response, b'ZmFrZQ==\r\n')  # b64 encoded 'fake'
 
+    @requires_hashdigest('md5')
     def test_login_cram_md5_bytes(self):
         class AuthHandler(SimpleIMAPHandler):
             capabilities = 'LOGINDISABLED AUTH=CRAM-MD5'
@@ -387,6 +389,7 @@ def cmd_AUTHENTICATE(self, tag, args):
         ret, _ = client.login_cram_md5("tim", b"tanstaaftanstaaf")
         self.assertEqual(ret, "OK")
 
+    @requires_hashdigest('md5')
     def test_login_cram_md5_plain_text(self):
         class AuthHandler(SimpleIMAPHandler):
             capabilities = 'LOGINDISABLED AUTH=CRAM-MD5'
@@ -797,6 +800,7 @@ def cmd_AUTHENTICATE(self, tag, args):
                              b'ZmFrZQ==\r\n')  # b64 encoded 'fake'
 
     @reap_threads
+    @requires_hashdigest('md5')
     def test_login_cram_md5(self):
 
         class AuthHandler(SimpleIMAPHandler):
diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py
index 20d4eeac12d4..7b1d854d1c08 100644
--- a/Lib/test/test_poplib.py
+++ b/Lib/test/test_poplib.py
@@ -309,9 +309,11 @@ def test_noop(self):
     def test_rpop(self):
         self.assertOK(self.client.rpop('foo'))
 
+    @test_support.requires_hashdigest('md5')
     def test_apop_normal(self):
         self.assertOK(self.client.apop('foo', 'dummypassword'))
 
+    @test_support.requires_hashdigest('md5')
     def test_apop_REDOS(self):
         # Replace welcome with very long evil welcome.
         # NB The upper bound on welcome length is currently 2048.
diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py
index f1332e9ef782..d0c9862edea9 100644
--- a/Lib/test/test_smtplib.py
+++ b/Lib/test/test_smtplib.py
@@ -4,6 +4,7 @@
 from email.message import EmailMessage
 from email.base64mime import body_encode as encode_base64
 import email.utils
+import hashlib
 import hmac
 import socket
 import smtpd
@@ -21,6 +22,7 @@
 from test import support, mock_socket
 from test.support import HOST
 from test.support import threading_setup, threading_cleanup, join_thread
+from test.support import requires_hashdigest
 from unittest.mock import Mock
 
 
@@ -1009,6 +1011,7 @@ def testAUTH_LOGIN(self):
         self.assertEqual(resp, (235, b'Authentication Succeeded'))
         smtp.close()
 
+    @requires_hashdigest('md5')
     def testAUTH_CRAM_MD5(self):
         self.serv.add_feature("AUTH CRAM-MD5")
         smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15)
@@ -1025,7 +1028,13 @@ def testAUTH_multiple(self):
         smtp.close()
 
     def test_auth_function(self):
-        supported = {'CRAM-MD5', 'PLAIN', 'LOGIN'}
+        supported = {'PLAIN', 'LOGIN'}
+        try:
+            hashlib.md5()
+        except ValueError:
+            pass
+        else:
+            supported.add('CRAM-MD5')
         for mechanism in supported:
             self.serv.add_feature("AUTH {}".format(mechanism))
         for mechanism in supported:
diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py
index 7e32cbccd6c5..15324a4e4881 100644
--- a/Lib/test/test_tarfile.py
+++ b/Lib/test/test_tarfile.py
@@ -1,7 +1,7 @@
 import sys
 import os
 import io
-from hashlib import md5
+from hashlib import sha256
 from contextlib import contextmanager
 from random import Random
 import pathlib
@@ -11,7 +11,7 @@
 import tarfile
 
 from test import support
-from test.support import script_helper
+from test.support import script_helper, requires_hashdigest
 
 # Check for our compression modules.
 try:
@@ -27,8 +27,8 @@
 except ImportError:
     lzma = None
 
-def md5sum(data):
-    return md5(data).hexdigest()
+def sha256sum(data):
+    return sha256(data).hexdigest()
 
 TEMPDIR = os.path.abspath(support.TESTFN) + "-tardir"
 tarextdir = TEMPDIR + '-extract-test'
@@ -39,8 +39,12 @@ def md5sum(data):
 tmpname = os.path.join(TEMPDIR, "tmp.tar")
 dotlessname = os.path.join(TEMPDIR, "testtar")
 
-md5_regtype = "65f477c818ad9e15f7feab0c6d37742f"
-md5_sparse = "a54fbc4ca4f4399a90e1b27164012fc6"
+sha256_regtype = (
+    "e09e4bc8b3c9d9177e77256353b36c159f5f040531bbd4b024a8f9b9196c71ce"
+)
+sha256_sparse = (
+    "4f05a776071146756345ceee937b33fc5644f5a96b9780d1c7d6a32cdf164d7b"
+)
 
 
 class TarTest:
@@ -95,7 +99,7 @@ def test_fileobj_regular_file(self):
             data = fobj.read()
             self.assertEqual(len(data), tarinfo.size,
                     "regular file extraction failed")
-            self.assertEqual(md5sum(data), md5_regtype,
+            self.assertEqual(sha256sum(data), sha256_regtype,
                     "regular file extraction failed")
 
     def test_fileobj_readlines(self):
@@ -178,7 +182,7 @@ def test_fileobj_text(self):
         with self.tar.extractfile("ustar/regtype") as fobj:
             fobj = io.TextIOWrapper(fobj)
             data = fobj.read().encode("iso8859-1")
-            self.assertEqual(md5sum(data), md5_regtype)
+            self.assertEqual(sha256sum(data), sha256_regtype)
             try:
                 fobj.seek(100)
             except AttributeError:
@@ -543,13 +547,13 @@ def test_extract_hardlink(self):
             self.addCleanup(support.unlink, os.path.join(TEMPDIR, "ustar/lnktype"))
             with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f:
                 data = f.read()
-            self.assertEqual(md5sum(data), md5_regtype)
+            self.assertEqual(sha256sum(data), sha256_regtype)
 
             tar.extract("ustar/symtype", TEMPDIR)
             self.addCleanup(support.unlink, os.path.join(TEMPDIR, "ustar/symtype"))
             with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f:
                 data = f.read()
-            self.assertEqual(md5sum(data), md5_regtype)
+            self.assertEqual(sha256sum(data), sha256_regtype)
 
     def test_extractall(self):
         # Test if extractall() correctly restores directory permissions
@@ -684,7 +688,7 @@ def test_fileobj_regular_file(self):
             data = fobj.read()
         self.assertEqual(len(data), tarinfo.size,
                 "regular file extraction failed")
-        self.assertEqual(md5sum(data), md5_regtype,
+        self.assertEqual(sha256sum(data), sha256_regtype,
                 "regular file extraction failed")
 
     def test_provoke_stream_error(self):
@@ -796,8 +800,8 @@ class MemberReadTest(ReadTest, unittest.TestCase):
     def _test_member(self, tarinfo, chksum=None, **kwargs):
         if chksum is not None:
             with self.tar.extractfile(tarinfo) as f:
-                self.assertEqual(md5sum(f.read()), chksum,
-                        "wrong md5sum for %s" % tarinfo.name)
+                self.assertEqual(sha256sum(f.read()), chksum,
+                        "wrong sha256sum for %s" % tarinfo.name)
 
         kwargs["mtime"] = 0o7606136617
         kwargs["uid"] = 1000
@@ -812,11 +816,11 @@ def _test_member(self, tarinfo, chksum=None, **kwargs):
 
     def test_find_regtype(self):
         tarinfo = self.tar.getmember("ustar/regtype")
-        self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
 
     def test_find_conttype(self):
         tarinfo = self.tar.getmember("ustar/conttype")
-        self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
 
     def test_find_dirtype(self):
         tarinfo = self.tar.getmember("ustar/dirtype")
@@ -848,28 +852,28 @@ def test_find_fifotype(self):
 
     def test_find_sparse(self):
         tarinfo = self.tar.getmember("ustar/sparse")
-        self._test_member(tarinfo, size=86016, chksum=md5_sparse)
+        self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
 
     def test_find_gnusparse(self):
         tarinfo = self.tar.getmember("gnu/sparse")
-        self._test_member(tarinfo, size=86016, chksum=md5_sparse)
+        self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
 
     def test_find_gnusparse_00(self):
         tarinfo = self.tar.getmember("gnu/sparse-0.0")
-        self._test_member(tarinfo, size=86016, chksum=md5_sparse)
+        self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
 
     def test_find_gnusparse_01(self):
         tarinfo = self.tar.getmember("gnu/sparse-0.1")
-        self._test_member(tarinfo, size=86016, chksum=md5_sparse)
+        self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
 
     def test_find_gnusparse_10(self):
         tarinfo = self.tar.getmember("gnu/sparse-1.0")
-        self._test_member(tarinfo, size=86016, chksum=md5_sparse)
+        self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
 
     def test_find_umlauts(self):
         tarinfo = self.tar.getmember("ustar/umlauts-"
                                      "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
-        self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
 
     def test_find_ustar_longname(self):
         name = "ustar/" + "12345/" * 39 + "1234567/longname"
@@ -877,7 +881,7 @@ def test_find_ustar_longname(self):
 
     def test_find_regtype_oldv7(self):
         tarinfo = self.tar.getmember("misc/regtype-old-v7")
-        self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
 
     def test_find_pax_umlauts(self):
         self.tar.close()
@@ -885,7 +889,7 @@ def test_find_pax_umlauts(self):
                                 encoding="iso8859-1")
         tarinfo = self.tar.getmember("pax/umlauts-"
                                      "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
-        self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
 
 
 class LongnameTest:
@@ -947,8 +951,8 @@ def _test_sparse_file(self, name):
         filename = os.path.join(TEMPDIR, name)
         with open(filename, "rb") as fobj:
             data = fobj.read()
-        self.assertEqual(md5sum(data), md5_sparse,
-                "wrong md5sum for %s" % name)
+        self.assertEqual(sha256sum(data), sha256_sparse,
+                "wrong sha256sum for %s" % name)
 
         if self._fs_supports_holes():
             s = os.stat(filename)
@@ -2443,7 +2447,7 @@ def _test_link_extraction(self, name):
         self.tar.extract(name, TEMPDIR)
         with open(os.path.join(TEMPDIR, name), "rb") as f:
             data = f.read()
-        self.assertEqual(md5sum(data), md5_regtype)
+        self.assertEqual(sha256sum(data), sha256_regtype)
 
     # See issues #1578269, #8879, and #17689 for some history on these skips
     @unittest.skipIf(hasattr(os.path, "islink"),
diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py
index 591b48d6d4d7..1cb358f8ddc8 100644
--- a/Lib/test/test_urllib2_localnet.py
+++ b/Lib/test/test_urllib2_localnet.py
@@ -322,6 +322,7 @@ class ProxyAuthTests(unittest.TestCase):
     PASSWD = "test123"
     REALM = "TestRealm"
 
+    @support.requires_hashdigest("md5")
     def setUp(self):
         super(ProxyAuthTests, self).setUp()
         # Ignore proxy bypass settings in the environment.
diff --git a/Misc/NEWS.d/next/Tests/2019-09-25-12-18-31.bpo-38270._x-9uH.rst b/Misc/NEWS.d/next/Tests/2019-09-25-12-18-31.bpo-38270._x-9uH.rst
new file mode 100644
index 000000000000..efb1b98b36d0
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-25-12-18-31.bpo-38270._x-9uH.rst
@@ -0,0 +1,4 @@
+test.support now has a helper function to check for availibility of a
+hash digest function. Several tests are refactored avoid MD5 and use
+SHA256 instead. Other tests are marked to use MD5 and skipped when MD5 is
+disabled.


From webhook-mailer at python.org  Wed Sep 25 11:55:07 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 25 Sep 2019 15:55:07 -0000
Subject: [Python-checkins] bpo-38271: encrypt private key test files with
 AES256 (GH-16385)
Message-ID: <mailman.83.1569426909.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/bfd0c963d88f3df69489ee250655e2b8f3d235bd
commit: bfd0c963d88f3df69489ee250655e2b8f3d235bd
branch: master
author: Christian Heimes <christian at python.org>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-25T08:55:02-07:00
summary:

bpo-38271: encrypt private key test files with AES256 (GH-16385)



The private keys for test_ssl were encrypted with 3DES in traditional
PKCS#5 format. 3DES and the digest algorithm of PKCS#5 are blocked by
some strict crypto policies. Use PKCS#8 format with AES256 encryption
instead.

Signed-off-by: Christian Heimes <christian at python.org>



https://bugs.python.org/issue38271



Automerge-Triggered-By: @tiran

files:
A Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst
M Lib/test/keycert.passwd.pem
M Lib/test/make_ssl_certs.py
M Lib/test/ssl_key.passwd.pem

diff --git a/Lib/test/keycert.passwd.pem b/Lib/test/keycert.passwd.pem
index cbb3c3bccd2c..c330c36d8f9f 100644
--- a/Lib/test/keycert.passwd.pem
+++ b/Lib/test/keycert.passwd.pem
@@ -1,45 +1,45 @@
------BEGIN RSA PRIVATE KEY-----
-Proc-Type: 4,ENCRYPTED
-DEK-Info: DES-EDE3-CBC,D134E931C96D9DEC
-
-nuGFEej7vIjkYWSMz5OJeVTNntDRQi6ZM4DBm3g8T7i/0odr3WFqGMMKZcIhLYQf
-rgRq7RSKtrJ1y5taVucMV+EuCjyfzDo0TsYt+ZrXv/D08eZhjRmkhoHnGVF0TqQm
-nQEXM/ERT4J2RM78dnG+homMkI76qOqxgGbRqQqJo6AiVRcAZ45y8s96bru2TAB8
-+pWjO/v0Je7AFVdwSU52N8OOY6uoSAygW+0UY1WVxbVGJF2XfRsNpPX+YQHYl6e+
-3xM5XBVCgr6kmdAyub5qUJ38X3TpdVGoR0i+CVS9GTr2pSRib1zURAeeHnlqiUZM
-4m0Gn9s72nJevU1wxED8pwOhR8fnHEmMKGD2HPhKoOCbzDhwwBZO27TNa1uWeM3f
-M5oixKDi2PqMn3y2cDx1NjJtP661688EcJ5a2Ih9BgO9xpnhSyzBWEKcAn0tJB0H
-/56M0FW6cdOOIzMveGGL7sHW5E+iOdI1n5e7C6KJUzew78Y9qJnhS53EdI6qTz9R
-wsIsj1i070Fk6RbPo6zpLlF6w7Zj8GlZaZA7OZZv9wo5VEV/0ST8gmiiBOBc4C6Y
-u9hyLIIu4dFEBKyQHRvBnQSLNpKx6or1OGFDVBay2In9Yh2BHh1+vOj/OIz/wq48
-EHOIV27fRJxLu4jeK5LIGDhuPnMJ8AJYQ0bQOUP6fd7p+TxWkAQZPB/Dx/cs3hxr
-nFEdzx+eO+IAsObx/b1EGZyEJyETBslu4GwYX7/KK3HsJhDJ1bdZ//28jOCaoir6
-ZOMT72GRwmVoQTJ0XpccfjHfKJDRLT7C1xvzo4Eibth0hpTZkA75IUYUp6qK/PuJ
-kH/qdiC7QIkRKtsrawW4vEDna3YtxIYhQqz9+KwO6u/0gzooZtv1RU4U3ifMDB5u
-5P5GAzACRqlY8QYBkM869lvWqzQPHvybC4ak9Yx6/heMO9ddjdIW9BaK8BLxvN/6
-UCD936Y4fWltt09jHZIoxWFykouBwmd7bXooNYXmDRNmjTdVhKJuOEOQw8hDzx7e
-pWFJ9Z/V4Qm1tvXbCD7QFqMCDoY3qFvVG8DBqXpmxe1yPfz21FWrT7IuqDXAD3ns
-vxfN/2a+Cy04U9FBNVCvWqWIs5AgNpdCMJC2FlXKTy+H3/7rIjNyFyvbX0vxIXtK
-liOVNXiyVM++KZXqktqMUDlsJENmIHV9B046luqbgW018fHkyEYlL3iRZGbYegwr
-XO9VVIKVPw1BEvJ8VNdGFGuZGepd8qX2ezfYADrNR+4t85HDm8inbjTobSjWuljs
-ftUNkOeCHqAvWCFQTLCfdykvV08EJfVY79y7yFPtfRV2gxYokXFifjo3su9sVQr1
-UiIS5ZAsIC1hBXWeXoBN7QVTkFi7Yto6E1q2k10LiT3obpUUUQ/oclhrJOCJVjrS
-oRcj2QBy8OT4T9slJr5maTWdgd7Lt6+I6cGQXPaDvjGOJl0eBYM14vhx4rRQWytJ
-k07hhHFO4+9CGCuHS8AAy2gR6acYFWt2ZiiNZ0z/iPIHNK4YEyy9aLf6uZH/KQjE
-jmHToo7XD6QvCAEC5qTHby3o3LfHIhyZi/4L+AhS4FKUHF6M0peeyYt4z3HaK2d2
-N6mHLPdjwNjra7GOmcns4gzcrdfoF+R293KpPal4PjknvR3dZL4kKP/ougTAM5zv
-qDIvRbkHzjP8ChTpoLcJsNVXykNcNkjcSi0GHtIpYjh6QX6P2uvR/S4+Bbb9p9rn
-hIy/ovu9tWN2hiPxGPe6torF6BulAxsTYlDercC204AyzsrdA0pr6HBgJH9C6ML1
-TchwodbFJqn9rSv91i1liusAGoOvE81AGBdrXY7LxfSNhYY1IK6yR/POJPTd53sA
-uX2/j6Rtoksd/2BHPM6AUnI/2B9slhuzWX2aCtWLeuwvXDS6rYuTigaQmLkzTRfM
-dlMI3s9KLXxgi5YVumUZleJWXwBNP7KiKajd+VTSD+7WAhyhM5FIG5wVOaxmy4G2
-TyqZ/Ax9d2VEjTQHWvQlLPQ4Mp0EIz0aEl94K/S8CK8bJRH6+PRkar+dJi1xqlL+
-BYb42At9mEJ8odLlFikvNi1+t7jqXk5jRi5C0xFKx3nTtzoH2zNUeuA3R6vSocVK
-45jnze9IkKmxMlJ4loR5sgszdpDCD3kXqjtCcbMTmcrGyzJek3HSOTpiEORoTFOe
-Rhg6jH5lm+QcC263oipojS0qEQcnsWJP2CylNYMYHR9O/9NQxT3o2lsRHqZTMELV
-uQa/SFH+paQNbZOj8MRwPSqqiIxJFuLswKte1R+W7LKn1yBSM7Pp39lNbzGvJD2E
-YRfnCwFpJ54voVAuQ4jXJvigCW2qeCjXlxeD6K2j4eGJEEOmIjIW1wjubyBY6OI3
------END RSA PRIVATE KEY-----
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIhD+rJdxqb6ECAggA
+MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDTdyjCP3riOSUfxix4aXEvBIIH
+ECGkbsFabrcFMZcplw5jHMaOlG7rYjUzwDJ80JM8uzbv2Jb8SvNlns2+xmnEvH/M
+mNvRmnXmplbVjH3XBMK8o2Psnr2V/a0j7/pgqpRxHykG+koOY4gzdt3MAg8JPbS2
+hymSl+Y5EpciO3xLfz4aFL1ZNqspQbO/TD13Ij7DUIy7xIRBMp4taoZCrP0cEBAZ
++wgu9m23I4dh3E8RUBzWyFFNic2MVVHrui6JbHc4dIHfyKLtXJDhUcS0vIC9PvcV
+jhorh3UZC4lM+/jjXV5AhzQ0VrJ2tXAUX2dA144XHzkSH2QmwfnajPsci7BL2CGC
+rjyTy4NfB/lDwU+55dqJZQSKXMxAapJMrtgw7LD5CKQcN6zmfhXGssJ7HQUXKkaX
+I1YOFzuUD7oo56BVCnVswv0jX9RxrE5QYNreMlOP9cS+kIYH65N+PAhlURuQC14K
+PgDkHn5knSa2UQA5tc5f7zdHOZhGRUfcjLP+KAWA3nh+/2OKw/X3zuPx75YT/FKe
+tACPw5hjEpl62m9Xa0eWepZXwqkIOkzHMmCyNCsbC0mmRoEjmvfnslfsmnh4Dg/c
+4YsTYMOLLIeCa+WIc38aA5W2lNO9lW0LwLhX1rP+GRVPv+TVHXlfoyaI+jp0iXrJ
+t3xxT0gaiIR/VznyS7Py68QV/zB7VdqbsNzS7LdquHK1k8+7OYiWjY3gqyU40Iu2
+d1eSnIoDvQJwyYp7XYXbOlXNLY+s1Qb7yxcW3vXm0Bg3gKT8r1XHWJ9rj+CxAn5r
+ysfkPs1JsesxzzQjwTiDNvHnBnZnwxuxfBr26ektEHmuAXSl8V6dzLN/aaPjpTj4
+CkE7KyqX3U9bLkp+ztl4xWKEmW44nskzm0+iqrtrxMyTfvvID4QrABjZL4zmWIqc
+e3ZfA3AYk9VDIegk/YKGC5VZ8YS7ZXQ0ASK652XqJ7QlMKTxxV7zda6Fp4uW6/qN
+ezt5wgbGGhZQXj2wDQmWNQYyG/juIgYTpCUA54U5XBIjuR6pg+Ytm0UrvNjsUoAC
+wGelyqaLDq8U8jdIFYVTJy9aJjQOYXjsUJ0dZN2aGHSlju0ZGIZc49cTIVQ9BTC5
+Yc0Vlwzpl+LuA25DzKZNSb/ci0lO/cQGJ2uXQQgaNgdsHlu8nukENGJhnIzx4fzK
+wEh3yHxhTRCzPPwDfXmx0IHXrPqJhSpAgaXBVIm8OjvmMxO+W75W4uLfNY/B7e2H
+3cjklGuvkofOf7sEOrGUYf4cb6Obg8FpvHgpKo5Twwmoh/qvEKckBFqNhZXDDl88
+GbGlSEgyaAV1Ig8s1NJKBolWFa0juyPAwJ8vT1T4iwW7kQ7KXKt2UNn96K/HxkLu
+pikvukz8oRHMlfVHa0R48UB1fFHwZLzPmwkpu6ancIxk3uO3yfhf6iDk3bmnyMlz
+g3k/b6MrLYaOVByRxay85jH3Vvgqfgn6wa6BJ7xQ81eZ8B45gFuTH0J5JtLL7SH8
+darRPLCYfA+Ums9/H6pU5EXfd3yfjMIbvhCXHkJrrljkZ+th3p8dyto6wmYqIY6I
+qR9sU+o6DhRaiP8tCICuhHxQpXylUM6WeJkJwduTJ8KWIvzsj4mReIKOl/oC2jSd
+gIdKhb9Q3zj9ce4N5m6v66tyvjxGZ+xf3BvUPDD+LwZeXgf7OBsNVbXzQbzto594
+nbCzPocFi3gERE50ru4K70eQCy08TPG5NpOz+DDdO5vpAuMLYEuI7O3L+3GjW40Q
+G5bu7H5/i7o/RWR67qhG/7p9kPw3nkUtYgnvnWaPMIuTfb4c2d069kjlfgWjIbbI
+tpSKmm5DHlqTE4/ECAbIEDtSaw9dXHCdL3nh5+n428xDdGbjN4lT86tfu17EYKzl
+ydH1RJ1LX3o3TEj9UkmDPt7LnftvwybMFEcP7hM2xD4lC++wKQs7Alg6dTkBnJV4
+5xU78WRntJkJTU7kFkpPKA0QfyCuSF1fAMoukDBkqUdOj6jE0BlJQlHk5iwgnJlt
+uEdkTjHZEjIUxWC6llPcAzaPNlmnD45AgfEW+Jn21IvutmJiQAz5lm9Z9PXaR0C8
+hXB6owRY67C0YKQwXhoNf6xQun2xGBGYy5rPEEezX1S1tUH5GR/KW1Lh+FzFqHXI
+ZEb5avfDqHKehGAjPON+Br7akuQ125M9LLjKuSyPaQzeeCAy356Xd7XzVwbPddbm
+9S9WSPqzaPgh10chIHoNoC8HMd33dB5j9/Q6jrbU/oPlptu/GlorWblvJdcTuBGI
+IVn45RFnkG8hCz0GJSNzW7+70YdESQbfJW79vssWMaiSjFE0pMyFXrFR5lBywBTx
+PiGEUWtvrKG94X1TMlGUzDzDJOQNZ9dT94bonNe9pVmP5BP4/DzwwiWh6qrzWk6p
+j8OE4cfCSh2WvHnhJbH7/N0v+JKjtxeIeJ16jx/K2oK5
+-----END ENCRYPTED PRIVATE KEY-----
 -----BEGIN CERTIFICATE-----
 MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV
 BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
@@ -66,3 +66,4 @@ jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu
 9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW
 HcVKQHyOeyvnINuBAQ==
 -----END CERTIFICATE-----
+
diff --git a/Lib/test/make_ssl_certs.py b/Lib/test/make_ssl_certs.py
index 362276583f50..41b5f46c882c 100644
--- a/Lib/test/make_ssl_certs.py
+++ b/Lib/test/make_ssl_certs.py
@@ -206,8 +206,8 @@ def print_cert(path):
     with open('ssl_key.pem', 'w') as f:
         f.write(key)
     print("password protecting ssl_key.pem in ssl_key.passwd.pem")
-    check_call(['openssl','rsa','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-des3','-passout','pass:somepass'])
-    check_call(['openssl','rsa','-in','ssl_key.pem','-out','keycert.passwd.pem','-des3','-passout','pass:somepass'])
+    check_call(['openssl','pkey','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-aes256','-passout','pass:somepass'])
+    check_call(['openssl','pkey','-in','ssl_key.pem','-out','keycert.passwd.pem','-aes256','-passout','pass:somepass'])
 
     with open('keycert.pem', 'w') as f:
         f.write(key)
diff --git a/Lib/test/ssl_key.passwd.pem b/Lib/test/ssl_key.passwd.pem
index e4f1370ab270..46de61ab85b1 100644
--- a/Lib/test/ssl_key.passwd.pem
+++ b/Lib/test/ssl_key.passwd.pem
@@ -1,42 +1,42 @@
------BEGIN RSA PRIVATE KEY-----
-Proc-Type: 4,ENCRYPTED
-DEK-Info: DES-EDE3-CBC,8064BE1494B24B13
-
-KJrffOMbo8M0I3PzcYxRZGMpKD1yB3Ii4+bT5XoanxjIJ+4fdx6LfZ0Rsx+riyzs
-tymsQu/iYY9j+4rCvN9+eetsL1X6iZpiimKsLexcid9M3fb0vxED5Sgw0dvunCUA
-xhqjLIKR92MKbODHf6KrDKCpsiPbjq4gZ7P+uCGXAMHL3MXIJSC0hW9rK7Ce6oyO
-CjpIcgB8x+GUWZZZhAFdlzIHMZrteNP2P5HK6QcaT71P034Dz1hhqoj4Q0t+Fta2
-4tfsM/bnTR/l6hwlhPa1e3Uj322tDTDWBScgWANn5+sEWldLmozMaWhZsn22pfk2
-KjRMGXG024JVheV882nbdOBvG7oq+lxkZ/ZP+vvqJqnvYtf7WtM8UivzYpe5Hz5b
-kVvWzPjBLUSZ9whM9rDLqSSqMPyPvDTuEmLkuq+xm7pYJmsLqIMP2klZLqRxLX6K
-uqwplb8UG440qauxgnQ905PId1l2fJEnRtV+7vXprA0L0QotgXLVHBhLmTFM+3PH
-9H3onf31dionUAPrn3nfVE36HhvVgRyvDBnBzJSIMighgq21Qx/d1dk0DRYi1hUI
-nCHl0YJPXheVcXR7JiSF2XQCAaFuS1Mr7NCXfWZOZQC/0dkvmHnl9DUAhuqq9BNZ
-1cKhZXcKHadg2/r0Zup/oDzmHPUEfTAXT0xbqoWlhkdwbF2veWQ96A/ncx3ISTb4
-PkXBlX9rdia8nmtyQDQRn4NuvchbaGkj4WKFC8pF8Hn7naHqwjpHaDUimBc0CoQW
-edNJqruKWwtSVLuwKHCC2gZFX9AXSKJXJz/QRSUlhFGOhuF/J6yKaXj6n5lxWNiQ
-54J+OP/hz2aS95CD2+Zf1SKpxdWiLZSIQqESpmmUrXROixNJZ/Z7gI74Dd9dSJOH
-W+3AU03vrrFZVrJVZhjcINHoH1Skh6JKscH18L6x4U868nSr4SrRLX8BhHllOQyD
-bmU+PZAjF8ZBIaCtTGulDXD29F73MeAZeTSsgQjFu0iKLj1wPiphbx8i/SUtR4YP
-X6PVA04g66r1NBw+3RQASVorZ3g1MSFvITHXcbKkBDeJH2z1+c6t/VVyTONnQhM5
-lLgRSk6HCbetvT9PKxWrWutA12pdBYEHdZhMHVf2+xclky7l09w8hg2/qqcdGRGe
-oAOZ72t0l5ObNyaruDKUS6f4AjOyWq/Xj5xuFtf1n3tQHyslSyCTPcAbQhDfTHUx
-vixb/V9qvYPt7OCn8py7v1M69NH42QVFAvwveDIFjZdqfIKBoJK2V4qPoevJI6uj
-Q5ByMt8OXOjSXNpHXpYQWUiWeCwOEBXJX8rzCHdMtg37jJ0zCmeErR1NTdg+EujM
-TWYgd06jlT67tURST0aB2kg4ijKgUJefD313LW1zC6gVsTbjSZxYyRbPfSP6flQB
-yCi1C19E2OsgleqbkBVC5GlYUzaJT7SGjCRmGx1eqtbrALu+LVH24Wceexlpjydl
-+s2nf/DZlKun/tlPh6YioifPCJjByZMQOCEfIox6BkemZETz8uYA4TTWimG13Z03
-gyDGC2jdpEW414J2qcQDvrdUgJ+HlhrAAHaWpMQDbXYxBGoZ+3+ORvQV4kAsCwL8
-k3EIrVpePdik+1xgOWsyLj6QxFXlTMvL6Wc5pnArFPORsgHEolJvxSPTf9aAHNPn
-V2WBvxiLBtYpGrujAUM40Syx/aN2RPtcXYPAusHUBw+S8/p+/8Kg8GZmnIXG3F89
-45Eepl2quZYIrou7a1fwIpIIZ0hFiBQ1mlHVMFtxwVHS1bQb3SU2GeO+JcGjdVXc
-04qeGuQ5M164eQ5C0T7ZQ1ULiUlFWKD30m+cjqmZzt3d7Q0mKpMKuESIuZJo/wpD
-Nas432aLKUhcNx/pOYLkKJRpGZKOupQoD5iUj/j44o8JoFkDK33v2S57XB5QGz28
-9Zuhx49b3W8mbM6EBanlQKLWJGCxXqc/jhYhFWn+b0MhidynFgA0oeWvf6ZDyt6H
-Yi5Etxsar09xp0Do3NxtQXLuSUu0ji2pQzSIKuoqQWKqldm6VrpwojiqJhy4WQBQ
-aVVyFeWBC7G3Zj76dO+yp2sfJ0itJUQ8AIB9Cg0f34rEZu+r9luPmqBoUeL95Tk7
-YvCOU3Jl8Iqysv8aNpVXT8sa8rrSbruWCByEePZ37RIdHLMVBwVY0eVaFQjrjU7E
-mXmM9eaoYLfXOllsQ+M2+qPFUITr/GU3Qig13DhK/+yC1R6V2a0l0WRhMltIPYKW
-Ztvvr4hK5LcYCeS113BLiMbDIMMZZYGDZGMdC8DnnVbT2loF0Rfmp80Af31KmMQ4
-6XvMatW9UDjBoY5a/YMpdm7SRwm+MgV2KNPpc2kST87/yi9oprGAb8qiarHiHTM0
------END RSA PRIVATE KEY-----
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQI072N7W+PDDMCAggA
+MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBA/AuaRNi4vE4KGqI4In+70BIIH
+ENGS5Vex5NID873frmd1UZEHZ+O/Bd0wDb+NUpIqesHkRYf7kKi6Gnr+nKQ/oVVn
+Lm3JjE7c8ECP0OkOOXmiXuWL1SkzBBWqCI4stSGUPvBiHsGwNnvJAaGjUffgMlcC
+aJOA2+dnejLkzblq4CB2LQdm06N3Xoe9tyqtQaUHxfzJAf5Ydd8uj7vpKN2MMhY7
+icIPJwSyh0N7S6XWVtHEokr9Kp4y2hS5a+BgCWV1/1z0aF7agnSVndmT1VR+nWmc
+lM14k+lethmHMB+fsNSjnqeJ7XOPlOTHqhiZ9bBSTgF/xr5Bck/NiKRzHjdovBox
+TKg+xchaBhpRh7wBPBIlNJeHmIjv+8obOKjKU98Ig/7R9+IryZaNcKAH0PuOT+Sw
+QHXiCGQbOiYHB9UyhDTWiB7YVjd8KHefOFxfHzOQb/iBhbv1x3bTl3DgepvRN6VO
+dIsPLoIZe42sdf9GeMsk8mGJyZUQ6AzsfhWk3grb/XscizPSvrNsJ2VL1R7YTyT3
+3WA4ZXR1EqvXnWL7N/raemQjy62iOG6t7fcF5IdP9CMbWP+Plpsz4cQW7FtesCTq
+a5ZXraochQz361ODFNIeBEGU+0qqXUtZDlmos/EySkZykSeU/L0bImS62VGE3afo
+YXBmznTTT9kkFkqv7H0MerfJsrE/wF8puP3GM01DW2JRgXRpSWlvbPV/2LnMtRuD
+II7iH4rWDtTjCN6BWKAgDOnPkc9sZ4XulqT32lcUeV6LTdMBfq8kMEc8eDij1vUT
+maVCRpuwaq8EIT3lVgNLufHiG96ojlyYtj3orzw22IjkgC/9ee8UDik9CqbMVmFf
+fVHhsw8LNSg8Q4bmwm5Eg2w2it2gtI68+mwr75oCxuJ/8OMjW21Prj8XDh5reie2
+c0lDKQOFZ9UnLU1bXR/6qUM+JFKR4DMq+fOCuoQSVoyVUEOsJpvBOYnYZN9cxsZm
+vh9dKafMEcKZ8flsbr+gOmOw7+Py2ifSlf25E/Frb1W4gtbTb0LQVHb6+drutrZj
+8HEu4CnHYFCD4ZnOJb26XlZCb8GFBddW86yJYyUqMMV6Q1aJfAOAglsTo1LjIMOZ
+byo0BTAmwUevU/iuOXQ4qRBXXcoidDcTCrxfUSPG9wdt9l+m5SdQpWqfQ+fx5O7m
+SLlrHyZCiPSFMtC9DxqjIklHjf5W3wslGLgaD30YXa4VDYkRihf3CNsxGQ+tVvef
+l0ZjoAitF7Gaua06IESmKnpHe23dkr1cjYq+u2IV+xGH8LeExdwsQ9kpuTeXPnQs
+JOA99SsFx1ct32RrwjxnDDsiNkaViTKo9GDkV3jQTfoFgAVqfSgg9wGXpqUqhNG7
+TiSIHCowllLny2zn4XrXCy2niD3VDt0skb3l/PaegHE2z7S5YY85nQtYwpLiwB9M
+SQ08DYKxPBZYKtS2iZ/fsA1gjSRQDPg/SIxMhUC3M3qH8iWny1Lzl25F2Uq7VVEX
+LdTUtaby49jRTT3CQGr5n6z7bMbUegiY7h8WmOekuThGDH+4xZp6+rDP4GFk4FeK
+JcF70vMQYIjQZhadic6olv+9VtUP42ltGG/yP9a3eWRkzfAf2eCh6B1rYdgEWwE8
+rlcZzwM+y6eUmeNF2FVWB8iWtTMQHy+dYNPM+Jtus1KQKxiiq/yCRs7nWvzWRFWA
+HRyqV0J6/lqgm4FvfktFt1T0W+mDoLJOR2/zIwMy2lgL5zeHuR3SaMJnCikJbqKS
+HB3UvrhAWUcZqdH29+FhVWeM7ybyF1Wccmf+IIC/ePLa6gjtqPV8lG/5kbpcpnB6
+UQY8WWaKMxyr3jJ9bAX5QKshchp04cDecOLZrpFGNNQngR8RxSEkiIgAqNxWunIu
+KrdBDrupv/XAgEOclmgToY3iywLJSV5gHAyHWDUhRH4cFCLiGPl4XIcnXOuTze3H
+3j+EYSiS3v3DhHjp33YU2pXlJDjiYsKzAXejEh66++Y8qaQdCAad3ruWRCzW3kgk
+Md0A1VGzntTnQsewvExQEMZH2LtYIsPv3KCYGeSAuLabX4tbGk79PswjnjLLEOr0
+Ghf6RF6qf5/iFyJoG4vrbKT8kx6ywh0InILCdjUunuDskIBxX6tEcr9XwajoIvb2
+kcmGdjam5kKLS7QOWQTl8/r/cuFes0dj34cX5Qpq+Gd7tRq/D+b0207926Cxvftv
+qQ1cVn8HiLxKkZzd3tpf2xnoV1zkTL0oHrNg+qzxoxXUTUcwtIf1d/HRbYEAhi/d
+bBBoFeftEHWNq+sJgS9bH+XNzo/yK4u04B5miOq8v4CSkJdzu+ZdF22d4cjiGmtQ
+8BTmcn0Unzm+u5H0+QSZe54QBHJGNXXOIKMTkgnOdW27g4DbI1y7fCqJiSMbRW6L
+oHmMfbdB3GWqGbsUkhY8i6h9op0MU6WOX7ea2Rxyt4t6
+-----END ENCRYPTED PRIVATE KEY-----
diff --git a/Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst b/Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst
new file mode 100644
index 000000000000..8f43d321d371
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst
@@ -0,0 +1,4 @@
+The private keys for test_ssl were encrypted with 3DES in traditional
+PKCS#5 format. 3DES and the digest algorithm of PKCS#5 are blocked by
+some strict crypto policies. Use PKCS#8 format with AES256 encryption
+instead.


From webhook-mailer at python.org  Wed Sep 25 11:56:09 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 25 Sep 2019 15:56:09 -0000
Subject: [Python-checkins] bpo-38005: Remove support of string argument in
 InterpreterID(). (GH-16227)
Message-ID: <mailman.84.1569426970.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/ca14f047b068cd0fb81ef3d5e7bea19fe8af1b58
commit: ca14f047b068cd0fb81ef3d5e7bea19fe8af1b58
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T08:56:05-07:00
summary:

bpo-38005: Remove support of string argument in InterpreterID(). (GH-16227)


Make negative interpreter id to raise ValueError instead of RuntimeError.
(cherry picked from commit 543a3951a1c96bae0ea839eacec71d3b1a563a10)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
M Lib/test/test__xxsubinterpreters.py
M Objects/interpreteridobject.c

diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py
index f5474f4101d0..207b5db5d8fb 100644
--- a/Lib/test/test__xxsubinterpreters.py
+++ b/Lib/test/test__xxsubinterpreters.py
@@ -514,7 +514,7 @@ def test_does_not_exist(self):
             interpreters.is_running(1_000_000)
 
     def test_bad_id(self):
-        with self.assertRaises(RuntimeError):
+        with self.assertRaises(ValueError):
             interpreters.is_running(-1)
 
 
@@ -530,18 +530,15 @@ class Int(str):
             def __index__(self):
                 return 10
 
-        for id in ('10', '1_0', Int()):
-            with self.subTest(id=id):
-                id = interpreters.InterpreterID(id, force=True)
-                self.assertEqual(int(id), 10)
+        id = interpreters.InterpreterID(Int(), force=True)
+        self.assertEqual(int(id), 10)
 
     def test_bad_id(self):
         self.assertRaises(TypeError, interpreters.InterpreterID, object())
         self.assertRaises(TypeError, interpreters.InterpreterID, 10.0)
+        self.assertRaises(TypeError, interpreters.InterpreterID, '10')
         self.assertRaises(TypeError, interpreters.InterpreterID, b'10')
         self.assertRaises(ValueError, interpreters.InterpreterID, -1)
-        self.assertRaises(ValueError, interpreters.InterpreterID, '-1')
-        self.assertRaises(ValueError, interpreters.InterpreterID, 'spam')
         self.assertRaises(OverflowError, interpreters.InterpreterID, 2**64)
 
     def test_does_not_exist(self):
@@ -720,7 +717,7 @@ def test_does_not_exist(self):
             interpreters.destroy(1_000_000)
 
     def test_bad_id(self):
-        with self.assertRaises(RuntimeError):
+        with self.assertRaises(ValueError):
             interpreters.destroy(-1)
 
     def test_from_current(self):
@@ -863,7 +860,7 @@ def test_does_not_exist(self):
             interpreters.run_string(id, 'print("spam")')
 
     def test_error_id(self):
-        with self.assertRaises(RuntimeError):
+        with self.assertRaises(ValueError):
             interpreters.run_string(-1, 'print("spam")')
 
     def test_bad_id(self):
diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c
index 3edbb85e6ac3..94f5dd709bbd 100644
--- a/Objects/interpreteridobject.c
+++ b/Objects/interpreteridobject.c
@@ -35,52 +35,45 @@ newinterpid(PyTypeObject *cls, int64_t id, int force)
     return self;
 }
 
-static PyObject *
-interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
+static int
+interp_id_converter(PyObject *arg, void *ptr)
 {
-    static char *kwlist[] = {"id", "force", NULL};
-    PyObject *idobj;
-    int force = 0;
-    if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O|$p:InterpreterID.__init__", kwlist,
-                                     &idobj, &force)) {
-        return NULL;
-    }
-
-    // Coerce and check the ID.
     int64_t id;
-    if (PyObject_TypeCheck(idobj, &_PyInterpreterID_Type)) {
-        id = ((interpid *)idobj)->id;
+    if (PyObject_TypeCheck(arg, &_PyInterpreterID_Type)) {
+        id = ((interpid *)arg)->id;
     }
-    else {
-        PyObject *pyid;
-        if (PyIndex_Check(idobj)) {
-            pyid = idobj;
-            Py_INCREF(pyid);
-        }
-        else if (PyUnicode_Check(idobj)) {
-            pyid = PyNumber_Long(idobj);
-            if (pyid == NULL) {
-                return NULL;
-            }
-        }
-        else {
-            PyErr_Format(PyExc_TypeError,
-                         "interpreter ID must be an int, got %.100s",
-                         idobj->ob_type->tp_name);
-            return NULL;
-        }
-        id = PyLong_AsLongLong(pyid);
-        Py_DECREF(pyid);
+    else if (PyIndex_Check(arg)) {
+        id = PyLong_AsLongLong(arg);
         if (id == -1 && PyErr_Occurred()) {
-            return NULL;
+            return 0;
         }
         if (id < 0) {
             PyErr_Format(PyExc_ValueError,
-                         "interpreter ID must be a non-negative int, got %R", idobj);
-            return NULL;
+                         "interpreter ID must be a non-negative int, got %R", arg);
+            return 0;
         }
     }
+    else {
+        PyErr_Format(PyExc_TypeError,
+                     "interpreter ID must be an int, got %.100s",
+                     arg->ob_type->tp_name);
+        return 0;
+    }
+    *(int64_t *)ptr = id;
+    return 1;
+}
+
+static PyObject *
+interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
+{
+    static char *kwlist[] = {"id", "force", NULL};
+    int64_t id;
+    int force = 0;
+    if (!PyArg_ParseTupleAndKeywords(args, kwds,
+                                     "O&|$p:InterpreterID.__init__", kwlist,
+                                     interp_id_converter, &id, &force)) {
+        return NULL;
+    }
 
     return (PyObject *)newinterpid(cls, id, force);
 }
@@ -287,19 +280,7 @@ PyInterpreterState *
 _PyInterpreterID_LookUp(PyObject *requested_id)
 {
     int64_t id;
-    if (PyObject_TypeCheck(requested_id, &_PyInterpreterID_Type)) {
-        id = ((interpid *)requested_id)->id;
-    }
-    else if (PyIndex_Check(requested_id)) {
-        id = PyLong_AsLongLong(requested_id);
-        if (id == -1 && PyErr_Occurred() != NULL) {
-            return NULL;
-        }
-        assert(id <= INT64_MAX);
-    }
-    else {
-        PyErr_Format(PyExc_TypeError, "interpreter ID must be an int, got %.100s",
-                     requested_id->ob_type->tp_name);
+    if (!interp_id_converter(requested_id, &id)) {
         return NULL;
     }
     return _PyInterpreterState_LookUpID(id);


From webhook-mailer at python.org  Wed Sep 25 12:13:10 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 25 Sep 2019 16:13:10 -0000
Subject: [Python-checkins] bpo-38271: encrypt private key test files with
 AES256 (GH-16385)
Message-ID: <mailman.85.1569427991.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/585798fb0eac38f32b54f1b8a86e418d76147d99
commit: 585798fb0eac38f32b54f1b8a86e418d76147d99
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T09:12:59-07:00
summary:

bpo-38271: encrypt private key test files with AES256 (GH-16385)


The private keys for test_ssl were encrypted with 3DES in traditional
PKCSGH-5 format. 3DES and the digest algorithm of PKCSGH-5 are blocked by
some strict crypto policies. Use PKCSGH-8 format with AES256 encryption
instead.

Signed-off-by: Christian Heimes <christian at python.org>

https://bugs.python.org/issue38271

Automerge-Triggered-By: @tiran
(cherry picked from commit bfd0c963d88f3df69489ee250655e2b8f3d235bd)

Co-authored-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst
M Lib/test/keycert.passwd.pem
M Lib/test/make_ssl_certs.py
M Lib/test/ssl_key.passwd.pem

diff --git a/Lib/test/keycert.passwd.pem b/Lib/test/keycert.passwd.pem
index cbb3c3bccd2c..c330c36d8f9f 100644
--- a/Lib/test/keycert.passwd.pem
+++ b/Lib/test/keycert.passwd.pem
@@ -1,45 +1,45 @@
------BEGIN RSA PRIVATE KEY-----
-Proc-Type: 4,ENCRYPTED
-DEK-Info: DES-EDE3-CBC,D134E931C96D9DEC
-
-nuGFEej7vIjkYWSMz5OJeVTNntDRQi6ZM4DBm3g8T7i/0odr3WFqGMMKZcIhLYQf
-rgRq7RSKtrJ1y5taVucMV+EuCjyfzDo0TsYt+ZrXv/D08eZhjRmkhoHnGVF0TqQm
-nQEXM/ERT4J2RM78dnG+homMkI76qOqxgGbRqQqJo6AiVRcAZ45y8s96bru2TAB8
-+pWjO/v0Je7AFVdwSU52N8OOY6uoSAygW+0UY1WVxbVGJF2XfRsNpPX+YQHYl6e+
-3xM5XBVCgr6kmdAyub5qUJ38X3TpdVGoR0i+CVS9GTr2pSRib1zURAeeHnlqiUZM
-4m0Gn9s72nJevU1wxED8pwOhR8fnHEmMKGD2HPhKoOCbzDhwwBZO27TNa1uWeM3f
-M5oixKDi2PqMn3y2cDx1NjJtP661688EcJ5a2Ih9BgO9xpnhSyzBWEKcAn0tJB0H
-/56M0FW6cdOOIzMveGGL7sHW5E+iOdI1n5e7C6KJUzew78Y9qJnhS53EdI6qTz9R
-wsIsj1i070Fk6RbPo6zpLlF6w7Zj8GlZaZA7OZZv9wo5VEV/0ST8gmiiBOBc4C6Y
-u9hyLIIu4dFEBKyQHRvBnQSLNpKx6or1OGFDVBay2In9Yh2BHh1+vOj/OIz/wq48
-EHOIV27fRJxLu4jeK5LIGDhuPnMJ8AJYQ0bQOUP6fd7p+TxWkAQZPB/Dx/cs3hxr
-nFEdzx+eO+IAsObx/b1EGZyEJyETBslu4GwYX7/KK3HsJhDJ1bdZ//28jOCaoir6
-ZOMT72GRwmVoQTJ0XpccfjHfKJDRLT7C1xvzo4Eibth0hpTZkA75IUYUp6qK/PuJ
-kH/qdiC7QIkRKtsrawW4vEDna3YtxIYhQqz9+KwO6u/0gzooZtv1RU4U3ifMDB5u
-5P5GAzACRqlY8QYBkM869lvWqzQPHvybC4ak9Yx6/heMO9ddjdIW9BaK8BLxvN/6
-UCD936Y4fWltt09jHZIoxWFykouBwmd7bXooNYXmDRNmjTdVhKJuOEOQw8hDzx7e
-pWFJ9Z/V4Qm1tvXbCD7QFqMCDoY3qFvVG8DBqXpmxe1yPfz21FWrT7IuqDXAD3ns
-vxfN/2a+Cy04U9FBNVCvWqWIs5AgNpdCMJC2FlXKTy+H3/7rIjNyFyvbX0vxIXtK
-liOVNXiyVM++KZXqktqMUDlsJENmIHV9B046luqbgW018fHkyEYlL3iRZGbYegwr
-XO9VVIKVPw1BEvJ8VNdGFGuZGepd8qX2ezfYADrNR+4t85HDm8inbjTobSjWuljs
-ftUNkOeCHqAvWCFQTLCfdykvV08EJfVY79y7yFPtfRV2gxYokXFifjo3su9sVQr1
-UiIS5ZAsIC1hBXWeXoBN7QVTkFi7Yto6E1q2k10LiT3obpUUUQ/oclhrJOCJVjrS
-oRcj2QBy8OT4T9slJr5maTWdgd7Lt6+I6cGQXPaDvjGOJl0eBYM14vhx4rRQWytJ
-k07hhHFO4+9CGCuHS8AAy2gR6acYFWt2ZiiNZ0z/iPIHNK4YEyy9aLf6uZH/KQjE
-jmHToo7XD6QvCAEC5qTHby3o3LfHIhyZi/4L+AhS4FKUHF6M0peeyYt4z3HaK2d2
-N6mHLPdjwNjra7GOmcns4gzcrdfoF+R293KpPal4PjknvR3dZL4kKP/ougTAM5zv
-qDIvRbkHzjP8ChTpoLcJsNVXykNcNkjcSi0GHtIpYjh6QX6P2uvR/S4+Bbb9p9rn
-hIy/ovu9tWN2hiPxGPe6torF6BulAxsTYlDercC204AyzsrdA0pr6HBgJH9C6ML1
-TchwodbFJqn9rSv91i1liusAGoOvE81AGBdrXY7LxfSNhYY1IK6yR/POJPTd53sA
-uX2/j6Rtoksd/2BHPM6AUnI/2B9slhuzWX2aCtWLeuwvXDS6rYuTigaQmLkzTRfM
-dlMI3s9KLXxgi5YVumUZleJWXwBNP7KiKajd+VTSD+7WAhyhM5FIG5wVOaxmy4G2
-TyqZ/Ax9d2VEjTQHWvQlLPQ4Mp0EIz0aEl94K/S8CK8bJRH6+PRkar+dJi1xqlL+
-BYb42At9mEJ8odLlFikvNi1+t7jqXk5jRi5C0xFKx3nTtzoH2zNUeuA3R6vSocVK
-45jnze9IkKmxMlJ4loR5sgszdpDCD3kXqjtCcbMTmcrGyzJek3HSOTpiEORoTFOe
-Rhg6jH5lm+QcC263oipojS0qEQcnsWJP2CylNYMYHR9O/9NQxT3o2lsRHqZTMELV
-uQa/SFH+paQNbZOj8MRwPSqqiIxJFuLswKte1R+W7LKn1yBSM7Pp39lNbzGvJD2E
-YRfnCwFpJ54voVAuQ4jXJvigCW2qeCjXlxeD6K2j4eGJEEOmIjIW1wjubyBY6OI3
------END RSA PRIVATE KEY-----
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIhD+rJdxqb6ECAggA
+MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDTdyjCP3riOSUfxix4aXEvBIIH
+ECGkbsFabrcFMZcplw5jHMaOlG7rYjUzwDJ80JM8uzbv2Jb8SvNlns2+xmnEvH/M
+mNvRmnXmplbVjH3XBMK8o2Psnr2V/a0j7/pgqpRxHykG+koOY4gzdt3MAg8JPbS2
+hymSl+Y5EpciO3xLfz4aFL1ZNqspQbO/TD13Ij7DUIy7xIRBMp4taoZCrP0cEBAZ
++wgu9m23I4dh3E8RUBzWyFFNic2MVVHrui6JbHc4dIHfyKLtXJDhUcS0vIC9PvcV
+jhorh3UZC4lM+/jjXV5AhzQ0VrJ2tXAUX2dA144XHzkSH2QmwfnajPsci7BL2CGC
+rjyTy4NfB/lDwU+55dqJZQSKXMxAapJMrtgw7LD5CKQcN6zmfhXGssJ7HQUXKkaX
+I1YOFzuUD7oo56BVCnVswv0jX9RxrE5QYNreMlOP9cS+kIYH65N+PAhlURuQC14K
+PgDkHn5knSa2UQA5tc5f7zdHOZhGRUfcjLP+KAWA3nh+/2OKw/X3zuPx75YT/FKe
+tACPw5hjEpl62m9Xa0eWepZXwqkIOkzHMmCyNCsbC0mmRoEjmvfnslfsmnh4Dg/c
+4YsTYMOLLIeCa+WIc38aA5W2lNO9lW0LwLhX1rP+GRVPv+TVHXlfoyaI+jp0iXrJ
+t3xxT0gaiIR/VznyS7Py68QV/zB7VdqbsNzS7LdquHK1k8+7OYiWjY3gqyU40Iu2
+d1eSnIoDvQJwyYp7XYXbOlXNLY+s1Qb7yxcW3vXm0Bg3gKT8r1XHWJ9rj+CxAn5r
+ysfkPs1JsesxzzQjwTiDNvHnBnZnwxuxfBr26ektEHmuAXSl8V6dzLN/aaPjpTj4
+CkE7KyqX3U9bLkp+ztl4xWKEmW44nskzm0+iqrtrxMyTfvvID4QrABjZL4zmWIqc
+e3ZfA3AYk9VDIegk/YKGC5VZ8YS7ZXQ0ASK652XqJ7QlMKTxxV7zda6Fp4uW6/qN
+ezt5wgbGGhZQXj2wDQmWNQYyG/juIgYTpCUA54U5XBIjuR6pg+Ytm0UrvNjsUoAC
+wGelyqaLDq8U8jdIFYVTJy9aJjQOYXjsUJ0dZN2aGHSlju0ZGIZc49cTIVQ9BTC5
+Yc0Vlwzpl+LuA25DzKZNSb/ci0lO/cQGJ2uXQQgaNgdsHlu8nukENGJhnIzx4fzK
+wEh3yHxhTRCzPPwDfXmx0IHXrPqJhSpAgaXBVIm8OjvmMxO+W75W4uLfNY/B7e2H
+3cjklGuvkofOf7sEOrGUYf4cb6Obg8FpvHgpKo5Twwmoh/qvEKckBFqNhZXDDl88
+GbGlSEgyaAV1Ig8s1NJKBolWFa0juyPAwJ8vT1T4iwW7kQ7KXKt2UNn96K/HxkLu
+pikvukz8oRHMlfVHa0R48UB1fFHwZLzPmwkpu6ancIxk3uO3yfhf6iDk3bmnyMlz
+g3k/b6MrLYaOVByRxay85jH3Vvgqfgn6wa6BJ7xQ81eZ8B45gFuTH0J5JtLL7SH8
+darRPLCYfA+Ums9/H6pU5EXfd3yfjMIbvhCXHkJrrljkZ+th3p8dyto6wmYqIY6I
+qR9sU+o6DhRaiP8tCICuhHxQpXylUM6WeJkJwduTJ8KWIvzsj4mReIKOl/oC2jSd
+gIdKhb9Q3zj9ce4N5m6v66tyvjxGZ+xf3BvUPDD+LwZeXgf7OBsNVbXzQbzto594
+nbCzPocFi3gERE50ru4K70eQCy08TPG5NpOz+DDdO5vpAuMLYEuI7O3L+3GjW40Q
+G5bu7H5/i7o/RWR67qhG/7p9kPw3nkUtYgnvnWaPMIuTfb4c2d069kjlfgWjIbbI
+tpSKmm5DHlqTE4/ECAbIEDtSaw9dXHCdL3nh5+n428xDdGbjN4lT86tfu17EYKzl
+ydH1RJ1LX3o3TEj9UkmDPt7LnftvwybMFEcP7hM2xD4lC++wKQs7Alg6dTkBnJV4
+5xU78WRntJkJTU7kFkpPKA0QfyCuSF1fAMoukDBkqUdOj6jE0BlJQlHk5iwgnJlt
+uEdkTjHZEjIUxWC6llPcAzaPNlmnD45AgfEW+Jn21IvutmJiQAz5lm9Z9PXaR0C8
+hXB6owRY67C0YKQwXhoNf6xQun2xGBGYy5rPEEezX1S1tUH5GR/KW1Lh+FzFqHXI
+ZEb5avfDqHKehGAjPON+Br7akuQ125M9LLjKuSyPaQzeeCAy356Xd7XzVwbPddbm
+9S9WSPqzaPgh10chIHoNoC8HMd33dB5j9/Q6jrbU/oPlptu/GlorWblvJdcTuBGI
+IVn45RFnkG8hCz0GJSNzW7+70YdESQbfJW79vssWMaiSjFE0pMyFXrFR5lBywBTx
+PiGEUWtvrKG94X1TMlGUzDzDJOQNZ9dT94bonNe9pVmP5BP4/DzwwiWh6qrzWk6p
+j8OE4cfCSh2WvHnhJbH7/N0v+JKjtxeIeJ16jx/K2oK5
+-----END ENCRYPTED PRIVATE KEY-----
 -----BEGIN CERTIFICATE-----
 MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV
 BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
@@ -66,3 +66,4 @@ jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu
 9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW
 HcVKQHyOeyvnINuBAQ==
 -----END CERTIFICATE-----
+
diff --git a/Lib/test/make_ssl_certs.py b/Lib/test/make_ssl_certs.py
index 362276583f50..41b5f46c882c 100644
--- a/Lib/test/make_ssl_certs.py
+++ b/Lib/test/make_ssl_certs.py
@@ -206,8 +206,8 @@ def print_cert(path):
     with open('ssl_key.pem', 'w') as f:
         f.write(key)
     print("password protecting ssl_key.pem in ssl_key.passwd.pem")
-    check_call(['openssl','rsa','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-des3','-passout','pass:somepass'])
-    check_call(['openssl','rsa','-in','ssl_key.pem','-out','keycert.passwd.pem','-des3','-passout','pass:somepass'])
+    check_call(['openssl','pkey','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-aes256','-passout','pass:somepass'])
+    check_call(['openssl','pkey','-in','ssl_key.pem','-out','keycert.passwd.pem','-aes256','-passout','pass:somepass'])
 
     with open('keycert.pem', 'w') as f:
         f.write(key)
diff --git a/Lib/test/ssl_key.passwd.pem b/Lib/test/ssl_key.passwd.pem
index e4f1370ab270..46de61ab85b1 100644
--- a/Lib/test/ssl_key.passwd.pem
+++ b/Lib/test/ssl_key.passwd.pem
@@ -1,42 +1,42 @@
------BEGIN RSA PRIVATE KEY-----
-Proc-Type: 4,ENCRYPTED
-DEK-Info: DES-EDE3-CBC,8064BE1494B24B13
-
-KJrffOMbo8M0I3PzcYxRZGMpKD1yB3Ii4+bT5XoanxjIJ+4fdx6LfZ0Rsx+riyzs
-tymsQu/iYY9j+4rCvN9+eetsL1X6iZpiimKsLexcid9M3fb0vxED5Sgw0dvunCUA
-xhqjLIKR92MKbODHf6KrDKCpsiPbjq4gZ7P+uCGXAMHL3MXIJSC0hW9rK7Ce6oyO
-CjpIcgB8x+GUWZZZhAFdlzIHMZrteNP2P5HK6QcaT71P034Dz1hhqoj4Q0t+Fta2
-4tfsM/bnTR/l6hwlhPa1e3Uj322tDTDWBScgWANn5+sEWldLmozMaWhZsn22pfk2
-KjRMGXG024JVheV882nbdOBvG7oq+lxkZ/ZP+vvqJqnvYtf7WtM8UivzYpe5Hz5b
-kVvWzPjBLUSZ9whM9rDLqSSqMPyPvDTuEmLkuq+xm7pYJmsLqIMP2klZLqRxLX6K
-uqwplb8UG440qauxgnQ905PId1l2fJEnRtV+7vXprA0L0QotgXLVHBhLmTFM+3PH
-9H3onf31dionUAPrn3nfVE36HhvVgRyvDBnBzJSIMighgq21Qx/d1dk0DRYi1hUI
-nCHl0YJPXheVcXR7JiSF2XQCAaFuS1Mr7NCXfWZOZQC/0dkvmHnl9DUAhuqq9BNZ
-1cKhZXcKHadg2/r0Zup/oDzmHPUEfTAXT0xbqoWlhkdwbF2veWQ96A/ncx3ISTb4
-PkXBlX9rdia8nmtyQDQRn4NuvchbaGkj4WKFC8pF8Hn7naHqwjpHaDUimBc0CoQW
-edNJqruKWwtSVLuwKHCC2gZFX9AXSKJXJz/QRSUlhFGOhuF/J6yKaXj6n5lxWNiQ
-54J+OP/hz2aS95CD2+Zf1SKpxdWiLZSIQqESpmmUrXROixNJZ/Z7gI74Dd9dSJOH
-W+3AU03vrrFZVrJVZhjcINHoH1Skh6JKscH18L6x4U868nSr4SrRLX8BhHllOQyD
-bmU+PZAjF8ZBIaCtTGulDXD29F73MeAZeTSsgQjFu0iKLj1wPiphbx8i/SUtR4YP
-X6PVA04g66r1NBw+3RQASVorZ3g1MSFvITHXcbKkBDeJH2z1+c6t/VVyTONnQhM5
-lLgRSk6HCbetvT9PKxWrWutA12pdBYEHdZhMHVf2+xclky7l09w8hg2/qqcdGRGe
-oAOZ72t0l5ObNyaruDKUS6f4AjOyWq/Xj5xuFtf1n3tQHyslSyCTPcAbQhDfTHUx
-vixb/V9qvYPt7OCn8py7v1M69NH42QVFAvwveDIFjZdqfIKBoJK2V4qPoevJI6uj
-Q5ByMt8OXOjSXNpHXpYQWUiWeCwOEBXJX8rzCHdMtg37jJ0zCmeErR1NTdg+EujM
-TWYgd06jlT67tURST0aB2kg4ijKgUJefD313LW1zC6gVsTbjSZxYyRbPfSP6flQB
-yCi1C19E2OsgleqbkBVC5GlYUzaJT7SGjCRmGx1eqtbrALu+LVH24Wceexlpjydl
-+s2nf/DZlKun/tlPh6YioifPCJjByZMQOCEfIox6BkemZETz8uYA4TTWimG13Z03
-gyDGC2jdpEW414J2qcQDvrdUgJ+HlhrAAHaWpMQDbXYxBGoZ+3+ORvQV4kAsCwL8
-k3EIrVpePdik+1xgOWsyLj6QxFXlTMvL6Wc5pnArFPORsgHEolJvxSPTf9aAHNPn
-V2WBvxiLBtYpGrujAUM40Syx/aN2RPtcXYPAusHUBw+S8/p+/8Kg8GZmnIXG3F89
-45Eepl2quZYIrou7a1fwIpIIZ0hFiBQ1mlHVMFtxwVHS1bQb3SU2GeO+JcGjdVXc
-04qeGuQ5M164eQ5C0T7ZQ1ULiUlFWKD30m+cjqmZzt3d7Q0mKpMKuESIuZJo/wpD
-Nas432aLKUhcNx/pOYLkKJRpGZKOupQoD5iUj/j44o8JoFkDK33v2S57XB5QGz28
-9Zuhx49b3W8mbM6EBanlQKLWJGCxXqc/jhYhFWn+b0MhidynFgA0oeWvf6ZDyt6H
-Yi5Etxsar09xp0Do3NxtQXLuSUu0ji2pQzSIKuoqQWKqldm6VrpwojiqJhy4WQBQ
-aVVyFeWBC7G3Zj76dO+yp2sfJ0itJUQ8AIB9Cg0f34rEZu+r9luPmqBoUeL95Tk7
-YvCOU3Jl8Iqysv8aNpVXT8sa8rrSbruWCByEePZ37RIdHLMVBwVY0eVaFQjrjU7E
-mXmM9eaoYLfXOllsQ+M2+qPFUITr/GU3Qig13DhK/+yC1R6V2a0l0WRhMltIPYKW
-Ztvvr4hK5LcYCeS113BLiMbDIMMZZYGDZGMdC8DnnVbT2loF0Rfmp80Af31KmMQ4
-6XvMatW9UDjBoY5a/YMpdm7SRwm+MgV2KNPpc2kST87/yi9oprGAb8qiarHiHTM0
------END RSA PRIVATE KEY-----
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQI072N7W+PDDMCAggA
+MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBA/AuaRNi4vE4KGqI4In+70BIIH
+ENGS5Vex5NID873frmd1UZEHZ+O/Bd0wDb+NUpIqesHkRYf7kKi6Gnr+nKQ/oVVn
+Lm3JjE7c8ECP0OkOOXmiXuWL1SkzBBWqCI4stSGUPvBiHsGwNnvJAaGjUffgMlcC
+aJOA2+dnejLkzblq4CB2LQdm06N3Xoe9tyqtQaUHxfzJAf5Ydd8uj7vpKN2MMhY7
+icIPJwSyh0N7S6XWVtHEokr9Kp4y2hS5a+BgCWV1/1z0aF7agnSVndmT1VR+nWmc
+lM14k+lethmHMB+fsNSjnqeJ7XOPlOTHqhiZ9bBSTgF/xr5Bck/NiKRzHjdovBox
+TKg+xchaBhpRh7wBPBIlNJeHmIjv+8obOKjKU98Ig/7R9+IryZaNcKAH0PuOT+Sw
+QHXiCGQbOiYHB9UyhDTWiB7YVjd8KHefOFxfHzOQb/iBhbv1x3bTl3DgepvRN6VO
+dIsPLoIZe42sdf9GeMsk8mGJyZUQ6AzsfhWk3grb/XscizPSvrNsJ2VL1R7YTyT3
+3WA4ZXR1EqvXnWL7N/raemQjy62iOG6t7fcF5IdP9CMbWP+Plpsz4cQW7FtesCTq
+a5ZXraochQz361ODFNIeBEGU+0qqXUtZDlmos/EySkZykSeU/L0bImS62VGE3afo
+YXBmznTTT9kkFkqv7H0MerfJsrE/wF8puP3GM01DW2JRgXRpSWlvbPV/2LnMtRuD
+II7iH4rWDtTjCN6BWKAgDOnPkc9sZ4XulqT32lcUeV6LTdMBfq8kMEc8eDij1vUT
+maVCRpuwaq8EIT3lVgNLufHiG96ojlyYtj3orzw22IjkgC/9ee8UDik9CqbMVmFf
+fVHhsw8LNSg8Q4bmwm5Eg2w2it2gtI68+mwr75oCxuJ/8OMjW21Prj8XDh5reie2
+c0lDKQOFZ9UnLU1bXR/6qUM+JFKR4DMq+fOCuoQSVoyVUEOsJpvBOYnYZN9cxsZm
+vh9dKafMEcKZ8flsbr+gOmOw7+Py2ifSlf25E/Frb1W4gtbTb0LQVHb6+drutrZj
+8HEu4CnHYFCD4ZnOJb26XlZCb8GFBddW86yJYyUqMMV6Q1aJfAOAglsTo1LjIMOZ
+byo0BTAmwUevU/iuOXQ4qRBXXcoidDcTCrxfUSPG9wdt9l+m5SdQpWqfQ+fx5O7m
+SLlrHyZCiPSFMtC9DxqjIklHjf5W3wslGLgaD30YXa4VDYkRihf3CNsxGQ+tVvef
+l0ZjoAitF7Gaua06IESmKnpHe23dkr1cjYq+u2IV+xGH8LeExdwsQ9kpuTeXPnQs
+JOA99SsFx1ct32RrwjxnDDsiNkaViTKo9GDkV3jQTfoFgAVqfSgg9wGXpqUqhNG7
+TiSIHCowllLny2zn4XrXCy2niD3VDt0skb3l/PaegHE2z7S5YY85nQtYwpLiwB9M
+SQ08DYKxPBZYKtS2iZ/fsA1gjSRQDPg/SIxMhUC3M3qH8iWny1Lzl25F2Uq7VVEX
+LdTUtaby49jRTT3CQGr5n6z7bMbUegiY7h8WmOekuThGDH+4xZp6+rDP4GFk4FeK
+JcF70vMQYIjQZhadic6olv+9VtUP42ltGG/yP9a3eWRkzfAf2eCh6B1rYdgEWwE8
+rlcZzwM+y6eUmeNF2FVWB8iWtTMQHy+dYNPM+Jtus1KQKxiiq/yCRs7nWvzWRFWA
+HRyqV0J6/lqgm4FvfktFt1T0W+mDoLJOR2/zIwMy2lgL5zeHuR3SaMJnCikJbqKS
+HB3UvrhAWUcZqdH29+FhVWeM7ybyF1Wccmf+IIC/ePLa6gjtqPV8lG/5kbpcpnB6
+UQY8WWaKMxyr3jJ9bAX5QKshchp04cDecOLZrpFGNNQngR8RxSEkiIgAqNxWunIu
+KrdBDrupv/XAgEOclmgToY3iywLJSV5gHAyHWDUhRH4cFCLiGPl4XIcnXOuTze3H
+3j+EYSiS3v3DhHjp33YU2pXlJDjiYsKzAXejEh66++Y8qaQdCAad3ruWRCzW3kgk
+Md0A1VGzntTnQsewvExQEMZH2LtYIsPv3KCYGeSAuLabX4tbGk79PswjnjLLEOr0
+Ghf6RF6qf5/iFyJoG4vrbKT8kx6ywh0InILCdjUunuDskIBxX6tEcr9XwajoIvb2
+kcmGdjam5kKLS7QOWQTl8/r/cuFes0dj34cX5Qpq+Gd7tRq/D+b0207926Cxvftv
+qQ1cVn8HiLxKkZzd3tpf2xnoV1zkTL0oHrNg+qzxoxXUTUcwtIf1d/HRbYEAhi/d
+bBBoFeftEHWNq+sJgS9bH+XNzo/yK4u04B5miOq8v4CSkJdzu+ZdF22d4cjiGmtQ
+8BTmcn0Unzm+u5H0+QSZe54QBHJGNXXOIKMTkgnOdW27g4DbI1y7fCqJiSMbRW6L
+oHmMfbdB3GWqGbsUkhY8i6h9op0MU6WOX7ea2Rxyt4t6
+-----END ENCRYPTED PRIVATE KEY-----
diff --git a/Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst b/Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst
new file mode 100644
index 000000000000..8f43d321d371
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst
@@ -0,0 +1,4 @@
+The private keys for test_ssl were encrypted with 3DES in traditional
+PKCS#5 format. 3DES and the digest algorithm of PKCS#5 are blocked by
+some strict crypto policies. Use PKCS#8 format with AES256 encryption
+instead.


From webhook-mailer at python.org  Wed Sep 25 12:13:58 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Wed, 25 Sep 2019 16:13:58 -0000
Subject: [Python-checkins] bpo-38271: encrypt private key test files with
 AES256 (GH-16385)
Message-ID: <mailman.86.1569428039.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/4267e8f72a7d8fe0433819511a5ae45d45e1c67b
commit: 4267e8f72a7d8fe0433819511a5ae45d45e1c67b
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T09:13:54-07:00
summary:

bpo-38271: encrypt private key test files with AES256 (GH-16385)


The private keys for test_ssl were encrypted with 3DES in traditional
PKCSGH-5 format. 3DES and the digest algorithm of PKCSGH-5 are blocked by
some strict crypto policies. Use PKCSGH-8 format with AES256 encryption
instead.

Signed-off-by: Christian Heimes <christian at python.org>

https://bugs.python.org/issue38271

Automerge-Triggered-By: @tiran
(cherry picked from commit bfd0c963d88f3df69489ee250655e2b8f3d235bd)

Co-authored-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst
M Lib/test/keycert.passwd.pem
M Lib/test/make_ssl_certs.py
M Lib/test/ssl_key.passwd.pem

diff --git a/Lib/test/keycert.passwd.pem b/Lib/test/keycert.passwd.pem
index cbb3c3bccd2c..c330c36d8f9f 100644
--- a/Lib/test/keycert.passwd.pem
+++ b/Lib/test/keycert.passwd.pem
@@ -1,45 +1,45 @@
------BEGIN RSA PRIVATE KEY-----
-Proc-Type: 4,ENCRYPTED
-DEK-Info: DES-EDE3-CBC,D134E931C96D9DEC
-
-nuGFEej7vIjkYWSMz5OJeVTNntDRQi6ZM4DBm3g8T7i/0odr3WFqGMMKZcIhLYQf
-rgRq7RSKtrJ1y5taVucMV+EuCjyfzDo0TsYt+ZrXv/D08eZhjRmkhoHnGVF0TqQm
-nQEXM/ERT4J2RM78dnG+homMkI76qOqxgGbRqQqJo6AiVRcAZ45y8s96bru2TAB8
-+pWjO/v0Je7AFVdwSU52N8OOY6uoSAygW+0UY1WVxbVGJF2XfRsNpPX+YQHYl6e+
-3xM5XBVCgr6kmdAyub5qUJ38X3TpdVGoR0i+CVS9GTr2pSRib1zURAeeHnlqiUZM
-4m0Gn9s72nJevU1wxED8pwOhR8fnHEmMKGD2HPhKoOCbzDhwwBZO27TNa1uWeM3f
-M5oixKDi2PqMn3y2cDx1NjJtP661688EcJ5a2Ih9BgO9xpnhSyzBWEKcAn0tJB0H
-/56M0FW6cdOOIzMveGGL7sHW5E+iOdI1n5e7C6KJUzew78Y9qJnhS53EdI6qTz9R
-wsIsj1i070Fk6RbPo6zpLlF6w7Zj8GlZaZA7OZZv9wo5VEV/0ST8gmiiBOBc4C6Y
-u9hyLIIu4dFEBKyQHRvBnQSLNpKx6or1OGFDVBay2In9Yh2BHh1+vOj/OIz/wq48
-EHOIV27fRJxLu4jeK5LIGDhuPnMJ8AJYQ0bQOUP6fd7p+TxWkAQZPB/Dx/cs3hxr
-nFEdzx+eO+IAsObx/b1EGZyEJyETBslu4GwYX7/KK3HsJhDJ1bdZ//28jOCaoir6
-ZOMT72GRwmVoQTJ0XpccfjHfKJDRLT7C1xvzo4Eibth0hpTZkA75IUYUp6qK/PuJ
-kH/qdiC7QIkRKtsrawW4vEDna3YtxIYhQqz9+KwO6u/0gzooZtv1RU4U3ifMDB5u
-5P5GAzACRqlY8QYBkM869lvWqzQPHvybC4ak9Yx6/heMO9ddjdIW9BaK8BLxvN/6
-UCD936Y4fWltt09jHZIoxWFykouBwmd7bXooNYXmDRNmjTdVhKJuOEOQw8hDzx7e
-pWFJ9Z/V4Qm1tvXbCD7QFqMCDoY3qFvVG8DBqXpmxe1yPfz21FWrT7IuqDXAD3ns
-vxfN/2a+Cy04U9FBNVCvWqWIs5AgNpdCMJC2FlXKTy+H3/7rIjNyFyvbX0vxIXtK
-liOVNXiyVM++KZXqktqMUDlsJENmIHV9B046luqbgW018fHkyEYlL3iRZGbYegwr
-XO9VVIKVPw1BEvJ8VNdGFGuZGepd8qX2ezfYADrNR+4t85HDm8inbjTobSjWuljs
-ftUNkOeCHqAvWCFQTLCfdykvV08EJfVY79y7yFPtfRV2gxYokXFifjo3su9sVQr1
-UiIS5ZAsIC1hBXWeXoBN7QVTkFi7Yto6E1q2k10LiT3obpUUUQ/oclhrJOCJVjrS
-oRcj2QBy8OT4T9slJr5maTWdgd7Lt6+I6cGQXPaDvjGOJl0eBYM14vhx4rRQWytJ
-k07hhHFO4+9CGCuHS8AAy2gR6acYFWt2ZiiNZ0z/iPIHNK4YEyy9aLf6uZH/KQjE
-jmHToo7XD6QvCAEC5qTHby3o3LfHIhyZi/4L+AhS4FKUHF6M0peeyYt4z3HaK2d2
-N6mHLPdjwNjra7GOmcns4gzcrdfoF+R293KpPal4PjknvR3dZL4kKP/ougTAM5zv
-qDIvRbkHzjP8ChTpoLcJsNVXykNcNkjcSi0GHtIpYjh6QX6P2uvR/S4+Bbb9p9rn
-hIy/ovu9tWN2hiPxGPe6torF6BulAxsTYlDercC204AyzsrdA0pr6HBgJH9C6ML1
-TchwodbFJqn9rSv91i1liusAGoOvE81AGBdrXY7LxfSNhYY1IK6yR/POJPTd53sA
-uX2/j6Rtoksd/2BHPM6AUnI/2B9slhuzWX2aCtWLeuwvXDS6rYuTigaQmLkzTRfM
-dlMI3s9KLXxgi5YVumUZleJWXwBNP7KiKajd+VTSD+7WAhyhM5FIG5wVOaxmy4G2
-TyqZ/Ax9d2VEjTQHWvQlLPQ4Mp0EIz0aEl94K/S8CK8bJRH6+PRkar+dJi1xqlL+
-BYb42At9mEJ8odLlFikvNi1+t7jqXk5jRi5C0xFKx3nTtzoH2zNUeuA3R6vSocVK
-45jnze9IkKmxMlJ4loR5sgszdpDCD3kXqjtCcbMTmcrGyzJek3HSOTpiEORoTFOe
-Rhg6jH5lm+QcC263oipojS0qEQcnsWJP2CylNYMYHR9O/9NQxT3o2lsRHqZTMELV
-uQa/SFH+paQNbZOj8MRwPSqqiIxJFuLswKte1R+W7LKn1yBSM7Pp39lNbzGvJD2E
-YRfnCwFpJ54voVAuQ4jXJvigCW2qeCjXlxeD6K2j4eGJEEOmIjIW1wjubyBY6OI3
------END RSA PRIVATE KEY-----
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIhD+rJdxqb6ECAggA
+MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDTdyjCP3riOSUfxix4aXEvBIIH
+ECGkbsFabrcFMZcplw5jHMaOlG7rYjUzwDJ80JM8uzbv2Jb8SvNlns2+xmnEvH/M
+mNvRmnXmplbVjH3XBMK8o2Psnr2V/a0j7/pgqpRxHykG+koOY4gzdt3MAg8JPbS2
+hymSl+Y5EpciO3xLfz4aFL1ZNqspQbO/TD13Ij7DUIy7xIRBMp4taoZCrP0cEBAZ
++wgu9m23I4dh3E8RUBzWyFFNic2MVVHrui6JbHc4dIHfyKLtXJDhUcS0vIC9PvcV
+jhorh3UZC4lM+/jjXV5AhzQ0VrJ2tXAUX2dA144XHzkSH2QmwfnajPsci7BL2CGC
+rjyTy4NfB/lDwU+55dqJZQSKXMxAapJMrtgw7LD5CKQcN6zmfhXGssJ7HQUXKkaX
+I1YOFzuUD7oo56BVCnVswv0jX9RxrE5QYNreMlOP9cS+kIYH65N+PAhlURuQC14K
+PgDkHn5knSa2UQA5tc5f7zdHOZhGRUfcjLP+KAWA3nh+/2OKw/X3zuPx75YT/FKe
+tACPw5hjEpl62m9Xa0eWepZXwqkIOkzHMmCyNCsbC0mmRoEjmvfnslfsmnh4Dg/c
+4YsTYMOLLIeCa+WIc38aA5W2lNO9lW0LwLhX1rP+GRVPv+TVHXlfoyaI+jp0iXrJ
+t3xxT0gaiIR/VznyS7Py68QV/zB7VdqbsNzS7LdquHK1k8+7OYiWjY3gqyU40Iu2
+d1eSnIoDvQJwyYp7XYXbOlXNLY+s1Qb7yxcW3vXm0Bg3gKT8r1XHWJ9rj+CxAn5r
+ysfkPs1JsesxzzQjwTiDNvHnBnZnwxuxfBr26ektEHmuAXSl8V6dzLN/aaPjpTj4
+CkE7KyqX3U9bLkp+ztl4xWKEmW44nskzm0+iqrtrxMyTfvvID4QrABjZL4zmWIqc
+e3ZfA3AYk9VDIegk/YKGC5VZ8YS7ZXQ0ASK652XqJ7QlMKTxxV7zda6Fp4uW6/qN
+ezt5wgbGGhZQXj2wDQmWNQYyG/juIgYTpCUA54U5XBIjuR6pg+Ytm0UrvNjsUoAC
+wGelyqaLDq8U8jdIFYVTJy9aJjQOYXjsUJ0dZN2aGHSlju0ZGIZc49cTIVQ9BTC5
+Yc0Vlwzpl+LuA25DzKZNSb/ci0lO/cQGJ2uXQQgaNgdsHlu8nukENGJhnIzx4fzK
+wEh3yHxhTRCzPPwDfXmx0IHXrPqJhSpAgaXBVIm8OjvmMxO+W75W4uLfNY/B7e2H
+3cjklGuvkofOf7sEOrGUYf4cb6Obg8FpvHgpKo5Twwmoh/qvEKckBFqNhZXDDl88
+GbGlSEgyaAV1Ig8s1NJKBolWFa0juyPAwJ8vT1T4iwW7kQ7KXKt2UNn96K/HxkLu
+pikvukz8oRHMlfVHa0R48UB1fFHwZLzPmwkpu6ancIxk3uO3yfhf6iDk3bmnyMlz
+g3k/b6MrLYaOVByRxay85jH3Vvgqfgn6wa6BJ7xQ81eZ8B45gFuTH0J5JtLL7SH8
+darRPLCYfA+Ums9/H6pU5EXfd3yfjMIbvhCXHkJrrljkZ+th3p8dyto6wmYqIY6I
+qR9sU+o6DhRaiP8tCICuhHxQpXylUM6WeJkJwduTJ8KWIvzsj4mReIKOl/oC2jSd
+gIdKhb9Q3zj9ce4N5m6v66tyvjxGZ+xf3BvUPDD+LwZeXgf7OBsNVbXzQbzto594
+nbCzPocFi3gERE50ru4K70eQCy08TPG5NpOz+DDdO5vpAuMLYEuI7O3L+3GjW40Q
+G5bu7H5/i7o/RWR67qhG/7p9kPw3nkUtYgnvnWaPMIuTfb4c2d069kjlfgWjIbbI
+tpSKmm5DHlqTE4/ECAbIEDtSaw9dXHCdL3nh5+n428xDdGbjN4lT86tfu17EYKzl
+ydH1RJ1LX3o3TEj9UkmDPt7LnftvwybMFEcP7hM2xD4lC++wKQs7Alg6dTkBnJV4
+5xU78WRntJkJTU7kFkpPKA0QfyCuSF1fAMoukDBkqUdOj6jE0BlJQlHk5iwgnJlt
+uEdkTjHZEjIUxWC6llPcAzaPNlmnD45AgfEW+Jn21IvutmJiQAz5lm9Z9PXaR0C8
+hXB6owRY67C0YKQwXhoNf6xQun2xGBGYy5rPEEezX1S1tUH5GR/KW1Lh+FzFqHXI
+ZEb5avfDqHKehGAjPON+Br7akuQ125M9LLjKuSyPaQzeeCAy356Xd7XzVwbPddbm
+9S9WSPqzaPgh10chIHoNoC8HMd33dB5j9/Q6jrbU/oPlptu/GlorWblvJdcTuBGI
+IVn45RFnkG8hCz0GJSNzW7+70YdESQbfJW79vssWMaiSjFE0pMyFXrFR5lBywBTx
+PiGEUWtvrKG94X1TMlGUzDzDJOQNZ9dT94bonNe9pVmP5BP4/DzwwiWh6qrzWk6p
+j8OE4cfCSh2WvHnhJbH7/N0v+JKjtxeIeJ16jx/K2oK5
+-----END ENCRYPTED PRIVATE KEY-----
 -----BEGIN CERTIFICATE-----
 MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV
 BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
@@ -66,3 +66,4 @@ jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu
 9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW
 HcVKQHyOeyvnINuBAQ==
 -----END CERTIFICATE-----
+
diff --git a/Lib/test/make_ssl_certs.py b/Lib/test/make_ssl_certs.py
index 362276583f50..41b5f46c882c 100644
--- a/Lib/test/make_ssl_certs.py
+++ b/Lib/test/make_ssl_certs.py
@@ -206,8 +206,8 @@ def print_cert(path):
     with open('ssl_key.pem', 'w') as f:
         f.write(key)
     print("password protecting ssl_key.pem in ssl_key.passwd.pem")
-    check_call(['openssl','rsa','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-des3','-passout','pass:somepass'])
-    check_call(['openssl','rsa','-in','ssl_key.pem','-out','keycert.passwd.pem','-des3','-passout','pass:somepass'])
+    check_call(['openssl','pkey','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-aes256','-passout','pass:somepass'])
+    check_call(['openssl','pkey','-in','ssl_key.pem','-out','keycert.passwd.pem','-aes256','-passout','pass:somepass'])
 
     with open('keycert.pem', 'w') as f:
         f.write(key)
diff --git a/Lib/test/ssl_key.passwd.pem b/Lib/test/ssl_key.passwd.pem
index e4f1370ab270..46de61ab85b1 100644
--- a/Lib/test/ssl_key.passwd.pem
+++ b/Lib/test/ssl_key.passwd.pem
@@ -1,42 +1,42 @@
------BEGIN RSA PRIVATE KEY-----
-Proc-Type: 4,ENCRYPTED
-DEK-Info: DES-EDE3-CBC,8064BE1494B24B13
-
-KJrffOMbo8M0I3PzcYxRZGMpKD1yB3Ii4+bT5XoanxjIJ+4fdx6LfZ0Rsx+riyzs
-tymsQu/iYY9j+4rCvN9+eetsL1X6iZpiimKsLexcid9M3fb0vxED5Sgw0dvunCUA
-xhqjLIKR92MKbODHf6KrDKCpsiPbjq4gZ7P+uCGXAMHL3MXIJSC0hW9rK7Ce6oyO
-CjpIcgB8x+GUWZZZhAFdlzIHMZrteNP2P5HK6QcaT71P034Dz1hhqoj4Q0t+Fta2
-4tfsM/bnTR/l6hwlhPa1e3Uj322tDTDWBScgWANn5+sEWldLmozMaWhZsn22pfk2
-KjRMGXG024JVheV882nbdOBvG7oq+lxkZ/ZP+vvqJqnvYtf7WtM8UivzYpe5Hz5b
-kVvWzPjBLUSZ9whM9rDLqSSqMPyPvDTuEmLkuq+xm7pYJmsLqIMP2klZLqRxLX6K
-uqwplb8UG440qauxgnQ905PId1l2fJEnRtV+7vXprA0L0QotgXLVHBhLmTFM+3PH
-9H3onf31dionUAPrn3nfVE36HhvVgRyvDBnBzJSIMighgq21Qx/d1dk0DRYi1hUI
-nCHl0YJPXheVcXR7JiSF2XQCAaFuS1Mr7NCXfWZOZQC/0dkvmHnl9DUAhuqq9BNZ
-1cKhZXcKHadg2/r0Zup/oDzmHPUEfTAXT0xbqoWlhkdwbF2veWQ96A/ncx3ISTb4
-PkXBlX9rdia8nmtyQDQRn4NuvchbaGkj4WKFC8pF8Hn7naHqwjpHaDUimBc0CoQW
-edNJqruKWwtSVLuwKHCC2gZFX9AXSKJXJz/QRSUlhFGOhuF/J6yKaXj6n5lxWNiQ
-54J+OP/hz2aS95CD2+Zf1SKpxdWiLZSIQqESpmmUrXROixNJZ/Z7gI74Dd9dSJOH
-W+3AU03vrrFZVrJVZhjcINHoH1Skh6JKscH18L6x4U868nSr4SrRLX8BhHllOQyD
-bmU+PZAjF8ZBIaCtTGulDXD29F73MeAZeTSsgQjFu0iKLj1wPiphbx8i/SUtR4YP
-X6PVA04g66r1NBw+3RQASVorZ3g1MSFvITHXcbKkBDeJH2z1+c6t/VVyTONnQhM5
-lLgRSk6HCbetvT9PKxWrWutA12pdBYEHdZhMHVf2+xclky7l09w8hg2/qqcdGRGe
-oAOZ72t0l5ObNyaruDKUS6f4AjOyWq/Xj5xuFtf1n3tQHyslSyCTPcAbQhDfTHUx
-vixb/V9qvYPt7OCn8py7v1M69NH42QVFAvwveDIFjZdqfIKBoJK2V4qPoevJI6uj
-Q5ByMt8OXOjSXNpHXpYQWUiWeCwOEBXJX8rzCHdMtg37jJ0zCmeErR1NTdg+EujM
-TWYgd06jlT67tURST0aB2kg4ijKgUJefD313LW1zC6gVsTbjSZxYyRbPfSP6flQB
-yCi1C19E2OsgleqbkBVC5GlYUzaJT7SGjCRmGx1eqtbrALu+LVH24Wceexlpjydl
-+s2nf/DZlKun/tlPh6YioifPCJjByZMQOCEfIox6BkemZETz8uYA4TTWimG13Z03
-gyDGC2jdpEW414J2qcQDvrdUgJ+HlhrAAHaWpMQDbXYxBGoZ+3+ORvQV4kAsCwL8
-k3EIrVpePdik+1xgOWsyLj6QxFXlTMvL6Wc5pnArFPORsgHEolJvxSPTf9aAHNPn
-V2WBvxiLBtYpGrujAUM40Syx/aN2RPtcXYPAusHUBw+S8/p+/8Kg8GZmnIXG3F89
-45Eepl2quZYIrou7a1fwIpIIZ0hFiBQ1mlHVMFtxwVHS1bQb3SU2GeO+JcGjdVXc
-04qeGuQ5M164eQ5C0T7ZQ1ULiUlFWKD30m+cjqmZzt3d7Q0mKpMKuESIuZJo/wpD
-Nas432aLKUhcNx/pOYLkKJRpGZKOupQoD5iUj/j44o8JoFkDK33v2S57XB5QGz28
-9Zuhx49b3W8mbM6EBanlQKLWJGCxXqc/jhYhFWn+b0MhidynFgA0oeWvf6ZDyt6H
-Yi5Etxsar09xp0Do3NxtQXLuSUu0ji2pQzSIKuoqQWKqldm6VrpwojiqJhy4WQBQ
-aVVyFeWBC7G3Zj76dO+yp2sfJ0itJUQ8AIB9Cg0f34rEZu+r9luPmqBoUeL95Tk7
-YvCOU3Jl8Iqysv8aNpVXT8sa8rrSbruWCByEePZ37RIdHLMVBwVY0eVaFQjrjU7E
-mXmM9eaoYLfXOllsQ+M2+qPFUITr/GU3Qig13DhK/+yC1R6V2a0l0WRhMltIPYKW
-Ztvvr4hK5LcYCeS113BLiMbDIMMZZYGDZGMdC8DnnVbT2loF0Rfmp80Af31KmMQ4
-6XvMatW9UDjBoY5a/YMpdm7SRwm+MgV2KNPpc2kST87/yi9oprGAb8qiarHiHTM0
------END RSA PRIVATE KEY-----
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQI072N7W+PDDMCAggA
+MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBA/AuaRNi4vE4KGqI4In+70BIIH
+ENGS5Vex5NID873frmd1UZEHZ+O/Bd0wDb+NUpIqesHkRYf7kKi6Gnr+nKQ/oVVn
+Lm3JjE7c8ECP0OkOOXmiXuWL1SkzBBWqCI4stSGUPvBiHsGwNnvJAaGjUffgMlcC
+aJOA2+dnejLkzblq4CB2LQdm06N3Xoe9tyqtQaUHxfzJAf5Ydd8uj7vpKN2MMhY7
+icIPJwSyh0N7S6XWVtHEokr9Kp4y2hS5a+BgCWV1/1z0aF7agnSVndmT1VR+nWmc
+lM14k+lethmHMB+fsNSjnqeJ7XOPlOTHqhiZ9bBSTgF/xr5Bck/NiKRzHjdovBox
+TKg+xchaBhpRh7wBPBIlNJeHmIjv+8obOKjKU98Ig/7R9+IryZaNcKAH0PuOT+Sw
+QHXiCGQbOiYHB9UyhDTWiB7YVjd8KHefOFxfHzOQb/iBhbv1x3bTl3DgepvRN6VO
+dIsPLoIZe42sdf9GeMsk8mGJyZUQ6AzsfhWk3grb/XscizPSvrNsJ2VL1R7YTyT3
+3WA4ZXR1EqvXnWL7N/raemQjy62iOG6t7fcF5IdP9CMbWP+Plpsz4cQW7FtesCTq
+a5ZXraochQz361ODFNIeBEGU+0qqXUtZDlmos/EySkZykSeU/L0bImS62VGE3afo
+YXBmznTTT9kkFkqv7H0MerfJsrE/wF8puP3GM01DW2JRgXRpSWlvbPV/2LnMtRuD
+II7iH4rWDtTjCN6BWKAgDOnPkc9sZ4XulqT32lcUeV6LTdMBfq8kMEc8eDij1vUT
+maVCRpuwaq8EIT3lVgNLufHiG96ojlyYtj3orzw22IjkgC/9ee8UDik9CqbMVmFf
+fVHhsw8LNSg8Q4bmwm5Eg2w2it2gtI68+mwr75oCxuJ/8OMjW21Prj8XDh5reie2
+c0lDKQOFZ9UnLU1bXR/6qUM+JFKR4DMq+fOCuoQSVoyVUEOsJpvBOYnYZN9cxsZm
+vh9dKafMEcKZ8flsbr+gOmOw7+Py2ifSlf25E/Frb1W4gtbTb0LQVHb6+drutrZj
+8HEu4CnHYFCD4ZnOJb26XlZCb8GFBddW86yJYyUqMMV6Q1aJfAOAglsTo1LjIMOZ
+byo0BTAmwUevU/iuOXQ4qRBXXcoidDcTCrxfUSPG9wdt9l+m5SdQpWqfQ+fx5O7m
+SLlrHyZCiPSFMtC9DxqjIklHjf5W3wslGLgaD30YXa4VDYkRihf3CNsxGQ+tVvef
+l0ZjoAitF7Gaua06IESmKnpHe23dkr1cjYq+u2IV+xGH8LeExdwsQ9kpuTeXPnQs
+JOA99SsFx1ct32RrwjxnDDsiNkaViTKo9GDkV3jQTfoFgAVqfSgg9wGXpqUqhNG7
+TiSIHCowllLny2zn4XrXCy2niD3VDt0skb3l/PaegHE2z7S5YY85nQtYwpLiwB9M
+SQ08DYKxPBZYKtS2iZ/fsA1gjSRQDPg/SIxMhUC3M3qH8iWny1Lzl25F2Uq7VVEX
+LdTUtaby49jRTT3CQGr5n6z7bMbUegiY7h8WmOekuThGDH+4xZp6+rDP4GFk4FeK
+JcF70vMQYIjQZhadic6olv+9VtUP42ltGG/yP9a3eWRkzfAf2eCh6B1rYdgEWwE8
+rlcZzwM+y6eUmeNF2FVWB8iWtTMQHy+dYNPM+Jtus1KQKxiiq/yCRs7nWvzWRFWA
+HRyqV0J6/lqgm4FvfktFt1T0W+mDoLJOR2/zIwMy2lgL5zeHuR3SaMJnCikJbqKS
+HB3UvrhAWUcZqdH29+FhVWeM7ybyF1Wccmf+IIC/ePLa6gjtqPV8lG/5kbpcpnB6
+UQY8WWaKMxyr3jJ9bAX5QKshchp04cDecOLZrpFGNNQngR8RxSEkiIgAqNxWunIu
+KrdBDrupv/XAgEOclmgToY3iywLJSV5gHAyHWDUhRH4cFCLiGPl4XIcnXOuTze3H
+3j+EYSiS3v3DhHjp33YU2pXlJDjiYsKzAXejEh66++Y8qaQdCAad3ruWRCzW3kgk
+Md0A1VGzntTnQsewvExQEMZH2LtYIsPv3KCYGeSAuLabX4tbGk79PswjnjLLEOr0
+Ghf6RF6qf5/iFyJoG4vrbKT8kx6ywh0InILCdjUunuDskIBxX6tEcr9XwajoIvb2
+kcmGdjam5kKLS7QOWQTl8/r/cuFes0dj34cX5Qpq+Gd7tRq/D+b0207926Cxvftv
+qQ1cVn8HiLxKkZzd3tpf2xnoV1zkTL0oHrNg+qzxoxXUTUcwtIf1d/HRbYEAhi/d
+bBBoFeftEHWNq+sJgS9bH+XNzo/yK4u04B5miOq8v4CSkJdzu+ZdF22d4cjiGmtQ
+8BTmcn0Unzm+u5H0+QSZe54QBHJGNXXOIKMTkgnOdW27g4DbI1y7fCqJiSMbRW6L
+oHmMfbdB3GWqGbsUkhY8i6h9op0MU6WOX7ea2Rxyt4t6
+-----END ENCRYPTED PRIVATE KEY-----
diff --git a/Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst b/Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst
new file mode 100644
index 000000000000..8f43d321d371
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-25-13-11-29.bpo-38271.iHXNIg.rst
@@ -0,0 +1,4 @@
+The private keys for test_ssl were encrypted with 3DES in traditional
+PKCS#5 format. 3DES and the digest algorithm of PKCS#5 are blocked by
+some strict crypto policies. Use PKCS#8 format with AES256 encryption
+instead.


From webhook-mailer at python.org  Wed Sep 25 15:57:27 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Wed, 25 Sep 2019 19:57:27 -0000
Subject: [Python-checkins] bpo-22273: Removed temporary test skipping on PPC
 platforms. (GH-16399)
Message-ID: <mailman.87.1569441448.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/cc28ed2421bf3953efc0fbde387f28722f3801e2
commit: cc28ed2421bf3953efc0fbde387f28722f3801e2
branch: master
author: Vinay Sajip <vinay_sajip at yahoo.co.uk>
committer: GitHub <noreply at github.com>
date: 2019-09-25T20:57:20+01:00
summary:

bpo-22273: Removed temporary test skipping on PPC platforms. (GH-16399)

files:
M Lib/ctypes/test/test_structures.py

diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py
index fedcae24dfd3..0c01c799042c 100644
--- a/Lib/ctypes/test/test_structures.py
+++ b/Lib/ctypes/test/test_structures.py
@@ -6,6 +6,9 @@
 import _ctypes_test
 from test import support
 
+# The following definition is meant to be used from time to time to assist
+# temporarily disabling tests on specific architectures while investigations
+# are in progress, to keep buildbots happy.
 MACHINE = platform.machine()
 
 class SubclassesTest(unittest.TestCase):
@@ -480,8 +483,6 @@ class X(Structure):
         self.assertEqual(s.first, got.first)
         self.assertEqual(s.second, got.second)
 
-    @unittest.skipIf(MACHINE.startswith('ppc'),
-                     'Test temporarily disabled on this architecture')
     def test_array_in_struct(self):
         # See bpo-22273
 


From webhook-mailer at python.org  Wed Sep 25 16:37:45 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Wed, 25 Sep 2019 20:37:45 -0000
Subject: [Python-checkins] [3.7] bpo-22273: Changed conditions for ctypes
 array-in-struct handling. (GH-16381) (GH-16400)
Message-ID: <mailman.88.1569443866.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/d015714f89af5c57b7071a74eeca618577b3dfe4
commit: d015714f89af5c57b7071a74eeca618577b3dfe4
branch: 3.7
author: Vinay Sajip <vinay_sajip at yahoo.co.uk>
committer: GitHub <noreply at github.com>
date: 2019-09-25T21:37:24+01:00
summary:

[3.7] bpo-22273: Changed conditions for ctypes array-in-struct handling. (GH-16381) (GH-16400)

(cherry picked from commit c64af8fad3c4f5751af624647fbb0ce023f525dc)

files:
M Modules/_ctypes/stgdict.c

diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c
index e584abc96f1a..5388a3154b50 100644
--- a/Modules/_ctypes/stgdict.c
+++ b/Modules/_ctypes/stgdict.c
@@ -344,9 +344,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     int pack = 0;
     Py_ssize_t ffi_ofs;
     int big_endian;
-#if defined(X86_64)
     int arrays_seen = 0;
-#endif
 
     /* HACK Alert: I cannot be bothered to fix ctypes.com, so there has to
        be a way to use the old, broken sematics: _fields_ are not extended
@@ -471,10 +469,8 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
             Py_XDECREF(pair);
             return -1;
         }
-#if defined(X86_64)
         if (PyCArrayTypeObject_Check(desc))
             arrays_seen = 1;
-#endif
         dict = PyType_stgdict(desc);
         if (dict == NULL) {
             Py_DECREF(pair);
@@ -615,8 +611,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     stgdict->align = total_align;
     stgdict->length = len;      /* ADD ffi_ofs? */
 
-#if defined(X86_64)
-
 #define MAX_ELEMENTS 16
 
     if (arrays_seen && (size <= 16)) {
@@ -636,6 +630,10 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
          * accurate set, to allow libffi to marshal them into registers
          * correctly. It means one more loop over the fields, but if we got
          * here, the structure is small, so there aren't too many of those.
+         *
+         * Although the passing in registers is specific to 64-bit Linux, the
+         * array-in-struct vs. pointer problem is general. But we restrict the
+         * type transformation to small structs nonetheless.
          */
         ffi_type *actual_types[MAX_ELEMENTS + 1];
         int actual_type_index = 0;
@@ -713,7 +711,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
         memcpy(&stgdict->ffi_type_pointer.elements[ffi_ofs], actual_types,
                actual_type_index * sizeof(ffi_type *));
     }
-#endif
 
     /* We did check that this flag was NOT set above, it must not
        have been set until now. */


From webhook-mailer at python.org  Wed Sep 25 17:03:38 2019
From: webhook-mailer at python.org (Christian Heimes)
Date: Wed, 25 Sep 2019 21:03:38 -0000
Subject: [Python-checkins] bpo-38142: Updated _hashopenssl.c to be PEP 384
 compliant (#16071)
Message-ID: <mailman.89.1569445419.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/df69e75edcc08475bc9a57a5a76df8a45bfc3c34
commit: df69e75edcc08475bc9a57a5a76df8a45bfc3c34
branch: master
author: Christian Heimes <christian at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-25T23:03:30+02:00
summary:

bpo-38142: Updated _hashopenssl.c to be PEP 384 compliant (#16071)

* Updated _hashopenssl.c to be PEP 384 compliant
* Remove refleak test from test_hashlib. The updated type no longer accepts random arguments to __init__.

files:
A Misc/NEWS.d/next/Library/2019-09-12-18-41-35.bpo-38142.1I0Ch0.rst
M Lib/test/test_hashlib.py
M Modules/_hashopenssl.c

diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index 9204b44bf4f9..d55de02f9164 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -167,16 +167,6 @@ def hash_constructors(self):
         constructors = self.constructors_to_test.values()
         return itertools.chain.from_iterable(constructors)
 
-    @support.refcount_test
-    @unittest.skipIf(c_hashlib is None, 'Require _hashlib module')
-    def test_refleaks_in_hash___init__(self):
-        gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount')
-        sha1_hash = c_hashlib.new('sha1')
-        refs_before = gettotalrefcount()
-        for i in range(100):
-            sha1_hash.__init__('sha1')
-        self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10)
-
     def test_hash_array(self):
         a = array.array("b", range(10))
         for cons in self.hash_constructors:
diff --git a/Misc/NEWS.d/next/Library/2019-09-12-18-41-35.bpo-38142.1I0Ch0.rst b/Misc/NEWS.d/next/Library/2019-09-12-18-41-35.bpo-38142.1I0Ch0.rst
new file mode 100644
index 000000000000..740b6f1cf096
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-12-18-41-35.bpo-38142.1I0Ch0.rst
@@ -0,0 +1 @@
+The _hashlib OpenSSL wrapper extension module is now PEP-384 compliant.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index d963c7a57221..48511f72bc63 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -46,6 +46,16 @@
 #define PY_OPENSSL_HAS_BLAKE2 1
 #endif
 
+static PyModuleDef _hashlibmodule;
+
+typedef struct {
+    PyTypeObject *EVPtype;
+} _hashlibstate;
+
+#define _hashlibstate(o) ((_hashlibstate *)PyModule_GetState(o))
+#define _hashlibstate_global ((_hashlibstate *)PyModule_GetState(PyState_FindModule(&_hashlibmodule)))
+
+
 typedef struct {
     PyObject_HEAD
     EVP_MD_CTX          *ctx;   /* OpenSSL message digest context */
@@ -53,14 +63,12 @@ typedef struct {
 } EVPobject;
 
 
-static PyTypeObject EVPtype;
-
 #include "clinic/_hashopenssl.c.h"
 /*[clinic input]
 module _hashlib
-class _hashlib.HASH "EVPobject *" "&EVPtype"
+class _hashlib.HASH "EVPobject *" "((_hashlibstate *)PyModule_GetState(module))->EVPtype"
 [clinic start generated code]*/
-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a881a5092eecad28]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=1adf85e8eb2ab979]*/
 
 
 /* LCOV_EXCL_START */
@@ -231,7 +239,9 @@ py_digest_by_name(const char *name)
 static EVPobject *
 newEVPobject(void)
 {
-    EVPobject *retval = (EVPobject *)PyObject_New(EVPobject, &EVPtype);
+    EVPobject *retval = (EVPobject *)PyObject_New(
+        EVPobject, _hashlibstate_global->EVPtype
+    );
     if (retval == NULL) {
         return NULL;
     }
@@ -273,10 +283,12 @@ EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len)
 static void
 EVP_dealloc(EVPobject *self)
 {
+    PyTypeObject *tp = Py_TYPE(self);
     if (self->lock != NULL)
         PyThread_free_lock(self->lock);
     EVP_MD_CTX_free(self->ctx);
     PyObject_Del(self);
+    Py_DECREF(tp);
 }
 
 static int
@@ -501,46 +513,23 @@ PyDoc_STRVAR(hashtype_doc,
 "name -- the hash algorithm being used by this object\n"
 "digest_size -- number of bytes in this hashes output");
 
-static PyTypeObject EVPtype = {
-    PyVarObject_HEAD_INIT(NULL, 0)
+static PyType_Slot EVPtype_slots[] = {
+    {Py_tp_dealloc, EVP_dealloc},
+    {Py_tp_repr, EVP_repr},
+    {Py_tp_doc, (char *)hashtype_doc},
+    {Py_tp_methods, EVP_methods},
+    {Py_tp_getset, EVP_getseters},
+    {0, 0},
+};
+
+static PyType_Spec EVPtype_spec = {
     "_hashlib.HASH",    /*tp_name*/
     sizeof(EVPobject),  /*tp_basicsize*/
     0,                  /*tp_itemsize*/
-    /* methods */
-    (destructor)EVP_dealloc, /*tp_dealloc*/
-    0,                  /*tp_vectorcall_offset*/
-    0,                  /*tp_getattr*/
-    0,                  /*tp_setattr*/
-    0,                  /*tp_as_async*/
-    (reprfunc)EVP_repr, /*tp_repr*/
-    0,                  /*tp_as_number*/
-    0,                  /*tp_as_sequence*/
-    0,                  /*tp_as_mapping*/
-    0,                  /*tp_hash*/
-    0,                  /*tp_call*/
-    0,                  /*tp_str*/
-    0,                  /*tp_getattro*/
-    0,                  /*tp_setattro*/
-    0,                  /*tp_as_buffer*/
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
-    hashtype_doc,       /*tp_doc*/
-    0,                  /*tp_traverse*/
-    0,                  /*tp_clear*/
-    0,                  /*tp_richcompare*/
-    0,                  /*tp_weaklistoffset*/
-    0,                  /*tp_iter*/
-    0,                  /*tp_iternext*/
-    EVP_methods,        /* tp_methods */
-    NULL,        /* tp_members */
-    EVP_getseters,      /* tp_getset */
-    0,                  /* tp_base */
-    0,                  /* tp_dict */
-    0,                  /* tp_descr_get */
-    0,                  /* tp_descr_set */
-    0,                  /* tp_dictoffset */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    EVPtype_slots
 };
 
-\
 static PyObject *
 EVPnew(const EVP_MD *digest,
        const unsigned char *cp, Py_ssize_t len, int usedforsecurity)
@@ -1120,17 +1109,39 @@ static struct PyMethodDef EVP_functions[] = {
 
 /* Initialize this module. */
 
+static int
+hashlib_traverse(PyObject *m, visitproc visit, void *arg)
+{
+    _hashlibstate *state = _hashlibstate(m);
+    Py_VISIT(state->EVPtype);
+    return 0;
+}
+
+static int
+hashlib_clear(PyObject *m)
+{
+    _hashlibstate *state = _hashlibstate(m);
+    Py_CLEAR(state->EVPtype);
+    return 0;
+}
+
+static void
+hashlib_free(void *m)
+{
+    hashlib_clear((PyObject *)m);
+}
+
 
 static struct PyModuleDef _hashlibmodule = {
     PyModuleDef_HEAD_INIT,
     "_hashlib",
     NULL,
-    -1,
+    sizeof(_hashlibstate),
     EVP_functions,
     NULL,
-    NULL,
-    NULL,
-    NULL
+    hashlib_traverse,
+    hashlib_clear,
+    hashlib_free
 };
 
 PyMODINIT_FUNC
@@ -1144,19 +1155,21 @@ PyInit__hashlib(void)
     ERR_load_crypto_strings();
 #endif
 
-    /* TODO build EVP_functions openssl_* entries dynamically based
-     * on what hashes are supported rather than listing many
-     * but having some be unsupported.  Only init appropriate
-     * constants. */
-
-    Py_TYPE(&EVPtype) = &PyType_Type;
-    if (PyType_Ready(&EVPtype) < 0)
-        return NULL;
+    m = PyState_FindModule(&_hashlibmodule);
+    if (m != NULL) {
+        Py_INCREF(m);
+        return m;
+    }
 
     m = PyModule_Create(&_hashlibmodule);
     if (m == NULL)
         return NULL;
 
+    PyTypeObject *EVPtype = (PyTypeObject *)PyType_FromSpec(&EVPtype_spec);
+    if (EVPtype == NULL)
+        return NULL;
+    _hashlibstate(m)->EVPtype = EVPtype;
+
     openssl_md_meth_names = generate_hash_name_list();
     if (openssl_md_meth_names == NULL) {
         Py_DECREF(m);
@@ -1167,8 +1180,9 @@ PyInit__hashlib(void)
         return NULL;
     }
 
-    Py_INCREF((PyObject *)&EVPtype);
-    PyModule_AddObject(m, "HASH", (PyObject *)&EVPtype);
+    Py_INCREF((PyObject *)_hashlibstate(m)->EVPtype);
+    PyModule_AddObject(m, "HASH", (PyObject *)_hashlibstate(m)->EVPtype);
 
+    PyState_AddModule(m, &_hashlibmodule);
     return m;
 }


From webhook-mailer at python.org  Wed Sep 25 17:41:09 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Wed, 25 Sep 2019 21:41:09 -0000
Subject: [Python-checkins] [3.8] bpo-22273: Changed conditions for ctypes
 array-in-struct handling. (GH-16381) (GH-16401)
Message-ID: <mailman.90.1569447671.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/b92b8c53f63d2e805288ac2c665bf5887d1b7ca6
commit: b92b8c53f63d2e805288ac2c665bf5887d1b7ca6
branch: 3.8
author: Vinay Sajip <vinay_sajip at yahoo.co.uk>
committer: GitHub <noreply at github.com>
date: 2019-09-25T22:41:02+01:00
summary:

[3.8] bpo-22273: Changed conditions for ctypes array-in-struct handling. (GH-16381) (GH-16401)

(cherry picked from commit c64af8fad3c4f5751af624647fbb0ce023f525dc)

files:
M Modules/_ctypes/stgdict.c

diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c
index 62dceceb4f4e..a27d1341dc3e 100644
--- a/Modules/_ctypes/stgdict.c
+++ b/Modules/_ctypes/stgdict.c
@@ -350,9 +350,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     int pack;
     Py_ssize_t ffi_ofs;
     int big_endian;
-#if defined(X86_64)
     int arrays_seen = 0;
-#endif
 
     /* HACK Alert: I cannot be bothered to fix ctypes.com, so there has to
        be a way to use the old, broken semantics: _fields_ are not extended
@@ -504,10 +502,8 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
             Py_XDECREF(pair);
             return -1;
         }
-#if defined(X86_64)
         if (PyCArrayTypeObject_Check(desc))
             arrays_seen = 1;
-#endif
         dict = PyType_stgdict(desc);
         if (dict == NULL) {
             Py_DECREF(pair);
@@ -648,8 +644,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     stgdict->align = total_align;
     stgdict->length = len;      /* ADD ffi_ofs? */
 
-#if defined(X86_64)
-
 #define MAX_ELEMENTS 16
 
     if (arrays_seen && (size <= 16)) {
@@ -669,6 +663,10 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
          * accurate set, to allow libffi to marshal them into registers
          * correctly. It means one more loop over the fields, but if we got
          * here, the structure is small, so there aren't too many of those.
+         *
+         * Although the passing in registers is specific to 64-bit Linux, the
+         * array-in-struct vs. pointer problem is general. But we restrict the
+         * type transformation to small structs nonetheless.
          */
         ffi_type *actual_types[MAX_ELEMENTS + 1];
         int actual_type_index = 0;
@@ -746,7 +744,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
         memcpy(&stgdict->ffi_type_pointer.elements[ffi_ofs], actual_types,
                actual_type_index * sizeof(ffi_type *));
     }
-#endif
 
     /* We did check that this flag was NOT set above, it must not
        have been set until now. */


From webhook-mailer at python.org  Wed Sep 25 20:22:49 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 26 Sep 2019 00:22:49 -0000
Subject: [Python-checkins] bpo-38234: Add test_init_setpath_config() to
 test_embed (GH-16402)
Message-ID: <mailman.91.1569457370.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/8bf39b606ef7b02c0279a80789f3c4824b0da5e9
commit: 8bf39b606ef7b02c0279a80789f3c4824b0da5e9
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-26T02:22:35+02:00
summary:

bpo-38234: Add test_init_setpath_config() to test_embed (GH-16402)

* Add test_embed.test_init_setpath_config(): test Py_SetPath()
  with PyConfig.
* test_init_setpath() and test_init_setpythonhome() no longer call
  Py_SetProgramName(), but use the default program name.
* _PyPathConfig: isolated, site_import  and base_executable
  fields are now only available on Windows.
* If executable is set explicitly in the configuration, ignore
  calculated base_executable: _PyConfig_InitPathConfig() copies
  executable to base_executable.
* Complete path config documentation.

files:
M Doc/c-api/init_config.rst
M Include/internal/pycore_pathconfig.h
M Lib/test/test_embed.py
M PC/getpathp.c
M Programs/_testembed.c
M Python/pathconfig.c

diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index bc24fa081317..d98bcda20747 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -864,29 +864,38 @@ Path Configuration
 
 :c:type:`PyConfig` contains multiple fields for the path configuration:
 
-* Path configuration input fields:
+* Path configuration inputs:
 
   * :c:member:`PyConfig.home`
   * :c:member:`PyConfig.pathconfig_warnings`
   * :c:member:`PyConfig.program_name`
   * :c:member:`PyConfig.pythonpath_env`
+  * current working directory: to get absolute paths
+  * ``PATH`` environment variable to get the program full path
+    (from :c:member:`PyConfig.program_name`)
+  * ``__PYVENV_LAUNCHER__`` environment variable
+  * (Windows only) Application paths in the registry under
+    "Software\Python\PythonCore\X.Y\PythonPath" of HKEY_CURRENT_USER and
+    HKEY_LOCAL_MACHINE (where X.Y is the Python version).
 
 * Path configuration output fields:
 
+  * :c:member:`PyConfig.base_exec_prefix`
   * :c:member:`PyConfig.base_executable`
+  * :c:member:`PyConfig.base_prefix`
   * :c:member:`PyConfig.exec_prefix`
   * :c:member:`PyConfig.executable`
-  * :c:member:`PyConfig.prefix`
   * :c:member:`PyConfig.module_search_paths_set`,
     :c:member:`PyConfig.module_search_paths`
+  * :c:member:`PyConfig.prefix`
 
-If at least one "output field" is not set, Python computes the path
+If at least one "output field" is not set, Python calculates the path
 configuration to fill unset fields. If
 :c:member:`~PyConfig.module_search_paths_set` is equal to 0,
 :c:member:`~PyConfig.module_search_paths` is overridden and
 :c:member:`~PyConfig.module_search_paths_set` is set to 1.
 
-It is possible to completely ignore the function computing the default
+It is possible to completely ignore the function calculating the default
 path configuration by setting explicitly all path configuration output
 fields listed above. A string is considered as set even if it is non-empty.
 ``module_search_paths`` is considered as set if
@@ -894,7 +903,7 @@ fields listed above. A string is considered as set even if it is non-empty.
 configuration input fields are ignored as well.
 
 Set :c:member:`~PyConfig.pathconfig_warnings` to 0 to suppress warnings when
-computing the path configuration (Unix only, Windows does not log any warning).
+calculating the path configuration (Unix only, Windows does not log any warning).
 
 If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix`
 fields are not set, they inherit their value from :c:member:`~PyConfig.prefix`
diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h
index 61b3790fe1f4..b5d114473654 100644
--- a/Include/internal/pycore_pathconfig.h
+++ b/Include/internal/pycore_pathconfig.h
@@ -19,6 +19,7 @@ typedef struct _PyPathConfig {
     wchar_t *program_name;
     /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */
     wchar_t *home;
+#ifdef MS_WINDOWS
     /* isolated and site_import are used to set Py_IsolatedFlag and
        Py_NoSiteFlag flags on Windows in read_pth_file(). These fields
        are ignored when their value are equal to -1 (unset). */
@@ -26,12 +27,18 @@ typedef struct _PyPathConfig {
     int site_import;
     /* Set when a venv is detected */
     wchar_t *base_executable;
+#endif
 } _PyPathConfig;
 
-#define _PyPathConfig_INIT \
-    {.module_search_path = NULL, \
-     .isolated = -1, \
-     .site_import = -1}
+#ifdef MS_WINDOWS
+#  define _PyPathConfig_INIT \
+      {.module_search_path = NULL, \
+       .isolated = -1, \
+       .site_import = -1}
+#else
+#  define _PyPathConfig_INIT \
+      {.module_search_path = NULL}
+#endif
 /* Note: _PyPathConfig_INIT sets other fields to 0/NULL */
 
 PyAPI_DATA(_PyPathConfig) _Py_path_config;
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index ed90fc0cbed0..8855e907c253 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -635,16 +635,19 @@ def check_global_config(self, configs):
         self.assertEqual(configs['global_config'], expected)
 
     def check_all_configs(self, testname, expected_config=None,
-                     expected_preconfig=None, modify_path_cb=None, stderr=None,
-                     *, api, env=None, ignore_stderr=False, cwd=None):
+                          expected_preconfig=None, modify_path_cb=None,
+                          stderr=None, *, api, preconfig_api=None,
+                          env=None, ignore_stderr=False, cwd=None):
         new_env = remove_python_envvars()
         if env is not None:
             new_env.update(env)
         env = new_env
 
-        if api == API_ISOLATED:
+        if preconfig_api is None:
+            preconfig_api = api
+        if preconfig_api == API_ISOLATED:
             default_preconfig = self.PRE_CONFIG_ISOLATED
-        elif api == API_PYTHON:
+        elif preconfig_api == API_PYTHON:
             default_preconfig = self.PRE_CONFIG_PYTHON
         else:
             default_preconfig = self.PRE_CONFIG_COMPAT
@@ -1002,8 +1005,21 @@ def test_init_dont_parse_argv(self):
         self.check_all_configs("test_init_dont_parse_argv", config, pre_config,
                                api=API_PYTHON)
 
+    def default_program_name(self, config):
+        if MS_WINDOWS:
+            program_name = 'python'
+            executable = self.test_exe
+        else:
+            program_name = 'python3'
+            executable = shutil.which(program_name) or ''
+        config.update({
+            'program_name': program_name,
+            'base_executable': executable,
+            'executable': executable,
+        })
+
     def test_init_setpath(self):
-        # Test Py_SetProgramName() + Py_SetPath()
+        # Test Py_SetPath()
         config = self._get_expected_config()
         paths = config['config']['module_search_paths']
 
@@ -1014,11 +1030,38 @@ def test_init_setpath(self):
             'exec_prefix': '',
             'base_exec_prefix': '',
         }
+        self.default_program_name(config)
         env = {'TESTPATH': os.path.pathsep.join(paths)}
         self.check_all_configs("test_init_setpath", config,
                                api=API_COMPAT, env=env,
                                ignore_stderr=True)
 
+    def test_init_setpath_config(self):
+        # Test Py_SetPath() with PyConfig
+        config = self._get_expected_config()
+        paths = config['config']['module_search_paths']
+
+        config = {
+            # set by Py_SetPath()
+            'module_search_paths': paths,
+            'prefix': '',
+            'base_prefix': '',
+            'exec_prefix': '',
+            'base_exec_prefix': '',
+            # overriden by PyConfig
+            'program_name': 'conf_program_name',
+            'base_executable': 'conf_executable',
+            'executable': 'conf_executable',
+        }
+        env = {'TESTPATH': os.path.pathsep.join(paths)}
+        # Py_SetPath() preinitialized Python using the compat API,
+        # so we need preconfig_api=API_COMPAT.
+        self.check_all_configs("test_init_setpath_config", config,
+                               api=API_PYTHON,
+                               preconfig_api=API_COMPAT,
+                               env=env,
+                               ignore_stderr=True)
+
     def module_search_paths(self, prefix=None, exec_prefix=None):
         config = self._get_expected_config()
         if prefix is None:
@@ -1067,8 +1110,7 @@ def tmpdir_with_python(self):
             yield tmpdir
 
     def test_init_setpythonhome(self):
-        # Test Py_SetPythonHome(home) + PYTHONPATH env var
-        # + Py_SetProgramName()
+        # Test Py_SetPythonHome(home) with PYTHONPATH env var
         config = self._get_expected_config()
         paths = config['config']['module_search_paths']
         paths_str = os.path.pathsep.join(paths)
@@ -1095,7 +1137,8 @@ def test_init_setpythonhome(self):
             'base_exec_prefix': exec_prefix,
             'pythonpath_env': paths_str,
         }
-        env = {'TESTHOME': home, 'TESTPATH': paths_str}
+        self.default_program_name(config)
+        env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
         self.check_all_configs("test_init_setpythonhome", config,
                                api=API_COMPAT, env=env)
 
diff --git a/PC/getpathp.c b/PC/getpathp.c
index 0eb75b8daf31..8bac592aefdb 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -1099,11 +1099,11 @@ calculate_free(PyCalculatePath *calculate)
    - __PYVENV_LAUNCHER__ environment variable
    - GetModuleFileNameW(NULL): fully qualified path of the executable file of
      the current process
-   - .pth configuration file
+   - ._pth configuration file
    - pyvenv.cfg configuration file
    - Registry key "Software\Python\PythonCore\X.Y\PythonPath"
-     of HKEY_LOCAL_MACHINE and HKEY_CURRENT_USER where X.Y is the Python
-     version (major.minor).
+     of HKEY_CURRENT_USER and HKEY_LOCAL_MACHINE where X.Y is the Python
+     version.
 
    Outputs, 'pathconfig' fields:
 
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index ed07606398d8..14fca24f318f 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -1425,8 +1425,6 @@ static int test_init_sys_add(void)
 
 static int test_init_setpath(void)
 {
-    Py_SetProgramName(PROGRAM_NAME);
-
     char *env = getenv("TESTPATH");
     if (!env) {
         fprintf(stderr, "missing TESTPATH env var\n");
@@ -1448,23 +1446,35 @@ static int test_init_setpath(void)
 }
 
 
-static int mysetenv(const char *name, const char *value)
+static int test_init_setpath_config(void)
 {
-    size_t len = strlen(name) + 1 + strlen(value) + 1;
-    char *env = PyMem_RawMalloc(len);
-    if (env == NULL) {
-        fprintf(stderr, "out of memory\n");
-        return -1;
+    char *env = getenv("TESTPATH");
+    if (!env) {
+        fprintf(stderr, "missing TESTPATH env var\n");
+        return 1;
     }
-    strcpy(env, name);
-    strcat(env, "=");
-    strcat(env, value);
+    wchar_t *path = Py_DecodeLocale(env, NULL);
+    if (path == NULL) {
+        fprintf(stderr, "failed to decode TESTPATH\n");
+        return 1;
+    }
+    Py_SetPath(path);
+    PyMem_RawFree(path);
+    putenv("TESTPATH=");
 
-    putenv(env);
+    PyStatus status;
+    PyConfig config;
 
-    /* Don't call PyMem_RawFree(env), but leak env memory block:
-       putenv() does not copy the string. */
+    status = PyConfig_InitPythonConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+    config_set_string(&config, &config.program_name, L"conf_program_name");
+    config_set_string(&config, &config.executable, L"conf_executable");
+    init_from_config_clear(&config);
 
+    dump_config();
+    Py_Finalize();
     return 0;
 }
 
@@ -1485,19 +1495,6 @@ static int test_init_setpythonhome(void)
     PyMem_RawFree(home);
     putenv("TESTHOME=");
 
-    char *path = getenv("TESTPATH");
-    if (!path) {
-        fprintf(stderr, "missing TESTPATH env var\n");
-        return 1;
-    }
-
-    if (mysetenv("PYTHONPATH", path) < 0) {
-        return 1;
-    }
-    putenv("TESTPATH=");
-
-    Py_SetProgramName(PROGRAM_NAME);
-
     Py_Initialize();
     dump_config();
     Py_Finalize();
@@ -1642,6 +1639,7 @@ static struct TestCase TestCases[] = {
     {"test_init_main", test_init_main},
     {"test_init_sys_add", test_init_sys_add},
     {"test_init_setpath", test_init_setpath},
+    {"test_init_setpath_config", test_init_setpath_config},
     {"test_init_setpythonhome", test_init_setpythonhome},
     {"test_run_main", test_run_main},
 
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index 8f76fa50c971..f4e149866878 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -58,7 +58,10 @@ pathconfig_clear(_PyPathConfig *config)
     CLEAR(config->module_search_path);
     CLEAR(config->program_name);
     CLEAR(config->home);
+#ifdef MS_WINDOWS
     CLEAR(config->base_executable);
+#endif
+
 #undef CLEAR
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
@@ -83,9 +86,11 @@ pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2)
     COPY_ATTR(module_search_path);
     COPY_ATTR(program_name);
     COPY_ATTR(home);
+#ifdef MS_WINDOWS
     config->isolated = config2->isolated;
     config->site_import = config2->site_import;
     COPY_ATTR(base_executable);
+#endif
 
 #undef COPY_ATTR
 
@@ -189,12 +194,14 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
             } \
         }
 
-    COPY_CONFIG(base_executable, base_executable);
     COPY_CONFIG(program_full_path, executable);
     COPY_CONFIG(prefix, prefix);
     COPY_CONFIG(exec_prefix, exec_prefix);
     COPY_CONFIG(program_name, program_name);
     COPY_CONFIG(home, home);
+#ifdef MS_WINDOWS
+    COPY_CONFIG(base_executable, base_executable);
+#endif
 
 #undef COPY_CONFIG
 
@@ -330,18 +337,32 @@ config_calculate_pathconfig(PyConfig *config)
             } \
         }
 
+#ifdef MS_WINDOWS
+    if (config->executable != NULL && config->base_executable == NULL) {
+        /* If executable is set explicitly in the configuration,
+           ignore calculated base_executable: _PyConfig_InitPathConfig()
+           will copy executable to base_executable */
+    }
+    else {
+        COPY_ATTR(base_executable, base_executable);
+    }
+#endif
+
     COPY_ATTR(program_full_path, executable);
     COPY_ATTR(prefix, prefix);
     COPY_ATTR(exec_prefix, exec_prefix);
-    COPY_ATTR(base_executable, base_executable);
+
 #undef COPY_ATTR
 
+#ifdef MS_WINDOWS
+    /* If a ._pth file is found: isolated and site_import are overriden */
     if (pathconfig.isolated != -1) {
         config->isolated = pathconfig.isolated;
     }
     if (pathconfig.site_import != -1) {
         config->site_import = pathconfig.site_import;
     }
+#endif
 
     status = _PyStatus_OK();
     goto done;
@@ -360,9 +381,9 @@ _PyConfig_InitPathConfig(PyConfig *config)
 {
     /* Do we need to calculate the path? */
     if (!config->module_search_paths_set
-        || (config->executable == NULL)
-        || (config->prefix == NULL)
-        || (config->exec_prefix == NULL))
+        || config->executable == NULL
+        || config->prefix == NULL
+        || config->exec_prefix == NULL)
     {
         PyStatus status = config_calculate_pathconfig(config);
         if (_PyStatus_EXCEPTION(status)) {
@@ -442,7 +463,9 @@ pathconfig_global_init(void)
     assert(_Py_path_config.module_search_path != NULL);
     assert(_Py_path_config.program_name != NULL);
     /* home can be NULL */
+#ifdef MS_WINDOWS
     assert(_Py_path_config.base_executable != NULL);
+#endif
 }
 
 


From webhook-mailer at python.org  Wed Sep 25 21:15:12 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 26 Sep 2019 01:15:12 -0000
Subject: [Python-checkins] bpo-38234: Complete init config documentation
 (GH-16404)
Message-ID: <mailman.92.1569460513.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/88feaecd46a8f427e30ef7ad8cfcddfe392a2402
commit: 88feaecd46a8f427e30ef7ad8cfcddfe392a2402
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-26T03:15:07+02:00
summary:

bpo-38234: Complete init config documentation (GH-16404)

files:
M Doc/c-api/init_config.rst

diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index d98bcda20747..0c3c725c841a 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -241,6 +241,7 @@ PyPreConfig
       locale to decide if it should be coerced.
 
    .. c:member:: int coerce_c_locale_warn
+
       If non-zero, emit a warning if the C locale is coerced.
 
    .. c:member:: int dev_mode
@@ -300,7 +301,7 @@ For :ref:`Python Configuration <init-python-config>`
 (:c:func:`PyPreConfig_InitPythonConfig`), if Python is initialized with
 command line arguments, the command line arguments must also be passed to
 preinitialize Python, since they have an effect on the pre-configuration
-like encodings. For example, the :option:`-X` ``utf8`` command line option
+like encodings. For example, the :option:`-X utf8 <-X>` command line option
 enables the UTF-8 Mode.
 
 ``PyMem_SetAllocator()`` can be called after :c:func:`Py_PreInitialize` and
@@ -464,7 +465,7 @@ PyConfig
 
    .. c:member:: int dev_mode
 
-      Development mode: see :option:`-X` ``dev``.
+      Development mode: see :option:`-X dev <-X>`.
 
    .. c:member:: int dump_refs
 
@@ -482,7 +483,7 @@ PyConfig
 
    .. c:member:: int faulthandler
 
-      If non-zero, call :func:`faulthandler.enable`.
+      If non-zero, call :func:`faulthandler.enable` at startup.
 
    .. c:member:: wchar_t* filesystem_encoding
 
@@ -504,6 +505,9 @@ PyConfig
 
       Python home directory.
 
+      Initialized from :envvar:`PYTHONHOME` environment variable value by
+      default.
+
    .. c:member:: int import_time
 
       If non-zero, profile import time.
@@ -561,7 +565,7 @@ PyConfig
 
       :data:`sys.path`. If :c:member:`~PyConfig.module_search_paths_set` is
       equal to 0, the :c:member:`~PyConfig.module_search_paths` is overridden
-      by the function computing the :ref:`Path Configuration
+      by the function calculating the :ref:`Path Configuration
       <init-path-config>`.
 
    .. c:member:: int optimization_level
@@ -586,9 +590,9 @@ PyConfig
 
    .. c:member:: int pathconfig_warnings
 
-      If equal to 0, suppress warnings when computing the path configuration
-      (Unix only, Windows does not log any warning). Otherwise, warnings are
-      written into ``stderr``.
+      If equal to 0, suppress warnings when calculating the :ref:`Path
+      Configuration <init-path-config>` (Unix only, Windows does not log any
+      warning). Otherwise, warnings are written into ``stderr``.
 
    .. c:member:: wchar_t* prefix
 
@@ -596,39 +600,46 @@ PyConfig
 
    .. c:member:: wchar_t* program_name
 
-      Program name.
+      Program name. Used to initialize :c:member:`~PyConfig.executable`, and in
+      early error messages.
 
    .. c:member:: wchar_t* pycache_prefix
 
-      ``.pyc`` cache prefix.
+      :data:`sys.pycache_prefix`: ``.pyc`` cache prefix.
+
+      If NULL, :data:`sys.pycache_prefix` is set to ``None``.
 
    .. c:member:: int quiet
 
       Quiet mode. For example, don't display the copyright and version messages
-      even in interactive mode.
+      in interactive mode.
 
    .. c:member:: wchar_t* run_command
 
-      ``python3 -c COMMAND`` argument.
+      ``python3 -c COMMAND`` argument. Used by :c:func:`Py_RunMain`.
 
    .. c:member:: wchar_t* run_filename
 
-      ``python3 FILENAME`` argument.
+      ``python3 FILENAME`` argument. Used by :c:func:`Py_RunMain`.
 
    .. c:member:: wchar_t* run_module
 
-      ``python3 -m MODULE`` argument.
+      ``python3 -m MODULE`` argument. Used by :c:func:`Py_RunMain`.
 
    .. c:member:: int show_alloc_count
 
       Show allocation counts at exit?
 
+      Set to 1 by :option:`-X showalloccount <-X>` command line option.
+
       Need a special Python build with ``COUNT_ALLOCS`` macro defined.
 
    .. c:member:: int show_ref_count
 
       Show total reference count at exit?
 
+      Set to 1 by :option:`-X showrefcount <-X>` command line option.
+
       Need a debug build of Python (``Py_REF_DEBUG`` macro must be defined).
 
    .. c:member:: int site_import
@@ -647,7 +658,7 @@ PyConfig
 
    .. c:member:: int tracemalloc
 
-      If non-zero, call :func:`tracemalloc.start`.
+      If non-zero, call :func:`tracemalloc.start` at startup.
 
    .. c:member:: int use_environment
 
@@ -669,6 +680,9 @@ PyConfig
 
       If non-zero, write ``.pyc`` files.
 
+      :data:`sys.dont_write_bytecode` is initialized to the inverted value of
+      :c:member:`~PyConfig.write_bytecode`.
+
    .. c:member:: PyWideStringList xoptions
 
       :data:`sys._xoptions`.
@@ -694,8 +708,8 @@ Function to initialize Python:
 The caller is responsible to handle exceptions (error or exit) using
 :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`.
 
-``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or
-``PyImport_ExtendInittab()`` is used: they must be set or called after Python
+If ``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or
+``PyImport_ExtendInittab()`` are used, they must be set or called after Python
 preinitialization and before the Python initialization.
 
 Example setting the program name::
@@ -760,7 +774,7 @@ configuration, and then override some parameters::
 
         /* Append our custom search path to sys.path */
         status = PyWideStringList_Append(&config.module_search_paths,
-                                      L"/path/to/more/modules");
+                                         L"/path/to/more/modules");
         if (PyStatus_Exception(status)) {
             goto done;
         }
@@ -791,9 +805,9 @@ isolate Python from the system. For example, to embed Python into an
 application.
 
 This configuration ignores global configuration variables, environments
-variables and command line arguments (:c:member:`PyConfig.argv` is not parsed).
-The C standard streams (ex: ``stdout``) and the LC_CTYPE locale are left
-unchanged by default.
+variables, command line arguments (:c:member:`PyConfig.argv` is not parsed)
+and user site directory. The C standard streams (ex: ``stdout``) and the
+LC_CTYPE locale are left unchanged. Signal handlers are not installed.
 
 Configuration files are still used with this configuration. Set the
 :ref:`Path Configuration <init-path-config>` ("output fields") to ignore these
@@ -970,7 +984,7 @@ initialization, the core feature of the :pep:`432`:
   * Builtin exceptions;
   * Builtin and frozen modules;
   * The :mod:`sys` module is only partially initialized
-    (ex: :data:`sys.path` doesn't exist yet);
+    (ex: :data:`sys.path` doesn't exist yet).
 
 * "Main" initialization phase, Python is fully initialized:
 
@@ -996,9 +1010,9 @@ No module is imported during the "Core" phase and the ``importlib`` module is
 not configured: the :ref:`Path Configuration <init-path-config>` is only
 applied during the "Main" phase. It may allow to customize Python in Python to
 override or tune the :ref:`Path Configuration <init-path-config>`, maybe
-install a custom sys.meta_path importer or an import hook, etc.
+install a custom :data:`sys.meta_path` importer or an import hook, etc.
 
-It may become possible to compute the :ref:`Path Configuration
+It may become possible to calculatin the :ref:`Path Configuration
 <init-path-config>` in Python, after the Core phase and before the Main phase,
 which is one of the :pep:`432` motivation.
 


From webhook-mailer at python.org  Wed Sep 25 22:01:54 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 26 Sep 2019 02:01:54 -0000
Subject: [Python-checkins] bpo-38234: Fix
 test_embed.test_init_setpath_config() on FreeBSD (GH-16406)
Message-ID: <mailman.93.1569463315.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/49d99f01e6e51acec5ca57a02e857f0796bc418b
commit: 49d99f01e6e51acec5ca57a02e857f0796bc418b
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-26T04:01:49+02:00
summary:

bpo-38234: Fix test_embed.test_init_setpath_config() on FreeBSD (GH-16406)

Explicitly preinitializes with a Python preconfiguration to avoid
Py_SetPath() implicit preinitialization with a compat
preconfiguration.

Fix also test_init_setpath() and test_init_setpythonhome() on macOS:
use self.test_exe as the executable (and base_executable), rather
than shutil.which('python3').

files:
M Lib/test/test_embed.py
M Programs/_testembed.c

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 8855e907c253..92b5136db7ea 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -15,6 +15,7 @@
 
 
 MS_WINDOWS = (os.name == 'nt')
+MACOS = (sys.platform == 'darwin')
 
 PYMEM_ALLOCATOR_NOT_SET = 0
 PYMEM_ALLOCATOR_DEBUG = 2
@@ -1011,7 +1012,10 @@ def default_program_name(self, config):
             executable = self.test_exe
         else:
             program_name = 'python3'
-            executable = shutil.which(program_name) or ''
+            if MACOS:
+                executable = self.test_exe
+            else:
+                executable = shutil.which(program_name) or ''
         config.update({
             'program_name': program_name,
             'base_executable': executable,
@@ -1054,13 +1058,8 @@ def test_init_setpath_config(self):
             'executable': 'conf_executable',
         }
         env = {'TESTPATH': os.path.pathsep.join(paths)}
-        # Py_SetPath() preinitialized Python using the compat API,
-        # so we need preconfig_api=API_COMPAT.
         self.check_all_configs("test_init_setpath_config", config,
-                               api=API_PYTHON,
-                               preconfig_api=API_COMPAT,
-                               env=env,
-                               ignore_stderr=True)
+                               api=API_PYTHON, env=env, ignore_stderr=True)
 
     def module_search_paths(self, prefix=None, exec_prefix=None):
         config = self._get_expected_config()
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index 14fca24f318f..83c266b885af 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -1448,6 +1448,17 @@ static int test_init_setpath(void)
 
 static int test_init_setpath_config(void)
 {
+    PyStatus status;
+    PyPreConfig preconfig;
+    PyPreConfig_InitPythonConfig(&preconfig);
+
+    /* Explicitly preinitializes with Python preconfiguration to avoid
+      Py_SetPath() implicit preinitialization with compat preconfiguration. */
+    status = Py_PreInitialize(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
     char *env = getenv("TESTPATH");
     if (!env) {
         fprintf(stderr, "missing TESTPATH env var\n");
@@ -1462,7 +1473,6 @@ static int test_init_setpath_config(void)
     PyMem_RawFree(path);
     putenv("TESTPATH=");
 
-    PyStatus status;
     PyConfig config;
 
     status = PyConfig_InitPythonConfig(&config);


From webhook-mailer at python.org  Thu Sep 26 00:34:04 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 26 Sep 2019 04:34:04 -0000
Subject: [Python-checkins] closes bpo-38174: Update vendored expat library
 to 2.2.8. (GH-16346)
Message-ID: <mailman.94.1569472445.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/52b940803860e37bcc3f6096b2d24e7c20a0e807
commit: 52b940803860e37bcc3f6096b2d24e7c20a0e807
branch: master
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-25T21:33:58-07:00
summary:

closes bpo-38174: Update vendored expat library to 2.2.8. (GH-16346)

Fixes CVE-2019-15903. See full changelog at https://github.com/libexpat/libexpat/blob/R_2_2_8/expat/Changes.

files:
A Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
D Modules/expat/loadlibrary.c
M Modules/expat/asciitab.h
M Modules/expat/expat.h
M Modules/expat/expat_external.h
M Modules/expat/iasciitab.h
M Modules/expat/internal.h
M Modules/expat/latin1tab.h
M Modules/expat/nametab.h
M Modules/expat/siphash.h
M Modules/expat/utf8tab.h
M Modules/expat/winconfig.h
M Modules/expat/xmlparse.c
M Modules/expat/xmlrole.c
M Modules/expat/xmlrole.h
M Modules/expat/xmltok.c
M Modules/expat/xmltok.h
M Modules/expat/xmltok_impl.c
M Modules/expat/xmltok_impl.h
M Modules/expat/xmltok_ns.c
M PCbuild/_elementtree.vcxproj
M PCbuild/_elementtree.vcxproj.filters
M PCbuild/pyexpat.vcxproj
M PCbuild/pyexpat.vcxproj.filters

diff --git a/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst b/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
new file mode 100644
index 000000000000..e3da6fcdd139
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
@@ -0,0 +1,2 @@
+Update vendorized expat library version to 2.2.8, which resolves
+CVE-2019-15903.
diff --git a/Modules/expat/asciitab.h b/Modules/expat/asciitab.h
index 2f59fd929065..63b1d1b4482e 100644
--- a/Modules/expat/asciitab.h
+++ b/Modules/expat/asciitab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
-/* 0x0C */ BT_NONXML, BT_CR, BT_NONXML, BT_NONXML,
-/* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
-/* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
-/* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
-/* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
-/* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
-/* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
-/* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
-/* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
-/* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
+    /* 0x0C */ BT_NONXML, BT_CR, BT_NONXML, BT_NONXML,
+    /* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
+    /* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
+    /* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
+    /* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
+    /* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
+    /* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
+    /* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
+    /* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
+    /* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h
index 56399da717bb..6c8eb1fda4a3 100644
--- a/Modules/expat/expat.h
+++ b/Modules/expat/expat.h
@@ -33,15 +33,6 @@
 #ifndef Expat_INCLUDED
 #define Expat_INCLUDED 1
 
-#ifdef __VMS
-/*      0        1         2         3      0        1         2         3
-        1234567890123456789012345678901     1234567890123456789012345678901 */
-#define XML_SetProcessingInstructionHandler XML_SetProcessingInstrHandler
-#define XML_SetUnparsedEntityDeclHandler    XML_SetUnparsedEntDeclHandler
-#define XML_SetStartNamespaceDeclHandler    XML_SetStartNamespcDeclHandler
-#define XML_SetExternalEntityRefHandlerArg  XML_SetExternalEntRefHandlerArg
-#endif
-
 #include <stdlib.h>
 #include "expat_external.h"
 
@@ -53,8 +44,8 @@ struct XML_ParserStruct;
 typedef struct XML_ParserStruct *XML_Parser;
 
 typedef unsigned char XML_Bool;
-#define XML_TRUE   ((XML_Bool) 1)
-#define XML_FALSE  ((XML_Bool) 0)
+#define XML_TRUE ((XML_Bool)1)
+#define XML_FALSE ((XML_Bool)0)
 
 /* The XML_Status enum gives the possible return values for several
    API functions.  The preprocessor #defines are included so this
@@ -164,25 +155,23 @@ enum XML_Content_Quant {
 typedef struct XML_cp XML_Content;
 
 struct XML_cp {
-  enum XML_Content_Type         type;
-  enum XML_Content_Quant        quant;
-  XML_Char *                    name;
-  unsigned int                  numchildren;
-  XML_Content *                 children;
+  enum XML_Content_Type type;
+  enum XML_Content_Quant quant;
+  XML_Char *name;
+  unsigned int numchildren;
+  XML_Content *children;
 };
 
-
 /* This is called for an element declaration. See above for
    description of the model argument. It's the caller's responsibility
    to free model when finished with it.
 */
-typedef void (XMLCALL *XML_ElementDeclHandler) (void *userData,
-                                                const XML_Char *name,
-                                                XML_Content *model);
+typedef void(XMLCALL *XML_ElementDeclHandler)(void *userData,
+                                              const XML_Char *name,
+                                              XML_Content *model);
 
 XMLPARSEAPI(void)
-XML_SetElementDeclHandler(XML_Parser parser,
-                          XML_ElementDeclHandler eldecl);
+XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl);
 
 /* The Attlist declaration handler is called for *each* attribute. So
    a single Attlist declaration with multiple attributes declared will
@@ -192,17 +181,12 @@ XML_SetElementDeclHandler(XML_Parser parser,
    value will be NULL in the case of "#REQUIRED". If "isrequired" is
    true and default is non-NULL, then this is a "#FIXED" default.
 */
-typedef void (XMLCALL *XML_AttlistDeclHandler) (
-                                    void            *userData,
-                                    const XML_Char  *elname,
-                                    const XML_Char  *attname,
-                                    const XML_Char  *att_type,
-                                    const XML_Char  *dflt,
-                                    int              isrequired);
+typedef void(XMLCALL *XML_AttlistDeclHandler)(
+    void *userData, const XML_Char *elname, const XML_Char *attname,
+    const XML_Char *att_type, const XML_Char *dflt, int isrequired);
 
 XMLPARSEAPI(void)
-XML_SetAttlistDeclHandler(XML_Parser parser,
-                          XML_AttlistDeclHandler attdecl);
+XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl);
 
 /* The XML declaration handler is called for *both* XML declarations
    and text declarations. The way to distinguish is that the version
@@ -212,15 +196,13 @@ XML_SetAttlistDeclHandler(XML_Parser parser,
    was no standalone parameter in the declaration, that it was given
    as no, or that it was given as yes.
 */
-typedef void (XMLCALL *XML_XmlDeclHandler) (void           *userData,
-                                            const XML_Char *version,
-                                            const XML_Char *encoding,
-                                            int             standalone);
+typedef void(XMLCALL *XML_XmlDeclHandler)(void *userData,
+                                          const XML_Char *version,
+                                          const XML_Char *encoding,
+                                          int standalone);
 
 XMLPARSEAPI(void)
-XML_SetXmlDeclHandler(XML_Parser parser,
-                      XML_XmlDeclHandler xmldecl);
-
+XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler xmldecl);
 
 typedef struct {
   void *(*malloc_fcn)(size_t size);
@@ -248,7 +230,6 @@ XML_ParserCreate(const XML_Char *encoding);
 XMLPARSEAPI(XML_Parser)
 XML_ParserCreateNS(const XML_Char *encoding, XML_Char namespaceSeparator);
 
-
 /* Constructs a new parser using the memory management suite referred to
    by memsuite. If memsuite is NULL, then use the standard library memory
    suite. If namespaceSeparator is non-NULL it creates a parser with
@@ -265,7 +246,7 @@ XML_ParserCreate_MM(const XML_Char *encoding,
 
 /* Prepare a parser object to be re-used.  This is particularly
    valuable when memory allocation overhead is disproportionately high,
-   such as when a large number of small documents need to be parsed.
+   such as when a large number of small documnents need to be parsed.
    All handlers are cleared from the parser, except for the
    unknownEncodingHandler. The parser's external state is re-initialized
    except for the values of ns and ns_triplets.
@@ -278,31 +259,27 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encoding);
 /* atts is array of name/value pairs, terminated by 0;
    names and values are 0 terminated.
 */
-typedef void (XMLCALL *XML_StartElementHandler) (void *userData,
-                                                 const XML_Char *name,
-                                                 const XML_Char **atts);
-
-typedef void (XMLCALL *XML_EndElementHandler) (void *userData,
-                                               const XML_Char *name);
+typedef void(XMLCALL *XML_StartElementHandler)(void *userData,
+                                               const XML_Char *name,
+                                               const XML_Char **atts);
 
+typedef void(XMLCALL *XML_EndElementHandler)(void *userData,
+                                             const XML_Char *name);
 
 /* s is not 0 terminated. */
-typedef void (XMLCALL *XML_CharacterDataHandler) (void *userData,
-                                                  const XML_Char *s,
-                                                  int len);
+typedef void(XMLCALL *XML_CharacterDataHandler)(void *userData,
+                                                const XML_Char *s, int len);
 
 /* target and data are 0 terminated */
-typedef void (XMLCALL *XML_ProcessingInstructionHandler) (
-                                                void *userData,
-                                                const XML_Char *target,
-                                                const XML_Char *data);
+typedef void(XMLCALL *XML_ProcessingInstructionHandler)(void *userData,
+                                                        const XML_Char *target,
+                                                        const XML_Char *data);
 
 /* data is 0 terminated */
-typedef void (XMLCALL *XML_CommentHandler) (void *userData,
-                                            const XML_Char *data);
+typedef void(XMLCALL *XML_CommentHandler)(void *userData, const XML_Char *data);
 
-typedef void (XMLCALL *XML_StartCdataSectionHandler) (void *userData);
-typedef void (XMLCALL *XML_EndCdataSectionHandler) (void *userData);
+typedef void(XMLCALL *XML_StartCdataSectionHandler)(void *userData);
+typedef void(XMLCALL *XML_EndCdataSectionHandler)(void *userData);
 
 /* This is called for any characters in the XML document for which
    there is no applicable handler.  This includes both characters that
@@ -317,25 +294,23 @@ typedef void (XMLCALL *XML_EndCdataSectionHandler) (void *userData);
    default handler: for example, a comment might be split between
    multiple calls.
 */
-typedef void (XMLCALL *XML_DefaultHandler) (void *userData,
-                                            const XML_Char *s,
-                                            int len);
+typedef void(XMLCALL *XML_DefaultHandler)(void *userData, const XML_Char *s,
+                                          int len);
 
 /* This is called for the start of the DOCTYPE declaration, before
    any DTD or internal subset is parsed.
 */
-typedef void (XMLCALL *XML_StartDoctypeDeclHandler) (
-                                            void *userData,
-                                            const XML_Char *doctypeName,
-                                            const XML_Char *sysid,
-                                            const XML_Char *pubid,
-                                            int has_internal_subset);
+typedef void(XMLCALL *XML_StartDoctypeDeclHandler)(void *userData,
+                                                   const XML_Char *doctypeName,
+                                                   const XML_Char *sysid,
+                                                   const XML_Char *pubid,
+                                                   int has_internal_subset);
 
 /* This is called for the start of the DOCTYPE declaration when the
    closing > is encountered, but after processing any external
    subset.
 */
-typedef void (XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
+typedef void(XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
 
 /* This is called for entity declarations. The is_parameter_entity
    argument will be non-zero if the entity is a parameter entity, zero
@@ -355,20 +330,14 @@ typedef void (XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
    Note that is_parameter_entity can't be changed to XML_Bool, since
    that would break binary compatibility.
 */
-typedef void (XMLCALL *XML_EntityDeclHandler) (
-                              void *userData,
-                              const XML_Char *entityName,
-                              int is_parameter_entity,
-                              const XML_Char *value,
-                              int value_length,
-                              const XML_Char *base,
-                              const XML_Char *systemId,
-                              const XML_Char *publicId,
-                              const XML_Char *notationName);
+typedef void(XMLCALL *XML_EntityDeclHandler)(
+    void *userData, const XML_Char *entityName, int is_parameter_entity,
+    const XML_Char *value, int value_length, const XML_Char *base,
+    const XML_Char *systemId, const XML_Char *publicId,
+    const XML_Char *notationName);
 
 XMLPARSEAPI(void)
-XML_SetEntityDeclHandler(XML_Parser parser,
-                         XML_EntityDeclHandler handler);
+XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler);
 
 /* OBSOLETE -- OBSOLETE -- OBSOLETE
    This handler has been superseded by the EntityDeclHandler above.
@@ -379,24 +348,20 @@ XML_SetEntityDeclHandler(XML_Parser parser,
    entityName, systemId and notationName arguments will never be
    NULL. The other arguments may be.
 */
-typedef void (XMLCALL *XML_UnparsedEntityDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *entityName,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId,
-                                    const XML_Char *notationName);
+typedef void(XMLCALL *XML_UnparsedEntityDeclHandler)(
+    void *userData, const XML_Char *entityName, const XML_Char *base,
+    const XML_Char *systemId, const XML_Char *publicId,
+    const XML_Char *notationName);
 
 /* This is called for a declaration of notation.  The base argument is
    whatever was set by XML_SetBase. The notationName will never be
    NULL.  The other arguments can be.
 */
-typedef void (XMLCALL *XML_NotationDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *notationName,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId);
+typedef void(XMLCALL *XML_NotationDeclHandler)(void *userData,
+                                               const XML_Char *notationName,
+                                               const XML_Char *base,
+                                               const XML_Char *systemId,
+                                               const XML_Char *publicId);
 
 /* When namespace processing is enabled, these are called once for
    each namespace declaration. The call to the start and end element
@@ -404,14 +369,12 @@ typedef void (XMLCALL *XML_NotationDeclHandler) (
    declaration handlers. For an xmlns attribute, prefix will be
    NULL.  For an xmlns="" attribute, uri will be NULL.
 */
-typedef void (XMLCALL *XML_StartNamespaceDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *prefix,
-                                    const XML_Char *uri);
+typedef void(XMLCALL *XML_StartNamespaceDeclHandler)(void *userData,
+                                                     const XML_Char *prefix,
+                                                     const XML_Char *uri);
 
-typedef void (XMLCALL *XML_EndNamespaceDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *prefix);
+typedef void(XMLCALL *XML_EndNamespaceDeclHandler)(void *userData,
+                                                   const XML_Char *prefix);
 
 /* This is called if the document is not standalone, that is, it has an
    external subset or a reference to a parameter entity, but does not
@@ -422,7 +385,7 @@ typedef void (XMLCALL *XML_EndNamespaceDeclHandler) (
    conditions above this handler will only be called if the referenced
    entity was actually read.
 */
-typedef int (XMLCALL *XML_NotStandaloneHandler) (void *userData);
+typedef int(XMLCALL *XML_NotStandaloneHandler)(void *userData);
 
 /* This is called for a reference to an external parsed general
    entity.  The referenced entity is not automatically parsed.  The
@@ -458,12 +421,11 @@ typedef int (XMLCALL *XML_NotStandaloneHandler) (void *userData);
    Note that unlike other handlers the first argument is the parser,
    not userData.
 */
-typedef int (XMLCALL *XML_ExternalEntityRefHandler) (
-                                    XML_Parser parser,
-                                    const XML_Char *context,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId);
+typedef int(XMLCALL *XML_ExternalEntityRefHandler)(XML_Parser parser,
+                                                   const XML_Char *context,
+                                                   const XML_Char *base,
+                                                   const XML_Char *systemId,
+                                                   const XML_Char *publicId);
 
 /* This is called in two situations:
    1) An entity reference is encountered for which no declaration
@@ -475,10 +437,9 @@ typedef int (XMLCALL *XML_ExternalEntityRefHandler) (
          the event would be out of sync with the reporting of the
          declarations or attribute values
 */
-typedef void (XMLCALL *XML_SkippedEntityHandler) (
-                                    void *userData,
-                                    const XML_Char *entityName,
-                                    int is_parameter_entity);
+typedef void(XMLCALL *XML_SkippedEntityHandler)(void *userData,
+                                                const XML_Char *entityName,
+                                                int is_parameter_entity);
 
 /* This structure is filled in by the XML_UnknownEncodingHandler to
    provide information to the parser about encodings that are unknown
@@ -535,8 +496,8 @@ typedef void (XMLCALL *XML_SkippedEntityHandler) (
 typedef struct {
   int map[256];
   void *data;
-  int (XMLCALL *convert)(void *data, const char *s);
-  void (XMLCALL *release)(void *data);
+  int(XMLCALL *convert)(void *data, const char *s);
+  void(XMLCALL *release)(void *data);
 } XML_Encoding;
 
 /* This is called for an encoding that is unknown to the parser.
@@ -554,23 +515,19 @@ typedef struct {
    If info does not describe a suitable encoding, then the parser will
    return an XML_UNKNOWN_ENCODING error.
 */
-typedef int (XMLCALL *XML_UnknownEncodingHandler) (
-                                    void *encodingHandlerData,
-                                    const XML_Char *name,
-                                    XML_Encoding *info);
+typedef int(XMLCALL *XML_UnknownEncodingHandler)(void *encodingHandlerData,
+                                                 const XML_Char *name,
+                                                 XML_Encoding *info);
 
 XMLPARSEAPI(void)
-XML_SetElementHandler(XML_Parser parser,
-                      XML_StartElementHandler start,
+XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start,
                       XML_EndElementHandler end);
 
 XMLPARSEAPI(void)
-XML_SetStartElementHandler(XML_Parser parser,
-                           XML_StartElementHandler handler);
+XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetEndElementHandler(XML_Parser parser,
-                         XML_EndElementHandler handler);
+XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetCharacterDataHandler(XML_Parser parser,
@@ -580,8 +537,7 @@ XMLPARSEAPI(void)
 XML_SetProcessingInstructionHandler(XML_Parser parser,
                                     XML_ProcessingInstructionHandler handler);
 XMLPARSEAPI(void)
-XML_SetCommentHandler(XML_Parser parser,
-                      XML_CommentHandler handler);
+XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetCdataSectionHandler(XML_Parser parser,
@@ -601,20 +557,17 @@ XML_SetEndCdataSectionHandler(XML_Parser parser,
    default handler, or to the skipped entity handler, if one is set.
 */
 XMLPARSEAPI(void)
-XML_SetDefaultHandler(XML_Parser parser,
-                      XML_DefaultHandler handler);
+XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler);
 
 /* This sets the default handler but does not inhibit expansion of
    internal entities.  The entity reference will not be passed to the
    default handler.
 */
 XMLPARSEAPI(void)
-XML_SetDefaultHandlerExpand(XML_Parser parser,
-                            XML_DefaultHandler handler);
+XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetDoctypeDeclHandler(XML_Parser parser,
-                          XML_StartDoctypeDeclHandler start,
+XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start,
                           XML_EndDoctypeDeclHandler end);
 
 XMLPARSEAPI(void)
@@ -622,16 +575,14 @@ XML_SetStartDoctypeDeclHandler(XML_Parser parser,
                                XML_StartDoctypeDeclHandler start);
 
 XMLPARSEAPI(void)
-XML_SetEndDoctypeDeclHandler(XML_Parser parser,
-                             XML_EndDoctypeDeclHandler end);
+XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end);
 
 XMLPARSEAPI(void)
 XML_SetUnparsedEntityDeclHandler(XML_Parser parser,
                                  XML_UnparsedEntityDeclHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetNotationDeclHandler(XML_Parser parser,
-                           XML_NotationDeclHandler handler);
+XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetNamespaceDeclHandler(XML_Parser parser,
@@ -659,8 +610,7 @@ XML_SetExternalEntityRefHandler(XML_Parser parser,
    instead of the parser object.
 */
 XMLPARSEAPI(void)
-XML_SetExternalEntityRefHandlerArg(XML_Parser parser,
-                                   void *arg);
+XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg);
 
 XMLPARSEAPI(void)
 XML_SetSkippedEntityHandler(XML_Parser parser,
@@ -740,7 +690,6 @@ XML_UseParserAsHandlerArg(XML_Parser parser);
 XMLPARSEAPI(enum XML_Error)
 XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD);
 
-
 /* Sets the base to be used for resolving relative URIs in system
    identifiers in declarations.  Resolving relative identifiers is
    left to the application: this value will be passed through as the
@@ -780,10 +729,10 @@ XML_GetIdAttributeIndex(XML_Parser parser);
    info->valueEnd - info->valueStart = 4 bytes.
 */
 typedef struct {
-  XML_Index  nameStart;  /* Offset to beginning of the attribute name. */
-  XML_Index  nameEnd;    /* Offset after the attribute name's last byte. */
-  XML_Index  valueStart; /* Offset to beginning of the attribute value. */
-  XML_Index  valueEnd;   /* Offset after the attribute value's last byte. */
+  XML_Index nameStart;  /* Offset to beginning of the attribute name. */
+  XML_Index nameEnd;    /* Offset after the attribute name's last byte. */
+  XML_Index valueStart; /* Offset to beginning of the attribute value. */
+  XML_Index valueEnd;   /* Offset after the attribute value's last byte. */
 } XML_AttrInfo;
 
 /* Returns an array of XML_AttrInfo structures for the attribute/value pairs
@@ -819,20 +768,20 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal);
    (resumable = 0) an already suspended parser. Some call-backs may
    still follow because they would otherwise get lost. Examples:
    - endElementHandler() for empty elements when stopped in
-     startElementHandler(), 
-   - endNameSpaceDeclHandler() when stopped in endElementHandler(), 
+     startElementHandler(),
+   - endNameSpaceDeclHandler() when stopped in endElementHandler(),
    and possibly others.
 
    Can be called from most handlers, including DTD related call-backs,
    except when parsing an external parameter entity and resumable != 0.
    Returns XML_STATUS_OK when successful, XML_STATUS_ERROR otherwise.
-   Possible error codes: 
+   Possible error codes:
    - XML_ERROR_SUSPENDED: when suspending an already suspended parser.
    - XML_ERROR_FINISHED: when the parser has already finished.
    - XML_ERROR_SUSPEND_PE: when suspending while parsing an external PE.
 
-   When resumable != 0 (true) then parsing is suspended, that is, 
-   XML_Parse() and XML_ParseBuffer() return XML_STATUS_SUSPENDED. 
+   When resumable != 0 (true) then parsing is suspended, that is,
+   XML_Parse() and XML_ParseBuffer() return XML_STATUS_SUSPENDED.
    Otherwise, parsing is aborted, that is, XML_Parse() and XML_ParseBuffer()
    return XML_STATUS_ERROR with error code XML_ERROR_ABORTED.
 
@@ -843,7 +792,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal);
    the externalEntityRefHandler() to call XML_StopParser() on the parent
    parser (recursively), if one wants to stop parsing altogether.
 
-   When suspended, parsing can be resumed by calling XML_ResumeParser(). 
+   When suspended, parsing can be resumed by calling XML_ResumeParser().
 */
 XMLPARSEAPI(enum XML_Status)
 XML_StopParser(XML_Parser parser, XML_Bool resumable);
@@ -851,7 +800,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable);
 /* Resumes parsing after it has been suspended with XML_StopParser().
    Must not be called from within a handler call-back. Returns same
    status codes as XML_Parse() or XML_ParseBuffer().
-   Additional error code XML_ERROR_NOT_SUSPENDED possible.   
+   Additional error code XML_ERROR_NOT_SUSPENDED possible.
 
    *Note*:
    This must be called on the most deeply nested child parser instance
@@ -863,12 +812,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable);
 XMLPARSEAPI(enum XML_Status)
 XML_ResumeParser(XML_Parser parser);
 
-enum XML_Parsing {
-  XML_INITIALIZED,
-  XML_PARSING,
-  XML_FINISHED,
-  XML_SUSPENDED
-};
+enum XML_Parsing { XML_INITIALIZED, XML_PARSING, XML_FINISHED, XML_SUSPENDED };
 
 typedef struct {
   enum XML_Parsing parsing;
@@ -900,8 +844,7 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status);
    Otherwise returns a new XML_Parser object.
 */
 XMLPARSEAPI(XML_Parser)
-XML_ExternalEntityParserCreate(XML_Parser parser,
-                               const XML_Char *context,
+XML_ExternalEntityParserCreate(XML_Parser parser, const XML_Char *context,
                                const XML_Char *encoding);
 
 enum XML_ParamEntityParsing {
@@ -945,8 +888,7 @@ XML_SetParamEntityParsing(XML_Parser parser,
    Note: If parser == NULL, the function will do nothing and return 0.
 */
 XMLPARSEAPI(int)
-XML_SetHashSalt(XML_Parser parser,
-                unsigned long hash_salt);
+XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt);
 
 /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then
    XML_GetErrorCode returns information about the error.
@@ -963,7 +905,7 @@ XML_GetErrorCode(XML_Parser parser);
    be within the relevant markup.  When called outside of the callback
    functions, the position indicated will be just past the last parse
    event (regardless of whether there was an associated callback).
-   
+
    They may also be called after returning from a call to XML_Parse
    or XML_ParseBuffer.  If the return value is XML_STATUS_ERROR then
    the location is the location of the character at which the error
@@ -995,14 +937,12 @@ XML_GetCurrentByteCount(XML_Parser parser);
    the handler that makes the call.
 */
 XMLPARSEAPI(const char *)
-XML_GetInputContext(XML_Parser parser,
-                    int *offset,
-                    int *size);
+XML_GetInputContext(XML_Parser parser, int *offset, int *size);
 
 /* For backwards compatibility with previous versions. */
-#define XML_GetErrorLineNumber   XML_GetCurrentLineNumber
+#define XML_GetErrorLineNumber XML_GetCurrentLineNumber
 #define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber
-#define XML_GetErrorByteIndex    XML_GetCurrentByteIndex
+#define XML_GetErrorByteIndex XML_GetCurrentByteIndex
 
 /* Frees the content model passed to the element declaration handler */
 XMLPARSEAPI(void)
@@ -1062,21 +1002,20 @@ enum XML_FeatureEnum {
 };
 
 typedef struct {
-  enum XML_FeatureEnum  feature;
-  const XML_LChar       *name;
-  long int              value;
+  enum XML_FeatureEnum feature;
+  const XML_LChar *name;
+  long int value;
 } XML_Feature;
 
 XMLPARSEAPI(const XML_Feature *)
 XML_GetFeatureList(void);
 
-
 /* Expat follows the semantic versioning convention.
    See http://semver.org.
 */
 #define XML_MAJOR_VERSION 2
 #define XML_MINOR_VERSION 2
-#define XML_MICRO_VERSION 7
+#define XML_MICRO_VERSION 8
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h
index fc63c2228d3e..f2b75dda8e27 100644
--- a/Modules/expat/expat_external.h
+++ b/Modules/expat/expat_external.h
@@ -35,10 +35,6 @@
 
 /* External API definitions */
 
-#if defined(_MSC_EXTENSIONS) && !defined(__BEOS__) && !defined(__CYGWIN__)
-# define XML_USE_MSC_EXTENSIONS 1
-#endif
-
 /* Expat tries very hard to make the API boundary very specifically
    defined.  There are two macros defined to control this boundary;
    each of these can be defined before including this header to
@@ -61,12 +57,17 @@
    compiled with the cdecl calling convention as the default since
    system headers may assume the cdecl convention.
 */
+
+/* Namespace external symbols to allow multiple libexpat version to
+   co-exist. */
+#include "pyexpatns.h"
+
 #ifndef XMLCALL
-# if defined(_MSC_VER)
-#  define XMLCALL __cdecl
-# elif defined(__GNUC__) && defined(__i386) && !defined(__INTEL_COMPILER)
-#  define XMLCALL __attribute__((cdecl))
-# else
+#  if defined(_MSC_VER)
+#    define XMLCALL __cdecl
+#  elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER)
+#    define XMLCALL __attribute__((cdecl))
+#  else
 /* For any platform which uses this definition and supports more than
    one calling convention, we need to extend this definition to
    declare the convention used on that platform, if it's possible to
@@ -77,49 +78,46 @@
    pre-processor and how to specify the same calling convention as the
    platform's malloc() implementation.
 */
-#  define XMLCALL
-# endif
-#endif  /* not defined XMLCALL */
-
-/* Namespace external symbols to allow multiple libexpat version to
-   co-exist. */
-#include "pyexpatns.h"
-
+#    define XMLCALL
+#  endif
+#endif /* not defined XMLCALL */
 
-#if !defined(XML_STATIC) && !defined(XMLIMPORT)
-# ifndef XML_BUILDING_EXPAT
+#if ! defined(XML_STATIC) && ! defined(XMLIMPORT)
+#  ifndef XML_BUILDING_EXPAT
 /* using Expat from an application */
 
-#  ifdef XML_USE_MSC_EXTENSIONS
-#   define XMLIMPORT __declspec(dllimport)
-#  endif
+#    if defined(_MSC_EXTENSIONS) && ! defined(__BEOS__) && ! defined(__CYGWIN__)
+#      define XMLIMPORT __declspec(dllimport)
+#    endif
 
-# endif
-#endif  /* not defined XML_STATIC */
+#  endif
+#endif /* not defined XML_STATIC */
 
 #ifndef XML_ENABLE_VISIBILITY
-# define XML_ENABLE_VISIBILITY 0
+#  define XML_ENABLE_VISIBILITY 0
 #endif
 
-#if !defined(XMLIMPORT) && XML_ENABLE_VISIBILITY
-# define XMLIMPORT __attribute__ ((visibility ("default")))
+#if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY
+#  define XMLIMPORT __attribute__((visibility("default")))
 #endif
 
 /* If we didn't define it above, define it away: */
 #ifndef XMLIMPORT
-# define XMLIMPORT
+#  define XMLIMPORT
 #endif
 
-#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
-# define XML_ATTR_MALLOC __attribute__((__malloc__))
+#if defined(__GNUC__)                                                          \
+    && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
+#  define XML_ATTR_MALLOC __attribute__((__malloc__))
 #else
-# define XML_ATTR_MALLOC
+#  define XML_ATTR_MALLOC
 #endif
 
-#if defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
-# define XML_ATTR_ALLOC_SIZE(x)  __attribute__((__alloc_size__(x)))
+#if defined(__GNUC__)                                                          \
+    && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
+#  define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x)))
 #else
-# define XML_ATTR_ALLOC_SIZE(x)
+#  define XML_ATTR_ALLOC_SIZE(x)
 #endif
 
 #define XMLPARSEAPI(type) XMLIMPORT type XMLCALL
@@ -129,35 +127,30 @@ extern "C" {
 #endif
 
 #ifdef XML_UNICODE_WCHAR_T
-# ifndef XML_UNICODE
-#  define XML_UNICODE
-# endif
-# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
-#  error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
-# endif
+#  ifndef XML_UNICODE
+#    define XML_UNICODE
+#  endif
+#  if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
+#    error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
+#  endif
 #endif
 
-#ifdef XML_UNICODE     /* Information is UTF-16 encoded. */
-# ifdef XML_UNICODE_WCHAR_T
+#ifdef XML_UNICODE /* Information is UTF-16 encoded. */
+#  ifdef XML_UNICODE_WCHAR_T
 typedef wchar_t XML_Char;
 typedef wchar_t XML_LChar;
-# else
+#  else
 typedef unsigned short XML_Char;
 typedef char XML_LChar;
-# endif /* XML_UNICODE_WCHAR_T */
-#else                  /* Information is UTF-8 encoded. */
+#  endif /* XML_UNICODE_WCHAR_T */
+#else    /* Information is UTF-8 encoded. */
 typedef char XML_Char;
 typedef char XML_LChar;
-#endif /* XML_UNICODE */
+#endif   /* XML_UNICODE */
 
-#ifdef XML_LARGE_SIZE  /* Use large integers for file/stream positions. */
-# if defined(XML_USE_MSC_EXTENSIONS) && _MSC_VER < 1400
-typedef __int64 XML_Index; 
-typedef unsigned __int64 XML_Size;
-# else
+#ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */
 typedef long long XML_Index;
 typedef unsigned long long XML_Size;
-# endif
 #else
 typedef long XML_Index;
 typedef unsigned long XML_Size;
diff --git a/Modules/expat/iasciitab.h b/Modules/expat/iasciitab.h
index ce4a4bf7edef..ea97cfcf678e 100644
--- a/Modules/expat/iasciitab.h
+++ b/Modules/expat/iasciitab.h
@@ -32,34 +32,34 @@
 
 /* Like asciitab.h, except that 0xD has code BT_S rather than BT_CR */
 /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
-/* 0x0C */ BT_NONXML, BT_S, BT_NONXML, BT_NONXML,
-/* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
-/* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
-/* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
-/* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
-/* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
-/* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
-/* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
-/* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
-/* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
+    /* 0x0C */ BT_NONXML, BT_S, BT_NONXML, BT_NONXML,
+    /* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
+    /* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
+    /* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
+    /* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
+    /* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
+    /* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
+    /* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
+    /* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
+    /* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h
index dc4ef0c7e258..60913dab762f 100644
--- a/Modules/expat/internal.h
+++ b/Modules/expat/internal.h
@@ -49,7 +49,7 @@
    USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
-#if defined(__GNUC__) && defined(__i386__) && !defined(__MINGW32__)
+#if defined(__GNUC__) && defined(__i386__) && ! defined(__MINGW32__)
 /* We'll use this version by default only where we know it helps.
 
    regparm() generates warnings on Solaris boxes.   See SF bug #692878.
@@ -59,8 +59,8 @@
    #define FASTCALL __attribute__((stdcall, regparm(3)))
    and let's try this:
 */
-#define FASTCALL __attribute__((regparm(3)))
-#define PTRFASTCALL __attribute__((regparm(3)))
+#  define FASTCALL __attribute__((regparm(3)))
+#  define PTRFASTCALL __attribute__((regparm(3)))
 #endif
 
 /* Using __fastcall seems to have an unexpected negative effect under
@@ -74,55 +74,49 @@
 /* Make sure all of these are defined if they aren't already. */
 
 #ifndef FASTCALL
-#define FASTCALL
+#  define FASTCALL
 #endif
 
 #ifndef PTRCALL
-#define PTRCALL
+#  define PTRCALL
 #endif
 
 #ifndef PTRFASTCALL
-#define PTRFASTCALL
+#  define PTRFASTCALL
 #endif
 
 #ifndef XML_MIN_SIZE
-#if !defined(__cplusplus) && !defined(inline)
-#ifdef __GNUC__
-#define inline __inline
-#endif /* __GNUC__ */
-#endif
+#  if ! defined(__cplusplus) && ! defined(inline)
+#    ifdef __GNUC__
+#      define inline __inline
+#    endif /* __GNUC__ */
+#  endif
 #endif /* XML_MIN_SIZE */
 
 #ifdef __cplusplus
-#define inline inline
+#  define inline inline
 #else
-#ifndef inline
-#define inline
-#endif
+#  ifndef inline
+#    define inline
+#  endif
 #endif
 
 #ifndef UNUSED_P
-# ifdef __GNUC__
-#  define UNUSED_P(p) UNUSED_ ## p __attribute__((__unused__))
-# else
-#  define UNUSED_P(p) UNUSED_ ## p
-# endif
+#  define UNUSED_P(p) (void)p
 #endif
 
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-
 #ifdef XML_ENABLE_VISIBILITY
-#if XML_ENABLE_VISIBILITY
-__attribute__ ((visibility ("default")))
-#endif
+#  if XML_ENABLE_VISIBILITY
+__attribute__((visibility("default")))
+#  endif
 #endif
 void
-_INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** fromLimRef);
-
+_INTERNAL_trim_to_complete_utf8_characters(const char *from,
+                                           const char **fromLimRef);
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/latin1tab.h b/Modules/expat/latin1tab.h
index 95dfa52b1fbe..6f916041355a 100644
--- a/Modules/expat/latin1tab.h
+++ b/Modules/expat/latin1tab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x80 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x84 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x88 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x8C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x90 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x94 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x98 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x9C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA4 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
-/* 0xAC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xB0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xB4 */ BT_OTHER, BT_NMSTRT, BT_OTHER, BT_NAME,
-/* 0xB8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
-/* 0xBC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xC0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xC4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xC8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xCC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xD0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xD4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0xD8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xDC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xEC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xF0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xF4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0xF8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xFC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x84 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x88 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x8C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x90 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x94 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x98 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x9C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA4 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
+    /* 0xAC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xB0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xB4 */ BT_OTHER, BT_NMSTRT, BT_OTHER, BT_NAME,
+    /* 0xB8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
+    /* 0xBC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xC0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xC4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xC8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xCC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xD0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xD4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0xD8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xDC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xEC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xF0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xF4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0xF8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xFC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
diff --git a/Modules/expat/loadlibrary.c b/Modules/expat/loadlibrary.c
deleted file mode 100644
index 2f80b3a379fd..000000000000
--- a/Modules/expat/loadlibrary.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/***************************************************************************
- *                                  _   _ ____  _
- *  Project                     ___| | | |  _ \| |
- *                             / __| | | | |_) | |
- *                            | (__| |_| |  _ <| |___
- *                             \___|\___/|_| \_\_____|
- *
- * Copyright (C) 2016 - 2017, Steve Holme, <steve_holme at hotmail.com>.
- * Copyright (C) 2017, Expat development team
- *
- * All rights reserved.
- * Licensed under the MIT license:
- *
- * Permission to  use, copy,  modify, and distribute  this software  for any
- * purpose with  or without fee is  hereby granted, provided that  the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE  SOFTWARE  IS  PROVIDED  "AS  IS",  WITHOUT  WARRANTY  OF  ANY  KIND,
- * EXPRESS  OR IMPLIED,  INCLUDING  BUT  NOT LIMITED  TO  THE WARRANTIES  OF
- * MERCHANTABILITY, FITNESS FOR A  PARTICULAR PURPOSE AND NONINFRINGEMENT OF
- * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
- * CONTRACT, TORT OR  OTHERWISE, ARISING FROM, OUT OF OR  IN CONNECTION WITH
- * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * Except as contained in this notice,  the name of a copyright holder shall
- * not be used in advertising or otherwise to promote the sale, use or other
- * dealings  in this  Software without  prior written  authorization of  the
- * copyright holder.
- *
- ***************************************************************************/
-
-#if defined(_WIN32)
-
-#include <windows.h>
-#include <tchar.h>
-
-
-HMODULE _Expat_LoadLibrary(LPCTSTR filename);
-
-
-#if !defined(LOAD_WITH_ALTERED_SEARCH_PATH)
-#define LOAD_WITH_ALTERED_SEARCH_PATH  0x00000008
-#endif
-
-#if !defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
-#define LOAD_LIBRARY_SEARCH_SYSTEM32   0x00000800
-#endif
-
-/* We use our own typedef here since some headers might lack these */
-typedef HMODULE (APIENTRY *LOADLIBRARYEX_FN)(LPCTSTR, HANDLE, DWORD);
-
-/* See function definitions in winbase.h */
-#ifdef UNICODE
-#  ifdef _WIN32_WCE
-#    define LOADLIBRARYEX  L"LoadLibraryExW"
-#  else
-#    define LOADLIBRARYEX  "LoadLibraryExW"
-#  endif
-#else
-#  define LOADLIBRARYEX    "LoadLibraryExA"
-#endif
-
-
-/*
- * _Expat_LoadLibrary()
- *
- * This is used to dynamically load DLLs using the most secure method available
- * for the version of Windows that we are running on.
- *
- * Parameters:
- *
- * filename  [in] - The filename or full path of the DLL to load. If only the
- *                  filename is passed then the DLL will be loaded from the
- *                  Windows system directory.
- *
- * Returns the handle of the module on success; otherwise NULL.
- */
-HMODULE _Expat_LoadLibrary(LPCTSTR filename)
-{
-  HMODULE hModule = NULL;
-  LOADLIBRARYEX_FN pLoadLibraryEx = NULL;
-
-  /* Get a handle to kernel32 so we can access it's functions at runtime */
-  HMODULE hKernel32 = GetModuleHandle(TEXT("kernel32"));
-  if(!hKernel32)
-    return NULL;  /* LCOV_EXCL_LINE */
-
-  /* Attempt to find LoadLibraryEx() which is only available on Windows 2000
-     and above */
-  pLoadLibraryEx = (LOADLIBRARYEX_FN) GetProcAddress(hKernel32, LOADLIBRARYEX);
-
-  /* Detect if there's already a path in the filename and load the library if
-     there is. Note: Both back slashes and forward slashes have been supported
-     since the earlier days of DOS at an API level although they are not
-     supported by command prompt */
-  if(_tcspbrk(filename, TEXT("\\/"))) {
-    /** !checksrc! disable BANNEDFUNC 1 **/
-    hModule = pLoadLibraryEx ?
-      pLoadLibraryEx(filename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) :
-      LoadLibrary(filename);
-  }
-  /* Detect if KB2533623 is installed, as LOAD_LIBARY_SEARCH_SYSTEM32 is only
-     supported on Windows Vista, Windows Server 2008, Windows 7 and Windows
-     Server 2008 R2 with this patch or natively on Windows 8 and above */
-  else if(pLoadLibraryEx && GetProcAddress(hKernel32, "AddDllDirectory")) {
-    /* Load the DLL from the Windows system directory */
-    hModule = pLoadLibraryEx(filename, NULL, LOAD_LIBRARY_SEARCH_SYSTEM32);
-  }
-  else {
-    /* Attempt to get the Windows system path */
-    UINT systemdirlen = GetSystemDirectory(NULL, 0);
-    if(systemdirlen) {
-      /* Allocate space for the full DLL path (Room for the null terminator
-         is included in systemdirlen) */
-      size_t filenamelen = _tcslen(filename);
-      TCHAR *path = malloc(sizeof(TCHAR) * (systemdirlen + 1 + filenamelen));
-      if(path && GetSystemDirectory(path, systemdirlen)) {
-        /* Calculate the full DLL path */
-        _tcscpy(path + _tcslen(path), TEXT("\\"));
-        _tcscpy(path + _tcslen(path), filename);
-
-        /* Load the DLL from the Windows system directory */
-        /** !checksrc! disable BANNEDFUNC 1 **/
-        hModule = pLoadLibraryEx ?
-          pLoadLibraryEx(path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) :
-          LoadLibrary(path);
-
-      }
-      free(path);
-    }
-  }
-
-  return hModule;
-}
-
-#else /* defined(_WIN32) */
-
-/* ISO C requires a translation unit to contain at least one declaration
-   [-Wempty-translation-unit] */
-typedef int _TRANSLATION_UNIT_LOAD_LIBRARY_C_NOT_EMTPY;
-
-#endif /* defined(_WIN32) */
diff --git a/Modules/expat/nametab.h b/Modules/expat/nametab.h
index bfa2bd38cd95..3681df348eeb 100644
--- a/Modules/expat/nametab.h
+++ b/Modules/expat/nametab.h
@@ -31,152 +31,106 @@
 */
 
 static const unsigned namingBitmap[] = {
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0x00000000, 0x04000000, 0x87FFFFFE, 0x07FFFFFE,
-0x00000000, 0x00000000, 0xFF7FFFFF, 0xFF7FFFFF,
-0xFFFFFFFF, 0x7FF3FFFF, 0xFFFFFDFE, 0x7FFFFFFF,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFE00F, 0xFC31FFFF,
-0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
-0xFFFFFFFF, 0xF80001FF, 0x00000003, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFD740, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
-0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
-0xFFFF0003, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF,
-0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
-0x0000007F, 0x00000000, 0xFFFF0000, 0x000707FF,
-0x00000000, 0x07FFFFFE, 0x000007FE, 0xFFFE0000,
-0xFFFFFFFF, 0x7CFFFFFF, 0x002F7FFF, 0x00000060,
-0xFFFFFFE0, 0x23FFFFFF, 0xFF000000, 0x00000003,
-0xFFF99FE0, 0x03C5FDFF, 0xB0000000, 0x00030003,
-0xFFF987E0, 0x036DFDFF, 0x5E000000, 0x001C0000,
-0xFFFBAFE0, 0x23EDFDFF, 0x00000000, 0x00000001,
-0xFFF99FE0, 0x23CDFDFF, 0xB0000000, 0x00000003,
-0xD63DC7E0, 0x03BFC718, 0x00000000, 0x00000000,
-0xFFFDDFE0, 0x03EFFDFF, 0x00000000, 0x00000003,
-0xFFFDDFE0, 0x03EFFDFF, 0x40000000, 0x00000003,
-0xFFFDDFE0, 0x03FFFDFF, 0x00000000, 0x00000003,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFE, 0x000D7FFF, 0x0000003F, 0x00000000,
-0xFEF02596, 0x200D6CAE, 0x0000001F, 0x00000000,
-0x00000000, 0x00000000, 0xFFFFFEFF, 0x000003FF,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0xFFFFFFFF, 0xFFFF003F, 0x007FFFFF,
-0x0007DAED, 0x50000000, 0x82315001, 0x002C62AB,
-0x40000000, 0xF580C900, 0x00000007, 0x02010800,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0x0FFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x03FFFFFF,
-0x3F3FFFFF, 0xFFFFFFFF, 0xAAFF3F3F, 0x3FFFFFFF,
-0xFFFFFFFF, 0x5FDFFFFF, 0x0FCF1FDC, 0x1FDC1FFF,
-0x00000000, 0x00004C40, 0x00000000, 0x00000000,
-0x00000007, 0x00000000, 0x00000000, 0x00000000,
-0x00000080, 0x000003FE, 0xFFFFFFFE, 0xFFFFFFFF,
-0x001FFFFF, 0xFFFFFFFE, 0xFFFFFFFF, 0x07FFFFFF,
-0xFFFFFFE0, 0x00001FFF, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0x0000003F, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0x0000000F, 0x00000000, 0x00000000,
-0x00000000, 0x07FF6000, 0x87FFFFFE, 0x07FFFFFE,
-0x00000000, 0x00800000, 0xFF7FFFFF, 0xFF7FFFFF,
-0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
-0xFFFFFFFF, 0xF80001FF, 0x00030003, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000003,
-0xFFFFD7C0, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
-0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
-0xFFFF007B, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF,
-0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
-0xFFFE007F, 0xBBFFFFFB, 0xFFFF0016, 0x000707FF,
-0x00000000, 0x07FFFFFE, 0x0007FFFF, 0xFFFF03FF,
-0xFFFFFFFF, 0x7CFFFFFF, 0xFFEF7FFF, 0x03FF3DFF,
-0xFFFFFFEE, 0xF3FFFFFF, 0xFF1E3FFF, 0x0000FFCF,
-0xFFF99FEE, 0xD3C5FDFF, 0xB080399F, 0x0003FFCF,
-0xFFF987E4, 0xD36DFDFF, 0x5E003987, 0x001FFFC0,
-0xFFFBAFEE, 0xF3EDFDFF, 0x00003BBF, 0x0000FFC1,
-0xFFF99FEE, 0xF3CDFDFF, 0xB0C0398F, 0x0000FFC3,
-0xD63DC7EC, 0xC3BFC718, 0x00803DC7, 0x0000FF80,
-0xFFFDDFEE, 0xC3EFFDFF, 0x00603DDF, 0x0000FFC3,
-0xFFFDDFEC, 0xC3EFFDFF, 0x40603DDF, 0x0000FFC3,
-0xFFFDDFEC, 0xC3FFFDFF, 0x00803DCF, 0x0000FFC3,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFE, 0x07FF7FFF, 0x03FF7FFF, 0x00000000,
-0xFEF02596, 0x3BFF6CAE, 0x03FF3F5F, 0x00000000,
-0x03000000, 0xC2A003FF, 0xFFFFFEFF, 0xFFFE03FF,
-0xFEBF0FDF, 0x02FE3FFF, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x1FFF0000, 0x00000002,
-0x000000A0, 0x003EFFFE, 0xFFFFFFFE, 0xFFFFFFFF,
-0x661FFFFF, 0xFFFFFFFE, 0xFFFFFFFF, 0x77FFFFFF,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x04000000,
+    0x87FFFFFE, 0x07FFFFFE, 0x00000000, 0x00000000, 0xFF7FFFFF, 0xFF7FFFFF,
+    0xFFFFFFFF, 0x7FF3FFFF, 0xFFFFFDFE, 0x7FFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFE00F, 0xFC31FFFF, 0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xF80001FF, 0x00000003, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFD740, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
+    0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF, 0xFFFF0003, 0xFFFFFFFF,
+    0xFFFF199F, 0x033FCFFF, 0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
+    0x0000007F, 0x00000000, 0xFFFF0000, 0x000707FF, 0x00000000, 0x07FFFFFE,
+    0x000007FE, 0xFFFE0000, 0xFFFFFFFF, 0x7CFFFFFF, 0x002F7FFF, 0x00000060,
+    0xFFFFFFE0, 0x23FFFFFF, 0xFF000000, 0x00000003, 0xFFF99FE0, 0x03C5FDFF,
+    0xB0000000, 0x00030003, 0xFFF987E0, 0x036DFDFF, 0x5E000000, 0x001C0000,
+    0xFFFBAFE0, 0x23EDFDFF, 0x00000000, 0x00000001, 0xFFF99FE0, 0x23CDFDFF,
+    0xB0000000, 0x00000003, 0xD63DC7E0, 0x03BFC718, 0x00000000, 0x00000000,
+    0xFFFDDFE0, 0x03EFFDFF, 0x00000000, 0x00000003, 0xFFFDDFE0, 0x03EFFDFF,
+    0x40000000, 0x00000003, 0xFFFDDFE0, 0x03FFFDFF, 0x00000000, 0x00000003,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFE, 0x000D7FFF,
+    0x0000003F, 0x00000000, 0xFEF02596, 0x200D6CAE, 0x0000001F, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFFEFF, 0x000003FF, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0xFFFFFFFF, 0xFFFF003F, 0x007FFFFF, 0x0007DAED, 0x50000000,
+    0x82315001, 0x002C62AB, 0x40000000, 0xF580C900, 0x00000007, 0x02010800,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0FFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0x03FFFFFF, 0x3F3FFFFF, 0xFFFFFFFF, 0xAAFF3F3F, 0x3FFFFFFF,
+    0xFFFFFFFF, 0x5FDFFFFF, 0x0FCF1FDC, 0x1FDC1FFF, 0x00000000, 0x00004C40,
+    0x00000000, 0x00000000, 0x00000007, 0x00000000, 0x00000000, 0x00000000,
+    0x00000080, 0x000003FE, 0xFFFFFFFE, 0xFFFFFFFF, 0x001FFFFF, 0xFFFFFFFE,
+    0xFFFFFFFF, 0x07FFFFFF, 0xFFFFFFE0, 0x00001FFF, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000000, 0x00000000,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000000F,
+    0x00000000, 0x00000000, 0x00000000, 0x07FF6000, 0x87FFFFFE, 0x07FFFFFE,
+    0x00000000, 0x00800000, 0xFF7FFFFF, 0xFF7FFFFF, 0x00FFFFFF, 0x00000000,
+    0xFFFF0000, 0xFFFFFFFF, 0xFFFFFFFF, 0xF80001FF, 0x00030003, 0x00000000,
+    0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000003, 0xFFFFD7C0, 0xFFFFFFFB,
+    0x547F7FFF, 0x000FFFFD, 0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
+    0xFFFF007B, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF, 0x00000000, 0xFFFE0000,
+    0x027FFFFF, 0xFFFFFFFE, 0xFFFE007F, 0xBBFFFFFB, 0xFFFF0016, 0x000707FF,
+    0x00000000, 0x07FFFFFE, 0x0007FFFF, 0xFFFF03FF, 0xFFFFFFFF, 0x7CFFFFFF,
+    0xFFEF7FFF, 0x03FF3DFF, 0xFFFFFFEE, 0xF3FFFFFF, 0xFF1E3FFF, 0x0000FFCF,
+    0xFFF99FEE, 0xD3C5FDFF, 0xB080399F, 0x0003FFCF, 0xFFF987E4, 0xD36DFDFF,
+    0x5E003987, 0x001FFFC0, 0xFFFBAFEE, 0xF3EDFDFF, 0x00003BBF, 0x0000FFC1,
+    0xFFF99FEE, 0xF3CDFDFF, 0xB0C0398F, 0x0000FFC3, 0xD63DC7EC, 0xC3BFC718,
+    0x00803DC7, 0x0000FF80, 0xFFFDDFEE, 0xC3EFFDFF, 0x00603DDF, 0x0000FFC3,
+    0xFFFDDFEC, 0xC3EFFDFF, 0x40603DDF, 0x0000FFC3, 0xFFFDDFEC, 0xC3FFFDFF,
+    0x00803DCF, 0x0000FFC3, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0xFFFFFFFE, 0x07FF7FFF, 0x03FF7FFF, 0x00000000, 0xFEF02596, 0x3BFF6CAE,
+    0x03FF3F5F, 0x00000000, 0x03000000, 0xC2A003FF, 0xFFFFFEFF, 0xFFFE03FF,
+    0xFEBF0FDF, 0x02FE3FFF, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x1FFF0000, 0x00000002,
+    0x000000A0, 0x003EFFFE, 0xFFFFFFFE, 0xFFFFFFFF, 0x661FFFFF, 0xFFFFFFFE,
+    0xFFFFFFFF, 0x77FFFFFF,
 };
 static const unsigned char nmstrtPages[] = {
-0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00,
-0x00, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
-0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13,
-0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x15, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x17,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x09, 0x0A, 0x0B,
+    0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13, 0x00, 0x14, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x15, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00,
 };
 static const unsigned char namePages[] = {
-0x19, 0x03, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x00,
-0x00, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25,
-0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13,
-0x26, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x27, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x17,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x19, 0x03, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x00, 0x00, 0x1F, 0x20, 0x21,
+    0x22, 0x23, 0x24, 0x25, 0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13, 0x26, 0x14, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x27, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00,
 };
diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h
index 4d6786d7839a..bfee65a332f1 100644
--- a/Modules/expat/siphash.h
+++ b/Modules/expat/siphash.h
@@ -11,6 +11,10 @@
  * --------------------------------------------------------------------------
  * HISTORY:
  *
+ * 2019-08-03  (Sebastian Pipping)
+ *   - Mark part of sip24_valid as to be excluded from clang-format
+ *   - Re-format code using clang-format 9
+ *
  * 2018-07-08  (Anton Maklakov)
  *   - Add "fall through" markers for GCC's -Wimplicit-fallthrough
  *
@@ -94,186 +98,186 @@
 #include <stddef.h> /* size_t */
 
 #if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER < 1600)
-  /* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */
-  typedef unsigned __int8   uint8_t;
-  typedef unsigned __int32 uint32_t;
-  typedef unsigned __int64 uint64_t;
+/* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */
+typedef unsigned __int8 uint8_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
 #else
- #include <stdint.h> /* uint64_t uint32_t uint8_t */
+#  include <stdint.h> /* uint64_t uint32_t uint8_t */
 #endif
 
-
 /*
  * Workaround to not require a C++11 compiler for using ULL suffix
  * if this code is included and compiled as C++; related GCC warning is:
  * warning: use of C++11 long long integer constant [-Wlong-long]
  */
-#define _SIP_ULL(high, low)  (((uint64_t)high << 32) | low)
-
-
-#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b))))
+#define _SIP_ULL(high, low) (((uint64_t)high << 32) | low)
 
-#define SIP_U32TO8_LE(p, v) \
-	(p)[0] = (uint8_t)((v) >>  0); (p)[1] = (uint8_t)((v) >>  8); \
-	(p)[2] = (uint8_t)((v) >> 16); (p)[3] = (uint8_t)((v) >> 24);
+#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
 
-#define SIP_U64TO8_LE(p, v) \
-	SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >>  0)); \
-	SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
+#define SIP_U32TO8_LE(p, v)                                                    \
+  (p)[0] = (uint8_t)((v) >> 0);                                                \
+  (p)[1] = (uint8_t)((v) >> 8);                                                \
+  (p)[2] = (uint8_t)((v) >> 16);                                               \
+  (p)[3] = (uint8_t)((v) >> 24);
 
-#define SIP_U8TO64_LE(p) \
-	(((uint64_t)((p)[0]) <<  0) | \
-	 ((uint64_t)((p)[1]) <<  8) | \
-	 ((uint64_t)((p)[2]) << 16) | \
-	 ((uint64_t)((p)[3]) << 24) | \
-	 ((uint64_t)((p)[4]) << 32) | \
-	 ((uint64_t)((p)[5]) << 40) | \
-	 ((uint64_t)((p)[6]) << 48) | \
-	 ((uint64_t)((p)[7]) << 56))
+#define SIP_U64TO8_LE(p, v)                                                    \
+  SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >> 0));                                \
+  SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
 
+#define SIP_U8TO64_LE(p)                                                       \
+  (((uint64_t)((p)[0]) << 0) | ((uint64_t)((p)[1]) << 8)                       \
+   | ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24)                   \
+   | ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40)                   \
+   | ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))
 
-#define SIPHASH_INITIALIZER { 0, 0, 0, 0, { 0 }, 0, 0 }
+#define SIPHASH_INITIALIZER                                                    \
+  { 0, 0, 0, 0, {0}, 0, 0 }
 
 struct siphash {
-	uint64_t v0, v1, v2, v3;
+  uint64_t v0, v1, v2, v3;
 
-	unsigned char buf[8], *p;
-	uint64_t c;
+  unsigned char buf[8], *p;
+  uint64_t c;
 }; /* struct siphash */
 
-
 #define SIP_KEYLEN 16
 
 struct sipkey {
-	uint64_t k[2];
+  uint64_t k[2];
 }; /* struct sipkey */
 
-#define sip_keyof(k) sip_tokey(&(struct sipkey){ { 0 } }, (k))
+#define sip_keyof(k) sip_tokey(&(struct sipkey){{0}}, (k))
 
-static struct sipkey *sip_tokey(struct sipkey *key, const void *src) {
-	key->k[0] = SIP_U8TO64_LE((const unsigned char *)src);
-	key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8);
-	return key;
+static struct sipkey *
+sip_tokey(struct sipkey *key, const void *src) {
+  key->k[0] = SIP_U8TO64_LE((const unsigned char *)src);
+  key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8);
+  return key;
 } /* sip_tokey() */
 
-
 #ifdef SIPHASH_TOBIN
 
-#define sip_binof(v) sip_tobin((unsigned char[8]){ 0 }, (v))
+#  define sip_binof(v) sip_tobin((unsigned char[8]){0}, (v))
 
-static void *sip_tobin(void *dst, uint64_t u64) {
-	SIP_U64TO8_LE((unsigned char *)dst, u64);
-	return dst;
+static void *
+sip_tobin(void *dst, uint64_t u64) {
+  SIP_U64TO8_LE((unsigned char *)dst, u64);
+  return dst;
 } /* sip_tobin() */
 
-#endif  /* SIPHASH_TOBIN */
-
+#endif /* SIPHASH_TOBIN */
 
-static void sip_round(struct siphash *H, const int rounds) {
-	int i;
+static void
+sip_round(struct siphash *H, const int rounds) {
+  int i;
 
-	for (i = 0; i < rounds; i++) {
-		H->v0 += H->v1;
-		H->v1 = SIP_ROTL(H->v1, 13);
-		H->v1 ^= H->v0;
-		H->v0 = SIP_ROTL(H->v0, 32);
+  for (i = 0; i < rounds; i++) {
+    H->v0 += H->v1;
+    H->v1 = SIP_ROTL(H->v1, 13);
+    H->v1 ^= H->v0;
+    H->v0 = SIP_ROTL(H->v0, 32);
 
-		H->v2 += H->v3;
-		H->v3 = SIP_ROTL(H->v3, 16);
-		H->v3 ^= H->v2;
+    H->v2 += H->v3;
+    H->v3 = SIP_ROTL(H->v3, 16);
+    H->v3 ^= H->v2;
 
-		H->v0 += H->v3;
-		H->v3 = SIP_ROTL(H->v3, 21);
-		H->v3 ^= H->v0;
+    H->v0 += H->v3;
+    H->v3 = SIP_ROTL(H->v3, 21);
+    H->v3 ^= H->v0;
 
-		H->v2 += H->v1;
-		H->v1 = SIP_ROTL(H->v1, 17);
-		H->v1 ^= H->v2;
-		H->v2 = SIP_ROTL(H->v2, 32);
-	}
+    H->v2 += H->v1;
+    H->v1 = SIP_ROTL(H->v1, 17);
+    H->v1 ^= H->v2;
+    H->v2 = SIP_ROTL(H->v2, 32);
+  }
 } /* sip_round() */
 
+static struct siphash *
+sip24_init(struct siphash *H, const struct sipkey *key) {
+  H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0];
+  H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1];
+  H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0];
+  H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1];
 
-static struct siphash *sip24_init(struct siphash *H,
-		const struct sipkey *key) {
-	H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0];
-	H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1];
-	H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0];
-	H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1];
+  H->p = H->buf;
+  H->c = 0;
 
-	H->p = H->buf;
-	H->c = 0;
-
-	return H;
+  return H;
 } /* sip24_init() */
 
+#define sip_endof(a) (&(a)[sizeof(a) / sizeof *(a)])
 
-#define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)])
-
-static struct siphash *sip24_update(struct siphash *H, const void *src,
-		size_t len) {
-	const unsigned char *p = (const unsigned char *)src, *pe = p + len;
-	uint64_t m;
+static struct siphash *
+sip24_update(struct siphash *H, const void *src, size_t len) {
+  const unsigned char *p = (const unsigned char *)src, *pe = p + len;
+  uint64_t m;
 
-	do {
-		while (p < pe && H->p < sip_endof(H->buf))
-			*H->p++ = *p++;
+  do {
+    while (p < pe && H->p < sip_endof(H->buf))
+      *H->p++ = *p++;
 
-		if (H->p < sip_endof(H->buf))
-			break;
+    if (H->p < sip_endof(H->buf))
+      break;
 
-		m = SIP_U8TO64_LE(H->buf);
-		H->v3 ^= m;
-		sip_round(H, 2);
-		H->v0 ^= m;
+    m = SIP_U8TO64_LE(H->buf);
+    H->v3 ^= m;
+    sip_round(H, 2);
+    H->v0 ^= m;
 
-		H->p = H->buf;
-		H->c += 8;
-	} while (p < pe);
+    H->p = H->buf;
+    H->c += 8;
+  } while (p < pe);
 
-	return H;
+  return H;
 } /* sip24_update() */
 
-
-static uint64_t sip24_final(struct siphash *H) {
-	const char left = (char)(H->p - H->buf);
-	uint64_t b = (H->c + left) << 56;
-
-	switch (left) {
-	case 7: b |= (uint64_t)H->buf[6] << 48;
-		/* fall through */
-	case 6: b |= (uint64_t)H->buf[5] << 40;
-		/* fall through */
-	case 5: b |= (uint64_t)H->buf[4] << 32;
-		/* fall through */
-	case 4: b |= (uint64_t)H->buf[3] << 24;
-		/* fall through */
-	case 3: b |= (uint64_t)H->buf[2] << 16;
-		/* fall through */
-	case 2: b |= (uint64_t)H->buf[1] << 8;
-		/* fall through */
-	case 1: b |= (uint64_t)H->buf[0] << 0;
-		/* fall through */
-	case 0: break;
-	}
-
-	H->v3 ^= b;
-	sip_round(H, 2);
-	H->v0 ^= b;
-	H->v2 ^= 0xff;
-	sip_round(H, 4);
-
-	return H->v0 ^ H->v1 ^ H->v2  ^ H->v3;
+static uint64_t
+sip24_final(struct siphash *H) {
+  const char left = (char)(H->p - H->buf);
+  uint64_t b = (H->c + left) << 56;
+
+  switch (left) {
+  case 7:
+    b |= (uint64_t)H->buf[6] << 48;
+    /* fall through */
+  case 6:
+    b |= (uint64_t)H->buf[5] << 40;
+    /* fall through */
+  case 5:
+    b |= (uint64_t)H->buf[4] << 32;
+    /* fall through */
+  case 4:
+    b |= (uint64_t)H->buf[3] << 24;
+    /* fall through */
+  case 3:
+    b |= (uint64_t)H->buf[2] << 16;
+    /* fall through */
+  case 2:
+    b |= (uint64_t)H->buf[1] << 8;
+    /* fall through */
+  case 1:
+    b |= (uint64_t)H->buf[0] << 0;
+    /* fall through */
+  case 0:
+    break;
+  }
+
+  H->v3 ^= b;
+  sip_round(H, 2);
+  H->v0 ^= b;
+  H->v2 ^= 0xff;
+  sip_round(H, 4);
+
+  return H->v0 ^ H->v1 ^ H->v2 ^ H->v3;
 } /* sip24_final() */
 
-
-static uint64_t siphash24(const void *src, size_t len,
-		const struct sipkey *key) {
-	struct siphash state = SIPHASH_INITIALIZER;
-	return sip24_final(sip24_update(sip24_init(&state, key), src, len));
+static uint64_t
+siphash24(const void *src, size_t len, const struct sipkey *key) {
+  struct siphash state = SIPHASH_INITIALIZER;
+  return sip24_final(sip24_update(sip24_init(&state, key), src, len));
 } /* siphash24() */
 
-
 /*
  * SipHash-2-4 output with
  * k = 00 01 02 ...
@@ -285,107 +289,110 @@ static uint64_t siphash24(const void *src, size_t len,
  * ...
  * in = 00 01 02 ... 3e (63 bytes)
  */
-static int sip24_valid(void) {
-	static const unsigned char vectors[64][8] = {
-		{ 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, },
-		{ 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, },
-		{ 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, },
-		{ 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, },
-		{ 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, },
-		{ 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, },
-		{ 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, },
-		{ 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, },
-		{ 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, },
-		{ 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, },
-		{ 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, },
-		{ 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, },
-		{ 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, },
-		{ 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, },
-		{ 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, },
-		{ 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, },
-		{ 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, },
-		{ 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, },
-		{ 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, },
-		{ 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, },
-		{ 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, },
-		{ 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, },
-		{ 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, },
-		{ 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, },
-		{ 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, },
-		{ 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, },
-		{ 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, },
-		{ 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, },
-		{ 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, },
-		{ 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, },
-		{ 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, },
-		{ 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, },
-		{ 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, },
-		{ 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, },
-		{ 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, },
-		{ 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, },
-		{ 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, },
-		{ 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, },
-		{ 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, },
-		{ 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, },
-		{ 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, },
-		{ 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, },
-		{ 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, },
-		{ 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, },
-		{ 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, },
-		{ 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, },
-		{ 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, },
-		{ 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, },
-		{ 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, },
-		{ 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, },
-		{ 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, },
-		{ 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, },
-		{ 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, },
-		{ 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, },
-		{ 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, },
-		{ 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, },
-		{ 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, },
-		{ 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, },
-		{ 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, },
-		{ 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, },
-		{ 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, },
-		{ 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, },
-		{ 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, },
-		{ 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, }
-	};
-	unsigned char in[64];
-	struct sipkey k;
-	size_t i;
-
-	sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011"
-			"\012\013\014\015\016\017");
-
-	for (i = 0; i < sizeof in; ++i) {
-		in[i] = (unsigned char)i;
-
-		if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i]))
-			return 0;
-	}
-
-	return 1;
+static int
+sip24_valid(void) {
+  /* clang-format off */
+  static const unsigned char vectors[64][8] = {
+    { 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, },
+    { 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, },
+    { 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, },
+    { 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, },
+    { 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, },
+    { 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, },
+    { 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, },
+    { 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, },
+    { 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, },
+    { 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, },
+    { 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, },
+    { 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, },
+    { 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, },
+    { 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, },
+    { 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, },
+    { 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, },
+    { 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, },
+    { 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, },
+    { 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, },
+    { 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, },
+    { 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, },
+    { 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, },
+    { 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, },
+    { 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, },
+    { 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, },
+    { 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, },
+    { 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, },
+    { 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, },
+    { 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, },
+    { 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, },
+    { 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, },
+    { 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, },
+    { 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, },
+    { 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, },
+    { 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, },
+    { 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, },
+    { 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, },
+    { 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, },
+    { 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, },
+    { 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, },
+    { 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, },
+    { 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, },
+    { 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, },
+    { 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, },
+    { 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, },
+    { 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, },
+    { 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, },
+    { 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, },
+    { 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, },
+    { 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, },
+    { 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, },
+    { 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, },
+    { 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, },
+    { 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, },
+    { 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, },
+    { 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, },
+    { 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, },
+    { 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, },
+    { 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, },
+    { 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, },
+    { 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, },
+    { 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, },
+    { 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, },
+    { 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, }
+  };
+  /* clang-format on */
+
+  unsigned char in[64];
+  struct sipkey k;
+  size_t i;
+
+  sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011"
+                "\012\013\014\015\016\017");
+
+  for (i = 0; i < sizeof in; ++i) {
+    in[i] = (unsigned char)i;
+
+    if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i]))
+      return 0;
+  }
+
+  return 1;
 } /* sip24_valid() */
 
-
 #ifdef SIPHASH_MAIN
 
-#include <stdio.h>
+#  include <stdio.h>
 
-int main(void) {
-	const int ok = sip24_valid();
+int
+main(void) {
+  const int ok = sip24_valid();
 
-	if (ok)
-		puts("OK");
-	else
-		puts("FAIL");
+  if (ok)
+    puts("OK");
+  else
+    puts("FAIL");
 
-	return !ok;
+  return ! ok;
 } /* main() */
 
 #endif /* SIPHASH_MAIN */
 
-
 #endif /* SIPHASH_H */
diff --git a/Modules/expat/utf8tab.h b/Modules/expat/utf8tab.h
index fa0bed6f5d75..a22986acbb95 100644
--- a/Modules/expat/utf8tab.h
+++ b/Modules/expat/utf8tab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x80 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x84 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x88 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x8C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x90 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x94 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x98 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x9C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xAC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xBC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xC0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xC4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xC8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xCC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xDC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xE0 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xE4 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xE8 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xEC */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xF0 */ BT_LEAD4, BT_LEAD4, BT_LEAD4, BT_LEAD4,
-/* 0xF4 */ BT_LEAD4, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0xF8 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0xFC */ BT_NONXML, BT_NONXML, BT_MALFORM, BT_MALFORM,
+    /* 0x84 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x88 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x8C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x90 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x94 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x98 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x9C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xAC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xBC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xC0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xC4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xC8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xCC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xDC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xE0 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xE4 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xE8 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xEC */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xF0 */ BT_LEAD4, BT_LEAD4, BT_LEAD4, BT_LEAD4,
+    /* 0xF4 */ BT_LEAD4, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0xF8 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0xFC */ BT_NONXML, BT_NONXML, BT_MALFORM, BT_MALFORM,
diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h
index 28a043c6044c..562a4a82dc1d 100644
--- a/Modules/expat/winconfig.h
+++ b/Modules/expat/winconfig.h
@@ -40,20 +40,17 @@
 #include <memory.h>
 #include <string.h>
 
+#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */
+#  include <expat_config.h>
+#else /* !defined(HAVE_EXPAT_CONFIG_H) */
 
-#if defined(HAVE_EXPAT_CONFIG_H)  /* e.g. MinGW */
-# include <expat_config.h>
-#else  /* !defined(HAVE_EXPAT_CONFIG_H) */
-
-
-#define XML_NS 1
-#define XML_DTD 1
-#define XML_CONTEXT_BYTES 1024
+#  define XML_NS 1
+#  define XML_DTD 1
+#  define XML_CONTEXT_BYTES 1024
 
 /* we will assume all Windows platforms are little endian */
-#define BYTEORDER 1234
+#  define BYTEORDER 1234
 
 #endif /* !defined(HAVE_EXPAT_CONFIG_H) */
 
-
 #endif /* ndef WINCONFIG_H */
diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c
index 311dbc4dfb9c..e740f0e19c7d 100644
--- a/Modules/expat/xmlparse.c
+++ b/Modules/expat/xmlparse.c
@@ -1,4 +1,4 @@
-/* 69df5be70289a11fb834869ce4a91c23c1d9dd04baffcbd10e86742d149a080c (2.2.7+)
+/* f2d0ab6d1d4422a08cf1cf3bbdfba96b49dea42fb5ff4615e03a2a25c306e769 (2.2.8+)
                             __  __            _
                          ___\ \/ /_ __   __ _| |_
                         / _ \\  /| '_ \ / _` | __|
@@ -30,33 +30,38 @@
    USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
-#if !defined(_GNU_SOURCE)
-# define _GNU_SOURCE 1                  /* syscall prototype */
+#if ! defined(_GNU_SOURCE)
+#  define _GNU_SOURCE 1 /* syscall prototype */
+#endif
+
+#ifdef _WIN32
+/* force stdlib to define rand_s() */
+#  define _CRT_RAND_S
 #endif
 
 #include <stddef.h>
-#include <string.h>                     /* memset(), memcpy() */
+#include <string.h> /* memset(), memcpy() */
 #include <assert.h>
-#include <limits.h>                     /* UINT_MAX */
-#include <stdio.h>                      /* fprintf */
-#include <stdlib.h>                     /* getenv */
+#include <limits.h> /* UINT_MAX */
+#include <stdio.h>  /* fprintf */
+#include <stdlib.h> /* getenv, rand_s */
 
 #ifdef _WIN32
-#define getpid GetCurrentProcessId
+#  define getpid GetCurrentProcessId
 #else
-#include <sys/time.h>                   /* gettimeofday() */
-#include <sys/types.h>                  /* getpid() */
-#include <unistd.h>                     /* getpid() */
-#include <fcntl.h>                      /* O_RDONLY */
-#include <errno.h>
+#  include <sys/time.h>  /* gettimeofday() */
+#  include <sys/types.h> /* getpid() */
+#  include <unistd.h>    /* getpid() */
+#  include <fcntl.h>     /* O_RDONLY */
+#  include <errno.h>
 #endif
 
 #define XML_BUILDING_EXPAT 1
 
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #elif defined(HAVE_EXPAT_CONFIG_H)
-#include <expat_config.h>
+#  include <expat_config.h>
 #endif /* ndef _WIN32 */
 
 #include "ascii.h"
@@ -64,33 +69,31 @@
 #include "siphash.h"
 
 #if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
-# if defined(HAVE_GETRANDOM)
-#  include <sys/random.h>    /* getrandom */
-# else
-#  include <unistd.h>        /* syscall */
-#  include <sys/syscall.h>   /* SYS_getrandom */
-# endif
-# if ! defined(GRND_NONBLOCK)
-#  define GRND_NONBLOCK  0x0001
-# endif  /* defined(GRND_NONBLOCK) */
-#endif  /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
-
-#if defined(HAVE_LIBBSD) \
+#  if defined(HAVE_GETRANDOM)
+#    include <sys/random.h> /* getrandom */
+#  else
+#    include <unistd.h>      /* syscall */
+#    include <sys/syscall.h> /* SYS_getrandom */
+#  endif
+#  if ! defined(GRND_NONBLOCK)
+#    define GRND_NONBLOCK 0x0001
+#  endif /* defined(GRND_NONBLOCK) */
+#endif   /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
+
+#if defined(HAVE_LIBBSD)                                                       \
     && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM))
-# include <bsd/stdlib.h>
+#  include <bsd/stdlib.h>
 #endif
 
-#if defined(_WIN32) && !defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
-# define LOAD_LIBRARY_SEARCH_SYSTEM32  0x00000800
+#if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
+#  define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800
 #endif
 
-#if !defined(HAVE_GETRANDOM) && !defined(HAVE_SYSCALL_GETRANDOM) \
-    && !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_ARC4RANDOM) \
-    && !defined(XML_DEV_URANDOM) \
-    && !defined(_WIN32) \
-    && !defined(XML_POOR_ENTROPY)
-# error  \
-    You do not have support for any sources of high quality entropy \
+#if ! defined(HAVE_GETRANDOM) && ! defined(HAVE_SYSCALL_GETRANDOM)             \
+    && ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)            \
+    && ! defined(XML_DEV_URANDOM) && ! defined(_WIN32)                         \
+    && ! defined(XML_POOR_ENTROPY)
+#  error You do not have support for any sources of high quality entropy \
     enabled.  For end user security, that is probably not what you want. \
     \
     Your options include: \
@@ -101,7 +104,7 @@
       * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \
       * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \
       * Linux / BSD / macOS (/dev/urandom): XML_DEV_URANDOM \
-      * Windows (RtlGenRandom): _WIN32. \
+      * Windows (rand_s): _WIN32. \
     \
     If insist on not using any of these, bypass this error by defining \
     XML_POOR_ENTROPY; you have been warned. \
@@ -110,56 +113,55 @@
     to the build system, please open a bug.  Thank you!
 #endif
 
-
 #ifdef XML_UNICODE
-#define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX
-#define XmlConvert XmlUtf16Convert
-#define XmlGetInternalEncoding XmlGetUtf16InternalEncoding
-#define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS
-#define XmlEncode XmlUtf16Encode
+#  define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX
+#  define XmlConvert XmlUtf16Convert
+#  define XmlGetInternalEncoding XmlGetUtf16InternalEncoding
+#  define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS
+#  define XmlEncode XmlUtf16Encode
 /* Using pointer subtraction to convert to integer type. */
-#define MUST_CONVERT(enc, s) (!(enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1))
+#  define MUST_CONVERT(enc, s)                                                 \
+    (! (enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1))
 typedef unsigned short ICHAR;
 #else
-#define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX
-#define XmlConvert XmlUtf8Convert
-#define XmlGetInternalEncoding XmlGetUtf8InternalEncoding
-#define XmlGetInternalEncodingNS XmlGetUtf8InternalEncodingNS
-#define XmlEncode XmlUtf8Encode
-#define MUST_CONVERT(enc, s) (!(enc)->isUtf8)
+#  define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX
+#  define XmlConvert XmlUtf8Convert
+#  define XmlGetInternalEncoding XmlGetUtf8InternalEncoding
+#  define XmlGetInternalEncodingNS XmlGetUtf8InternalEncodingNS
+#  define XmlEncode XmlUtf8Encode
+#  define MUST_CONVERT(enc, s) (! (enc)->isUtf8)
 typedef char ICHAR;
 #endif
 
-
 #ifndef XML_NS
 
-#define XmlInitEncodingNS XmlInitEncoding
-#define XmlInitUnknownEncodingNS XmlInitUnknownEncoding
-#undef XmlGetInternalEncodingNS
-#define XmlGetInternalEncodingNS XmlGetInternalEncoding
-#define XmlParseXmlDeclNS XmlParseXmlDecl
+#  define XmlInitEncodingNS XmlInitEncoding
+#  define XmlInitUnknownEncodingNS XmlInitUnknownEncoding
+#  undef XmlGetInternalEncodingNS
+#  define XmlGetInternalEncodingNS XmlGetInternalEncoding
+#  define XmlParseXmlDeclNS XmlParseXmlDecl
 
 #endif
 
 #ifdef XML_UNICODE
 
-#ifdef XML_UNICODE_WCHAR_T
-#define XML_T(x) (const wchar_t)x
-#define XML_L(x) L ## x
-#else
-#define XML_T(x) (const unsigned short)x
-#define XML_L(x) x
-#endif
+#  ifdef XML_UNICODE_WCHAR_T
+#    define XML_T(x) (const wchar_t) x
+#    define XML_L(x) L##x
+#  else
+#    define XML_T(x) (const unsigned short)x
+#    define XML_L(x) x
+#  endif
 
 #else
 
-#define XML_T(x) x
-#define XML_L(x) x
+#  define XML_T(x) x
+#  define XML_L(x) x
 
 #endif
 
 /* Round up n to be a multiple of sz, where sz is a power of 2. */
-#define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1))
+#define ROUND_UP(n, sz) (((n) + ((sz)-1)) & ~((sz)-1))
 
 /* Do safe (NULL-aware) pointer arithmetic */
 #define EXPAT_SAFE_PTR_DIFF(p, q) (((p) && (q)) ? ((p) - (q)) : 0)
@@ -182,11 +184,9 @@ typedef struct {
   const XML_Memory_Handling_Suite *mem;
 } HASH_TABLE;
 
-static size_t
-keylen(KEY s);
+static size_t keylen(KEY s);
 
-static void
-copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key);
+static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key);
 
 /* For probing (after a collision) we need a step size relative prime
    to the hash table size, which is a power of 2. We use double-hashing,
@@ -196,9 +196,9 @@ copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key);
    We limit the maximum step size to table->size / 4 (mask >> 2) and make
    it odd, since odd numbers are always relative prime to a power of 2.
 */
-#define SECOND_HASH(hash, mask, power) \
-  ((((hash) & ~(mask)) >> ((power) - 1)) & ((mask) >> 2))
-#define PROBE_STEP(hash, mask, power) \
+#define SECOND_HASH(hash, mask, power)                                         \
+  ((((hash) & ~(mask)) >> ((power)-1)) & ((mask) >> 2))
+#define PROBE_STEP(hash, mask, power)                                          \
   ((unsigned char)((SECOND_HASH(hash, mask, power)) | 1))
 
 typedef struct {
@@ -206,7 +206,7 @@ typedef struct {
   NAMED **end;
 } HASH_TABLE_ITER;
 
-#define INIT_TAG_BUF_SIZE 32  /* must be a multiple of sizeof(XML_Char) */
+#define INIT_TAG_BUF_SIZE 32 /* must be a multiple of sizeof(XML_Char) */
 #define INIT_DATA_BUF_SIZE 1024
 #define INIT_ATTS_SIZE 16
 #define INIT_ATTS_VERSION 0xFFFFFFFF
@@ -253,20 +253,20 @@ typedef struct {
    TAG objects in a free list.
 */
 typedef struct tag {
-  struct tag *parent;           /* parent of this element */
-  const char *rawName;          /* tagName in the original encoding */
+  struct tag *parent;  /* parent of this element */
+  const char *rawName; /* tagName in the original encoding */
   int rawNameLength;
-  TAG_NAME name;                /* tagName in the API encoding */
-  char *buf;                    /* buffer for name components */
-  char *bufEnd;                 /* end of the buffer */
+  TAG_NAME name; /* tagName in the API encoding */
+  char *buf;     /* buffer for name components */
+  char *bufEnd;  /* end of the buffer */
   BINDING *bindings;
 } TAG;
 
 typedef struct {
   const XML_Char *name;
   const XML_Char *textPtr;
-  int textLen;                  /* length in XML_Chars */
-  int processed;                /* # of processed bytes - when suspended */
+  int textLen;   /* length in XML_Chars */
+  int processed; /* # of processed bytes - when suspended */
   const XML_Char *systemId;
   const XML_Char *base;
   const XML_Char *publicId;
@@ -277,13 +277,13 @@ typedef struct {
 } ENTITY;
 
 typedef struct {
-  enum XML_Content_Type         type;
-  enum XML_Content_Quant        quant;
-  const XML_Char *              name;
-  int                           firstchild;
-  int                           lastchild;
-  int                           childcnt;
-  int                           nextsib;
+  enum XML_Content_Type type;
+  enum XML_Content_Quant quant;
+  const XML_Char *name;
+  int firstchild;
+  int lastchild;
+  int childcnt;
+  int nextsib;
 } CONTENT_SCAFFOLD;
 
 #define INIT_SCAFFOLD_ELEMENTS 32
@@ -371,10 +371,8 @@ typedef struct open_internal_entity {
   XML_Bool betweenDecl; /* WFC: PE Between Declarations */
 } OPEN_INTERNAL_ENTITY;
 
-typedef enum XML_Error PTRCALL Processor(XML_Parser parser,
-                                         const char *start,
-                                         const char *end,
-                                         const char **endPtr);
+typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start,
+                                         const char *end, const char **endPtr);
 
 static Processor prologProcessor;
 static Processor prologInitProcessor;
@@ -395,118 +393,101 @@ static Processor externalEntityInitProcessor3;
 static Processor externalEntityContentProcessor;
 static Processor internalEntityProcessor;
 
-static enum XML_Error
-handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName);
-static enum XML_Error
-processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
-               const char *s, const char *next);
-static enum XML_Error
-initializeEncoding(XML_Parser parser);
-static enum XML_Error
-doProlog(XML_Parser parser, const ENCODING *enc, const char *s,
-         const char *end, int tok, const char *next, const char **nextPtr,
-         XML_Bool haveMore);
-static enum XML_Error
-processInternalEntity(XML_Parser parser, ENTITY *entity,
-                      XML_Bool betweenDecl);
-static enum XML_Error
-doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
-          const char *start, const char *end, const char **endPtr,
-          XML_Bool haveMore);
-static enum XML_Error
-doCdataSection(XML_Parser parser, const ENCODING *, const char **startPtr,
-               const char *end, const char **nextPtr, XML_Bool haveMore);
+static enum XML_Error handleUnknownEncoding(XML_Parser parser,
+                                            const XML_Char *encodingName);
+static enum XML_Error processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
+                                     const char *s, const char *next);
+static enum XML_Error initializeEncoding(XML_Parser parser);
+static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc,
+                               const char *s, const char *end, int tok,
+                               const char *next, const char **nextPtr,
+                               XML_Bool haveMore, XML_Bool allowClosingDoctype);
+static enum XML_Error processInternalEntity(XML_Parser parser, ENTITY *entity,
+                                            XML_Bool betweenDecl);
+static enum XML_Error doContent(XML_Parser parser, int startTagLevel,
+                                const ENCODING *enc, const char *start,
+                                const char *end, const char **endPtr,
+                                XML_Bool haveMore);
+static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *,
+                                     const char **startPtr, const char *end,
+                                     const char **nextPtr, XML_Bool haveMore);
 #ifdef XML_DTD
-static enum XML_Error
-doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr,
-                const char *end, const char **nextPtr, XML_Bool haveMore);
+static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *,
+                                      const char **startPtr, const char *end,
+                                      const char **nextPtr, XML_Bool haveMore);
 #endif /* XML_DTD */
 
-static void
-freeBindings(XML_Parser parser, BINDING *bindings);
-static enum XML_Error
-storeAtts(XML_Parser parser, const ENCODING *, const char *s,
-          TAG_NAME *tagNamePtr, BINDING **bindingsPtr);
-static enum XML_Error
-addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
-           const XML_Char *uri, BINDING **bindingsPtr);
-static int
-defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata,
-                XML_Bool isId, const XML_Char *dfltValue, XML_Parser parser);
-static enum XML_Error
-storeAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata,
-                    const char *, const char *, STRING_POOL *);
-static enum XML_Error
-appendAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata,
-                     const char *, const char *, STRING_POOL *);
-static ATTRIBUTE_ID *
-getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start,
-               const char *end);
-static int
-setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *);
-static enum XML_Error
-storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *start,
-                 const char *end);
-static int
-reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
-                            const char *start, const char *end);
-static int
-reportComment(XML_Parser parser, const ENCODING *enc, const char *start,
-              const char *end);
-static void
-reportDefault(XML_Parser parser, const ENCODING *enc, const char *start,
-              const char *end);
-
-static const XML_Char * getContext(XML_Parser parser);
-static XML_Bool
-setContext(XML_Parser parser, const XML_Char *context);
+static void freeBindings(XML_Parser parser, BINDING *bindings);
+static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *,
+                                const char *s, TAG_NAME *tagNamePtr,
+                                BINDING **bindingsPtr);
+static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix,
+                                 const ATTRIBUTE_ID *attId, const XML_Char *uri,
+                                 BINDING **bindingsPtr);
+static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata,
+                           XML_Bool isId, const XML_Char *dfltValue,
+                           XML_Parser parser);
+static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *,
+                                          XML_Bool isCdata, const char *,
+                                          const char *, STRING_POOL *);
+static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *,
+                                           XML_Bool isCdata, const char *,
+                                           const char *, STRING_POOL *);
+static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc,
+                                    const char *start, const char *end);
+static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *);
+static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc,
+                                       const char *start, const char *end);
+static int reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
+                                       const char *start, const char *end);
+static int reportComment(XML_Parser parser, const ENCODING *enc,
+                         const char *start, const char *end);
+static void reportDefault(XML_Parser parser, const ENCODING *enc,
+                          const char *start, const char *end);
+
+static const XML_Char *getContext(XML_Parser parser);
+static XML_Bool setContext(XML_Parser parser, const XML_Char *context);
 
 static void FASTCALL normalizePublicId(XML_Char *s);
 
-static DTD * dtdCreate(const XML_Memory_Handling_Suite *ms);
+static DTD *dtdCreate(const XML_Memory_Handling_Suite *ms);
 /* do not call if m_parentParser != NULL */
 static void dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms);
-static void
-dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms);
-static int
-dtdCopy(XML_Parser oldParser,
-        DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms);
-static int
-copyEntityTable(XML_Parser oldParser,
-                HASH_TABLE *, STRING_POOL *, const HASH_TABLE *);
-static NAMED *
-lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize);
-static void FASTCALL
-hashTableInit(HASH_TABLE *, const XML_Memory_Handling_Suite *ms);
+static void dtdDestroy(DTD *p, XML_Bool isDocEntity,
+                       const XML_Memory_Handling_Suite *ms);
+static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
+                   const XML_Memory_Handling_Suite *ms);
+static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *, STRING_POOL *,
+                           const HASH_TABLE *);
+static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name,
+                     size_t createSize);
+static void FASTCALL hashTableInit(HASH_TABLE *,
+                                   const XML_Memory_Handling_Suite *ms);
 static void FASTCALL hashTableClear(HASH_TABLE *);
 static void FASTCALL hashTableDestroy(HASH_TABLE *);
-static void FASTCALL
-hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *);
-static NAMED * FASTCALL hashTableIterNext(HASH_TABLE_ITER *);
+static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *);
+static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *);
 
-static void FASTCALL
-poolInit(STRING_POOL *, const XML_Memory_Handling_Suite *ms);
+static void FASTCALL poolInit(STRING_POOL *,
+                              const XML_Memory_Handling_Suite *ms);
 static void FASTCALL poolClear(STRING_POOL *);
 static void FASTCALL poolDestroy(STRING_POOL *);
-static XML_Char *
-poolAppend(STRING_POOL *pool, const ENCODING *enc,
-           const char *ptr, const char *end);
-static XML_Char *
-poolStoreString(STRING_POOL *pool, const ENCODING *enc,
-                const char *ptr, const char *end);
+static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc,
+                            const char *ptr, const char *end);
+static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc,
+                                 const char *ptr, const char *end);
 static XML_Bool FASTCALL poolGrow(STRING_POOL *pool);
-static const XML_Char * FASTCALL
-poolCopyString(STRING_POOL *pool, const XML_Char *s);
-static const XML_Char *
-poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n);
-static const XML_Char * FASTCALL
-poolAppendString(STRING_POOL *pool, const XML_Char *s);
+static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool,
+                                               const XML_Char *s);
+static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s,
+                                       int n);
+static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool,
+                                                 const XML_Char *s);
 
 static int FASTCALL nextScaffoldPart(XML_Parser parser);
-static XML_Content * build_model(XML_Parser parser);
-static ELEMENT_TYPE *
-getElementType(XML_Parser parser, const ENCODING *enc,
-               const char *ptr, const char *end);
+static XML_Content *build_model(XML_Parser parser);
+static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc,
+                                    const char *ptr, const char *end);
 
 static XML_Char *copyString(const XML_Char *s,
                             const XML_Memory_Handling_Suite *memsuite);
@@ -514,14 +495,11 @@ static XML_Char *copyString(const XML_Char *s,
 static unsigned long generate_hash_secret_salt(XML_Parser parser);
 static XML_Bool startParsing(XML_Parser parser);
 
-static XML_Parser
-parserCreate(const XML_Char *encodingName,
-             const XML_Memory_Handling_Suite *memsuite,
-             const XML_Char *nameSep,
-             DTD *dtd);
+static XML_Parser parserCreate(const XML_Char *encodingName,
+                               const XML_Memory_Handling_Suite *memsuite,
+                               const XML_Char *nameSep, DTD *dtd);
 
-static void
-parserInit(XML_Parser parser, const XML_Char *encodingName);
+static void parserInit(XML_Parser parser, const XML_Char *encodingName);
 
 #define poolStart(pool) ((pool)->start)
 #define poolEnd(pool) ((pool)->ptr)
@@ -530,10 +508,10 @@ parserInit(XML_Parser parser, const XML_Char *encodingName);
 #define poolLastChar(pool) (((pool)->ptr)[-1])
 #define poolDiscard(pool) ((pool)->ptr = (pool)->start)
 #define poolFinish(pool) ((pool)->start = (pool)->ptr)
-#define poolAppendChar(pool, c) \
-  (((pool)->ptr == (pool)->end && !poolGrow(pool)) \
-   ? 0 \
-   : ((*((pool)->ptr)++ = c), 1))
+#define poolAppendChar(pool, c)                                                \
+  (((pool)->ptr == (pool)->end && ! poolGrow(pool))                            \
+       ? 0                                                                     \
+       : ((*((pool)->ptr)++ = c), 1))
 
 struct XML_ParserStruct {
   /* The first member must be m_userData so that the XML_GetUserData
@@ -584,7 +562,7 @@ struct XML_ParserStruct {
   void *m_unknownEncodingMem;
   void *m_unknownEncodingData;
   void *m_unknownEncodingHandlerData;
-  void (XMLCALL *m_unknownEncodingRelease)(void *);
+  void(XMLCALL *m_unknownEncodingRelease)(void *);
   PROLOG_STATE m_prologState;
   Processor *m_processor;
   enum XML_Error m_errorCode;
@@ -638,57 +616,55 @@ struct XML_ParserStruct {
   unsigned long m_hash_secret_salt;
 };
 
-#define MALLOC(parser, s)      (parser->m_mem.malloc_fcn((s)))
-#define REALLOC(parser, p, s)  (parser->m_mem.realloc_fcn((p),(s)))
-#define FREE(parser, p)        (parser->m_mem.free_fcn((p)))
-
+#define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s)))
+#define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s)))
+#define FREE(parser, p) (parser->m_mem.free_fcn((p)))
 
 XML_Parser XMLCALL
-XML_ParserCreate(const XML_Char *encodingName)
-{
+XML_ParserCreate(const XML_Char *encodingName) {
   return XML_ParserCreate_MM(encodingName, NULL, NULL);
 }
 
 XML_Parser XMLCALL
-XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep)
-{
+XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) {
   XML_Char tmp[2];
   *tmp = nsSep;
   return XML_ParserCreate_MM(encodingName, NULL, tmp);
 }
 
-static const XML_Char implicitContext[] = {
-  ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h, ASCII_t, ASCII_t, ASCII_p,
-  ASCII_COLON, ASCII_SLASH, ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w,
-  ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g,
-  ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L, ASCII_SLASH, ASCII_1, ASCII_9,
-  ASCII_9, ASCII_8, ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e,
-  ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'
-};
-
+static const XML_Char implicitContext[]
+    = {ASCII_x,     ASCII_m,     ASCII_l,      ASCII_EQUALS, ASCII_h,
+       ASCII_t,     ASCII_t,     ASCII_p,      ASCII_COLON,  ASCII_SLASH,
+       ASCII_SLASH, ASCII_w,     ASCII_w,      ASCII_w,      ASCII_PERIOD,
+       ASCII_w,     ASCII_3,     ASCII_PERIOD, ASCII_o,      ASCII_r,
+       ASCII_g,     ASCII_SLASH, ASCII_X,      ASCII_M,      ASCII_L,
+       ASCII_SLASH, ASCII_1,     ASCII_9,      ASCII_9,      ASCII_8,
+       ASCII_SLASH, ASCII_n,     ASCII_a,      ASCII_m,      ASCII_e,
+       ASCII_s,     ASCII_p,     ASCII_a,      ASCII_c,      ASCII_e,
+       '\0'};
 
 /* To avoid warnings about unused functions: */
 #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)
 
-#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
+#  if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
 
 /* Obtain entropy on Linux 3.17+ */
 static int
-writeRandomBytes_getrandom_nonblock(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
+writeRandomBytes_getrandom_nonblock(void *target, size_t count) {
+  int success = 0; /* full count bytes written? */
   size_t bytesWrittenTotal = 0;
   const unsigned int getrandomFlags = GRND_NONBLOCK;
 
   do {
-    void * const currentTarget = (void*)((char*)target + bytesWrittenTotal);
+    void *const currentTarget = (void *)((char *)target + bytesWrittenTotal);
     const size_t bytesToWrite = count - bytesWrittenTotal;
 
     const int bytesWrittenMore =
-#if defined(HAVE_GETRANDOM)
+#    if defined(HAVE_GETRANDOM)
         getrandom(currentTarget, bytesToWrite, getrandomFlags);
-#else
+#    else
         syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags);
-#endif
+#    endif
 
     if (bytesWrittenMore > 0) {
       bytesWrittenTotal += bytesWrittenMore;
@@ -700,15 +676,14 @@ writeRandomBytes_getrandom_nonblock(void * target, size_t count) {
   return success;
 }
 
-#endif  /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
-
+#  endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
 
-#if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
+#  if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
 
 /* Extract entropy from /dev/urandom */
 static int
-writeRandomBytes_dev_urandom(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
+writeRandomBytes_dev_urandom(void *target, size_t count) {
+  int success = 0; /* full count bytes written? */
   size_t bytesWrittenTotal = 0;
 
   const int fd = open("/dev/urandom", O_RDONLY);
@@ -717,7 +692,7 @@ writeRandomBytes_dev_urandom(void * target, size_t count) {
   }
 
   do {
-    void * const currentTarget = (void*)((char*)target + bytesWrittenTotal);
+    void *const currentTarget = (void *)((char *)target + bytesWrittenTotal);
     const size_t bytesToWrite = count - bytesWrittenTotal;
 
     const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite);
@@ -733,15 +708,14 @@ writeRandomBytes_dev_urandom(void * target, size_t count) {
   return success;
 }
 
-#endif  /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
-
-#endif  /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
+#  endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
 
+#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
 
 #if defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF)
 
 static void
-writeRandomBytes_arc4random(void * target, size_t count) {
+writeRandomBytes_arc4random(void *target, size_t count) {
   size_t bytesWrittenTotal = 0;
 
   while (bytesWrittenTotal < count) {
@@ -749,93 +723,82 @@ writeRandomBytes_arc4random(void * target, size_t count) {
     size_t i = 0;
 
     for (; (i < sizeof(random32)) && (bytesWrittenTotal < count);
-        i++, bytesWrittenTotal++) {
+         i++, bytesWrittenTotal++) {
       const uint8_t random8 = (uint8_t)(random32 >> (i * 8));
       ((uint8_t *)target)[bytesWrittenTotal] = random8;
     }
   }
 }
 
-#endif  /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */
-
+#endif /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */
 
 #ifdef _WIN32
 
-typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG);
-HMODULE _Expat_LoadLibrary(LPCTSTR filename);  /* see loadlibrary.c */
-
-/* Obtain entropy on Windows XP / Windows Server 2003 and later.
- * Hint on RtlGenRandom and the following article from libsodium.
- *
- * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI
- * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographically-secure-random-number-on-windows-without-using-cryptoapi/
+/* Obtain entropy on Windows using the rand_s() function which
+ * generates cryptographically secure random numbers.  Internally it
+ * uses RtlGenRandom API which is present in Windows XP and later.
  */
 static int
-writeRandomBytes_RtlGenRandom(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
-  const HMODULE advapi32 = _Expat_LoadLibrary(TEXT("ADVAPI32.DLL"));
-
-  if (advapi32) {
-    const RTLGENRANDOM_FUNC RtlGenRandom
-        = (RTLGENRANDOM_FUNC)GetProcAddress(advapi32, "SystemFunction036");
-    if (RtlGenRandom) {
-      if (RtlGenRandom((PVOID)target, (ULONG)count) == TRUE) {
-        success = 1;
-      }
+writeRandomBytes_rand_s(void *target, size_t count) {
+  size_t bytesWrittenTotal = 0;
+
+  while (bytesWrittenTotal < count) {
+    unsigned int random32 = 0;
+    size_t i = 0;
+
+    if (rand_s(&random32))
+      return 0; /* failure */
+
+    for (; (i < sizeof(random32)) && (bytesWrittenTotal < count);
+         i++, bytesWrittenTotal++) {
+      const uint8_t random8 = (uint8_t)(random32 >> (i * 8));
+      ((uint8_t *)target)[bytesWrittenTotal] = random8;
     }
-    FreeLibrary(advapi32);
   }
-
-  return success;
+  return 1; /* success */
 }
 
 #endif /* _WIN32 */
 
-
 #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)
 
 static unsigned long
-gather_time_entropy(void)
-{
-#ifdef _WIN32
+gather_time_entropy(void) {
+#  ifdef _WIN32
   FILETIME ft;
   GetSystemTimeAsFileTime(&ft); /* never fails */
   return ft.dwHighDateTime ^ ft.dwLowDateTime;
-#else
+#  else
   struct timeval tv;
   int gettimeofday_res;
 
   gettimeofday_res = gettimeofday(&tv, NULL);
 
-#if defined(NDEBUG)
+#    if defined(NDEBUG)
   (void)gettimeofday_res;
-#else
-  assert (gettimeofday_res == 0);
-#endif  /* defined(NDEBUG) */
+#    else
+  assert(gettimeofday_res == 0);
+#    endif /* defined(NDEBUG) */
 
   /* Microseconds time is <20 bits entropy */
   return tv.tv_usec;
-#endif
+#  endif
 }
 
-#endif  /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
-
+#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
 
 static unsigned long
-ENTROPY_DEBUG(const char * label, unsigned long entropy) {
-  const char * const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG");
+ENTROPY_DEBUG(const char *label, unsigned long entropy) {
+  const char *const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG");
   if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) {
-    fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n",
-        label,
-        (int)sizeof(entropy) * 2, entropy,
-        (unsigned long)sizeof(entropy));
+    fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", label,
+            (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy));
   }
   return entropy;
 }
 
 static unsigned long
-generate_hash_secret_salt(XML_Parser parser)
-{
+generate_hash_secret_salt(XML_Parser parser) {
   unsigned long entropy;
   (void)parser;
 
@@ -848,20 +811,20 @@ generate_hash_secret_salt(XML_Parser parser)
   return ENTROPY_DEBUG("arc4random", entropy);
 #else
   /* Try high quality providers first .. */
-#ifdef _WIN32
-  if (writeRandomBytes_RtlGenRandom((void *)&entropy, sizeof(entropy))) {
-    return ENTROPY_DEBUG("RtlGenRandom", entropy);
+#  ifdef _WIN32
+  if (writeRandomBytes_rand_s((void *)&entropy, sizeof(entropy))) {
+    return ENTROPY_DEBUG("rand_s", entropy);
   }
-#elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
+#  elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
   if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) {
     return ENTROPY_DEBUG("getrandom", entropy);
   }
-#endif
-#if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
+#  endif
+#  if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
   if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) {
     return ENTROPY_DEBUG("/dev/urandom", entropy);
   }
-#endif  /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
+#  endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
   /* .. and self-made low quality for backup: */
 
   /* Process ID is 0 bits entropy if attacker has local access */
@@ -872,7 +835,7 @@ generate_hash_secret_salt(XML_Parser parser)
     return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647);
   } else {
     return ENTROPY_DEBUG("fallback(8)",
-        entropy * (unsigned long)2305843009213693951ULL);
+                         entropy * (unsigned long)2305843009213693951ULL);
   }
 #endif
 }
@@ -884,49 +847,43 @@ get_hash_secret_salt(XML_Parser parser) {
   return parser->m_hash_secret_salt;
 }
 
-static XML_Bool  /* only valid for root parser */
-startParsing(XML_Parser parser)
-{
-    /* hash functions must be initialized before setContext() is called */
-    if (parser->m_hash_secret_salt == 0)
-      parser->m_hash_secret_salt = generate_hash_secret_salt(parser);
-    if (parser->m_ns) {
-      /* implicit context only set for root parser, since child
-         parsers (i.e. external entity parsers) will inherit it
-      */
-      return setContext(parser, implicitContext);
-    }
-    return XML_TRUE;
+static XML_Bool /* only valid for root parser */
+startParsing(XML_Parser parser) {
+  /* hash functions must be initialized before setContext() is called */
+  if (parser->m_hash_secret_salt == 0)
+    parser->m_hash_secret_salt = generate_hash_secret_salt(parser);
+  if (parser->m_ns) {
+    /* implicit context only set for root parser, since child
+       parsers (i.e. external entity parsers) will inherit it
+    */
+    return setContext(parser, implicitContext);
+  }
+  return XML_TRUE;
 }
 
 XML_Parser XMLCALL
 XML_ParserCreate_MM(const XML_Char *encodingName,
                     const XML_Memory_Handling_Suite *memsuite,
-                    const XML_Char *nameSep)
-{
+                    const XML_Char *nameSep) {
   return parserCreate(encodingName, memsuite, nameSep, NULL);
 }
 
 static XML_Parser
 parserCreate(const XML_Char *encodingName,
-             const XML_Memory_Handling_Suite *memsuite,
-             const XML_Char *nameSep,
-             DTD *dtd)
-{
+             const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep,
+             DTD *dtd) {
   XML_Parser parser;
 
   if (memsuite) {
     XML_Memory_Handling_Suite *mtemp;
-    parser = (XML_Parser)
-      memsuite->malloc_fcn(sizeof(struct XML_ParserStruct));
+    parser = (XML_Parser)memsuite->malloc_fcn(sizeof(struct XML_ParserStruct));
     if (parser != NULL) {
       mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem);
       mtemp->malloc_fcn = memsuite->malloc_fcn;
       mtemp->realloc_fcn = memsuite->realloc_fcn;
       mtemp->free_fcn = memsuite->free_fcn;
     }
-  }
-  else {
+  } else {
     XML_Memory_Handling_Suite *mtemp;
     parser = (XML_Parser)malloc(sizeof(struct XML_ParserStruct));
     if (parser != NULL) {
@@ -937,27 +894,30 @@ parserCreate(const XML_Char *encodingName,
     }
   }
 
-  if (!parser)
+  if (! parser)
     return parser;
 
   parser->m_buffer = NULL;
   parser->m_bufferLim = NULL;
 
   parser->m_attsSize = INIT_ATTS_SIZE;
-  parser->m_atts = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE));
+  parser->m_atts
+      = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE));
   if (parser->m_atts == NULL) {
     FREE(parser, parser);
     return NULL;
   }
 #ifdef XML_ATTR_INFO
-  parser->m_attInfo = (XML_AttrInfo*)MALLOC(parser, parser->m_attsSize * sizeof(XML_AttrInfo));
+  parser->m_attInfo = (XML_AttrInfo *)MALLOC(
+      parser, parser->m_attsSize * sizeof(XML_AttrInfo));
   if (parser->m_attInfo == NULL) {
     FREE(parser, parser->m_atts);
     FREE(parser, parser);
     return NULL;
   }
 #endif
-  parser->m_dataBuf = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char));
+  parser->m_dataBuf
+      = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char));
   if (parser->m_dataBuf == NULL) {
     FREE(parser, parser->m_atts);
 #ifdef XML_ATTR_INFO
@@ -1007,7 +967,7 @@ parserCreate(const XML_Char *encodingName,
   poolInit(&parser->m_temp2Pool, &(parser->m_mem));
   parserInit(parser, encodingName);
 
-  if (encodingName && !parser->m_protocolEncodingName) {
+  if (encodingName && ! parser->m_protocolEncodingName) {
     XML_ParserFree(parser);
     return NULL;
   }
@@ -1016,8 +976,7 @@ parserCreate(const XML_Char *encodingName,
     parser->m_ns = XML_TRUE;
     parser->m_internalEncoding = XmlGetInternalEncodingNS();
     parser->m_namespaceSeparator = *nameSep;
-  }
-  else {
+  } else {
     parser->m_internalEncoding = XmlGetInternalEncoding();
   }
 
@@ -1025,8 +984,7 @@ parserCreate(const XML_Char *encodingName,
 }
 
 static void
-parserInit(XML_Parser parser, const XML_Char *encodingName)
-{
+parserInit(XML_Parser parser, const XML_Char *encodingName) {
   parser->m_processor = prologInitProcessor;
   XmlPrologStateInit(&parser->m_prologState);
   if (encodingName != NULL) {
@@ -1099,8 +1057,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName)
 
 /* moves list of bindings to m_freeBindingList */
 static void FASTCALL
-moveToFreeBindingList(XML_Parser parser, BINDING *bindings)
-{
+moveToFreeBindingList(XML_Parser parser, BINDING *bindings) {
   while (bindings) {
     BINDING *b = bindings;
     bindings = bindings->nextTagBinding;
@@ -1110,13 +1067,12 @@ moveToFreeBindingList(XML_Parser parser, BINDING *bindings)
 }
 
 XML_Bool XMLCALL
-XML_ParserReset(XML_Parser parser, const XML_Char *encodingName)
-{
+XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) {
   TAG *tStk;
   OPEN_INTERNAL_ENTITY *openEntityList;
 
   if (parser == NULL)
-      return XML_FALSE;
+    return XML_FALSE;
 
   if (parser->m_parentParser)
     return XML_FALSE;
@@ -1152,15 +1108,15 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName)
 }
 
 enum XML_Status XMLCALL
-XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName)
-{
+XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) {
   if (parser == NULL)
-      return XML_STATUS_ERROR;
+    return XML_STATUS_ERROR;
   /* Block after XML_Parse()/XML_ParseBuffer() has been called.
      XXX There's no way for the caller to determine which of the
      XXX possible error cases caused the XML_STATUS_ERROR return.
   */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return XML_STATUS_ERROR;
 
   /* Get rid of any previous encoding name */
@@ -1172,17 +1128,15 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName)
   else {
     /* Copy the new encoding name into allocated memory */
     parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem));
-    if (!parser->m_protocolEncodingName)
+    if (! parser->m_protocolEncodingName)
       return XML_STATUS_ERROR;
   }
   return XML_STATUS_OK;
 }
 
 XML_Parser XMLCALL
-XML_ExternalEntityParserCreate(XML_Parser oldParser,
-                               const XML_Char *context,
-                               const XML_Char *encodingName)
-{
+XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
+                               const XML_Char *encodingName) {
   XML_Parser parser = oldParser;
   DTD *newDtd = NULL;
   DTD *oldDtd;
@@ -1206,7 +1160,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   XML_AttlistDeclHandler oldAttlistDeclHandler;
   XML_EntityDeclHandler oldEntityDeclHandler;
   XML_XmlDeclHandler oldXmlDeclHandler;
-  ELEMENT_TYPE * oldDeclElementType;
+  ELEMENT_TYPE *oldDeclElementType;
 
   void *oldUserData;
   void *oldHandlerArg;
@@ -1269,7 +1223,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   oldhash_secret_salt = parser->m_hash_secret_salt;
 
 #ifdef XML_DTD
-  if (!context)
+  if (! context)
     newDtd = oldDtd;
 #endif /* XML_DTD */
 
@@ -1282,12 +1236,11 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
     XML_Char tmp[2];
     *tmp = parser->m_namespaceSeparator;
     parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd);
-  }
-  else {
+  } else {
     parser = parserCreate(encodingName, &parser->m_mem, NULL, newDtd);
   }
 
-  if (!parser)
+  if (! parser)
     return NULL;
 
   parser->m_startElementHandler = oldStartElementHandler;
@@ -1327,21 +1280,20 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   parser->m_prologState.inEntityValue = oldInEntityValue;
   if (context) {
 #endif /* XML_DTD */
-    if (!dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem)
-      || !setContext(parser, context)) {
+    if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem)
+        || ! setContext(parser, context)) {
       XML_ParserFree(parser);
       return NULL;
     }
     parser->m_processor = externalEntityInitProcessor;
 #ifdef XML_DTD
-  }
-  else {
-    /* The DTD instance referenced by parser->m_dtd is shared between the document's
-       root parser and external PE parsers, therefore one does not need to
-       call setContext. In addition, one also *must* not call setContext,
-       because this would overwrite existing prefix->binding pointers in
-       parser->m_dtd with ones that get destroyed with the external PE parser.
-       This would leave those prefixes with dangling pointers.
+  } else {
+    /* The DTD instance referenced by parser->m_dtd is shared between the
+       document's root parser and external PE parsers, therefore one does not
+       need to call setContext. In addition, one also *must* not call
+       setContext, because this would overwrite existing prefix->binding
+       pointers in parser->m_dtd with ones that get destroyed with the external
+       PE parser. This would leave those prefixes with dangling pointers.
     */
     parser->m_isParamEntity = XML_TRUE;
     XmlPrologStateInitExternalEntity(&parser->m_prologState);
@@ -1352,11 +1304,10 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
 }
 
 static void FASTCALL
-destroyBindings(BINDING *bindings, XML_Parser parser)
-{
+destroyBindings(BINDING *bindings, XML_Parser parser) {
   for (;;) {
     BINDING *b = bindings;
-    if (!b)
+    if (! b)
       break;
     bindings = b->nextTagBinding;
     FREE(parser, b->uri);
@@ -1365,8 +1316,7 @@ destroyBindings(BINDING *bindings, XML_Parser parser)
 }
 
 void XMLCALL
-XML_ParserFree(XML_Parser parser)
-{
+XML_ParserFree(XML_Parser parser) {
   TAG *tagList;
   OPEN_INTERNAL_ENTITY *entityList;
   if (parser == NULL)
@@ -1411,11 +1361,12 @@ XML_ParserFree(XML_Parser parser)
   /* external parameter entity parsers share the DTD structure
      parser->m_dtd with the root parser, so we must not destroy it
   */
-  if (!parser->m_isParamEntity && parser->m_dtd)
+  if (! parser->m_isParamEntity && parser->m_dtd)
 #else
   if (parser->m_dtd)
 #endif /* XML_DTD */
-    dtdDestroy(parser->m_dtd, (XML_Bool)!parser->m_parentParser, &parser->m_mem);
+    dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser,
+               &parser->m_mem);
   FREE(parser, (void *)parser->m_atts);
 #ifdef XML_ATTR_INFO
   FREE(parser, (void *)parser->m_attInfo);
@@ -1431,20 +1382,19 @@ XML_ParserFree(XML_Parser parser)
 }
 
 void XMLCALL
-XML_UseParserAsHandlerArg(XML_Parser parser)
-{
+XML_UseParserAsHandlerArg(XML_Parser parser) {
   if (parser != NULL)
     parser->m_handlerArg = parser;
 }
 
 enum XML_Error XMLCALL
-XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD)
-{
+XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) {
   if (parser == NULL)
     return XML_ERROR_INVALID_ARGUMENT;
 #ifdef XML_DTD
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING;
   parser->m_useForeignDTD = useDTD;
   return XML_ERROR_NONE;
@@ -1454,19 +1404,18 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD)
 }
 
 void XMLCALL
-XML_SetReturnNSTriplet(XML_Parser parser, int do_nst)
-{
+XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) {
   if (parser == NULL)
     return;
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return;
   parser->m_ns_triplets = do_nst ? XML_TRUE : XML_FALSE;
 }
 
 void XMLCALL
-XML_SetUserData(XML_Parser parser, void *p)
-{
+XML_SetUserData(XML_Parser parser, void *p) {
   if (parser == NULL)
     return;
   if (parser->m_handlerArg == parser->m_userData)
@@ -1476,49 +1425,43 @@ XML_SetUserData(XML_Parser parser, void *p)
 }
 
 enum XML_Status XMLCALL
-XML_SetBase(XML_Parser parser, const XML_Char *p)
-{
+XML_SetBase(XML_Parser parser, const XML_Char *p) {
   if (parser == NULL)
     return XML_STATUS_ERROR;
   if (p) {
     p = poolCopyString(&parser->m_dtd->pool, p);
-    if (!p)
+    if (! p)
       return XML_STATUS_ERROR;
     parser->m_curBase = p;
-  }
-  else
+  } else
     parser->m_curBase = NULL;
   return XML_STATUS_OK;
 }
 
-const XML_Char * XMLCALL
-XML_GetBase(XML_Parser parser)
-{
+const XML_Char *XMLCALL
+XML_GetBase(XML_Parser parser) {
   if (parser == NULL)
     return NULL;
   return parser->m_curBase;
 }
 
 int XMLCALL
-XML_GetSpecifiedAttributeCount(XML_Parser parser)
-{
+XML_GetSpecifiedAttributeCount(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   return parser->m_nSpecifiedAtts;
 }
 
 int XMLCALL
-XML_GetIdAttributeIndex(XML_Parser parser)
-{
+XML_GetIdAttributeIndex(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   return parser->m_idAttIndex;
 }
 
 #ifdef XML_ATTR_INFO
-const XML_AttrInfo * XMLCALL
-XML_GetAttributeInfo(XML_Parser parser)
-{
+const XML_AttrInfo *XMLCALL
+XML_GetAttributeInfo(XML_Parser parser) {
   if (parser == NULL)
     return NULL;
   return parser->m_attInfo;
@@ -1526,10 +1469,8 @@ XML_GetAttributeInfo(XML_Parser parser)
 #endif
 
 void XMLCALL
-XML_SetElementHandler(XML_Parser parser,
-                      XML_StartElementHandler start,
-                      XML_EndElementHandler end)
-{
+XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start,
+                      XML_EndElementHandler end) {
   if (parser == NULL)
     return;
   parser->m_startElementHandler = start;
@@ -1537,39 +1478,33 @@ XML_SetElementHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetStartElementHandler(XML_Parser parser,
-                           XML_StartElementHandler start) {
+XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler start) {
   if (parser != NULL)
     parser->m_startElementHandler = start;
 }
 
 void XMLCALL
-XML_SetEndElementHandler(XML_Parser parser,
-                         XML_EndElementHandler end) {
+XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler end) {
   if (parser != NULL)
     parser->m_endElementHandler = end;
 }
 
 void XMLCALL
 XML_SetCharacterDataHandler(XML_Parser parser,
-                            XML_CharacterDataHandler handler)
-{
+                            XML_CharacterDataHandler handler) {
   if (parser != NULL)
     parser->m_characterDataHandler = handler;
 }
 
 void XMLCALL
 XML_SetProcessingInstructionHandler(XML_Parser parser,
-                                    XML_ProcessingInstructionHandler handler)
-{
+                                    XML_ProcessingInstructionHandler handler) {
   if (parser != NULL)
     parser->m_processingInstructionHandler = handler;
 }
 
 void XMLCALL
-XML_SetCommentHandler(XML_Parser parser,
-                      XML_CommentHandler handler)
-{
+XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler) {
   if (parser != NULL)
     parser->m_commentHandler = handler;
 }
@@ -1577,8 +1512,7 @@ XML_SetCommentHandler(XML_Parser parser,
 void XMLCALL
 XML_SetCdataSectionHandler(XML_Parser parser,
                            XML_StartCdataSectionHandler start,
-                           XML_EndCdataSectionHandler end)
-{
+                           XML_EndCdataSectionHandler end) {
   if (parser == NULL)
     return;
   parser->m_startCdataSectionHandler = start;
@@ -1600,9 +1534,7 @@ XML_SetEndCdataSectionHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDefaultHandler(XML_Parser parser,
-                      XML_DefaultHandler handler)
-{
+XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler) {
   if (parser == NULL)
     return;
   parser->m_defaultHandler = handler;
@@ -1610,9 +1542,7 @@ XML_SetDefaultHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDefaultHandlerExpand(XML_Parser parser,
-                            XML_DefaultHandler handler)
-{
+XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler) {
   if (parser == NULL)
     return;
   parser->m_defaultHandler = handler;
@@ -1620,10 +1550,8 @@ XML_SetDefaultHandlerExpand(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDoctypeDeclHandler(XML_Parser parser,
-                          XML_StartDoctypeDeclHandler start,
-                          XML_EndDoctypeDeclHandler end)
-{
+XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start,
+                          XML_EndDoctypeDeclHandler end) {
   if (parser == NULL)
     return;
   parser->m_startDoctypeDeclHandler = start;
@@ -1638,24 +1566,20 @@ XML_SetStartDoctypeDeclHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetEndDoctypeDeclHandler(XML_Parser parser,
-                             XML_EndDoctypeDeclHandler end) {
+XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end) {
   if (parser != NULL)
     parser->m_endDoctypeDeclHandler = end;
 }
 
 void XMLCALL
 XML_SetUnparsedEntityDeclHandler(XML_Parser parser,
-                                 XML_UnparsedEntityDeclHandler handler)
-{
+                                 XML_UnparsedEntityDeclHandler handler) {
   if (parser != NULL)
     parser->m_unparsedEntityDeclHandler = handler;
 }
 
 void XMLCALL
-XML_SetNotationDeclHandler(XML_Parser parser,
-                           XML_NotationDeclHandler handler)
-{
+XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler) {
   if (parser != NULL)
     parser->m_notationDeclHandler = handler;
 }
@@ -1663,8 +1587,7 @@ XML_SetNotationDeclHandler(XML_Parser parser,
 void XMLCALL
 XML_SetNamespaceDeclHandler(XML_Parser parser,
                             XML_StartNamespaceDeclHandler start,
-                            XML_EndNamespaceDeclHandler end)
-{
+                            XML_EndNamespaceDeclHandler end) {
   if (parser == NULL)
     return;
   parser->m_startNamespaceDeclHandler = start;
@@ -1687,23 +1610,20 @@ XML_SetEndNamespaceDeclHandler(XML_Parser parser,
 
 void XMLCALL
 XML_SetNotStandaloneHandler(XML_Parser parser,
-                            XML_NotStandaloneHandler handler)
-{
+                            XML_NotStandaloneHandler handler) {
   if (parser != NULL)
     parser->m_notStandaloneHandler = handler;
 }
 
 void XMLCALL
 XML_SetExternalEntityRefHandler(XML_Parser parser,
-                                XML_ExternalEntityRefHandler handler)
-{
+                                XML_ExternalEntityRefHandler handler) {
   if (parser != NULL)
     parser->m_externalEntityRefHandler = handler;
 }
 
 void XMLCALL
-XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg)
-{
+XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg) {
   if (parser == NULL)
     return;
   if (arg)
@@ -1714,17 +1634,14 @@ XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg)
 
 void XMLCALL
 XML_SetSkippedEntityHandler(XML_Parser parser,
-                            XML_SkippedEntityHandler handler)
-{
+                            XML_SkippedEntityHandler handler) {
   if (parser != NULL)
     parser->m_skippedEntityHandler = handler;
 }
 
 void XMLCALL
 XML_SetUnknownEncodingHandler(XML_Parser parser,
-                              XML_UnknownEncodingHandler handler,
-                              void *data)
-{
+                              XML_UnknownEncodingHandler handler, void *data) {
   if (parser == NULL)
     return;
   parser->m_unknownEncodingHandler = handler;
@@ -1732,44 +1649,37 @@ XML_SetUnknownEncodingHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetElementDeclHandler(XML_Parser parser,
-                          XML_ElementDeclHandler eldecl)
-{
+XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl) {
   if (parser != NULL)
     parser->m_elementDeclHandler = eldecl;
 }
 
 void XMLCALL
-XML_SetAttlistDeclHandler(XML_Parser parser,
-                          XML_AttlistDeclHandler attdecl)
-{
+XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl) {
   if (parser != NULL)
     parser->m_attlistDeclHandler = attdecl;
 }
 
 void XMLCALL
-XML_SetEntityDeclHandler(XML_Parser parser,
-                         XML_EntityDeclHandler handler)
-{
+XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler) {
   if (parser != NULL)
     parser->m_entityDeclHandler = handler;
 }
 
 void XMLCALL
-XML_SetXmlDeclHandler(XML_Parser parser,
-                      XML_XmlDeclHandler handler) {
+XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler handler) {
   if (parser != NULL)
     parser->m_xmlDeclHandler = handler;
 }
 
 int XMLCALL
 XML_SetParamEntityParsing(XML_Parser parser,
-                          enum XML_ParamEntityParsing peParsing)
-{
+                          enum XML_ParamEntityParsing peParsing) {
   if (parser == NULL)
     return 0;
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return 0;
 #ifdef XML_DTD
   parser->m_paramEntityParsing = peParsing;
@@ -1780,23 +1690,21 @@ XML_SetParamEntityParsing(XML_Parser parser,
 }
 
 int XMLCALL
-XML_SetHashSalt(XML_Parser parser,
-                unsigned long hash_salt)
-{
+XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) {
   if (parser == NULL)
     return 0;
   if (parser->m_parentParser)
     return XML_SetHashSalt(parser->m_parentParser, hash_salt);
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return 0;
   parser->m_hash_secret_salt = hash_salt;
   return 1;
 }
 
 enum XML_Status XMLCALL
-XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
-{
+XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) {
   if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) {
     if (parser != NULL)
       parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT;
@@ -1810,7 +1718,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     parser->m_errorCode = XML_ERROR_FINISHED;
     return XML_STATUS_ERROR;
   case XML_INITIALIZED:
-    if (parser->m_parentParser == NULL && !startParsing(parser)) {
+    if (parser->m_parentParser == NULL && ! startParsing(parser)) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return XML_STATUS_ERROR;
     }
@@ -1821,7 +1729,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
 
   if (len == 0) {
     parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
-    if (!isFinal)
+    if (! isFinal)
       return XML_STATUS_OK;
     parser->m_positionPtr = parser->m_bufferPtr;
     parser->m_parseEndPtr = parser->m_bufferEnd;
@@ -1830,7 +1738,9 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
        data are the final chunk of input, then we have to check them again
        to detect errors based on that fact.
     */
-    parser->m_errorCode = parser->m_processor(parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
+    parser->m_errorCode
+        = parser->m_processor(parser, parser->m_bufferPtr,
+                              parser->m_parseEndPtr, &parser->m_bufferPtr);
 
     if (parser->m_errorCode == XML_ERROR_NONE) {
       switch (parser->m_parsingStatus.parsing) {
@@ -1847,7 +1757,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
          *
          * LCOV_EXCL_START
          */
-        XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+        XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                          parser->m_bufferPtr, &parser->m_position);
         parser->m_positionPtr = parser->m_bufferPtr;
         return XML_STATUS_SUSPENDED;
         /* LCOV_EXCL_STOP */
@@ -1870,23 +1781,23 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     enum XML_Status result;
     /* Detect overflow (a+b > MAX <==> b > MAX-a) */
     if (len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) {
-       parser->m_errorCode = XML_ERROR_NO_MEMORY;
-       parser->m_eventPtr = parser->m_eventEndPtr = NULL;
-       parser->m_processor = errorProcessor;
-       return XML_STATUS_ERROR;
+      parser->m_errorCode = XML_ERROR_NO_MEMORY;
+      parser->m_eventPtr = parser->m_eventEndPtr = NULL;
+      parser->m_processor = errorProcessor;
+      return XML_STATUS_ERROR;
     }
     parser->m_parseEndByteIndex += len;
     parser->m_positionPtr = s;
     parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
 
-    parser->m_errorCode = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end);
+    parser->m_errorCode
+        = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end);
 
     if (parser->m_errorCode != XML_ERROR_NONE) {
       parser->m_eventEndPtr = parser->m_eventPtr;
       parser->m_processor = errorProcessor;
       return XML_STATUS_ERROR;
-    }
-    else {
+    } else {
       switch (parser->m_parsingStatus.parsing) {
       case XML_SUSPENDED:
         result = XML_STATUS_SUSPENDED;
@@ -1903,10 +1814,12 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
       }
     }
 
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, end, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, end,
+                      &parser->m_position);
     nLeftOver = s + len - end;
     if (nLeftOver) {
-      if (parser->m_buffer == NULL || nLeftOver > parser->m_bufferLim - parser->m_buffer) {
+      if (parser->m_buffer == NULL
+          || nLeftOver > parser->m_bufferLim - parser->m_buffer) {
         /* avoid _signed_ integer overflow */
         char *temp = NULL;
         const int bytesToAllocate = (int)((unsigned)len * 2U);
@@ -1932,7 +1845,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     parser->m_eventEndPtr = parser->m_bufferPtr;
     return result;
   }
-#endif  /* not defined XML_CONTEXT_BYTES */
+#endif /* not defined XML_CONTEXT_BYTES */
   else {
     void *buff = XML_GetBuffer(parser, len);
     if (buff == NULL)
@@ -1945,8 +1858,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
 }
 
 enum XML_Status XMLCALL
-XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
-{
+XML_ParseBuffer(XML_Parser parser, int len, int isFinal) {
   const char *start;
   enum XML_Status result = XML_STATUS_OK;
 
@@ -1960,7 +1872,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
     parser->m_errorCode = XML_ERROR_FINISHED;
     return XML_STATUS_ERROR;
   case XML_INITIALIZED:
-    if (parser->m_parentParser == NULL && !startParsing(parser)) {
+    if (parser->m_parentParser == NULL && ! startParsing(parser)) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return XML_STATUS_ERROR;
     }
@@ -1976,14 +1888,14 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
   parser->m_parseEndByteIndex += len;
   parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
 
-  parser->m_errorCode = parser->m_processor(parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr);
+  parser->m_errorCode = parser->m_processor(
+      parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr);
 
   if (parser->m_errorCode != XML_ERROR_NONE) {
     parser->m_eventEndPtr = parser->m_eventPtr;
     parser->m_processor = errorProcessor;
     return XML_STATUS_ERROR;
-  }
-  else {
+  } else {
     switch (parser->m_parsingStatus.parsing) {
     case XML_SUSPENDED:
       result = XML_STATUS_SUSPENDED;
@@ -1994,18 +1906,18 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
         parser->m_parsingStatus.parsing = XML_FINISHED;
         return result;
       }
-    default: ;  /* should not happen */
+    default:; /* should not happen */
     }
   }
 
-  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                    parser->m_bufferPtr, &parser->m_position);
   parser->m_positionPtr = parser->m_bufferPtr;
   return result;
 }
 
-void * XMLCALL
-XML_GetBuffer(XML_Parser parser, int len)
-{
+void *XMLCALL
+XML_GetBuffer(XML_Parser parser, int len) {
   if (parser == NULL)
     return NULL;
   if (len < 0) {
@@ -2019,17 +1931,17 @@ XML_GetBuffer(XML_Parser parser, int len)
   case XML_FINISHED:
     parser->m_errorCode = XML_ERROR_FINISHED;
     return NULL;
-  default: ;
+  default:;
   }
 
   if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) {
 #ifdef XML_CONTEXT_BYTES
     int keep;
-#endif  /* defined XML_CONTEXT_BYTES */
+#endif /* defined XML_CONTEXT_BYTES */
     /* Do not invoke signed arithmetic overflow: */
-    int neededSize = (int) ((unsigned)len +
-                            (unsigned)EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd,
-                                                          parser->m_bufferPtr));
+    int neededSize = (int)((unsigned)len
+                           + (unsigned)EXPAT_SAFE_PTR_DIFF(
+                               parser->m_bufferEnd, parser->m_bufferPtr));
     if (neededSize < 0) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return NULL;
@@ -2039,13 +1951,18 @@ XML_GetBuffer(XML_Parser parser, int len)
     if (keep > XML_CONTEXT_BYTES)
       keep = XML_CONTEXT_BYTES;
     neededSize += keep;
-#endif  /* defined XML_CONTEXT_BYTES */
-    if (neededSize <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) {
+#endif /* defined XML_CONTEXT_BYTES */
+    if (neededSize
+        <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) {
 #ifdef XML_CONTEXT_BYTES
       if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) {
-          int offset = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) - keep;
-        /* The buffer pointers cannot be NULL here; we have at least some bytes in the buffer */
-        memmove(parser->m_buffer, &parser->m_buffer[offset], parser->m_bufferEnd - parser->m_bufferPtr + keep);
+        int offset
+            = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)
+              - keep;
+        /* The buffer pointers cannot be NULL here; we have at least some bytes
+         * in the buffer */
+        memmove(parser->m_buffer, &parser->m_buffer[offset],
+                parser->m_bufferEnd - parser->m_bufferPtr + keep);
         parser->m_bufferEnd -= offset;
         parser->m_bufferPtr -= offset;
       }
@@ -2053,20 +1970,21 @@ XML_GetBuffer(XML_Parser parser, int len)
       if (parser->m_buffer && parser->m_bufferPtr) {
         memmove(parser->m_buffer, parser->m_bufferPtr,
                 EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr));
-        parser->m_bufferEnd = parser->m_buffer +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
+        parser->m_bufferEnd
+            = parser->m_buffer
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
         parser->m_bufferPtr = parser->m_buffer;
       }
-#endif  /* not defined XML_CONTEXT_BYTES */
-    }
-    else {
+#endif /* not defined XML_CONTEXT_BYTES */
+    } else {
       char *newBuf;
-      int bufferSize = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr);
+      int bufferSize
+          = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr);
       if (bufferSize == 0)
         bufferSize = INIT_BUFFER_SIZE;
       do {
         /* Do not invoke signed arithmetic overflow: */
-        bufferSize = (int) (2U * (unsigned) bufferSize);
+        bufferSize = (int)(2U * (unsigned)bufferSize);
       } while (bufferSize < neededSize && bufferSize > 0);
       if (bufferSize <= 0) {
         parser->m_errorCode = XML_ERROR_NO_MEMORY;
@@ -2080,18 +1998,17 @@ XML_GetBuffer(XML_Parser parser, int len)
       parser->m_bufferLim = newBuf + bufferSize;
 #ifdef XML_CONTEXT_BYTES
       if (parser->m_bufferPtr) {
-        int keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer);
-        if (keep > XML_CONTEXT_BYTES)
-          keep = XML_CONTEXT_BYTES;
         memcpy(newBuf, &parser->m_bufferPtr[-keep],
-               EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) + keep);
+               EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)
+                   + keep);
         FREE(parser, parser->m_buffer);
         parser->m_buffer = newBuf;
-        parser->m_bufferEnd = parser->m_buffer +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) + keep;
+        parser->m_bufferEnd
+            = parser->m_buffer
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)
+              + keep;
         parser->m_bufferPtr = parser->m_buffer + keep;
-      }
-      else {
+      } else {
         /* This must be a brand new buffer with no data in it yet */
         parser->m_bufferEnd = newBuf;
         parser->m_bufferPtr = parser->m_buffer = newBuf;
@@ -2101,15 +2018,15 @@ XML_GetBuffer(XML_Parser parser, int len)
         memcpy(newBuf, parser->m_bufferPtr,
                EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr));
         FREE(parser, parser->m_buffer);
-        parser->m_bufferEnd = newBuf +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
-      }
-      else {
+        parser->m_bufferEnd
+            = newBuf
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
+      } else {
         /* This must be a brand new buffer with no data in it yet */
         parser->m_bufferEnd = newBuf;
       }
       parser->m_bufferPtr = parser->m_buffer = newBuf;
-#endif  /* not defined XML_CONTEXT_BYTES */
+#endif /* not defined XML_CONTEXT_BYTES */
     }
     parser->m_eventPtr = parser->m_eventEndPtr = NULL;
     parser->m_positionPtr = NULL;
@@ -2118,8 +2035,7 @@ XML_GetBuffer(XML_Parser parser, int len)
 }
 
 enum XML_Status XMLCALL
-XML_StopParser(XML_Parser parser, XML_Bool resumable)
-{
+XML_StopParser(XML_Parser parser, XML_Bool resumable) {
   if (parser == NULL)
     return XML_STATUS_ERROR;
   switch (parser->m_parsingStatus.parsing) {
@@ -2142,16 +2058,14 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable)
       }
 #endif
       parser->m_parsingStatus.parsing = XML_SUSPENDED;
-    }
-    else
+    } else
       parser->m_parsingStatus.parsing = XML_FINISHED;
   }
   return XML_STATUS_OK;
 }
 
 enum XML_Status XMLCALL
-XML_ResumeParser(XML_Parser parser)
-{
+XML_ResumeParser(XML_Parser parser) {
   enum XML_Status result = XML_STATUS_OK;
 
   if (parser == NULL)
@@ -2162,14 +2076,14 @@ XML_ResumeParser(XML_Parser parser)
   }
   parser->m_parsingStatus.parsing = XML_PARSING;
 
-  parser->m_errorCode = parser->m_processor(parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
+  parser->m_errorCode = parser->m_processor(
+      parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
 
   if (parser->m_errorCode != XML_ERROR_NONE) {
     parser->m_eventEndPtr = parser->m_eventPtr;
     parser->m_processor = errorProcessor;
     return XML_STATUS_ERROR;
-  }
-  else {
+  } else {
     switch (parser->m_parsingStatus.parsing) {
     case XML_SUSPENDED:
       result = XML_STATUS_SUSPENDED;
@@ -2180,18 +2094,18 @@ XML_ResumeParser(XML_Parser parser)
         parser->m_parsingStatus.parsing = XML_FINISHED;
         return result;
       }
-    default: ;
+    default:;
     }
   }
 
-  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                    parser->m_bufferPtr, &parser->m_position);
   parser->m_positionPtr = parser->m_bufferPtr;
   return result;
 }
 
 void XMLCALL
-XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status)
-{
+XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) {
   if (parser == NULL)
     return;
   assert(status != NULL);
@@ -2199,26 +2113,24 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status)
 }
 
 enum XML_Error XMLCALL
-XML_GetErrorCode(XML_Parser parser)
-{
+XML_GetErrorCode(XML_Parser parser) {
   if (parser == NULL)
     return XML_ERROR_INVALID_ARGUMENT;
   return parser->m_errorCode;
 }
 
 XML_Index XMLCALL
-XML_GetCurrentByteIndex(XML_Parser parser)
-{
+XML_GetCurrentByteIndex(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   if (parser->m_eventPtr)
-    return (XML_Index)(parser->m_parseEndByteIndex - (parser->m_parseEndPtr - parser->m_eventPtr));
+    return (XML_Index)(parser->m_parseEndByteIndex
+                       - (parser->m_parseEndPtr - parser->m_eventPtr));
   return -1;
 }
 
 int XMLCALL
-XML_GetCurrentByteCount(XML_Parser parser)
-{
+XML_GetCurrentByteCount(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventEndPtr && parser->m_eventPtr)
@@ -2226,9 +2138,8 @@ XML_GetCurrentByteCount(XML_Parser parser)
   return 0;
 }
 
-const char * XMLCALL
-XML_GetInputContext(XML_Parser parser, int *offset, int *size)
-{
+const char *XMLCALL
+XML_GetInputContext(XML_Parser parser, int *offset, int *size) {
 #ifdef XML_CONTEXT_BYTES
   if (parser == NULL)
     return NULL;
@@ -2236,7 +2147,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size)
     if (offset != NULL)
       *offset = (int)(parser->m_eventPtr - parser->m_buffer);
     if (size != NULL)
-      *size   = (int)(parser->m_bufferEnd - parser->m_buffer);
+      *size = (int)(parser->m_bufferEnd - parser->m_buffer);
     return parser->m_buffer;
   }
 #else
@@ -2244,82 +2155,76 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size)
   (void)offset;
   (void)size;
 #endif /* defined XML_CONTEXT_BYTES */
-  return (char *) 0;
+  return (char *)0;
 }
 
 XML_Size XMLCALL
-XML_GetCurrentLineNumber(XML_Parser parser)
-{
+XML_GetCurrentLineNumber(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) {
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_eventPtr, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                      parser->m_eventPtr, &parser->m_position);
     parser->m_positionPtr = parser->m_eventPtr;
   }
   return parser->m_position.lineNumber + 1;
 }
 
 XML_Size XMLCALL
-XML_GetCurrentColumnNumber(XML_Parser parser)
-{
+XML_GetCurrentColumnNumber(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) {
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_eventPtr, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                      parser->m_eventPtr, &parser->m_position);
     parser->m_positionPtr = parser->m_eventPtr;
   }
   return parser->m_position.columnNumber;
 }
 
 void XMLCALL
-XML_FreeContentModel(XML_Parser parser, XML_Content *model)
-{
+XML_FreeContentModel(XML_Parser parser, XML_Content *model) {
   if (parser != NULL)
     FREE(parser, model);
 }
 
-void * XMLCALL
-XML_MemMalloc(XML_Parser parser, size_t size)
-{
+void *XMLCALL
+XML_MemMalloc(XML_Parser parser, size_t size) {
   if (parser == NULL)
     return NULL;
   return MALLOC(parser, size);
 }
 
-void * XMLCALL
-XML_MemRealloc(XML_Parser parser, void *ptr, size_t size)
-{
+void *XMLCALL
+XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) {
   if (parser == NULL)
     return NULL;
   return REALLOC(parser, ptr, size);
 }
 
 void XMLCALL
-XML_MemFree(XML_Parser parser, void *ptr)
-{
+XML_MemFree(XML_Parser parser, void *ptr) {
   if (parser != NULL)
     FREE(parser, ptr);
 }
 
 void XMLCALL
-XML_DefaultCurrent(XML_Parser parser)
-{
+XML_DefaultCurrent(XML_Parser parser) {
   if (parser == NULL)
     return;
   if (parser->m_defaultHandler) {
     if (parser->m_openInternalEntities)
-      reportDefault(parser,
-                    parser->m_internalEncoding,
+      reportDefault(parser, parser->m_internalEncoding,
                     parser->m_openInternalEntities->internalEventPtr,
                     parser->m_openInternalEntities->internalEventEndPtr);
     else
-      reportDefault(parser, parser->m_encoding, parser->m_eventPtr, parser->m_eventEndPtr);
+      reportDefault(parser, parser->m_encoding, parser->m_eventPtr,
+                    parser->m_eventEndPtr);
   }
 }
 
-const XML_LChar * XMLCALL
-XML_ErrorString(enum XML_Error code)
-{
+const XML_LChar *XMLCALL
+XML_ErrorString(enum XML_Error code) {
   switch (code) {
   case XML_ERROR_NONE:
     return NULL;
@@ -2401,21 +2306,22 @@ XML_ErrorString(enum XML_Error code)
     return XML_L("cannot suspend in external parameter entity");
   /* Added in 2.0.0. */
   case XML_ERROR_RESERVED_PREFIX_XML:
-    return XML_L("reserved prefix (xml) must not be undeclared or bound to another namespace name");
+    return XML_L(
+        "reserved prefix (xml) must not be undeclared or bound to another namespace name");
   case XML_ERROR_RESERVED_PREFIX_XMLNS:
     return XML_L("reserved prefix (xmlns) must not be declared or undeclared");
   case XML_ERROR_RESERVED_NAMESPACE_URI:
-    return XML_L("prefix must not be bound to one of the reserved namespace names");
+    return XML_L(
+        "prefix must not be bound to one of the reserved namespace names");
   /* Added in 2.2.5. */
-  case XML_ERROR_INVALID_ARGUMENT:  /* Constant added in 2.2.1, already */
+  case XML_ERROR_INVALID_ARGUMENT: /* Constant added in 2.2.1, already */
     return XML_L("invalid argument");
   }
   return NULL;
 }
 
-const XML_LChar * XMLCALL
+const XML_LChar *XMLCALL
 XML_ExpatVersion(void) {
-
   /* V1 is used to string-ize the version number. However, it would
      string-ize the actual version macro *names* unless we get them
      substituted before being passed to V1. CPP is defined to expand
@@ -2424,8 +2330,8 @@ XML_ExpatVersion(void) {
      with the correct numerals. */
   /* ### I'm assuming cpp is portable in this respect... */
 
-#define V1(a,b,c) XML_L(#a)XML_L(".")XML_L(#b)XML_L(".")XML_L(#c)
-#define V2(a,b,c) XML_L("expat_")V1(a,b,c)
+#define V1(a, b, c) XML_L(#a) XML_L(".") XML_L(#b) XML_L(".") XML_L(#c)
+#define V2(a, b, c) XML_L("expat_") V1(a, b, c)
 
   return V2(XML_MAJOR_VERSION, XML_MINOR_VERSION, XML_MICRO_VERSION);
 
@@ -2434,8 +2340,7 @@ XML_ExpatVersion(void) {
 }
 
 XML_Expat_Version XMLCALL
-XML_ExpatVersionInfo(void)
-{
+XML_ExpatVersionInfo(void) {
   XML_Expat_Version version;
 
   version.major = XML_MAJOR_VERSION;
@@ -2445,41 +2350,39 @@ XML_ExpatVersionInfo(void)
   return version;
 }
 
-const XML_Feature * XMLCALL
-XML_GetFeatureList(void)
-{
-  static const XML_Feature features[] = {
-    {XML_FEATURE_SIZEOF_XML_CHAR,  XML_L("sizeof(XML_Char)"),
-     sizeof(XML_Char)},
-    {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"),
-     sizeof(XML_LChar)},
+const XML_Feature *XMLCALL
+XML_GetFeatureList(void) {
+  static const XML_Feature features[]
+      = {{XML_FEATURE_SIZEOF_XML_CHAR, XML_L("sizeof(XML_Char)"),
+          sizeof(XML_Char)},
+         {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"),
+          sizeof(XML_LChar)},
 #ifdef XML_UNICODE
-    {XML_FEATURE_UNICODE,          XML_L("XML_UNICODE"), 0},
+         {XML_FEATURE_UNICODE, XML_L("XML_UNICODE"), 0},
 #endif
 #ifdef XML_UNICODE_WCHAR_T
-    {XML_FEATURE_UNICODE_WCHAR_T,  XML_L("XML_UNICODE_WCHAR_T"), 0},
+         {XML_FEATURE_UNICODE_WCHAR_T, XML_L("XML_UNICODE_WCHAR_T"), 0},
 #endif
 #ifdef XML_DTD
-    {XML_FEATURE_DTD,              XML_L("XML_DTD"), 0},
+         {XML_FEATURE_DTD, XML_L("XML_DTD"), 0},
 #endif
 #ifdef XML_CONTEXT_BYTES
-    {XML_FEATURE_CONTEXT_BYTES,    XML_L("XML_CONTEXT_BYTES"),
-     XML_CONTEXT_BYTES},
+         {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"),
+          XML_CONTEXT_BYTES},
 #endif
 #ifdef XML_MIN_SIZE
-    {XML_FEATURE_MIN_SIZE,         XML_L("XML_MIN_SIZE"), 0},
+         {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0},
 #endif
 #ifdef XML_NS
-    {XML_FEATURE_NS,               XML_L("XML_NS"), 0},
+         {XML_FEATURE_NS, XML_L("XML_NS"), 0},
 #endif
 #ifdef XML_LARGE_SIZE
-    {XML_FEATURE_LARGE_SIZE,       XML_L("XML_LARGE_SIZE"), 0},
+         {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0},
 #endif
 #ifdef XML_ATTR_INFO
-    {XML_FEATURE_ATTR_INFO,        XML_L("XML_ATTR_INFO"), 0},
+         {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0},
 #endif
-    {XML_FEATURE_END,              NULL, 0}
-  };
+         {XML_FEATURE_END, NULL, 0}};
 
   return features;
 }
@@ -2490,8 +2393,7 @@ XML_GetFeatureList(void)
    permanent location, since the parse buffer is about to be discarded.
 */
 static XML_Bool
-storeRawNames(XML_Parser parser)
-{
+storeRawNames(XML_Parser parser) {
   TAG *tag = parser->m_tagStack;
   while (tag) {
     int bufSize;
@@ -2521,8 +2423,8 @@ storeRawNames(XML_Parser parser)
          then update it as well, since it will always point into tag->buf
       */
       if (tag->name.localPart)
-        tag->name.localPart = (XML_Char *)temp + (tag->name.localPart -
-                                                  (XML_Char *)tag->buf);
+        tag->name.localPart
+            = (XML_Char *)temp + (tag->name.localPart - (XML_Char *)tag->buf);
       tag->buf = temp;
       tag->bufEnd = temp + bufSize;
       rawNameBuf = temp + nameLen;
@@ -2535,26 +2437,21 @@ storeRawNames(XML_Parser parser)
 }
 
 static enum XML_Error PTRCALL
-contentProcessor(XML_Parser parser,
-                 const char *start,
-                 const char *end,
-                 const char **endPtr)
-{
-  enum XML_Error result = doContent(parser, 0, parser->m_encoding, start, end,
-                                    endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+contentProcessor(XML_Parser parser, const char *start, const char *end,
+                 const char **endPtr) {
+  enum XML_Error result
+      = doContent(parser, 0, parser->m_encoding, start, end, endPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result == XML_ERROR_NONE) {
-    if (!storeRawNames(parser))
+    if (! storeRawNames(parser))
       return XML_ERROR_NO_MEMORY;
   }
   return result;
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor(XML_Parser parser,
-                            const char *start,
-                            const char *end,
-                            const char **endPtr)
-{
+externalEntityInitProcessor(XML_Parser parser, const char *start,
+                            const char *end, const char **endPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -2563,11 +2460,8 @@ externalEntityInitProcessor(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor2(XML_Parser parser,
-                             const char *start,
-                             const char *end,
-                             const char **endPtr)
-{
+externalEntityInitProcessor2(XML_Parser parser, const char *start,
+                             const char *end, const char **endPtr) {
   const char *next = start; /* XmlContentTok doesn't always set the last arg */
   int tok = XmlContentTok(parser->m_encoding, start, end, &next);
   switch (tok) {
@@ -2577,21 +2471,21 @@ externalEntityInitProcessor2(XML_Parser parser,
        doContent (by detecting XML_TOK_NONE) without processing any xml text
        declaration - causing the error XML_ERROR_MISPLACED_XML_PI in doContent.
     */
-    if (next == end && !parser->m_parsingStatus.finalBuffer) {
+    if (next == end && ! parser->m_parsingStatus.finalBuffer) {
       *endPtr = next;
       return XML_ERROR_NONE;
     }
     start = next;
     break;
   case XML_TOK_PARTIAL:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
     parser->m_eventPtr = start;
     return XML_ERROR_UNCLOSED_TOKEN;
   case XML_TOK_PARTIAL_CHAR:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
@@ -2603,11 +2497,8 @@ externalEntityInitProcessor2(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor3(XML_Parser parser,
-                             const char *start,
-                             const char *end,
-                             const char **endPtr)
-{
+externalEntityInitProcessor3(XML_Parser parser, const char *start,
+                             const char *end, const char **endPtr) {
   int tok;
   const char *next = start; /* XmlContentTok doesn't always set the last arg */
   parser->m_eventPtr = start;
@@ -2615,31 +2506,29 @@ externalEntityInitProcessor3(XML_Parser parser,
   parser->m_eventEndPtr = next;
 
   switch (tok) {
-  case XML_TOK_XML_DECL:
-    {
-      enum XML_Error result;
-      result = processXmlDecl(parser, 1, start, next);
-      if (result != XML_ERROR_NONE)
-        return result;
-      switch (parser->m_parsingStatus.parsing) {
-      case XML_SUSPENDED:
-        *endPtr = next;
-        return XML_ERROR_NONE;
-      case XML_FINISHED:
-        return XML_ERROR_ABORTED;
-      default:
-        start = next;
-      }
+  case XML_TOK_XML_DECL: {
+    enum XML_Error result;
+    result = processXmlDecl(parser, 1, start, next);
+    if (result != XML_ERROR_NONE)
+      return result;
+    switch (parser->m_parsingStatus.parsing) {
+    case XML_SUSPENDED:
+      *endPtr = next;
+      return XML_ERROR_NONE;
+    case XML_FINISHED:
+      return XML_ERROR_ABORTED;
+    default:
+      start = next;
     }
-    break;
+  } break;
   case XML_TOK_PARTIAL:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
     return XML_ERROR_UNCLOSED_TOKEN;
   case XML_TOK_PARTIAL_CHAR:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
@@ -2651,39 +2540,31 @@ externalEntityInitProcessor3(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityContentProcessor(XML_Parser parser,
-                               const char *start,
-                               const char *end,
-                               const char **endPtr)
-{
-  enum XML_Error result = doContent(parser, 1, parser->m_encoding, start, end,
-                                    endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+externalEntityContentProcessor(XML_Parser parser, const char *start,
+                               const char *end, const char **endPtr) {
+  enum XML_Error result
+      = doContent(parser, 1, parser->m_encoding, start, end, endPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result == XML_ERROR_NONE) {
-    if (!storeRawNames(parser))
+    if (! storeRawNames(parser))
       return XML_ERROR_NO_MEMORY;
   }
   return result;
 }
 
 static enum XML_Error
-doContent(XML_Parser parser,
-          int startTagLevel,
-          const ENCODING *enc,
-          const char *s,
-          const char *end,
-          const char **nextPtr,
-          XML_Bool haveMore)
-{
+doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
+          const char *s, const char *end, const char **nextPtr,
+          XML_Bool haveMore) {
   /* save one level of indirection */
-  DTD * const dtd = parser->m_dtd;
+  DTD *const dtd = parser->m_dtd;
 
   const char **eventPP;
   const char **eventEndPP;
   if (enc == parser->m_encoding) {
     eventPP = &parser->m_eventPtr;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -2703,8 +2584,7 @@ doContent(XML_Parser parser,
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, end);
       /* We are at the end of the final buffer, should we check for
          XML_SUSPENDED, XML_FINISHED?
@@ -2742,185 +2622,178 @@ doContent(XML_Parser parser,
         return XML_ERROR_NONE;
       }
       return XML_ERROR_PARTIAL_CHAR;
-    case XML_TOK_ENTITY_REF:
-      {
-        const XML_Char *name;
-        ENTITY *entity;
-        XML_Char ch = (XML_Char) XmlPredefinedEntityName(enc,
-                                              s + enc->minBytesPerChar,
-                                              next - enc->minBytesPerChar);
-        if (ch) {
-          if (parser->m_characterDataHandler)
-            parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1);
-          else if (parser->m_defaultHandler)
-            reportDefault(parser, enc, s, next);
-          break;
-        }
-        name = poolStoreString(&dtd->pool, enc,
-                                s + enc->minBytesPerChar,
-                                next - enc->minBytesPerChar);
-        if (!name)
-          return XML_ERROR_NO_MEMORY;
-        entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
-        poolDiscard(&dtd->pool);
-        /* First, determine if a check for an existing declaration is needed;
-           if yes, check that the entity exists, and that it is internal,
-           otherwise call the skipped entity or default handler.
-        */
-        if (!dtd->hasParamEntityRefs || dtd->standalone) {
-          if (!entity)
-            return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal)
-            return XML_ERROR_ENTITY_DECLARED_IN_PE;
-        }
-        else if (!entity) {
+    case XML_TOK_ENTITY_REF: {
+      const XML_Char *name;
+      ENTITY *entity;
+      XML_Char ch = (XML_Char)XmlPredefinedEntityName(
+          enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar);
+      if (ch) {
+        if (parser->m_characterDataHandler)
+          parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1);
+        else if (parser->m_defaultHandler)
+          reportDefault(parser, enc, s, next);
+        break;
+      }
+      name = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                             next - enc->minBytesPerChar);
+      if (! name)
+        return XML_ERROR_NO_MEMORY;
+      entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
+      poolDiscard(&dtd->pool);
+      /* First, determine if a check for an existing declaration is needed;
+         if yes, check that the entity exists, and that it is internal,
+         otherwise call the skipped entity or default handler.
+      */
+      if (! dtd->hasParamEntityRefs || dtd->standalone) {
+        if (! entity)
+          return XML_ERROR_UNDEFINED_ENTITY;
+        else if (! entity->is_internal)
+          return XML_ERROR_ENTITY_DECLARED_IN_PE;
+      } else if (! entity) {
+        if (parser->m_skippedEntityHandler)
+          parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+        else if (parser->m_defaultHandler)
+          reportDefault(parser, enc, s, next);
+        break;
+      }
+      if (entity->open)
+        return XML_ERROR_RECURSIVE_ENTITY_REF;
+      if (entity->notation)
+        return XML_ERROR_BINARY_ENTITY_REF;
+      if (entity->textPtr) {
+        enum XML_Error result;
+        if (! parser->m_defaultExpandInternalEntities) {
           if (parser->m_skippedEntityHandler)
-            parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+            parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name,
+                                           0);
           else if (parser->m_defaultHandler)
             reportDefault(parser, enc, s, next);
           break;
         }
-        if (entity->open)
-          return XML_ERROR_RECURSIVE_ENTITY_REF;
-        if (entity->notation)
-          return XML_ERROR_BINARY_ENTITY_REF;
-        if (entity->textPtr) {
-          enum XML_Error result;
-          if (!parser->m_defaultExpandInternalEntities) {
-            if (parser->m_skippedEntityHandler)
-              parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name, 0);
-            else if (parser->m_defaultHandler)
-              reportDefault(parser, enc, s, next);
-            break;
-          }
-          result = processInternalEntity(parser, entity, XML_FALSE);
-          if (result != XML_ERROR_NONE)
-            return result;
-        }
-        else if (parser->m_externalEntityRefHandler) {
-          const XML_Char *context;
-          entity->open = XML_TRUE;
-          context = getContext(parser);
-          entity->open = XML_FALSE;
-          if (!context)
-            return XML_ERROR_NO_MEMORY;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        context,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
-            return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
-          poolDiscard(&parser->m_tempPool);
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        break;
-      }
+        result = processInternalEntity(parser, entity, XML_FALSE);
+        if (result != XML_ERROR_NONE)
+          return result;
+      } else if (parser->m_externalEntityRefHandler) {
+        const XML_Char *context;
+        entity->open = XML_TRUE;
+        context = getContext(parser);
+        entity->open = XML_FALSE;
+        if (! context)
+          return XML_ERROR_NO_MEMORY;
+        if (! parser->m_externalEntityRefHandler(
+                parser->m_externalEntityRefHandlerArg, context, entity->base,
+                entity->systemId, entity->publicId))
+          return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
+        poolDiscard(&parser->m_tempPool);
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      break;
+    }
     case XML_TOK_START_TAG_NO_ATTS:
       /* fall through */
-    case XML_TOK_START_TAG_WITH_ATTS:
-      {
-        TAG *tag;
-        enum XML_Error result;
-        XML_Char *toPtr;
-        if (parser->m_freeTagList) {
-          tag = parser->m_freeTagList;
-          parser->m_freeTagList = parser->m_freeTagList->parent;
+    case XML_TOK_START_TAG_WITH_ATTS: {
+      TAG *tag;
+      enum XML_Error result;
+      XML_Char *toPtr;
+      if (parser->m_freeTagList) {
+        tag = parser->m_freeTagList;
+        parser->m_freeTagList = parser->m_freeTagList->parent;
+      } else {
+        tag = (TAG *)MALLOC(parser, sizeof(TAG));
+        if (! tag)
+          return XML_ERROR_NO_MEMORY;
+        tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE);
+        if (! tag->buf) {
+          FREE(parser, tag);
+          return XML_ERROR_NO_MEMORY;
         }
-        else {
-          tag = (TAG *)MALLOC(parser, sizeof(TAG));
-          if (!tag)
-            return XML_ERROR_NO_MEMORY;
-          tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE);
-          if (!tag->buf) {
-            FREE(parser, tag);
-            return XML_ERROR_NO_MEMORY;
+        tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE;
+      }
+      tag->bindings = NULL;
+      tag->parent = parser->m_tagStack;
+      parser->m_tagStack = tag;
+      tag->name.localPart = NULL;
+      tag->name.prefix = NULL;
+      tag->rawName = s + enc->minBytesPerChar;
+      tag->rawNameLength = XmlNameLength(enc, tag->rawName);
+      ++parser->m_tagLevel;
+      {
+        const char *rawNameEnd = tag->rawName + tag->rawNameLength;
+        const char *fromPtr = tag->rawName;
+        toPtr = (XML_Char *)tag->buf;
+        for (;;) {
+          int bufSize;
+          int convLen;
+          const enum XML_Convert_Result convert_res
+              = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr,
+                           (ICHAR *)tag->bufEnd - 1);
+          convLen = (int)(toPtr - (XML_Char *)tag->buf);
+          if ((fromPtr >= rawNameEnd)
+              || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) {
+            tag->name.strLen = convLen;
+            break;
           }
-          tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE;
-        }
-        tag->bindings = NULL;
-        tag->parent = parser->m_tagStack;
-        parser->m_tagStack = tag;
-        tag->name.localPart = NULL;
-        tag->name.prefix = NULL;
-        tag->rawName = s + enc->minBytesPerChar;
-        tag->rawNameLength = XmlNameLength(enc, tag->rawName);
-        ++parser->m_tagLevel;
-        {
-          const char *rawNameEnd = tag->rawName + tag->rawNameLength;
-          const char *fromPtr = tag->rawName;
-          toPtr = (XML_Char *)tag->buf;
-          for (;;) {
-            int bufSize;
-            int convLen;
-            const enum XML_Convert_Result convert_res = XmlConvert(enc,
-                       &fromPtr, rawNameEnd,
-                       (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1);
-            convLen = (int)(toPtr - (XML_Char *)tag->buf);
-            if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) {
-              tag->name.strLen = convLen;
-              break;
-            }
-            bufSize = (int)(tag->bufEnd - tag->buf) << 1;
-            {
-              char *temp = (char *)REALLOC(parser, tag->buf, bufSize);
-              if (temp == NULL)
-                return XML_ERROR_NO_MEMORY;
-              tag->buf = temp;
-              tag->bufEnd = temp + bufSize;
-              toPtr = (XML_Char *)temp + convLen;
-            }
+          bufSize = (int)(tag->bufEnd - tag->buf) << 1;
+          {
+            char *temp = (char *)REALLOC(parser, tag->buf, bufSize);
+            if (temp == NULL)
+              return XML_ERROR_NO_MEMORY;
+            tag->buf = temp;
+            tag->bufEnd = temp + bufSize;
+            toPtr = (XML_Char *)temp + convLen;
           }
         }
-        tag->name.str = (XML_Char *)tag->buf;
-        *toPtr = XML_T('\0');
-        result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings));
-        if (result)
-          return result;
-        if (parser->m_startElementHandler)
-          parser->m_startElementHandler(parser->m_handlerArg, tag->name.str,
-                              (const XML_Char **)parser->m_atts);
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        poolClear(&parser->m_tempPool);
-        break;
       }
+      tag->name.str = (XML_Char *)tag->buf;
+      *toPtr = XML_T('\0');
+      result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings));
+      if (result)
+        return result;
+      if (parser->m_startElementHandler)
+        parser->m_startElementHandler(parser->m_handlerArg, tag->name.str,
+                                      (const XML_Char **)parser->m_atts);
+      else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      poolClear(&parser->m_tempPool);
+      break;
+    }
     case XML_TOK_EMPTY_ELEMENT_NO_ATTS:
       /* fall through */
-    case XML_TOK_EMPTY_ELEMENT_WITH_ATTS:
-      {
-        const char *rawName = s + enc->minBytesPerChar;
-        enum XML_Error result;
-        BINDING *bindings = NULL;
-        XML_Bool noElmHandlers = XML_TRUE;
-        TAG_NAME name;
-        name.str = poolStoreString(&parser->m_tempPool, enc, rawName,
-                                   rawName + XmlNameLength(enc, rawName));
-        if (!name.str)
-          return XML_ERROR_NO_MEMORY;
-        poolFinish(&parser->m_tempPool);
-        result = storeAtts(parser, enc, s, &name, &bindings);
-        if (result != XML_ERROR_NONE) {
-          freeBindings(parser, bindings);
-          return result;
-        }
-        poolFinish(&parser->m_tempPool);
-        if (parser->m_startElementHandler) {
-          parser->m_startElementHandler(parser->m_handlerArg, name.str, (const XML_Char **)parser->m_atts);
-          noElmHandlers = XML_FALSE;
-        }
-        if (parser->m_endElementHandler) {
-          if (parser->m_startElementHandler)
-            *eventPP = *eventEndPP;
-          parser->m_endElementHandler(parser->m_handlerArg, name.str);
-          noElmHandlers = XML_FALSE;
-        }
-        if (noElmHandlers && parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        poolClear(&parser->m_tempPool);
+    case XML_TOK_EMPTY_ELEMENT_WITH_ATTS: {
+      const char *rawName = s + enc->minBytesPerChar;
+      enum XML_Error result;
+      BINDING *bindings = NULL;
+      XML_Bool noElmHandlers = XML_TRUE;
+      TAG_NAME name;
+      name.str = poolStoreString(&parser->m_tempPool, enc, rawName,
+                                 rawName + XmlNameLength(enc, rawName));
+      if (! name.str)
+        return XML_ERROR_NO_MEMORY;
+      poolFinish(&parser->m_tempPool);
+      result = storeAtts(parser, enc, s, &name, &bindings);
+      if (result != XML_ERROR_NONE) {
         freeBindings(parser, bindings);
+        return result;
+      }
+      poolFinish(&parser->m_tempPool);
+      if (parser->m_startElementHandler) {
+        parser->m_startElementHandler(parser->m_handlerArg, name.str,
+                                      (const XML_Char **)parser->m_atts);
+        noElmHandlers = XML_FALSE;
+      }
+      if (parser->m_endElementHandler) {
+        if (parser->m_startElementHandler)
+          *eventPP = *eventEndPP;
+        parser->m_endElementHandler(parser->m_handlerArg, name.str);
+        noElmHandlers = XML_FALSE;
       }
-      if ((parser->m_tagLevel == 0) && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
+      if (noElmHandlers && parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      poolClear(&parser->m_tempPool);
+      freeBindings(parser, bindings);
+    }
+      if ((parser->m_tagLevel == 0)
+          && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
         if (parser->m_parsingStatus.parsing == XML_SUSPENDED)
           parser->m_processor = epilogProcessor;
         else
@@ -2937,7 +2810,7 @@ doContent(XML_Parser parser,
         parser->m_tagStack = tag->parent;
         tag->parent = parser->m_freeTagList;
         parser->m_freeTagList = tag;
-        rawName = s + enc->minBytesPerChar*2;
+        rawName = s + enc->minBytesPerChar * 2;
         len = XmlNameLength(enc, rawName);
         if (len != tag->rawNameLength
             || memcmp(tag->rawName, rawName, len) != 0) {
@@ -2957,86 +2830,89 @@ doContent(XML_Parser parser,
             */
             uri = (XML_Char *)tag->name.str + tag->name.uriLen;
             /* don't need to check for space - already done in storeAtts() */
-            while (*localPart) *uri++ = *localPart++;
+            while (*localPart)
+              *uri++ = *localPart++;
             prefix = (XML_Char *)tag->name.prefix;
             if (parser->m_ns_triplets && prefix) {
               *uri++ = parser->m_namespaceSeparator;
-              while (*prefix) *uri++ = *prefix++;
-             }
+              while (*prefix)
+                *uri++ = *prefix++;
+            }
             *uri = XML_T('\0');
           }
           parser->m_endElementHandler(parser->m_handlerArg, tag->name.str);
-        }
-        else if (parser->m_defaultHandler)
+        } else if (parser->m_defaultHandler)
           reportDefault(parser, enc, s, next);
         while (tag->bindings) {
           BINDING *b = tag->bindings;
           if (parser->m_endNamespaceDeclHandler)
-            parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
+            parser->m_endNamespaceDeclHandler(parser->m_handlerArg,
+                                              b->prefix->name);
           tag->bindings = tag->bindings->nextTagBinding;
           b->nextTagBinding = parser->m_freeBindingList;
           parser->m_freeBindingList = b;
           b->prefix->binding = b->prevPrefixBinding;
         }
-        if (parser->m_tagLevel == 0)
-          return epilogProcessor(parser, next, end, nextPtr);
-      }
-      break;
-    case XML_TOK_CHAR_REF:
-      {
-        int n = XmlCharRefNumber(enc, s);
-        if (n < 0)
-          return XML_ERROR_BAD_CHAR_REF;
-        if (parser->m_characterDataHandler) {
-          XML_Char buf[XML_ENCODE_MAX];
-          parser->m_characterDataHandler(parser->m_handlerArg, buf, XmlEncode(n, (ICHAR *)buf));
+        if ((parser->m_tagLevel == 0)
+            && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
+          if (parser->m_parsingStatus.parsing == XML_SUSPENDED)
+            parser->m_processor = epilogProcessor;
+          else
+            return epilogProcessor(parser, next, end, nextPtr);
         }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
       }
       break;
+    case XML_TOK_CHAR_REF: {
+      int n = XmlCharRefNumber(enc, s);
+      if (n < 0)
+        return XML_ERROR_BAD_CHAR_REF;
+      if (parser->m_characterDataHandler) {
+        XML_Char buf[XML_ENCODE_MAX];
+        parser->m_characterDataHandler(parser->m_handlerArg, buf,
+                                       XmlEncode(n, (ICHAR *)buf));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_XML_DECL:
       return XML_ERROR_MISPLACED_XML_PI;
     case XML_TOK_DATA_NEWLINE:
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       break;
-    case XML_TOK_CDATA_SECT_OPEN:
-      {
-        enum XML_Error result;
-        if (parser->m_startCdataSectionHandler)
-          parser->m_startCdataSectionHandler(parser->m_handlerArg);
-/* BEGIN disabled code */
-        /* Suppose you doing a transformation on a document that involves
-           changing only the character data.  You set up a defaultHandler
-           and a characterDataHandler.  The defaultHandler simply copies
-           characters through.  The characterDataHandler does the
-           transformation and writes the characters out escaping them as
-           necessary.  This case will fail to work if we leave out the
-           following two lines (because & and < inside CDATA sections will
-           be incorrectly escaped).
-
-           However, now we have a start/endCdataSectionHandler, so it seems
-           easier to let the user deal with this.
-        */
-        else if (0 && parser->m_characterDataHandler)
-          parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0);
-/* END disabled code */
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore);
-        if (result != XML_ERROR_NONE)
-          return result;
-        else if (!next) {
-          parser->m_processor = cdataSectionProcessor;
-          return result;
-        }
+    case XML_TOK_CDATA_SECT_OPEN: {
+      enum XML_Error result;
+      if (parser->m_startCdataSectionHandler)
+        parser->m_startCdataSectionHandler(parser->m_handlerArg);
+      /* BEGIN disabled code */
+      /* Suppose you doing a transformation on a document that involves
+         changing only the character data.  You set up a defaultHandler
+         and a characterDataHandler.  The defaultHandler simply copies
+         characters through.  The characterDataHandler does the
+         transformation and writes the characters out escaping them as
+         necessary.  This case will fail to work if we leave out the
+         following two lines (because & and < inside CDATA sections will
+         be incorrectly escaped).
+
+         However, now we have a start/endCdataSectionHandler, so it seems
+         easier to let the user deal with this.
+      */
+      else if (0 && parser->m_characterDataHandler)
+        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                                       0);
+      /* END disabled code */
+      else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore);
+      if (result != XML_ERROR_NONE)
+        return result;
+      else if (! next) {
+        parser->m_processor = cdataSectionProcessor;
+        return result;
       }
-      break;
+    } break;
     case XML_TOK_TRAILING_RSQB:
       if (haveMore) {
         *nextPtr = s;
@@ -3046,15 +2922,14 @@ doContent(XML_Parser parser,
         if (MUST_CONVERT(enc, s)) {
           ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
           XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-          parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                               (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-        }
-        else
-          parser->m_characterDataHandler(parser->m_handlerArg,
-                               (XML_Char *)s,
-                               (int)((XML_Char *)end - (XML_Char *)s));
-      }
-      else if (parser->m_defaultHandler)
+          parser->m_characterDataHandler(
+              parser->m_handlerArg, parser->m_dataBuf,
+              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+        } else
+          parser->m_characterDataHandler(
+              parser->m_handlerArg, (XML_Char *)s,
+              (int)((XML_Char *)end - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, end);
       /* We are at the end of the final buffer, should we check for
          XML_SUSPENDED, XML_FINISHED?
@@ -3069,37 +2944,34 @@ doContent(XML_Parser parser,
       }
       *nextPtr = end;
       return XML_ERROR_NONE;
-    case XML_TOK_DATA_CHARS:
-      {
-        XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
-        if (charDataHandler) {
-          if (MUST_CONVERT(enc, s)) {
-            for (;;) {
-              ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-              const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-              *eventEndPP = s;
-              charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-              if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
-                break;
-              *eventPP = s;
-            }
+    case XML_TOK_DATA_CHARS: {
+      XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
+      if (charDataHandler) {
+        if (MUST_CONVERT(enc, s)) {
+          for (;;) {
+            ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
+            const enum XML_Convert_Result convert_res = XmlConvert(
+                enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+            *eventEndPP = s;
+            charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                            (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+            if ((convert_res == XML_CONVERT_COMPLETED)
+                || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+              break;
+            *eventPP = s;
           }
-          else
-            charDataHandler(parser->m_handlerArg,
-                            (XML_Char *)s,
-                            (int)((XML_Char *)next - (XML_Char *)s));
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-      }
-      break;
+        } else
+          charDataHandler(parser->m_handlerArg, (XML_Char *)s,
+                          (int)((XML_Char *)next - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_PI:
-      if (!reportProcessingInstruction(parser, enc, s, next))
+      if (! reportProcessingInstruction(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_COMMENT:
-      if (!reportComment(parser, enc, s, next))
+      if (! reportComment(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     default:
@@ -3122,7 +2994,7 @@ doContent(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
   /* not reached */
@@ -3133,8 +3005,7 @@ doContent(XML_Parser parser,
  * reused as appropriate.
  */
 static void
-freeBindings(XML_Parser parser, BINDING *bindings)
-{
+freeBindings(XML_Parser parser, BINDING *bindings) {
   while (bindings) {
     BINDING *b = bindings;
 
@@ -3142,7 +3013,7 @@ freeBindings(XML_Parser parser, BINDING *bindings)
      * binding in addBindings(), so call the end handler now.
      */
     if (parser->m_endNamespaceDeclHandler)
-        parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
+      parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
 
     bindings = bindings->nextTagBinding;
     b->nextTagBinding = parser->m_freeBindingList;
@@ -3162,14 +3033,12 @@ freeBindings(XML_Parser parser, BINDING *bindings)
    - generate namespace aware element name (URI, prefix)
 */
 static enum XML_Error
-storeAtts(XML_Parser parser, const ENCODING *enc,
-          const char *attStr, TAG_NAME *tagNamePtr,
-          BINDING **bindingsPtr)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
+          TAG_NAME *tagNamePtr, BINDING **bindingsPtr) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   ELEMENT_TYPE *elementType;
   int nDefaultAtts;
-  const XML_Char **appAtts;   /* the attribute list for the application */
+  const XML_Char **appAtts; /* the attribute list for the application */
   int attIndex = 0;
   int prefixLen;
   int i;
@@ -3180,16 +3049,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   const XML_Char *localPart;
 
   /* lookup the element type name */
-  elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str,0);
-  if (!elementType) {
+  elementType
+      = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str, 0);
+  if (! elementType) {
     const XML_Char *name = poolCopyString(&dtd->pool, tagNamePtr->str);
-    if (!name)
+    if (! name)
       return XML_ERROR_NO_MEMORY;
     elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, name,
                                          sizeof(ELEMENT_TYPE));
-    if (!elementType)
+    if (! elementType)
       return XML_ERROR_NO_MEMORY;
-    if (parser->m_ns && !setElementTypePrefix(parser, elementType))
+    if (parser->m_ns && ! setElementTypePrefix(parser, elementType))
       return XML_ERROR_NO_MEMORY;
   }
   nDefaultAtts = elementType->nDefaultAtts;
@@ -3203,14 +3073,16 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     XML_AttrInfo *temp2;
 #endif
     parser->m_attsSize = n + nDefaultAtts + INIT_ATTS_SIZE;
-    temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts, parser->m_attsSize * sizeof(ATTRIBUTE));
+    temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts,
+                                parser->m_attsSize * sizeof(ATTRIBUTE));
     if (temp == NULL) {
       parser->m_attsSize = oldAttsSize;
       return XML_ERROR_NO_MEMORY;
     }
     parser->m_atts = temp;
 #ifdef XML_ATTR_INFO
-    temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo, parser->m_attsSize * sizeof(XML_AttrInfo));
+    temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo,
+                                    parser->m_attsSize * sizeof(XML_AttrInfo));
     if (temp2 == NULL) {
       parser->m_attsSize = oldAttsSize;
       return XML_ERROR_NO_MEMORY;
@@ -3228,18 +3100,20 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     XML_AttrInfo *currAttInfo = &parser->m_attInfo[i];
 #endif
     /* add the name and value to the attribute list */
-    ATTRIBUTE_ID *attId = getAttributeId(parser, enc, currAtt->name,
-                                         currAtt->name
-                                         + XmlNameLength(enc, currAtt->name));
-    if (!attId)
+    ATTRIBUTE_ID *attId
+        = getAttributeId(parser, enc, currAtt->name,
+                         currAtt->name + XmlNameLength(enc, currAtt->name));
+    if (! attId)
       return XML_ERROR_NO_MEMORY;
 #ifdef XML_ATTR_INFO
-    currAttInfo->nameStart = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name);
-    currAttInfo->nameEnd = currAttInfo->nameStart +
-                           XmlNameLength(enc, currAtt->name);
-    currAttInfo->valueStart = parser->m_parseEndByteIndex -
-                            (parser->m_parseEndPtr - currAtt->valuePtr);
-    currAttInfo->valueEnd = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->valueEnd);
+    currAttInfo->nameStart
+        = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name);
+    currAttInfo->nameEnd
+        = currAttInfo->nameStart + XmlNameLength(enc, currAtt->name);
+    currAttInfo->valueStart = parser->m_parseEndByteIndex
+                              - (parser->m_parseEndPtr - currAtt->valuePtr);
+    currAttInfo->valueEnd = parser->m_parseEndByteIndex
+                            - (parser->m_parseEndPtr - currAtt->valueEnd);
 #endif
     /* Detect duplicate attributes by their QNames. This does not work when
        namespace processing is turned on and different prefixes for the same
@@ -3252,7 +3126,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     }
     (attId->name)[-1] = 1;
     appAtts[attIndex++] = attId->name;
-    if (!parser->m_atts[i].normalized) {
+    if (! parser->m_atts[i].normalized) {
       enum XML_Error result;
       XML_Bool isCdata = XML_TRUE;
 
@@ -3268,17 +3142,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
       }
 
       /* normalize the attribute value */
-      result = storeAttributeValue(parser, enc, isCdata,
-                                   parser->m_atts[i].valuePtr, parser->m_atts[i].valueEnd,
-                                   &parser->m_tempPool);
+      result = storeAttributeValue(
+          parser, enc, isCdata, parser->m_atts[i].valuePtr,
+          parser->m_atts[i].valueEnd, &parser->m_tempPool);
       if (result)
         return result;
       appAtts[attIndex] = poolStart(&parser->m_tempPool);
       poolFinish(&parser->m_tempPool);
-    }
-    else {
+    } else {
       /* the value did not need normalizing */
-      appAtts[attIndex] = poolStoreString(&parser->m_tempPool, enc, parser->m_atts[i].valuePtr,
+      appAtts[attIndex] = poolStoreString(&parser->m_tempPool, enc,
+                                          parser->m_atts[i].valuePtr,
                                           parser->m_atts[i].valueEnd);
       if (appAtts[attIndex] == 0)
         return XML_ERROR_NO_MEMORY;
@@ -3293,15 +3167,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         if (result)
           return result;
         --attIndex;
-      }
-      else {
+      } else {
         /* deal with other prefixed names later */
         attIndex++;
         nPrefixes++;
         (attId->name)[-1] = 2;
       }
-    }
-    else
+    } else
       attIndex++;
   }
 
@@ -3313,29 +3185,26 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         parser->m_idAttIndex = i;
         break;
       }
-  }
-  else
+  } else
     parser->m_idAttIndex = -1;
 
   /* do attribute defaulting */
   for (i = 0; i < nDefaultAtts; i++) {
     const DEFAULT_ATTRIBUTE *da = elementType->defaultAtts + i;
-    if (!(da->id->name)[-1] && da->value) {
+    if (! (da->id->name)[-1] && da->value) {
       if (da->id->prefix) {
         if (da->id->xmlns) {
           enum XML_Error result = addBinding(parser, da->id->prefix, da->id,
                                              da->value, bindingsPtr);
           if (result)
             return result;
-        }
-        else {
+        } else {
           (da->id->name)[-1] = 2;
           nPrefixes++;
           appAtts[attIndex++] = da->id->name;
           appAtts[attIndex++] = da->value;
         }
-      }
-      else {
+      } else {
         (da->id->name)[-1] = 1;
         appAtts[attIndex++] = da->id->name;
         appAtts[attIndex++] = da->value;
@@ -3348,31 +3217,34 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
      and clear flags that say whether attributes were specified */
   i = 0;
   if (nPrefixes) {
-    int j;  /* hash table index */
+    int j; /* hash table index */
     unsigned long version = parser->m_nsAttsVersion;
     int nsAttsSize = (int)1 << parser->m_nsAttsPower;
     unsigned char oldNsAttsPower = parser->m_nsAttsPower;
     /* size of hash table must be at least 2 * (# of prefixed attributes) */
-    if ((nPrefixes << 1) >> parser->m_nsAttsPower) {  /* true for m_nsAttsPower = 0 */
+    if ((nPrefixes << 1)
+        >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */
       NS_ATT *temp;
       /* hash table size must also be a power of 2 and >= 8 */
-      while (nPrefixes >> parser->m_nsAttsPower++);
+      while (nPrefixes >> parser->m_nsAttsPower++)
+        ;
       if (parser->m_nsAttsPower < 3)
         parser->m_nsAttsPower = 3;
       nsAttsSize = (int)1 << parser->m_nsAttsPower;
-      temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT));
-      if (!temp) {
+      temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts,
+                               nsAttsSize * sizeof(NS_ATT));
+      if (! temp) {
         /* Restore actual size of memory in m_nsAtts */
         parser->m_nsAttsPower = oldNsAttsPower;
         return XML_ERROR_NO_MEMORY;
       }
       parser->m_nsAtts = temp;
-      version = 0;  /* force re-initialization of m_nsAtts hash table */
+      version = 0; /* force re-initialization of m_nsAtts hash table */
     }
     /* using a version flag saves us from initializing m_nsAtts every time */
-    if (!version) {  /* initialize version flags when version wraps around */
+    if (! version) { /* initialize version flags when version wraps around */
       version = INIT_ATTS_VERSION;
-      for (j = nsAttsSize; j != 0; )
+      for (j = nsAttsSize; j != 0;)
         parser->m_nsAtts[--j].version = version;
     }
     parser->m_nsAttsVersion = --version;
@@ -3380,7 +3252,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     /* expand prefixed names and check for duplicates */
     for (; i < attIndex; i += 2) {
       const XML_Char *s = appAtts[i];
-      if (s[-1] == 2) {  /* prefixed */
+      if (s[-1] == 2) { /* prefixed */
         ATTRIBUTE_ID *id;
         const BINDING *b;
         unsigned long uriHash;
@@ -3390,16 +3262,16 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         copy_salt_to_sipkey(parser, &sip_key);
         sip24_init(&sip_state, &sip_key);
 
-        ((XML_Char *)s)[-1] = 0;  /* clear flag */
+        ((XML_Char *)s)[-1] = 0; /* clear flag */
         id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0);
-        if (!id || !id->prefix) {
+        if (! id || ! id->prefix) {
           /* This code is walking through the appAtts array, dealing
            * with (in this case) a prefixed attribute name.  To be in
            * the array, the attribute must have already been bound, so
            * has to have passed through the hash table lookup once
            * already.  That implies that an entry for it already
            * exists, so the lookup above will return a pointer to
-           * already allocated memory.  There is no opportunity for
+           * already allocated memory.  There is no opportunaity for
            * the allocator to fail, so the condition above cannot be
            * fulfilled.
            *
@@ -3410,12 +3282,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
           return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */
         }
         b = id->prefix->binding;
-        if (!b)
+        if (! b)
           return XML_ERROR_UNBOUND_PREFIX;
 
         for (j = 0; j < b->uriLen; j++) {
           const XML_Char c = b->uri[j];
-          if (!poolAppendChar(&parser->m_tempPool, c))
+          if (! poolAppendChar(&parser->m_tempPool, c))
             return XML_ERROR_NO_MEMORY;
         }
 
@@ -3426,8 +3298,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
 
         sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char));
 
-        do {  /* copies null terminator */
-          if (!poolAppendChar(&parser->m_tempPool, *s))
+        do { /* copies null terminator */
+          if (! poolAppendChar(&parser->m_tempPool, *s))
             return XML_ERROR_NO_MEMORY;
         } while (*s++);
 
@@ -3438,28 +3310,29 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
           */
           unsigned char step = 0;
           unsigned long mask = nsAttsSize - 1;
-          j = uriHash & mask;  /* index into hash table */
+          j = uriHash & mask; /* index into hash table */
           while (parser->m_nsAtts[j].version == version) {
             /* for speed we compare stored hash values first */
             if (uriHash == parser->m_nsAtts[j].hash) {
               const XML_Char *s1 = poolStart(&parser->m_tempPool);
               const XML_Char *s2 = parser->m_nsAtts[j].uriName;
               /* s1 is null terminated, but not s2 */
-              for (; *s1 == *s2 && *s1 != 0; s1++, s2++);
+              for (; *s1 == *s2 && *s1 != 0; s1++, s2++)
+                ;
               if (*s1 == 0)
                 return XML_ERROR_DUPLICATE_ATTRIBUTE;
             }
-            if (!step)
+            if (! step)
               step = PROBE_STEP(uriHash, mask, parser->m_nsAttsPower);
             j < step ? (j += nsAttsSize - step) : (j -= step);
           }
         }
 
-        if (parser->m_ns_triplets) {  /* append namespace separator and prefix */
+        if (parser->m_ns_triplets) { /* append namespace separator and prefix */
           parser->m_tempPool.ptr[-1] = parser->m_namespaceSeparator;
           s = b->prefix->name;
           do {
-            if (!poolAppendChar(&parser->m_tempPool, *s))
+            if (! poolAppendChar(&parser->m_tempPool, *s))
               return XML_ERROR_NO_MEMORY;
           } while (*s++);
         }
@@ -3474,13 +3347,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         parser->m_nsAtts[j].hash = uriHash;
         parser->m_nsAtts[j].uriName = s;
 
-        if (!--nPrefixes) {
+        if (! --nPrefixes) {
           i += 2;
           break;
         }
-      }
-      else  /* not prefixed */
-        ((XML_Char *)s)[-1] = 0;  /* clear flag */
+      } else                     /* not prefixed */
+        ((XML_Char *)s)[-1] = 0; /* clear flag */
     }
   }
   /* clear flags for the remaining attributes */
@@ -3489,40 +3361,38 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   for (binding = *bindingsPtr; binding; binding = binding->nextTagBinding)
     binding->attId->name[-1] = 0;
 
-  if (!parser->m_ns)
+  if (! parser->m_ns)
     return XML_ERROR_NONE;
 
   /* expand the element type name */
   if (elementType->prefix) {
     binding = elementType->prefix->binding;
-    if (!binding)
+    if (! binding)
       return XML_ERROR_UNBOUND_PREFIX;
     localPart = tagNamePtr->str;
     while (*localPart++ != XML_T(ASCII_COLON))
       ;
-  }
-  else if (dtd->defaultPrefix.binding) {
+  } else if (dtd->defaultPrefix.binding) {
     binding = dtd->defaultPrefix.binding;
     localPart = tagNamePtr->str;
-  }
-  else
+  } else
     return XML_ERROR_NONE;
   prefixLen = 0;
   if (parser->m_ns_triplets && binding->prefix->name) {
     for (; binding->prefix->name[prefixLen++];)
-      ;  /* prefixLen includes null terminator */
+      ; /* prefixLen includes null terminator */
   }
   tagNamePtr->localPart = localPart;
   tagNamePtr->uriLen = binding->uriLen;
   tagNamePtr->prefix = binding->prefix->name;
   tagNamePtr->prefixLen = prefixLen;
   for (i = 0; localPart[i++];)
-    ;  /* i includes null terminator */
+    ; /* i includes null terminator */
   n = i + binding->uriLen + prefixLen;
   if (n > binding->uriAlloc) {
     TAG *p;
     uri = (XML_Char *)MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char));
-    if (!uri)
+    if (! uri)
       return XML_ERROR_NO_MEMORY;
     binding->uriAlloc = n + EXPAND_SPARE;
     memcpy(uri, binding->uri, binding->uriLen * sizeof(XML_Char));
@@ -3538,7 +3408,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   /* we always have a namespace separator between localPart and prefix */
   if (prefixLen) {
     uri += i - 1;
-    *uri = parser->m_namespaceSeparator;  /* replace null terminator */
+    *uri = parser->m_namespaceSeparator; /* replace null terminator */
     memcpy(uri + 1, binding->prefix->name, prefixLen * sizeof(XML_Char));
   }
   tagNamePtr->str = binding->uri;
@@ -3550,27 +3420,25 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
 */
 static enum XML_Error
 addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
-           const XML_Char *uri, BINDING **bindingsPtr)
-{
-  static const XML_Char xmlNamespace[] = {
-    ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, ASCII_SLASH,
-    ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD,
-    ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L,
-    ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8, ASCII_SLASH,
-    ASCII_n, ASCII_a, ASCII_m, ASCII_e, ASCII_s, ASCII_p, ASCII_a, ASCII_c,
-    ASCII_e, '\0'
-  };
-  static const int xmlLen =
-    (int)sizeof(xmlNamespace)/sizeof(XML_Char) - 1;
-  static const XML_Char xmlnsNamespace[] = {
-    ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, ASCII_SLASH,
-    ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD,
-    ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_2, ASCII_0, ASCII_0,
-    ASCII_0, ASCII_SLASH, ASCII_x, ASCII_m, ASCII_l, ASCII_n, ASCII_s,
-    ASCII_SLASH, '\0'
-  };
-  static const int xmlnsLen =
-    (int)sizeof(xmlnsNamespace)/sizeof(XML_Char) - 1;
+           const XML_Char *uri, BINDING **bindingsPtr) {
+  static const XML_Char xmlNamespace[]
+      = {ASCII_h,      ASCII_t,     ASCII_t,     ASCII_p,      ASCII_COLON,
+         ASCII_SLASH,  ASCII_SLASH, ASCII_w,     ASCII_w,      ASCII_w,
+         ASCII_PERIOD, ASCII_w,     ASCII_3,     ASCII_PERIOD, ASCII_o,
+         ASCII_r,      ASCII_g,     ASCII_SLASH, ASCII_X,      ASCII_M,
+         ASCII_L,      ASCII_SLASH, ASCII_1,     ASCII_9,      ASCII_9,
+         ASCII_8,      ASCII_SLASH, ASCII_n,     ASCII_a,      ASCII_m,
+         ASCII_e,      ASCII_s,     ASCII_p,     ASCII_a,      ASCII_c,
+         ASCII_e,      '\0'};
+  static const int xmlLen = (int)sizeof(xmlNamespace) / sizeof(XML_Char) - 1;
+  static const XML_Char xmlnsNamespace[]
+      = {ASCII_h,     ASCII_t,      ASCII_t, ASCII_p, ASCII_COLON,  ASCII_SLASH,
+         ASCII_SLASH, ASCII_w,      ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w,
+         ASCII_3,     ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g,      ASCII_SLASH,
+         ASCII_2,     ASCII_0,      ASCII_0, ASCII_0, ASCII_SLASH,  ASCII_x,
+         ASCII_m,     ASCII_l,      ASCII_n, ASCII_s, ASCII_SLASH,  '\0'};
+  static const int xmlnsLen
+      = (int)sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1;
 
   XML_Bool mustBeXML = XML_FALSE;
   XML_Bool isXML = XML_TRUE;
@@ -3583,14 +3451,11 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   if (*uri == XML_T('\0') && prefix->name)
     return XML_ERROR_UNDECLARING_PREFIX;
 
-  if (prefix->name
-      && prefix->name[0] == XML_T(ASCII_x)
+  if (prefix->name && prefix->name[0] == XML_T(ASCII_x)
       && prefix->name[1] == XML_T(ASCII_m)
       && prefix->name[2] == XML_T(ASCII_l)) {
-
     /* Not allowed to bind xmlns */
-    if (prefix->name[3] == XML_T(ASCII_n)
-        && prefix->name[4] == XML_T(ASCII_s)
+    if (prefix->name[3] == XML_T(ASCII_n) && prefix->name[4] == XML_T(ASCII_s)
         && prefix->name[5] == XML_T('\0'))
       return XML_ERROR_RESERVED_PREFIX_XMLNS;
 
@@ -3602,7 +3467,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
     if (isXML && (len > xmlLen || uri[len] != xmlNamespace[len]))
       isXML = XML_FALSE;
 
-    if (!mustBeXML && isXMLNS
+    if (! mustBeXML && isXMLNS
         && (len > xmlnsLen || uri[len] != xmlnsNamespace[len]))
       isXMLNS = XML_FALSE;
   }
@@ -3621,21 +3486,21 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   if (parser->m_freeBindingList) {
     b = parser->m_freeBindingList;
     if (len > b->uriAlloc) {
-      XML_Char *temp = (XML_Char *)REALLOC(parser, b->uri,
-                          sizeof(XML_Char) * (len + EXPAND_SPARE));
+      XML_Char *temp = (XML_Char *)REALLOC(
+          parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE));
       if (temp == NULL)
         return XML_ERROR_NO_MEMORY;
       b->uri = temp;
       b->uriAlloc = len + EXPAND_SPARE;
     }
     parser->m_freeBindingList = b->nextTagBinding;
-  }
-  else {
+  } else {
     b = (BINDING *)MALLOC(parser, sizeof(BINDING));
-    if (!b)
+    if (! b)
       return XML_ERROR_NO_MEMORY;
-    b->uri = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE));
-    if (!b->uri) {
+    b->uri
+        = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE));
+    if (! b->uri) {
       FREE(parser, b);
       return XML_ERROR_NO_MEMORY;
     }
@@ -3658,7 +3523,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   /* if attId == NULL then we are not starting a namespace scope */
   if (attId && parser->m_startNamespaceDeclHandler)
     parser->m_startNamespaceDeclHandler(parser->m_handlerArg, prefix->name,
-                              prefix->binding ? uri : 0);
+                                        prefix->binding ? uri : 0);
   return XML_ERROR_NONE;
 }
 
@@ -3666,21 +3531,18 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
    the whole file is parsed with one call.
 */
 static enum XML_Error PTRCALL
-cdataSectionProcessor(XML_Parser parser,
-                      const char *start,
-                      const char *end,
-                      const char **endPtr)
-{
-  enum XML_Error result = doCdataSection(parser, parser->m_encoding, &start, end,
-                                         endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+cdataSectionProcessor(XML_Parser parser, const char *start, const char *end,
+                      const char **endPtr) {
+  enum XML_Error result
+      = doCdataSection(parser, parser->m_encoding, &start, end, endPtr,
+                       (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result != XML_ERROR_NONE)
     return result;
   if (start) {
-    if (parser->m_parentParser) {  /* we are parsing an external entity */
+    if (parser->m_parentParser) { /* we are parsing an external entity */
       parser->m_processor = externalEntityContentProcessor;
       return externalEntityContentProcessor(parser, start, end, endPtr);
-    }
-    else {
+    } else {
       parser->m_processor = contentProcessor;
       return contentProcessor(parser, start, end, endPtr);
     }
@@ -3692,13 +3554,8 @@ cdataSectionProcessor(XML_Parser parser,
    the section is not yet closed.
 */
 static enum XML_Error
-doCdataSection(XML_Parser parser,
-               const ENCODING *enc,
-               const char **startPtr,
-               const char *end,
-               const char **nextPtr,
-               XML_Bool haveMore)
-{
+doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
+               const char *end, const char **nextPtr, XML_Bool haveMore) {
   const char *s = *startPtr;
   const char **eventPP;
   const char **eventEndPP;
@@ -3706,8 +3563,7 @@ doCdataSection(XML_Parser parser,
     eventPP = &parser->m_eventPtr;
     *eventPP = s;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -3722,11 +3578,12 @@ doCdataSection(XML_Parser parser,
     case XML_TOK_CDATA_SECT_CLOSE:
       if (parser->m_endCdataSectionHandler)
         parser->m_endCdataSectionHandler(parser->m_handlerArg);
-/* BEGIN disabled code */
+      /* BEGIN disabled code */
       /* see comment under XML_TOK_CDATA_SECT_OPEN */
       else if (0 && parser->m_characterDataHandler)
-        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0);
-/* END disabled code */
+        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                                       0);
+      /* END disabled code */
       else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       *startPtr = next;
@@ -3739,35 +3596,31 @@ doCdataSection(XML_Parser parser,
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       break;
-    case XML_TOK_DATA_CHARS:
-      {
-        XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
-        if (charDataHandler) {
-          if (MUST_CONVERT(enc, s)) {
-            for (;;) {
-              ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-              const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-              *eventEndPP = next;
-              charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-              if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
-                break;
-              *eventPP = s;
-            }
+    case XML_TOK_DATA_CHARS: {
+      XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
+      if (charDataHandler) {
+        if (MUST_CONVERT(enc, s)) {
+          for (;;) {
+            ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
+            const enum XML_Convert_Result convert_res = XmlConvert(
+                enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+            *eventEndPP = next;
+            charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                            (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+            if ((convert_res == XML_CONVERT_COMPLETED)
+                || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+              break;
+            *eventPP = s;
           }
-          else
-            charDataHandler(parser->m_handlerArg,
-                            (XML_Char *)s,
-                            (int)((XML_Char *)next - (XML_Char *)s));
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-      }
-      break;
+        } else
+          charDataHandler(parser->m_handlerArg, (XML_Char *)s,
+                          (int)((XML_Char *)next - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_INVALID:
       *eventPP = next;
       return XML_ERROR_INVALID_TOKEN;
@@ -3791,7 +3644,7 @@ doCdataSection(XML_Parser parser,
        * statistics.
        *
        * LCOV_EXCL_START
-      */
+       */
       *eventPP = next;
       return XML_ERROR_UNEXPECTED_STATE;
       /* LCOV_EXCL_STOP */
@@ -3804,7 +3657,7 @@ doCdataSection(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
   /* not reached */
@@ -3816,13 +3669,11 @@ doCdataSection(XML_Parser parser,
    the whole file is parsed with one call.
 */
 static enum XML_Error PTRCALL
-ignoreSectionProcessor(XML_Parser parser,
-                       const char *start,
-                       const char *end,
-                       const char **endPtr)
-{
-  enum XML_Error result = doIgnoreSection(parser, parser->m_encoding, &start, end,
-                                          endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end,
+                       const char **endPtr) {
+  enum XML_Error result
+      = doIgnoreSection(parser, parser->m_encoding, &start, end, endPtr,
+                        (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result != XML_ERROR_NONE)
     return result;
   if (start) {
@@ -3836,13 +3687,8 @@ ignoreSectionProcessor(XML_Parser parser,
    if the section is not yet closed.
 */
 static enum XML_Error
-doIgnoreSection(XML_Parser parser,
-                const ENCODING *enc,
-                const char **startPtr,
-                const char *end,
-                const char **nextPtr,
-                XML_Bool haveMore)
-{
+doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
+                const char *end, const char **nextPtr, XML_Bool haveMore) {
   const char *next;
   int tok;
   const char *s = *startPtr;
@@ -3852,8 +3698,7 @@ doIgnoreSection(XML_Parser parser,
     eventPP = &parser->m_eventPtr;
     *eventPP = s;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     /* It's not entirely clear, but it seems the following two lines
      * of code cannot be executed.  The only occasions on which 'enc'
      * is not 'encoding' are when this function is called
@@ -3917,13 +3762,12 @@ doIgnoreSection(XML_Parser parser,
 #endif /* XML_DTD */
 
 static enum XML_Error
-initializeEncoding(XML_Parser parser)
-{
+initializeEncoding(XML_Parser parser) {
   const char *s;
 #ifdef XML_UNICODE
   char encodingBuf[128];
   /* See comments abount `protoclEncodingName` in parserInit() */
-  if (!parser->m_protocolEncodingName)
+  if (! parser->m_protocolEncodingName)
     s = NULL;
   else {
     int i;
@@ -3941,15 +3785,15 @@ initializeEncoding(XML_Parser parser)
 #else
   s = parser->m_protocolEncodingName;
 #endif
-  if ((parser->m_ns ? XmlInitEncodingNS : XmlInitEncoding)(&parser->m_initEncoding, &parser->m_encoding, s))
+  if ((parser->m_ns ? XmlInitEncodingNS : XmlInitEncoding)(
+          &parser->m_initEncoding, &parser->m_encoding, s))
     return XML_ERROR_NONE;
   return handleUnknownEncoding(parser, parser->m_protocolEncodingName);
 }
 
 static enum XML_Error
-processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
-               const char *s, const char *next)
-{
+processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s,
+               const char *next) {
   const char *encodingName = NULL;
   const XML_Char *storedEncName = NULL;
   const ENCODING *newEncoding = NULL;
@@ -3957,52 +3801,41 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
   const char *versionend;
   const XML_Char *storedversion = NULL;
   int standalone = -1;
-  if (!(parser->m_ns
-        ? XmlParseXmlDeclNS
-        : XmlParseXmlDecl)(isGeneralTextEntity,
-                           parser->m_encoding,
-                           s,
-                           next,
-                           &parser->m_eventPtr,
-                           &version,
-                           &versionend,
-                           &encodingName,
-                           &newEncoding,
-                           &standalone)) {
+  if (! (parser->m_ns ? XmlParseXmlDeclNS : XmlParseXmlDecl)(
+          isGeneralTextEntity, parser->m_encoding, s, next, &parser->m_eventPtr,
+          &version, &versionend, &encodingName, &newEncoding, &standalone)) {
     if (isGeneralTextEntity)
       return XML_ERROR_TEXT_DECL;
     else
       return XML_ERROR_XML_DECL;
   }
-  if (!isGeneralTextEntity && standalone == 1) {
+  if (! isGeneralTextEntity && standalone == 1) {
     parser->m_dtd->standalone = XML_TRUE;
 #ifdef XML_DTD
-    if (parser->m_paramEntityParsing == XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE)
+    if (parser->m_paramEntityParsing
+        == XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE)
       parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER;
 #endif /* XML_DTD */
   }
   if (parser->m_xmlDeclHandler) {
     if (encodingName != NULL) {
-      storedEncName = poolStoreString(&parser->m_temp2Pool,
-                                      parser->m_encoding,
-                                      encodingName,
-                                      encodingName
-                                      + XmlNameLength(parser->m_encoding, encodingName));
-      if (!storedEncName)
-              return XML_ERROR_NO_MEMORY;
+      storedEncName = poolStoreString(
+          &parser->m_temp2Pool, parser->m_encoding, encodingName,
+          encodingName + XmlNameLength(parser->m_encoding, encodingName));
+      if (! storedEncName)
+        return XML_ERROR_NO_MEMORY;
       poolFinish(&parser->m_temp2Pool);
     }
     if (version) {
-      storedversion = poolStoreString(&parser->m_temp2Pool,
-                                      parser->m_encoding,
-                                      version,
-                                      versionend - parser->m_encoding->minBytesPerChar);
-      if (!storedversion)
+      storedversion
+          = poolStoreString(&parser->m_temp2Pool, parser->m_encoding, version,
+                            versionend - parser->m_encoding->minBytesPerChar);
+      if (! storedversion)
         return XML_ERROR_NO_MEMORY;
     }
-    parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName, standalone);
-  }
-  else if (parser->m_defaultHandler)
+    parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName,
+                             standalone);
+  } else if (parser->m_defaultHandler)
     reportDefault(parser, parser->m_encoding, s, next);
   if (parser->m_protocolEncodingName == NULL) {
     if (newEncoding) {
@@ -4012,20 +3845,19 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
        * this is UTF-16, is it the same endianness?
        */
       if (newEncoding->minBytesPerChar != parser->m_encoding->minBytesPerChar
-          || (newEncoding->minBytesPerChar == 2 &&
-              newEncoding != parser->m_encoding)) {
+          || (newEncoding->minBytesPerChar == 2
+              && newEncoding != parser->m_encoding)) {
         parser->m_eventPtr = encodingName;
         return XML_ERROR_INCORRECT_ENCODING;
       }
       parser->m_encoding = newEncoding;
-    }
-    else if (encodingName) {
+    } else if (encodingName) {
       enum XML_Error result;
-      if (!storedEncName) {
+      if (! storedEncName) {
         storedEncName = poolStoreString(
-          &parser->m_temp2Pool, parser->m_encoding, encodingName,
-          encodingName + XmlNameLength(parser->m_encoding, encodingName));
-        if (!storedEncName)
+            &parser->m_temp2Pool, parser->m_encoding, encodingName,
+            encodingName + XmlNameLength(parser->m_encoding, encodingName));
+        if (! storedEncName)
           return XML_ERROR_NO_MEMORY;
       }
       result = handleUnknownEncoding(parser, storedEncName);
@@ -4043,8 +3875,7 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
 }
 
 static enum XML_Error
-handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
-{
+handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) {
   if (parser->m_unknownEncodingHandler) {
     XML_Encoding info;
     int i;
@@ -4053,21 +3884,17 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
     info.convert = NULL;
     info.data = NULL;
     info.release = NULL;
-    if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData, encodingName,
-                               &info)) {
+    if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData,
+                                         encodingName, &info)) {
       ENCODING *enc;
       parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding());
-      if (!parser->m_unknownEncodingMem) {
+      if (! parser->m_unknownEncodingMem) {
         if (info.release)
           info.release(info.data);
         return XML_ERROR_NO_MEMORY;
       }
-      enc = (parser->m_ns
-             ? XmlInitUnknownEncodingNS
-             : XmlInitUnknownEncoding)(parser->m_unknownEncodingMem,
-                                       info.map,
-                                       info.convert,
-                                       info.data);
+      enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)(
+          parser->m_unknownEncodingMem, info.map, info.convert, info.data);
       if (enc) {
         parser->m_unknownEncodingData = info.data;
         parser->m_unknownEncodingRelease = info.release;
@@ -4082,11 +3909,8 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
 }
 
 static enum XML_Error PTRCALL
-prologInitProcessor(XML_Parser parser,
-                    const char *s,
-                    const char *end,
-                    const char **nextPtr)
-{
+prologInitProcessor(XML_Parser parser, const char *s, const char *end,
+                    const char **nextPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -4097,11 +3921,8 @@ prologInitProcessor(XML_Parser parser,
 #ifdef XML_DTD
 
 static enum XML_Error PTRCALL
-externalParEntInitProcessor(XML_Parser parser,
-                            const char *s,
-                            const char *end,
-                            const char **nextPtr)
-{
+externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end,
+                            const char **nextPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -4113,19 +3934,15 @@ externalParEntInitProcessor(XML_Parser parser,
   if (parser->m_prologState.inEntityValue) {
     parser->m_processor = entityValueInitProcessor;
     return entityValueInitProcessor(parser, s, end, nextPtr);
-  }
-  else {
+  } else {
     parser->m_processor = externalParEntProcessor;
     return externalParEntProcessor(parser, s, end, nextPtr);
   }
 }
 
 static enum XML_Error PTRCALL
-entityValueInitProcessor(XML_Parser parser,
-                         const char *s,
-                         const char *end,
-                         const char **nextPtr)
-{
+entityValueInitProcessor(XML_Parser parser, const char *s, const char *end,
+                         const char **nextPtr) {
   int tok;
   const char *start = s;
   const char *next = start;
@@ -4135,7 +3952,7 @@ entityValueInitProcessor(XML_Parser parser,
     tok = XmlPrologTok(parser->m_encoding, start, end, &next);
     parser->m_eventEndPtr = next;
     if (tok <= 0) {
-      if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+      if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -4146,22 +3963,21 @@ entityValueInitProcessor(XML_Parser parser,
         return XML_ERROR_UNCLOSED_TOKEN;
       case XML_TOK_PARTIAL_CHAR:
         return XML_ERROR_PARTIAL_CHAR;
-      case XML_TOK_NONE:   /* start == end */
+      case XML_TOK_NONE: /* start == end */
       default:
         break;
       }
       /* found end of entity value - can store it now */
       return storeEntityValue(parser, parser->m_encoding, s, end);
-    }
-    else if (tok == XML_TOK_XML_DECL) {
+    } else if (tok == XML_TOK_XML_DECL) {
       enum XML_Error result;
       result = processXmlDecl(parser, 0, start, next);
       if (result != XML_ERROR_NONE)
         return result;
-      /* At this point, m_parsingStatus.parsing cannot be XML_SUSPENDED.  For that
-       * to happen, a parameter entity parsing handler must have
-       * attempted to suspend the parser, which fails and raises an
-       * error.  The parser can be aborted, but can't be suspended.
+      /* At this point, m_parsingStatus.parsing cannot be XML_SUSPENDED.  For
+       * that to happen, a parameter entity parsing handler must have attempted
+       * to suspend the parser, which fails and raises an error.  The parser can
+       * be aborted, but can't be suspended.
        */
       if (parser->m_parsingStatus.parsing == XML_FINISHED)
         return XML_ERROR_ABORTED;
@@ -4177,7 +3993,8 @@ entityValueInitProcessor(XML_Parser parser,
        then, when this routine is entered the next time, XmlPrologTok will
        return XML_TOK_INVALID, since the BOM is still in the buffer
     */
-    else if (tok == XML_TOK_BOM && next == end && !parser->m_parsingStatus.finalBuffer) {
+    else if (tok == XML_TOK_BOM && next == end
+             && ! parser->m_parsingStatus.finalBuffer) {
       *nextPtr = next;
       return XML_ERROR_NONE;
     }
@@ -4195,17 +4012,14 @@ entityValueInitProcessor(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalParEntProcessor(XML_Parser parser,
-                        const char *s,
-                        const char *end,
-                        const char **nextPtr)
-{
+externalParEntProcessor(XML_Parser parser, const char *s, const char *end,
+                        const char **nextPtr) {
   const char *next = s;
   int tok;
 
   tok = XmlPrologTok(parser->m_encoding, s, end, &next);
   if (tok <= 0) {
-    if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+    if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
       *nextPtr = s;
       return XML_ERROR_NONE;
     }
@@ -4216,7 +4030,7 @@ externalParEntProcessor(XML_Parser parser,
       return XML_ERROR_UNCLOSED_TOKEN;
     case XML_TOK_PARTIAL_CHAR:
       return XML_ERROR_PARTIAL_CHAR;
-    case XML_TOK_NONE:   /* start == end */
+    case XML_TOK_NONE: /* start == end */
     default:
       break;
     }
@@ -4231,16 +4045,13 @@ externalParEntProcessor(XML_Parser parser,
   }
 
   parser->m_processor = prologProcessor;
-  return doProlog(parser, parser->m_encoding, s, end, tok, next,
-                  nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+  return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
 }
 
 static enum XML_Error PTRCALL
-entityValueProcessor(XML_Parser parser,
-                     const char *s,
-                     const char *end,
-                     const char **nextPtr)
-{
+entityValueProcessor(XML_Parser parser, const char *s, const char *end,
+                     const char **nextPtr) {
   const char *start = s;
   const char *next = s;
   const ENCODING *enc = parser->m_encoding;
@@ -4249,7 +4060,7 @@ entityValueProcessor(XML_Parser parser,
   for (;;) {
     tok = XmlPrologTok(enc, start, end, &next);
     if (tok <= 0) {
-      if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+      if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -4260,7 +4071,7 @@ entityValueProcessor(XML_Parser parser,
         return XML_ERROR_UNCLOSED_TOKEN;
       case XML_TOK_PARTIAL_CHAR:
         return XML_ERROR_PARTIAL_CHAR;
-      case XML_TOK_NONE:   /* start == end */
+      case XML_TOK_NONE: /* start == end */
       default:
         break;
       }
@@ -4274,52 +4085,46 @@ entityValueProcessor(XML_Parser parser,
 #endif /* XML_DTD */
 
 static enum XML_Error PTRCALL
-prologProcessor(XML_Parser parser,
-                const char *s,
-                const char *end,
-                const char **nextPtr)
-{
+prologProcessor(XML_Parser parser, const char *s, const char *end,
+                const char **nextPtr) {
   const char *next = s;
   int tok = XmlPrologTok(parser->m_encoding, s, end, &next);
-  return doProlog(parser, parser->m_encoding, s, end, tok, next,
-                  nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+  return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
 }
 
 static enum XML_Error
-doProlog(XML_Parser parser,
-         const ENCODING *enc,
-         const char *s,
-         const char *end,
-         int tok,
-         const char *next,
-         const char **nextPtr,
-         XML_Bool haveMore)
-{
+doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
+         int tok, const char *next, const char **nextPtr, XML_Bool haveMore,
+         XML_Bool allowClosingDoctype) {
 #ifdef XML_DTD
-  static const XML_Char externalSubsetName[] = { ASCII_HASH , '\0' };
+  static const XML_Char externalSubsetName[] = {ASCII_HASH, '\0'};
 #endif /* XML_DTD */
-  static const XML_Char atypeCDATA[] =
-      { ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-  static const XML_Char atypeID[] = { ASCII_I, ASCII_D, '\0' };
-  static const XML_Char atypeIDREF[] =
-      { ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0' };
-  static const XML_Char atypeIDREFS[] =
-      { ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0' };
-  static const XML_Char atypeENTITY[] =
-      { ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0' };
-  static const XML_Char atypeENTITIES[] = { ASCII_E, ASCII_N,
-      ASCII_T, ASCII_I, ASCII_T, ASCII_I, ASCII_E, ASCII_S, '\0' };
-  static const XML_Char atypeNMTOKEN[] = {
-      ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0' };
-  static const XML_Char atypeNMTOKENS[] = { ASCII_N, ASCII_M, ASCII_T,
-      ASCII_O, ASCII_K, ASCII_E, ASCII_N, ASCII_S, '\0' };
-  static const XML_Char notationPrefix[] = { ASCII_N, ASCII_O, ASCII_T,
-      ASCII_A, ASCII_T, ASCII_I, ASCII_O, ASCII_N, ASCII_LPAREN, '\0' };
-  static const XML_Char enumValueSep[] = { ASCII_PIPE, '\0' };
-  static const XML_Char enumValueStart[] = { ASCII_LPAREN, '\0' };
+  static const XML_Char atypeCDATA[]
+      = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+  static const XML_Char atypeID[] = {ASCII_I, ASCII_D, '\0'};
+  static const XML_Char atypeIDREF[]
+      = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0'};
+  static const XML_Char atypeIDREFS[]
+      = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0'};
+  static const XML_Char atypeENTITY[]
+      = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0'};
+  static const XML_Char atypeENTITIES[]
+      = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T,
+         ASCII_I, ASCII_E, ASCII_S, '\0'};
+  static const XML_Char atypeNMTOKEN[]
+      = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0'};
+  static const XML_Char atypeNMTOKENS[]
+      = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K,
+         ASCII_E, ASCII_N, ASCII_S, '\0'};
+  static const XML_Char notationPrefix[]
+      = {ASCII_N, ASCII_O, ASCII_T, ASCII_A,      ASCII_T,
+         ASCII_I, ASCII_O, ASCII_N, ASCII_LPAREN, '\0'};
+  static const XML_Char enumValueSep[] = {ASCII_PIPE, '\0'};
+  static const XML_Char enumValueStart[] = {ASCII_LPAREN, '\0'};
 
   /* save one level of indirection */
-  DTD * const dtd = parser->m_dtd;
+  DTD *const dtd = parser->m_dtd;
 
   const char **eventPP;
   const char **eventEndPP;
@@ -4328,8 +4133,7 @@ doProlog(XML_Parser parser,
   if (enc == parser->m_encoding) {
     eventPP = &parser->m_eventPtr;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -4358,7 +4162,8 @@ doProlog(XML_Parser parser,
       case XML_TOK_NONE:
 #ifdef XML_DTD
         /* for internal PE NOT referenced between declarations */
-        if (enc != parser->m_encoding && !parser->m_openInternalEntities->betweenDecl) {
+        if (enc != parser->m_encoding
+            && ! parser->m_openInternalEntities->betweenDecl) {
           *nextPtr = s;
           return XML_ERROR_NONE;
         }
@@ -4383,19 +4188,18 @@ doProlog(XML_Parser parser,
     }
     role = XmlTokenRole(&parser->m_prologState, tok, s, next, enc);
     switch (role) {
-    case XML_ROLE_XML_DECL:
-      {
-        enum XML_Error result = processXmlDecl(parser, 0, s, next);
-        if (result != XML_ERROR_NONE)
-          return result;
-        enc = parser->m_encoding;
-        handleDefault = XML_FALSE;
-      }
-      break;
+    case XML_ROLE_XML_DECL: {
+      enum XML_Error result = processXmlDecl(parser, 0, s, next);
+      if (result != XML_ERROR_NONE)
+        return result;
+      enc = parser->m_encoding;
+      handleDefault = XML_FALSE;
+    } break;
     case XML_ROLE_DOCTYPE_NAME:
       if (parser->m_startDoctypeDeclHandler) {
-        parser->m_doctypeName = poolStoreString(&parser->m_tempPool, enc, s, next);
-        if (!parser->m_doctypeName)
+        parser->m_doctypeName
+            = poolStoreString(&parser->m_tempPool, enc, s, next);
+        if (! parser->m_doctypeName)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
         parser->m_doctypePubid = NULL;
@@ -4405,43 +4209,40 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_DOCTYPE_INTERNAL_SUBSET:
       if (parser->m_startDoctypeDeclHandler) {
-        parser->m_startDoctypeDeclHandler(parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
-                                parser->m_doctypePubid, 1);
+        parser->m_startDoctypeDeclHandler(
+            parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
+            parser->m_doctypePubid, 1);
         parser->m_doctypeName = NULL;
         poolClear(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
       break;
 #ifdef XML_DTD
-    case XML_ROLE_TEXT_DECL:
-      {
-        enum XML_Error result = processXmlDecl(parser, 1, s, next);
-        if (result != XML_ERROR_NONE)
-          return result;
-        enc = parser->m_encoding;
-        handleDefault = XML_FALSE;
-      }
-      break;
+    case XML_ROLE_TEXT_DECL: {
+      enum XML_Error result = processXmlDecl(parser, 1, s, next);
+      if (result != XML_ERROR_NONE)
+        return result;
+      enc = parser->m_encoding;
+      handleDefault = XML_FALSE;
+    } break;
 #endif /* XML_DTD */
     case XML_ROLE_DOCTYPE_PUBLIC_ID:
 #ifdef XML_DTD
       parser->m_useForeignDTD = XML_FALSE;
-      parser->m_declEntity = (ENTITY *)lookup(parser,
-                                    &dtd->paramEntities,
-                                    externalSubsetName,
-                                    sizeof(ENTITY));
-      if (!parser->m_declEntity)
+      parser->m_declEntity = (ENTITY *)lookup(
+          parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY));
+      if (! parser->m_declEntity)
         return XML_ERROR_NO_MEMORY;
 #endif /* XML_DTD */
       dtd->hasParamEntityRefs = XML_TRUE;
       if (parser->m_startDoctypeDeclHandler) {
         XML_Char *pubId;
-        if (!XmlIsPublicId(enc, s, next, eventPP))
+        if (! XmlIsPublicId(enc, s, next, eventPP))
           return XML_ERROR_PUBLICID;
         pubId = poolStoreString(&parser->m_tempPool, enc,
                                 s + enc->minBytesPerChar,
                                 next - enc->minBytesPerChar);
-        if (!pubId)
+        if (! pubId)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(pubId);
         poolFinish(&parser->m_tempPool);
@@ -4451,15 +4252,14 @@ doProlog(XML_Parser parser,
       }
       /* fall through */
     case XML_ROLE_ENTITY_PUBLIC_ID:
-      if (!XmlIsPublicId(enc, s, next, eventPP))
+      if (! XmlIsPublicId(enc, s, next, eventPP))
         return XML_ERROR_PUBLICID;
     alreadyChecked:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        XML_Char *tem = poolStoreString(&dtd->pool,
-                                        enc,
-                                        s + enc->minBytesPerChar,
-                                        next - enc->minBytesPerChar);
-        if (!tem)
+        XML_Char *tem
+            = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                              next - enc->minBytesPerChar);
+        if (! tem)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(tem);
         parser->m_declEntity->publicId = tem;
@@ -4472,9 +4272,15 @@ doProlog(XML_Parser parser,
       }
       break;
     case XML_ROLE_DOCTYPE_CLOSE:
+      if (allowClosingDoctype != XML_TRUE) {
+        /* Must not close doctype from within expanded parameter entities */
+        return XML_ERROR_INVALID_TOKEN;
+      }
+
       if (parser->m_doctypeName) {
-        parser->m_startDoctypeDeclHandler(parser->m_handlerArg, parser->m_doctypeName,
-                                parser->m_doctypeSysid, parser->m_doctypePubid, 0);
+        parser->m_startDoctypeDeclHandler(
+            parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
+            parser->m_doctypePubid, 0);
         poolClear(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
@@ -4486,12 +4292,11 @@ doProlog(XML_Parser parser,
       if (parser->m_doctypeSysid || parser->m_useForeignDTD) {
         XML_Bool hadParamEntityRefs = dtd->hasParamEntityRefs;
         dtd->hasParamEntityRefs = XML_TRUE;
-        if (parser->m_paramEntityParsing && parser->m_externalEntityRefHandler) {
-          ENTITY *entity = (ENTITY *)lookup(parser,
-                                            &dtd->paramEntities,
-                                            externalSubsetName,
-                                            sizeof(ENTITY));
-          if (!entity) {
+        if (parser->m_paramEntityParsing
+            && parser->m_externalEntityRefHandler) {
+          ENTITY *entity = (ENTITY *)lookup(parser, &dtd->paramEntities,
+                                            externalSubsetName, sizeof(ENTITY));
+          if (! entity) {
             /* The external subset name "#" will have already been
              * inserted into the hash table at the start of the
              * external entity parsing, so no allocation will happen
@@ -4502,22 +4307,19 @@ doProlog(XML_Parser parser,
           if (parser->m_useForeignDTD)
             entity->base = parser->m_curBase;
           dtd->paramEntityRead = XML_FALSE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId))
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           if (dtd->paramEntityRead) {
-            if (!dtd->standalone &&
-                parser->m_notStandaloneHandler &&
-                !parser->m_notStandaloneHandler(parser->m_handlerArg))
+            if (! dtd->standalone && parser->m_notStandaloneHandler
+                && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
               return XML_ERROR_NOT_STANDALONE;
           }
           /* if we didn't read the foreign DTD then this means that there
              is no external subset and we must reset dtd->hasParamEntityRefs
           */
-          else if (!parser->m_doctypeSysid)
+          else if (! parser->m_doctypeSysid)
             dtd->hasParamEntityRefs = hadParamEntityRefs;
           /* end of DTD - no need to update dtd->keepProcessing */
         }
@@ -4537,24 +4339,21 @@ doProlog(XML_Parser parser,
       if (parser->m_useForeignDTD) {
         XML_Bool hadParamEntityRefs = dtd->hasParamEntityRefs;
         dtd->hasParamEntityRefs = XML_TRUE;
-        if (parser->m_paramEntityParsing && parser->m_externalEntityRefHandler) {
+        if (parser->m_paramEntityParsing
+            && parser->m_externalEntityRefHandler) {
           ENTITY *entity = (ENTITY *)lookup(parser, &dtd->paramEntities,
-                                            externalSubsetName,
-                                            sizeof(ENTITY));
-          if (!entity)
+                                            externalSubsetName, sizeof(ENTITY));
+          if (! entity)
             return XML_ERROR_NO_MEMORY;
           entity->base = parser->m_curBase;
           dtd->paramEntityRead = XML_FALSE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId))
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           if (dtd->paramEntityRead) {
-            if (!dtd->standalone &&
-                parser->m_notStandaloneHandler &&
-                !parser->m_notStandaloneHandler(parser->m_handlerArg))
+            if (! dtd->standalone && parser->m_notStandaloneHandler
+                && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
               return XML_ERROR_NOT_STANDALONE;
           }
           /* if we didn't read the foreign DTD then this means that there
@@ -4570,12 +4369,12 @@ doProlog(XML_Parser parser,
       return contentProcessor(parser, s, end, nextPtr);
     case XML_ROLE_ATTLIST_ELEMENT_NAME:
       parser->m_declElementType = getElementType(parser, enc, s, next);
-      if (!parser->m_declElementType)
+      if (! parser->m_declElementType)
         return XML_ERROR_NO_MEMORY;
       goto checkAttListDeclHandler;
     case XML_ROLE_ATTRIBUTE_NAME:
       parser->m_declAttributeId = getAttributeId(parser, enc, s, next);
-      if (!parser->m_declAttributeId)
+      if (! parser->m_declAttributeId)
         return XML_ERROR_NO_MEMORY;
       parser->m_declAttributeIsCdata = XML_FALSE;
       parser->m_declAttributeType = NULL;
@@ -4616,15 +4415,13 @@ doProlog(XML_Parser parser,
         const XML_Char *prefix;
         if (parser->m_declAttributeType) {
           prefix = enumValueSep;
+        } else {
+          prefix = (role == XML_ROLE_ATTRIBUTE_NOTATION_VALUE ? notationPrefix
+                                                              : enumValueStart);
         }
-        else {
-          prefix = (role == XML_ROLE_ATTRIBUTE_NOTATION_VALUE
-                    ? notationPrefix
-                    : enumValueStart);
-        }
-        if (!poolAppendString(&parser->m_tempPool, prefix))
+        if (! poolAppendString(&parser->m_tempPool, prefix))
           return XML_ERROR_NO_MEMORY;
-        if (!poolAppend(&parser->m_tempPool, enc, s, next))
+        if (! poolAppend(&parser->m_tempPool, enc, s, next))
           return XML_ERROR_NO_MEMORY;
         parser->m_declAttributeType = parser->m_tempPool.start;
         handleDefault = XML_FALSE;
@@ -4633,25 +4430,27 @@ doProlog(XML_Parser parser,
     case XML_ROLE_IMPLIED_ATTRIBUTE_VALUE:
     case XML_ROLE_REQUIRED_ATTRIBUTE_VALUE:
       if (dtd->keepProcessing) {
-        if (!defineAttribute(parser->m_declElementType, parser->m_declAttributeId,
-                             parser->m_declAttributeIsCdata, parser->m_declAttributeIsId,
-                             0, parser))
+        if (! defineAttribute(parser->m_declElementType,
+                              parser->m_declAttributeId,
+                              parser->m_declAttributeIsCdata,
+                              parser->m_declAttributeIsId, 0, parser))
           return XML_ERROR_NO_MEMORY;
         if (parser->m_attlistDeclHandler && parser->m_declAttributeType) {
           if (*parser->m_declAttributeType == XML_T(ASCII_LPAREN)
               || (*parser->m_declAttributeType == XML_T(ASCII_N)
                   && parser->m_declAttributeType[1] == XML_T(ASCII_O))) {
             /* Enumerated or Notation type */
-            if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
-                || !poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+            if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
+                || ! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
               return XML_ERROR_NO_MEMORY;
             parser->m_declAttributeType = parser->m_tempPool.start;
             poolFinish(&parser->m_tempPool);
           }
           *eventEndPP = s;
-          parser->m_attlistDeclHandler(parser->m_handlerArg, parser->m_declElementType->name,
-                             parser->m_declAttributeId->name, parser->m_declAttributeType,
-                             0, role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE);
+          parser->m_attlistDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name,
+              parser->m_declAttributeId->name, parser->m_declAttributeType, 0,
+              role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE);
           poolClear(&parser->m_tempPool);
           handleDefault = XML_FALSE;
         }
@@ -4661,35 +4460,34 @@ doProlog(XML_Parser parser,
     case XML_ROLE_FIXED_ATTRIBUTE_VALUE:
       if (dtd->keepProcessing) {
         const XML_Char *attVal;
-        enum XML_Error result =
-          storeAttributeValue(parser, enc, parser->m_declAttributeIsCdata,
-                              s + enc->minBytesPerChar,
-                              next - enc->minBytesPerChar,
-                              &dtd->pool);
+        enum XML_Error result = storeAttributeValue(
+            parser, enc, parser->m_declAttributeIsCdata,
+            s + enc->minBytesPerChar, next - enc->minBytesPerChar, &dtd->pool);
         if (result)
           return result;
         attVal = poolStart(&dtd->pool);
         poolFinish(&dtd->pool);
         /* ID attributes aren't allowed to have a default */
-        if (!defineAttribute(parser->m_declElementType, parser->m_declAttributeId,
-                             parser->m_declAttributeIsCdata, XML_FALSE, attVal, parser))
+        if (! defineAttribute(
+                parser->m_declElementType, parser->m_declAttributeId,
+                parser->m_declAttributeIsCdata, XML_FALSE, attVal, parser))
           return XML_ERROR_NO_MEMORY;
         if (parser->m_attlistDeclHandler && parser->m_declAttributeType) {
           if (*parser->m_declAttributeType == XML_T(ASCII_LPAREN)
               || (*parser->m_declAttributeType == XML_T(ASCII_N)
                   && parser->m_declAttributeType[1] == XML_T(ASCII_O))) {
             /* Enumerated or Notation type */
-            if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
-                || !poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+            if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
+                || ! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
               return XML_ERROR_NO_MEMORY;
             parser->m_declAttributeType = parser->m_tempPool.start;
             poolFinish(&parser->m_tempPool);
           }
           *eventEndPP = s;
-          parser->m_attlistDeclHandler(parser->m_handlerArg, parser->m_declElementType->name,
-                             parser->m_declAttributeId->name, parser->m_declAttributeType,
-                             attVal,
-                             role == XML_ROLE_FIXED_ATTRIBUTE_VALUE);
+          parser->m_attlistDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name,
+              parser->m_declAttributeId->name, parser->m_declAttributeType,
+              attVal, role == XML_ROLE_FIXED_ATTRIBUTE_VALUE);
           poolClear(&parser->m_tempPool);
           handleDefault = XML_FALSE;
         }
@@ -4697,25 +4495,22 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_ENTITY_VALUE:
       if (dtd->keepProcessing) {
-        enum XML_Error result = storeEntityValue(parser, enc,
-                                            s + enc->minBytesPerChar,
-                                            next - enc->minBytesPerChar);
+        enum XML_Error result = storeEntityValue(
+            parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar);
         if (parser->m_declEntity) {
           parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool);
-          parser->m_declEntity->textLen = (int)(poolLength(&dtd->entityValuePool));
+          parser->m_declEntity->textLen
+              = (int)(poolLength(&dtd->entityValuePool));
           poolFinish(&dtd->entityValuePool);
           if (parser->m_entityDeclHandler) {
             *eventEndPP = s;
-            parser->m_entityDeclHandler(parser->m_handlerArg,
-                              parser->m_declEntity->name,
-                              parser->m_declEntity->is_param,
-                              parser->m_declEntity->textPtr,
-                              parser->m_declEntity->textLen,
-                              parser->m_curBase, 0, 0, 0);
+            parser->m_entityDeclHandler(
+                parser->m_handlerArg, parser->m_declEntity->name,
+                parser->m_declEntity->is_param, parser->m_declEntity->textPtr,
+                parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0);
             handleDefault = XML_FALSE;
           }
-        }
-        else
+        } else
           poolDiscard(&dtd->entityValuePool);
         if (result != XML_ERROR_NONE)
           return result;
@@ -4728,8 +4523,8 @@ doProlog(XML_Parser parser,
       dtd->hasParamEntityRefs = XML_TRUE;
       if (parser->m_startDoctypeDeclHandler) {
         parser->m_doctypeSysid = poolStoreString(&parser->m_tempPool, enc,
-                                       s + enc->minBytesPerChar,
-                                       next - enc->minBytesPerChar);
+                                                 s + enc->minBytesPerChar,
+                                                 next - enc->minBytesPerChar);
         if (parser->m_doctypeSysid == NULL)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
@@ -4741,22 +4536,20 @@ doProlog(XML_Parser parser,
            for the case where no parser->m_startDoctypeDeclHandler is set */
         parser->m_doctypeSysid = externalSubsetName;
 #endif /* XML_DTD */
-      if (!dtd->standalone
+      if (! dtd->standalone
 #ifdef XML_DTD
-          && !parser->m_paramEntityParsing
+          && ! parser->m_paramEntityParsing
 #endif /* XML_DTD */
           && parser->m_notStandaloneHandler
-          && !parser->m_notStandaloneHandler(parser->m_handlerArg))
+          && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
         return XML_ERROR_NOT_STANDALONE;
 #ifndef XML_DTD
       break;
-#else /* XML_DTD */
-      if (!parser->m_declEntity) {
-        parser->m_declEntity = (ENTITY *)lookup(parser,
-                                      &dtd->paramEntities,
-                                      externalSubsetName,
-                                      sizeof(ENTITY));
-        if (!parser->m_declEntity)
+#else  /* XML_DTD */
+      if (! parser->m_declEntity) {
+        parser->m_declEntity = (ENTITY *)lookup(
+            parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY));
+        if (! parser->m_declEntity)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity->publicId = NULL;
       }
@@ -4764,10 +4557,10 @@ doProlog(XML_Parser parser,
       /* fall through */
     case XML_ROLE_ENTITY_SYSTEM_ID:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        parser->m_declEntity->systemId = poolStoreString(&dtd->pool, enc,
-                                               s + enc->minBytesPerChar,
-                                               next - enc->minBytesPerChar);
-        if (!parser->m_declEntity->systemId)
+        parser->m_declEntity->systemId
+            = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                              next - enc->minBytesPerChar);
+        if (! parser->m_declEntity->systemId)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity->base = parser->m_curBase;
         poolFinish(&dtd->pool);
@@ -4779,115 +4572,103 @@ doProlog(XML_Parser parser,
       }
       break;
     case XML_ROLE_ENTITY_COMPLETE:
-      if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) {
+      if (dtd->keepProcessing && parser->m_declEntity
+          && parser->m_entityDeclHandler) {
         *eventEndPP = s;
-        parser->m_entityDeclHandler(parser->m_handlerArg,
-                          parser->m_declEntity->name,
-                          parser->m_declEntity->is_param,
-                          0,0,
-                          parser->m_declEntity->base,
-                          parser->m_declEntity->systemId,
-                          parser->m_declEntity->publicId,
-                          0);
+        parser->m_entityDeclHandler(
+            parser->m_handlerArg, parser->m_declEntity->name,
+            parser->m_declEntity->is_param, 0, 0, parser->m_declEntity->base,
+            parser->m_declEntity->systemId, parser->m_declEntity->publicId, 0);
         handleDefault = XML_FALSE;
       }
       break;
     case XML_ROLE_ENTITY_NOTATION_NAME:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        parser->m_declEntity->notation = poolStoreString(&dtd->pool, enc, s, next);
-        if (!parser->m_declEntity->notation)
+        parser->m_declEntity->notation
+            = poolStoreString(&dtd->pool, enc, s, next);
+        if (! parser->m_declEntity->notation)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&dtd->pool);
         if (parser->m_unparsedEntityDeclHandler) {
           *eventEndPP = s;
-          parser->m_unparsedEntityDeclHandler(parser->m_handlerArg,
-                                    parser->m_declEntity->name,
-                                    parser->m_declEntity->base,
-                                    parser->m_declEntity->systemId,
-                                    parser->m_declEntity->publicId,
-                                    parser->m_declEntity->notation);
+          parser->m_unparsedEntityDeclHandler(
+              parser->m_handlerArg, parser->m_declEntity->name,
+              parser->m_declEntity->base, parser->m_declEntity->systemId,
+              parser->m_declEntity->publicId, parser->m_declEntity->notation);
           handleDefault = XML_FALSE;
-        }
-        else if (parser->m_entityDeclHandler) {
+        } else if (parser->m_entityDeclHandler) {
           *eventEndPP = s;
-          parser->m_entityDeclHandler(parser->m_handlerArg,
-                            parser->m_declEntity->name,
-                            0,0,0,
-                            parser->m_declEntity->base,
-                            parser->m_declEntity->systemId,
-                            parser->m_declEntity->publicId,
-                            parser->m_declEntity->notation);
+          parser->m_entityDeclHandler(
+              parser->m_handlerArg, parser->m_declEntity->name, 0, 0, 0,
+              parser->m_declEntity->base, parser->m_declEntity->systemId,
+              parser->m_declEntity->publicId, parser->m_declEntity->notation);
           handleDefault = XML_FALSE;
         }
       }
       break;
-    case XML_ROLE_GENERAL_ENTITY_NAME:
-      {
-        if (XmlPredefinedEntityName(enc, s, next)) {
-          parser->m_declEntity = NULL;
-          break;
-        }
-        if (dtd->keepProcessing) {
-          const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
-          if (!name)
-            return XML_ERROR_NO_MEMORY;
-          parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->generalEntities, name,
-                                        sizeof(ENTITY));
-          if (!parser->m_declEntity)
-            return XML_ERROR_NO_MEMORY;
-          if (parser->m_declEntity->name != name) {
-            poolDiscard(&dtd->pool);
-            parser->m_declEntity = NULL;
-          }
-          else {
-            poolFinish(&dtd->pool);
-            parser->m_declEntity->publicId = NULL;
-            parser->m_declEntity->is_param = XML_FALSE;
-            /* if we have a parent parser or are reading an internal parameter
-               entity, then the entity declaration is not considered "internal"
-            */
-            parser->m_declEntity->is_internal = !(parser->m_parentParser || parser->m_openInternalEntities);
-            if (parser->m_entityDeclHandler)
-              handleDefault = XML_FALSE;
-          }
-        }
-        else {
+    case XML_ROLE_GENERAL_ENTITY_NAME: {
+      if (XmlPredefinedEntityName(enc, s, next)) {
+        parser->m_declEntity = NULL;
+        break;
+      }
+      if (dtd->keepProcessing) {
+        const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
+        if (! name)
+          return XML_ERROR_NO_MEMORY;
+        parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->generalEntities,
+                                                name, sizeof(ENTITY));
+        if (! parser->m_declEntity)
+          return XML_ERROR_NO_MEMORY;
+        if (parser->m_declEntity->name != name) {
           poolDiscard(&dtd->pool);
           parser->m_declEntity = NULL;
+        } else {
+          poolFinish(&dtd->pool);
+          parser->m_declEntity->publicId = NULL;
+          parser->m_declEntity->is_param = XML_FALSE;
+          /* if we have a parent parser or are reading an internal parameter
+             entity, then the entity declaration is not considered "internal"
+          */
+          parser->m_declEntity->is_internal
+              = ! (parser->m_parentParser || parser->m_openInternalEntities);
+          if (parser->m_entityDeclHandler)
+            handleDefault = XML_FALSE;
         }
+      } else {
+        poolDiscard(&dtd->pool);
+        parser->m_declEntity = NULL;
       }
-      break;
+    } break;
     case XML_ROLE_PARAM_ENTITY_NAME:
 #ifdef XML_DTD
       if (dtd->keepProcessing) {
         const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
-        if (!name)
+        if (! name)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->paramEntities,
-                                           name, sizeof(ENTITY));
-        if (!parser->m_declEntity)
+                                                name, sizeof(ENTITY));
+        if (! parser->m_declEntity)
           return XML_ERROR_NO_MEMORY;
         if (parser->m_declEntity->name != name) {
           poolDiscard(&dtd->pool);
           parser->m_declEntity = NULL;
-        }
-        else {
+        } else {
           poolFinish(&dtd->pool);
           parser->m_declEntity->publicId = NULL;
           parser->m_declEntity->is_param = XML_TRUE;
           /* if we have a parent parser or are reading an internal parameter
              entity, then the entity declaration is not considered "internal"
           */
-          parser->m_declEntity->is_internal = !(parser->m_parentParser || parser->m_openInternalEntities);
+          parser->m_declEntity->is_internal
+              = ! (parser->m_parentParser || parser->m_openInternalEntities);
           if (parser->m_entityDeclHandler)
             handleDefault = XML_FALSE;
         }
-      }
-      else {
+      } else {
         poolDiscard(&dtd->pool);
         parser->m_declEntity = NULL;
       }
-#else /* not XML_DTD */
+#else  /* not XML_DTD */
       parser->m_declEntity = NULL;
 #endif /* XML_DTD */
       break;
@@ -4895,22 +4676,23 @@ doProlog(XML_Parser parser,
       parser->m_declNotationPublicId = NULL;
       parser->m_declNotationName = NULL;
       if (parser->m_notationDeclHandler) {
-        parser->m_declNotationName = poolStoreString(&parser->m_tempPool, enc, s, next);
-        if (!parser->m_declNotationName)
+        parser->m_declNotationName
+            = poolStoreString(&parser->m_tempPool, enc, s, next);
+        if (! parser->m_declNotationName)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
       break;
     case XML_ROLE_NOTATION_PUBLIC_ID:
-      if (!XmlIsPublicId(enc, s, next, eventPP))
+      if (! XmlIsPublicId(enc, s, next, eventPP))
         return XML_ERROR_PUBLICID;
-      if (parser->m_declNotationName) {  /* means m_notationDeclHandler != NULL */
-        XML_Char *tem = poolStoreString(&parser->m_tempPool,
-                                        enc,
+      if (parser
+              ->m_declNotationName) { /* means m_notationDeclHandler != NULL */
+        XML_Char *tem = poolStoreString(&parser->m_tempPool, enc,
                                         s + enc->minBytesPerChar,
                                         next - enc->minBytesPerChar);
-        if (!tem)
+        if (! tem)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(tem);
         parser->m_declNotationPublicId = tem;
@@ -4920,18 +4702,15 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_NOTATION_SYSTEM_ID:
       if (parser->m_declNotationName && parser->m_notationDeclHandler) {
-        const XML_Char *systemId
-          = poolStoreString(&parser->m_tempPool, enc,
-                            s + enc->minBytesPerChar,
-                            next - enc->minBytesPerChar);
-        if (!systemId)
+        const XML_Char *systemId = poolStoreString(&parser->m_tempPool, enc,
+                                                   s + enc->minBytesPerChar,
+                                                   next - enc->minBytesPerChar);
+        if (! systemId)
           return XML_ERROR_NO_MEMORY;
         *eventEndPP = s;
-        parser->m_notationDeclHandler(parser->m_handlerArg,
-                            parser->m_declNotationName,
-                            parser->m_curBase,
-                            systemId,
-                            parser->m_declNotationPublicId);
+        parser->m_notationDeclHandler(
+            parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase,
+            systemId, parser->m_declNotationPublicId);
         handleDefault = XML_FALSE;
       }
       poolClear(&parser->m_tempPool);
@@ -4939,11 +4718,9 @@ doProlog(XML_Parser parser,
     case XML_ROLE_NOTATION_NO_SYSTEM_ID:
       if (parser->m_declNotationPublicId && parser->m_notationDeclHandler) {
         *eventEndPP = s;
-        parser->m_notationDeclHandler(parser->m_handlerArg,
-                            parser->m_declNotationName,
-                            parser->m_curBase,
-                            0,
-                            parser->m_declNotationPublicId);
+        parser->m_notationDeclHandler(
+            parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase,
+            0, parser->m_declNotationPublicId);
         handleDefault = XML_FALSE;
       }
       poolClear(&parser->m_tempPool);
@@ -4960,42 +4737,44 @@ doProlog(XML_Parser parser,
         return XML_ERROR_SYNTAX;
       }
 #ifdef XML_DTD
-    case XML_ROLE_IGNORE_SECT:
-      {
-        enum XML_Error result;
-        if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        handleDefault = XML_FALSE;
-        result = doIgnoreSection(parser, enc, &next, end, nextPtr, haveMore);
-        if (result != XML_ERROR_NONE)
-          return result;
-        else if (!next) {
-          parser->m_processor = ignoreSectionProcessor;
-          return result;
-        }
+    case XML_ROLE_IGNORE_SECT: {
+      enum XML_Error result;
+      if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      handleDefault = XML_FALSE;
+      result = doIgnoreSection(parser, enc, &next, end, nextPtr, haveMore);
+      if (result != XML_ERROR_NONE)
+        return result;
+      else if (! next) {
+        parser->m_processor = ignoreSectionProcessor;
+        return result;
       }
-      break;
+    } break;
 #endif /* XML_DTD */
     case XML_ROLE_GROUP_OPEN:
       if (parser->m_prologState.level >= parser->m_groupSize) {
         if (parser->m_groupSize) {
-          char *temp = (char *)REALLOC(parser, parser->m_groupConnector, parser->m_groupSize *= 2);
-          if (temp == NULL) {
-            parser->m_groupSize /= 2;
-            return XML_ERROR_NO_MEMORY;
+          {
+            char *const new_connector = (char *)REALLOC(
+                parser, parser->m_groupConnector, parser->m_groupSize *= 2);
+            if (new_connector == NULL) {
+              parser->m_groupSize /= 2;
+              return XML_ERROR_NO_MEMORY;
+            }
+            parser->m_groupConnector = new_connector;
           }
-          parser->m_groupConnector = temp;
+
           if (dtd->scaffIndex) {
-            int *temp = (int *)REALLOC(parser, dtd->scaffIndex,
-                          parser->m_groupSize * sizeof(int));
-            if (temp == NULL)
+            int *const new_scaff_index = (int *)REALLOC(
+                parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int));
+            if (new_scaff_index == NULL)
               return XML_ERROR_NO_MEMORY;
-            dtd->scaffIndex = temp;
+            dtd->scaffIndex = new_scaff_index;
           }
-        }
-        else {
-          parser->m_groupConnector = (char *)MALLOC(parser, parser->m_groupSize = 32);
-          if (!parser->m_groupConnector) {
+        } else {
+          parser->m_groupConnector
+              = (char *)MALLOC(parser, parser->m_groupSize = 32);
+          if (! parser->m_groupConnector) {
             parser->m_groupSize = 0;
             return XML_ERROR_NO_MEMORY;
           }
@@ -5006,6 +4785,7 @@ doProlog(XML_Parser parser,
         int myindex = nextScaffoldPart(parser);
         if (myindex < 0)
           return XML_ERROR_NO_MEMORY;
+        assert(dtd->scaffIndex != NULL);
         dtd->scaffIndex[dtd->scaffLevel] = myindex;
         dtd->scaffLevel++;
         dtd->scaffold[myindex].type = XML_CTYPE_SEQ;
@@ -5024,10 +4804,9 @@ doProlog(XML_Parser parser,
       if (parser->m_groupConnector[parser->m_prologState.level] == ASCII_COMMA)
         return XML_ERROR_SYNTAX;
       if (dtd->in_eldecl
-          && !parser->m_groupConnector[parser->m_prologState.level]
+          && ! parser->m_groupConnector[parser->m_prologState.level]
           && (dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]].type
-              != XML_CTYPE_MIXED)
-          ) {
+              != XML_CTYPE_MIXED)) {
         dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]].type
             = XML_CTYPE_CHOICE;
         if (parser->m_elementDeclHandler)
@@ -5039,15 +4818,14 @@ doProlog(XML_Parser parser,
 #ifdef XML_DTD
     case XML_ROLE_INNER_PARAM_ENTITY_REF:
       dtd->hasParamEntityRefs = XML_TRUE;
-      if (!parser->m_paramEntityParsing)
+      if (! parser->m_paramEntityParsing)
         dtd->keepProcessing = dtd->standalone;
       else {
         const XML_Char *name;
         ENTITY *entity;
-        name = poolStoreString(&dtd->pool, enc,
-                                s + enc->minBytesPerChar,
-                                next - enc->minBytesPerChar);
-        if (!name)
+        name = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                               next - enc->minBytesPerChar);
+        if (! name)
           return XML_ERROR_NO_MEMORY;
         entity = (ENTITY *)lookup(parser, &dtd->paramEntities, name, 0);
         poolDiscard(&dtd->pool);
@@ -5055,13 +4833,12 @@ doProlog(XML_Parser parser,
            if yes, check that the entity exists, and that it is internal,
            otherwise call the skipped entity handler
         */
-        if (parser->m_prologState.documentEntity &&
-            (dtd->standalone
-             ? !parser->m_openInternalEntities
-             : !dtd->hasParamEntityRefs)) {
-          if (!entity)
+        if (parser->m_prologState.documentEntity
+            && (dtd->standalone ? ! parser->m_openInternalEntities
+                                : ! dtd->hasParamEntityRefs)) {
+          if (! entity)
             return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal) {
+          else if (! entity->is_internal) {
             /* It's hard to exhaustively search the code to be sure,
              * but there doesn't seem to be a way of executing the
              * following line.  There are two cases:
@@ -5084,11 +4861,11 @@ doProlog(XML_Parser parser,
              */
             return XML_ERROR_ENTITY_DECLARED_IN_PE; /* LCOV_EXCL_LINE */
           }
-        }
-        else if (!entity) {
+        } else if (! entity) {
           dtd->keepProcessing = dtd->standalone;
           /* cannot report skipped entities in declarations */
-          if ((role == XML_ROLE_PARAM_ENTITY_REF) && parser->m_skippedEntityHandler) {
+          if ((role == XML_ROLE_PARAM_ENTITY_REF)
+              && parser->m_skippedEntityHandler) {
             parser->m_skippedEntityHandler(parser->m_handlerArg, name, 1);
             handleDefault = XML_FALSE;
           }
@@ -5098,8 +4875,8 @@ doProlog(XML_Parser parser,
           return XML_ERROR_RECURSIVE_ENTITY_REF;
         if (entity->textPtr) {
           enum XML_Error result;
-          XML_Bool betweenDecl =
-            (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE);
+          XML_Bool betweenDecl
+              = (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE);
           result = processInternalEntity(parser, entity, betweenDecl);
           if (result != XML_ERROR_NONE)
             return result;
@@ -5109,39 +4886,35 @@ doProlog(XML_Parser parser,
         if (parser->m_externalEntityRefHandler) {
           dtd->paramEntityRead = XML_FALSE;
           entity->open = XML_TRUE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId)) {
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId)) {
             entity->open = XML_FALSE;
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           }
           entity->open = XML_FALSE;
           handleDefault = XML_FALSE;
-          if (!dtd->paramEntityRead) {
+          if (! dtd->paramEntityRead) {
             dtd->keepProcessing = dtd->standalone;
             break;
           }
-        }
-        else {
+        } else {
           dtd->keepProcessing = dtd->standalone;
           break;
         }
       }
 #endif /* XML_DTD */
-      if (!dtd->standalone &&
-          parser->m_notStandaloneHandler &&
-          !parser->m_notStandaloneHandler(parser->m_handlerArg))
+      if (! dtd->standalone && parser->m_notStandaloneHandler
+          && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
         return XML_ERROR_NOT_STANDALONE;
       break;
 
-    /* Element declaration stuff */
+      /* Element declaration stuff */
 
     case XML_ROLE_ELEMENT_NAME:
       if (parser->m_elementDeclHandler) {
         parser->m_declElementType = getElementType(parser, enc, s, next);
-        if (!parser->m_declElementType)
+        if (! parser->m_declElementType)
           return XML_ERROR_NO_MEMORY;
         dtd->scaffLevel = 0;
         dtd->scaffCount = 0;
@@ -5154,18 +4927,19 @@ doProlog(XML_Parser parser,
     case XML_ROLE_CONTENT_EMPTY:
       if (dtd->in_eldecl) {
         if (parser->m_elementDeclHandler) {
-          XML_Content * content = (XML_Content *) MALLOC(parser, sizeof(XML_Content));
-          if (!content)
+          XML_Content *content
+              = (XML_Content *)MALLOC(parser, sizeof(XML_Content));
+          if (! content)
             return XML_ERROR_NO_MEMORY;
           content->quant = XML_CQUANT_NONE;
           content->name = NULL;
           content->numchildren = 0;
           content->children = NULL;
-          content->type = ((role == XML_ROLE_CONTENT_ANY) ?
-                           XML_CTYPE_ANY :
-                           XML_CTYPE_EMPTY);
+          content->type = ((role == XML_ROLE_CONTENT_ANY) ? XML_CTYPE_ANY
+                                                          : XML_CTYPE_EMPTY);
           *eventEndPP = s;
-          parser->m_elementDeclHandler(parser->m_handlerArg, parser->m_declElementType->name, content);
+          parser->m_elementDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name, content);
           handleDefault = XML_FALSE;
         }
         dtd->in_eldecl = XML_FALSE;
@@ -5197,22 +4971,22 @@ doProlog(XML_Parser parser,
         ELEMENT_TYPE *el;
         const XML_Char *name;
         int nameLen;
-        const char *nxt = (quant == XML_CQUANT_NONE
-                           ? next
-                           : next - enc->minBytesPerChar);
+        const char *nxt
+            = (quant == XML_CQUANT_NONE ? next : next - enc->minBytesPerChar);
         int myindex = nextScaffoldPart(parser);
         if (myindex < 0)
           return XML_ERROR_NO_MEMORY;
         dtd->scaffold[myindex].type = XML_CTYPE_NAME;
         dtd->scaffold[myindex].quant = quant;
         el = getElementType(parser, enc, s, nxt);
-        if (!el)
+        if (! el)
           return XML_ERROR_NO_MEMORY;
         name = el->name;
         dtd->scaffold[myindex].name = name;
         nameLen = 0;
-        for (; name[nameLen++]; );
-        dtd->contentStringLen +=  nameLen;
+        for (; name[nameLen++];)
+          ;
+        dtd->contentStringLen += nameLen;
         if (parser->m_elementDeclHandler)
           handleDefault = XML_FALSE;
       }
@@ -5236,12 +5010,13 @@ doProlog(XML_Parser parser,
         dtd->scaffLevel--;
         dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel]].quant = quant;
         if (dtd->scaffLevel == 0) {
-          if (!handleDefault) {
+          if (! handleDefault) {
             XML_Content *model = build_model(parser);
-            if (!model)
+            if (! model)
               return XML_ERROR_NO_MEMORY;
             *eventEndPP = s;
-            parser->m_elementDeclHandler(parser->m_handlerArg, parser->m_declElementType->name, model);
+            parser->m_elementDeclHandler(
+                parser->m_handlerArg, parser->m_declElementType->name, model);
           }
           dtd->in_eldecl = XML_FALSE;
           dtd->contentStringLen = 0;
@@ -5251,12 +5026,12 @@ doProlog(XML_Parser parser,
       /* End element declaration stuff */
 
     case XML_ROLE_PI:
-      if (!reportProcessingInstruction(parser, enc, s, next))
+      if (! reportProcessingInstruction(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       handleDefault = XML_FALSE;
       break;
     case XML_ROLE_COMMENT:
-      if (!reportComment(parser, enc, s, next))
+      if (! reportComment(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       handleDefault = XML_FALSE;
       break;
@@ -5307,11 +5082,8 @@ doProlog(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-epilogProcessor(XML_Parser parser,
-                const char *s,
-                const char *end,
-                const char **nextPtr)
-{
+epilogProcessor(XML_Parser parser, const char *s, const char *end,
+                const char **nextPtr) {
   parser->m_processor = epilogProcessor;
   parser->m_eventPtr = s;
   for (;;) {
@@ -5336,24 +5108,24 @@ epilogProcessor(XML_Parser parser,
         reportDefault(parser, parser->m_encoding, s, next);
       break;
     case XML_TOK_PI:
-      if (!reportProcessingInstruction(parser, parser->m_encoding, s, next))
+      if (! reportProcessingInstruction(parser, parser->m_encoding, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_COMMENT:
-      if (!reportComment(parser, parser->m_encoding, s, next))
+      if (! reportComment(parser, parser->m_encoding, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_INVALID:
       parser->m_eventPtr = next;
       return XML_ERROR_INVALID_TOKEN;
     case XML_TOK_PARTIAL:
-      if (!parser->m_parsingStatus.finalBuffer) {
+      if (! parser->m_parsingStatus.finalBuffer) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
       return XML_ERROR_UNCLOSED_TOKEN;
     case XML_TOK_PARTIAL_CHAR:
-      if (!parser->m_parsingStatus.finalBuffer) {
+      if (! parser->m_parsingStatus.finalBuffer) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -5368,15 +5140,13 @@ epilogProcessor(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
 }
 
 static enum XML_Error
-processInternalEntity(XML_Parser parser, ENTITY *entity,
-                      XML_Bool betweenDecl)
-{
+processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) {
   const char *textStart, *textEnd;
   const char *next;
   enum XML_Error result;
@@ -5385,10 +5155,10 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
   if (parser->m_freeInternalEntities) {
     openEntity = parser->m_freeInternalEntities;
     parser->m_freeInternalEntities = openEntity->next;
-  }
-  else {
-    openEntity = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY));
-    if (!openEntity)
+  } else {
+    openEntity
+        = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY));
+    if (! openEntity)
       return XML_ERROR_NO_MEMORY;
   }
   entity->open = XML_TRUE;
@@ -5407,21 +5177,20 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
 
 #ifdef XML_DTD
   if (entity->is_param) {
-    int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
-    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok,
-                      next, &next, XML_FALSE);
-  }
-  else
+    int tok
+        = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
+    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
+                      tok, next, &next, XML_FALSE, XML_FALSE);
+  } else
 #endif /* XML_DTD */
-    result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, textStart,
-                       textEnd, &next, XML_FALSE);
+    result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding,
+                       textStart, textEnd, &next, XML_FALSE);
 
   if (result == XML_ERROR_NONE) {
     if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
       entity->processed = (int)(next - textStart);
       parser->m_processor = internalEntityProcessor;
-    }
-    else {
+    } else {
       entity->open = XML_FALSE;
       parser->m_openInternalEntities = openEntity->next;
       /* put openEntity back in list of free instances */
@@ -5433,17 +5202,14 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
 }
 
 static enum XML_Error PTRCALL
-internalEntityProcessor(XML_Parser parser,
-                        const char *s,
-                        const char *end,
-                        const char **nextPtr)
-{
+internalEntityProcessor(XML_Parser parser, const char *s, const char *end,
+                        const char **nextPtr) {
   ENTITY *entity;
   const char *textStart, *textEnd;
   const char *next;
   enum XML_Error result;
   OPEN_INTERNAL_ENTITY *openEntity = parser->m_openInternalEntities;
-  if (!openEntity)
+  if (! openEntity)
     return XML_ERROR_UNEXPECTED_STATE;
 
   entity = openEntity->entity;
@@ -5454,22 +5220,23 @@ internalEntityProcessor(XML_Parser parser,
 
 #ifdef XML_DTD
   if (entity->is_param) {
-    int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
-    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok,
-                      next, &next, XML_FALSE);
-  }
-  else
+    int tok
+        = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
+    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
+                      tok, next, &next, XML_FALSE, XML_TRUE);
+  } else
 #endif /* XML_DTD */
-    result = doContent(parser, openEntity->startTagLevel, parser->m_internalEncoding,
-                       textStart, textEnd, &next, XML_FALSE);
+    result = doContent(parser, openEntity->startTagLevel,
+                       parser->m_internalEncoding, textStart, textEnd, &next,
+                       XML_FALSE);
 
   if (result != XML_ERROR_NONE)
     return result;
-  else if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
+  else if (textEnd != next
+           && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
     entity->processed = (int)(next - (char *)entity->textPtr);
     return result;
-  }
-  else {
+  } else {
     entity->open = XML_FALSE;
     parser->m_openInternalEntities = openEntity->next;
     /* put openEntity back in list of free instances */
@@ -5483,49 +5250,45 @@ internalEntityProcessor(XML_Parser parser,
     parser->m_processor = prologProcessor;
     tok = XmlPrologTok(parser->m_encoding, s, end, &next);
     return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
-                    (XML_Bool)!parser->m_parsingStatus.finalBuffer);
-  }
-  else
+                    (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
+  } else
 #endif /* XML_DTD */
   {
     parser->m_processor = contentProcessor;
     /* see externalEntityContentProcessor vs contentProcessor */
-    return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding, s, end,
-                     nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+    return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding,
+                     s, end, nextPtr,
+                     (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   }
 }
 
 static enum XML_Error PTRCALL
-errorProcessor(XML_Parser parser,
-               const char *UNUSED_P(s),
-               const char *UNUSED_P(end),
-               const char **UNUSED_P(nextPtr))
-{
+errorProcessor(XML_Parser parser, const char *s, const char *end,
+               const char **nextPtr) {
+  UNUSED_P(s);
+  UNUSED_P(end);
+  UNUSED_P(nextPtr);
   return parser->m_errorCode;
 }
 
 static enum XML_Error
 storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
-                    const char *ptr, const char *end,
-                    STRING_POOL *pool)
-{
-  enum XML_Error result = appendAttributeValue(parser, enc, isCdata, ptr,
-                                               end, pool);
+                    const char *ptr, const char *end, STRING_POOL *pool) {
+  enum XML_Error result
+      = appendAttributeValue(parser, enc, isCdata, ptr, end, pool);
   if (result)
     return result;
-  if (!isCdata && poolLength(pool) && poolLastChar(pool) == 0x20)
+  if (! isCdata && poolLength(pool) && poolLastChar(pool) == 0x20)
     poolChop(pool);
-  if (!poolAppendChar(pool, XML_T('\0')))
+  if (! poolAppendChar(pool, XML_T('\0')))
     return XML_ERROR_NO_MEMORY;
   return XML_ERROR_NONE;
 }
 
 static enum XML_Error
 appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
-                     const char *ptr, const char *end,
-                     STRING_POOL *pool)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+                     const char *ptr, const char *end, STRING_POOL *pool) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   for (;;) {
     const char *next;
     int tok = XmlAttributeValueTok(enc, ptr, end, &next);
@@ -5540,38 +5303,35 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
       if (enc == parser->m_encoding)
         parser->m_eventPtr = ptr;
       return XML_ERROR_INVALID_TOKEN;
-    case XML_TOK_CHAR_REF:
-      {
-        XML_Char buf[XML_ENCODE_MAX];
-        int i;
-        int n = XmlCharRefNumber(enc, ptr);
-        if (n < 0) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_BAD_CHAR_REF;
-        }
-        if (!isCdata
-            && n == 0x20 /* space */
-            && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
-          break;
-        n = XmlEncode(n, (ICHAR *)buf);
-        /* The XmlEncode() functions can never return 0 here.  That
-         * error return happens if the code point passed in is either
-         * negative or greater than or equal to 0x110000.  The
-         * XmlCharRefNumber() functions will all return a number
-         * strictly less than 0x110000 or a negative value if an error
-         * occurred.  The negative value is intercepted above, so
-         * XmlEncode() is never passed a value it might return an
-         * error for.
-         */
-        for (i = 0; i < n; i++) {
-          if (!poolAppendChar(pool, buf[i]))
-            return XML_ERROR_NO_MEMORY;
-        }
+    case XML_TOK_CHAR_REF: {
+      XML_Char buf[XML_ENCODE_MAX];
+      int i;
+      int n = XmlCharRefNumber(enc, ptr);
+      if (n < 0) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_BAD_CHAR_REF;
       }
-      break;
+      if (! isCdata && n == 0x20 /* space */
+          && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
+        break;
+      n = XmlEncode(n, (ICHAR *)buf);
+      /* The XmlEncode() functions can never return 0 here.  That
+       * error return happens if the code point passed in is either
+       * negative or greater than or equal to 0x110000.  The
+       * XmlCharRefNumber() functions will all return a number
+       * strictly less than 0x110000 or a negative value if an error
+       * occurred.  The negative value is intercepted above, so
+       * XmlEncode() is never passed a value it might return an
+       * error for.
+       */
+      for (i = 0; i < n; i++) {
+        if (! poolAppendChar(pool, buf[i]))
+          return XML_ERROR_NO_MEMORY;
+      }
+    } break;
     case XML_TOK_DATA_CHARS:
-      if (!poolAppend(pool, enc, ptr, next))
+      if (! poolAppend(pool, enc, ptr, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_TRAILING_CR:
@@ -5579,109 +5339,103 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
       /* fall through */
     case XML_TOK_ATTRIBUTE_VALUE_S:
     case XML_TOK_DATA_NEWLINE:
-      if (!isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
+      if (! isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
         break;
-      if (!poolAppendChar(pool, 0x20))
+      if (! poolAppendChar(pool, 0x20))
         return XML_ERROR_NO_MEMORY;
       break;
-    case XML_TOK_ENTITY_REF:
-      {
-        const XML_Char *name;
-        ENTITY *entity;
-        char checkEntityDecl;
-        XML_Char ch = (XML_Char) XmlPredefinedEntityName(enc,
-                                              ptr + enc->minBytesPerChar,
-                                              next - enc->minBytesPerChar);
-        if (ch) {
-          if (!poolAppendChar(pool, ch))
-                return XML_ERROR_NO_MEMORY;
-          break;
-        }
-        name = poolStoreString(&parser->m_temp2Pool, enc,
-                               ptr + enc->minBytesPerChar,
-                               next - enc->minBytesPerChar);
-        if (!name)
+    case XML_TOK_ENTITY_REF: {
+      const XML_Char *name;
+      ENTITY *entity;
+      char checkEntityDecl;
+      XML_Char ch = (XML_Char)XmlPredefinedEntityName(
+          enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar);
+      if (ch) {
+        if (! poolAppendChar(pool, ch))
           return XML_ERROR_NO_MEMORY;
-        entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
-        poolDiscard(&parser->m_temp2Pool);
-        /* First, determine if a check for an existing declaration is needed;
-           if yes, check that the entity exists, and that it is internal.
-        */
-        if (pool == &dtd->pool)  /* are we called from prolog? */
-          checkEntityDecl =
+        break;
+      }
+      name = poolStoreString(&parser->m_temp2Pool, enc,
+                             ptr + enc->minBytesPerChar,
+                             next - enc->minBytesPerChar);
+      if (! name)
+        return XML_ERROR_NO_MEMORY;
+      entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
+      poolDiscard(&parser->m_temp2Pool);
+      /* First, determine if a check for an existing declaration is needed;
+         if yes, check that the entity exists, and that it is internal.
+      */
+      if (pool == &dtd->pool) /* are we called from prolog? */
+        checkEntityDecl =
 #ifdef XML_DTD
-              parser->m_prologState.documentEntity &&
+            parser->m_prologState.documentEntity &&
 #endif /* XML_DTD */
-              (dtd->standalone
-               ? !parser->m_openInternalEntities
-               : !dtd->hasParamEntityRefs);
-        else /* if (pool == &parser->m_tempPool): we are called from content */
-          checkEntityDecl = !dtd->hasParamEntityRefs || dtd->standalone;
-        if (checkEntityDecl) {
-          if (!entity)
-            return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal)
-            return XML_ERROR_ENTITY_DECLARED_IN_PE;
-        }
-        else if (!entity) {
-          /* Cannot report skipped entity here - see comments on
-             parser->m_skippedEntityHandler.
-          if (parser->m_skippedEntityHandler)
-            parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
-          */
-          /* Cannot call the default handler because this would be
-             out of sync with the call to the startElementHandler.
-          if ((pool == &parser->m_tempPool) && parser->m_defaultHandler)
-            reportDefault(parser, enc, ptr, next);
-          */
-          break;
-        }
-        if (entity->open) {
-          if (enc == parser->m_encoding) {
-            /* It does not appear that this line can be executed.
-             *
-             * The "if (entity->open)" check catches recursive entity
-             * definitions.  In order to be called with an open
-             * entity, it must have gone through this code before and
-             * been through the recursive call to
-             * appendAttributeValue() some lines below.  That call
-             * sets the local encoding ("enc") to the parser's
-             * internal encoding (internal_utf8 or internal_utf16),
-             * which can never be the same as the principle encoding.
-             * It doesn't appear there is another code path that gets
-             * here with entity->open being TRUE.
-             *
-             * Since it is not certain that this logic is watertight,
-             * we keep the line and merely exclude it from coverage
-             * tests.
-             */
-            parser->m_eventPtr = ptr; /* LCOV_EXCL_LINE */
-          }
-          return XML_ERROR_RECURSIVE_ENTITY_REF;
-        }
-        if (entity->notation) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_BINARY_ENTITY_REF;
-        }
-        if (!entity->textPtr) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF;
-        }
-        else {
-          enum XML_Error result;
-          const XML_Char *textEnd = entity->textPtr + entity->textLen;
-          entity->open = XML_TRUE;
-          result = appendAttributeValue(parser, parser->m_internalEncoding, isCdata,
-                                        (char *)entity->textPtr,
-                                        (char *)textEnd, pool);
-          entity->open = XML_FALSE;
-          if (result)
-            return result;
+            (dtd->standalone ? ! parser->m_openInternalEntities
+                             : ! dtd->hasParamEntityRefs);
+      else /* if (pool == &parser->m_tempPool): we are called from content */
+        checkEntityDecl = ! dtd->hasParamEntityRefs || dtd->standalone;
+      if (checkEntityDecl) {
+        if (! entity)
+          return XML_ERROR_UNDEFINED_ENTITY;
+        else if (! entity->is_internal)
+          return XML_ERROR_ENTITY_DECLARED_IN_PE;
+      } else if (! entity) {
+        /* Cannot report skipped entity here - see comments on
+           parser->m_skippedEntityHandler.
+        if (parser->m_skippedEntityHandler)
+          parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+        */
+        /* Cannot call the default handler because this would be
+           out of sync with the call to the startElementHandler.
+        if ((pool == &parser->m_tempPool) && parser->m_defaultHandler)
+          reportDefault(parser, enc, ptr, next);
+        */
+        break;
+      }
+      if (entity->open) {
+        if (enc == parser->m_encoding) {
+          /* It does not appear that this line can be executed.
+           *
+           * The "if (entity->open)" check catches recursive entity
+           * definitions.  In order to be called with an open
+           * entity, it must have gone through this code before and
+           * been through the recursive call to
+           * appendAttributeValue() some lines below.  That call
+           * sets the local encoding ("enc") to the parser's
+           * internal encoding (internal_utf8 or internal_utf16),
+           * which can never be the same as the principle encoding.
+           * It doesn't appear there is another code path that gets
+           * here with entity->open being TRUE.
+           *
+           * Since it is not certain that this logic is watertight,
+           * we keep the line and merely exclude it from coverage
+           * tests.
+           */
+          parser->m_eventPtr = ptr; /* LCOV_EXCL_LINE */
         }
+        return XML_ERROR_RECURSIVE_ENTITY_REF;
       }
-      break;
+      if (entity->notation) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_BINARY_ENTITY_REF;
+      }
+      if (! entity->textPtr) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF;
+      } else {
+        enum XML_Error result;
+        const XML_Char *textEnd = entity->textPtr + entity->textLen;
+        entity->open = XML_TRUE;
+        result = appendAttributeValue(parser, parser->m_internalEncoding,
+                                      isCdata, (char *)entity->textPtr,
+                                      (char *)textEnd, pool);
+        entity->open = XML_FALSE;
+        if (result)
+          return result;
+      }
+    } break;
     default:
       /* The only token returned by XmlAttributeValueTok() that does
        * not have an explicit case here is XML_TOK_PARTIAL_CHAR.
@@ -5705,12 +5459,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
 }
 
 static enum XML_Error
-storeEntityValue(XML_Parser parser,
-                 const ENCODING *enc,
-                 const char *entityTextPtr,
-                 const char *entityTextEnd)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+storeEntityValue(XML_Parser parser, const ENCODING *enc,
+                 const char *entityTextPtr, const char *entityTextEnd) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   STRING_POOL *pool = &(dtd->entityValuePool);
   enum XML_Error result = XML_ERROR_NONE;
 #ifdef XML_DTD
@@ -5720,8 +5471,8 @@ storeEntityValue(XML_Parser parser,
   /* never return Null for the value argument in EntityDeclHandler,
      since this would indicate an external entity; therefore we
      have to make sure that entityValuePool.start is not null */
-  if (!pool->blocks) {
-    if (!poolGrow(pool))
+  if (! pool->blocks) {
+    if (! poolGrow(pool))
       return XML_ERROR_NO_MEMORY;
   }
 
@@ -5737,13 +5488,13 @@ storeEntityValue(XML_Parser parser,
         name = poolStoreString(&parser->m_tempPool, enc,
                                entityTextPtr + enc->minBytesPerChar,
                                next - enc->minBytesPerChar);
-        if (!name) {
+        if (! name) {
           result = XML_ERROR_NO_MEMORY;
           goto endEntityValue;
         }
         entity = (ENTITY *)lookup(parser, &dtd->paramEntities, name, 0);
         poolDiscard(&parser->m_tempPool);
-        if (!entity) {
+        if (! entity) {
           /* not a well-formedness error - see XML 1.0: WFC Entity Declared */
           /* cannot report skipped entity here - see comments on
              parser->m_skippedEntityHandler
@@ -5763,29 +5514,23 @@ storeEntityValue(XML_Parser parser,
           if (parser->m_externalEntityRefHandler) {
             dtd->paramEntityRead = XML_FALSE;
             entity->open = XML_TRUE;
-            if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                          0,
-                                          entity->base,
-                                          entity->systemId,
-                                          entity->publicId)) {
+            if (! parser->m_externalEntityRefHandler(
+                    parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                    entity->systemId, entity->publicId)) {
               entity->open = XML_FALSE;
               result = XML_ERROR_EXTERNAL_ENTITY_HANDLING;
               goto endEntityValue;
             }
             entity->open = XML_FALSE;
-            if (!dtd->paramEntityRead)
+            if (! dtd->paramEntityRead)
               dtd->keepProcessing = dtd->standalone;
-          }
-          else
+          } else
             dtd->keepProcessing = dtd->standalone;
-        }
-        else {
+        } else {
           entity->open = XML_TRUE;
-          result = storeEntityValue(parser,
-                                    parser->m_internalEncoding,
-                                    (char *)entity->textPtr,
-                                    (char *)(entity->textPtr
-                                             + entity->textLen));
+          result = storeEntityValue(
+              parser, parser->m_internalEncoding, (char *)entity->textPtr,
+              (char *)(entity->textPtr + entity->textLen));
           entity->open = XML_FALSE;
           if (result)
             goto endEntityValue;
@@ -5803,7 +5548,7 @@ storeEntityValue(XML_Parser parser,
       goto endEntityValue;
     case XML_TOK_ENTITY_REF:
     case XML_TOK_DATA_CHARS:
-      if (!poolAppend(pool, enc, entityTextPtr, next)) {
+      if (! poolAppend(pool, enc, entityTextPtr, next)) {
         result = XML_ERROR_NO_MEMORY;
         goto endEntityValue;
       }
@@ -5812,42 +5557,40 @@ storeEntityValue(XML_Parser parser,
       next = entityTextPtr + enc->minBytesPerChar;
       /* fall through */
     case XML_TOK_DATA_NEWLINE:
-      if (pool->end == pool->ptr && !poolGrow(pool)) {
-              result = XML_ERROR_NO_MEMORY;
+      if (pool->end == pool->ptr && ! poolGrow(pool)) {
+        result = XML_ERROR_NO_MEMORY;
         goto endEntityValue;
       }
       *(pool->ptr)++ = 0xA;
       break;
-    case XML_TOK_CHAR_REF:
-      {
-        XML_Char buf[XML_ENCODE_MAX];
-        int i;
-        int n = XmlCharRefNumber(enc, entityTextPtr);
-        if (n < 0) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = entityTextPtr;
-          result = XML_ERROR_BAD_CHAR_REF;
+    case XML_TOK_CHAR_REF: {
+      XML_Char buf[XML_ENCODE_MAX];
+      int i;
+      int n = XmlCharRefNumber(enc, entityTextPtr);
+      if (n < 0) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = entityTextPtr;
+        result = XML_ERROR_BAD_CHAR_REF;
+        goto endEntityValue;
+      }
+      n = XmlEncode(n, (ICHAR *)buf);
+      /* The XmlEncode() functions can never return 0 here.  That
+       * error return happens if the code point passed in is either
+       * negative or greater than or equal to 0x110000.  The
+       * XmlCharRefNumber() functions will all return a number
+       * strictly less than 0x110000 or a negative value if an error
+       * occurred.  The negative value is intercepted above, so
+       * XmlEncode() is never passed a value it might return an
+       * error for.
+       */
+      for (i = 0; i < n; i++) {
+        if (pool->end == pool->ptr && ! poolGrow(pool)) {
+          result = XML_ERROR_NO_MEMORY;
           goto endEntityValue;
         }
-        n = XmlEncode(n, (ICHAR *)buf);
-        /* The XmlEncode() functions can never return 0 here.  That
-         * error return happens if the code point passed in is either
-         * negative or greater than or equal to 0x110000.  The
-         * XmlCharRefNumber() functions will all return a number
-         * strictly less than 0x110000 or a negative value if an error
-         * occurred.  The negative value is intercepted above, so
-         * XmlEncode() is never passed a value it might return an
-         * error for.
-         */
-        for (i = 0; i < n; i++) {
-          if (pool->end == pool->ptr && !poolGrow(pool)) {
-            result = XML_ERROR_NO_MEMORY;
-            goto endEntityValue;
-          }
-          *(pool->ptr)++ = buf[i];
-        }
+        *(pool->ptr)++ = buf[i];
       }
-      break;
+    } break;
     case XML_TOK_PARTIAL:
       if (enc == parser->m_encoding)
         parser->m_eventPtr = entityTextPtr;
@@ -5882,8 +5625,7 @@ storeEntityValue(XML_Parser parser,
 }
 
 static void FASTCALL
-normalizeLines(XML_Char *s)
-{
+normalizeLines(XML_Char *s) {
   XML_Char *p;
   for (;; s++) {
     if (*s == XML_T('\0'))
@@ -5897,8 +5639,7 @@ normalizeLines(XML_Char *s)
       *p++ = 0xA;
       if (*++s == 0xA)
         s++;
-    }
-    else
+    } else
       *p++ = *s++;
   } while (*s);
   *p = XML_T('\0');
@@ -5906,12 +5647,11 @@ normalizeLines(XML_Char *s)
 
 static int
 reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
-                            const char *start, const char *end)
-{
+                            const char *start, const char *end) {
   const XML_Char *target;
   XML_Char *data;
   const char *tem;
-  if (!parser->m_processingInstructionHandler) {
+  if (! parser->m_processingInstructionHandler) {
     if (parser->m_defaultHandler)
       reportDefault(parser, enc, start, end);
     return 1;
@@ -5919,13 +5659,12 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
   start += enc->minBytesPerChar * 2;
   tem = start + XmlNameLength(enc, start);
   target = poolStoreString(&parser->m_tempPool, enc, start, tem);
-  if (!target)
+  if (! target)
     return 0;
   poolFinish(&parser->m_tempPool);
-  data = poolStoreString(&parser->m_tempPool, enc,
-                        XmlSkipS(enc, tem),
-                        end - enc->minBytesPerChar*2);
-  if (!data)
+  data = poolStoreString(&parser->m_tempPool, enc, XmlSkipS(enc, tem),
+                         end - enc->minBytesPerChar * 2);
+  if (! data)
     return 0;
   normalizeLines(data);
   parser->m_processingInstructionHandler(parser->m_handlerArg, target, data);
@@ -5934,20 +5673,18 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
 }
 
 static int
-reportComment(XML_Parser parser, const ENCODING *enc,
-              const char *start, const char *end)
-{
+reportComment(XML_Parser parser, const ENCODING *enc, const char *start,
+              const char *end) {
   XML_Char *data;
-  if (!parser->m_commentHandler) {
+  if (! parser->m_commentHandler) {
     if (parser->m_defaultHandler)
       reportDefault(parser, enc, start, end);
     return 1;
   }
-  data = poolStoreString(&parser->m_tempPool,
-                         enc,
+  data = poolStoreString(&parser->m_tempPool, enc,
                          start + enc->minBytesPerChar * 4,
                          end - enc->minBytesPerChar * 3);
-  if (!data)
+  if (! data)
     return 0;
   normalizeLines(data);
   parser->m_commentHandler(parser->m_handlerArg, data);
@@ -5956,9 +5693,8 @@ reportComment(XML_Parser parser, const ENCODING *enc,
 }
 
 static void
-reportDefault(XML_Parser parser, const ENCODING *enc,
-              const char *s, const char *end)
-{
+reportDefault(XML_Parser parser, const ENCODING *enc, const char *s,
+              const char *end) {
   if (MUST_CONVERT(enc, s)) {
     enum XML_Convert_Result convert_res;
     const char **eventPP;
@@ -5966,8 +5702,7 @@ reportDefault(XML_Parser parser, const ENCODING *enc,
     if (enc == parser->m_encoding) {
       eventPP = &parser->m_eventPtr;
       eventEndPP = &parser->m_eventEndPtr;
-    }
-    else {
+    } else {
       /* To get here, two things must be true; the parser must be
        * using a character encoding that is not the same as the
        * encoding passed in, and the encoding passed in must need
@@ -5990,21 +5725,22 @@ reportDefault(XML_Parser parser, const ENCODING *enc,
     }
     do {
       ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-      convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+      convert_res
+          = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
       *eventEndPP = s;
-      parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+      parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf,
+                               (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
       *eventPP = s;
-    } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE));
-  }
-  else
-    parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s, (int)((XML_Char *)end - (XML_Char *)s));
+    } while ((convert_res != XML_CONVERT_COMPLETED)
+             && (convert_res != XML_CONVERT_INPUT_INCOMPLETE));
+  } else
+    parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s,
+                             (int)((XML_Char *)end - (XML_Char *)s));
 }
 
-
 static int
 defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
-                XML_Bool isId, const XML_Char *value, XML_Parser parser)
-{
+                XML_Bool isId, const XML_Char *value, XML_Parser parser) {
   DEFAULT_ATTRIBUTE *att;
   if (value || isId) {
     /* The handling of default attributes gets messed up if we have
@@ -6013,24 +5749,23 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
     for (i = 0; i < type->nDefaultAtts; i++)
       if (attId == type->defaultAtts[i].id)
         return 1;
-    if (isId && !type->idAtt && !attId->xmlns)
+    if (isId && ! type->idAtt && ! attId->xmlns)
       type->idAtt = attId;
   }
   if (type->nDefaultAtts == type->allocDefaultAtts) {
     if (type->allocDefaultAtts == 0) {
       type->allocDefaultAtts = 8;
-      type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC(parser, type->allocDefaultAtts
-                            * sizeof(DEFAULT_ATTRIBUTE));
-      if (!type->defaultAtts) {
+      type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC(
+          parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
+      if (! type->defaultAtts) {
         type->allocDefaultAtts = 0;
         return 0;
       }
-    }
-    else {
+    } else {
       DEFAULT_ATTRIBUTE *temp;
       int count = type->allocDefaultAtts * 2;
-      temp = (DEFAULT_ATTRIBUTE *)
-        REALLOC(parser, type->defaultAtts, (count * sizeof(DEFAULT_ATTRIBUTE)));
+      temp = (DEFAULT_ATTRIBUTE *)REALLOC(parser, type->defaultAtts,
+                                          (count * sizeof(DEFAULT_ATTRIBUTE)));
       if (temp == NULL)
         return 0;
       type->allocDefaultAtts = count;
@@ -6041,30 +5776,29 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
   att->id = attId;
   att->value = value;
   att->isCdata = isCdata;
-  if (!isCdata)
+  if (! isCdata)
     attId->maybeTokenized = XML_TRUE;
   type->nDefaultAtts += 1;
   return 1;
 }
 
 static int
-setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *name;
   for (name = elementType->name; *name; name++) {
     if (*name == XML_T(ASCII_COLON)) {
       PREFIX *prefix;
       const XML_Char *s;
       for (s = elementType->name; s != name; s++) {
-        if (!poolAppendChar(&dtd->pool, *s))
+        if (! poolAppendChar(&dtd->pool, *s))
           return 0;
       }
-      if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+      if (! poolAppendChar(&dtd->pool, XML_T('\0')))
         return 0;
       prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&dtd->pool),
                                 sizeof(PREFIX));
-      if (!prefix)
+      if (! prefix)
         return 0;
       if (prefix->name == poolStart(&dtd->pool))
         poolFinish(&dtd->pool);
@@ -6078,55 +5812,53 @@ setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType)
 }
 
 static ATTRIBUTE_ID *
-getAttributeId(XML_Parser parser, const ENCODING *enc,
-               const char *start, const char *end)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start,
+               const char *end) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   ATTRIBUTE_ID *id;
   const XML_Char *name;
-  if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+  if (! poolAppendChar(&dtd->pool, XML_T('\0')))
     return NULL;
   name = poolStoreString(&dtd->pool, enc, start, end);
-  if (!name)
+  if (! name)
     return NULL;
   /* skip quotation mark - its storage will be re-used (like in name[-1]) */
   ++name;
-  id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name, sizeof(ATTRIBUTE_ID));
-  if (!id)
+  id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name,
+                              sizeof(ATTRIBUTE_ID));
+  if (! id)
     return NULL;
   if (id->name != name)
     poolDiscard(&dtd->pool);
   else {
     poolFinish(&dtd->pool);
-    if (!parser->m_ns)
+    if (! parser->m_ns)
       ;
-    else if (name[0] == XML_T(ASCII_x)
-        && name[1] == XML_T(ASCII_m)
-        && name[2] == XML_T(ASCII_l)
-        && name[3] == XML_T(ASCII_n)
-        && name[4] == XML_T(ASCII_s)
-        && (name[5] == XML_T('\0') || name[5] == XML_T(ASCII_COLON))) {
+    else if (name[0] == XML_T(ASCII_x) && name[1] == XML_T(ASCII_m)
+             && name[2] == XML_T(ASCII_l) && name[3] == XML_T(ASCII_n)
+             && name[4] == XML_T(ASCII_s)
+             && (name[5] == XML_T('\0') || name[5] == XML_T(ASCII_COLON))) {
       if (name[5] == XML_T('\0'))
         id->prefix = &dtd->defaultPrefix;
       else
-        id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6, sizeof(PREFIX));
+        id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6,
+                                      sizeof(PREFIX));
       id->xmlns = XML_TRUE;
-    }
-    else {
+    } else {
       int i;
       for (i = 0; name[i]; i++) {
         /* attributes without prefix are *not* in the default namespace */
         if (name[i] == XML_T(ASCII_COLON)) {
           int j;
           for (j = 0; j < i; j++) {
-            if (!poolAppendChar(&dtd->pool, name[j]))
+            if (! poolAppendChar(&dtd->pool, name[j]))
               return NULL;
           }
-          if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+          if (! poolAppendChar(&dtd->pool, XML_T('\0')))
             return NULL;
-          id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&dtd->pool),
-                                        sizeof(PREFIX));
-          if (!id->prefix)
+          id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes,
+                                        poolStart(&dtd->pool), sizeof(PREFIX));
+          if (! id->prefix)
             return NULL;
           if (id->prefix->name == poolStart(&dtd->pool))
             poolFinish(&dtd->pool);
@@ -6143,22 +5875,22 @@ getAttributeId(XML_Parser parser, const ENCODING *enc,
 #define CONTEXT_SEP XML_T(ASCII_FF)
 
 static const XML_Char *
-getContext(XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getContext(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   HASH_TABLE_ITER iter;
   XML_Bool needSep = XML_FALSE;
 
   if (dtd->defaultPrefix.binding) {
     int i;
     int len;
-    if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
+    if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
       return NULL;
     len = dtd->defaultPrefix.binding->uriLen;
     if (parser->m_namespaceSeparator)
       len--;
     for (i = 0; i < len; i++) {
-      if (!poolAppendChar(&parser->m_tempPool, dtd->defaultPrefix.binding->uri[i])) {
+      if (! poolAppendChar(&parser->m_tempPool,
+                           dtd->defaultPrefix.binding->uri[i])) {
         /* Because of memory caching, I don't believe this line can be
          * executed.
          *
@@ -6190,9 +5922,9 @@ getContext(XML_Parser parser)
     int len;
     const XML_Char *s;
     PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter);
-    if (!prefix)
+    if (! prefix)
       break;
-    if (!prefix->binding) {
+    if (! prefix->binding) {
       /* This test appears to be (justifiable) paranoia.  There does
        * not seem to be a way of injecting a prefix without a binding
        * that doesn't get errored long before this function is called.
@@ -6201,98 +5933,96 @@ getContext(XML_Parser parser)
        */
       continue; /* LCOV_EXCL_LINE */
     }
-    if (needSep && !poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
+    if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
       return NULL;
     for (s = prefix->name; *s; s++)
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return NULL;
-    if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
+    if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
       return NULL;
     len = prefix->binding->uriLen;
     if (parser->m_namespaceSeparator)
       len--;
     for (i = 0; i < len; i++)
-      if (!poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i]))
+      if (! poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i]))
         return NULL;
     needSep = XML_TRUE;
   }
 
-
   hashTableIterInit(&iter, &(dtd->generalEntities));
   for (;;) {
     const XML_Char *s;
     ENTITY *e = (ENTITY *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
-    if (!e->open)
+    if (! e->open)
       continue;
-    if (needSep && !poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
+    if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
       return NULL;
     for (s = e->name; *s; s++)
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return 0;
     needSep = XML_TRUE;
   }
 
-  if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+  if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
     return NULL;
   return parser->m_tempPool.start;
 }
 
 static XML_Bool
-setContext(XML_Parser parser, const XML_Char *context)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+setContext(XML_Parser parser, const XML_Char *context) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *s = context;
 
   while (*context != XML_T('\0')) {
     if (*s == CONTEXT_SEP || *s == XML_T('\0')) {
       ENTITY *e;
-      if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+      if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
         return XML_FALSE;
-      e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0);
+      e = (ENTITY *)lookup(parser, &dtd->generalEntities,
+                           poolStart(&parser->m_tempPool), 0);
       if (e)
         e->open = XML_TRUE;
       if (*s != XML_T('\0'))
         s++;
       context = s;
       poolDiscard(&parser->m_tempPool);
-    }
-    else if (*s == XML_T(ASCII_EQUALS)) {
+    } else if (*s == XML_T(ASCII_EQUALS)) {
       PREFIX *prefix;
       if (poolLength(&parser->m_tempPool) == 0)
         prefix = &dtd->defaultPrefix;
       else {
-        if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+        if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
           return XML_FALSE;
-        prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&parser->m_tempPool),
-                                  sizeof(PREFIX));
-        if (!prefix)
+        prefix
+            = (PREFIX *)lookup(parser, &dtd->prefixes,
+                               poolStart(&parser->m_tempPool), sizeof(PREFIX));
+        if (! prefix)
           return XML_FALSE;
         if (prefix->name == poolStart(&parser->m_tempPool)) {
           prefix->name = poolCopyString(&dtd->pool, prefix->name);
-          if (!prefix->name)
+          if (! prefix->name)
             return XML_FALSE;
         }
         poolDiscard(&parser->m_tempPool);
       }
-      for (context = s + 1;
-           *context != CONTEXT_SEP && *context != XML_T('\0');
+      for (context = s + 1; *context != CONTEXT_SEP && *context != XML_T('\0');
            context++)
-        if (!poolAppendChar(&parser->m_tempPool, *context))
+        if (! poolAppendChar(&parser->m_tempPool, *context))
           return XML_FALSE;
-      if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+      if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
         return XML_FALSE;
       if (addBinding(parser, prefix, NULL, poolStart(&parser->m_tempPool),
-                     &parser->m_inheritedBindings) != XML_ERROR_NONE)
+                     &parser->m_inheritedBindings)
+          != XML_ERROR_NONE)
         return XML_FALSE;
       poolDiscard(&parser->m_tempPool);
       if (*context != XML_T('\0'))
         ++context;
       s = context;
-    }
-    else {
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+    } else {
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return XML_FALSE;
       s++;
     }
@@ -6301,8 +6031,7 @@ setContext(XML_Parser parser, const XML_Char *context)
 }
 
 static void FASTCALL
-normalizePublicId(XML_Char *publicId)
-{
+normalizePublicId(XML_Char *publicId) {
   XML_Char *p = publicId;
   XML_Char *s;
   for (s = publicId; *s; s++) {
@@ -6323,8 +6052,7 @@ normalizePublicId(XML_Char *publicId)
 }
 
 static DTD *
-dtdCreate(const XML_Memory_Handling_Suite *ms)
-{
+dtdCreate(const XML_Memory_Handling_Suite *ms) {
   DTD *p = (DTD *)ms->malloc_fcn(sizeof(DTD));
   if (p == NULL)
     return p;
@@ -6356,13 +6084,12 @@ dtdCreate(const XML_Memory_Handling_Suite *ms)
 }
 
 static void
-dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms)
-{
+dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
   hashTableIterInit(&iter, &(p->elementTypes));
   for (;;) {
     ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
     if (e->allocDefaultAtts != 0)
       ms->free_fcn(e->defaultAtts);
@@ -6398,13 +6125,12 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms)
 }
 
 static void
-dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms)
-{
+dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
   hashTableIterInit(&iter, &(p->elementTypes));
   for (;;) {
     ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
     if (e->allocDefaultAtts != 0)
       ms->free_fcn(e->defaultAtts);
@@ -6429,8 +6155,8 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms)
    The new DTD has already been initialized.
 */
 static int
-dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms)
-{
+dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
+        const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
 
   /* Copy the prefix table. */
@@ -6439,12 +6165,12 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
   for (;;) {
     const XML_Char *name;
     const PREFIX *oldP = (PREFIX *)hashTableIterNext(&iter);
-    if (!oldP)
+    if (! oldP)
       break;
     name = poolCopyString(&(newDtd->pool), oldP->name);
-    if (!name)
+    if (! name)
       return 0;
-    if (!lookup(oldParser, &(newDtd->prefixes), name, sizeof(PREFIX)))
+    if (! lookup(oldParser, &(newDtd->prefixes), name, sizeof(PREFIX)))
       return 0;
   }
 
@@ -6457,18 +6183,18 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
     const XML_Char *name;
     const ATTRIBUTE_ID *oldA = (ATTRIBUTE_ID *)hashTableIterNext(&iter);
 
-    if (!oldA)
+    if (! oldA)
       break;
     /* Remember to allocate the scratch byte before the name. */
-    if (!poolAppendChar(&(newDtd->pool), XML_T('\0')))
+    if (! poolAppendChar(&(newDtd->pool), XML_T('\0')))
       return 0;
     name = poolCopyString(&(newDtd->pool), oldA->name);
-    if (!name)
+    if (! name)
       return 0;
     ++name;
     newA = (ATTRIBUTE_ID *)lookup(oldParser, &(newDtd->attributeIds), name,
                                   sizeof(ATTRIBUTE_ID));
-    if (!newA)
+    if (! newA)
       return 0;
     newA->maybeTokenized = oldA->maybeTokenized;
     if (oldA->prefix) {
@@ -6490,57 +6216,52 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
     ELEMENT_TYPE *newE;
     const XML_Char *name;
     const ELEMENT_TYPE *oldE = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!oldE)
+    if (! oldE)
       break;
     name = poolCopyString(&(newDtd->pool), oldE->name);
-    if (!name)
+    if (! name)
       return 0;
     newE = (ELEMENT_TYPE *)lookup(oldParser, &(newDtd->elementTypes), name,
                                   sizeof(ELEMENT_TYPE));
-    if (!newE)
+    if (! newE)
       return 0;
     if (oldE->nDefaultAtts) {
-      newE->defaultAtts = (DEFAULT_ATTRIBUTE *)
-          ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
-      if (!newE->defaultAtts) {
+      newE->defaultAtts = (DEFAULT_ATTRIBUTE *)ms->malloc_fcn(
+          oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
+      if (! newE->defaultAtts) {
         return 0;
       }
     }
     if (oldE->idAtt)
-      newE->idAtt = (ATTRIBUTE_ID *)
-          lookup(oldParser, &(newDtd->attributeIds), oldE->idAtt->name, 0);
+      newE->idAtt = (ATTRIBUTE_ID *)lookup(oldParser, &(newDtd->attributeIds),
+                                           oldE->idAtt->name, 0);
     newE->allocDefaultAtts = newE->nDefaultAtts = oldE->nDefaultAtts;
     if (oldE->prefix)
       newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes),
                                       oldE->prefix->name, 0);
     for (i = 0; i < newE->nDefaultAtts; i++) {
-      newE->defaultAtts[i].id = (ATTRIBUTE_ID *)
-          lookup(oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0);
+      newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup(
+          oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0);
       newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata;
       if (oldE->defaultAtts[i].value) {
         newE->defaultAtts[i].value
             = poolCopyString(&(newDtd->pool), oldE->defaultAtts[i].value);
-        if (!newE->defaultAtts[i].value)
+        if (! newE->defaultAtts[i].value)
           return 0;
-      }
-      else
+      } else
         newE->defaultAtts[i].value = NULL;
     }
   }
 
   /* Copy the entity tables. */
-  if (!copyEntityTable(oldParser,
-                       &(newDtd->generalEntities),
-                       &(newDtd->pool),
-                       &(oldDtd->generalEntities)))
-      return 0;
+  if (! copyEntityTable(oldParser, &(newDtd->generalEntities), &(newDtd->pool),
+                        &(oldDtd->generalEntities)))
+    return 0;
 
 #ifdef XML_DTD
-  if (!copyEntityTable(oldParser,
-                       &(newDtd->paramEntities),
-                       &(newDtd->pool),
-                       &(oldDtd->paramEntities)))
-      return 0;
+  if (! copyEntityTable(oldParser, &(newDtd->paramEntities), &(newDtd->pool),
+                        &(oldDtd->paramEntities)))
+    return 0;
   newDtd->paramEntityRead = oldDtd->paramEntityRead;
 #endif /* XML_DTD */
 
@@ -6557,14 +6278,11 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
   newDtd->scaffIndex = oldDtd->scaffIndex;
 
   return 1;
-}  /* End dtdCopy */
+} /* End dtdCopy */
 
 static int
-copyEntityTable(XML_Parser oldParser,
-                HASH_TABLE *newTable,
-                STRING_POOL *newPool,
-                const HASH_TABLE *oldTable)
-{
+copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable,
+                STRING_POOL *newPool, const HASH_TABLE *oldTable) {
   HASH_TABLE_ITER iter;
   const XML_Char *cachedOldBase = NULL;
   const XML_Char *cachedNewBase = NULL;
@@ -6575,17 +6293,17 @@ copyEntityTable(XML_Parser oldParser,
     ENTITY *newE;
     const XML_Char *name;
     const ENTITY *oldE = (ENTITY *)hashTableIterNext(&iter);
-    if (!oldE)
+    if (! oldE)
       break;
     name = poolCopyString(newPool, oldE->name);
-    if (!name)
+    if (! name)
       return 0;
     newE = (ENTITY *)lookup(oldParser, newTable, name, sizeof(ENTITY));
-    if (!newE)
+    if (! newE)
       return 0;
     if (oldE->systemId) {
       const XML_Char *tem = poolCopyString(newPool, oldE->systemId);
-      if (!tem)
+      if (! tem)
         return 0;
       newE->systemId = tem;
       if (oldE->base) {
@@ -6594,29 +6312,28 @@ copyEntityTable(XML_Parser oldParser,
         else {
           cachedOldBase = oldE->base;
           tem = poolCopyString(newPool, cachedOldBase);
-          if (!tem)
+          if (! tem)
             return 0;
           cachedNewBase = newE->base = tem;
         }
       }
       if (oldE->publicId) {
         tem = poolCopyString(newPool, oldE->publicId);
-        if (!tem)
+        if (! tem)
           return 0;
         newE->publicId = tem;
       }
-    }
-    else {
-      const XML_Char *tem = poolCopyStringN(newPool, oldE->textPtr,
-                                            oldE->textLen);
-      if (!tem)
+    } else {
+      const XML_Char *tem
+          = poolCopyStringN(newPool, oldE->textPtr, oldE->textLen);
+      if (! tem)
         return 0;
       newE->textPtr = tem;
       newE->textLen = oldE->textLen;
     }
     if (oldE->notation) {
       const XML_Char *tem = poolCopyString(newPool, oldE->notation);
-      if (!tem)
+      if (! tem)
         return 0;
       newE->notation = tem;
     }
@@ -6629,8 +6346,7 @@ copyEntityTable(XML_Parser oldParser,
 #define INIT_POWER 6
 
 static XML_Bool FASTCALL
-keyeq(KEY s1, KEY s2)
-{
+keyeq(KEY s1, KEY s2) {
   for (; *s1 == *s2; s1++, s2++)
     if (*s1 == 0)
       return XML_TRUE;
@@ -6638,23 +6354,21 @@ keyeq(KEY s1, KEY s2)
 }
 
 static size_t
-keylen(KEY s)
-{
+keylen(KEY s) {
   size_t len = 0;
-  for (; *s; s++, len++);
+  for (; *s; s++, len++)
+    ;
   return len;
 }
 
 static void
-copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key)
-{
+copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) {
   key->k[0] = 0;
   key->k[1] = get_hash_secret_salt(parser);
 }
 
 static unsigned long FASTCALL
-hash(XML_Parser parser, KEY s)
-{
+hash(XML_Parser parser, KEY s) {
   struct siphash state;
   struct sipkey key;
   (void)sip24_valid;
@@ -6665,26 +6379,24 @@ hash(XML_Parser parser, KEY s)
 }
 
 static NAMED *
-lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
-{
+lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) {
   size_t i;
   if (table->size == 0) {
     size_t tsize;
-    if (!createSize)
+    if (! createSize)
       return NULL;
     table->power = INIT_POWER;
     /* table->size is a power of 2 */
     table->size = (size_t)1 << INIT_POWER;
     tsize = table->size * sizeof(NAMED *);
     table->v = (NAMED **)table->mem->malloc_fcn(tsize);
-    if (!table->v) {
+    if (! table->v) {
       table->size = 0;
       return NULL;
     }
     memset(table->v, 0, tsize);
     i = hash(parser, name) & ((unsigned long)table->size - 1);
-  }
-  else {
+  } else {
     unsigned long h = hash(parser, name);
     unsigned long mask = (unsigned long)table->size - 1;
     unsigned char step = 0;
@@ -6692,11 +6404,11 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
     while (table->v[i]) {
       if (keyeq(name, table->v[i]->name))
         return table->v[i];
-      if (!step)
+      if (! step)
         step = PROBE_STEP(h, mask, table->power);
       i < step ? (i += table->size - step) : (i -= step);
     }
-    if (!createSize)
+    if (! createSize)
       return NULL;
 
     /* check for overflow (table is half full) */
@@ -6706,7 +6418,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
       unsigned long newMask = (unsigned long)newSize - 1;
       size_t tsize = newSize * sizeof(NAMED *);
       NAMED **newV = (NAMED **)table->mem->malloc_fcn(tsize);
-      if (!newV)
+      if (! newV)
         return NULL;
       memset(newV, 0, tsize);
       for (i = 0; i < table->size; i++)
@@ -6715,7 +6427,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
           size_t j = newHash & newMask;
           step = 0;
           while (newV[j]) {
-            if (!step)
+            if (! step)
               step = PROBE_STEP(newHash, newMask, newPower);
             j < step ? (j += newSize - step) : (j -= step);
           }
@@ -6728,14 +6440,14 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
       i = h & newMask;
       step = 0;
       while (table->v[i]) {
-        if (!step)
+        if (! step)
           step = PROBE_STEP(h, newMask, newPower);
         i < step ? (i += newSize - step) : (i -= step);
       }
     }
   }
   table->v[i] = (NAMED *)table->mem->malloc_fcn(createSize);
-  if (!table->v[i])
+  if (! table->v[i])
     return NULL;
   memset(table->v[i], 0, createSize);
   table->v[i]->name = name;
@@ -6744,8 +6456,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
 }
 
 static void FASTCALL
-hashTableClear(HASH_TABLE *table)
-{
+hashTableClear(HASH_TABLE *table) {
   size_t i;
   for (i = 0; i < table->size; i++) {
     table->mem->free_fcn(table->v[i]);
@@ -6755,8 +6466,7 @@ hashTableClear(HASH_TABLE *table)
 }
 
 static void FASTCALL
-hashTableDestroy(HASH_TABLE *table)
-{
+hashTableDestroy(HASH_TABLE *table) {
   size_t i;
   for (i = 0; i < table->size; i++)
     table->mem->free_fcn(table->v[i]);
@@ -6764,8 +6474,7 @@ hashTableDestroy(HASH_TABLE *table)
 }
 
 static void FASTCALL
-hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms)
-{
+hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) {
   p->power = 0;
   p->size = 0;
   p->used = 0;
@@ -6774,15 +6483,13 @@ hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms)
 }
 
 static void FASTCALL
-hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table)
-{
+hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) {
   iter->p = table->v;
   iter->end = iter->p + table->size;
 }
 
-static NAMED * FASTCALL
-hashTableIterNext(HASH_TABLE_ITER *iter)
-{
+static NAMED *FASTCALL
+hashTableIterNext(HASH_TABLE_ITER *iter) {
   while (iter->p != iter->end) {
     NAMED *tem = *(iter->p)++;
     if (tem)
@@ -6792,8 +6499,7 @@ hashTableIterNext(HASH_TABLE_ITER *iter)
 }
 
 static void FASTCALL
-poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms)
-{
+poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms) {
   pool->blocks = NULL;
   pool->freeBlocks = NULL;
   pool->start = NULL;
@@ -6803,9 +6509,8 @@ poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms)
 }
 
 static void FASTCALL
-poolClear(STRING_POOL *pool)
-{
-  if (!pool->freeBlocks)
+poolClear(STRING_POOL *pool) {
+  if (! pool->freeBlocks)
     pool->freeBlocks = pool->blocks;
   else {
     BLOCK *p = pool->blocks;
@@ -6823,8 +6528,7 @@ poolClear(STRING_POOL *pool)
 }
 
 static void FASTCALL
-poolDestroy(STRING_POOL *pool)
-{
+poolDestroy(STRING_POOL *pool) {
   BLOCK *p = pool->blocks;
   while (p) {
     BLOCK *tem = p->next;
@@ -6840,26 +6544,26 @@ poolDestroy(STRING_POOL *pool)
 }
 
 static XML_Char *
-poolAppend(STRING_POOL *pool, const ENCODING *enc,
-           const char *ptr, const char *end)
-{
-  if (!pool->ptr && !poolGrow(pool))
+poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr,
+           const char *end) {
+  if (! pool->ptr && ! poolGrow(pool))
     return NULL;
   for (;;) {
-    const enum XML_Convert_Result convert_res = XmlConvert(enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end);
-    if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+    const enum XML_Convert_Result convert_res = XmlConvert(
+        enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end);
+    if ((convert_res == XML_CONVERT_COMPLETED)
+        || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
       break;
-    if (!poolGrow(pool))
+    if (! poolGrow(pool))
       return NULL;
   }
   return pool->start;
 }
 
-static const XML_Char * FASTCALL
-poolCopyString(STRING_POOL *pool, const XML_Char *s)
-{
+static const XML_Char *FASTCALL
+poolCopyString(STRING_POOL *pool, const XML_Char *s) {
   do {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
   } while (*s++);
   s = pool->start;
@@ -6868,9 +6572,8 @@ poolCopyString(STRING_POOL *pool, const XML_Char *s)
 }
 
 static const XML_Char *
-poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
-{
-  if (!pool->ptr && !poolGrow(pool)) {
+poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) {
+  if (! pool->ptr && ! poolGrow(pool)) {
     /* The following line is unreachable given the current usage of
      * poolCopyStringN().  Currently it is called from exactly one
      * place to copy the text of a simple general entity.  By that
@@ -6885,7 +6588,7 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
     return NULL; /* LCOV_EXCL_LINE */
   }
   for (; n > 0; --n, s++) {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
   }
   s = pool->start;
@@ -6893,11 +6596,10 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
   return s;
 }
 
-static const XML_Char * FASTCALL
-poolAppendString(STRING_POOL *pool, const XML_Char *s)
-{
+static const XML_Char *FASTCALL
+poolAppendString(STRING_POOL *pool, const XML_Char *s) {
   while (*s) {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
     s++;
   }
@@ -6905,20 +6607,18 @@ poolAppendString(STRING_POOL *pool, const XML_Char *s)
 }
 
 static XML_Char *
-poolStoreString(STRING_POOL *pool, const ENCODING *enc,
-                const char *ptr, const char *end)
-{
-  if (!poolAppend(pool, enc, ptr, end))
+poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr,
+                const char *end) {
+  if (! poolAppend(pool, enc, ptr, end))
     return NULL;
-  if (pool->ptr == pool->end && !poolGrow(pool))
+  if (pool->ptr == pool->end && ! poolGrow(pool))
     return NULL;
   *(pool->ptr)++ = 0;
   return pool->start;
 }
 
 static size_t
-poolBytesToAllocateFor(int blockSize)
-{
+poolBytesToAllocateFor(int blockSize) {
   /* Unprotected math would be:
   ** return offsetof(BLOCK, s) + blockSize * sizeof(XML_Char);
   **
@@ -6926,7 +6626,7 @@ poolBytesToAllocateFor(int blockSize)
   ** For a + b * c we check b * c in isolation first, so that addition of a
   ** on top has no chance of making us accept a small non-negative number
   */
-  const size_t stretch = sizeof(XML_Char);  /* can be 4 bytes */
+  const size_t stretch = sizeof(XML_Char); /* can be 4 bytes */
 
   if (blockSize <= 0)
     return 0;
@@ -6936,8 +6636,8 @@ poolBytesToAllocateFor(int blockSize)
 
   {
     const int stretchedBlockSize = blockSize * (int)stretch;
-    const int bytesToAllocate = (int)(
-        offsetof(BLOCK, s) + (unsigned)stretchedBlockSize);
+    const int bytesToAllocate
+        = (int)(offsetof(BLOCK, s) + (unsigned)stretchedBlockSize);
     if (bytesToAllocate < 0)
       return 0;
 
@@ -6946,8 +6646,7 @@ poolBytesToAllocateFor(int blockSize)
 }
 
 static XML_Bool FASTCALL
-poolGrow(STRING_POOL *pool)
-{
+poolGrow(STRING_POOL *pool) {
   if (pool->freeBlocks) {
     if (pool->start == 0) {
       pool->blocks = pool->freeBlocks;
@@ -6973,7 +6672,7 @@ poolGrow(STRING_POOL *pool)
   }
   if (pool->blocks && pool->start == pool->blocks->s) {
     BLOCK *temp;
-    int blockSize = (int)((unsigned)(pool->end - pool->start)*2U);
+    int blockSize = (int)((unsigned)(pool->end - pool->start) * 2U);
     size_t bytesToAllocate;
 
     /* NOTE: Needs to be calculated prior to calling `realloc`
@@ -6994,8 +6693,8 @@ poolGrow(STRING_POOL *pool)
     if (bytesToAllocate == 0)
       return XML_FALSE;
 
-    temp = (BLOCK *)
-      pool->mem->realloc_fcn(pool->blocks, (unsigned)bytesToAllocate);
+    temp = (BLOCK *)pool->mem->realloc_fcn(pool->blocks,
+                                           (unsigned)bytesToAllocate);
     if (temp == NULL)
       return XML_FALSE;
     pool->blocks = temp;
@@ -7003,8 +6702,7 @@ poolGrow(STRING_POOL *pool)
     pool->ptr = pool->blocks->s + offsetInsideBlock;
     pool->start = pool->blocks->s;
     pool->end = pool->start + blockSize;
-  }
-  else {
+  } else {
     BLOCK *tem;
     int blockSize = (int)(pool->end - pool->start);
     size_t bytesToAllocate;
@@ -7019,7 +6717,7 @@ poolGrow(STRING_POOL *pool)
        * function).  Either way it isn't readily testable, so we
        * exclude it from the coverage statistics.
        */
-      return XML_FALSE;  /* LCOV_EXCL_LINE */
+      return XML_FALSE; /* LCOV_EXCL_LINE */
     }
 
     if (blockSize < INIT_BLOCK_SIZE)
@@ -7037,14 +6735,13 @@ poolGrow(STRING_POOL *pool)
       return XML_FALSE;
 
     tem = (BLOCK *)pool->mem->malloc_fcn(bytesToAllocate);
-    if (!tem)
+    if (! tem)
       return XML_FALSE;
     tem->size = blockSize;
     tem->next = pool->blocks;
     pool->blocks = tem;
     if (pool->ptr != pool->start)
-      memcpy(tem->s, pool->start,
-             (pool->ptr - pool->start) * sizeof(XML_Char));
+      memcpy(tem->s, pool->start, (pool->ptr - pool->start) * sizeof(XML_Char));
     pool->ptr = tem->s + (pool->ptr - pool->start);
     pool->start = tem->s;
     pool->end = tem->s + blockSize;
@@ -7053,15 +6750,14 @@ poolGrow(STRING_POOL *pool)
 }
 
 static int FASTCALL
-nextScaffoldPart(XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
-  CONTENT_SCAFFOLD * me;
+nextScaffoldPart(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
+  CONTENT_SCAFFOLD *me;
   int next;
 
-  if (!dtd->scaffIndex) {
+  if (! dtd->scaffIndex) {
     dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int));
-    if (!dtd->scaffIndex)
+    if (! dtd->scaffIndex)
       return -1;
     dtd->scaffIndex[0] = 0;
   }
@@ -7069,15 +6765,14 @@ nextScaffoldPart(XML_Parser parser)
   if (dtd->scaffCount >= dtd->scaffSize) {
     CONTENT_SCAFFOLD *temp;
     if (dtd->scaffold) {
-      temp = (CONTENT_SCAFFOLD *)
-        REALLOC(parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD));
+      temp = (CONTENT_SCAFFOLD *)REALLOC(
+          parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD));
       if (temp == NULL)
         return -1;
       dtd->scaffSize *= 2;
-    }
-    else {
+    } else {
       temp = (CONTENT_SCAFFOLD *)MALLOC(parser, INIT_SCAFFOLD_ELEMENTS
-                                        * sizeof(CONTENT_SCAFFOLD));
+                                                    * sizeof(CONTENT_SCAFFOLD));
       if (temp == NULL)
         return -1;
       dtd->scaffSize = INIT_SCAFFOLD_ELEMENTS;
@@ -7087,11 +6782,12 @@ nextScaffoldPart(XML_Parser parser)
   next = dtd->scaffCount++;
   me = &dtd->scaffold[next];
   if (dtd->scaffLevel) {
-    CONTENT_SCAFFOLD *parent = &dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel-1]];
+    CONTENT_SCAFFOLD *parent
+        = &dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]];
     if (parent->lastchild) {
       dtd->scaffold[parent->lastchild].nextsib = next;
     }
-    if (!parent->childcnt)
+    if (! parent->childcnt)
       parent->firstchild = next;
     parent->lastchild = next;
     parent->childcnt++;
@@ -7101,13 +6797,9 @@ nextScaffoldPart(XML_Parser parser)
 }
 
 static void
-build_node(XML_Parser parser,
-           int src_node,
-           XML_Content *dest,
-           XML_Content **contpos,
-           XML_Char **strpos)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+build_node(XML_Parser parser, int src_node, XML_Content *dest,
+           XML_Content **contpos, XML_Char **strpos) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   dest->type = dtd->scaffold[src_node].type;
   dest->quant = dtd->scaffold[src_node].quant;
   if (dest->type == XML_CTYPE_NAME) {
@@ -7116,21 +6808,19 @@ build_node(XML_Parser parser,
     src = dtd->scaffold[src_node].name;
     for (;;) {
       *(*strpos)++ = *src;
-      if (!*src)
+      if (! *src)
         break;
       src++;
     }
     dest->numchildren = 0;
     dest->children = NULL;
-  }
-  else {
+  } else {
     unsigned int i;
     int cn;
     dest->numchildren = dtd->scaffold[src_node].childcnt;
     dest->children = *contpos;
     *contpos += dest->numchildren;
-    for (i = 0, cn = dtd->scaffold[src_node].firstchild;
-         i < dest->numchildren;
+    for (i = 0, cn = dtd->scaffold[src_node].firstchild; i < dest->numchildren;
          i++, cn = dtd->scaffold[cn].nextsib) {
       build_node(parser, cn, &(dest->children[i]), contpos, strpos);
     }
@@ -7139,20 +6829,19 @@ build_node(XML_Parser parser,
 }
 
 static XML_Content *
-build_model (XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+build_model(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   XML_Content *ret;
   XML_Content *cpos;
-  XML_Char * str;
+  XML_Char *str;
   int allocsize = (dtd->scaffCount * sizeof(XML_Content)
                    + (dtd->contentStringLen * sizeof(XML_Char)));
 
   ret = (XML_Content *)MALLOC(parser, allocsize);
-  if (!ret)
+  if (! ret)
     return NULL;
 
-  str =  (XML_Char *) (&ret[dtd->scaffCount]);
+  str = (XML_Char *)(&ret[dtd->scaffCount]);
   cpos = &ret[1];
 
   build_node(parser, 0, ret, &cpos, &str);
@@ -7160,49 +6849,45 @@ build_model (XML_Parser parser)
 }
 
 static ELEMENT_TYPE *
-getElementType(XML_Parser parser,
-               const ENCODING *enc,
-               const char *ptr,
-               const char *end)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr,
+               const char *end) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *name = poolStoreString(&dtd->pool, enc, ptr, end);
   ELEMENT_TYPE *ret;
 
-  if (!name)
+  if (! name)
     return NULL;
-  ret = (ELEMENT_TYPE *) lookup(parser, &dtd->elementTypes, name, sizeof(ELEMENT_TYPE));
-  if (!ret)
+  ret = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, name,
+                               sizeof(ELEMENT_TYPE));
+  if (! ret)
     return NULL;
   if (ret->name != name)
     poolDiscard(&dtd->pool);
   else {
     poolFinish(&dtd->pool);
-    if (!setElementTypePrefix(parser, ret))
+    if (! setElementTypePrefix(parser, ret))
       return NULL;
   }
   return ret;
 }
 
 static XML_Char *
-copyString(const XML_Char *s,
-           const XML_Memory_Handling_Suite *memsuite)
-{
-    int charsRequired = 0;
-    XML_Char *result;
-
-    /* First determine how long the string is */
-    while (s[charsRequired] != 0) {
-      charsRequired++;
-    }
-    /* Include the terminator */
+copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) {
+  int charsRequired = 0;
+  XML_Char *result;
+
+  /* First determine how long the string is */
+  while (s[charsRequired] != 0) {
     charsRequired++;
+  }
+  /* Include the terminator */
+  charsRequired++;
 
-    /* Now allocate space for the copy */
-    result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char));
-    if (result == NULL)
-        return NULL;
-    /* Copy the original into place */
-    memcpy(result, s, charsRequired * sizeof(XML_Char));
-    return result;
+  /* Now allocate space for the copy */
+  result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char));
+  if (result == NULL)
+    return NULL;
+  /* Copy the original into place */
+  memcpy(result, s, charsRequired * sizeof(XML_Char));
+  return result;
 }
diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c
index 708507d575be..4d3e3e86e9e8 100644
--- a/Modules/expat/xmlrole.c
+++ b/Modules/expat/xmlrole.c
@@ -33,11 +33,11 @@
 #include <stddef.h>
 
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #else
-#ifdef HAVE_EXPAT_CONFIG_H
-#include <expat_config.h>
-#endif
+#  ifdef HAVE_EXPAT_CONFIG_H
+#    include <expat_config.h>
+#  endif
 #endif /* ndef _WIN32 */
 
 #include "expat_external.h"
@@ -52,107 +52,88 @@
 
 */
 
-static const char KW_ANY[] = {
-    ASCII_A, ASCII_N, ASCII_Y, '\0' };
-static const char KW_ATTLIST[] = {
-    ASCII_A, ASCII_T, ASCII_T, ASCII_L, ASCII_I, ASCII_S, ASCII_T, '\0' };
-static const char KW_CDATA[] = {
-    ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_DOCTYPE[] = {
-    ASCII_D, ASCII_O, ASCII_C, ASCII_T, ASCII_Y, ASCII_P, ASCII_E, '\0' };
-static const char KW_ELEMENT[] = {
-    ASCII_E, ASCII_L, ASCII_E, ASCII_M, ASCII_E, ASCII_N, ASCII_T, '\0' };
-static const char KW_EMPTY[] = {
-    ASCII_E, ASCII_M, ASCII_P, ASCII_T, ASCII_Y, '\0' };
-static const char KW_ENTITIES[] = {
-    ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_I, ASCII_E, ASCII_S,
-    '\0' };
-static const char KW_ENTITY[] = {
-    ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0' };
-static const char KW_FIXED[] = {
-    ASCII_F, ASCII_I, ASCII_X, ASCII_E, ASCII_D, '\0' };
-static const char KW_ID[] = {
-    ASCII_I, ASCII_D, '\0' };
-static const char KW_IDREF[] = {
-    ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0' };
-static const char KW_IDREFS[] = {
-    ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0' };
+static const char KW_ANY[] = {ASCII_A, ASCII_N, ASCII_Y, '\0'};
+static const char KW_ATTLIST[]
+    = {ASCII_A, ASCII_T, ASCII_T, ASCII_L, ASCII_I, ASCII_S, ASCII_T, '\0'};
+static const char KW_CDATA[]
+    = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_DOCTYPE[]
+    = {ASCII_D, ASCII_O, ASCII_C, ASCII_T, ASCII_Y, ASCII_P, ASCII_E, '\0'};
+static const char KW_ELEMENT[]
+    = {ASCII_E, ASCII_L, ASCII_E, ASCII_M, ASCII_E, ASCII_N, ASCII_T, '\0'};
+static const char KW_EMPTY[]
+    = {ASCII_E, ASCII_M, ASCII_P, ASCII_T, ASCII_Y, '\0'};
+static const char KW_ENTITIES[] = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T,
+                                   ASCII_I, ASCII_E, ASCII_S, '\0'};
+static const char KW_ENTITY[]
+    = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0'};
+static const char KW_FIXED[]
+    = {ASCII_F, ASCII_I, ASCII_X, ASCII_E, ASCII_D, '\0'};
+static const char KW_ID[] = {ASCII_I, ASCII_D, '\0'};
+static const char KW_IDREF[]
+    = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0'};
+static const char KW_IDREFS[]
+    = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0'};
 #ifdef XML_DTD
-static const char KW_IGNORE[] = {
-    ASCII_I, ASCII_G, ASCII_N, ASCII_O, ASCII_R, ASCII_E, '\0' };
+static const char KW_IGNORE[]
+    = {ASCII_I, ASCII_G, ASCII_N, ASCII_O, ASCII_R, ASCII_E, '\0'};
 #endif
-static const char KW_IMPLIED[] = {
-    ASCII_I, ASCII_M, ASCII_P, ASCII_L, ASCII_I, ASCII_E, ASCII_D, '\0' };
+static const char KW_IMPLIED[]
+    = {ASCII_I, ASCII_M, ASCII_P, ASCII_L, ASCII_I, ASCII_E, ASCII_D, '\0'};
 #ifdef XML_DTD
-static const char KW_INCLUDE[] = {
-    ASCII_I, ASCII_N, ASCII_C, ASCII_L, ASCII_U, ASCII_D, ASCII_E, '\0' };
+static const char KW_INCLUDE[]
+    = {ASCII_I, ASCII_N, ASCII_C, ASCII_L, ASCII_U, ASCII_D, ASCII_E, '\0'};
 #endif
-static const char KW_NDATA[] = {
-    ASCII_N, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_NMTOKEN[] = {
-    ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0' };
-static const char KW_NMTOKENS[] = {
-    ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, ASCII_S,
-    '\0' };
-static const char KW_NOTATION[] =
-    { ASCII_N, ASCII_O, ASCII_T, ASCII_A, ASCII_T, ASCII_I, ASCII_O, ASCII_N,
-      '\0' };
-static const char KW_PCDATA[] = {
-    ASCII_P, ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_PUBLIC[] = {
-    ASCII_P, ASCII_U, ASCII_B, ASCII_L, ASCII_I, ASCII_C, '\0' };
-static const char KW_REQUIRED[] = {
-    ASCII_R, ASCII_E, ASCII_Q, ASCII_U, ASCII_I, ASCII_R, ASCII_E, ASCII_D,
-    '\0' };
-static const char KW_SYSTEM[] = {
-    ASCII_S, ASCII_Y, ASCII_S, ASCII_T, ASCII_E, ASCII_M, '\0' };
+static const char KW_NDATA[]
+    = {ASCII_N, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_NMTOKEN[]
+    = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0'};
+static const char KW_NMTOKENS[] = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K,
+                                   ASCII_E, ASCII_N, ASCII_S, '\0'};
+static const char KW_NOTATION[] = {ASCII_N, ASCII_O, ASCII_T, ASCII_A, ASCII_T,
+                                   ASCII_I, ASCII_O, ASCII_N, '\0'};
+static const char KW_PCDATA[]
+    = {ASCII_P, ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_PUBLIC[]
+    = {ASCII_P, ASCII_U, ASCII_B, ASCII_L, ASCII_I, ASCII_C, '\0'};
+static const char KW_REQUIRED[] = {ASCII_R, ASCII_E, ASCII_Q, ASCII_U, ASCII_I,
+                                   ASCII_R, ASCII_E, ASCII_D, '\0'};
+static const char KW_SYSTEM[]
+    = {ASCII_S, ASCII_Y, ASCII_S, ASCII_T, ASCII_E, ASCII_M, '\0'};
 
 #ifndef MIN_BYTES_PER_CHAR
-#define MIN_BYTES_PER_CHAR(enc) ((enc)->minBytesPerChar)
+#  define MIN_BYTES_PER_CHAR(enc) ((enc)->minBytesPerChar)
 #endif
 
 #ifdef XML_DTD
-#define setTopLevel(state) \
-  ((state)->handler = ((state)->documentEntity \
-                       ? internalSubset \
-                       : externalSubset1))
+#  define setTopLevel(state)                                                   \
+    ((state)->handler                                                          \
+     = ((state)->documentEntity ? internalSubset : externalSubset1))
 #else /* not XML_DTD */
-#define setTopLevel(state) ((state)->handler = internalSubset)
+#  define setTopLevel(state) ((state)->handler = internalSubset)
 #endif /* not XML_DTD */
 
-typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state,
-                                   int tok,
-                                   const char *ptr,
-                                   const char *end,
+typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state, int tok,
+                                   const char *ptr, const char *end,
                                    const ENCODING *enc);
 
-static PROLOG_HANDLER
-  prolog0, prolog1, prolog2,
-  doctype0, doctype1, doctype2, doctype3, doctype4, doctype5,
-  internalSubset,
-  entity0, entity1, entity2, entity3, entity4, entity5, entity6,
-  entity7, entity8, entity9, entity10,
-  notation0, notation1, notation2, notation3, notation4,
-  attlist0, attlist1, attlist2, attlist3, attlist4, attlist5, attlist6,
-  attlist7, attlist8, attlist9,
-  element0, element1, element2, element3, element4, element5, element6,
-  element7,
+static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2,
+    doctype3, doctype4, doctype5, internalSubset, entity0, entity1, entity2,
+    entity3, entity4, entity5, entity6, entity7, entity8, entity9, entity10,
+    notation0, notation1, notation2, notation3, notation4, attlist0, attlist1,
+    attlist2, attlist3, attlist4, attlist5, attlist6, attlist7, attlist8,
+    attlist9, element0, element1, element2, element3, element4, element5,
+    element6, element7,
 #ifdef XML_DTD
-  externalSubset0, externalSubset1,
-  condSect0, condSect1, condSect2,
+    externalSubset0, externalSubset1, condSect0, condSect1, condSect2,
 #endif /* XML_DTD */
-  declClose,
-  error;
+    declClose, error;
 
 static int FASTCALL common(PROLOG_STATE *state, int tok);
 
 static int PTRCALL
-prolog0(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     state->handler = prolog1;
@@ -169,10 +150,8 @@ prolog0(PROLOG_STATE *state,
   case XML_TOK_BOM:
     return XML_ROLE_NONE;
   case XML_TOK_DECL_OPEN:
-    if (!XmlNameMatchesAscii(enc,
-                             ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                             end,
-                             KW_DOCTYPE))
+    if (! XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
+                              KW_DOCTYPE))
       break;
     state->handler = doctype0;
     return XML_ROLE_DOCTYPE_NONE;
@@ -184,12 +163,8 @@ prolog0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-prolog1(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -207,10 +182,8 @@ prolog1(PROLOG_STATE *state,
      */
     return XML_ROLE_NONE; /* LCOV_EXCL_LINE */
   case XML_TOK_DECL_OPEN:
-    if (!XmlNameMatchesAscii(enc,
-                             ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                             end,
-                             KW_DOCTYPE))
+    if (! XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
+                              KW_DOCTYPE))
       break;
     state->handler = doctype0;
     return XML_ROLE_DOCTYPE_NONE;
@@ -222,12 +195,11 @@ prolog1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-prolog2(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -243,12 +215,11 @@ prolog2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -261,12 +232,8 @@ doctype0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype1(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -291,12 +258,11 @@ doctype1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype2(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -308,12 +274,11 @@ doctype2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -325,12 +290,11 @@ doctype3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -345,12 +309,11 @@ doctype4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -362,40 +325,28 @@ doctype5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-internalSubset(PROLOG_STATE *state,
-               int tok,
-               const char *ptr,
-               const char *end,
-               const ENCODING *enc)
-{
+internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+               const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
   case XML_TOK_DECL_OPEN:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ENTITY)) {
       state->handler = entity0;
       return XML_ROLE_ENTITY_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ATTLIST)) {
       state->handler = attlist0;
       return XML_ROLE_ATTLIST_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ELEMENT)) {
       state->handler = element0;
       return XML_ROLE_ELEMENT_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_NOTATION)) {
       state->handler = notation0;
       return XML_ROLE_NOTATION_NONE;
@@ -419,12 +370,8 @@ internalSubset(PROLOG_STATE *state,
 #ifdef XML_DTD
 
 static int PTRCALL
-externalSubset0(PROLOG_STATE *state,
-                int tok,
-                const char *ptr,
-                const char *end,
-                const ENCODING *enc)
-{
+externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+                const ENCODING *enc) {
   state->handler = externalSubset1;
   if (tok == XML_TOK_XML_DECL)
     return XML_ROLE_TEXT_DECL;
@@ -432,12 +379,8 @@ externalSubset0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-externalSubset1(PROLOG_STATE *state,
-                int tok,
-                const char *ptr,
-                const char *end,
-                const ENCODING *enc)
-{
+externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+                const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_COND_SECT_OPEN:
     state->handler = condSect0;
@@ -464,12 +407,11 @@ externalSubset1(PROLOG_STATE *state,
 #endif /* XML_DTD */
 
 static int PTRCALL
-entity0(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -484,12 +426,11 @@ entity0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity1(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -501,12 +442,8 @@ entity1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity2(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -529,12 +466,11 @@ entity2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity3(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -546,12 +482,11 @@ entity3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity4(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -563,12 +498,8 @@ entity4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity5(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -586,12 +517,11 @@ entity5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity6(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -604,12 +534,8 @@ entity6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity7(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -632,12 +558,11 @@ entity7(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity8(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -649,12 +574,11 @@ entity8(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity9(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -666,12 +590,11 @@ entity9(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity10(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -683,12 +606,11 @@ entity10(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation0(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -700,12 +622,8 @@ notation0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation1(PROLOG_STATE *state,
-          int tok,
-          const char *ptr,
-          const char *end,
-          const ENCODING *enc)
-{
+notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -724,12 +642,11 @@ notation1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation2(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -741,12 +658,11 @@ notation2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation3(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -759,12 +675,11 @@ notation3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation4(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -780,12 +695,11 @@ notation4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -798,12 +712,11 @@ attlist0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist1(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -819,34 +732,23 @@ attlist1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist2(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
-  case XML_TOK_NAME:
-    {
-      static const char * const types[] = {
-        KW_CDATA,
-        KW_ID,
-        KW_IDREF,
-        KW_IDREFS,
-        KW_ENTITY,
-        KW_ENTITIES,
-        KW_NMTOKEN,
-        KW_NMTOKENS,
-      };
-      int i;
-      for (i = 0; i < (int)(sizeof(types)/sizeof(types[0])); i++)
-        if (XmlNameMatchesAscii(enc, ptr, end, types[i])) {
-          state->handler = attlist8;
-          return XML_ROLE_ATTRIBUTE_TYPE_CDATA + i;
-        }
-    }
+  case XML_TOK_NAME: {
+    static const char *const types[] = {
+        KW_CDATA,  KW_ID,       KW_IDREF,   KW_IDREFS,
+        KW_ENTITY, KW_ENTITIES, KW_NMTOKEN, KW_NMTOKENS,
+    };
+    int i;
+    for (i = 0; i < (int)(sizeof(types) / sizeof(types[0])); i++)
+      if (XmlNameMatchesAscii(enc, ptr, end, types[i])) {
+        state->handler = attlist8;
+        return XML_ROLE_ATTRIBUTE_TYPE_CDATA + i;
+      }
+  }
     if (XmlNameMatchesAscii(enc, ptr, end, KW_NOTATION)) {
       state->handler = attlist5;
       return XML_ROLE_ATTLIST_NONE;
@@ -860,12 +762,11 @@ attlist2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -879,12 +780,11 @@ attlist3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -899,12 +799,11 @@ attlist4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -916,12 +815,11 @@ attlist5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist6(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -933,12 +831,11 @@ attlist6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist7(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -954,33 +851,23 @@ attlist7(PROLOG_STATE *state,
 
 /* default value */
 static int PTRCALL
-attlist8(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
   case XML_TOK_POUND_NAME:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_IMPLIED)) {
       state->handler = attlist1;
       return XML_ROLE_IMPLIED_ATTRIBUTE_VALUE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_REQUIRED)) {
       state->handler = attlist1;
       return XML_ROLE_REQUIRED_ATTRIBUTE_VALUE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_FIXED)) {
       state->handler = attlist9;
       return XML_ROLE_ATTLIST_NONE;
@@ -994,12 +881,11 @@ attlist8(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist9(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -1011,12 +897,11 @@ attlist9(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1029,12 +914,8 @@ element0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element1(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1059,19 +940,13 @@ element1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element2(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
   case XML_TOK_POUND_NAME:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_PCDATA)) {
       state->handler = element3;
       return XML_ROLE_CONTENT_PCDATA;
@@ -1099,12 +974,11 @@ element2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1124,12 +998,11 @@ element3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1142,12 +1015,11 @@ element4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1163,12 +1035,11 @@ element5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element6(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1193,12 +1064,11 @@ element6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element7(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1243,12 +1113,8 @@ element7(PROLOG_STATE *state,
 #ifdef XML_DTD
 
 static int PTRCALL
-condSect0(PROLOG_STATE *state,
-          int tok,
-          const char *ptr,
-          const char *end,
-          const ENCODING *enc)
-{
+condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1267,12 +1133,11 @@ condSect0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-condSect1(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1285,12 +1150,11 @@ condSect1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-condSect2(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1304,12 +1168,11 @@ condSect2(PROLOG_STATE *state,
 #endif /* XML_DTD */
 
 static int PTRCALL
-declClose(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return state->role_none;
@@ -1341,21 +1204,21 @@ declClose(PROLOG_STATE *state,
  * LCOV_EXCL_START
  */
 static int PTRCALL
-error(PROLOG_STATE *UNUSED_P(state),
-      int UNUSED_P(tok),
-      const char *UNUSED_P(ptr),
-      const char *UNUSED_P(end),
-      const ENCODING *UNUSED_P(enc))
-{
+error(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+      const ENCODING *enc) {
+  UNUSED_P(state);
+  UNUSED_P(tok);
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   return XML_ROLE_NONE;
 }
 /* LCOV_EXCL_STOP */
 
 static int FASTCALL
-common(PROLOG_STATE *state, int tok)
-{
+common(PROLOG_STATE *state, int tok) {
 #ifdef XML_DTD
-  if (!state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF)
+  if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF)
     return XML_ROLE_INNER_PARAM_ENTITY_REF;
 #endif
   state->handler = error;
@@ -1363,8 +1226,7 @@ common(PROLOG_STATE *state, int tok)
 }
 
 void
-XmlPrologStateInit(PROLOG_STATE *state)
-{
+XmlPrologStateInit(PROLOG_STATE *state) {
   state->handler = prolog0;
 #ifdef XML_DTD
   state->documentEntity = 1;
@@ -1376,8 +1238,7 @@ XmlPrologStateInit(PROLOG_STATE *state)
 #ifdef XML_DTD
 
 void
-XmlPrologStateInitExternalEntity(PROLOG_STATE *state)
-{
+XmlPrologStateInitExternalEntity(PROLOG_STATE *state) {
   state->handler = externalSubset0;
   state->documentEntity = 0;
   state->includeLevel = 0;
diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h
index e5f048eab55c..036aba64fd29 100644
--- a/Modules/expat/xmlrole.h
+++ b/Modules/expat/xmlrole.h
@@ -36,7 +36,7 @@
 #ifdef __VMS
 /*      0        1         2         3      0        1         2         3
         1234567890123456789012345678901     1234567890123456789012345678901 */
-#define XmlPrologStateInitExternalEntity    XmlPrologStateInitExternalEnt
+#  define XmlPrologStateInitExternalEntity XmlPrologStateInitExternalEnt
 #endif
 
 #include "xmltok.h"
@@ -113,11 +113,8 @@ enum {
 };
 
 typedef struct prolog_state {
-  int (PTRCALL *handler) (struct prolog_state *state,
-                          int tok,
-                          const char *ptr,
-                          const char *end,
-                          const ENCODING *enc);
+  int(PTRCALL *handler)(struct prolog_state *state, int tok, const char *ptr,
+                        const char *end, const ENCODING *enc);
   unsigned level;
   int role_none;
 #ifdef XML_DTD
@@ -132,8 +129,8 @@ void XmlPrologStateInit(PROLOG_STATE *);
 void XmlPrologStateInitExternalEntity(PROLOG_STATE *);
 #endif /* XML_DTD */
 
-#define XmlTokenRole(state, tok, ptr, end, enc) \
- (((state)->handler)(state, tok, ptr, end, enc))
+#define XmlTokenRole(state, tok, ptr, end, enc)                                \
+  (((state)->handler)(state, tok, ptr, end, enc))
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c
index 6b415d83972c..11e9d1ccdad4 100644
--- a/Modules/expat/xmltok.c
+++ b/Modules/expat/xmltok.c
@@ -31,24 +31,23 @@
 */
 
 #include <stddef.h>
-#include <string.h>  /* memcpy */
+#include <string.h> /* memcpy */
 
 #if defined(_MSC_VER) && (_MSC_VER <= 1700)
-  /* for vs2012/11.0/1700 and earlier Visual Studio compilers */
-# define bool   int
-# define false  0
-# define true   1
+/* for vs2012/11.0/1700 and earlier Visual Studio compilers */
+#  define bool int
+#  define false 0
+#  define true 1
 #else
-# include <stdbool.h>
+#  include <stdbool.h>
 #endif
 
-
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #else
-#ifdef HAVE_EXPAT_CONFIG_H
-#include <expat_config.h>
-#endif
+#  ifdef HAVE_EXPAT_CONFIG_H
+#    include <expat_config.h>
+#  endif
 #endif /* ndef _WIN32 */
 
 #include "expat_external.h"
@@ -57,58 +56,49 @@
 #include "nametab.h"
 
 #ifdef XML_DTD
-#define IGNORE_SECTION_TOK_VTABLE , PREFIX(ignoreSectionTok)
+#  define IGNORE_SECTION_TOK_VTABLE , PREFIX(ignoreSectionTok)
 #else
-#define IGNORE_SECTION_TOK_VTABLE /* as nothing */
+#  define IGNORE_SECTION_TOK_VTABLE /* as nothing */
 #endif
 
-#define VTABLE1 \
-  { PREFIX(prologTok), PREFIX(contentTok), \
-    PREFIX(cdataSectionTok) IGNORE_SECTION_TOK_VTABLE }, \
-  { PREFIX(attributeValueTok), PREFIX(entityValueTok) }, \
-  PREFIX(nameMatchesAscii), \
-  PREFIX(nameLength), \
-  PREFIX(skipS), \
-  PREFIX(getAtts), \
-  PREFIX(charRefNumber), \
-  PREFIX(predefinedEntityName), \
-  PREFIX(updatePosition), \
-  PREFIX(isPublicId)
+#define VTABLE1                                                                \
+  {PREFIX(prologTok), PREFIX(contentTok),                                      \
+   PREFIX(cdataSectionTok) IGNORE_SECTION_TOK_VTABLE},                         \
+      {PREFIX(attributeValueTok), PREFIX(entityValueTok)},                     \
+      PREFIX(nameMatchesAscii), PREFIX(nameLength), PREFIX(skipS),             \
+      PREFIX(getAtts), PREFIX(charRefNumber), PREFIX(predefinedEntityName),    \
+      PREFIX(updatePosition), PREFIX(isPublicId)
 
 #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16)
 
-#define UCS2_GET_NAMING(pages, hi, lo) \
-   (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo) & 0x1F)))
+#define UCS2_GET_NAMING(pages, hi, lo)                                         \
+  (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo)&0x1F)))
 
 /* A 2 byte UTF-8 representation splits the characters 11 bits between
    the bottom 5 and 6 bits of the bytes.  We need 8 bits to index into
    pages, 3 bits to add to that index and 5 bits to generate the mask.
 */
-#define UTF8_GET_NAMING2(pages, byte) \
-    (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3) \
-                      + ((((byte)[0]) & 3) << 1) \
-                      + ((((byte)[1]) >> 5) & 1)] \
-         & (1u << (((byte)[1]) & 0x1F)))
+#define UTF8_GET_NAMING2(pages, byte)                                          \
+  (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3)                         \
+                + ((((byte)[0]) & 3) << 1) + ((((byte)[1]) >> 5) & 1)]         \
+   & (1u << (((byte)[1]) & 0x1F)))
 
 /* A 3 byte UTF-8 representation splits the characters 16 bits between
    the bottom 4, 6 and 6 bits of the bytes.  We need 8 bits to index
    into pages, 3 bits to add to that index and 5 bits to generate the
    mask.
 */
-#define UTF8_GET_NAMING3(pages, byte) \
-  (namingBitmap[((pages)[((((byte)[0]) & 0xF) << 4) \
-                             + ((((byte)[1]) >> 2) & 0xF)] \
-                       << 3) \
-                      + ((((byte)[1]) & 3) << 1) \
-                      + ((((byte)[2]) >> 5) & 1)] \
-         & (1u << (((byte)[2]) & 0x1F)))
-
-#define UTF8_GET_NAMING(pages, p, n) \
-  ((n) == 2 \
-  ? UTF8_GET_NAMING2(pages, (const unsigned char *)(p)) \
-  : ((n) == 3 \
-     ? UTF8_GET_NAMING3(pages, (const unsigned char *)(p)) \
-     : 0))
+#define UTF8_GET_NAMING3(pages, byte)                                          \
+  (namingBitmap                                                                \
+       [((pages)[((((byte)[0]) & 0xF) << 4) + ((((byte)[1]) >> 2) & 0xF)]      \
+         << 3)                                                                 \
+        + ((((byte)[1]) & 3) << 1) + ((((byte)[2]) >> 5) & 1)]                 \
+   & (1u << (((byte)[2]) & 0x1F)))
+
+#define UTF8_GET_NAMING(pages, p, n)                                           \
+  ((n) == 2                                                                    \
+       ? UTF8_GET_NAMING2(pages, (const unsigned char *)(p))                   \
+       : ((n) == 3 ? UTF8_GET_NAMING3(pages, (const unsigned char *)(p)) : 0))
 
 /* Detection of invalid UTF-8 sequences is based on Table 3.1B
    of Unicode 3.2: http://www.unicode.org/unicode/reports/tr28/
@@ -120,88 +110,76 @@
      (A & 0xC0) == 0xC0  means A > 0xBF
 */
 
-#define UTF8_INVALID2(p) \
+#define UTF8_INVALID2(p)                                                       \
   ((*p) < 0xC2 || ((p)[1] & 0x80) == 0 || ((p)[1] & 0xC0) == 0xC0)
 
-#define UTF8_INVALID3(p) \
-  (((p)[2] & 0x80) == 0 \
-  || \
-  ((*p) == 0xEF && (p)[1] == 0xBF \
-    ? \
-    (p)[2] > 0xBD \
-    : \
-    ((p)[2] & 0xC0) == 0xC0) \
-  || \
-  ((*p) == 0xE0 \
-    ? \
-    (p)[1] < 0xA0 || ((p)[1] & 0xC0) == 0xC0 \
-    : \
-    ((p)[1] & 0x80) == 0 \
-    || \
-    ((*p) == 0xED ? (p)[1] > 0x9F : ((p)[1] & 0xC0) == 0xC0)))
-
-#define UTF8_INVALID4(p) \
-  (((p)[3] & 0x80) == 0 || ((p)[3] & 0xC0) == 0xC0 \
-  || \
-  ((p)[2] & 0x80) == 0 || ((p)[2] & 0xC0) == 0xC0 \
-  || \
-  ((*p) == 0xF0 \
-    ? \
-    (p)[1] < 0x90 || ((p)[1] & 0xC0) == 0xC0 \
-    : \
-    ((p)[1] & 0x80) == 0 \
-    || \
-    ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0)))
+#define UTF8_INVALID3(p)                                                       \
+  (((p)[2] & 0x80) == 0                                                        \
+   || ((*p) == 0xEF && (p)[1] == 0xBF ? (p)[2] > 0xBD                          \
+                                      : ((p)[2] & 0xC0) == 0xC0)               \
+   || ((*p) == 0xE0                                                            \
+           ? (p)[1] < 0xA0 || ((p)[1] & 0xC0) == 0xC0                          \
+           : ((p)[1] & 0x80) == 0                                              \
+                 || ((*p) == 0xED ? (p)[1] > 0x9F : ((p)[1] & 0xC0) == 0xC0)))
+
+#define UTF8_INVALID4(p)                                                       \
+  (((p)[3] & 0x80) == 0 || ((p)[3] & 0xC0) == 0xC0 || ((p)[2] & 0x80) == 0     \
+   || ((p)[2] & 0xC0) == 0xC0                                                  \
+   || ((*p) == 0xF0                                                            \
+           ? (p)[1] < 0x90 || ((p)[1] & 0xC0) == 0xC0                          \
+           : ((p)[1] & 0x80) == 0                                              \
+                 || ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0)))
 
 static int PTRFASTCALL
-isNever(const ENCODING *UNUSED_P(enc), const char *UNUSED_P(p))
-{
+isNever(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  UNUSED_P(p);
   return 0;
 }
 
 static int PTRFASTCALL
-utf8_isName2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isName2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING2(namePages, (const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isName3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isName3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING3(namePages, (const unsigned char *)p);
 }
 
 #define utf8_isName4 isNever
 
 static int PTRFASTCALL
-utf8_isNmstrt2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isNmstrt2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isNmstrt3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isNmstrt3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p);
 }
 
 #define utf8_isNmstrt4 isNever
 
 static int PTRFASTCALL
-utf8_isInvalid2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID2((const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isInvalid3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID3((const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isInvalid4(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid4(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID4((const unsigned char *)p);
 }
 
@@ -209,61 +187,44 @@ struct normal_encoding {
   ENCODING enc;
   unsigned char type[256];
 #ifdef XML_MIN_SIZE
-  int (PTRFASTCALL *byteType)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNameMin)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *);
-  int (PTRFASTCALL *byteToAscii)(const ENCODING *, const char *);
-  int (PTRCALL *charMatches)(const ENCODING *, const char *, int);
+  int(PTRFASTCALL *byteType)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNameMin)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *);
+  int(PTRFASTCALL *byteToAscii)(const ENCODING *, const char *);
+  int(PTRCALL *charMatches)(const ENCODING *, const char *, int);
 #endif /* XML_MIN_SIZE */
-  int (PTRFASTCALL *isName2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isName3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isName4)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid4)(const ENCODING *, const char *);
 };
 
-#define AS_NORMAL_ENCODING(enc)   ((const struct normal_encoding *) (enc))
+#define AS_NORMAL_ENCODING(enc) ((const struct normal_encoding *)(enc))
 
 #ifdef XML_MIN_SIZE
 
-#define STANDARD_VTABLE(E) \
- E ## byteType, \
- E ## isNameMin, \
- E ## isNmstrtMin, \
- E ## byteToAscii, \
- E ## charMatches,
+#  define STANDARD_VTABLE(E)                                                   \
+    E##byteType, E##isNameMin, E##isNmstrtMin, E##byteToAscii, E##charMatches,
 
 #else
 
-#define STANDARD_VTABLE(E) /* as nothing */
+#  define STANDARD_VTABLE(E) /* as nothing */
 
 #endif
 
-#define NORMAL_VTABLE(E) \
- E ## isName2, \
- E ## isName3, \
- E ## isName4, \
- E ## isNmstrt2, \
- E ## isNmstrt3, \
- E ## isNmstrt4, \
- E ## isInvalid2, \
- E ## isInvalid3, \
- E ## isInvalid4
-
-#define NULL_VTABLE \
- /* isName2 */ NULL, \
- /* isName3 */ NULL, \
- /* isName4 */ NULL, \
- /* isNmstrt2 */ NULL, \
- /* isNmstrt3 */ NULL, \
- /* isNmstrt4 */ NULL, \
- /* isInvalid2 */ NULL, \
- /* isInvalid3 */ NULL, \
- /* isInvalid4 */ NULL
+#define NORMAL_VTABLE(E)                                                       \
+  E##isName2, E##isName3, E##isName4, E##isNmstrt2, E##isNmstrt3,              \
+      E##isNmstrt4, E##isInvalid2, E##isInvalid3, E##isInvalid4
+
+#define NULL_VTABLE                                                            \
+  /* isName2 */ NULL, /* isName3 */ NULL, /* isName4 */ NULL,                  \
+      /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL,        \
+      /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL
 
 static int FASTCALL checkCharRefNumber(int);
 
@@ -271,75 +232,70 @@ static int FASTCALL checkCharRefNumber(int);
 #include "ascii.h"
 
 #ifdef XML_MIN_SIZE
-#define sb_isNameMin isNever
-#define sb_isNmstrtMin isNever
+#  define sb_isNameMin isNever
+#  define sb_isNmstrtMin isNever
 #endif
 
 #ifdef XML_MIN_SIZE
-#define MINBPC(enc) ((enc)->minBytesPerChar)
+#  define MINBPC(enc) ((enc)->minBytesPerChar)
 #else
 /* minimum bytes per character */
-#define MINBPC(enc) 1
+#  define MINBPC(enc) 1
 #endif
 
-#define SB_BYTE_TYPE(enc, p) \
+#define SB_BYTE_TYPE(enc, p)                                                   \
   (((struct normal_encoding *)(enc))->type[(unsigned char)*(p)])
 
 #ifdef XML_MIN_SIZE
 static int PTRFASTCALL
-sb_byteType(const ENCODING *enc, const char *p)
-{
+sb_byteType(const ENCODING *enc, const char *p) {
   return SB_BYTE_TYPE(enc, p);
 }
-#define BYTE_TYPE(enc, p) \
- (AS_NORMAL_ENCODING(enc)->byteType(enc, p))
+#  define BYTE_TYPE(enc, p) (AS_NORMAL_ENCODING(enc)->byteType(enc, p))
 #else
-#define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
+#  define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
 #endif
 
 #ifdef XML_MIN_SIZE
-#define BYTE_TO_ASCII(enc, p) \
- (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p))
+#  define BYTE_TO_ASCII(enc, p) (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p))
 static int PTRFASTCALL
-sb_byteToAscii(const ENCODING *enc, const char *p)
-{
+sb_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return *p;
 }
 #else
-#define BYTE_TO_ASCII(enc, p) (*(p))
+#  define BYTE_TO_ASCII(enc, p) (*(p))
 #endif
 
-#define IS_NAME_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isName ## n(enc, p))
-#define IS_NMSTRT_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isNmstrt ## n(enc, p))
-#define IS_INVALID_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isInvalid ## n(enc, p))
+#define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p))
+#define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p))
+#define IS_INVALID_CHAR(enc, p, n)                                             \
+  (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p))
 
 #ifdef XML_MIN_SIZE
-#define IS_NAME_CHAR_MINBPC(enc, p) \
- (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p))
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) \
- (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p))
+#  define IS_NAME_CHAR_MINBPC(enc, p)                                          \
+    (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p))
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p)                                        \
+    (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p))
 #else
-#define IS_NAME_CHAR_MINBPC(enc, p) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
+#  define IS_NAME_CHAR_MINBPC(enc, p) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
 #endif
 
 #ifdef XML_MIN_SIZE
-#define CHAR_MATCHES(enc, p, c) \
- (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c))
+#  define CHAR_MATCHES(enc, p, c)                                              \
+    (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c))
 static int PTRCALL
-sb_charMatches(const ENCODING *enc, const char *p, int c)
-{
+sb_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
   return *p == c;
 }
 #else
 /* c is an ASCII character */
-#define CHAR_MATCHES(enc, p, c) (*(p) == c)
+#  define CHAR_MATCHES(enc, p, c) (*(p) == c)
 #endif
 
-#define PREFIX(ident) normal_ ## ident
+#define PREFIX(ident) normal_##ident
 #define XML_TOK_IMPL_C
 #include "xmltok_impl.c"
 #undef XML_TOK_IMPL_C
@@ -354,42 +310,46 @@ sb_charMatches(const ENCODING *enc, const char *p, int c)
 #undef IS_NMSTRT_CHAR_MINBPC
 #undef IS_INVALID_CHAR
 
-enum {  /* UTF8_cvalN is value of masked first byte of N byte sequence */
-  UTF8_cval1 = 0x00,
-  UTF8_cval2 = 0xc0,
-  UTF8_cval3 = 0xe0,
-  UTF8_cval4 = 0xf0
+enum { /* UTF8_cvalN is value of masked first byte of N byte sequence */
+       UTF8_cval1 = 0x00,
+       UTF8_cval2 = 0xc0,
+       UTF8_cval3 = 0xe0,
+       UTF8_cval4 = 0xf0
 };
 
 void
-_INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** fromLimRef)
-{
-  const char * fromLim = *fromLimRef;
+_INTERNAL_trim_to_complete_utf8_characters(const char *from,
+                                           const char **fromLimRef) {
+  const char *fromLim = *fromLimRef;
   size_t walked = 0;
   for (; fromLim > from; fromLim--, walked++) {
     const unsigned char prev = (unsigned char)fromLim[-1];
-    if ((prev & 0xf8u) == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */
+    if ((prev & 0xf8u)
+        == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */
       if (walked + 1 >= 4) {
         fromLim += 4 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0xf0u) == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */
+    } else if ((prev & 0xf0u)
+               == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */
       if (walked + 1 >= 3) {
         fromLim += 3 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0xe0u) == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */
+    } else if ((prev & 0xe0u)
+               == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */
       if (walked + 1 >= 2) {
         fromLim += 2 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0x80u) == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */
+    } else if ((prev & 0x80u)
+               == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */
       break;
     }
   }
@@ -397,16 +357,15 @@ _INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** from
 }
 
 static enum XML_Convert_Result PTRCALL
-utf8_toUtf8(const ENCODING *UNUSED_P(enc),
-            const char **fromP, const char *fromLim,
-            char **toP, const char *toLim)
-{
+utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+            char **toP, const char *toLim) {
   bool input_incomplete = false;
   bool output_exhausted = false;
 
   /* Avoid copying partial characters (due to limited space). */
   const ptrdiff_t bytesAvailable = fromLim - *fromP;
   const ptrdiff_t bytesStorable = toLim - *toP;
+  UNUSED_P(enc);
   if (bytesAvailable > bytesStorable) {
     fromLim = *fromP + bytesStorable;
     output_exhausted = true;
@@ -414,7 +373,7 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
 
   /* Avoid copying partial characters (from incomplete input). */
   {
-    const char * const fromLimBefore = fromLim;
+    const char *const fromLimBefore = fromLim;
     _INTERNAL_trim_to_complete_utf8_characters(*fromP, &fromLim);
     if (fromLim < fromLimBefore) {
       input_incomplete = true;
@@ -428,7 +387,7 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
     *toP += bytesToCopy;
   }
 
-  if (output_exhausted)  /* needs to go first */
+  if (output_exhausted) /* needs to go first */
     return XML_CONVERT_OUTPUT_EXHAUSTED;
   else if (input_incomplete)
     return XML_CONVERT_INPUT_INCOMPLETE;
@@ -437,10 +396,8 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
 }
 
 static enum XML_Convert_Result PTRCALL
-utf8_toUtf16(const ENCODING *enc,
-             const char **fromP, const char *fromLim,
-             unsigned short **toP, const unsigned short *toLim)
-{
+utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+             unsigned short **toP, const unsigned short *toLim) {
   enum XML_Convert_Result res = XML_CONVERT_COMPLETED;
   unsigned short *to = *toP;
   const char *from = *fromP;
@@ -459,30 +416,28 @@ utf8_toUtf16(const ENCODING *enc,
         res = XML_CONVERT_INPUT_INCOMPLETE;
         goto after;
       }
-      *to++ = (unsigned short)(((from[0] & 0xf) << 12)
-                               | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f));
+      *to++ = (unsigned short)(((from[0] & 0xf) << 12) | ((from[1] & 0x3f) << 6)
+                               | (from[2] & 0x3f));
       from += 3;
       break;
-    case BT_LEAD4:
-      {
-        unsigned long n;
-        if (toLim - to < 2) {
-          res = XML_CONVERT_OUTPUT_EXHAUSTED;
-          goto after;
-        }
-        if (fromLim - from < 4) {
-          res = XML_CONVERT_INPUT_INCOMPLETE;
-          goto after;
-        }
-        n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12)
-            | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f);
-        n -= 0x10000;
-        to[0] = (unsigned short)((n >> 10) | 0xD800);
-        to[1] = (unsigned short)((n & 0x3FF) | 0xDC00);
-        to += 2;
-        from += 4;
+    case BT_LEAD4: {
+      unsigned long n;
+      if (toLim - to < 2) {
+        res = XML_CONVERT_OUTPUT_EXHAUSTED;
+        goto after;
       }
-      break;
+      if (fromLim - from < 4) {
+        res = XML_CONVERT_INPUT_INCOMPLETE;
+        goto after;
+      }
+      n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12)
+          | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f);
+      n -= 0x10000;
+      to[0] = (unsigned short)((n >> 10) | 0xD800);
+      to[1] = (unsigned short)((n & 0x3FF) | 0xDC00);
+      to += 2;
+      from += 4;
+    } break;
     default:
       *to++ = *from++;
       break;
@@ -497,56 +452,51 @@ utf8_toUtf16(const ENCODING *enc,
 }
 
 #ifdef XML_NS
-static const struct normal_encoding utf8_encoding_ns = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
-#include "asciitab.h"
-#include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+static const struct normal_encoding utf8_encoding_ns
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
+#  include "asciitab.h"
+#  include "utf8tab.h"
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 #endif
 
-static const struct normal_encoding utf8_encoding = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding utf8_encoding
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 #ifdef XML_NS
 
-static const struct normal_encoding internal_utf8_encoding_ns = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
-#include "iasciitab.h"
-#include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+static const struct normal_encoding internal_utf8_encoding_ns
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
+#  include "iasciitab.h"
+#  include "utf8tab.h"
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 #endif
 
-static const struct normal_encoding internal_utf8_encoding = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding internal_utf8_encoding
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "iasciitab.h"
 #undef BT_COLON
 #include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 static enum XML_Convert_Result PTRCALL
-latin1_toUtf8(const ENCODING *UNUSED_P(enc),
-              const char **fromP, const char *fromLim,
-              char **toP, const char *toLim)
-{
+latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+              char **toP, const char *toLim) {
+  UNUSED_P(enc);
   for (;;) {
     unsigned char c;
     if (*fromP == fromLim)
@@ -558,8 +508,7 @@ latin1_toUtf8(const ENCODING *UNUSED_P(enc),
       *(*toP)++ = (char)((c >> 6) | UTF8_cval2);
       *(*toP)++ = (char)((c & 0x3f) | 0x80);
       (*fromP)++;
-    }
-    else {
+    } else {
       if (*toP == toLim)
         return XML_CONVERT_OUTPUT_EXHAUSTED;
       *(*toP)++ = *(*fromP)++;
@@ -568,10 +517,9 @@ latin1_toUtf8(const ENCODING *UNUSED_P(enc),
 }
 
 static enum XML_Convert_Result PTRCALL
-latin1_toUtf16(const ENCODING *UNUSED_P(enc),
-               const char **fromP, const char *fromLim,
-               unsigned short **toP, const unsigned short *toLim)
-{
+latin1_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+               unsigned short **toP, const unsigned short *toLim) {
+  UNUSED_P(enc);
   while (*fromP < fromLim && *toP < toLim)
     *(*toP)++ = (unsigned char)*(*fromP)++;
 
@@ -583,33 +531,30 @@ latin1_toUtf16(const ENCODING *UNUSED_P(enc),
 
 #ifdef XML_NS
 
-static const struct normal_encoding latin1_encoding_ns = {
-  { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+static const struct normal_encoding latin1_encoding_ns
+    = {{VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0},
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding latin1_encoding = {
-  { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
-  {
+static const struct normal_encoding latin1_encoding
+    = {{VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 static enum XML_Convert_Result PTRCALL
-ascii_toUtf8(const ENCODING *UNUSED_P(enc),
-             const char **fromP, const char *fromLim,
-             char **toP, const char *toLim)
-{
+ascii_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+             char **toP, const char *toLim) {
+  UNUSED_P(enc);
   while (*fromP < fromLim && *toP < toLim)
     *(*toP)++ = *(*fromP)++;
 
@@ -621,40 +566,45 @@ ascii_toUtf8(const ENCODING *UNUSED_P(enc),
 
 #ifdef XML_NS
 
-static const struct normal_encoding ascii_encoding_ns = {
-  { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
-  {
-#include "asciitab.h"
-/* BT_NONXML == 0 */
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+static const struct normal_encoding ascii_encoding_ns
+    = {{VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0},
+       {
+#  include "asciitab.h"
+           /* BT_NONXML == 0 */
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding ascii_encoding = {
-  { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding ascii_encoding
+    = {{VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
-/* BT_NONXML == 0 */
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+           /* BT_NONXML == 0 */
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 static int PTRFASTCALL
-unicode_byte_type(char hi, char lo)
-{
+unicode_byte_type(char hi, char lo) {
   switch ((unsigned char)hi) {
-  case 0xD8: case 0xD9: case 0xDA: case 0xDB:
+  /* 0xD800?0xDBFF first 16-bit code unit or high surrogate (W1) */
+  case 0xD8:
+  case 0xD9:
+  case 0xDA:
+  case 0xDB:
     return BT_LEAD4;
-  case 0xDC: case 0xDD: case 0xDE: case 0xDF:
+  /* 0xDC00?0xDFFF second 16-bit code unit or low surrogate (W2) */
+  case 0xDC:
+  case 0xDD:
+  case 0xDE:
+  case 0xDF:
     return BT_TRAIL;
   case 0xFF:
     switch ((unsigned char)lo) {
-    case 0xFF:
-    case 0xFE:
+    case 0xFF: /* noncharacter-FFFF */
+    case 0xFE: /* noncharacter-FFFE */
       return BT_NONXML;
     }
     break;
@@ -662,102 +612,105 @@ unicode_byte_type(char hi, char lo)
   return BT_NONASCII;
 }
 
-#define DEFINE_UTF16_TO_UTF8(E) \
-static enum XML_Convert_Result  PTRCALL \
-E ## toUtf8(const ENCODING *UNUSED_P(enc), \
-            const char **fromP, const char *fromLim, \
-            char **toP, const char *toLim) \
-{ \
-  const char *from = *fromP; \
-  fromLim = from + (((fromLim - from) >> 1) << 1);  /* shrink to even */ \
-  for (; from < fromLim; from += 2) { \
-    int plane; \
-    unsigned char lo2; \
-    unsigned char lo = GET_LO(from); \
-    unsigned char hi = GET_HI(from); \
-    switch (hi) { \
-    case 0: \
-      if (lo < 0x80) { \
-        if (*toP == toLim) { \
-          *fromP = from; \
-          return XML_CONVERT_OUTPUT_EXHAUSTED; \
-        } \
-        *(*toP)++ = lo; \
-        break; \
-      } \
-      /* fall through */ \
-    case 0x1: case 0x2: case 0x3: \
-    case 0x4: case 0x5: case 0x6: case 0x7: \
-      if (toLim -  *toP < 2) { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      *(*toP)++ = ((lo >> 6) | (hi << 2) |  UTF8_cval2); \
-      *(*toP)++ = ((lo & 0x3f) | 0x80); \
-      break; \
-    default: \
-      if (toLim -  *toP < 3)  { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      /* 16 bits divided 4, 6, 6 amongst 3 bytes */ \
-      *(*toP)++ = ((hi >> 4) | UTF8_cval3); \
-      *(*toP)++ = (((hi & 0xf) << 2) | (lo >> 6) | 0x80); \
-      *(*toP)++ = ((lo & 0x3f) | 0x80); \
-      break; \
-    case 0xD8: case 0xD9: case 0xDA: case 0xDB: \
-      if (toLim -  *toP < 4) { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      if (fromLim - from < 4) { \
-        *fromP = from; \
-        return XML_CONVERT_INPUT_INCOMPLETE; \
-      } \
-      plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1; \
-      *(*toP)++ = ((plane >> 2) | UTF8_cval4); \
-      *(*toP)++ = (((lo >> 2) & 0xF) | ((plane & 0x3) << 4) | 0x80); \
-      from += 2; \
-      lo2 = GET_LO(from); \
-      *(*toP)++ = (((lo & 0x3) << 4) \
-                   | ((GET_HI(from) & 0x3) << 2) \
-                   | (lo2 >> 6) \
-                   | 0x80); \
-      *(*toP)++ = ((lo2 & 0x3f) | 0x80); \
-      break; \
-    } \
-  } \
-  *fromP = from; \
-  if (from < fromLim) \
-    return XML_CONVERT_INPUT_INCOMPLETE; \
-  else \
-    return XML_CONVERT_COMPLETED; \
-}
+#define DEFINE_UTF16_TO_UTF8(E)                                                \
+  static enum XML_Convert_Result PTRCALL E##toUtf8(                            \
+      const ENCODING *enc, const char **fromP, const char *fromLim,            \
+      char **toP, const char *toLim) {                                         \
+    const char *from = *fromP;                                                 \
+    UNUSED_P(enc);                                                             \
+    fromLim = from + (((fromLim - from) >> 1) << 1); /* shrink to even */      \
+    for (; from < fromLim; from += 2) {                                        \
+      int plane;                                                               \
+      unsigned char lo2;                                                       \
+      unsigned char lo = GET_LO(from);                                         \
+      unsigned char hi = GET_HI(from);                                         \
+      switch (hi) {                                                            \
+      case 0:                                                                  \
+        if (lo < 0x80) {                                                       \
+          if (*toP == toLim) {                                                 \
+            *fromP = from;                                                     \
+            return XML_CONVERT_OUTPUT_EXHAUSTED;                               \
+          }                                                                    \
+          *(*toP)++ = lo;                                                      \
+          break;                                                               \
+        }                                                                      \
+        /* fall through */                                                     \
+      case 0x1:                                                                \
+      case 0x2:                                                                \
+      case 0x3:                                                                \
+      case 0x4:                                                                \
+      case 0x5:                                                                \
+      case 0x6:                                                                \
+      case 0x7:                                                                \
+        if (toLim - *toP < 2) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        *(*toP)++ = ((lo >> 6) | (hi << 2) | UTF8_cval2);                      \
+        *(*toP)++ = ((lo & 0x3f) | 0x80);                                      \
+        break;                                                                 \
+      default:                                                                 \
+        if (toLim - *toP < 3) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        /* 16 bits divided 4, 6, 6 amongst 3 bytes */                          \
+        *(*toP)++ = ((hi >> 4) | UTF8_cval3);                                  \
+        *(*toP)++ = (((hi & 0xf) << 2) | (lo >> 6) | 0x80);                    \
+        *(*toP)++ = ((lo & 0x3f) | 0x80);                                      \
+        break;                                                                 \
+      case 0xD8:                                                               \
+      case 0xD9:                                                               \
+      case 0xDA:                                                               \
+      case 0xDB:                                                               \
+        if (toLim - *toP < 4) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        if (fromLim - from < 4) {                                              \
+          *fromP = from;                                                       \
+          return XML_CONVERT_INPUT_INCOMPLETE;                                 \
+        }                                                                      \
+        plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1;                   \
+        *(*toP)++ = (char)((plane >> 2) | UTF8_cval4);                         \
+        *(*toP)++ = (((lo >> 2) & 0xF) | ((plane & 0x3) << 4) | 0x80);         \
+        from += 2;                                                             \
+        lo2 = GET_LO(from);                                                    \
+        *(*toP)++ = (((lo & 0x3) << 4) | ((GET_HI(from) & 0x3) << 2)           \
+                     | (lo2 >> 6) | 0x80);                                     \
+        *(*toP)++ = ((lo2 & 0x3f) | 0x80);                                     \
+        break;                                                                 \
+      }                                                                        \
+    }                                                                          \
+    *fromP = from;                                                             \
+    if (from < fromLim)                                                        \
+      return XML_CONVERT_INPUT_INCOMPLETE;                                     \
+    else                                                                       \
+      return XML_CONVERT_COMPLETED;                                            \
+  }
 
-#define DEFINE_UTF16_TO_UTF16(E) \
-static enum XML_Convert_Result  PTRCALL \
-E ## toUtf16(const ENCODING *UNUSED_P(enc), \
-             const char **fromP, const char *fromLim, \
-             unsigned short **toP, const unsigned short *toLim) \
-{ \
-  enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \
-  fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1);  /* shrink to even */ \
-  /* Avoid copying first half only of surrogate */ \
-  if (fromLim - *fromP > ((toLim - *toP) << 1) \
-      && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \
-    fromLim -= 2; \
-    res = XML_CONVERT_INPUT_INCOMPLETE; \
-  } \
-  for (; *fromP < fromLim && *toP < toLim; *fromP += 2) \
-    *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP); \
-  if ((*toP == toLim) && (*fromP < fromLim)) \
-    return XML_CONVERT_OUTPUT_EXHAUSTED; \
-  else \
-    return res; \
-}
+#define DEFINE_UTF16_TO_UTF16(E)                                               \
+  static enum XML_Convert_Result PTRCALL E##toUtf16(                           \
+      const ENCODING *enc, const char **fromP, const char *fromLim,            \
+      unsigned short **toP, const unsigned short *toLim) {                     \
+    enum XML_Convert_Result res = XML_CONVERT_COMPLETED;                       \
+    UNUSED_P(enc);                                                             \
+    fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */  \
+    /* Avoid copying first half only of surrogate */                           \
+    if (fromLim - *fromP > ((toLim - *toP) << 1)                               \
+        && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) {                             \
+      fromLim -= 2;                                                            \
+      res = XML_CONVERT_INPUT_INCOMPLETE;                                      \
+    }                                                                          \
+    for (; *fromP < fromLim && *toP < toLim; *fromP += 2)                      \
+      *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP);                      \
+    if ((*toP == toLim) && (*fromP < fromLim))                                 \
+      return XML_CONVERT_OUTPUT_EXHAUSTED;                                     \
+    else                                                                       \
+      return res;                                                              \
+  }
 
-#define SET2(ptr, ch) \
-  (((ptr)[0] = ((ch) & 0xff)), ((ptr)[1] = ((ch) >> 8)))
+#define SET2(ptr, ch) (((ptr)[0] = ((ch)&0xff)), ((ptr)[1] = ((ch) >> 8)))
 #define GET_LO(ptr) ((unsigned char)(ptr)[0])
 #define GET_HI(ptr) ((unsigned char)(ptr)[1])
 
@@ -768,8 +721,7 @@ DEFINE_UTF16_TO_UTF16(little2_)
 #undef GET_LO
 #undef GET_HI
 
-#define SET2(ptr, ch) \
-  (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch) & 0xFF)))
+#define SET2(ptr, ch) (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch)&0xFF)))
 #define GET_LO(ptr) ((unsigned char)(ptr)[1])
 #define GET_HI(ptr) ((unsigned char)(ptr)[0])
 
@@ -780,292 +732,279 @@ DEFINE_UTF16_TO_UTF16(big2_)
 #undef GET_LO
 #undef GET_HI
 
-#define LITTLE2_BYTE_TYPE(enc, p) \
- ((p)[1] == 0 \
-  ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)] \
-  : unicode_byte_type((p)[1], (p)[0]))
-#define LITTLE2_BYTE_TO_ASCII(enc, p) ((p)[1] == 0 ? (p)[0] : -1)
-#define LITTLE2_CHAR_MATCHES(enc, p, c) ((p)[1] == 0 && (p)[0] == c)
-#define LITTLE2_IS_NAME_CHAR_MINBPC(enc, p) \
+#define LITTLE2_BYTE_TYPE(enc, p)                                              \
+  ((p)[1] == 0 ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)]  \
+               : unicode_byte_type((p)[1], (p)[0]))
+#define LITTLE2_BYTE_TO_ASCII(p) ((p)[1] == 0 ? (p)[0] : -1)
+#define LITTLE2_CHAR_MATCHES(p, c) ((p)[1] == 0 && (p)[0] == c)
+#define LITTLE2_IS_NAME_CHAR_MINBPC(p)                                         \
   UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0])
-#define LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
+#define LITTLE2_IS_NMSTRT_CHAR_MINBPC(p)                                       \
   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[1], (unsigned char)p[0])
 
 #ifdef XML_MIN_SIZE
 
 static int PTRFASTCALL
-little2_byteType(const ENCODING *enc, const char *p)
-{
+little2_byteType(const ENCODING *enc, const char *p) {
   return LITTLE2_BYTE_TYPE(enc, p);
 }
 
 static int PTRFASTCALL
-little2_byteToAscii(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_BYTE_TO_ASCII(enc, p);
+little2_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_BYTE_TO_ASCII(p);
 }
 
 static int PTRCALL
-little2_charMatches(const ENCODING *enc, const char *p, int c)
-{
-  return LITTLE2_CHAR_MATCHES(enc, p, c);
+little2_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
+  return LITTLE2_CHAR_MATCHES(p, c);
 }
 
 static int PTRFASTCALL
-little2_isNameMin(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_IS_NAME_CHAR_MINBPC(enc, p);
+little2_isNameMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_IS_NAME_CHAR_MINBPC(p);
 }
 
 static int PTRFASTCALL
-little2_isNmstrtMin(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p);
+little2_isNmstrtMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_IS_NMSTRT_CHAR_MINBPC(p);
 }
 
-#undef VTABLE
-#define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
+#  undef VTABLE
+#  define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
 
 #else /* not XML_MIN_SIZE */
 
-#undef PREFIX
-#define PREFIX(ident) little2_ ## ident
-#define MINBPC(enc) 2
+#  undef PREFIX
+#  define PREFIX(ident) little2_##ident
+#  define MINBPC(enc) 2
 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
-#define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
-#define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(enc, p)
-#define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(enc, p, c)
-#define IS_NAME_CHAR(enc, p, n) 0
-#define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(enc, p)
-#define IS_NMSTRT_CHAR(enc, p, n) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p)
-
-#define XML_TOK_IMPL_C
-#include "xmltok_impl.c"
-#undef XML_TOK_IMPL_C
-
-#undef MINBPC
-#undef BYTE_TYPE
-#undef BYTE_TO_ASCII
-#undef CHAR_MATCHES
-#undef IS_NAME_CHAR
-#undef IS_NAME_CHAR_MINBPC
-#undef IS_NMSTRT_CHAR
-#undef IS_NMSTRT_CHAR_MINBPC
-#undef IS_INVALID_CHAR
+#  define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
+#  define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p)
+#  define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c)
+#  define IS_NAME_CHAR(enc, p, n) 0
+#  define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p)
+#  define IS_NMSTRT_CHAR(enc, p, n) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p)
+
+#  define XML_TOK_IMPL_C
+#  include "xmltok_impl.c"
+#  undef XML_TOK_IMPL_C
+
+#  undef MINBPC
+#  undef BYTE_TYPE
+#  undef BYTE_TO_ASCII
+#  undef CHAR_MATCHES
+#  undef IS_NAME_CHAR
+#  undef IS_NAME_CHAR_MINBPC
+#  undef IS_NMSTRT_CHAR
+#  undef IS_NMSTRT_CHAR_MINBPC
+#  undef IS_INVALID_CHAR
 
 #endif /* not XML_MIN_SIZE */
 
 #ifdef XML_NS
 
-static const struct normal_encoding little2_encoding_ns = {
-  { VTABLE, 2, 0,
-#if BYTEORDER == 1234
-    1
-#else
-    0
-#endif
-  },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding little2_encoding_ns
+    = {{VTABLE, 2, 0,
+#  if BYTEORDER == 1234
+        1
+#  else
+        0
+#  endif
+       },
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding little2_encoding = {
-  { VTABLE, 2, 0,
+static const struct normal_encoding little2_encoding
+    = {{VTABLE, 2, 0,
 #if BYTEORDER == 1234
-    1
+        1
 #else
-    0
+        0
 #endif
-  },
-  {
+       },
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #if BYTEORDER != 4321
 
-#ifdef XML_NS
+#  ifdef XML_NS
 
-static const struct normal_encoding internal_little2_encoding_ns = {
-  { VTABLE, 2, 0, 1 },
-  {
-#include "iasciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_little2_encoding_ns
+    = {{VTABLE, 2, 0, 1},
+       {
+#    include "iasciitab.h"
+#    include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
-#endif
+#  endif
 
-static const struct normal_encoding internal_little2_encoding = {
-  { VTABLE, 2, 0, 1 },
-  {
-#define BT_COLON BT_NMSTRT
-#include "iasciitab.h"
-#undef BT_COLON
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_little2_encoding
+    = {{VTABLE, 2, 0, 1},
+       {
+#  define BT_COLON BT_NMSTRT
+#  include "iasciitab.h"
+#  undef BT_COLON
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #endif
 
-
-#define BIG2_BYTE_TYPE(enc, p) \
- ((p)[0] == 0 \
-  ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]] \
-  : unicode_byte_type((p)[0], (p)[1]))
-#define BIG2_BYTE_TO_ASCII(enc, p) ((p)[0] == 0 ? (p)[1] : -1)
-#define BIG2_CHAR_MATCHES(enc, p, c) ((p)[0] == 0 && (p)[1] == c)
-#define BIG2_IS_NAME_CHAR_MINBPC(enc, p) \
+#define BIG2_BYTE_TYPE(enc, p)                                                 \
+  ((p)[0] == 0                                                                 \
+       ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]]        \
+       : unicode_byte_type((p)[0], (p)[1]))
+#define BIG2_BYTE_TO_ASCII(p) ((p)[0] == 0 ? (p)[1] : -1)
+#define BIG2_CHAR_MATCHES(p, c) ((p)[0] == 0 && (p)[1] == c)
+#define BIG2_IS_NAME_CHAR_MINBPC(p)                                            \
   UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1])
-#define BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
+#define BIG2_IS_NMSTRT_CHAR_MINBPC(p)                                          \
   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[0], (unsigned char)p[1])
 
 #ifdef XML_MIN_SIZE
 
 static int PTRFASTCALL
-big2_byteType(const ENCODING *enc, const char *p)
-{
+big2_byteType(const ENCODING *enc, const char *p) {
   return BIG2_BYTE_TYPE(enc, p);
 }
 
 static int PTRFASTCALL
-big2_byteToAscii(const ENCODING *enc, const char *p)
-{
-  return BIG2_BYTE_TO_ASCII(enc, p);
+big2_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_BYTE_TO_ASCII(p);
 }
 
 static int PTRCALL
-big2_charMatches(const ENCODING *enc, const char *p, int c)
-{
-  return BIG2_CHAR_MATCHES(enc, p, c);
+big2_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
+  return BIG2_CHAR_MATCHES(p, c);
 }
 
 static int PTRFASTCALL
-big2_isNameMin(const ENCODING *enc, const char *p)
-{
-  return BIG2_IS_NAME_CHAR_MINBPC(enc, p);
+big2_isNameMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_IS_NAME_CHAR_MINBPC(p);
 }
 
 static int PTRFASTCALL
-big2_isNmstrtMin(const ENCODING *enc, const char *p)
-{
-  return BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p);
+big2_isNmstrtMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_IS_NMSTRT_CHAR_MINBPC(p);
 }
 
-#undef VTABLE
-#define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
+#  undef VTABLE
+#  define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
 
 #else /* not XML_MIN_SIZE */
 
-#undef PREFIX
-#define PREFIX(ident) big2_ ## ident
-#define MINBPC(enc) 2
+#  undef PREFIX
+#  define PREFIX(ident) big2_##ident
+#  define MINBPC(enc) 2
 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
-#define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
-#define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(enc, p)
-#define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(enc, p, c)
-#define IS_NAME_CHAR(enc, p, n) 0
-#define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(enc, p)
-#define IS_NMSTRT_CHAR(enc, p, n) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p)
-
-#define XML_TOK_IMPL_C
-#include "xmltok_impl.c"
-#undef XML_TOK_IMPL_C
-
-#undef MINBPC
-#undef BYTE_TYPE
-#undef BYTE_TO_ASCII
-#undef CHAR_MATCHES
-#undef IS_NAME_CHAR
-#undef IS_NAME_CHAR_MINBPC
-#undef IS_NMSTRT_CHAR
-#undef IS_NMSTRT_CHAR_MINBPC
-#undef IS_INVALID_CHAR
+#  define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
+#  define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p)
+#  define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c)
+#  define IS_NAME_CHAR(enc, p, n) 0
+#  define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p)
+#  define IS_NMSTRT_CHAR(enc, p, n) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p)
+
+#  define XML_TOK_IMPL_C
+#  include "xmltok_impl.c"
+#  undef XML_TOK_IMPL_C
+
+#  undef MINBPC
+#  undef BYTE_TYPE
+#  undef BYTE_TO_ASCII
+#  undef CHAR_MATCHES
+#  undef IS_NAME_CHAR
+#  undef IS_NAME_CHAR_MINBPC
+#  undef IS_NMSTRT_CHAR
+#  undef IS_NMSTRT_CHAR_MINBPC
+#  undef IS_INVALID_CHAR
 
 #endif /* not XML_MIN_SIZE */
 
 #ifdef XML_NS
 
-static const struct normal_encoding big2_encoding_ns = {
-  { VTABLE, 2, 0,
-#if BYTEORDER == 4321
-  1
-#else
-  0
-#endif
-  },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding big2_encoding_ns
+    = {{VTABLE, 2, 0,
+#  if BYTEORDER == 4321
+        1
+#  else
+        0
+#  endif
+       },
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding big2_encoding = {
-  { VTABLE, 2, 0,
+static const struct normal_encoding big2_encoding
+    = {{VTABLE, 2, 0,
 #if BYTEORDER == 4321
-  1
+        1
 #else
-  0
+        0
 #endif
-  },
-  {
+       },
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #if BYTEORDER != 1234
 
-#ifdef XML_NS
+#  ifdef XML_NS
 
-static const struct normal_encoding internal_big2_encoding_ns = {
-  { VTABLE, 2, 0, 1 },
-  {
-#include "iasciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_big2_encoding_ns
+    = {{VTABLE, 2, 0, 1},
+       {
+#    include "iasciitab.h"
+#    include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
-#endif
+#  endif
 
-static const struct normal_encoding internal_big2_encoding = {
-  { VTABLE, 2, 0, 1 },
-  {
-#define BT_COLON BT_NMSTRT
-#include "iasciitab.h"
-#undef BT_COLON
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_big2_encoding
+    = {{VTABLE, 2, 0, 1},
+       {
+#  define BT_COLON BT_NMSTRT
+#  include "iasciitab.h"
+#  undef BT_COLON
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #endif
 
 #undef PREFIX
 
 static int FASTCALL
-streqci(const char *s1, const char *s2)
-{
+streqci(const char *s1, const char *s2) {
   for (;;) {
     char c1 = *s1++;
     char c2 = *s2++;
@@ -1079,22 +1018,21 @@ streqci(const char *s1, const char *s2)
       c2 += ASCII_A - ASCII_a; /* LCOV_EXCL_LINE */
     if (c1 != c2)
       return 0;
-    if (!c1)
+    if (! c1)
       break;
   }
   return 1;
 }
 
 static void PTRCALL
-initUpdatePosition(const ENCODING *UNUSED_P(enc), const char *ptr,
-                   const char *end, POSITION *pos)
-{
+initUpdatePosition(const ENCODING *enc, const char *ptr, const char *end,
+                   POSITION *pos) {
+  UNUSED_P(enc);
   normal_updatePosition(&utf8_encoding.enc, ptr, end, pos);
 }
 
 static int
-toAscii(const ENCODING *enc, const char *ptr, const char *end)
-{
+toAscii(const ENCODING *enc, const char *ptr, const char *end) {
   char buf[1];
   char *p = buf;
   XmlUtf8Convert(enc, &ptr, end, &p, p + 1);
@@ -1105,8 +1043,7 @@ toAscii(const ENCODING *enc, const char *ptr, const char *end)
 }
 
 static int FASTCALL
-isSpace(int c)
-{
+isSpace(int c) {
   switch (c) {
   case 0x20:
   case 0xD:
@@ -1121,21 +1058,16 @@ isSpace(int c)
    followed by name=val.
 */
 static int
-parsePseudoAttribute(const ENCODING *enc,
-                     const char *ptr,
-                     const char *end,
-                     const char **namePtr,
-                     const char **nameEndPtr,
-                     const char **valPtr,
-                     const char **nextTokPtr)
-{
+parsePseudoAttribute(const ENCODING *enc, const char *ptr, const char *end,
+                     const char **namePtr, const char **nameEndPtr,
+                     const char **valPtr, const char **nextTokPtr) {
   int c;
   char open;
   if (ptr == end) {
     *namePtr = NULL;
     return 1;
   }
-  if (!isSpace(toAscii(enc, ptr, end))) {
+  if (! isSpace(toAscii(enc, ptr, end))) {
     *nextTokPtr = ptr;
     return 0;
   }
@@ -1191,12 +1123,9 @@ parsePseudoAttribute(const ENCODING *enc,
     c = toAscii(enc, ptr, end);
     if (c == open)
       break;
-    if (!(ASCII_a <= c && c <= ASCII_z)
-        && !(ASCII_A <= c && c <= ASCII_Z)
-        && !(ASCII_0 <= c && c <= ASCII_9)
-        && c != ASCII_PERIOD
-        && c != ASCII_MINUS
-        && c != ASCII_UNDERSCORE) {
+    if (! (ASCII_a <= c && c <= ASCII_z) && ! (ASCII_A <= c && c <= ASCII_Z)
+        && ! (ASCII_0 <= c && c <= ASCII_9) && c != ASCII_PERIOD
+        && c != ASCII_MINUS && c != ASCII_UNDERSCORE) {
       *nextTokPtr = ptr;
       return 0;
     }
@@ -1205,68 +1134,52 @@ parsePseudoAttribute(const ENCODING *enc,
   return 1;
 }
 
-static const char KW_version[] = {
-  ASCII_v, ASCII_e, ASCII_r, ASCII_s, ASCII_i, ASCII_o, ASCII_n, '\0'
-};
+static const char KW_version[]
+    = {ASCII_v, ASCII_e, ASCII_r, ASCII_s, ASCII_i, ASCII_o, ASCII_n, '\0'};
 
-static const char KW_encoding[] = {
-  ASCII_e, ASCII_n, ASCII_c, ASCII_o, ASCII_d, ASCII_i, ASCII_n, ASCII_g, '\0'
-};
+static const char KW_encoding[] = {ASCII_e, ASCII_n, ASCII_c, ASCII_o, ASCII_d,
+                                   ASCII_i, ASCII_n, ASCII_g, '\0'};
 
-static const char KW_standalone[] = {
-  ASCII_s, ASCII_t, ASCII_a, ASCII_n, ASCII_d, ASCII_a, ASCII_l, ASCII_o,
-  ASCII_n, ASCII_e, '\0'
-};
+static const char KW_standalone[]
+    = {ASCII_s, ASCII_t, ASCII_a, ASCII_n, ASCII_d, ASCII_a,
+       ASCII_l, ASCII_o, ASCII_n, ASCII_e, '\0'};
 
-static const char KW_yes[] = {
-  ASCII_y, ASCII_e, ASCII_s,  '\0'
-};
+static const char KW_yes[] = {ASCII_y, ASCII_e, ASCII_s, '\0'};
 
-static const char KW_no[] = {
-  ASCII_n, ASCII_o,  '\0'
-};
+static const char KW_no[] = {ASCII_n, ASCII_o, '\0'};
 
 static int
-doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
-                                                 const char *,
+doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *,
                                                  const char *),
-               int isGeneralTextEntity,
-               const ENCODING *enc,
-               const char *ptr,
-               const char *end,
-               const char **badPtr,
-               const char **versionPtr,
-               const char **versionEndPtr,
-               const char **encodingName,
-               const ENCODING **encoding,
-               int *standalone)
-{
+               int isGeneralTextEntity, const ENCODING *enc, const char *ptr,
+               const char *end, const char **badPtr, const char **versionPtr,
+               const char **versionEndPtr, const char **encodingName,
+               const ENCODING **encoding, int *standalone) {
   const char *val = NULL;
   const char *name = NULL;
   const char *nameEnd = NULL;
   ptr += 5 * enc->minBytesPerChar;
   end -= 2 * enc->minBytesPerChar;
-  if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)
-      || !name) {
+  if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)
+      || ! name) {
     *badPtr = ptr;
     return 0;
   }
-  if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_version)) {
-    if (!isGeneralTextEntity) {
+  if (! XmlNameMatchesAscii(enc, name, nameEnd, KW_version)) {
+    if (! isGeneralTextEntity) {
       *badPtr = name;
       return 0;
     }
-  }
-  else {
+  } else {
     if (versionPtr)
       *versionPtr = val;
     if (versionEndPtr)
       *versionEndPtr = ptr;
-    if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
+    if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
       *badPtr = ptr;
       return 0;
     }
-    if (!name) {
+    if (! name) {
       if (isGeneralTextEntity) {
         /* a TextDecl must have an EncodingDecl */
         *badPtr = ptr;
@@ -1277,7 +1190,7 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
   }
   if (XmlNameMatchesAscii(enc, name, nameEnd, KW_encoding)) {
     int c = toAscii(enc, val, end);
-    if (!(ASCII_a <= c && c <= ASCII_z) && !(ASCII_A <= c && c <= ASCII_Z)) {
+    if (! (ASCII_a <= c && c <= ASCII_z) && ! (ASCII_A <= c && c <= ASCII_Z)) {
       *badPtr = val;
       return 0;
     }
@@ -1285,14 +1198,14 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
       *encodingName = val;
     if (encoding)
       *encoding = encodingFinder(enc, val, ptr - enc->minBytesPerChar);
-    if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
+    if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
       *badPtr = ptr;
       return 0;
     }
-    if (!name)
+    if (! name)
       return 1;
   }
-  if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_standalone)
+  if (! XmlNameMatchesAscii(enc, name, nameEnd, KW_standalone)
       || isGeneralTextEntity) {
     *badPtr = name;
     return 0;
@@ -1300,12 +1213,10 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
   if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_yes)) {
     if (standalone)
       *standalone = 1;
-  }
-  else if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_no)) {
+  } else if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_no)) {
     if (standalone)
       *standalone = 0;
-  }
-  else {
+  } else {
     *badPtr = val;
     return 0;
   }
@@ -1319,11 +1230,16 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
 }
 
 static int FASTCALL
-checkCharRefNumber(int result)
-{
+checkCharRefNumber(int result) {
   switch (result >> 8) {
-  case 0xD8: case 0xD9: case 0xDA: case 0xDB:
-  case 0xDC: case 0xDD: case 0xDE: case 0xDF:
+  case 0xD8:
+  case 0xD9:
+  case 0xDA:
+  case 0xDB:
+  case 0xDC:
+  case 0xDD:
+  case 0xDE:
+  case 0xDF:
     return -1;
   case 0:
     if (latin1_encoding.type[result] == BT_NONXML)
@@ -1338,8 +1254,7 @@ checkCharRefNumber(int result)
 }
 
 int FASTCALL
-XmlUtf8Encode(int c, char *buf)
-{
+XmlUtf8Encode(int c, char *buf) {
   enum {
     /* minN is minimum legal resulting value for N byte sequence */
     min2 = 0x80,
@@ -1375,8 +1290,7 @@ XmlUtf8Encode(int c, char *buf)
 }
 
 int FASTCALL
-XmlUtf16Encode(int charNum, unsigned short *buf)
-{
+XmlUtf16Encode(int charNum, unsigned short *buf) {
   if (charNum < 0)
     return 0;
   if (charNum < 0x10000) {
@@ -1400,17 +1314,15 @@ struct unknown_encoding {
   char utf8[256][4];
 };
 
-#define AS_UNKNOWN_ENCODING(enc)  ((const struct unknown_encoding *) (enc))
+#define AS_UNKNOWN_ENCODING(enc) ((const struct unknown_encoding *)(enc))
 
 int
-XmlSizeOfUnknownEncoding(void)
-{
+XmlSizeOfUnknownEncoding(void) {
   return sizeof(struct unknown_encoding);
 }
 
 static int PTRFASTCALL
-unknown_isName(const ENCODING *enc, const char *p)
-{
+unknown_isName(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   if (c & ~0xFFFF)
@@ -1419,8 +1331,7 @@ unknown_isName(const ENCODING *enc, const char *p)
 }
 
 static int PTRFASTCALL
-unknown_isNmstrt(const ENCODING *enc, const char *p)
-{
+unknown_isNmstrt(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   if (c & ~0xFFFF)
@@ -1429,18 +1340,15 @@ unknown_isNmstrt(const ENCODING *enc, const char *p)
 }
 
 static int PTRFASTCALL
-unknown_isInvalid(const ENCODING *enc, const char *p)
-{
+unknown_isInvalid(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   return (c & ~0xFFFF) || checkCharRefNumber(c) < 0;
 }
 
 static enum XML_Convert_Result PTRCALL
-unknown_toUtf8(const ENCODING *enc,
-               const char **fromP, const char *fromLim,
-               char **toP, const char *toLim)
-{
+unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+               char **toP, const char *toLim) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   char buf[XML_UTF8_ENCODE_MAX];
   for (;;) {
@@ -1458,8 +1366,7 @@ unknown_toUtf8(const ENCODING *enc,
       utf8 = buf;
       *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP]
                  - (BT_LEAD2 - 2));
-    }
-    else {
+    } else {
       if (n > toLim - *toP)
         return XML_CONVERT_OUTPUT_EXHAUSTED;
       (*fromP)++;
@@ -1470,20 +1377,16 @@ unknown_toUtf8(const ENCODING *enc,
 }
 
 static enum XML_Convert_Result PTRCALL
-unknown_toUtf16(const ENCODING *enc,
-                const char **fromP, const char *fromLim,
-                unsigned short **toP, const unsigned short *toLim)
-{
+unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+                unsigned short **toP, const unsigned short *toLim) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   while (*fromP < fromLim && *toP < toLim) {
     unsigned short c = uenc->utf16[(unsigned char)**fromP];
     if (c == 0) {
-      c = (unsigned short)
-          uenc->convert(uenc->userData, *fromP);
+      c = (unsigned short)uenc->convert(uenc->userData, *fromP);
       *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP]
                  - (BT_LEAD2 - 2));
-    }
-    else
+    } else
       (*fromP)++;
     *(*toP)++ = c;
   }
@@ -1495,19 +1398,14 @@ unknown_toUtf16(const ENCODING *enc,
 }
 
 ENCODING *
-XmlInitUnknownEncoding(void *mem,
-                       int *table,
-                       CONVERTER convert,
-                       void *userData)
-{
+XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert,
+                       void *userData) {
   int i;
   struct unknown_encoding *e = (struct unknown_encoding *)mem;
-  for (i = 0; i < (int)sizeof(struct normal_encoding); i++)
-    ((char *)mem)[i] = ((char *)&latin1_encoding)[i];
+  memcpy(mem, &latin1_encoding, sizeof(struct normal_encoding));
   for (i = 0; i < 128; i++)
     if (latin1_encoding.type[i] != BT_OTHER
-        && latin1_encoding.type[i] != BT_NONXML
-        && table[i] != i)
+        && latin1_encoding.type[i] != BT_NONXML && table[i] != i)
       return 0;
   for (i = 0; i < 256; i++) {
     int c = table[i];
@@ -1517,35 +1415,30 @@ XmlInitUnknownEncoding(void *mem,
       e->utf16[i] = 0xFFFF;
       e->utf8[i][0] = 1;
       e->utf8[i][1] = 0;
-    }
-    else if (c < 0) {
+    } else if (c < 0) {
       if (c < -4)
         return 0;
       /* Multi-byte sequences need a converter function */
-      if (!convert)
+      if (! convert)
         return 0;
       e->normal.type[i] = (unsigned char)(BT_LEAD2 - (c + 2));
       e->utf8[i][0] = 0;
       e->utf16[i] = 0;
-    }
-    else if (c < 0x80) {
+    } else if (c < 0x80) {
       if (latin1_encoding.type[c] != BT_OTHER
-          && latin1_encoding.type[c] != BT_NONXML
-          && c != i)
+          && latin1_encoding.type[c] != BT_NONXML && c != i)
         return 0;
       e->normal.type[i] = latin1_encoding.type[c];
       e->utf8[i][0] = 1;
       e->utf8[i][1] = (char)c;
       e->utf16[i] = (unsigned short)(c == 0 ? 0xFFFF : c);
-    }
-    else if (checkCharRefNumber(c) < 0) {
+    } else if (checkCharRefNumber(c) < 0) {
       e->normal.type[i] = BT_NONXML;
       /* This shouldn't really get used. */
       e->utf16[i] = 0xFFFF;
       e->utf8[i][0] = 1;
       e->utf8[i][1] = 0;
-    }
-    else {
+    } else {
       if (c > 0xFFFF)
         return 0;
       if (UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xff))
@@ -1590,44 +1483,32 @@ enum {
   NO_ENC
 };
 
-static const char KW_ISO_8859_1[] = {
-  ASCII_I, ASCII_S, ASCII_O, ASCII_MINUS, ASCII_8, ASCII_8, ASCII_5, ASCII_9,
-  ASCII_MINUS, ASCII_1, '\0'
-};
-static const char KW_US_ASCII[] = {
-  ASCII_U, ASCII_S, ASCII_MINUS, ASCII_A, ASCII_S, ASCII_C, ASCII_I, ASCII_I,
-  '\0'
-};
-static const char KW_UTF_8[] =  {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_8, '\0'
-};
-static const char KW_UTF_16[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, '\0'
-};
-static const char KW_UTF_16BE[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_B, ASCII_E,
-  '\0'
-};
-static const char KW_UTF_16LE[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_L, ASCII_E,
-  '\0'
-};
+static const char KW_ISO_8859_1[]
+    = {ASCII_I, ASCII_S, ASCII_O,     ASCII_MINUS, ASCII_8, ASCII_8,
+       ASCII_5, ASCII_9, ASCII_MINUS, ASCII_1,     '\0'};
+static const char KW_US_ASCII[]
+    = {ASCII_U, ASCII_S, ASCII_MINUS, ASCII_A, ASCII_S,
+       ASCII_C, ASCII_I, ASCII_I,     '\0'};
+static const char KW_UTF_8[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_8, '\0'};
+static const char KW_UTF_16[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, '\0'};
+static const char KW_UTF_16BE[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1,
+       ASCII_6, ASCII_B, ASCII_E, '\0'};
+static const char KW_UTF_16LE[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1,
+       ASCII_6, ASCII_L, ASCII_E, '\0'};
 
 static int FASTCALL
-getEncodingIndex(const char *name)
-{
-  static const char * const encodingNames[] = {
-    KW_ISO_8859_1,
-    KW_US_ASCII,
-    KW_UTF_8,
-    KW_UTF_16,
-    KW_UTF_16BE,
-    KW_UTF_16LE,
+getEncodingIndex(const char *name) {
+  static const char *const encodingNames[] = {
+      KW_ISO_8859_1, KW_US_ASCII, KW_UTF_8, KW_UTF_16, KW_UTF_16BE, KW_UTF_16LE,
   };
   int i;
   if (name == NULL)
     return NO_ENC;
-  for (i = 0; i < (int)(sizeof(encodingNames)/sizeof(encodingNames[0])); i++)
+  for (i = 0; i < (int)(sizeof(encodingNames) / sizeof(encodingNames[0])); i++)
     if (streqci(name, encodingNames[i]))
       return i;
   return UNKNOWN_ENC;
@@ -1647,15 +1528,9 @@ getEncodingIndex(const char *name)
    XML_PROLOG_STATE otherwise.
 */
 
-
 static int
-initScan(const ENCODING * const *encodingTable,
-         const INIT_ENCODING *enc,
-         int state,
-         const char *ptr,
-         const char *end,
-         const char **nextTokPtr)
-{
+initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc,
+         int state, const char *ptr, const char *end, const char **nextTokPtr) {
   const ENCODING **encPtr;
 
   if (ptr >= end)
@@ -1680,20 +1555,17 @@ initScan(const ENCODING * const *encodingTable,
     case 0xFE:
     case 0xFF:
     case 0xEF: /* possibly first byte of UTF-8 BOM */
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       /* fall through */
     case 0x00:
     case 0x3C:
       return XML_TOK_PARTIAL;
     }
-  }
-  else {
+  } else {
     switch (((unsigned char)ptr[0] << 8) | (unsigned char)ptr[1]) {
     case 0xFEFF:
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       *nextTokPtr = ptr + 2;
       *encPtr = encodingTable[UTF_16BE_ENC];
@@ -1707,8 +1579,7 @@ initScan(const ENCODING * const *encodingTable,
       *encPtr = encodingTable[UTF_16LE_ENC];
       return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
     case 0xFFFE:
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       *nextTokPtr = ptr + 2;
       *encPtr = encodingTable[UTF_16LE_ENC];
@@ -1723,8 +1594,8 @@ initScan(const ENCODING * const *encodingTable,
       */
       if (state == XML_CONTENT_STATE) {
         int e = INIT_ENC_INDEX(enc);
-        if (e == ISO_8859_1_ENC || e == UTF_16BE_ENC
-            || e == UTF_16LE_ENC || e == UTF_16_ENC)
+        if (e == ISO_8859_1_ENC || e == UTF_16BE_ENC || e == UTF_16LE_ENC
+            || e == UTF_16_ENC)
           break;
       }
       if (ptr + 2 == end)
@@ -1747,8 +1618,7 @@ initScan(const ENCODING * const *encodingTable,
           break;
         *encPtr = encodingTable[UTF_16BE_ENC];
         return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
-      }
-      else if (ptr[1] == '\0') {
+      } else if (ptr[1] == '\0') {
         /* We could recover here in the case:
             - parsing an external entity
             - second byte is 0
@@ -1770,7 +1640,6 @@ initScan(const ENCODING * const *encodingTable,
   return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
 }
 
-
 #define NS(x) x
 #define ns(x) x
 #define XML_TOK_NS_C
@@ -1781,22 +1650,19 @@ initScan(const ENCODING * const *encodingTable,
 
 #ifdef XML_NS
 
-#define NS(x) x ## NS
-#define ns(x) x ## _ns
+#  define NS(x) x##NS
+#  define ns(x) x##_ns
 
-#define XML_TOK_NS_C
-#include "xmltok_ns.c"
-#undef XML_TOK_NS_C
+#  define XML_TOK_NS_C
+#  include "xmltok_ns.c"
+#  undef XML_TOK_NS_C
 
-#undef NS
-#undef ns
+#  undef NS
+#  undef ns
 
 ENCODING *
-XmlInitUnknownEncodingNS(void *mem,
-                         int *table,
-                         CONVERTER convert,
-                         void *userData)
-{
+XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert,
+                         void *userData) {
   ENCODING *enc = XmlInitUnknownEncoding(mem, table, convert, userData);
   if (enc)
     ((struct normal_encoding *)enc)->type[ASCII_COLON] = BT_COLON;
diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h
index 50926f38ab32..2adbf5307bef 100644
--- a/Modules/expat/xmltok.h
+++ b/Modules/expat/xmltok.h
@@ -38,16 +38,18 @@ extern "C" {
 #endif
 
 /* The following token may be returned by XmlContentTok */
-#define XML_TOK_TRAILING_RSQB -5 /* ] or ]] at the end of the scan; might be
-                                    start of illegal ]]> sequence */
+#define XML_TOK_TRAILING_RSQB                                                  \
+  -5 /* ] or ]] at the end of the scan; might be                               \
+        start of illegal ]]> sequence */
 /* The following tokens may be returned by both XmlPrologTok and
    XmlContentTok.
 */
-#define XML_TOK_NONE -4          /* The string to be scanned is empty */
-#define XML_TOK_TRAILING_CR -3   /* A CR at the end of the scan;
-                                    might be part of CRLF sequence */
-#define XML_TOK_PARTIAL_CHAR -2  /* only part of a multibyte sequence */
-#define XML_TOK_PARTIAL -1       /* only part of a token */
+#define XML_TOK_NONE -4 /* The string to be scanned is empty */
+#define XML_TOK_TRAILING_CR                                                    \
+  -3                            /* A CR at the end of the scan;                \
+                                   might be part of CRLF sequence */
+#define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */
+#define XML_TOK_PARTIAL -1      /* only part of a token */
 #define XML_TOK_INVALID 0
 
 /* The following tokens are returned by XmlContentTok; some are also
@@ -62,24 +64,24 @@ extern "C" {
 #define XML_TOK_DATA_NEWLINE 7
 #define XML_TOK_CDATA_SECT_OPEN 8
 #define XML_TOK_ENTITY_REF 9
-#define XML_TOK_CHAR_REF 10               /* numeric character reference */
+#define XML_TOK_CHAR_REF 10 /* numeric character reference */
 
 /* The following tokens may be returned by both XmlPrologTok and
    XmlContentTok.
 */
-#define XML_TOK_PI 11                     /* processing instruction */
-#define XML_TOK_XML_DECL 12               /* XML decl or text decl */
+#define XML_TOK_PI 11       /* processing instruction */
+#define XML_TOK_XML_DECL 12 /* XML decl or text decl */
 #define XML_TOK_COMMENT 13
-#define XML_TOK_BOM 14                    /* Byte order mark */
+#define XML_TOK_BOM 14 /* Byte order mark */
 
 /* The following tokens are returned only by XmlPrologTok */
 #define XML_TOK_PROLOG_S 15
-#define XML_TOK_DECL_OPEN 16              /* <!foo */
-#define XML_TOK_DECL_CLOSE 17             /* > */
+#define XML_TOK_DECL_OPEN 16  /* <!foo */
+#define XML_TOK_DECL_CLOSE 17 /* > */
 #define XML_TOK_NAME 18
 #define XML_TOK_NMTOKEN 19
-#define XML_TOK_POUND_NAME 20             /* #name */
-#define XML_TOK_OR 21                     /* | */
+#define XML_TOK_POUND_NAME 20 /* #name */
+#define XML_TOK_OR 21         /* | */
 #define XML_TOK_PERCENT 22
 #define XML_TOK_OPEN_PAREN 23
 #define XML_TOK_CLOSE_PAREN 24
@@ -90,14 +92,14 @@ extern "C" {
 #define XML_TOK_INSTANCE_START 29
 
 /* The following occur only in element type declarations */
-#define XML_TOK_NAME_QUESTION 30          /* name? */
-#define XML_TOK_NAME_ASTERISK 31          /* name* */
-#define XML_TOK_NAME_PLUS 32              /* name+ */
-#define XML_TOK_COND_SECT_OPEN 33         /* <![ */
-#define XML_TOK_COND_SECT_CLOSE 34        /* ]]> */
-#define XML_TOK_CLOSE_PAREN_QUESTION 35   /* )? */
-#define XML_TOK_CLOSE_PAREN_ASTERISK 36   /* )* */
-#define XML_TOK_CLOSE_PAREN_PLUS 37       /* )+ */
+#define XML_TOK_NAME_QUESTION 30        /* name? */
+#define XML_TOK_NAME_ASTERISK 31        /* name* */
+#define XML_TOK_NAME_PLUS 32            /* name+ */
+#define XML_TOK_COND_SECT_OPEN 33       /* <![ */
+#define XML_TOK_COND_SECT_CLOSE 34      /* ]]> */
+#define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */
+#define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */
+#define XML_TOK_CLOSE_PAREN_PLUS 37     /* )+ */
 #define XML_TOK_COMMA 38
 
 /* The following token is returned only by XmlAttributeValueTok */
@@ -112,20 +114,20 @@ extern "C" {
 #define XML_TOK_PREFIXED_NAME 41
 
 #ifdef XML_DTD
-#define XML_TOK_IGNORE_SECT 42
+#  define XML_TOK_IGNORE_SECT 42
 #endif /* XML_DTD */
 
 #ifdef XML_DTD
-#define XML_N_STATES 4
+#  define XML_N_STATES 4
 #else /* not XML_DTD */
-#define XML_N_STATES 3
+#  define XML_N_STATES 3
 #endif /* not XML_DTD */
 
 #define XML_PROLOG_STATE 0
 #define XML_CONTENT_STATE 1
 #define XML_CDATA_SECTION_STATE 2
 #ifdef XML_DTD
-#define XML_IGNORE_SECTION_STATE 3
+#  define XML_IGNORE_SECTION_STATE 3
 #endif /* XML_DTD */
 
 #define XML_N_LITERAL_TYPES 2
@@ -153,52 +155,41 @@ typedef struct {
 struct encoding;
 typedef struct encoding ENCODING;
 
-typedef int (PTRCALL *SCANNER)(const ENCODING *,
-                               const char *,
-                               const char *,
-                               const char **);
+typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *,
+                              const char **);
 
 enum XML_Convert_Result {
   XML_CONVERT_COMPLETED = 0,
   XML_CONVERT_INPUT_INCOMPLETE = 1,
-  XML_CONVERT_OUTPUT_EXHAUSTED = 2  /* and therefore potentially input remaining as well */
+  XML_CONVERT_OUTPUT_EXHAUSTED
+  = 2 /* and therefore potentially input remaining as well */
 };
 
 struct encoding {
   SCANNER scanners[XML_N_STATES];
   SCANNER literalScanners[XML_N_LITERAL_TYPES];
-  int (PTRCALL *nameMatchesAscii)(const ENCODING *,
-                                  const char *,
-                                  const char *,
-                                  const char *);
-  int (PTRFASTCALL *nameLength)(const ENCODING *, const char *);
+  int(PTRCALL *nameMatchesAscii)(const ENCODING *, const char *, const char *,
+                                 const char *);
+  int(PTRFASTCALL *nameLength)(const ENCODING *, const char *);
   const char *(PTRFASTCALL *skipS)(const ENCODING *, const char *);
-  int (PTRCALL *getAtts)(const ENCODING *enc,
-                         const char *ptr,
-                         int attsMax,
-                         ATTRIBUTE *atts);
-  int (PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr);
-  int (PTRCALL *predefinedEntityName)(const ENCODING *,
-                                      const char *,
-                                      const char *);
-  void (PTRCALL *updatePosition)(const ENCODING *,
-                                 const char *ptr,
-                                 const char *end,
-                                 POSITION *);
-  int (PTRCALL *isPublicId)(const ENCODING *enc,
-                            const char *ptr,
-                            const char *end,
-                            const char **badPtr);
-  enum XML_Convert_Result (PTRCALL *utf8Convert)(const ENCODING *enc,
-                              const char **fromP,
-                              const char *fromLim,
-                              char **toP,
-                              const char *toLim);
-  enum XML_Convert_Result (PTRCALL *utf16Convert)(const ENCODING *enc,
-                               const char **fromP,
-                               const char *fromLim,
-                               unsigned short **toP,
-                               const unsigned short *toLim);
+  int(PTRCALL *getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
+                        ATTRIBUTE *atts);
+  int(PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr);
+  int(PTRCALL *predefinedEntityName)(const ENCODING *, const char *,
+                                     const char *);
+  void(PTRCALL *updatePosition)(const ENCODING *, const char *ptr,
+                                const char *end, POSITION *);
+  int(PTRCALL *isPublicId)(const ENCODING *enc, const char *ptr,
+                           const char *end, const char **badPtr);
+  enum XML_Convert_Result(PTRCALL *utf8Convert)(const ENCODING *enc,
+                                                const char **fromP,
+                                                const char *fromLim, char **toP,
+                                                const char *toLim);
+  enum XML_Convert_Result(PTRCALL *utf16Convert)(const ENCODING *enc,
+                                                 const char **fromP,
+                                                 const char *fromLim,
+                                                 unsigned short **toP,
+                                                 const unsigned short *toLim);
   int minBytesPerChar;
   char isUtf8;
   char isUtf16;
@@ -225,66 +216,62 @@ struct encoding {
    the prolog outside literals, comments and processing instructions.
 */
 
-
-#define XmlTok(enc, state, ptr, end, nextTokPtr) \
+#define XmlTok(enc, state, ptr, end, nextTokPtr)                               \
   (((enc)->scanners[state])(enc, ptr, end, nextTokPtr))
 
-#define XmlPrologTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr)
+#define XmlPrologTok(enc, ptr, end, nextTokPtr)                                \
+  XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr)
 
-#define XmlContentTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr)
+#define XmlContentTok(enc, ptr, end, nextTokPtr)                               \
+  XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr)
 
-#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr)
+#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr)                          \
+  XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr)
 
 #ifdef XML_DTD
 
-#define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr)
+#  define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr)                       \
+    XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr)
 
 #endif /* XML_DTD */
 
 /* This is used for performing a 2nd-level tokenization on the content
    of a literal that has already been returned by XmlTok.
 */
-#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \
+#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr)                  \
   (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr))
 
-#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \
-   XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr)
+#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr)                        \
+  XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr)
 
-#define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \
-   XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr)
+#define XmlEntityValueTok(enc, ptr, end, nextTokPtr)                           \
+  XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr)
 
-#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \
+#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2)                             \
   (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2))
 
-#define XmlNameLength(enc, ptr) \
-  (((enc)->nameLength)(enc, ptr))
+#define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr))
 
-#define XmlSkipS(enc, ptr) \
-  (((enc)->skipS)(enc, ptr))
+#define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr))
 
-#define XmlGetAttributes(enc, ptr, attsMax, atts) \
+#define XmlGetAttributes(enc, ptr, attsMax, atts)                              \
   (((enc)->getAtts)(enc, ptr, attsMax, atts))
 
-#define XmlCharRefNumber(enc, ptr) \
-  (((enc)->charRefNumber)(enc, ptr))
+#define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr))
 
-#define XmlPredefinedEntityName(enc, ptr, end) \
+#define XmlPredefinedEntityName(enc, ptr, end)                                 \
   (((enc)->predefinedEntityName)(enc, ptr, end))
 
-#define XmlUpdatePosition(enc, ptr, end, pos) \
+#define XmlUpdatePosition(enc, ptr, end, pos)                                  \
   (((enc)->updatePosition)(enc, ptr, end, pos))
 
-#define XmlIsPublicId(enc, ptr, end, badPtr) \
+#define XmlIsPublicId(enc, ptr, end, badPtr)                                   \
   (((enc)->isPublicId)(enc, ptr, end, badPtr))
 
-#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \
+#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim)                        \
   (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim))
 
-#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \
+#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim)                       \
   (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim))
 
 typedef struct {
@@ -292,16 +279,11 @@ typedef struct {
   const ENCODING **encPtr;
 } INIT_ENCODING;
 
-int XmlParseXmlDecl(int isGeneralTextEntity,
-                    const ENCODING *enc,
-                    const char *ptr,
-                    const char *end,
-                    const char **badPtr,
-                    const char **versionPtr,
-                    const char **versionEndPtr,
+int XmlParseXmlDecl(int isGeneralTextEntity, const ENCODING *enc,
+                    const char *ptr, const char *end, const char **badPtr,
+                    const char **versionPtr, const char **versionEndPtr,
                     const char **encodingNamePtr,
-                    const ENCODING **namedEncodingPtr,
-                    int *standalonePtr);
+                    const ENCODING **namedEncodingPtr, int *standalonePtr);
 
 int XmlInitEncoding(INIT_ENCODING *, const ENCODING **, const char *name);
 const ENCODING *XmlGetUtf8InternalEncoding(void);
@@ -310,34 +292,22 @@ int FASTCALL XmlUtf8Encode(int charNumber, char *buf);
 int FASTCALL XmlUtf16Encode(int charNumber, unsigned short *buf);
 int XmlSizeOfUnknownEncoding(void);
 
+typedef int(XMLCALL *CONVERTER)(void *userData, const char *p);
 
-typedef int (XMLCALL *CONVERTER) (void *userData, const char *p);
-
-ENCODING *
-XmlInitUnknownEncoding(void *mem,
-                       int *table,
-                       CONVERTER convert,
-                       void *userData);
+ENCODING *XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert,
+                                 void *userData);
 
-int XmlParseXmlDeclNS(int isGeneralTextEntity,
-                      const ENCODING *enc,
-                      const char *ptr,
-                      const char *end,
-                      const char **badPtr,
-                      const char **versionPtr,
-                      const char **versionEndPtr,
+int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc,
+                      const char *ptr, const char *end, const char **badPtr,
+                      const char **versionPtr, const char **versionEndPtr,
                       const char **encodingNamePtr,
-                      const ENCODING **namedEncodingPtr,
-                      int *standalonePtr);
+                      const ENCODING **namedEncodingPtr, int *standalonePtr);
 
 int XmlInitEncodingNS(INIT_ENCODING *, const ENCODING **, const char *name);
 const ENCODING *XmlGetUtf8InternalEncodingNS(void);
 const ENCODING *XmlGetUtf16InternalEncodingNS(void);
-ENCODING *
-XmlInitUnknownEncodingNS(void *mem,
-                         int *table,
-                         CONVERTER convert,
-                         void *userData);
+ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert,
+                                   void *userData);
 #ifdef __cplusplus
 }
 #endif
diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c
index 4d9ae7dc3896..c209221cd79d 100644
--- a/Modules/expat/xmltok_impl.c
+++ b/Modules/expat/xmltok_impl.c
@@ -32,130 +32,124 @@
 
 #ifdef XML_TOK_IMPL_C
 
-#ifndef IS_INVALID_CHAR
-#define IS_INVALID_CHAR(enc, ptr, n) (0)
-#endif
-
-#define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n) \
-        return XML_TOK_PARTIAL_CHAR; \
-      if (IS_INVALID_CHAR(enc, ptr, n)) { \
-        *(nextTokPtr) = (ptr); \
-        return XML_TOK_INVALID; \
-      } \
-      ptr += n; \
-      break;
+#  ifndef IS_INVALID_CHAR
+#    define IS_INVALID_CHAR(enc, ptr, n) (0)
+#  endif
+
+#  define INVALID_LEAD_CASE(n, ptr, nextTokPtr)                                \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (IS_INVALID_CHAR(enc, ptr, n)) {                                        \
+      *(nextTokPtr) = (ptr);                                                   \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define INVALID_CASES(ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(2, ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(3, ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(4, ptr, nextTokPtr) \
-  case BT_NONXML: \
-  case BT_MALFORM: \
-  case BT_TRAIL: \
-    *(nextTokPtr) = (ptr); \
+#  define INVALID_CASES(ptr, nextTokPtr)                                       \
+    INVALID_LEAD_CASE(2, ptr, nextTokPtr)                                      \
+    INVALID_LEAD_CASE(3, ptr, nextTokPtr)                                      \
+    INVALID_LEAD_CASE(4, ptr, nextTokPtr)                                      \
+  case BT_NONXML:                                                              \
+  case BT_MALFORM:                                                             \
+  case BT_TRAIL:                                                               \
+    *(nextTokPtr) = (ptr);                                                     \
     return XML_TOK_INVALID;
 
-#define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr) \
-   case BT_LEAD ## n: \
-     if (end - ptr < n) \
-       return XML_TOK_PARTIAL_CHAR; \
-     if (!IS_NAME_CHAR(enc, ptr, n)) { \
-       *nextTokPtr = ptr; \
-       return XML_TOK_INVALID; \
-     } \
-     ptr += n; \
-     break;
-
-#define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) \
-  case BT_NONASCII: \
-    if (!IS_NAME_CHAR_MINBPC(enc, ptr)) { \
-      *nextTokPtr = ptr; \
-      return XML_TOK_INVALID; \
-    } \
-    /* fall through */ \
-  case BT_NMSTRT: \
-  case BT_HEX: \
-  case BT_DIGIT: \
-  case BT_NAME: \
-  case BT_MINUS: \
-    ptr += MINBPC(enc); \
-    break; \
-  CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr) \
-  CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr) \
-  CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)
-
-#define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr) \
-   case BT_LEAD ## n: \
-     if (end - ptr < n) \
-       return XML_TOK_PARTIAL_CHAR; \
-     if (!IS_NMSTRT_CHAR(enc, ptr, n)) { \
-       *nextTokPtr = ptr; \
-       return XML_TOK_INVALID; \
-     } \
-     ptr += n; \
-     break;
-
-#define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) \
-  case BT_NONASCII: \
-    if (!IS_NMSTRT_CHAR_MINBPC(enc, ptr)) { \
-      *nextTokPtr = ptr; \
-      return XML_TOK_INVALID; \
-    } \
-    /* fall through */ \
-  case BT_NMSTRT: \
-  case BT_HEX: \
-    ptr += MINBPC(enc); \
-    break; \
-  CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr) \
-  CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr) \
-  CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)
-
-#ifndef PREFIX
-#define PREFIX(ident) ident
-#endif
-
-
-#define HAS_CHARS(enc, ptr, end, count) \
-    (end - ptr >= count * MINBPC(enc))
+#  define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr)                        \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (! IS_NAME_CHAR(enc, ptr, n)) {                                         \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define HAS_CHAR(enc, ptr, end) \
-    HAS_CHARS(enc, ptr, end, 1)
+#  define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)                          \
+  case BT_NONASCII:                                                            \
+    if (! IS_NAME_CHAR_MINBPC(enc, ptr)) {                                     \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    /* fall through */                                                         \
+  case BT_NMSTRT:                                                              \
+  case BT_HEX:                                                                 \
+  case BT_DIGIT:                                                               \
+  case BT_NAME:                                                                \
+  case BT_MINUS:                                                               \
+    ptr += MINBPC(enc);                                                        \
+    break;                                                                     \
+    CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr)                              \
+    CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr)                              \
+    CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)
+
+#  define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr)                      \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (! IS_NMSTRT_CHAR(enc, ptr, n)) {                                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define REQUIRE_CHARS(enc, ptr, end, count) \
-    { \
-      if (! HAS_CHARS(enc, ptr, end, count)) { \
-        return XML_TOK_PARTIAL; \
-      } \
+#  define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)                        \
+  case BT_NONASCII:                                                            \
+    if (! IS_NMSTRT_CHAR_MINBPC(enc, ptr)) {                                   \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    /* fall through */                                                         \
+  case BT_NMSTRT:                                                              \
+  case BT_HEX:                                                                 \
+    ptr += MINBPC(enc);                                                        \
+    break;                                                                     \
+    CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr)                            \
+    CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr)                            \
+    CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)
+
+#  ifndef PREFIX
+#    define PREFIX(ident) ident
+#  endif
+
+#  define HAS_CHARS(enc, ptr, end, count) (end - ptr >= count * MINBPC(enc))
+
+#  define HAS_CHAR(enc, ptr, end) HAS_CHARS(enc, ptr, end, 1)
+
+#  define REQUIRE_CHARS(enc, ptr, end, count)                                  \
+    {                                                                          \
+      if (! HAS_CHARS(enc, ptr, end, count)) {                                 \
+        return XML_TOK_PARTIAL;                                                \
+      }                                                                        \
     }
 
-#define REQUIRE_CHAR(enc, ptr, end) \
-    REQUIRE_CHARS(enc, ptr, end, 1)
-
+#  define REQUIRE_CHAR(enc, ptr, end) REQUIRE_CHARS(enc, ptr, end, 1)
 
 /* ptr points to character following "<!-" */
 
 static int PTRCALL
-PREFIX(scanComment)(const ENCODING *enc, const char *ptr,
-                    const char *end, const char **nextTokPtr)
-{
+PREFIX(scanComment)(const ENCODING *enc, const char *ptr, const char *end,
+                    const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
-    if (!CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
     }
     ptr += MINBPC(enc);
     while (HAS_CHAR(enc, ptr, end)) {
       switch (BYTE_TYPE(enc, ptr)) {
-      INVALID_CASES(ptr, nextTokPtr)
+        INVALID_CASES(ptr, nextTokPtr)
       case BT_MINUS:
         ptr += MINBPC(enc);
         REQUIRE_CHAR(enc, ptr, end);
         if (CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
-          if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+          if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
             *nextTokPtr = ptr;
             return XML_TOK_INVALID;
           }
@@ -175,9 +169,8 @@ PREFIX(scanComment)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "<!" */
 
 static int PTRCALL
-PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
-                 const char *end, const char **nextTokPtr)
-{
+PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, const char *end,
+                 const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
   case BT_MINUS:
@@ -199,12 +192,17 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
       REQUIRE_CHARS(enc, ptr, end, 2);
       /* don't allow <!ENTITY% foo "whatever"> */
       switch (BYTE_TYPE(enc, ptr + MINBPC(enc))) {
-      case BT_S: case BT_CR: case BT_LF: case BT_PERCNT:
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+      case BT_PERCNT:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       /* fall through */
-    case BT_S: case BT_CR: case BT_LF:
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       *nextTokPtr = ptr;
       return XML_TOK_DECL_OPEN;
     case BT_NMSTRT:
@@ -220,12 +218,12 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                      const char *end, int *tokPtr)
-{
+PREFIX(checkPiTarget)(const ENCODING *enc, const char *ptr, const char *end,
+                      int *tokPtr) {
   int upper = 0;
+  UNUSED_P(enc);
   *tokPtr = XML_TOK_PI;
-  if (end - ptr != MINBPC(enc)*3)
+  if (end - ptr != MINBPC(enc) * 3)
     return 1;
   switch (BYTE_TO_ASCII(enc, ptr)) {
   case ASCII_x:
@@ -265,30 +263,31 @@ PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr,
 /* ptr points to character following "<?" */
 
 static int PTRCALL
-PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
-               const char *end, const char **nextTokPtr)
-{
+PREFIX(scanPi)(const ENCODING *enc, const char *ptr, const char *end,
+               const char **nextTokPtr) {
   int tok;
   const char *target = ptr;
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_S: case BT_CR: case BT_LF:
-      if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
+      if (! PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       ptr += MINBPC(enc);
       while (HAS_CHAR(enc, ptr, end)) {
         switch (BYTE_TYPE(enc, ptr)) {
-        INVALID_CASES(ptr, nextTokPtr)
+          INVALID_CASES(ptr, nextTokPtr)
         case BT_QUEST:
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
@@ -304,7 +303,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
       }
       return XML_TOK_PARTIAL;
     case BT_QUEST:
-      if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
+      if (! PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -324,16 +323,16 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                         const char *end, const char **nextTokPtr)
-{
-  static const char CDATA_LSQB[] = { ASCII_C, ASCII_D, ASCII_A,
-                                     ASCII_T, ASCII_A, ASCII_LSQB };
+PREFIX(scanCdataSection)(const ENCODING *enc, const char *ptr, const char *end,
+                         const char **nextTokPtr) {
+  static const char CDATA_LSQB[]
+      = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, ASCII_LSQB};
   int i;
+  UNUSED_P(enc);
   /* CDATA[ */
   REQUIRE_CHARS(enc, ptr, end, 6);
   for (i = 0; i < 6; i++, ptr += MINBPC(enc)) {
-    if (!CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {
+    if (! CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
     }
@@ -343,9 +342,8 @@ PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
-                        const char *end, const char **nextTokPtr)
-{
+PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
+                        const char **nextTokPtr) {
   if (ptr >= end)
     return XML_TOK_NONE;
   if (MINBPC(enc) > 1) {
@@ -361,11 +359,11 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
   case BT_RSQB:
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
-    if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
+    if (! CHAR_MATCHES(enc, ptr, ASCII_RSQB))
       break;
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
-    if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
       ptr -= MINBPC(enc);
       break;
     }
@@ -381,23 +379,25 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
   case BT_LF:
     *nextTokPtr = ptr + MINBPC(enc);
     return XML_TOK_DATA_NEWLINE;
-  INVALID_CASES(ptr, nextTokPtr)
+    INVALID_CASES(ptr, nextTokPtr)
   default:
     ptr += MINBPC(enc);
     break;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \
-        *nextTokPtr = ptr; \
-        return XML_TOK_DATA_CHARS; \
-      } \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) {                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_DATA_CHARS;                                               \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONXML:
     case BT_MALFORM:
     case BT_TRAIL:
@@ -418,23 +418,26 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "</" */
 
 static int PTRCALL
-PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
-                   const char *end, const char **nextTokPtr)
-{
+PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr, const char *end,
+                   const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_S: case BT_CR: case BT_LF:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) {
         switch (BYTE_TYPE(enc, ptr)) {
-        case BT_S: case BT_CR: case BT_LF:
+        case BT_S:
+        case BT_CR:
+        case BT_LF:
           break;
         case BT_GT:
           *nextTokPtr = ptr + MINBPC(enc);
@@ -445,13 +448,13 @@ PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
         }
       }
       return XML_TOK_PARTIAL;
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
       /* no need to check qname syntax here,
          since end-tag must match exactly */
       ptr += MINBPC(enc);
       break;
-#endif
+#  endif
     case BT_GT:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_END_TAG;
@@ -466,9 +469,8 @@ PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "&#X" */
 
 static int PTRCALL
-PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr,
-                       const char *end, const char **nextTokPtr)
-{
+PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, const char *end,
+                       const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
     case BT_DIGIT:
@@ -498,9 +500,8 @@ PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "&#" */
 
 static int PTRCALL
-PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr,
-                    const char *end, const char **nextTokPtr)
-{
+PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, const char *end,
+                    const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
     if (CHAR_MATCHES(enc, ptr, ASCII_x))
       return PREFIX(scanHexCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -531,11 +532,10 @@ PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr,
 
 static int PTRCALL
 PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
-                const char **nextTokPtr)
-{
+                const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   case BT_NUM:
     return PREFIX(scanCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
   default:
@@ -544,7 +544,7 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
     case BT_SEMI:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_ENTITY_REF;
@@ -560,15 +560,14 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
-                 const char **nextTokPtr)
-{
-#ifdef XML_NS
+                 const char **nextTokPtr) {
+#  ifdef XML_NS
   int hadColon = 0;
-#endif
+#  endif
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-#ifdef XML_NS
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+#  ifdef XML_NS
     case BT_COLON:
       if (hadColon) {
         *nextTokPtr = ptr;
@@ -578,14 +577,16 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
       switch (BYTE_TYPE(enc, ptr)) {
-      CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+        CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
       default:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       break;
-#endif
-    case BT_S: case BT_CR: case BT_LF:
+#  endif
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       for (;;) {
         int t;
 
@@ -605,101 +606,101 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
         }
       }
       /* fall through */
-    case BT_EQUALS:
-      {
-        int open;
-#ifdef XML_NS
-        hadColon = 0;
-#endif
-        for (;;) {
-          ptr += MINBPC(enc);
-          REQUIRE_CHAR(enc, ptr, end);
-          open = BYTE_TYPE(enc, ptr);
-          if (open == BT_QUOT || open == BT_APOS)
-            break;
-          switch (open) {
-          case BT_S:
-          case BT_LF:
-          case BT_CR:
-            break;
-          default:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          }
-        }
+    case BT_EQUALS: {
+      int open;
+#  ifdef XML_NS
+      hadColon = 0;
+#  endif
+      for (;;) {
         ptr += MINBPC(enc);
-        /* in attribute value */
-        for (;;) {
-          int t;
-          REQUIRE_CHAR(enc, ptr, end);
-          t = BYTE_TYPE(enc, ptr);
-          if (t == open)
-            break;
-          switch (t) {
+        REQUIRE_CHAR(enc, ptr, end);
+        open = BYTE_TYPE(enc, ptr);
+        if (open == BT_QUOT || open == BT_APOS)
+          break;
+        switch (open) {
+        case BT_S:
+        case BT_LF:
+        case BT_CR:
+          break;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
+        }
+      }
+      ptr += MINBPC(enc);
+      /* in attribute value */
+      for (;;) {
+        int t;
+        REQUIRE_CHAR(enc, ptr, end);
+        t = BYTE_TYPE(enc, ptr);
+        if (t == open)
+          break;
+        switch (t) {
           INVALID_CASES(ptr, nextTokPtr)
-          case BT_AMP:
-            {
-              int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);
-              if (tok <= 0) {
-                if (tok == XML_TOK_INVALID)
-                  *nextTokPtr = ptr;
-                return tok;
-              }
-              break;
-            }
-          case BT_LT:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          default:
-            ptr += MINBPC(enc);
-            break;
+        case BT_AMP: {
+          int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);
+          if (tok <= 0) {
+            if (tok == XML_TOK_INVALID)
+              *nextTokPtr = ptr;
+            return tok;
           }
+          break;
+        }
+        case BT_LT:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
+        default:
+          ptr += MINBPC(enc);
+          break;
         }
+      }
+      ptr += MINBPC(enc);
+      REQUIRE_CHAR(enc, ptr, end);
+      switch (BYTE_TYPE(enc, ptr)) {
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+        break;
+      case BT_SOL:
+        goto sol;
+      case BT_GT:
+        goto gt;
+      default:
+        *nextTokPtr = ptr;
+        return XML_TOK_INVALID;
+      }
+      /* ptr points to closing quote */
+      for (;;) {
         ptr += MINBPC(enc);
         REQUIRE_CHAR(enc, ptr, end);
         switch (BYTE_TYPE(enc, ptr)) {
+          CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
         case BT_S:
         case BT_CR:
         case BT_LF:
-          break;
-        case BT_SOL:
-          goto sol;
+          continue;
         case BT_GT:
-          goto gt;
-        default:
-          *nextTokPtr = ptr;
-          return XML_TOK_INVALID;
-        }
-        /* ptr points to closing quote */
-        for (;;) {
+        gt:
+          *nextTokPtr = ptr + MINBPC(enc);
+          return XML_TOK_START_TAG_WITH_ATTS;
+        case BT_SOL:
+        sol:
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
-          switch (BYTE_TYPE(enc, ptr)) {
-          CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-          case BT_S: case BT_CR: case BT_LF:
-            continue;
-          case BT_GT:
-          gt:
-            *nextTokPtr = ptr + MINBPC(enc);
-            return XML_TOK_START_TAG_WITH_ATTS;
-          case BT_SOL:
-          sol:
-            ptr += MINBPC(enc);
-            REQUIRE_CHAR(enc, ptr, end);
-            if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
-              *nextTokPtr = ptr;
-              return XML_TOK_INVALID;
-            }
-            *nextTokPtr = ptr + MINBPC(enc);
-            return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;
-          default:
+          if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
             *nextTokPtr = ptr;
             return XML_TOK_INVALID;
           }
-          break;
+          *nextTokPtr = ptr + MINBPC(enc);
+          return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
         }
         break;
       }
+      break;
+    }
     default:
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
@@ -712,14 +713,13 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
-               const char **nextTokPtr)
-{
-#ifdef XML_NS
+               const char **nextTokPtr) {
+#  ifdef XML_NS
   int hadColon;
-#endif
+#  endif
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   case BT_EXCL:
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
@@ -727,8 +727,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_MINUS:
       return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);
     case BT_LSQB:
-      return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc),
-                                      end, nextTokPtr);
+      return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc), end, nextTokPtr);
     }
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
@@ -740,14 +739,14 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
-#ifdef XML_NS
+#  ifdef XML_NS
   hadColon = 0;
-#endif
+#  endif
   /* we have a start-tag */
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-#ifdef XML_NS
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+#  ifdef XML_NS
     case BT_COLON:
       if (hadColon) {
         *nextTokPtr = ptr;
@@ -757,34 +756,37 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
       switch (BYTE_TYPE(enc, ptr)) {
-      CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+        CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
       default:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       break;
-#endif
-    case BT_S: case BT_CR: case BT_LF:
-      {
-        ptr += MINBPC(enc);
-        while (HAS_CHAR(enc, ptr, end)) {
-          switch (BYTE_TYPE(enc, ptr)) {
+#  endif
+    case BT_S:
+    case BT_CR:
+    case BT_LF: {
+      ptr += MINBPC(enc);
+      while (HAS_CHAR(enc, ptr, end)) {
+        switch (BYTE_TYPE(enc, ptr)) {
           CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-          case BT_GT:
-            goto gt;
-          case BT_SOL:
-            goto sol;
-          case BT_S: case BT_CR: case BT_LF:
-            ptr += MINBPC(enc);
-            continue;
-          default:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          }
-          return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);
+        case BT_GT:
+          goto gt;
+        case BT_SOL:
+          goto sol;
+        case BT_S:
+        case BT_CR:
+        case BT_LF:
+          ptr += MINBPC(enc);
+          continue;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
         }
-        return XML_TOK_PARTIAL;
+        return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);
       }
+      return XML_TOK_PARTIAL;
+    }
     case BT_GT:
     gt:
       *nextTokPtr = ptr + MINBPC(enc);
@@ -793,7 +795,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     sol:
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
-      if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+      if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -809,8 +811,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **nextTokPtr)
-{
+                   const char **nextTokPtr) {
   if (ptr >= end)
     return XML_TOK_NONE;
   if (MINBPC(enc) > 1) {
@@ -842,48 +843,50 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
     ptr += MINBPC(enc);
     if (! HAS_CHAR(enc, ptr, end))
       return XML_TOK_TRAILING_RSQB;
-    if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
+    if (! CHAR_MATCHES(enc, ptr, ASCII_RSQB))
       break;
     ptr += MINBPC(enc);
     if (! HAS_CHAR(enc, ptr, end))
       return XML_TOK_TRAILING_RSQB;
-    if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
       ptr -= MINBPC(enc);
       break;
     }
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
-  INVALID_CASES(ptr, nextTokPtr)
+    INVALID_CASES(ptr, nextTokPtr)
   default:
     ptr += MINBPC(enc);
     break;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \
-        *nextTokPtr = ptr; \
-        return XML_TOK_DATA_CHARS; \
-      } \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) {                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_DATA_CHARS;                                               \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_RSQB:
       if (HAS_CHARS(enc, ptr, end, 2)) {
-         if (!CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {
-           ptr += MINBPC(enc);
-           break;
-         }
-         if (HAS_CHARS(enc, ptr, end, 3)) {
-           if (!CHAR_MATCHES(enc, ptr + 2*MINBPC(enc), ASCII_GT)) {
-             ptr += MINBPC(enc);
-             break;
-           }
-           *nextTokPtr = ptr + 2*MINBPC(enc);
-           return XML_TOK_INVALID;
-         }
+        if (! CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {
+          ptr += MINBPC(enc);
+          break;
+        }
+        if (HAS_CHARS(enc, ptr, end, 3)) {
+          if (! CHAR_MATCHES(enc, ptr + 2 * MINBPC(enc), ASCII_GT)) {
+            ptr += MINBPC(enc);
+            break;
+          }
+          *nextTokPtr = ptr + 2 * MINBPC(enc);
+          return XML_TOK_INVALID;
+        }
       }
       /* fall through */
     case BT_AMP:
@@ -908,12 +911,14 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
-                    const char **nextTokPtr)
-{
+                    const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-  case BT_S: case BT_LF: case BT_CR: case BT_PERCNT:
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+  case BT_S:
+  case BT_LF:
+  case BT_CR:
+  case BT_PERCNT:
     *nextTokPtr = ptr;
     return XML_TOK_PERCENT;
   default:
@@ -922,7 +927,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
     case BT_SEMI:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_PARAM_ENTITY_REF;
@@ -936,20 +941,24 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,
-                      const char **nextTokPtr)
-{
+                      const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_CR: case BT_LF: case BT_S:
-    case BT_RPAR: case BT_GT: case BT_PERCNT: case BT_VERBAR:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_CR:
+    case BT_LF:
+    case BT_S:
+    case BT_RPAR:
+    case BT_GT:
+    case BT_PERCNT:
+    case BT_VERBAR:
       *nextTokPtr = ptr;
       return XML_TOK_POUND_NAME;
     default:
@@ -961,14 +970,12 @@ PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,
 }
 
 static int PTRCALL
-PREFIX(scanLit)(int open, const ENCODING *enc,
-                const char *ptr, const char *end,
-                const char **nextTokPtr)
-{
+PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end,
+                const char **nextTokPtr) {
   while (HAS_CHAR(enc, ptr, end)) {
     int t = BYTE_TYPE(enc, ptr);
     switch (t) {
-    INVALID_CASES(ptr, nextTokPtr)
+      INVALID_CASES(ptr, nextTokPtr)
     case BT_QUOT:
     case BT_APOS:
       ptr += MINBPC(enc);
@@ -978,8 +985,12 @@ PREFIX(scanLit)(int open, const ENCODING *enc,
         return -XML_TOK_LITERAL;
       *nextTokPtr = ptr;
       switch (BYTE_TYPE(enc, ptr)) {
-      case BT_S: case BT_CR: case BT_LF:
-      case BT_GT: case BT_PERCNT: case BT_LSQB:
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+      case BT_GT:
+      case BT_PERCNT:
+      case BT_LSQB:
         return XML_TOK_LITERAL;
       default:
         return XML_TOK_INVALID;
@@ -994,8 +1005,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc,
 
 static int PTRCALL
 PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
-                  const char **nextTokPtr)
-{
+                  const char **nextTokPtr) {
   int tok;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1013,27 +1023,26 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     return PREFIX(scanLit)(BT_QUOT, enc, ptr + MINBPC(enc), end, nextTokPtr);
   case BT_APOS:
     return PREFIX(scanLit)(BT_APOS, enc, ptr + MINBPC(enc), end, nextTokPtr);
-  case BT_LT:
-    {
-      ptr += MINBPC(enc);
-      REQUIRE_CHAR(enc, ptr, end);
-      switch (BYTE_TYPE(enc, ptr)) {
-      case BT_EXCL:
-        return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-      case BT_QUEST:
-        return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-      case BT_NMSTRT:
-      case BT_HEX:
-      case BT_NONASCII:
-      case BT_LEAD2:
-      case BT_LEAD3:
-      case BT_LEAD4:
-        *nextTokPtr = ptr - MINBPC(enc);
-        return XML_TOK_INSTANCE_START;
-      }
-      *nextTokPtr = ptr;
-      return XML_TOK_INVALID;
+  case BT_LT: {
+    ptr += MINBPC(enc);
+    REQUIRE_CHAR(enc, ptr, end);
+    switch (BYTE_TYPE(enc, ptr)) {
+    case BT_EXCL:
+      return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);
+    case BT_QUEST:
+      return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
+    case BT_NMSTRT:
+    case BT_HEX:
+    case BT_NONASCII:
+    case BT_LEAD2:
+    case BT_LEAD3:
+    case BT_LEAD4:
+      *nextTokPtr = ptr - MINBPC(enc);
+      return XML_TOK_INSTANCE_START;
     }
+    *nextTokPtr = ptr;
+    return XML_TOK_INVALID;
+  }
   case BT_CR:
     if (ptr + MINBPC(enc) == end) {
       *nextTokPtr = end;
@@ -1041,13 +1050,15 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
       return -XML_TOK_PROLOG_S;
     }
     /* fall through */
-  case BT_S: case BT_LF:
+  case BT_S:
+  case BT_LF:
     for (;;) {
       ptr += MINBPC(enc);
       if (! HAS_CHAR(enc, ptr, end))
         break;
       switch (BYTE_TYPE(enc, ptr)) {
-      case BT_S: case BT_LF:
+      case BT_S:
+      case BT_LF:
         break;
       case BT_CR:
         /* don't split CR/LF pair */
@@ -1076,7 +1087,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) {
       REQUIRE_CHARS(enc, ptr, end, 2);
       if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_GT)) {
-        *nextTokPtr = ptr + 2*MINBPC(enc);
+        *nextTokPtr = ptr + 2 * MINBPC(enc);
         return XML_TOK_COND_SECT_CLOSE;
       }
     }
@@ -1099,8 +1110,12 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_PLUS:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_CLOSE_PAREN_PLUS;
-    case BT_CR: case BT_LF: case BT_S:
-    case BT_GT: case BT_COMMA: case BT_VERBAR:
+    case BT_CR:
+    case BT_LF:
+    case BT_S:
+    case BT_GT:
+    case BT_COMMA:
+    case BT_VERBAR:
     case BT_RPAR:
       *nextTokPtr = ptr;
       return XML_TOK_CLOSE_PAREN;
@@ -1115,24 +1130,26 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     return XML_TOK_DECL_CLOSE;
   case BT_NUM:
     return PREFIX(scanPoundName)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-#define LEAD_CASE(n) \
-  case BT_LEAD ## n: \
-    if (end - ptr < n) \
-      return XML_TOK_PARTIAL_CHAR; \
-    if (IS_NMSTRT_CHAR(enc, ptr, n)) { \
-      ptr += n; \
-      tok = XML_TOK_NAME; \
-      break; \
-    } \
-    if (IS_NAME_CHAR(enc, ptr, n)) { \
-      ptr += n; \
-      tok = XML_TOK_NMTOKEN; \
-      break; \
-    } \
-    *nextTokPtr = ptr; \
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (IS_NMSTRT_CHAR(enc, ptr, n)) {                                         \
+      ptr += n;                                                                \
+      tok = XML_TOK_NAME;                                                      \
+      break;                                                                   \
+    }                                                                          \
+    if (IS_NAME_CHAR(enc, ptr, n)) {                                           \
+      ptr += n;                                                                \
+      tok = XML_TOK_NMTOKEN;                                                   \
+      break;                                                                   \
+    }                                                                          \
+    *nextTokPtr = ptr;                                                         \
     return XML_TOK_INVALID;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+    LEAD_CASE(2)
+    LEAD_CASE(3)
+    LEAD_CASE(4)
+#  undef LEAD_CASE
   case BT_NMSTRT:
   case BT_HEX:
     tok = XML_TOK_NAME;
@@ -1141,9 +1158,9 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
   case BT_DIGIT:
   case BT_NAME:
   case BT_MINUS:
-#ifdef XML_NS
+#  ifdef XML_NS
   case BT_COLON:
-#endif
+#  endif
     tok = XML_TOK_NMTOKEN;
     ptr += MINBPC(enc);
     break;
@@ -1165,13 +1182,19 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_GT: case BT_RPAR: case BT_COMMA:
-    case BT_VERBAR: case BT_LSQB: case BT_PERCNT:
-    case BT_S: case BT_CR: case BT_LF:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_GT:
+    case BT_RPAR:
+    case BT_COMMA:
+    case BT_VERBAR:
+    case BT_LSQB:
+    case BT_PERCNT:
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       *nextTokPtr = ptr;
       return tok;
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
       ptr += MINBPC(enc);
       switch (tok) {
@@ -1179,7 +1202,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
         REQUIRE_CHAR(enc, ptr, end);
         tok = XML_TOK_PREFIXED_NAME;
         switch (BYTE_TYPE(enc, ptr)) {
-        CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+          CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
         default:
           tok = XML_TOK_NMTOKEN;
           break;
@@ -1190,23 +1213,23 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
         break;
       }
       break;
-#endif
+#  endif
     case BT_PLUS:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_NAME_PLUS;
     case BT_AST:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_NAME_ASTERISK;
     case BT_QUEST:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -1221,9 +1244,8 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
 }
 
 static int PTRCALL
-PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
-                          const char *end, const char **nextTokPtr)
-{
+PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end,
+                          const char **nextTokPtr) {
   const char *start;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1238,10 +1260,14 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
   start = ptr;
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_AMP:
       if (ptr == start)
         return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -1287,9 +1313,8 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
-                       const char *end, const char **nextTokPtr)
-{
+PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end,
+                       const char **nextTokPtr) {
   const char *start;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1304,10 +1329,14 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
   start = ptr;
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_AMP:
       if (ptr == start)
         return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -1315,8 +1344,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
       return XML_TOK_DATA_CHARS;
     case BT_PERCNT:
       if (ptr == start) {
-        int tok =  PREFIX(scanPercent)(enc, ptr + MINBPC(enc),
-                                       end, nextTokPtr);
+        int tok = PREFIX(scanPercent)(enc, ptr + MINBPC(enc), end, nextTokPtr);
         return (tok == XML_TOK_PERCENT) ? XML_TOK_INVALID : tok;
       }
       *nextTokPtr = ptr;
@@ -1349,12 +1377,11 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
   return XML_TOK_DATA_CHARS;
 }
 
-#ifdef XML_DTD
+#  ifdef XML_DTD
 
 static int PTRCALL
-PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
-                         const char *end, const char **nextTokPtr)
-{
+PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
+                         const char **nextTokPtr) {
   int level = 0;
   if (MINBPC(enc) > 1) {
     size_t n = end - ptr;
@@ -1365,7 +1392,7 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    INVALID_CASES(ptr, nextTokPtr)
+      INVALID_CASES(ptr, nextTokPtr)
     case BT_LT:
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
@@ -1402,12 +1429,11 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
   return XML_TOK_PARTIAL;
 }
 
-#endif /* XML_DTD */
+#  endif /* XML_DTD */
 
 static int PTRCALL
 PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **badPtr)
-{
+                   const char **badPtr) {
   ptr += MINBPC(enc);
   end -= MINBPC(enc);
   for (; HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) {
@@ -1430,9 +1456,9 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_AST:
     case BT_PERCNT:
     case BT_NUM:
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
-#endif
+#  endif
       break;
     case BT_S:
       if (CHAR_MATCHES(enc, ptr, ASCII_TAB)) {
@@ -1442,7 +1468,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
       break;
     case BT_NAME:
     case BT_NMSTRT:
-      if (!(BYTE_TO_ASCII(enc, ptr) & ~0x7f))
+      if (! (BYTE_TO_ASCII(enc, ptr) & ~0x7f))
         break;
       /* fall through */
     default:
@@ -1466,9 +1492,8 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
 */
 
 static int PTRCALL
-PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
-                int attsMax, ATTRIBUTE *atts)
-{
+PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
+                ATTRIBUTE *atts) {
   enum { other, inName, inValue } state = inName;
   int nAtts = 0;
   int open = 0; /* defined when state == inValue;
@@ -1476,32 +1501,35 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
 
   for (ptr += MINBPC(enc);; ptr += MINBPC(enc)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define START_NAME \
-      if (state == other) { \
-        if (nAtts < attsMax) { \
-          atts[nAtts].name = ptr; \
-          atts[nAtts].normalized = 1; \
-        } \
-        state = inName; \
-      }
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: START_NAME ptr += (n - MINBPC(enc)); break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define START_NAME                                                           \
+    if (state == other) {                                                      \
+      if (nAtts < attsMax) {                                                   \
+        atts[nAtts].name = ptr;                                                \
+        atts[nAtts].normalized = 1;                                            \
+      }                                                                        \
+      state = inName;                                                          \
+    }
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    START_NAME ptr += (n - MINBPC(enc));                                       \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONASCII:
     case BT_NMSTRT:
     case BT_HEX:
       START_NAME
       break;
-#undef START_NAME
+#  undef START_NAME
     case BT_QUOT:
       if (state != inValue) {
         if (nAtts < attsMax)
           atts[nAtts].valuePtr = ptr + MINBPC(enc);
         state = inValue;
         open = BT_QUOT;
-      }
-      else if (open == BT_QUOT) {
+      } else if (open == BT_QUOT) {
         state = other;
         if (nAtts < attsMax)
           atts[nAtts].valueEnd = ptr;
@@ -1514,8 +1542,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
           atts[nAtts].valuePtr = ptr + MINBPC(enc);
         state = inValue;
         open = BT_APOS;
-      }
-      else if (open == BT_APOS) {
+      } else if (open == BT_APOS) {
         state = other;
         if (nAtts < attsMax)
           atts[nAtts].valueEnd = ptr;
@@ -1529,16 +1556,15 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
     case BT_S:
       if (state == inName)
         state = other;
-      else if (state == inValue
-               && nAtts < attsMax
-               && atts[nAtts].normalized
+      else if (state == inValue && nAtts < attsMax && atts[nAtts].normalized
                && (ptr == atts[nAtts].valuePtr
                    || BYTE_TO_ASCII(enc, ptr) != ASCII_SPACE
                    || BYTE_TO_ASCII(enc, ptr + MINBPC(enc)) == ASCII_SPACE
                    || BYTE_TYPE(enc, ptr + MINBPC(enc)) == open))
         atts[nAtts].normalized = 0;
       break;
-    case BT_CR: case BT_LF:
+    case BT_CR:
+    case BT_LF:
       /* This case ensures that the first attribute name is counted
          Apart from that we could just change state on the quote. */
       if (state == inName)
@@ -1559,29 +1585,44 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRFASTCALL
-PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
-{
+PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) {
   int result = 0;
   /* skip &# */
-  ptr += 2*MINBPC(enc);
+  UNUSED_P(enc);
+  ptr += 2 * MINBPC(enc);
   if (CHAR_MATCHES(enc, ptr, ASCII_x)) {
-    for (ptr += MINBPC(enc);
-         !CHAR_MATCHES(enc, ptr, ASCII_SEMI);
+    for (ptr += MINBPC(enc); ! CHAR_MATCHES(enc, ptr, ASCII_SEMI);
          ptr += MINBPC(enc)) {
       int c = BYTE_TO_ASCII(enc, ptr);
       switch (c) {
-      case ASCII_0: case ASCII_1: case ASCII_2: case ASCII_3: case ASCII_4:
-      case ASCII_5: case ASCII_6: case ASCII_7: case ASCII_8: case ASCII_9:
+      case ASCII_0:
+      case ASCII_1:
+      case ASCII_2:
+      case ASCII_3:
+      case ASCII_4:
+      case ASCII_5:
+      case ASCII_6:
+      case ASCII_7:
+      case ASCII_8:
+      case ASCII_9:
         result <<= 4;
         result |= (c - ASCII_0);
         break;
-      case ASCII_A: case ASCII_B: case ASCII_C:
-      case ASCII_D: case ASCII_E: case ASCII_F:
+      case ASCII_A:
+      case ASCII_B:
+      case ASCII_C:
+      case ASCII_D:
+      case ASCII_E:
+      case ASCII_F:
         result <<= 4;
         result += 10 + (c - ASCII_A);
         break;
-      case ASCII_a: case ASCII_b: case ASCII_c:
-      case ASCII_d: case ASCII_e: case ASCII_f:
+      case ASCII_a:
+      case ASCII_b:
+      case ASCII_c:
+      case ASCII_d:
+      case ASCII_e:
+      case ASCII_f:
         result <<= 4;
         result += 10 + (c - ASCII_a);
         break;
@@ -1589,9 +1630,8 @@ PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
       if (result >= 0x110000)
         return -1;
     }
-  }
-  else {
-    for (; !CHAR_MATCHES(enc, ptr, ASCII_SEMI); ptr += MINBPC(enc)) {
+  } else {
+    for (; ! CHAR_MATCHES(enc, ptr, ASCII_SEMI); ptr += MINBPC(enc)) {
       int c = BYTE_TO_ASCII(enc, ptr);
       result *= 10;
       result += (c - ASCII_0);
@@ -1603,10 +1643,10 @@ PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
 }
 
 static int PTRCALL
-PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                             const char *end)
-{
-  switch ((end - ptr)/MINBPC(enc)) {
+PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr,
+                             const char *end) {
+  UNUSED_P(enc);
+  switch ((end - ptr) / MINBPC(enc)) {
   case 2:
     if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_t)) {
       switch (BYTE_TO_ASCII(enc, ptr)) {
@@ -1657,9 +1697,9 @@ PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1,
-                         const char *end1, const char *ptr2)
-{
+PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1,
+                         const char *end1, const char *ptr2) {
+  UNUSED_P(enc);
   for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) {
     if (end1 - ptr1 < MINBPC(enc)) {
       /* This line cannot be executed.  The incoming data has already
@@ -1669,27 +1709,30 @@ PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1,
        */
       return 0; /* LCOV_EXCL_LINE */
     }
-    if (!CHAR_MATCHES(enc, ptr1, *ptr2))
+    if (! CHAR_MATCHES(enc, ptr1, *ptr2))
       return 0;
   }
   return ptr1 == end1;
 }
 
 static int PTRFASTCALL
-PREFIX(nameLength)(const ENCODING *enc, const char *ptr)
-{
+PREFIX(nameLength)(const ENCODING *enc, const char *ptr) {
   const char *start = ptr;
   for (;;) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONASCII:
     case BT_NMSTRT:
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
-#endif
+#  endif
     case BT_HEX:
     case BT_DIGIT:
     case BT_NAME:
@@ -1702,9 +1745,8 @@ PREFIX(nameLength)(const ENCODING *enc, const char *ptr)
   }
 }
 
-static const char * PTRFASTCALL
-PREFIX(skipS)(const ENCODING *enc, const char *ptr)
-{
+static const char *PTRFASTCALL
+PREFIX(skipS)(const ENCODING *enc, const char *ptr) {
   for (;;) {
     switch (BYTE_TYPE(enc, ptr)) {
     case BT_LF:
@@ -1719,19 +1761,18 @@ PREFIX(skipS)(const ENCODING *enc, const char *ptr)
 }
 
 static void PTRCALL
-PREFIX(updatePosition)(const ENCODING *enc,
-                       const char *ptr,
-                       const char *end,
-                       POSITION *pos)
-{
+PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end,
+                       POSITION *pos) {
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_LF:
       pos->columnNumber = (XML_Size)-1;
       pos->lineNumber++;
@@ -1752,12 +1793,12 @@ PREFIX(updatePosition)(const ENCODING *enc,
   }
 }
 
-#undef DO_LEAD_CASE
-#undef MULTIBYTE_CASES
-#undef INVALID_CASES
-#undef CHECK_NAME_CASE
-#undef CHECK_NAME_CASES
-#undef CHECK_NMSTRT_CASE
-#undef CHECK_NMSTRT_CASES
+#  undef DO_LEAD_CASE
+#  undef MULTIBYTE_CASES
+#  undef INVALID_CASES
+#  undef CHECK_NAME_CASE
+#  undef CHECK_NAME_CASES
+#  undef CHECK_NMSTRT_CASE
+#  undef CHECK_NMSTRT_CASES
 
 #endif /* XML_TOK_IMPL_C */
diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h
index a6420f48eedc..e925dbc7e2c8 100644
--- a/Modules/expat/xmltok_impl.h
+++ b/Modules/expat/xmltok_impl.h
@@ -31,43 +31,43 @@
 */
 
 enum {
-  BT_NONXML,
-  BT_MALFORM,
-  BT_LT,
-  BT_AMP,
-  BT_RSQB,
-  BT_LEAD2,
-  BT_LEAD3,
-  BT_LEAD4,
-  BT_TRAIL,
-  BT_CR,
-  BT_LF,
-  BT_GT,
-  BT_QUOT,
-  BT_APOS,
-  BT_EQUALS,
-  BT_QUEST,
-  BT_EXCL,
-  BT_SOL,
-  BT_SEMI,
-  BT_NUM,
-  BT_LSQB,
-  BT_S,
-  BT_NMSTRT,
-  BT_COLON,
-  BT_HEX,
-  BT_DIGIT,
-  BT_NAME,
-  BT_MINUS,
-  BT_OTHER, /* known not to be a name or name start character */
+  BT_NONXML,   /* e.g. noncharacter-FFFF */
+  BT_MALFORM,  /* illegal, with regard to encoding */
+  BT_LT,       /* less than = "<" */
+  BT_AMP,      /* ampersand = "&" */
+  BT_RSQB,     /* right square bracket = "[" */
+  BT_LEAD2,    /* lead byte of a 2-byte UTF-8 character */
+  BT_LEAD3,    /* lead byte of a 3-byte UTF-8 character */
+  BT_LEAD4,    /* lead byte of a 4-byte UTF-8 character */
+  BT_TRAIL,    /* trailing unit, e.g. second 16-bit unit of a 4-byte char. */
+  BT_CR,       /* carriage return = "\r" */
+  BT_LF,       /* line feed = "\n" */
+  BT_GT,       /* greater than = ">" */
+  BT_QUOT,     /* quotation character = "\"" */
+  BT_APOS,     /* aposthrophe = "'" */
+  BT_EQUALS,   /* equal sign = "=" */
+  BT_QUEST,    /* question mark = "?" */
+  BT_EXCL,     /* exclamation mark = "!" */
+  BT_SOL,      /* solidus, slash = "/" */
+  BT_SEMI,     /* semicolon = ";" */
+  BT_NUM,      /* number sign = "#" */
+  BT_LSQB,     /* left square bracket = "[" */
+  BT_S,        /* white space, e.g. "\t", " "[, "\r"] */
+  BT_NMSTRT,   /* non-hex name start letter = "G".."Z" + "g".."z" + "_" */
+  BT_COLON,    /* colon = ":" */
+  BT_HEX,      /* hex letter = "A".."F" + "a".."f" */
+  BT_DIGIT,    /* digit = "0".."9" */
+  BT_NAME,     /* dot and middle dot = "." + chr(0xb7) */
+  BT_MINUS,    /* minus = "-" */
+  BT_OTHER,    /* known not to be a name or name start character */
   BT_NONASCII, /* might be a name or name start character */
-  BT_PERCNT,
-  BT_LPAR,
-  BT_RPAR,
-  BT_AST,
-  BT_PLUS,
-  BT_COMMA,
-  BT_VERBAR
+  BT_PERCNT,   /* percent sign = "%" */
+  BT_LPAR,     /* left parenthesis = "(" */
+  BT_RPAR,     /* right parenthesis = "(" */
+  BT_AST,      /* asterisk = "*" */
+  BT_PLUS,     /* plus sign = "+" */
+  BT_COMMA,    /* comma = "," */
+  BT_VERBAR    /* vertical bar = "|" */
 };
 
 #include <stddef.h>
diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c
index 23d31e8e4249..919c74e9f97f 100644
--- a/Modules/expat/xmltok_ns.c
+++ b/Modules/expat/xmltok_ns.c
@@ -33,56 +33,47 @@
 #ifdef XML_TOK_NS_C
 
 const ENCODING *
-NS(XmlGetUtf8InternalEncoding)(void)
-{
+NS(XmlGetUtf8InternalEncoding)(void) {
   return &ns(internal_utf8_encoding).enc;
 }
 
 const ENCODING *
-NS(XmlGetUtf16InternalEncoding)(void)
-{
-#if BYTEORDER == 1234
+NS(XmlGetUtf16InternalEncoding)(void) {
+#  if BYTEORDER == 1234
   return &ns(internal_little2_encoding).enc;
-#elif BYTEORDER == 4321
+#  elif BYTEORDER == 4321
   return &ns(internal_big2_encoding).enc;
-#else
+#  else
   const short n = 1;
-  return (*(const char *)&n
-          ? &ns(internal_little2_encoding).enc
-          : &ns(internal_big2_encoding).enc);
-#endif
+  return (*(const char *)&n ? &ns(internal_little2_encoding).enc
+                            : &ns(internal_big2_encoding).enc);
+#  endif
 }
 
-static const ENCODING * const NS(encodings)[] = {
-  &ns(latin1_encoding).enc,
-  &ns(ascii_encoding).enc,
-  &ns(utf8_encoding).enc,
-  &ns(big2_encoding).enc,
-  &ns(big2_encoding).enc,
-  &ns(little2_encoding).enc,
-  &ns(utf8_encoding).enc /* NO_ENC */
+static const ENCODING *const NS(encodings)[] = {
+    &ns(latin1_encoding).enc, &ns(ascii_encoding).enc,
+    &ns(utf8_encoding).enc,   &ns(big2_encoding).enc,
+    &ns(big2_encoding).enc,   &ns(little2_encoding).enc,
+    &ns(utf8_encoding).enc /* NO_ENC */
 };
 
 static int PTRCALL
 NS(initScanProlog)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **nextTokPtr)
-{
-  return initScan(NS(encodings), (const INIT_ENCODING *)enc,
-                  XML_PROLOG_STATE, ptr, end, nextTokPtr);
+                   const char **nextTokPtr) {
+  return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_PROLOG_STATE,
+                  ptr, end, nextTokPtr);
 }
 
 static int PTRCALL
 NS(initScanContent)(const ENCODING *enc, const char *ptr, const char *end,
-                    const char **nextTokPtr)
-{
-  return initScan(NS(encodings), (const INIT_ENCODING *)enc,
-                  XML_CONTENT_STATE, ptr, end, nextTokPtr);
+                    const char **nextTokPtr) {
+  return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_CONTENT_STATE,
+                  ptr, end, nextTokPtr);
 }
 
 int
 NS(XmlInitEncoding)(INIT_ENCODING *p, const ENCODING **encPtr,
-                    const char *name)
-{
+                    const char *name) {
   int i = getEncodingIndex(name);
   if (i == UNKNOWN_ENC)
     return 0;
@@ -96,9 +87,8 @@ NS(XmlInitEncoding)(INIT_ENCODING *p, const ENCODING **encPtr,
 }
 
 static const ENCODING *
-NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end)
-{
-#define ENCODING_MAX 128
+NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end) {
+#  define ENCODING_MAX 128
   char buf[ENCODING_MAX];
   char *p = buf;
   int i;
@@ -115,28 +105,14 @@ NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end)
 }
 
 int
-NS(XmlParseXmlDecl)(int isGeneralTextEntity,
-                    const ENCODING *enc,
-                    const char *ptr,
-                    const char *end,
-                    const char **badPtr,
-                    const char **versionPtr,
-                    const char **versionEndPtr,
-                    const char **encodingName,
-                    const ENCODING **encoding,
-                    int *standalone)
-{
-  return doParseXmlDecl(NS(findEncoding),
-                        isGeneralTextEntity,
-                        enc,
-                        ptr,
-                        end,
-                        badPtr,
-                        versionPtr,
-                        versionEndPtr,
-                        encodingName,
-                        encoding,
-                        standalone);
+NS(XmlParseXmlDecl)(int isGeneralTextEntity, const ENCODING *enc,
+                    const char *ptr, const char *end, const char **badPtr,
+                    const char **versionPtr, const char **versionEndPtr,
+                    const char **encodingName, const ENCODING **encoding,
+                    int *standalone) {
+  return doParseXmlDecl(NS(findEncoding), isGeneralTextEntity, enc, ptr, end,
+                        badPtr, versionPtr, versionEndPtr, encodingName,
+                        encoding, standalone);
 }
 
 #endif /* XML_TOK_NS_C */
diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj
index 33a017327189..4a125b243b78 100644
--- a/PCbuild/_elementtree.vcxproj
+++ b/PCbuild/_elementtree.vcxproj
@@ -116,7 +116,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\_elementtree.c" />
-    <ClCompile Include="..\Modules\expat\loadlibrary.c" />
     <ClCompile Include="..\Modules\expat\xmlparse.c" />
     <ClCompile Include="..\Modules\expat\xmlrole.c" />
     <ClCompile Include="..\Modules\expat\xmltok.c" />
diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters
index 4597ee521b33..6acdf35846ab 100644
--- a/PCbuild/_elementtree.vcxproj.filters
+++ b/PCbuild/_elementtree.vcxproj.filters
@@ -33,9 +33,6 @@
     <ClInclude Include="..\Modules\expat\latin1tab.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\expat\loadlibrary.c">
-      <Filter>Header Files</Filter>
-    </ClInclude>
     <ClInclude Include="..\Modules\expat\macconfig.h">
       <Filter>Header Files</Filter>
     </ClInclude>
diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj
index 28a11a82936e..b2d9f5d57d49 100644
--- a/PCbuild/pyexpat.vcxproj
+++ b/PCbuild/pyexpat.vcxproj
@@ -100,7 +100,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\pyexpat.c" />
-    <ClCompile Include="..\Modules\expat\loadlibrary.c" />
     <ClCompile Include="..\Modules\expat\xmlparse.c" />
     <ClCompile Include="..\Modules\expat\xmlrole.c" />
     <ClCompile Include="..\Modules\expat\xmltok.c" />
diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters
index cb02847980c6..f8d46026c9c2 100644
--- a/PCbuild/pyexpat.vcxproj.filters
+++ b/PCbuild/pyexpat.vcxproj.filters
@@ -20,9 +20,6 @@
     <ClCompile Include="..\Modules\pyexpat.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\expat\loadlibrary.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Modules\expat\xmlparse.c">
       <Filter>Source Files</Filter>
     </ClCompile>


From webhook-mailer at python.org  Thu Sep 26 00:49:09 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 26 Sep 2019 04:49:09 -0000
Subject: [Python-checkins] [2.7] closes bpo-38174: Update vendored expat
 library to 2.2.8. (GH-16408)
Message-ID: <mailman.95.1569473351.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/e73b93ab3e0bc2e5a3591f125f465353e025ecaf
commit: e73b93ab3e0bc2e5a3591f125f465353e025ecaf
branch: 2.7
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-25T21:49:04-07:00
summary:

[2.7] closes bpo-38174: Update vendored expat library to 2.2.8. (GH-16408)

Fixes CVE-2019-15903. See full changelog at https://github.com/libexpat/libexpat/blob/R_2_2_8/expat/Changes..
(cherry picked from commit 52b940803860e37bcc3f6096b2d24e7c20a0e807)

Co-authored-by: Benjamin Peterson <benjamin at python.org>

files:
A Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
D Modules/expat/loadlibrary.c
M Modules/expat/asciitab.h
M Modules/expat/expat.h
M Modules/expat/expat_external.h
M Modules/expat/iasciitab.h
M Modules/expat/internal.h
M Modules/expat/latin1tab.h
M Modules/expat/nametab.h
M Modules/expat/siphash.h
M Modules/expat/utf8tab.h
M Modules/expat/winconfig.h
M Modules/expat/xmlparse.c
M Modules/expat/xmlrole.c
M Modules/expat/xmlrole.h
M Modules/expat/xmltok.c
M Modules/expat/xmltok.h
M Modules/expat/xmltok_impl.c
M Modules/expat/xmltok_impl.h
M Modules/expat/xmltok_ns.c
M PCbuild/_elementtree.vcxproj
M PCbuild/_elementtree.vcxproj.filters
M PCbuild/pyexpat.vcxproj
M PCbuild/pyexpat.vcxproj.filters

diff --git a/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst b/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
new file mode 100644
index 000000000000..e3da6fcdd139
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
@@ -0,0 +1,2 @@
+Update vendorized expat library version to 2.2.8, which resolves
+CVE-2019-15903.
diff --git a/Modules/expat/asciitab.h b/Modules/expat/asciitab.h
index 2f59fd929065..63b1d1b4482e 100644
--- a/Modules/expat/asciitab.h
+++ b/Modules/expat/asciitab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
-/* 0x0C */ BT_NONXML, BT_CR, BT_NONXML, BT_NONXML,
-/* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
-/* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
-/* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
-/* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
-/* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
-/* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
-/* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
-/* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
-/* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
+    /* 0x0C */ BT_NONXML, BT_CR, BT_NONXML, BT_NONXML,
+    /* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
+    /* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
+    /* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
+    /* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
+    /* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
+    /* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
+    /* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
+    /* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
+    /* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h
index c050f1d918ce..6c8eb1fda4a3 100644
--- a/Modules/expat/expat.h
+++ b/Modules/expat/expat.h
@@ -33,15 +33,6 @@
 #ifndef Expat_INCLUDED
 #define Expat_INCLUDED 1
 
-#ifdef __VMS
-/*      0        1         2         3      0        1         2         3
-        1234567890123456789012345678901     1234567890123456789012345678901 */
-#define XML_SetProcessingInstructionHandler XML_SetProcessingInstrHandler
-#define XML_SetUnparsedEntityDeclHandler    XML_SetUnparsedEntDeclHandler
-#define XML_SetStartNamespaceDeclHandler    XML_SetStartNamespcDeclHandler
-#define XML_SetExternalEntityRefHandlerArg  XML_SetExternalEntRefHandlerArg
-#endif
-
 #include <stdlib.h>
 #include "expat_external.h"
 
@@ -53,8 +44,8 @@ struct XML_ParserStruct;
 typedef struct XML_ParserStruct *XML_Parser;
 
 typedef unsigned char XML_Bool;
-#define XML_TRUE   ((XML_Bool) 1)
-#define XML_FALSE  ((XML_Bool) 0)
+#define XML_TRUE ((XML_Bool)1)
+#define XML_FALSE ((XML_Bool)0)
 
 /* The XML_Status enum gives the possible return values for several
    API functions.  The preprocessor #defines are included so this
@@ -164,25 +155,23 @@ enum XML_Content_Quant {
 typedef struct XML_cp XML_Content;
 
 struct XML_cp {
-  enum XML_Content_Type         type;
-  enum XML_Content_Quant        quant;
-  XML_Char *                    name;
-  unsigned int                  numchildren;
-  XML_Content *                 children;
+  enum XML_Content_Type type;
+  enum XML_Content_Quant quant;
+  XML_Char *name;
+  unsigned int numchildren;
+  XML_Content *children;
 };
 
-
 /* This is called for an element declaration. See above for
    description of the model argument. It's the caller's responsibility
    to free model when finished with it.
 */
-typedef void (XMLCALL *XML_ElementDeclHandler) (void *userData,
-                                                const XML_Char *name,
-                                                XML_Content *model);
+typedef void(XMLCALL *XML_ElementDeclHandler)(void *userData,
+                                              const XML_Char *name,
+                                              XML_Content *model);
 
 XMLPARSEAPI(void)
-XML_SetElementDeclHandler(XML_Parser parser,
-                          XML_ElementDeclHandler eldecl);
+XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl);
 
 /* The Attlist declaration handler is called for *each* attribute. So
    a single Attlist declaration with multiple attributes declared will
@@ -192,17 +181,12 @@ XML_SetElementDeclHandler(XML_Parser parser,
    value will be NULL in the case of "#REQUIRED". If "isrequired" is
    true and default is non-NULL, then this is a "#FIXED" default.
 */
-typedef void (XMLCALL *XML_AttlistDeclHandler) (
-                                    void            *userData,
-                                    const XML_Char  *elname,
-                                    const XML_Char  *attname,
-                                    const XML_Char  *att_type,
-                                    const XML_Char  *dflt,
-                                    int              isrequired);
+typedef void(XMLCALL *XML_AttlistDeclHandler)(
+    void *userData, const XML_Char *elname, const XML_Char *attname,
+    const XML_Char *att_type, const XML_Char *dflt, int isrequired);
 
 XMLPARSEAPI(void)
-XML_SetAttlistDeclHandler(XML_Parser parser,
-                          XML_AttlistDeclHandler attdecl);
+XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl);
 
 /* The XML declaration handler is called for *both* XML declarations
    and text declarations. The way to distinguish is that the version
@@ -212,15 +196,13 @@ XML_SetAttlistDeclHandler(XML_Parser parser,
    was no standalone parameter in the declaration, that it was given
    as no, or that it was given as yes.
 */
-typedef void (XMLCALL *XML_XmlDeclHandler) (void           *userData,
-                                            const XML_Char *version,
-                                            const XML_Char *encoding,
-                                            int             standalone);
+typedef void(XMLCALL *XML_XmlDeclHandler)(void *userData,
+                                          const XML_Char *version,
+                                          const XML_Char *encoding,
+                                          int standalone);
 
 XMLPARSEAPI(void)
-XML_SetXmlDeclHandler(XML_Parser parser,
-                      XML_XmlDeclHandler xmldecl);
-
+XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler xmldecl);
 
 typedef struct {
   void *(*malloc_fcn)(size_t size);
@@ -248,7 +230,6 @@ XML_ParserCreate(const XML_Char *encoding);
 XMLPARSEAPI(XML_Parser)
 XML_ParserCreateNS(const XML_Char *encoding, XML_Char namespaceSeparator);
 
-
 /* Constructs a new parser using the memory management suite referred to
    by memsuite. If memsuite is NULL, then use the standard library memory
    suite. If namespaceSeparator is non-NULL it creates a parser with
@@ -278,31 +259,27 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encoding);
 /* atts is array of name/value pairs, terminated by 0;
    names and values are 0 terminated.
 */
-typedef void (XMLCALL *XML_StartElementHandler) (void *userData,
-                                                 const XML_Char *name,
-                                                 const XML_Char **atts);
-
-typedef void (XMLCALL *XML_EndElementHandler) (void *userData,
-                                               const XML_Char *name);
+typedef void(XMLCALL *XML_StartElementHandler)(void *userData,
+                                               const XML_Char *name,
+                                               const XML_Char **atts);
 
+typedef void(XMLCALL *XML_EndElementHandler)(void *userData,
+                                             const XML_Char *name);
 
 /* s is not 0 terminated. */
-typedef void (XMLCALL *XML_CharacterDataHandler) (void *userData,
-                                                  const XML_Char *s,
-                                                  int len);
+typedef void(XMLCALL *XML_CharacterDataHandler)(void *userData,
+                                                const XML_Char *s, int len);
 
 /* target and data are 0 terminated */
-typedef void (XMLCALL *XML_ProcessingInstructionHandler) (
-                                                void *userData,
-                                                const XML_Char *target,
-                                                const XML_Char *data);
+typedef void(XMLCALL *XML_ProcessingInstructionHandler)(void *userData,
+                                                        const XML_Char *target,
+                                                        const XML_Char *data);
 
 /* data is 0 terminated */
-typedef void (XMLCALL *XML_CommentHandler) (void *userData,
-                                            const XML_Char *data);
+typedef void(XMLCALL *XML_CommentHandler)(void *userData, const XML_Char *data);
 
-typedef void (XMLCALL *XML_StartCdataSectionHandler) (void *userData);
-typedef void (XMLCALL *XML_EndCdataSectionHandler) (void *userData);
+typedef void(XMLCALL *XML_StartCdataSectionHandler)(void *userData);
+typedef void(XMLCALL *XML_EndCdataSectionHandler)(void *userData);
 
 /* This is called for any characters in the XML document for which
    there is no applicable handler.  This includes both characters that
@@ -317,25 +294,23 @@ typedef void (XMLCALL *XML_EndCdataSectionHandler) (void *userData);
    default handler: for example, a comment might be split between
    multiple calls.
 */
-typedef void (XMLCALL *XML_DefaultHandler) (void *userData,
-                                            const XML_Char *s,
-                                            int len);
+typedef void(XMLCALL *XML_DefaultHandler)(void *userData, const XML_Char *s,
+                                          int len);
 
 /* This is called for the start of the DOCTYPE declaration, before
    any DTD or internal subset is parsed.
 */
-typedef void (XMLCALL *XML_StartDoctypeDeclHandler) (
-                                            void *userData,
-                                            const XML_Char *doctypeName,
-                                            const XML_Char *sysid,
-                                            const XML_Char *pubid,
-                                            int has_internal_subset);
+typedef void(XMLCALL *XML_StartDoctypeDeclHandler)(void *userData,
+                                                   const XML_Char *doctypeName,
+                                                   const XML_Char *sysid,
+                                                   const XML_Char *pubid,
+                                                   int has_internal_subset);
 
 /* This is called for the start of the DOCTYPE declaration when the
    closing > is encountered, but after processing any external
    subset.
 */
-typedef void (XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
+typedef void(XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
 
 /* This is called for entity declarations. The is_parameter_entity
    argument will be non-zero if the entity is a parameter entity, zero
@@ -355,20 +330,14 @@ typedef void (XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
    Note that is_parameter_entity can't be changed to XML_Bool, since
    that would break binary compatibility.
 */
-typedef void (XMLCALL *XML_EntityDeclHandler) (
-                              void *userData,
-                              const XML_Char *entityName,
-                              int is_parameter_entity,
-                              const XML_Char *value,
-                              int value_length,
-                              const XML_Char *base,
-                              const XML_Char *systemId,
-                              const XML_Char *publicId,
-                              const XML_Char *notationName);
+typedef void(XMLCALL *XML_EntityDeclHandler)(
+    void *userData, const XML_Char *entityName, int is_parameter_entity,
+    const XML_Char *value, int value_length, const XML_Char *base,
+    const XML_Char *systemId, const XML_Char *publicId,
+    const XML_Char *notationName);
 
 XMLPARSEAPI(void)
-XML_SetEntityDeclHandler(XML_Parser parser,
-                         XML_EntityDeclHandler handler);
+XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler);
 
 /* OBSOLETE -- OBSOLETE -- OBSOLETE
    This handler has been superseded by the EntityDeclHandler above.
@@ -379,24 +348,20 @@ XML_SetEntityDeclHandler(XML_Parser parser,
    entityName, systemId and notationName arguments will never be
    NULL. The other arguments may be.
 */
-typedef void (XMLCALL *XML_UnparsedEntityDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *entityName,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId,
-                                    const XML_Char *notationName);
+typedef void(XMLCALL *XML_UnparsedEntityDeclHandler)(
+    void *userData, const XML_Char *entityName, const XML_Char *base,
+    const XML_Char *systemId, const XML_Char *publicId,
+    const XML_Char *notationName);
 
 /* This is called for a declaration of notation.  The base argument is
    whatever was set by XML_SetBase. The notationName will never be
    NULL.  The other arguments can be.
 */
-typedef void (XMLCALL *XML_NotationDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *notationName,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId);
+typedef void(XMLCALL *XML_NotationDeclHandler)(void *userData,
+                                               const XML_Char *notationName,
+                                               const XML_Char *base,
+                                               const XML_Char *systemId,
+                                               const XML_Char *publicId);
 
 /* When namespace processing is enabled, these are called once for
    each namespace declaration. The call to the start and end element
@@ -404,14 +369,12 @@ typedef void (XMLCALL *XML_NotationDeclHandler) (
    declaration handlers. For an xmlns attribute, prefix will be
    NULL.  For an xmlns="" attribute, uri will be NULL.
 */
-typedef void (XMLCALL *XML_StartNamespaceDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *prefix,
-                                    const XML_Char *uri);
+typedef void(XMLCALL *XML_StartNamespaceDeclHandler)(void *userData,
+                                                     const XML_Char *prefix,
+                                                     const XML_Char *uri);
 
-typedef void (XMLCALL *XML_EndNamespaceDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *prefix);
+typedef void(XMLCALL *XML_EndNamespaceDeclHandler)(void *userData,
+                                                   const XML_Char *prefix);
 
 /* This is called if the document is not standalone, that is, it has an
    external subset or a reference to a parameter entity, but does not
@@ -422,7 +385,7 @@ typedef void (XMLCALL *XML_EndNamespaceDeclHandler) (
    conditions above this handler will only be called if the referenced
    entity was actually read.
 */
-typedef int (XMLCALL *XML_NotStandaloneHandler) (void *userData);
+typedef int(XMLCALL *XML_NotStandaloneHandler)(void *userData);
 
 /* This is called for a reference to an external parsed general
    entity.  The referenced entity is not automatically parsed.  The
@@ -458,12 +421,11 @@ typedef int (XMLCALL *XML_NotStandaloneHandler) (void *userData);
    Note that unlike other handlers the first argument is the parser,
    not userData.
 */
-typedef int (XMLCALL *XML_ExternalEntityRefHandler) (
-                                    XML_Parser parser,
-                                    const XML_Char *context,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId);
+typedef int(XMLCALL *XML_ExternalEntityRefHandler)(XML_Parser parser,
+                                                   const XML_Char *context,
+                                                   const XML_Char *base,
+                                                   const XML_Char *systemId,
+                                                   const XML_Char *publicId);
 
 /* This is called in two situations:
    1) An entity reference is encountered for which no declaration
@@ -475,10 +437,9 @@ typedef int (XMLCALL *XML_ExternalEntityRefHandler) (
          the event would be out of sync with the reporting of the
          declarations or attribute values
 */
-typedef void (XMLCALL *XML_SkippedEntityHandler) (
-                                    void *userData,
-                                    const XML_Char *entityName,
-                                    int is_parameter_entity);
+typedef void(XMLCALL *XML_SkippedEntityHandler)(void *userData,
+                                                const XML_Char *entityName,
+                                                int is_parameter_entity);
 
 /* This structure is filled in by the XML_UnknownEncodingHandler to
    provide information to the parser about encodings that are unknown
@@ -535,8 +496,8 @@ typedef void (XMLCALL *XML_SkippedEntityHandler) (
 typedef struct {
   int map[256];
   void *data;
-  int (XMLCALL *convert)(void *data, const char *s);
-  void (XMLCALL *release)(void *data);
+  int(XMLCALL *convert)(void *data, const char *s);
+  void(XMLCALL *release)(void *data);
 } XML_Encoding;
 
 /* This is called for an encoding that is unknown to the parser.
@@ -554,23 +515,19 @@ typedef struct {
    If info does not describe a suitable encoding, then the parser will
    return an XML_UNKNOWN_ENCODING error.
 */
-typedef int (XMLCALL *XML_UnknownEncodingHandler) (
-                                    void *encodingHandlerData,
-                                    const XML_Char *name,
-                                    XML_Encoding *info);
+typedef int(XMLCALL *XML_UnknownEncodingHandler)(void *encodingHandlerData,
+                                                 const XML_Char *name,
+                                                 XML_Encoding *info);
 
 XMLPARSEAPI(void)
-XML_SetElementHandler(XML_Parser parser,
-                      XML_StartElementHandler start,
+XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start,
                       XML_EndElementHandler end);
 
 XMLPARSEAPI(void)
-XML_SetStartElementHandler(XML_Parser parser,
-                           XML_StartElementHandler handler);
+XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetEndElementHandler(XML_Parser parser,
-                         XML_EndElementHandler handler);
+XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetCharacterDataHandler(XML_Parser parser,
@@ -580,8 +537,7 @@ XMLPARSEAPI(void)
 XML_SetProcessingInstructionHandler(XML_Parser parser,
                                     XML_ProcessingInstructionHandler handler);
 XMLPARSEAPI(void)
-XML_SetCommentHandler(XML_Parser parser,
-                      XML_CommentHandler handler);
+XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetCdataSectionHandler(XML_Parser parser,
@@ -601,20 +557,17 @@ XML_SetEndCdataSectionHandler(XML_Parser parser,
    default handler, or to the skipped entity handler, if one is set.
 */
 XMLPARSEAPI(void)
-XML_SetDefaultHandler(XML_Parser parser,
-                      XML_DefaultHandler handler);
+XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler);
 
 /* This sets the default handler but does not inhibit expansion of
    internal entities.  The entity reference will not be passed to the
    default handler.
 */
 XMLPARSEAPI(void)
-XML_SetDefaultHandlerExpand(XML_Parser parser,
-                            XML_DefaultHandler handler);
+XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetDoctypeDeclHandler(XML_Parser parser,
-                          XML_StartDoctypeDeclHandler start,
+XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start,
                           XML_EndDoctypeDeclHandler end);
 
 XMLPARSEAPI(void)
@@ -622,16 +575,14 @@ XML_SetStartDoctypeDeclHandler(XML_Parser parser,
                                XML_StartDoctypeDeclHandler start);
 
 XMLPARSEAPI(void)
-XML_SetEndDoctypeDeclHandler(XML_Parser parser,
-                             XML_EndDoctypeDeclHandler end);
+XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end);
 
 XMLPARSEAPI(void)
 XML_SetUnparsedEntityDeclHandler(XML_Parser parser,
                                  XML_UnparsedEntityDeclHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetNotationDeclHandler(XML_Parser parser,
-                           XML_NotationDeclHandler handler);
+XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetNamespaceDeclHandler(XML_Parser parser,
@@ -659,8 +610,7 @@ XML_SetExternalEntityRefHandler(XML_Parser parser,
    instead of the parser object.
 */
 XMLPARSEAPI(void)
-XML_SetExternalEntityRefHandlerArg(XML_Parser parser,
-                                   void *arg);
+XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg);
 
 XMLPARSEAPI(void)
 XML_SetSkippedEntityHandler(XML_Parser parser,
@@ -740,7 +690,6 @@ XML_UseParserAsHandlerArg(XML_Parser parser);
 XMLPARSEAPI(enum XML_Error)
 XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD);
 
-
 /* Sets the base to be used for resolving relative URIs in system
    identifiers in declarations.  Resolving relative identifiers is
    left to the application: this value will be passed through as the
@@ -780,10 +729,10 @@ XML_GetIdAttributeIndex(XML_Parser parser);
    info->valueEnd - info->valueStart = 4 bytes.
 */
 typedef struct {
-  XML_Index  nameStart;  /* Offset to beginning of the attribute name. */
-  XML_Index  nameEnd;    /* Offset after the attribute name's last byte. */
-  XML_Index  valueStart; /* Offset to beginning of the attribute value. */
-  XML_Index  valueEnd;   /* Offset after the attribute value's last byte. */
+  XML_Index nameStart;  /* Offset to beginning of the attribute name. */
+  XML_Index nameEnd;    /* Offset after the attribute name's last byte. */
+  XML_Index valueStart; /* Offset to beginning of the attribute value. */
+  XML_Index valueEnd;   /* Offset after the attribute value's last byte. */
 } XML_AttrInfo;
 
 /* Returns an array of XML_AttrInfo structures for the attribute/value pairs
@@ -819,20 +768,20 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal);
    (resumable = 0) an already suspended parser. Some call-backs may
    still follow because they would otherwise get lost. Examples:
    - endElementHandler() for empty elements when stopped in
-     startElementHandler(), 
-   - endNameSpaceDeclHandler() when stopped in endElementHandler(), 
+     startElementHandler(),
+   - endNameSpaceDeclHandler() when stopped in endElementHandler(),
    and possibly others.
 
    Can be called from most handlers, including DTD related call-backs,
    except when parsing an external parameter entity and resumable != 0.
    Returns XML_STATUS_OK when successful, XML_STATUS_ERROR otherwise.
-   Possible error codes: 
+   Possible error codes:
    - XML_ERROR_SUSPENDED: when suspending an already suspended parser.
    - XML_ERROR_FINISHED: when the parser has already finished.
    - XML_ERROR_SUSPEND_PE: when suspending while parsing an external PE.
 
-   When resumable != 0 (true) then parsing is suspended, that is, 
-   XML_Parse() and XML_ParseBuffer() return XML_STATUS_SUSPENDED. 
+   When resumable != 0 (true) then parsing is suspended, that is,
+   XML_Parse() and XML_ParseBuffer() return XML_STATUS_SUSPENDED.
    Otherwise, parsing is aborted, that is, XML_Parse() and XML_ParseBuffer()
    return XML_STATUS_ERROR with error code XML_ERROR_ABORTED.
 
@@ -843,7 +792,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal);
    the externalEntityRefHandler() to call XML_StopParser() on the parent
    parser (recursively), if one wants to stop parsing altogether.
 
-   When suspended, parsing can be resumed by calling XML_ResumeParser(). 
+   When suspended, parsing can be resumed by calling XML_ResumeParser().
 */
 XMLPARSEAPI(enum XML_Status)
 XML_StopParser(XML_Parser parser, XML_Bool resumable);
@@ -851,7 +800,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable);
 /* Resumes parsing after it has been suspended with XML_StopParser().
    Must not be called from within a handler call-back. Returns same
    status codes as XML_Parse() or XML_ParseBuffer().
-   Additional error code XML_ERROR_NOT_SUSPENDED possible.   
+   Additional error code XML_ERROR_NOT_SUSPENDED possible.
 
    *Note*:
    This must be called on the most deeply nested child parser instance
@@ -863,12 +812,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable);
 XMLPARSEAPI(enum XML_Status)
 XML_ResumeParser(XML_Parser parser);
 
-enum XML_Parsing {
-  XML_INITIALIZED,
-  XML_PARSING,
-  XML_FINISHED,
-  XML_SUSPENDED
-};
+enum XML_Parsing { XML_INITIALIZED, XML_PARSING, XML_FINISHED, XML_SUSPENDED };
 
 typedef struct {
   enum XML_Parsing parsing;
@@ -900,8 +844,7 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status);
    Otherwise returns a new XML_Parser object.
 */
 XMLPARSEAPI(XML_Parser)
-XML_ExternalEntityParserCreate(XML_Parser parser,
-                               const XML_Char *context,
+XML_ExternalEntityParserCreate(XML_Parser parser, const XML_Char *context,
                                const XML_Char *encoding);
 
 enum XML_ParamEntityParsing {
@@ -945,8 +888,7 @@ XML_SetParamEntityParsing(XML_Parser parser,
    Note: If parser == NULL, the function will do nothing and return 0.
 */
 XMLPARSEAPI(int)
-XML_SetHashSalt(XML_Parser parser,
-                unsigned long hash_salt);
+XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt);
 
 /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then
    XML_GetErrorCode returns information about the error.
@@ -963,7 +905,7 @@ XML_GetErrorCode(XML_Parser parser);
    be within the relevant markup.  When called outside of the callback
    functions, the position indicated will be just past the last parse
    event (regardless of whether there was an associated callback).
-   
+
    They may also be called after returning from a call to XML_Parse
    or XML_ParseBuffer.  If the return value is XML_STATUS_ERROR then
    the location is the location of the character at which the error
@@ -995,14 +937,12 @@ XML_GetCurrentByteCount(XML_Parser parser);
    the handler that makes the call.
 */
 XMLPARSEAPI(const char *)
-XML_GetInputContext(XML_Parser parser,
-                    int *offset,
-                    int *size);
+XML_GetInputContext(XML_Parser parser, int *offset, int *size);
 
 /* For backwards compatibility with previous versions. */
-#define XML_GetErrorLineNumber   XML_GetCurrentLineNumber
+#define XML_GetErrorLineNumber XML_GetCurrentLineNumber
 #define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber
-#define XML_GetErrorByteIndex    XML_GetCurrentByteIndex
+#define XML_GetErrorByteIndex XML_GetCurrentByteIndex
 
 /* Frees the content model passed to the element declaration handler */
 XMLPARSEAPI(void)
@@ -1062,21 +1002,20 @@ enum XML_FeatureEnum {
 };
 
 typedef struct {
-  enum XML_FeatureEnum  feature;
-  const XML_LChar       *name;
-  long int              value;
+  enum XML_FeatureEnum feature;
+  const XML_LChar *name;
+  long int value;
 } XML_Feature;
 
 XMLPARSEAPI(const XML_Feature *)
 XML_GetFeatureList(void);
 
-
 /* Expat follows the semantic versioning convention.
    See http://semver.org.
 */
 #define XML_MAJOR_VERSION 2
 #define XML_MINOR_VERSION 2
-#define XML_MICRO_VERSION 7
+#define XML_MICRO_VERSION 8
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h
index fc63c2228d3e..f2b75dda8e27 100644
--- a/Modules/expat/expat_external.h
+++ b/Modules/expat/expat_external.h
@@ -35,10 +35,6 @@
 
 /* External API definitions */
 
-#if defined(_MSC_EXTENSIONS) && !defined(__BEOS__) && !defined(__CYGWIN__)
-# define XML_USE_MSC_EXTENSIONS 1
-#endif
-
 /* Expat tries very hard to make the API boundary very specifically
    defined.  There are two macros defined to control this boundary;
    each of these can be defined before including this header to
@@ -61,12 +57,17 @@
    compiled with the cdecl calling convention as the default since
    system headers may assume the cdecl convention.
 */
+
+/* Namespace external symbols to allow multiple libexpat version to
+   co-exist. */
+#include "pyexpatns.h"
+
 #ifndef XMLCALL
-# if defined(_MSC_VER)
-#  define XMLCALL __cdecl
-# elif defined(__GNUC__) && defined(__i386) && !defined(__INTEL_COMPILER)
-#  define XMLCALL __attribute__((cdecl))
-# else
+#  if defined(_MSC_VER)
+#    define XMLCALL __cdecl
+#  elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER)
+#    define XMLCALL __attribute__((cdecl))
+#  else
 /* For any platform which uses this definition and supports more than
    one calling convention, we need to extend this definition to
    declare the convention used on that platform, if it's possible to
@@ -77,49 +78,46 @@
    pre-processor and how to specify the same calling convention as the
    platform's malloc() implementation.
 */
-#  define XMLCALL
-# endif
-#endif  /* not defined XMLCALL */
-
-/* Namespace external symbols to allow multiple libexpat version to
-   co-exist. */
-#include "pyexpatns.h"
-
+#    define XMLCALL
+#  endif
+#endif /* not defined XMLCALL */
 
-#if !defined(XML_STATIC) && !defined(XMLIMPORT)
-# ifndef XML_BUILDING_EXPAT
+#if ! defined(XML_STATIC) && ! defined(XMLIMPORT)
+#  ifndef XML_BUILDING_EXPAT
 /* using Expat from an application */
 
-#  ifdef XML_USE_MSC_EXTENSIONS
-#   define XMLIMPORT __declspec(dllimport)
-#  endif
+#    if defined(_MSC_EXTENSIONS) && ! defined(__BEOS__) && ! defined(__CYGWIN__)
+#      define XMLIMPORT __declspec(dllimport)
+#    endif
 
-# endif
-#endif  /* not defined XML_STATIC */
+#  endif
+#endif /* not defined XML_STATIC */
 
 #ifndef XML_ENABLE_VISIBILITY
-# define XML_ENABLE_VISIBILITY 0
+#  define XML_ENABLE_VISIBILITY 0
 #endif
 
-#if !defined(XMLIMPORT) && XML_ENABLE_VISIBILITY
-# define XMLIMPORT __attribute__ ((visibility ("default")))
+#if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY
+#  define XMLIMPORT __attribute__((visibility("default")))
 #endif
 
 /* If we didn't define it above, define it away: */
 #ifndef XMLIMPORT
-# define XMLIMPORT
+#  define XMLIMPORT
 #endif
 
-#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
-# define XML_ATTR_MALLOC __attribute__((__malloc__))
+#if defined(__GNUC__)                                                          \
+    && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
+#  define XML_ATTR_MALLOC __attribute__((__malloc__))
 #else
-# define XML_ATTR_MALLOC
+#  define XML_ATTR_MALLOC
 #endif
 
-#if defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
-# define XML_ATTR_ALLOC_SIZE(x)  __attribute__((__alloc_size__(x)))
+#if defined(__GNUC__)                                                          \
+    && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
+#  define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x)))
 #else
-# define XML_ATTR_ALLOC_SIZE(x)
+#  define XML_ATTR_ALLOC_SIZE(x)
 #endif
 
 #define XMLPARSEAPI(type) XMLIMPORT type XMLCALL
@@ -129,35 +127,30 @@ extern "C" {
 #endif
 
 #ifdef XML_UNICODE_WCHAR_T
-# ifndef XML_UNICODE
-#  define XML_UNICODE
-# endif
-# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
-#  error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
-# endif
+#  ifndef XML_UNICODE
+#    define XML_UNICODE
+#  endif
+#  if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
+#    error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
+#  endif
 #endif
 
-#ifdef XML_UNICODE     /* Information is UTF-16 encoded. */
-# ifdef XML_UNICODE_WCHAR_T
+#ifdef XML_UNICODE /* Information is UTF-16 encoded. */
+#  ifdef XML_UNICODE_WCHAR_T
 typedef wchar_t XML_Char;
 typedef wchar_t XML_LChar;
-# else
+#  else
 typedef unsigned short XML_Char;
 typedef char XML_LChar;
-# endif /* XML_UNICODE_WCHAR_T */
-#else                  /* Information is UTF-8 encoded. */
+#  endif /* XML_UNICODE_WCHAR_T */
+#else    /* Information is UTF-8 encoded. */
 typedef char XML_Char;
 typedef char XML_LChar;
-#endif /* XML_UNICODE */
+#endif   /* XML_UNICODE */
 
-#ifdef XML_LARGE_SIZE  /* Use large integers for file/stream positions. */
-# if defined(XML_USE_MSC_EXTENSIONS) && _MSC_VER < 1400
-typedef __int64 XML_Index; 
-typedef unsigned __int64 XML_Size;
-# else
+#ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */
 typedef long long XML_Index;
 typedef unsigned long long XML_Size;
-# endif
 #else
 typedef long XML_Index;
 typedef unsigned long XML_Size;
diff --git a/Modules/expat/iasciitab.h b/Modules/expat/iasciitab.h
index ce4a4bf7edef..ea97cfcf678e 100644
--- a/Modules/expat/iasciitab.h
+++ b/Modules/expat/iasciitab.h
@@ -32,34 +32,34 @@
 
 /* Like asciitab.h, except that 0xD has code BT_S rather than BT_CR */
 /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
-/* 0x0C */ BT_NONXML, BT_S, BT_NONXML, BT_NONXML,
-/* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
-/* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
-/* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
-/* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
-/* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
-/* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
-/* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
-/* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
-/* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
+    /* 0x0C */ BT_NONXML, BT_S, BT_NONXML, BT_NONXML,
+    /* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
+    /* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
+    /* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
+    /* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
+    /* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
+    /* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
+    /* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
+    /* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
+    /* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h
index dc4ef0c7e258..60913dab762f 100644
--- a/Modules/expat/internal.h
+++ b/Modules/expat/internal.h
@@ -49,7 +49,7 @@
    USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
-#if defined(__GNUC__) && defined(__i386__) && !defined(__MINGW32__)
+#if defined(__GNUC__) && defined(__i386__) && ! defined(__MINGW32__)
 /* We'll use this version by default only where we know it helps.
 
    regparm() generates warnings on Solaris boxes.   See SF bug #692878.
@@ -59,8 +59,8 @@
    #define FASTCALL __attribute__((stdcall, regparm(3)))
    and let's try this:
 */
-#define FASTCALL __attribute__((regparm(3)))
-#define PTRFASTCALL __attribute__((regparm(3)))
+#  define FASTCALL __attribute__((regparm(3)))
+#  define PTRFASTCALL __attribute__((regparm(3)))
 #endif
 
 /* Using __fastcall seems to have an unexpected negative effect under
@@ -74,55 +74,49 @@
 /* Make sure all of these are defined if they aren't already. */
 
 #ifndef FASTCALL
-#define FASTCALL
+#  define FASTCALL
 #endif
 
 #ifndef PTRCALL
-#define PTRCALL
+#  define PTRCALL
 #endif
 
 #ifndef PTRFASTCALL
-#define PTRFASTCALL
+#  define PTRFASTCALL
 #endif
 
 #ifndef XML_MIN_SIZE
-#if !defined(__cplusplus) && !defined(inline)
-#ifdef __GNUC__
-#define inline __inline
-#endif /* __GNUC__ */
-#endif
+#  if ! defined(__cplusplus) && ! defined(inline)
+#    ifdef __GNUC__
+#      define inline __inline
+#    endif /* __GNUC__ */
+#  endif
 #endif /* XML_MIN_SIZE */
 
 #ifdef __cplusplus
-#define inline inline
+#  define inline inline
 #else
-#ifndef inline
-#define inline
-#endif
+#  ifndef inline
+#    define inline
+#  endif
 #endif
 
 #ifndef UNUSED_P
-# ifdef __GNUC__
-#  define UNUSED_P(p) UNUSED_ ## p __attribute__((__unused__))
-# else
-#  define UNUSED_P(p) UNUSED_ ## p
-# endif
+#  define UNUSED_P(p) (void)p
 #endif
 
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-
 #ifdef XML_ENABLE_VISIBILITY
-#if XML_ENABLE_VISIBILITY
-__attribute__ ((visibility ("default")))
-#endif
+#  if XML_ENABLE_VISIBILITY
+__attribute__((visibility("default")))
+#  endif
 #endif
 void
-_INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** fromLimRef);
-
+_INTERNAL_trim_to_complete_utf8_characters(const char *from,
+                                           const char **fromLimRef);
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/latin1tab.h b/Modules/expat/latin1tab.h
index 95dfa52b1fbe..6f916041355a 100644
--- a/Modules/expat/latin1tab.h
+++ b/Modules/expat/latin1tab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x80 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x84 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x88 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x8C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x90 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x94 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x98 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x9C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA4 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
-/* 0xAC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xB0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xB4 */ BT_OTHER, BT_NMSTRT, BT_OTHER, BT_NAME,
-/* 0xB8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
-/* 0xBC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xC0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xC4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xC8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xCC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xD0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xD4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0xD8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xDC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xEC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xF0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xF4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0xF8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xFC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x84 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x88 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x8C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x90 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x94 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x98 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x9C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA4 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
+    /* 0xAC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xB0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xB4 */ BT_OTHER, BT_NMSTRT, BT_OTHER, BT_NAME,
+    /* 0xB8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
+    /* 0xBC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xC0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xC4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xC8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xCC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xD0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xD4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0xD8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xDC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xEC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xF0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xF4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0xF8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xFC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
diff --git a/Modules/expat/loadlibrary.c b/Modules/expat/loadlibrary.c
deleted file mode 100644
index 35fdf98bce6c..000000000000
--- a/Modules/expat/loadlibrary.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/***************************************************************************
- *                                  _   _ ____  _
- *  Project                     ___| | | |  _ \| |
- *                             / __| | | | |_) | |
- *                            | (__| |_| |  _ <| |___
- *                             \___|\___/|_| \_\_____|
- *
- * Copyright (C) 2016 - 2017, Steve Holme, <steve_holme at hotmail.com>.
- * Copyright (C) 2017, Expat development team
- *
- * All rights reserved.
- * Licensed under the MIT license:
- *
- * Permission to  use, copy,  modify, and distribute  this software  for any
- * purpose with  or without fee is  hereby granted, provided that  the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE  SOFTWARE  IS  PROVIDED  "AS  IS",  WITHOUT  WARRANTY  OF  ANY  KIND,
- * EXPRESS  OR IMPLIED,  INCLUDING  BUT  NOT LIMITED  TO  THE WARRANTIES  OF
- * MERCHANTABILITY, FITNESS FOR A  PARTICULAR PURPOSE AND NONINFRINGEMENT OF
- * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
- * CONTRACT, TORT OR  OTHERWISE, ARISING FROM, OUT OF OR  IN CONNECTION WITH
- * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * Except as contained in this notice,  the name of a copyright holder shall
- * not be used in advertising or otherwise to promote the sale, use or other
- * dealings  in this  Software without  prior written  authorization of  the
- * copyright holder.
- *
- ***************************************************************************/
-
-#if defined(_WIN32)
-
-#include <windows.h>
-#include <tchar.h>
-
-
-HMODULE _Expat_LoadLibrary(LPCTSTR filename);
-
-
-#if !defined(LOAD_WITH_ALTERED_SEARCH_PATH)
-#define LOAD_WITH_ALTERED_SEARCH_PATH  0x00000008
-#endif
-
-#if !defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
-#define LOAD_LIBRARY_SEARCH_SYSTEM32   0x00000800
-#endif
-
-/* We use our own typedef here since some headers might lack these */
-typedef HMODULE (APIENTRY *LOADLIBRARYEX_FN)(LPCTSTR, HANDLE, DWORD);
-
-/* See function definitions in winbase.h */
-#ifdef UNICODE
-#  ifdef _WIN32_WCE
-#    define LOADLIBARYEX  L"LoadLibraryExW"
-#  else
-#    define LOADLIBARYEX  "LoadLibraryExW"
-#  endif
-#else
-#  define LOADLIBARYEX    "LoadLibraryExA"
-#endif
-
-
-/*
- * _Expat_LoadLibrary()
- *
- * This is used to dynamically load DLLs using the most secure method available
- * for the version of Windows that we are running on.
- *
- * Parameters:
- *
- * filename  [in] - The filename or full path of the DLL to load. If only the
- *                  filename is passed then the DLL will be loaded from the
- *                  Windows system directory.
- *
- * Returns the handle of the module on success; otherwise NULL.
- */
-HMODULE _Expat_LoadLibrary(LPCTSTR filename)
-{
-  HMODULE hModule = NULL;
-  LOADLIBRARYEX_FN pLoadLibraryEx = NULL;
-
-  /* Get a handle to kernel32 so we can access it's functions at runtime */
-  HMODULE hKernel32 = GetModuleHandle(TEXT("kernel32"));
-  if(!hKernel32)
-    return NULL;  /* LCOV_EXCL_LINE */
-
-  /* Attempt to find LoadLibraryEx() which is only available on Windows 2000
-     and above */
-  pLoadLibraryEx = (LOADLIBRARYEX_FN) GetProcAddress(hKernel32, LOADLIBARYEX);
-
-  /* Detect if there's already a path in the filename and load the library if
-     there is. Note: Both back slashes and forward slashes have been supported
-     since the earlier days of DOS at an API level although they are not
-     supported by command prompt */
-  if(_tcspbrk(filename, TEXT("\\/"))) {
-    /** !checksrc! disable BANNEDFUNC 1 **/
-    hModule = pLoadLibraryEx ?
-      pLoadLibraryEx(filename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) :
-      LoadLibrary(filename);
-  }
-  /* Detect if KB2533623 is installed, as LOAD_LIBARY_SEARCH_SYSTEM32 is only
-     supported on Windows Vista, Windows Server 2008, Windows 7 and Windows
-     Server 2008 R2 with this patch or natively on Windows 8 and above */
-  else if(pLoadLibraryEx && GetProcAddress(hKernel32, "AddDllDirectory")) {
-    /* Load the DLL from the Windows system directory */
-    hModule = pLoadLibraryEx(filename, NULL, LOAD_LIBRARY_SEARCH_SYSTEM32);
-  }
-  else {
-    /* Attempt to get the Windows system path */
-    UINT systemdirlen = GetSystemDirectory(NULL, 0);
-    if(systemdirlen) {
-      /* Allocate space for the full DLL path (Room for the null terminator
-         is included in systemdirlen) */
-      size_t filenamelen = _tcslen(filename);
-      TCHAR *path = malloc(sizeof(TCHAR) * (systemdirlen + 1 + filenamelen));
-      if(path && GetSystemDirectory(path, systemdirlen)) {
-        /* Calculate the full DLL path */
-        _tcscpy(path + _tcslen(path), TEXT("\\"));
-        _tcscpy(path + _tcslen(path), filename);
-
-        /* Load the DLL from the Windows system directory */
-        /** !checksrc! disable BANNEDFUNC 1 **/
-        hModule = pLoadLibraryEx ?
-          pLoadLibraryEx(path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) :
-          LoadLibrary(path);
-
-      }
-      free(path);
-    }
-  }
-
-  return hModule;
-}
-
-#else /* defined(_WIN32) */
-
-/* ISO C requires a translation unit to contain at least one declaration
-   [-Wempty-translation-unit] */
-typedef int _TRANSLATION_UNIT_LOAD_LIBRARY_C_NOT_EMTPY;
-
-#endif /* defined(_WIN32) */
diff --git a/Modules/expat/nametab.h b/Modules/expat/nametab.h
index bfa2bd38cd95..3681df348eeb 100644
--- a/Modules/expat/nametab.h
+++ b/Modules/expat/nametab.h
@@ -31,152 +31,106 @@
 */
 
 static const unsigned namingBitmap[] = {
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0x00000000, 0x04000000, 0x87FFFFFE, 0x07FFFFFE,
-0x00000000, 0x00000000, 0xFF7FFFFF, 0xFF7FFFFF,
-0xFFFFFFFF, 0x7FF3FFFF, 0xFFFFFDFE, 0x7FFFFFFF,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFE00F, 0xFC31FFFF,
-0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
-0xFFFFFFFF, 0xF80001FF, 0x00000003, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFD740, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
-0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
-0xFFFF0003, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF,
-0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
-0x0000007F, 0x00000000, 0xFFFF0000, 0x000707FF,
-0x00000000, 0x07FFFFFE, 0x000007FE, 0xFFFE0000,
-0xFFFFFFFF, 0x7CFFFFFF, 0x002F7FFF, 0x00000060,
-0xFFFFFFE0, 0x23FFFFFF, 0xFF000000, 0x00000003,
-0xFFF99FE0, 0x03C5FDFF, 0xB0000000, 0x00030003,
-0xFFF987E0, 0x036DFDFF, 0x5E000000, 0x001C0000,
-0xFFFBAFE0, 0x23EDFDFF, 0x00000000, 0x00000001,
-0xFFF99FE0, 0x23CDFDFF, 0xB0000000, 0x00000003,
-0xD63DC7E0, 0x03BFC718, 0x00000000, 0x00000000,
-0xFFFDDFE0, 0x03EFFDFF, 0x00000000, 0x00000003,
-0xFFFDDFE0, 0x03EFFDFF, 0x40000000, 0x00000003,
-0xFFFDDFE0, 0x03FFFDFF, 0x00000000, 0x00000003,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFE, 0x000D7FFF, 0x0000003F, 0x00000000,
-0xFEF02596, 0x200D6CAE, 0x0000001F, 0x00000000,
-0x00000000, 0x00000000, 0xFFFFFEFF, 0x000003FF,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0xFFFFFFFF, 0xFFFF003F, 0x007FFFFF,
-0x0007DAED, 0x50000000, 0x82315001, 0x002C62AB,
-0x40000000, 0xF580C900, 0x00000007, 0x02010800,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0x0FFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x03FFFFFF,
-0x3F3FFFFF, 0xFFFFFFFF, 0xAAFF3F3F, 0x3FFFFFFF,
-0xFFFFFFFF, 0x5FDFFFFF, 0x0FCF1FDC, 0x1FDC1FFF,
-0x00000000, 0x00004C40, 0x00000000, 0x00000000,
-0x00000007, 0x00000000, 0x00000000, 0x00000000,
-0x00000080, 0x000003FE, 0xFFFFFFFE, 0xFFFFFFFF,
-0x001FFFFF, 0xFFFFFFFE, 0xFFFFFFFF, 0x07FFFFFF,
-0xFFFFFFE0, 0x00001FFF, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0x0000003F, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0x0000000F, 0x00000000, 0x00000000,
-0x00000000, 0x07FF6000, 0x87FFFFFE, 0x07FFFFFE,
-0x00000000, 0x00800000, 0xFF7FFFFF, 0xFF7FFFFF,
-0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
-0xFFFFFFFF, 0xF80001FF, 0x00030003, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000003,
-0xFFFFD7C0, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
-0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
-0xFFFF007B, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF,
-0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
-0xFFFE007F, 0xBBFFFFFB, 0xFFFF0016, 0x000707FF,
-0x00000000, 0x07FFFFFE, 0x0007FFFF, 0xFFFF03FF,
-0xFFFFFFFF, 0x7CFFFFFF, 0xFFEF7FFF, 0x03FF3DFF,
-0xFFFFFFEE, 0xF3FFFFFF, 0xFF1E3FFF, 0x0000FFCF,
-0xFFF99FEE, 0xD3C5FDFF, 0xB080399F, 0x0003FFCF,
-0xFFF987E4, 0xD36DFDFF, 0x5E003987, 0x001FFFC0,
-0xFFFBAFEE, 0xF3EDFDFF, 0x00003BBF, 0x0000FFC1,
-0xFFF99FEE, 0xF3CDFDFF, 0xB0C0398F, 0x0000FFC3,
-0xD63DC7EC, 0xC3BFC718, 0x00803DC7, 0x0000FF80,
-0xFFFDDFEE, 0xC3EFFDFF, 0x00603DDF, 0x0000FFC3,
-0xFFFDDFEC, 0xC3EFFDFF, 0x40603DDF, 0x0000FFC3,
-0xFFFDDFEC, 0xC3FFFDFF, 0x00803DCF, 0x0000FFC3,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFE, 0x07FF7FFF, 0x03FF7FFF, 0x00000000,
-0xFEF02596, 0x3BFF6CAE, 0x03FF3F5F, 0x00000000,
-0x03000000, 0xC2A003FF, 0xFFFFFEFF, 0xFFFE03FF,
-0xFEBF0FDF, 0x02FE3FFF, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x1FFF0000, 0x00000002,
-0x000000A0, 0x003EFFFE, 0xFFFFFFFE, 0xFFFFFFFF,
-0x661FFFFF, 0xFFFFFFFE, 0xFFFFFFFF, 0x77FFFFFF,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x04000000,
+    0x87FFFFFE, 0x07FFFFFE, 0x00000000, 0x00000000, 0xFF7FFFFF, 0xFF7FFFFF,
+    0xFFFFFFFF, 0x7FF3FFFF, 0xFFFFFDFE, 0x7FFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFE00F, 0xFC31FFFF, 0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xF80001FF, 0x00000003, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFD740, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
+    0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF, 0xFFFF0003, 0xFFFFFFFF,
+    0xFFFF199F, 0x033FCFFF, 0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
+    0x0000007F, 0x00000000, 0xFFFF0000, 0x000707FF, 0x00000000, 0x07FFFFFE,
+    0x000007FE, 0xFFFE0000, 0xFFFFFFFF, 0x7CFFFFFF, 0x002F7FFF, 0x00000060,
+    0xFFFFFFE0, 0x23FFFFFF, 0xFF000000, 0x00000003, 0xFFF99FE0, 0x03C5FDFF,
+    0xB0000000, 0x00030003, 0xFFF987E0, 0x036DFDFF, 0x5E000000, 0x001C0000,
+    0xFFFBAFE0, 0x23EDFDFF, 0x00000000, 0x00000001, 0xFFF99FE0, 0x23CDFDFF,
+    0xB0000000, 0x00000003, 0xD63DC7E0, 0x03BFC718, 0x00000000, 0x00000000,
+    0xFFFDDFE0, 0x03EFFDFF, 0x00000000, 0x00000003, 0xFFFDDFE0, 0x03EFFDFF,
+    0x40000000, 0x00000003, 0xFFFDDFE0, 0x03FFFDFF, 0x00000000, 0x00000003,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFE, 0x000D7FFF,
+    0x0000003F, 0x00000000, 0xFEF02596, 0x200D6CAE, 0x0000001F, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFFEFF, 0x000003FF, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0xFFFFFFFF, 0xFFFF003F, 0x007FFFFF, 0x0007DAED, 0x50000000,
+    0x82315001, 0x002C62AB, 0x40000000, 0xF580C900, 0x00000007, 0x02010800,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0FFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0x03FFFFFF, 0x3F3FFFFF, 0xFFFFFFFF, 0xAAFF3F3F, 0x3FFFFFFF,
+    0xFFFFFFFF, 0x5FDFFFFF, 0x0FCF1FDC, 0x1FDC1FFF, 0x00000000, 0x00004C40,
+    0x00000000, 0x00000000, 0x00000007, 0x00000000, 0x00000000, 0x00000000,
+    0x00000080, 0x000003FE, 0xFFFFFFFE, 0xFFFFFFFF, 0x001FFFFF, 0xFFFFFFFE,
+    0xFFFFFFFF, 0x07FFFFFF, 0xFFFFFFE0, 0x00001FFF, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000000, 0x00000000,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000000F,
+    0x00000000, 0x00000000, 0x00000000, 0x07FF6000, 0x87FFFFFE, 0x07FFFFFE,
+    0x00000000, 0x00800000, 0xFF7FFFFF, 0xFF7FFFFF, 0x00FFFFFF, 0x00000000,
+    0xFFFF0000, 0xFFFFFFFF, 0xFFFFFFFF, 0xF80001FF, 0x00030003, 0x00000000,
+    0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000003, 0xFFFFD7C0, 0xFFFFFFFB,
+    0x547F7FFF, 0x000FFFFD, 0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
+    0xFFFF007B, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF, 0x00000000, 0xFFFE0000,
+    0x027FFFFF, 0xFFFFFFFE, 0xFFFE007F, 0xBBFFFFFB, 0xFFFF0016, 0x000707FF,
+    0x00000000, 0x07FFFFFE, 0x0007FFFF, 0xFFFF03FF, 0xFFFFFFFF, 0x7CFFFFFF,
+    0xFFEF7FFF, 0x03FF3DFF, 0xFFFFFFEE, 0xF3FFFFFF, 0xFF1E3FFF, 0x0000FFCF,
+    0xFFF99FEE, 0xD3C5FDFF, 0xB080399F, 0x0003FFCF, 0xFFF987E4, 0xD36DFDFF,
+    0x5E003987, 0x001FFFC0, 0xFFFBAFEE, 0xF3EDFDFF, 0x00003BBF, 0x0000FFC1,
+    0xFFF99FEE, 0xF3CDFDFF, 0xB0C0398F, 0x0000FFC3, 0xD63DC7EC, 0xC3BFC718,
+    0x00803DC7, 0x0000FF80, 0xFFFDDFEE, 0xC3EFFDFF, 0x00603DDF, 0x0000FFC3,
+    0xFFFDDFEC, 0xC3EFFDFF, 0x40603DDF, 0x0000FFC3, 0xFFFDDFEC, 0xC3FFFDFF,
+    0x00803DCF, 0x0000FFC3, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0xFFFFFFFE, 0x07FF7FFF, 0x03FF7FFF, 0x00000000, 0xFEF02596, 0x3BFF6CAE,
+    0x03FF3F5F, 0x00000000, 0x03000000, 0xC2A003FF, 0xFFFFFEFF, 0xFFFE03FF,
+    0xFEBF0FDF, 0x02FE3FFF, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x1FFF0000, 0x00000002,
+    0x000000A0, 0x003EFFFE, 0xFFFFFFFE, 0xFFFFFFFF, 0x661FFFFF, 0xFFFFFFFE,
+    0xFFFFFFFF, 0x77FFFFFF,
 };
 static const unsigned char nmstrtPages[] = {
-0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00,
-0x00, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
-0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13,
-0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x15, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x17,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x09, 0x0A, 0x0B,
+    0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13, 0x00, 0x14, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x15, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00,
 };
 static const unsigned char namePages[] = {
-0x19, 0x03, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x00,
-0x00, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25,
-0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13,
-0x26, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x27, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x17,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x19, 0x03, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x00, 0x00, 0x1F, 0x20, 0x21,
+    0x22, 0x23, 0x24, 0x25, 0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13, 0x26, 0x14, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x27, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00,
 };
diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h
index 4d6786d7839a..bfee65a332f1 100644
--- a/Modules/expat/siphash.h
+++ b/Modules/expat/siphash.h
@@ -11,6 +11,10 @@
  * --------------------------------------------------------------------------
  * HISTORY:
  *
+ * 2019-08-03  (Sebastian Pipping)
+ *   - Mark part of sip24_valid as to be excluded from clang-format
+ *   - Re-format code using clang-format 9
+ *
  * 2018-07-08  (Anton Maklakov)
  *   - Add "fall through" markers for GCC's -Wimplicit-fallthrough
  *
@@ -94,186 +98,186 @@
 #include <stddef.h> /* size_t */
 
 #if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER < 1600)
-  /* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */
-  typedef unsigned __int8   uint8_t;
-  typedef unsigned __int32 uint32_t;
-  typedef unsigned __int64 uint64_t;
+/* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */
+typedef unsigned __int8 uint8_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
 #else
- #include <stdint.h> /* uint64_t uint32_t uint8_t */
+#  include <stdint.h> /* uint64_t uint32_t uint8_t */
 #endif
 
-
 /*
  * Workaround to not require a C++11 compiler for using ULL suffix
  * if this code is included and compiled as C++; related GCC warning is:
  * warning: use of C++11 long long integer constant [-Wlong-long]
  */
-#define _SIP_ULL(high, low)  (((uint64_t)high << 32) | low)
-
-
-#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b))))
+#define _SIP_ULL(high, low) (((uint64_t)high << 32) | low)
 
-#define SIP_U32TO8_LE(p, v) \
-	(p)[0] = (uint8_t)((v) >>  0); (p)[1] = (uint8_t)((v) >>  8); \
-	(p)[2] = (uint8_t)((v) >> 16); (p)[3] = (uint8_t)((v) >> 24);
+#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
 
-#define SIP_U64TO8_LE(p, v) \
-	SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >>  0)); \
-	SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
+#define SIP_U32TO8_LE(p, v)                                                    \
+  (p)[0] = (uint8_t)((v) >> 0);                                                \
+  (p)[1] = (uint8_t)((v) >> 8);                                                \
+  (p)[2] = (uint8_t)((v) >> 16);                                               \
+  (p)[3] = (uint8_t)((v) >> 24);
 
-#define SIP_U8TO64_LE(p) \
-	(((uint64_t)((p)[0]) <<  0) | \
-	 ((uint64_t)((p)[1]) <<  8) | \
-	 ((uint64_t)((p)[2]) << 16) | \
-	 ((uint64_t)((p)[3]) << 24) | \
-	 ((uint64_t)((p)[4]) << 32) | \
-	 ((uint64_t)((p)[5]) << 40) | \
-	 ((uint64_t)((p)[6]) << 48) | \
-	 ((uint64_t)((p)[7]) << 56))
+#define SIP_U64TO8_LE(p, v)                                                    \
+  SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >> 0));                                \
+  SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
 
+#define SIP_U8TO64_LE(p)                                                       \
+  (((uint64_t)((p)[0]) << 0) | ((uint64_t)((p)[1]) << 8)                       \
+   | ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24)                   \
+   | ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40)                   \
+   | ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))
 
-#define SIPHASH_INITIALIZER { 0, 0, 0, 0, { 0 }, 0, 0 }
+#define SIPHASH_INITIALIZER                                                    \
+  { 0, 0, 0, 0, {0}, 0, 0 }
 
 struct siphash {
-	uint64_t v0, v1, v2, v3;
+  uint64_t v0, v1, v2, v3;
 
-	unsigned char buf[8], *p;
-	uint64_t c;
+  unsigned char buf[8], *p;
+  uint64_t c;
 }; /* struct siphash */
 
-
 #define SIP_KEYLEN 16
 
 struct sipkey {
-	uint64_t k[2];
+  uint64_t k[2];
 }; /* struct sipkey */
 
-#define sip_keyof(k) sip_tokey(&(struct sipkey){ { 0 } }, (k))
+#define sip_keyof(k) sip_tokey(&(struct sipkey){{0}}, (k))
 
-static struct sipkey *sip_tokey(struct sipkey *key, const void *src) {
-	key->k[0] = SIP_U8TO64_LE((const unsigned char *)src);
-	key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8);
-	return key;
+static struct sipkey *
+sip_tokey(struct sipkey *key, const void *src) {
+  key->k[0] = SIP_U8TO64_LE((const unsigned char *)src);
+  key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8);
+  return key;
 } /* sip_tokey() */
 
-
 #ifdef SIPHASH_TOBIN
 
-#define sip_binof(v) sip_tobin((unsigned char[8]){ 0 }, (v))
+#  define sip_binof(v) sip_tobin((unsigned char[8]){0}, (v))
 
-static void *sip_tobin(void *dst, uint64_t u64) {
-	SIP_U64TO8_LE((unsigned char *)dst, u64);
-	return dst;
+static void *
+sip_tobin(void *dst, uint64_t u64) {
+  SIP_U64TO8_LE((unsigned char *)dst, u64);
+  return dst;
 } /* sip_tobin() */
 
-#endif  /* SIPHASH_TOBIN */
-
+#endif /* SIPHASH_TOBIN */
 
-static void sip_round(struct siphash *H, const int rounds) {
-	int i;
+static void
+sip_round(struct siphash *H, const int rounds) {
+  int i;
 
-	for (i = 0; i < rounds; i++) {
-		H->v0 += H->v1;
-		H->v1 = SIP_ROTL(H->v1, 13);
-		H->v1 ^= H->v0;
-		H->v0 = SIP_ROTL(H->v0, 32);
+  for (i = 0; i < rounds; i++) {
+    H->v0 += H->v1;
+    H->v1 = SIP_ROTL(H->v1, 13);
+    H->v1 ^= H->v0;
+    H->v0 = SIP_ROTL(H->v0, 32);
 
-		H->v2 += H->v3;
-		H->v3 = SIP_ROTL(H->v3, 16);
-		H->v3 ^= H->v2;
+    H->v2 += H->v3;
+    H->v3 = SIP_ROTL(H->v3, 16);
+    H->v3 ^= H->v2;
 
-		H->v0 += H->v3;
-		H->v3 = SIP_ROTL(H->v3, 21);
-		H->v3 ^= H->v0;
+    H->v0 += H->v3;
+    H->v3 = SIP_ROTL(H->v3, 21);
+    H->v3 ^= H->v0;
 
-		H->v2 += H->v1;
-		H->v1 = SIP_ROTL(H->v1, 17);
-		H->v1 ^= H->v2;
-		H->v2 = SIP_ROTL(H->v2, 32);
-	}
+    H->v2 += H->v1;
+    H->v1 = SIP_ROTL(H->v1, 17);
+    H->v1 ^= H->v2;
+    H->v2 = SIP_ROTL(H->v2, 32);
+  }
 } /* sip_round() */
 
+static struct siphash *
+sip24_init(struct siphash *H, const struct sipkey *key) {
+  H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0];
+  H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1];
+  H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0];
+  H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1];
 
-static struct siphash *sip24_init(struct siphash *H,
-		const struct sipkey *key) {
-	H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0];
-	H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1];
-	H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0];
-	H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1];
+  H->p = H->buf;
+  H->c = 0;
 
-	H->p = H->buf;
-	H->c = 0;
-
-	return H;
+  return H;
 } /* sip24_init() */
 
+#define sip_endof(a) (&(a)[sizeof(a) / sizeof *(a)])
 
-#define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)])
-
-static struct siphash *sip24_update(struct siphash *H, const void *src,
-		size_t len) {
-	const unsigned char *p = (const unsigned char *)src, *pe = p + len;
-	uint64_t m;
+static struct siphash *
+sip24_update(struct siphash *H, const void *src, size_t len) {
+  const unsigned char *p = (const unsigned char *)src, *pe = p + len;
+  uint64_t m;
 
-	do {
-		while (p < pe && H->p < sip_endof(H->buf))
-			*H->p++ = *p++;
+  do {
+    while (p < pe && H->p < sip_endof(H->buf))
+      *H->p++ = *p++;
 
-		if (H->p < sip_endof(H->buf))
-			break;
+    if (H->p < sip_endof(H->buf))
+      break;
 
-		m = SIP_U8TO64_LE(H->buf);
-		H->v3 ^= m;
-		sip_round(H, 2);
-		H->v0 ^= m;
+    m = SIP_U8TO64_LE(H->buf);
+    H->v3 ^= m;
+    sip_round(H, 2);
+    H->v0 ^= m;
 
-		H->p = H->buf;
-		H->c += 8;
-	} while (p < pe);
+    H->p = H->buf;
+    H->c += 8;
+  } while (p < pe);
 
-	return H;
+  return H;
 } /* sip24_update() */
 
-
-static uint64_t sip24_final(struct siphash *H) {
-	const char left = (char)(H->p - H->buf);
-	uint64_t b = (H->c + left) << 56;
-
-	switch (left) {
-	case 7: b |= (uint64_t)H->buf[6] << 48;
-		/* fall through */
-	case 6: b |= (uint64_t)H->buf[5] << 40;
-		/* fall through */
-	case 5: b |= (uint64_t)H->buf[4] << 32;
-		/* fall through */
-	case 4: b |= (uint64_t)H->buf[3] << 24;
-		/* fall through */
-	case 3: b |= (uint64_t)H->buf[2] << 16;
-		/* fall through */
-	case 2: b |= (uint64_t)H->buf[1] << 8;
-		/* fall through */
-	case 1: b |= (uint64_t)H->buf[0] << 0;
-		/* fall through */
-	case 0: break;
-	}
-
-	H->v3 ^= b;
-	sip_round(H, 2);
-	H->v0 ^= b;
-	H->v2 ^= 0xff;
-	sip_round(H, 4);
-
-	return H->v0 ^ H->v1 ^ H->v2  ^ H->v3;
+static uint64_t
+sip24_final(struct siphash *H) {
+  const char left = (char)(H->p - H->buf);
+  uint64_t b = (H->c + left) << 56;
+
+  switch (left) {
+  case 7:
+    b |= (uint64_t)H->buf[6] << 48;
+    /* fall through */
+  case 6:
+    b |= (uint64_t)H->buf[5] << 40;
+    /* fall through */
+  case 5:
+    b |= (uint64_t)H->buf[4] << 32;
+    /* fall through */
+  case 4:
+    b |= (uint64_t)H->buf[3] << 24;
+    /* fall through */
+  case 3:
+    b |= (uint64_t)H->buf[2] << 16;
+    /* fall through */
+  case 2:
+    b |= (uint64_t)H->buf[1] << 8;
+    /* fall through */
+  case 1:
+    b |= (uint64_t)H->buf[0] << 0;
+    /* fall through */
+  case 0:
+    break;
+  }
+
+  H->v3 ^= b;
+  sip_round(H, 2);
+  H->v0 ^= b;
+  H->v2 ^= 0xff;
+  sip_round(H, 4);
+
+  return H->v0 ^ H->v1 ^ H->v2 ^ H->v3;
 } /* sip24_final() */
 
-
-static uint64_t siphash24(const void *src, size_t len,
-		const struct sipkey *key) {
-	struct siphash state = SIPHASH_INITIALIZER;
-	return sip24_final(sip24_update(sip24_init(&state, key), src, len));
+static uint64_t
+siphash24(const void *src, size_t len, const struct sipkey *key) {
+  struct siphash state = SIPHASH_INITIALIZER;
+  return sip24_final(sip24_update(sip24_init(&state, key), src, len));
 } /* siphash24() */
 
-
 /*
  * SipHash-2-4 output with
  * k = 00 01 02 ...
@@ -285,107 +289,110 @@ static uint64_t siphash24(const void *src, size_t len,
  * ...
  * in = 00 01 02 ... 3e (63 bytes)
  */
-static int sip24_valid(void) {
-	static const unsigned char vectors[64][8] = {
-		{ 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, },
-		{ 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, },
-		{ 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, },
-		{ 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, },
-		{ 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, },
-		{ 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, },
-		{ 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, },
-		{ 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, },
-		{ 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, },
-		{ 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, },
-		{ 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, },
-		{ 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, },
-		{ 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, },
-		{ 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, },
-		{ 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, },
-		{ 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, },
-		{ 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, },
-		{ 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, },
-		{ 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, },
-		{ 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, },
-		{ 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, },
-		{ 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, },
-		{ 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, },
-		{ 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, },
-		{ 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, },
-		{ 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, },
-		{ 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, },
-		{ 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, },
-		{ 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, },
-		{ 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, },
-		{ 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, },
-		{ 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, },
-		{ 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, },
-		{ 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, },
-		{ 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, },
-		{ 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, },
-		{ 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, },
-		{ 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, },
-		{ 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, },
-		{ 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, },
-		{ 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, },
-		{ 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, },
-		{ 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, },
-		{ 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, },
-		{ 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, },
-		{ 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, },
-		{ 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, },
-		{ 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, },
-		{ 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, },
-		{ 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, },
-		{ 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, },
-		{ 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, },
-		{ 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, },
-		{ 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, },
-		{ 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, },
-		{ 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, },
-		{ 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, },
-		{ 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, },
-		{ 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, },
-		{ 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, },
-		{ 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, },
-		{ 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, },
-		{ 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, },
-		{ 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, }
-	};
-	unsigned char in[64];
-	struct sipkey k;
-	size_t i;
-
-	sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011"
-			"\012\013\014\015\016\017");
-
-	for (i = 0; i < sizeof in; ++i) {
-		in[i] = (unsigned char)i;
-
-		if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i]))
-			return 0;
-	}
-
-	return 1;
+static int
+sip24_valid(void) {
+  /* clang-format off */
+  static const unsigned char vectors[64][8] = {
+    { 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, },
+    { 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, },
+    { 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, },
+    { 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, },
+    { 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, },
+    { 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, },
+    { 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, },
+    { 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, },
+    { 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, },
+    { 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, },
+    { 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, },
+    { 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, },
+    { 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, },
+    { 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, },
+    { 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, },
+    { 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, },
+    { 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, },
+    { 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, },
+    { 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, },
+    { 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, },
+    { 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, },
+    { 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, },
+    { 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, },
+    { 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, },
+    { 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, },
+    { 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, },
+    { 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, },
+    { 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, },
+    { 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, },
+    { 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, },
+    { 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, },
+    { 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, },
+    { 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, },
+    { 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, },
+    { 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, },
+    { 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, },
+    { 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, },
+    { 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, },
+    { 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, },
+    { 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, },
+    { 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, },
+    { 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, },
+    { 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, },
+    { 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, },
+    { 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, },
+    { 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, },
+    { 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, },
+    { 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, },
+    { 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, },
+    { 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, },
+    { 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, },
+    { 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, },
+    { 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, },
+    { 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, },
+    { 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, },
+    { 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, },
+    { 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, },
+    { 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, },
+    { 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, },
+    { 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, },
+    { 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, },
+    { 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, },
+    { 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, },
+    { 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, }
+  };
+  /* clang-format on */
+
+  unsigned char in[64];
+  struct sipkey k;
+  size_t i;
+
+  sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011"
+                "\012\013\014\015\016\017");
+
+  for (i = 0; i < sizeof in; ++i) {
+    in[i] = (unsigned char)i;
+
+    if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i]))
+      return 0;
+  }
+
+  return 1;
 } /* sip24_valid() */
 
-
 #ifdef SIPHASH_MAIN
 
-#include <stdio.h>
+#  include <stdio.h>
 
-int main(void) {
-	const int ok = sip24_valid();
+int
+main(void) {
+  const int ok = sip24_valid();
 
-	if (ok)
-		puts("OK");
-	else
-		puts("FAIL");
+  if (ok)
+    puts("OK");
+  else
+    puts("FAIL");
 
-	return !ok;
+  return ! ok;
 } /* main() */
 
 #endif /* SIPHASH_MAIN */
 
-
 #endif /* SIPHASH_H */
diff --git a/Modules/expat/utf8tab.h b/Modules/expat/utf8tab.h
index fa0bed6f5d75..a22986acbb95 100644
--- a/Modules/expat/utf8tab.h
+++ b/Modules/expat/utf8tab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x80 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x84 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x88 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x8C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x90 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x94 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x98 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x9C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xAC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xBC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xC0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xC4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xC8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xCC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xDC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xE0 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xE4 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xE8 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xEC */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xF0 */ BT_LEAD4, BT_LEAD4, BT_LEAD4, BT_LEAD4,
-/* 0xF4 */ BT_LEAD4, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0xF8 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0xFC */ BT_NONXML, BT_NONXML, BT_MALFORM, BT_MALFORM,
+    /* 0x84 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x88 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x8C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x90 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x94 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x98 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x9C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xAC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xBC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xC0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xC4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xC8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xCC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xDC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xE0 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xE4 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xE8 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xEC */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xF0 */ BT_LEAD4, BT_LEAD4, BT_LEAD4, BT_LEAD4,
+    /* 0xF4 */ BT_LEAD4, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0xF8 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0xFC */ BT_NONXML, BT_NONXML, BT_MALFORM, BT_MALFORM,
diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h
index 28a043c6044c..562a4a82dc1d 100644
--- a/Modules/expat/winconfig.h
+++ b/Modules/expat/winconfig.h
@@ -40,20 +40,17 @@
 #include <memory.h>
 #include <string.h>
 
+#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */
+#  include <expat_config.h>
+#else /* !defined(HAVE_EXPAT_CONFIG_H) */
 
-#if defined(HAVE_EXPAT_CONFIG_H)  /* e.g. MinGW */
-# include <expat_config.h>
-#else  /* !defined(HAVE_EXPAT_CONFIG_H) */
-
-
-#define XML_NS 1
-#define XML_DTD 1
-#define XML_CONTEXT_BYTES 1024
+#  define XML_NS 1
+#  define XML_DTD 1
+#  define XML_CONTEXT_BYTES 1024
 
 /* we will assume all Windows platforms are little endian */
-#define BYTEORDER 1234
+#  define BYTEORDER 1234
 
 #endif /* !defined(HAVE_EXPAT_CONFIG_H) */
 
-
 #endif /* ndef WINCONFIG_H */
diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c
index 4957ad543e28..e740f0e19c7d 100644
--- a/Modules/expat/xmlparse.c
+++ b/Modules/expat/xmlparse.c
@@ -1,4 +1,4 @@
-/* 69df5be70289a11fb834869ce4a91c23c1d9dd04baffcbd10e86742d149a080c (2.2.7+)
+/* f2d0ab6d1d4422a08cf1cf3bbdfba96b49dea42fb5ff4615e03a2a25c306e769 (2.2.8+)
                             __  __            _
                          ___\ \/ /_ __   __ _| |_
                         / _ \\  /| '_ \ / _` | __|
@@ -30,36 +30,38 @@
    USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
-#if !defined(_WIN32) && defined(HAVE_EXPAT_CONFIG_H)
-#  include <pyconfig.h>
+#if ! defined(_GNU_SOURCE)
+#  define _GNU_SOURCE 1 /* syscall prototype */
 #endif
-#if !defined(_GNU_SOURCE)
-# define _GNU_SOURCE 1                  /* syscall prototype */
+
+#ifdef _WIN32
+/* force stdlib to define rand_s() */
+#  define _CRT_RAND_S
 #endif
 
 #include <stddef.h>
-#include <string.h>                     /* memset(), memcpy() */
+#include <string.h> /* memset(), memcpy() */
 #include <assert.h>
-#include <limits.h>                     /* UINT_MAX */
-#include <stdio.h>                      /* fprintf */
-#include <stdlib.h>                     /* getenv */
+#include <limits.h> /* UINT_MAX */
+#include <stdio.h>  /* fprintf */
+#include <stdlib.h> /* getenv, rand_s */
 
 #ifdef _WIN32
-#define getpid GetCurrentProcessId
+#  define getpid GetCurrentProcessId
 #else
-#include <sys/time.h>                   /* gettimeofday() */
-#include <sys/types.h>                  /* getpid() */
-#include <unistd.h>                     /* getpid() */
-#include <fcntl.h>                      /* O_RDONLY */
-#include <errno.h>
+#  include <sys/time.h>  /* gettimeofday() */
+#  include <sys/types.h> /* getpid() */
+#  include <unistd.h>    /* getpid() */
+#  include <fcntl.h>     /* O_RDONLY */
+#  include <errno.h>
 #endif
 
 #define XML_BUILDING_EXPAT 1
 
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #elif defined(HAVE_EXPAT_CONFIG_H)
-#include <expat_config.h>
+#  include <expat_config.h>
 #endif /* ndef _WIN32 */
 
 #include "ascii.h"
@@ -67,33 +69,31 @@
 #include "siphash.h"
 
 #if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
-# if defined(HAVE_GETRANDOM)
-#  include <sys/random.h>    /* getrandom */
-# else
-#  include <unistd.h>        /* syscall */
-#  include <sys/syscall.h>   /* SYS_getrandom */
-# endif
-# if ! defined(GRND_NONBLOCK)
-#  define GRND_NONBLOCK  0x0001
-# endif  /* defined(GRND_NONBLOCK) */
-#endif  /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
-
-#if defined(HAVE_LIBBSD) \
+#  if defined(HAVE_GETRANDOM)
+#    include <sys/random.h> /* getrandom */
+#  else
+#    include <unistd.h>      /* syscall */
+#    include <sys/syscall.h> /* SYS_getrandom */
+#  endif
+#  if ! defined(GRND_NONBLOCK)
+#    define GRND_NONBLOCK 0x0001
+#  endif /* defined(GRND_NONBLOCK) */
+#endif   /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
+
+#if defined(HAVE_LIBBSD)                                                       \
     && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM))
-# include <bsd/stdlib.h>
+#  include <bsd/stdlib.h>
 #endif
 
-#if defined(_WIN32) && !defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
-# define LOAD_LIBRARY_SEARCH_SYSTEM32  0x00000800
+#if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
+#  define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800
 #endif
 
-#if !defined(HAVE_GETRANDOM) && !defined(HAVE_SYSCALL_GETRANDOM) \
-    && !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_ARC4RANDOM) \
-    && !defined(XML_DEV_URANDOM) \
-    && !defined(_WIN32) \
-    && !defined(XML_POOR_ENTROPY)
-# error  \
-    You do not have support for any sources of high quality entropy \
+#if ! defined(HAVE_GETRANDOM) && ! defined(HAVE_SYSCALL_GETRANDOM)             \
+    && ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)            \
+    && ! defined(XML_DEV_URANDOM) && ! defined(_WIN32)                         \
+    && ! defined(XML_POOR_ENTROPY)
+#  error You do not have support for any sources of high quality entropy \
     enabled.  For end user security, that is probably not what you want. \
     \
     Your options include: \
@@ -104,7 +104,7 @@
       * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \
       * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \
       * Linux / BSD / macOS (/dev/urandom): XML_DEV_URANDOM \
-      * Windows (RtlGenRandom): _WIN32. \
+      * Windows (rand_s): _WIN32. \
     \
     If insist on not using any of these, bypass this error by defining \
     XML_POOR_ENTROPY; you have been warned. \
@@ -113,56 +113,55 @@
     to the build system, please open a bug.  Thank you!
 #endif
 
-
 #ifdef XML_UNICODE
-#define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX
-#define XmlConvert XmlUtf16Convert
-#define XmlGetInternalEncoding XmlGetUtf16InternalEncoding
-#define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS
-#define XmlEncode XmlUtf16Encode
+#  define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX
+#  define XmlConvert XmlUtf16Convert
+#  define XmlGetInternalEncoding XmlGetUtf16InternalEncoding
+#  define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS
+#  define XmlEncode XmlUtf16Encode
 /* Using pointer subtraction to convert to integer type. */
-#define MUST_CONVERT(enc, s) (!(enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1))
+#  define MUST_CONVERT(enc, s)                                                 \
+    (! (enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1))
 typedef unsigned short ICHAR;
 #else
-#define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX
-#define XmlConvert XmlUtf8Convert
-#define XmlGetInternalEncoding XmlGetUtf8InternalEncoding
-#define XmlGetInternalEncodingNS XmlGetUtf8InternalEncodingNS
-#define XmlEncode XmlUtf8Encode
-#define MUST_CONVERT(enc, s) (!(enc)->isUtf8)
+#  define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX
+#  define XmlConvert XmlUtf8Convert
+#  define XmlGetInternalEncoding XmlGetUtf8InternalEncoding
+#  define XmlGetInternalEncodingNS XmlGetUtf8InternalEncodingNS
+#  define XmlEncode XmlUtf8Encode
+#  define MUST_CONVERT(enc, s) (! (enc)->isUtf8)
 typedef char ICHAR;
 #endif
 
-
 #ifndef XML_NS
 
-#define XmlInitEncodingNS XmlInitEncoding
-#define XmlInitUnknownEncodingNS XmlInitUnknownEncoding
-#undef XmlGetInternalEncodingNS
-#define XmlGetInternalEncodingNS XmlGetInternalEncoding
-#define XmlParseXmlDeclNS XmlParseXmlDecl
+#  define XmlInitEncodingNS XmlInitEncoding
+#  define XmlInitUnknownEncodingNS XmlInitUnknownEncoding
+#  undef XmlGetInternalEncodingNS
+#  define XmlGetInternalEncodingNS XmlGetInternalEncoding
+#  define XmlParseXmlDeclNS XmlParseXmlDecl
 
 #endif
 
 #ifdef XML_UNICODE
 
-#ifdef XML_UNICODE_WCHAR_T
-#define XML_T(x) (const wchar_t)x
-#define XML_L(x) L ## x
-#else
-#define XML_T(x) (const unsigned short)x
-#define XML_L(x) x
-#endif
+#  ifdef XML_UNICODE_WCHAR_T
+#    define XML_T(x) (const wchar_t) x
+#    define XML_L(x) L##x
+#  else
+#    define XML_T(x) (const unsigned short)x
+#    define XML_L(x) x
+#  endif
 
 #else
 
-#define XML_T(x) x
-#define XML_L(x) x
+#  define XML_T(x) x
+#  define XML_L(x) x
 
 #endif
 
 /* Round up n to be a multiple of sz, where sz is a power of 2. */
-#define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1))
+#define ROUND_UP(n, sz) (((n) + ((sz)-1)) & ~((sz)-1))
 
 /* Do safe (NULL-aware) pointer arithmetic */
 #define EXPAT_SAFE_PTR_DIFF(p, q) (((p) && (q)) ? ((p) - (q)) : 0)
@@ -185,11 +184,9 @@ typedef struct {
   const XML_Memory_Handling_Suite *mem;
 } HASH_TABLE;
 
-static size_t
-keylen(KEY s);
+static size_t keylen(KEY s);
 
-static void
-copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key);
+static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key);
 
 /* For probing (after a collision) we need a step size relative prime
    to the hash table size, which is a power of 2. We use double-hashing,
@@ -199,9 +196,9 @@ copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key);
    We limit the maximum step size to table->size / 4 (mask >> 2) and make
    it odd, since odd numbers are always relative prime to a power of 2.
 */
-#define SECOND_HASH(hash, mask, power) \
-  ((((hash) & ~(mask)) >> ((power) - 1)) & ((mask) >> 2))
-#define PROBE_STEP(hash, mask, power) \
+#define SECOND_HASH(hash, mask, power)                                         \
+  ((((hash) & ~(mask)) >> ((power)-1)) & ((mask) >> 2))
+#define PROBE_STEP(hash, mask, power)                                          \
   ((unsigned char)((SECOND_HASH(hash, mask, power)) | 1))
 
 typedef struct {
@@ -209,7 +206,7 @@ typedef struct {
   NAMED **end;
 } HASH_TABLE_ITER;
 
-#define INIT_TAG_BUF_SIZE 32  /* must be a multiple of sizeof(XML_Char) */
+#define INIT_TAG_BUF_SIZE 32 /* must be a multiple of sizeof(XML_Char) */
 #define INIT_DATA_BUF_SIZE 1024
 #define INIT_ATTS_SIZE 16
 #define INIT_ATTS_VERSION 0xFFFFFFFF
@@ -256,20 +253,20 @@ typedef struct {
    TAG objects in a free list.
 */
 typedef struct tag {
-  struct tag *parent;           /* parent of this element */
-  const char *rawName;          /* tagName in the original encoding */
+  struct tag *parent;  /* parent of this element */
+  const char *rawName; /* tagName in the original encoding */
   int rawNameLength;
-  TAG_NAME name;                /* tagName in the API encoding */
-  char *buf;                    /* buffer for name components */
-  char *bufEnd;                 /* end of the buffer */
+  TAG_NAME name; /* tagName in the API encoding */
+  char *buf;     /* buffer for name components */
+  char *bufEnd;  /* end of the buffer */
   BINDING *bindings;
 } TAG;
 
 typedef struct {
   const XML_Char *name;
   const XML_Char *textPtr;
-  int textLen;                  /* length in XML_Chars */
-  int processed;                /* # of processed bytes - when suspended */
+  int textLen;   /* length in XML_Chars */
+  int processed; /* # of processed bytes - when suspended */
   const XML_Char *systemId;
   const XML_Char *base;
   const XML_Char *publicId;
@@ -280,13 +277,13 @@ typedef struct {
 } ENTITY;
 
 typedef struct {
-  enum XML_Content_Type         type;
-  enum XML_Content_Quant        quant;
-  const XML_Char *              name;
-  int                           firstchild;
-  int                           lastchild;
-  int                           childcnt;
-  int                           nextsib;
+  enum XML_Content_Type type;
+  enum XML_Content_Quant quant;
+  const XML_Char *name;
+  int firstchild;
+  int lastchild;
+  int childcnt;
+  int nextsib;
 } CONTENT_SCAFFOLD;
 
 #define INIT_SCAFFOLD_ELEMENTS 32
@@ -374,10 +371,8 @@ typedef struct open_internal_entity {
   XML_Bool betweenDecl; /* WFC: PE Between Declarations */
 } OPEN_INTERNAL_ENTITY;
 
-typedef enum XML_Error PTRCALL Processor(XML_Parser parser,
-                                         const char *start,
-                                         const char *end,
-                                         const char **endPtr);
+typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start,
+                                         const char *end, const char **endPtr);
 
 static Processor prologProcessor;
 static Processor prologInitProcessor;
@@ -398,118 +393,101 @@ static Processor externalEntityInitProcessor3;
 static Processor externalEntityContentProcessor;
 static Processor internalEntityProcessor;
 
-static enum XML_Error
-handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName);
-static enum XML_Error
-processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
-               const char *s, const char *next);
-static enum XML_Error
-initializeEncoding(XML_Parser parser);
-static enum XML_Error
-doProlog(XML_Parser parser, const ENCODING *enc, const char *s,
-         const char *end, int tok, const char *next, const char **nextPtr,
-         XML_Bool haveMore);
-static enum XML_Error
-processInternalEntity(XML_Parser parser, ENTITY *entity,
-                      XML_Bool betweenDecl);
-static enum XML_Error
-doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
-          const char *start, const char *end, const char **endPtr,
-          XML_Bool haveMore);
-static enum XML_Error
-doCdataSection(XML_Parser parser, const ENCODING *, const char **startPtr,
-               const char *end, const char **nextPtr, XML_Bool haveMore);
+static enum XML_Error handleUnknownEncoding(XML_Parser parser,
+                                            const XML_Char *encodingName);
+static enum XML_Error processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
+                                     const char *s, const char *next);
+static enum XML_Error initializeEncoding(XML_Parser parser);
+static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc,
+                               const char *s, const char *end, int tok,
+                               const char *next, const char **nextPtr,
+                               XML_Bool haveMore, XML_Bool allowClosingDoctype);
+static enum XML_Error processInternalEntity(XML_Parser parser, ENTITY *entity,
+                                            XML_Bool betweenDecl);
+static enum XML_Error doContent(XML_Parser parser, int startTagLevel,
+                                const ENCODING *enc, const char *start,
+                                const char *end, const char **endPtr,
+                                XML_Bool haveMore);
+static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *,
+                                     const char **startPtr, const char *end,
+                                     const char **nextPtr, XML_Bool haveMore);
 #ifdef XML_DTD
-static enum XML_Error
-doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr,
-                const char *end, const char **nextPtr, XML_Bool haveMore);
+static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *,
+                                      const char **startPtr, const char *end,
+                                      const char **nextPtr, XML_Bool haveMore);
 #endif /* XML_DTD */
 
-static void
-freeBindings(XML_Parser parser, BINDING *bindings);
-static enum XML_Error
-storeAtts(XML_Parser parser, const ENCODING *, const char *s,
-          TAG_NAME *tagNamePtr, BINDING **bindingsPtr);
-static enum XML_Error
-addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
-           const XML_Char *uri, BINDING **bindingsPtr);
-static int
-defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata,
-                XML_Bool isId, const XML_Char *dfltValue, XML_Parser parser);
-static enum XML_Error
-storeAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata,
-                    const char *, const char *, STRING_POOL *);
-static enum XML_Error
-appendAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata,
-                     const char *, const char *, STRING_POOL *);
-static ATTRIBUTE_ID *
-getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start,
-               const char *end);
-static int
-setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *);
-static enum XML_Error
-storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *start,
-                 const char *end);
-static int
-reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
-                            const char *start, const char *end);
-static int
-reportComment(XML_Parser parser, const ENCODING *enc, const char *start,
-              const char *end);
-static void
-reportDefault(XML_Parser parser, const ENCODING *enc, const char *start,
-              const char *end);
-
-static const XML_Char * getContext(XML_Parser parser);
-static XML_Bool
-setContext(XML_Parser parser, const XML_Char *context);
+static void freeBindings(XML_Parser parser, BINDING *bindings);
+static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *,
+                                const char *s, TAG_NAME *tagNamePtr,
+                                BINDING **bindingsPtr);
+static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix,
+                                 const ATTRIBUTE_ID *attId, const XML_Char *uri,
+                                 BINDING **bindingsPtr);
+static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata,
+                           XML_Bool isId, const XML_Char *dfltValue,
+                           XML_Parser parser);
+static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *,
+                                          XML_Bool isCdata, const char *,
+                                          const char *, STRING_POOL *);
+static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *,
+                                           XML_Bool isCdata, const char *,
+                                           const char *, STRING_POOL *);
+static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc,
+                                    const char *start, const char *end);
+static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *);
+static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc,
+                                       const char *start, const char *end);
+static int reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
+                                       const char *start, const char *end);
+static int reportComment(XML_Parser parser, const ENCODING *enc,
+                         const char *start, const char *end);
+static void reportDefault(XML_Parser parser, const ENCODING *enc,
+                          const char *start, const char *end);
+
+static const XML_Char *getContext(XML_Parser parser);
+static XML_Bool setContext(XML_Parser parser, const XML_Char *context);
 
 static void FASTCALL normalizePublicId(XML_Char *s);
 
-static DTD * dtdCreate(const XML_Memory_Handling_Suite *ms);
+static DTD *dtdCreate(const XML_Memory_Handling_Suite *ms);
 /* do not call if m_parentParser != NULL */
 static void dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms);
-static void
-dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms);
-static int
-dtdCopy(XML_Parser oldParser,
-        DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms);
-static int
-copyEntityTable(XML_Parser oldParser,
-                HASH_TABLE *, STRING_POOL *, const HASH_TABLE *);
-static NAMED *
-lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize);
-static void FASTCALL
-hashTableInit(HASH_TABLE *, const XML_Memory_Handling_Suite *ms);
+static void dtdDestroy(DTD *p, XML_Bool isDocEntity,
+                       const XML_Memory_Handling_Suite *ms);
+static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
+                   const XML_Memory_Handling_Suite *ms);
+static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *, STRING_POOL *,
+                           const HASH_TABLE *);
+static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name,
+                     size_t createSize);
+static void FASTCALL hashTableInit(HASH_TABLE *,
+                                   const XML_Memory_Handling_Suite *ms);
 static void FASTCALL hashTableClear(HASH_TABLE *);
 static void FASTCALL hashTableDestroy(HASH_TABLE *);
-static void FASTCALL
-hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *);
-static NAMED * FASTCALL hashTableIterNext(HASH_TABLE_ITER *);
+static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *);
+static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *);
 
-static void FASTCALL
-poolInit(STRING_POOL *, const XML_Memory_Handling_Suite *ms);
+static void FASTCALL poolInit(STRING_POOL *,
+                              const XML_Memory_Handling_Suite *ms);
 static void FASTCALL poolClear(STRING_POOL *);
 static void FASTCALL poolDestroy(STRING_POOL *);
-static XML_Char *
-poolAppend(STRING_POOL *pool, const ENCODING *enc,
-           const char *ptr, const char *end);
-static XML_Char *
-poolStoreString(STRING_POOL *pool, const ENCODING *enc,
-                const char *ptr, const char *end);
+static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc,
+                            const char *ptr, const char *end);
+static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc,
+                                 const char *ptr, const char *end);
 static XML_Bool FASTCALL poolGrow(STRING_POOL *pool);
-static const XML_Char * FASTCALL
-poolCopyString(STRING_POOL *pool, const XML_Char *s);
-static const XML_Char *
-poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n);
-static const XML_Char * FASTCALL
-poolAppendString(STRING_POOL *pool, const XML_Char *s);
+static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool,
+                                               const XML_Char *s);
+static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s,
+                                       int n);
+static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool,
+                                                 const XML_Char *s);
 
 static int FASTCALL nextScaffoldPart(XML_Parser parser);
-static XML_Content * build_model(XML_Parser parser);
-static ELEMENT_TYPE *
-getElementType(XML_Parser parser, const ENCODING *enc,
-               const char *ptr, const char *end);
+static XML_Content *build_model(XML_Parser parser);
+static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc,
+                                    const char *ptr, const char *end);
 
 static XML_Char *copyString(const XML_Char *s,
                             const XML_Memory_Handling_Suite *memsuite);
@@ -517,14 +495,11 @@ static XML_Char *copyString(const XML_Char *s,
 static unsigned long generate_hash_secret_salt(XML_Parser parser);
 static XML_Bool startParsing(XML_Parser parser);
 
-static XML_Parser
-parserCreate(const XML_Char *encodingName,
-             const XML_Memory_Handling_Suite *memsuite,
-             const XML_Char *nameSep,
-             DTD *dtd);
+static XML_Parser parserCreate(const XML_Char *encodingName,
+                               const XML_Memory_Handling_Suite *memsuite,
+                               const XML_Char *nameSep, DTD *dtd);
 
-static void
-parserInit(XML_Parser parser, const XML_Char *encodingName);
+static void parserInit(XML_Parser parser, const XML_Char *encodingName);
 
 #define poolStart(pool) ((pool)->start)
 #define poolEnd(pool) ((pool)->ptr)
@@ -533,10 +508,10 @@ parserInit(XML_Parser parser, const XML_Char *encodingName);
 #define poolLastChar(pool) (((pool)->ptr)[-1])
 #define poolDiscard(pool) ((pool)->ptr = (pool)->start)
 #define poolFinish(pool) ((pool)->start = (pool)->ptr)
-#define poolAppendChar(pool, c) \
-  (((pool)->ptr == (pool)->end && !poolGrow(pool)) \
-   ? 0 \
-   : ((*((pool)->ptr)++ = c), 1))
+#define poolAppendChar(pool, c)                                                \
+  (((pool)->ptr == (pool)->end && ! poolGrow(pool))                            \
+       ? 0                                                                     \
+       : ((*((pool)->ptr)++ = c), 1))
 
 struct XML_ParserStruct {
   /* The first member must be m_userData so that the XML_GetUserData
@@ -587,7 +562,7 @@ struct XML_ParserStruct {
   void *m_unknownEncodingMem;
   void *m_unknownEncodingData;
   void *m_unknownEncodingHandlerData;
-  void (XMLCALL *m_unknownEncodingRelease)(void *);
+  void(XMLCALL *m_unknownEncodingRelease)(void *);
   PROLOG_STATE m_prologState;
   Processor *m_processor;
   enum XML_Error m_errorCode;
@@ -641,57 +616,55 @@ struct XML_ParserStruct {
   unsigned long m_hash_secret_salt;
 };
 
-#define MALLOC(parser, s)      (parser->m_mem.malloc_fcn((s)))
-#define REALLOC(parser, p, s)  (parser->m_mem.realloc_fcn((p),(s)))
-#define FREE(parser, p)        (parser->m_mem.free_fcn((p)))
-
+#define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s)))
+#define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s)))
+#define FREE(parser, p) (parser->m_mem.free_fcn((p)))
 
 XML_Parser XMLCALL
-XML_ParserCreate(const XML_Char *encodingName)
-{
+XML_ParserCreate(const XML_Char *encodingName) {
   return XML_ParserCreate_MM(encodingName, NULL, NULL);
 }
 
 XML_Parser XMLCALL
-XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep)
-{
+XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) {
   XML_Char tmp[2];
   *tmp = nsSep;
   return XML_ParserCreate_MM(encodingName, NULL, tmp);
 }
 
-static const XML_Char implicitContext[] = {
-  ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h, ASCII_t, ASCII_t, ASCII_p,
-  ASCII_COLON, ASCII_SLASH, ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w,
-  ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g,
-  ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L, ASCII_SLASH, ASCII_1, ASCII_9,
-  ASCII_9, ASCII_8, ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e,
-  ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'
-};
-
+static const XML_Char implicitContext[]
+    = {ASCII_x,     ASCII_m,     ASCII_l,      ASCII_EQUALS, ASCII_h,
+       ASCII_t,     ASCII_t,     ASCII_p,      ASCII_COLON,  ASCII_SLASH,
+       ASCII_SLASH, ASCII_w,     ASCII_w,      ASCII_w,      ASCII_PERIOD,
+       ASCII_w,     ASCII_3,     ASCII_PERIOD, ASCII_o,      ASCII_r,
+       ASCII_g,     ASCII_SLASH, ASCII_X,      ASCII_M,      ASCII_L,
+       ASCII_SLASH, ASCII_1,     ASCII_9,      ASCII_9,      ASCII_8,
+       ASCII_SLASH, ASCII_n,     ASCII_a,      ASCII_m,      ASCII_e,
+       ASCII_s,     ASCII_p,     ASCII_a,      ASCII_c,      ASCII_e,
+       '\0'};
 
 /* To avoid warnings about unused functions: */
 #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)
 
-#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
+#  if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
 
 /* Obtain entropy on Linux 3.17+ */
 static int
-writeRandomBytes_getrandom_nonblock(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
+writeRandomBytes_getrandom_nonblock(void *target, size_t count) {
+  int success = 0; /* full count bytes written? */
   size_t bytesWrittenTotal = 0;
   const unsigned int getrandomFlags = GRND_NONBLOCK;
 
   do {
-    void * const currentTarget = (void*)((char*)target + bytesWrittenTotal);
+    void *const currentTarget = (void *)((char *)target + bytesWrittenTotal);
     const size_t bytesToWrite = count - bytesWrittenTotal;
 
     const int bytesWrittenMore =
-#if defined(HAVE_GETRANDOM)
+#    if defined(HAVE_GETRANDOM)
         getrandom(currentTarget, bytesToWrite, getrandomFlags);
-#else
+#    else
         syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags);
-#endif
+#    endif
 
     if (bytesWrittenMore > 0) {
       bytesWrittenTotal += bytesWrittenMore;
@@ -703,15 +676,14 @@ writeRandomBytes_getrandom_nonblock(void * target, size_t count) {
   return success;
 }
 
-#endif  /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
-
+#  endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
 
-#if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
+#  if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
 
 /* Extract entropy from /dev/urandom */
 static int
-writeRandomBytes_dev_urandom(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
+writeRandomBytes_dev_urandom(void *target, size_t count) {
+  int success = 0; /* full count bytes written? */
   size_t bytesWrittenTotal = 0;
 
   const int fd = open("/dev/urandom", O_RDONLY);
@@ -720,7 +692,7 @@ writeRandomBytes_dev_urandom(void * target, size_t count) {
   }
 
   do {
-    void * const currentTarget = (void*)((char*)target + bytesWrittenTotal);
+    void *const currentTarget = (void *)((char *)target + bytesWrittenTotal);
     const size_t bytesToWrite = count - bytesWrittenTotal;
 
     const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite);
@@ -736,15 +708,14 @@ writeRandomBytes_dev_urandom(void * target, size_t count) {
   return success;
 }
 
-#endif  /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
-
-#endif  /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
+#  endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
 
+#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
 
 #if defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF)
 
 static void
-writeRandomBytes_arc4random(void * target, size_t count) {
+writeRandomBytes_arc4random(void *target, size_t count) {
   size_t bytesWrittenTotal = 0;
 
   while (bytesWrittenTotal < count) {
@@ -752,93 +723,82 @@ writeRandomBytes_arc4random(void * target, size_t count) {
     size_t i = 0;
 
     for (; (i < sizeof(random32)) && (bytesWrittenTotal < count);
-        i++, bytesWrittenTotal++) {
+         i++, bytesWrittenTotal++) {
       const uint8_t random8 = (uint8_t)(random32 >> (i * 8));
       ((uint8_t *)target)[bytesWrittenTotal] = random8;
     }
   }
 }
 
-#endif  /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */
-
+#endif /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */
 
 #ifdef _WIN32
 
-typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG);
-HMODULE _Expat_LoadLibrary(LPCTSTR filename);  /* see loadlibrary.c */
-
-/* Obtain entropy on Windows XP / Windows Server 2003 and later.
- * Hint on RtlGenRandom and the following article from libsodium.
- *
- * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI
- * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographically-secure-random-number-on-windows-without-using-cryptoapi/
+/* Obtain entropy on Windows using the rand_s() function which
+ * generates cryptographically secure random numbers.  Internally it
+ * uses RtlGenRandom API which is present in Windows XP and later.
  */
 static int
-writeRandomBytes_RtlGenRandom(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
-  const HMODULE advapi32 = _Expat_LoadLibrary(TEXT("ADVAPI32.DLL"));
-
-  if (advapi32) {
-    const RTLGENRANDOM_FUNC RtlGenRandom
-        = (RTLGENRANDOM_FUNC)GetProcAddress(advapi32, "SystemFunction036");
-    if (RtlGenRandom) {
-      if (RtlGenRandom((PVOID)target, (ULONG)count) == TRUE) {
-        success = 1;
-      }
+writeRandomBytes_rand_s(void *target, size_t count) {
+  size_t bytesWrittenTotal = 0;
+
+  while (bytesWrittenTotal < count) {
+    unsigned int random32 = 0;
+    size_t i = 0;
+
+    if (rand_s(&random32))
+      return 0; /* failure */
+
+    for (; (i < sizeof(random32)) && (bytesWrittenTotal < count);
+         i++, bytesWrittenTotal++) {
+      const uint8_t random8 = (uint8_t)(random32 >> (i * 8));
+      ((uint8_t *)target)[bytesWrittenTotal] = random8;
     }
-    FreeLibrary(advapi32);
   }
-
-  return success;
+  return 1; /* success */
 }
 
 #endif /* _WIN32 */
 
-
 #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)
 
 static unsigned long
-gather_time_entropy(void)
-{
-#ifdef _WIN32
+gather_time_entropy(void) {
+#  ifdef _WIN32
   FILETIME ft;
   GetSystemTimeAsFileTime(&ft); /* never fails */
   return ft.dwHighDateTime ^ ft.dwLowDateTime;
-#else
+#  else
   struct timeval tv;
   int gettimeofday_res;
 
   gettimeofday_res = gettimeofday(&tv, NULL);
 
-#if defined(NDEBUG)
+#    if defined(NDEBUG)
   (void)gettimeofday_res;
-#else
-  assert (gettimeofday_res == 0);
-#endif  /* defined(NDEBUG) */
+#    else
+  assert(gettimeofday_res == 0);
+#    endif /* defined(NDEBUG) */
 
   /* Microseconds time is <20 bits entropy */
   return tv.tv_usec;
-#endif
+#  endif
 }
 
-#endif  /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
-
+#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
 
 static unsigned long
-ENTROPY_DEBUG(const char * label, unsigned long entropy) {
-  const char * const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG");
+ENTROPY_DEBUG(const char *label, unsigned long entropy) {
+  const char *const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG");
   if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) {
-    fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n",
-        label,
-        (int)sizeof(entropy) * 2, entropy,
-        (unsigned long)sizeof(entropy));
+    fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", label,
+            (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy));
   }
   return entropy;
 }
 
 static unsigned long
-generate_hash_secret_salt(XML_Parser parser)
-{
+generate_hash_secret_salt(XML_Parser parser) {
   unsigned long entropy;
   (void)parser;
 
@@ -851,20 +811,20 @@ generate_hash_secret_salt(XML_Parser parser)
   return ENTROPY_DEBUG("arc4random", entropy);
 #else
   /* Try high quality providers first .. */
-#ifdef _WIN32
-  if (writeRandomBytes_RtlGenRandom((void *)&entropy, sizeof(entropy))) {
-    return ENTROPY_DEBUG("RtlGenRandom", entropy);
+#  ifdef _WIN32
+  if (writeRandomBytes_rand_s((void *)&entropy, sizeof(entropy))) {
+    return ENTROPY_DEBUG("rand_s", entropy);
   }
-#elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
+#  elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
   if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) {
     return ENTROPY_DEBUG("getrandom", entropy);
   }
-#endif
-#if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
+#  endif
+#  if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
   if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) {
     return ENTROPY_DEBUG("/dev/urandom", entropy);
   }
-#endif  /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
+#  endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
   /* .. and self-made low quality for backup: */
 
   /* Process ID is 0 bits entropy if attacker has local access */
@@ -875,7 +835,7 @@ generate_hash_secret_salt(XML_Parser parser)
     return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647);
   } else {
     return ENTROPY_DEBUG("fallback(8)",
-        entropy * (unsigned long)2305843009213693951ULL);
+                         entropy * (unsigned long)2305843009213693951ULL);
   }
 #endif
 }
@@ -887,49 +847,43 @@ get_hash_secret_salt(XML_Parser parser) {
   return parser->m_hash_secret_salt;
 }
 
-static XML_Bool  /* only valid for root parser */
-startParsing(XML_Parser parser)
-{
-    /* hash functions must be initialized before setContext() is called */
-    if (parser->m_hash_secret_salt == 0)
-      parser->m_hash_secret_salt = generate_hash_secret_salt(parser);
-    if (parser->m_ns) {
-      /* implicit context only set for root parser, since child
-         parsers (i.e. external entity parsers) will inherit it
-      */
-      return setContext(parser, implicitContext);
-    }
-    return XML_TRUE;
+static XML_Bool /* only valid for root parser */
+startParsing(XML_Parser parser) {
+  /* hash functions must be initialized before setContext() is called */
+  if (parser->m_hash_secret_salt == 0)
+    parser->m_hash_secret_salt = generate_hash_secret_salt(parser);
+  if (parser->m_ns) {
+    /* implicit context only set for root parser, since child
+       parsers (i.e. external entity parsers) will inherit it
+    */
+    return setContext(parser, implicitContext);
+  }
+  return XML_TRUE;
 }
 
 XML_Parser XMLCALL
 XML_ParserCreate_MM(const XML_Char *encodingName,
                     const XML_Memory_Handling_Suite *memsuite,
-                    const XML_Char *nameSep)
-{
+                    const XML_Char *nameSep) {
   return parserCreate(encodingName, memsuite, nameSep, NULL);
 }
 
 static XML_Parser
 parserCreate(const XML_Char *encodingName,
-             const XML_Memory_Handling_Suite *memsuite,
-             const XML_Char *nameSep,
-             DTD *dtd)
-{
+             const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep,
+             DTD *dtd) {
   XML_Parser parser;
 
   if (memsuite) {
     XML_Memory_Handling_Suite *mtemp;
-    parser = (XML_Parser)
-      memsuite->malloc_fcn(sizeof(struct XML_ParserStruct));
+    parser = (XML_Parser)memsuite->malloc_fcn(sizeof(struct XML_ParserStruct));
     if (parser != NULL) {
       mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem);
       mtemp->malloc_fcn = memsuite->malloc_fcn;
       mtemp->realloc_fcn = memsuite->realloc_fcn;
       mtemp->free_fcn = memsuite->free_fcn;
     }
-  }
-  else {
+  } else {
     XML_Memory_Handling_Suite *mtemp;
     parser = (XML_Parser)malloc(sizeof(struct XML_ParserStruct));
     if (parser != NULL) {
@@ -940,27 +894,30 @@ parserCreate(const XML_Char *encodingName,
     }
   }
 
-  if (!parser)
+  if (! parser)
     return parser;
 
   parser->m_buffer = NULL;
   parser->m_bufferLim = NULL;
 
   parser->m_attsSize = INIT_ATTS_SIZE;
-  parser->m_atts = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE));
+  parser->m_atts
+      = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE));
   if (parser->m_atts == NULL) {
     FREE(parser, parser);
     return NULL;
   }
 #ifdef XML_ATTR_INFO
-  parser->m_attInfo = (XML_AttrInfo*)MALLOC(parser, parser->m_attsSize * sizeof(XML_AttrInfo));
+  parser->m_attInfo = (XML_AttrInfo *)MALLOC(
+      parser, parser->m_attsSize * sizeof(XML_AttrInfo));
   if (parser->m_attInfo == NULL) {
     FREE(parser, parser->m_atts);
     FREE(parser, parser);
     return NULL;
   }
 #endif
-  parser->m_dataBuf = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char));
+  parser->m_dataBuf
+      = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char));
   if (parser->m_dataBuf == NULL) {
     FREE(parser, parser->m_atts);
 #ifdef XML_ATTR_INFO
@@ -1010,7 +967,7 @@ parserCreate(const XML_Char *encodingName,
   poolInit(&parser->m_temp2Pool, &(parser->m_mem));
   parserInit(parser, encodingName);
 
-  if (encodingName && !parser->m_protocolEncodingName) {
+  if (encodingName && ! parser->m_protocolEncodingName) {
     XML_ParserFree(parser);
     return NULL;
   }
@@ -1019,8 +976,7 @@ parserCreate(const XML_Char *encodingName,
     parser->m_ns = XML_TRUE;
     parser->m_internalEncoding = XmlGetInternalEncodingNS();
     parser->m_namespaceSeparator = *nameSep;
-  }
-  else {
+  } else {
     parser->m_internalEncoding = XmlGetInternalEncoding();
   }
 
@@ -1028,8 +984,7 @@ parserCreate(const XML_Char *encodingName,
 }
 
 static void
-parserInit(XML_Parser parser, const XML_Char *encodingName)
-{
+parserInit(XML_Parser parser, const XML_Char *encodingName) {
   parser->m_processor = prologInitProcessor;
   XmlPrologStateInit(&parser->m_prologState);
   if (encodingName != NULL) {
@@ -1102,8 +1057,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName)
 
 /* moves list of bindings to m_freeBindingList */
 static void FASTCALL
-moveToFreeBindingList(XML_Parser parser, BINDING *bindings)
-{
+moveToFreeBindingList(XML_Parser parser, BINDING *bindings) {
   while (bindings) {
     BINDING *b = bindings;
     bindings = bindings->nextTagBinding;
@@ -1113,13 +1067,12 @@ moveToFreeBindingList(XML_Parser parser, BINDING *bindings)
 }
 
 XML_Bool XMLCALL
-XML_ParserReset(XML_Parser parser, const XML_Char *encodingName)
-{
+XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) {
   TAG *tStk;
   OPEN_INTERNAL_ENTITY *openEntityList;
 
   if (parser == NULL)
-      return XML_FALSE;
+    return XML_FALSE;
 
   if (parser->m_parentParser)
     return XML_FALSE;
@@ -1155,15 +1108,15 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName)
 }
 
 enum XML_Status XMLCALL
-XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName)
-{
+XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) {
   if (parser == NULL)
-      return XML_STATUS_ERROR;
+    return XML_STATUS_ERROR;
   /* Block after XML_Parse()/XML_ParseBuffer() has been called.
      XXX There's no way for the caller to determine which of the
      XXX possible error cases caused the XML_STATUS_ERROR return.
   */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return XML_STATUS_ERROR;
 
   /* Get rid of any previous encoding name */
@@ -1175,17 +1128,15 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName)
   else {
     /* Copy the new encoding name into allocated memory */
     parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem));
-    if (!parser->m_protocolEncodingName)
+    if (! parser->m_protocolEncodingName)
       return XML_STATUS_ERROR;
   }
   return XML_STATUS_OK;
 }
 
 XML_Parser XMLCALL
-XML_ExternalEntityParserCreate(XML_Parser oldParser,
-                               const XML_Char *context,
-                               const XML_Char *encodingName)
-{
+XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
+                               const XML_Char *encodingName) {
   XML_Parser parser = oldParser;
   DTD *newDtd = NULL;
   DTD *oldDtd;
@@ -1209,7 +1160,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   XML_AttlistDeclHandler oldAttlistDeclHandler;
   XML_EntityDeclHandler oldEntityDeclHandler;
   XML_XmlDeclHandler oldXmlDeclHandler;
-  ELEMENT_TYPE * oldDeclElementType;
+  ELEMENT_TYPE *oldDeclElementType;
 
   void *oldUserData;
   void *oldHandlerArg;
@@ -1272,7 +1223,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   oldhash_secret_salt = parser->m_hash_secret_salt;
 
 #ifdef XML_DTD
-  if (!context)
+  if (! context)
     newDtd = oldDtd;
 #endif /* XML_DTD */
 
@@ -1285,12 +1236,11 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
     XML_Char tmp[2];
     *tmp = parser->m_namespaceSeparator;
     parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd);
-  }
-  else {
+  } else {
     parser = parserCreate(encodingName, &parser->m_mem, NULL, newDtd);
   }
 
-  if (!parser)
+  if (! parser)
     return NULL;
 
   parser->m_startElementHandler = oldStartElementHandler;
@@ -1330,21 +1280,20 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   parser->m_prologState.inEntityValue = oldInEntityValue;
   if (context) {
 #endif /* XML_DTD */
-    if (!dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem)
-      || !setContext(parser, context)) {
+    if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem)
+        || ! setContext(parser, context)) {
       XML_ParserFree(parser);
       return NULL;
     }
     parser->m_processor = externalEntityInitProcessor;
 #ifdef XML_DTD
-  }
-  else {
-    /* The DTD instance referenced by parser->m_dtd is shared between the document's
-       root parser and external PE parsers, therefore one does not need to
-       call setContext. In addition, one also *must* not call setContext,
-       because this would overwrite existing prefix->binding pointers in
-       parser->m_dtd with ones that get destroyed with the external PE parser.
-       This would leave those prefixes with dangling pointers.
+  } else {
+    /* The DTD instance referenced by parser->m_dtd is shared between the
+       document's root parser and external PE parsers, therefore one does not
+       need to call setContext. In addition, one also *must* not call
+       setContext, because this would overwrite existing prefix->binding
+       pointers in parser->m_dtd with ones that get destroyed with the external
+       PE parser. This would leave those prefixes with dangling pointers.
     */
     parser->m_isParamEntity = XML_TRUE;
     XmlPrologStateInitExternalEntity(&parser->m_prologState);
@@ -1355,11 +1304,10 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
 }
 
 static void FASTCALL
-destroyBindings(BINDING *bindings, XML_Parser parser)
-{
+destroyBindings(BINDING *bindings, XML_Parser parser) {
   for (;;) {
     BINDING *b = bindings;
-    if (!b)
+    if (! b)
       break;
     bindings = b->nextTagBinding;
     FREE(parser, b->uri);
@@ -1368,8 +1316,7 @@ destroyBindings(BINDING *bindings, XML_Parser parser)
 }
 
 void XMLCALL
-XML_ParserFree(XML_Parser parser)
-{
+XML_ParserFree(XML_Parser parser) {
   TAG *tagList;
   OPEN_INTERNAL_ENTITY *entityList;
   if (parser == NULL)
@@ -1414,11 +1361,12 @@ XML_ParserFree(XML_Parser parser)
   /* external parameter entity parsers share the DTD structure
      parser->m_dtd with the root parser, so we must not destroy it
   */
-  if (!parser->m_isParamEntity && parser->m_dtd)
+  if (! parser->m_isParamEntity && parser->m_dtd)
 #else
   if (parser->m_dtd)
 #endif /* XML_DTD */
-    dtdDestroy(parser->m_dtd, (XML_Bool)!parser->m_parentParser, &parser->m_mem);
+    dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser,
+               &parser->m_mem);
   FREE(parser, (void *)parser->m_atts);
 #ifdef XML_ATTR_INFO
   FREE(parser, (void *)parser->m_attInfo);
@@ -1434,20 +1382,19 @@ XML_ParserFree(XML_Parser parser)
 }
 
 void XMLCALL
-XML_UseParserAsHandlerArg(XML_Parser parser)
-{
+XML_UseParserAsHandlerArg(XML_Parser parser) {
   if (parser != NULL)
     parser->m_handlerArg = parser;
 }
 
 enum XML_Error XMLCALL
-XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD)
-{
+XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) {
   if (parser == NULL)
     return XML_ERROR_INVALID_ARGUMENT;
 #ifdef XML_DTD
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING;
   parser->m_useForeignDTD = useDTD;
   return XML_ERROR_NONE;
@@ -1457,19 +1404,18 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD)
 }
 
 void XMLCALL
-XML_SetReturnNSTriplet(XML_Parser parser, int do_nst)
-{
+XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) {
   if (parser == NULL)
     return;
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return;
   parser->m_ns_triplets = do_nst ? XML_TRUE : XML_FALSE;
 }
 
 void XMLCALL
-XML_SetUserData(XML_Parser parser, void *p)
-{
+XML_SetUserData(XML_Parser parser, void *p) {
   if (parser == NULL)
     return;
   if (parser->m_handlerArg == parser->m_userData)
@@ -1479,49 +1425,43 @@ XML_SetUserData(XML_Parser parser, void *p)
 }
 
 enum XML_Status XMLCALL
-XML_SetBase(XML_Parser parser, const XML_Char *p)
-{
+XML_SetBase(XML_Parser parser, const XML_Char *p) {
   if (parser == NULL)
     return XML_STATUS_ERROR;
   if (p) {
     p = poolCopyString(&parser->m_dtd->pool, p);
-    if (!p)
+    if (! p)
       return XML_STATUS_ERROR;
     parser->m_curBase = p;
-  }
-  else
+  } else
     parser->m_curBase = NULL;
   return XML_STATUS_OK;
 }
 
-const XML_Char * XMLCALL
-XML_GetBase(XML_Parser parser)
-{
+const XML_Char *XMLCALL
+XML_GetBase(XML_Parser parser) {
   if (parser == NULL)
     return NULL;
   return parser->m_curBase;
 }
 
 int XMLCALL
-XML_GetSpecifiedAttributeCount(XML_Parser parser)
-{
+XML_GetSpecifiedAttributeCount(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   return parser->m_nSpecifiedAtts;
 }
 
 int XMLCALL
-XML_GetIdAttributeIndex(XML_Parser parser)
-{
+XML_GetIdAttributeIndex(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   return parser->m_idAttIndex;
 }
 
 #ifdef XML_ATTR_INFO
-const XML_AttrInfo * XMLCALL
-XML_GetAttributeInfo(XML_Parser parser)
-{
+const XML_AttrInfo *XMLCALL
+XML_GetAttributeInfo(XML_Parser parser) {
   if (parser == NULL)
     return NULL;
   return parser->m_attInfo;
@@ -1529,10 +1469,8 @@ XML_GetAttributeInfo(XML_Parser parser)
 #endif
 
 void XMLCALL
-XML_SetElementHandler(XML_Parser parser,
-                      XML_StartElementHandler start,
-                      XML_EndElementHandler end)
-{
+XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start,
+                      XML_EndElementHandler end) {
   if (parser == NULL)
     return;
   parser->m_startElementHandler = start;
@@ -1540,39 +1478,33 @@ XML_SetElementHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetStartElementHandler(XML_Parser parser,
-                           XML_StartElementHandler start) {
+XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler start) {
   if (parser != NULL)
     parser->m_startElementHandler = start;
 }
 
 void XMLCALL
-XML_SetEndElementHandler(XML_Parser parser,
-                         XML_EndElementHandler end) {
+XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler end) {
   if (parser != NULL)
     parser->m_endElementHandler = end;
 }
 
 void XMLCALL
 XML_SetCharacterDataHandler(XML_Parser parser,
-                            XML_CharacterDataHandler handler)
-{
+                            XML_CharacterDataHandler handler) {
   if (parser != NULL)
     parser->m_characterDataHandler = handler;
 }
 
 void XMLCALL
 XML_SetProcessingInstructionHandler(XML_Parser parser,
-                                    XML_ProcessingInstructionHandler handler)
-{
+                                    XML_ProcessingInstructionHandler handler) {
   if (parser != NULL)
     parser->m_processingInstructionHandler = handler;
 }
 
 void XMLCALL
-XML_SetCommentHandler(XML_Parser parser,
-                      XML_CommentHandler handler)
-{
+XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler) {
   if (parser != NULL)
     parser->m_commentHandler = handler;
 }
@@ -1580,8 +1512,7 @@ XML_SetCommentHandler(XML_Parser parser,
 void XMLCALL
 XML_SetCdataSectionHandler(XML_Parser parser,
                            XML_StartCdataSectionHandler start,
-                           XML_EndCdataSectionHandler end)
-{
+                           XML_EndCdataSectionHandler end) {
   if (parser == NULL)
     return;
   parser->m_startCdataSectionHandler = start;
@@ -1603,9 +1534,7 @@ XML_SetEndCdataSectionHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDefaultHandler(XML_Parser parser,
-                      XML_DefaultHandler handler)
-{
+XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler) {
   if (parser == NULL)
     return;
   parser->m_defaultHandler = handler;
@@ -1613,9 +1542,7 @@ XML_SetDefaultHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDefaultHandlerExpand(XML_Parser parser,
-                            XML_DefaultHandler handler)
-{
+XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler) {
   if (parser == NULL)
     return;
   parser->m_defaultHandler = handler;
@@ -1623,10 +1550,8 @@ XML_SetDefaultHandlerExpand(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDoctypeDeclHandler(XML_Parser parser,
-                          XML_StartDoctypeDeclHandler start,
-                          XML_EndDoctypeDeclHandler end)
-{
+XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start,
+                          XML_EndDoctypeDeclHandler end) {
   if (parser == NULL)
     return;
   parser->m_startDoctypeDeclHandler = start;
@@ -1641,24 +1566,20 @@ XML_SetStartDoctypeDeclHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetEndDoctypeDeclHandler(XML_Parser parser,
-                             XML_EndDoctypeDeclHandler end) {
+XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end) {
   if (parser != NULL)
     parser->m_endDoctypeDeclHandler = end;
 }
 
 void XMLCALL
 XML_SetUnparsedEntityDeclHandler(XML_Parser parser,
-                                 XML_UnparsedEntityDeclHandler handler)
-{
+                                 XML_UnparsedEntityDeclHandler handler) {
   if (parser != NULL)
     parser->m_unparsedEntityDeclHandler = handler;
 }
 
 void XMLCALL
-XML_SetNotationDeclHandler(XML_Parser parser,
-                           XML_NotationDeclHandler handler)
-{
+XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler) {
   if (parser != NULL)
     parser->m_notationDeclHandler = handler;
 }
@@ -1666,8 +1587,7 @@ XML_SetNotationDeclHandler(XML_Parser parser,
 void XMLCALL
 XML_SetNamespaceDeclHandler(XML_Parser parser,
                             XML_StartNamespaceDeclHandler start,
-                            XML_EndNamespaceDeclHandler end)
-{
+                            XML_EndNamespaceDeclHandler end) {
   if (parser == NULL)
     return;
   parser->m_startNamespaceDeclHandler = start;
@@ -1690,23 +1610,20 @@ XML_SetEndNamespaceDeclHandler(XML_Parser parser,
 
 void XMLCALL
 XML_SetNotStandaloneHandler(XML_Parser parser,
-                            XML_NotStandaloneHandler handler)
-{
+                            XML_NotStandaloneHandler handler) {
   if (parser != NULL)
     parser->m_notStandaloneHandler = handler;
 }
 
 void XMLCALL
 XML_SetExternalEntityRefHandler(XML_Parser parser,
-                                XML_ExternalEntityRefHandler handler)
-{
+                                XML_ExternalEntityRefHandler handler) {
   if (parser != NULL)
     parser->m_externalEntityRefHandler = handler;
 }
 
 void XMLCALL
-XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg)
-{
+XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg) {
   if (parser == NULL)
     return;
   if (arg)
@@ -1717,17 +1634,14 @@ XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg)
 
 void XMLCALL
 XML_SetSkippedEntityHandler(XML_Parser parser,
-                            XML_SkippedEntityHandler handler)
-{
+                            XML_SkippedEntityHandler handler) {
   if (parser != NULL)
     parser->m_skippedEntityHandler = handler;
 }
 
 void XMLCALL
 XML_SetUnknownEncodingHandler(XML_Parser parser,
-                              XML_UnknownEncodingHandler handler,
-                              void *data)
-{
+                              XML_UnknownEncodingHandler handler, void *data) {
   if (parser == NULL)
     return;
   parser->m_unknownEncodingHandler = handler;
@@ -1735,44 +1649,37 @@ XML_SetUnknownEncodingHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetElementDeclHandler(XML_Parser parser,
-                          XML_ElementDeclHandler eldecl)
-{
+XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl) {
   if (parser != NULL)
     parser->m_elementDeclHandler = eldecl;
 }
 
 void XMLCALL
-XML_SetAttlistDeclHandler(XML_Parser parser,
-                          XML_AttlistDeclHandler attdecl)
-{
+XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl) {
   if (parser != NULL)
     parser->m_attlistDeclHandler = attdecl;
 }
 
 void XMLCALL
-XML_SetEntityDeclHandler(XML_Parser parser,
-                         XML_EntityDeclHandler handler)
-{
+XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler) {
   if (parser != NULL)
     parser->m_entityDeclHandler = handler;
 }
 
 void XMLCALL
-XML_SetXmlDeclHandler(XML_Parser parser,
-                      XML_XmlDeclHandler handler) {
+XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler handler) {
   if (parser != NULL)
     parser->m_xmlDeclHandler = handler;
 }
 
 int XMLCALL
 XML_SetParamEntityParsing(XML_Parser parser,
-                          enum XML_ParamEntityParsing peParsing)
-{
+                          enum XML_ParamEntityParsing peParsing) {
   if (parser == NULL)
     return 0;
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return 0;
 #ifdef XML_DTD
   parser->m_paramEntityParsing = peParsing;
@@ -1783,23 +1690,21 @@ XML_SetParamEntityParsing(XML_Parser parser,
 }
 
 int XMLCALL
-XML_SetHashSalt(XML_Parser parser,
-                unsigned long hash_salt)
-{
+XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) {
   if (parser == NULL)
     return 0;
   if (parser->m_parentParser)
     return XML_SetHashSalt(parser->m_parentParser, hash_salt);
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return 0;
   parser->m_hash_secret_salt = hash_salt;
   return 1;
 }
 
 enum XML_Status XMLCALL
-XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
-{
+XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) {
   if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) {
     if (parser != NULL)
       parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT;
@@ -1813,7 +1718,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     parser->m_errorCode = XML_ERROR_FINISHED;
     return XML_STATUS_ERROR;
   case XML_INITIALIZED:
-    if (parser->m_parentParser == NULL && !startParsing(parser)) {
+    if (parser->m_parentParser == NULL && ! startParsing(parser)) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return XML_STATUS_ERROR;
     }
@@ -1824,7 +1729,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
 
   if (len == 0) {
     parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
-    if (!isFinal)
+    if (! isFinal)
       return XML_STATUS_OK;
     parser->m_positionPtr = parser->m_bufferPtr;
     parser->m_parseEndPtr = parser->m_bufferEnd;
@@ -1833,7 +1738,9 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
        data are the final chunk of input, then we have to check them again
        to detect errors based on that fact.
     */
-    parser->m_errorCode = parser->m_processor(parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
+    parser->m_errorCode
+        = parser->m_processor(parser, parser->m_bufferPtr,
+                              parser->m_parseEndPtr, &parser->m_bufferPtr);
 
     if (parser->m_errorCode == XML_ERROR_NONE) {
       switch (parser->m_parsingStatus.parsing) {
@@ -1850,7 +1757,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
          *
          * LCOV_EXCL_START
          */
-        XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+        XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                          parser->m_bufferPtr, &parser->m_position);
         parser->m_positionPtr = parser->m_bufferPtr;
         return XML_STATUS_SUSPENDED;
         /* LCOV_EXCL_STOP */
@@ -1873,23 +1781,23 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     enum XML_Status result;
     /* Detect overflow (a+b > MAX <==> b > MAX-a) */
     if (len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) {
-       parser->m_errorCode = XML_ERROR_NO_MEMORY;
-       parser->m_eventPtr = parser->m_eventEndPtr = NULL;
-       parser->m_processor = errorProcessor;
-       return XML_STATUS_ERROR;
+      parser->m_errorCode = XML_ERROR_NO_MEMORY;
+      parser->m_eventPtr = parser->m_eventEndPtr = NULL;
+      parser->m_processor = errorProcessor;
+      return XML_STATUS_ERROR;
     }
     parser->m_parseEndByteIndex += len;
     parser->m_positionPtr = s;
     parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
 
-    parser->m_errorCode = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end);
+    parser->m_errorCode
+        = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end);
 
     if (parser->m_errorCode != XML_ERROR_NONE) {
       parser->m_eventEndPtr = parser->m_eventPtr;
       parser->m_processor = errorProcessor;
       return XML_STATUS_ERROR;
-    }
-    else {
+    } else {
       switch (parser->m_parsingStatus.parsing) {
       case XML_SUSPENDED:
         result = XML_STATUS_SUSPENDED;
@@ -1906,10 +1814,12 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
       }
     }
 
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, end, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, end,
+                      &parser->m_position);
     nLeftOver = s + len - end;
     if (nLeftOver) {
-      if (parser->m_buffer == NULL || nLeftOver > parser->m_bufferLim - parser->m_buffer) {
+      if (parser->m_buffer == NULL
+          || nLeftOver > parser->m_bufferLim - parser->m_buffer) {
         /* avoid _signed_ integer overflow */
         char *temp = NULL;
         const int bytesToAllocate = (int)((unsigned)len * 2U);
@@ -1935,7 +1845,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     parser->m_eventEndPtr = parser->m_bufferPtr;
     return result;
   }
-#endif  /* not defined XML_CONTEXT_BYTES */
+#endif /* not defined XML_CONTEXT_BYTES */
   else {
     void *buff = XML_GetBuffer(parser, len);
     if (buff == NULL)
@@ -1948,8 +1858,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
 }
 
 enum XML_Status XMLCALL
-XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
-{
+XML_ParseBuffer(XML_Parser parser, int len, int isFinal) {
   const char *start;
   enum XML_Status result = XML_STATUS_OK;
 
@@ -1963,7 +1872,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
     parser->m_errorCode = XML_ERROR_FINISHED;
     return XML_STATUS_ERROR;
   case XML_INITIALIZED:
-    if (parser->m_parentParser == NULL && !startParsing(parser)) {
+    if (parser->m_parentParser == NULL && ! startParsing(parser)) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return XML_STATUS_ERROR;
     }
@@ -1979,14 +1888,14 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
   parser->m_parseEndByteIndex += len;
   parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
 
-  parser->m_errorCode = parser->m_processor(parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr);
+  parser->m_errorCode = parser->m_processor(
+      parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr);
 
   if (parser->m_errorCode != XML_ERROR_NONE) {
     parser->m_eventEndPtr = parser->m_eventPtr;
     parser->m_processor = errorProcessor;
     return XML_STATUS_ERROR;
-  }
-  else {
+  } else {
     switch (parser->m_parsingStatus.parsing) {
     case XML_SUSPENDED:
       result = XML_STATUS_SUSPENDED;
@@ -1997,18 +1906,18 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
         parser->m_parsingStatus.parsing = XML_FINISHED;
         return result;
       }
-    default: ;  /* should not happen */
+    default:; /* should not happen */
     }
   }
 
-  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                    parser->m_bufferPtr, &parser->m_position);
   parser->m_positionPtr = parser->m_bufferPtr;
   return result;
 }
 
-void * XMLCALL
-XML_GetBuffer(XML_Parser parser, int len)
-{
+void *XMLCALL
+XML_GetBuffer(XML_Parser parser, int len) {
   if (parser == NULL)
     return NULL;
   if (len < 0) {
@@ -2022,17 +1931,17 @@ XML_GetBuffer(XML_Parser parser, int len)
   case XML_FINISHED:
     parser->m_errorCode = XML_ERROR_FINISHED;
     return NULL;
-  default: ;
+  default:;
   }
 
   if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) {
 #ifdef XML_CONTEXT_BYTES
     int keep;
-#endif  /* defined XML_CONTEXT_BYTES */
+#endif /* defined XML_CONTEXT_BYTES */
     /* Do not invoke signed arithmetic overflow: */
-    int neededSize = (int) ((unsigned)len +
-                            (unsigned)EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd,
-                                                          parser->m_bufferPtr));
+    int neededSize = (int)((unsigned)len
+                           + (unsigned)EXPAT_SAFE_PTR_DIFF(
+                               parser->m_bufferEnd, parser->m_bufferPtr));
     if (neededSize < 0) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return NULL;
@@ -2042,13 +1951,18 @@ XML_GetBuffer(XML_Parser parser, int len)
     if (keep > XML_CONTEXT_BYTES)
       keep = XML_CONTEXT_BYTES;
     neededSize += keep;
-#endif  /* defined XML_CONTEXT_BYTES */
-    if (neededSize <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) {
+#endif /* defined XML_CONTEXT_BYTES */
+    if (neededSize
+        <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) {
 #ifdef XML_CONTEXT_BYTES
       if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) {
-          int offset = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) - keep;
-        /* The buffer pointers cannot be NULL here; we have at least some bytes in the buffer */
-        memmove(parser->m_buffer, &parser->m_buffer[offset], parser->m_bufferEnd - parser->m_bufferPtr + keep);
+        int offset
+            = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)
+              - keep;
+        /* The buffer pointers cannot be NULL here; we have at least some bytes
+         * in the buffer */
+        memmove(parser->m_buffer, &parser->m_buffer[offset],
+                parser->m_bufferEnd - parser->m_bufferPtr + keep);
         parser->m_bufferEnd -= offset;
         parser->m_bufferPtr -= offset;
       }
@@ -2056,20 +1970,21 @@ XML_GetBuffer(XML_Parser parser, int len)
       if (parser->m_buffer && parser->m_bufferPtr) {
         memmove(parser->m_buffer, parser->m_bufferPtr,
                 EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr));
-        parser->m_bufferEnd = parser->m_buffer +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
+        parser->m_bufferEnd
+            = parser->m_buffer
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
         parser->m_bufferPtr = parser->m_buffer;
       }
-#endif  /* not defined XML_CONTEXT_BYTES */
-    }
-    else {
+#endif /* not defined XML_CONTEXT_BYTES */
+    } else {
       char *newBuf;
-      int bufferSize = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr);
+      int bufferSize
+          = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr);
       if (bufferSize == 0)
         bufferSize = INIT_BUFFER_SIZE;
       do {
         /* Do not invoke signed arithmetic overflow: */
-        bufferSize = (int) (2U * (unsigned) bufferSize);
+        bufferSize = (int)(2U * (unsigned)bufferSize);
       } while (bufferSize < neededSize && bufferSize > 0);
       if (bufferSize <= 0) {
         parser->m_errorCode = XML_ERROR_NO_MEMORY;
@@ -2083,18 +1998,17 @@ XML_GetBuffer(XML_Parser parser, int len)
       parser->m_bufferLim = newBuf + bufferSize;
 #ifdef XML_CONTEXT_BYTES
       if (parser->m_bufferPtr) {
-        int keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer);
-        if (keep > XML_CONTEXT_BYTES)
-          keep = XML_CONTEXT_BYTES;
         memcpy(newBuf, &parser->m_bufferPtr[-keep],
-               EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) + keep);
+               EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)
+                   + keep);
         FREE(parser, parser->m_buffer);
         parser->m_buffer = newBuf;
-        parser->m_bufferEnd = parser->m_buffer +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) + keep;
+        parser->m_bufferEnd
+            = parser->m_buffer
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)
+              + keep;
         parser->m_bufferPtr = parser->m_buffer + keep;
-      }
-      else {
+      } else {
         /* This must be a brand new buffer with no data in it yet */
         parser->m_bufferEnd = newBuf;
         parser->m_bufferPtr = parser->m_buffer = newBuf;
@@ -2104,15 +2018,15 @@ XML_GetBuffer(XML_Parser parser, int len)
         memcpy(newBuf, parser->m_bufferPtr,
                EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr));
         FREE(parser, parser->m_buffer);
-        parser->m_bufferEnd = newBuf +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
-      }
-      else {
+        parser->m_bufferEnd
+            = newBuf
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
+      } else {
         /* This must be a brand new buffer with no data in it yet */
         parser->m_bufferEnd = newBuf;
       }
       parser->m_bufferPtr = parser->m_buffer = newBuf;
-#endif  /* not defined XML_CONTEXT_BYTES */
+#endif /* not defined XML_CONTEXT_BYTES */
     }
     parser->m_eventPtr = parser->m_eventEndPtr = NULL;
     parser->m_positionPtr = NULL;
@@ -2121,8 +2035,7 @@ XML_GetBuffer(XML_Parser parser, int len)
 }
 
 enum XML_Status XMLCALL
-XML_StopParser(XML_Parser parser, XML_Bool resumable)
-{
+XML_StopParser(XML_Parser parser, XML_Bool resumable) {
   if (parser == NULL)
     return XML_STATUS_ERROR;
   switch (parser->m_parsingStatus.parsing) {
@@ -2145,16 +2058,14 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable)
       }
 #endif
       parser->m_parsingStatus.parsing = XML_SUSPENDED;
-    }
-    else
+    } else
       parser->m_parsingStatus.parsing = XML_FINISHED;
   }
   return XML_STATUS_OK;
 }
 
 enum XML_Status XMLCALL
-XML_ResumeParser(XML_Parser parser)
-{
+XML_ResumeParser(XML_Parser parser) {
   enum XML_Status result = XML_STATUS_OK;
 
   if (parser == NULL)
@@ -2165,14 +2076,14 @@ XML_ResumeParser(XML_Parser parser)
   }
   parser->m_parsingStatus.parsing = XML_PARSING;
 
-  parser->m_errorCode = parser->m_processor(parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
+  parser->m_errorCode = parser->m_processor(
+      parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
 
   if (parser->m_errorCode != XML_ERROR_NONE) {
     parser->m_eventEndPtr = parser->m_eventPtr;
     parser->m_processor = errorProcessor;
     return XML_STATUS_ERROR;
-  }
-  else {
+  } else {
     switch (parser->m_parsingStatus.parsing) {
     case XML_SUSPENDED:
       result = XML_STATUS_SUSPENDED;
@@ -2183,18 +2094,18 @@ XML_ResumeParser(XML_Parser parser)
         parser->m_parsingStatus.parsing = XML_FINISHED;
         return result;
       }
-    default: ;
+    default:;
     }
   }
 
-  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                    parser->m_bufferPtr, &parser->m_position);
   parser->m_positionPtr = parser->m_bufferPtr;
   return result;
 }
 
 void XMLCALL
-XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status)
-{
+XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) {
   if (parser == NULL)
     return;
   assert(status != NULL);
@@ -2202,26 +2113,24 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status)
 }
 
 enum XML_Error XMLCALL
-XML_GetErrorCode(XML_Parser parser)
-{
+XML_GetErrorCode(XML_Parser parser) {
   if (parser == NULL)
     return XML_ERROR_INVALID_ARGUMENT;
   return parser->m_errorCode;
 }
 
 XML_Index XMLCALL
-XML_GetCurrentByteIndex(XML_Parser parser)
-{
+XML_GetCurrentByteIndex(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   if (parser->m_eventPtr)
-    return (XML_Index)(parser->m_parseEndByteIndex - (parser->m_parseEndPtr - parser->m_eventPtr));
+    return (XML_Index)(parser->m_parseEndByteIndex
+                       - (parser->m_parseEndPtr - parser->m_eventPtr));
   return -1;
 }
 
 int XMLCALL
-XML_GetCurrentByteCount(XML_Parser parser)
-{
+XML_GetCurrentByteCount(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventEndPtr && parser->m_eventPtr)
@@ -2229,9 +2138,8 @@ XML_GetCurrentByteCount(XML_Parser parser)
   return 0;
 }
 
-const char * XMLCALL
-XML_GetInputContext(XML_Parser parser, int *offset, int *size)
-{
+const char *XMLCALL
+XML_GetInputContext(XML_Parser parser, int *offset, int *size) {
 #ifdef XML_CONTEXT_BYTES
   if (parser == NULL)
     return NULL;
@@ -2239,7 +2147,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size)
     if (offset != NULL)
       *offset = (int)(parser->m_eventPtr - parser->m_buffer);
     if (size != NULL)
-      *size   = (int)(parser->m_bufferEnd - parser->m_buffer);
+      *size = (int)(parser->m_bufferEnd - parser->m_buffer);
     return parser->m_buffer;
   }
 #else
@@ -2247,82 +2155,76 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size)
   (void)offset;
   (void)size;
 #endif /* defined XML_CONTEXT_BYTES */
-  return (char *) 0;
+  return (char *)0;
 }
 
 XML_Size XMLCALL
-XML_GetCurrentLineNumber(XML_Parser parser)
-{
+XML_GetCurrentLineNumber(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) {
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_eventPtr, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                      parser->m_eventPtr, &parser->m_position);
     parser->m_positionPtr = parser->m_eventPtr;
   }
   return parser->m_position.lineNumber + 1;
 }
 
 XML_Size XMLCALL
-XML_GetCurrentColumnNumber(XML_Parser parser)
-{
+XML_GetCurrentColumnNumber(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) {
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_eventPtr, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                      parser->m_eventPtr, &parser->m_position);
     parser->m_positionPtr = parser->m_eventPtr;
   }
   return parser->m_position.columnNumber;
 }
 
 void XMLCALL
-XML_FreeContentModel(XML_Parser parser, XML_Content *model)
-{
+XML_FreeContentModel(XML_Parser parser, XML_Content *model) {
   if (parser != NULL)
     FREE(parser, model);
 }
 
-void * XMLCALL
-XML_MemMalloc(XML_Parser parser, size_t size)
-{
+void *XMLCALL
+XML_MemMalloc(XML_Parser parser, size_t size) {
   if (parser == NULL)
     return NULL;
   return MALLOC(parser, size);
 }
 
-void * XMLCALL
-XML_MemRealloc(XML_Parser parser, void *ptr, size_t size)
-{
+void *XMLCALL
+XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) {
   if (parser == NULL)
     return NULL;
   return REALLOC(parser, ptr, size);
 }
 
 void XMLCALL
-XML_MemFree(XML_Parser parser, void *ptr)
-{
+XML_MemFree(XML_Parser parser, void *ptr) {
   if (parser != NULL)
     FREE(parser, ptr);
 }
 
 void XMLCALL
-XML_DefaultCurrent(XML_Parser parser)
-{
+XML_DefaultCurrent(XML_Parser parser) {
   if (parser == NULL)
     return;
   if (parser->m_defaultHandler) {
     if (parser->m_openInternalEntities)
-      reportDefault(parser,
-                    parser->m_internalEncoding,
+      reportDefault(parser, parser->m_internalEncoding,
                     parser->m_openInternalEntities->internalEventPtr,
                     parser->m_openInternalEntities->internalEventEndPtr);
     else
-      reportDefault(parser, parser->m_encoding, parser->m_eventPtr, parser->m_eventEndPtr);
+      reportDefault(parser, parser->m_encoding, parser->m_eventPtr,
+                    parser->m_eventEndPtr);
   }
 }
 
-const XML_LChar * XMLCALL
-XML_ErrorString(enum XML_Error code)
-{
+const XML_LChar *XMLCALL
+XML_ErrorString(enum XML_Error code) {
   switch (code) {
   case XML_ERROR_NONE:
     return NULL;
@@ -2404,21 +2306,22 @@ XML_ErrorString(enum XML_Error code)
     return XML_L("cannot suspend in external parameter entity");
   /* Added in 2.0.0. */
   case XML_ERROR_RESERVED_PREFIX_XML:
-    return XML_L("reserved prefix (xml) must not be undeclared or bound to another namespace name");
+    return XML_L(
+        "reserved prefix (xml) must not be undeclared or bound to another namespace name");
   case XML_ERROR_RESERVED_PREFIX_XMLNS:
     return XML_L("reserved prefix (xmlns) must not be declared or undeclared");
   case XML_ERROR_RESERVED_NAMESPACE_URI:
-    return XML_L("prefix must not be bound to one of the reserved namespace names");
+    return XML_L(
+        "prefix must not be bound to one of the reserved namespace names");
   /* Added in 2.2.5. */
-  case XML_ERROR_INVALID_ARGUMENT:  /* Constant added in 2.2.1, already */
+  case XML_ERROR_INVALID_ARGUMENT: /* Constant added in 2.2.1, already */
     return XML_L("invalid argument");
   }
   return NULL;
 }
 
-const XML_LChar * XMLCALL
+const XML_LChar *XMLCALL
 XML_ExpatVersion(void) {
-
   /* V1 is used to string-ize the version number. However, it would
      string-ize the actual version macro *names* unless we get them
      substituted before being passed to V1. CPP is defined to expand
@@ -2427,8 +2330,8 @@ XML_ExpatVersion(void) {
      with the correct numerals. */
   /* ### I'm assuming cpp is portable in this respect... */
 
-#define V1(a,b,c) XML_L(#a)XML_L(".")XML_L(#b)XML_L(".")XML_L(#c)
-#define V2(a,b,c) XML_L("expat_")V1(a,b,c)
+#define V1(a, b, c) XML_L(#a) XML_L(".") XML_L(#b) XML_L(".") XML_L(#c)
+#define V2(a, b, c) XML_L("expat_") V1(a, b, c)
 
   return V2(XML_MAJOR_VERSION, XML_MINOR_VERSION, XML_MICRO_VERSION);
 
@@ -2437,8 +2340,7 @@ XML_ExpatVersion(void) {
 }
 
 XML_Expat_Version XMLCALL
-XML_ExpatVersionInfo(void)
-{
+XML_ExpatVersionInfo(void) {
   XML_Expat_Version version;
 
   version.major = XML_MAJOR_VERSION;
@@ -2448,41 +2350,39 @@ XML_ExpatVersionInfo(void)
   return version;
 }
 
-const XML_Feature * XMLCALL
-XML_GetFeatureList(void)
-{
-  static const XML_Feature features[] = {
-    {XML_FEATURE_SIZEOF_XML_CHAR,  XML_L("sizeof(XML_Char)"),
-     sizeof(XML_Char)},
-    {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"),
-     sizeof(XML_LChar)},
+const XML_Feature *XMLCALL
+XML_GetFeatureList(void) {
+  static const XML_Feature features[]
+      = {{XML_FEATURE_SIZEOF_XML_CHAR, XML_L("sizeof(XML_Char)"),
+          sizeof(XML_Char)},
+         {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"),
+          sizeof(XML_LChar)},
 #ifdef XML_UNICODE
-    {XML_FEATURE_UNICODE,          XML_L("XML_UNICODE"), 0},
+         {XML_FEATURE_UNICODE, XML_L("XML_UNICODE"), 0},
 #endif
 #ifdef XML_UNICODE_WCHAR_T
-    {XML_FEATURE_UNICODE_WCHAR_T,  XML_L("XML_UNICODE_WCHAR_T"), 0},
+         {XML_FEATURE_UNICODE_WCHAR_T, XML_L("XML_UNICODE_WCHAR_T"), 0},
 #endif
 #ifdef XML_DTD
-    {XML_FEATURE_DTD,              XML_L("XML_DTD"), 0},
+         {XML_FEATURE_DTD, XML_L("XML_DTD"), 0},
 #endif
 #ifdef XML_CONTEXT_BYTES
-    {XML_FEATURE_CONTEXT_BYTES,    XML_L("XML_CONTEXT_BYTES"),
-     XML_CONTEXT_BYTES},
+         {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"),
+          XML_CONTEXT_BYTES},
 #endif
 #ifdef XML_MIN_SIZE
-    {XML_FEATURE_MIN_SIZE,         XML_L("XML_MIN_SIZE"), 0},
+         {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0},
 #endif
 #ifdef XML_NS
-    {XML_FEATURE_NS,               XML_L("XML_NS"), 0},
+         {XML_FEATURE_NS, XML_L("XML_NS"), 0},
 #endif
 #ifdef XML_LARGE_SIZE
-    {XML_FEATURE_LARGE_SIZE,       XML_L("XML_LARGE_SIZE"), 0},
+         {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0},
 #endif
 #ifdef XML_ATTR_INFO
-    {XML_FEATURE_ATTR_INFO,        XML_L("XML_ATTR_INFO"), 0},
+         {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0},
 #endif
-    {XML_FEATURE_END,              NULL, 0}
-  };
+         {XML_FEATURE_END, NULL, 0}};
 
   return features;
 }
@@ -2493,8 +2393,7 @@ XML_GetFeatureList(void)
    permanent location, since the parse buffer is about to be discarded.
 */
 static XML_Bool
-storeRawNames(XML_Parser parser)
-{
+storeRawNames(XML_Parser parser) {
   TAG *tag = parser->m_tagStack;
   while (tag) {
     int bufSize;
@@ -2524,8 +2423,8 @@ storeRawNames(XML_Parser parser)
          then update it as well, since it will always point into tag->buf
       */
       if (tag->name.localPart)
-        tag->name.localPart = (XML_Char *)temp + (tag->name.localPart -
-                                                  (XML_Char *)tag->buf);
+        tag->name.localPart
+            = (XML_Char *)temp + (tag->name.localPart - (XML_Char *)tag->buf);
       tag->buf = temp;
       tag->bufEnd = temp + bufSize;
       rawNameBuf = temp + nameLen;
@@ -2538,26 +2437,21 @@ storeRawNames(XML_Parser parser)
 }
 
 static enum XML_Error PTRCALL
-contentProcessor(XML_Parser parser,
-                 const char *start,
-                 const char *end,
-                 const char **endPtr)
-{
-  enum XML_Error result = doContent(parser, 0, parser->m_encoding, start, end,
-                                    endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+contentProcessor(XML_Parser parser, const char *start, const char *end,
+                 const char **endPtr) {
+  enum XML_Error result
+      = doContent(parser, 0, parser->m_encoding, start, end, endPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result == XML_ERROR_NONE) {
-    if (!storeRawNames(parser))
+    if (! storeRawNames(parser))
       return XML_ERROR_NO_MEMORY;
   }
   return result;
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor(XML_Parser parser,
-                            const char *start,
-                            const char *end,
-                            const char **endPtr)
-{
+externalEntityInitProcessor(XML_Parser parser, const char *start,
+                            const char *end, const char **endPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -2566,11 +2460,8 @@ externalEntityInitProcessor(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor2(XML_Parser parser,
-                             const char *start,
-                             const char *end,
-                             const char **endPtr)
-{
+externalEntityInitProcessor2(XML_Parser parser, const char *start,
+                             const char *end, const char **endPtr) {
   const char *next = start; /* XmlContentTok doesn't always set the last arg */
   int tok = XmlContentTok(parser->m_encoding, start, end, &next);
   switch (tok) {
@@ -2580,21 +2471,21 @@ externalEntityInitProcessor2(XML_Parser parser,
        doContent (by detecting XML_TOK_NONE) without processing any xml text
        declaration - causing the error XML_ERROR_MISPLACED_XML_PI in doContent.
     */
-    if (next == end && !parser->m_parsingStatus.finalBuffer) {
+    if (next == end && ! parser->m_parsingStatus.finalBuffer) {
       *endPtr = next;
       return XML_ERROR_NONE;
     }
     start = next;
     break;
   case XML_TOK_PARTIAL:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
     parser->m_eventPtr = start;
     return XML_ERROR_UNCLOSED_TOKEN;
   case XML_TOK_PARTIAL_CHAR:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
@@ -2606,11 +2497,8 @@ externalEntityInitProcessor2(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor3(XML_Parser parser,
-                             const char *start,
-                             const char *end,
-                             const char **endPtr)
-{
+externalEntityInitProcessor3(XML_Parser parser, const char *start,
+                             const char *end, const char **endPtr) {
   int tok;
   const char *next = start; /* XmlContentTok doesn't always set the last arg */
   parser->m_eventPtr = start;
@@ -2618,31 +2506,29 @@ externalEntityInitProcessor3(XML_Parser parser,
   parser->m_eventEndPtr = next;
 
   switch (tok) {
-  case XML_TOK_XML_DECL:
-    {
-      enum XML_Error result;
-      result = processXmlDecl(parser, 1, start, next);
-      if (result != XML_ERROR_NONE)
-        return result;
-      switch (parser->m_parsingStatus.parsing) {
-      case XML_SUSPENDED:
-        *endPtr = next;
-        return XML_ERROR_NONE;
-      case XML_FINISHED:
-        return XML_ERROR_ABORTED;
-      default:
-        start = next;
-      }
+  case XML_TOK_XML_DECL: {
+    enum XML_Error result;
+    result = processXmlDecl(parser, 1, start, next);
+    if (result != XML_ERROR_NONE)
+      return result;
+    switch (parser->m_parsingStatus.parsing) {
+    case XML_SUSPENDED:
+      *endPtr = next;
+      return XML_ERROR_NONE;
+    case XML_FINISHED:
+      return XML_ERROR_ABORTED;
+    default:
+      start = next;
     }
-    break;
+  } break;
   case XML_TOK_PARTIAL:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
     return XML_ERROR_UNCLOSED_TOKEN;
   case XML_TOK_PARTIAL_CHAR:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
@@ -2654,39 +2540,31 @@ externalEntityInitProcessor3(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityContentProcessor(XML_Parser parser,
-                               const char *start,
-                               const char *end,
-                               const char **endPtr)
-{
-  enum XML_Error result = doContent(parser, 1, parser->m_encoding, start, end,
-                                    endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+externalEntityContentProcessor(XML_Parser parser, const char *start,
+                               const char *end, const char **endPtr) {
+  enum XML_Error result
+      = doContent(parser, 1, parser->m_encoding, start, end, endPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result == XML_ERROR_NONE) {
-    if (!storeRawNames(parser))
+    if (! storeRawNames(parser))
       return XML_ERROR_NO_MEMORY;
   }
   return result;
 }
 
 static enum XML_Error
-doContent(XML_Parser parser,
-          int startTagLevel,
-          const ENCODING *enc,
-          const char *s,
-          const char *end,
-          const char **nextPtr,
-          XML_Bool haveMore)
-{
+doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
+          const char *s, const char *end, const char **nextPtr,
+          XML_Bool haveMore) {
   /* save one level of indirection */
-  DTD * const dtd = parser->m_dtd;
+  DTD *const dtd = parser->m_dtd;
 
   const char **eventPP;
   const char **eventEndPP;
   if (enc == parser->m_encoding) {
     eventPP = &parser->m_eventPtr;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -2706,8 +2584,7 @@ doContent(XML_Parser parser,
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, end);
       /* We are at the end of the final buffer, should we check for
          XML_SUSPENDED, XML_FINISHED?
@@ -2745,185 +2622,178 @@ doContent(XML_Parser parser,
         return XML_ERROR_NONE;
       }
       return XML_ERROR_PARTIAL_CHAR;
-    case XML_TOK_ENTITY_REF:
-      {
-        const XML_Char *name;
-        ENTITY *entity;
-        XML_Char ch = (XML_Char) XmlPredefinedEntityName(enc,
-                                              s + enc->minBytesPerChar,
-                                              next - enc->minBytesPerChar);
-        if (ch) {
-          if (parser->m_characterDataHandler)
-            parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1);
-          else if (parser->m_defaultHandler)
-            reportDefault(parser, enc, s, next);
-          break;
-        }
-        name = poolStoreString(&dtd->pool, enc,
-                                s + enc->minBytesPerChar,
-                                next - enc->minBytesPerChar);
-        if (!name)
-          return XML_ERROR_NO_MEMORY;
-        entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
-        poolDiscard(&dtd->pool);
-        /* First, determine if a check for an existing declaration is needed;
-           if yes, check that the entity exists, and that it is internal,
-           otherwise call the skipped entity or default handler.
-        */
-        if (!dtd->hasParamEntityRefs || dtd->standalone) {
-          if (!entity)
-            return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal)
-            return XML_ERROR_ENTITY_DECLARED_IN_PE;
-        }
-        else if (!entity) {
+    case XML_TOK_ENTITY_REF: {
+      const XML_Char *name;
+      ENTITY *entity;
+      XML_Char ch = (XML_Char)XmlPredefinedEntityName(
+          enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar);
+      if (ch) {
+        if (parser->m_characterDataHandler)
+          parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1);
+        else if (parser->m_defaultHandler)
+          reportDefault(parser, enc, s, next);
+        break;
+      }
+      name = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                             next - enc->minBytesPerChar);
+      if (! name)
+        return XML_ERROR_NO_MEMORY;
+      entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
+      poolDiscard(&dtd->pool);
+      /* First, determine if a check for an existing declaration is needed;
+         if yes, check that the entity exists, and that it is internal,
+         otherwise call the skipped entity or default handler.
+      */
+      if (! dtd->hasParamEntityRefs || dtd->standalone) {
+        if (! entity)
+          return XML_ERROR_UNDEFINED_ENTITY;
+        else if (! entity->is_internal)
+          return XML_ERROR_ENTITY_DECLARED_IN_PE;
+      } else if (! entity) {
+        if (parser->m_skippedEntityHandler)
+          parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+        else if (parser->m_defaultHandler)
+          reportDefault(parser, enc, s, next);
+        break;
+      }
+      if (entity->open)
+        return XML_ERROR_RECURSIVE_ENTITY_REF;
+      if (entity->notation)
+        return XML_ERROR_BINARY_ENTITY_REF;
+      if (entity->textPtr) {
+        enum XML_Error result;
+        if (! parser->m_defaultExpandInternalEntities) {
           if (parser->m_skippedEntityHandler)
-            parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+            parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name,
+                                           0);
           else if (parser->m_defaultHandler)
             reportDefault(parser, enc, s, next);
           break;
         }
-        if (entity->open)
-          return XML_ERROR_RECURSIVE_ENTITY_REF;
-        if (entity->notation)
-          return XML_ERROR_BINARY_ENTITY_REF;
-        if (entity->textPtr) {
-          enum XML_Error result;
-          if (!parser->m_defaultExpandInternalEntities) {
-            if (parser->m_skippedEntityHandler)
-              parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name, 0);
-            else if (parser->m_defaultHandler)
-              reportDefault(parser, enc, s, next);
-            break;
-          }
-          result = processInternalEntity(parser, entity, XML_FALSE);
-          if (result != XML_ERROR_NONE)
-            return result;
-        }
-        else if (parser->m_externalEntityRefHandler) {
-          const XML_Char *context;
-          entity->open = XML_TRUE;
-          context = getContext(parser);
-          entity->open = XML_FALSE;
-          if (!context)
-            return XML_ERROR_NO_MEMORY;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        context,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
-            return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
-          poolDiscard(&parser->m_tempPool);
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        break;
-      }
+        result = processInternalEntity(parser, entity, XML_FALSE);
+        if (result != XML_ERROR_NONE)
+          return result;
+      } else if (parser->m_externalEntityRefHandler) {
+        const XML_Char *context;
+        entity->open = XML_TRUE;
+        context = getContext(parser);
+        entity->open = XML_FALSE;
+        if (! context)
+          return XML_ERROR_NO_MEMORY;
+        if (! parser->m_externalEntityRefHandler(
+                parser->m_externalEntityRefHandlerArg, context, entity->base,
+                entity->systemId, entity->publicId))
+          return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
+        poolDiscard(&parser->m_tempPool);
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      break;
+    }
     case XML_TOK_START_TAG_NO_ATTS:
       /* fall through */
-    case XML_TOK_START_TAG_WITH_ATTS:
-      {
-        TAG *tag;
-        enum XML_Error result;
-        XML_Char *toPtr;
-        if (parser->m_freeTagList) {
-          tag = parser->m_freeTagList;
-          parser->m_freeTagList = parser->m_freeTagList->parent;
+    case XML_TOK_START_TAG_WITH_ATTS: {
+      TAG *tag;
+      enum XML_Error result;
+      XML_Char *toPtr;
+      if (parser->m_freeTagList) {
+        tag = parser->m_freeTagList;
+        parser->m_freeTagList = parser->m_freeTagList->parent;
+      } else {
+        tag = (TAG *)MALLOC(parser, sizeof(TAG));
+        if (! tag)
+          return XML_ERROR_NO_MEMORY;
+        tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE);
+        if (! tag->buf) {
+          FREE(parser, tag);
+          return XML_ERROR_NO_MEMORY;
         }
-        else {
-          tag = (TAG *)MALLOC(parser, sizeof(TAG));
-          if (!tag)
-            return XML_ERROR_NO_MEMORY;
-          tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE);
-          if (!tag->buf) {
-            FREE(parser, tag);
-            return XML_ERROR_NO_MEMORY;
+        tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE;
+      }
+      tag->bindings = NULL;
+      tag->parent = parser->m_tagStack;
+      parser->m_tagStack = tag;
+      tag->name.localPart = NULL;
+      tag->name.prefix = NULL;
+      tag->rawName = s + enc->minBytesPerChar;
+      tag->rawNameLength = XmlNameLength(enc, tag->rawName);
+      ++parser->m_tagLevel;
+      {
+        const char *rawNameEnd = tag->rawName + tag->rawNameLength;
+        const char *fromPtr = tag->rawName;
+        toPtr = (XML_Char *)tag->buf;
+        for (;;) {
+          int bufSize;
+          int convLen;
+          const enum XML_Convert_Result convert_res
+              = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr,
+                           (ICHAR *)tag->bufEnd - 1);
+          convLen = (int)(toPtr - (XML_Char *)tag->buf);
+          if ((fromPtr >= rawNameEnd)
+              || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) {
+            tag->name.strLen = convLen;
+            break;
           }
-          tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE;
-        }
-        tag->bindings = NULL;
-        tag->parent = parser->m_tagStack;
-        parser->m_tagStack = tag;
-        tag->name.localPart = NULL;
-        tag->name.prefix = NULL;
-        tag->rawName = s + enc->minBytesPerChar;
-        tag->rawNameLength = XmlNameLength(enc, tag->rawName);
-        ++parser->m_tagLevel;
-        {
-          const char *rawNameEnd = tag->rawName + tag->rawNameLength;
-          const char *fromPtr = tag->rawName;
-          toPtr = (XML_Char *)tag->buf;
-          for (;;) {
-            int bufSize;
-            int convLen;
-            const enum XML_Convert_Result convert_res = XmlConvert(enc,
-                       &fromPtr, rawNameEnd,
-                       (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1);
-            convLen = (int)(toPtr - (XML_Char *)tag->buf);
-            if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) {
-              tag->name.strLen = convLen;
-              break;
-            }
-            bufSize = (int)(tag->bufEnd - tag->buf) << 1;
-            {
-              char *temp = (char *)REALLOC(parser, tag->buf, bufSize);
-              if (temp == NULL)
-                return XML_ERROR_NO_MEMORY;
-              tag->buf = temp;
-              tag->bufEnd = temp + bufSize;
-              toPtr = (XML_Char *)temp + convLen;
-            }
+          bufSize = (int)(tag->bufEnd - tag->buf) << 1;
+          {
+            char *temp = (char *)REALLOC(parser, tag->buf, bufSize);
+            if (temp == NULL)
+              return XML_ERROR_NO_MEMORY;
+            tag->buf = temp;
+            tag->bufEnd = temp + bufSize;
+            toPtr = (XML_Char *)temp + convLen;
           }
         }
-        tag->name.str = (XML_Char *)tag->buf;
-        *toPtr = XML_T('\0');
-        result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings));
-        if (result)
-          return result;
-        if (parser->m_startElementHandler)
-          parser->m_startElementHandler(parser->m_handlerArg, tag->name.str,
-                              (const XML_Char **)parser->m_atts);
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        poolClear(&parser->m_tempPool);
-        break;
       }
+      tag->name.str = (XML_Char *)tag->buf;
+      *toPtr = XML_T('\0');
+      result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings));
+      if (result)
+        return result;
+      if (parser->m_startElementHandler)
+        parser->m_startElementHandler(parser->m_handlerArg, tag->name.str,
+                                      (const XML_Char **)parser->m_atts);
+      else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      poolClear(&parser->m_tempPool);
+      break;
+    }
     case XML_TOK_EMPTY_ELEMENT_NO_ATTS:
       /* fall through */
-    case XML_TOK_EMPTY_ELEMENT_WITH_ATTS:
-      {
-        const char *rawName = s + enc->minBytesPerChar;
-        enum XML_Error result;
-        BINDING *bindings = NULL;
-        XML_Bool noElmHandlers = XML_TRUE;
-        TAG_NAME name;
-        name.str = poolStoreString(&parser->m_tempPool, enc, rawName,
-                                   rawName + XmlNameLength(enc, rawName));
-        if (!name.str)
-          return XML_ERROR_NO_MEMORY;
-        poolFinish(&parser->m_tempPool);
-        result = storeAtts(parser, enc, s, &name, &bindings);
-        if (result != XML_ERROR_NONE) {
-          freeBindings(parser, bindings);
-          return result;
-        }
-        poolFinish(&parser->m_tempPool);
-        if (parser->m_startElementHandler) {
-          parser->m_startElementHandler(parser->m_handlerArg, name.str, (const XML_Char **)parser->m_atts);
-          noElmHandlers = XML_FALSE;
-        }
-        if (parser->m_endElementHandler) {
-          if (parser->m_startElementHandler)
-            *eventPP = *eventEndPP;
-          parser->m_endElementHandler(parser->m_handlerArg, name.str);
-          noElmHandlers = XML_FALSE;
-        }
-        if (noElmHandlers && parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        poolClear(&parser->m_tempPool);
+    case XML_TOK_EMPTY_ELEMENT_WITH_ATTS: {
+      const char *rawName = s + enc->minBytesPerChar;
+      enum XML_Error result;
+      BINDING *bindings = NULL;
+      XML_Bool noElmHandlers = XML_TRUE;
+      TAG_NAME name;
+      name.str = poolStoreString(&parser->m_tempPool, enc, rawName,
+                                 rawName + XmlNameLength(enc, rawName));
+      if (! name.str)
+        return XML_ERROR_NO_MEMORY;
+      poolFinish(&parser->m_tempPool);
+      result = storeAtts(parser, enc, s, &name, &bindings);
+      if (result != XML_ERROR_NONE) {
         freeBindings(parser, bindings);
+        return result;
+      }
+      poolFinish(&parser->m_tempPool);
+      if (parser->m_startElementHandler) {
+        parser->m_startElementHandler(parser->m_handlerArg, name.str,
+                                      (const XML_Char **)parser->m_atts);
+        noElmHandlers = XML_FALSE;
+      }
+      if (parser->m_endElementHandler) {
+        if (parser->m_startElementHandler)
+          *eventPP = *eventEndPP;
+        parser->m_endElementHandler(parser->m_handlerArg, name.str);
+        noElmHandlers = XML_FALSE;
       }
-      if ((parser->m_tagLevel == 0) && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
+      if (noElmHandlers && parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      poolClear(&parser->m_tempPool);
+      freeBindings(parser, bindings);
+    }
+      if ((parser->m_tagLevel == 0)
+          && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
         if (parser->m_parsingStatus.parsing == XML_SUSPENDED)
           parser->m_processor = epilogProcessor;
         else
@@ -2940,7 +2810,7 @@ doContent(XML_Parser parser,
         parser->m_tagStack = tag->parent;
         tag->parent = parser->m_freeTagList;
         parser->m_freeTagList = tag;
-        rawName = s + enc->minBytesPerChar*2;
+        rawName = s + enc->minBytesPerChar * 2;
         len = XmlNameLength(enc, rawName);
         if (len != tag->rawNameLength
             || memcmp(tag->rawName, rawName, len) != 0) {
@@ -2960,86 +2830,89 @@ doContent(XML_Parser parser,
             */
             uri = (XML_Char *)tag->name.str + tag->name.uriLen;
             /* don't need to check for space - already done in storeAtts() */
-            while (*localPart) *uri++ = *localPart++;
+            while (*localPart)
+              *uri++ = *localPart++;
             prefix = (XML_Char *)tag->name.prefix;
             if (parser->m_ns_triplets && prefix) {
               *uri++ = parser->m_namespaceSeparator;
-              while (*prefix) *uri++ = *prefix++;
-             }
+              while (*prefix)
+                *uri++ = *prefix++;
+            }
             *uri = XML_T('\0');
           }
           parser->m_endElementHandler(parser->m_handlerArg, tag->name.str);
-        }
-        else if (parser->m_defaultHandler)
+        } else if (parser->m_defaultHandler)
           reportDefault(parser, enc, s, next);
         while (tag->bindings) {
           BINDING *b = tag->bindings;
           if (parser->m_endNamespaceDeclHandler)
-            parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
+            parser->m_endNamespaceDeclHandler(parser->m_handlerArg,
+                                              b->prefix->name);
           tag->bindings = tag->bindings->nextTagBinding;
           b->nextTagBinding = parser->m_freeBindingList;
           parser->m_freeBindingList = b;
           b->prefix->binding = b->prevPrefixBinding;
         }
-        if (parser->m_tagLevel == 0)
-          return epilogProcessor(parser, next, end, nextPtr);
-      }
-      break;
-    case XML_TOK_CHAR_REF:
-      {
-        int n = XmlCharRefNumber(enc, s);
-        if (n < 0)
-          return XML_ERROR_BAD_CHAR_REF;
-        if (parser->m_characterDataHandler) {
-          XML_Char buf[XML_ENCODE_MAX];
-          parser->m_characterDataHandler(parser->m_handlerArg, buf, XmlEncode(n, (ICHAR *)buf));
+        if ((parser->m_tagLevel == 0)
+            && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
+          if (parser->m_parsingStatus.parsing == XML_SUSPENDED)
+            parser->m_processor = epilogProcessor;
+          else
+            return epilogProcessor(parser, next, end, nextPtr);
         }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
       }
       break;
+    case XML_TOK_CHAR_REF: {
+      int n = XmlCharRefNumber(enc, s);
+      if (n < 0)
+        return XML_ERROR_BAD_CHAR_REF;
+      if (parser->m_characterDataHandler) {
+        XML_Char buf[XML_ENCODE_MAX];
+        parser->m_characterDataHandler(parser->m_handlerArg, buf,
+                                       XmlEncode(n, (ICHAR *)buf));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_XML_DECL:
       return XML_ERROR_MISPLACED_XML_PI;
     case XML_TOK_DATA_NEWLINE:
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       break;
-    case XML_TOK_CDATA_SECT_OPEN:
-      {
-        enum XML_Error result;
-        if (parser->m_startCdataSectionHandler)
-          parser->m_startCdataSectionHandler(parser->m_handlerArg);
-/* BEGIN disabled code */
-        /* Suppose you doing a transformation on a document that involves
-           changing only the character data.  You set up a defaultHandler
-           and a characterDataHandler.  The defaultHandler simply copies
-           characters through.  The characterDataHandler does the
-           transformation and writes the characters out escaping them as
-           necessary.  This case will fail to work if we leave out the
-           following two lines (because & and < inside CDATA sections will
-           be incorrectly escaped).
-
-           However, now we have a start/endCdataSectionHandler, so it seems
-           easier to let the user deal with this.
-        */
-        else if (0 && parser->m_characterDataHandler)
-          parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0);
-/* END disabled code */
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore);
-        if (result != XML_ERROR_NONE)
-          return result;
-        else if (!next) {
-          parser->m_processor = cdataSectionProcessor;
-          return result;
-        }
+    case XML_TOK_CDATA_SECT_OPEN: {
+      enum XML_Error result;
+      if (parser->m_startCdataSectionHandler)
+        parser->m_startCdataSectionHandler(parser->m_handlerArg);
+      /* BEGIN disabled code */
+      /* Suppose you doing a transformation on a document that involves
+         changing only the character data.  You set up a defaultHandler
+         and a characterDataHandler.  The defaultHandler simply copies
+         characters through.  The characterDataHandler does the
+         transformation and writes the characters out escaping them as
+         necessary.  This case will fail to work if we leave out the
+         following two lines (because & and < inside CDATA sections will
+         be incorrectly escaped).
+
+         However, now we have a start/endCdataSectionHandler, so it seems
+         easier to let the user deal with this.
+      */
+      else if (0 && parser->m_characterDataHandler)
+        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                                       0);
+      /* END disabled code */
+      else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore);
+      if (result != XML_ERROR_NONE)
+        return result;
+      else if (! next) {
+        parser->m_processor = cdataSectionProcessor;
+        return result;
       }
-      break;
+    } break;
     case XML_TOK_TRAILING_RSQB:
       if (haveMore) {
         *nextPtr = s;
@@ -3049,15 +2922,14 @@ doContent(XML_Parser parser,
         if (MUST_CONVERT(enc, s)) {
           ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
           XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-          parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                               (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-        }
-        else
-          parser->m_characterDataHandler(parser->m_handlerArg,
-                               (XML_Char *)s,
-                               (int)((XML_Char *)end - (XML_Char *)s));
-      }
-      else if (parser->m_defaultHandler)
+          parser->m_characterDataHandler(
+              parser->m_handlerArg, parser->m_dataBuf,
+              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+        } else
+          parser->m_characterDataHandler(
+              parser->m_handlerArg, (XML_Char *)s,
+              (int)((XML_Char *)end - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, end);
       /* We are at the end of the final buffer, should we check for
          XML_SUSPENDED, XML_FINISHED?
@@ -3072,37 +2944,34 @@ doContent(XML_Parser parser,
       }
       *nextPtr = end;
       return XML_ERROR_NONE;
-    case XML_TOK_DATA_CHARS:
-      {
-        XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
-        if (charDataHandler) {
-          if (MUST_CONVERT(enc, s)) {
-            for (;;) {
-              ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-              const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-              *eventEndPP = s;
-              charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-              if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
-                break;
-              *eventPP = s;
-            }
+    case XML_TOK_DATA_CHARS: {
+      XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
+      if (charDataHandler) {
+        if (MUST_CONVERT(enc, s)) {
+          for (;;) {
+            ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
+            const enum XML_Convert_Result convert_res = XmlConvert(
+                enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+            *eventEndPP = s;
+            charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                            (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+            if ((convert_res == XML_CONVERT_COMPLETED)
+                || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+              break;
+            *eventPP = s;
           }
-          else
-            charDataHandler(parser->m_handlerArg,
-                            (XML_Char *)s,
-                            (int)((XML_Char *)next - (XML_Char *)s));
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-      }
-      break;
+        } else
+          charDataHandler(parser->m_handlerArg, (XML_Char *)s,
+                          (int)((XML_Char *)next - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_PI:
-      if (!reportProcessingInstruction(parser, enc, s, next))
+      if (! reportProcessingInstruction(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_COMMENT:
-      if (!reportComment(parser, enc, s, next))
+      if (! reportComment(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     default:
@@ -3125,7 +2994,7 @@ doContent(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
   /* not reached */
@@ -3136,8 +3005,7 @@ doContent(XML_Parser parser,
  * reused as appropriate.
  */
 static void
-freeBindings(XML_Parser parser, BINDING *bindings)
-{
+freeBindings(XML_Parser parser, BINDING *bindings) {
   while (bindings) {
     BINDING *b = bindings;
 
@@ -3145,7 +3013,7 @@ freeBindings(XML_Parser parser, BINDING *bindings)
      * binding in addBindings(), so call the end handler now.
      */
     if (parser->m_endNamespaceDeclHandler)
-        parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
+      parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
 
     bindings = bindings->nextTagBinding;
     b->nextTagBinding = parser->m_freeBindingList;
@@ -3165,14 +3033,12 @@ freeBindings(XML_Parser parser, BINDING *bindings)
    - generate namespace aware element name (URI, prefix)
 */
 static enum XML_Error
-storeAtts(XML_Parser parser, const ENCODING *enc,
-          const char *attStr, TAG_NAME *tagNamePtr,
-          BINDING **bindingsPtr)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
+          TAG_NAME *tagNamePtr, BINDING **bindingsPtr) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   ELEMENT_TYPE *elementType;
   int nDefaultAtts;
-  const XML_Char **appAtts;   /* the attribute list for the application */
+  const XML_Char **appAtts; /* the attribute list for the application */
   int attIndex = 0;
   int prefixLen;
   int i;
@@ -3183,16 +3049,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   const XML_Char *localPart;
 
   /* lookup the element type name */
-  elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str,0);
-  if (!elementType) {
+  elementType
+      = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str, 0);
+  if (! elementType) {
     const XML_Char *name = poolCopyString(&dtd->pool, tagNamePtr->str);
-    if (!name)
+    if (! name)
       return XML_ERROR_NO_MEMORY;
     elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, name,
                                          sizeof(ELEMENT_TYPE));
-    if (!elementType)
+    if (! elementType)
       return XML_ERROR_NO_MEMORY;
-    if (parser->m_ns && !setElementTypePrefix(parser, elementType))
+    if (parser->m_ns && ! setElementTypePrefix(parser, elementType))
       return XML_ERROR_NO_MEMORY;
   }
   nDefaultAtts = elementType->nDefaultAtts;
@@ -3206,14 +3073,16 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     XML_AttrInfo *temp2;
 #endif
     parser->m_attsSize = n + nDefaultAtts + INIT_ATTS_SIZE;
-    temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts, parser->m_attsSize * sizeof(ATTRIBUTE));
+    temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts,
+                                parser->m_attsSize * sizeof(ATTRIBUTE));
     if (temp == NULL) {
       parser->m_attsSize = oldAttsSize;
       return XML_ERROR_NO_MEMORY;
     }
     parser->m_atts = temp;
 #ifdef XML_ATTR_INFO
-    temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo, parser->m_attsSize * sizeof(XML_AttrInfo));
+    temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo,
+                                    parser->m_attsSize * sizeof(XML_AttrInfo));
     if (temp2 == NULL) {
       parser->m_attsSize = oldAttsSize;
       return XML_ERROR_NO_MEMORY;
@@ -3231,18 +3100,20 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     XML_AttrInfo *currAttInfo = &parser->m_attInfo[i];
 #endif
     /* add the name and value to the attribute list */
-    ATTRIBUTE_ID *attId = getAttributeId(parser, enc, currAtt->name,
-                                         currAtt->name
-                                         + XmlNameLength(enc, currAtt->name));
-    if (!attId)
+    ATTRIBUTE_ID *attId
+        = getAttributeId(parser, enc, currAtt->name,
+                         currAtt->name + XmlNameLength(enc, currAtt->name));
+    if (! attId)
       return XML_ERROR_NO_MEMORY;
 #ifdef XML_ATTR_INFO
-    currAttInfo->nameStart = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name);
-    currAttInfo->nameEnd = currAttInfo->nameStart +
-                           XmlNameLength(enc, currAtt->name);
-    currAttInfo->valueStart = parser->m_parseEndByteIndex -
-                            (parser->m_parseEndPtr - currAtt->valuePtr);
-    currAttInfo->valueEnd = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->valueEnd);
+    currAttInfo->nameStart
+        = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name);
+    currAttInfo->nameEnd
+        = currAttInfo->nameStart + XmlNameLength(enc, currAtt->name);
+    currAttInfo->valueStart = parser->m_parseEndByteIndex
+                              - (parser->m_parseEndPtr - currAtt->valuePtr);
+    currAttInfo->valueEnd = parser->m_parseEndByteIndex
+                            - (parser->m_parseEndPtr - currAtt->valueEnd);
 #endif
     /* Detect duplicate attributes by their QNames. This does not work when
        namespace processing is turned on and different prefixes for the same
@@ -3255,7 +3126,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     }
     (attId->name)[-1] = 1;
     appAtts[attIndex++] = attId->name;
-    if (!parser->m_atts[i].normalized) {
+    if (! parser->m_atts[i].normalized) {
       enum XML_Error result;
       XML_Bool isCdata = XML_TRUE;
 
@@ -3271,17 +3142,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
       }
 
       /* normalize the attribute value */
-      result = storeAttributeValue(parser, enc, isCdata,
-                                   parser->m_atts[i].valuePtr, parser->m_atts[i].valueEnd,
-                                   &parser->m_tempPool);
+      result = storeAttributeValue(
+          parser, enc, isCdata, parser->m_atts[i].valuePtr,
+          parser->m_atts[i].valueEnd, &parser->m_tempPool);
       if (result)
         return result;
       appAtts[attIndex] = poolStart(&parser->m_tempPool);
       poolFinish(&parser->m_tempPool);
-    }
-    else {
+    } else {
       /* the value did not need normalizing */
-      appAtts[attIndex] = poolStoreString(&parser->m_tempPool, enc, parser->m_atts[i].valuePtr,
+      appAtts[attIndex] = poolStoreString(&parser->m_tempPool, enc,
+                                          parser->m_atts[i].valuePtr,
                                           parser->m_atts[i].valueEnd);
       if (appAtts[attIndex] == 0)
         return XML_ERROR_NO_MEMORY;
@@ -3296,15 +3167,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         if (result)
           return result;
         --attIndex;
-      }
-      else {
+      } else {
         /* deal with other prefixed names later */
         attIndex++;
         nPrefixes++;
         (attId->name)[-1] = 2;
       }
-    }
-    else
+    } else
       attIndex++;
   }
 
@@ -3316,29 +3185,26 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         parser->m_idAttIndex = i;
         break;
       }
-  }
-  else
+  } else
     parser->m_idAttIndex = -1;
 
   /* do attribute defaulting */
   for (i = 0; i < nDefaultAtts; i++) {
     const DEFAULT_ATTRIBUTE *da = elementType->defaultAtts + i;
-    if (!(da->id->name)[-1] && da->value) {
+    if (! (da->id->name)[-1] && da->value) {
       if (da->id->prefix) {
         if (da->id->xmlns) {
           enum XML_Error result = addBinding(parser, da->id->prefix, da->id,
                                              da->value, bindingsPtr);
           if (result)
             return result;
-        }
-        else {
+        } else {
           (da->id->name)[-1] = 2;
           nPrefixes++;
           appAtts[attIndex++] = da->id->name;
           appAtts[attIndex++] = da->value;
         }
-      }
-      else {
+      } else {
         (da->id->name)[-1] = 1;
         appAtts[attIndex++] = da->id->name;
         appAtts[attIndex++] = da->value;
@@ -3351,31 +3217,34 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
      and clear flags that say whether attributes were specified */
   i = 0;
   if (nPrefixes) {
-    int j;  /* hash table index */
+    int j; /* hash table index */
     unsigned long version = parser->m_nsAttsVersion;
     int nsAttsSize = (int)1 << parser->m_nsAttsPower;
     unsigned char oldNsAttsPower = parser->m_nsAttsPower;
     /* size of hash table must be at least 2 * (# of prefixed attributes) */
-    if ((nPrefixes << 1) >> parser->m_nsAttsPower) {  /* true for m_nsAttsPower = 0 */
+    if ((nPrefixes << 1)
+        >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */
       NS_ATT *temp;
       /* hash table size must also be a power of 2 and >= 8 */
-      while (nPrefixes >> parser->m_nsAttsPower++);
+      while (nPrefixes >> parser->m_nsAttsPower++)
+        ;
       if (parser->m_nsAttsPower < 3)
         parser->m_nsAttsPower = 3;
       nsAttsSize = (int)1 << parser->m_nsAttsPower;
-      temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT));
-      if (!temp) {
+      temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts,
+                               nsAttsSize * sizeof(NS_ATT));
+      if (! temp) {
         /* Restore actual size of memory in m_nsAtts */
         parser->m_nsAttsPower = oldNsAttsPower;
         return XML_ERROR_NO_MEMORY;
       }
       parser->m_nsAtts = temp;
-      version = 0;  /* force re-initialization of m_nsAtts hash table */
+      version = 0; /* force re-initialization of m_nsAtts hash table */
     }
     /* using a version flag saves us from initializing m_nsAtts every time */
-    if (!version) {  /* initialize version flags when version wraps around */
+    if (! version) { /* initialize version flags when version wraps around */
       version = INIT_ATTS_VERSION;
-      for (j = nsAttsSize; j != 0; )
+      for (j = nsAttsSize; j != 0;)
         parser->m_nsAtts[--j].version = version;
     }
     parser->m_nsAttsVersion = --version;
@@ -3383,7 +3252,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     /* expand prefixed names and check for duplicates */
     for (; i < attIndex; i += 2) {
       const XML_Char *s = appAtts[i];
-      if (s[-1] == 2) {  /* prefixed */
+      if (s[-1] == 2) { /* prefixed */
         ATTRIBUTE_ID *id;
         const BINDING *b;
         unsigned long uriHash;
@@ -3393,9 +3262,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         copy_salt_to_sipkey(parser, &sip_key);
         sip24_init(&sip_state, &sip_key);
 
-        ((XML_Char *)s)[-1] = 0;  /* clear flag */
+        ((XML_Char *)s)[-1] = 0; /* clear flag */
         id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0);
-        if (!id || !id->prefix) {
+        if (! id || ! id->prefix) {
           /* This code is walking through the appAtts array, dealing
            * with (in this case) a prefixed attribute name.  To be in
            * the array, the attribute must have already been bound, so
@@ -3413,12 +3282,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
           return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */
         }
         b = id->prefix->binding;
-        if (!b)
+        if (! b)
           return XML_ERROR_UNBOUND_PREFIX;
 
         for (j = 0; j < b->uriLen; j++) {
           const XML_Char c = b->uri[j];
-          if (!poolAppendChar(&parser->m_tempPool, c))
+          if (! poolAppendChar(&parser->m_tempPool, c))
             return XML_ERROR_NO_MEMORY;
         }
 
@@ -3429,8 +3298,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
 
         sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char));
 
-        do {  /* copies null terminator */
-          if (!poolAppendChar(&parser->m_tempPool, *s))
+        do { /* copies null terminator */
+          if (! poolAppendChar(&parser->m_tempPool, *s))
             return XML_ERROR_NO_MEMORY;
         } while (*s++);
 
@@ -3441,28 +3310,29 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
           */
           unsigned char step = 0;
           unsigned long mask = nsAttsSize - 1;
-          j = uriHash & mask;  /* index into hash table */
+          j = uriHash & mask; /* index into hash table */
           while (parser->m_nsAtts[j].version == version) {
             /* for speed we compare stored hash values first */
             if (uriHash == parser->m_nsAtts[j].hash) {
               const XML_Char *s1 = poolStart(&parser->m_tempPool);
               const XML_Char *s2 = parser->m_nsAtts[j].uriName;
               /* s1 is null terminated, but not s2 */
-              for (; *s1 == *s2 && *s1 != 0; s1++, s2++);
+              for (; *s1 == *s2 && *s1 != 0; s1++, s2++)
+                ;
               if (*s1 == 0)
                 return XML_ERROR_DUPLICATE_ATTRIBUTE;
             }
-            if (!step)
+            if (! step)
               step = PROBE_STEP(uriHash, mask, parser->m_nsAttsPower);
             j < step ? (j += nsAttsSize - step) : (j -= step);
           }
         }
 
-        if (parser->m_ns_triplets) {  /* append namespace separator and prefix */
+        if (parser->m_ns_triplets) { /* append namespace separator and prefix */
           parser->m_tempPool.ptr[-1] = parser->m_namespaceSeparator;
           s = b->prefix->name;
           do {
-            if (!poolAppendChar(&parser->m_tempPool, *s))
+            if (! poolAppendChar(&parser->m_tempPool, *s))
               return XML_ERROR_NO_MEMORY;
           } while (*s++);
         }
@@ -3477,13 +3347,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         parser->m_nsAtts[j].hash = uriHash;
         parser->m_nsAtts[j].uriName = s;
 
-        if (!--nPrefixes) {
+        if (! --nPrefixes) {
           i += 2;
           break;
         }
-      }
-      else  /* not prefixed */
-        ((XML_Char *)s)[-1] = 0;  /* clear flag */
+      } else                     /* not prefixed */
+        ((XML_Char *)s)[-1] = 0; /* clear flag */
     }
   }
   /* clear flags for the remaining attributes */
@@ -3492,40 +3361,38 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   for (binding = *bindingsPtr; binding; binding = binding->nextTagBinding)
     binding->attId->name[-1] = 0;
 
-  if (!parser->m_ns)
+  if (! parser->m_ns)
     return XML_ERROR_NONE;
 
   /* expand the element type name */
   if (elementType->prefix) {
     binding = elementType->prefix->binding;
-    if (!binding)
+    if (! binding)
       return XML_ERROR_UNBOUND_PREFIX;
     localPart = tagNamePtr->str;
     while (*localPart++ != XML_T(ASCII_COLON))
       ;
-  }
-  else if (dtd->defaultPrefix.binding) {
+  } else if (dtd->defaultPrefix.binding) {
     binding = dtd->defaultPrefix.binding;
     localPart = tagNamePtr->str;
-  }
-  else
+  } else
     return XML_ERROR_NONE;
   prefixLen = 0;
   if (parser->m_ns_triplets && binding->prefix->name) {
     for (; binding->prefix->name[prefixLen++];)
-      ;  /* prefixLen includes null terminator */
+      ; /* prefixLen includes null terminator */
   }
   tagNamePtr->localPart = localPart;
   tagNamePtr->uriLen = binding->uriLen;
   tagNamePtr->prefix = binding->prefix->name;
   tagNamePtr->prefixLen = prefixLen;
   for (i = 0; localPart[i++];)
-    ;  /* i includes null terminator */
+    ; /* i includes null terminator */
   n = i + binding->uriLen + prefixLen;
   if (n > binding->uriAlloc) {
     TAG *p;
     uri = (XML_Char *)MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char));
-    if (!uri)
+    if (! uri)
       return XML_ERROR_NO_MEMORY;
     binding->uriAlloc = n + EXPAND_SPARE;
     memcpy(uri, binding->uri, binding->uriLen * sizeof(XML_Char));
@@ -3541,7 +3408,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   /* we always have a namespace separator between localPart and prefix */
   if (prefixLen) {
     uri += i - 1;
-    *uri = parser->m_namespaceSeparator;  /* replace null terminator */
+    *uri = parser->m_namespaceSeparator; /* replace null terminator */
     memcpy(uri + 1, binding->prefix->name, prefixLen * sizeof(XML_Char));
   }
   tagNamePtr->str = binding->uri;
@@ -3553,27 +3420,25 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
 */
 static enum XML_Error
 addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
-           const XML_Char *uri, BINDING **bindingsPtr)
-{
-  static const XML_Char xmlNamespace[] = {
-    ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, ASCII_SLASH,
-    ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD,
-    ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L,
-    ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8, ASCII_SLASH,
-    ASCII_n, ASCII_a, ASCII_m, ASCII_e, ASCII_s, ASCII_p, ASCII_a, ASCII_c,
-    ASCII_e, '\0'
-  };
-  static const int xmlLen =
-    (int)sizeof(xmlNamespace)/sizeof(XML_Char) - 1;
-  static const XML_Char xmlnsNamespace[] = {
-    ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, ASCII_SLASH,
-    ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD,
-    ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_2, ASCII_0, ASCII_0,
-    ASCII_0, ASCII_SLASH, ASCII_x, ASCII_m, ASCII_l, ASCII_n, ASCII_s,
-    ASCII_SLASH, '\0'
-  };
-  static const int xmlnsLen =
-    (int)sizeof(xmlnsNamespace)/sizeof(XML_Char) - 1;
+           const XML_Char *uri, BINDING **bindingsPtr) {
+  static const XML_Char xmlNamespace[]
+      = {ASCII_h,      ASCII_t,     ASCII_t,     ASCII_p,      ASCII_COLON,
+         ASCII_SLASH,  ASCII_SLASH, ASCII_w,     ASCII_w,      ASCII_w,
+         ASCII_PERIOD, ASCII_w,     ASCII_3,     ASCII_PERIOD, ASCII_o,
+         ASCII_r,      ASCII_g,     ASCII_SLASH, ASCII_X,      ASCII_M,
+         ASCII_L,      ASCII_SLASH, ASCII_1,     ASCII_9,      ASCII_9,
+         ASCII_8,      ASCII_SLASH, ASCII_n,     ASCII_a,      ASCII_m,
+         ASCII_e,      ASCII_s,     ASCII_p,     ASCII_a,      ASCII_c,
+         ASCII_e,      '\0'};
+  static const int xmlLen = (int)sizeof(xmlNamespace) / sizeof(XML_Char) - 1;
+  static const XML_Char xmlnsNamespace[]
+      = {ASCII_h,     ASCII_t,      ASCII_t, ASCII_p, ASCII_COLON,  ASCII_SLASH,
+         ASCII_SLASH, ASCII_w,      ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w,
+         ASCII_3,     ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g,      ASCII_SLASH,
+         ASCII_2,     ASCII_0,      ASCII_0, ASCII_0, ASCII_SLASH,  ASCII_x,
+         ASCII_m,     ASCII_l,      ASCII_n, ASCII_s, ASCII_SLASH,  '\0'};
+  static const int xmlnsLen
+      = (int)sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1;
 
   XML_Bool mustBeXML = XML_FALSE;
   XML_Bool isXML = XML_TRUE;
@@ -3586,14 +3451,11 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   if (*uri == XML_T('\0') && prefix->name)
     return XML_ERROR_UNDECLARING_PREFIX;
 
-  if (prefix->name
-      && prefix->name[0] == XML_T(ASCII_x)
+  if (prefix->name && prefix->name[0] == XML_T(ASCII_x)
       && prefix->name[1] == XML_T(ASCII_m)
       && prefix->name[2] == XML_T(ASCII_l)) {
-
     /* Not allowed to bind xmlns */
-    if (prefix->name[3] == XML_T(ASCII_n)
-        && prefix->name[4] == XML_T(ASCII_s)
+    if (prefix->name[3] == XML_T(ASCII_n) && prefix->name[4] == XML_T(ASCII_s)
         && prefix->name[5] == XML_T('\0'))
       return XML_ERROR_RESERVED_PREFIX_XMLNS;
 
@@ -3605,7 +3467,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
     if (isXML && (len > xmlLen || uri[len] != xmlNamespace[len]))
       isXML = XML_FALSE;
 
-    if (!mustBeXML && isXMLNS
+    if (! mustBeXML && isXMLNS
         && (len > xmlnsLen || uri[len] != xmlnsNamespace[len]))
       isXMLNS = XML_FALSE;
   }
@@ -3624,21 +3486,21 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   if (parser->m_freeBindingList) {
     b = parser->m_freeBindingList;
     if (len > b->uriAlloc) {
-      XML_Char *temp = (XML_Char *)REALLOC(parser, b->uri,
-                          sizeof(XML_Char) * (len + EXPAND_SPARE));
+      XML_Char *temp = (XML_Char *)REALLOC(
+          parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE));
       if (temp == NULL)
         return XML_ERROR_NO_MEMORY;
       b->uri = temp;
       b->uriAlloc = len + EXPAND_SPARE;
     }
     parser->m_freeBindingList = b->nextTagBinding;
-  }
-  else {
+  } else {
     b = (BINDING *)MALLOC(parser, sizeof(BINDING));
-    if (!b)
+    if (! b)
       return XML_ERROR_NO_MEMORY;
-    b->uri = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE));
-    if (!b->uri) {
+    b->uri
+        = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE));
+    if (! b->uri) {
       FREE(parser, b);
       return XML_ERROR_NO_MEMORY;
     }
@@ -3661,7 +3523,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   /* if attId == NULL then we are not starting a namespace scope */
   if (attId && parser->m_startNamespaceDeclHandler)
     parser->m_startNamespaceDeclHandler(parser->m_handlerArg, prefix->name,
-                              prefix->binding ? uri : 0);
+                                        prefix->binding ? uri : 0);
   return XML_ERROR_NONE;
 }
 
@@ -3669,21 +3531,18 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
    the whole file is parsed with one call.
 */
 static enum XML_Error PTRCALL
-cdataSectionProcessor(XML_Parser parser,
-                      const char *start,
-                      const char *end,
-                      const char **endPtr)
-{
-  enum XML_Error result = doCdataSection(parser, parser->m_encoding, &start, end,
-                                         endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+cdataSectionProcessor(XML_Parser parser, const char *start, const char *end,
+                      const char **endPtr) {
+  enum XML_Error result
+      = doCdataSection(parser, parser->m_encoding, &start, end, endPtr,
+                       (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result != XML_ERROR_NONE)
     return result;
   if (start) {
-    if (parser->m_parentParser) {  /* we are parsing an external entity */
+    if (parser->m_parentParser) { /* we are parsing an external entity */
       parser->m_processor = externalEntityContentProcessor;
       return externalEntityContentProcessor(parser, start, end, endPtr);
-    }
-    else {
+    } else {
       parser->m_processor = contentProcessor;
       return contentProcessor(parser, start, end, endPtr);
     }
@@ -3695,13 +3554,8 @@ cdataSectionProcessor(XML_Parser parser,
    the section is not yet closed.
 */
 static enum XML_Error
-doCdataSection(XML_Parser parser,
-               const ENCODING *enc,
-               const char **startPtr,
-               const char *end,
-               const char **nextPtr,
-               XML_Bool haveMore)
-{
+doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
+               const char *end, const char **nextPtr, XML_Bool haveMore) {
   const char *s = *startPtr;
   const char **eventPP;
   const char **eventEndPP;
@@ -3709,8 +3563,7 @@ doCdataSection(XML_Parser parser,
     eventPP = &parser->m_eventPtr;
     *eventPP = s;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -3725,11 +3578,12 @@ doCdataSection(XML_Parser parser,
     case XML_TOK_CDATA_SECT_CLOSE:
       if (parser->m_endCdataSectionHandler)
         parser->m_endCdataSectionHandler(parser->m_handlerArg);
-/* BEGIN disabled code */
+      /* BEGIN disabled code */
       /* see comment under XML_TOK_CDATA_SECT_OPEN */
       else if (0 && parser->m_characterDataHandler)
-        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0);
-/* END disabled code */
+        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                                       0);
+      /* END disabled code */
       else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       *startPtr = next;
@@ -3742,35 +3596,31 @@ doCdataSection(XML_Parser parser,
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       break;
-    case XML_TOK_DATA_CHARS:
-      {
-        XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
-        if (charDataHandler) {
-          if (MUST_CONVERT(enc, s)) {
-            for (;;) {
-              ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-              const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-              *eventEndPP = next;
-              charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-              if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
-                break;
-              *eventPP = s;
-            }
+    case XML_TOK_DATA_CHARS: {
+      XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
+      if (charDataHandler) {
+        if (MUST_CONVERT(enc, s)) {
+          for (;;) {
+            ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
+            const enum XML_Convert_Result convert_res = XmlConvert(
+                enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+            *eventEndPP = next;
+            charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                            (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+            if ((convert_res == XML_CONVERT_COMPLETED)
+                || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+              break;
+            *eventPP = s;
           }
-          else
-            charDataHandler(parser->m_handlerArg,
-                            (XML_Char *)s,
-                            (int)((XML_Char *)next - (XML_Char *)s));
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-      }
-      break;
+        } else
+          charDataHandler(parser->m_handlerArg, (XML_Char *)s,
+                          (int)((XML_Char *)next - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_INVALID:
       *eventPP = next;
       return XML_ERROR_INVALID_TOKEN;
@@ -3794,7 +3644,7 @@ doCdataSection(XML_Parser parser,
        * statistics.
        *
        * LCOV_EXCL_START
-      */
+       */
       *eventPP = next;
       return XML_ERROR_UNEXPECTED_STATE;
       /* LCOV_EXCL_STOP */
@@ -3807,7 +3657,7 @@ doCdataSection(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
   /* not reached */
@@ -3819,13 +3669,11 @@ doCdataSection(XML_Parser parser,
    the whole file is parsed with one call.
 */
 static enum XML_Error PTRCALL
-ignoreSectionProcessor(XML_Parser parser,
-                       const char *start,
-                       const char *end,
-                       const char **endPtr)
-{
-  enum XML_Error result = doIgnoreSection(parser, parser->m_encoding, &start, end,
-                                          endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end,
+                       const char **endPtr) {
+  enum XML_Error result
+      = doIgnoreSection(parser, parser->m_encoding, &start, end, endPtr,
+                        (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result != XML_ERROR_NONE)
     return result;
   if (start) {
@@ -3839,13 +3687,8 @@ ignoreSectionProcessor(XML_Parser parser,
    if the section is not yet closed.
 */
 static enum XML_Error
-doIgnoreSection(XML_Parser parser,
-                const ENCODING *enc,
-                const char **startPtr,
-                const char *end,
-                const char **nextPtr,
-                XML_Bool haveMore)
-{
+doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
+                const char *end, const char **nextPtr, XML_Bool haveMore) {
   const char *next;
   int tok;
   const char *s = *startPtr;
@@ -3855,8 +3698,7 @@ doIgnoreSection(XML_Parser parser,
     eventPP = &parser->m_eventPtr;
     *eventPP = s;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     /* It's not entirely clear, but it seems the following two lines
      * of code cannot be executed.  The only occasions on which 'enc'
      * is not 'encoding' are when this function is called
@@ -3920,13 +3762,12 @@ doIgnoreSection(XML_Parser parser,
 #endif /* XML_DTD */
 
 static enum XML_Error
-initializeEncoding(XML_Parser parser)
-{
+initializeEncoding(XML_Parser parser) {
   const char *s;
 #ifdef XML_UNICODE
   char encodingBuf[128];
   /* See comments abount `protoclEncodingName` in parserInit() */
-  if (!parser->m_protocolEncodingName)
+  if (! parser->m_protocolEncodingName)
     s = NULL;
   else {
     int i;
@@ -3944,15 +3785,15 @@ initializeEncoding(XML_Parser parser)
 #else
   s = parser->m_protocolEncodingName;
 #endif
-  if ((parser->m_ns ? XmlInitEncodingNS : XmlInitEncoding)(&parser->m_initEncoding, &parser->m_encoding, s))
+  if ((parser->m_ns ? XmlInitEncodingNS : XmlInitEncoding)(
+          &parser->m_initEncoding, &parser->m_encoding, s))
     return XML_ERROR_NONE;
   return handleUnknownEncoding(parser, parser->m_protocolEncodingName);
 }
 
 static enum XML_Error
-processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
-               const char *s, const char *next)
-{
+processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s,
+               const char *next) {
   const char *encodingName = NULL;
   const XML_Char *storedEncName = NULL;
   const ENCODING *newEncoding = NULL;
@@ -3960,52 +3801,41 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
   const char *versionend;
   const XML_Char *storedversion = NULL;
   int standalone = -1;
-  if (!(parser->m_ns
-        ? XmlParseXmlDeclNS
-        : XmlParseXmlDecl)(isGeneralTextEntity,
-                           parser->m_encoding,
-                           s,
-                           next,
-                           &parser->m_eventPtr,
-                           &version,
-                           &versionend,
-                           &encodingName,
-                           &newEncoding,
-                           &standalone)) {
+  if (! (parser->m_ns ? XmlParseXmlDeclNS : XmlParseXmlDecl)(
+          isGeneralTextEntity, parser->m_encoding, s, next, &parser->m_eventPtr,
+          &version, &versionend, &encodingName, &newEncoding, &standalone)) {
     if (isGeneralTextEntity)
       return XML_ERROR_TEXT_DECL;
     else
       return XML_ERROR_XML_DECL;
   }
-  if (!isGeneralTextEntity && standalone == 1) {
+  if (! isGeneralTextEntity && standalone == 1) {
     parser->m_dtd->standalone = XML_TRUE;
 #ifdef XML_DTD
-    if (parser->m_paramEntityParsing == XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE)
+    if (parser->m_paramEntityParsing
+        == XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE)
       parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER;
 #endif /* XML_DTD */
   }
   if (parser->m_xmlDeclHandler) {
     if (encodingName != NULL) {
-      storedEncName = poolStoreString(&parser->m_temp2Pool,
-                                      parser->m_encoding,
-                                      encodingName,
-                                      encodingName
-                                      + XmlNameLength(parser->m_encoding, encodingName));
-      if (!storedEncName)
-              return XML_ERROR_NO_MEMORY;
+      storedEncName = poolStoreString(
+          &parser->m_temp2Pool, parser->m_encoding, encodingName,
+          encodingName + XmlNameLength(parser->m_encoding, encodingName));
+      if (! storedEncName)
+        return XML_ERROR_NO_MEMORY;
       poolFinish(&parser->m_temp2Pool);
     }
     if (version) {
-      storedversion = poolStoreString(&parser->m_temp2Pool,
-                                      parser->m_encoding,
-                                      version,
-                                      versionend - parser->m_encoding->minBytesPerChar);
-      if (!storedversion)
+      storedversion
+          = poolStoreString(&parser->m_temp2Pool, parser->m_encoding, version,
+                            versionend - parser->m_encoding->minBytesPerChar);
+      if (! storedversion)
         return XML_ERROR_NO_MEMORY;
     }
-    parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName, standalone);
-  }
-  else if (parser->m_defaultHandler)
+    parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName,
+                             standalone);
+  } else if (parser->m_defaultHandler)
     reportDefault(parser, parser->m_encoding, s, next);
   if (parser->m_protocolEncodingName == NULL) {
     if (newEncoding) {
@@ -4015,20 +3845,19 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
        * this is UTF-16, is it the same endianness?
        */
       if (newEncoding->minBytesPerChar != parser->m_encoding->minBytesPerChar
-          || (newEncoding->minBytesPerChar == 2 &&
-              newEncoding != parser->m_encoding)) {
+          || (newEncoding->minBytesPerChar == 2
+              && newEncoding != parser->m_encoding)) {
         parser->m_eventPtr = encodingName;
         return XML_ERROR_INCORRECT_ENCODING;
       }
       parser->m_encoding = newEncoding;
-    }
-    else if (encodingName) {
+    } else if (encodingName) {
       enum XML_Error result;
-      if (!storedEncName) {
+      if (! storedEncName) {
         storedEncName = poolStoreString(
-          &parser->m_temp2Pool, parser->m_encoding, encodingName,
-          encodingName + XmlNameLength(parser->m_encoding, encodingName));
-        if (!storedEncName)
+            &parser->m_temp2Pool, parser->m_encoding, encodingName,
+            encodingName + XmlNameLength(parser->m_encoding, encodingName));
+        if (! storedEncName)
           return XML_ERROR_NO_MEMORY;
       }
       result = handleUnknownEncoding(parser, storedEncName);
@@ -4046,8 +3875,7 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
 }
 
 static enum XML_Error
-handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
-{
+handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) {
   if (parser->m_unknownEncodingHandler) {
     XML_Encoding info;
     int i;
@@ -4056,21 +3884,17 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
     info.convert = NULL;
     info.data = NULL;
     info.release = NULL;
-    if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData, encodingName,
-                               &info)) {
+    if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData,
+                                         encodingName, &info)) {
       ENCODING *enc;
       parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding());
-      if (!parser->m_unknownEncodingMem) {
+      if (! parser->m_unknownEncodingMem) {
         if (info.release)
           info.release(info.data);
         return XML_ERROR_NO_MEMORY;
       }
-      enc = (parser->m_ns
-             ? XmlInitUnknownEncodingNS
-             : XmlInitUnknownEncoding)(parser->m_unknownEncodingMem,
-                                       info.map,
-                                       info.convert,
-                                       info.data);
+      enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)(
+          parser->m_unknownEncodingMem, info.map, info.convert, info.data);
       if (enc) {
         parser->m_unknownEncodingData = info.data;
         parser->m_unknownEncodingRelease = info.release;
@@ -4085,11 +3909,8 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
 }
 
 static enum XML_Error PTRCALL
-prologInitProcessor(XML_Parser parser,
-                    const char *s,
-                    const char *end,
-                    const char **nextPtr)
-{
+prologInitProcessor(XML_Parser parser, const char *s, const char *end,
+                    const char **nextPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -4100,11 +3921,8 @@ prologInitProcessor(XML_Parser parser,
 #ifdef XML_DTD
 
 static enum XML_Error PTRCALL
-externalParEntInitProcessor(XML_Parser parser,
-                            const char *s,
-                            const char *end,
-                            const char **nextPtr)
-{
+externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end,
+                            const char **nextPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -4116,19 +3934,15 @@ externalParEntInitProcessor(XML_Parser parser,
   if (parser->m_prologState.inEntityValue) {
     parser->m_processor = entityValueInitProcessor;
     return entityValueInitProcessor(parser, s, end, nextPtr);
-  }
-  else {
+  } else {
     parser->m_processor = externalParEntProcessor;
     return externalParEntProcessor(parser, s, end, nextPtr);
   }
 }
 
 static enum XML_Error PTRCALL
-entityValueInitProcessor(XML_Parser parser,
-                         const char *s,
-                         const char *end,
-                         const char **nextPtr)
-{
+entityValueInitProcessor(XML_Parser parser, const char *s, const char *end,
+                         const char **nextPtr) {
   int tok;
   const char *start = s;
   const char *next = start;
@@ -4138,7 +3952,7 @@ entityValueInitProcessor(XML_Parser parser,
     tok = XmlPrologTok(parser->m_encoding, start, end, &next);
     parser->m_eventEndPtr = next;
     if (tok <= 0) {
-      if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+      if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -4149,22 +3963,21 @@ entityValueInitProcessor(XML_Parser parser,
         return XML_ERROR_UNCLOSED_TOKEN;
       case XML_TOK_PARTIAL_CHAR:
         return XML_ERROR_PARTIAL_CHAR;
-      case XML_TOK_NONE:   /* start == end */
+      case XML_TOK_NONE: /* start == end */
       default:
         break;
       }
       /* found end of entity value - can store it now */
       return storeEntityValue(parser, parser->m_encoding, s, end);
-    }
-    else if (tok == XML_TOK_XML_DECL) {
+    } else if (tok == XML_TOK_XML_DECL) {
       enum XML_Error result;
       result = processXmlDecl(parser, 0, start, next);
       if (result != XML_ERROR_NONE)
         return result;
-      /* At this point, m_parsingStatus.parsing cannot be XML_SUSPENDED.  For that
-       * to happen, a parameter entity parsing handler must have
-       * attempted to suspend the parser, which fails and raises an
-       * error.  The parser can be aborted, but can't be suspended.
+      /* At this point, m_parsingStatus.parsing cannot be XML_SUSPENDED.  For
+       * that to happen, a parameter entity parsing handler must have attempted
+       * to suspend the parser, which fails and raises an error.  The parser can
+       * be aborted, but can't be suspended.
        */
       if (parser->m_parsingStatus.parsing == XML_FINISHED)
         return XML_ERROR_ABORTED;
@@ -4180,7 +3993,8 @@ entityValueInitProcessor(XML_Parser parser,
        then, when this routine is entered the next time, XmlPrologTok will
        return XML_TOK_INVALID, since the BOM is still in the buffer
     */
-    else if (tok == XML_TOK_BOM && next == end && !parser->m_parsingStatus.finalBuffer) {
+    else if (tok == XML_TOK_BOM && next == end
+             && ! parser->m_parsingStatus.finalBuffer) {
       *nextPtr = next;
       return XML_ERROR_NONE;
     }
@@ -4198,17 +4012,14 @@ entityValueInitProcessor(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalParEntProcessor(XML_Parser parser,
-                        const char *s,
-                        const char *end,
-                        const char **nextPtr)
-{
+externalParEntProcessor(XML_Parser parser, const char *s, const char *end,
+                        const char **nextPtr) {
   const char *next = s;
   int tok;
 
   tok = XmlPrologTok(parser->m_encoding, s, end, &next);
   if (tok <= 0) {
-    if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+    if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
       *nextPtr = s;
       return XML_ERROR_NONE;
     }
@@ -4219,7 +4030,7 @@ externalParEntProcessor(XML_Parser parser,
       return XML_ERROR_UNCLOSED_TOKEN;
     case XML_TOK_PARTIAL_CHAR:
       return XML_ERROR_PARTIAL_CHAR;
-    case XML_TOK_NONE:   /* start == end */
+    case XML_TOK_NONE: /* start == end */
     default:
       break;
     }
@@ -4234,16 +4045,13 @@ externalParEntProcessor(XML_Parser parser,
   }
 
   parser->m_processor = prologProcessor;
-  return doProlog(parser, parser->m_encoding, s, end, tok, next,
-                  nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+  return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
 }
 
 static enum XML_Error PTRCALL
-entityValueProcessor(XML_Parser parser,
-                     const char *s,
-                     const char *end,
-                     const char **nextPtr)
-{
+entityValueProcessor(XML_Parser parser, const char *s, const char *end,
+                     const char **nextPtr) {
   const char *start = s;
   const char *next = s;
   const ENCODING *enc = parser->m_encoding;
@@ -4252,7 +4060,7 @@ entityValueProcessor(XML_Parser parser,
   for (;;) {
     tok = XmlPrologTok(enc, start, end, &next);
     if (tok <= 0) {
-      if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+      if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -4263,7 +4071,7 @@ entityValueProcessor(XML_Parser parser,
         return XML_ERROR_UNCLOSED_TOKEN;
       case XML_TOK_PARTIAL_CHAR:
         return XML_ERROR_PARTIAL_CHAR;
-      case XML_TOK_NONE:   /* start == end */
+      case XML_TOK_NONE: /* start == end */
       default:
         break;
       }
@@ -4277,52 +4085,46 @@ entityValueProcessor(XML_Parser parser,
 #endif /* XML_DTD */
 
 static enum XML_Error PTRCALL
-prologProcessor(XML_Parser parser,
-                const char *s,
-                const char *end,
-                const char **nextPtr)
-{
+prologProcessor(XML_Parser parser, const char *s, const char *end,
+                const char **nextPtr) {
   const char *next = s;
   int tok = XmlPrologTok(parser->m_encoding, s, end, &next);
-  return doProlog(parser, parser->m_encoding, s, end, tok, next,
-                  nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+  return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
 }
 
 static enum XML_Error
-doProlog(XML_Parser parser,
-         const ENCODING *enc,
-         const char *s,
-         const char *end,
-         int tok,
-         const char *next,
-         const char **nextPtr,
-         XML_Bool haveMore)
-{
+doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
+         int tok, const char *next, const char **nextPtr, XML_Bool haveMore,
+         XML_Bool allowClosingDoctype) {
 #ifdef XML_DTD
-  static const XML_Char externalSubsetName[] = { ASCII_HASH , '\0' };
+  static const XML_Char externalSubsetName[] = {ASCII_HASH, '\0'};
 #endif /* XML_DTD */
-  static const XML_Char atypeCDATA[] =
-      { ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-  static const XML_Char atypeID[] = { ASCII_I, ASCII_D, '\0' };
-  static const XML_Char atypeIDREF[] =
-      { ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0' };
-  static const XML_Char atypeIDREFS[] =
-      { ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0' };
-  static const XML_Char atypeENTITY[] =
-      { ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0' };
-  static const XML_Char atypeENTITIES[] = { ASCII_E, ASCII_N,
-      ASCII_T, ASCII_I, ASCII_T, ASCII_I, ASCII_E, ASCII_S, '\0' };
-  static const XML_Char atypeNMTOKEN[] = {
-      ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0' };
-  static const XML_Char atypeNMTOKENS[] = { ASCII_N, ASCII_M, ASCII_T,
-      ASCII_O, ASCII_K, ASCII_E, ASCII_N, ASCII_S, '\0' };
-  static const XML_Char notationPrefix[] = { ASCII_N, ASCII_O, ASCII_T,
-      ASCII_A, ASCII_T, ASCII_I, ASCII_O, ASCII_N, ASCII_LPAREN, '\0' };
-  static const XML_Char enumValueSep[] = { ASCII_PIPE, '\0' };
-  static const XML_Char enumValueStart[] = { ASCII_LPAREN, '\0' };
+  static const XML_Char atypeCDATA[]
+      = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+  static const XML_Char atypeID[] = {ASCII_I, ASCII_D, '\0'};
+  static const XML_Char atypeIDREF[]
+      = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0'};
+  static const XML_Char atypeIDREFS[]
+      = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0'};
+  static const XML_Char atypeENTITY[]
+      = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0'};
+  static const XML_Char atypeENTITIES[]
+      = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T,
+         ASCII_I, ASCII_E, ASCII_S, '\0'};
+  static const XML_Char atypeNMTOKEN[]
+      = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0'};
+  static const XML_Char atypeNMTOKENS[]
+      = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K,
+         ASCII_E, ASCII_N, ASCII_S, '\0'};
+  static const XML_Char notationPrefix[]
+      = {ASCII_N, ASCII_O, ASCII_T, ASCII_A,      ASCII_T,
+         ASCII_I, ASCII_O, ASCII_N, ASCII_LPAREN, '\0'};
+  static const XML_Char enumValueSep[] = {ASCII_PIPE, '\0'};
+  static const XML_Char enumValueStart[] = {ASCII_LPAREN, '\0'};
 
   /* save one level of indirection */
-  DTD * const dtd = parser->m_dtd;
+  DTD *const dtd = parser->m_dtd;
 
   const char **eventPP;
   const char **eventEndPP;
@@ -4331,8 +4133,7 @@ doProlog(XML_Parser parser,
   if (enc == parser->m_encoding) {
     eventPP = &parser->m_eventPtr;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -4361,7 +4162,8 @@ doProlog(XML_Parser parser,
       case XML_TOK_NONE:
 #ifdef XML_DTD
         /* for internal PE NOT referenced between declarations */
-        if (enc != parser->m_encoding && !parser->m_openInternalEntities->betweenDecl) {
+        if (enc != parser->m_encoding
+            && ! parser->m_openInternalEntities->betweenDecl) {
           *nextPtr = s;
           return XML_ERROR_NONE;
         }
@@ -4386,19 +4188,18 @@ doProlog(XML_Parser parser,
     }
     role = XmlTokenRole(&parser->m_prologState, tok, s, next, enc);
     switch (role) {
-    case XML_ROLE_XML_DECL:
-      {
-        enum XML_Error result = processXmlDecl(parser, 0, s, next);
-        if (result != XML_ERROR_NONE)
-          return result;
-        enc = parser->m_encoding;
-        handleDefault = XML_FALSE;
-      }
-      break;
+    case XML_ROLE_XML_DECL: {
+      enum XML_Error result = processXmlDecl(parser, 0, s, next);
+      if (result != XML_ERROR_NONE)
+        return result;
+      enc = parser->m_encoding;
+      handleDefault = XML_FALSE;
+    } break;
     case XML_ROLE_DOCTYPE_NAME:
       if (parser->m_startDoctypeDeclHandler) {
-        parser->m_doctypeName = poolStoreString(&parser->m_tempPool, enc, s, next);
-        if (!parser->m_doctypeName)
+        parser->m_doctypeName
+            = poolStoreString(&parser->m_tempPool, enc, s, next);
+        if (! parser->m_doctypeName)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
         parser->m_doctypePubid = NULL;
@@ -4408,43 +4209,40 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_DOCTYPE_INTERNAL_SUBSET:
       if (parser->m_startDoctypeDeclHandler) {
-        parser->m_startDoctypeDeclHandler(parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
-                                parser->m_doctypePubid, 1);
+        parser->m_startDoctypeDeclHandler(
+            parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
+            parser->m_doctypePubid, 1);
         parser->m_doctypeName = NULL;
         poolClear(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
       break;
 #ifdef XML_DTD
-    case XML_ROLE_TEXT_DECL:
-      {
-        enum XML_Error result = processXmlDecl(parser, 1, s, next);
-        if (result != XML_ERROR_NONE)
-          return result;
-        enc = parser->m_encoding;
-        handleDefault = XML_FALSE;
-      }
-      break;
+    case XML_ROLE_TEXT_DECL: {
+      enum XML_Error result = processXmlDecl(parser, 1, s, next);
+      if (result != XML_ERROR_NONE)
+        return result;
+      enc = parser->m_encoding;
+      handleDefault = XML_FALSE;
+    } break;
 #endif /* XML_DTD */
     case XML_ROLE_DOCTYPE_PUBLIC_ID:
 #ifdef XML_DTD
       parser->m_useForeignDTD = XML_FALSE;
-      parser->m_declEntity = (ENTITY *)lookup(parser,
-                                    &dtd->paramEntities,
-                                    externalSubsetName,
-                                    sizeof(ENTITY));
-      if (!parser->m_declEntity)
+      parser->m_declEntity = (ENTITY *)lookup(
+          parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY));
+      if (! parser->m_declEntity)
         return XML_ERROR_NO_MEMORY;
 #endif /* XML_DTD */
       dtd->hasParamEntityRefs = XML_TRUE;
       if (parser->m_startDoctypeDeclHandler) {
         XML_Char *pubId;
-        if (!XmlIsPublicId(enc, s, next, eventPP))
+        if (! XmlIsPublicId(enc, s, next, eventPP))
           return XML_ERROR_PUBLICID;
         pubId = poolStoreString(&parser->m_tempPool, enc,
                                 s + enc->minBytesPerChar,
                                 next - enc->minBytesPerChar);
-        if (!pubId)
+        if (! pubId)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(pubId);
         poolFinish(&parser->m_tempPool);
@@ -4454,15 +4252,14 @@ doProlog(XML_Parser parser,
       }
       /* fall through */
     case XML_ROLE_ENTITY_PUBLIC_ID:
-      if (!XmlIsPublicId(enc, s, next, eventPP))
+      if (! XmlIsPublicId(enc, s, next, eventPP))
         return XML_ERROR_PUBLICID;
     alreadyChecked:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        XML_Char *tem = poolStoreString(&dtd->pool,
-                                        enc,
-                                        s + enc->minBytesPerChar,
-                                        next - enc->minBytesPerChar);
-        if (!tem)
+        XML_Char *tem
+            = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                              next - enc->minBytesPerChar);
+        if (! tem)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(tem);
         parser->m_declEntity->publicId = tem;
@@ -4475,9 +4272,15 @@ doProlog(XML_Parser parser,
       }
       break;
     case XML_ROLE_DOCTYPE_CLOSE:
+      if (allowClosingDoctype != XML_TRUE) {
+        /* Must not close doctype from within expanded parameter entities */
+        return XML_ERROR_INVALID_TOKEN;
+      }
+
       if (parser->m_doctypeName) {
-        parser->m_startDoctypeDeclHandler(parser->m_handlerArg, parser->m_doctypeName,
-                                parser->m_doctypeSysid, parser->m_doctypePubid, 0);
+        parser->m_startDoctypeDeclHandler(
+            parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
+            parser->m_doctypePubid, 0);
         poolClear(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
@@ -4489,12 +4292,11 @@ doProlog(XML_Parser parser,
       if (parser->m_doctypeSysid || parser->m_useForeignDTD) {
         XML_Bool hadParamEntityRefs = dtd->hasParamEntityRefs;
         dtd->hasParamEntityRefs = XML_TRUE;
-        if (parser->m_paramEntityParsing && parser->m_externalEntityRefHandler) {
-          ENTITY *entity = (ENTITY *)lookup(parser,
-                                            &dtd->paramEntities,
-                                            externalSubsetName,
-                                            sizeof(ENTITY));
-          if (!entity) {
+        if (parser->m_paramEntityParsing
+            && parser->m_externalEntityRefHandler) {
+          ENTITY *entity = (ENTITY *)lookup(parser, &dtd->paramEntities,
+                                            externalSubsetName, sizeof(ENTITY));
+          if (! entity) {
             /* The external subset name "#" will have already been
              * inserted into the hash table at the start of the
              * external entity parsing, so no allocation will happen
@@ -4505,22 +4307,19 @@ doProlog(XML_Parser parser,
           if (parser->m_useForeignDTD)
             entity->base = parser->m_curBase;
           dtd->paramEntityRead = XML_FALSE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId))
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           if (dtd->paramEntityRead) {
-            if (!dtd->standalone &&
-                parser->m_notStandaloneHandler &&
-                !parser->m_notStandaloneHandler(parser->m_handlerArg))
+            if (! dtd->standalone && parser->m_notStandaloneHandler
+                && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
               return XML_ERROR_NOT_STANDALONE;
           }
           /* if we didn't read the foreign DTD then this means that there
              is no external subset and we must reset dtd->hasParamEntityRefs
           */
-          else if (!parser->m_doctypeSysid)
+          else if (! parser->m_doctypeSysid)
             dtd->hasParamEntityRefs = hadParamEntityRefs;
           /* end of DTD - no need to update dtd->keepProcessing */
         }
@@ -4540,24 +4339,21 @@ doProlog(XML_Parser parser,
       if (parser->m_useForeignDTD) {
         XML_Bool hadParamEntityRefs = dtd->hasParamEntityRefs;
         dtd->hasParamEntityRefs = XML_TRUE;
-        if (parser->m_paramEntityParsing && parser->m_externalEntityRefHandler) {
+        if (parser->m_paramEntityParsing
+            && parser->m_externalEntityRefHandler) {
           ENTITY *entity = (ENTITY *)lookup(parser, &dtd->paramEntities,
-                                            externalSubsetName,
-                                            sizeof(ENTITY));
-          if (!entity)
+                                            externalSubsetName, sizeof(ENTITY));
+          if (! entity)
             return XML_ERROR_NO_MEMORY;
           entity->base = parser->m_curBase;
           dtd->paramEntityRead = XML_FALSE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId))
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           if (dtd->paramEntityRead) {
-            if (!dtd->standalone &&
-                parser->m_notStandaloneHandler &&
-                !parser->m_notStandaloneHandler(parser->m_handlerArg))
+            if (! dtd->standalone && parser->m_notStandaloneHandler
+                && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
               return XML_ERROR_NOT_STANDALONE;
           }
           /* if we didn't read the foreign DTD then this means that there
@@ -4573,12 +4369,12 @@ doProlog(XML_Parser parser,
       return contentProcessor(parser, s, end, nextPtr);
     case XML_ROLE_ATTLIST_ELEMENT_NAME:
       parser->m_declElementType = getElementType(parser, enc, s, next);
-      if (!parser->m_declElementType)
+      if (! parser->m_declElementType)
         return XML_ERROR_NO_MEMORY;
       goto checkAttListDeclHandler;
     case XML_ROLE_ATTRIBUTE_NAME:
       parser->m_declAttributeId = getAttributeId(parser, enc, s, next);
-      if (!parser->m_declAttributeId)
+      if (! parser->m_declAttributeId)
         return XML_ERROR_NO_MEMORY;
       parser->m_declAttributeIsCdata = XML_FALSE;
       parser->m_declAttributeType = NULL;
@@ -4619,15 +4415,13 @@ doProlog(XML_Parser parser,
         const XML_Char *prefix;
         if (parser->m_declAttributeType) {
           prefix = enumValueSep;
+        } else {
+          prefix = (role == XML_ROLE_ATTRIBUTE_NOTATION_VALUE ? notationPrefix
+                                                              : enumValueStart);
         }
-        else {
-          prefix = (role == XML_ROLE_ATTRIBUTE_NOTATION_VALUE
-                    ? notationPrefix
-                    : enumValueStart);
-        }
-        if (!poolAppendString(&parser->m_tempPool, prefix))
+        if (! poolAppendString(&parser->m_tempPool, prefix))
           return XML_ERROR_NO_MEMORY;
-        if (!poolAppend(&parser->m_tempPool, enc, s, next))
+        if (! poolAppend(&parser->m_tempPool, enc, s, next))
           return XML_ERROR_NO_MEMORY;
         parser->m_declAttributeType = parser->m_tempPool.start;
         handleDefault = XML_FALSE;
@@ -4636,25 +4430,27 @@ doProlog(XML_Parser parser,
     case XML_ROLE_IMPLIED_ATTRIBUTE_VALUE:
     case XML_ROLE_REQUIRED_ATTRIBUTE_VALUE:
       if (dtd->keepProcessing) {
-        if (!defineAttribute(parser->m_declElementType, parser->m_declAttributeId,
-                             parser->m_declAttributeIsCdata, parser->m_declAttributeIsId,
-                             0, parser))
+        if (! defineAttribute(parser->m_declElementType,
+                              parser->m_declAttributeId,
+                              parser->m_declAttributeIsCdata,
+                              parser->m_declAttributeIsId, 0, parser))
           return XML_ERROR_NO_MEMORY;
         if (parser->m_attlistDeclHandler && parser->m_declAttributeType) {
           if (*parser->m_declAttributeType == XML_T(ASCII_LPAREN)
               || (*parser->m_declAttributeType == XML_T(ASCII_N)
                   && parser->m_declAttributeType[1] == XML_T(ASCII_O))) {
             /* Enumerated or Notation type */
-            if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
-                || !poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+            if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
+                || ! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
               return XML_ERROR_NO_MEMORY;
             parser->m_declAttributeType = parser->m_tempPool.start;
             poolFinish(&parser->m_tempPool);
           }
           *eventEndPP = s;
-          parser->m_attlistDeclHandler(parser->m_handlerArg, parser->m_declElementType->name,
-                             parser->m_declAttributeId->name, parser->m_declAttributeType,
-                             0, role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE);
+          parser->m_attlistDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name,
+              parser->m_declAttributeId->name, parser->m_declAttributeType, 0,
+              role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE);
           poolClear(&parser->m_tempPool);
           handleDefault = XML_FALSE;
         }
@@ -4664,35 +4460,34 @@ doProlog(XML_Parser parser,
     case XML_ROLE_FIXED_ATTRIBUTE_VALUE:
       if (dtd->keepProcessing) {
         const XML_Char *attVal;
-        enum XML_Error result =
-          storeAttributeValue(parser, enc, parser->m_declAttributeIsCdata,
-                              s + enc->minBytesPerChar,
-                              next - enc->minBytesPerChar,
-                              &dtd->pool);
+        enum XML_Error result = storeAttributeValue(
+            parser, enc, parser->m_declAttributeIsCdata,
+            s + enc->minBytesPerChar, next - enc->minBytesPerChar, &dtd->pool);
         if (result)
           return result;
         attVal = poolStart(&dtd->pool);
         poolFinish(&dtd->pool);
         /* ID attributes aren't allowed to have a default */
-        if (!defineAttribute(parser->m_declElementType, parser->m_declAttributeId,
-                             parser->m_declAttributeIsCdata, XML_FALSE, attVal, parser))
+        if (! defineAttribute(
+                parser->m_declElementType, parser->m_declAttributeId,
+                parser->m_declAttributeIsCdata, XML_FALSE, attVal, parser))
           return XML_ERROR_NO_MEMORY;
         if (parser->m_attlistDeclHandler && parser->m_declAttributeType) {
           if (*parser->m_declAttributeType == XML_T(ASCII_LPAREN)
               || (*parser->m_declAttributeType == XML_T(ASCII_N)
                   && parser->m_declAttributeType[1] == XML_T(ASCII_O))) {
             /* Enumerated or Notation type */
-            if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
-                || !poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+            if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
+                || ! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
               return XML_ERROR_NO_MEMORY;
             parser->m_declAttributeType = parser->m_tempPool.start;
             poolFinish(&parser->m_tempPool);
           }
           *eventEndPP = s;
-          parser->m_attlistDeclHandler(parser->m_handlerArg, parser->m_declElementType->name,
-                             parser->m_declAttributeId->name, parser->m_declAttributeType,
-                             attVal,
-                             role == XML_ROLE_FIXED_ATTRIBUTE_VALUE);
+          parser->m_attlistDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name,
+              parser->m_declAttributeId->name, parser->m_declAttributeType,
+              attVal, role == XML_ROLE_FIXED_ATTRIBUTE_VALUE);
           poolClear(&parser->m_tempPool);
           handleDefault = XML_FALSE;
         }
@@ -4700,25 +4495,22 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_ENTITY_VALUE:
       if (dtd->keepProcessing) {
-        enum XML_Error result = storeEntityValue(parser, enc,
-                                            s + enc->minBytesPerChar,
-                                            next - enc->minBytesPerChar);
+        enum XML_Error result = storeEntityValue(
+            parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar);
         if (parser->m_declEntity) {
           parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool);
-          parser->m_declEntity->textLen = (int)(poolLength(&dtd->entityValuePool));
+          parser->m_declEntity->textLen
+              = (int)(poolLength(&dtd->entityValuePool));
           poolFinish(&dtd->entityValuePool);
           if (parser->m_entityDeclHandler) {
             *eventEndPP = s;
-            parser->m_entityDeclHandler(parser->m_handlerArg,
-                              parser->m_declEntity->name,
-                              parser->m_declEntity->is_param,
-                              parser->m_declEntity->textPtr,
-                              parser->m_declEntity->textLen,
-                              parser->m_curBase, 0, 0, 0);
+            parser->m_entityDeclHandler(
+                parser->m_handlerArg, parser->m_declEntity->name,
+                parser->m_declEntity->is_param, parser->m_declEntity->textPtr,
+                parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0);
             handleDefault = XML_FALSE;
           }
-        }
-        else
+        } else
           poolDiscard(&dtd->entityValuePool);
         if (result != XML_ERROR_NONE)
           return result;
@@ -4731,8 +4523,8 @@ doProlog(XML_Parser parser,
       dtd->hasParamEntityRefs = XML_TRUE;
       if (parser->m_startDoctypeDeclHandler) {
         parser->m_doctypeSysid = poolStoreString(&parser->m_tempPool, enc,
-                                       s + enc->minBytesPerChar,
-                                       next - enc->minBytesPerChar);
+                                                 s + enc->minBytesPerChar,
+                                                 next - enc->minBytesPerChar);
         if (parser->m_doctypeSysid == NULL)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
@@ -4744,22 +4536,20 @@ doProlog(XML_Parser parser,
            for the case where no parser->m_startDoctypeDeclHandler is set */
         parser->m_doctypeSysid = externalSubsetName;
 #endif /* XML_DTD */
-      if (!dtd->standalone
+      if (! dtd->standalone
 #ifdef XML_DTD
-          && !parser->m_paramEntityParsing
+          && ! parser->m_paramEntityParsing
 #endif /* XML_DTD */
           && parser->m_notStandaloneHandler
-          && !parser->m_notStandaloneHandler(parser->m_handlerArg))
+          && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
         return XML_ERROR_NOT_STANDALONE;
 #ifndef XML_DTD
       break;
-#else /* XML_DTD */
-      if (!parser->m_declEntity) {
-        parser->m_declEntity = (ENTITY *)lookup(parser,
-                                      &dtd->paramEntities,
-                                      externalSubsetName,
-                                      sizeof(ENTITY));
-        if (!parser->m_declEntity)
+#else  /* XML_DTD */
+      if (! parser->m_declEntity) {
+        parser->m_declEntity = (ENTITY *)lookup(
+            parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY));
+        if (! parser->m_declEntity)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity->publicId = NULL;
       }
@@ -4767,10 +4557,10 @@ doProlog(XML_Parser parser,
       /* fall through */
     case XML_ROLE_ENTITY_SYSTEM_ID:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        parser->m_declEntity->systemId = poolStoreString(&dtd->pool, enc,
-                                               s + enc->minBytesPerChar,
-                                               next - enc->minBytesPerChar);
-        if (!parser->m_declEntity->systemId)
+        parser->m_declEntity->systemId
+            = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                              next - enc->minBytesPerChar);
+        if (! parser->m_declEntity->systemId)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity->base = parser->m_curBase;
         poolFinish(&dtd->pool);
@@ -4782,115 +4572,103 @@ doProlog(XML_Parser parser,
       }
       break;
     case XML_ROLE_ENTITY_COMPLETE:
-      if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) {
+      if (dtd->keepProcessing && parser->m_declEntity
+          && parser->m_entityDeclHandler) {
         *eventEndPP = s;
-        parser->m_entityDeclHandler(parser->m_handlerArg,
-                          parser->m_declEntity->name,
-                          parser->m_declEntity->is_param,
-                          0,0,
-                          parser->m_declEntity->base,
-                          parser->m_declEntity->systemId,
-                          parser->m_declEntity->publicId,
-                          0);
+        parser->m_entityDeclHandler(
+            parser->m_handlerArg, parser->m_declEntity->name,
+            parser->m_declEntity->is_param, 0, 0, parser->m_declEntity->base,
+            parser->m_declEntity->systemId, parser->m_declEntity->publicId, 0);
         handleDefault = XML_FALSE;
       }
       break;
     case XML_ROLE_ENTITY_NOTATION_NAME:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        parser->m_declEntity->notation = poolStoreString(&dtd->pool, enc, s, next);
-        if (!parser->m_declEntity->notation)
+        parser->m_declEntity->notation
+            = poolStoreString(&dtd->pool, enc, s, next);
+        if (! parser->m_declEntity->notation)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&dtd->pool);
         if (parser->m_unparsedEntityDeclHandler) {
           *eventEndPP = s;
-          parser->m_unparsedEntityDeclHandler(parser->m_handlerArg,
-                                    parser->m_declEntity->name,
-                                    parser->m_declEntity->base,
-                                    parser->m_declEntity->systemId,
-                                    parser->m_declEntity->publicId,
-                                    parser->m_declEntity->notation);
+          parser->m_unparsedEntityDeclHandler(
+              parser->m_handlerArg, parser->m_declEntity->name,
+              parser->m_declEntity->base, parser->m_declEntity->systemId,
+              parser->m_declEntity->publicId, parser->m_declEntity->notation);
           handleDefault = XML_FALSE;
-        }
-        else if (parser->m_entityDeclHandler) {
+        } else if (parser->m_entityDeclHandler) {
           *eventEndPP = s;
-          parser->m_entityDeclHandler(parser->m_handlerArg,
-                            parser->m_declEntity->name,
-                            0,0,0,
-                            parser->m_declEntity->base,
-                            parser->m_declEntity->systemId,
-                            parser->m_declEntity->publicId,
-                            parser->m_declEntity->notation);
+          parser->m_entityDeclHandler(
+              parser->m_handlerArg, parser->m_declEntity->name, 0, 0, 0,
+              parser->m_declEntity->base, parser->m_declEntity->systemId,
+              parser->m_declEntity->publicId, parser->m_declEntity->notation);
           handleDefault = XML_FALSE;
         }
       }
       break;
-    case XML_ROLE_GENERAL_ENTITY_NAME:
-      {
-        if (XmlPredefinedEntityName(enc, s, next)) {
-          parser->m_declEntity = NULL;
-          break;
-        }
-        if (dtd->keepProcessing) {
-          const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
-          if (!name)
-            return XML_ERROR_NO_MEMORY;
-          parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->generalEntities, name,
-                                        sizeof(ENTITY));
-          if (!parser->m_declEntity)
-            return XML_ERROR_NO_MEMORY;
-          if (parser->m_declEntity->name != name) {
-            poolDiscard(&dtd->pool);
-            parser->m_declEntity = NULL;
-          }
-          else {
-            poolFinish(&dtd->pool);
-            parser->m_declEntity->publicId = NULL;
-            parser->m_declEntity->is_param = XML_FALSE;
-            /* if we have a parent parser or are reading an internal parameter
-               entity, then the entity declaration is not considered "internal"
-            */
-            parser->m_declEntity->is_internal = !(parser->m_parentParser || parser->m_openInternalEntities);
-            if (parser->m_entityDeclHandler)
-              handleDefault = XML_FALSE;
-          }
-        }
-        else {
+    case XML_ROLE_GENERAL_ENTITY_NAME: {
+      if (XmlPredefinedEntityName(enc, s, next)) {
+        parser->m_declEntity = NULL;
+        break;
+      }
+      if (dtd->keepProcessing) {
+        const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
+        if (! name)
+          return XML_ERROR_NO_MEMORY;
+        parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->generalEntities,
+                                                name, sizeof(ENTITY));
+        if (! parser->m_declEntity)
+          return XML_ERROR_NO_MEMORY;
+        if (parser->m_declEntity->name != name) {
           poolDiscard(&dtd->pool);
           parser->m_declEntity = NULL;
+        } else {
+          poolFinish(&dtd->pool);
+          parser->m_declEntity->publicId = NULL;
+          parser->m_declEntity->is_param = XML_FALSE;
+          /* if we have a parent parser or are reading an internal parameter
+             entity, then the entity declaration is not considered "internal"
+          */
+          parser->m_declEntity->is_internal
+              = ! (parser->m_parentParser || parser->m_openInternalEntities);
+          if (parser->m_entityDeclHandler)
+            handleDefault = XML_FALSE;
         }
+      } else {
+        poolDiscard(&dtd->pool);
+        parser->m_declEntity = NULL;
       }
-      break;
+    } break;
     case XML_ROLE_PARAM_ENTITY_NAME:
 #ifdef XML_DTD
       if (dtd->keepProcessing) {
         const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
-        if (!name)
+        if (! name)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->paramEntities,
-                                           name, sizeof(ENTITY));
-        if (!parser->m_declEntity)
+                                                name, sizeof(ENTITY));
+        if (! parser->m_declEntity)
           return XML_ERROR_NO_MEMORY;
         if (parser->m_declEntity->name != name) {
           poolDiscard(&dtd->pool);
           parser->m_declEntity = NULL;
-        }
-        else {
+        } else {
           poolFinish(&dtd->pool);
           parser->m_declEntity->publicId = NULL;
           parser->m_declEntity->is_param = XML_TRUE;
           /* if we have a parent parser or are reading an internal parameter
              entity, then the entity declaration is not considered "internal"
           */
-          parser->m_declEntity->is_internal = !(parser->m_parentParser || parser->m_openInternalEntities);
+          parser->m_declEntity->is_internal
+              = ! (parser->m_parentParser || parser->m_openInternalEntities);
           if (parser->m_entityDeclHandler)
             handleDefault = XML_FALSE;
         }
-      }
-      else {
+      } else {
         poolDiscard(&dtd->pool);
         parser->m_declEntity = NULL;
       }
-#else /* not XML_DTD */
+#else  /* not XML_DTD */
       parser->m_declEntity = NULL;
 #endif /* XML_DTD */
       break;
@@ -4898,22 +4676,23 @@ doProlog(XML_Parser parser,
       parser->m_declNotationPublicId = NULL;
       parser->m_declNotationName = NULL;
       if (parser->m_notationDeclHandler) {
-        parser->m_declNotationName = poolStoreString(&parser->m_tempPool, enc, s, next);
-        if (!parser->m_declNotationName)
+        parser->m_declNotationName
+            = poolStoreString(&parser->m_tempPool, enc, s, next);
+        if (! parser->m_declNotationName)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
       break;
     case XML_ROLE_NOTATION_PUBLIC_ID:
-      if (!XmlIsPublicId(enc, s, next, eventPP))
+      if (! XmlIsPublicId(enc, s, next, eventPP))
         return XML_ERROR_PUBLICID;
-      if (parser->m_declNotationName) {  /* means m_notationDeclHandler != NULL */
-        XML_Char *tem = poolStoreString(&parser->m_tempPool,
-                                        enc,
+      if (parser
+              ->m_declNotationName) { /* means m_notationDeclHandler != NULL */
+        XML_Char *tem = poolStoreString(&parser->m_tempPool, enc,
                                         s + enc->minBytesPerChar,
                                         next - enc->minBytesPerChar);
-        if (!tem)
+        if (! tem)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(tem);
         parser->m_declNotationPublicId = tem;
@@ -4923,18 +4702,15 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_NOTATION_SYSTEM_ID:
       if (parser->m_declNotationName && parser->m_notationDeclHandler) {
-        const XML_Char *systemId
-          = poolStoreString(&parser->m_tempPool, enc,
-                            s + enc->minBytesPerChar,
-                            next - enc->minBytesPerChar);
-        if (!systemId)
+        const XML_Char *systemId = poolStoreString(&parser->m_tempPool, enc,
+                                                   s + enc->minBytesPerChar,
+                                                   next - enc->minBytesPerChar);
+        if (! systemId)
           return XML_ERROR_NO_MEMORY;
         *eventEndPP = s;
-        parser->m_notationDeclHandler(parser->m_handlerArg,
-                            parser->m_declNotationName,
-                            parser->m_curBase,
-                            systemId,
-                            parser->m_declNotationPublicId);
+        parser->m_notationDeclHandler(
+            parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase,
+            systemId, parser->m_declNotationPublicId);
         handleDefault = XML_FALSE;
       }
       poolClear(&parser->m_tempPool);
@@ -4942,11 +4718,9 @@ doProlog(XML_Parser parser,
     case XML_ROLE_NOTATION_NO_SYSTEM_ID:
       if (parser->m_declNotationPublicId && parser->m_notationDeclHandler) {
         *eventEndPP = s;
-        parser->m_notationDeclHandler(parser->m_handlerArg,
-                            parser->m_declNotationName,
-                            parser->m_curBase,
-                            0,
-                            parser->m_declNotationPublicId);
+        parser->m_notationDeclHandler(
+            parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase,
+            0, parser->m_declNotationPublicId);
         handleDefault = XML_FALSE;
       }
       poolClear(&parser->m_tempPool);
@@ -4963,42 +4737,44 @@ doProlog(XML_Parser parser,
         return XML_ERROR_SYNTAX;
       }
 #ifdef XML_DTD
-    case XML_ROLE_IGNORE_SECT:
-      {
-        enum XML_Error result;
-        if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        handleDefault = XML_FALSE;
-        result = doIgnoreSection(parser, enc, &next, end, nextPtr, haveMore);
-        if (result != XML_ERROR_NONE)
-          return result;
-        else if (!next) {
-          parser->m_processor = ignoreSectionProcessor;
-          return result;
-        }
+    case XML_ROLE_IGNORE_SECT: {
+      enum XML_Error result;
+      if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      handleDefault = XML_FALSE;
+      result = doIgnoreSection(parser, enc, &next, end, nextPtr, haveMore);
+      if (result != XML_ERROR_NONE)
+        return result;
+      else if (! next) {
+        parser->m_processor = ignoreSectionProcessor;
+        return result;
       }
-      break;
+    } break;
 #endif /* XML_DTD */
     case XML_ROLE_GROUP_OPEN:
       if (parser->m_prologState.level >= parser->m_groupSize) {
         if (parser->m_groupSize) {
-          char *temp = (char *)REALLOC(parser, parser->m_groupConnector, parser->m_groupSize *= 2);
-          if (temp == NULL) {
-            parser->m_groupSize /= 2;
-            return XML_ERROR_NO_MEMORY;
+          {
+            char *const new_connector = (char *)REALLOC(
+                parser, parser->m_groupConnector, parser->m_groupSize *= 2);
+            if (new_connector == NULL) {
+              parser->m_groupSize /= 2;
+              return XML_ERROR_NO_MEMORY;
+            }
+            parser->m_groupConnector = new_connector;
           }
-          parser->m_groupConnector = temp;
+
           if (dtd->scaffIndex) {
-            int *temp = (int *)REALLOC(parser, dtd->scaffIndex,
-                          parser->m_groupSize * sizeof(int));
-            if (temp == NULL)
+            int *const new_scaff_index = (int *)REALLOC(
+                parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int));
+            if (new_scaff_index == NULL)
               return XML_ERROR_NO_MEMORY;
-            dtd->scaffIndex = temp;
+            dtd->scaffIndex = new_scaff_index;
           }
-        }
-        else {
-          parser->m_groupConnector = (char *)MALLOC(parser, parser->m_groupSize = 32);
-          if (!parser->m_groupConnector) {
+        } else {
+          parser->m_groupConnector
+              = (char *)MALLOC(parser, parser->m_groupSize = 32);
+          if (! parser->m_groupConnector) {
             parser->m_groupSize = 0;
             return XML_ERROR_NO_MEMORY;
           }
@@ -5009,6 +4785,7 @@ doProlog(XML_Parser parser,
         int myindex = nextScaffoldPart(parser);
         if (myindex < 0)
           return XML_ERROR_NO_MEMORY;
+        assert(dtd->scaffIndex != NULL);
         dtd->scaffIndex[dtd->scaffLevel] = myindex;
         dtd->scaffLevel++;
         dtd->scaffold[myindex].type = XML_CTYPE_SEQ;
@@ -5027,10 +4804,9 @@ doProlog(XML_Parser parser,
       if (parser->m_groupConnector[parser->m_prologState.level] == ASCII_COMMA)
         return XML_ERROR_SYNTAX;
       if (dtd->in_eldecl
-          && !parser->m_groupConnector[parser->m_prologState.level]
+          && ! parser->m_groupConnector[parser->m_prologState.level]
           && (dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]].type
-              != XML_CTYPE_MIXED)
-          ) {
+              != XML_CTYPE_MIXED)) {
         dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]].type
             = XML_CTYPE_CHOICE;
         if (parser->m_elementDeclHandler)
@@ -5042,15 +4818,14 @@ doProlog(XML_Parser parser,
 #ifdef XML_DTD
     case XML_ROLE_INNER_PARAM_ENTITY_REF:
       dtd->hasParamEntityRefs = XML_TRUE;
-      if (!parser->m_paramEntityParsing)
+      if (! parser->m_paramEntityParsing)
         dtd->keepProcessing = dtd->standalone;
       else {
         const XML_Char *name;
         ENTITY *entity;
-        name = poolStoreString(&dtd->pool, enc,
-                                s + enc->minBytesPerChar,
-                                next - enc->minBytesPerChar);
-        if (!name)
+        name = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                               next - enc->minBytesPerChar);
+        if (! name)
           return XML_ERROR_NO_MEMORY;
         entity = (ENTITY *)lookup(parser, &dtd->paramEntities, name, 0);
         poolDiscard(&dtd->pool);
@@ -5058,13 +4833,12 @@ doProlog(XML_Parser parser,
            if yes, check that the entity exists, and that it is internal,
            otherwise call the skipped entity handler
         */
-        if (parser->m_prologState.documentEntity &&
-            (dtd->standalone
-             ? !parser->m_openInternalEntities
-             : !dtd->hasParamEntityRefs)) {
-          if (!entity)
+        if (parser->m_prologState.documentEntity
+            && (dtd->standalone ? ! parser->m_openInternalEntities
+                                : ! dtd->hasParamEntityRefs)) {
+          if (! entity)
             return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal) {
+          else if (! entity->is_internal) {
             /* It's hard to exhaustively search the code to be sure,
              * but there doesn't seem to be a way of executing the
              * following line.  There are two cases:
@@ -5087,11 +4861,11 @@ doProlog(XML_Parser parser,
              */
             return XML_ERROR_ENTITY_DECLARED_IN_PE; /* LCOV_EXCL_LINE */
           }
-        }
-        else if (!entity) {
+        } else if (! entity) {
           dtd->keepProcessing = dtd->standalone;
           /* cannot report skipped entities in declarations */
-          if ((role == XML_ROLE_PARAM_ENTITY_REF) && parser->m_skippedEntityHandler) {
+          if ((role == XML_ROLE_PARAM_ENTITY_REF)
+              && parser->m_skippedEntityHandler) {
             parser->m_skippedEntityHandler(parser->m_handlerArg, name, 1);
             handleDefault = XML_FALSE;
           }
@@ -5101,8 +4875,8 @@ doProlog(XML_Parser parser,
           return XML_ERROR_RECURSIVE_ENTITY_REF;
         if (entity->textPtr) {
           enum XML_Error result;
-          XML_Bool betweenDecl =
-            (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE);
+          XML_Bool betweenDecl
+              = (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE);
           result = processInternalEntity(parser, entity, betweenDecl);
           if (result != XML_ERROR_NONE)
             return result;
@@ -5112,39 +4886,35 @@ doProlog(XML_Parser parser,
         if (parser->m_externalEntityRefHandler) {
           dtd->paramEntityRead = XML_FALSE;
           entity->open = XML_TRUE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId)) {
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId)) {
             entity->open = XML_FALSE;
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           }
           entity->open = XML_FALSE;
           handleDefault = XML_FALSE;
-          if (!dtd->paramEntityRead) {
+          if (! dtd->paramEntityRead) {
             dtd->keepProcessing = dtd->standalone;
             break;
           }
-        }
-        else {
+        } else {
           dtd->keepProcessing = dtd->standalone;
           break;
         }
       }
 #endif /* XML_DTD */
-      if (!dtd->standalone &&
-          parser->m_notStandaloneHandler &&
-          !parser->m_notStandaloneHandler(parser->m_handlerArg))
+      if (! dtd->standalone && parser->m_notStandaloneHandler
+          && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
         return XML_ERROR_NOT_STANDALONE;
       break;
 
-    /* Element declaration stuff */
+      /* Element declaration stuff */
 
     case XML_ROLE_ELEMENT_NAME:
       if (parser->m_elementDeclHandler) {
         parser->m_declElementType = getElementType(parser, enc, s, next);
-        if (!parser->m_declElementType)
+        if (! parser->m_declElementType)
           return XML_ERROR_NO_MEMORY;
         dtd->scaffLevel = 0;
         dtd->scaffCount = 0;
@@ -5157,18 +4927,19 @@ doProlog(XML_Parser parser,
     case XML_ROLE_CONTENT_EMPTY:
       if (dtd->in_eldecl) {
         if (parser->m_elementDeclHandler) {
-          XML_Content * content = (XML_Content *) MALLOC(parser, sizeof(XML_Content));
-          if (!content)
+          XML_Content *content
+              = (XML_Content *)MALLOC(parser, sizeof(XML_Content));
+          if (! content)
             return XML_ERROR_NO_MEMORY;
           content->quant = XML_CQUANT_NONE;
           content->name = NULL;
           content->numchildren = 0;
           content->children = NULL;
-          content->type = ((role == XML_ROLE_CONTENT_ANY) ?
-                           XML_CTYPE_ANY :
-                           XML_CTYPE_EMPTY);
+          content->type = ((role == XML_ROLE_CONTENT_ANY) ? XML_CTYPE_ANY
+                                                          : XML_CTYPE_EMPTY);
           *eventEndPP = s;
-          parser->m_elementDeclHandler(parser->m_handlerArg, parser->m_declElementType->name, content);
+          parser->m_elementDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name, content);
           handleDefault = XML_FALSE;
         }
         dtd->in_eldecl = XML_FALSE;
@@ -5200,22 +4971,22 @@ doProlog(XML_Parser parser,
         ELEMENT_TYPE *el;
         const XML_Char *name;
         int nameLen;
-        const char *nxt = (quant == XML_CQUANT_NONE
-                           ? next
-                           : next - enc->minBytesPerChar);
+        const char *nxt
+            = (quant == XML_CQUANT_NONE ? next : next - enc->minBytesPerChar);
         int myindex = nextScaffoldPart(parser);
         if (myindex < 0)
           return XML_ERROR_NO_MEMORY;
         dtd->scaffold[myindex].type = XML_CTYPE_NAME;
         dtd->scaffold[myindex].quant = quant;
         el = getElementType(parser, enc, s, nxt);
-        if (!el)
+        if (! el)
           return XML_ERROR_NO_MEMORY;
         name = el->name;
         dtd->scaffold[myindex].name = name;
         nameLen = 0;
-        for (; name[nameLen++]; );
-        dtd->contentStringLen +=  nameLen;
+        for (; name[nameLen++];)
+          ;
+        dtd->contentStringLen += nameLen;
         if (parser->m_elementDeclHandler)
           handleDefault = XML_FALSE;
       }
@@ -5239,12 +5010,13 @@ doProlog(XML_Parser parser,
         dtd->scaffLevel--;
         dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel]].quant = quant;
         if (dtd->scaffLevel == 0) {
-          if (!handleDefault) {
+          if (! handleDefault) {
             XML_Content *model = build_model(parser);
-            if (!model)
+            if (! model)
               return XML_ERROR_NO_MEMORY;
             *eventEndPP = s;
-            parser->m_elementDeclHandler(parser->m_handlerArg, parser->m_declElementType->name, model);
+            parser->m_elementDeclHandler(
+                parser->m_handlerArg, parser->m_declElementType->name, model);
           }
           dtd->in_eldecl = XML_FALSE;
           dtd->contentStringLen = 0;
@@ -5254,12 +5026,12 @@ doProlog(XML_Parser parser,
       /* End element declaration stuff */
 
     case XML_ROLE_PI:
-      if (!reportProcessingInstruction(parser, enc, s, next))
+      if (! reportProcessingInstruction(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       handleDefault = XML_FALSE;
       break;
     case XML_ROLE_COMMENT:
-      if (!reportComment(parser, enc, s, next))
+      if (! reportComment(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       handleDefault = XML_FALSE;
       break;
@@ -5310,11 +5082,8 @@ doProlog(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-epilogProcessor(XML_Parser parser,
-                const char *s,
-                const char *end,
-                const char **nextPtr)
-{
+epilogProcessor(XML_Parser parser, const char *s, const char *end,
+                const char **nextPtr) {
   parser->m_processor = epilogProcessor;
   parser->m_eventPtr = s;
   for (;;) {
@@ -5339,24 +5108,24 @@ epilogProcessor(XML_Parser parser,
         reportDefault(parser, parser->m_encoding, s, next);
       break;
     case XML_TOK_PI:
-      if (!reportProcessingInstruction(parser, parser->m_encoding, s, next))
+      if (! reportProcessingInstruction(parser, parser->m_encoding, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_COMMENT:
-      if (!reportComment(parser, parser->m_encoding, s, next))
+      if (! reportComment(parser, parser->m_encoding, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_INVALID:
       parser->m_eventPtr = next;
       return XML_ERROR_INVALID_TOKEN;
     case XML_TOK_PARTIAL:
-      if (!parser->m_parsingStatus.finalBuffer) {
+      if (! parser->m_parsingStatus.finalBuffer) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
       return XML_ERROR_UNCLOSED_TOKEN;
     case XML_TOK_PARTIAL_CHAR:
-      if (!parser->m_parsingStatus.finalBuffer) {
+      if (! parser->m_parsingStatus.finalBuffer) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -5371,15 +5140,13 @@ epilogProcessor(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
 }
 
 static enum XML_Error
-processInternalEntity(XML_Parser parser, ENTITY *entity,
-                      XML_Bool betweenDecl)
-{
+processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) {
   const char *textStart, *textEnd;
   const char *next;
   enum XML_Error result;
@@ -5388,10 +5155,10 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
   if (parser->m_freeInternalEntities) {
     openEntity = parser->m_freeInternalEntities;
     parser->m_freeInternalEntities = openEntity->next;
-  }
-  else {
-    openEntity = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY));
-    if (!openEntity)
+  } else {
+    openEntity
+        = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY));
+    if (! openEntity)
       return XML_ERROR_NO_MEMORY;
   }
   entity->open = XML_TRUE;
@@ -5410,21 +5177,20 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
 
 #ifdef XML_DTD
   if (entity->is_param) {
-    int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
-    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok,
-                      next, &next, XML_FALSE);
-  }
-  else
+    int tok
+        = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
+    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
+                      tok, next, &next, XML_FALSE, XML_FALSE);
+  } else
 #endif /* XML_DTD */
-    result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, textStart,
-                       textEnd, &next, XML_FALSE);
+    result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding,
+                       textStart, textEnd, &next, XML_FALSE);
 
   if (result == XML_ERROR_NONE) {
     if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
       entity->processed = (int)(next - textStart);
       parser->m_processor = internalEntityProcessor;
-    }
-    else {
+    } else {
       entity->open = XML_FALSE;
       parser->m_openInternalEntities = openEntity->next;
       /* put openEntity back in list of free instances */
@@ -5436,17 +5202,14 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
 }
 
 static enum XML_Error PTRCALL
-internalEntityProcessor(XML_Parser parser,
-                        const char *s,
-                        const char *end,
-                        const char **nextPtr)
-{
+internalEntityProcessor(XML_Parser parser, const char *s, const char *end,
+                        const char **nextPtr) {
   ENTITY *entity;
   const char *textStart, *textEnd;
   const char *next;
   enum XML_Error result;
   OPEN_INTERNAL_ENTITY *openEntity = parser->m_openInternalEntities;
-  if (!openEntity)
+  if (! openEntity)
     return XML_ERROR_UNEXPECTED_STATE;
 
   entity = openEntity->entity;
@@ -5457,22 +5220,23 @@ internalEntityProcessor(XML_Parser parser,
 
 #ifdef XML_DTD
   if (entity->is_param) {
-    int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
-    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok,
-                      next, &next, XML_FALSE);
-  }
-  else
+    int tok
+        = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
+    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
+                      tok, next, &next, XML_FALSE, XML_TRUE);
+  } else
 #endif /* XML_DTD */
-    result = doContent(parser, openEntity->startTagLevel, parser->m_internalEncoding,
-                       textStart, textEnd, &next, XML_FALSE);
+    result = doContent(parser, openEntity->startTagLevel,
+                       parser->m_internalEncoding, textStart, textEnd, &next,
+                       XML_FALSE);
 
   if (result != XML_ERROR_NONE)
     return result;
-  else if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
+  else if (textEnd != next
+           && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
     entity->processed = (int)(next - (char *)entity->textPtr);
     return result;
-  }
-  else {
+  } else {
     entity->open = XML_FALSE;
     parser->m_openInternalEntities = openEntity->next;
     /* put openEntity back in list of free instances */
@@ -5486,49 +5250,45 @@ internalEntityProcessor(XML_Parser parser,
     parser->m_processor = prologProcessor;
     tok = XmlPrologTok(parser->m_encoding, s, end, &next);
     return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
-                    (XML_Bool)!parser->m_parsingStatus.finalBuffer);
-  }
-  else
+                    (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
+  } else
 #endif /* XML_DTD */
   {
     parser->m_processor = contentProcessor;
     /* see externalEntityContentProcessor vs contentProcessor */
-    return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding, s, end,
-                     nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+    return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding,
+                     s, end, nextPtr,
+                     (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   }
 }
 
 static enum XML_Error PTRCALL
-errorProcessor(XML_Parser parser,
-               const char *UNUSED_P(s),
-               const char *UNUSED_P(end),
-               const char **UNUSED_P(nextPtr))
-{
+errorProcessor(XML_Parser parser, const char *s, const char *end,
+               const char **nextPtr) {
+  UNUSED_P(s);
+  UNUSED_P(end);
+  UNUSED_P(nextPtr);
   return parser->m_errorCode;
 }
 
 static enum XML_Error
 storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
-                    const char *ptr, const char *end,
-                    STRING_POOL *pool)
-{
-  enum XML_Error result = appendAttributeValue(parser, enc, isCdata, ptr,
-                                               end, pool);
+                    const char *ptr, const char *end, STRING_POOL *pool) {
+  enum XML_Error result
+      = appendAttributeValue(parser, enc, isCdata, ptr, end, pool);
   if (result)
     return result;
-  if (!isCdata && poolLength(pool) && poolLastChar(pool) == 0x20)
+  if (! isCdata && poolLength(pool) && poolLastChar(pool) == 0x20)
     poolChop(pool);
-  if (!poolAppendChar(pool, XML_T('\0')))
+  if (! poolAppendChar(pool, XML_T('\0')))
     return XML_ERROR_NO_MEMORY;
   return XML_ERROR_NONE;
 }
 
 static enum XML_Error
 appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
-                     const char *ptr, const char *end,
-                     STRING_POOL *pool)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+                     const char *ptr, const char *end, STRING_POOL *pool) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   for (;;) {
     const char *next;
     int tok = XmlAttributeValueTok(enc, ptr, end, &next);
@@ -5543,38 +5303,35 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
       if (enc == parser->m_encoding)
         parser->m_eventPtr = ptr;
       return XML_ERROR_INVALID_TOKEN;
-    case XML_TOK_CHAR_REF:
-      {
-        XML_Char buf[XML_ENCODE_MAX];
-        int i;
-        int n = XmlCharRefNumber(enc, ptr);
-        if (n < 0) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_BAD_CHAR_REF;
-        }
-        if (!isCdata
-            && n == 0x20 /* space */
-            && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
-          break;
-        n = XmlEncode(n, (ICHAR *)buf);
-        /* The XmlEncode() functions can never return 0 here.  That
-         * error return happens if the code point passed in is either
-         * negative or greater than or equal to 0x110000.  The
-         * XmlCharRefNumber() functions will all return a number
-         * strictly less than 0x110000 or a negative value if an error
-         * occurred.  The negative value is intercepted above, so
-         * XmlEncode() is never passed a value it might return an
-         * error for.
-         */
-        for (i = 0; i < n; i++) {
-          if (!poolAppendChar(pool, buf[i]))
-            return XML_ERROR_NO_MEMORY;
-        }
+    case XML_TOK_CHAR_REF: {
+      XML_Char buf[XML_ENCODE_MAX];
+      int i;
+      int n = XmlCharRefNumber(enc, ptr);
+      if (n < 0) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_BAD_CHAR_REF;
       }
-      break;
+      if (! isCdata && n == 0x20 /* space */
+          && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
+        break;
+      n = XmlEncode(n, (ICHAR *)buf);
+      /* The XmlEncode() functions can never return 0 here.  That
+       * error return happens if the code point passed in is either
+       * negative or greater than or equal to 0x110000.  The
+       * XmlCharRefNumber() functions will all return a number
+       * strictly less than 0x110000 or a negative value if an error
+       * occurred.  The negative value is intercepted above, so
+       * XmlEncode() is never passed a value it might return an
+       * error for.
+       */
+      for (i = 0; i < n; i++) {
+        if (! poolAppendChar(pool, buf[i]))
+          return XML_ERROR_NO_MEMORY;
+      }
+    } break;
     case XML_TOK_DATA_CHARS:
-      if (!poolAppend(pool, enc, ptr, next))
+      if (! poolAppend(pool, enc, ptr, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_TRAILING_CR:
@@ -5582,109 +5339,103 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
       /* fall through */
     case XML_TOK_ATTRIBUTE_VALUE_S:
     case XML_TOK_DATA_NEWLINE:
-      if (!isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
+      if (! isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
         break;
-      if (!poolAppendChar(pool, 0x20))
+      if (! poolAppendChar(pool, 0x20))
         return XML_ERROR_NO_MEMORY;
       break;
-    case XML_TOK_ENTITY_REF:
-      {
-        const XML_Char *name;
-        ENTITY *entity;
-        char checkEntityDecl;
-        XML_Char ch = (XML_Char) XmlPredefinedEntityName(enc,
-                                              ptr + enc->minBytesPerChar,
-                                              next - enc->minBytesPerChar);
-        if (ch) {
-          if (!poolAppendChar(pool, ch))
-                return XML_ERROR_NO_MEMORY;
-          break;
-        }
-        name = poolStoreString(&parser->m_temp2Pool, enc,
-                               ptr + enc->minBytesPerChar,
-                               next - enc->minBytesPerChar);
-        if (!name)
+    case XML_TOK_ENTITY_REF: {
+      const XML_Char *name;
+      ENTITY *entity;
+      char checkEntityDecl;
+      XML_Char ch = (XML_Char)XmlPredefinedEntityName(
+          enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar);
+      if (ch) {
+        if (! poolAppendChar(pool, ch))
           return XML_ERROR_NO_MEMORY;
-        entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
-        poolDiscard(&parser->m_temp2Pool);
-        /* First, determine if a check for an existing declaration is needed;
-           if yes, check that the entity exists, and that it is internal.
-        */
-        if (pool == &dtd->pool)  /* are we called from prolog? */
-          checkEntityDecl =
+        break;
+      }
+      name = poolStoreString(&parser->m_temp2Pool, enc,
+                             ptr + enc->minBytesPerChar,
+                             next - enc->minBytesPerChar);
+      if (! name)
+        return XML_ERROR_NO_MEMORY;
+      entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
+      poolDiscard(&parser->m_temp2Pool);
+      /* First, determine if a check for an existing declaration is needed;
+         if yes, check that the entity exists, and that it is internal.
+      */
+      if (pool == &dtd->pool) /* are we called from prolog? */
+        checkEntityDecl =
 #ifdef XML_DTD
-              parser->m_prologState.documentEntity &&
+            parser->m_prologState.documentEntity &&
 #endif /* XML_DTD */
-              (dtd->standalone
-               ? !parser->m_openInternalEntities
-               : !dtd->hasParamEntityRefs);
-        else /* if (pool == &parser->m_tempPool): we are called from content */
-          checkEntityDecl = !dtd->hasParamEntityRefs || dtd->standalone;
-        if (checkEntityDecl) {
-          if (!entity)
-            return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal)
-            return XML_ERROR_ENTITY_DECLARED_IN_PE;
-        }
-        else if (!entity) {
-          /* Cannot report skipped entity here - see comments on
-             parser->m_skippedEntityHandler.
-          if (parser->m_skippedEntityHandler)
-            parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
-          */
-          /* Cannot call the default handler because this would be
-             out of sync with the call to the startElementHandler.
-          if ((pool == &parser->m_tempPool) && parser->m_defaultHandler)
-            reportDefault(parser, enc, ptr, next);
-          */
-          break;
-        }
-        if (entity->open) {
-          if (enc == parser->m_encoding) {
-            /* It does not appear that this line can be executed.
-             *
-             * The "if (entity->open)" check catches recursive entity
-             * definitions.  In order to be called with an open
-             * entity, it must have gone through this code before and
-             * been through the recursive call to
-             * appendAttributeValue() some lines below.  That call
-             * sets the local encoding ("enc") to the parser's
-             * internal encoding (internal_utf8 or internal_utf16),
-             * which can never be the same as the principle encoding.
-             * It doesn't appear there is another code path that gets
-             * here with entity->open being TRUE.
-             *
-             * Since it is not certain that this logic is watertight,
-             * we keep the line and merely exclude it from coverage
-             * tests.
-             */
-            parser->m_eventPtr = ptr; /* LCOV_EXCL_LINE */
-          }
-          return XML_ERROR_RECURSIVE_ENTITY_REF;
-        }
-        if (entity->notation) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_BINARY_ENTITY_REF;
-        }
-        if (!entity->textPtr) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF;
-        }
-        else {
-          enum XML_Error result;
-          const XML_Char *textEnd = entity->textPtr + entity->textLen;
-          entity->open = XML_TRUE;
-          result = appendAttributeValue(parser, parser->m_internalEncoding, isCdata,
-                                        (char *)entity->textPtr,
-                                        (char *)textEnd, pool);
-          entity->open = XML_FALSE;
-          if (result)
-            return result;
+            (dtd->standalone ? ! parser->m_openInternalEntities
+                             : ! dtd->hasParamEntityRefs);
+      else /* if (pool == &parser->m_tempPool): we are called from content */
+        checkEntityDecl = ! dtd->hasParamEntityRefs || dtd->standalone;
+      if (checkEntityDecl) {
+        if (! entity)
+          return XML_ERROR_UNDEFINED_ENTITY;
+        else if (! entity->is_internal)
+          return XML_ERROR_ENTITY_DECLARED_IN_PE;
+      } else if (! entity) {
+        /* Cannot report skipped entity here - see comments on
+           parser->m_skippedEntityHandler.
+        if (parser->m_skippedEntityHandler)
+          parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+        */
+        /* Cannot call the default handler because this would be
+           out of sync with the call to the startElementHandler.
+        if ((pool == &parser->m_tempPool) && parser->m_defaultHandler)
+          reportDefault(parser, enc, ptr, next);
+        */
+        break;
+      }
+      if (entity->open) {
+        if (enc == parser->m_encoding) {
+          /* It does not appear that this line can be executed.
+           *
+           * The "if (entity->open)" check catches recursive entity
+           * definitions.  In order to be called with an open
+           * entity, it must have gone through this code before and
+           * been through the recursive call to
+           * appendAttributeValue() some lines below.  That call
+           * sets the local encoding ("enc") to the parser's
+           * internal encoding (internal_utf8 or internal_utf16),
+           * which can never be the same as the principle encoding.
+           * It doesn't appear there is another code path that gets
+           * here with entity->open being TRUE.
+           *
+           * Since it is not certain that this logic is watertight,
+           * we keep the line and merely exclude it from coverage
+           * tests.
+           */
+          parser->m_eventPtr = ptr; /* LCOV_EXCL_LINE */
         }
+        return XML_ERROR_RECURSIVE_ENTITY_REF;
       }
-      break;
+      if (entity->notation) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_BINARY_ENTITY_REF;
+      }
+      if (! entity->textPtr) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF;
+      } else {
+        enum XML_Error result;
+        const XML_Char *textEnd = entity->textPtr + entity->textLen;
+        entity->open = XML_TRUE;
+        result = appendAttributeValue(parser, parser->m_internalEncoding,
+                                      isCdata, (char *)entity->textPtr,
+                                      (char *)textEnd, pool);
+        entity->open = XML_FALSE;
+        if (result)
+          return result;
+      }
+    } break;
     default:
       /* The only token returned by XmlAttributeValueTok() that does
        * not have an explicit case here is XML_TOK_PARTIAL_CHAR.
@@ -5708,12 +5459,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
 }
 
 static enum XML_Error
-storeEntityValue(XML_Parser parser,
-                 const ENCODING *enc,
-                 const char *entityTextPtr,
-                 const char *entityTextEnd)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+storeEntityValue(XML_Parser parser, const ENCODING *enc,
+                 const char *entityTextPtr, const char *entityTextEnd) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   STRING_POOL *pool = &(dtd->entityValuePool);
   enum XML_Error result = XML_ERROR_NONE;
 #ifdef XML_DTD
@@ -5723,8 +5471,8 @@ storeEntityValue(XML_Parser parser,
   /* never return Null for the value argument in EntityDeclHandler,
      since this would indicate an external entity; therefore we
      have to make sure that entityValuePool.start is not null */
-  if (!pool->blocks) {
-    if (!poolGrow(pool))
+  if (! pool->blocks) {
+    if (! poolGrow(pool))
       return XML_ERROR_NO_MEMORY;
   }
 
@@ -5740,13 +5488,13 @@ storeEntityValue(XML_Parser parser,
         name = poolStoreString(&parser->m_tempPool, enc,
                                entityTextPtr + enc->minBytesPerChar,
                                next - enc->minBytesPerChar);
-        if (!name) {
+        if (! name) {
           result = XML_ERROR_NO_MEMORY;
           goto endEntityValue;
         }
         entity = (ENTITY *)lookup(parser, &dtd->paramEntities, name, 0);
         poolDiscard(&parser->m_tempPool);
-        if (!entity) {
+        if (! entity) {
           /* not a well-formedness error - see XML 1.0: WFC Entity Declared */
           /* cannot report skipped entity here - see comments on
              parser->m_skippedEntityHandler
@@ -5766,29 +5514,23 @@ storeEntityValue(XML_Parser parser,
           if (parser->m_externalEntityRefHandler) {
             dtd->paramEntityRead = XML_FALSE;
             entity->open = XML_TRUE;
-            if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                          0,
-                                          entity->base,
-                                          entity->systemId,
-                                          entity->publicId)) {
+            if (! parser->m_externalEntityRefHandler(
+                    parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                    entity->systemId, entity->publicId)) {
               entity->open = XML_FALSE;
               result = XML_ERROR_EXTERNAL_ENTITY_HANDLING;
               goto endEntityValue;
             }
             entity->open = XML_FALSE;
-            if (!dtd->paramEntityRead)
+            if (! dtd->paramEntityRead)
               dtd->keepProcessing = dtd->standalone;
-          }
-          else
+          } else
             dtd->keepProcessing = dtd->standalone;
-        }
-        else {
+        } else {
           entity->open = XML_TRUE;
-          result = storeEntityValue(parser,
-                                    parser->m_internalEncoding,
-                                    (char *)entity->textPtr,
-                                    (char *)(entity->textPtr
-                                             + entity->textLen));
+          result = storeEntityValue(
+              parser, parser->m_internalEncoding, (char *)entity->textPtr,
+              (char *)(entity->textPtr + entity->textLen));
           entity->open = XML_FALSE;
           if (result)
             goto endEntityValue;
@@ -5806,7 +5548,7 @@ storeEntityValue(XML_Parser parser,
       goto endEntityValue;
     case XML_TOK_ENTITY_REF:
     case XML_TOK_DATA_CHARS:
-      if (!poolAppend(pool, enc, entityTextPtr, next)) {
+      if (! poolAppend(pool, enc, entityTextPtr, next)) {
         result = XML_ERROR_NO_MEMORY;
         goto endEntityValue;
       }
@@ -5815,42 +5557,40 @@ storeEntityValue(XML_Parser parser,
       next = entityTextPtr + enc->minBytesPerChar;
       /* fall through */
     case XML_TOK_DATA_NEWLINE:
-      if (pool->end == pool->ptr && !poolGrow(pool)) {
-              result = XML_ERROR_NO_MEMORY;
+      if (pool->end == pool->ptr && ! poolGrow(pool)) {
+        result = XML_ERROR_NO_MEMORY;
         goto endEntityValue;
       }
       *(pool->ptr)++ = 0xA;
       break;
-    case XML_TOK_CHAR_REF:
-      {
-        XML_Char buf[XML_ENCODE_MAX];
-        int i;
-        int n = XmlCharRefNumber(enc, entityTextPtr);
-        if (n < 0) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = entityTextPtr;
-          result = XML_ERROR_BAD_CHAR_REF;
+    case XML_TOK_CHAR_REF: {
+      XML_Char buf[XML_ENCODE_MAX];
+      int i;
+      int n = XmlCharRefNumber(enc, entityTextPtr);
+      if (n < 0) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = entityTextPtr;
+        result = XML_ERROR_BAD_CHAR_REF;
+        goto endEntityValue;
+      }
+      n = XmlEncode(n, (ICHAR *)buf);
+      /* The XmlEncode() functions can never return 0 here.  That
+       * error return happens if the code point passed in is either
+       * negative or greater than or equal to 0x110000.  The
+       * XmlCharRefNumber() functions will all return a number
+       * strictly less than 0x110000 or a negative value if an error
+       * occurred.  The negative value is intercepted above, so
+       * XmlEncode() is never passed a value it might return an
+       * error for.
+       */
+      for (i = 0; i < n; i++) {
+        if (pool->end == pool->ptr && ! poolGrow(pool)) {
+          result = XML_ERROR_NO_MEMORY;
           goto endEntityValue;
         }
-        n = XmlEncode(n, (ICHAR *)buf);
-        /* The XmlEncode() functions can never return 0 here.  That
-         * error return happens if the code point passed in is either
-         * negative or greater than or equal to 0x110000.  The
-         * XmlCharRefNumber() functions will all return a number
-         * strictly less than 0x110000 or a negative value if an error
-         * occurred.  The negative value is intercepted above, so
-         * XmlEncode() is never passed a value it might return an
-         * error for.
-         */
-        for (i = 0; i < n; i++) {
-          if (pool->end == pool->ptr && !poolGrow(pool)) {
-            result = XML_ERROR_NO_MEMORY;
-            goto endEntityValue;
-          }
-          *(pool->ptr)++ = buf[i];
-        }
+        *(pool->ptr)++ = buf[i];
       }
-      break;
+    } break;
     case XML_TOK_PARTIAL:
       if (enc == parser->m_encoding)
         parser->m_eventPtr = entityTextPtr;
@@ -5885,8 +5625,7 @@ storeEntityValue(XML_Parser parser,
 }
 
 static void FASTCALL
-normalizeLines(XML_Char *s)
-{
+normalizeLines(XML_Char *s) {
   XML_Char *p;
   for (;; s++) {
     if (*s == XML_T('\0'))
@@ -5900,8 +5639,7 @@ normalizeLines(XML_Char *s)
       *p++ = 0xA;
       if (*++s == 0xA)
         s++;
-    }
-    else
+    } else
       *p++ = *s++;
   } while (*s);
   *p = XML_T('\0');
@@ -5909,12 +5647,11 @@ normalizeLines(XML_Char *s)
 
 static int
 reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
-                            const char *start, const char *end)
-{
+                            const char *start, const char *end) {
   const XML_Char *target;
   XML_Char *data;
   const char *tem;
-  if (!parser->m_processingInstructionHandler) {
+  if (! parser->m_processingInstructionHandler) {
     if (parser->m_defaultHandler)
       reportDefault(parser, enc, start, end);
     return 1;
@@ -5922,13 +5659,12 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
   start += enc->minBytesPerChar * 2;
   tem = start + XmlNameLength(enc, start);
   target = poolStoreString(&parser->m_tempPool, enc, start, tem);
-  if (!target)
+  if (! target)
     return 0;
   poolFinish(&parser->m_tempPool);
-  data = poolStoreString(&parser->m_tempPool, enc,
-                        XmlSkipS(enc, tem),
-                        end - enc->minBytesPerChar*2);
-  if (!data)
+  data = poolStoreString(&parser->m_tempPool, enc, XmlSkipS(enc, tem),
+                         end - enc->minBytesPerChar * 2);
+  if (! data)
     return 0;
   normalizeLines(data);
   parser->m_processingInstructionHandler(parser->m_handlerArg, target, data);
@@ -5937,20 +5673,18 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
 }
 
 static int
-reportComment(XML_Parser parser, const ENCODING *enc,
-              const char *start, const char *end)
-{
+reportComment(XML_Parser parser, const ENCODING *enc, const char *start,
+              const char *end) {
   XML_Char *data;
-  if (!parser->m_commentHandler) {
+  if (! parser->m_commentHandler) {
     if (parser->m_defaultHandler)
       reportDefault(parser, enc, start, end);
     return 1;
   }
-  data = poolStoreString(&parser->m_tempPool,
-                         enc,
+  data = poolStoreString(&parser->m_tempPool, enc,
                          start + enc->minBytesPerChar * 4,
                          end - enc->minBytesPerChar * 3);
-  if (!data)
+  if (! data)
     return 0;
   normalizeLines(data);
   parser->m_commentHandler(parser->m_handlerArg, data);
@@ -5959,9 +5693,8 @@ reportComment(XML_Parser parser, const ENCODING *enc,
 }
 
 static void
-reportDefault(XML_Parser parser, const ENCODING *enc,
-              const char *s, const char *end)
-{
+reportDefault(XML_Parser parser, const ENCODING *enc, const char *s,
+              const char *end) {
   if (MUST_CONVERT(enc, s)) {
     enum XML_Convert_Result convert_res;
     const char **eventPP;
@@ -5969,8 +5702,7 @@ reportDefault(XML_Parser parser, const ENCODING *enc,
     if (enc == parser->m_encoding) {
       eventPP = &parser->m_eventPtr;
       eventEndPP = &parser->m_eventEndPtr;
-    }
-    else {
+    } else {
       /* To get here, two things must be true; the parser must be
        * using a character encoding that is not the same as the
        * encoding passed in, and the encoding passed in must need
@@ -5993,21 +5725,22 @@ reportDefault(XML_Parser parser, const ENCODING *enc,
     }
     do {
       ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-      convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+      convert_res
+          = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
       *eventEndPP = s;
-      parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+      parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf,
+                               (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
       *eventPP = s;
-    } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE));
-  }
-  else
-    parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s, (int)((XML_Char *)end - (XML_Char *)s));
+    } while ((convert_res != XML_CONVERT_COMPLETED)
+             && (convert_res != XML_CONVERT_INPUT_INCOMPLETE));
+  } else
+    parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s,
+                             (int)((XML_Char *)end - (XML_Char *)s));
 }
 
-
 static int
 defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
-                XML_Bool isId, const XML_Char *value, XML_Parser parser)
-{
+                XML_Bool isId, const XML_Char *value, XML_Parser parser) {
   DEFAULT_ATTRIBUTE *att;
   if (value || isId) {
     /* The handling of default attributes gets messed up if we have
@@ -6016,24 +5749,23 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
     for (i = 0; i < type->nDefaultAtts; i++)
       if (attId == type->defaultAtts[i].id)
         return 1;
-    if (isId && !type->idAtt && !attId->xmlns)
+    if (isId && ! type->idAtt && ! attId->xmlns)
       type->idAtt = attId;
   }
   if (type->nDefaultAtts == type->allocDefaultAtts) {
     if (type->allocDefaultAtts == 0) {
       type->allocDefaultAtts = 8;
-      type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC(parser, type->allocDefaultAtts
-                            * sizeof(DEFAULT_ATTRIBUTE));
-      if (!type->defaultAtts) {
+      type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC(
+          parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
+      if (! type->defaultAtts) {
         type->allocDefaultAtts = 0;
         return 0;
       }
-    }
-    else {
+    } else {
       DEFAULT_ATTRIBUTE *temp;
       int count = type->allocDefaultAtts * 2;
-      temp = (DEFAULT_ATTRIBUTE *)
-        REALLOC(parser, type->defaultAtts, (count * sizeof(DEFAULT_ATTRIBUTE)));
+      temp = (DEFAULT_ATTRIBUTE *)REALLOC(parser, type->defaultAtts,
+                                          (count * sizeof(DEFAULT_ATTRIBUTE)));
       if (temp == NULL)
         return 0;
       type->allocDefaultAtts = count;
@@ -6044,30 +5776,29 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
   att->id = attId;
   att->value = value;
   att->isCdata = isCdata;
-  if (!isCdata)
+  if (! isCdata)
     attId->maybeTokenized = XML_TRUE;
   type->nDefaultAtts += 1;
   return 1;
 }
 
 static int
-setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *name;
   for (name = elementType->name; *name; name++) {
     if (*name == XML_T(ASCII_COLON)) {
       PREFIX *prefix;
       const XML_Char *s;
       for (s = elementType->name; s != name; s++) {
-        if (!poolAppendChar(&dtd->pool, *s))
+        if (! poolAppendChar(&dtd->pool, *s))
           return 0;
       }
-      if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+      if (! poolAppendChar(&dtd->pool, XML_T('\0')))
         return 0;
       prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&dtd->pool),
                                 sizeof(PREFIX));
-      if (!prefix)
+      if (! prefix)
         return 0;
       if (prefix->name == poolStart(&dtd->pool))
         poolFinish(&dtd->pool);
@@ -6081,55 +5812,53 @@ setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType)
 }
 
 static ATTRIBUTE_ID *
-getAttributeId(XML_Parser parser, const ENCODING *enc,
-               const char *start, const char *end)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start,
+               const char *end) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   ATTRIBUTE_ID *id;
   const XML_Char *name;
-  if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+  if (! poolAppendChar(&dtd->pool, XML_T('\0')))
     return NULL;
   name = poolStoreString(&dtd->pool, enc, start, end);
-  if (!name)
+  if (! name)
     return NULL;
   /* skip quotation mark - its storage will be re-used (like in name[-1]) */
   ++name;
-  id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name, sizeof(ATTRIBUTE_ID));
-  if (!id)
+  id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name,
+                              sizeof(ATTRIBUTE_ID));
+  if (! id)
     return NULL;
   if (id->name != name)
     poolDiscard(&dtd->pool);
   else {
     poolFinish(&dtd->pool);
-    if (!parser->m_ns)
+    if (! parser->m_ns)
       ;
-    else if (name[0] == XML_T(ASCII_x)
-        && name[1] == XML_T(ASCII_m)
-        && name[2] == XML_T(ASCII_l)
-        && name[3] == XML_T(ASCII_n)
-        && name[4] == XML_T(ASCII_s)
-        && (name[5] == XML_T('\0') || name[5] == XML_T(ASCII_COLON))) {
+    else if (name[0] == XML_T(ASCII_x) && name[1] == XML_T(ASCII_m)
+             && name[2] == XML_T(ASCII_l) && name[3] == XML_T(ASCII_n)
+             && name[4] == XML_T(ASCII_s)
+             && (name[5] == XML_T('\0') || name[5] == XML_T(ASCII_COLON))) {
       if (name[5] == XML_T('\0'))
         id->prefix = &dtd->defaultPrefix;
       else
-        id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6, sizeof(PREFIX));
+        id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6,
+                                      sizeof(PREFIX));
       id->xmlns = XML_TRUE;
-    }
-    else {
+    } else {
       int i;
       for (i = 0; name[i]; i++) {
         /* attributes without prefix are *not* in the default namespace */
         if (name[i] == XML_T(ASCII_COLON)) {
           int j;
           for (j = 0; j < i; j++) {
-            if (!poolAppendChar(&dtd->pool, name[j]))
+            if (! poolAppendChar(&dtd->pool, name[j]))
               return NULL;
           }
-          if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+          if (! poolAppendChar(&dtd->pool, XML_T('\0')))
             return NULL;
-          id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&dtd->pool),
-                                        sizeof(PREFIX));
-          if (!id->prefix)
+          id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes,
+                                        poolStart(&dtd->pool), sizeof(PREFIX));
+          if (! id->prefix)
             return NULL;
           if (id->prefix->name == poolStart(&dtd->pool))
             poolFinish(&dtd->pool);
@@ -6146,22 +5875,22 @@ getAttributeId(XML_Parser parser, const ENCODING *enc,
 #define CONTEXT_SEP XML_T(ASCII_FF)
 
 static const XML_Char *
-getContext(XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getContext(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   HASH_TABLE_ITER iter;
   XML_Bool needSep = XML_FALSE;
 
   if (dtd->defaultPrefix.binding) {
     int i;
     int len;
-    if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
+    if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
       return NULL;
     len = dtd->defaultPrefix.binding->uriLen;
     if (parser->m_namespaceSeparator)
       len--;
     for (i = 0; i < len; i++) {
-      if (!poolAppendChar(&parser->m_tempPool, dtd->defaultPrefix.binding->uri[i])) {
+      if (! poolAppendChar(&parser->m_tempPool,
+                           dtd->defaultPrefix.binding->uri[i])) {
         /* Because of memory caching, I don't believe this line can be
          * executed.
          *
@@ -6193,9 +5922,9 @@ getContext(XML_Parser parser)
     int len;
     const XML_Char *s;
     PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter);
-    if (!prefix)
+    if (! prefix)
       break;
-    if (!prefix->binding) {
+    if (! prefix->binding) {
       /* This test appears to be (justifiable) paranoia.  There does
        * not seem to be a way of injecting a prefix without a binding
        * that doesn't get errored long before this function is called.
@@ -6204,98 +5933,96 @@ getContext(XML_Parser parser)
        */
       continue; /* LCOV_EXCL_LINE */
     }
-    if (needSep && !poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
+    if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
       return NULL;
     for (s = prefix->name; *s; s++)
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return NULL;
-    if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
+    if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
       return NULL;
     len = prefix->binding->uriLen;
     if (parser->m_namespaceSeparator)
       len--;
     for (i = 0; i < len; i++)
-      if (!poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i]))
+      if (! poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i]))
         return NULL;
     needSep = XML_TRUE;
   }
 
-
   hashTableIterInit(&iter, &(dtd->generalEntities));
   for (;;) {
     const XML_Char *s;
     ENTITY *e = (ENTITY *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
-    if (!e->open)
+    if (! e->open)
       continue;
-    if (needSep && !poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
+    if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
       return NULL;
     for (s = e->name; *s; s++)
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return 0;
     needSep = XML_TRUE;
   }
 
-  if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+  if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
     return NULL;
   return parser->m_tempPool.start;
 }
 
 static XML_Bool
-setContext(XML_Parser parser, const XML_Char *context)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+setContext(XML_Parser parser, const XML_Char *context) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *s = context;
 
   while (*context != XML_T('\0')) {
     if (*s == CONTEXT_SEP || *s == XML_T('\0')) {
       ENTITY *e;
-      if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+      if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
         return XML_FALSE;
-      e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0);
+      e = (ENTITY *)lookup(parser, &dtd->generalEntities,
+                           poolStart(&parser->m_tempPool), 0);
       if (e)
         e->open = XML_TRUE;
       if (*s != XML_T('\0'))
         s++;
       context = s;
       poolDiscard(&parser->m_tempPool);
-    }
-    else if (*s == XML_T(ASCII_EQUALS)) {
+    } else if (*s == XML_T(ASCII_EQUALS)) {
       PREFIX *prefix;
       if (poolLength(&parser->m_tempPool) == 0)
         prefix = &dtd->defaultPrefix;
       else {
-        if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+        if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
           return XML_FALSE;
-        prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&parser->m_tempPool),
-                                  sizeof(PREFIX));
-        if (!prefix)
+        prefix
+            = (PREFIX *)lookup(parser, &dtd->prefixes,
+                               poolStart(&parser->m_tempPool), sizeof(PREFIX));
+        if (! prefix)
           return XML_FALSE;
         if (prefix->name == poolStart(&parser->m_tempPool)) {
           prefix->name = poolCopyString(&dtd->pool, prefix->name);
-          if (!prefix->name)
+          if (! prefix->name)
             return XML_FALSE;
         }
         poolDiscard(&parser->m_tempPool);
       }
-      for (context = s + 1;
-           *context != CONTEXT_SEP && *context != XML_T('\0');
+      for (context = s + 1; *context != CONTEXT_SEP && *context != XML_T('\0');
            context++)
-        if (!poolAppendChar(&parser->m_tempPool, *context))
+        if (! poolAppendChar(&parser->m_tempPool, *context))
           return XML_FALSE;
-      if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+      if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
         return XML_FALSE;
       if (addBinding(parser, prefix, NULL, poolStart(&parser->m_tempPool),
-                     &parser->m_inheritedBindings) != XML_ERROR_NONE)
+                     &parser->m_inheritedBindings)
+          != XML_ERROR_NONE)
         return XML_FALSE;
       poolDiscard(&parser->m_tempPool);
       if (*context != XML_T('\0'))
         ++context;
       s = context;
-    }
-    else {
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+    } else {
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return XML_FALSE;
       s++;
     }
@@ -6304,8 +6031,7 @@ setContext(XML_Parser parser, const XML_Char *context)
 }
 
 static void FASTCALL
-normalizePublicId(XML_Char *publicId)
-{
+normalizePublicId(XML_Char *publicId) {
   XML_Char *p = publicId;
   XML_Char *s;
   for (s = publicId; *s; s++) {
@@ -6326,8 +6052,7 @@ normalizePublicId(XML_Char *publicId)
 }
 
 static DTD *
-dtdCreate(const XML_Memory_Handling_Suite *ms)
-{
+dtdCreate(const XML_Memory_Handling_Suite *ms) {
   DTD *p = (DTD *)ms->malloc_fcn(sizeof(DTD));
   if (p == NULL)
     return p;
@@ -6359,13 +6084,12 @@ dtdCreate(const XML_Memory_Handling_Suite *ms)
 }
 
 static void
-dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms)
-{
+dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
   hashTableIterInit(&iter, &(p->elementTypes));
   for (;;) {
     ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
     if (e->allocDefaultAtts != 0)
       ms->free_fcn(e->defaultAtts);
@@ -6401,13 +6125,12 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms)
 }
 
 static void
-dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms)
-{
+dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
   hashTableIterInit(&iter, &(p->elementTypes));
   for (;;) {
     ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
     if (e->allocDefaultAtts != 0)
       ms->free_fcn(e->defaultAtts);
@@ -6432,8 +6155,8 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms)
    The new DTD has already been initialized.
 */
 static int
-dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms)
-{
+dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
+        const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
 
   /* Copy the prefix table. */
@@ -6442,12 +6165,12 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
   for (;;) {
     const XML_Char *name;
     const PREFIX *oldP = (PREFIX *)hashTableIterNext(&iter);
-    if (!oldP)
+    if (! oldP)
       break;
     name = poolCopyString(&(newDtd->pool), oldP->name);
-    if (!name)
+    if (! name)
       return 0;
-    if (!lookup(oldParser, &(newDtd->prefixes), name, sizeof(PREFIX)))
+    if (! lookup(oldParser, &(newDtd->prefixes), name, sizeof(PREFIX)))
       return 0;
   }
 
@@ -6460,18 +6183,18 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
     const XML_Char *name;
     const ATTRIBUTE_ID *oldA = (ATTRIBUTE_ID *)hashTableIterNext(&iter);
 
-    if (!oldA)
+    if (! oldA)
       break;
     /* Remember to allocate the scratch byte before the name. */
-    if (!poolAppendChar(&(newDtd->pool), XML_T('\0')))
+    if (! poolAppendChar(&(newDtd->pool), XML_T('\0')))
       return 0;
     name = poolCopyString(&(newDtd->pool), oldA->name);
-    if (!name)
+    if (! name)
       return 0;
     ++name;
     newA = (ATTRIBUTE_ID *)lookup(oldParser, &(newDtd->attributeIds), name,
                                   sizeof(ATTRIBUTE_ID));
-    if (!newA)
+    if (! newA)
       return 0;
     newA->maybeTokenized = oldA->maybeTokenized;
     if (oldA->prefix) {
@@ -6493,57 +6216,52 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
     ELEMENT_TYPE *newE;
     const XML_Char *name;
     const ELEMENT_TYPE *oldE = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!oldE)
+    if (! oldE)
       break;
     name = poolCopyString(&(newDtd->pool), oldE->name);
-    if (!name)
+    if (! name)
       return 0;
     newE = (ELEMENT_TYPE *)lookup(oldParser, &(newDtd->elementTypes), name,
                                   sizeof(ELEMENT_TYPE));
-    if (!newE)
+    if (! newE)
       return 0;
     if (oldE->nDefaultAtts) {
-      newE->defaultAtts = (DEFAULT_ATTRIBUTE *)
-          ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
-      if (!newE->defaultAtts) {
+      newE->defaultAtts = (DEFAULT_ATTRIBUTE *)ms->malloc_fcn(
+          oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
+      if (! newE->defaultAtts) {
         return 0;
       }
     }
     if (oldE->idAtt)
-      newE->idAtt = (ATTRIBUTE_ID *)
-          lookup(oldParser, &(newDtd->attributeIds), oldE->idAtt->name, 0);
+      newE->idAtt = (ATTRIBUTE_ID *)lookup(oldParser, &(newDtd->attributeIds),
+                                           oldE->idAtt->name, 0);
     newE->allocDefaultAtts = newE->nDefaultAtts = oldE->nDefaultAtts;
     if (oldE->prefix)
       newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes),
                                       oldE->prefix->name, 0);
     for (i = 0; i < newE->nDefaultAtts; i++) {
-      newE->defaultAtts[i].id = (ATTRIBUTE_ID *)
-          lookup(oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0);
+      newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup(
+          oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0);
       newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata;
       if (oldE->defaultAtts[i].value) {
         newE->defaultAtts[i].value
             = poolCopyString(&(newDtd->pool), oldE->defaultAtts[i].value);
-        if (!newE->defaultAtts[i].value)
+        if (! newE->defaultAtts[i].value)
           return 0;
-      }
-      else
+      } else
         newE->defaultAtts[i].value = NULL;
     }
   }
 
   /* Copy the entity tables. */
-  if (!copyEntityTable(oldParser,
-                       &(newDtd->generalEntities),
-                       &(newDtd->pool),
-                       &(oldDtd->generalEntities)))
-      return 0;
+  if (! copyEntityTable(oldParser, &(newDtd->generalEntities), &(newDtd->pool),
+                        &(oldDtd->generalEntities)))
+    return 0;
 
 #ifdef XML_DTD
-  if (!copyEntityTable(oldParser,
-                       &(newDtd->paramEntities),
-                       &(newDtd->pool),
-                       &(oldDtd->paramEntities)))
-      return 0;
+  if (! copyEntityTable(oldParser, &(newDtd->paramEntities), &(newDtd->pool),
+                        &(oldDtd->paramEntities)))
+    return 0;
   newDtd->paramEntityRead = oldDtd->paramEntityRead;
 #endif /* XML_DTD */
 
@@ -6560,14 +6278,11 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
   newDtd->scaffIndex = oldDtd->scaffIndex;
 
   return 1;
-}  /* End dtdCopy */
+} /* End dtdCopy */
 
 static int
-copyEntityTable(XML_Parser oldParser,
-                HASH_TABLE *newTable,
-                STRING_POOL *newPool,
-                const HASH_TABLE *oldTable)
-{
+copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable,
+                STRING_POOL *newPool, const HASH_TABLE *oldTable) {
   HASH_TABLE_ITER iter;
   const XML_Char *cachedOldBase = NULL;
   const XML_Char *cachedNewBase = NULL;
@@ -6578,17 +6293,17 @@ copyEntityTable(XML_Parser oldParser,
     ENTITY *newE;
     const XML_Char *name;
     const ENTITY *oldE = (ENTITY *)hashTableIterNext(&iter);
-    if (!oldE)
+    if (! oldE)
       break;
     name = poolCopyString(newPool, oldE->name);
-    if (!name)
+    if (! name)
       return 0;
     newE = (ENTITY *)lookup(oldParser, newTable, name, sizeof(ENTITY));
-    if (!newE)
+    if (! newE)
       return 0;
     if (oldE->systemId) {
       const XML_Char *tem = poolCopyString(newPool, oldE->systemId);
-      if (!tem)
+      if (! tem)
         return 0;
       newE->systemId = tem;
       if (oldE->base) {
@@ -6597,29 +6312,28 @@ copyEntityTable(XML_Parser oldParser,
         else {
           cachedOldBase = oldE->base;
           tem = poolCopyString(newPool, cachedOldBase);
-          if (!tem)
+          if (! tem)
             return 0;
           cachedNewBase = newE->base = tem;
         }
       }
       if (oldE->publicId) {
         tem = poolCopyString(newPool, oldE->publicId);
-        if (!tem)
+        if (! tem)
           return 0;
         newE->publicId = tem;
       }
-    }
-    else {
-      const XML_Char *tem = poolCopyStringN(newPool, oldE->textPtr,
-                                            oldE->textLen);
-      if (!tem)
+    } else {
+      const XML_Char *tem
+          = poolCopyStringN(newPool, oldE->textPtr, oldE->textLen);
+      if (! tem)
         return 0;
       newE->textPtr = tem;
       newE->textLen = oldE->textLen;
     }
     if (oldE->notation) {
       const XML_Char *tem = poolCopyString(newPool, oldE->notation);
-      if (!tem)
+      if (! tem)
         return 0;
       newE->notation = tem;
     }
@@ -6632,8 +6346,7 @@ copyEntityTable(XML_Parser oldParser,
 #define INIT_POWER 6
 
 static XML_Bool FASTCALL
-keyeq(KEY s1, KEY s2)
-{
+keyeq(KEY s1, KEY s2) {
   for (; *s1 == *s2; s1++, s2++)
     if (*s1 == 0)
       return XML_TRUE;
@@ -6641,23 +6354,21 @@ keyeq(KEY s1, KEY s2)
 }
 
 static size_t
-keylen(KEY s)
-{
+keylen(KEY s) {
   size_t len = 0;
-  for (; *s; s++, len++);
+  for (; *s; s++, len++)
+    ;
   return len;
 }
 
 static void
-copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key)
-{
+copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) {
   key->k[0] = 0;
   key->k[1] = get_hash_secret_salt(parser);
 }
 
 static unsigned long FASTCALL
-hash(XML_Parser parser, KEY s)
-{
+hash(XML_Parser parser, KEY s) {
   struct siphash state;
   struct sipkey key;
   (void)sip24_valid;
@@ -6668,26 +6379,24 @@ hash(XML_Parser parser, KEY s)
 }
 
 static NAMED *
-lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
-{
+lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) {
   size_t i;
   if (table->size == 0) {
     size_t tsize;
-    if (!createSize)
+    if (! createSize)
       return NULL;
     table->power = INIT_POWER;
     /* table->size is a power of 2 */
     table->size = (size_t)1 << INIT_POWER;
     tsize = table->size * sizeof(NAMED *);
     table->v = (NAMED **)table->mem->malloc_fcn(tsize);
-    if (!table->v) {
+    if (! table->v) {
       table->size = 0;
       return NULL;
     }
     memset(table->v, 0, tsize);
     i = hash(parser, name) & ((unsigned long)table->size - 1);
-  }
-  else {
+  } else {
     unsigned long h = hash(parser, name);
     unsigned long mask = (unsigned long)table->size - 1;
     unsigned char step = 0;
@@ -6695,11 +6404,11 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
     while (table->v[i]) {
       if (keyeq(name, table->v[i]->name))
         return table->v[i];
-      if (!step)
+      if (! step)
         step = PROBE_STEP(h, mask, table->power);
       i < step ? (i += table->size - step) : (i -= step);
     }
-    if (!createSize)
+    if (! createSize)
       return NULL;
 
     /* check for overflow (table is half full) */
@@ -6709,7 +6418,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
       unsigned long newMask = (unsigned long)newSize - 1;
       size_t tsize = newSize * sizeof(NAMED *);
       NAMED **newV = (NAMED **)table->mem->malloc_fcn(tsize);
-      if (!newV)
+      if (! newV)
         return NULL;
       memset(newV, 0, tsize);
       for (i = 0; i < table->size; i++)
@@ -6718,7 +6427,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
           size_t j = newHash & newMask;
           step = 0;
           while (newV[j]) {
-            if (!step)
+            if (! step)
               step = PROBE_STEP(newHash, newMask, newPower);
             j < step ? (j += newSize - step) : (j -= step);
           }
@@ -6731,14 +6440,14 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
       i = h & newMask;
       step = 0;
       while (table->v[i]) {
-        if (!step)
+        if (! step)
           step = PROBE_STEP(h, newMask, newPower);
         i < step ? (i += newSize - step) : (i -= step);
       }
     }
   }
   table->v[i] = (NAMED *)table->mem->malloc_fcn(createSize);
-  if (!table->v[i])
+  if (! table->v[i])
     return NULL;
   memset(table->v[i], 0, createSize);
   table->v[i]->name = name;
@@ -6747,8 +6456,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
 }
 
 static void FASTCALL
-hashTableClear(HASH_TABLE *table)
-{
+hashTableClear(HASH_TABLE *table) {
   size_t i;
   for (i = 0; i < table->size; i++) {
     table->mem->free_fcn(table->v[i]);
@@ -6758,8 +6466,7 @@ hashTableClear(HASH_TABLE *table)
 }
 
 static void FASTCALL
-hashTableDestroy(HASH_TABLE *table)
-{
+hashTableDestroy(HASH_TABLE *table) {
   size_t i;
   for (i = 0; i < table->size; i++)
     table->mem->free_fcn(table->v[i]);
@@ -6767,8 +6474,7 @@ hashTableDestroy(HASH_TABLE *table)
 }
 
 static void FASTCALL
-hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms)
-{
+hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) {
   p->power = 0;
   p->size = 0;
   p->used = 0;
@@ -6777,15 +6483,13 @@ hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms)
 }
 
 static void FASTCALL
-hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table)
-{
+hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) {
   iter->p = table->v;
   iter->end = iter->p + table->size;
 }
 
-static NAMED * FASTCALL
-hashTableIterNext(HASH_TABLE_ITER *iter)
-{
+static NAMED *FASTCALL
+hashTableIterNext(HASH_TABLE_ITER *iter) {
   while (iter->p != iter->end) {
     NAMED *tem = *(iter->p)++;
     if (tem)
@@ -6795,8 +6499,7 @@ hashTableIterNext(HASH_TABLE_ITER *iter)
 }
 
 static void FASTCALL
-poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms)
-{
+poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms) {
   pool->blocks = NULL;
   pool->freeBlocks = NULL;
   pool->start = NULL;
@@ -6806,9 +6509,8 @@ poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms)
 }
 
 static void FASTCALL
-poolClear(STRING_POOL *pool)
-{
-  if (!pool->freeBlocks)
+poolClear(STRING_POOL *pool) {
+  if (! pool->freeBlocks)
     pool->freeBlocks = pool->blocks;
   else {
     BLOCK *p = pool->blocks;
@@ -6826,8 +6528,7 @@ poolClear(STRING_POOL *pool)
 }
 
 static void FASTCALL
-poolDestroy(STRING_POOL *pool)
-{
+poolDestroy(STRING_POOL *pool) {
   BLOCK *p = pool->blocks;
   while (p) {
     BLOCK *tem = p->next;
@@ -6843,26 +6544,26 @@ poolDestroy(STRING_POOL *pool)
 }
 
 static XML_Char *
-poolAppend(STRING_POOL *pool, const ENCODING *enc,
-           const char *ptr, const char *end)
-{
-  if (!pool->ptr && !poolGrow(pool))
+poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr,
+           const char *end) {
+  if (! pool->ptr && ! poolGrow(pool))
     return NULL;
   for (;;) {
-    const enum XML_Convert_Result convert_res = XmlConvert(enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end);
-    if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+    const enum XML_Convert_Result convert_res = XmlConvert(
+        enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end);
+    if ((convert_res == XML_CONVERT_COMPLETED)
+        || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
       break;
-    if (!poolGrow(pool))
+    if (! poolGrow(pool))
       return NULL;
   }
   return pool->start;
 }
 
-static const XML_Char * FASTCALL
-poolCopyString(STRING_POOL *pool, const XML_Char *s)
-{
+static const XML_Char *FASTCALL
+poolCopyString(STRING_POOL *pool, const XML_Char *s) {
   do {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
   } while (*s++);
   s = pool->start;
@@ -6871,9 +6572,8 @@ poolCopyString(STRING_POOL *pool, const XML_Char *s)
 }
 
 static const XML_Char *
-poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
-{
-  if (!pool->ptr && !poolGrow(pool)) {
+poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) {
+  if (! pool->ptr && ! poolGrow(pool)) {
     /* The following line is unreachable given the current usage of
      * poolCopyStringN().  Currently it is called from exactly one
      * place to copy the text of a simple general entity.  By that
@@ -6888,7 +6588,7 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
     return NULL; /* LCOV_EXCL_LINE */
   }
   for (; n > 0; --n, s++) {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
   }
   s = pool->start;
@@ -6896,11 +6596,10 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
   return s;
 }
 
-static const XML_Char * FASTCALL
-poolAppendString(STRING_POOL *pool, const XML_Char *s)
-{
+static const XML_Char *FASTCALL
+poolAppendString(STRING_POOL *pool, const XML_Char *s) {
   while (*s) {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
     s++;
   }
@@ -6908,20 +6607,18 @@ poolAppendString(STRING_POOL *pool, const XML_Char *s)
 }
 
 static XML_Char *
-poolStoreString(STRING_POOL *pool, const ENCODING *enc,
-                const char *ptr, const char *end)
-{
-  if (!poolAppend(pool, enc, ptr, end))
+poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr,
+                const char *end) {
+  if (! poolAppend(pool, enc, ptr, end))
     return NULL;
-  if (pool->ptr == pool->end && !poolGrow(pool))
+  if (pool->ptr == pool->end && ! poolGrow(pool))
     return NULL;
   *(pool->ptr)++ = 0;
   return pool->start;
 }
 
 static size_t
-poolBytesToAllocateFor(int blockSize)
-{
+poolBytesToAllocateFor(int blockSize) {
   /* Unprotected math would be:
   ** return offsetof(BLOCK, s) + blockSize * sizeof(XML_Char);
   **
@@ -6929,7 +6626,7 @@ poolBytesToAllocateFor(int blockSize)
   ** For a + b * c we check b * c in isolation first, so that addition of a
   ** on top has no chance of making us accept a small non-negative number
   */
-  const size_t stretch = sizeof(XML_Char);  /* can be 4 bytes */
+  const size_t stretch = sizeof(XML_Char); /* can be 4 bytes */
 
   if (blockSize <= 0)
     return 0;
@@ -6939,8 +6636,8 @@ poolBytesToAllocateFor(int blockSize)
 
   {
     const int stretchedBlockSize = blockSize * (int)stretch;
-    const int bytesToAllocate = (int)(
-        offsetof(BLOCK, s) + (unsigned)stretchedBlockSize);
+    const int bytesToAllocate
+        = (int)(offsetof(BLOCK, s) + (unsigned)stretchedBlockSize);
     if (bytesToAllocate < 0)
       return 0;
 
@@ -6949,8 +6646,7 @@ poolBytesToAllocateFor(int blockSize)
 }
 
 static XML_Bool FASTCALL
-poolGrow(STRING_POOL *pool)
-{
+poolGrow(STRING_POOL *pool) {
   if (pool->freeBlocks) {
     if (pool->start == 0) {
       pool->blocks = pool->freeBlocks;
@@ -6976,7 +6672,7 @@ poolGrow(STRING_POOL *pool)
   }
   if (pool->blocks && pool->start == pool->blocks->s) {
     BLOCK *temp;
-    int blockSize = (int)((unsigned)(pool->end - pool->start)*2U);
+    int blockSize = (int)((unsigned)(pool->end - pool->start) * 2U);
     size_t bytesToAllocate;
 
     /* NOTE: Needs to be calculated prior to calling `realloc`
@@ -6997,8 +6693,8 @@ poolGrow(STRING_POOL *pool)
     if (bytesToAllocate == 0)
       return XML_FALSE;
 
-    temp = (BLOCK *)
-      pool->mem->realloc_fcn(pool->blocks, (unsigned)bytesToAllocate);
+    temp = (BLOCK *)pool->mem->realloc_fcn(pool->blocks,
+                                           (unsigned)bytesToAllocate);
     if (temp == NULL)
       return XML_FALSE;
     pool->blocks = temp;
@@ -7006,8 +6702,7 @@ poolGrow(STRING_POOL *pool)
     pool->ptr = pool->blocks->s + offsetInsideBlock;
     pool->start = pool->blocks->s;
     pool->end = pool->start + blockSize;
-  }
-  else {
+  } else {
     BLOCK *tem;
     int blockSize = (int)(pool->end - pool->start);
     size_t bytesToAllocate;
@@ -7022,7 +6717,7 @@ poolGrow(STRING_POOL *pool)
        * function).  Either way it isn't readily testable, so we
        * exclude it from the coverage statistics.
        */
-      return XML_FALSE;  /* LCOV_EXCL_LINE */
+      return XML_FALSE; /* LCOV_EXCL_LINE */
     }
 
     if (blockSize < INIT_BLOCK_SIZE)
@@ -7040,14 +6735,13 @@ poolGrow(STRING_POOL *pool)
       return XML_FALSE;
 
     tem = (BLOCK *)pool->mem->malloc_fcn(bytesToAllocate);
-    if (!tem)
+    if (! tem)
       return XML_FALSE;
     tem->size = blockSize;
     tem->next = pool->blocks;
     pool->blocks = tem;
     if (pool->ptr != pool->start)
-      memcpy(tem->s, pool->start,
-             (pool->ptr - pool->start) * sizeof(XML_Char));
+      memcpy(tem->s, pool->start, (pool->ptr - pool->start) * sizeof(XML_Char));
     pool->ptr = tem->s + (pool->ptr - pool->start);
     pool->start = tem->s;
     pool->end = tem->s + blockSize;
@@ -7056,15 +6750,14 @@ poolGrow(STRING_POOL *pool)
 }
 
 static int FASTCALL
-nextScaffoldPart(XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
-  CONTENT_SCAFFOLD * me;
+nextScaffoldPart(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
+  CONTENT_SCAFFOLD *me;
   int next;
 
-  if (!dtd->scaffIndex) {
+  if (! dtd->scaffIndex) {
     dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int));
-    if (!dtd->scaffIndex)
+    if (! dtd->scaffIndex)
       return -1;
     dtd->scaffIndex[0] = 0;
   }
@@ -7072,15 +6765,14 @@ nextScaffoldPart(XML_Parser parser)
   if (dtd->scaffCount >= dtd->scaffSize) {
     CONTENT_SCAFFOLD *temp;
     if (dtd->scaffold) {
-      temp = (CONTENT_SCAFFOLD *)
-        REALLOC(parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD));
+      temp = (CONTENT_SCAFFOLD *)REALLOC(
+          parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD));
       if (temp == NULL)
         return -1;
       dtd->scaffSize *= 2;
-    }
-    else {
+    } else {
       temp = (CONTENT_SCAFFOLD *)MALLOC(parser, INIT_SCAFFOLD_ELEMENTS
-                                        * sizeof(CONTENT_SCAFFOLD));
+                                                    * sizeof(CONTENT_SCAFFOLD));
       if (temp == NULL)
         return -1;
       dtd->scaffSize = INIT_SCAFFOLD_ELEMENTS;
@@ -7090,11 +6782,12 @@ nextScaffoldPart(XML_Parser parser)
   next = dtd->scaffCount++;
   me = &dtd->scaffold[next];
   if (dtd->scaffLevel) {
-    CONTENT_SCAFFOLD *parent = &dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel-1]];
+    CONTENT_SCAFFOLD *parent
+        = &dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]];
     if (parent->lastchild) {
       dtd->scaffold[parent->lastchild].nextsib = next;
     }
-    if (!parent->childcnt)
+    if (! parent->childcnt)
       parent->firstchild = next;
     parent->lastchild = next;
     parent->childcnt++;
@@ -7104,13 +6797,9 @@ nextScaffoldPart(XML_Parser parser)
 }
 
 static void
-build_node(XML_Parser parser,
-           int src_node,
-           XML_Content *dest,
-           XML_Content **contpos,
-           XML_Char **strpos)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+build_node(XML_Parser parser, int src_node, XML_Content *dest,
+           XML_Content **contpos, XML_Char **strpos) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   dest->type = dtd->scaffold[src_node].type;
   dest->quant = dtd->scaffold[src_node].quant;
   if (dest->type == XML_CTYPE_NAME) {
@@ -7119,21 +6808,19 @@ build_node(XML_Parser parser,
     src = dtd->scaffold[src_node].name;
     for (;;) {
       *(*strpos)++ = *src;
-      if (!*src)
+      if (! *src)
         break;
       src++;
     }
     dest->numchildren = 0;
     dest->children = NULL;
-  }
-  else {
+  } else {
     unsigned int i;
     int cn;
     dest->numchildren = dtd->scaffold[src_node].childcnt;
     dest->children = *contpos;
     *contpos += dest->numchildren;
-    for (i = 0, cn = dtd->scaffold[src_node].firstchild;
-         i < dest->numchildren;
+    for (i = 0, cn = dtd->scaffold[src_node].firstchild; i < dest->numchildren;
          i++, cn = dtd->scaffold[cn].nextsib) {
       build_node(parser, cn, &(dest->children[i]), contpos, strpos);
     }
@@ -7142,20 +6829,19 @@ build_node(XML_Parser parser,
 }
 
 static XML_Content *
-build_model (XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+build_model(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   XML_Content *ret;
   XML_Content *cpos;
-  XML_Char * str;
+  XML_Char *str;
   int allocsize = (dtd->scaffCount * sizeof(XML_Content)
                    + (dtd->contentStringLen * sizeof(XML_Char)));
 
   ret = (XML_Content *)MALLOC(parser, allocsize);
-  if (!ret)
+  if (! ret)
     return NULL;
 
-  str =  (XML_Char *) (&ret[dtd->scaffCount]);
+  str = (XML_Char *)(&ret[dtd->scaffCount]);
   cpos = &ret[1];
 
   build_node(parser, 0, ret, &cpos, &str);
@@ -7163,49 +6849,45 @@ build_model (XML_Parser parser)
 }
 
 static ELEMENT_TYPE *
-getElementType(XML_Parser parser,
-               const ENCODING *enc,
-               const char *ptr,
-               const char *end)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr,
+               const char *end) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *name = poolStoreString(&dtd->pool, enc, ptr, end);
   ELEMENT_TYPE *ret;
 
-  if (!name)
+  if (! name)
     return NULL;
-  ret = (ELEMENT_TYPE *) lookup(parser, &dtd->elementTypes, name, sizeof(ELEMENT_TYPE));
-  if (!ret)
+  ret = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, name,
+                               sizeof(ELEMENT_TYPE));
+  if (! ret)
     return NULL;
   if (ret->name != name)
     poolDiscard(&dtd->pool);
   else {
     poolFinish(&dtd->pool);
-    if (!setElementTypePrefix(parser, ret))
+    if (! setElementTypePrefix(parser, ret))
       return NULL;
   }
   return ret;
 }
 
 static XML_Char *
-copyString(const XML_Char *s,
-           const XML_Memory_Handling_Suite *memsuite)
-{
-    int charsRequired = 0;
-    XML_Char *result;
-
-    /* First determine how long the string is */
-    while (s[charsRequired] != 0) {
-      charsRequired++;
-    }
-    /* Include the terminator */
+copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) {
+  int charsRequired = 0;
+  XML_Char *result;
+
+  /* First determine how long the string is */
+  while (s[charsRequired] != 0) {
     charsRequired++;
+  }
+  /* Include the terminator */
+  charsRequired++;
 
-    /* Now allocate space for the copy */
-    result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char));
-    if (result == NULL)
-        return NULL;
-    /* Copy the original into place */
-    memcpy(result, s, charsRequired * sizeof(XML_Char));
-    return result;
+  /* Now allocate space for the copy */
+  result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char));
+  if (result == NULL)
+    return NULL;
+  /* Copy the original into place */
+  memcpy(result, s, charsRequired * sizeof(XML_Char));
+  return result;
 }
diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c
index 708507d575be..4d3e3e86e9e8 100644
--- a/Modules/expat/xmlrole.c
+++ b/Modules/expat/xmlrole.c
@@ -33,11 +33,11 @@
 #include <stddef.h>
 
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #else
-#ifdef HAVE_EXPAT_CONFIG_H
-#include <expat_config.h>
-#endif
+#  ifdef HAVE_EXPAT_CONFIG_H
+#    include <expat_config.h>
+#  endif
 #endif /* ndef _WIN32 */
 
 #include "expat_external.h"
@@ -52,107 +52,88 @@
 
 */
 
-static const char KW_ANY[] = {
-    ASCII_A, ASCII_N, ASCII_Y, '\0' };
-static const char KW_ATTLIST[] = {
-    ASCII_A, ASCII_T, ASCII_T, ASCII_L, ASCII_I, ASCII_S, ASCII_T, '\0' };
-static const char KW_CDATA[] = {
-    ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_DOCTYPE[] = {
-    ASCII_D, ASCII_O, ASCII_C, ASCII_T, ASCII_Y, ASCII_P, ASCII_E, '\0' };
-static const char KW_ELEMENT[] = {
-    ASCII_E, ASCII_L, ASCII_E, ASCII_M, ASCII_E, ASCII_N, ASCII_T, '\0' };
-static const char KW_EMPTY[] = {
-    ASCII_E, ASCII_M, ASCII_P, ASCII_T, ASCII_Y, '\0' };
-static const char KW_ENTITIES[] = {
-    ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_I, ASCII_E, ASCII_S,
-    '\0' };
-static const char KW_ENTITY[] = {
-    ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0' };
-static const char KW_FIXED[] = {
-    ASCII_F, ASCII_I, ASCII_X, ASCII_E, ASCII_D, '\0' };
-static const char KW_ID[] = {
-    ASCII_I, ASCII_D, '\0' };
-static const char KW_IDREF[] = {
-    ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0' };
-static const char KW_IDREFS[] = {
-    ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0' };
+static const char KW_ANY[] = {ASCII_A, ASCII_N, ASCII_Y, '\0'};
+static const char KW_ATTLIST[]
+    = {ASCII_A, ASCII_T, ASCII_T, ASCII_L, ASCII_I, ASCII_S, ASCII_T, '\0'};
+static const char KW_CDATA[]
+    = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_DOCTYPE[]
+    = {ASCII_D, ASCII_O, ASCII_C, ASCII_T, ASCII_Y, ASCII_P, ASCII_E, '\0'};
+static const char KW_ELEMENT[]
+    = {ASCII_E, ASCII_L, ASCII_E, ASCII_M, ASCII_E, ASCII_N, ASCII_T, '\0'};
+static const char KW_EMPTY[]
+    = {ASCII_E, ASCII_M, ASCII_P, ASCII_T, ASCII_Y, '\0'};
+static const char KW_ENTITIES[] = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T,
+                                   ASCII_I, ASCII_E, ASCII_S, '\0'};
+static const char KW_ENTITY[]
+    = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0'};
+static const char KW_FIXED[]
+    = {ASCII_F, ASCII_I, ASCII_X, ASCII_E, ASCII_D, '\0'};
+static const char KW_ID[] = {ASCII_I, ASCII_D, '\0'};
+static const char KW_IDREF[]
+    = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0'};
+static const char KW_IDREFS[]
+    = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0'};
 #ifdef XML_DTD
-static const char KW_IGNORE[] = {
-    ASCII_I, ASCII_G, ASCII_N, ASCII_O, ASCII_R, ASCII_E, '\0' };
+static const char KW_IGNORE[]
+    = {ASCII_I, ASCII_G, ASCII_N, ASCII_O, ASCII_R, ASCII_E, '\0'};
 #endif
-static const char KW_IMPLIED[] = {
-    ASCII_I, ASCII_M, ASCII_P, ASCII_L, ASCII_I, ASCII_E, ASCII_D, '\0' };
+static const char KW_IMPLIED[]
+    = {ASCII_I, ASCII_M, ASCII_P, ASCII_L, ASCII_I, ASCII_E, ASCII_D, '\0'};
 #ifdef XML_DTD
-static const char KW_INCLUDE[] = {
-    ASCII_I, ASCII_N, ASCII_C, ASCII_L, ASCII_U, ASCII_D, ASCII_E, '\0' };
+static const char KW_INCLUDE[]
+    = {ASCII_I, ASCII_N, ASCII_C, ASCII_L, ASCII_U, ASCII_D, ASCII_E, '\0'};
 #endif
-static const char KW_NDATA[] = {
-    ASCII_N, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_NMTOKEN[] = {
-    ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0' };
-static const char KW_NMTOKENS[] = {
-    ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, ASCII_S,
-    '\0' };
-static const char KW_NOTATION[] =
-    { ASCII_N, ASCII_O, ASCII_T, ASCII_A, ASCII_T, ASCII_I, ASCII_O, ASCII_N,
-      '\0' };
-static const char KW_PCDATA[] = {
-    ASCII_P, ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_PUBLIC[] = {
-    ASCII_P, ASCII_U, ASCII_B, ASCII_L, ASCII_I, ASCII_C, '\0' };
-static const char KW_REQUIRED[] = {
-    ASCII_R, ASCII_E, ASCII_Q, ASCII_U, ASCII_I, ASCII_R, ASCII_E, ASCII_D,
-    '\0' };
-static const char KW_SYSTEM[] = {
-    ASCII_S, ASCII_Y, ASCII_S, ASCII_T, ASCII_E, ASCII_M, '\0' };
+static const char KW_NDATA[]
+    = {ASCII_N, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_NMTOKEN[]
+    = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0'};
+static const char KW_NMTOKENS[] = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K,
+                                   ASCII_E, ASCII_N, ASCII_S, '\0'};
+static const char KW_NOTATION[] = {ASCII_N, ASCII_O, ASCII_T, ASCII_A, ASCII_T,
+                                   ASCII_I, ASCII_O, ASCII_N, '\0'};
+static const char KW_PCDATA[]
+    = {ASCII_P, ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_PUBLIC[]
+    = {ASCII_P, ASCII_U, ASCII_B, ASCII_L, ASCII_I, ASCII_C, '\0'};
+static const char KW_REQUIRED[] = {ASCII_R, ASCII_E, ASCII_Q, ASCII_U, ASCII_I,
+                                   ASCII_R, ASCII_E, ASCII_D, '\0'};
+static const char KW_SYSTEM[]
+    = {ASCII_S, ASCII_Y, ASCII_S, ASCII_T, ASCII_E, ASCII_M, '\0'};
 
 #ifndef MIN_BYTES_PER_CHAR
-#define MIN_BYTES_PER_CHAR(enc) ((enc)->minBytesPerChar)
+#  define MIN_BYTES_PER_CHAR(enc) ((enc)->minBytesPerChar)
 #endif
 
 #ifdef XML_DTD
-#define setTopLevel(state) \
-  ((state)->handler = ((state)->documentEntity \
-                       ? internalSubset \
-                       : externalSubset1))
+#  define setTopLevel(state)                                                   \
+    ((state)->handler                                                          \
+     = ((state)->documentEntity ? internalSubset : externalSubset1))
 #else /* not XML_DTD */
-#define setTopLevel(state) ((state)->handler = internalSubset)
+#  define setTopLevel(state) ((state)->handler = internalSubset)
 #endif /* not XML_DTD */
 
-typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state,
-                                   int tok,
-                                   const char *ptr,
-                                   const char *end,
+typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state, int tok,
+                                   const char *ptr, const char *end,
                                    const ENCODING *enc);
 
-static PROLOG_HANDLER
-  prolog0, prolog1, prolog2,
-  doctype0, doctype1, doctype2, doctype3, doctype4, doctype5,
-  internalSubset,
-  entity0, entity1, entity2, entity3, entity4, entity5, entity6,
-  entity7, entity8, entity9, entity10,
-  notation0, notation1, notation2, notation3, notation4,
-  attlist0, attlist1, attlist2, attlist3, attlist4, attlist5, attlist6,
-  attlist7, attlist8, attlist9,
-  element0, element1, element2, element3, element4, element5, element6,
-  element7,
+static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2,
+    doctype3, doctype4, doctype5, internalSubset, entity0, entity1, entity2,
+    entity3, entity4, entity5, entity6, entity7, entity8, entity9, entity10,
+    notation0, notation1, notation2, notation3, notation4, attlist0, attlist1,
+    attlist2, attlist3, attlist4, attlist5, attlist6, attlist7, attlist8,
+    attlist9, element0, element1, element2, element3, element4, element5,
+    element6, element7,
 #ifdef XML_DTD
-  externalSubset0, externalSubset1,
-  condSect0, condSect1, condSect2,
+    externalSubset0, externalSubset1, condSect0, condSect1, condSect2,
 #endif /* XML_DTD */
-  declClose,
-  error;
+    declClose, error;
 
 static int FASTCALL common(PROLOG_STATE *state, int tok);
 
 static int PTRCALL
-prolog0(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     state->handler = prolog1;
@@ -169,10 +150,8 @@ prolog0(PROLOG_STATE *state,
   case XML_TOK_BOM:
     return XML_ROLE_NONE;
   case XML_TOK_DECL_OPEN:
-    if (!XmlNameMatchesAscii(enc,
-                             ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                             end,
-                             KW_DOCTYPE))
+    if (! XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
+                              KW_DOCTYPE))
       break;
     state->handler = doctype0;
     return XML_ROLE_DOCTYPE_NONE;
@@ -184,12 +163,8 @@ prolog0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-prolog1(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -207,10 +182,8 @@ prolog1(PROLOG_STATE *state,
      */
     return XML_ROLE_NONE; /* LCOV_EXCL_LINE */
   case XML_TOK_DECL_OPEN:
-    if (!XmlNameMatchesAscii(enc,
-                             ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                             end,
-                             KW_DOCTYPE))
+    if (! XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
+                              KW_DOCTYPE))
       break;
     state->handler = doctype0;
     return XML_ROLE_DOCTYPE_NONE;
@@ -222,12 +195,11 @@ prolog1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-prolog2(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -243,12 +215,11 @@ prolog2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -261,12 +232,8 @@ doctype0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype1(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -291,12 +258,11 @@ doctype1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype2(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -308,12 +274,11 @@ doctype2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -325,12 +290,11 @@ doctype3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -345,12 +309,11 @@ doctype4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -362,40 +325,28 @@ doctype5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-internalSubset(PROLOG_STATE *state,
-               int tok,
-               const char *ptr,
-               const char *end,
-               const ENCODING *enc)
-{
+internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+               const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
   case XML_TOK_DECL_OPEN:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ENTITY)) {
       state->handler = entity0;
       return XML_ROLE_ENTITY_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ATTLIST)) {
       state->handler = attlist0;
       return XML_ROLE_ATTLIST_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ELEMENT)) {
       state->handler = element0;
       return XML_ROLE_ELEMENT_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_NOTATION)) {
       state->handler = notation0;
       return XML_ROLE_NOTATION_NONE;
@@ -419,12 +370,8 @@ internalSubset(PROLOG_STATE *state,
 #ifdef XML_DTD
 
 static int PTRCALL
-externalSubset0(PROLOG_STATE *state,
-                int tok,
-                const char *ptr,
-                const char *end,
-                const ENCODING *enc)
-{
+externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+                const ENCODING *enc) {
   state->handler = externalSubset1;
   if (tok == XML_TOK_XML_DECL)
     return XML_ROLE_TEXT_DECL;
@@ -432,12 +379,8 @@ externalSubset0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-externalSubset1(PROLOG_STATE *state,
-                int tok,
-                const char *ptr,
-                const char *end,
-                const ENCODING *enc)
-{
+externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+                const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_COND_SECT_OPEN:
     state->handler = condSect0;
@@ -464,12 +407,11 @@ externalSubset1(PROLOG_STATE *state,
 #endif /* XML_DTD */
 
 static int PTRCALL
-entity0(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -484,12 +426,11 @@ entity0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity1(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -501,12 +442,8 @@ entity1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity2(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -529,12 +466,11 @@ entity2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity3(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -546,12 +482,11 @@ entity3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity4(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -563,12 +498,8 @@ entity4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity5(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -586,12 +517,11 @@ entity5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity6(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -604,12 +534,8 @@ entity6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity7(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -632,12 +558,11 @@ entity7(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity8(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -649,12 +574,11 @@ entity8(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity9(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -666,12 +590,11 @@ entity9(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity10(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -683,12 +606,11 @@ entity10(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation0(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -700,12 +622,8 @@ notation0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation1(PROLOG_STATE *state,
-          int tok,
-          const char *ptr,
-          const char *end,
-          const ENCODING *enc)
-{
+notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -724,12 +642,11 @@ notation1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation2(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -741,12 +658,11 @@ notation2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation3(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -759,12 +675,11 @@ notation3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation4(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -780,12 +695,11 @@ notation4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -798,12 +712,11 @@ attlist0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist1(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -819,34 +732,23 @@ attlist1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist2(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
-  case XML_TOK_NAME:
-    {
-      static const char * const types[] = {
-        KW_CDATA,
-        KW_ID,
-        KW_IDREF,
-        KW_IDREFS,
-        KW_ENTITY,
-        KW_ENTITIES,
-        KW_NMTOKEN,
-        KW_NMTOKENS,
-      };
-      int i;
-      for (i = 0; i < (int)(sizeof(types)/sizeof(types[0])); i++)
-        if (XmlNameMatchesAscii(enc, ptr, end, types[i])) {
-          state->handler = attlist8;
-          return XML_ROLE_ATTRIBUTE_TYPE_CDATA + i;
-        }
-    }
+  case XML_TOK_NAME: {
+    static const char *const types[] = {
+        KW_CDATA,  KW_ID,       KW_IDREF,   KW_IDREFS,
+        KW_ENTITY, KW_ENTITIES, KW_NMTOKEN, KW_NMTOKENS,
+    };
+    int i;
+    for (i = 0; i < (int)(sizeof(types) / sizeof(types[0])); i++)
+      if (XmlNameMatchesAscii(enc, ptr, end, types[i])) {
+        state->handler = attlist8;
+        return XML_ROLE_ATTRIBUTE_TYPE_CDATA + i;
+      }
+  }
     if (XmlNameMatchesAscii(enc, ptr, end, KW_NOTATION)) {
       state->handler = attlist5;
       return XML_ROLE_ATTLIST_NONE;
@@ -860,12 +762,11 @@ attlist2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -879,12 +780,11 @@ attlist3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -899,12 +799,11 @@ attlist4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -916,12 +815,11 @@ attlist5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist6(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -933,12 +831,11 @@ attlist6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist7(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -954,33 +851,23 @@ attlist7(PROLOG_STATE *state,
 
 /* default value */
 static int PTRCALL
-attlist8(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
   case XML_TOK_POUND_NAME:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_IMPLIED)) {
       state->handler = attlist1;
       return XML_ROLE_IMPLIED_ATTRIBUTE_VALUE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_REQUIRED)) {
       state->handler = attlist1;
       return XML_ROLE_REQUIRED_ATTRIBUTE_VALUE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_FIXED)) {
       state->handler = attlist9;
       return XML_ROLE_ATTLIST_NONE;
@@ -994,12 +881,11 @@ attlist8(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist9(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -1011,12 +897,11 @@ attlist9(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1029,12 +914,8 @@ element0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element1(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1059,19 +940,13 @@ element1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element2(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
   case XML_TOK_POUND_NAME:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_PCDATA)) {
       state->handler = element3;
       return XML_ROLE_CONTENT_PCDATA;
@@ -1099,12 +974,11 @@ element2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1124,12 +998,11 @@ element3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1142,12 +1015,11 @@ element4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1163,12 +1035,11 @@ element5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element6(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1193,12 +1064,11 @@ element6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element7(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1243,12 +1113,8 @@ element7(PROLOG_STATE *state,
 #ifdef XML_DTD
 
 static int PTRCALL
-condSect0(PROLOG_STATE *state,
-          int tok,
-          const char *ptr,
-          const char *end,
-          const ENCODING *enc)
-{
+condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1267,12 +1133,11 @@ condSect0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-condSect1(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1285,12 +1150,11 @@ condSect1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-condSect2(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1304,12 +1168,11 @@ condSect2(PROLOG_STATE *state,
 #endif /* XML_DTD */
 
 static int PTRCALL
-declClose(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return state->role_none;
@@ -1341,21 +1204,21 @@ declClose(PROLOG_STATE *state,
  * LCOV_EXCL_START
  */
 static int PTRCALL
-error(PROLOG_STATE *UNUSED_P(state),
-      int UNUSED_P(tok),
-      const char *UNUSED_P(ptr),
-      const char *UNUSED_P(end),
-      const ENCODING *UNUSED_P(enc))
-{
+error(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+      const ENCODING *enc) {
+  UNUSED_P(state);
+  UNUSED_P(tok);
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   return XML_ROLE_NONE;
 }
 /* LCOV_EXCL_STOP */
 
 static int FASTCALL
-common(PROLOG_STATE *state, int tok)
-{
+common(PROLOG_STATE *state, int tok) {
 #ifdef XML_DTD
-  if (!state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF)
+  if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF)
     return XML_ROLE_INNER_PARAM_ENTITY_REF;
 #endif
   state->handler = error;
@@ -1363,8 +1226,7 @@ common(PROLOG_STATE *state, int tok)
 }
 
 void
-XmlPrologStateInit(PROLOG_STATE *state)
-{
+XmlPrologStateInit(PROLOG_STATE *state) {
   state->handler = prolog0;
 #ifdef XML_DTD
   state->documentEntity = 1;
@@ -1376,8 +1238,7 @@ XmlPrologStateInit(PROLOG_STATE *state)
 #ifdef XML_DTD
 
 void
-XmlPrologStateInitExternalEntity(PROLOG_STATE *state)
-{
+XmlPrologStateInitExternalEntity(PROLOG_STATE *state) {
   state->handler = externalSubset0;
   state->documentEntity = 0;
   state->includeLevel = 0;
diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h
index e5f048eab55c..036aba64fd29 100644
--- a/Modules/expat/xmlrole.h
+++ b/Modules/expat/xmlrole.h
@@ -36,7 +36,7 @@
 #ifdef __VMS
 /*      0        1         2         3      0        1         2         3
         1234567890123456789012345678901     1234567890123456789012345678901 */
-#define XmlPrologStateInitExternalEntity    XmlPrologStateInitExternalEnt
+#  define XmlPrologStateInitExternalEntity XmlPrologStateInitExternalEnt
 #endif
 
 #include "xmltok.h"
@@ -113,11 +113,8 @@ enum {
 };
 
 typedef struct prolog_state {
-  int (PTRCALL *handler) (struct prolog_state *state,
-                          int tok,
-                          const char *ptr,
-                          const char *end,
-                          const ENCODING *enc);
+  int(PTRCALL *handler)(struct prolog_state *state, int tok, const char *ptr,
+                        const char *end, const ENCODING *enc);
   unsigned level;
   int role_none;
 #ifdef XML_DTD
@@ -132,8 +129,8 @@ void XmlPrologStateInit(PROLOG_STATE *);
 void XmlPrologStateInitExternalEntity(PROLOG_STATE *);
 #endif /* XML_DTD */
 
-#define XmlTokenRole(state, tok, ptr, end, enc) \
- (((state)->handler)(state, tok, ptr, end, enc))
+#define XmlTokenRole(state, tok, ptr, end, enc)                                \
+  (((state)->handler)(state, tok, ptr, end, enc))
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c
index 6b415d83972c..11e9d1ccdad4 100644
--- a/Modules/expat/xmltok.c
+++ b/Modules/expat/xmltok.c
@@ -31,24 +31,23 @@
 */
 
 #include <stddef.h>
-#include <string.h>  /* memcpy */
+#include <string.h> /* memcpy */
 
 #if defined(_MSC_VER) && (_MSC_VER <= 1700)
-  /* for vs2012/11.0/1700 and earlier Visual Studio compilers */
-# define bool   int
-# define false  0
-# define true   1
+/* for vs2012/11.0/1700 and earlier Visual Studio compilers */
+#  define bool int
+#  define false 0
+#  define true 1
 #else
-# include <stdbool.h>
+#  include <stdbool.h>
 #endif
 
-
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #else
-#ifdef HAVE_EXPAT_CONFIG_H
-#include <expat_config.h>
-#endif
+#  ifdef HAVE_EXPAT_CONFIG_H
+#    include <expat_config.h>
+#  endif
 #endif /* ndef _WIN32 */
 
 #include "expat_external.h"
@@ -57,58 +56,49 @@
 #include "nametab.h"
 
 #ifdef XML_DTD
-#define IGNORE_SECTION_TOK_VTABLE , PREFIX(ignoreSectionTok)
+#  define IGNORE_SECTION_TOK_VTABLE , PREFIX(ignoreSectionTok)
 #else
-#define IGNORE_SECTION_TOK_VTABLE /* as nothing */
+#  define IGNORE_SECTION_TOK_VTABLE /* as nothing */
 #endif
 
-#define VTABLE1 \
-  { PREFIX(prologTok), PREFIX(contentTok), \
-    PREFIX(cdataSectionTok) IGNORE_SECTION_TOK_VTABLE }, \
-  { PREFIX(attributeValueTok), PREFIX(entityValueTok) }, \
-  PREFIX(nameMatchesAscii), \
-  PREFIX(nameLength), \
-  PREFIX(skipS), \
-  PREFIX(getAtts), \
-  PREFIX(charRefNumber), \
-  PREFIX(predefinedEntityName), \
-  PREFIX(updatePosition), \
-  PREFIX(isPublicId)
+#define VTABLE1                                                                \
+  {PREFIX(prologTok), PREFIX(contentTok),                                      \
+   PREFIX(cdataSectionTok) IGNORE_SECTION_TOK_VTABLE},                         \
+      {PREFIX(attributeValueTok), PREFIX(entityValueTok)},                     \
+      PREFIX(nameMatchesAscii), PREFIX(nameLength), PREFIX(skipS),             \
+      PREFIX(getAtts), PREFIX(charRefNumber), PREFIX(predefinedEntityName),    \
+      PREFIX(updatePosition), PREFIX(isPublicId)
 
 #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16)
 
-#define UCS2_GET_NAMING(pages, hi, lo) \
-   (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo) & 0x1F)))
+#define UCS2_GET_NAMING(pages, hi, lo)                                         \
+  (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo)&0x1F)))
 
 /* A 2 byte UTF-8 representation splits the characters 11 bits between
    the bottom 5 and 6 bits of the bytes.  We need 8 bits to index into
    pages, 3 bits to add to that index and 5 bits to generate the mask.
 */
-#define UTF8_GET_NAMING2(pages, byte) \
-    (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3) \
-                      + ((((byte)[0]) & 3) << 1) \
-                      + ((((byte)[1]) >> 5) & 1)] \
-         & (1u << (((byte)[1]) & 0x1F)))
+#define UTF8_GET_NAMING2(pages, byte)                                          \
+  (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3)                         \
+                + ((((byte)[0]) & 3) << 1) + ((((byte)[1]) >> 5) & 1)]         \
+   & (1u << (((byte)[1]) & 0x1F)))
 
 /* A 3 byte UTF-8 representation splits the characters 16 bits between
    the bottom 4, 6 and 6 bits of the bytes.  We need 8 bits to index
    into pages, 3 bits to add to that index and 5 bits to generate the
    mask.
 */
-#define UTF8_GET_NAMING3(pages, byte) \
-  (namingBitmap[((pages)[((((byte)[0]) & 0xF) << 4) \
-                             + ((((byte)[1]) >> 2) & 0xF)] \
-                       << 3) \
-                      + ((((byte)[1]) & 3) << 1) \
-                      + ((((byte)[2]) >> 5) & 1)] \
-         & (1u << (((byte)[2]) & 0x1F)))
-
-#define UTF8_GET_NAMING(pages, p, n) \
-  ((n) == 2 \
-  ? UTF8_GET_NAMING2(pages, (const unsigned char *)(p)) \
-  : ((n) == 3 \
-     ? UTF8_GET_NAMING3(pages, (const unsigned char *)(p)) \
-     : 0))
+#define UTF8_GET_NAMING3(pages, byte)                                          \
+  (namingBitmap                                                                \
+       [((pages)[((((byte)[0]) & 0xF) << 4) + ((((byte)[1]) >> 2) & 0xF)]      \
+         << 3)                                                                 \
+        + ((((byte)[1]) & 3) << 1) + ((((byte)[2]) >> 5) & 1)]                 \
+   & (1u << (((byte)[2]) & 0x1F)))
+
+#define UTF8_GET_NAMING(pages, p, n)                                           \
+  ((n) == 2                                                                    \
+       ? UTF8_GET_NAMING2(pages, (const unsigned char *)(p))                   \
+       : ((n) == 3 ? UTF8_GET_NAMING3(pages, (const unsigned char *)(p)) : 0))
 
 /* Detection of invalid UTF-8 sequences is based on Table 3.1B
    of Unicode 3.2: http://www.unicode.org/unicode/reports/tr28/
@@ -120,88 +110,76 @@
      (A & 0xC0) == 0xC0  means A > 0xBF
 */
 
-#define UTF8_INVALID2(p) \
+#define UTF8_INVALID2(p)                                                       \
   ((*p) < 0xC2 || ((p)[1] & 0x80) == 0 || ((p)[1] & 0xC0) == 0xC0)
 
-#define UTF8_INVALID3(p) \
-  (((p)[2] & 0x80) == 0 \
-  || \
-  ((*p) == 0xEF && (p)[1] == 0xBF \
-    ? \
-    (p)[2] > 0xBD \
-    : \
-    ((p)[2] & 0xC0) == 0xC0) \
-  || \
-  ((*p) == 0xE0 \
-    ? \
-    (p)[1] < 0xA0 || ((p)[1] & 0xC0) == 0xC0 \
-    : \
-    ((p)[1] & 0x80) == 0 \
-    || \
-    ((*p) == 0xED ? (p)[1] > 0x9F : ((p)[1] & 0xC0) == 0xC0)))
-
-#define UTF8_INVALID4(p) \
-  (((p)[3] & 0x80) == 0 || ((p)[3] & 0xC0) == 0xC0 \
-  || \
-  ((p)[2] & 0x80) == 0 || ((p)[2] & 0xC0) == 0xC0 \
-  || \
-  ((*p) == 0xF0 \
-    ? \
-    (p)[1] < 0x90 || ((p)[1] & 0xC0) == 0xC0 \
-    : \
-    ((p)[1] & 0x80) == 0 \
-    || \
-    ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0)))
+#define UTF8_INVALID3(p)                                                       \
+  (((p)[2] & 0x80) == 0                                                        \
+   || ((*p) == 0xEF && (p)[1] == 0xBF ? (p)[2] > 0xBD                          \
+                                      : ((p)[2] & 0xC0) == 0xC0)               \
+   || ((*p) == 0xE0                                                            \
+           ? (p)[1] < 0xA0 || ((p)[1] & 0xC0) == 0xC0                          \
+           : ((p)[1] & 0x80) == 0                                              \
+                 || ((*p) == 0xED ? (p)[1] > 0x9F : ((p)[1] & 0xC0) == 0xC0)))
+
+#define UTF8_INVALID4(p)                                                       \
+  (((p)[3] & 0x80) == 0 || ((p)[3] & 0xC0) == 0xC0 || ((p)[2] & 0x80) == 0     \
+   || ((p)[2] & 0xC0) == 0xC0                                                  \
+   || ((*p) == 0xF0                                                            \
+           ? (p)[1] < 0x90 || ((p)[1] & 0xC0) == 0xC0                          \
+           : ((p)[1] & 0x80) == 0                                              \
+                 || ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0)))
 
 static int PTRFASTCALL
-isNever(const ENCODING *UNUSED_P(enc), const char *UNUSED_P(p))
-{
+isNever(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  UNUSED_P(p);
   return 0;
 }
 
 static int PTRFASTCALL
-utf8_isName2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isName2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING2(namePages, (const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isName3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isName3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING3(namePages, (const unsigned char *)p);
 }
 
 #define utf8_isName4 isNever
 
 static int PTRFASTCALL
-utf8_isNmstrt2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isNmstrt2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isNmstrt3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isNmstrt3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p);
 }
 
 #define utf8_isNmstrt4 isNever
 
 static int PTRFASTCALL
-utf8_isInvalid2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID2((const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isInvalid3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID3((const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isInvalid4(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid4(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID4((const unsigned char *)p);
 }
 
@@ -209,61 +187,44 @@ struct normal_encoding {
   ENCODING enc;
   unsigned char type[256];
 #ifdef XML_MIN_SIZE
-  int (PTRFASTCALL *byteType)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNameMin)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *);
-  int (PTRFASTCALL *byteToAscii)(const ENCODING *, const char *);
-  int (PTRCALL *charMatches)(const ENCODING *, const char *, int);
+  int(PTRFASTCALL *byteType)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNameMin)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *);
+  int(PTRFASTCALL *byteToAscii)(const ENCODING *, const char *);
+  int(PTRCALL *charMatches)(const ENCODING *, const char *, int);
 #endif /* XML_MIN_SIZE */
-  int (PTRFASTCALL *isName2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isName3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isName4)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid4)(const ENCODING *, const char *);
 };
 
-#define AS_NORMAL_ENCODING(enc)   ((const struct normal_encoding *) (enc))
+#define AS_NORMAL_ENCODING(enc) ((const struct normal_encoding *)(enc))
 
 #ifdef XML_MIN_SIZE
 
-#define STANDARD_VTABLE(E) \
- E ## byteType, \
- E ## isNameMin, \
- E ## isNmstrtMin, \
- E ## byteToAscii, \
- E ## charMatches,
+#  define STANDARD_VTABLE(E)                                                   \
+    E##byteType, E##isNameMin, E##isNmstrtMin, E##byteToAscii, E##charMatches,
 
 #else
 
-#define STANDARD_VTABLE(E) /* as nothing */
+#  define STANDARD_VTABLE(E) /* as nothing */
 
 #endif
 
-#define NORMAL_VTABLE(E) \
- E ## isName2, \
- E ## isName3, \
- E ## isName4, \
- E ## isNmstrt2, \
- E ## isNmstrt3, \
- E ## isNmstrt4, \
- E ## isInvalid2, \
- E ## isInvalid3, \
- E ## isInvalid4
-
-#define NULL_VTABLE \
- /* isName2 */ NULL, \
- /* isName3 */ NULL, \
- /* isName4 */ NULL, \
- /* isNmstrt2 */ NULL, \
- /* isNmstrt3 */ NULL, \
- /* isNmstrt4 */ NULL, \
- /* isInvalid2 */ NULL, \
- /* isInvalid3 */ NULL, \
- /* isInvalid4 */ NULL
+#define NORMAL_VTABLE(E)                                                       \
+  E##isName2, E##isName3, E##isName4, E##isNmstrt2, E##isNmstrt3,              \
+      E##isNmstrt4, E##isInvalid2, E##isInvalid3, E##isInvalid4
+
+#define NULL_VTABLE                                                            \
+  /* isName2 */ NULL, /* isName3 */ NULL, /* isName4 */ NULL,                  \
+      /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL,        \
+      /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL
 
 static int FASTCALL checkCharRefNumber(int);
 
@@ -271,75 +232,70 @@ static int FASTCALL checkCharRefNumber(int);
 #include "ascii.h"
 
 #ifdef XML_MIN_SIZE
-#define sb_isNameMin isNever
-#define sb_isNmstrtMin isNever
+#  define sb_isNameMin isNever
+#  define sb_isNmstrtMin isNever
 #endif
 
 #ifdef XML_MIN_SIZE
-#define MINBPC(enc) ((enc)->minBytesPerChar)
+#  define MINBPC(enc) ((enc)->minBytesPerChar)
 #else
 /* minimum bytes per character */
-#define MINBPC(enc) 1
+#  define MINBPC(enc) 1
 #endif
 
-#define SB_BYTE_TYPE(enc, p) \
+#define SB_BYTE_TYPE(enc, p)                                                   \
   (((struct normal_encoding *)(enc))->type[(unsigned char)*(p)])
 
 #ifdef XML_MIN_SIZE
 static int PTRFASTCALL
-sb_byteType(const ENCODING *enc, const char *p)
-{
+sb_byteType(const ENCODING *enc, const char *p) {
   return SB_BYTE_TYPE(enc, p);
 }
-#define BYTE_TYPE(enc, p) \
- (AS_NORMAL_ENCODING(enc)->byteType(enc, p))
+#  define BYTE_TYPE(enc, p) (AS_NORMAL_ENCODING(enc)->byteType(enc, p))
 #else
-#define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
+#  define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
 #endif
 
 #ifdef XML_MIN_SIZE
-#define BYTE_TO_ASCII(enc, p) \
- (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p))
+#  define BYTE_TO_ASCII(enc, p) (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p))
 static int PTRFASTCALL
-sb_byteToAscii(const ENCODING *enc, const char *p)
-{
+sb_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return *p;
 }
 #else
-#define BYTE_TO_ASCII(enc, p) (*(p))
+#  define BYTE_TO_ASCII(enc, p) (*(p))
 #endif
 
-#define IS_NAME_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isName ## n(enc, p))
-#define IS_NMSTRT_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isNmstrt ## n(enc, p))
-#define IS_INVALID_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isInvalid ## n(enc, p))
+#define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p))
+#define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p))
+#define IS_INVALID_CHAR(enc, p, n)                                             \
+  (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p))
 
 #ifdef XML_MIN_SIZE
-#define IS_NAME_CHAR_MINBPC(enc, p) \
- (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p))
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) \
- (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p))
+#  define IS_NAME_CHAR_MINBPC(enc, p)                                          \
+    (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p))
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p)                                        \
+    (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p))
 #else
-#define IS_NAME_CHAR_MINBPC(enc, p) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
+#  define IS_NAME_CHAR_MINBPC(enc, p) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
 #endif
 
 #ifdef XML_MIN_SIZE
-#define CHAR_MATCHES(enc, p, c) \
- (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c))
+#  define CHAR_MATCHES(enc, p, c)                                              \
+    (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c))
 static int PTRCALL
-sb_charMatches(const ENCODING *enc, const char *p, int c)
-{
+sb_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
   return *p == c;
 }
 #else
 /* c is an ASCII character */
-#define CHAR_MATCHES(enc, p, c) (*(p) == c)
+#  define CHAR_MATCHES(enc, p, c) (*(p) == c)
 #endif
 
-#define PREFIX(ident) normal_ ## ident
+#define PREFIX(ident) normal_##ident
 #define XML_TOK_IMPL_C
 #include "xmltok_impl.c"
 #undef XML_TOK_IMPL_C
@@ -354,42 +310,46 @@ sb_charMatches(const ENCODING *enc, const char *p, int c)
 #undef IS_NMSTRT_CHAR_MINBPC
 #undef IS_INVALID_CHAR
 
-enum {  /* UTF8_cvalN is value of masked first byte of N byte sequence */
-  UTF8_cval1 = 0x00,
-  UTF8_cval2 = 0xc0,
-  UTF8_cval3 = 0xe0,
-  UTF8_cval4 = 0xf0
+enum { /* UTF8_cvalN is value of masked first byte of N byte sequence */
+       UTF8_cval1 = 0x00,
+       UTF8_cval2 = 0xc0,
+       UTF8_cval3 = 0xe0,
+       UTF8_cval4 = 0xf0
 };
 
 void
-_INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** fromLimRef)
-{
-  const char * fromLim = *fromLimRef;
+_INTERNAL_trim_to_complete_utf8_characters(const char *from,
+                                           const char **fromLimRef) {
+  const char *fromLim = *fromLimRef;
   size_t walked = 0;
   for (; fromLim > from; fromLim--, walked++) {
     const unsigned char prev = (unsigned char)fromLim[-1];
-    if ((prev & 0xf8u) == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */
+    if ((prev & 0xf8u)
+        == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */
       if (walked + 1 >= 4) {
         fromLim += 4 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0xf0u) == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */
+    } else if ((prev & 0xf0u)
+               == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */
       if (walked + 1 >= 3) {
         fromLim += 3 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0xe0u) == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */
+    } else if ((prev & 0xe0u)
+               == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */
       if (walked + 1 >= 2) {
         fromLim += 2 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0x80u) == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */
+    } else if ((prev & 0x80u)
+               == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */
       break;
     }
   }
@@ -397,16 +357,15 @@ _INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** from
 }
 
 static enum XML_Convert_Result PTRCALL
-utf8_toUtf8(const ENCODING *UNUSED_P(enc),
-            const char **fromP, const char *fromLim,
-            char **toP, const char *toLim)
-{
+utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+            char **toP, const char *toLim) {
   bool input_incomplete = false;
   bool output_exhausted = false;
 
   /* Avoid copying partial characters (due to limited space). */
   const ptrdiff_t bytesAvailable = fromLim - *fromP;
   const ptrdiff_t bytesStorable = toLim - *toP;
+  UNUSED_P(enc);
   if (bytesAvailable > bytesStorable) {
     fromLim = *fromP + bytesStorable;
     output_exhausted = true;
@@ -414,7 +373,7 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
 
   /* Avoid copying partial characters (from incomplete input). */
   {
-    const char * const fromLimBefore = fromLim;
+    const char *const fromLimBefore = fromLim;
     _INTERNAL_trim_to_complete_utf8_characters(*fromP, &fromLim);
     if (fromLim < fromLimBefore) {
       input_incomplete = true;
@@ -428,7 +387,7 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
     *toP += bytesToCopy;
   }
 
-  if (output_exhausted)  /* needs to go first */
+  if (output_exhausted) /* needs to go first */
     return XML_CONVERT_OUTPUT_EXHAUSTED;
   else if (input_incomplete)
     return XML_CONVERT_INPUT_INCOMPLETE;
@@ -437,10 +396,8 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
 }
 
 static enum XML_Convert_Result PTRCALL
-utf8_toUtf16(const ENCODING *enc,
-             const char **fromP, const char *fromLim,
-             unsigned short **toP, const unsigned short *toLim)
-{
+utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+             unsigned short **toP, const unsigned short *toLim) {
   enum XML_Convert_Result res = XML_CONVERT_COMPLETED;
   unsigned short *to = *toP;
   const char *from = *fromP;
@@ -459,30 +416,28 @@ utf8_toUtf16(const ENCODING *enc,
         res = XML_CONVERT_INPUT_INCOMPLETE;
         goto after;
       }
-      *to++ = (unsigned short)(((from[0] & 0xf) << 12)
-                               | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f));
+      *to++ = (unsigned short)(((from[0] & 0xf) << 12) | ((from[1] & 0x3f) << 6)
+                               | (from[2] & 0x3f));
       from += 3;
       break;
-    case BT_LEAD4:
-      {
-        unsigned long n;
-        if (toLim - to < 2) {
-          res = XML_CONVERT_OUTPUT_EXHAUSTED;
-          goto after;
-        }
-        if (fromLim - from < 4) {
-          res = XML_CONVERT_INPUT_INCOMPLETE;
-          goto after;
-        }
-        n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12)
-            | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f);
-        n -= 0x10000;
-        to[0] = (unsigned short)((n >> 10) | 0xD800);
-        to[1] = (unsigned short)((n & 0x3FF) | 0xDC00);
-        to += 2;
-        from += 4;
+    case BT_LEAD4: {
+      unsigned long n;
+      if (toLim - to < 2) {
+        res = XML_CONVERT_OUTPUT_EXHAUSTED;
+        goto after;
       }
-      break;
+      if (fromLim - from < 4) {
+        res = XML_CONVERT_INPUT_INCOMPLETE;
+        goto after;
+      }
+      n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12)
+          | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f);
+      n -= 0x10000;
+      to[0] = (unsigned short)((n >> 10) | 0xD800);
+      to[1] = (unsigned short)((n & 0x3FF) | 0xDC00);
+      to += 2;
+      from += 4;
+    } break;
     default:
       *to++ = *from++;
       break;
@@ -497,56 +452,51 @@ utf8_toUtf16(const ENCODING *enc,
 }
 
 #ifdef XML_NS
-static const struct normal_encoding utf8_encoding_ns = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
-#include "asciitab.h"
-#include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+static const struct normal_encoding utf8_encoding_ns
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
+#  include "asciitab.h"
+#  include "utf8tab.h"
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 #endif
 
-static const struct normal_encoding utf8_encoding = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding utf8_encoding
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 #ifdef XML_NS
 
-static const struct normal_encoding internal_utf8_encoding_ns = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
-#include "iasciitab.h"
-#include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+static const struct normal_encoding internal_utf8_encoding_ns
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
+#  include "iasciitab.h"
+#  include "utf8tab.h"
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 #endif
 
-static const struct normal_encoding internal_utf8_encoding = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding internal_utf8_encoding
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "iasciitab.h"
 #undef BT_COLON
 #include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 static enum XML_Convert_Result PTRCALL
-latin1_toUtf8(const ENCODING *UNUSED_P(enc),
-              const char **fromP, const char *fromLim,
-              char **toP, const char *toLim)
-{
+latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+              char **toP, const char *toLim) {
+  UNUSED_P(enc);
   for (;;) {
     unsigned char c;
     if (*fromP == fromLim)
@@ -558,8 +508,7 @@ latin1_toUtf8(const ENCODING *UNUSED_P(enc),
       *(*toP)++ = (char)((c >> 6) | UTF8_cval2);
       *(*toP)++ = (char)((c & 0x3f) | 0x80);
       (*fromP)++;
-    }
-    else {
+    } else {
       if (*toP == toLim)
         return XML_CONVERT_OUTPUT_EXHAUSTED;
       *(*toP)++ = *(*fromP)++;
@@ -568,10 +517,9 @@ latin1_toUtf8(const ENCODING *UNUSED_P(enc),
 }
 
 static enum XML_Convert_Result PTRCALL
-latin1_toUtf16(const ENCODING *UNUSED_P(enc),
-               const char **fromP, const char *fromLim,
-               unsigned short **toP, const unsigned short *toLim)
-{
+latin1_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+               unsigned short **toP, const unsigned short *toLim) {
+  UNUSED_P(enc);
   while (*fromP < fromLim && *toP < toLim)
     *(*toP)++ = (unsigned char)*(*fromP)++;
 
@@ -583,33 +531,30 @@ latin1_toUtf16(const ENCODING *UNUSED_P(enc),
 
 #ifdef XML_NS
 
-static const struct normal_encoding latin1_encoding_ns = {
-  { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+static const struct normal_encoding latin1_encoding_ns
+    = {{VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0},
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding latin1_encoding = {
-  { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
-  {
+static const struct normal_encoding latin1_encoding
+    = {{VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 static enum XML_Convert_Result PTRCALL
-ascii_toUtf8(const ENCODING *UNUSED_P(enc),
-             const char **fromP, const char *fromLim,
-             char **toP, const char *toLim)
-{
+ascii_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+             char **toP, const char *toLim) {
+  UNUSED_P(enc);
   while (*fromP < fromLim && *toP < toLim)
     *(*toP)++ = *(*fromP)++;
 
@@ -621,40 +566,45 @@ ascii_toUtf8(const ENCODING *UNUSED_P(enc),
 
 #ifdef XML_NS
 
-static const struct normal_encoding ascii_encoding_ns = {
-  { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
-  {
-#include "asciitab.h"
-/* BT_NONXML == 0 */
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+static const struct normal_encoding ascii_encoding_ns
+    = {{VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0},
+       {
+#  include "asciitab.h"
+           /* BT_NONXML == 0 */
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding ascii_encoding = {
-  { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding ascii_encoding
+    = {{VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
-/* BT_NONXML == 0 */
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+           /* BT_NONXML == 0 */
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 static int PTRFASTCALL
-unicode_byte_type(char hi, char lo)
-{
+unicode_byte_type(char hi, char lo) {
   switch ((unsigned char)hi) {
-  case 0xD8: case 0xD9: case 0xDA: case 0xDB:
+  /* 0xD800?0xDBFF first 16-bit code unit or high surrogate (W1) */
+  case 0xD8:
+  case 0xD9:
+  case 0xDA:
+  case 0xDB:
     return BT_LEAD4;
-  case 0xDC: case 0xDD: case 0xDE: case 0xDF:
+  /* 0xDC00?0xDFFF second 16-bit code unit or low surrogate (W2) */
+  case 0xDC:
+  case 0xDD:
+  case 0xDE:
+  case 0xDF:
     return BT_TRAIL;
   case 0xFF:
     switch ((unsigned char)lo) {
-    case 0xFF:
-    case 0xFE:
+    case 0xFF: /* noncharacter-FFFF */
+    case 0xFE: /* noncharacter-FFFE */
       return BT_NONXML;
     }
     break;
@@ -662,102 +612,105 @@ unicode_byte_type(char hi, char lo)
   return BT_NONASCII;
 }
 
-#define DEFINE_UTF16_TO_UTF8(E) \
-static enum XML_Convert_Result  PTRCALL \
-E ## toUtf8(const ENCODING *UNUSED_P(enc), \
-            const char **fromP, const char *fromLim, \
-            char **toP, const char *toLim) \
-{ \
-  const char *from = *fromP; \
-  fromLim = from + (((fromLim - from) >> 1) << 1);  /* shrink to even */ \
-  for (; from < fromLim; from += 2) { \
-    int plane; \
-    unsigned char lo2; \
-    unsigned char lo = GET_LO(from); \
-    unsigned char hi = GET_HI(from); \
-    switch (hi) { \
-    case 0: \
-      if (lo < 0x80) { \
-        if (*toP == toLim) { \
-          *fromP = from; \
-          return XML_CONVERT_OUTPUT_EXHAUSTED; \
-        } \
-        *(*toP)++ = lo; \
-        break; \
-      } \
-      /* fall through */ \
-    case 0x1: case 0x2: case 0x3: \
-    case 0x4: case 0x5: case 0x6: case 0x7: \
-      if (toLim -  *toP < 2) { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      *(*toP)++ = ((lo >> 6) | (hi << 2) |  UTF8_cval2); \
-      *(*toP)++ = ((lo & 0x3f) | 0x80); \
-      break; \
-    default: \
-      if (toLim -  *toP < 3)  { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      /* 16 bits divided 4, 6, 6 amongst 3 bytes */ \
-      *(*toP)++ = ((hi >> 4) | UTF8_cval3); \
-      *(*toP)++ = (((hi & 0xf) << 2) | (lo >> 6) | 0x80); \
-      *(*toP)++ = ((lo & 0x3f) | 0x80); \
-      break; \
-    case 0xD8: case 0xD9: case 0xDA: case 0xDB: \
-      if (toLim -  *toP < 4) { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      if (fromLim - from < 4) { \
-        *fromP = from; \
-        return XML_CONVERT_INPUT_INCOMPLETE; \
-      } \
-      plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1; \
-      *(*toP)++ = ((plane >> 2) | UTF8_cval4); \
-      *(*toP)++ = (((lo >> 2) & 0xF) | ((plane & 0x3) << 4) | 0x80); \
-      from += 2; \
-      lo2 = GET_LO(from); \
-      *(*toP)++ = (((lo & 0x3) << 4) \
-                   | ((GET_HI(from) & 0x3) << 2) \
-                   | (lo2 >> 6) \
-                   | 0x80); \
-      *(*toP)++ = ((lo2 & 0x3f) | 0x80); \
-      break; \
-    } \
-  } \
-  *fromP = from; \
-  if (from < fromLim) \
-    return XML_CONVERT_INPUT_INCOMPLETE; \
-  else \
-    return XML_CONVERT_COMPLETED; \
-}
+#define DEFINE_UTF16_TO_UTF8(E)                                                \
+  static enum XML_Convert_Result PTRCALL E##toUtf8(                            \
+      const ENCODING *enc, const char **fromP, const char *fromLim,            \
+      char **toP, const char *toLim) {                                         \
+    const char *from = *fromP;                                                 \
+    UNUSED_P(enc);                                                             \
+    fromLim = from + (((fromLim - from) >> 1) << 1); /* shrink to even */      \
+    for (; from < fromLim; from += 2) {                                        \
+      int plane;                                                               \
+      unsigned char lo2;                                                       \
+      unsigned char lo = GET_LO(from);                                         \
+      unsigned char hi = GET_HI(from);                                         \
+      switch (hi) {                                                            \
+      case 0:                                                                  \
+        if (lo < 0x80) {                                                       \
+          if (*toP == toLim) {                                                 \
+            *fromP = from;                                                     \
+            return XML_CONVERT_OUTPUT_EXHAUSTED;                               \
+          }                                                                    \
+          *(*toP)++ = lo;                                                      \
+          break;                                                               \
+        }                                                                      \
+        /* fall through */                                                     \
+      case 0x1:                                                                \
+      case 0x2:                                                                \
+      case 0x3:                                                                \
+      case 0x4:                                                                \
+      case 0x5:                                                                \
+      case 0x6:                                                                \
+      case 0x7:                                                                \
+        if (toLim - *toP < 2) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        *(*toP)++ = ((lo >> 6) | (hi << 2) | UTF8_cval2);                      \
+        *(*toP)++ = ((lo & 0x3f) | 0x80);                                      \
+        break;                                                                 \
+      default:                                                                 \
+        if (toLim - *toP < 3) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        /* 16 bits divided 4, 6, 6 amongst 3 bytes */                          \
+        *(*toP)++ = ((hi >> 4) | UTF8_cval3);                                  \
+        *(*toP)++ = (((hi & 0xf) << 2) | (lo >> 6) | 0x80);                    \
+        *(*toP)++ = ((lo & 0x3f) | 0x80);                                      \
+        break;                                                                 \
+      case 0xD8:                                                               \
+      case 0xD9:                                                               \
+      case 0xDA:                                                               \
+      case 0xDB:                                                               \
+        if (toLim - *toP < 4) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        if (fromLim - from < 4) {                                              \
+          *fromP = from;                                                       \
+          return XML_CONVERT_INPUT_INCOMPLETE;                                 \
+        }                                                                      \
+        plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1;                   \
+        *(*toP)++ = (char)((plane >> 2) | UTF8_cval4);                         \
+        *(*toP)++ = (((lo >> 2) & 0xF) | ((plane & 0x3) << 4) | 0x80);         \
+        from += 2;                                                             \
+        lo2 = GET_LO(from);                                                    \
+        *(*toP)++ = (((lo & 0x3) << 4) | ((GET_HI(from) & 0x3) << 2)           \
+                     | (lo2 >> 6) | 0x80);                                     \
+        *(*toP)++ = ((lo2 & 0x3f) | 0x80);                                     \
+        break;                                                                 \
+      }                                                                        \
+    }                                                                          \
+    *fromP = from;                                                             \
+    if (from < fromLim)                                                        \
+      return XML_CONVERT_INPUT_INCOMPLETE;                                     \
+    else                                                                       \
+      return XML_CONVERT_COMPLETED;                                            \
+  }
 
-#define DEFINE_UTF16_TO_UTF16(E) \
-static enum XML_Convert_Result  PTRCALL \
-E ## toUtf16(const ENCODING *UNUSED_P(enc), \
-             const char **fromP, const char *fromLim, \
-             unsigned short **toP, const unsigned short *toLim) \
-{ \
-  enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \
-  fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1);  /* shrink to even */ \
-  /* Avoid copying first half only of surrogate */ \
-  if (fromLim - *fromP > ((toLim - *toP) << 1) \
-      && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \
-    fromLim -= 2; \
-    res = XML_CONVERT_INPUT_INCOMPLETE; \
-  } \
-  for (; *fromP < fromLim && *toP < toLim; *fromP += 2) \
-    *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP); \
-  if ((*toP == toLim) && (*fromP < fromLim)) \
-    return XML_CONVERT_OUTPUT_EXHAUSTED; \
-  else \
-    return res; \
-}
+#define DEFINE_UTF16_TO_UTF16(E)                                               \
+  static enum XML_Convert_Result PTRCALL E##toUtf16(                           \
+      const ENCODING *enc, const char **fromP, const char *fromLim,            \
+      unsigned short **toP, const unsigned short *toLim) {                     \
+    enum XML_Convert_Result res = XML_CONVERT_COMPLETED;                       \
+    UNUSED_P(enc);                                                             \
+    fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */  \
+    /* Avoid copying first half only of surrogate */                           \
+    if (fromLim - *fromP > ((toLim - *toP) << 1)                               \
+        && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) {                             \
+      fromLim -= 2;                                                            \
+      res = XML_CONVERT_INPUT_INCOMPLETE;                                      \
+    }                                                                          \
+    for (; *fromP < fromLim && *toP < toLim; *fromP += 2)                      \
+      *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP);                      \
+    if ((*toP == toLim) && (*fromP < fromLim))                                 \
+      return XML_CONVERT_OUTPUT_EXHAUSTED;                                     \
+    else                                                                       \
+      return res;                                                              \
+  }
 
-#define SET2(ptr, ch) \
-  (((ptr)[0] = ((ch) & 0xff)), ((ptr)[1] = ((ch) >> 8)))
+#define SET2(ptr, ch) (((ptr)[0] = ((ch)&0xff)), ((ptr)[1] = ((ch) >> 8)))
 #define GET_LO(ptr) ((unsigned char)(ptr)[0])
 #define GET_HI(ptr) ((unsigned char)(ptr)[1])
 
@@ -768,8 +721,7 @@ DEFINE_UTF16_TO_UTF16(little2_)
 #undef GET_LO
 #undef GET_HI
 
-#define SET2(ptr, ch) \
-  (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch) & 0xFF)))
+#define SET2(ptr, ch) (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch)&0xFF)))
 #define GET_LO(ptr) ((unsigned char)(ptr)[1])
 #define GET_HI(ptr) ((unsigned char)(ptr)[0])
 
@@ -780,292 +732,279 @@ DEFINE_UTF16_TO_UTF16(big2_)
 #undef GET_LO
 #undef GET_HI
 
-#define LITTLE2_BYTE_TYPE(enc, p) \
- ((p)[1] == 0 \
-  ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)] \
-  : unicode_byte_type((p)[1], (p)[0]))
-#define LITTLE2_BYTE_TO_ASCII(enc, p) ((p)[1] == 0 ? (p)[0] : -1)
-#define LITTLE2_CHAR_MATCHES(enc, p, c) ((p)[1] == 0 && (p)[0] == c)
-#define LITTLE2_IS_NAME_CHAR_MINBPC(enc, p) \
+#define LITTLE2_BYTE_TYPE(enc, p)                                              \
+  ((p)[1] == 0 ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)]  \
+               : unicode_byte_type((p)[1], (p)[0]))
+#define LITTLE2_BYTE_TO_ASCII(p) ((p)[1] == 0 ? (p)[0] : -1)
+#define LITTLE2_CHAR_MATCHES(p, c) ((p)[1] == 0 && (p)[0] == c)
+#define LITTLE2_IS_NAME_CHAR_MINBPC(p)                                         \
   UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0])
-#define LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
+#define LITTLE2_IS_NMSTRT_CHAR_MINBPC(p)                                       \
   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[1], (unsigned char)p[0])
 
 #ifdef XML_MIN_SIZE
 
 static int PTRFASTCALL
-little2_byteType(const ENCODING *enc, const char *p)
-{
+little2_byteType(const ENCODING *enc, const char *p) {
   return LITTLE2_BYTE_TYPE(enc, p);
 }
 
 static int PTRFASTCALL
-little2_byteToAscii(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_BYTE_TO_ASCII(enc, p);
+little2_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_BYTE_TO_ASCII(p);
 }
 
 static int PTRCALL
-little2_charMatches(const ENCODING *enc, const char *p, int c)
-{
-  return LITTLE2_CHAR_MATCHES(enc, p, c);
+little2_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
+  return LITTLE2_CHAR_MATCHES(p, c);
 }
 
 static int PTRFASTCALL
-little2_isNameMin(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_IS_NAME_CHAR_MINBPC(enc, p);
+little2_isNameMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_IS_NAME_CHAR_MINBPC(p);
 }
 
 static int PTRFASTCALL
-little2_isNmstrtMin(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p);
+little2_isNmstrtMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_IS_NMSTRT_CHAR_MINBPC(p);
 }
 
-#undef VTABLE
-#define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
+#  undef VTABLE
+#  define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
 
 #else /* not XML_MIN_SIZE */
 
-#undef PREFIX
-#define PREFIX(ident) little2_ ## ident
-#define MINBPC(enc) 2
+#  undef PREFIX
+#  define PREFIX(ident) little2_##ident
+#  define MINBPC(enc) 2
 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
-#define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
-#define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(enc, p)
-#define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(enc, p, c)
-#define IS_NAME_CHAR(enc, p, n) 0
-#define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(enc, p)
-#define IS_NMSTRT_CHAR(enc, p, n) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p)
-
-#define XML_TOK_IMPL_C
-#include "xmltok_impl.c"
-#undef XML_TOK_IMPL_C
-
-#undef MINBPC
-#undef BYTE_TYPE
-#undef BYTE_TO_ASCII
-#undef CHAR_MATCHES
-#undef IS_NAME_CHAR
-#undef IS_NAME_CHAR_MINBPC
-#undef IS_NMSTRT_CHAR
-#undef IS_NMSTRT_CHAR_MINBPC
-#undef IS_INVALID_CHAR
+#  define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
+#  define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p)
+#  define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c)
+#  define IS_NAME_CHAR(enc, p, n) 0
+#  define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p)
+#  define IS_NMSTRT_CHAR(enc, p, n) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p)
+
+#  define XML_TOK_IMPL_C
+#  include "xmltok_impl.c"
+#  undef XML_TOK_IMPL_C
+
+#  undef MINBPC
+#  undef BYTE_TYPE
+#  undef BYTE_TO_ASCII
+#  undef CHAR_MATCHES
+#  undef IS_NAME_CHAR
+#  undef IS_NAME_CHAR_MINBPC
+#  undef IS_NMSTRT_CHAR
+#  undef IS_NMSTRT_CHAR_MINBPC
+#  undef IS_INVALID_CHAR
 
 #endif /* not XML_MIN_SIZE */
 
 #ifdef XML_NS
 
-static const struct normal_encoding little2_encoding_ns = {
-  { VTABLE, 2, 0,
-#if BYTEORDER == 1234
-    1
-#else
-    0
-#endif
-  },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding little2_encoding_ns
+    = {{VTABLE, 2, 0,
+#  if BYTEORDER == 1234
+        1
+#  else
+        0
+#  endif
+       },
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding little2_encoding = {
-  { VTABLE, 2, 0,
+static const struct normal_encoding little2_encoding
+    = {{VTABLE, 2, 0,
 #if BYTEORDER == 1234
-    1
+        1
 #else
-    0
+        0
 #endif
-  },
-  {
+       },
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #if BYTEORDER != 4321
 
-#ifdef XML_NS
+#  ifdef XML_NS
 
-static const struct normal_encoding internal_little2_encoding_ns = {
-  { VTABLE, 2, 0, 1 },
-  {
-#include "iasciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_little2_encoding_ns
+    = {{VTABLE, 2, 0, 1},
+       {
+#    include "iasciitab.h"
+#    include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
-#endif
+#  endif
 
-static const struct normal_encoding internal_little2_encoding = {
-  { VTABLE, 2, 0, 1 },
-  {
-#define BT_COLON BT_NMSTRT
-#include "iasciitab.h"
-#undef BT_COLON
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_little2_encoding
+    = {{VTABLE, 2, 0, 1},
+       {
+#  define BT_COLON BT_NMSTRT
+#  include "iasciitab.h"
+#  undef BT_COLON
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #endif
 
-
-#define BIG2_BYTE_TYPE(enc, p) \
- ((p)[0] == 0 \
-  ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]] \
-  : unicode_byte_type((p)[0], (p)[1]))
-#define BIG2_BYTE_TO_ASCII(enc, p) ((p)[0] == 0 ? (p)[1] : -1)
-#define BIG2_CHAR_MATCHES(enc, p, c) ((p)[0] == 0 && (p)[1] == c)
-#define BIG2_IS_NAME_CHAR_MINBPC(enc, p) \
+#define BIG2_BYTE_TYPE(enc, p)                                                 \
+  ((p)[0] == 0                                                                 \
+       ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]]        \
+       : unicode_byte_type((p)[0], (p)[1]))
+#define BIG2_BYTE_TO_ASCII(p) ((p)[0] == 0 ? (p)[1] : -1)
+#define BIG2_CHAR_MATCHES(p, c) ((p)[0] == 0 && (p)[1] == c)
+#define BIG2_IS_NAME_CHAR_MINBPC(p)                                            \
   UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1])
-#define BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
+#define BIG2_IS_NMSTRT_CHAR_MINBPC(p)                                          \
   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[0], (unsigned char)p[1])
 
 #ifdef XML_MIN_SIZE
 
 static int PTRFASTCALL
-big2_byteType(const ENCODING *enc, const char *p)
-{
+big2_byteType(const ENCODING *enc, const char *p) {
   return BIG2_BYTE_TYPE(enc, p);
 }
 
 static int PTRFASTCALL
-big2_byteToAscii(const ENCODING *enc, const char *p)
-{
-  return BIG2_BYTE_TO_ASCII(enc, p);
+big2_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_BYTE_TO_ASCII(p);
 }
 
 static int PTRCALL
-big2_charMatches(const ENCODING *enc, const char *p, int c)
-{
-  return BIG2_CHAR_MATCHES(enc, p, c);
+big2_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
+  return BIG2_CHAR_MATCHES(p, c);
 }
 
 static int PTRFASTCALL
-big2_isNameMin(const ENCODING *enc, const char *p)
-{
-  return BIG2_IS_NAME_CHAR_MINBPC(enc, p);
+big2_isNameMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_IS_NAME_CHAR_MINBPC(p);
 }
 
 static int PTRFASTCALL
-big2_isNmstrtMin(const ENCODING *enc, const char *p)
-{
-  return BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p);
+big2_isNmstrtMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_IS_NMSTRT_CHAR_MINBPC(p);
 }
 
-#undef VTABLE
-#define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
+#  undef VTABLE
+#  define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
 
 #else /* not XML_MIN_SIZE */
 
-#undef PREFIX
-#define PREFIX(ident) big2_ ## ident
-#define MINBPC(enc) 2
+#  undef PREFIX
+#  define PREFIX(ident) big2_##ident
+#  define MINBPC(enc) 2
 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
-#define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
-#define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(enc, p)
-#define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(enc, p, c)
-#define IS_NAME_CHAR(enc, p, n) 0
-#define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(enc, p)
-#define IS_NMSTRT_CHAR(enc, p, n) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p)
-
-#define XML_TOK_IMPL_C
-#include "xmltok_impl.c"
-#undef XML_TOK_IMPL_C
-
-#undef MINBPC
-#undef BYTE_TYPE
-#undef BYTE_TO_ASCII
-#undef CHAR_MATCHES
-#undef IS_NAME_CHAR
-#undef IS_NAME_CHAR_MINBPC
-#undef IS_NMSTRT_CHAR
-#undef IS_NMSTRT_CHAR_MINBPC
-#undef IS_INVALID_CHAR
+#  define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
+#  define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p)
+#  define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c)
+#  define IS_NAME_CHAR(enc, p, n) 0
+#  define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p)
+#  define IS_NMSTRT_CHAR(enc, p, n) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p)
+
+#  define XML_TOK_IMPL_C
+#  include "xmltok_impl.c"
+#  undef XML_TOK_IMPL_C
+
+#  undef MINBPC
+#  undef BYTE_TYPE
+#  undef BYTE_TO_ASCII
+#  undef CHAR_MATCHES
+#  undef IS_NAME_CHAR
+#  undef IS_NAME_CHAR_MINBPC
+#  undef IS_NMSTRT_CHAR
+#  undef IS_NMSTRT_CHAR_MINBPC
+#  undef IS_INVALID_CHAR
 
 #endif /* not XML_MIN_SIZE */
 
 #ifdef XML_NS
 
-static const struct normal_encoding big2_encoding_ns = {
-  { VTABLE, 2, 0,
-#if BYTEORDER == 4321
-  1
-#else
-  0
-#endif
-  },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding big2_encoding_ns
+    = {{VTABLE, 2, 0,
+#  if BYTEORDER == 4321
+        1
+#  else
+        0
+#  endif
+       },
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding big2_encoding = {
-  { VTABLE, 2, 0,
+static const struct normal_encoding big2_encoding
+    = {{VTABLE, 2, 0,
 #if BYTEORDER == 4321
-  1
+        1
 #else
-  0
+        0
 #endif
-  },
-  {
+       },
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #if BYTEORDER != 1234
 
-#ifdef XML_NS
+#  ifdef XML_NS
 
-static const struct normal_encoding internal_big2_encoding_ns = {
-  { VTABLE, 2, 0, 1 },
-  {
-#include "iasciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_big2_encoding_ns
+    = {{VTABLE, 2, 0, 1},
+       {
+#    include "iasciitab.h"
+#    include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
-#endif
+#  endif
 
-static const struct normal_encoding internal_big2_encoding = {
-  { VTABLE, 2, 0, 1 },
-  {
-#define BT_COLON BT_NMSTRT
-#include "iasciitab.h"
-#undef BT_COLON
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_big2_encoding
+    = {{VTABLE, 2, 0, 1},
+       {
+#  define BT_COLON BT_NMSTRT
+#  include "iasciitab.h"
+#  undef BT_COLON
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #endif
 
 #undef PREFIX
 
 static int FASTCALL
-streqci(const char *s1, const char *s2)
-{
+streqci(const char *s1, const char *s2) {
   for (;;) {
     char c1 = *s1++;
     char c2 = *s2++;
@@ -1079,22 +1018,21 @@ streqci(const char *s1, const char *s2)
       c2 += ASCII_A - ASCII_a; /* LCOV_EXCL_LINE */
     if (c1 != c2)
       return 0;
-    if (!c1)
+    if (! c1)
       break;
   }
   return 1;
 }
 
 static void PTRCALL
-initUpdatePosition(const ENCODING *UNUSED_P(enc), const char *ptr,
-                   const char *end, POSITION *pos)
-{
+initUpdatePosition(const ENCODING *enc, const char *ptr, const char *end,
+                   POSITION *pos) {
+  UNUSED_P(enc);
   normal_updatePosition(&utf8_encoding.enc, ptr, end, pos);
 }
 
 static int
-toAscii(const ENCODING *enc, const char *ptr, const char *end)
-{
+toAscii(const ENCODING *enc, const char *ptr, const char *end) {
   char buf[1];
   char *p = buf;
   XmlUtf8Convert(enc, &ptr, end, &p, p + 1);
@@ -1105,8 +1043,7 @@ toAscii(const ENCODING *enc, const char *ptr, const char *end)
 }
 
 static int FASTCALL
-isSpace(int c)
-{
+isSpace(int c) {
   switch (c) {
   case 0x20:
   case 0xD:
@@ -1121,21 +1058,16 @@ isSpace(int c)
    followed by name=val.
 */
 static int
-parsePseudoAttribute(const ENCODING *enc,
-                     const char *ptr,
-                     const char *end,
-                     const char **namePtr,
-                     const char **nameEndPtr,
-                     const char **valPtr,
-                     const char **nextTokPtr)
-{
+parsePseudoAttribute(const ENCODING *enc, const char *ptr, const char *end,
+                     const char **namePtr, const char **nameEndPtr,
+                     const char **valPtr, const char **nextTokPtr) {
   int c;
   char open;
   if (ptr == end) {
     *namePtr = NULL;
     return 1;
   }
-  if (!isSpace(toAscii(enc, ptr, end))) {
+  if (! isSpace(toAscii(enc, ptr, end))) {
     *nextTokPtr = ptr;
     return 0;
   }
@@ -1191,12 +1123,9 @@ parsePseudoAttribute(const ENCODING *enc,
     c = toAscii(enc, ptr, end);
     if (c == open)
       break;
-    if (!(ASCII_a <= c && c <= ASCII_z)
-        && !(ASCII_A <= c && c <= ASCII_Z)
-        && !(ASCII_0 <= c && c <= ASCII_9)
-        && c != ASCII_PERIOD
-        && c != ASCII_MINUS
-        && c != ASCII_UNDERSCORE) {
+    if (! (ASCII_a <= c && c <= ASCII_z) && ! (ASCII_A <= c && c <= ASCII_Z)
+        && ! (ASCII_0 <= c && c <= ASCII_9) && c != ASCII_PERIOD
+        && c != ASCII_MINUS && c != ASCII_UNDERSCORE) {
       *nextTokPtr = ptr;
       return 0;
     }
@@ -1205,68 +1134,52 @@ parsePseudoAttribute(const ENCODING *enc,
   return 1;
 }
 
-static const char KW_version[] = {
-  ASCII_v, ASCII_e, ASCII_r, ASCII_s, ASCII_i, ASCII_o, ASCII_n, '\0'
-};
+static const char KW_version[]
+    = {ASCII_v, ASCII_e, ASCII_r, ASCII_s, ASCII_i, ASCII_o, ASCII_n, '\0'};
 
-static const char KW_encoding[] = {
-  ASCII_e, ASCII_n, ASCII_c, ASCII_o, ASCII_d, ASCII_i, ASCII_n, ASCII_g, '\0'
-};
+static const char KW_encoding[] = {ASCII_e, ASCII_n, ASCII_c, ASCII_o, ASCII_d,
+                                   ASCII_i, ASCII_n, ASCII_g, '\0'};
 
-static const char KW_standalone[] = {
-  ASCII_s, ASCII_t, ASCII_a, ASCII_n, ASCII_d, ASCII_a, ASCII_l, ASCII_o,
-  ASCII_n, ASCII_e, '\0'
-};
+static const char KW_standalone[]
+    = {ASCII_s, ASCII_t, ASCII_a, ASCII_n, ASCII_d, ASCII_a,
+       ASCII_l, ASCII_o, ASCII_n, ASCII_e, '\0'};
 
-static const char KW_yes[] = {
-  ASCII_y, ASCII_e, ASCII_s,  '\0'
-};
+static const char KW_yes[] = {ASCII_y, ASCII_e, ASCII_s, '\0'};
 
-static const char KW_no[] = {
-  ASCII_n, ASCII_o,  '\0'
-};
+static const char KW_no[] = {ASCII_n, ASCII_o, '\0'};
 
 static int
-doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
-                                                 const char *,
+doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *,
                                                  const char *),
-               int isGeneralTextEntity,
-               const ENCODING *enc,
-               const char *ptr,
-               const char *end,
-               const char **badPtr,
-               const char **versionPtr,
-               const char **versionEndPtr,
-               const char **encodingName,
-               const ENCODING **encoding,
-               int *standalone)
-{
+               int isGeneralTextEntity, const ENCODING *enc, const char *ptr,
+               const char *end, const char **badPtr, const char **versionPtr,
+               const char **versionEndPtr, const char **encodingName,
+               const ENCODING **encoding, int *standalone) {
   const char *val = NULL;
   const char *name = NULL;
   const char *nameEnd = NULL;
   ptr += 5 * enc->minBytesPerChar;
   end -= 2 * enc->minBytesPerChar;
-  if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)
-      || !name) {
+  if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)
+      || ! name) {
     *badPtr = ptr;
     return 0;
   }
-  if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_version)) {
-    if (!isGeneralTextEntity) {
+  if (! XmlNameMatchesAscii(enc, name, nameEnd, KW_version)) {
+    if (! isGeneralTextEntity) {
       *badPtr = name;
       return 0;
     }
-  }
-  else {
+  } else {
     if (versionPtr)
       *versionPtr = val;
     if (versionEndPtr)
       *versionEndPtr = ptr;
-    if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
+    if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
       *badPtr = ptr;
       return 0;
     }
-    if (!name) {
+    if (! name) {
       if (isGeneralTextEntity) {
         /* a TextDecl must have an EncodingDecl */
         *badPtr = ptr;
@@ -1277,7 +1190,7 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
   }
   if (XmlNameMatchesAscii(enc, name, nameEnd, KW_encoding)) {
     int c = toAscii(enc, val, end);
-    if (!(ASCII_a <= c && c <= ASCII_z) && !(ASCII_A <= c && c <= ASCII_Z)) {
+    if (! (ASCII_a <= c && c <= ASCII_z) && ! (ASCII_A <= c && c <= ASCII_Z)) {
       *badPtr = val;
       return 0;
     }
@@ -1285,14 +1198,14 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
       *encodingName = val;
     if (encoding)
       *encoding = encodingFinder(enc, val, ptr - enc->minBytesPerChar);
-    if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
+    if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
       *badPtr = ptr;
       return 0;
     }
-    if (!name)
+    if (! name)
       return 1;
   }
-  if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_standalone)
+  if (! XmlNameMatchesAscii(enc, name, nameEnd, KW_standalone)
       || isGeneralTextEntity) {
     *badPtr = name;
     return 0;
@@ -1300,12 +1213,10 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
   if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_yes)) {
     if (standalone)
       *standalone = 1;
-  }
-  else if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_no)) {
+  } else if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_no)) {
     if (standalone)
       *standalone = 0;
-  }
-  else {
+  } else {
     *badPtr = val;
     return 0;
   }
@@ -1319,11 +1230,16 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
 }
 
 static int FASTCALL
-checkCharRefNumber(int result)
-{
+checkCharRefNumber(int result) {
   switch (result >> 8) {
-  case 0xD8: case 0xD9: case 0xDA: case 0xDB:
-  case 0xDC: case 0xDD: case 0xDE: case 0xDF:
+  case 0xD8:
+  case 0xD9:
+  case 0xDA:
+  case 0xDB:
+  case 0xDC:
+  case 0xDD:
+  case 0xDE:
+  case 0xDF:
     return -1;
   case 0:
     if (latin1_encoding.type[result] == BT_NONXML)
@@ -1338,8 +1254,7 @@ checkCharRefNumber(int result)
 }
 
 int FASTCALL
-XmlUtf8Encode(int c, char *buf)
-{
+XmlUtf8Encode(int c, char *buf) {
   enum {
     /* minN is minimum legal resulting value for N byte sequence */
     min2 = 0x80,
@@ -1375,8 +1290,7 @@ XmlUtf8Encode(int c, char *buf)
 }
 
 int FASTCALL
-XmlUtf16Encode(int charNum, unsigned short *buf)
-{
+XmlUtf16Encode(int charNum, unsigned short *buf) {
   if (charNum < 0)
     return 0;
   if (charNum < 0x10000) {
@@ -1400,17 +1314,15 @@ struct unknown_encoding {
   char utf8[256][4];
 };
 
-#define AS_UNKNOWN_ENCODING(enc)  ((const struct unknown_encoding *) (enc))
+#define AS_UNKNOWN_ENCODING(enc) ((const struct unknown_encoding *)(enc))
 
 int
-XmlSizeOfUnknownEncoding(void)
-{
+XmlSizeOfUnknownEncoding(void) {
   return sizeof(struct unknown_encoding);
 }
 
 static int PTRFASTCALL
-unknown_isName(const ENCODING *enc, const char *p)
-{
+unknown_isName(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   if (c & ~0xFFFF)
@@ -1419,8 +1331,7 @@ unknown_isName(const ENCODING *enc, const char *p)
 }
 
 static int PTRFASTCALL
-unknown_isNmstrt(const ENCODING *enc, const char *p)
-{
+unknown_isNmstrt(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   if (c & ~0xFFFF)
@@ -1429,18 +1340,15 @@ unknown_isNmstrt(const ENCODING *enc, const char *p)
 }
 
 static int PTRFASTCALL
-unknown_isInvalid(const ENCODING *enc, const char *p)
-{
+unknown_isInvalid(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   return (c & ~0xFFFF) || checkCharRefNumber(c) < 0;
 }
 
 static enum XML_Convert_Result PTRCALL
-unknown_toUtf8(const ENCODING *enc,
-               const char **fromP, const char *fromLim,
-               char **toP, const char *toLim)
-{
+unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+               char **toP, const char *toLim) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   char buf[XML_UTF8_ENCODE_MAX];
   for (;;) {
@@ -1458,8 +1366,7 @@ unknown_toUtf8(const ENCODING *enc,
       utf8 = buf;
       *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP]
                  - (BT_LEAD2 - 2));
-    }
-    else {
+    } else {
       if (n > toLim - *toP)
         return XML_CONVERT_OUTPUT_EXHAUSTED;
       (*fromP)++;
@@ -1470,20 +1377,16 @@ unknown_toUtf8(const ENCODING *enc,
 }
 
 static enum XML_Convert_Result PTRCALL
-unknown_toUtf16(const ENCODING *enc,
-                const char **fromP, const char *fromLim,
-                unsigned short **toP, const unsigned short *toLim)
-{
+unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+                unsigned short **toP, const unsigned short *toLim) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   while (*fromP < fromLim && *toP < toLim) {
     unsigned short c = uenc->utf16[(unsigned char)**fromP];
     if (c == 0) {
-      c = (unsigned short)
-          uenc->convert(uenc->userData, *fromP);
+      c = (unsigned short)uenc->convert(uenc->userData, *fromP);
       *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP]
                  - (BT_LEAD2 - 2));
-    }
-    else
+    } else
       (*fromP)++;
     *(*toP)++ = c;
   }
@@ -1495,19 +1398,14 @@ unknown_toUtf16(const ENCODING *enc,
 }
 
 ENCODING *
-XmlInitUnknownEncoding(void *mem,
-                       int *table,
-                       CONVERTER convert,
-                       void *userData)
-{
+XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert,
+                       void *userData) {
   int i;
   struct unknown_encoding *e = (struct unknown_encoding *)mem;
-  for (i = 0; i < (int)sizeof(struct normal_encoding); i++)
-    ((char *)mem)[i] = ((char *)&latin1_encoding)[i];
+  memcpy(mem, &latin1_encoding, sizeof(struct normal_encoding));
   for (i = 0; i < 128; i++)
     if (latin1_encoding.type[i] != BT_OTHER
-        && latin1_encoding.type[i] != BT_NONXML
-        && table[i] != i)
+        && latin1_encoding.type[i] != BT_NONXML && table[i] != i)
       return 0;
   for (i = 0; i < 256; i++) {
     int c = table[i];
@@ -1517,35 +1415,30 @@ XmlInitUnknownEncoding(void *mem,
       e->utf16[i] = 0xFFFF;
       e->utf8[i][0] = 1;
       e->utf8[i][1] = 0;
-    }
-    else if (c < 0) {
+    } else if (c < 0) {
       if (c < -4)
         return 0;
       /* Multi-byte sequences need a converter function */
-      if (!convert)
+      if (! convert)
         return 0;
       e->normal.type[i] = (unsigned char)(BT_LEAD2 - (c + 2));
       e->utf8[i][0] = 0;
       e->utf16[i] = 0;
-    }
-    else if (c < 0x80) {
+    } else if (c < 0x80) {
       if (latin1_encoding.type[c] != BT_OTHER
-          && latin1_encoding.type[c] != BT_NONXML
-          && c != i)
+          && latin1_encoding.type[c] != BT_NONXML && c != i)
         return 0;
       e->normal.type[i] = latin1_encoding.type[c];
       e->utf8[i][0] = 1;
       e->utf8[i][1] = (char)c;
       e->utf16[i] = (unsigned short)(c == 0 ? 0xFFFF : c);
-    }
-    else if (checkCharRefNumber(c) < 0) {
+    } else if (checkCharRefNumber(c) < 0) {
       e->normal.type[i] = BT_NONXML;
       /* This shouldn't really get used. */
       e->utf16[i] = 0xFFFF;
       e->utf8[i][0] = 1;
       e->utf8[i][1] = 0;
-    }
-    else {
+    } else {
       if (c > 0xFFFF)
         return 0;
       if (UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xff))
@@ -1590,44 +1483,32 @@ enum {
   NO_ENC
 };
 
-static const char KW_ISO_8859_1[] = {
-  ASCII_I, ASCII_S, ASCII_O, ASCII_MINUS, ASCII_8, ASCII_8, ASCII_5, ASCII_9,
-  ASCII_MINUS, ASCII_1, '\0'
-};
-static const char KW_US_ASCII[] = {
-  ASCII_U, ASCII_S, ASCII_MINUS, ASCII_A, ASCII_S, ASCII_C, ASCII_I, ASCII_I,
-  '\0'
-};
-static const char KW_UTF_8[] =  {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_8, '\0'
-};
-static const char KW_UTF_16[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, '\0'
-};
-static const char KW_UTF_16BE[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_B, ASCII_E,
-  '\0'
-};
-static const char KW_UTF_16LE[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_L, ASCII_E,
-  '\0'
-};
+static const char KW_ISO_8859_1[]
+    = {ASCII_I, ASCII_S, ASCII_O,     ASCII_MINUS, ASCII_8, ASCII_8,
+       ASCII_5, ASCII_9, ASCII_MINUS, ASCII_1,     '\0'};
+static const char KW_US_ASCII[]
+    = {ASCII_U, ASCII_S, ASCII_MINUS, ASCII_A, ASCII_S,
+       ASCII_C, ASCII_I, ASCII_I,     '\0'};
+static const char KW_UTF_8[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_8, '\0'};
+static const char KW_UTF_16[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, '\0'};
+static const char KW_UTF_16BE[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1,
+       ASCII_6, ASCII_B, ASCII_E, '\0'};
+static const char KW_UTF_16LE[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1,
+       ASCII_6, ASCII_L, ASCII_E, '\0'};
 
 static int FASTCALL
-getEncodingIndex(const char *name)
-{
-  static const char * const encodingNames[] = {
-    KW_ISO_8859_1,
-    KW_US_ASCII,
-    KW_UTF_8,
-    KW_UTF_16,
-    KW_UTF_16BE,
-    KW_UTF_16LE,
+getEncodingIndex(const char *name) {
+  static const char *const encodingNames[] = {
+      KW_ISO_8859_1, KW_US_ASCII, KW_UTF_8, KW_UTF_16, KW_UTF_16BE, KW_UTF_16LE,
   };
   int i;
   if (name == NULL)
     return NO_ENC;
-  for (i = 0; i < (int)(sizeof(encodingNames)/sizeof(encodingNames[0])); i++)
+  for (i = 0; i < (int)(sizeof(encodingNames) / sizeof(encodingNames[0])); i++)
     if (streqci(name, encodingNames[i]))
       return i;
   return UNKNOWN_ENC;
@@ -1647,15 +1528,9 @@ getEncodingIndex(const char *name)
    XML_PROLOG_STATE otherwise.
 */
 
-
 static int
-initScan(const ENCODING * const *encodingTable,
-         const INIT_ENCODING *enc,
-         int state,
-         const char *ptr,
-         const char *end,
-         const char **nextTokPtr)
-{
+initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc,
+         int state, const char *ptr, const char *end, const char **nextTokPtr) {
   const ENCODING **encPtr;
 
   if (ptr >= end)
@@ -1680,20 +1555,17 @@ initScan(const ENCODING * const *encodingTable,
     case 0xFE:
     case 0xFF:
     case 0xEF: /* possibly first byte of UTF-8 BOM */
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       /* fall through */
     case 0x00:
     case 0x3C:
       return XML_TOK_PARTIAL;
     }
-  }
-  else {
+  } else {
     switch (((unsigned char)ptr[0] << 8) | (unsigned char)ptr[1]) {
     case 0xFEFF:
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       *nextTokPtr = ptr + 2;
       *encPtr = encodingTable[UTF_16BE_ENC];
@@ -1707,8 +1579,7 @@ initScan(const ENCODING * const *encodingTable,
       *encPtr = encodingTable[UTF_16LE_ENC];
       return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
     case 0xFFFE:
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       *nextTokPtr = ptr + 2;
       *encPtr = encodingTable[UTF_16LE_ENC];
@@ -1723,8 +1594,8 @@ initScan(const ENCODING * const *encodingTable,
       */
       if (state == XML_CONTENT_STATE) {
         int e = INIT_ENC_INDEX(enc);
-        if (e == ISO_8859_1_ENC || e == UTF_16BE_ENC
-            || e == UTF_16LE_ENC || e == UTF_16_ENC)
+        if (e == ISO_8859_1_ENC || e == UTF_16BE_ENC || e == UTF_16LE_ENC
+            || e == UTF_16_ENC)
           break;
       }
       if (ptr + 2 == end)
@@ -1747,8 +1618,7 @@ initScan(const ENCODING * const *encodingTable,
           break;
         *encPtr = encodingTable[UTF_16BE_ENC];
         return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
-      }
-      else if (ptr[1] == '\0') {
+      } else if (ptr[1] == '\0') {
         /* We could recover here in the case:
             - parsing an external entity
             - second byte is 0
@@ -1770,7 +1640,6 @@ initScan(const ENCODING * const *encodingTable,
   return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
 }
 
-
 #define NS(x) x
 #define ns(x) x
 #define XML_TOK_NS_C
@@ -1781,22 +1650,19 @@ initScan(const ENCODING * const *encodingTable,
 
 #ifdef XML_NS
 
-#define NS(x) x ## NS
-#define ns(x) x ## _ns
+#  define NS(x) x##NS
+#  define ns(x) x##_ns
 
-#define XML_TOK_NS_C
-#include "xmltok_ns.c"
-#undef XML_TOK_NS_C
+#  define XML_TOK_NS_C
+#  include "xmltok_ns.c"
+#  undef XML_TOK_NS_C
 
-#undef NS
-#undef ns
+#  undef NS
+#  undef ns
 
 ENCODING *
-XmlInitUnknownEncodingNS(void *mem,
-                         int *table,
-                         CONVERTER convert,
-                         void *userData)
-{
+XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert,
+                         void *userData) {
   ENCODING *enc = XmlInitUnknownEncoding(mem, table, convert, userData);
   if (enc)
     ((struct normal_encoding *)enc)->type[ASCII_COLON] = BT_COLON;
diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h
index 50926f38ab32..2adbf5307bef 100644
--- a/Modules/expat/xmltok.h
+++ b/Modules/expat/xmltok.h
@@ -38,16 +38,18 @@ extern "C" {
 #endif
 
 /* The following token may be returned by XmlContentTok */
-#define XML_TOK_TRAILING_RSQB -5 /* ] or ]] at the end of the scan; might be
-                                    start of illegal ]]> sequence */
+#define XML_TOK_TRAILING_RSQB                                                  \
+  -5 /* ] or ]] at the end of the scan; might be                               \
+        start of illegal ]]> sequence */
 /* The following tokens may be returned by both XmlPrologTok and
    XmlContentTok.
 */
-#define XML_TOK_NONE -4          /* The string to be scanned is empty */
-#define XML_TOK_TRAILING_CR -3   /* A CR at the end of the scan;
-                                    might be part of CRLF sequence */
-#define XML_TOK_PARTIAL_CHAR -2  /* only part of a multibyte sequence */
-#define XML_TOK_PARTIAL -1       /* only part of a token */
+#define XML_TOK_NONE -4 /* The string to be scanned is empty */
+#define XML_TOK_TRAILING_CR                                                    \
+  -3                            /* A CR at the end of the scan;                \
+                                   might be part of CRLF sequence */
+#define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */
+#define XML_TOK_PARTIAL -1      /* only part of a token */
 #define XML_TOK_INVALID 0
 
 /* The following tokens are returned by XmlContentTok; some are also
@@ -62,24 +64,24 @@ extern "C" {
 #define XML_TOK_DATA_NEWLINE 7
 #define XML_TOK_CDATA_SECT_OPEN 8
 #define XML_TOK_ENTITY_REF 9
-#define XML_TOK_CHAR_REF 10               /* numeric character reference */
+#define XML_TOK_CHAR_REF 10 /* numeric character reference */
 
 /* The following tokens may be returned by both XmlPrologTok and
    XmlContentTok.
 */
-#define XML_TOK_PI 11                     /* processing instruction */
-#define XML_TOK_XML_DECL 12               /* XML decl or text decl */
+#define XML_TOK_PI 11       /* processing instruction */
+#define XML_TOK_XML_DECL 12 /* XML decl or text decl */
 #define XML_TOK_COMMENT 13
-#define XML_TOK_BOM 14                    /* Byte order mark */
+#define XML_TOK_BOM 14 /* Byte order mark */
 
 /* The following tokens are returned only by XmlPrologTok */
 #define XML_TOK_PROLOG_S 15
-#define XML_TOK_DECL_OPEN 16              /* <!foo */
-#define XML_TOK_DECL_CLOSE 17             /* > */
+#define XML_TOK_DECL_OPEN 16  /* <!foo */
+#define XML_TOK_DECL_CLOSE 17 /* > */
 #define XML_TOK_NAME 18
 #define XML_TOK_NMTOKEN 19
-#define XML_TOK_POUND_NAME 20             /* #name */
-#define XML_TOK_OR 21                     /* | */
+#define XML_TOK_POUND_NAME 20 /* #name */
+#define XML_TOK_OR 21         /* | */
 #define XML_TOK_PERCENT 22
 #define XML_TOK_OPEN_PAREN 23
 #define XML_TOK_CLOSE_PAREN 24
@@ -90,14 +92,14 @@ extern "C" {
 #define XML_TOK_INSTANCE_START 29
 
 /* The following occur only in element type declarations */
-#define XML_TOK_NAME_QUESTION 30          /* name? */
-#define XML_TOK_NAME_ASTERISK 31          /* name* */
-#define XML_TOK_NAME_PLUS 32              /* name+ */
-#define XML_TOK_COND_SECT_OPEN 33         /* <![ */
-#define XML_TOK_COND_SECT_CLOSE 34        /* ]]> */
-#define XML_TOK_CLOSE_PAREN_QUESTION 35   /* )? */
-#define XML_TOK_CLOSE_PAREN_ASTERISK 36   /* )* */
-#define XML_TOK_CLOSE_PAREN_PLUS 37       /* )+ */
+#define XML_TOK_NAME_QUESTION 30        /* name? */
+#define XML_TOK_NAME_ASTERISK 31        /* name* */
+#define XML_TOK_NAME_PLUS 32            /* name+ */
+#define XML_TOK_COND_SECT_OPEN 33       /* <![ */
+#define XML_TOK_COND_SECT_CLOSE 34      /* ]]> */
+#define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */
+#define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */
+#define XML_TOK_CLOSE_PAREN_PLUS 37     /* )+ */
 #define XML_TOK_COMMA 38
 
 /* The following token is returned only by XmlAttributeValueTok */
@@ -112,20 +114,20 @@ extern "C" {
 #define XML_TOK_PREFIXED_NAME 41
 
 #ifdef XML_DTD
-#define XML_TOK_IGNORE_SECT 42
+#  define XML_TOK_IGNORE_SECT 42
 #endif /* XML_DTD */
 
 #ifdef XML_DTD
-#define XML_N_STATES 4
+#  define XML_N_STATES 4
 #else /* not XML_DTD */
-#define XML_N_STATES 3
+#  define XML_N_STATES 3
 #endif /* not XML_DTD */
 
 #define XML_PROLOG_STATE 0
 #define XML_CONTENT_STATE 1
 #define XML_CDATA_SECTION_STATE 2
 #ifdef XML_DTD
-#define XML_IGNORE_SECTION_STATE 3
+#  define XML_IGNORE_SECTION_STATE 3
 #endif /* XML_DTD */
 
 #define XML_N_LITERAL_TYPES 2
@@ -153,52 +155,41 @@ typedef struct {
 struct encoding;
 typedef struct encoding ENCODING;
 
-typedef int (PTRCALL *SCANNER)(const ENCODING *,
-                               const char *,
-                               const char *,
-                               const char **);
+typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *,
+                              const char **);
 
 enum XML_Convert_Result {
   XML_CONVERT_COMPLETED = 0,
   XML_CONVERT_INPUT_INCOMPLETE = 1,
-  XML_CONVERT_OUTPUT_EXHAUSTED = 2  /* and therefore potentially input remaining as well */
+  XML_CONVERT_OUTPUT_EXHAUSTED
+  = 2 /* and therefore potentially input remaining as well */
 };
 
 struct encoding {
   SCANNER scanners[XML_N_STATES];
   SCANNER literalScanners[XML_N_LITERAL_TYPES];
-  int (PTRCALL *nameMatchesAscii)(const ENCODING *,
-                                  const char *,
-                                  const char *,
-                                  const char *);
-  int (PTRFASTCALL *nameLength)(const ENCODING *, const char *);
+  int(PTRCALL *nameMatchesAscii)(const ENCODING *, const char *, const char *,
+                                 const char *);
+  int(PTRFASTCALL *nameLength)(const ENCODING *, const char *);
   const char *(PTRFASTCALL *skipS)(const ENCODING *, const char *);
-  int (PTRCALL *getAtts)(const ENCODING *enc,
-                         const char *ptr,
-                         int attsMax,
-                         ATTRIBUTE *atts);
-  int (PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr);
-  int (PTRCALL *predefinedEntityName)(const ENCODING *,
-                                      const char *,
-                                      const char *);
-  void (PTRCALL *updatePosition)(const ENCODING *,
-                                 const char *ptr,
-                                 const char *end,
-                                 POSITION *);
-  int (PTRCALL *isPublicId)(const ENCODING *enc,
-                            const char *ptr,
-                            const char *end,
-                            const char **badPtr);
-  enum XML_Convert_Result (PTRCALL *utf8Convert)(const ENCODING *enc,
-                              const char **fromP,
-                              const char *fromLim,
-                              char **toP,
-                              const char *toLim);
-  enum XML_Convert_Result (PTRCALL *utf16Convert)(const ENCODING *enc,
-                               const char **fromP,
-                               const char *fromLim,
-                               unsigned short **toP,
-                               const unsigned short *toLim);
+  int(PTRCALL *getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
+                        ATTRIBUTE *atts);
+  int(PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr);
+  int(PTRCALL *predefinedEntityName)(const ENCODING *, const char *,
+                                     const char *);
+  void(PTRCALL *updatePosition)(const ENCODING *, const char *ptr,
+                                const char *end, POSITION *);
+  int(PTRCALL *isPublicId)(const ENCODING *enc, const char *ptr,
+                           const char *end, const char **badPtr);
+  enum XML_Convert_Result(PTRCALL *utf8Convert)(const ENCODING *enc,
+                                                const char **fromP,
+                                                const char *fromLim, char **toP,
+                                                const char *toLim);
+  enum XML_Convert_Result(PTRCALL *utf16Convert)(const ENCODING *enc,
+                                                 const char **fromP,
+                                                 const char *fromLim,
+                                                 unsigned short **toP,
+                                                 const unsigned short *toLim);
   int minBytesPerChar;
   char isUtf8;
   char isUtf16;
@@ -225,66 +216,62 @@ struct encoding {
    the prolog outside literals, comments and processing instructions.
 */
 
-
-#define XmlTok(enc, state, ptr, end, nextTokPtr) \
+#define XmlTok(enc, state, ptr, end, nextTokPtr)                               \
   (((enc)->scanners[state])(enc, ptr, end, nextTokPtr))
 
-#define XmlPrologTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr)
+#define XmlPrologTok(enc, ptr, end, nextTokPtr)                                \
+  XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr)
 
-#define XmlContentTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr)
+#define XmlContentTok(enc, ptr, end, nextTokPtr)                               \
+  XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr)
 
-#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr)
+#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr)                          \
+  XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr)
 
 #ifdef XML_DTD
 
-#define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr)
+#  define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr)                       \
+    XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr)
 
 #endif /* XML_DTD */
 
 /* This is used for performing a 2nd-level tokenization on the content
    of a literal that has already been returned by XmlTok.
 */
-#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \
+#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr)                  \
   (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr))
 
-#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \
-   XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr)
+#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr)                        \
+  XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr)
 
-#define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \
-   XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr)
+#define XmlEntityValueTok(enc, ptr, end, nextTokPtr)                           \
+  XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr)
 
-#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \
+#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2)                             \
   (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2))
 
-#define XmlNameLength(enc, ptr) \
-  (((enc)->nameLength)(enc, ptr))
+#define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr))
 
-#define XmlSkipS(enc, ptr) \
-  (((enc)->skipS)(enc, ptr))
+#define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr))
 
-#define XmlGetAttributes(enc, ptr, attsMax, atts) \
+#define XmlGetAttributes(enc, ptr, attsMax, atts)                              \
   (((enc)->getAtts)(enc, ptr, attsMax, atts))
 
-#define XmlCharRefNumber(enc, ptr) \
-  (((enc)->charRefNumber)(enc, ptr))
+#define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr))
 
-#define XmlPredefinedEntityName(enc, ptr, end) \
+#define XmlPredefinedEntityName(enc, ptr, end)                                 \
   (((enc)->predefinedEntityName)(enc, ptr, end))
 
-#define XmlUpdatePosition(enc, ptr, end, pos) \
+#define XmlUpdatePosition(enc, ptr, end, pos)                                  \
   (((enc)->updatePosition)(enc, ptr, end, pos))
 
-#define XmlIsPublicId(enc, ptr, end, badPtr) \
+#define XmlIsPublicId(enc, ptr, end, badPtr)                                   \
   (((enc)->isPublicId)(enc, ptr, end, badPtr))
 
-#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \
+#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim)                        \
   (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim))
 
-#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \
+#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim)                       \
   (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim))
 
 typedef struct {
@@ -292,16 +279,11 @@ typedef struct {
   const ENCODING **encPtr;
 } INIT_ENCODING;
 
-int XmlParseXmlDecl(int isGeneralTextEntity,
-                    const ENCODING *enc,
-                    const char *ptr,
-                    const char *end,
-                    const char **badPtr,
-                    const char **versionPtr,
-                    const char **versionEndPtr,
+int XmlParseXmlDecl(int isGeneralTextEntity, const ENCODING *enc,
+                    const char *ptr, const char *end, const char **badPtr,
+                    const char **versionPtr, const char **versionEndPtr,
                     const char **encodingNamePtr,
-                    const ENCODING **namedEncodingPtr,
-                    int *standalonePtr);
+                    const ENCODING **namedEncodingPtr, int *standalonePtr);
 
 int XmlInitEncoding(INIT_ENCODING *, const ENCODING **, const char *name);
 const ENCODING *XmlGetUtf8InternalEncoding(void);
@@ -310,34 +292,22 @@ int FASTCALL XmlUtf8Encode(int charNumber, char *buf);
 int FASTCALL XmlUtf16Encode(int charNumber, unsigned short *buf);
 int XmlSizeOfUnknownEncoding(void);
 
+typedef int(XMLCALL *CONVERTER)(void *userData, const char *p);
 
-typedef int (XMLCALL *CONVERTER) (void *userData, const char *p);
-
-ENCODING *
-XmlInitUnknownEncoding(void *mem,
-                       int *table,
-                       CONVERTER convert,
-                       void *userData);
+ENCODING *XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert,
+                                 void *userData);
 
-int XmlParseXmlDeclNS(int isGeneralTextEntity,
-                      const ENCODING *enc,
-                      const char *ptr,
-                      const char *end,
-                      const char **badPtr,
-                      const char **versionPtr,
-                      const char **versionEndPtr,
+int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc,
+                      const char *ptr, const char *end, const char **badPtr,
+                      const char **versionPtr, const char **versionEndPtr,
                       const char **encodingNamePtr,
-                      const ENCODING **namedEncodingPtr,
-                      int *standalonePtr);
+                      const ENCODING **namedEncodingPtr, int *standalonePtr);
 
 int XmlInitEncodingNS(INIT_ENCODING *, const ENCODING **, const char *name);
 const ENCODING *XmlGetUtf8InternalEncodingNS(void);
 const ENCODING *XmlGetUtf16InternalEncodingNS(void);
-ENCODING *
-XmlInitUnknownEncodingNS(void *mem,
-                         int *table,
-                         CONVERTER convert,
-                         void *userData);
+ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert,
+                                   void *userData);
 #ifdef __cplusplus
 }
 #endif
diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c
index 4d9ae7dc3896..c209221cd79d 100644
--- a/Modules/expat/xmltok_impl.c
+++ b/Modules/expat/xmltok_impl.c
@@ -32,130 +32,124 @@
 
 #ifdef XML_TOK_IMPL_C
 
-#ifndef IS_INVALID_CHAR
-#define IS_INVALID_CHAR(enc, ptr, n) (0)
-#endif
-
-#define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n) \
-        return XML_TOK_PARTIAL_CHAR; \
-      if (IS_INVALID_CHAR(enc, ptr, n)) { \
-        *(nextTokPtr) = (ptr); \
-        return XML_TOK_INVALID; \
-      } \
-      ptr += n; \
-      break;
+#  ifndef IS_INVALID_CHAR
+#    define IS_INVALID_CHAR(enc, ptr, n) (0)
+#  endif
+
+#  define INVALID_LEAD_CASE(n, ptr, nextTokPtr)                                \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (IS_INVALID_CHAR(enc, ptr, n)) {                                        \
+      *(nextTokPtr) = (ptr);                                                   \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define INVALID_CASES(ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(2, ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(3, ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(4, ptr, nextTokPtr) \
-  case BT_NONXML: \
-  case BT_MALFORM: \
-  case BT_TRAIL: \
-    *(nextTokPtr) = (ptr); \
+#  define INVALID_CASES(ptr, nextTokPtr)                                       \
+    INVALID_LEAD_CASE(2, ptr, nextTokPtr)                                      \
+    INVALID_LEAD_CASE(3, ptr, nextTokPtr)                                      \
+    INVALID_LEAD_CASE(4, ptr, nextTokPtr)                                      \
+  case BT_NONXML:                                                              \
+  case BT_MALFORM:                                                             \
+  case BT_TRAIL:                                                               \
+    *(nextTokPtr) = (ptr);                                                     \
     return XML_TOK_INVALID;
 
-#define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr) \
-   case BT_LEAD ## n: \
-     if (end - ptr < n) \
-       return XML_TOK_PARTIAL_CHAR; \
-     if (!IS_NAME_CHAR(enc, ptr, n)) { \
-       *nextTokPtr = ptr; \
-       return XML_TOK_INVALID; \
-     } \
-     ptr += n; \
-     break;
-
-#define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) \
-  case BT_NONASCII: \
-    if (!IS_NAME_CHAR_MINBPC(enc, ptr)) { \
-      *nextTokPtr = ptr; \
-      return XML_TOK_INVALID; \
-    } \
-    /* fall through */ \
-  case BT_NMSTRT: \
-  case BT_HEX: \
-  case BT_DIGIT: \
-  case BT_NAME: \
-  case BT_MINUS: \
-    ptr += MINBPC(enc); \
-    break; \
-  CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr) \
-  CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr) \
-  CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)
-
-#define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr) \
-   case BT_LEAD ## n: \
-     if (end - ptr < n) \
-       return XML_TOK_PARTIAL_CHAR; \
-     if (!IS_NMSTRT_CHAR(enc, ptr, n)) { \
-       *nextTokPtr = ptr; \
-       return XML_TOK_INVALID; \
-     } \
-     ptr += n; \
-     break;
-
-#define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) \
-  case BT_NONASCII: \
-    if (!IS_NMSTRT_CHAR_MINBPC(enc, ptr)) { \
-      *nextTokPtr = ptr; \
-      return XML_TOK_INVALID; \
-    } \
-    /* fall through */ \
-  case BT_NMSTRT: \
-  case BT_HEX: \
-    ptr += MINBPC(enc); \
-    break; \
-  CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr) \
-  CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr) \
-  CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)
-
-#ifndef PREFIX
-#define PREFIX(ident) ident
-#endif
-
-
-#define HAS_CHARS(enc, ptr, end, count) \
-    (end - ptr >= count * MINBPC(enc))
+#  define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr)                        \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (! IS_NAME_CHAR(enc, ptr, n)) {                                         \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define HAS_CHAR(enc, ptr, end) \
-    HAS_CHARS(enc, ptr, end, 1)
+#  define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)                          \
+  case BT_NONASCII:                                                            \
+    if (! IS_NAME_CHAR_MINBPC(enc, ptr)) {                                     \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    /* fall through */                                                         \
+  case BT_NMSTRT:                                                              \
+  case BT_HEX:                                                                 \
+  case BT_DIGIT:                                                               \
+  case BT_NAME:                                                                \
+  case BT_MINUS:                                                               \
+    ptr += MINBPC(enc);                                                        \
+    break;                                                                     \
+    CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr)                              \
+    CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr)                              \
+    CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)
+
+#  define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr)                      \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (! IS_NMSTRT_CHAR(enc, ptr, n)) {                                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define REQUIRE_CHARS(enc, ptr, end, count) \
-    { \
-      if (! HAS_CHARS(enc, ptr, end, count)) { \
-        return XML_TOK_PARTIAL; \
-      } \
+#  define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)                        \
+  case BT_NONASCII:                                                            \
+    if (! IS_NMSTRT_CHAR_MINBPC(enc, ptr)) {                                   \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    /* fall through */                                                         \
+  case BT_NMSTRT:                                                              \
+  case BT_HEX:                                                                 \
+    ptr += MINBPC(enc);                                                        \
+    break;                                                                     \
+    CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr)                            \
+    CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr)                            \
+    CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)
+
+#  ifndef PREFIX
+#    define PREFIX(ident) ident
+#  endif
+
+#  define HAS_CHARS(enc, ptr, end, count) (end - ptr >= count * MINBPC(enc))
+
+#  define HAS_CHAR(enc, ptr, end) HAS_CHARS(enc, ptr, end, 1)
+
+#  define REQUIRE_CHARS(enc, ptr, end, count)                                  \
+    {                                                                          \
+      if (! HAS_CHARS(enc, ptr, end, count)) {                                 \
+        return XML_TOK_PARTIAL;                                                \
+      }                                                                        \
     }
 
-#define REQUIRE_CHAR(enc, ptr, end) \
-    REQUIRE_CHARS(enc, ptr, end, 1)
-
+#  define REQUIRE_CHAR(enc, ptr, end) REQUIRE_CHARS(enc, ptr, end, 1)
 
 /* ptr points to character following "<!-" */
 
 static int PTRCALL
-PREFIX(scanComment)(const ENCODING *enc, const char *ptr,
-                    const char *end, const char **nextTokPtr)
-{
+PREFIX(scanComment)(const ENCODING *enc, const char *ptr, const char *end,
+                    const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
-    if (!CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
     }
     ptr += MINBPC(enc);
     while (HAS_CHAR(enc, ptr, end)) {
       switch (BYTE_TYPE(enc, ptr)) {
-      INVALID_CASES(ptr, nextTokPtr)
+        INVALID_CASES(ptr, nextTokPtr)
       case BT_MINUS:
         ptr += MINBPC(enc);
         REQUIRE_CHAR(enc, ptr, end);
         if (CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
-          if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+          if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
             *nextTokPtr = ptr;
             return XML_TOK_INVALID;
           }
@@ -175,9 +169,8 @@ PREFIX(scanComment)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "<!" */
 
 static int PTRCALL
-PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
-                 const char *end, const char **nextTokPtr)
-{
+PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, const char *end,
+                 const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
   case BT_MINUS:
@@ -199,12 +192,17 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
       REQUIRE_CHARS(enc, ptr, end, 2);
       /* don't allow <!ENTITY% foo "whatever"> */
       switch (BYTE_TYPE(enc, ptr + MINBPC(enc))) {
-      case BT_S: case BT_CR: case BT_LF: case BT_PERCNT:
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+      case BT_PERCNT:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       /* fall through */
-    case BT_S: case BT_CR: case BT_LF:
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       *nextTokPtr = ptr;
       return XML_TOK_DECL_OPEN;
     case BT_NMSTRT:
@@ -220,12 +218,12 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                      const char *end, int *tokPtr)
-{
+PREFIX(checkPiTarget)(const ENCODING *enc, const char *ptr, const char *end,
+                      int *tokPtr) {
   int upper = 0;
+  UNUSED_P(enc);
   *tokPtr = XML_TOK_PI;
-  if (end - ptr != MINBPC(enc)*3)
+  if (end - ptr != MINBPC(enc) * 3)
     return 1;
   switch (BYTE_TO_ASCII(enc, ptr)) {
   case ASCII_x:
@@ -265,30 +263,31 @@ PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr,
 /* ptr points to character following "<?" */
 
 static int PTRCALL
-PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
-               const char *end, const char **nextTokPtr)
-{
+PREFIX(scanPi)(const ENCODING *enc, const char *ptr, const char *end,
+               const char **nextTokPtr) {
   int tok;
   const char *target = ptr;
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_S: case BT_CR: case BT_LF:
-      if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
+      if (! PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       ptr += MINBPC(enc);
       while (HAS_CHAR(enc, ptr, end)) {
         switch (BYTE_TYPE(enc, ptr)) {
-        INVALID_CASES(ptr, nextTokPtr)
+          INVALID_CASES(ptr, nextTokPtr)
         case BT_QUEST:
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
@@ -304,7 +303,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
       }
       return XML_TOK_PARTIAL;
     case BT_QUEST:
-      if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
+      if (! PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -324,16 +323,16 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                         const char *end, const char **nextTokPtr)
-{
-  static const char CDATA_LSQB[] = { ASCII_C, ASCII_D, ASCII_A,
-                                     ASCII_T, ASCII_A, ASCII_LSQB };
+PREFIX(scanCdataSection)(const ENCODING *enc, const char *ptr, const char *end,
+                         const char **nextTokPtr) {
+  static const char CDATA_LSQB[]
+      = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, ASCII_LSQB};
   int i;
+  UNUSED_P(enc);
   /* CDATA[ */
   REQUIRE_CHARS(enc, ptr, end, 6);
   for (i = 0; i < 6; i++, ptr += MINBPC(enc)) {
-    if (!CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {
+    if (! CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
     }
@@ -343,9 +342,8 @@ PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
-                        const char *end, const char **nextTokPtr)
-{
+PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
+                        const char **nextTokPtr) {
   if (ptr >= end)
     return XML_TOK_NONE;
   if (MINBPC(enc) > 1) {
@@ -361,11 +359,11 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
   case BT_RSQB:
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
-    if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
+    if (! CHAR_MATCHES(enc, ptr, ASCII_RSQB))
       break;
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
-    if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
       ptr -= MINBPC(enc);
       break;
     }
@@ -381,23 +379,25 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
   case BT_LF:
     *nextTokPtr = ptr + MINBPC(enc);
     return XML_TOK_DATA_NEWLINE;
-  INVALID_CASES(ptr, nextTokPtr)
+    INVALID_CASES(ptr, nextTokPtr)
   default:
     ptr += MINBPC(enc);
     break;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \
-        *nextTokPtr = ptr; \
-        return XML_TOK_DATA_CHARS; \
-      } \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) {                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_DATA_CHARS;                                               \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONXML:
     case BT_MALFORM:
     case BT_TRAIL:
@@ -418,23 +418,26 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "</" */
 
 static int PTRCALL
-PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
-                   const char *end, const char **nextTokPtr)
-{
+PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr, const char *end,
+                   const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_S: case BT_CR: case BT_LF:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) {
         switch (BYTE_TYPE(enc, ptr)) {
-        case BT_S: case BT_CR: case BT_LF:
+        case BT_S:
+        case BT_CR:
+        case BT_LF:
           break;
         case BT_GT:
           *nextTokPtr = ptr + MINBPC(enc);
@@ -445,13 +448,13 @@ PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
         }
       }
       return XML_TOK_PARTIAL;
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
       /* no need to check qname syntax here,
          since end-tag must match exactly */
       ptr += MINBPC(enc);
       break;
-#endif
+#  endif
     case BT_GT:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_END_TAG;
@@ -466,9 +469,8 @@ PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "&#X" */
 
 static int PTRCALL
-PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr,
-                       const char *end, const char **nextTokPtr)
-{
+PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, const char *end,
+                       const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
     case BT_DIGIT:
@@ -498,9 +500,8 @@ PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "&#" */
 
 static int PTRCALL
-PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr,
-                    const char *end, const char **nextTokPtr)
-{
+PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, const char *end,
+                    const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
     if (CHAR_MATCHES(enc, ptr, ASCII_x))
       return PREFIX(scanHexCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -531,11 +532,10 @@ PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr,
 
 static int PTRCALL
 PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
-                const char **nextTokPtr)
-{
+                const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   case BT_NUM:
     return PREFIX(scanCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
   default:
@@ -544,7 +544,7 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
     case BT_SEMI:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_ENTITY_REF;
@@ -560,15 +560,14 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
-                 const char **nextTokPtr)
-{
-#ifdef XML_NS
+                 const char **nextTokPtr) {
+#  ifdef XML_NS
   int hadColon = 0;
-#endif
+#  endif
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-#ifdef XML_NS
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+#  ifdef XML_NS
     case BT_COLON:
       if (hadColon) {
         *nextTokPtr = ptr;
@@ -578,14 +577,16 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
       switch (BYTE_TYPE(enc, ptr)) {
-      CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+        CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
       default:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       break;
-#endif
-    case BT_S: case BT_CR: case BT_LF:
+#  endif
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       for (;;) {
         int t;
 
@@ -605,101 +606,101 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
         }
       }
       /* fall through */
-    case BT_EQUALS:
-      {
-        int open;
-#ifdef XML_NS
-        hadColon = 0;
-#endif
-        for (;;) {
-          ptr += MINBPC(enc);
-          REQUIRE_CHAR(enc, ptr, end);
-          open = BYTE_TYPE(enc, ptr);
-          if (open == BT_QUOT || open == BT_APOS)
-            break;
-          switch (open) {
-          case BT_S:
-          case BT_LF:
-          case BT_CR:
-            break;
-          default:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          }
-        }
+    case BT_EQUALS: {
+      int open;
+#  ifdef XML_NS
+      hadColon = 0;
+#  endif
+      for (;;) {
         ptr += MINBPC(enc);
-        /* in attribute value */
-        for (;;) {
-          int t;
-          REQUIRE_CHAR(enc, ptr, end);
-          t = BYTE_TYPE(enc, ptr);
-          if (t == open)
-            break;
-          switch (t) {
+        REQUIRE_CHAR(enc, ptr, end);
+        open = BYTE_TYPE(enc, ptr);
+        if (open == BT_QUOT || open == BT_APOS)
+          break;
+        switch (open) {
+        case BT_S:
+        case BT_LF:
+        case BT_CR:
+          break;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
+        }
+      }
+      ptr += MINBPC(enc);
+      /* in attribute value */
+      for (;;) {
+        int t;
+        REQUIRE_CHAR(enc, ptr, end);
+        t = BYTE_TYPE(enc, ptr);
+        if (t == open)
+          break;
+        switch (t) {
           INVALID_CASES(ptr, nextTokPtr)
-          case BT_AMP:
-            {
-              int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);
-              if (tok <= 0) {
-                if (tok == XML_TOK_INVALID)
-                  *nextTokPtr = ptr;
-                return tok;
-              }
-              break;
-            }
-          case BT_LT:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          default:
-            ptr += MINBPC(enc);
-            break;
+        case BT_AMP: {
+          int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);
+          if (tok <= 0) {
+            if (tok == XML_TOK_INVALID)
+              *nextTokPtr = ptr;
+            return tok;
           }
+          break;
+        }
+        case BT_LT:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
+        default:
+          ptr += MINBPC(enc);
+          break;
         }
+      }
+      ptr += MINBPC(enc);
+      REQUIRE_CHAR(enc, ptr, end);
+      switch (BYTE_TYPE(enc, ptr)) {
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+        break;
+      case BT_SOL:
+        goto sol;
+      case BT_GT:
+        goto gt;
+      default:
+        *nextTokPtr = ptr;
+        return XML_TOK_INVALID;
+      }
+      /* ptr points to closing quote */
+      for (;;) {
         ptr += MINBPC(enc);
         REQUIRE_CHAR(enc, ptr, end);
         switch (BYTE_TYPE(enc, ptr)) {
+          CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
         case BT_S:
         case BT_CR:
         case BT_LF:
-          break;
-        case BT_SOL:
-          goto sol;
+          continue;
         case BT_GT:
-          goto gt;
-        default:
-          *nextTokPtr = ptr;
-          return XML_TOK_INVALID;
-        }
-        /* ptr points to closing quote */
-        for (;;) {
+        gt:
+          *nextTokPtr = ptr + MINBPC(enc);
+          return XML_TOK_START_TAG_WITH_ATTS;
+        case BT_SOL:
+        sol:
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
-          switch (BYTE_TYPE(enc, ptr)) {
-          CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-          case BT_S: case BT_CR: case BT_LF:
-            continue;
-          case BT_GT:
-          gt:
-            *nextTokPtr = ptr + MINBPC(enc);
-            return XML_TOK_START_TAG_WITH_ATTS;
-          case BT_SOL:
-          sol:
-            ptr += MINBPC(enc);
-            REQUIRE_CHAR(enc, ptr, end);
-            if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
-              *nextTokPtr = ptr;
-              return XML_TOK_INVALID;
-            }
-            *nextTokPtr = ptr + MINBPC(enc);
-            return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;
-          default:
+          if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
             *nextTokPtr = ptr;
             return XML_TOK_INVALID;
           }
-          break;
+          *nextTokPtr = ptr + MINBPC(enc);
+          return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
         }
         break;
       }
+      break;
+    }
     default:
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
@@ -712,14 +713,13 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
-               const char **nextTokPtr)
-{
-#ifdef XML_NS
+               const char **nextTokPtr) {
+#  ifdef XML_NS
   int hadColon;
-#endif
+#  endif
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   case BT_EXCL:
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
@@ -727,8 +727,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_MINUS:
       return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);
     case BT_LSQB:
-      return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc),
-                                      end, nextTokPtr);
+      return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc), end, nextTokPtr);
     }
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
@@ -740,14 +739,14 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
-#ifdef XML_NS
+#  ifdef XML_NS
   hadColon = 0;
-#endif
+#  endif
   /* we have a start-tag */
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-#ifdef XML_NS
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+#  ifdef XML_NS
     case BT_COLON:
       if (hadColon) {
         *nextTokPtr = ptr;
@@ -757,34 +756,37 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
       switch (BYTE_TYPE(enc, ptr)) {
-      CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+        CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
       default:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       break;
-#endif
-    case BT_S: case BT_CR: case BT_LF:
-      {
-        ptr += MINBPC(enc);
-        while (HAS_CHAR(enc, ptr, end)) {
-          switch (BYTE_TYPE(enc, ptr)) {
+#  endif
+    case BT_S:
+    case BT_CR:
+    case BT_LF: {
+      ptr += MINBPC(enc);
+      while (HAS_CHAR(enc, ptr, end)) {
+        switch (BYTE_TYPE(enc, ptr)) {
           CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-          case BT_GT:
-            goto gt;
-          case BT_SOL:
-            goto sol;
-          case BT_S: case BT_CR: case BT_LF:
-            ptr += MINBPC(enc);
-            continue;
-          default:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          }
-          return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);
+        case BT_GT:
+          goto gt;
+        case BT_SOL:
+          goto sol;
+        case BT_S:
+        case BT_CR:
+        case BT_LF:
+          ptr += MINBPC(enc);
+          continue;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
         }
-        return XML_TOK_PARTIAL;
+        return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);
       }
+      return XML_TOK_PARTIAL;
+    }
     case BT_GT:
     gt:
       *nextTokPtr = ptr + MINBPC(enc);
@@ -793,7 +795,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     sol:
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
-      if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+      if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -809,8 +811,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **nextTokPtr)
-{
+                   const char **nextTokPtr) {
   if (ptr >= end)
     return XML_TOK_NONE;
   if (MINBPC(enc) > 1) {
@@ -842,48 +843,50 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
     ptr += MINBPC(enc);
     if (! HAS_CHAR(enc, ptr, end))
       return XML_TOK_TRAILING_RSQB;
-    if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
+    if (! CHAR_MATCHES(enc, ptr, ASCII_RSQB))
       break;
     ptr += MINBPC(enc);
     if (! HAS_CHAR(enc, ptr, end))
       return XML_TOK_TRAILING_RSQB;
-    if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
       ptr -= MINBPC(enc);
       break;
     }
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
-  INVALID_CASES(ptr, nextTokPtr)
+    INVALID_CASES(ptr, nextTokPtr)
   default:
     ptr += MINBPC(enc);
     break;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \
-        *nextTokPtr = ptr; \
-        return XML_TOK_DATA_CHARS; \
-      } \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) {                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_DATA_CHARS;                                               \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_RSQB:
       if (HAS_CHARS(enc, ptr, end, 2)) {
-         if (!CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {
-           ptr += MINBPC(enc);
-           break;
-         }
-         if (HAS_CHARS(enc, ptr, end, 3)) {
-           if (!CHAR_MATCHES(enc, ptr + 2*MINBPC(enc), ASCII_GT)) {
-             ptr += MINBPC(enc);
-             break;
-           }
-           *nextTokPtr = ptr + 2*MINBPC(enc);
-           return XML_TOK_INVALID;
-         }
+        if (! CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {
+          ptr += MINBPC(enc);
+          break;
+        }
+        if (HAS_CHARS(enc, ptr, end, 3)) {
+          if (! CHAR_MATCHES(enc, ptr + 2 * MINBPC(enc), ASCII_GT)) {
+            ptr += MINBPC(enc);
+            break;
+          }
+          *nextTokPtr = ptr + 2 * MINBPC(enc);
+          return XML_TOK_INVALID;
+        }
       }
       /* fall through */
     case BT_AMP:
@@ -908,12 +911,14 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
-                    const char **nextTokPtr)
-{
+                    const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-  case BT_S: case BT_LF: case BT_CR: case BT_PERCNT:
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+  case BT_S:
+  case BT_LF:
+  case BT_CR:
+  case BT_PERCNT:
     *nextTokPtr = ptr;
     return XML_TOK_PERCENT;
   default:
@@ -922,7 +927,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
     case BT_SEMI:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_PARAM_ENTITY_REF;
@@ -936,20 +941,24 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,
-                      const char **nextTokPtr)
-{
+                      const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_CR: case BT_LF: case BT_S:
-    case BT_RPAR: case BT_GT: case BT_PERCNT: case BT_VERBAR:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_CR:
+    case BT_LF:
+    case BT_S:
+    case BT_RPAR:
+    case BT_GT:
+    case BT_PERCNT:
+    case BT_VERBAR:
       *nextTokPtr = ptr;
       return XML_TOK_POUND_NAME;
     default:
@@ -961,14 +970,12 @@ PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,
 }
 
 static int PTRCALL
-PREFIX(scanLit)(int open, const ENCODING *enc,
-                const char *ptr, const char *end,
-                const char **nextTokPtr)
-{
+PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end,
+                const char **nextTokPtr) {
   while (HAS_CHAR(enc, ptr, end)) {
     int t = BYTE_TYPE(enc, ptr);
     switch (t) {
-    INVALID_CASES(ptr, nextTokPtr)
+      INVALID_CASES(ptr, nextTokPtr)
     case BT_QUOT:
     case BT_APOS:
       ptr += MINBPC(enc);
@@ -978,8 +985,12 @@ PREFIX(scanLit)(int open, const ENCODING *enc,
         return -XML_TOK_LITERAL;
       *nextTokPtr = ptr;
       switch (BYTE_TYPE(enc, ptr)) {
-      case BT_S: case BT_CR: case BT_LF:
-      case BT_GT: case BT_PERCNT: case BT_LSQB:
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+      case BT_GT:
+      case BT_PERCNT:
+      case BT_LSQB:
         return XML_TOK_LITERAL;
       default:
         return XML_TOK_INVALID;
@@ -994,8 +1005,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc,
 
 static int PTRCALL
 PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
-                  const char **nextTokPtr)
-{
+                  const char **nextTokPtr) {
   int tok;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1013,27 +1023,26 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     return PREFIX(scanLit)(BT_QUOT, enc, ptr + MINBPC(enc), end, nextTokPtr);
   case BT_APOS:
     return PREFIX(scanLit)(BT_APOS, enc, ptr + MINBPC(enc), end, nextTokPtr);
-  case BT_LT:
-    {
-      ptr += MINBPC(enc);
-      REQUIRE_CHAR(enc, ptr, end);
-      switch (BYTE_TYPE(enc, ptr)) {
-      case BT_EXCL:
-        return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-      case BT_QUEST:
-        return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-      case BT_NMSTRT:
-      case BT_HEX:
-      case BT_NONASCII:
-      case BT_LEAD2:
-      case BT_LEAD3:
-      case BT_LEAD4:
-        *nextTokPtr = ptr - MINBPC(enc);
-        return XML_TOK_INSTANCE_START;
-      }
-      *nextTokPtr = ptr;
-      return XML_TOK_INVALID;
+  case BT_LT: {
+    ptr += MINBPC(enc);
+    REQUIRE_CHAR(enc, ptr, end);
+    switch (BYTE_TYPE(enc, ptr)) {
+    case BT_EXCL:
+      return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);
+    case BT_QUEST:
+      return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
+    case BT_NMSTRT:
+    case BT_HEX:
+    case BT_NONASCII:
+    case BT_LEAD2:
+    case BT_LEAD3:
+    case BT_LEAD4:
+      *nextTokPtr = ptr - MINBPC(enc);
+      return XML_TOK_INSTANCE_START;
     }
+    *nextTokPtr = ptr;
+    return XML_TOK_INVALID;
+  }
   case BT_CR:
     if (ptr + MINBPC(enc) == end) {
       *nextTokPtr = end;
@@ -1041,13 +1050,15 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
       return -XML_TOK_PROLOG_S;
     }
     /* fall through */
-  case BT_S: case BT_LF:
+  case BT_S:
+  case BT_LF:
     for (;;) {
       ptr += MINBPC(enc);
       if (! HAS_CHAR(enc, ptr, end))
         break;
       switch (BYTE_TYPE(enc, ptr)) {
-      case BT_S: case BT_LF:
+      case BT_S:
+      case BT_LF:
         break;
       case BT_CR:
         /* don't split CR/LF pair */
@@ -1076,7 +1087,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) {
       REQUIRE_CHARS(enc, ptr, end, 2);
       if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_GT)) {
-        *nextTokPtr = ptr + 2*MINBPC(enc);
+        *nextTokPtr = ptr + 2 * MINBPC(enc);
         return XML_TOK_COND_SECT_CLOSE;
       }
     }
@@ -1099,8 +1110,12 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_PLUS:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_CLOSE_PAREN_PLUS;
-    case BT_CR: case BT_LF: case BT_S:
-    case BT_GT: case BT_COMMA: case BT_VERBAR:
+    case BT_CR:
+    case BT_LF:
+    case BT_S:
+    case BT_GT:
+    case BT_COMMA:
+    case BT_VERBAR:
     case BT_RPAR:
       *nextTokPtr = ptr;
       return XML_TOK_CLOSE_PAREN;
@@ -1115,24 +1130,26 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     return XML_TOK_DECL_CLOSE;
   case BT_NUM:
     return PREFIX(scanPoundName)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-#define LEAD_CASE(n) \
-  case BT_LEAD ## n: \
-    if (end - ptr < n) \
-      return XML_TOK_PARTIAL_CHAR; \
-    if (IS_NMSTRT_CHAR(enc, ptr, n)) { \
-      ptr += n; \
-      tok = XML_TOK_NAME; \
-      break; \
-    } \
-    if (IS_NAME_CHAR(enc, ptr, n)) { \
-      ptr += n; \
-      tok = XML_TOK_NMTOKEN; \
-      break; \
-    } \
-    *nextTokPtr = ptr; \
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (IS_NMSTRT_CHAR(enc, ptr, n)) {                                         \
+      ptr += n;                                                                \
+      tok = XML_TOK_NAME;                                                      \
+      break;                                                                   \
+    }                                                                          \
+    if (IS_NAME_CHAR(enc, ptr, n)) {                                           \
+      ptr += n;                                                                \
+      tok = XML_TOK_NMTOKEN;                                                   \
+      break;                                                                   \
+    }                                                                          \
+    *nextTokPtr = ptr;                                                         \
     return XML_TOK_INVALID;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+    LEAD_CASE(2)
+    LEAD_CASE(3)
+    LEAD_CASE(4)
+#  undef LEAD_CASE
   case BT_NMSTRT:
   case BT_HEX:
     tok = XML_TOK_NAME;
@@ -1141,9 +1158,9 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
   case BT_DIGIT:
   case BT_NAME:
   case BT_MINUS:
-#ifdef XML_NS
+#  ifdef XML_NS
   case BT_COLON:
-#endif
+#  endif
     tok = XML_TOK_NMTOKEN;
     ptr += MINBPC(enc);
     break;
@@ -1165,13 +1182,19 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_GT: case BT_RPAR: case BT_COMMA:
-    case BT_VERBAR: case BT_LSQB: case BT_PERCNT:
-    case BT_S: case BT_CR: case BT_LF:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_GT:
+    case BT_RPAR:
+    case BT_COMMA:
+    case BT_VERBAR:
+    case BT_LSQB:
+    case BT_PERCNT:
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       *nextTokPtr = ptr;
       return tok;
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
       ptr += MINBPC(enc);
       switch (tok) {
@@ -1179,7 +1202,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
         REQUIRE_CHAR(enc, ptr, end);
         tok = XML_TOK_PREFIXED_NAME;
         switch (BYTE_TYPE(enc, ptr)) {
-        CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+          CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
         default:
           tok = XML_TOK_NMTOKEN;
           break;
@@ -1190,23 +1213,23 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
         break;
       }
       break;
-#endif
+#  endif
     case BT_PLUS:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_NAME_PLUS;
     case BT_AST:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_NAME_ASTERISK;
     case BT_QUEST:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -1221,9 +1244,8 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
 }
 
 static int PTRCALL
-PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
-                          const char *end, const char **nextTokPtr)
-{
+PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end,
+                          const char **nextTokPtr) {
   const char *start;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1238,10 +1260,14 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
   start = ptr;
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_AMP:
       if (ptr == start)
         return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -1287,9 +1313,8 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
-                       const char *end, const char **nextTokPtr)
-{
+PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end,
+                       const char **nextTokPtr) {
   const char *start;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1304,10 +1329,14 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
   start = ptr;
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_AMP:
       if (ptr == start)
         return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -1315,8 +1344,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
       return XML_TOK_DATA_CHARS;
     case BT_PERCNT:
       if (ptr == start) {
-        int tok =  PREFIX(scanPercent)(enc, ptr + MINBPC(enc),
-                                       end, nextTokPtr);
+        int tok = PREFIX(scanPercent)(enc, ptr + MINBPC(enc), end, nextTokPtr);
         return (tok == XML_TOK_PERCENT) ? XML_TOK_INVALID : tok;
       }
       *nextTokPtr = ptr;
@@ -1349,12 +1377,11 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
   return XML_TOK_DATA_CHARS;
 }
 
-#ifdef XML_DTD
+#  ifdef XML_DTD
 
 static int PTRCALL
-PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
-                         const char *end, const char **nextTokPtr)
-{
+PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
+                         const char **nextTokPtr) {
   int level = 0;
   if (MINBPC(enc) > 1) {
     size_t n = end - ptr;
@@ -1365,7 +1392,7 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    INVALID_CASES(ptr, nextTokPtr)
+      INVALID_CASES(ptr, nextTokPtr)
     case BT_LT:
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
@@ -1402,12 +1429,11 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
   return XML_TOK_PARTIAL;
 }
 
-#endif /* XML_DTD */
+#  endif /* XML_DTD */
 
 static int PTRCALL
 PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **badPtr)
-{
+                   const char **badPtr) {
   ptr += MINBPC(enc);
   end -= MINBPC(enc);
   for (; HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) {
@@ -1430,9 +1456,9 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_AST:
     case BT_PERCNT:
     case BT_NUM:
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
-#endif
+#  endif
       break;
     case BT_S:
       if (CHAR_MATCHES(enc, ptr, ASCII_TAB)) {
@@ -1442,7 +1468,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
       break;
     case BT_NAME:
     case BT_NMSTRT:
-      if (!(BYTE_TO_ASCII(enc, ptr) & ~0x7f))
+      if (! (BYTE_TO_ASCII(enc, ptr) & ~0x7f))
         break;
       /* fall through */
     default:
@@ -1466,9 +1492,8 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
 */
 
 static int PTRCALL
-PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
-                int attsMax, ATTRIBUTE *atts)
-{
+PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
+                ATTRIBUTE *atts) {
   enum { other, inName, inValue } state = inName;
   int nAtts = 0;
   int open = 0; /* defined when state == inValue;
@@ -1476,32 +1501,35 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
 
   for (ptr += MINBPC(enc);; ptr += MINBPC(enc)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define START_NAME \
-      if (state == other) { \
-        if (nAtts < attsMax) { \
-          atts[nAtts].name = ptr; \
-          atts[nAtts].normalized = 1; \
-        } \
-        state = inName; \
-      }
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: START_NAME ptr += (n - MINBPC(enc)); break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define START_NAME                                                           \
+    if (state == other) {                                                      \
+      if (nAtts < attsMax) {                                                   \
+        atts[nAtts].name = ptr;                                                \
+        atts[nAtts].normalized = 1;                                            \
+      }                                                                        \
+      state = inName;                                                          \
+    }
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    START_NAME ptr += (n - MINBPC(enc));                                       \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONASCII:
     case BT_NMSTRT:
     case BT_HEX:
       START_NAME
       break;
-#undef START_NAME
+#  undef START_NAME
     case BT_QUOT:
       if (state != inValue) {
         if (nAtts < attsMax)
           atts[nAtts].valuePtr = ptr + MINBPC(enc);
         state = inValue;
         open = BT_QUOT;
-      }
-      else if (open == BT_QUOT) {
+      } else if (open == BT_QUOT) {
         state = other;
         if (nAtts < attsMax)
           atts[nAtts].valueEnd = ptr;
@@ -1514,8 +1542,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
           atts[nAtts].valuePtr = ptr + MINBPC(enc);
         state = inValue;
         open = BT_APOS;
-      }
-      else if (open == BT_APOS) {
+      } else if (open == BT_APOS) {
         state = other;
         if (nAtts < attsMax)
           atts[nAtts].valueEnd = ptr;
@@ -1529,16 +1556,15 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
     case BT_S:
       if (state == inName)
         state = other;
-      else if (state == inValue
-               && nAtts < attsMax
-               && atts[nAtts].normalized
+      else if (state == inValue && nAtts < attsMax && atts[nAtts].normalized
                && (ptr == atts[nAtts].valuePtr
                    || BYTE_TO_ASCII(enc, ptr) != ASCII_SPACE
                    || BYTE_TO_ASCII(enc, ptr + MINBPC(enc)) == ASCII_SPACE
                    || BYTE_TYPE(enc, ptr + MINBPC(enc)) == open))
         atts[nAtts].normalized = 0;
       break;
-    case BT_CR: case BT_LF:
+    case BT_CR:
+    case BT_LF:
       /* This case ensures that the first attribute name is counted
          Apart from that we could just change state on the quote. */
       if (state == inName)
@@ -1559,29 +1585,44 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRFASTCALL
-PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
-{
+PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) {
   int result = 0;
   /* skip &# */
-  ptr += 2*MINBPC(enc);
+  UNUSED_P(enc);
+  ptr += 2 * MINBPC(enc);
   if (CHAR_MATCHES(enc, ptr, ASCII_x)) {
-    for (ptr += MINBPC(enc);
-         !CHAR_MATCHES(enc, ptr, ASCII_SEMI);
+    for (ptr += MINBPC(enc); ! CHAR_MATCHES(enc, ptr, ASCII_SEMI);
          ptr += MINBPC(enc)) {
       int c = BYTE_TO_ASCII(enc, ptr);
       switch (c) {
-      case ASCII_0: case ASCII_1: case ASCII_2: case ASCII_3: case ASCII_4:
-      case ASCII_5: case ASCII_6: case ASCII_7: case ASCII_8: case ASCII_9:
+      case ASCII_0:
+      case ASCII_1:
+      case ASCII_2:
+      case ASCII_3:
+      case ASCII_4:
+      case ASCII_5:
+      case ASCII_6:
+      case ASCII_7:
+      case ASCII_8:
+      case ASCII_9:
         result <<= 4;
         result |= (c - ASCII_0);
         break;
-      case ASCII_A: case ASCII_B: case ASCII_C:
-      case ASCII_D: case ASCII_E: case ASCII_F:
+      case ASCII_A:
+      case ASCII_B:
+      case ASCII_C:
+      case ASCII_D:
+      case ASCII_E:
+      case ASCII_F:
         result <<= 4;
         result += 10 + (c - ASCII_A);
         break;
-      case ASCII_a: case ASCII_b: case ASCII_c:
-      case ASCII_d: case ASCII_e: case ASCII_f:
+      case ASCII_a:
+      case ASCII_b:
+      case ASCII_c:
+      case ASCII_d:
+      case ASCII_e:
+      case ASCII_f:
         result <<= 4;
         result += 10 + (c - ASCII_a);
         break;
@@ -1589,9 +1630,8 @@ PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
       if (result >= 0x110000)
         return -1;
     }
-  }
-  else {
-    for (; !CHAR_MATCHES(enc, ptr, ASCII_SEMI); ptr += MINBPC(enc)) {
+  } else {
+    for (; ! CHAR_MATCHES(enc, ptr, ASCII_SEMI); ptr += MINBPC(enc)) {
       int c = BYTE_TO_ASCII(enc, ptr);
       result *= 10;
       result += (c - ASCII_0);
@@ -1603,10 +1643,10 @@ PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
 }
 
 static int PTRCALL
-PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                             const char *end)
-{
-  switch ((end - ptr)/MINBPC(enc)) {
+PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr,
+                             const char *end) {
+  UNUSED_P(enc);
+  switch ((end - ptr) / MINBPC(enc)) {
   case 2:
     if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_t)) {
       switch (BYTE_TO_ASCII(enc, ptr)) {
@@ -1657,9 +1697,9 @@ PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1,
-                         const char *end1, const char *ptr2)
-{
+PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1,
+                         const char *end1, const char *ptr2) {
+  UNUSED_P(enc);
   for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) {
     if (end1 - ptr1 < MINBPC(enc)) {
       /* This line cannot be executed.  The incoming data has already
@@ -1669,27 +1709,30 @@ PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1,
        */
       return 0; /* LCOV_EXCL_LINE */
     }
-    if (!CHAR_MATCHES(enc, ptr1, *ptr2))
+    if (! CHAR_MATCHES(enc, ptr1, *ptr2))
       return 0;
   }
   return ptr1 == end1;
 }
 
 static int PTRFASTCALL
-PREFIX(nameLength)(const ENCODING *enc, const char *ptr)
-{
+PREFIX(nameLength)(const ENCODING *enc, const char *ptr) {
   const char *start = ptr;
   for (;;) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONASCII:
     case BT_NMSTRT:
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
-#endif
+#  endif
     case BT_HEX:
     case BT_DIGIT:
     case BT_NAME:
@@ -1702,9 +1745,8 @@ PREFIX(nameLength)(const ENCODING *enc, const char *ptr)
   }
 }
 
-static const char * PTRFASTCALL
-PREFIX(skipS)(const ENCODING *enc, const char *ptr)
-{
+static const char *PTRFASTCALL
+PREFIX(skipS)(const ENCODING *enc, const char *ptr) {
   for (;;) {
     switch (BYTE_TYPE(enc, ptr)) {
     case BT_LF:
@@ -1719,19 +1761,18 @@ PREFIX(skipS)(const ENCODING *enc, const char *ptr)
 }
 
 static void PTRCALL
-PREFIX(updatePosition)(const ENCODING *enc,
-                       const char *ptr,
-                       const char *end,
-                       POSITION *pos)
-{
+PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end,
+                       POSITION *pos) {
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_LF:
       pos->columnNumber = (XML_Size)-1;
       pos->lineNumber++;
@@ -1752,12 +1793,12 @@ PREFIX(updatePosition)(const ENCODING *enc,
   }
 }
 
-#undef DO_LEAD_CASE
-#undef MULTIBYTE_CASES
-#undef INVALID_CASES
-#undef CHECK_NAME_CASE
-#undef CHECK_NAME_CASES
-#undef CHECK_NMSTRT_CASE
-#undef CHECK_NMSTRT_CASES
+#  undef DO_LEAD_CASE
+#  undef MULTIBYTE_CASES
+#  undef INVALID_CASES
+#  undef CHECK_NAME_CASE
+#  undef CHECK_NAME_CASES
+#  undef CHECK_NMSTRT_CASE
+#  undef CHECK_NMSTRT_CASES
 
 #endif /* XML_TOK_IMPL_C */
diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h
index a6420f48eedc..e925dbc7e2c8 100644
--- a/Modules/expat/xmltok_impl.h
+++ b/Modules/expat/xmltok_impl.h
@@ -31,43 +31,43 @@
 */
 
 enum {
-  BT_NONXML,
-  BT_MALFORM,
-  BT_LT,
-  BT_AMP,
-  BT_RSQB,
-  BT_LEAD2,
-  BT_LEAD3,
-  BT_LEAD4,
-  BT_TRAIL,
-  BT_CR,
-  BT_LF,
-  BT_GT,
-  BT_QUOT,
-  BT_APOS,
-  BT_EQUALS,
-  BT_QUEST,
-  BT_EXCL,
-  BT_SOL,
-  BT_SEMI,
-  BT_NUM,
-  BT_LSQB,
-  BT_S,
-  BT_NMSTRT,
-  BT_COLON,
-  BT_HEX,
-  BT_DIGIT,
-  BT_NAME,
-  BT_MINUS,
-  BT_OTHER, /* known not to be a name or name start character */
+  BT_NONXML,   /* e.g. noncharacter-FFFF */
+  BT_MALFORM,  /* illegal, with regard to encoding */
+  BT_LT,       /* less than = "<" */
+  BT_AMP,      /* ampersand = "&" */
+  BT_RSQB,     /* right square bracket = "[" */
+  BT_LEAD2,    /* lead byte of a 2-byte UTF-8 character */
+  BT_LEAD3,    /* lead byte of a 3-byte UTF-8 character */
+  BT_LEAD4,    /* lead byte of a 4-byte UTF-8 character */
+  BT_TRAIL,    /* trailing unit, e.g. second 16-bit unit of a 4-byte char. */
+  BT_CR,       /* carriage return = "\r" */
+  BT_LF,       /* line feed = "\n" */
+  BT_GT,       /* greater than = ">" */
+  BT_QUOT,     /* quotation character = "\"" */
+  BT_APOS,     /* aposthrophe = "'" */
+  BT_EQUALS,   /* equal sign = "=" */
+  BT_QUEST,    /* question mark = "?" */
+  BT_EXCL,     /* exclamation mark = "!" */
+  BT_SOL,      /* solidus, slash = "/" */
+  BT_SEMI,     /* semicolon = ";" */
+  BT_NUM,      /* number sign = "#" */
+  BT_LSQB,     /* left square bracket = "[" */
+  BT_S,        /* white space, e.g. "\t", " "[, "\r"] */
+  BT_NMSTRT,   /* non-hex name start letter = "G".."Z" + "g".."z" + "_" */
+  BT_COLON,    /* colon = ":" */
+  BT_HEX,      /* hex letter = "A".."F" + "a".."f" */
+  BT_DIGIT,    /* digit = "0".."9" */
+  BT_NAME,     /* dot and middle dot = "." + chr(0xb7) */
+  BT_MINUS,    /* minus = "-" */
+  BT_OTHER,    /* known not to be a name or name start character */
   BT_NONASCII, /* might be a name or name start character */
-  BT_PERCNT,
-  BT_LPAR,
-  BT_RPAR,
-  BT_AST,
-  BT_PLUS,
-  BT_COMMA,
-  BT_VERBAR
+  BT_PERCNT,   /* percent sign = "%" */
+  BT_LPAR,     /* left parenthesis = "(" */
+  BT_RPAR,     /* right parenthesis = "(" */
+  BT_AST,      /* asterisk = "*" */
+  BT_PLUS,     /* plus sign = "+" */
+  BT_COMMA,    /* comma = "," */
+  BT_VERBAR    /* vertical bar = "|" */
 };
 
 #include <stddef.h>
diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c
index 23d31e8e4249..919c74e9f97f 100644
--- a/Modules/expat/xmltok_ns.c
+++ b/Modules/expat/xmltok_ns.c
@@ -33,56 +33,47 @@
 #ifdef XML_TOK_NS_C
 
 const ENCODING *
-NS(XmlGetUtf8InternalEncoding)(void)
-{
+NS(XmlGetUtf8InternalEncoding)(void) {
   return &ns(internal_utf8_encoding).enc;
 }
 
 const ENCODING *
-NS(XmlGetUtf16InternalEncoding)(void)
-{
-#if BYTEORDER == 1234
+NS(XmlGetUtf16InternalEncoding)(void) {
+#  if BYTEORDER == 1234
   return &ns(internal_little2_encoding).enc;
-#elif BYTEORDER == 4321
+#  elif BYTEORDER == 4321
   return &ns(internal_big2_encoding).enc;
-#else
+#  else
   const short n = 1;
-  return (*(const char *)&n
-          ? &ns(internal_little2_encoding).enc
-          : &ns(internal_big2_encoding).enc);
-#endif
+  return (*(const char *)&n ? &ns(internal_little2_encoding).enc
+                            : &ns(internal_big2_encoding).enc);
+#  endif
 }
 
-static const ENCODING * const NS(encodings)[] = {
-  &ns(latin1_encoding).enc,
-  &ns(ascii_encoding).enc,
-  &ns(utf8_encoding).enc,
-  &ns(big2_encoding).enc,
-  &ns(big2_encoding).enc,
-  &ns(little2_encoding).enc,
-  &ns(utf8_encoding).enc /* NO_ENC */
+static const ENCODING *const NS(encodings)[] = {
+    &ns(latin1_encoding).enc, &ns(ascii_encoding).enc,
+    &ns(utf8_encoding).enc,   &ns(big2_encoding).enc,
+    &ns(big2_encoding).enc,   &ns(little2_encoding).enc,
+    &ns(utf8_encoding).enc /* NO_ENC */
 };
 
 static int PTRCALL
 NS(initScanProlog)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **nextTokPtr)
-{
-  return initScan(NS(encodings), (const INIT_ENCODING *)enc,
-                  XML_PROLOG_STATE, ptr, end, nextTokPtr);
+                   const char **nextTokPtr) {
+  return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_PROLOG_STATE,
+                  ptr, end, nextTokPtr);
 }
 
 static int PTRCALL
 NS(initScanContent)(const ENCODING *enc, const char *ptr, const char *end,
-                    const char **nextTokPtr)
-{
-  return initScan(NS(encodings), (const INIT_ENCODING *)enc,
-                  XML_CONTENT_STATE, ptr, end, nextTokPtr);
+                    const char **nextTokPtr) {
+  return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_CONTENT_STATE,
+                  ptr, end, nextTokPtr);
 }
 
 int
 NS(XmlInitEncoding)(INIT_ENCODING *p, const ENCODING **encPtr,
-                    const char *name)
-{
+                    const char *name) {
   int i = getEncodingIndex(name);
   if (i == UNKNOWN_ENC)
     return 0;
@@ -96,9 +87,8 @@ NS(XmlInitEncoding)(INIT_ENCODING *p, const ENCODING **encPtr,
 }
 
 static const ENCODING *
-NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end)
-{
-#define ENCODING_MAX 128
+NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end) {
+#  define ENCODING_MAX 128
   char buf[ENCODING_MAX];
   char *p = buf;
   int i;
@@ -115,28 +105,14 @@ NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end)
 }
 
 int
-NS(XmlParseXmlDecl)(int isGeneralTextEntity,
-                    const ENCODING *enc,
-                    const char *ptr,
-                    const char *end,
-                    const char **badPtr,
-                    const char **versionPtr,
-                    const char **versionEndPtr,
-                    const char **encodingName,
-                    const ENCODING **encoding,
-                    int *standalone)
-{
-  return doParseXmlDecl(NS(findEncoding),
-                        isGeneralTextEntity,
-                        enc,
-                        ptr,
-                        end,
-                        badPtr,
-                        versionPtr,
-                        versionEndPtr,
-                        encodingName,
-                        encoding,
-                        standalone);
+NS(XmlParseXmlDecl)(int isGeneralTextEntity, const ENCODING *enc,
+                    const char *ptr, const char *end, const char **badPtr,
+                    const char **versionPtr, const char **versionEndPtr,
+                    const char **encodingName, const ENCODING **encoding,
+                    int *standalone) {
+  return doParseXmlDecl(NS(findEncoding), isGeneralTextEntity, enc, ptr, end,
+                        badPtr, versionPtr, versionEndPtr, encodingName,
+                        encoding, standalone);
 }
 
 #endif /* XML_TOK_NS_C */
diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj
index 1b024b9d854c..511e26bb60db 100644
--- a/PCbuild/_elementtree.vcxproj
+++ b/PCbuild/_elementtree.vcxproj
@@ -87,7 +87,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\_elementtree.c" />
-    <ClCompile Include="..\Modules\expat\loadlibrary.c" />
     <ClCompile Include="..\Modules\expat\xmlparse.c" />
     <ClCompile Include="..\Modules\expat\xmlrole.c" />
     <ClCompile Include="..\Modules\expat\xmltok.c" />
diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters
index 4597ee521b33..6acdf35846ab 100644
--- a/PCbuild/_elementtree.vcxproj.filters
+++ b/PCbuild/_elementtree.vcxproj.filters
@@ -33,9 +33,6 @@
     <ClInclude Include="..\Modules\expat\latin1tab.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\expat\loadlibrary.c">
-      <Filter>Header Files</Filter>
-    </ClInclude>
     <ClInclude Include="..\Modules\expat\macconfig.h">
       <Filter>Header Files</Filter>
     </ClInclude>
diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj
index e8be337c2590..61a2697e1166 100644
--- a/PCbuild/pyexpat.vcxproj
+++ b/PCbuild/pyexpat.vcxproj
@@ -68,7 +68,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\pyexpat.c" />
-    <ClCompile Include="..\Modules\expat\loadlibrary.c" />
     <ClCompile Include="..\Modules\expat\xmlparse.c" />
     <ClCompile Include="..\Modules\expat\xmlrole.c" />
     <ClCompile Include="..\Modules\expat\xmltok.c" />
diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters
index cb02847980c6..f8d46026c9c2 100644
--- a/PCbuild/pyexpat.vcxproj.filters
+++ b/PCbuild/pyexpat.vcxproj.filters
@@ -20,9 +20,6 @@
     <ClCompile Include="..\Modules\pyexpat.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\expat\loadlibrary.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Modules\expat\xmlparse.c">
       <Filter>Source Files</Filter>
     </ClCompile>


From webhook-mailer at python.org  Thu Sep 26 00:55:04 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 26 Sep 2019 04:55:04 -0000
Subject: [Python-checkins] [3.7] closes bpo-38174: Update vendored expat
 library to 2.2.8. (GH-16407)
Message-ID: <mailman.96.1569473706.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/8e4622ea898bdda687eb2fb0747721c0cd8b0d02
commit: 8e4622ea898bdda687eb2fb0747721c0cd8b0d02
branch: 3.7
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-25T21:54:59-07:00
summary:

[3.7] closes bpo-38174: Update vendored expat library to 2.2.8. (GH-16407)

Fixes CVE-2019-15903. See full changelog at https://github.com/libexpat/libexpat/blob/R_2_2_8/expat/Changes..
(cherry picked from commit 52b940803860e37bcc3f6096b2d24e7c20a0e807)

Co-authored-by: Benjamin Peterson <benjamin at python.org>

files:
A Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
D Modules/expat/loadlibrary.c
M Modules/expat/asciitab.h
M Modules/expat/expat.h
M Modules/expat/expat_external.h
M Modules/expat/iasciitab.h
M Modules/expat/internal.h
M Modules/expat/latin1tab.h
M Modules/expat/nametab.h
M Modules/expat/siphash.h
M Modules/expat/utf8tab.h
M Modules/expat/winconfig.h
M Modules/expat/xmlparse.c
M Modules/expat/xmlrole.c
M Modules/expat/xmlrole.h
M Modules/expat/xmltok.c
M Modules/expat/xmltok.h
M Modules/expat/xmltok_impl.c
M Modules/expat/xmltok_impl.h
M Modules/expat/xmltok_ns.c
M PCbuild/_elementtree.vcxproj
M PCbuild/_elementtree.vcxproj.filters
M PCbuild/pyexpat.vcxproj
M PCbuild/pyexpat.vcxproj.filters

diff --git a/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst b/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
new file mode 100644
index 000000000000..e3da6fcdd139
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
@@ -0,0 +1,2 @@
+Update vendorized expat library version to 2.2.8, which resolves
+CVE-2019-15903.
diff --git a/Modules/expat/asciitab.h b/Modules/expat/asciitab.h
index 2f59fd929065..63b1d1b4482e 100644
--- a/Modules/expat/asciitab.h
+++ b/Modules/expat/asciitab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
-/* 0x0C */ BT_NONXML, BT_CR, BT_NONXML, BT_NONXML,
-/* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
-/* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
-/* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
-/* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
-/* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
-/* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
-/* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
-/* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
-/* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
+    /* 0x0C */ BT_NONXML, BT_CR, BT_NONXML, BT_NONXML,
+    /* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
+    /* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
+    /* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
+    /* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
+    /* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
+    /* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
+    /* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
+    /* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
+    /* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h
index c050f1d918ce..6c8eb1fda4a3 100644
--- a/Modules/expat/expat.h
+++ b/Modules/expat/expat.h
@@ -33,15 +33,6 @@
 #ifndef Expat_INCLUDED
 #define Expat_INCLUDED 1
 
-#ifdef __VMS
-/*      0        1         2         3      0        1         2         3
-        1234567890123456789012345678901     1234567890123456789012345678901 */
-#define XML_SetProcessingInstructionHandler XML_SetProcessingInstrHandler
-#define XML_SetUnparsedEntityDeclHandler    XML_SetUnparsedEntDeclHandler
-#define XML_SetStartNamespaceDeclHandler    XML_SetStartNamespcDeclHandler
-#define XML_SetExternalEntityRefHandlerArg  XML_SetExternalEntRefHandlerArg
-#endif
-
 #include <stdlib.h>
 #include "expat_external.h"
 
@@ -53,8 +44,8 @@ struct XML_ParserStruct;
 typedef struct XML_ParserStruct *XML_Parser;
 
 typedef unsigned char XML_Bool;
-#define XML_TRUE   ((XML_Bool) 1)
-#define XML_FALSE  ((XML_Bool) 0)
+#define XML_TRUE ((XML_Bool)1)
+#define XML_FALSE ((XML_Bool)0)
 
 /* The XML_Status enum gives the possible return values for several
    API functions.  The preprocessor #defines are included so this
@@ -164,25 +155,23 @@ enum XML_Content_Quant {
 typedef struct XML_cp XML_Content;
 
 struct XML_cp {
-  enum XML_Content_Type         type;
-  enum XML_Content_Quant        quant;
-  XML_Char *                    name;
-  unsigned int                  numchildren;
-  XML_Content *                 children;
+  enum XML_Content_Type type;
+  enum XML_Content_Quant quant;
+  XML_Char *name;
+  unsigned int numchildren;
+  XML_Content *children;
 };
 
-
 /* This is called for an element declaration. See above for
    description of the model argument. It's the caller's responsibility
    to free model when finished with it.
 */
-typedef void (XMLCALL *XML_ElementDeclHandler) (void *userData,
-                                                const XML_Char *name,
-                                                XML_Content *model);
+typedef void(XMLCALL *XML_ElementDeclHandler)(void *userData,
+                                              const XML_Char *name,
+                                              XML_Content *model);
 
 XMLPARSEAPI(void)
-XML_SetElementDeclHandler(XML_Parser parser,
-                          XML_ElementDeclHandler eldecl);
+XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl);
 
 /* The Attlist declaration handler is called for *each* attribute. So
    a single Attlist declaration with multiple attributes declared will
@@ -192,17 +181,12 @@ XML_SetElementDeclHandler(XML_Parser parser,
    value will be NULL in the case of "#REQUIRED". If "isrequired" is
    true and default is non-NULL, then this is a "#FIXED" default.
 */
-typedef void (XMLCALL *XML_AttlistDeclHandler) (
-                                    void            *userData,
-                                    const XML_Char  *elname,
-                                    const XML_Char  *attname,
-                                    const XML_Char  *att_type,
-                                    const XML_Char  *dflt,
-                                    int              isrequired);
+typedef void(XMLCALL *XML_AttlistDeclHandler)(
+    void *userData, const XML_Char *elname, const XML_Char *attname,
+    const XML_Char *att_type, const XML_Char *dflt, int isrequired);
 
 XMLPARSEAPI(void)
-XML_SetAttlistDeclHandler(XML_Parser parser,
-                          XML_AttlistDeclHandler attdecl);
+XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl);
 
 /* The XML declaration handler is called for *both* XML declarations
    and text declarations. The way to distinguish is that the version
@@ -212,15 +196,13 @@ XML_SetAttlistDeclHandler(XML_Parser parser,
    was no standalone parameter in the declaration, that it was given
    as no, or that it was given as yes.
 */
-typedef void (XMLCALL *XML_XmlDeclHandler) (void           *userData,
-                                            const XML_Char *version,
-                                            const XML_Char *encoding,
-                                            int             standalone);
+typedef void(XMLCALL *XML_XmlDeclHandler)(void *userData,
+                                          const XML_Char *version,
+                                          const XML_Char *encoding,
+                                          int standalone);
 
 XMLPARSEAPI(void)
-XML_SetXmlDeclHandler(XML_Parser parser,
-                      XML_XmlDeclHandler xmldecl);
-
+XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler xmldecl);
 
 typedef struct {
   void *(*malloc_fcn)(size_t size);
@@ -248,7 +230,6 @@ XML_ParserCreate(const XML_Char *encoding);
 XMLPARSEAPI(XML_Parser)
 XML_ParserCreateNS(const XML_Char *encoding, XML_Char namespaceSeparator);
 
-
 /* Constructs a new parser using the memory management suite referred to
    by memsuite. If memsuite is NULL, then use the standard library memory
    suite. If namespaceSeparator is non-NULL it creates a parser with
@@ -278,31 +259,27 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encoding);
 /* atts is array of name/value pairs, terminated by 0;
    names and values are 0 terminated.
 */
-typedef void (XMLCALL *XML_StartElementHandler) (void *userData,
-                                                 const XML_Char *name,
-                                                 const XML_Char **atts);
-
-typedef void (XMLCALL *XML_EndElementHandler) (void *userData,
-                                               const XML_Char *name);
+typedef void(XMLCALL *XML_StartElementHandler)(void *userData,
+                                               const XML_Char *name,
+                                               const XML_Char **atts);
 
+typedef void(XMLCALL *XML_EndElementHandler)(void *userData,
+                                             const XML_Char *name);
 
 /* s is not 0 terminated. */
-typedef void (XMLCALL *XML_CharacterDataHandler) (void *userData,
-                                                  const XML_Char *s,
-                                                  int len);
+typedef void(XMLCALL *XML_CharacterDataHandler)(void *userData,
+                                                const XML_Char *s, int len);
 
 /* target and data are 0 terminated */
-typedef void (XMLCALL *XML_ProcessingInstructionHandler) (
-                                                void *userData,
-                                                const XML_Char *target,
-                                                const XML_Char *data);
+typedef void(XMLCALL *XML_ProcessingInstructionHandler)(void *userData,
+                                                        const XML_Char *target,
+                                                        const XML_Char *data);
 
 /* data is 0 terminated */
-typedef void (XMLCALL *XML_CommentHandler) (void *userData,
-                                            const XML_Char *data);
+typedef void(XMLCALL *XML_CommentHandler)(void *userData, const XML_Char *data);
 
-typedef void (XMLCALL *XML_StartCdataSectionHandler) (void *userData);
-typedef void (XMLCALL *XML_EndCdataSectionHandler) (void *userData);
+typedef void(XMLCALL *XML_StartCdataSectionHandler)(void *userData);
+typedef void(XMLCALL *XML_EndCdataSectionHandler)(void *userData);
 
 /* This is called for any characters in the XML document for which
    there is no applicable handler.  This includes both characters that
@@ -317,25 +294,23 @@ typedef void (XMLCALL *XML_EndCdataSectionHandler) (void *userData);
    default handler: for example, a comment might be split between
    multiple calls.
 */
-typedef void (XMLCALL *XML_DefaultHandler) (void *userData,
-                                            const XML_Char *s,
-                                            int len);
+typedef void(XMLCALL *XML_DefaultHandler)(void *userData, const XML_Char *s,
+                                          int len);
 
 /* This is called for the start of the DOCTYPE declaration, before
    any DTD or internal subset is parsed.
 */
-typedef void (XMLCALL *XML_StartDoctypeDeclHandler) (
-                                            void *userData,
-                                            const XML_Char *doctypeName,
-                                            const XML_Char *sysid,
-                                            const XML_Char *pubid,
-                                            int has_internal_subset);
+typedef void(XMLCALL *XML_StartDoctypeDeclHandler)(void *userData,
+                                                   const XML_Char *doctypeName,
+                                                   const XML_Char *sysid,
+                                                   const XML_Char *pubid,
+                                                   int has_internal_subset);
 
 /* This is called for the start of the DOCTYPE declaration when the
    closing > is encountered, but after processing any external
    subset.
 */
-typedef void (XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
+typedef void(XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
 
 /* This is called for entity declarations. The is_parameter_entity
    argument will be non-zero if the entity is a parameter entity, zero
@@ -355,20 +330,14 @@ typedef void (XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
    Note that is_parameter_entity can't be changed to XML_Bool, since
    that would break binary compatibility.
 */
-typedef void (XMLCALL *XML_EntityDeclHandler) (
-                              void *userData,
-                              const XML_Char *entityName,
-                              int is_parameter_entity,
-                              const XML_Char *value,
-                              int value_length,
-                              const XML_Char *base,
-                              const XML_Char *systemId,
-                              const XML_Char *publicId,
-                              const XML_Char *notationName);
+typedef void(XMLCALL *XML_EntityDeclHandler)(
+    void *userData, const XML_Char *entityName, int is_parameter_entity,
+    const XML_Char *value, int value_length, const XML_Char *base,
+    const XML_Char *systemId, const XML_Char *publicId,
+    const XML_Char *notationName);
 
 XMLPARSEAPI(void)
-XML_SetEntityDeclHandler(XML_Parser parser,
-                         XML_EntityDeclHandler handler);
+XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler);
 
 /* OBSOLETE -- OBSOLETE -- OBSOLETE
    This handler has been superseded by the EntityDeclHandler above.
@@ -379,24 +348,20 @@ XML_SetEntityDeclHandler(XML_Parser parser,
    entityName, systemId and notationName arguments will never be
    NULL. The other arguments may be.
 */
-typedef void (XMLCALL *XML_UnparsedEntityDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *entityName,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId,
-                                    const XML_Char *notationName);
+typedef void(XMLCALL *XML_UnparsedEntityDeclHandler)(
+    void *userData, const XML_Char *entityName, const XML_Char *base,
+    const XML_Char *systemId, const XML_Char *publicId,
+    const XML_Char *notationName);
 
 /* This is called for a declaration of notation.  The base argument is
    whatever was set by XML_SetBase. The notationName will never be
    NULL.  The other arguments can be.
 */
-typedef void (XMLCALL *XML_NotationDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *notationName,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId);
+typedef void(XMLCALL *XML_NotationDeclHandler)(void *userData,
+                                               const XML_Char *notationName,
+                                               const XML_Char *base,
+                                               const XML_Char *systemId,
+                                               const XML_Char *publicId);
 
 /* When namespace processing is enabled, these are called once for
    each namespace declaration. The call to the start and end element
@@ -404,14 +369,12 @@ typedef void (XMLCALL *XML_NotationDeclHandler) (
    declaration handlers. For an xmlns attribute, prefix will be
    NULL.  For an xmlns="" attribute, uri will be NULL.
 */
-typedef void (XMLCALL *XML_StartNamespaceDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *prefix,
-                                    const XML_Char *uri);
+typedef void(XMLCALL *XML_StartNamespaceDeclHandler)(void *userData,
+                                                     const XML_Char *prefix,
+                                                     const XML_Char *uri);
 
-typedef void (XMLCALL *XML_EndNamespaceDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *prefix);
+typedef void(XMLCALL *XML_EndNamespaceDeclHandler)(void *userData,
+                                                   const XML_Char *prefix);
 
 /* This is called if the document is not standalone, that is, it has an
    external subset or a reference to a parameter entity, but does not
@@ -422,7 +385,7 @@ typedef void (XMLCALL *XML_EndNamespaceDeclHandler) (
    conditions above this handler will only be called if the referenced
    entity was actually read.
 */
-typedef int (XMLCALL *XML_NotStandaloneHandler) (void *userData);
+typedef int(XMLCALL *XML_NotStandaloneHandler)(void *userData);
 
 /* This is called for a reference to an external parsed general
    entity.  The referenced entity is not automatically parsed.  The
@@ -458,12 +421,11 @@ typedef int (XMLCALL *XML_NotStandaloneHandler) (void *userData);
    Note that unlike other handlers the first argument is the parser,
    not userData.
 */
-typedef int (XMLCALL *XML_ExternalEntityRefHandler) (
-                                    XML_Parser parser,
-                                    const XML_Char *context,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId);
+typedef int(XMLCALL *XML_ExternalEntityRefHandler)(XML_Parser parser,
+                                                   const XML_Char *context,
+                                                   const XML_Char *base,
+                                                   const XML_Char *systemId,
+                                                   const XML_Char *publicId);
 
 /* This is called in two situations:
    1) An entity reference is encountered for which no declaration
@@ -475,10 +437,9 @@ typedef int (XMLCALL *XML_ExternalEntityRefHandler) (
          the event would be out of sync with the reporting of the
          declarations or attribute values
 */
-typedef void (XMLCALL *XML_SkippedEntityHandler) (
-                                    void *userData,
-                                    const XML_Char *entityName,
-                                    int is_parameter_entity);
+typedef void(XMLCALL *XML_SkippedEntityHandler)(void *userData,
+                                                const XML_Char *entityName,
+                                                int is_parameter_entity);
 
 /* This structure is filled in by the XML_UnknownEncodingHandler to
    provide information to the parser about encodings that are unknown
@@ -535,8 +496,8 @@ typedef void (XMLCALL *XML_SkippedEntityHandler) (
 typedef struct {
   int map[256];
   void *data;
-  int (XMLCALL *convert)(void *data, const char *s);
-  void (XMLCALL *release)(void *data);
+  int(XMLCALL *convert)(void *data, const char *s);
+  void(XMLCALL *release)(void *data);
 } XML_Encoding;
 
 /* This is called for an encoding that is unknown to the parser.
@@ -554,23 +515,19 @@ typedef struct {
    If info does not describe a suitable encoding, then the parser will
    return an XML_UNKNOWN_ENCODING error.
 */
-typedef int (XMLCALL *XML_UnknownEncodingHandler) (
-                                    void *encodingHandlerData,
-                                    const XML_Char *name,
-                                    XML_Encoding *info);
+typedef int(XMLCALL *XML_UnknownEncodingHandler)(void *encodingHandlerData,
+                                                 const XML_Char *name,
+                                                 XML_Encoding *info);
 
 XMLPARSEAPI(void)
-XML_SetElementHandler(XML_Parser parser,
-                      XML_StartElementHandler start,
+XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start,
                       XML_EndElementHandler end);
 
 XMLPARSEAPI(void)
-XML_SetStartElementHandler(XML_Parser parser,
-                           XML_StartElementHandler handler);
+XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetEndElementHandler(XML_Parser parser,
-                         XML_EndElementHandler handler);
+XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetCharacterDataHandler(XML_Parser parser,
@@ -580,8 +537,7 @@ XMLPARSEAPI(void)
 XML_SetProcessingInstructionHandler(XML_Parser parser,
                                     XML_ProcessingInstructionHandler handler);
 XMLPARSEAPI(void)
-XML_SetCommentHandler(XML_Parser parser,
-                      XML_CommentHandler handler);
+XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetCdataSectionHandler(XML_Parser parser,
@@ -601,20 +557,17 @@ XML_SetEndCdataSectionHandler(XML_Parser parser,
    default handler, or to the skipped entity handler, if one is set.
 */
 XMLPARSEAPI(void)
-XML_SetDefaultHandler(XML_Parser parser,
-                      XML_DefaultHandler handler);
+XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler);
 
 /* This sets the default handler but does not inhibit expansion of
    internal entities.  The entity reference will not be passed to the
    default handler.
 */
 XMLPARSEAPI(void)
-XML_SetDefaultHandlerExpand(XML_Parser parser,
-                            XML_DefaultHandler handler);
+XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetDoctypeDeclHandler(XML_Parser parser,
-                          XML_StartDoctypeDeclHandler start,
+XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start,
                           XML_EndDoctypeDeclHandler end);
 
 XMLPARSEAPI(void)
@@ -622,16 +575,14 @@ XML_SetStartDoctypeDeclHandler(XML_Parser parser,
                                XML_StartDoctypeDeclHandler start);
 
 XMLPARSEAPI(void)
-XML_SetEndDoctypeDeclHandler(XML_Parser parser,
-                             XML_EndDoctypeDeclHandler end);
+XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end);
 
 XMLPARSEAPI(void)
 XML_SetUnparsedEntityDeclHandler(XML_Parser parser,
                                  XML_UnparsedEntityDeclHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetNotationDeclHandler(XML_Parser parser,
-                           XML_NotationDeclHandler handler);
+XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetNamespaceDeclHandler(XML_Parser parser,
@@ -659,8 +610,7 @@ XML_SetExternalEntityRefHandler(XML_Parser parser,
    instead of the parser object.
 */
 XMLPARSEAPI(void)
-XML_SetExternalEntityRefHandlerArg(XML_Parser parser,
-                                   void *arg);
+XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg);
 
 XMLPARSEAPI(void)
 XML_SetSkippedEntityHandler(XML_Parser parser,
@@ -740,7 +690,6 @@ XML_UseParserAsHandlerArg(XML_Parser parser);
 XMLPARSEAPI(enum XML_Error)
 XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD);
 
-
 /* Sets the base to be used for resolving relative URIs in system
    identifiers in declarations.  Resolving relative identifiers is
    left to the application: this value will be passed through as the
@@ -780,10 +729,10 @@ XML_GetIdAttributeIndex(XML_Parser parser);
    info->valueEnd - info->valueStart = 4 bytes.
 */
 typedef struct {
-  XML_Index  nameStart;  /* Offset to beginning of the attribute name. */
-  XML_Index  nameEnd;    /* Offset after the attribute name's last byte. */
-  XML_Index  valueStart; /* Offset to beginning of the attribute value. */
-  XML_Index  valueEnd;   /* Offset after the attribute value's last byte. */
+  XML_Index nameStart;  /* Offset to beginning of the attribute name. */
+  XML_Index nameEnd;    /* Offset after the attribute name's last byte. */
+  XML_Index valueStart; /* Offset to beginning of the attribute value. */
+  XML_Index valueEnd;   /* Offset after the attribute value's last byte. */
 } XML_AttrInfo;
 
 /* Returns an array of XML_AttrInfo structures for the attribute/value pairs
@@ -819,20 +768,20 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal);
    (resumable = 0) an already suspended parser. Some call-backs may
    still follow because they would otherwise get lost. Examples:
    - endElementHandler() for empty elements when stopped in
-     startElementHandler(), 
-   - endNameSpaceDeclHandler() when stopped in endElementHandler(), 
+     startElementHandler(),
+   - endNameSpaceDeclHandler() when stopped in endElementHandler(),
    and possibly others.
 
    Can be called from most handlers, including DTD related call-backs,
    except when parsing an external parameter entity and resumable != 0.
    Returns XML_STATUS_OK when successful, XML_STATUS_ERROR otherwise.
-   Possible error codes: 
+   Possible error codes:
    - XML_ERROR_SUSPENDED: when suspending an already suspended parser.
    - XML_ERROR_FINISHED: when the parser has already finished.
    - XML_ERROR_SUSPEND_PE: when suspending while parsing an external PE.
 
-   When resumable != 0 (true) then parsing is suspended, that is, 
-   XML_Parse() and XML_ParseBuffer() return XML_STATUS_SUSPENDED. 
+   When resumable != 0 (true) then parsing is suspended, that is,
+   XML_Parse() and XML_ParseBuffer() return XML_STATUS_SUSPENDED.
    Otherwise, parsing is aborted, that is, XML_Parse() and XML_ParseBuffer()
    return XML_STATUS_ERROR with error code XML_ERROR_ABORTED.
 
@@ -843,7 +792,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal);
    the externalEntityRefHandler() to call XML_StopParser() on the parent
    parser (recursively), if one wants to stop parsing altogether.
 
-   When suspended, parsing can be resumed by calling XML_ResumeParser(). 
+   When suspended, parsing can be resumed by calling XML_ResumeParser().
 */
 XMLPARSEAPI(enum XML_Status)
 XML_StopParser(XML_Parser parser, XML_Bool resumable);
@@ -851,7 +800,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable);
 /* Resumes parsing after it has been suspended with XML_StopParser().
    Must not be called from within a handler call-back. Returns same
    status codes as XML_Parse() or XML_ParseBuffer().
-   Additional error code XML_ERROR_NOT_SUSPENDED possible.   
+   Additional error code XML_ERROR_NOT_SUSPENDED possible.
 
    *Note*:
    This must be called on the most deeply nested child parser instance
@@ -863,12 +812,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable);
 XMLPARSEAPI(enum XML_Status)
 XML_ResumeParser(XML_Parser parser);
 
-enum XML_Parsing {
-  XML_INITIALIZED,
-  XML_PARSING,
-  XML_FINISHED,
-  XML_SUSPENDED
-};
+enum XML_Parsing { XML_INITIALIZED, XML_PARSING, XML_FINISHED, XML_SUSPENDED };
 
 typedef struct {
   enum XML_Parsing parsing;
@@ -900,8 +844,7 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status);
    Otherwise returns a new XML_Parser object.
 */
 XMLPARSEAPI(XML_Parser)
-XML_ExternalEntityParserCreate(XML_Parser parser,
-                               const XML_Char *context,
+XML_ExternalEntityParserCreate(XML_Parser parser, const XML_Char *context,
                                const XML_Char *encoding);
 
 enum XML_ParamEntityParsing {
@@ -945,8 +888,7 @@ XML_SetParamEntityParsing(XML_Parser parser,
    Note: If parser == NULL, the function will do nothing and return 0.
 */
 XMLPARSEAPI(int)
-XML_SetHashSalt(XML_Parser parser,
-                unsigned long hash_salt);
+XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt);
 
 /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then
    XML_GetErrorCode returns information about the error.
@@ -963,7 +905,7 @@ XML_GetErrorCode(XML_Parser parser);
    be within the relevant markup.  When called outside of the callback
    functions, the position indicated will be just past the last parse
    event (regardless of whether there was an associated callback).
-   
+
    They may also be called after returning from a call to XML_Parse
    or XML_ParseBuffer.  If the return value is XML_STATUS_ERROR then
    the location is the location of the character at which the error
@@ -995,14 +937,12 @@ XML_GetCurrentByteCount(XML_Parser parser);
    the handler that makes the call.
 */
 XMLPARSEAPI(const char *)
-XML_GetInputContext(XML_Parser parser,
-                    int *offset,
-                    int *size);
+XML_GetInputContext(XML_Parser parser, int *offset, int *size);
 
 /* For backwards compatibility with previous versions. */
-#define XML_GetErrorLineNumber   XML_GetCurrentLineNumber
+#define XML_GetErrorLineNumber XML_GetCurrentLineNumber
 #define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber
-#define XML_GetErrorByteIndex    XML_GetCurrentByteIndex
+#define XML_GetErrorByteIndex XML_GetCurrentByteIndex
 
 /* Frees the content model passed to the element declaration handler */
 XMLPARSEAPI(void)
@@ -1062,21 +1002,20 @@ enum XML_FeatureEnum {
 };
 
 typedef struct {
-  enum XML_FeatureEnum  feature;
-  const XML_LChar       *name;
-  long int              value;
+  enum XML_FeatureEnum feature;
+  const XML_LChar *name;
+  long int value;
 } XML_Feature;
 
 XMLPARSEAPI(const XML_Feature *)
 XML_GetFeatureList(void);
 
-
 /* Expat follows the semantic versioning convention.
    See http://semver.org.
 */
 #define XML_MAJOR_VERSION 2
 #define XML_MINOR_VERSION 2
-#define XML_MICRO_VERSION 7
+#define XML_MICRO_VERSION 8
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h
index fc63c2228d3e..f2b75dda8e27 100644
--- a/Modules/expat/expat_external.h
+++ b/Modules/expat/expat_external.h
@@ -35,10 +35,6 @@
 
 /* External API definitions */
 
-#if defined(_MSC_EXTENSIONS) && !defined(__BEOS__) && !defined(__CYGWIN__)
-# define XML_USE_MSC_EXTENSIONS 1
-#endif
-
 /* Expat tries very hard to make the API boundary very specifically
    defined.  There are two macros defined to control this boundary;
    each of these can be defined before including this header to
@@ -61,12 +57,17 @@
    compiled with the cdecl calling convention as the default since
    system headers may assume the cdecl convention.
 */
+
+/* Namespace external symbols to allow multiple libexpat version to
+   co-exist. */
+#include "pyexpatns.h"
+
 #ifndef XMLCALL
-# if defined(_MSC_VER)
-#  define XMLCALL __cdecl
-# elif defined(__GNUC__) && defined(__i386) && !defined(__INTEL_COMPILER)
-#  define XMLCALL __attribute__((cdecl))
-# else
+#  if defined(_MSC_VER)
+#    define XMLCALL __cdecl
+#  elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER)
+#    define XMLCALL __attribute__((cdecl))
+#  else
 /* For any platform which uses this definition and supports more than
    one calling convention, we need to extend this definition to
    declare the convention used on that platform, if it's possible to
@@ -77,49 +78,46 @@
    pre-processor and how to specify the same calling convention as the
    platform's malloc() implementation.
 */
-#  define XMLCALL
-# endif
-#endif  /* not defined XMLCALL */
-
-/* Namespace external symbols to allow multiple libexpat version to
-   co-exist. */
-#include "pyexpatns.h"
-
+#    define XMLCALL
+#  endif
+#endif /* not defined XMLCALL */
 
-#if !defined(XML_STATIC) && !defined(XMLIMPORT)
-# ifndef XML_BUILDING_EXPAT
+#if ! defined(XML_STATIC) && ! defined(XMLIMPORT)
+#  ifndef XML_BUILDING_EXPAT
 /* using Expat from an application */
 
-#  ifdef XML_USE_MSC_EXTENSIONS
-#   define XMLIMPORT __declspec(dllimport)
-#  endif
+#    if defined(_MSC_EXTENSIONS) && ! defined(__BEOS__) && ! defined(__CYGWIN__)
+#      define XMLIMPORT __declspec(dllimport)
+#    endif
 
-# endif
-#endif  /* not defined XML_STATIC */
+#  endif
+#endif /* not defined XML_STATIC */
 
 #ifndef XML_ENABLE_VISIBILITY
-# define XML_ENABLE_VISIBILITY 0
+#  define XML_ENABLE_VISIBILITY 0
 #endif
 
-#if !defined(XMLIMPORT) && XML_ENABLE_VISIBILITY
-# define XMLIMPORT __attribute__ ((visibility ("default")))
+#if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY
+#  define XMLIMPORT __attribute__((visibility("default")))
 #endif
 
 /* If we didn't define it above, define it away: */
 #ifndef XMLIMPORT
-# define XMLIMPORT
+#  define XMLIMPORT
 #endif
 
-#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
-# define XML_ATTR_MALLOC __attribute__((__malloc__))
+#if defined(__GNUC__)                                                          \
+    && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
+#  define XML_ATTR_MALLOC __attribute__((__malloc__))
 #else
-# define XML_ATTR_MALLOC
+#  define XML_ATTR_MALLOC
 #endif
 
-#if defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
-# define XML_ATTR_ALLOC_SIZE(x)  __attribute__((__alloc_size__(x)))
+#if defined(__GNUC__)                                                          \
+    && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
+#  define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x)))
 #else
-# define XML_ATTR_ALLOC_SIZE(x)
+#  define XML_ATTR_ALLOC_SIZE(x)
 #endif
 
 #define XMLPARSEAPI(type) XMLIMPORT type XMLCALL
@@ -129,35 +127,30 @@ extern "C" {
 #endif
 
 #ifdef XML_UNICODE_WCHAR_T
-# ifndef XML_UNICODE
-#  define XML_UNICODE
-# endif
-# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
-#  error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
-# endif
+#  ifndef XML_UNICODE
+#    define XML_UNICODE
+#  endif
+#  if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
+#    error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
+#  endif
 #endif
 
-#ifdef XML_UNICODE     /* Information is UTF-16 encoded. */
-# ifdef XML_UNICODE_WCHAR_T
+#ifdef XML_UNICODE /* Information is UTF-16 encoded. */
+#  ifdef XML_UNICODE_WCHAR_T
 typedef wchar_t XML_Char;
 typedef wchar_t XML_LChar;
-# else
+#  else
 typedef unsigned short XML_Char;
 typedef char XML_LChar;
-# endif /* XML_UNICODE_WCHAR_T */
-#else                  /* Information is UTF-8 encoded. */
+#  endif /* XML_UNICODE_WCHAR_T */
+#else    /* Information is UTF-8 encoded. */
 typedef char XML_Char;
 typedef char XML_LChar;
-#endif /* XML_UNICODE */
+#endif   /* XML_UNICODE */
 
-#ifdef XML_LARGE_SIZE  /* Use large integers for file/stream positions. */
-# if defined(XML_USE_MSC_EXTENSIONS) && _MSC_VER < 1400
-typedef __int64 XML_Index; 
-typedef unsigned __int64 XML_Size;
-# else
+#ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */
 typedef long long XML_Index;
 typedef unsigned long long XML_Size;
-# endif
 #else
 typedef long XML_Index;
 typedef unsigned long XML_Size;
diff --git a/Modules/expat/iasciitab.h b/Modules/expat/iasciitab.h
index ce4a4bf7edef..ea97cfcf678e 100644
--- a/Modules/expat/iasciitab.h
+++ b/Modules/expat/iasciitab.h
@@ -32,34 +32,34 @@
 
 /* Like asciitab.h, except that 0xD has code BT_S rather than BT_CR */
 /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
-/* 0x0C */ BT_NONXML, BT_S, BT_NONXML, BT_NONXML,
-/* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
-/* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
-/* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
-/* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
-/* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
-/* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
-/* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
-/* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
-/* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
+    /* 0x0C */ BT_NONXML, BT_S, BT_NONXML, BT_NONXML,
+    /* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
+    /* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
+    /* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
+    /* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
+    /* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
+    /* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
+    /* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
+    /* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
+    /* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h
index dc4ef0c7e258..60913dab762f 100644
--- a/Modules/expat/internal.h
+++ b/Modules/expat/internal.h
@@ -49,7 +49,7 @@
    USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
-#if defined(__GNUC__) && defined(__i386__) && !defined(__MINGW32__)
+#if defined(__GNUC__) && defined(__i386__) && ! defined(__MINGW32__)
 /* We'll use this version by default only where we know it helps.
 
    regparm() generates warnings on Solaris boxes.   See SF bug #692878.
@@ -59,8 +59,8 @@
    #define FASTCALL __attribute__((stdcall, regparm(3)))
    and let's try this:
 */
-#define FASTCALL __attribute__((regparm(3)))
-#define PTRFASTCALL __attribute__((regparm(3)))
+#  define FASTCALL __attribute__((regparm(3)))
+#  define PTRFASTCALL __attribute__((regparm(3)))
 #endif
 
 /* Using __fastcall seems to have an unexpected negative effect under
@@ -74,55 +74,49 @@
 /* Make sure all of these are defined if they aren't already. */
 
 #ifndef FASTCALL
-#define FASTCALL
+#  define FASTCALL
 #endif
 
 #ifndef PTRCALL
-#define PTRCALL
+#  define PTRCALL
 #endif
 
 #ifndef PTRFASTCALL
-#define PTRFASTCALL
+#  define PTRFASTCALL
 #endif
 
 #ifndef XML_MIN_SIZE
-#if !defined(__cplusplus) && !defined(inline)
-#ifdef __GNUC__
-#define inline __inline
-#endif /* __GNUC__ */
-#endif
+#  if ! defined(__cplusplus) && ! defined(inline)
+#    ifdef __GNUC__
+#      define inline __inline
+#    endif /* __GNUC__ */
+#  endif
 #endif /* XML_MIN_SIZE */
 
 #ifdef __cplusplus
-#define inline inline
+#  define inline inline
 #else
-#ifndef inline
-#define inline
-#endif
+#  ifndef inline
+#    define inline
+#  endif
 #endif
 
 #ifndef UNUSED_P
-# ifdef __GNUC__
-#  define UNUSED_P(p) UNUSED_ ## p __attribute__((__unused__))
-# else
-#  define UNUSED_P(p) UNUSED_ ## p
-# endif
+#  define UNUSED_P(p) (void)p
 #endif
 
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-
 #ifdef XML_ENABLE_VISIBILITY
-#if XML_ENABLE_VISIBILITY
-__attribute__ ((visibility ("default")))
-#endif
+#  if XML_ENABLE_VISIBILITY
+__attribute__((visibility("default")))
+#  endif
 #endif
 void
-_INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** fromLimRef);
-
+_INTERNAL_trim_to_complete_utf8_characters(const char *from,
+                                           const char **fromLimRef);
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/latin1tab.h b/Modules/expat/latin1tab.h
index 95dfa52b1fbe..6f916041355a 100644
--- a/Modules/expat/latin1tab.h
+++ b/Modules/expat/latin1tab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x80 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x84 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x88 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x8C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x90 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x94 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x98 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x9C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA4 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
-/* 0xAC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xB0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xB4 */ BT_OTHER, BT_NMSTRT, BT_OTHER, BT_NAME,
-/* 0xB8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
-/* 0xBC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xC0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xC4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xC8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xCC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xD0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xD4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0xD8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xDC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xEC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xF0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xF4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0xF8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xFC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x84 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x88 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x8C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x90 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x94 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x98 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x9C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA4 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
+    /* 0xAC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xB0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xB4 */ BT_OTHER, BT_NMSTRT, BT_OTHER, BT_NAME,
+    /* 0xB8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
+    /* 0xBC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xC0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xC4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xC8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xCC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xD0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xD4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0xD8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xDC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xEC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xF0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xF4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0xF8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xFC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
diff --git a/Modules/expat/loadlibrary.c b/Modules/expat/loadlibrary.c
deleted file mode 100644
index 35fdf98bce6c..000000000000
--- a/Modules/expat/loadlibrary.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/***************************************************************************
- *                                  _   _ ____  _
- *  Project                     ___| | | |  _ \| |
- *                             / __| | | | |_) | |
- *                            | (__| |_| |  _ <| |___
- *                             \___|\___/|_| \_\_____|
- *
- * Copyright (C) 2016 - 2017, Steve Holme, <steve_holme at hotmail.com>.
- * Copyright (C) 2017, Expat development team
- *
- * All rights reserved.
- * Licensed under the MIT license:
- *
- * Permission to  use, copy,  modify, and distribute  this software  for any
- * purpose with  or without fee is  hereby granted, provided that  the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE  SOFTWARE  IS  PROVIDED  "AS  IS",  WITHOUT  WARRANTY  OF  ANY  KIND,
- * EXPRESS  OR IMPLIED,  INCLUDING  BUT  NOT LIMITED  TO  THE WARRANTIES  OF
- * MERCHANTABILITY, FITNESS FOR A  PARTICULAR PURPOSE AND NONINFRINGEMENT OF
- * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
- * CONTRACT, TORT OR  OTHERWISE, ARISING FROM, OUT OF OR  IN CONNECTION WITH
- * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * Except as contained in this notice,  the name of a copyright holder shall
- * not be used in advertising or otherwise to promote the sale, use or other
- * dealings  in this  Software without  prior written  authorization of  the
- * copyright holder.
- *
- ***************************************************************************/
-
-#if defined(_WIN32)
-
-#include <windows.h>
-#include <tchar.h>
-
-
-HMODULE _Expat_LoadLibrary(LPCTSTR filename);
-
-
-#if !defined(LOAD_WITH_ALTERED_SEARCH_PATH)
-#define LOAD_WITH_ALTERED_SEARCH_PATH  0x00000008
-#endif
-
-#if !defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
-#define LOAD_LIBRARY_SEARCH_SYSTEM32   0x00000800
-#endif
-
-/* We use our own typedef here since some headers might lack these */
-typedef HMODULE (APIENTRY *LOADLIBRARYEX_FN)(LPCTSTR, HANDLE, DWORD);
-
-/* See function definitions in winbase.h */
-#ifdef UNICODE
-#  ifdef _WIN32_WCE
-#    define LOADLIBARYEX  L"LoadLibraryExW"
-#  else
-#    define LOADLIBARYEX  "LoadLibraryExW"
-#  endif
-#else
-#  define LOADLIBARYEX    "LoadLibraryExA"
-#endif
-
-
-/*
- * _Expat_LoadLibrary()
- *
- * This is used to dynamically load DLLs using the most secure method available
- * for the version of Windows that we are running on.
- *
- * Parameters:
- *
- * filename  [in] - The filename or full path of the DLL to load. If only the
- *                  filename is passed then the DLL will be loaded from the
- *                  Windows system directory.
- *
- * Returns the handle of the module on success; otherwise NULL.
- */
-HMODULE _Expat_LoadLibrary(LPCTSTR filename)
-{
-  HMODULE hModule = NULL;
-  LOADLIBRARYEX_FN pLoadLibraryEx = NULL;
-
-  /* Get a handle to kernel32 so we can access it's functions at runtime */
-  HMODULE hKernel32 = GetModuleHandle(TEXT("kernel32"));
-  if(!hKernel32)
-    return NULL;  /* LCOV_EXCL_LINE */
-
-  /* Attempt to find LoadLibraryEx() which is only available on Windows 2000
-     and above */
-  pLoadLibraryEx = (LOADLIBRARYEX_FN) GetProcAddress(hKernel32, LOADLIBARYEX);
-
-  /* Detect if there's already a path in the filename and load the library if
-     there is. Note: Both back slashes and forward slashes have been supported
-     since the earlier days of DOS at an API level although they are not
-     supported by command prompt */
-  if(_tcspbrk(filename, TEXT("\\/"))) {
-    /** !checksrc! disable BANNEDFUNC 1 **/
-    hModule = pLoadLibraryEx ?
-      pLoadLibraryEx(filename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) :
-      LoadLibrary(filename);
-  }
-  /* Detect if KB2533623 is installed, as LOAD_LIBARY_SEARCH_SYSTEM32 is only
-     supported on Windows Vista, Windows Server 2008, Windows 7 and Windows
-     Server 2008 R2 with this patch or natively on Windows 8 and above */
-  else if(pLoadLibraryEx && GetProcAddress(hKernel32, "AddDllDirectory")) {
-    /* Load the DLL from the Windows system directory */
-    hModule = pLoadLibraryEx(filename, NULL, LOAD_LIBRARY_SEARCH_SYSTEM32);
-  }
-  else {
-    /* Attempt to get the Windows system path */
-    UINT systemdirlen = GetSystemDirectory(NULL, 0);
-    if(systemdirlen) {
-      /* Allocate space for the full DLL path (Room for the null terminator
-         is included in systemdirlen) */
-      size_t filenamelen = _tcslen(filename);
-      TCHAR *path = malloc(sizeof(TCHAR) * (systemdirlen + 1 + filenamelen));
-      if(path && GetSystemDirectory(path, systemdirlen)) {
-        /* Calculate the full DLL path */
-        _tcscpy(path + _tcslen(path), TEXT("\\"));
-        _tcscpy(path + _tcslen(path), filename);
-
-        /* Load the DLL from the Windows system directory */
-        /** !checksrc! disable BANNEDFUNC 1 **/
-        hModule = pLoadLibraryEx ?
-          pLoadLibraryEx(path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) :
-          LoadLibrary(path);
-
-      }
-      free(path);
-    }
-  }
-
-  return hModule;
-}
-
-#else /* defined(_WIN32) */
-
-/* ISO C requires a translation unit to contain at least one declaration
-   [-Wempty-translation-unit] */
-typedef int _TRANSLATION_UNIT_LOAD_LIBRARY_C_NOT_EMTPY;
-
-#endif /* defined(_WIN32) */
diff --git a/Modules/expat/nametab.h b/Modules/expat/nametab.h
index bfa2bd38cd95..3681df348eeb 100644
--- a/Modules/expat/nametab.h
+++ b/Modules/expat/nametab.h
@@ -31,152 +31,106 @@
 */
 
 static const unsigned namingBitmap[] = {
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0x00000000, 0x04000000, 0x87FFFFFE, 0x07FFFFFE,
-0x00000000, 0x00000000, 0xFF7FFFFF, 0xFF7FFFFF,
-0xFFFFFFFF, 0x7FF3FFFF, 0xFFFFFDFE, 0x7FFFFFFF,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFE00F, 0xFC31FFFF,
-0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
-0xFFFFFFFF, 0xF80001FF, 0x00000003, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFD740, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
-0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
-0xFFFF0003, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF,
-0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
-0x0000007F, 0x00000000, 0xFFFF0000, 0x000707FF,
-0x00000000, 0x07FFFFFE, 0x000007FE, 0xFFFE0000,
-0xFFFFFFFF, 0x7CFFFFFF, 0x002F7FFF, 0x00000060,
-0xFFFFFFE0, 0x23FFFFFF, 0xFF000000, 0x00000003,
-0xFFF99FE0, 0x03C5FDFF, 0xB0000000, 0x00030003,
-0xFFF987E0, 0x036DFDFF, 0x5E000000, 0x001C0000,
-0xFFFBAFE0, 0x23EDFDFF, 0x00000000, 0x00000001,
-0xFFF99FE0, 0x23CDFDFF, 0xB0000000, 0x00000003,
-0xD63DC7E0, 0x03BFC718, 0x00000000, 0x00000000,
-0xFFFDDFE0, 0x03EFFDFF, 0x00000000, 0x00000003,
-0xFFFDDFE0, 0x03EFFDFF, 0x40000000, 0x00000003,
-0xFFFDDFE0, 0x03FFFDFF, 0x00000000, 0x00000003,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFE, 0x000D7FFF, 0x0000003F, 0x00000000,
-0xFEF02596, 0x200D6CAE, 0x0000001F, 0x00000000,
-0x00000000, 0x00000000, 0xFFFFFEFF, 0x000003FF,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0xFFFFFFFF, 0xFFFF003F, 0x007FFFFF,
-0x0007DAED, 0x50000000, 0x82315001, 0x002C62AB,
-0x40000000, 0xF580C900, 0x00000007, 0x02010800,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0x0FFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x03FFFFFF,
-0x3F3FFFFF, 0xFFFFFFFF, 0xAAFF3F3F, 0x3FFFFFFF,
-0xFFFFFFFF, 0x5FDFFFFF, 0x0FCF1FDC, 0x1FDC1FFF,
-0x00000000, 0x00004C40, 0x00000000, 0x00000000,
-0x00000007, 0x00000000, 0x00000000, 0x00000000,
-0x00000080, 0x000003FE, 0xFFFFFFFE, 0xFFFFFFFF,
-0x001FFFFF, 0xFFFFFFFE, 0xFFFFFFFF, 0x07FFFFFF,
-0xFFFFFFE0, 0x00001FFF, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0x0000003F, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0x0000000F, 0x00000000, 0x00000000,
-0x00000000, 0x07FF6000, 0x87FFFFFE, 0x07FFFFFE,
-0x00000000, 0x00800000, 0xFF7FFFFF, 0xFF7FFFFF,
-0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
-0xFFFFFFFF, 0xF80001FF, 0x00030003, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000003,
-0xFFFFD7C0, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
-0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
-0xFFFF007B, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF,
-0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
-0xFFFE007F, 0xBBFFFFFB, 0xFFFF0016, 0x000707FF,
-0x00000000, 0x07FFFFFE, 0x0007FFFF, 0xFFFF03FF,
-0xFFFFFFFF, 0x7CFFFFFF, 0xFFEF7FFF, 0x03FF3DFF,
-0xFFFFFFEE, 0xF3FFFFFF, 0xFF1E3FFF, 0x0000FFCF,
-0xFFF99FEE, 0xD3C5FDFF, 0xB080399F, 0x0003FFCF,
-0xFFF987E4, 0xD36DFDFF, 0x5E003987, 0x001FFFC0,
-0xFFFBAFEE, 0xF3EDFDFF, 0x00003BBF, 0x0000FFC1,
-0xFFF99FEE, 0xF3CDFDFF, 0xB0C0398F, 0x0000FFC3,
-0xD63DC7EC, 0xC3BFC718, 0x00803DC7, 0x0000FF80,
-0xFFFDDFEE, 0xC3EFFDFF, 0x00603DDF, 0x0000FFC3,
-0xFFFDDFEC, 0xC3EFFDFF, 0x40603DDF, 0x0000FFC3,
-0xFFFDDFEC, 0xC3FFFDFF, 0x00803DCF, 0x0000FFC3,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFE, 0x07FF7FFF, 0x03FF7FFF, 0x00000000,
-0xFEF02596, 0x3BFF6CAE, 0x03FF3F5F, 0x00000000,
-0x03000000, 0xC2A003FF, 0xFFFFFEFF, 0xFFFE03FF,
-0xFEBF0FDF, 0x02FE3FFF, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x1FFF0000, 0x00000002,
-0x000000A0, 0x003EFFFE, 0xFFFFFFFE, 0xFFFFFFFF,
-0x661FFFFF, 0xFFFFFFFE, 0xFFFFFFFF, 0x77FFFFFF,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x04000000,
+    0x87FFFFFE, 0x07FFFFFE, 0x00000000, 0x00000000, 0xFF7FFFFF, 0xFF7FFFFF,
+    0xFFFFFFFF, 0x7FF3FFFF, 0xFFFFFDFE, 0x7FFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFE00F, 0xFC31FFFF, 0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xF80001FF, 0x00000003, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFD740, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
+    0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF, 0xFFFF0003, 0xFFFFFFFF,
+    0xFFFF199F, 0x033FCFFF, 0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
+    0x0000007F, 0x00000000, 0xFFFF0000, 0x000707FF, 0x00000000, 0x07FFFFFE,
+    0x000007FE, 0xFFFE0000, 0xFFFFFFFF, 0x7CFFFFFF, 0x002F7FFF, 0x00000060,
+    0xFFFFFFE0, 0x23FFFFFF, 0xFF000000, 0x00000003, 0xFFF99FE0, 0x03C5FDFF,
+    0xB0000000, 0x00030003, 0xFFF987E0, 0x036DFDFF, 0x5E000000, 0x001C0000,
+    0xFFFBAFE0, 0x23EDFDFF, 0x00000000, 0x00000001, 0xFFF99FE0, 0x23CDFDFF,
+    0xB0000000, 0x00000003, 0xD63DC7E0, 0x03BFC718, 0x00000000, 0x00000000,
+    0xFFFDDFE0, 0x03EFFDFF, 0x00000000, 0x00000003, 0xFFFDDFE0, 0x03EFFDFF,
+    0x40000000, 0x00000003, 0xFFFDDFE0, 0x03FFFDFF, 0x00000000, 0x00000003,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFE, 0x000D7FFF,
+    0x0000003F, 0x00000000, 0xFEF02596, 0x200D6CAE, 0x0000001F, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFFEFF, 0x000003FF, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0xFFFFFFFF, 0xFFFF003F, 0x007FFFFF, 0x0007DAED, 0x50000000,
+    0x82315001, 0x002C62AB, 0x40000000, 0xF580C900, 0x00000007, 0x02010800,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0FFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0x03FFFFFF, 0x3F3FFFFF, 0xFFFFFFFF, 0xAAFF3F3F, 0x3FFFFFFF,
+    0xFFFFFFFF, 0x5FDFFFFF, 0x0FCF1FDC, 0x1FDC1FFF, 0x00000000, 0x00004C40,
+    0x00000000, 0x00000000, 0x00000007, 0x00000000, 0x00000000, 0x00000000,
+    0x00000080, 0x000003FE, 0xFFFFFFFE, 0xFFFFFFFF, 0x001FFFFF, 0xFFFFFFFE,
+    0xFFFFFFFF, 0x07FFFFFF, 0xFFFFFFE0, 0x00001FFF, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000000, 0x00000000,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000000F,
+    0x00000000, 0x00000000, 0x00000000, 0x07FF6000, 0x87FFFFFE, 0x07FFFFFE,
+    0x00000000, 0x00800000, 0xFF7FFFFF, 0xFF7FFFFF, 0x00FFFFFF, 0x00000000,
+    0xFFFF0000, 0xFFFFFFFF, 0xFFFFFFFF, 0xF80001FF, 0x00030003, 0x00000000,
+    0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000003, 0xFFFFD7C0, 0xFFFFFFFB,
+    0x547F7FFF, 0x000FFFFD, 0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
+    0xFFFF007B, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF, 0x00000000, 0xFFFE0000,
+    0x027FFFFF, 0xFFFFFFFE, 0xFFFE007F, 0xBBFFFFFB, 0xFFFF0016, 0x000707FF,
+    0x00000000, 0x07FFFFFE, 0x0007FFFF, 0xFFFF03FF, 0xFFFFFFFF, 0x7CFFFFFF,
+    0xFFEF7FFF, 0x03FF3DFF, 0xFFFFFFEE, 0xF3FFFFFF, 0xFF1E3FFF, 0x0000FFCF,
+    0xFFF99FEE, 0xD3C5FDFF, 0xB080399F, 0x0003FFCF, 0xFFF987E4, 0xD36DFDFF,
+    0x5E003987, 0x001FFFC0, 0xFFFBAFEE, 0xF3EDFDFF, 0x00003BBF, 0x0000FFC1,
+    0xFFF99FEE, 0xF3CDFDFF, 0xB0C0398F, 0x0000FFC3, 0xD63DC7EC, 0xC3BFC718,
+    0x00803DC7, 0x0000FF80, 0xFFFDDFEE, 0xC3EFFDFF, 0x00603DDF, 0x0000FFC3,
+    0xFFFDDFEC, 0xC3EFFDFF, 0x40603DDF, 0x0000FFC3, 0xFFFDDFEC, 0xC3FFFDFF,
+    0x00803DCF, 0x0000FFC3, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0xFFFFFFFE, 0x07FF7FFF, 0x03FF7FFF, 0x00000000, 0xFEF02596, 0x3BFF6CAE,
+    0x03FF3F5F, 0x00000000, 0x03000000, 0xC2A003FF, 0xFFFFFEFF, 0xFFFE03FF,
+    0xFEBF0FDF, 0x02FE3FFF, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x1FFF0000, 0x00000002,
+    0x000000A0, 0x003EFFFE, 0xFFFFFFFE, 0xFFFFFFFF, 0x661FFFFF, 0xFFFFFFFE,
+    0xFFFFFFFF, 0x77FFFFFF,
 };
 static const unsigned char nmstrtPages[] = {
-0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00,
-0x00, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
-0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13,
-0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x15, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x17,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x09, 0x0A, 0x0B,
+    0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13, 0x00, 0x14, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x15, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00,
 };
 static const unsigned char namePages[] = {
-0x19, 0x03, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x00,
-0x00, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25,
-0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13,
-0x26, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x27, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x17,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x19, 0x03, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x00, 0x00, 0x1F, 0x20, 0x21,
+    0x22, 0x23, 0x24, 0x25, 0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13, 0x26, 0x14, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x27, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00,
 };
diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h
index 4d6786d7839a..bfee65a332f1 100644
--- a/Modules/expat/siphash.h
+++ b/Modules/expat/siphash.h
@@ -11,6 +11,10 @@
  * --------------------------------------------------------------------------
  * HISTORY:
  *
+ * 2019-08-03  (Sebastian Pipping)
+ *   - Mark part of sip24_valid as to be excluded from clang-format
+ *   - Re-format code using clang-format 9
+ *
  * 2018-07-08  (Anton Maklakov)
  *   - Add "fall through" markers for GCC's -Wimplicit-fallthrough
  *
@@ -94,186 +98,186 @@
 #include <stddef.h> /* size_t */
 
 #if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER < 1600)
-  /* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */
-  typedef unsigned __int8   uint8_t;
-  typedef unsigned __int32 uint32_t;
-  typedef unsigned __int64 uint64_t;
+/* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */
+typedef unsigned __int8 uint8_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
 #else
- #include <stdint.h> /* uint64_t uint32_t uint8_t */
+#  include <stdint.h> /* uint64_t uint32_t uint8_t */
 #endif
 
-
 /*
  * Workaround to not require a C++11 compiler for using ULL suffix
  * if this code is included and compiled as C++; related GCC warning is:
  * warning: use of C++11 long long integer constant [-Wlong-long]
  */
-#define _SIP_ULL(high, low)  (((uint64_t)high << 32) | low)
-
-
-#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b))))
+#define _SIP_ULL(high, low) (((uint64_t)high << 32) | low)
 
-#define SIP_U32TO8_LE(p, v) \
-	(p)[0] = (uint8_t)((v) >>  0); (p)[1] = (uint8_t)((v) >>  8); \
-	(p)[2] = (uint8_t)((v) >> 16); (p)[3] = (uint8_t)((v) >> 24);
+#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
 
-#define SIP_U64TO8_LE(p, v) \
-	SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >>  0)); \
-	SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
+#define SIP_U32TO8_LE(p, v)                                                    \
+  (p)[0] = (uint8_t)((v) >> 0);                                                \
+  (p)[1] = (uint8_t)((v) >> 8);                                                \
+  (p)[2] = (uint8_t)((v) >> 16);                                               \
+  (p)[3] = (uint8_t)((v) >> 24);
 
-#define SIP_U8TO64_LE(p) \
-	(((uint64_t)((p)[0]) <<  0) | \
-	 ((uint64_t)((p)[1]) <<  8) | \
-	 ((uint64_t)((p)[2]) << 16) | \
-	 ((uint64_t)((p)[3]) << 24) | \
-	 ((uint64_t)((p)[4]) << 32) | \
-	 ((uint64_t)((p)[5]) << 40) | \
-	 ((uint64_t)((p)[6]) << 48) | \
-	 ((uint64_t)((p)[7]) << 56))
+#define SIP_U64TO8_LE(p, v)                                                    \
+  SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >> 0));                                \
+  SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
 
+#define SIP_U8TO64_LE(p)                                                       \
+  (((uint64_t)((p)[0]) << 0) | ((uint64_t)((p)[1]) << 8)                       \
+   | ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24)                   \
+   | ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40)                   \
+   | ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))
 
-#define SIPHASH_INITIALIZER { 0, 0, 0, 0, { 0 }, 0, 0 }
+#define SIPHASH_INITIALIZER                                                    \
+  { 0, 0, 0, 0, {0}, 0, 0 }
 
 struct siphash {
-	uint64_t v0, v1, v2, v3;
+  uint64_t v0, v1, v2, v3;
 
-	unsigned char buf[8], *p;
-	uint64_t c;
+  unsigned char buf[8], *p;
+  uint64_t c;
 }; /* struct siphash */
 
-
 #define SIP_KEYLEN 16
 
 struct sipkey {
-	uint64_t k[2];
+  uint64_t k[2];
 }; /* struct sipkey */
 
-#define sip_keyof(k) sip_tokey(&(struct sipkey){ { 0 } }, (k))
+#define sip_keyof(k) sip_tokey(&(struct sipkey){{0}}, (k))
 
-static struct sipkey *sip_tokey(struct sipkey *key, const void *src) {
-	key->k[0] = SIP_U8TO64_LE((const unsigned char *)src);
-	key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8);
-	return key;
+static struct sipkey *
+sip_tokey(struct sipkey *key, const void *src) {
+  key->k[0] = SIP_U8TO64_LE((const unsigned char *)src);
+  key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8);
+  return key;
 } /* sip_tokey() */
 
-
 #ifdef SIPHASH_TOBIN
 
-#define sip_binof(v) sip_tobin((unsigned char[8]){ 0 }, (v))
+#  define sip_binof(v) sip_tobin((unsigned char[8]){0}, (v))
 
-static void *sip_tobin(void *dst, uint64_t u64) {
-	SIP_U64TO8_LE((unsigned char *)dst, u64);
-	return dst;
+static void *
+sip_tobin(void *dst, uint64_t u64) {
+  SIP_U64TO8_LE((unsigned char *)dst, u64);
+  return dst;
 } /* sip_tobin() */
 
-#endif  /* SIPHASH_TOBIN */
-
+#endif /* SIPHASH_TOBIN */
 
-static void sip_round(struct siphash *H, const int rounds) {
-	int i;
+static void
+sip_round(struct siphash *H, const int rounds) {
+  int i;
 
-	for (i = 0; i < rounds; i++) {
-		H->v0 += H->v1;
-		H->v1 = SIP_ROTL(H->v1, 13);
-		H->v1 ^= H->v0;
-		H->v0 = SIP_ROTL(H->v0, 32);
+  for (i = 0; i < rounds; i++) {
+    H->v0 += H->v1;
+    H->v1 = SIP_ROTL(H->v1, 13);
+    H->v1 ^= H->v0;
+    H->v0 = SIP_ROTL(H->v0, 32);
 
-		H->v2 += H->v3;
-		H->v3 = SIP_ROTL(H->v3, 16);
-		H->v3 ^= H->v2;
+    H->v2 += H->v3;
+    H->v3 = SIP_ROTL(H->v3, 16);
+    H->v3 ^= H->v2;
 
-		H->v0 += H->v3;
-		H->v3 = SIP_ROTL(H->v3, 21);
-		H->v3 ^= H->v0;
+    H->v0 += H->v3;
+    H->v3 = SIP_ROTL(H->v3, 21);
+    H->v3 ^= H->v0;
 
-		H->v2 += H->v1;
-		H->v1 = SIP_ROTL(H->v1, 17);
-		H->v1 ^= H->v2;
-		H->v2 = SIP_ROTL(H->v2, 32);
-	}
+    H->v2 += H->v1;
+    H->v1 = SIP_ROTL(H->v1, 17);
+    H->v1 ^= H->v2;
+    H->v2 = SIP_ROTL(H->v2, 32);
+  }
 } /* sip_round() */
 
+static struct siphash *
+sip24_init(struct siphash *H, const struct sipkey *key) {
+  H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0];
+  H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1];
+  H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0];
+  H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1];
 
-static struct siphash *sip24_init(struct siphash *H,
-		const struct sipkey *key) {
-	H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0];
-	H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1];
-	H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0];
-	H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1];
+  H->p = H->buf;
+  H->c = 0;
 
-	H->p = H->buf;
-	H->c = 0;
-
-	return H;
+  return H;
 } /* sip24_init() */
 
+#define sip_endof(a) (&(a)[sizeof(a) / sizeof *(a)])
 
-#define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)])
-
-static struct siphash *sip24_update(struct siphash *H, const void *src,
-		size_t len) {
-	const unsigned char *p = (const unsigned char *)src, *pe = p + len;
-	uint64_t m;
+static struct siphash *
+sip24_update(struct siphash *H, const void *src, size_t len) {
+  const unsigned char *p = (const unsigned char *)src, *pe = p + len;
+  uint64_t m;
 
-	do {
-		while (p < pe && H->p < sip_endof(H->buf))
-			*H->p++ = *p++;
+  do {
+    while (p < pe && H->p < sip_endof(H->buf))
+      *H->p++ = *p++;
 
-		if (H->p < sip_endof(H->buf))
-			break;
+    if (H->p < sip_endof(H->buf))
+      break;
 
-		m = SIP_U8TO64_LE(H->buf);
-		H->v3 ^= m;
-		sip_round(H, 2);
-		H->v0 ^= m;
+    m = SIP_U8TO64_LE(H->buf);
+    H->v3 ^= m;
+    sip_round(H, 2);
+    H->v0 ^= m;
 
-		H->p = H->buf;
-		H->c += 8;
-	} while (p < pe);
+    H->p = H->buf;
+    H->c += 8;
+  } while (p < pe);
 
-	return H;
+  return H;
 } /* sip24_update() */
 
-
-static uint64_t sip24_final(struct siphash *H) {
-	const char left = (char)(H->p - H->buf);
-	uint64_t b = (H->c + left) << 56;
-
-	switch (left) {
-	case 7: b |= (uint64_t)H->buf[6] << 48;
-		/* fall through */
-	case 6: b |= (uint64_t)H->buf[5] << 40;
-		/* fall through */
-	case 5: b |= (uint64_t)H->buf[4] << 32;
-		/* fall through */
-	case 4: b |= (uint64_t)H->buf[3] << 24;
-		/* fall through */
-	case 3: b |= (uint64_t)H->buf[2] << 16;
-		/* fall through */
-	case 2: b |= (uint64_t)H->buf[1] << 8;
-		/* fall through */
-	case 1: b |= (uint64_t)H->buf[0] << 0;
-		/* fall through */
-	case 0: break;
-	}
-
-	H->v3 ^= b;
-	sip_round(H, 2);
-	H->v0 ^= b;
-	H->v2 ^= 0xff;
-	sip_round(H, 4);
-
-	return H->v0 ^ H->v1 ^ H->v2  ^ H->v3;
+static uint64_t
+sip24_final(struct siphash *H) {
+  const char left = (char)(H->p - H->buf);
+  uint64_t b = (H->c + left) << 56;
+
+  switch (left) {
+  case 7:
+    b |= (uint64_t)H->buf[6] << 48;
+    /* fall through */
+  case 6:
+    b |= (uint64_t)H->buf[5] << 40;
+    /* fall through */
+  case 5:
+    b |= (uint64_t)H->buf[4] << 32;
+    /* fall through */
+  case 4:
+    b |= (uint64_t)H->buf[3] << 24;
+    /* fall through */
+  case 3:
+    b |= (uint64_t)H->buf[2] << 16;
+    /* fall through */
+  case 2:
+    b |= (uint64_t)H->buf[1] << 8;
+    /* fall through */
+  case 1:
+    b |= (uint64_t)H->buf[0] << 0;
+    /* fall through */
+  case 0:
+    break;
+  }
+
+  H->v3 ^= b;
+  sip_round(H, 2);
+  H->v0 ^= b;
+  H->v2 ^= 0xff;
+  sip_round(H, 4);
+
+  return H->v0 ^ H->v1 ^ H->v2 ^ H->v3;
 } /* sip24_final() */
 
-
-static uint64_t siphash24(const void *src, size_t len,
-		const struct sipkey *key) {
-	struct siphash state = SIPHASH_INITIALIZER;
-	return sip24_final(sip24_update(sip24_init(&state, key), src, len));
+static uint64_t
+siphash24(const void *src, size_t len, const struct sipkey *key) {
+  struct siphash state = SIPHASH_INITIALIZER;
+  return sip24_final(sip24_update(sip24_init(&state, key), src, len));
 } /* siphash24() */
 
-
 /*
  * SipHash-2-4 output with
  * k = 00 01 02 ...
@@ -285,107 +289,110 @@ static uint64_t siphash24(const void *src, size_t len,
  * ...
  * in = 00 01 02 ... 3e (63 bytes)
  */
-static int sip24_valid(void) {
-	static const unsigned char vectors[64][8] = {
-		{ 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, },
-		{ 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, },
-		{ 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, },
-		{ 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, },
-		{ 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, },
-		{ 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, },
-		{ 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, },
-		{ 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, },
-		{ 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, },
-		{ 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, },
-		{ 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, },
-		{ 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, },
-		{ 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, },
-		{ 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, },
-		{ 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, },
-		{ 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, },
-		{ 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, },
-		{ 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, },
-		{ 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, },
-		{ 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, },
-		{ 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, },
-		{ 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, },
-		{ 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, },
-		{ 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, },
-		{ 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, },
-		{ 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, },
-		{ 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, },
-		{ 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, },
-		{ 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, },
-		{ 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, },
-		{ 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, },
-		{ 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, },
-		{ 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, },
-		{ 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, },
-		{ 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, },
-		{ 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, },
-		{ 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, },
-		{ 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, },
-		{ 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, },
-		{ 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, },
-		{ 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, },
-		{ 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, },
-		{ 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, },
-		{ 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, },
-		{ 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, },
-		{ 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, },
-		{ 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, },
-		{ 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, },
-		{ 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, },
-		{ 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, },
-		{ 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, },
-		{ 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, },
-		{ 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, },
-		{ 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, },
-		{ 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, },
-		{ 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, },
-		{ 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, },
-		{ 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, },
-		{ 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, },
-		{ 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, },
-		{ 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, },
-		{ 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, },
-		{ 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, },
-		{ 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, }
-	};
-	unsigned char in[64];
-	struct sipkey k;
-	size_t i;
-
-	sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011"
-			"\012\013\014\015\016\017");
-
-	for (i = 0; i < sizeof in; ++i) {
-		in[i] = (unsigned char)i;
-
-		if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i]))
-			return 0;
-	}
-
-	return 1;
+static int
+sip24_valid(void) {
+  /* clang-format off */
+  static const unsigned char vectors[64][8] = {
+    { 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, },
+    { 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, },
+    { 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, },
+    { 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, },
+    { 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, },
+    { 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, },
+    { 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, },
+    { 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, },
+    { 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, },
+    { 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, },
+    { 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, },
+    { 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, },
+    { 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, },
+    { 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, },
+    { 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, },
+    { 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, },
+    { 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, },
+    { 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, },
+    { 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, },
+    { 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, },
+    { 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, },
+    { 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, },
+    { 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, },
+    { 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, },
+    { 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, },
+    { 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, },
+    { 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, },
+    { 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, },
+    { 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, },
+    { 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, },
+    { 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, },
+    { 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, },
+    { 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, },
+    { 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, },
+    { 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, },
+    { 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, },
+    { 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, },
+    { 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, },
+    { 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, },
+    { 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, },
+    { 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, },
+    { 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, },
+    { 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, },
+    { 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, },
+    { 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, },
+    { 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, },
+    { 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, },
+    { 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, },
+    { 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, },
+    { 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, },
+    { 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, },
+    { 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, },
+    { 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, },
+    { 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, },
+    { 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, },
+    { 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, },
+    { 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, },
+    { 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, },
+    { 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, },
+    { 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, },
+    { 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, },
+    { 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, },
+    { 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, },
+    { 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, }
+  };
+  /* clang-format on */
+
+  unsigned char in[64];
+  struct sipkey k;
+  size_t i;
+
+  sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011"
+                "\012\013\014\015\016\017");
+
+  for (i = 0; i < sizeof in; ++i) {
+    in[i] = (unsigned char)i;
+
+    if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i]))
+      return 0;
+  }
+
+  return 1;
 } /* sip24_valid() */
 
-
 #ifdef SIPHASH_MAIN
 
-#include <stdio.h>
+#  include <stdio.h>
 
-int main(void) {
-	const int ok = sip24_valid();
+int
+main(void) {
+  const int ok = sip24_valid();
 
-	if (ok)
-		puts("OK");
-	else
-		puts("FAIL");
+  if (ok)
+    puts("OK");
+  else
+    puts("FAIL");
 
-	return !ok;
+  return ! ok;
 } /* main() */
 
 #endif /* SIPHASH_MAIN */
 
-
 #endif /* SIPHASH_H */
diff --git a/Modules/expat/utf8tab.h b/Modules/expat/utf8tab.h
index fa0bed6f5d75..a22986acbb95 100644
--- a/Modules/expat/utf8tab.h
+++ b/Modules/expat/utf8tab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x80 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x84 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x88 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x8C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x90 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x94 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x98 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x9C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xAC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xBC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xC0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xC4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xC8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xCC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xDC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xE0 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xE4 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xE8 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xEC */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xF0 */ BT_LEAD4, BT_LEAD4, BT_LEAD4, BT_LEAD4,
-/* 0xF4 */ BT_LEAD4, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0xF8 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0xFC */ BT_NONXML, BT_NONXML, BT_MALFORM, BT_MALFORM,
+    /* 0x84 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x88 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x8C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x90 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x94 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x98 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x9C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xAC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xBC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xC0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xC4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xC8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xCC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xDC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xE0 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xE4 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xE8 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xEC */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xF0 */ BT_LEAD4, BT_LEAD4, BT_LEAD4, BT_LEAD4,
+    /* 0xF4 */ BT_LEAD4, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0xF8 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0xFC */ BT_NONXML, BT_NONXML, BT_MALFORM, BT_MALFORM,
diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h
index 28a043c6044c..562a4a82dc1d 100644
--- a/Modules/expat/winconfig.h
+++ b/Modules/expat/winconfig.h
@@ -40,20 +40,17 @@
 #include <memory.h>
 #include <string.h>
 
+#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */
+#  include <expat_config.h>
+#else /* !defined(HAVE_EXPAT_CONFIG_H) */
 
-#if defined(HAVE_EXPAT_CONFIG_H)  /* e.g. MinGW */
-# include <expat_config.h>
-#else  /* !defined(HAVE_EXPAT_CONFIG_H) */
-
-
-#define XML_NS 1
-#define XML_DTD 1
-#define XML_CONTEXT_BYTES 1024
+#  define XML_NS 1
+#  define XML_DTD 1
+#  define XML_CONTEXT_BYTES 1024
 
 /* we will assume all Windows platforms are little endian */
-#define BYTEORDER 1234
+#  define BYTEORDER 1234
 
 #endif /* !defined(HAVE_EXPAT_CONFIG_H) */
 
-
 #endif /* ndef WINCONFIG_H */
diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c
index 9c0987f4f6d8..e740f0e19c7d 100644
--- a/Modules/expat/xmlparse.c
+++ b/Modules/expat/xmlparse.c
@@ -1,4 +1,4 @@
-/* 69df5be70289a11fb834869ce4a91c23c1d9dd04baffcbd10e86742d149a080c (2.2.7+)
+/* f2d0ab6d1d4422a08cf1cf3bbdfba96b49dea42fb5ff4615e03a2a25c306e769 (2.2.8+)
                             __  __            _
                          ___\ \/ /_ __   __ _| |_
                         / _ \\  /| '_ \ / _` | __|
@@ -30,33 +30,38 @@
    USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
-#if !defined(_GNU_SOURCE)
-# define _GNU_SOURCE 1                  /* syscall prototype */
+#if ! defined(_GNU_SOURCE)
+#  define _GNU_SOURCE 1 /* syscall prototype */
+#endif
+
+#ifdef _WIN32
+/* force stdlib to define rand_s() */
+#  define _CRT_RAND_S
 #endif
 
 #include <stddef.h>
-#include <string.h>                     /* memset(), memcpy() */
+#include <string.h> /* memset(), memcpy() */
 #include <assert.h>
-#include <limits.h>                     /* UINT_MAX */
-#include <stdio.h>                      /* fprintf */
-#include <stdlib.h>                     /* getenv */
+#include <limits.h> /* UINT_MAX */
+#include <stdio.h>  /* fprintf */
+#include <stdlib.h> /* getenv, rand_s */
 
 #ifdef _WIN32
-#define getpid GetCurrentProcessId
+#  define getpid GetCurrentProcessId
 #else
-#include <sys/time.h>                   /* gettimeofday() */
-#include <sys/types.h>                  /* getpid() */
-#include <unistd.h>                     /* getpid() */
-#include <fcntl.h>                      /* O_RDONLY */
-#include <errno.h>
+#  include <sys/time.h>  /* gettimeofday() */
+#  include <sys/types.h> /* getpid() */
+#  include <unistd.h>    /* getpid() */
+#  include <fcntl.h>     /* O_RDONLY */
+#  include <errno.h>
 #endif
 
 #define XML_BUILDING_EXPAT 1
 
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #elif defined(HAVE_EXPAT_CONFIG_H)
-#include <expat_config.h>
+#  include <expat_config.h>
 #endif /* ndef _WIN32 */
 
 #include "ascii.h"
@@ -64,33 +69,31 @@
 #include "siphash.h"
 
 #if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
-# if defined(HAVE_GETRANDOM)
-#  include <sys/random.h>    /* getrandom */
-# else
-#  include <unistd.h>        /* syscall */
-#  include <sys/syscall.h>   /* SYS_getrandom */
-# endif
-# if ! defined(GRND_NONBLOCK)
-#  define GRND_NONBLOCK  0x0001
-# endif  /* defined(GRND_NONBLOCK) */
-#endif  /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
-
-#if defined(HAVE_LIBBSD) \
+#  if defined(HAVE_GETRANDOM)
+#    include <sys/random.h> /* getrandom */
+#  else
+#    include <unistd.h>      /* syscall */
+#    include <sys/syscall.h> /* SYS_getrandom */
+#  endif
+#  if ! defined(GRND_NONBLOCK)
+#    define GRND_NONBLOCK 0x0001
+#  endif /* defined(GRND_NONBLOCK) */
+#endif   /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
+
+#if defined(HAVE_LIBBSD)                                                       \
     && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM))
-# include <bsd/stdlib.h>
+#  include <bsd/stdlib.h>
 #endif
 
-#if defined(_WIN32) && !defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
-# define LOAD_LIBRARY_SEARCH_SYSTEM32  0x00000800
+#if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
+#  define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800
 #endif
 
-#if !defined(HAVE_GETRANDOM) && !defined(HAVE_SYSCALL_GETRANDOM) \
-    && !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_ARC4RANDOM) \
-    && !defined(XML_DEV_URANDOM) \
-    && !defined(_WIN32) \
-    && !defined(XML_POOR_ENTROPY)
-# error  \
-    You do not have support for any sources of high quality entropy \
+#if ! defined(HAVE_GETRANDOM) && ! defined(HAVE_SYSCALL_GETRANDOM)             \
+    && ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)            \
+    && ! defined(XML_DEV_URANDOM) && ! defined(_WIN32)                         \
+    && ! defined(XML_POOR_ENTROPY)
+#  error You do not have support for any sources of high quality entropy \
     enabled.  For end user security, that is probably not what you want. \
     \
     Your options include: \
@@ -101,7 +104,7 @@
       * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \
       * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \
       * Linux / BSD / macOS (/dev/urandom): XML_DEV_URANDOM \
-      * Windows (RtlGenRandom): _WIN32. \
+      * Windows (rand_s): _WIN32. \
     \
     If insist on not using any of these, bypass this error by defining \
     XML_POOR_ENTROPY; you have been warned. \
@@ -110,56 +113,55 @@
     to the build system, please open a bug.  Thank you!
 #endif
 
-
 #ifdef XML_UNICODE
-#define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX
-#define XmlConvert XmlUtf16Convert
-#define XmlGetInternalEncoding XmlGetUtf16InternalEncoding
-#define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS
-#define XmlEncode XmlUtf16Encode
+#  define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX
+#  define XmlConvert XmlUtf16Convert
+#  define XmlGetInternalEncoding XmlGetUtf16InternalEncoding
+#  define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS
+#  define XmlEncode XmlUtf16Encode
 /* Using pointer subtraction to convert to integer type. */
-#define MUST_CONVERT(enc, s) (!(enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1))
+#  define MUST_CONVERT(enc, s)                                                 \
+    (! (enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1))
 typedef unsigned short ICHAR;
 #else
-#define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX
-#define XmlConvert XmlUtf8Convert
-#define XmlGetInternalEncoding XmlGetUtf8InternalEncoding
-#define XmlGetInternalEncodingNS XmlGetUtf8InternalEncodingNS
-#define XmlEncode XmlUtf8Encode
-#define MUST_CONVERT(enc, s) (!(enc)->isUtf8)
+#  define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX
+#  define XmlConvert XmlUtf8Convert
+#  define XmlGetInternalEncoding XmlGetUtf8InternalEncoding
+#  define XmlGetInternalEncodingNS XmlGetUtf8InternalEncodingNS
+#  define XmlEncode XmlUtf8Encode
+#  define MUST_CONVERT(enc, s) (! (enc)->isUtf8)
 typedef char ICHAR;
 #endif
 
-
 #ifndef XML_NS
 
-#define XmlInitEncodingNS XmlInitEncoding
-#define XmlInitUnknownEncodingNS XmlInitUnknownEncoding
-#undef XmlGetInternalEncodingNS
-#define XmlGetInternalEncodingNS XmlGetInternalEncoding
-#define XmlParseXmlDeclNS XmlParseXmlDecl
+#  define XmlInitEncodingNS XmlInitEncoding
+#  define XmlInitUnknownEncodingNS XmlInitUnknownEncoding
+#  undef XmlGetInternalEncodingNS
+#  define XmlGetInternalEncodingNS XmlGetInternalEncoding
+#  define XmlParseXmlDeclNS XmlParseXmlDecl
 
 #endif
 
 #ifdef XML_UNICODE
 
-#ifdef XML_UNICODE_WCHAR_T
-#define XML_T(x) (const wchar_t)x
-#define XML_L(x) L ## x
-#else
-#define XML_T(x) (const unsigned short)x
-#define XML_L(x) x
-#endif
+#  ifdef XML_UNICODE_WCHAR_T
+#    define XML_T(x) (const wchar_t) x
+#    define XML_L(x) L##x
+#  else
+#    define XML_T(x) (const unsigned short)x
+#    define XML_L(x) x
+#  endif
 
 #else
 
-#define XML_T(x) x
-#define XML_L(x) x
+#  define XML_T(x) x
+#  define XML_L(x) x
 
 #endif
 
 /* Round up n to be a multiple of sz, where sz is a power of 2. */
-#define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1))
+#define ROUND_UP(n, sz) (((n) + ((sz)-1)) & ~((sz)-1))
 
 /* Do safe (NULL-aware) pointer arithmetic */
 #define EXPAT_SAFE_PTR_DIFF(p, q) (((p) && (q)) ? ((p) - (q)) : 0)
@@ -182,11 +184,9 @@ typedef struct {
   const XML_Memory_Handling_Suite *mem;
 } HASH_TABLE;
 
-static size_t
-keylen(KEY s);
+static size_t keylen(KEY s);
 
-static void
-copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key);
+static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key);
 
 /* For probing (after a collision) we need a step size relative prime
    to the hash table size, which is a power of 2. We use double-hashing,
@@ -196,9 +196,9 @@ copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key);
    We limit the maximum step size to table->size / 4 (mask >> 2) and make
    it odd, since odd numbers are always relative prime to a power of 2.
 */
-#define SECOND_HASH(hash, mask, power) \
-  ((((hash) & ~(mask)) >> ((power) - 1)) & ((mask) >> 2))
-#define PROBE_STEP(hash, mask, power) \
+#define SECOND_HASH(hash, mask, power)                                         \
+  ((((hash) & ~(mask)) >> ((power)-1)) & ((mask) >> 2))
+#define PROBE_STEP(hash, mask, power)                                          \
   ((unsigned char)((SECOND_HASH(hash, mask, power)) | 1))
 
 typedef struct {
@@ -206,7 +206,7 @@ typedef struct {
   NAMED **end;
 } HASH_TABLE_ITER;
 
-#define INIT_TAG_BUF_SIZE 32  /* must be a multiple of sizeof(XML_Char) */
+#define INIT_TAG_BUF_SIZE 32 /* must be a multiple of sizeof(XML_Char) */
 #define INIT_DATA_BUF_SIZE 1024
 #define INIT_ATTS_SIZE 16
 #define INIT_ATTS_VERSION 0xFFFFFFFF
@@ -253,20 +253,20 @@ typedef struct {
    TAG objects in a free list.
 */
 typedef struct tag {
-  struct tag *parent;           /* parent of this element */
-  const char *rawName;          /* tagName in the original encoding */
+  struct tag *parent;  /* parent of this element */
+  const char *rawName; /* tagName in the original encoding */
   int rawNameLength;
-  TAG_NAME name;                /* tagName in the API encoding */
-  char *buf;                    /* buffer for name components */
-  char *bufEnd;                 /* end of the buffer */
+  TAG_NAME name; /* tagName in the API encoding */
+  char *buf;     /* buffer for name components */
+  char *bufEnd;  /* end of the buffer */
   BINDING *bindings;
 } TAG;
 
 typedef struct {
   const XML_Char *name;
   const XML_Char *textPtr;
-  int textLen;                  /* length in XML_Chars */
-  int processed;                /* # of processed bytes - when suspended */
+  int textLen;   /* length in XML_Chars */
+  int processed; /* # of processed bytes - when suspended */
   const XML_Char *systemId;
   const XML_Char *base;
   const XML_Char *publicId;
@@ -277,13 +277,13 @@ typedef struct {
 } ENTITY;
 
 typedef struct {
-  enum XML_Content_Type         type;
-  enum XML_Content_Quant        quant;
-  const XML_Char *              name;
-  int                           firstchild;
-  int                           lastchild;
-  int                           childcnt;
-  int                           nextsib;
+  enum XML_Content_Type type;
+  enum XML_Content_Quant quant;
+  const XML_Char *name;
+  int firstchild;
+  int lastchild;
+  int childcnt;
+  int nextsib;
 } CONTENT_SCAFFOLD;
 
 #define INIT_SCAFFOLD_ELEMENTS 32
@@ -371,10 +371,8 @@ typedef struct open_internal_entity {
   XML_Bool betweenDecl; /* WFC: PE Between Declarations */
 } OPEN_INTERNAL_ENTITY;
 
-typedef enum XML_Error PTRCALL Processor(XML_Parser parser,
-                                         const char *start,
-                                         const char *end,
-                                         const char **endPtr);
+typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start,
+                                         const char *end, const char **endPtr);
 
 static Processor prologProcessor;
 static Processor prologInitProcessor;
@@ -395,118 +393,101 @@ static Processor externalEntityInitProcessor3;
 static Processor externalEntityContentProcessor;
 static Processor internalEntityProcessor;
 
-static enum XML_Error
-handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName);
-static enum XML_Error
-processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
-               const char *s, const char *next);
-static enum XML_Error
-initializeEncoding(XML_Parser parser);
-static enum XML_Error
-doProlog(XML_Parser parser, const ENCODING *enc, const char *s,
-         const char *end, int tok, const char *next, const char **nextPtr,
-         XML_Bool haveMore);
-static enum XML_Error
-processInternalEntity(XML_Parser parser, ENTITY *entity,
-                      XML_Bool betweenDecl);
-static enum XML_Error
-doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
-          const char *start, const char *end, const char **endPtr,
-          XML_Bool haveMore);
-static enum XML_Error
-doCdataSection(XML_Parser parser, const ENCODING *, const char **startPtr,
-               const char *end, const char **nextPtr, XML_Bool haveMore);
+static enum XML_Error handleUnknownEncoding(XML_Parser parser,
+                                            const XML_Char *encodingName);
+static enum XML_Error processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
+                                     const char *s, const char *next);
+static enum XML_Error initializeEncoding(XML_Parser parser);
+static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc,
+                               const char *s, const char *end, int tok,
+                               const char *next, const char **nextPtr,
+                               XML_Bool haveMore, XML_Bool allowClosingDoctype);
+static enum XML_Error processInternalEntity(XML_Parser parser, ENTITY *entity,
+                                            XML_Bool betweenDecl);
+static enum XML_Error doContent(XML_Parser parser, int startTagLevel,
+                                const ENCODING *enc, const char *start,
+                                const char *end, const char **endPtr,
+                                XML_Bool haveMore);
+static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *,
+                                     const char **startPtr, const char *end,
+                                     const char **nextPtr, XML_Bool haveMore);
 #ifdef XML_DTD
-static enum XML_Error
-doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr,
-                const char *end, const char **nextPtr, XML_Bool haveMore);
+static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *,
+                                      const char **startPtr, const char *end,
+                                      const char **nextPtr, XML_Bool haveMore);
 #endif /* XML_DTD */
 
-static void
-freeBindings(XML_Parser parser, BINDING *bindings);
-static enum XML_Error
-storeAtts(XML_Parser parser, const ENCODING *, const char *s,
-          TAG_NAME *tagNamePtr, BINDING **bindingsPtr);
-static enum XML_Error
-addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
-           const XML_Char *uri, BINDING **bindingsPtr);
-static int
-defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata,
-                XML_Bool isId, const XML_Char *dfltValue, XML_Parser parser);
-static enum XML_Error
-storeAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata,
-                    const char *, const char *, STRING_POOL *);
-static enum XML_Error
-appendAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata,
-                     const char *, const char *, STRING_POOL *);
-static ATTRIBUTE_ID *
-getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start,
-               const char *end);
-static int
-setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *);
-static enum XML_Error
-storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *start,
-                 const char *end);
-static int
-reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
-                            const char *start, const char *end);
-static int
-reportComment(XML_Parser parser, const ENCODING *enc, const char *start,
-              const char *end);
-static void
-reportDefault(XML_Parser parser, const ENCODING *enc, const char *start,
-              const char *end);
-
-static const XML_Char * getContext(XML_Parser parser);
-static XML_Bool
-setContext(XML_Parser parser, const XML_Char *context);
+static void freeBindings(XML_Parser parser, BINDING *bindings);
+static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *,
+                                const char *s, TAG_NAME *tagNamePtr,
+                                BINDING **bindingsPtr);
+static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix,
+                                 const ATTRIBUTE_ID *attId, const XML_Char *uri,
+                                 BINDING **bindingsPtr);
+static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata,
+                           XML_Bool isId, const XML_Char *dfltValue,
+                           XML_Parser parser);
+static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *,
+                                          XML_Bool isCdata, const char *,
+                                          const char *, STRING_POOL *);
+static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *,
+                                           XML_Bool isCdata, const char *,
+                                           const char *, STRING_POOL *);
+static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc,
+                                    const char *start, const char *end);
+static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *);
+static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc,
+                                       const char *start, const char *end);
+static int reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
+                                       const char *start, const char *end);
+static int reportComment(XML_Parser parser, const ENCODING *enc,
+                         const char *start, const char *end);
+static void reportDefault(XML_Parser parser, const ENCODING *enc,
+                          const char *start, const char *end);
+
+static const XML_Char *getContext(XML_Parser parser);
+static XML_Bool setContext(XML_Parser parser, const XML_Char *context);
 
 static void FASTCALL normalizePublicId(XML_Char *s);
 
-static DTD * dtdCreate(const XML_Memory_Handling_Suite *ms);
+static DTD *dtdCreate(const XML_Memory_Handling_Suite *ms);
 /* do not call if m_parentParser != NULL */
 static void dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms);
-static void
-dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms);
-static int
-dtdCopy(XML_Parser oldParser,
-        DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms);
-static int
-copyEntityTable(XML_Parser oldParser,
-                HASH_TABLE *, STRING_POOL *, const HASH_TABLE *);
-static NAMED *
-lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize);
-static void FASTCALL
-hashTableInit(HASH_TABLE *, const XML_Memory_Handling_Suite *ms);
+static void dtdDestroy(DTD *p, XML_Bool isDocEntity,
+                       const XML_Memory_Handling_Suite *ms);
+static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
+                   const XML_Memory_Handling_Suite *ms);
+static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *, STRING_POOL *,
+                           const HASH_TABLE *);
+static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name,
+                     size_t createSize);
+static void FASTCALL hashTableInit(HASH_TABLE *,
+                                   const XML_Memory_Handling_Suite *ms);
 static void FASTCALL hashTableClear(HASH_TABLE *);
 static void FASTCALL hashTableDestroy(HASH_TABLE *);
-static void FASTCALL
-hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *);
-static NAMED * FASTCALL hashTableIterNext(HASH_TABLE_ITER *);
+static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *);
+static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *);
 
-static void FASTCALL
-poolInit(STRING_POOL *, const XML_Memory_Handling_Suite *ms);
+static void FASTCALL poolInit(STRING_POOL *,
+                              const XML_Memory_Handling_Suite *ms);
 static void FASTCALL poolClear(STRING_POOL *);
 static void FASTCALL poolDestroy(STRING_POOL *);
-static XML_Char *
-poolAppend(STRING_POOL *pool, const ENCODING *enc,
-           const char *ptr, const char *end);
-static XML_Char *
-poolStoreString(STRING_POOL *pool, const ENCODING *enc,
-                const char *ptr, const char *end);
+static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc,
+                            const char *ptr, const char *end);
+static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc,
+                                 const char *ptr, const char *end);
 static XML_Bool FASTCALL poolGrow(STRING_POOL *pool);
-static const XML_Char * FASTCALL
-poolCopyString(STRING_POOL *pool, const XML_Char *s);
-static const XML_Char *
-poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n);
-static const XML_Char * FASTCALL
-poolAppendString(STRING_POOL *pool, const XML_Char *s);
+static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool,
+                                               const XML_Char *s);
+static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s,
+                                       int n);
+static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool,
+                                                 const XML_Char *s);
 
 static int FASTCALL nextScaffoldPart(XML_Parser parser);
-static XML_Content * build_model(XML_Parser parser);
-static ELEMENT_TYPE *
-getElementType(XML_Parser parser, const ENCODING *enc,
-               const char *ptr, const char *end);
+static XML_Content *build_model(XML_Parser parser);
+static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc,
+                                    const char *ptr, const char *end);
 
 static XML_Char *copyString(const XML_Char *s,
                             const XML_Memory_Handling_Suite *memsuite);
@@ -514,14 +495,11 @@ static XML_Char *copyString(const XML_Char *s,
 static unsigned long generate_hash_secret_salt(XML_Parser parser);
 static XML_Bool startParsing(XML_Parser parser);
 
-static XML_Parser
-parserCreate(const XML_Char *encodingName,
-             const XML_Memory_Handling_Suite *memsuite,
-             const XML_Char *nameSep,
-             DTD *dtd);
+static XML_Parser parserCreate(const XML_Char *encodingName,
+                               const XML_Memory_Handling_Suite *memsuite,
+                               const XML_Char *nameSep, DTD *dtd);
 
-static void
-parserInit(XML_Parser parser, const XML_Char *encodingName);
+static void parserInit(XML_Parser parser, const XML_Char *encodingName);
 
 #define poolStart(pool) ((pool)->start)
 #define poolEnd(pool) ((pool)->ptr)
@@ -530,10 +508,10 @@ parserInit(XML_Parser parser, const XML_Char *encodingName);
 #define poolLastChar(pool) (((pool)->ptr)[-1])
 #define poolDiscard(pool) ((pool)->ptr = (pool)->start)
 #define poolFinish(pool) ((pool)->start = (pool)->ptr)
-#define poolAppendChar(pool, c) \
-  (((pool)->ptr == (pool)->end && !poolGrow(pool)) \
-   ? 0 \
-   : ((*((pool)->ptr)++ = c), 1))
+#define poolAppendChar(pool, c)                                                \
+  (((pool)->ptr == (pool)->end && ! poolGrow(pool))                            \
+       ? 0                                                                     \
+       : ((*((pool)->ptr)++ = c), 1))
 
 struct XML_ParserStruct {
   /* The first member must be m_userData so that the XML_GetUserData
@@ -584,7 +562,7 @@ struct XML_ParserStruct {
   void *m_unknownEncodingMem;
   void *m_unknownEncodingData;
   void *m_unknownEncodingHandlerData;
-  void (XMLCALL *m_unknownEncodingRelease)(void *);
+  void(XMLCALL *m_unknownEncodingRelease)(void *);
   PROLOG_STATE m_prologState;
   Processor *m_processor;
   enum XML_Error m_errorCode;
@@ -638,57 +616,55 @@ struct XML_ParserStruct {
   unsigned long m_hash_secret_salt;
 };
 
-#define MALLOC(parser, s)      (parser->m_mem.malloc_fcn((s)))
-#define REALLOC(parser, p, s)  (parser->m_mem.realloc_fcn((p),(s)))
-#define FREE(parser, p)        (parser->m_mem.free_fcn((p)))
-
+#define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s)))
+#define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s)))
+#define FREE(parser, p) (parser->m_mem.free_fcn((p)))
 
 XML_Parser XMLCALL
-XML_ParserCreate(const XML_Char *encodingName)
-{
+XML_ParserCreate(const XML_Char *encodingName) {
   return XML_ParserCreate_MM(encodingName, NULL, NULL);
 }
 
 XML_Parser XMLCALL
-XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep)
-{
+XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) {
   XML_Char tmp[2];
   *tmp = nsSep;
   return XML_ParserCreate_MM(encodingName, NULL, tmp);
 }
 
-static const XML_Char implicitContext[] = {
-  ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h, ASCII_t, ASCII_t, ASCII_p,
-  ASCII_COLON, ASCII_SLASH, ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w,
-  ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g,
-  ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L, ASCII_SLASH, ASCII_1, ASCII_9,
-  ASCII_9, ASCII_8, ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e,
-  ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'
-};
-
+static const XML_Char implicitContext[]
+    = {ASCII_x,     ASCII_m,     ASCII_l,      ASCII_EQUALS, ASCII_h,
+       ASCII_t,     ASCII_t,     ASCII_p,      ASCII_COLON,  ASCII_SLASH,
+       ASCII_SLASH, ASCII_w,     ASCII_w,      ASCII_w,      ASCII_PERIOD,
+       ASCII_w,     ASCII_3,     ASCII_PERIOD, ASCII_o,      ASCII_r,
+       ASCII_g,     ASCII_SLASH, ASCII_X,      ASCII_M,      ASCII_L,
+       ASCII_SLASH, ASCII_1,     ASCII_9,      ASCII_9,      ASCII_8,
+       ASCII_SLASH, ASCII_n,     ASCII_a,      ASCII_m,      ASCII_e,
+       ASCII_s,     ASCII_p,     ASCII_a,      ASCII_c,      ASCII_e,
+       '\0'};
 
 /* To avoid warnings about unused functions: */
 #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)
 
-#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
+#  if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
 
 /* Obtain entropy on Linux 3.17+ */
 static int
-writeRandomBytes_getrandom_nonblock(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
+writeRandomBytes_getrandom_nonblock(void *target, size_t count) {
+  int success = 0; /* full count bytes written? */
   size_t bytesWrittenTotal = 0;
   const unsigned int getrandomFlags = GRND_NONBLOCK;
 
   do {
-    void * const currentTarget = (void*)((char*)target + bytesWrittenTotal);
+    void *const currentTarget = (void *)((char *)target + bytesWrittenTotal);
     const size_t bytesToWrite = count - bytesWrittenTotal;
 
     const int bytesWrittenMore =
-#if defined(HAVE_GETRANDOM)
+#    if defined(HAVE_GETRANDOM)
         getrandom(currentTarget, bytesToWrite, getrandomFlags);
-#else
+#    else
         syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags);
-#endif
+#    endif
 
     if (bytesWrittenMore > 0) {
       bytesWrittenTotal += bytesWrittenMore;
@@ -700,15 +676,14 @@ writeRandomBytes_getrandom_nonblock(void * target, size_t count) {
   return success;
 }
 
-#endif  /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
-
+#  endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
 
-#if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
+#  if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
 
 /* Extract entropy from /dev/urandom */
 static int
-writeRandomBytes_dev_urandom(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
+writeRandomBytes_dev_urandom(void *target, size_t count) {
+  int success = 0; /* full count bytes written? */
   size_t bytesWrittenTotal = 0;
 
   const int fd = open("/dev/urandom", O_RDONLY);
@@ -717,7 +692,7 @@ writeRandomBytes_dev_urandom(void * target, size_t count) {
   }
 
   do {
-    void * const currentTarget = (void*)((char*)target + bytesWrittenTotal);
+    void *const currentTarget = (void *)((char *)target + bytesWrittenTotal);
     const size_t bytesToWrite = count - bytesWrittenTotal;
 
     const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite);
@@ -733,15 +708,14 @@ writeRandomBytes_dev_urandom(void * target, size_t count) {
   return success;
 }
 
-#endif  /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
-
-#endif  /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
+#  endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
 
+#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
 
 #if defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF)
 
 static void
-writeRandomBytes_arc4random(void * target, size_t count) {
+writeRandomBytes_arc4random(void *target, size_t count) {
   size_t bytesWrittenTotal = 0;
 
   while (bytesWrittenTotal < count) {
@@ -749,93 +723,82 @@ writeRandomBytes_arc4random(void * target, size_t count) {
     size_t i = 0;
 
     for (; (i < sizeof(random32)) && (bytesWrittenTotal < count);
-        i++, bytesWrittenTotal++) {
+         i++, bytesWrittenTotal++) {
       const uint8_t random8 = (uint8_t)(random32 >> (i * 8));
       ((uint8_t *)target)[bytesWrittenTotal] = random8;
     }
   }
 }
 
-#endif  /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */
-
+#endif /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */
 
 #ifdef _WIN32
 
-typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG);
-HMODULE _Expat_LoadLibrary(LPCTSTR filename);  /* see loadlibrary.c */
-
-/* Obtain entropy on Windows XP / Windows Server 2003 and later.
- * Hint on RtlGenRandom and the following article from libsodium.
- *
- * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI
- * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographically-secure-random-number-on-windows-without-using-cryptoapi/
+/* Obtain entropy on Windows using the rand_s() function which
+ * generates cryptographically secure random numbers.  Internally it
+ * uses RtlGenRandom API which is present in Windows XP and later.
  */
 static int
-writeRandomBytes_RtlGenRandom(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
-  const HMODULE advapi32 = _Expat_LoadLibrary(TEXT("ADVAPI32.DLL"));
-
-  if (advapi32) {
-    const RTLGENRANDOM_FUNC RtlGenRandom
-        = (RTLGENRANDOM_FUNC)GetProcAddress(advapi32, "SystemFunction036");
-    if (RtlGenRandom) {
-      if (RtlGenRandom((PVOID)target, (ULONG)count) == TRUE) {
-        success = 1;
-      }
+writeRandomBytes_rand_s(void *target, size_t count) {
+  size_t bytesWrittenTotal = 0;
+
+  while (bytesWrittenTotal < count) {
+    unsigned int random32 = 0;
+    size_t i = 0;
+
+    if (rand_s(&random32))
+      return 0; /* failure */
+
+    for (; (i < sizeof(random32)) && (bytesWrittenTotal < count);
+         i++, bytesWrittenTotal++) {
+      const uint8_t random8 = (uint8_t)(random32 >> (i * 8));
+      ((uint8_t *)target)[bytesWrittenTotal] = random8;
     }
-    FreeLibrary(advapi32);
   }
-
-  return success;
+  return 1; /* success */
 }
 
 #endif /* _WIN32 */
 
-
 #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)
 
 static unsigned long
-gather_time_entropy(void)
-{
-#ifdef _WIN32
+gather_time_entropy(void) {
+#  ifdef _WIN32
   FILETIME ft;
   GetSystemTimeAsFileTime(&ft); /* never fails */
   return ft.dwHighDateTime ^ ft.dwLowDateTime;
-#else
+#  else
   struct timeval tv;
   int gettimeofday_res;
 
   gettimeofday_res = gettimeofday(&tv, NULL);
 
-#if defined(NDEBUG)
+#    if defined(NDEBUG)
   (void)gettimeofday_res;
-#else
-  assert (gettimeofday_res == 0);
-#endif  /* defined(NDEBUG) */
+#    else
+  assert(gettimeofday_res == 0);
+#    endif /* defined(NDEBUG) */
 
   /* Microseconds time is <20 bits entropy */
   return tv.tv_usec;
-#endif
+#  endif
 }
 
-#endif  /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
-
+#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
 
 static unsigned long
-ENTROPY_DEBUG(const char * label, unsigned long entropy) {
-  const char * const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG");
+ENTROPY_DEBUG(const char *label, unsigned long entropy) {
+  const char *const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG");
   if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) {
-    fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n",
-        label,
-        (int)sizeof(entropy) * 2, entropy,
-        (unsigned long)sizeof(entropy));
+    fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", label,
+            (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy));
   }
   return entropy;
 }
 
 static unsigned long
-generate_hash_secret_salt(XML_Parser parser)
-{
+generate_hash_secret_salt(XML_Parser parser) {
   unsigned long entropy;
   (void)parser;
 
@@ -848,20 +811,20 @@ generate_hash_secret_salt(XML_Parser parser)
   return ENTROPY_DEBUG("arc4random", entropy);
 #else
   /* Try high quality providers first .. */
-#ifdef _WIN32
-  if (writeRandomBytes_RtlGenRandom((void *)&entropy, sizeof(entropy))) {
-    return ENTROPY_DEBUG("RtlGenRandom", entropy);
+#  ifdef _WIN32
+  if (writeRandomBytes_rand_s((void *)&entropy, sizeof(entropy))) {
+    return ENTROPY_DEBUG("rand_s", entropy);
   }
-#elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
+#  elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
   if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) {
     return ENTROPY_DEBUG("getrandom", entropy);
   }
-#endif
-#if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
+#  endif
+#  if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
   if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) {
     return ENTROPY_DEBUG("/dev/urandom", entropy);
   }
-#endif  /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
+#  endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
   /* .. and self-made low quality for backup: */
 
   /* Process ID is 0 bits entropy if attacker has local access */
@@ -872,7 +835,7 @@ generate_hash_secret_salt(XML_Parser parser)
     return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647);
   } else {
     return ENTROPY_DEBUG("fallback(8)",
-        entropy * (unsigned long)2305843009213693951ULL);
+                         entropy * (unsigned long)2305843009213693951ULL);
   }
 #endif
 }
@@ -884,49 +847,43 @@ get_hash_secret_salt(XML_Parser parser) {
   return parser->m_hash_secret_salt;
 }
 
-static XML_Bool  /* only valid for root parser */
-startParsing(XML_Parser parser)
-{
-    /* hash functions must be initialized before setContext() is called */
-    if (parser->m_hash_secret_salt == 0)
-      parser->m_hash_secret_salt = generate_hash_secret_salt(parser);
-    if (parser->m_ns) {
-      /* implicit context only set for root parser, since child
-         parsers (i.e. external entity parsers) will inherit it
-      */
-      return setContext(parser, implicitContext);
-    }
-    return XML_TRUE;
+static XML_Bool /* only valid for root parser */
+startParsing(XML_Parser parser) {
+  /* hash functions must be initialized before setContext() is called */
+  if (parser->m_hash_secret_salt == 0)
+    parser->m_hash_secret_salt = generate_hash_secret_salt(parser);
+  if (parser->m_ns) {
+    /* implicit context only set for root parser, since child
+       parsers (i.e. external entity parsers) will inherit it
+    */
+    return setContext(parser, implicitContext);
+  }
+  return XML_TRUE;
 }
 
 XML_Parser XMLCALL
 XML_ParserCreate_MM(const XML_Char *encodingName,
                     const XML_Memory_Handling_Suite *memsuite,
-                    const XML_Char *nameSep)
-{
+                    const XML_Char *nameSep) {
   return parserCreate(encodingName, memsuite, nameSep, NULL);
 }
 
 static XML_Parser
 parserCreate(const XML_Char *encodingName,
-             const XML_Memory_Handling_Suite *memsuite,
-             const XML_Char *nameSep,
-             DTD *dtd)
-{
+             const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep,
+             DTD *dtd) {
   XML_Parser parser;
 
   if (memsuite) {
     XML_Memory_Handling_Suite *mtemp;
-    parser = (XML_Parser)
-      memsuite->malloc_fcn(sizeof(struct XML_ParserStruct));
+    parser = (XML_Parser)memsuite->malloc_fcn(sizeof(struct XML_ParserStruct));
     if (parser != NULL) {
       mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem);
       mtemp->malloc_fcn = memsuite->malloc_fcn;
       mtemp->realloc_fcn = memsuite->realloc_fcn;
       mtemp->free_fcn = memsuite->free_fcn;
     }
-  }
-  else {
+  } else {
     XML_Memory_Handling_Suite *mtemp;
     parser = (XML_Parser)malloc(sizeof(struct XML_ParserStruct));
     if (parser != NULL) {
@@ -937,27 +894,30 @@ parserCreate(const XML_Char *encodingName,
     }
   }
 
-  if (!parser)
+  if (! parser)
     return parser;
 
   parser->m_buffer = NULL;
   parser->m_bufferLim = NULL;
 
   parser->m_attsSize = INIT_ATTS_SIZE;
-  parser->m_atts = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE));
+  parser->m_atts
+      = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE));
   if (parser->m_atts == NULL) {
     FREE(parser, parser);
     return NULL;
   }
 #ifdef XML_ATTR_INFO
-  parser->m_attInfo = (XML_AttrInfo*)MALLOC(parser, parser->m_attsSize * sizeof(XML_AttrInfo));
+  parser->m_attInfo = (XML_AttrInfo *)MALLOC(
+      parser, parser->m_attsSize * sizeof(XML_AttrInfo));
   if (parser->m_attInfo == NULL) {
     FREE(parser, parser->m_atts);
     FREE(parser, parser);
     return NULL;
   }
 #endif
-  parser->m_dataBuf = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char));
+  parser->m_dataBuf
+      = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char));
   if (parser->m_dataBuf == NULL) {
     FREE(parser, parser->m_atts);
 #ifdef XML_ATTR_INFO
@@ -1007,7 +967,7 @@ parserCreate(const XML_Char *encodingName,
   poolInit(&parser->m_temp2Pool, &(parser->m_mem));
   parserInit(parser, encodingName);
 
-  if (encodingName && !parser->m_protocolEncodingName) {
+  if (encodingName && ! parser->m_protocolEncodingName) {
     XML_ParserFree(parser);
     return NULL;
   }
@@ -1016,8 +976,7 @@ parserCreate(const XML_Char *encodingName,
     parser->m_ns = XML_TRUE;
     parser->m_internalEncoding = XmlGetInternalEncodingNS();
     parser->m_namespaceSeparator = *nameSep;
-  }
-  else {
+  } else {
     parser->m_internalEncoding = XmlGetInternalEncoding();
   }
 
@@ -1025,8 +984,7 @@ parserCreate(const XML_Char *encodingName,
 }
 
 static void
-parserInit(XML_Parser parser, const XML_Char *encodingName)
-{
+parserInit(XML_Parser parser, const XML_Char *encodingName) {
   parser->m_processor = prologInitProcessor;
   XmlPrologStateInit(&parser->m_prologState);
   if (encodingName != NULL) {
@@ -1099,8 +1057,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName)
 
 /* moves list of bindings to m_freeBindingList */
 static void FASTCALL
-moveToFreeBindingList(XML_Parser parser, BINDING *bindings)
-{
+moveToFreeBindingList(XML_Parser parser, BINDING *bindings) {
   while (bindings) {
     BINDING *b = bindings;
     bindings = bindings->nextTagBinding;
@@ -1110,13 +1067,12 @@ moveToFreeBindingList(XML_Parser parser, BINDING *bindings)
 }
 
 XML_Bool XMLCALL
-XML_ParserReset(XML_Parser parser, const XML_Char *encodingName)
-{
+XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) {
   TAG *tStk;
   OPEN_INTERNAL_ENTITY *openEntityList;
 
   if (parser == NULL)
-      return XML_FALSE;
+    return XML_FALSE;
 
   if (parser->m_parentParser)
     return XML_FALSE;
@@ -1152,15 +1108,15 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName)
 }
 
 enum XML_Status XMLCALL
-XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName)
-{
+XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) {
   if (parser == NULL)
-      return XML_STATUS_ERROR;
+    return XML_STATUS_ERROR;
   /* Block after XML_Parse()/XML_ParseBuffer() has been called.
      XXX There's no way for the caller to determine which of the
      XXX possible error cases caused the XML_STATUS_ERROR return.
   */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return XML_STATUS_ERROR;
 
   /* Get rid of any previous encoding name */
@@ -1172,17 +1128,15 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName)
   else {
     /* Copy the new encoding name into allocated memory */
     parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem));
-    if (!parser->m_protocolEncodingName)
+    if (! parser->m_protocolEncodingName)
       return XML_STATUS_ERROR;
   }
   return XML_STATUS_OK;
 }
 
 XML_Parser XMLCALL
-XML_ExternalEntityParserCreate(XML_Parser oldParser,
-                               const XML_Char *context,
-                               const XML_Char *encodingName)
-{
+XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
+                               const XML_Char *encodingName) {
   XML_Parser parser = oldParser;
   DTD *newDtd = NULL;
   DTD *oldDtd;
@@ -1206,7 +1160,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   XML_AttlistDeclHandler oldAttlistDeclHandler;
   XML_EntityDeclHandler oldEntityDeclHandler;
   XML_XmlDeclHandler oldXmlDeclHandler;
-  ELEMENT_TYPE * oldDeclElementType;
+  ELEMENT_TYPE *oldDeclElementType;
 
   void *oldUserData;
   void *oldHandlerArg;
@@ -1269,7 +1223,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   oldhash_secret_salt = parser->m_hash_secret_salt;
 
 #ifdef XML_DTD
-  if (!context)
+  if (! context)
     newDtd = oldDtd;
 #endif /* XML_DTD */
 
@@ -1282,12 +1236,11 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
     XML_Char tmp[2];
     *tmp = parser->m_namespaceSeparator;
     parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd);
-  }
-  else {
+  } else {
     parser = parserCreate(encodingName, &parser->m_mem, NULL, newDtd);
   }
 
-  if (!parser)
+  if (! parser)
     return NULL;
 
   parser->m_startElementHandler = oldStartElementHandler;
@@ -1327,21 +1280,20 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   parser->m_prologState.inEntityValue = oldInEntityValue;
   if (context) {
 #endif /* XML_DTD */
-    if (!dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem)
-      || !setContext(parser, context)) {
+    if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem)
+        || ! setContext(parser, context)) {
       XML_ParserFree(parser);
       return NULL;
     }
     parser->m_processor = externalEntityInitProcessor;
 #ifdef XML_DTD
-  }
-  else {
-    /* The DTD instance referenced by parser->m_dtd is shared between the document's
-       root parser and external PE parsers, therefore one does not need to
-       call setContext. In addition, one also *must* not call setContext,
-       because this would overwrite existing prefix->binding pointers in
-       parser->m_dtd with ones that get destroyed with the external PE parser.
-       This would leave those prefixes with dangling pointers.
+  } else {
+    /* The DTD instance referenced by parser->m_dtd is shared between the
+       document's root parser and external PE parsers, therefore one does not
+       need to call setContext. In addition, one also *must* not call
+       setContext, because this would overwrite existing prefix->binding
+       pointers in parser->m_dtd with ones that get destroyed with the external
+       PE parser. This would leave those prefixes with dangling pointers.
     */
     parser->m_isParamEntity = XML_TRUE;
     XmlPrologStateInitExternalEntity(&parser->m_prologState);
@@ -1352,11 +1304,10 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
 }
 
 static void FASTCALL
-destroyBindings(BINDING *bindings, XML_Parser parser)
-{
+destroyBindings(BINDING *bindings, XML_Parser parser) {
   for (;;) {
     BINDING *b = bindings;
-    if (!b)
+    if (! b)
       break;
     bindings = b->nextTagBinding;
     FREE(parser, b->uri);
@@ -1365,8 +1316,7 @@ destroyBindings(BINDING *bindings, XML_Parser parser)
 }
 
 void XMLCALL
-XML_ParserFree(XML_Parser parser)
-{
+XML_ParserFree(XML_Parser parser) {
   TAG *tagList;
   OPEN_INTERNAL_ENTITY *entityList;
   if (parser == NULL)
@@ -1411,11 +1361,12 @@ XML_ParserFree(XML_Parser parser)
   /* external parameter entity parsers share the DTD structure
      parser->m_dtd with the root parser, so we must not destroy it
   */
-  if (!parser->m_isParamEntity && parser->m_dtd)
+  if (! parser->m_isParamEntity && parser->m_dtd)
 #else
   if (parser->m_dtd)
 #endif /* XML_DTD */
-    dtdDestroy(parser->m_dtd, (XML_Bool)!parser->m_parentParser, &parser->m_mem);
+    dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser,
+               &parser->m_mem);
   FREE(parser, (void *)parser->m_atts);
 #ifdef XML_ATTR_INFO
   FREE(parser, (void *)parser->m_attInfo);
@@ -1431,20 +1382,19 @@ XML_ParserFree(XML_Parser parser)
 }
 
 void XMLCALL
-XML_UseParserAsHandlerArg(XML_Parser parser)
-{
+XML_UseParserAsHandlerArg(XML_Parser parser) {
   if (parser != NULL)
     parser->m_handlerArg = parser;
 }
 
 enum XML_Error XMLCALL
-XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD)
-{
+XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) {
   if (parser == NULL)
     return XML_ERROR_INVALID_ARGUMENT;
 #ifdef XML_DTD
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING;
   parser->m_useForeignDTD = useDTD;
   return XML_ERROR_NONE;
@@ -1454,19 +1404,18 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD)
 }
 
 void XMLCALL
-XML_SetReturnNSTriplet(XML_Parser parser, int do_nst)
-{
+XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) {
   if (parser == NULL)
     return;
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return;
   parser->m_ns_triplets = do_nst ? XML_TRUE : XML_FALSE;
 }
 
 void XMLCALL
-XML_SetUserData(XML_Parser parser, void *p)
-{
+XML_SetUserData(XML_Parser parser, void *p) {
   if (parser == NULL)
     return;
   if (parser->m_handlerArg == parser->m_userData)
@@ -1476,49 +1425,43 @@ XML_SetUserData(XML_Parser parser, void *p)
 }
 
 enum XML_Status XMLCALL
-XML_SetBase(XML_Parser parser, const XML_Char *p)
-{
+XML_SetBase(XML_Parser parser, const XML_Char *p) {
   if (parser == NULL)
     return XML_STATUS_ERROR;
   if (p) {
     p = poolCopyString(&parser->m_dtd->pool, p);
-    if (!p)
+    if (! p)
       return XML_STATUS_ERROR;
     parser->m_curBase = p;
-  }
-  else
+  } else
     parser->m_curBase = NULL;
   return XML_STATUS_OK;
 }
 
-const XML_Char * XMLCALL
-XML_GetBase(XML_Parser parser)
-{
+const XML_Char *XMLCALL
+XML_GetBase(XML_Parser parser) {
   if (parser == NULL)
     return NULL;
   return parser->m_curBase;
 }
 
 int XMLCALL
-XML_GetSpecifiedAttributeCount(XML_Parser parser)
-{
+XML_GetSpecifiedAttributeCount(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   return parser->m_nSpecifiedAtts;
 }
 
 int XMLCALL
-XML_GetIdAttributeIndex(XML_Parser parser)
-{
+XML_GetIdAttributeIndex(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   return parser->m_idAttIndex;
 }
 
 #ifdef XML_ATTR_INFO
-const XML_AttrInfo * XMLCALL
-XML_GetAttributeInfo(XML_Parser parser)
-{
+const XML_AttrInfo *XMLCALL
+XML_GetAttributeInfo(XML_Parser parser) {
   if (parser == NULL)
     return NULL;
   return parser->m_attInfo;
@@ -1526,10 +1469,8 @@ XML_GetAttributeInfo(XML_Parser parser)
 #endif
 
 void XMLCALL
-XML_SetElementHandler(XML_Parser parser,
-                      XML_StartElementHandler start,
-                      XML_EndElementHandler end)
-{
+XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start,
+                      XML_EndElementHandler end) {
   if (parser == NULL)
     return;
   parser->m_startElementHandler = start;
@@ -1537,39 +1478,33 @@ XML_SetElementHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetStartElementHandler(XML_Parser parser,
-                           XML_StartElementHandler start) {
+XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler start) {
   if (parser != NULL)
     parser->m_startElementHandler = start;
 }
 
 void XMLCALL
-XML_SetEndElementHandler(XML_Parser parser,
-                         XML_EndElementHandler end) {
+XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler end) {
   if (parser != NULL)
     parser->m_endElementHandler = end;
 }
 
 void XMLCALL
 XML_SetCharacterDataHandler(XML_Parser parser,
-                            XML_CharacterDataHandler handler)
-{
+                            XML_CharacterDataHandler handler) {
   if (parser != NULL)
     parser->m_characterDataHandler = handler;
 }
 
 void XMLCALL
 XML_SetProcessingInstructionHandler(XML_Parser parser,
-                                    XML_ProcessingInstructionHandler handler)
-{
+                                    XML_ProcessingInstructionHandler handler) {
   if (parser != NULL)
     parser->m_processingInstructionHandler = handler;
 }
 
 void XMLCALL
-XML_SetCommentHandler(XML_Parser parser,
-                      XML_CommentHandler handler)
-{
+XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler) {
   if (parser != NULL)
     parser->m_commentHandler = handler;
 }
@@ -1577,8 +1512,7 @@ XML_SetCommentHandler(XML_Parser parser,
 void XMLCALL
 XML_SetCdataSectionHandler(XML_Parser parser,
                            XML_StartCdataSectionHandler start,
-                           XML_EndCdataSectionHandler end)
-{
+                           XML_EndCdataSectionHandler end) {
   if (parser == NULL)
     return;
   parser->m_startCdataSectionHandler = start;
@@ -1600,9 +1534,7 @@ XML_SetEndCdataSectionHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDefaultHandler(XML_Parser parser,
-                      XML_DefaultHandler handler)
-{
+XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler) {
   if (parser == NULL)
     return;
   parser->m_defaultHandler = handler;
@@ -1610,9 +1542,7 @@ XML_SetDefaultHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDefaultHandlerExpand(XML_Parser parser,
-                            XML_DefaultHandler handler)
-{
+XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler) {
   if (parser == NULL)
     return;
   parser->m_defaultHandler = handler;
@@ -1620,10 +1550,8 @@ XML_SetDefaultHandlerExpand(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDoctypeDeclHandler(XML_Parser parser,
-                          XML_StartDoctypeDeclHandler start,
-                          XML_EndDoctypeDeclHandler end)
-{
+XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start,
+                          XML_EndDoctypeDeclHandler end) {
   if (parser == NULL)
     return;
   parser->m_startDoctypeDeclHandler = start;
@@ -1638,24 +1566,20 @@ XML_SetStartDoctypeDeclHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetEndDoctypeDeclHandler(XML_Parser parser,
-                             XML_EndDoctypeDeclHandler end) {
+XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end) {
   if (parser != NULL)
     parser->m_endDoctypeDeclHandler = end;
 }
 
 void XMLCALL
 XML_SetUnparsedEntityDeclHandler(XML_Parser parser,
-                                 XML_UnparsedEntityDeclHandler handler)
-{
+                                 XML_UnparsedEntityDeclHandler handler) {
   if (parser != NULL)
     parser->m_unparsedEntityDeclHandler = handler;
 }
 
 void XMLCALL
-XML_SetNotationDeclHandler(XML_Parser parser,
-                           XML_NotationDeclHandler handler)
-{
+XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler) {
   if (parser != NULL)
     parser->m_notationDeclHandler = handler;
 }
@@ -1663,8 +1587,7 @@ XML_SetNotationDeclHandler(XML_Parser parser,
 void XMLCALL
 XML_SetNamespaceDeclHandler(XML_Parser parser,
                             XML_StartNamespaceDeclHandler start,
-                            XML_EndNamespaceDeclHandler end)
-{
+                            XML_EndNamespaceDeclHandler end) {
   if (parser == NULL)
     return;
   parser->m_startNamespaceDeclHandler = start;
@@ -1687,23 +1610,20 @@ XML_SetEndNamespaceDeclHandler(XML_Parser parser,
 
 void XMLCALL
 XML_SetNotStandaloneHandler(XML_Parser parser,
-                            XML_NotStandaloneHandler handler)
-{
+                            XML_NotStandaloneHandler handler) {
   if (parser != NULL)
     parser->m_notStandaloneHandler = handler;
 }
 
 void XMLCALL
 XML_SetExternalEntityRefHandler(XML_Parser parser,
-                                XML_ExternalEntityRefHandler handler)
-{
+                                XML_ExternalEntityRefHandler handler) {
   if (parser != NULL)
     parser->m_externalEntityRefHandler = handler;
 }
 
 void XMLCALL
-XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg)
-{
+XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg) {
   if (parser == NULL)
     return;
   if (arg)
@@ -1714,17 +1634,14 @@ XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg)
 
 void XMLCALL
 XML_SetSkippedEntityHandler(XML_Parser parser,
-                            XML_SkippedEntityHandler handler)
-{
+                            XML_SkippedEntityHandler handler) {
   if (parser != NULL)
     parser->m_skippedEntityHandler = handler;
 }
 
 void XMLCALL
 XML_SetUnknownEncodingHandler(XML_Parser parser,
-                              XML_UnknownEncodingHandler handler,
-                              void *data)
-{
+                              XML_UnknownEncodingHandler handler, void *data) {
   if (parser == NULL)
     return;
   parser->m_unknownEncodingHandler = handler;
@@ -1732,44 +1649,37 @@ XML_SetUnknownEncodingHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetElementDeclHandler(XML_Parser parser,
-                          XML_ElementDeclHandler eldecl)
-{
+XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl) {
   if (parser != NULL)
     parser->m_elementDeclHandler = eldecl;
 }
 
 void XMLCALL
-XML_SetAttlistDeclHandler(XML_Parser parser,
-                          XML_AttlistDeclHandler attdecl)
-{
+XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl) {
   if (parser != NULL)
     parser->m_attlistDeclHandler = attdecl;
 }
 
 void XMLCALL
-XML_SetEntityDeclHandler(XML_Parser parser,
-                         XML_EntityDeclHandler handler)
-{
+XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler) {
   if (parser != NULL)
     parser->m_entityDeclHandler = handler;
 }
 
 void XMLCALL
-XML_SetXmlDeclHandler(XML_Parser parser,
-                      XML_XmlDeclHandler handler) {
+XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler handler) {
   if (parser != NULL)
     parser->m_xmlDeclHandler = handler;
 }
 
 int XMLCALL
 XML_SetParamEntityParsing(XML_Parser parser,
-                          enum XML_ParamEntityParsing peParsing)
-{
+                          enum XML_ParamEntityParsing peParsing) {
   if (parser == NULL)
     return 0;
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return 0;
 #ifdef XML_DTD
   parser->m_paramEntityParsing = peParsing;
@@ -1780,23 +1690,21 @@ XML_SetParamEntityParsing(XML_Parser parser,
 }
 
 int XMLCALL
-XML_SetHashSalt(XML_Parser parser,
-                unsigned long hash_salt)
-{
+XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) {
   if (parser == NULL)
     return 0;
   if (parser->m_parentParser)
     return XML_SetHashSalt(parser->m_parentParser, hash_salt);
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return 0;
   parser->m_hash_secret_salt = hash_salt;
   return 1;
 }
 
 enum XML_Status XMLCALL
-XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
-{
+XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) {
   if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) {
     if (parser != NULL)
       parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT;
@@ -1810,7 +1718,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     parser->m_errorCode = XML_ERROR_FINISHED;
     return XML_STATUS_ERROR;
   case XML_INITIALIZED:
-    if (parser->m_parentParser == NULL && !startParsing(parser)) {
+    if (parser->m_parentParser == NULL && ! startParsing(parser)) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return XML_STATUS_ERROR;
     }
@@ -1821,7 +1729,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
 
   if (len == 0) {
     parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
-    if (!isFinal)
+    if (! isFinal)
       return XML_STATUS_OK;
     parser->m_positionPtr = parser->m_bufferPtr;
     parser->m_parseEndPtr = parser->m_bufferEnd;
@@ -1830,7 +1738,9 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
        data are the final chunk of input, then we have to check them again
        to detect errors based on that fact.
     */
-    parser->m_errorCode = parser->m_processor(parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
+    parser->m_errorCode
+        = parser->m_processor(parser, parser->m_bufferPtr,
+                              parser->m_parseEndPtr, &parser->m_bufferPtr);
 
     if (parser->m_errorCode == XML_ERROR_NONE) {
       switch (parser->m_parsingStatus.parsing) {
@@ -1847,7 +1757,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
          *
          * LCOV_EXCL_START
          */
-        XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+        XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                          parser->m_bufferPtr, &parser->m_position);
         parser->m_positionPtr = parser->m_bufferPtr;
         return XML_STATUS_SUSPENDED;
         /* LCOV_EXCL_STOP */
@@ -1870,23 +1781,23 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     enum XML_Status result;
     /* Detect overflow (a+b > MAX <==> b > MAX-a) */
     if (len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) {
-       parser->m_errorCode = XML_ERROR_NO_MEMORY;
-       parser->m_eventPtr = parser->m_eventEndPtr = NULL;
-       parser->m_processor = errorProcessor;
-       return XML_STATUS_ERROR;
+      parser->m_errorCode = XML_ERROR_NO_MEMORY;
+      parser->m_eventPtr = parser->m_eventEndPtr = NULL;
+      parser->m_processor = errorProcessor;
+      return XML_STATUS_ERROR;
     }
     parser->m_parseEndByteIndex += len;
     parser->m_positionPtr = s;
     parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
 
-    parser->m_errorCode = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end);
+    parser->m_errorCode
+        = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end);
 
     if (parser->m_errorCode != XML_ERROR_NONE) {
       parser->m_eventEndPtr = parser->m_eventPtr;
       parser->m_processor = errorProcessor;
       return XML_STATUS_ERROR;
-    }
-    else {
+    } else {
       switch (parser->m_parsingStatus.parsing) {
       case XML_SUSPENDED:
         result = XML_STATUS_SUSPENDED;
@@ -1903,10 +1814,12 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
       }
     }
 
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, end, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, end,
+                      &parser->m_position);
     nLeftOver = s + len - end;
     if (nLeftOver) {
-      if (parser->m_buffer == NULL || nLeftOver > parser->m_bufferLim - parser->m_buffer) {
+      if (parser->m_buffer == NULL
+          || nLeftOver > parser->m_bufferLim - parser->m_buffer) {
         /* avoid _signed_ integer overflow */
         char *temp = NULL;
         const int bytesToAllocate = (int)((unsigned)len * 2U);
@@ -1932,7 +1845,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     parser->m_eventEndPtr = parser->m_bufferPtr;
     return result;
   }
-#endif  /* not defined XML_CONTEXT_BYTES */
+#endif /* not defined XML_CONTEXT_BYTES */
   else {
     void *buff = XML_GetBuffer(parser, len);
     if (buff == NULL)
@@ -1945,8 +1858,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
 }
 
 enum XML_Status XMLCALL
-XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
-{
+XML_ParseBuffer(XML_Parser parser, int len, int isFinal) {
   const char *start;
   enum XML_Status result = XML_STATUS_OK;
 
@@ -1960,7 +1872,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
     parser->m_errorCode = XML_ERROR_FINISHED;
     return XML_STATUS_ERROR;
   case XML_INITIALIZED:
-    if (parser->m_parentParser == NULL && !startParsing(parser)) {
+    if (parser->m_parentParser == NULL && ! startParsing(parser)) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return XML_STATUS_ERROR;
     }
@@ -1976,14 +1888,14 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
   parser->m_parseEndByteIndex += len;
   parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
 
-  parser->m_errorCode = parser->m_processor(parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr);
+  parser->m_errorCode = parser->m_processor(
+      parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr);
 
   if (parser->m_errorCode != XML_ERROR_NONE) {
     parser->m_eventEndPtr = parser->m_eventPtr;
     parser->m_processor = errorProcessor;
     return XML_STATUS_ERROR;
-  }
-  else {
+  } else {
     switch (parser->m_parsingStatus.parsing) {
     case XML_SUSPENDED:
       result = XML_STATUS_SUSPENDED;
@@ -1994,18 +1906,18 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
         parser->m_parsingStatus.parsing = XML_FINISHED;
         return result;
       }
-    default: ;  /* should not happen */
+    default:; /* should not happen */
     }
   }
 
-  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                    parser->m_bufferPtr, &parser->m_position);
   parser->m_positionPtr = parser->m_bufferPtr;
   return result;
 }
 
-void * XMLCALL
-XML_GetBuffer(XML_Parser parser, int len)
-{
+void *XMLCALL
+XML_GetBuffer(XML_Parser parser, int len) {
   if (parser == NULL)
     return NULL;
   if (len < 0) {
@@ -2019,17 +1931,17 @@ XML_GetBuffer(XML_Parser parser, int len)
   case XML_FINISHED:
     parser->m_errorCode = XML_ERROR_FINISHED;
     return NULL;
-  default: ;
+  default:;
   }
 
   if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) {
 #ifdef XML_CONTEXT_BYTES
     int keep;
-#endif  /* defined XML_CONTEXT_BYTES */
+#endif /* defined XML_CONTEXT_BYTES */
     /* Do not invoke signed arithmetic overflow: */
-    int neededSize = (int) ((unsigned)len +
-                            (unsigned)EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd,
-                                                          parser->m_bufferPtr));
+    int neededSize = (int)((unsigned)len
+                           + (unsigned)EXPAT_SAFE_PTR_DIFF(
+                               parser->m_bufferEnd, parser->m_bufferPtr));
     if (neededSize < 0) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return NULL;
@@ -2039,13 +1951,18 @@ XML_GetBuffer(XML_Parser parser, int len)
     if (keep > XML_CONTEXT_BYTES)
       keep = XML_CONTEXT_BYTES;
     neededSize += keep;
-#endif  /* defined XML_CONTEXT_BYTES */
-    if (neededSize <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) {
+#endif /* defined XML_CONTEXT_BYTES */
+    if (neededSize
+        <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) {
 #ifdef XML_CONTEXT_BYTES
       if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) {
-          int offset = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) - keep;
-        /* The buffer pointers cannot be NULL here; we have at least some bytes in the buffer */
-        memmove(parser->m_buffer, &parser->m_buffer[offset], parser->m_bufferEnd - parser->m_bufferPtr + keep);
+        int offset
+            = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)
+              - keep;
+        /* The buffer pointers cannot be NULL here; we have at least some bytes
+         * in the buffer */
+        memmove(parser->m_buffer, &parser->m_buffer[offset],
+                parser->m_bufferEnd - parser->m_bufferPtr + keep);
         parser->m_bufferEnd -= offset;
         parser->m_bufferPtr -= offset;
       }
@@ -2053,20 +1970,21 @@ XML_GetBuffer(XML_Parser parser, int len)
       if (parser->m_buffer && parser->m_bufferPtr) {
         memmove(parser->m_buffer, parser->m_bufferPtr,
                 EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr));
-        parser->m_bufferEnd = parser->m_buffer +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
+        parser->m_bufferEnd
+            = parser->m_buffer
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
         parser->m_bufferPtr = parser->m_buffer;
       }
-#endif  /* not defined XML_CONTEXT_BYTES */
-    }
-    else {
+#endif /* not defined XML_CONTEXT_BYTES */
+    } else {
       char *newBuf;
-      int bufferSize = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr);
+      int bufferSize
+          = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr);
       if (bufferSize == 0)
         bufferSize = INIT_BUFFER_SIZE;
       do {
         /* Do not invoke signed arithmetic overflow: */
-        bufferSize = (int) (2U * (unsigned) bufferSize);
+        bufferSize = (int)(2U * (unsigned)bufferSize);
       } while (bufferSize < neededSize && bufferSize > 0);
       if (bufferSize <= 0) {
         parser->m_errorCode = XML_ERROR_NO_MEMORY;
@@ -2080,18 +1998,17 @@ XML_GetBuffer(XML_Parser parser, int len)
       parser->m_bufferLim = newBuf + bufferSize;
 #ifdef XML_CONTEXT_BYTES
       if (parser->m_bufferPtr) {
-        int keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer);
-        if (keep > XML_CONTEXT_BYTES)
-          keep = XML_CONTEXT_BYTES;
         memcpy(newBuf, &parser->m_bufferPtr[-keep],
-               EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) + keep);
+               EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)
+                   + keep);
         FREE(parser, parser->m_buffer);
         parser->m_buffer = newBuf;
-        parser->m_bufferEnd = parser->m_buffer +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) + keep;
+        parser->m_bufferEnd
+            = parser->m_buffer
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)
+              + keep;
         parser->m_bufferPtr = parser->m_buffer + keep;
-      }
-      else {
+      } else {
         /* This must be a brand new buffer with no data in it yet */
         parser->m_bufferEnd = newBuf;
         parser->m_bufferPtr = parser->m_buffer = newBuf;
@@ -2101,15 +2018,15 @@ XML_GetBuffer(XML_Parser parser, int len)
         memcpy(newBuf, parser->m_bufferPtr,
                EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr));
         FREE(parser, parser->m_buffer);
-        parser->m_bufferEnd = newBuf +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
-      }
-      else {
+        parser->m_bufferEnd
+            = newBuf
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
+      } else {
         /* This must be a brand new buffer with no data in it yet */
         parser->m_bufferEnd = newBuf;
       }
       parser->m_bufferPtr = parser->m_buffer = newBuf;
-#endif  /* not defined XML_CONTEXT_BYTES */
+#endif /* not defined XML_CONTEXT_BYTES */
     }
     parser->m_eventPtr = parser->m_eventEndPtr = NULL;
     parser->m_positionPtr = NULL;
@@ -2118,8 +2035,7 @@ XML_GetBuffer(XML_Parser parser, int len)
 }
 
 enum XML_Status XMLCALL
-XML_StopParser(XML_Parser parser, XML_Bool resumable)
-{
+XML_StopParser(XML_Parser parser, XML_Bool resumable) {
   if (parser == NULL)
     return XML_STATUS_ERROR;
   switch (parser->m_parsingStatus.parsing) {
@@ -2142,16 +2058,14 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable)
       }
 #endif
       parser->m_parsingStatus.parsing = XML_SUSPENDED;
-    }
-    else
+    } else
       parser->m_parsingStatus.parsing = XML_FINISHED;
   }
   return XML_STATUS_OK;
 }
 
 enum XML_Status XMLCALL
-XML_ResumeParser(XML_Parser parser)
-{
+XML_ResumeParser(XML_Parser parser) {
   enum XML_Status result = XML_STATUS_OK;
 
   if (parser == NULL)
@@ -2162,14 +2076,14 @@ XML_ResumeParser(XML_Parser parser)
   }
   parser->m_parsingStatus.parsing = XML_PARSING;
 
-  parser->m_errorCode = parser->m_processor(parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
+  parser->m_errorCode = parser->m_processor(
+      parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
 
   if (parser->m_errorCode != XML_ERROR_NONE) {
     parser->m_eventEndPtr = parser->m_eventPtr;
     parser->m_processor = errorProcessor;
     return XML_STATUS_ERROR;
-  }
-  else {
+  } else {
     switch (parser->m_parsingStatus.parsing) {
     case XML_SUSPENDED:
       result = XML_STATUS_SUSPENDED;
@@ -2180,18 +2094,18 @@ XML_ResumeParser(XML_Parser parser)
         parser->m_parsingStatus.parsing = XML_FINISHED;
         return result;
       }
-    default: ;
+    default:;
     }
   }
 
-  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                    parser->m_bufferPtr, &parser->m_position);
   parser->m_positionPtr = parser->m_bufferPtr;
   return result;
 }
 
 void XMLCALL
-XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status)
-{
+XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) {
   if (parser == NULL)
     return;
   assert(status != NULL);
@@ -2199,26 +2113,24 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status)
 }
 
 enum XML_Error XMLCALL
-XML_GetErrorCode(XML_Parser parser)
-{
+XML_GetErrorCode(XML_Parser parser) {
   if (parser == NULL)
     return XML_ERROR_INVALID_ARGUMENT;
   return parser->m_errorCode;
 }
 
 XML_Index XMLCALL
-XML_GetCurrentByteIndex(XML_Parser parser)
-{
+XML_GetCurrentByteIndex(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   if (parser->m_eventPtr)
-    return (XML_Index)(parser->m_parseEndByteIndex - (parser->m_parseEndPtr - parser->m_eventPtr));
+    return (XML_Index)(parser->m_parseEndByteIndex
+                       - (parser->m_parseEndPtr - parser->m_eventPtr));
   return -1;
 }
 
 int XMLCALL
-XML_GetCurrentByteCount(XML_Parser parser)
-{
+XML_GetCurrentByteCount(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventEndPtr && parser->m_eventPtr)
@@ -2226,9 +2138,8 @@ XML_GetCurrentByteCount(XML_Parser parser)
   return 0;
 }
 
-const char * XMLCALL
-XML_GetInputContext(XML_Parser parser, int *offset, int *size)
-{
+const char *XMLCALL
+XML_GetInputContext(XML_Parser parser, int *offset, int *size) {
 #ifdef XML_CONTEXT_BYTES
   if (parser == NULL)
     return NULL;
@@ -2236,7 +2147,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size)
     if (offset != NULL)
       *offset = (int)(parser->m_eventPtr - parser->m_buffer);
     if (size != NULL)
-      *size   = (int)(parser->m_bufferEnd - parser->m_buffer);
+      *size = (int)(parser->m_bufferEnd - parser->m_buffer);
     return parser->m_buffer;
   }
 #else
@@ -2244,82 +2155,76 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size)
   (void)offset;
   (void)size;
 #endif /* defined XML_CONTEXT_BYTES */
-  return (char *) 0;
+  return (char *)0;
 }
 
 XML_Size XMLCALL
-XML_GetCurrentLineNumber(XML_Parser parser)
-{
+XML_GetCurrentLineNumber(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) {
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_eventPtr, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                      parser->m_eventPtr, &parser->m_position);
     parser->m_positionPtr = parser->m_eventPtr;
   }
   return parser->m_position.lineNumber + 1;
 }
 
 XML_Size XMLCALL
-XML_GetCurrentColumnNumber(XML_Parser parser)
-{
+XML_GetCurrentColumnNumber(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) {
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_eventPtr, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                      parser->m_eventPtr, &parser->m_position);
     parser->m_positionPtr = parser->m_eventPtr;
   }
   return parser->m_position.columnNumber;
 }
 
 void XMLCALL
-XML_FreeContentModel(XML_Parser parser, XML_Content *model)
-{
+XML_FreeContentModel(XML_Parser parser, XML_Content *model) {
   if (parser != NULL)
     FREE(parser, model);
 }
 
-void * XMLCALL
-XML_MemMalloc(XML_Parser parser, size_t size)
-{
+void *XMLCALL
+XML_MemMalloc(XML_Parser parser, size_t size) {
   if (parser == NULL)
     return NULL;
   return MALLOC(parser, size);
 }
 
-void * XMLCALL
-XML_MemRealloc(XML_Parser parser, void *ptr, size_t size)
-{
+void *XMLCALL
+XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) {
   if (parser == NULL)
     return NULL;
   return REALLOC(parser, ptr, size);
 }
 
 void XMLCALL
-XML_MemFree(XML_Parser parser, void *ptr)
-{
+XML_MemFree(XML_Parser parser, void *ptr) {
   if (parser != NULL)
     FREE(parser, ptr);
 }
 
 void XMLCALL
-XML_DefaultCurrent(XML_Parser parser)
-{
+XML_DefaultCurrent(XML_Parser parser) {
   if (parser == NULL)
     return;
   if (parser->m_defaultHandler) {
     if (parser->m_openInternalEntities)
-      reportDefault(parser,
-                    parser->m_internalEncoding,
+      reportDefault(parser, parser->m_internalEncoding,
                     parser->m_openInternalEntities->internalEventPtr,
                     parser->m_openInternalEntities->internalEventEndPtr);
     else
-      reportDefault(parser, parser->m_encoding, parser->m_eventPtr, parser->m_eventEndPtr);
+      reportDefault(parser, parser->m_encoding, parser->m_eventPtr,
+                    parser->m_eventEndPtr);
   }
 }
 
-const XML_LChar * XMLCALL
-XML_ErrorString(enum XML_Error code)
-{
+const XML_LChar *XMLCALL
+XML_ErrorString(enum XML_Error code) {
   switch (code) {
   case XML_ERROR_NONE:
     return NULL;
@@ -2401,21 +2306,22 @@ XML_ErrorString(enum XML_Error code)
     return XML_L("cannot suspend in external parameter entity");
   /* Added in 2.0.0. */
   case XML_ERROR_RESERVED_PREFIX_XML:
-    return XML_L("reserved prefix (xml) must not be undeclared or bound to another namespace name");
+    return XML_L(
+        "reserved prefix (xml) must not be undeclared or bound to another namespace name");
   case XML_ERROR_RESERVED_PREFIX_XMLNS:
     return XML_L("reserved prefix (xmlns) must not be declared or undeclared");
   case XML_ERROR_RESERVED_NAMESPACE_URI:
-    return XML_L("prefix must not be bound to one of the reserved namespace names");
+    return XML_L(
+        "prefix must not be bound to one of the reserved namespace names");
   /* Added in 2.2.5. */
-  case XML_ERROR_INVALID_ARGUMENT:  /* Constant added in 2.2.1, already */
+  case XML_ERROR_INVALID_ARGUMENT: /* Constant added in 2.2.1, already */
     return XML_L("invalid argument");
   }
   return NULL;
 }
 
-const XML_LChar * XMLCALL
+const XML_LChar *XMLCALL
 XML_ExpatVersion(void) {
-
   /* V1 is used to string-ize the version number. However, it would
      string-ize the actual version macro *names* unless we get them
      substituted before being passed to V1. CPP is defined to expand
@@ -2424,8 +2330,8 @@ XML_ExpatVersion(void) {
      with the correct numerals. */
   /* ### I'm assuming cpp is portable in this respect... */
 
-#define V1(a,b,c) XML_L(#a)XML_L(".")XML_L(#b)XML_L(".")XML_L(#c)
-#define V2(a,b,c) XML_L("expat_")V1(a,b,c)
+#define V1(a, b, c) XML_L(#a) XML_L(".") XML_L(#b) XML_L(".") XML_L(#c)
+#define V2(a, b, c) XML_L("expat_") V1(a, b, c)
 
   return V2(XML_MAJOR_VERSION, XML_MINOR_VERSION, XML_MICRO_VERSION);
 
@@ -2434,8 +2340,7 @@ XML_ExpatVersion(void) {
 }
 
 XML_Expat_Version XMLCALL
-XML_ExpatVersionInfo(void)
-{
+XML_ExpatVersionInfo(void) {
   XML_Expat_Version version;
 
   version.major = XML_MAJOR_VERSION;
@@ -2445,41 +2350,39 @@ XML_ExpatVersionInfo(void)
   return version;
 }
 
-const XML_Feature * XMLCALL
-XML_GetFeatureList(void)
-{
-  static const XML_Feature features[] = {
-    {XML_FEATURE_SIZEOF_XML_CHAR,  XML_L("sizeof(XML_Char)"),
-     sizeof(XML_Char)},
-    {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"),
-     sizeof(XML_LChar)},
+const XML_Feature *XMLCALL
+XML_GetFeatureList(void) {
+  static const XML_Feature features[]
+      = {{XML_FEATURE_SIZEOF_XML_CHAR, XML_L("sizeof(XML_Char)"),
+          sizeof(XML_Char)},
+         {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"),
+          sizeof(XML_LChar)},
 #ifdef XML_UNICODE
-    {XML_FEATURE_UNICODE,          XML_L("XML_UNICODE"), 0},
+         {XML_FEATURE_UNICODE, XML_L("XML_UNICODE"), 0},
 #endif
 #ifdef XML_UNICODE_WCHAR_T
-    {XML_FEATURE_UNICODE_WCHAR_T,  XML_L("XML_UNICODE_WCHAR_T"), 0},
+         {XML_FEATURE_UNICODE_WCHAR_T, XML_L("XML_UNICODE_WCHAR_T"), 0},
 #endif
 #ifdef XML_DTD
-    {XML_FEATURE_DTD,              XML_L("XML_DTD"), 0},
+         {XML_FEATURE_DTD, XML_L("XML_DTD"), 0},
 #endif
 #ifdef XML_CONTEXT_BYTES
-    {XML_FEATURE_CONTEXT_BYTES,    XML_L("XML_CONTEXT_BYTES"),
-     XML_CONTEXT_BYTES},
+         {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"),
+          XML_CONTEXT_BYTES},
 #endif
 #ifdef XML_MIN_SIZE
-    {XML_FEATURE_MIN_SIZE,         XML_L("XML_MIN_SIZE"), 0},
+         {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0},
 #endif
 #ifdef XML_NS
-    {XML_FEATURE_NS,               XML_L("XML_NS"), 0},
+         {XML_FEATURE_NS, XML_L("XML_NS"), 0},
 #endif
 #ifdef XML_LARGE_SIZE
-    {XML_FEATURE_LARGE_SIZE,       XML_L("XML_LARGE_SIZE"), 0},
+         {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0},
 #endif
 #ifdef XML_ATTR_INFO
-    {XML_FEATURE_ATTR_INFO,        XML_L("XML_ATTR_INFO"), 0},
+         {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0},
 #endif
-    {XML_FEATURE_END,              NULL, 0}
-  };
+         {XML_FEATURE_END, NULL, 0}};
 
   return features;
 }
@@ -2490,8 +2393,7 @@ XML_GetFeatureList(void)
    permanent location, since the parse buffer is about to be discarded.
 */
 static XML_Bool
-storeRawNames(XML_Parser parser)
-{
+storeRawNames(XML_Parser parser) {
   TAG *tag = parser->m_tagStack;
   while (tag) {
     int bufSize;
@@ -2521,8 +2423,8 @@ storeRawNames(XML_Parser parser)
          then update it as well, since it will always point into tag->buf
       */
       if (tag->name.localPart)
-        tag->name.localPart = (XML_Char *)temp + (tag->name.localPart -
-                                                  (XML_Char *)tag->buf);
+        tag->name.localPart
+            = (XML_Char *)temp + (tag->name.localPart - (XML_Char *)tag->buf);
       tag->buf = temp;
       tag->bufEnd = temp + bufSize;
       rawNameBuf = temp + nameLen;
@@ -2535,26 +2437,21 @@ storeRawNames(XML_Parser parser)
 }
 
 static enum XML_Error PTRCALL
-contentProcessor(XML_Parser parser,
-                 const char *start,
-                 const char *end,
-                 const char **endPtr)
-{
-  enum XML_Error result = doContent(parser, 0, parser->m_encoding, start, end,
-                                    endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+contentProcessor(XML_Parser parser, const char *start, const char *end,
+                 const char **endPtr) {
+  enum XML_Error result
+      = doContent(parser, 0, parser->m_encoding, start, end, endPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result == XML_ERROR_NONE) {
-    if (!storeRawNames(parser))
+    if (! storeRawNames(parser))
       return XML_ERROR_NO_MEMORY;
   }
   return result;
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor(XML_Parser parser,
-                            const char *start,
-                            const char *end,
-                            const char **endPtr)
-{
+externalEntityInitProcessor(XML_Parser parser, const char *start,
+                            const char *end, const char **endPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -2563,11 +2460,8 @@ externalEntityInitProcessor(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor2(XML_Parser parser,
-                             const char *start,
-                             const char *end,
-                             const char **endPtr)
-{
+externalEntityInitProcessor2(XML_Parser parser, const char *start,
+                             const char *end, const char **endPtr) {
   const char *next = start; /* XmlContentTok doesn't always set the last arg */
   int tok = XmlContentTok(parser->m_encoding, start, end, &next);
   switch (tok) {
@@ -2577,21 +2471,21 @@ externalEntityInitProcessor2(XML_Parser parser,
        doContent (by detecting XML_TOK_NONE) without processing any xml text
        declaration - causing the error XML_ERROR_MISPLACED_XML_PI in doContent.
     */
-    if (next == end && !parser->m_parsingStatus.finalBuffer) {
+    if (next == end && ! parser->m_parsingStatus.finalBuffer) {
       *endPtr = next;
       return XML_ERROR_NONE;
     }
     start = next;
     break;
   case XML_TOK_PARTIAL:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
     parser->m_eventPtr = start;
     return XML_ERROR_UNCLOSED_TOKEN;
   case XML_TOK_PARTIAL_CHAR:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
@@ -2603,11 +2497,8 @@ externalEntityInitProcessor2(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor3(XML_Parser parser,
-                             const char *start,
-                             const char *end,
-                             const char **endPtr)
-{
+externalEntityInitProcessor3(XML_Parser parser, const char *start,
+                             const char *end, const char **endPtr) {
   int tok;
   const char *next = start; /* XmlContentTok doesn't always set the last arg */
   parser->m_eventPtr = start;
@@ -2615,31 +2506,29 @@ externalEntityInitProcessor3(XML_Parser parser,
   parser->m_eventEndPtr = next;
 
   switch (tok) {
-  case XML_TOK_XML_DECL:
-    {
-      enum XML_Error result;
-      result = processXmlDecl(parser, 1, start, next);
-      if (result != XML_ERROR_NONE)
-        return result;
-      switch (parser->m_parsingStatus.parsing) {
-      case XML_SUSPENDED:
-        *endPtr = next;
-        return XML_ERROR_NONE;
-      case XML_FINISHED:
-        return XML_ERROR_ABORTED;
-      default:
-        start = next;
-      }
+  case XML_TOK_XML_DECL: {
+    enum XML_Error result;
+    result = processXmlDecl(parser, 1, start, next);
+    if (result != XML_ERROR_NONE)
+      return result;
+    switch (parser->m_parsingStatus.parsing) {
+    case XML_SUSPENDED:
+      *endPtr = next;
+      return XML_ERROR_NONE;
+    case XML_FINISHED:
+      return XML_ERROR_ABORTED;
+    default:
+      start = next;
     }
-    break;
+  } break;
   case XML_TOK_PARTIAL:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
     return XML_ERROR_UNCLOSED_TOKEN;
   case XML_TOK_PARTIAL_CHAR:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
@@ -2651,39 +2540,31 @@ externalEntityInitProcessor3(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityContentProcessor(XML_Parser parser,
-                               const char *start,
-                               const char *end,
-                               const char **endPtr)
-{
-  enum XML_Error result = doContent(parser, 1, parser->m_encoding, start, end,
-                                    endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+externalEntityContentProcessor(XML_Parser parser, const char *start,
+                               const char *end, const char **endPtr) {
+  enum XML_Error result
+      = doContent(parser, 1, parser->m_encoding, start, end, endPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result == XML_ERROR_NONE) {
-    if (!storeRawNames(parser))
+    if (! storeRawNames(parser))
       return XML_ERROR_NO_MEMORY;
   }
   return result;
 }
 
 static enum XML_Error
-doContent(XML_Parser parser,
-          int startTagLevel,
-          const ENCODING *enc,
-          const char *s,
-          const char *end,
-          const char **nextPtr,
-          XML_Bool haveMore)
-{
+doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
+          const char *s, const char *end, const char **nextPtr,
+          XML_Bool haveMore) {
   /* save one level of indirection */
-  DTD * const dtd = parser->m_dtd;
+  DTD *const dtd = parser->m_dtd;
 
   const char **eventPP;
   const char **eventEndPP;
   if (enc == parser->m_encoding) {
     eventPP = &parser->m_eventPtr;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -2703,8 +2584,7 @@ doContent(XML_Parser parser,
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, end);
       /* We are at the end of the final buffer, should we check for
          XML_SUSPENDED, XML_FINISHED?
@@ -2742,185 +2622,178 @@ doContent(XML_Parser parser,
         return XML_ERROR_NONE;
       }
       return XML_ERROR_PARTIAL_CHAR;
-    case XML_TOK_ENTITY_REF:
-      {
-        const XML_Char *name;
-        ENTITY *entity;
-        XML_Char ch = (XML_Char) XmlPredefinedEntityName(enc,
-                                              s + enc->minBytesPerChar,
-                                              next - enc->minBytesPerChar);
-        if (ch) {
-          if (parser->m_characterDataHandler)
-            parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1);
-          else if (parser->m_defaultHandler)
-            reportDefault(parser, enc, s, next);
-          break;
-        }
-        name = poolStoreString(&dtd->pool, enc,
-                                s + enc->minBytesPerChar,
-                                next - enc->minBytesPerChar);
-        if (!name)
-          return XML_ERROR_NO_MEMORY;
-        entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
-        poolDiscard(&dtd->pool);
-        /* First, determine if a check for an existing declaration is needed;
-           if yes, check that the entity exists, and that it is internal,
-           otherwise call the skipped entity or default handler.
-        */
-        if (!dtd->hasParamEntityRefs || dtd->standalone) {
-          if (!entity)
-            return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal)
-            return XML_ERROR_ENTITY_DECLARED_IN_PE;
-        }
-        else if (!entity) {
+    case XML_TOK_ENTITY_REF: {
+      const XML_Char *name;
+      ENTITY *entity;
+      XML_Char ch = (XML_Char)XmlPredefinedEntityName(
+          enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar);
+      if (ch) {
+        if (parser->m_characterDataHandler)
+          parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1);
+        else if (parser->m_defaultHandler)
+          reportDefault(parser, enc, s, next);
+        break;
+      }
+      name = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                             next - enc->minBytesPerChar);
+      if (! name)
+        return XML_ERROR_NO_MEMORY;
+      entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
+      poolDiscard(&dtd->pool);
+      /* First, determine if a check for an existing declaration is needed;
+         if yes, check that the entity exists, and that it is internal,
+         otherwise call the skipped entity or default handler.
+      */
+      if (! dtd->hasParamEntityRefs || dtd->standalone) {
+        if (! entity)
+          return XML_ERROR_UNDEFINED_ENTITY;
+        else if (! entity->is_internal)
+          return XML_ERROR_ENTITY_DECLARED_IN_PE;
+      } else if (! entity) {
+        if (parser->m_skippedEntityHandler)
+          parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+        else if (parser->m_defaultHandler)
+          reportDefault(parser, enc, s, next);
+        break;
+      }
+      if (entity->open)
+        return XML_ERROR_RECURSIVE_ENTITY_REF;
+      if (entity->notation)
+        return XML_ERROR_BINARY_ENTITY_REF;
+      if (entity->textPtr) {
+        enum XML_Error result;
+        if (! parser->m_defaultExpandInternalEntities) {
           if (parser->m_skippedEntityHandler)
-            parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+            parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name,
+                                           0);
           else if (parser->m_defaultHandler)
             reportDefault(parser, enc, s, next);
           break;
         }
-        if (entity->open)
-          return XML_ERROR_RECURSIVE_ENTITY_REF;
-        if (entity->notation)
-          return XML_ERROR_BINARY_ENTITY_REF;
-        if (entity->textPtr) {
-          enum XML_Error result;
-          if (!parser->m_defaultExpandInternalEntities) {
-            if (parser->m_skippedEntityHandler)
-              parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name, 0);
-            else if (parser->m_defaultHandler)
-              reportDefault(parser, enc, s, next);
-            break;
-          }
-          result = processInternalEntity(parser, entity, XML_FALSE);
-          if (result != XML_ERROR_NONE)
-            return result;
-        }
-        else if (parser->m_externalEntityRefHandler) {
-          const XML_Char *context;
-          entity->open = XML_TRUE;
-          context = getContext(parser);
-          entity->open = XML_FALSE;
-          if (!context)
-            return XML_ERROR_NO_MEMORY;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        context,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
-            return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
-          poolDiscard(&parser->m_tempPool);
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        break;
-      }
+        result = processInternalEntity(parser, entity, XML_FALSE);
+        if (result != XML_ERROR_NONE)
+          return result;
+      } else if (parser->m_externalEntityRefHandler) {
+        const XML_Char *context;
+        entity->open = XML_TRUE;
+        context = getContext(parser);
+        entity->open = XML_FALSE;
+        if (! context)
+          return XML_ERROR_NO_MEMORY;
+        if (! parser->m_externalEntityRefHandler(
+                parser->m_externalEntityRefHandlerArg, context, entity->base,
+                entity->systemId, entity->publicId))
+          return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
+        poolDiscard(&parser->m_tempPool);
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      break;
+    }
     case XML_TOK_START_TAG_NO_ATTS:
       /* fall through */
-    case XML_TOK_START_TAG_WITH_ATTS:
-      {
-        TAG *tag;
-        enum XML_Error result;
-        XML_Char *toPtr;
-        if (parser->m_freeTagList) {
-          tag = parser->m_freeTagList;
-          parser->m_freeTagList = parser->m_freeTagList->parent;
+    case XML_TOK_START_TAG_WITH_ATTS: {
+      TAG *tag;
+      enum XML_Error result;
+      XML_Char *toPtr;
+      if (parser->m_freeTagList) {
+        tag = parser->m_freeTagList;
+        parser->m_freeTagList = parser->m_freeTagList->parent;
+      } else {
+        tag = (TAG *)MALLOC(parser, sizeof(TAG));
+        if (! tag)
+          return XML_ERROR_NO_MEMORY;
+        tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE);
+        if (! tag->buf) {
+          FREE(parser, tag);
+          return XML_ERROR_NO_MEMORY;
         }
-        else {
-          tag = (TAG *)MALLOC(parser, sizeof(TAG));
-          if (!tag)
-            return XML_ERROR_NO_MEMORY;
-          tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE);
-          if (!tag->buf) {
-            FREE(parser, tag);
-            return XML_ERROR_NO_MEMORY;
+        tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE;
+      }
+      tag->bindings = NULL;
+      tag->parent = parser->m_tagStack;
+      parser->m_tagStack = tag;
+      tag->name.localPart = NULL;
+      tag->name.prefix = NULL;
+      tag->rawName = s + enc->minBytesPerChar;
+      tag->rawNameLength = XmlNameLength(enc, tag->rawName);
+      ++parser->m_tagLevel;
+      {
+        const char *rawNameEnd = tag->rawName + tag->rawNameLength;
+        const char *fromPtr = tag->rawName;
+        toPtr = (XML_Char *)tag->buf;
+        for (;;) {
+          int bufSize;
+          int convLen;
+          const enum XML_Convert_Result convert_res
+              = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr,
+                           (ICHAR *)tag->bufEnd - 1);
+          convLen = (int)(toPtr - (XML_Char *)tag->buf);
+          if ((fromPtr >= rawNameEnd)
+              || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) {
+            tag->name.strLen = convLen;
+            break;
           }
-          tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE;
-        }
-        tag->bindings = NULL;
-        tag->parent = parser->m_tagStack;
-        parser->m_tagStack = tag;
-        tag->name.localPart = NULL;
-        tag->name.prefix = NULL;
-        tag->rawName = s + enc->minBytesPerChar;
-        tag->rawNameLength = XmlNameLength(enc, tag->rawName);
-        ++parser->m_tagLevel;
-        {
-          const char *rawNameEnd = tag->rawName + tag->rawNameLength;
-          const char *fromPtr = tag->rawName;
-          toPtr = (XML_Char *)tag->buf;
-          for (;;) {
-            int bufSize;
-            int convLen;
-            const enum XML_Convert_Result convert_res = XmlConvert(enc,
-                       &fromPtr, rawNameEnd,
-                       (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1);
-            convLen = (int)(toPtr - (XML_Char *)tag->buf);
-            if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) {
-              tag->name.strLen = convLen;
-              break;
-            }
-            bufSize = (int)(tag->bufEnd - tag->buf) << 1;
-            {
-              char *temp = (char *)REALLOC(parser, tag->buf, bufSize);
-              if (temp == NULL)
-                return XML_ERROR_NO_MEMORY;
-              tag->buf = temp;
-              tag->bufEnd = temp + bufSize;
-              toPtr = (XML_Char *)temp + convLen;
-            }
+          bufSize = (int)(tag->bufEnd - tag->buf) << 1;
+          {
+            char *temp = (char *)REALLOC(parser, tag->buf, bufSize);
+            if (temp == NULL)
+              return XML_ERROR_NO_MEMORY;
+            tag->buf = temp;
+            tag->bufEnd = temp + bufSize;
+            toPtr = (XML_Char *)temp + convLen;
           }
         }
-        tag->name.str = (XML_Char *)tag->buf;
-        *toPtr = XML_T('\0');
-        result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings));
-        if (result)
-          return result;
-        if (parser->m_startElementHandler)
-          parser->m_startElementHandler(parser->m_handlerArg, tag->name.str,
-                              (const XML_Char **)parser->m_atts);
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        poolClear(&parser->m_tempPool);
-        break;
       }
+      tag->name.str = (XML_Char *)tag->buf;
+      *toPtr = XML_T('\0');
+      result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings));
+      if (result)
+        return result;
+      if (parser->m_startElementHandler)
+        parser->m_startElementHandler(parser->m_handlerArg, tag->name.str,
+                                      (const XML_Char **)parser->m_atts);
+      else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      poolClear(&parser->m_tempPool);
+      break;
+    }
     case XML_TOK_EMPTY_ELEMENT_NO_ATTS:
       /* fall through */
-    case XML_TOK_EMPTY_ELEMENT_WITH_ATTS:
-      {
-        const char *rawName = s + enc->minBytesPerChar;
-        enum XML_Error result;
-        BINDING *bindings = NULL;
-        XML_Bool noElmHandlers = XML_TRUE;
-        TAG_NAME name;
-        name.str = poolStoreString(&parser->m_tempPool, enc, rawName,
-                                   rawName + XmlNameLength(enc, rawName));
-        if (!name.str)
-          return XML_ERROR_NO_MEMORY;
-        poolFinish(&parser->m_tempPool);
-        result = storeAtts(parser, enc, s, &name, &bindings);
-        if (result != XML_ERROR_NONE) {
-          freeBindings(parser, bindings);
-          return result;
-        }
-        poolFinish(&parser->m_tempPool);
-        if (parser->m_startElementHandler) {
-          parser->m_startElementHandler(parser->m_handlerArg, name.str, (const XML_Char **)parser->m_atts);
-          noElmHandlers = XML_FALSE;
-        }
-        if (parser->m_endElementHandler) {
-          if (parser->m_startElementHandler)
-            *eventPP = *eventEndPP;
-          parser->m_endElementHandler(parser->m_handlerArg, name.str);
-          noElmHandlers = XML_FALSE;
-        }
-        if (noElmHandlers && parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        poolClear(&parser->m_tempPool);
+    case XML_TOK_EMPTY_ELEMENT_WITH_ATTS: {
+      const char *rawName = s + enc->minBytesPerChar;
+      enum XML_Error result;
+      BINDING *bindings = NULL;
+      XML_Bool noElmHandlers = XML_TRUE;
+      TAG_NAME name;
+      name.str = poolStoreString(&parser->m_tempPool, enc, rawName,
+                                 rawName + XmlNameLength(enc, rawName));
+      if (! name.str)
+        return XML_ERROR_NO_MEMORY;
+      poolFinish(&parser->m_tempPool);
+      result = storeAtts(parser, enc, s, &name, &bindings);
+      if (result != XML_ERROR_NONE) {
         freeBindings(parser, bindings);
+        return result;
+      }
+      poolFinish(&parser->m_tempPool);
+      if (parser->m_startElementHandler) {
+        parser->m_startElementHandler(parser->m_handlerArg, name.str,
+                                      (const XML_Char **)parser->m_atts);
+        noElmHandlers = XML_FALSE;
+      }
+      if (parser->m_endElementHandler) {
+        if (parser->m_startElementHandler)
+          *eventPP = *eventEndPP;
+        parser->m_endElementHandler(parser->m_handlerArg, name.str);
+        noElmHandlers = XML_FALSE;
       }
-      if ((parser->m_tagLevel == 0) && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
+      if (noElmHandlers && parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      poolClear(&parser->m_tempPool);
+      freeBindings(parser, bindings);
+    }
+      if ((parser->m_tagLevel == 0)
+          && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
         if (parser->m_parsingStatus.parsing == XML_SUSPENDED)
           parser->m_processor = epilogProcessor;
         else
@@ -2937,7 +2810,7 @@ doContent(XML_Parser parser,
         parser->m_tagStack = tag->parent;
         tag->parent = parser->m_freeTagList;
         parser->m_freeTagList = tag;
-        rawName = s + enc->minBytesPerChar*2;
+        rawName = s + enc->minBytesPerChar * 2;
         len = XmlNameLength(enc, rawName);
         if (len != tag->rawNameLength
             || memcmp(tag->rawName, rawName, len) != 0) {
@@ -2957,86 +2830,89 @@ doContent(XML_Parser parser,
             */
             uri = (XML_Char *)tag->name.str + tag->name.uriLen;
             /* don't need to check for space - already done in storeAtts() */
-            while (*localPart) *uri++ = *localPart++;
+            while (*localPart)
+              *uri++ = *localPart++;
             prefix = (XML_Char *)tag->name.prefix;
             if (parser->m_ns_triplets && prefix) {
               *uri++ = parser->m_namespaceSeparator;
-              while (*prefix) *uri++ = *prefix++;
-             }
+              while (*prefix)
+                *uri++ = *prefix++;
+            }
             *uri = XML_T('\0');
           }
           parser->m_endElementHandler(parser->m_handlerArg, tag->name.str);
-        }
-        else if (parser->m_defaultHandler)
+        } else if (parser->m_defaultHandler)
           reportDefault(parser, enc, s, next);
         while (tag->bindings) {
           BINDING *b = tag->bindings;
           if (parser->m_endNamespaceDeclHandler)
-            parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
+            parser->m_endNamespaceDeclHandler(parser->m_handlerArg,
+                                              b->prefix->name);
           tag->bindings = tag->bindings->nextTagBinding;
           b->nextTagBinding = parser->m_freeBindingList;
           parser->m_freeBindingList = b;
           b->prefix->binding = b->prevPrefixBinding;
         }
-        if (parser->m_tagLevel == 0)
-          return epilogProcessor(parser, next, end, nextPtr);
-      }
-      break;
-    case XML_TOK_CHAR_REF:
-      {
-        int n = XmlCharRefNumber(enc, s);
-        if (n < 0)
-          return XML_ERROR_BAD_CHAR_REF;
-        if (parser->m_characterDataHandler) {
-          XML_Char buf[XML_ENCODE_MAX];
-          parser->m_characterDataHandler(parser->m_handlerArg, buf, XmlEncode(n, (ICHAR *)buf));
+        if ((parser->m_tagLevel == 0)
+            && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
+          if (parser->m_parsingStatus.parsing == XML_SUSPENDED)
+            parser->m_processor = epilogProcessor;
+          else
+            return epilogProcessor(parser, next, end, nextPtr);
         }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
       }
       break;
+    case XML_TOK_CHAR_REF: {
+      int n = XmlCharRefNumber(enc, s);
+      if (n < 0)
+        return XML_ERROR_BAD_CHAR_REF;
+      if (parser->m_characterDataHandler) {
+        XML_Char buf[XML_ENCODE_MAX];
+        parser->m_characterDataHandler(parser->m_handlerArg, buf,
+                                       XmlEncode(n, (ICHAR *)buf));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_XML_DECL:
       return XML_ERROR_MISPLACED_XML_PI;
     case XML_TOK_DATA_NEWLINE:
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       break;
-    case XML_TOK_CDATA_SECT_OPEN:
-      {
-        enum XML_Error result;
-        if (parser->m_startCdataSectionHandler)
-          parser->m_startCdataSectionHandler(parser->m_handlerArg);
-/* BEGIN disabled code */
-        /* Suppose you doing a transformation on a document that involves
-           changing only the character data.  You set up a defaultHandler
-           and a characterDataHandler.  The defaultHandler simply copies
-           characters through.  The characterDataHandler does the
-           transformation and writes the characters out escaping them as
-           necessary.  This case will fail to work if we leave out the
-           following two lines (because & and < inside CDATA sections will
-           be incorrectly escaped).
-
-           However, now we have a start/endCdataSectionHandler, so it seems
-           easier to let the user deal with this.
-        */
-        else if (0 && parser->m_characterDataHandler)
-          parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0);
-/* END disabled code */
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore);
-        if (result != XML_ERROR_NONE)
-          return result;
-        else if (!next) {
-          parser->m_processor = cdataSectionProcessor;
-          return result;
-        }
+    case XML_TOK_CDATA_SECT_OPEN: {
+      enum XML_Error result;
+      if (parser->m_startCdataSectionHandler)
+        parser->m_startCdataSectionHandler(parser->m_handlerArg);
+      /* BEGIN disabled code */
+      /* Suppose you doing a transformation on a document that involves
+         changing only the character data.  You set up a defaultHandler
+         and a characterDataHandler.  The defaultHandler simply copies
+         characters through.  The characterDataHandler does the
+         transformation and writes the characters out escaping them as
+         necessary.  This case will fail to work if we leave out the
+         following two lines (because & and < inside CDATA sections will
+         be incorrectly escaped).
+
+         However, now we have a start/endCdataSectionHandler, so it seems
+         easier to let the user deal with this.
+      */
+      else if (0 && parser->m_characterDataHandler)
+        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                                       0);
+      /* END disabled code */
+      else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore);
+      if (result != XML_ERROR_NONE)
+        return result;
+      else if (! next) {
+        parser->m_processor = cdataSectionProcessor;
+        return result;
       }
-      break;
+    } break;
     case XML_TOK_TRAILING_RSQB:
       if (haveMore) {
         *nextPtr = s;
@@ -3046,15 +2922,14 @@ doContent(XML_Parser parser,
         if (MUST_CONVERT(enc, s)) {
           ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
           XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-          parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                               (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-        }
-        else
-          parser->m_characterDataHandler(parser->m_handlerArg,
-                               (XML_Char *)s,
-                               (int)((XML_Char *)end - (XML_Char *)s));
-      }
-      else if (parser->m_defaultHandler)
+          parser->m_characterDataHandler(
+              parser->m_handlerArg, parser->m_dataBuf,
+              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+        } else
+          parser->m_characterDataHandler(
+              parser->m_handlerArg, (XML_Char *)s,
+              (int)((XML_Char *)end - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, end);
       /* We are at the end of the final buffer, should we check for
          XML_SUSPENDED, XML_FINISHED?
@@ -3069,37 +2944,34 @@ doContent(XML_Parser parser,
       }
       *nextPtr = end;
       return XML_ERROR_NONE;
-    case XML_TOK_DATA_CHARS:
-      {
-        XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
-        if (charDataHandler) {
-          if (MUST_CONVERT(enc, s)) {
-            for (;;) {
-              ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-              const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-              *eventEndPP = s;
-              charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-              if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
-                break;
-              *eventPP = s;
-            }
+    case XML_TOK_DATA_CHARS: {
+      XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
+      if (charDataHandler) {
+        if (MUST_CONVERT(enc, s)) {
+          for (;;) {
+            ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
+            const enum XML_Convert_Result convert_res = XmlConvert(
+                enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+            *eventEndPP = s;
+            charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                            (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+            if ((convert_res == XML_CONVERT_COMPLETED)
+                || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+              break;
+            *eventPP = s;
           }
-          else
-            charDataHandler(parser->m_handlerArg,
-                            (XML_Char *)s,
-                            (int)((XML_Char *)next - (XML_Char *)s));
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-      }
-      break;
+        } else
+          charDataHandler(parser->m_handlerArg, (XML_Char *)s,
+                          (int)((XML_Char *)next - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_PI:
-      if (!reportProcessingInstruction(parser, enc, s, next))
+      if (! reportProcessingInstruction(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_COMMENT:
-      if (!reportComment(parser, enc, s, next))
+      if (! reportComment(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     default:
@@ -3122,7 +2994,7 @@ doContent(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
   /* not reached */
@@ -3133,8 +3005,7 @@ doContent(XML_Parser parser,
  * reused as appropriate.
  */
 static void
-freeBindings(XML_Parser parser, BINDING *bindings)
-{
+freeBindings(XML_Parser parser, BINDING *bindings) {
   while (bindings) {
     BINDING *b = bindings;
 
@@ -3142,7 +3013,7 @@ freeBindings(XML_Parser parser, BINDING *bindings)
      * binding in addBindings(), so call the end handler now.
      */
     if (parser->m_endNamespaceDeclHandler)
-        parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
+      parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
 
     bindings = bindings->nextTagBinding;
     b->nextTagBinding = parser->m_freeBindingList;
@@ -3162,14 +3033,12 @@ freeBindings(XML_Parser parser, BINDING *bindings)
    - generate namespace aware element name (URI, prefix)
 */
 static enum XML_Error
-storeAtts(XML_Parser parser, const ENCODING *enc,
-          const char *attStr, TAG_NAME *tagNamePtr,
-          BINDING **bindingsPtr)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
+          TAG_NAME *tagNamePtr, BINDING **bindingsPtr) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   ELEMENT_TYPE *elementType;
   int nDefaultAtts;
-  const XML_Char **appAtts;   /* the attribute list for the application */
+  const XML_Char **appAtts; /* the attribute list for the application */
   int attIndex = 0;
   int prefixLen;
   int i;
@@ -3180,16 +3049,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   const XML_Char *localPart;
 
   /* lookup the element type name */
-  elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str,0);
-  if (!elementType) {
+  elementType
+      = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str, 0);
+  if (! elementType) {
     const XML_Char *name = poolCopyString(&dtd->pool, tagNamePtr->str);
-    if (!name)
+    if (! name)
       return XML_ERROR_NO_MEMORY;
     elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, name,
                                          sizeof(ELEMENT_TYPE));
-    if (!elementType)
+    if (! elementType)
       return XML_ERROR_NO_MEMORY;
-    if (parser->m_ns && !setElementTypePrefix(parser, elementType))
+    if (parser->m_ns && ! setElementTypePrefix(parser, elementType))
       return XML_ERROR_NO_MEMORY;
   }
   nDefaultAtts = elementType->nDefaultAtts;
@@ -3203,14 +3073,16 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     XML_AttrInfo *temp2;
 #endif
     parser->m_attsSize = n + nDefaultAtts + INIT_ATTS_SIZE;
-    temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts, parser->m_attsSize * sizeof(ATTRIBUTE));
+    temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts,
+                                parser->m_attsSize * sizeof(ATTRIBUTE));
     if (temp == NULL) {
       parser->m_attsSize = oldAttsSize;
       return XML_ERROR_NO_MEMORY;
     }
     parser->m_atts = temp;
 #ifdef XML_ATTR_INFO
-    temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo, parser->m_attsSize * sizeof(XML_AttrInfo));
+    temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo,
+                                    parser->m_attsSize * sizeof(XML_AttrInfo));
     if (temp2 == NULL) {
       parser->m_attsSize = oldAttsSize;
       return XML_ERROR_NO_MEMORY;
@@ -3228,18 +3100,20 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     XML_AttrInfo *currAttInfo = &parser->m_attInfo[i];
 #endif
     /* add the name and value to the attribute list */
-    ATTRIBUTE_ID *attId = getAttributeId(parser, enc, currAtt->name,
-                                         currAtt->name
-                                         + XmlNameLength(enc, currAtt->name));
-    if (!attId)
+    ATTRIBUTE_ID *attId
+        = getAttributeId(parser, enc, currAtt->name,
+                         currAtt->name + XmlNameLength(enc, currAtt->name));
+    if (! attId)
       return XML_ERROR_NO_MEMORY;
 #ifdef XML_ATTR_INFO
-    currAttInfo->nameStart = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name);
-    currAttInfo->nameEnd = currAttInfo->nameStart +
-                           XmlNameLength(enc, currAtt->name);
-    currAttInfo->valueStart = parser->m_parseEndByteIndex -
-                            (parser->m_parseEndPtr - currAtt->valuePtr);
-    currAttInfo->valueEnd = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->valueEnd);
+    currAttInfo->nameStart
+        = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name);
+    currAttInfo->nameEnd
+        = currAttInfo->nameStart + XmlNameLength(enc, currAtt->name);
+    currAttInfo->valueStart = parser->m_parseEndByteIndex
+                              - (parser->m_parseEndPtr - currAtt->valuePtr);
+    currAttInfo->valueEnd = parser->m_parseEndByteIndex
+                            - (parser->m_parseEndPtr - currAtt->valueEnd);
 #endif
     /* Detect duplicate attributes by their QNames. This does not work when
        namespace processing is turned on and different prefixes for the same
@@ -3252,7 +3126,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     }
     (attId->name)[-1] = 1;
     appAtts[attIndex++] = attId->name;
-    if (!parser->m_atts[i].normalized) {
+    if (! parser->m_atts[i].normalized) {
       enum XML_Error result;
       XML_Bool isCdata = XML_TRUE;
 
@@ -3268,17 +3142,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
       }
 
       /* normalize the attribute value */
-      result = storeAttributeValue(parser, enc, isCdata,
-                                   parser->m_atts[i].valuePtr, parser->m_atts[i].valueEnd,
-                                   &parser->m_tempPool);
+      result = storeAttributeValue(
+          parser, enc, isCdata, parser->m_atts[i].valuePtr,
+          parser->m_atts[i].valueEnd, &parser->m_tempPool);
       if (result)
         return result;
       appAtts[attIndex] = poolStart(&parser->m_tempPool);
       poolFinish(&parser->m_tempPool);
-    }
-    else {
+    } else {
       /* the value did not need normalizing */
-      appAtts[attIndex] = poolStoreString(&parser->m_tempPool, enc, parser->m_atts[i].valuePtr,
+      appAtts[attIndex] = poolStoreString(&parser->m_tempPool, enc,
+                                          parser->m_atts[i].valuePtr,
                                           parser->m_atts[i].valueEnd);
       if (appAtts[attIndex] == 0)
         return XML_ERROR_NO_MEMORY;
@@ -3293,15 +3167,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         if (result)
           return result;
         --attIndex;
-      }
-      else {
+      } else {
         /* deal with other prefixed names later */
         attIndex++;
         nPrefixes++;
         (attId->name)[-1] = 2;
       }
-    }
-    else
+    } else
       attIndex++;
   }
 
@@ -3313,29 +3185,26 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         parser->m_idAttIndex = i;
         break;
       }
-  }
-  else
+  } else
     parser->m_idAttIndex = -1;
 
   /* do attribute defaulting */
   for (i = 0; i < nDefaultAtts; i++) {
     const DEFAULT_ATTRIBUTE *da = elementType->defaultAtts + i;
-    if (!(da->id->name)[-1] && da->value) {
+    if (! (da->id->name)[-1] && da->value) {
       if (da->id->prefix) {
         if (da->id->xmlns) {
           enum XML_Error result = addBinding(parser, da->id->prefix, da->id,
                                              da->value, bindingsPtr);
           if (result)
             return result;
-        }
-        else {
+        } else {
           (da->id->name)[-1] = 2;
           nPrefixes++;
           appAtts[attIndex++] = da->id->name;
           appAtts[attIndex++] = da->value;
         }
-      }
-      else {
+      } else {
         (da->id->name)[-1] = 1;
         appAtts[attIndex++] = da->id->name;
         appAtts[attIndex++] = da->value;
@@ -3348,31 +3217,34 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
      and clear flags that say whether attributes were specified */
   i = 0;
   if (nPrefixes) {
-    int j;  /* hash table index */
+    int j; /* hash table index */
     unsigned long version = parser->m_nsAttsVersion;
     int nsAttsSize = (int)1 << parser->m_nsAttsPower;
     unsigned char oldNsAttsPower = parser->m_nsAttsPower;
     /* size of hash table must be at least 2 * (# of prefixed attributes) */
-    if ((nPrefixes << 1) >> parser->m_nsAttsPower) {  /* true for m_nsAttsPower = 0 */
+    if ((nPrefixes << 1)
+        >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */
       NS_ATT *temp;
       /* hash table size must also be a power of 2 and >= 8 */
-      while (nPrefixes >> parser->m_nsAttsPower++);
+      while (nPrefixes >> parser->m_nsAttsPower++)
+        ;
       if (parser->m_nsAttsPower < 3)
         parser->m_nsAttsPower = 3;
       nsAttsSize = (int)1 << parser->m_nsAttsPower;
-      temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT));
-      if (!temp) {
+      temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts,
+                               nsAttsSize * sizeof(NS_ATT));
+      if (! temp) {
         /* Restore actual size of memory in m_nsAtts */
         parser->m_nsAttsPower = oldNsAttsPower;
         return XML_ERROR_NO_MEMORY;
       }
       parser->m_nsAtts = temp;
-      version = 0;  /* force re-initialization of m_nsAtts hash table */
+      version = 0; /* force re-initialization of m_nsAtts hash table */
     }
     /* using a version flag saves us from initializing m_nsAtts every time */
-    if (!version) {  /* initialize version flags when version wraps around */
+    if (! version) { /* initialize version flags when version wraps around */
       version = INIT_ATTS_VERSION;
-      for (j = nsAttsSize; j != 0; )
+      for (j = nsAttsSize; j != 0;)
         parser->m_nsAtts[--j].version = version;
     }
     parser->m_nsAttsVersion = --version;
@@ -3380,7 +3252,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     /* expand prefixed names and check for duplicates */
     for (; i < attIndex; i += 2) {
       const XML_Char *s = appAtts[i];
-      if (s[-1] == 2) {  /* prefixed */
+      if (s[-1] == 2) { /* prefixed */
         ATTRIBUTE_ID *id;
         const BINDING *b;
         unsigned long uriHash;
@@ -3390,9 +3262,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         copy_salt_to_sipkey(parser, &sip_key);
         sip24_init(&sip_state, &sip_key);
 
-        ((XML_Char *)s)[-1] = 0;  /* clear flag */
+        ((XML_Char *)s)[-1] = 0; /* clear flag */
         id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0);
-        if (!id || !id->prefix) {
+        if (! id || ! id->prefix) {
           /* This code is walking through the appAtts array, dealing
            * with (in this case) a prefixed attribute name.  To be in
            * the array, the attribute must have already been bound, so
@@ -3410,12 +3282,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
           return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */
         }
         b = id->prefix->binding;
-        if (!b)
+        if (! b)
           return XML_ERROR_UNBOUND_PREFIX;
 
         for (j = 0; j < b->uriLen; j++) {
           const XML_Char c = b->uri[j];
-          if (!poolAppendChar(&parser->m_tempPool, c))
+          if (! poolAppendChar(&parser->m_tempPool, c))
             return XML_ERROR_NO_MEMORY;
         }
 
@@ -3426,8 +3298,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
 
         sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char));
 
-        do {  /* copies null terminator */
-          if (!poolAppendChar(&parser->m_tempPool, *s))
+        do { /* copies null terminator */
+          if (! poolAppendChar(&parser->m_tempPool, *s))
             return XML_ERROR_NO_MEMORY;
         } while (*s++);
 
@@ -3438,28 +3310,29 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
           */
           unsigned char step = 0;
           unsigned long mask = nsAttsSize - 1;
-          j = uriHash & mask;  /* index into hash table */
+          j = uriHash & mask; /* index into hash table */
           while (parser->m_nsAtts[j].version == version) {
             /* for speed we compare stored hash values first */
             if (uriHash == parser->m_nsAtts[j].hash) {
               const XML_Char *s1 = poolStart(&parser->m_tempPool);
               const XML_Char *s2 = parser->m_nsAtts[j].uriName;
               /* s1 is null terminated, but not s2 */
-              for (; *s1 == *s2 && *s1 != 0; s1++, s2++);
+              for (; *s1 == *s2 && *s1 != 0; s1++, s2++)
+                ;
               if (*s1 == 0)
                 return XML_ERROR_DUPLICATE_ATTRIBUTE;
             }
-            if (!step)
+            if (! step)
               step = PROBE_STEP(uriHash, mask, parser->m_nsAttsPower);
             j < step ? (j += nsAttsSize - step) : (j -= step);
           }
         }
 
-        if (parser->m_ns_triplets) {  /* append namespace separator and prefix */
+        if (parser->m_ns_triplets) { /* append namespace separator and prefix */
           parser->m_tempPool.ptr[-1] = parser->m_namespaceSeparator;
           s = b->prefix->name;
           do {
-            if (!poolAppendChar(&parser->m_tempPool, *s))
+            if (! poolAppendChar(&parser->m_tempPool, *s))
               return XML_ERROR_NO_MEMORY;
           } while (*s++);
         }
@@ -3474,13 +3347,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         parser->m_nsAtts[j].hash = uriHash;
         parser->m_nsAtts[j].uriName = s;
 
-        if (!--nPrefixes) {
+        if (! --nPrefixes) {
           i += 2;
           break;
         }
-      }
-      else  /* not prefixed */
-        ((XML_Char *)s)[-1] = 0;  /* clear flag */
+      } else                     /* not prefixed */
+        ((XML_Char *)s)[-1] = 0; /* clear flag */
     }
   }
   /* clear flags for the remaining attributes */
@@ -3489,40 +3361,38 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   for (binding = *bindingsPtr; binding; binding = binding->nextTagBinding)
     binding->attId->name[-1] = 0;
 
-  if (!parser->m_ns)
+  if (! parser->m_ns)
     return XML_ERROR_NONE;
 
   /* expand the element type name */
   if (elementType->prefix) {
     binding = elementType->prefix->binding;
-    if (!binding)
+    if (! binding)
       return XML_ERROR_UNBOUND_PREFIX;
     localPart = tagNamePtr->str;
     while (*localPart++ != XML_T(ASCII_COLON))
       ;
-  }
-  else if (dtd->defaultPrefix.binding) {
+  } else if (dtd->defaultPrefix.binding) {
     binding = dtd->defaultPrefix.binding;
     localPart = tagNamePtr->str;
-  }
-  else
+  } else
     return XML_ERROR_NONE;
   prefixLen = 0;
   if (parser->m_ns_triplets && binding->prefix->name) {
     for (; binding->prefix->name[prefixLen++];)
-      ;  /* prefixLen includes null terminator */
+      ; /* prefixLen includes null terminator */
   }
   tagNamePtr->localPart = localPart;
   tagNamePtr->uriLen = binding->uriLen;
   tagNamePtr->prefix = binding->prefix->name;
   tagNamePtr->prefixLen = prefixLen;
   for (i = 0; localPart[i++];)
-    ;  /* i includes null terminator */
+    ; /* i includes null terminator */
   n = i + binding->uriLen + prefixLen;
   if (n > binding->uriAlloc) {
     TAG *p;
     uri = (XML_Char *)MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char));
-    if (!uri)
+    if (! uri)
       return XML_ERROR_NO_MEMORY;
     binding->uriAlloc = n + EXPAND_SPARE;
     memcpy(uri, binding->uri, binding->uriLen * sizeof(XML_Char));
@@ -3538,7 +3408,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   /* we always have a namespace separator between localPart and prefix */
   if (prefixLen) {
     uri += i - 1;
-    *uri = parser->m_namespaceSeparator;  /* replace null terminator */
+    *uri = parser->m_namespaceSeparator; /* replace null terminator */
     memcpy(uri + 1, binding->prefix->name, prefixLen * sizeof(XML_Char));
   }
   tagNamePtr->str = binding->uri;
@@ -3550,27 +3420,25 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
 */
 static enum XML_Error
 addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
-           const XML_Char *uri, BINDING **bindingsPtr)
-{
-  static const XML_Char xmlNamespace[] = {
-    ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, ASCII_SLASH,
-    ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD,
-    ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L,
-    ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8, ASCII_SLASH,
-    ASCII_n, ASCII_a, ASCII_m, ASCII_e, ASCII_s, ASCII_p, ASCII_a, ASCII_c,
-    ASCII_e, '\0'
-  };
-  static const int xmlLen =
-    (int)sizeof(xmlNamespace)/sizeof(XML_Char) - 1;
-  static const XML_Char xmlnsNamespace[] = {
-    ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, ASCII_SLASH,
-    ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD,
-    ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_2, ASCII_0, ASCII_0,
-    ASCII_0, ASCII_SLASH, ASCII_x, ASCII_m, ASCII_l, ASCII_n, ASCII_s,
-    ASCII_SLASH, '\0'
-  };
-  static const int xmlnsLen =
-    (int)sizeof(xmlnsNamespace)/sizeof(XML_Char) - 1;
+           const XML_Char *uri, BINDING **bindingsPtr) {
+  static const XML_Char xmlNamespace[]
+      = {ASCII_h,      ASCII_t,     ASCII_t,     ASCII_p,      ASCII_COLON,
+         ASCII_SLASH,  ASCII_SLASH, ASCII_w,     ASCII_w,      ASCII_w,
+         ASCII_PERIOD, ASCII_w,     ASCII_3,     ASCII_PERIOD, ASCII_o,
+         ASCII_r,      ASCII_g,     ASCII_SLASH, ASCII_X,      ASCII_M,
+         ASCII_L,      ASCII_SLASH, ASCII_1,     ASCII_9,      ASCII_9,
+         ASCII_8,      ASCII_SLASH, ASCII_n,     ASCII_a,      ASCII_m,
+         ASCII_e,      ASCII_s,     ASCII_p,     ASCII_a,      ASCII_c,
+         ASCII_e,      '\0'};
+  static const int xmlLen = (int)sizeof(xmlNamespace) / sizeof(XML_Char) - 1;
+  static const XML_Char xmlnsNamespace[]
+      = {ASCII_h,     ASCII_t,      ASCII_t, ASCII_p, ASCII_COLON,  ASCII_SLASH,
+         ASCII_SLASH, ASCII_w,      ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w,
+         ASCII_3,     ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g,      ASCII_SLASH,
+         ASCII_2,     ASCII_0,      ASCII_0, ASCII_0, ASCII_SLASH,  ASCII_x,
+         ASCII_m,     ASCII_l,      ASCII_n, ASCII_s, ASCII_SLASH,  '\0'};
+  static const int xmlnsLen
+      = (int)sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1;
 
   XML_Bool mustBeXML = XML_FALSE;
   XML_Bool isXML = XML_TRUE;
@@ -3583,14 +3451,11 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   if (*uri == XML_T('\0') && prefix->name)
     return XML_ERROR_UNDECLARING_PREFIX;
 
-  if (prefix->name
-      && prefix->name[0] == XML_T(ASCII_x)
+  if (prefix->name && prefix->name[0] == XML_T(ASCII_x)
       && prefix->name[1] == XML_T(ASCII_m)
       && prefix->name[2] == XML_T(ASCII_l)) {
-
     /* Not allowed to bind xmlns */
-    if (prefix->name[3] == XML_T(ASCII_n)
-        && prefix->name[4] == XML_T(ASCII_s)
+    if (prefix->name[3] == XML_T(ASCII_n) && prefix->name[4] == XML_T(ASCII_s)
         && prefix->name[5] == XML_T('\0'))
       return XML_ERROR_RESERVED_PREFIX_XMLNS;
 
@@ -3602,7 +3467,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
     if (isXML && (len > xmlLen || uri[len] != xmlNamespace[len]))
       isXML = XML_FALSE;
 
-    if (!mustBeXML && isXMLNS
+    if (! mustBeXML && isXMLNS
         && (len > xmlnsLen || uri[len] != xmlnsNamespace[len]))
       isXMLNS = XML_FALSE;
   }
@@ -3621,21 +3486,21 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   if (parser->m_freeBindingList) {
     b = parser->m_freeBindingList;
     if (len > b->uriAlloc) {
-      XML_Char *temp = (XML_Char *)REALLOC(parser, b->uri,
-                          sizeof(XML_Char) * (len + EXPAND_SPARE));
+      XML_Char *temp = (XML_Char *)REALLOC(
+          parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE));
       if (temp == NULL)
         return XML_ERROR_NO_MEMORY;
       b->uri = temp;
       b->uriAlloc = len + EXPAND_SPARE;
     }
     parser->m_freeBindingList = b->nextTagBinding;
-  }
-  else {
+  } else {
     b = (BINDING *)MALLOC(parser, sizeof(BINDING));
-    if (!b)
+    if (! b)
       return XML_ERROR_NO_MEMORY;
-    b->uri = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE));
-    if (!b->uri) {
+    b->uri
+        = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE));
+    if (! b->uri) {
       FREE(parser, b);
       return XML_ERROR_NO_MEMORY;
     }
@@ -3658,7 +3523,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   /* if attId == NULL then we are not starting a namespace scope */
   if (attId && parser->m_startNamespaceDeclHandler)
     parser->m_startNamespaceDeclHandler(parser->m_handlerArg, prefix->name,
-                              prefix->binding ? uri : 0);
+                                        prefix->binding ? uri : 0);
   return XML_ERROR_NONE;
 }
 
@@ -3666,21 +3531,18 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
    the whole file is parsed with one call.
 */
 static enum XML_Error PTRCALL
-cdataSectionProcessor(XML_Parser parser,
-                      const char *start,
-                      const char *end,
-                      const char **endPtr)
-{
-  enum XML_Error result = doCdataSection(parser, parser->m_encoding, &start, end,
-                                         endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+cdataSectionProcessor(XML_Parser parser, const char *start, const char *end,
+                      const char **endPtr) {
+  enum XML_Error result
+      = doCdataSection(parser, parser->m_encoding, &start, end, endPtr,
+                       (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result != XML_ERROR_NONE)
     return result;
   if (start) {
-    if (parser->m_parentParser) {  /* we are parsing an external entity */
+    if (parser->m_parentParser) { /* we are parsing an external entity */
       parser->m_processor = externalEntityContentProcessor;
       return externalEntityContentProcessor(parser, start, end, endPtr);
-    }
-    else {
+    } else {
       parser->m_processor = contentProcessor;
       return contentProcessor(parser, start, end, endPtr);
     }
@@ -3692,13 +3554,8 @@ cdataSectionProcessor(XML_Parser parser,
    the section is not yet closed.
 */
 static enum XML_Error
-doCdataSection(XML_Parser parser,
-               const ENCODING *enc,
-               const char **startPtr,
-               const char *end,
-               const char **nextPtr,
-               XML_Bool haveMore)
-{
+doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
+               const char *end, const char **nextPtr, XML_Bool haveMore) {
   const char *s = *startPtr;
   const char **eventPP;
   const char **eventEndPP;
@@ -3706,8 +3563,7 @@ doCdataSection(XML_Parser parser,
     eventPP = &parser->m_eventPtr;
     *eventPP = s;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -3722,11 +3578,12 @@ doCdataSection(XML_Parser parser,
     case XML_TOK_CDATA_SECT_CLOSE:
       if (parser->m_endCdataSectionHandler)
         parser->m_endCdataSectionHandler(parser->m_handlerArg);
-/* BEGIN disabled code */
+      /* BEGIN disabled code */
       /* see comment under XML_TOK_CDATA_SECT_OPEN */
       else if (0 && parser->m_characterDataHandler)
-        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0);
-/* END disabled code */
+        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                                       0);
+      /* END disabled code */
       else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       *startPtr = next;
@@ -3739,35 +3596,31 @@ doCdataSection(XML_Parser parser,
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       break;
-    case XML_TOK_DATA_CHARS:
-      {
-        XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
-        if (charDataHandler) {
-          if (MUST_CONVERT(enc, s)) {
-            for (;;) {
-              ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-              const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-              *eventEndPP = next;
-              charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-              if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
-                break;
-              *eventPP = s;
-            }
+    case XML_TOK_DATA_CHARS: {
+      XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
+      if (charDataHandler) {
+        if (MUST_CONVERT(enc, s)) {
+          for (;;) {
+            ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
+            const enum XML_Convert_Result convert_res = XmlConvert(
+                enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+            *eventEndPP = next;
+            charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                            (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+            if ((convert_res == XML_CONVERT_COMPLETED)
+                || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+              break;
+            *eventPP = s;
           }
-          else
-            charDataHandler(parser->m_handlerArg,
-                            (XML_Char *)s,
-                            (int)((XML_Char *)next - (XML_Char *)s));
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-      }
-      break;
+        } else
+          charDataHandler(parser->m_handlerArg, (XML_Char *)s,
+                          (int)((XML_Char *)next - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_INVALID:
       *eventPP = next;
       return XML_ERROR_INVALID_TOKEN;
@@ -3791,7 +3644,7 @@ doCdataSection(XML_Parser parser,
        * statistics.
        *
        * LCOV_EXCL_START
-      */
+       */
       *eventPP = next;
       return XML_ERROR_UNEXPECTED_STATE;
       /* LCOV_EXCL_STOP */
@@ -3804,7 +3657,7 @@ doCdataSection(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
   /* not reached */
@@ -3816,13 +3669,11 @@ doCdataSection(XML_Parser parser,
    the whole file is parsed with one call.
 */
 static enum XML_Error PTRCALL
-ignoreSectionProcessor(XML_Parser parser,
-                       const char *start,
-                       const char *end,
-                       const char **endPtr)
-{
-  enum XML_Error result = doIgnoreSection(parser, parser->m_encoding, &start, end,
-                                          endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end,
+                       const char **endPtr) {
+  enum XML_Error result
+      = doIgnoreSection(parser, parser->m_encoding, &start, end, endPtr,
+                        (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result != XML_ERROR_NONE)
     return result;
   if (start) {
@@ -3836,13 +3687,8 @@ ignoreSectionProcessor(XML_Parser parser,
    if the section is not yet closed.
 */
 static enum XML_Error
-doIgnoreSection(XML_Parser parser,
-                const ENCODING *enc,
-                const char **startPtr,
-                const char *end,
-                const char **nextPtr,
-                XML_Bool haveMore)
-{
+doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
+                const char *end, const char **nextPtr, XML_Bool haveMore) {
   const char *next;
   int tok;
   const char *s = *startPtr;
@@ -3852,8 +3698,7 @@ doIgnoreSection(XML_Parser parser,
     eventPP = &parser->m_eventPtr;
     *eventPP = s;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     /* It's not entirely clear, but it seems the following two lines
      * of code cannot be executed.  The only occasions on which 'enc'
      * is not 'encoding' are when this function is called
@@ -3917,13 +3762,12 @@ doIgnoreSection(XML_Parser parser,
 #endif /* XML_DTD */
 
 static enum XML_Error
-initializeEncoding(XML_Parser parser)
-{
+initializeEncoding(XML_Parser parser) {
   const char *s;
 #ifdef XML_UNICODE
   char encodingBuf[128];
   /* See comments abount `protoclEncodingName` in parserInit() */
-  if (!parser->m_protocolEncodingName)
+  if (! parser->m_protocolEncodingName)
     s = NULL;
   else {
     int i;
@@ -3941,15 +3785,15 @@ initializeEncoding(XML_Parser parser)
 #else
   s = parser->m_protocolEncodingName;
 #endif
-  if ((parser->m_ns ? XmlInitEncodingNS : XmlInitEncoding)(&parser->m_initEncoding, &parser->m_encoding, s))
+  if ((parser->m_ns ? XmlInitEncodingNS : XmlInitEncoding)(
+          &parser->m_initEncoding, &parser->m_encoding, s))
     return XML_ERROR_NONE;
   return handleUnknownEncoding(parser, parser->m_protocolEncodingName);
 }
 
 static enum XML_Error
-processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
-               const char *s, const char *next)
-{
+processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s,
+               const char *next) {
   const char *encodingName = NULL;
   const XML_Char *storedEncName = NULL;
   const ENCODING *newEncoding = NULL;
@@ -3957,52 +3801,41 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
   const char *versionend;
   const XML_Char *storedversion = NULL;
   int standalone = -1;
-  if (!(parser->m_ns
-        ? XmlParseXmlDeclNS
-        : XmlParseXmlDecl)(isGeneralTextEntity,
-                           parser->m_encoding,
-                           s,
-                           next,
-                           &parser->m_eventPtr,
-                           &version,
-                           &versionend,
-                           &encodingName,
-                           &newEncoding,
-                           &standalone)) {
+  if (! (parser->m_ns ? XmlParseXmlDeclNS : XmlParseXmlDecl)(
+          isGeneralTextEntity, parser->m_encoding, s, next, &parser->m_eventPtr,
+          &version, &versionend, &encodingName, &newEncoding, &standalone)) {
     if (isGeneralTextEntity)
       return XML_ERROR_TEXT_DECL;
     else
       return XML_ERROR_XML_DECL;
   }
-  if (!isGeneralTextEntity && standalone == 1) {
+  if (! isGeneralTextEntity && standalone == 1) {
     parser->m_dtd->standalone = XML_TRUE;
 #ifdef XML_DTD
-    if (parser->m_paramEntityParsing == XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE)
+    if (parser->m_paramEntityParsing
+        == XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE)
       parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER;
 #endif /* XML_DTD */
   }
   if (parser->m_xmlDeclHandler) {
     if (encodingName != NULL) {
-      storedEncName = poolStoreString(&parser->m_temp2Pool,
-                                      parser->m_encoding,
-                                      encodingName,
-                                      encodingName
-                                      + XmlNameLength(parser->m_encoding, encodingName));
-      if (!storedEncName)
-              return XML_ERROR_NO_MEMORY;
+      storedEncName = poolStoreString(
+          &parser->m_temp2Pool, parser->m_encoding, encodingName,
+          encodingName + XmlNameLength(parser->m_encoding, encodingName));
+      if (! storedEncName)
+        return XML_ERROR_NO_MEMORY;
       poolFinish(&parser->m_temp2Pool);
     }
     if (version) {
-      storedversion = poolStoreString(&parser->m_temp2Pool,
-                                      parser->m_encoding,
-                                      version,
-                                      versionend - parser->m_encoding->minBytesPerChar);
-      if (!storedversion)
+      storedversion
+          = poolStoreString(&parser->m_temp2Pool, parser->m_encoding, version,
+                            versionend - parser->m_encoding->minBytesPerChar);
+      if (! storedversion)
         return XML_ERROR_NO_MEMORY;
     }
-    parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName, standalone);
-  }
-  else if (parser->m_defaultHandler)
+    parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName,
+                             standalone);
+  } else if (parser->m_defaultHandler)
     reportDefault(parser, parser->m_encoding, s, next);
   if (parser->m_protocolEncodingName == NULL) {
     if (newEncoding) {
@@ -4012,20 +3845,19 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
        * this is UTF-16, is it the same endianness?
        */
       if (newEncoding->minBytesPerChar != parser->m_encoding->minBytesPerChar
-          || (newEncoding->minBytesPerChar == 2 &&
-              newEncoding != parser->m_encoding)) {
+          || (newEncoding->minBytesPerChar == 2
+              && newEncoding != parser->m_encoding)) {
         parser->m_eventPtr = encodingName;
         return XML_ERROR_INCORRECT_ENCODING;
       }
       parser->m_encoding = newEncoding;
-    }
-    else if (encodingName) {
+    } else if (encodingName) {
       enum XML_Error result;
-      if (!storedEncName) {
+      if (! storedEncName) {
         storedEncName = poolStoreString(
-          &parser->m_temp2Pool, parser->m_encoding, encodingName,
-          encodingName + XmlNameLength(parser->m_encoding, encodingName));
-        if (!storedEncName)
+            &parser->m_temp2Pool, parser->m_encoding, encodingName,
+            encodingName + XmlNameLength(parser->m_encoding, encodingName));
+        if (! storedEncName)
           return XML_ERROR_NO_MEMORY;
       }
       result = handleUnknownEncoding(parser, storedEncName);
@@ -4043,8 +3875,7 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
 }
 
 static enum XML_Error
-handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
-{
+handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) {
   if (parser->m_unknownEncodingHandler) {
     XML_Encoding info;
     int i;
@@ -4053,21 +3884,17 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
     info.convert = NULL;
     info.data = NULL;
     info.release = NULL;
-    if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData, encodingName,
-                               &info)) {
+    if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData,
+                                         encodingName, &info)) {
       ENCODING *enc;
       parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding());
-      if (!parser->m_unknownEncodingMem) {
+      if (! parser->m_unknownEncodingMem) {
         if (info.release)
           info.release(info.data);
         return XML_ERROR_NO_MEMORY;
       }
-      enc = (parser->m_ns
-             ? XmlInitUnknownEncodingNS
-             : XmlInitUnknownEncoding)(parser->m_unknownEncodingMem,
-                                       info.map,
-                                       info.convert,
-                                       info.data);
+      enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)(
+          parser->m_unknownEncodingMem, info.map, info.convert, info.data);
       if (enc) {
         parser->m_unknownEncodingData = info.data;
         parser->m_unknownEncodingRelease = info.release;
@@ -4082,11 +3909,8 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
 }
 
 static enum XML_Error PTRCALL
-prologInitProcessor(XML_Parser parser,
-                    const char *s,
-                    const char *end,
-                    const char **nextPtr)
-{
+prologInitProcessor(XML_Parser parser, const char *s, const char *end,
+                    const char **nextPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -4097,11 +3921,8 @@ prologInitProcessor(XML_Parser parser,
 #ifdef XML_DTD
 
 static enum XML_Error PTRCALL
-externalParEntInitProcessor(XML_Parser parser,
-                            const char *s,
-                            const char *end,
-                            const char **nextPtr)
-{
+externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end,
+                            const char **nextPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -4113,19 +3934,15 @@ externalParEntInitProcessor(XML_Parser parser,
   if (parser->m_prologState.inEntityValue) {
     parser->m_processor = entityValueInitProcessor;
     return entityValueInitProcessor(parser, s, end, nextPtr);
-  }
-  else {
+  } else {
     parser->m_processor = externalParEntProcessor;
     return externalParEntProcessor(parser, s, end, nextPtr);
   }
 }
 
 static enum XML_Error PTRCALL
-entityValueInitProcessor(XML_Parser parser,
-                         const char *s,
-                         const char *end,
-                         const char **nextPtr)
-{
+entityValueInitProcessor(XML_Parser parser, const char *s, const char *end,
+                         const char **nextPtr) {
   int tok;
   const char *start = s;
   const char *next = start;
@@ -4135,7 +3952,7 @@ entityValueInitProcessor(XML_Parser parser,
     tok = XmlPrologTok(parser->m_encoding, start, end, &next);
     parser->m_eventEndPtr = next;
     if (tok <= 0) {
-      if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+      if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -4146,22 +3963,21 @@ entityValueInitProcessor(XML_Parser parser,
         return XML_ERROR_UNCLOSED_TOKEN;
       case XML_TOK_PARTIAL_CHAR:
         return XML_ERROR_PARTIAL_CHAR;
-      case XML_TOK_NONE:   /* start == end */
+      case XML_TOK_NONE: /* start == end */
       default:
         break;
       }
       /* found end of entity value - can store it now */
       return storeEntityValue(parser, parser->m_encoding, s, end);
-    }
-    else if (tok == XML_TOK_XML_DECL) {
+    } else if (tok == XML_TOK_XML_DECL) {
       enum XML_Error result;
       result = processXmlDecl(parser, 0, start, next);
       if (result != XML_ERROR_NONE)
         return result;
-      /* At this point, m_parsingStatus.parsing cannot be XML_SUSPENDED.  For that
-       * to happen, a parameter entity parsing handler must have
-       * attempted to suspend the parser, which fails and raises an
-       * error.  The parser can be aborted, but can't be suspended.
+      /* At this point, m_parsingStatus.parsing cannot be XML_SUSPENDED.  For
+       * that to happen, a parameter entity parsing handler must have attempted
+       * to suspend the parser, which fails and raises an error.  The parser can
+       * be aborted, but can't be suspended.
        */
       if (parser->m_parsingStatus.parsing == XML_FINISHED)
         return XML_ERROR_ABORTED;
@@ -4177,7 +3993,8 @@ entityValueInitProcessor(XML_Parser parser,
        then, when this routine is entered the next time, XmlPrologTok will
        return XML_TOK_INVALID, since the BOM is still in the buffer
     */
-    else if (tok == XML_TOK_BOM && next == end && !parser->m_parsingStatus.finalBuffer) {
+    else if (tok == XML_TOK_BOM && next == end
+             && ! parser->m_parsingStatus.finalBuffer) {
       *nextPtr = next;
       return XML_ERROR_NONE;
     }
@@ -4195,17 +4012,14 @@ entityValueInitProcessor(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalParEntProcessor(XML_Parser parser,
-                        const char *s,
-                        const char *end,
-                        const char **nextPtr)
-{
+externalParEntProcessor(XML_Parser parser, const char *s, const char *end,
+                        const char **nextPtr) {
   const char *next = s;
   int tok;
 
   tok = XmlPrologTok(parser->m_encoding, s, end, &next);
   if (tok <= 0) {
-    if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+    if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
       *nextPtr = s;
       return XML_ERROR_NONE;
     }
@@ -4216,7 +4030,7 @@ externalParEntProcessor(XML_Parser parser,
       return XML_ERROR_UNCLOSED_TOKEN;
     case XML_TOK_PARTIAL_CHAR:
       return XML_ERROR_PARTIAL_CHAR;
-    case XML_TOK_NONE:   /* start == end */
+    case XML_TOK_NONE: /* start == end */
     default:
       break;
     }
@@ -4231,16 +4045,13 @@ externalParEntProcessor(XML_Parser parser,
   }
 
   parser->m_processor = prologProcessor;
-  return doProlog(parser, parser->m_encoding, s, end, tok, next,
-                  nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+  return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
 }
 
 static enum XML_Error PTRCALL
-entityValueProcessor(XML_Parser parser,
-                     const char *s,
-                     const char *end,
-                     const char **nextPtr)
-{
+entityValueProcessor(XML_Parser parser, const char *s, const char *end,
+                     const char **nextPtr) {
   const char *start = s;
   const char *next = s;
   const ENCODING *enc = parser->m_encoding;
@@ -4249,7 +4060,7 @@ entityValueProcessor(XML_Parser parser,
   for (;;) {
     tok = XmlPrologTok(enc, start, end, &next);
     if (tok <= 0) {
-      if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+      if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -4260,7 +4071,7 @@ entityValueProcessor(XML_Parser parser,
         return XML_ERROR_UNCLOSED_TOKEN;
       case XML_TOK_PARTIAL_CHAR:
         return XML_ERROR_PARTIAL_CHAR;
-      case XML_TOK_NONE:   /* start == end */
+      case XML_TOK_NONE: /* start == end */
       default:
         break;
       }
@@ -4274,52 +4085,46 @@ entityValueProcessor(XML_Parser parser,
 #endif /* XML_DTD */
 
 static enum XML_Error PTRCALL
-prologProcessor(XML_Parser parser,
-                const char *s,
-                const char *end,
-                const char **nextPtr)
-{
+prologProcessor(XML_Parser parser, const char *s, const char *end,
+                const char **nextPtr) {
   const char *next = s;
   int tok = XmlPrologTok(parser->m_encoding, s, end, &next);
-  return doProlog(parser, parser->m_encoding, s, end, tok, next,
-                  nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+  return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
 }
 
 static enum XML_Error
-doProlog(XML_Parser parser,
-         const ENCODING *enc,
-         const char *s,
-         const char *end,
-         int tok,
-         const char *next,
-         const char **nextPtr,
-         XML_Bool haveMore)
-{
+doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
+         int tok, const char *next, const char **nextPtr, XML_Bool haveMore,
+         XML_Bool allowClosingDoctype) {
 #ifdef XML_DTD
-  static const XML_Char externalSubsetName[] = { ASCII_HASH , '\0' };
+  static const XML_Char externalSubsetName[] = {ASCII_HASH, '\0'};
 #endif /* XML_DTD */
-  static const XML_Char atypeCDATA[] =
-      { ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-  static const XML_Char atypeID[] = { ASCII_I, ASCII_D, '\0' };
-  static const XML_Char atypeIDREF[] =
-      { ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0' };
-  static const XML_Char atypeIDREFS[] =
-      { ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0' };
-  static const XML_Char atypeENTITY[] =
-      { ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0' };
-  static const XML_Char atypeENTITIES[] = { ASCII_E, ASCII_N,
-      ASCII_T, ASCII_I, ASCII_T, ASCII_I, ASCII_E, ASCII_S, '\0' };
-  static const XML_Char atypeNMTOKEN[] = {
-      ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0' };
-  static const XML_Char atypeNMTOKENS[] = { ASCII_N, ASCII_M, ASCII_T,
-      ASCII_O, ASCII_K, ASCII_E, ASCII_N, ASCII_S, '\0' };
-  static const XML_Char notationPrefix[] = { ASCII_N, ASCII_O, ASCII_T,
-      ASCII_A, ASCII_T, ASCII_I, ASCII_O, ASCII_N, ASCII_LPAREN, '\0' };
-  static const XML_Char enumValueSep[] = { ASCII_PIPE, '\0' };
-  static const XML_Char enumValueStart[] = { ASCII_LPAREN, '\0' };
+  static const XML_Char atypeCDATA[]
+      = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+  static const XML_Char atypeID[] = {ASCII_I, ASCII_D, '\0'};
+  static const XML_Char atypeIDREF[]
+      = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0'};
+  static const XML_Char atypeIDREFS[]
+      = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0'};
+  static const XML_Char atypeENTITY[]
+      = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0'};
+  static const XML_Char atypeENTITIES[]
+      = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T,
+         ASCII_I, ASCII_E, ASCII_S, '\0'};
+  static const XML_Char atypeNMTOKEN[]
+      = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0'};
+  static const XML_Char atypeNMTOKENS[]
+      = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K,
+         ASCII_E, ASCII_N, ASCII_S, '\0'};
+  static const XML_Char notationPrefix[]
+      = {ASCII_N, ASCII_O, ASCII_T, ASCII_A,      ASCII_T,
+         ASCII_I, ASCII_O, ASCII_N, ASCII_LPAREN, '\0'};
+  static const XML_Char enumValueSep[] = {ASCII_PIPE, '\0'};
+  static const XML_Char enumValueStart[] = {ASCII_LPAREN, '\0'};
 
   /* save one level of indirection */
-  DTD * const dtd = parser->m_dtd;
+  DTD *const dtd = parser->m_dtd;
 
   const char **eventPP;
   const char **eventEndPP;
@@ -4328,8 +4133,7 @@ doProlog(XML_Parser parser,
   if (enc == parser->m_encoding) {
     eventPP = &parser->m_eventPtr;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -4358,7 +4162,8 @@ doProlog(XML_Parser parser,
       case XML_TOK_NONE:
 #ifdef XML_DTD
         /* for internal PE NOT referenced between declarations */
-        if (enc != parser->m_encoding && !parser->m_openInternalEntities->betweenDecl) {
+        if (enc != parser->m_encoding
+            && ! parser->m_openInternalEntities->betweenDecl) {
           *nextPtr = s;
           return XML_ERROR_NONE;
         }
@@ -4383,19 +4188,18 @@ doProlog(XML_Parser parser,
     }
     role = XmlTokenRole(&parser->m_prologState, tok, s, next, enc);
     switch (role) {
-    case XML_ROLE_XML_DECL:
-      {
-        enum XML_Error result = processXmlDecl(parser, 0, s, next);
-        if (result != XML_ERROR_NONE)
-          return result;
-        enc = parser->m_encoding;
-        handleDefault = XML_FALSE;
-      }
-      break;
+    case XML_ROLE_XML_DECL: {
+      enum XML_Error result = processXmlDecl(parser, 0, s, next);
+      if (result != XML_ERROR_NONE)
+        return result;
+      enc = parser->m_encoding;
+      handleDefault = XML_FALSE;
+    } break;
     case XML_ROLE_DOCTYPE_NAME:
       if (parser->m_startDoctypeDeclHandler) {
-        parser->m_doctypeName = poolStoreString(&parser->m_tempPool, enc, s, next);
-        if (!parser->m_doctypeName)
+        parser->m_doctypeName
+            = poolStoreString(&parser->m_tempPool, enc, s, next);
+        if (! parser->m_doctypeName)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
         parser->m_doctypePubid = NULL;
@@ -4405,43 +4209,40 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_DOCTYPE_INTERNAL_SUBSET:
       if (parser->m_startDoctypeDeclHandler) {
-        parser->m_startDoctypeDeclHandler(parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
-                                parser->m_doctypePubid, 1);
+        parser->m_startDoctypeDeclHandler(
+            parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
+            parser->m_doctypePubid, 1);
         parser->m_doctypeName = NULL;
         poolClear(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
       break;
 #ifdef XML_DTD
-    case XML_ROLE_TEXT_DECL:
-      {
-        enum XML_Error result = processXmlDecl(parser, 1, s, next);
-        if (result != XML_ERROR_NONE)
-          return result;
-        enc = parser->m_encoding;
-        handleDefault = XML_FALSE;
-      }
-      break;
+    case XML_ROLE_TEXT_DECL: {
+      enum XML_Error result = processXmlDecl(parser, 1, s, next);
+      if (result != XML_ERROR_NONE)
+        return result;
+      enc = parser->m_encoding;
+      handleDefault = XML_FALSE;
+    } break;
 #endif /* XML_DTD */
     case XML_ROLE_DOCTYPE_PUBLIC_ID:
 #ifdef XML_DTD
       parser->m_useForeignDTD = XML_FALSE;
-      parser->m_declEntity = (ENTITY *)lookup(parser,
-                                    &dtd->paramEntities,
-                                    externalSubsetName,
-                                    sizeof(ENTITY));
-      if (!parser->m_declEntity)
+      parser->m_declEntity = (ENTITY *)lookup(
+          parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY));
+      if (! parser->m_declEntity)
         return XML_ERROR_NO_MEMORY;
 #endif /* XML_DTD */
       dtd->hasParamEntityRefs = XML_TRUE;
       if (parser->m_startDoctypeDeclHandler) {
         XML_Char *pubId;
-        if (!XmlIsPublicId(enc, s, next, eventPP))
+        if (! XmlIsPublicId(enc, s, next, eventPP))
           return XML_ERROR_PUBLICID;
         pubId = poolStoreString(&parser->m_tempPool, enc,
                                 s + enc->minBytesPerChar,
                                 next - enc->minBytesPerChar);
-        if (!pubId)
+        if (! pubId)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(pubId);
         poolFinish(&parser->m_tempPool);
@@ -4451,15 +4252,14 @@ doProlog(XML_Parser parser,
       }
       /* fall through */
     case XML_ROLE_ENTITY_PUBLIC_ID:
-      if (!XmlIsPublicId(enc, s, next, eventPP))
+      if (! XmlIsPublicId(enc, s, next, eventPP))
         return XML_ERROR_PUBLICID;
     alreadyChecked:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        XML_Char *tem = poolStoreString(&dtd->pool,
-                                        enc,
-                                        s + enc->minBytesPerChar,
-                                        next - enc->minBytesPerChar);
-        if (!tem)
+        XML_Char *tem
+            = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                              next - enc->minBytesPerChar);
+        if (! tem)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(tem);
         parser->m_declEntity->publicId = tem;
@@ -4472,9 +4272,15 @@ doProlog(XML_Parser parser,
       }
       break;
     case XML_ROLE_DOCTYPE_CLOSE:
+      if (allowClosingDoctype != XML_TRUE) {
+        /* Must not close doctype from within expanded parameter entities */
+        return XML_ERROR_INVALID_TOKEN;
+      }
+
       if (parser->m_doctypeName) {
-        parser->m_startDoctypeDeclHandler(parser->m_handlerArg, parser->m_doctypeName,
-                                parser->m_doctypeSysid, parser->m_doctypePubid, 0);
+        parser->m_startDoctypeDeclHandler(
+            parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
+            parser->m_doctypePubid, 0);
         poolClear(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
@@ -4486,12 +4292,11 @@ doProlog(XML_Parser parser,
       if (parser->m_doctypeSysid || parser->m_useForeignDTD) {
         XML_Bool hadParamEntityRefs = dtd->hasParamEntityRefs;
         dtd->hasParamEntityRefs = XML_TRUE;
-        if (parser->m_paramEntityParsing && parser->m_externalEntityRefHandler) {
-          ENTITY *entity = (ENTITY *)lookup(parser,
-                                            &dtd->paramEntities,
-                                            externalSubsetName,
-                                            sizeof(ENTITY));
-          if (!entity) {
+        if (parser->m_paramEntityParsing
+            && parser->m_externalEntityRefHandler) {
+          ENTITY *entity = (ENTITY *)lookup(parser, &dtd->paramEntities,
+                                            externalSubsetName, sizeof(ENTITY));
+          if (! entity) {
             /* The external subset name "#" will have already been
              * inserted into the hash table at the start of the
              * external entity parsing, so no allocation will happen
@@ -4502,22 +4307,19 @@ doProlog(XML_Parser parser,
           if (parser->m_useForeignDTD)
             entity->base = parser->m_curBase;
           dtd->paramEntityRead = XML_FALSE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId))
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           if (dtd->paramEntityRead) {
-            if (!dtd->standalone &&
-                parser->m_notStandaloneHandler &&
-                !parser->m_notStandaloneHandler(parser->m_handlerArg))
+            if (! dtd->standalone && parser->m_notStandaloneHandler
+                && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
               return XML_ERROR_NOT_STANDALONE;
           }
           /* if we didn't read the foreign DTD then this means that there
              is no external subset and we must reset dtd->hasParamEntityRefs
           */
-          else if (!parser->m_doctypeSysid)
+          else if (! parser->m_doctypeSysid)
             dtd->hasParamEntityRefs = hadParamEntityRefs;
           /* end of DTD - no need to update dtd->keepProcessing */
         }
@@ -4537,24 +4339,21 @@ doProlog(XML_Parser parser,
       if (parser->m_useForeignDTD) {
         XML_Bool hadParamEntityRefs = dtd->hasParamEntityRefs;
         dtd->hasParamEntityRefs = XML_TRUE;
-        if (parser->m_paramEntityParsing && parser->m_externalEntityRefHandler) {
+        if (parser->m_paramEntityParsing
+            && parser->m_externalEntityRefHandler) {
           ENTITY *entity = (ENTITY *)lookup(parser, &dtd->paramEntities,
-                                            externalSubsetName,
-                                            sizeof(ENTITY));
-          if (!entity)
+                                            externalSubsetName, sizeof(ENTITY));
+          if (! entity)
             return XML_ERROR_NO_MEMORY;
           entity->base = parser->m_curBase;
           dtd->paramEntityRead = XML_FALSE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId))
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           if (dtd->paramEntityRead) {
-            if (!dtd->standalone &&
-                parser->m_notStandaloneHandler &&
-                !parser->m_notStandaloneHandler(parser->m_handlerArg))
+            if (! dtd->standalone && parser->m_notStandaloneHandler
+                && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
               return XML_ERROR_NOT_STANDALONE;
           }
           /* if we didn't read the foreign DTD then this means that there
@@ -4570,12 +4369,12 @@ doProlog(XML_Parser parser,
       return contentProcessor(parser, s, end, nextPtr);
     case XML_ROLE_ATTLIST_ELEMENT_NAME:
       parser->m_declElementType = getElementType(parser, enc, s, next);
-      if (!parser->m_declElementType)
+      if (! parser->m_declElementType)
         return XML_ERROR_NO_MEMORY;
       goto checkAttListDeclHandler;
     case XML_ROLE_ATTRIBUTE_NAME:
       parser->m_declAttributeId = getAttributeId(parser, enc, s, next);
-      if (!parser->m_declAttributeId)
+      if (! parser->m_declAttributeId)
         return XML_ERROR_NO_MEMORY;
       parser->m_declAttributeIsCdata = XML_FALSE;
       parser->m_declAttributeType = NULL;
@@ -4616,15 +4415,13 @@ doProlog(XML_Parser parser,
         const XML_Char *prefix;
         if (parser->m_declAttributeType) {
           prefix = enumValueSep;
+        } else {
+          prefix = (role == XML_ROLE_ATTRIBUTE_NOTATION_VALUE ? notationPrefix
+                                                              : enumValueStart);
         }
-        else {
-          prefix = (role == XML_ROLE_ATTRIBUTE_NOTATION_VALUE
-                    ? notationPrefix
-                    : enumValueStart);
-        }
-        if (!poolAppendString(&parser->m_tempPool, prefix))
+        if (! poolAppendString(&parser->m_tempPool, prefix))
           return XML_ERROR_NO_MEMORY;
-        if (!poolAppend(&parser->m_tempPool, enc, s, next))
+        if (! poolAppend(&parser->m_tempPool, enc, s, next))
           return XML_ERROR_NO_MEMORY;
         parser->m_declAttributeType = parser->m_tempPool.start;
         handleDefault = XML_FALSE;
@@ -4633,25 +4430,27 @@ doProlog(XML_Parser parser,
     case XML_ROLE_IMPLIED_ATTRIBUTE_VALUE:
     case XML_ROLE_REQUIRED_ATTRIBUTE_VALUE:
       if (dtd->keepProcessing) {
-        if (!defineAttribute(parser->m_declElementType, parser->m_declAttributeId,
-                             parser->m_declAttributeIsCdata, parser->m_declAttributeIsId,
-                             0, parser))
+        if (! defineAttribute(parser->m_declElementType,
+                              parser->m_declAttributeId,
+                              parser->m_declAttributeIsCdata,
+                              parser->m_declAttributeIsId, 0, parser))
           return XML_ERROR_NO_MEMORY;
         if (parser->m_attlistDeclHandler && parser->m_declAttributeType) {
           if (*parser->m_declAttributeType == XML_T(ASCII_LPAREN)
               || (*parser->m_declAttributeType == XML_T(ASCII_N)
                   && parser->m_declAttributeType[1] == XML_T(ASCII_O))) {
             /* Enumerated or Notation type */
-            if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
-                || !poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+            if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
+                || ! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
               return XML_ERROR_NO_MEMORY;
             parser->m_declAttributeType = parser->m_tempPool.start;
             poolFinish(&parser->m_tempPool);
           }
           *eventEndPP = s;
-          parser->m_attlistDeclHandler(parser->m_handlerArg, parser->m_declElementType->name,
-                             parser->m_declAttributeId->name, parser->m_declAttributeType,
-                             0, role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE);
+          parser->m_attlistDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name,
+              parser->m_declAttributeId->name, parser->m_declAttributeType, 0,
+              role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE);
           poolClear(&parser->m_tempPool);
           handleDefault = XML_FALSE;
         }
@@ -4661,35 +4460,34 @@ doProlog(XML_Parser parser,
     case XML_ROLE_FIXED_ATTRIBUTE_VALUE:
       if (dtd->keepProcessing) {
         const XML_Char *attVal;
-        enum XML_Error result =
-          storeAttributeValue(parser, enc, parser->m_declAttributeIsCdata,
-                              s + enc->minBytesPerChar,
-                              next - enc->minBytesPerChar,
-                              &dtd->pool);
+        enum XML_Error result = storeAttributeValue(
+            parser, enc, parser->m_declAttributeIsCdata,
+            s + enc->minBytesPerChar, next - enc->minBytesPerChar, &dtd->pool);
         if (result)
           return result;
         attVal = poolStart(&dtd->pool);
         poolFinish(&dtd->pool);
         /* ID attributes aren't allowed to have a default */
-        if (!defineAttribute(parser->m_declElementType, parser->m_declAttributeId,
-                             parser->m_declAttributeIsCdata, XML_FALSE, attVal, parser))
+        if (! defineAttribute(
+                parser->m_declElementType, parser->m_declAttributeId,
+                parser->m_declAttributeIsCdata, XML_FALSE, attVal, parser))
           return XML_ERROR_NO_MEMORY;
         if (parser->m_attlistDeclHandler && parser->m_declAttributeType) {
           if (*parser->m_declAttributeType == XML_T(ASCII_LPAREN)
               || (*parser->m_declAttributeType == XML_T(ASCII_N)
                   && parser->m_declAttributeType[1] == XML_T(ASCII_O))) {
             /* Enumerated or Notation type */
-            if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
-                || !poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+            if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
+                || ! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
               return XML_ERROR_NO_MEMORY;
             parser->m_declAttributeType = parser->m_tempPool.start;
             poolFinish(&parser->m_tempPool);
           }
           *eventEndPP = s;
-          parser->m_attlistDeclHandler(parser->m_handlerArg, parser->m_declElementType->name,
-                             parser->m_declAttributeId->name, parser->m_declAttributeType,
-                             attVal,
-                             role == XML_ROLE_FIXED_ATTRIBUTE_VALUE);
+          parser->m_attlistDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name,
+              parser->m_declAttributeId->name, parser->m_declAttributeType,
+              attVal, role == XML_ROLE_FIXED_ATTRIBUTE_VALUE);
           poolClear(&parser->m_tempPool);
           handleDefault = XML_FALSE;
         }
@@ -4697,25 +4495,22 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_ENTITY_VALUE:
       if (dtd->keepProcessing) {
-        enum XML_Error result = storeEntityValue(parser, enc,
-                                            s + enc->minBytesPerChar,
-                                            next - enc->minBytesPerChar);
+        enum XML_Error result = storeEntityValue(
+            parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar);
         if (parser->m_declEntity) {
           parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool);
-          parser->m_declEntity->textLen = (int)(poolLength(&dtd->entityValuePool));
+          parser->m_declEntity->textLen
+              = (int)(poolLength(&dtd->entityValuePool));
           poolFinish(&dtd->entityValuePool);
           if (parser->m_entityDeclHandler) {
             *eventEndPP = s;
-            parser->m_entityDeclHandler(parser->m_handlerArg,
-                              parser->m_declEntity->name,
-                              parser->m_declEntity->is_param,
-                              parser->m_declEntity->textPtr,
-                              parser->m_declEntity->textLen,
-                              parser->m_curBase, 0, 0, 0);
+            parser->m_entityDeclHandler(
+                parser->m_handlerArg, parser->m_declEntity->name,
+                parser->m_declEntity->is_param, parser->m_declEntity->textPtr,
+                parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0);
             handleDefault = XML_FALSE;
           }
-        }
-        else
+        } else
           poolDiscard(&dtd->entityValuePool);
         if (result != XML_ERROR_NONE)
           return result;
@@ -4728,8 +4523,8 @@ doProlog(XML_Parser parser,
       dtd->hasParamEntityRefs = XML_TRUE;
       if (parser->m_startDoctypeDeclHandler) {
         parser->m_doctypeSysid = poolStoreString(&parser->m_tempPool, enc,
-                                       s + enc->minBytesPerChar,
-                                       next - enc->minBytesPerChar);
+                                                 s + enc->minBytesPerChar,
+                                                 next - enc->minBytesPerChar);
         if (parser->m_doctypeSysid == NULL)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
@@ -4741,22 +4536,20 @@ doProlog(XML_Parser parser,
            for the case where no parser->m_startDoctypeDeclHandler is set */
         parser->m_doctypeSysid = externalSubsetName;
 #endif /* XML_DTD */
-      if (!dtd->standalone
+      if (! dtd->standalone
 #ifdef XML_DTD
-          && !parser->m_paramEntityParsing
+          && ! parser->m_paramEntityParsing
 #endif /* XML_DTD */
           && parser->m_notStandaloneHandler
-          && !parser->m_notStandaloneHandler(parser->m_handlerArg))
+          && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
         return XML_ERROR_NOT_STANDALONE;
 #ifndef XML_DTD
       break;
-#else /* XML_DTD */
-      if (!parser->m_declEntity) {
-        parser->m_declEntity = (ENTITY *)lookup(parser,
-                                      &dtd->paramEntities,
-                                      externalSubsetName,
-                                      sizeof(ENTITY));
-        if (!parser->m_declEntity)
+#else  /* XML_DTD */
+      if (! parser->m_declEntity) {
+        parser->m_declEntity = (ENTITY *)lookup(
+            parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY));
+        if (! parser->m_declEntity)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity->publicId = NULL;
       }
@@ -4764,10 +4557,10 @@ doProlog(XML_Parser parser,
       /* fall through */
     case XML_ROLE_ENTITY_SYSTEM_ID:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        parser->m_declEntity->systemId = poolStoreString(&dtd->pool, enc,
-                                               s + enc->minBytesPerChar,
-                                               next - enc->minBytesPerChar);
-        if (!parser->m_declEntity->systemId)
+        parser->m_declEntity->systemId
+            = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                              next - enc->minBytesPerChar);
+        if (! parser->m_declEntity->systemId)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity->base = parser->m_curBase;
         poolFinish(&dtd->pool);
@@ -4779,115 +4572,103 @@ doProlog(XML_Parser parser,
       }
       break;
     case XML_ROLE_ENTITY_COMPLETE:
-      if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) {
+      if (dtd->keepProcessing && parser->m_declEntity
+          && parser->m_entityDeclHandler) {
         *eventEndPP = s;
-        parser->m_entityDeclHandler(parser->m_handlerArg,
-                          parser->m_declEntity->name,
-                          parser->m_declEntity->is_param,
-                          0,0,
-                          parser->m_declEntity->base,
-                          parser->m_declEntity->systemId,
-                          parser->m_declEntity->publicId,
-                          0);
+        parser->m_entityDeclHandler(
+            parser->m_handlerArg, parser->m_declEntity->name,
+            parser->m_declEntity->is_param, 0, 0, parser->m_declEntity->base,
+            parser->m_declEntity->systemId, parser->m_declEntity->publicId, 0);
         handleDefault = XML_FALSE;
       }
       break;
     case XML_ROLE_ENTITY_NOTATION_NAME:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        parser->m_declEntity->notation = poolStoreString(&dtd->pool, enc, s, next);
-        if (!parser->m_declEntity->notation)
+        parser->m_declEntity->notation
+            = poolStoreString(&dtd->pool, enc, s, next);
+        if (! parser->m_declEntity->notation)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&dtd->pool);
         if (parser->m_unparsedEntityDeclHandler) {
           *eventEndPP = s;
-          parser->m_unparsedEntityDeclHandler(parser->m_handlerArg,
-                                    parser->m_declEntity->name,
-                                    parser->m_declEntity->base,
-                                    parser->m_declEntity->systemId,
-                                    parser->m_declEntity->publicId,
-                                    parser->m_declEntity->notation);
+          parser->m_unparsedEntityDeclHandler(
+              parser->m_handlerArg, parser->m_declEntity->name,
+              parser->m_declEntity->base, parser->m_declEntity->systemId,
+              parser->m_declEntity->publicId, parser->m_declEntity->notation);
           handleDefault = XML_FALSE;
-        }
-        else if (parser->m_entityDeclHandler) {
+        } else if (parser->m_entityDeclHandler) {
           *eventEndPP = s;
-          parser->m_entityDeclHandler(parser->m_handlerArg,
-                            parser->m_declEntity->name,
-                            0,0,0,
-                            parser->m_declEntity->base,
-                            parser->m_declEntity->systemId,
-                            parser->m_declEntity->publicId,
-                            parser->m_declEntity->notation);
+          parser->m_entityDeclHandler(
+              parser->m_handlerArg, parser->m_declEntity->name, 0, 0, 0,
+              parser->m_declEntity->base, parser->m_declEntity->systemId,
+              parser->m_declEntity->publicId, parser->m_declEntity->notation);
           handleDefault = XML_FALSE;
         }
       }
       break;
-    case XML_ROLE_GENERAL_ENTITY_NAME:
-      {
-        if (XmlPredefinedEntityName(enc, s, next)) {
-          parser->m_declEntity = NULL;
-          break;
-        }
-        if (dtd->keepProcessing) {
-          const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
-          if (!name)
-            return XML_ERROR_NO_MEMORY;
-          parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->generalEntities, name,
-                                        sizeof(ENTITY));
-          if (!parser->m_declEntity)
-            return XML_ERROR_NO_MEMORY;
-          if (parser->m_declEntity->name != name) {
-            poolDiscard(&dtd->pool);
-            parser->m_declEntity = NULL;
-          }
-          else {
-            poolFinish(&dtd->pool);
-            parser->m_declEntity->publicId = NULL;
-            parser->m_declEntity->is_param = XML_FALSE;
-            /* if we have a parent parser or are reading an internal parameter
-               entity, then the entity declaration is not considered "internal"
-            */
-            parser->m_declEntity->is_internal = !(parser->m_parentParser || parser->m_openInternalEntities);
-            if (parser->m_entityDeclHandler)
-              handleDefault = XML_FALSE;
-          }
-        }
-        else {
+    case XML_ROLE_GENERAL_ENTITY_NAME: {
+      if (XmlPredefinedEntityName(enc, s, next)) {
+        parser->m_declEntity = NULL;
+        break;
+      }
+      if (dtd->keepProcessing) {
+        const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
+        if (! name)
+          return XML_ERROR_NO_MEMORY;
+        parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->generalEntities,
+                                                name, sizeof(ENTITY));
+        if (! parser->m_declEntity)
+          return XML_ERROR_NO_MEMORY;
+        if (parser->m_declEntity->name != name) {
           poolDiscard(&dtd->pool);
           parser->m_declEntity = NULL;
+        } else {
+          poolFinish(&dtd->pool);
+          parser->m_declEntity->publicId = NULL;
+          parser->m_declEntity->is_param = XML_FALSE;
+          /* if we have a parent parser or are reading an internal parameter
+             entity, then the entity declaration is not considered "internal"
+          */
+          parser->m_declEntity->is_internal
+              = ! (parser->m_parentParser || parser->m_openInternalEntities);
+          if (parser->m_entityDeclHandler)
+            handleDefault = XML_FALSE;
         }
+      } else {
+        poolDiscard(&dtd->pool);
+        parser->m_declEntity = NULL;
       }
-      break;
+    } break;
     case XML_ROLE_PARAM_ENTITY_NAME:
 #ifdef XML_DTD
       if (dtd->keepProcessing) {
         const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
-        if (!name)
+        if (! name)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->paramEntities,
-                                           name, sizeof(ENTITY));
-        if (!parser->m_declEntity)
+                                                name, sizeof(ENTITY));
+        if (! parser->m_declEntity)
           return XML_ERROR_NO_MEMORY;
         if (parser->m_declEntity->name != name) {
           poolDiscard(&dtd->pool);
           parser->m_declEntity = NULL;
-        }
-        else {
+        } else {
           poolFinish(&dtd->pool);
           parser->m_declEntity->publicId = NULL;
           parser->m_declEntity->is_param = XML_TRUE;
           /* if we have a parent parser or are reading an internal parameter
              entity, then the entity declaration is not considered "internal"
           */
-          parser->m_declEntity->is_internal = !(parser->m_parentParser || parser->m_openInternalEntities);
+          parser->m_declEntity->is_internal
+              = ! (parser->m_parentParser || parser->m_openInternalEntities);
           if (parser->m_entityDeclHandler)
             handleDefault = XML_FALSE;
         }
-      }
-      else {
+      } else {
         poolDiscard(&dtd->pool);
         parser->m_declEntity = NULL;
       }
-#else /* not XML_DTD */
+#else  /* not XML_DTD */
       parser->m_declEntity = NULL;
 #endif /* XML_DTD */
       break;
@@ -4895,22 +4676,23 @@ doProlog(XML_Parser parser,
       parser->m_declNotationPublicId = NULL;
       parser->m_declNotationName = NULL;
       if (parser->m_notationDeclHandler) {
-        parser->m_declNotationName = poolStoreString(&parser->m_tempPool, enc, s, next);
-        if (!parser->m_declNotationName)
+        parser->m_declNotationName
+            = poolStoreString(&parser->m_tempPool, enc, s, next);
+        if (! parser->m_declNotationName)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
       break;
     case XML_ROLE_NOTATION_PUBLIC_ID:
-      if (!XmlIsPublicId(enc, s, next, eventPP))
+      if (! XmlIsPublicId(enc, s, next, eventPP))
         return XML_ERROR_PUBLICID;
-      if (parser->m_declNotationName) {  /* means m_notationDeclHandler != NULL */
-        XML_Char *tem = poolStoreString(&parser->m_tempPool,
-                                        enc,
+      if (parser
+              ->m_declNotationName) { /* means m_notationDeclHandler != NULL */
+        XML_Char *tem = poolStoreString(&parser->m_tempPool, enc,
                                         s + enc->minBytesPerChar,
                                         next - enc->minBytesPerChar);
-        if (!tem)
+        if (! tem)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(tem);
         parser->m_declNotationPublicId = tem;
@@ -4920,18 +4702,15 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_NOTATION_SYSTEM_ID:
       if (parser->m_declNotationName && parser->m_notationDeclHandler) {
-        const XML_Char *systemId
-          = poolStoreString(&parser->m_tempPool, enc,
-                            s + enc->minBytesPerChar,
-                            next - enc->minBytesPerChar);
-        if (!systemId)
+        const XML_Char *systemId = poolStoreString(&parser->m_tempPool, enc,
+                                                   s + enc->minBytesPerChar,
+                                                   next - enc->minBytesPerChar);
+        if (! systemId)
           return XML_ERROR_NO_MEMORY;
         *eventEndPP = s;
-        parser->m_notationDeclHandler(parser->m_handlerArg,
-                            parser->m_declNotationName,
-                            parser->m_curBase,
-                            systemId,
-                            parser->m_declNotationPublicId);
+        parser->m_notationDeclHandler(
+            parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase,
+            systemId, parser->m_declNotationPublicId);
         handleDefault = XML_FALSE;
       }
       poolClear(&parser->m_tempPool);
@@ -4939,11 +4718,9 @@ doProlog(XML_Parser parser,
     case XML_ROLE_NOTATION_NO_SYSTEM_ID:
       if (parser->m_declNotationPublicId && parser->m_notationDeclHandler) {
         *eventEndPP = s;
-        parser->m_notationDeclHandler(parser->m_handlerArg,
-                            parser->m_declNotationName,
-                            parser->m_curBase,
-                            0,
-                            parser->m_declNotationPublicId);
+        parser->m_notationDeclHandler(
+            parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase,
+            0, parser->m_declNotationPublicId);
         handleDefault = XML_FALSE;
       }
       poolClear(&parser->m_tempPool);
@@ -4960,42 +4737,44 @@ doProlog(XML_Parser parser,
         return XML_ERROR_SYNTAX;
       }
 #ifdef XML_DTD
-    case XML_ROLE_IGNORE_SECT:
-      {
-        enum XML_Error result;
-        if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        handleDefault = XML_FALSE;
-        result = doIgnoreSection(parser, enc, &next, end, nextPtr, haveMore);
-        if (result != XML_ERROR_NONE)
-          return result;
-        else if (!next) {
-          parser->m_processor = ignoreSectionProcessor;
-          return result;
-        }
+    case XML_ROLE_IGNORE_SECT: {
+      enum XML_Error result;
+      if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      handleDefault = XML_FALSE;
+      result = doIgnoreSection(parser, enc, &next, end, nextPtr, haveMore);
+      if (result != XML_ERROR_NONE)
+        return result;
+      else if (! next) {
+        parser->m_processor = ignoreSectionProcessor;
+        return result;
       }
-      break;
+    } break;
 #endif /* XML_DTD */
     case XML_ROLE_GROUP_OPEN:
       if (parser->m_prologState.level >= parser->m_groupSize) {
         if (parser->m_groupSize) {
-          char *temp = (char *)REALLOC(parser, parser->m_groupConnector, parser->m_groupSize *= 2);
-          if (temp == NULL) {
-            parser->m_groupSize /= 2;
-            return XML_ERROR_NO_MEMORY;
+          {
+            char *const new_connector = (char *)REALLOC(
+                parser, parser->m_groupConnector, parser->m_groupSize *= 2);
+            if (new_connector == NULL) {
+              parser->m_groupSize /= 2;
+              return XML_ERROR_NO_MEMORY;
+            }
+            parser->m_groupConnector = new_connector;
           }
-          parser->m_groupConnector = temp;
+
           if (dtd->scaffIndex) {
-            int *temp = (int *)REALLOC(parser, dtd->scaffIndex,
-                          parser->m_groupSize * sizeof(int));
-            if (temp == NULL)
+            int *const new_scaff_index = (int *)REALLOC(
+                parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int));
+            if (new_scaff_index == NULL)
               return XML_ERROR_NO_MEMORY;
-            dtd->scaffIndex = temp;
+            dtd->scaffIndex = new_scaff_index;
           }
-        }
-        else {
-          parser->m_groupConnector = (char *)MALLOC(parser, parser->m_groupSize = 32);
-          if (!parser->m_groupConnector) {
+        } else {
+          parser->m_groupConnector
+              = (char *)MALLOC(parser, parser->m_groupSize = 32);
+          if (! parser->m_groupConnector) {
             parser->m_groupSize = 0;
             return XML_ERROR_NO_MEMORY;
           }
@@ -5006,6 +4785,7 @@ doProlog(XML_Parser parser,
         int myindex = nextScaffoldPart(parser);
         if (myindex < 0)
           return XML_ERROR_NO_MEMORY;
+        assert(dtd->scaffIndex != NULL);
         dtd->scaffIndex[dtd->scaffLevel] = myindex;
         dtd->scaffLevel++;
         dtd->scaffold[myindex].type = XML_CTYPE_SEQ;
@@ -5024,10 +4804,9 @@ doProlog(XML_Parser parser,
       if (parser->m_groupConnector[parser->m_prologState.level] == ASCII_COMMA)
         return XML_ERROR_SYNTAX;
       if (dtd->in_eldecl
-          && !parser->m_groupConnector[parser->m_prologState.level]
+          && ! parser->m_groupConnector[parser->m_prologState.level]
           && (dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]].type
-              != XML_CTYPE_MIXED)
-          ) {
+              != XML_CTYPE_MIXED)) {
         dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]].type
             = XML_CTYPE_CHOICE;
         if (parser->m_elementDeclHandler)
@@ -5039,15 +4818,14 @@ doProlog(XML_Parser parser,
 #ifdef XML_DTD
     case XML_ROLE_INNER_PARAM_ENTITY_REF:
       dtd->hasParamEntityRefs = XML_TRUE;
-      if (!parser->m_paramEntityParsing)
+      if (! parser->m_paramEntityParsing)
         dtd->keepProcessing = dtd->standalone;
       else {
         const XML_Char *name;
         ENTITY *entity;
-        name = poolStoreString(&dtd->pool, enc,
-                                s + enc->minBytesPerChar,
-                                next - enc->minBytesPerChar);
-        if (!name)
+        name = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                               next - enc->minBytesPerChar);
+        if (! name)
           return XML_ERROR_NO_MEMORY;
         entity = (ENTITY *)lookup(parser, &dtd->paramEntities, name, 0);
         poolDiscard(&dtd->pool);
@@ -5055,13 +4833,12 @@ doProlog(XML_Parser parser,
            if yes, check that the entity exists, and that it is internal,
            otherwise call the skipped entity handler
         */
-        if (parser->m_prologState.documentEntity &&
-            (dtd->standalone
-             ? !parser->m_openInternalEntities
-             : !dtd->hasParamEntityRefs)) {
-          if (!entity)
+        if (parser->m_prologState.documentEntity
+            && (dtd->standalone ? ! parser->m_openInternalEntities
+                                : ! dtd->hasParamEntityRefs)) {
+          if (! entity)
             return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal) {
+          else if (! entity->is_internal) {
             /* It's hard to exhaustively search the code to be sure,
              * but there doesn't seem to be a way of executing the
              * following line.  There are two cases:
@@ -5084,11 +4861,11 @@ doProlog(XML_Parser parser,
              */
             return XML_ERROR_ENTITY_DECLARED_IN_PE; /* LCOV_EXCL_LINE */
           }
-        }
-        else if (!entity) {
+        } else if (! entity) {
           dtd->keepProcessing = dtd->standalone;
           /* cannot report skipped entities in declarations */
-          if ((role == XML_ROLE_PARAM_ENTITY_REF) && parser->m_skippedEntityHandler) {
+          if ((role == XML_ROLE_PARAM_ENTITY_REF)
+              && parser->m_skippedEntityHandler) {
             parser->m_skippedEntityHandler(parser->m_handlerArg, name, 1);
             handleDefault = XML_FALSE;
           }
@@ -5098,8 +4875,8 @@ doProlog(XML_Parser parser,
           return XML_ERROR_RECURSIVE_ENTITY_REF;
         if (entity->textPtr) {
           enum XML_Error result;
-          XML_Bool betweenDecl =
-            (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE);
+          XML_Bool betweenDecl
+              = (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE);
           result = processInternalEntity(parser, entity, betweenDecl);
           if (result != XML_ERROR_NONE)
             return result;
@@ -5109,39 +4886,35 @@ doProlog(XML_Parser parser,
         if (parser->m_externalEntityRefHandler) {
           dtd->paramEntityRead = XML_FALSE;
           entity->open = XML_TRUE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId)) {
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId)) {
             entity->open = XML_FALSE;
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           }
           entity->open = XML_FALSE;
           handleDefault = XML_FALSE;
-          if (!dtd->paramEntityRead) {
+          if (! dtd->paramEntityRead) {
             dtd->keepProcessing = dtd->standalone;
             break;
           }
-        }
-        else {
+        } else {
           dtd->keepProcessing = dtd->standalone;
           break;
         }
       }
 #endif /* XML_DTD */
-      if (!dtd->standalone &&
-          parser->m_notStandaloneHandler &&
-          !parser->m_notStandaloneHandler(parser->m_handlerArg))
+      if (! dtd->standalone && parser->m_notStandaloneHandler
+          && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
         return XML_ERROR_NOT_STANDALONE;
       break;
 
-    /* Element declaration stuff */
+      /* Element declaration stuff */
 
     case XML_ROLE_ELEMENT_NAME:
       if (parser->m_elementDeclHandler) {
         parser->m_declElementType = getElementType(parser, enc, s, next);
-        if (!parser->m_declElementType)
+        if (! parser->m_declElementType)
           return XML_ERROR_NO_MEMORY;
         dtd->scaffLevel = 0;
         dtd->scaffCount = 0;
@@ -5154,18 +4927,19 @@ doProlog(XML_Parser parser,
     case XML_ROLE_CONTENT_EMPTY:
       if (dtd->in_eldecl) {
         if (parser->m_elementDeclHandler) {
-          XML_Content * content = (XML_Content *) MALLOC(parser, sizeof(XML_Content));
-          if (!content)
+          XML_Content *content
+              = (XML_Content *)MALLOC(parser, sizeof(XML_Content));
+          if (! content)
             return XML_ERROR_NO_MEMORY;
           content->quant = XML_CQUANT_NONE;
           content->name = NULL;
           content->numchildren = 0;
           content->children = NULL;
-          content->type = ((role == XML_ROLE_CONTENT_ANY) ?
-                           XML_CTYPE_ANY :
-                           XML_CTYPE_EMPTY);
+          content->type = ((role == XML_ROLE_CONTENT_ANY) ? XML_CTYPE_ANY
+                                                          : XML_CTYPE_EMPTY);
           *eventEndPP = s;
-          parser->m_elementDeclHandler(parser->m_handlerArg, parser->m_declElementType->name, content);
+          parser->m_elementDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name, content);
           handleDefault = XML_FALSE;
         }
         dtd->in_eldecl = XML_FALSE;
@@ -5197,22 +4971,22 @@ doProlog(XML_Parser parser,
         ELEMENT_TYPE *el;
         const XML_Char *name;
         int nameLen;
-        const char *nxt = (quant == XML_CQUANT_NONE
-                           ? next
-                           : next - enc->minBytesPerChar);
+        const char *nxt
+            = (quant == XML_CQUANT_NONE ? next : next - enc->minBytesPerChar);
         int myindex = nextScaffoldPart(parser);
         if (myindex < 0)
           return XML_ERROR_NO_MEMORY;
         dtd->scaffold[myindex].type = XML_CTYPE_NAME;
         dtd->scaffold[myindex].quant = quant;
         el = getElementType(parser, enc, s, nxt);
-        if (!el)
+        if (! el)
           return XML_ERROR_NO_MEMORY;
         name = el->name;
         dtd->scaffold[myindex].name = name;
         nameLen = 0;
-        for (; name[nameLen++]; );
-        dtd->contentStringLen +=  nameLen;
+        for (; name[nameLen++];)
+          ;
+        dtd->contentStringLen += nameLen;
         if (parser->m_elementDeclHandler)
           handleDefault = XML_FALSE;
       }
@@ -5236,12 +5010,13 @@ doProlog(XML_Parser parser,
         dtd->scaffLevel--;
         dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel]].quant = quant;
         if (dtd->scaffLevel == 0) {
-          if (!handleDefault) {
+          if (! handleDefault) {
             XML_Content *model = build_model(parser);
-            if (!model)
+            if (! model)
               return XML_ERROR_NO_MEMORY;
             *eventEndPP = s;
-            parser->m_elementDeclHandler(parser->m_handlerArg, parser->m_declElementType->name, model);
+            parser->m_elementDeclHandler(
+                parser->m_handlerArg, parser->m_declElementType->name, model);
           }
           dtd->in_eldecl = XML_FALSE;
           dtd->contentStringLen = 0;
@@ -5251,12 +5026,12 @@ doProlog(XML_Parser parser,
       /* End element declaration stuff */
 
     case XML_ROLE_PI:
-      if (!reportProcessingInstruction(parser, enc, s, next))
+      if (! reportProcessingInstruction(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       handleDefault = XML_FALSE;
       break;
     case XML_ROLE_COMMENT:
-      if (!reportComment(parser, enc, s, next))
+      if (! reportComment(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       handleDefault = XML_FALSE;
       break;
@@ -5307,11 +5082,8 @@ doProlog(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-epilogProcessor(XML_Parser parser,
-                const char *s,
-                const char *end,
-                const char **nextPtr)
-{
+epilogProcessor(XML_Parser parser, const char *s, const char *end,
+                const char **nextPtr) {
   parser->m_processor = epilogProcessor;
   parser->m_eventPtr = s;
   for (;;) {
@@ -5336,24 +5108,24 @@ epilogProcessor(XML_Parser parser,
         reportDefault(parser, parser->m_encoding, s, next);
       break;
     case XML_TOK_PI:
-      if (!reportProcessingInstruction(parser, parser->m_encoding, s, next))
+      if (! reportProcessingInstruction(parser, parser->m_encoding, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_COMMENT:
-      if (!reportComment(parser, parser->m_encoding, s, next))
+      if (! reportComment(parser, parser->m_encoding, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_INVALID:
       parser->m_eventPtr = next;
       return XML_ERROR_INVALID_TOKEN;
     case XML_TOK_PARTIAL:
-      if (!parser->m_parsingStatus.finalBuffer) {
+      if (! parser->m_parsingStatus.finalBuffer) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
       return XML_ERROR_UNCLOSED_TOKEN;
     case XML_TOK_PARTIAL_CHAR:
-      if (!parser->m_parsingStatus.finalBuffer) {
+      if (! parser->m_parsingStatus.finalBuffer) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -5368,15 +5140,13 @@ epilogProcessor(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
 }
 
 static enum XML_Error
-processInternalEntity(XML_Parser parser, ENTITY *entity,
-                      XML_Bool betweenDecl)
-{
+processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) {
   const char *textStart, *textEnd;
   const char *next;
   enum XML_Error result;
@@ -5385,10 +5155,10 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
   if (parser->m_freeInternalEntities) {
     openEntity = parser->m_freeInternalEntities;
     parser->m_freeInternalEntities = openEntity->next;
-  }
-  else {
-    openEntity = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY));
-    if (!openEntity)
+  } else {
+    openEntity
+        = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY));
+    if (! openEntity)
       return XML_ERROR_NO_MEMORY;
   }
   entity->open = XML_TRUE;
@@ -5407,21 +5177,20 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
 
 #ifdef XML_DTD
   if (entity->is_param) {
-    int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
-    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok,
-                      next, &next, XML_FALSE);
-  }
-  else
+    int tok
+        = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
+    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
+                      tok, next, &next, XML_FALSE, XML_FALSE);
+  } else
 #endif /* XML_DTD */
-    result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, textStart,
-                       textEnd, &next, XML_FALSE);
+    result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding,
+                       textStart, textEnd, &next, XML_FALSE);
 
   if (result == XML_ERROR_NONE) {
     if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
       entity->processed = (int)(next - textStart);
       parser->m_processor = internalEntityProcessor;
-    }
-    else {
+    } else {
       entity->open = XML_FALSE;
       parser->m_openInternalEntities = openEntity->next;
       /* put openEntity back in list of free instances */
@@ -5433,17 +5202,14 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
 }
 
 static enum XML_Error PTRCALL
-internalEntityProcessor(XML_Parser parser,
-                        const char *s,
-                        const char *end,
-                        const char **nextPtr)
-{
+internalEntityProcessor(XML_Parser parser, const char *s, const char *end,
+                        const char **nextPtr) {
   ENTITY *entity;
   const char *textStart, *textEnd;
   const char *next;
   enum XML_Error result;
   OPEN_INTERNAL_ENTITY *openEntity = parser->m_openInternalEntities;
-  if (!openEntity)
+  if (! openEntity)
     return XML_ERROR_UNEXPECTED_STATE;
 
   entity = openEntity->entity;
@@ -5454,22 +5220,23 @@ internalEntityProcessor(XML_Parser parser,
 
 #ifdef XML_DTD
   if (entity->is_param) {
-    int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
-    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok,
-                      next, &next, XML_FALSE);
-  }
-  else
+    int tok
+        = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
+    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
+                      tok, next, &next, XML_FALSE, XML_TRUE);
+  } else
 #endif /* XML_DTD */
-    result = doContent(parser, openEntity->startTagLevel, parser->m_internalEncoding,
-                       textStart, textEnd, &next, XML_FALSE);
+    result = doContent(parser, openEntity->startTagLevel,
+                       parser->m_internalEncoding, textStart, textEnd, &next,
+                       XML_FALSE);
 
   if (result != XML_ERROR_NONE)
     return result;
-  else if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
+  else if (textEnd != next
+           && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
     entity->processed = (int)(next - (char *)entity->textPtr);
     return result;
-  }
-  else {
+  } else {
     entity->open = XML_FALSE;
     parser->m_openInternalEntities = openEntity->next;
     /* put openEntity back in list of free instances */
@@ -5483,49 +5250,45 @@ internalEntityProcessor(XML_Parser parser,
     parser->m_processor = prologProcessor;
     tok = XmlPrologTok(parser->m_encoding, s, end, &next);
     return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
-                    (XML_Bool)!parser->m_parsingStatus.finalBuffer);
-  }
-  else
+                    (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
+  } else
 #endif /* XML_DTD */
   {
     parser->m_processor = contentProcessor;
     /* see externalEntityContentProcessor vs contentProcessor */
-    return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding, s, end,
-                     nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+    return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding,
+                     s, end, nextPtr,
+                     (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   }
 }
 
 static enum XML_Error PTRCALL
-errorProcessor(XML_Parser parser,
-               const char *UNUSED_P(s),
-               const char *UNUSED_P(end),
-               const char **UNUSED_P(nextPtr))
-{
+errorProcessor(XML_Parser parser, const char *s, const char *end,
+               const char **nextPtr) {
+  UNUSED_P(s);
+  UNUSED_P(end);
+  UNUSED_P(nextPtr);
   return parser->m_errorCode;
 }
 
 static enum XML_Error
 storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
-                    const char *ptr, const char *end,
-                    STRING_POOL *pool)
-{
-  enum XML_Error result = appendAttributeValue(parser, enc, isCdata, ptr,
-                                               end, pool);
+                    const char *ptr, const char *end, STRING_POOL *pool) {
+  enum XML_Error result
+      = appendAttributeValue(parser, enc, isCdata, ptr, end, pool);
   if (result)
     return result;
-  if (!isCdata && poolLength(pool) && poolLastChar(pool) == 0x20)
+  if (! isCdata && poolLength(pool) && poolLastChar(pool) == 0x20)
     poolChop(pool);
-  if (!poolAppendChar(pool, XML_T('\0')))
+  if (! poolAppendChar(pool, XML_T('\0')))
     return XML_ERROR_NO_MEMORY;
   return XML_ERROR_NONE;
 }
 
 static enum XML_Error
 appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
-                     const char *ptr, const char *end,
-                     STRING_POOL *pool)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+                     const char *ptr, const char *end, STRING_POOL *pool) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   for (;;) {
     const char *next;
     int tok = XmlAttributeValueTok(enc, ptr, end, &next);
@@ -5540,38 +5303,35 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
       if (enc == parser->m_encoding)
         parser->m_eventPtr = ptr;
       return XML_ERROR_INVALID_TOKEN;
-    case XML_TOK_CHAR_REF:
-      {
-        XML_Char buf[XML_ENCODE_MAX];
-        int i;
-        int n = XmlCharRefNumber(enc, ptr);
-        if (n < 0) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_BAD_CHAR_REF;
-        }
-        if (!isCdata
-            && n == 0x20 /* space */
-            && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
-          break;
-        n = XmlEncode(n, (ICHAR *)buf);
-        /* The XmlEncode() functions can never return 0 here.  That
-         * error return happens if the code point passed in is either
-         * negative or greater than or equal to 0x110000.  The
-         * XmlCharRefNumber() functions will all return a number
-         * strictly less than 0x110000 or a negative value if an error
-         * occurred.  The negative value is intercepted above, so
-         * XmlEncode() is never passed a value it might return an
-         * error for.
-         */
-        for (i = 0; i < n; i++) {
-          if (!poolAppendChar(pool, buf[i]))
-            return XML_ERROR_NO_MEMORY;
-        }
+    case XML_TOK_CHAR_REF: {
+      XML_Char buf[XML_ENCODE_MAX];
+      int i;
+      int n = XmlCharRefNumber(enc, ptr);
+      if (n < 0) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_BAD_CHAR_REF;
       }
-      break;
+      if (! isCdata && n == 0x20 /* space */
+          && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
+        break;
+      n = XmlEncode(n, (ICHAR *)buf);
+      /* The XmlEncode() functions can never return 0 here.  That
+       * error return happens if the code point passed in is either
+       * negative or greater than or equal to 0x110000.  The
+       * XmlCharRefNumber() functions will all return a number
+       * strictly less than 0x110000 or a negative value if an error
+       * occurred.  The negative value is intercepted above, so
+       * XmlEncode() is never passed a value it might return an
+       * error for.
+       */
+      for (i = 0; i < n; i++) {
+        if (! poolAppendChar(pool, buf[i]))
+          return XML_ERROR_NO_MEMORY;
+      }
+    } break;
     case XML_TOK_DATA_CHARS:
-      if (!poolAppend(pool, enc, ptr, next))
+      if (! poolAppend(pool, enc, ptr, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_TRAILING_CR:
@@ -5579,109 +5339,103 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
       /* fall through */
     case XML_TOK_ATTRIBUTE_VALUE_S:
     case XML_TOK_DATA_NEWLINE:
-      if (!isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
+      if (! isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
         break;
-      if (!poolAppendChar(pool, 0x20))
+      if (! poolAppendChar(pool, 0x20))
         return XML_ERROR_NO_MEMORY;
       break;
-    case XML_TOK_ENTITY_REF:
-      {
-        const XML_Char *name;
-        ENTITY *entity;
-        char checkEntityDecl;
-        XML_Char ch = (XML_Char) XmlPredefinedEntityName(enc,
-                                              ptr + enc->minBytesPerChar,
-                                              next - enc->minBytesPerChar);
-        if (ch) {
-          if (!poolAppendChar(pool, ch))
-                return XML_ERROR_NO_MEMORY;
-          break;
-        }
-        name = poolStoreString(&parser->m_temp2Pool, enc,
-                               ptr + enc->minBytesPerChar,
-                               next - enc->minBytesPerChar);
-        if (!name)
+    case XML_TOK_ENTITY_REF: {
+      const XML_Char *name;
+      ENTITY *entity;
+      char checkEntityDecl;
+      XML_Char ch = (XML_Char)XmlPredefinedEntityName(
+          enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar);
+      if (ch) {
+        if (! poolAppendChar(pool, ch))
           return XML_ERROR_NO_MEMORY;
-        entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
-        poolDiscard(&parser->m_temp2Pool);
-        /* First, determine if a check for an existing declaration is needed;
-           if yes, check that the entity exists, and that it is internal.
-        */
-        if (pool == &dtd->pool)  /* are we called from prolog? */
-          checkEntityDecl =
+        break;
+      }
+      name = poolStoreString(&parser->m_temp2Pool, enc,
+                             ptr + enc->minBytesPerChar,
+                             next - enc->minBytesPerChar);
+      if (! name)
+        return XML_ERROR_NO_MEMORY;
+      entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
+      poolDiscard(&parser->m_temp2Pool);
+      /* First, determine if a check for an existing declaration is needed;
+         if yes, check that the entity exists, and that it is internal.
+      */
+      if (pool == &dtd->pool) /* are we called from prolog? */
+        checkEntityDecl =
 #ifdef XML_DTD
-              parser->m_prologState.documentEntity &&
+            parser->m_prologState.documentEntity &&
 #endif /* XML_DTD */
-              (dtd->standalone
-               ? !parser->m_openInternalEntities
-               : !dtd->hasParamEntityRefs);
-        else /* if (pool == &parser->m_tempPool): we are called from content */
-          checkEntityDecl = !dtd->hasParamEntityRefs || dtd->standalone;
-        if (checkEntityDecl) {
-          if (!entity)
-            return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal)
-            return XML_ERROR_ENTITY_DECLARED_IN_PE;
-        }
-        else if (!entity) {
-          /* Cannot report skipped entity here - see comments on
-             parser->m_skippedEntityHandler.
-          if (parser->m_skippedEntityHandler)
-            parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
-          */
-          /* Cannot call the default handler because this would be
-             out of sync with the call to the startElementHandler.
-          if ((pool == &parser->m_tempPool) && parser->m_defaultHandler)
-            reportDefault(parser, enc, ptr, next);
-          */
-          break;
-        }
-        if (entity->open) {
-          if (enc == parser->m_encoding) {
-            /* It does not appear that this line can be executed.
-             *
-             * The "if (entity->open)" check catches recursive entity
-             * definitions.  In order to be called with an open
-             * entity, it must have gone through this code before and
-             * been through the recursive call to
-             * appendAttributeValue() some lines below.  That call
-             * sets the local encoding ("enc") to the parser's
-             * internal encoding (internal_utf8 or internal_utf16),
-             * which can never be the same as the principle encoding.
-             * It doesn't appear there is another code path that gets
-             * here with entity->open being TRUE.
-             *
-             * Since it is not certain that this logic is watertight,
-             * we keep the line and merely exclude it from coverage
-             * tests.
-             */
-            parser->m_eventPtr = ptr; /* LCOV_EXCL_LINE */
-          }
-          return XML_ERROR_RECURSIVE_ENTITY_REF;
-        }
-        if (entity->notation) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_BINARY_ENTITY_REF;
-        }
-        if (!entity->textPtr) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF;
-        }
-        else {
-          enum XML_Error result;
-          const XML_Char *textEnd = entity->textPtr + entity->textLen;
-          entity->open = XML_TRUE;
-          result = appendAttributeValue(parser, parser->m_internalEncoding, isCdata,
-                                        (char *)entity->textPtr,
-                                        (char *)textEnd, pool);
-          entity->open = XML_FALSE;
-          if (result)
-            return result;
+            (dtd->standalone ? ! parser->m_openInternalEntities
+                             : ! dtd->hasParamEntityRefs);
+      else /* if (pool == &parser->m_tempPool): we are called from content */
+        checkEntityDecl = ! dtd->hasParamEntityRefs || dtd->standalone;
+      if (checkEntityDecl) {
+        if (! entity)
+          return XML_ERROR_UNDEFINED_ENTITY;
+        else if (! entity->is_internal)
+          return XML_ERROR_ENTITY_DECLARED_IN_PE;
+      } else if (! entity) {
+        /* Cannot report skipped entity here - see comments on
+           parser->m_skippedEntityHandler.
+        if (parser->m_skippedEntityHandler)
+          parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+        */
+        /* Cannot call the default handler because this would be
+           out of sync with the call to the startElementHandler.
+        if ((pool == &parser->m_tempPool) && parser->m_defaultHandler)
+          reportDefault(parser, enc, ptr, next);
+        */
+        break;
+      }
+      if (entity->open) {
+        if (enc == parser->m_encoding) {
+          /* It does not appear that this line can be executed.
+           *
+           * The "if (entity->open)" check catches recursive entity
+           * definitions.  In order to be called with an open
+           * entity, it must have gone through this code before and
+           * been through the recursive call to
+           * appendAttributeValue() some lines below.  That call
+           * sets the local encoding ("enc") to the parser's
+           * internal encoding (internal_utf8 or internal_utf16),
+           * which can never be the same as the principle encoding.
+           * It doesn't appear there is another code path that gets
+           * here with entity->open being TRUE.
+           *
+           * Since it is not certain that this logic is watertight,
+           * we keep the line and merely exclude it from coverage
+           * tests.
+           */
+          parser->m_eventPtr = ptr; /* LCOV_EXCL_LINE */
         }
+        return XML_ERROR_RECURSIVE_ENTITY_REF;
       }
-      break;
+      if (entity->notation) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_BINARY_ENTITY_REF;
+      }
+      if (! entity->textPtr) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF;
+      } else {
+        enum XML_Error result;
+        const XML_Char *textEnd = entity->textPtr + entity->textLen;
+        entity->open = XML_TRUE;
+        result = appendAttributeValue(parser, parser->m_internalEncoding,
+                                      isCdata, (char *)entity->textPtr,
+                                      (char *)textEnd, pool);
+        entity->open = XML_FALSE;
+        if (result)
+          return result;
+      }
+    } break;
     default:
       /* The only token returned by XmlAttributeValueTok() that does
        * not have an explicit case here is XML_TOK_PARTIAL_CHAR.
@@ -5705,12 +5459,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
 }
 
 static enum XML_Error
-storeEntityValue(XML_Parser parser,
-                 const ENCODING *enc,
-                 const char *entityTextPtr,
-                 const char *entityTextEnd)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+storeEntityValue(XML_Parser parser, const ENCODING *enc,
+                 const char *entityTextPtr, const char *entityTextEnd) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   STRING_POOL *pool = &(dtd->entityValuePool);
   enum XML_Error result = XML_ERROR_NONE;
 #ifdef XML_DTD
@@ -5720,8 +5471,8 @@ storeEntityValue(XML_Parser parser,
   /* never return Null for the value argument in EntityDeclHandler,
      since this would indicate an external entity; therefore we
      have to make sure that entityValuePool.start is not null */
-  if (!pool->blocks) {
-    if (!poolGrow(pool))
+  if (! pool->blocks) {
+    if (! poolGrow(pool))
       return XML_ERROR_NO_MEMORY;
   }
 
@@ -5737,13 +5488,13 @@ storeEntityValue(XML_Parser parser,
         name = poolStoreString(&parser->m_tempPool, enc,
                                entityTextPtr + enc->minBytesPerChar,
                                next - enc->minBytesPerChar);
-        if (!name) {
+        if (! name) {
           result = XML_ERROR_NO_MEMORY;
           goto endEntityValue;
         }
         entity = (ENTITY *)lookup(parser, &dtd->paramEntities, name, 0);
         poolDiscard(&parser->m_tempPool);
-        if (!entity) {
+        if (! entity) {
           /* not a well-formedness error - see XML 1.0: WFC Entity Declared */
           /* cannot report skipped entity here - see comments on
              parser->m_skippedEntityHandler
@@ -5763,29 +5514,23 @@ storeEntityValue(XML_Parser parser,
           if (parser->m_externalEntityRefHandler) {
             dtd->paramEntityRead = XML_FALSE;
             entity->open = XML_TRUE;
-            if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                          0,
-                                          entity->base,
-                                          entity->systemId,
-                                          entity->publicId)) {
+            if (! parser->m_externalEntityRefHandler(
+                    parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                    entity->systemId, entity->publicId)) {
               entity->open = XML_FALSE;
               result = XML_ERROR_EXTERNAL_ENTITY_HANDLING;
               goto endEntityValue;
             }
             entity->open = XML_FALSE;
-            if (!dtd->paramEntityRead)
+            if (! dtd->paramEntityRead)
               dtd->keepProcessing = dtd->standalone;
-          }
-          else
+          } else
             dtd->keepProcessing = dtd->standalone;
-        }
-        else {
+        } else {
           entity->open = XML_TRUE;
-          result = storeEntityValue(parser,
-                                    parser->m_internalEncoding,
-                                    (char *)entity->textPtr,
-                                    (char *)(entity->textPtr
-                                             + entity->textLen));
+          result = storeEntityValue(
+              parser, parser->m_internalEncoding, (char *)entity->textPtr,
+              (char *)(entity->textPtr + entity->textLen));
           entity->open = XML_FALSE;
           if (result)
             goto endEntityValue;
@@ -5803,7 +5548,7 @@ storeEntityValue(XML_Parser parser,
       goto endEntityValue;
     case XML_TOK_ENTITY_REF:
     case XML_TOK_DATA_CHARS:
-      if (!poolAppend(pool, enc, entityTextPtr, next)) {
+      if (! poolAppend(pool, enc, entityTextPtr, next)) {
         result = XML_ERROR_NO_MEMORY;
         goto endEntityValue;
       }
@@ -5812,42 +5557,40 @@ storeEntityValue(XML_Parser parser,
       next = entityTextPtr + enc->minBytesPerChar;
       /* fall through */
     case XML_TOK_DATA_NEWLINE:
-      if (pool->end == pool->ptr && !poolGrow(pool)) {
-              result = XML_ERROR_NO_MEMORY;
+      if (pool->end == pool->ptr && ! poolGrow(pool)) {
+        result = XML_ERROR_NO_MEMORY;
         goto endEntityValue;
       }
       *(pool->ptr)++ = 0xA;
       break;
-    case XML_TOK_CHAR_REF:
-      {
-        XML_Char buf[XML_ENCODE_MAX];
-        int i;
-        int n = XmlCharRefNumber(enc, entityTextPtr);
-        if (n < 0) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = entityTextPtr;
-          result = XML_ERROR_BAD_CHAR_REF;
+    case XML_TOK_CHAR_REF: {
+      XML_Char buf[XML_ENCODE_MAX];
+      int i;
+      int n = XmlCharRefNumber(enc, entityTextPtr);
+      if (n < 0) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = entityTextPtr;
+        result = XML_ERROR_BAD_CHAR_REF;
+        goto endEntityValue;
+      }
+      n = XmlEncode(n, (ICHAR *)buf);
+      /* The XmlEncode() functions can never return 0 here.  That
+       * error return happens if the code point passed in is either
+       * negative or greater than or equal to 0x110000.  The
+       * XmlCharRefNumber() functions will all return a number
+       * strictly less than 0x110000 or a negative value if an error
+       * occurred.  The negative value is intercepted above, so
+       * XmlEncode() is never passed a value it might return an
+       * error for.
+       */
+      for (i = 0; i < n; i++) {
+        if (pool->end == pool->ptr && ! poolGrow(pool)) {
+          result = XML_ERROR_NO_MEMORY;
           goto endEntityValue;
         }
-        n = XmlEncode(n, (ICHAR *)buf);
-        /* The XmlEncode() functions can never return 0 here.  That
-         * error return happens if the code point passed in is either
-         * negative or greater than or equal to 0x110000.  The
-         * XmlCharRefNumber() functions will all return a number
-         * strictly less than 0x110000 or a negative value if an error
-         * occurred.  The negative value is intercepted above, so
-         * XmlEncode() is never passed a value it might return an
-         * error for.
-         */
-        for (i = 0; i < n; i++) {
-          if (pool->end == pool->ptr && !poolGrow(pool)) {
-            result = XML_ERROR_NO_MEMORY;
-            goto endEntityValue;
-          }
-          *(pool->ptr)++ = buf[i];
-        }
+        *(pool->ptr)++ = buf[i];
       }
-      break;
+    } break;
     case XML_TOK_PARTIAL:
       if (enc == parser->m_encoding)
         parser->m_eventPtr = entityTextPtr;
@@ -5882,8 +5625,7 @@ storeEntityValue(XML_Parser parser,
 }
 
 static void FASTCALL
-normalizeLines(XML_Char *s)
-{
+normalizeLines(XML_Char *s) {
   XML_Char *p;
   for (;; s++) {
     if (*s == XML_T('\0'))
@@ -5897,8 +5639,7 @@ normalizeLines(XML_Char *s)
       *p++ = 0xA;
       if (*++s == 0xA)
         s++;
-    }
-    else
+    } else
       *p++ = *s++;
   } while (*s);
   *p = XML_T('\0');
@@ -5906,12 +5647,11 @@ normalizeLines(XML_Char *s)
 
 static int
 reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
-                            const char *start, const char *end)
-{
+                            const char *start, const char *end) {
   const XML_Char *target;
   XML_Char *data;
   const char *tem;
-  if (!parser->m_processingInstructionHandler) {
+  if (! parser->m_processingInstructionHandler) {
     if (parser->m_defaultHandler)
       reportDefault(parser, enc, start, end);
     return 1;
@@ -5919,13 +5659,12 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
   start += enc->minBytesPerChar * 2;
   tem = start + XmlNameLength(enc, start);
   target = poolStoreString(&parser->m_tempPool, enc, start, tem);
-  if (!target)
+  if (! target)
     return 0;
   poolFinish(&parser->m_tempPool);
-  data = poolStoreString(&parser->m_tempPool, enc,
-                        XmlSkipS(enc, tem),
-                        end - enc->minBytesPerChar*2);
-  if (!data)
+  data = poolStoreString(&parser->m_tempPool, enc, XmlSkipS(enc, tem),
+                         end - enc->minBytesPerChar * 2);
+  if (! data)
     return 0;
   normalizeLines(data);
   parser->m_processingInstructionHandler(parser->m_handlerArg, target, data);
@@ -5934,20 +5673,18 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
 }
 
 static int
-reportComment(XML_Parser parser, const ENCODING *enc,
-              const char *start, const char *end)
-{
+reportComment(XML_Parser parser, const ENCODING *enc, const char *start,
+              const char *end) {
   XML_Char *data;
-  if (!parser->m_commentHandler) {
+  if (! parser->m_commentHandler) {
     if (parser->m_defaultHandler)
       reportDefault(parser, enc, start, end);
     return 1;
   }
-  data = poolStoreString(&parser->m_tempPool,
-                         enc,
+  data = poolStoreString(&parser->m_tempPool, enc,
                          start + enc->minBytesPerChar * 4,
                          end - enc->minBytesPerChar * 3);
-  if (!data)
+  if (! data)
     return 0;
   normalizeLines(data);
   parser->m_commentHandler(parser->m_handlerArg, data);
@@ -5956,9 +5693,8 @@ reportComment(XML_Parser parser, const ENCODING *enc,
 }
 
 static void
-reportDefault(XML_Parser parser, const ENCODING *enc,
-              const char *s, const char *end)
-{
+reportDefault(XML_Parser parser, const ENCODING *enc, const char *s,
+              const char *end) {
   if (MUST_CONVERT(enc, s)) {
     enum XML_Convert_Result convert_res;
     const char **eventPP;
@@ -5966,8 +5702,7 @@ reportDefault(XML_Parser parser, const ENCODING *enc,
     if (enc == parser->m_encoding) {
       eventPP = &parser->m_eventPtr;
       eventEndPP = &parser->m_eventEndPtr;
-    }
-    else {
+    } else {
       /* To get here, two things must be true; the parser must be
        * using a character encoding that is not the same as the
        * encoding passed in, and the encoding passed in must need
@@ -5990,21 +5725,22 @@ reportDefault(XML_Parser parser, const ENCODING *enc,
     }
     do {
       ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-      convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+      convert_res
+          = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
       *eventEndPP = s;
-      parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+      parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf,
+                               (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
       *eventPP = s;
-    } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE));
-  }
-  else
-    parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s, (int)((XML_Char *)end - (XML_Char *)s));
+    } while ((convert_res != XML_CONVERT_COMPLETED)
+             && (convert_res != XML_CONVERT_INPUT_INCOMPLETE));
+  } else
+    parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s,
+                             (int)((XML_Char *)end - (XML_Char *)s));
 }
 
-
 static int
 defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
-                XML_Bool isId, const XML_Char *value, XML_Parser parser)
-{
+                XML_Bool isId, const XML_Char *value, XML_Parser parser) {
   DEFAULT_ATTRIBUTE *att;
   if (value || isId) {
     /* The handling of default attributes gets messed up if we have
@@ -6013,24 +5749,23 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
     for (i = 0; i < type->nDefaultAtts; i++)
       if (attId == type->defaultAtts[i].id)
         return 1;
-    if (isId && !type->idAtt && !attId->xmlns)
+    if (isId && ! type->idAtt && ! attId->xmlns)
       type->idAtt = attId;
   }
   if (type->nDefaultAtts == type->allocDefaultAtts) {
     if (type->allocDefaultAtts == 0) {
       type->allocDefaultAtts = 8;
-      type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC(parser, type->allocDefaultAtts
-                            * sizeof(DEFAULT_ATTRIBUTE));
-      if (!type->defaultAtts) {
+      type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC(
+          parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
+      if (! type->defaultAtts) {
         type->allocDefaultAtts = 0;
         return 0;
       }
-    }
-    else {
+    } else {
       DEFAULT_ATTRIBUTE *temp;
       int count = type->allocDefaultAtts * 2;
-      temp = (DEFAULT_ATTRIBUTE *)
-        REALLOC(parser, type->defaultAtts, (count * sizeof(DEFAULT_ATTRIBUTE)));
+      temp = (DEFAULT_ATTRIBUTE *)REALLOC(parser, type->defaultAtts,
+                                          (count * sizeof(DEFAULT_ATTRIBUTE)));
       if (temp == NULL)
         return 0;
       type->allocDefaultAtts = count;
@@ -6041,30 +5776,29 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
   att->id = attId;
   att->value = value;
   att->isCdata = isCdata;
-  if (!isCdata)
+  if (! isCdata)
     attId->maybeTokenized = XML_TRUE;
   type->nDefaultAtts += 1;
   return 1;
 }
 
 static int
-setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *name;
   for (name = elementType->name; *name; name++) {
     if (*name == XML_T(ASCII_COLON)) {
       PREFIX *prefix;
       const XML_Char *s;
       for (s = elementType->name; s != name; s++) {
-        if (!poolAppendChar(&dtd->pool, *s))
+        if (! poolAppendChar(&dtd->pool, *s))
           return 0;
       }
-      if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+      if (! poolAppendChar(&dtd->pool, XML_T('\0')))
         return 0;
       prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&dtd->pool),
                                 sizeof(PREFIX));
-      if (!prefix)
+      if (! prefix)
         return 0;
       if (prefix->name == poolStart(&dtd->pool))
         poolFinish(&dtd->pool);
@@ -6078,55 +5812,53 @@ setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType)
 }
 
 static ATTRIBUTE_ID *
-getAttributeId(XML_Parser parser, const ENCODING *enc,
-               const char *start, const char *end)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start,
+               const char *end) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   ATTRIBUTE_ID *id;
   const XML_Char *name;
-  if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+  if (! poolAppendChar(&dtd->pool, XML_T('\0')))
     return NULL;
   name = poolStoreString(&dtd->pool, enc, start, end);
-  if (!name)
+  if (! name)
     return NULL;
   /* skip quotation mark - its storage will be re-used (like in name[-1]) */
   ++name;
-  id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name, sizeof(ATTRIBUTE_ID));
-  if (!id)
+  id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name,
+                              sizeof(ATTRIBUTE_ID));
+  if (! id)
     return NULL;
   if (id->name != name)
     poolDiscard(&dtd->pool);
   else {
     poolFinish(&dtd->pool);
-    if (!parser->m_ns)
+    if (! parser->m_ns)
       ;
-    else if (name[0] == XML_T(ASCII_x)
-        && name[1] == XML_T(ASCII_m)
-        && name[2] == XML_T(ASCII_l)
-        && name[3] == XML_T(ASCII_n)
-        && name[4] == XML_T(ASCII_s)
-        && (name[5] == XML_T('\0') || name[5] == XML_T(ASCII_COLON))) {
+    else if (name[0] == XML_T(ASCII_x) && name[1] == XML_T(ASCII_m)
+             && name[2] == XML_T(ASCII_l) && name[3] == XML_T(ASCII_n)
+             && name[4] == XML_T(ASCII_s)
+             && (name[5] == XML_T('\0') || name[5] == XML_T(ASCII_COLON))) {
       if (name[5] == XML_T('\0'))
         id->prefix = &dtd->defaultPrefix;
       else
-        id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6, sizeof(PREFIX));
+        id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6,
+                                      sizeof(PREFIX));
       id->xmlns = XML_TRUE;
-    }
-    else {
+    } else {
       int i;
       for (i = 0; name[i]; i++) {
         /* attributes without prefix are *not* in the default namespace */
         if (name[i] == XML_T(ASCII_COLON)) {
           int j;
           for (j = 0; j < i; j++) {
-            if (!poolAppendChar(&dtd->pool, name[j]))
+            if (! poolAppendChar(&dtd->pool, name[j]))
               return NULL;
           }
-          if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+          if (! poolAppendChar(&dtd->pool, XML_T('\0')))
             return NULL;
-          id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&dtd->pool),
-                                        sizeof(PREFIX));
-          if (!id->prefix)
+          id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes,
+                                        poolStart(&dtd->pool), sizeof(PREFIX));
+          if (! id->prefix)
             return NULL;
           if (id->prefix->name == poolStart(&dtd->pool))
             poolFinish(&dtd->pool);
@@ -6143,22 +5875,22 @@ getAttributeId(XML_Parser parser, const ENCODING *enc,
 #define CONTEXT_SEP XML_T(ASCII_FF)
 
 static const XML_Char *
-getContext(XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getContext(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   HASH_TABLE_ITER iter;
   XML_Bool needSep = XML_FALSE;
 
   if (dtd->defaultPrefix.binding) {
     int i;
     int len;
-    if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
+    if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
       return NULL;
     len = dtd->defaultPrefix.binding->uriLen;
     if (parser->m_namespaceSeparator)
       len--;
     for (i = 0; i < len; i++) {
-      if (!poolAppendChar(&parser->m_tempPool, dtd->defaultPrefix.binding->uri[i])) {
+      if (! poolAppendChar(&parser->m_tempPool,
+                           dtd->defaultPrefix.binding->uri[i])) {
         /* Because of memory caching, I don't believe this line can be
          * executed.
          *
@@ -6190,9 +5922,9 @@ getContext(XML_Parser parser)
     int len;
     const XML_Char *s;
     PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter);
-    if (!prefix)
+    if (! prefix)
       break;
-    if (!prefix->binding) {
+    if (! prefix->binding) {
       /* This test appears to be (justifiable) paranoia.  There does
        * not seem to be a way of injecting a prefix without a binding
        * that doesn't get errored long before this function is called.
@@ -6201,98 +5933,96 @@ getContext(XML_Parser parser)
        */
       continue; /* LCOV_EXCL_LINE */
     }
-    if (needSep && !poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
+    if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
       return NULL;
     for (s = prefix->name; *s; s++)
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return NULL;
-    if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
+    if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
       return NULL;
     len = prefix->binding->uriLen;
     if (parser->m_namespaceSeparator)
       len--;
     for (i = 0; i < len; i++)
-      if (!poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i]))
+      if (! poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i]))
         return NULL;
     needSep = XML_TRUE;
   }
 
-
   hashTableIterInit(&iter, &(dtd->generalEntities));
   for (;;) {
     const XML_Char *s;
     ENTITY *e = (ENTITY *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
-    if (!e->open)
+    if (! e->open)
       continue;
-    if (needSep && !poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
+    if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
       return NULL;
     for (s = e->name; *s; s++)
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return 0;
     needSep = XML_TRUE;
   }
 
-  if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+  if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
     return NULL;
   return parser->m_tempPool.start;
 }
 
 static XML_Bool
-setContext(XML_Parser parser, const XML_Char *context)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+setContext(XML_Parser parser, const XML_Char *context) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *s = context;
 
   while (*context != XML_T('\0')) {
     if (*s == CONTEXT_SEP || *s == XML_T('\0')) {
       ENTITY *e;
-      if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+      if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
         return XML_FALSE;
-      e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0);
+      e = (ENTITY *)lookup(parser, &dtd->generalEntities,
+                           poolStart(&parser->m_tempPool), 0);
       if (e)
         e->open = XML_TRUE;
       if (*s != XML_T('\0'))
         s++;
       context = s;
       poolDiscard(&parser->m_tempPool);
-    }
-    else if (*s == XML_T(ASCII_EQUALS)) {
+    } else if (*s == XML_T(ASCII_EQUALS)) {
       PREFIX *prefix;
       if (poolLength(&parser->m_tempPool) == 0)
         prefix = &dtd->defaultPrefix;
       else {
-        if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+        if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
           return XML_FALSE;
-        prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&parser->m_tempPool),
-                                  sizeof(PREFIX));
-        if (!prefix)
+        prefix
+            = (PREFIX *)lookup(parser, &dtd->prefixes,
+                               poolStart(&parser->m_tempPool), sizeof(PREFIX));
+        if (! prefix)
           return XML_FALSE;
         if (prefix->name == poolStart(&parser->m_tempPool)) {
           prefix->name = poolCopyString(&dtd->pool, prefix->name);
-          if (!prefix->name)
+          if (! prefix->name)
             return XML_FALSE;
         }
         poolDiscard(&parser->m_tempPool);
       }
-      for (context = s + 1;
-           *context != CONTEXT_SEP && *context != XML_T('\0');
+      for (context = s + 1; *context != CONTEXT_SEP && *context != XML_T('\0');
            context++)
-        if (!poolAppendChar(&parser->m_tempPool, *context))
+        if (! poolAppendChar(&parser->m_tempPool, *context))
           return XML_FALSE;
-      if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+      if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
         return XML_FALSE;
       if (addBinding(parser, prefix, NULL, poolStart(&parser->m_tempPool),
-                     &parser->m_inheritedBindings) != XML_ERROR_NONE)
+                     &parser->m_inheritedBindings)
+          != XML_ERROR_NONE)
         return XML_FALSE;
       poolDiscard(&parser->m_tempPool);
       if (*context != XML_T('\0'))
         ++context;
       s = context;
-    }
-    else {
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+    } else {
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return XML_FALSE;
       s++;
     }
@@ -6301,8 +6031,7 @@ setContext(XML_Parser parser, const XML_Char *context)
 }
 
 static void FASTCALL
-normalizePublicId(XML_Char *publicId)
-{
+normalizePublicId(XML_Char *publicId) {
   XML_Char *p = publicId;
   XML_Char *s;
   for (s = publicId; *s; s++) {
@@ -6323,8 +6052,7 @@ normalizePublicId(XML_Char *publicId)
 }
 
 static DTD *
-dtdCreate(const XML_Memory_Handling_Suite *ms)
-{
+dtdCreate(const XML_Memory_Handling_Suite *ms) {
   DTD *p = (DTD *)ms->malloc_fcn(sizeof(DTD));
   if (p == NULL)
     return p;
@@ -6356,13 +6084,12 @@ dtdCreate(const XML_Memory_Handling_Suite *ms)
 }
 
 static void
-dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms)
-{
+dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
   hashTableIterInit(&iter, &(p->elementTypes));
   for (;;) {
     ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
     if (e->allocDefaultAtts != 0)
       ms->free_fcn(e->defaultAtts);
@@ -6398,13 +6125,12 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms)
 }
 
 static void
-dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms)
-{
+dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
   hashTableIterInit(&iter, &(p->elementTypes));
   for (;;) {
     ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
     if (e->allocDefaultAtts != 0)
       ms->free_fcn(e->defaultAtts);
@@ -6429,8 +6155,8 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms)
    The new DTD has already been initialized.
 */
 static int
-dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms)
-{
+dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
+        const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
 
   /* Copy the prefix table. */
@@ -6439,12 +6165,12 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
   for (;;) {
     const XML_Char *name;
     const PREFIX *oldP = (PREFIX *)hashTableIterNext(&iter);
-    if (!oldP)
+    if (! oldP)
       break;
     name = poolCopyString(&(newDtd->pool), oldP->name);
-    if (!name)
+    if (! name)
       return 0;
-    if (!lookup(oldParser, &(newDtd->prefixes), name, sizeof(PREFIX)))
+    if (! lookup(oldParser, &(newDtd->prefixes), name, sizeof(PREFIX)))
       return 0;
   }
 
@@ -6457,18 +6183,18 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
     const XML_Char *name;
     const ATTRIBUTE_ID *oldA = (ATTRIBUTE_ID *)hashTableIterNext(&iter);
 
-    if (!oldA)
+    if (! oldA)
       break;
     /* Remember to allocate the scratch byte before the name. */
-    if (!poolAppendChar(&(newDtd->pool), XML_T('\0')))
+    if (! poolAppendChar(&(newDtd->pool), XML_T('\0')))
       return 0;
     name = poolCopyString(&(newDtd->pool), oldA->name);
-    if (!name)
+    if (! name)
       return 0;
     ++name;
     newA = (ATTRIBUTE_ID *)lookup(oldParser, &(newDtd->attributeIds), name,
                                   sizeof(ATTRIBUTE_ID));
-    if (!newA)
+    if (! newA)
       return 0;
     newA->maybeTokenized = oldA->maybeTokenized;
     if (oldA->prefix) {
@@ -6490,57 +6216,52 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
     ELEMENT_TYPE *newE;
     const XML_Char *name;
     const ELEMENT_TYPE *oldE = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!oldE)
+    if (! oldE)
       break;
     name = poolCopyString(&(newDtd->pool), oldE->name);
-    if (!name)
+    if (! name)
       return 0;
     newE = (ELEMENT_TYPE *)lookup(oldParser, &(newDtd->elementTypes), name,
                                   sizeof(ELEMENT_TYPE));
-    if (!newE)
+    if (! newE)
       return 0;
     if (oldE->nDefaultAtts) {
-      newE->defaultAtts = (DEFAULT_ATTRIBUTE *)
-          ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
-      if (!newE->defaultAtts) {
+      newE->defaultAtts = (DEFAULT_ATTRIBUTE *)ms->malloc_fcn(
+          oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
+      if (! newE->defaultAtts) {
         return 0;
       }
     }
     if (oldE->idAtt)
-      newE->idAtt = (ATTRIBUTE_ID *)
-          lookup(oldParser, &(newDtd->attributeIds), oldE->idAtt->name, 0);
+      newE->idAtt = (ATTRIBUTE_ID *)lookup(oldParser, &(newDtd->attributeIds),
+                                           oldE->idAtt->name, 0);
     newE->allocDefaultAtts = newE->nDefaultAtts = oldE->nDefaultAtts;
     if (oldE->prefix)
       newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes),
                                       oldE->prefix->name, 0);
     for (i = 0; i < newE->nDefaultAtts; i++) {
-      newE->defaultAtts[i].id = (ATTRIBUTE_ID *)
-          lookup(oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0);
+      newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup(
+          oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0);
       newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata;
       if (oldE->defaultAtts[i].value) {
         newE->defaultAtts[i].value
             = poolCopyString(&(newDtd->pool), oldE->defaultAtts[i].value);
-        if (!newE->defaultAtts[i].value)
+        if (! newE->defaultAtts[i].value)
           return 0;
-      }
-      else
+      } else
         newE->defaultAtts[i].value = NULL;
     }
   }
 
   /* Copy the entity tables. */
-  if (!copyEntityTable(oldParser,
-                       &(newDtd->generalEntities),
-                       &(newDtd->pool),
-                       &(oldDtd->generalEntities)))
-      return 0;
+  if (! copyEntityTable(oldParser, &(newDtd->generalEntities), &(newDtd->pool),
+                        &(oldDtd->generalEntities)))
+    return 0;
 
 #ifdef XML_DTD
-  if (!copyEntityTable(oldParser,
-                       &(newDtd->paramEntities),
-                       &(newDtd->pool),
-                       &(oldDtd->paramEntities)))
-      return 0;
+  if (! copyEntityTable(oldParser, &(newDtd->paramEntities), &(newDtd->pool),
+                        &(oldDtd->paramEntities)))
+    return 0;
   newDtd->paramEntityRead = oldDtd->paramEntityRead;
 #endif /* XML_DTD */
 
@@ -6557,14 +6278,11 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
   newDtd->scaffIndex = oldDtd->scaffIndex;
 
   return 1;
-}  /* End dtdCopy */
+} /* End dtdCopy */
 
 static int
-copyEntityTable(XML_Parser oldParser,
-                HASH_TABLE *newTable,
-                STRING_POOL *newPool,
-                const HASH_TABLE *oldTable)
-{
+copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable,
+                STRING_POOL *newPool, const HASH_TABLE *oldTable) {
   HASH_TABLE_ITER iter;
   const XML_Char *cachedOldBase = NULL;
   const XML_Char *cachedNewBase = NULL;
@@ -6575,17 +6293,17 @@ copyEntityTable(XML_Parser oldParser,
     ENTITY *newE;
     const XML_Char *name;
     const ENTITY *oldE = (ENTITY *)hashTableIterNext(&iter);
-    if (!oldE)
+    if (! oldE)
       break;
     name = poolCopyString(newPool, oldE->name);
-    if (!name)
+    if (! name)
       return 0;
     newE = (ENTITY *)lookup(oldParser, newTable, name, sizeof(ENTITY));
-    if (!newE)
+    if (! newE)
       return 0;
     if (oldE->systemId) {
       const XML_Char *tem = poolCopyString(newPool, oldE->systemId);
-      if (!tem)
+      if (! tem)
         return 0;
       newE->systemId = tem;
       if (oldE->base) {
@@ -6594,29 +6312,28 @@ copyEntityTable(XML_Parser oldParser,
         else {
           cachedOldBase = oldE->base;
           tem = poolCopyString(newPool, cachedOldBase);
-          if (!tem)
+          if (! tem)
             return 0;
           cachedNewBase = newE->base = tem;
         }
       }
       if (oldE->publicId) {
         tem = poolCopyString(newPool, oldE->publicId);
-        if (!tem)
+        if (! tem)
           return 0;
         newE->publicId = tem;
       }
-    }
-    else {
-      const XML_Char *tem = poolCopyStringN(newPool, oldE->textPtr,
-                                            oldE->textLen);
-      if (!tem)
+    } else {
+      const XML_Char *tem
+          = poolCopyStringN(newPool, oldE->textPtr, oldE->textLen);
+      if (! tem)
         return 0;
       newE->textPtr = tem;
       newE->textLen = oldE->textLen;
     }
     if (oldE->notation) {
       const XML_Char *tem = poolCopyString(newPool, oldE->notation);
-      if (!tem)
+      if (! tem)
         return 0;
       newE->notation = tem;
     }
@@ -6629,8 +6346,7 @@ copyEntityTable(XML_Parser oldParser,
 #define INIT_POWER 6
 
 static XML_Bool FASTCALL
-keyeq(KEY s1, KEY s2)
-{
+keyeq(KEY s1, KEY s2) {
   for (; *s1 == *s2; s1++, s2++)
     if (*s1 == 0)
       return XML_TRUE;
@@ -6638,23 +6354,21 @@ keyeq(KEY s1, KEY s2)
 }
 
 static size_t
-keylen(KEY s)
-{
+keylen(KEY s) {
   size_t len = 0;
-  for (; *s; s++, len++);
+  for (; *s; s++, len++)
+    ;
   return len;
 }
 
 static void
-copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key)
-{
+copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) {
   key->k[0] = 0;
   key->k[1] = get_hash_secret_salt(parser);
 }
 
 static unsigned long FASTCALL
-hash(XML_Parser parser, KEY s)
-{
+hash(XML_Parser parser, KEY s) {
   struct siphash state;
   struct sipkey key;
   (void)sip24_valid;
@@ -6665,26 +6379,24 @@ hash(XML_Parser parser, KEY s)
 }
 
 static NAMED *
-lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
-{
+lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) {
   size_t i;
   if (table->size == 0) {
     size_t tsize;
-    if (!createSize)
+    if (! createSize)
       return NULL;
     table->power = INIT_POWER;
     /* table->size is a power of 2 */
     table->size = (size_t)1 << INIT_POWER;
     tsize = table->size * sizeof(NAMED *);
     table->v = (NAMED **)table->mem->malloc_fcn(tsize);
-    if (!table->v) {
+    if (! table->v) {
       table->size = 0;
       return NULL;
     }
     memset(table->v, 0, tsize);
     i = hash(parser, name) & ((unsigned long)table->size - 1);
-  }
-  else {
+  } else {
     unsigned long h = hash(parser, name);
     unsigned long mask = (unsigned long)table->size - 1;
     unsigned char step = 0;
@@ -6692,11 +6404,11 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
     while (table->v[i]) {
       if (keyeq(name, table->v[i]->name))
         return table->v[i];
-      if (!step)
+      if (! step)
         step = PROBE_STEP(h, mask, table->power);
       i < step ? (i += table->size - step) : (i -= step);
     }
-    if (!createSize)
+    if (! createSize)
       return NULL;
 
     /* check for overflow (table is half full) */
@@ -6706,7 +6418,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
       unsigned long newMask = (unsigned long)newSize - 1;
       size_t tsize = newSize * sizeof(NAMED *);
       NAMED **newV = (NAMED **)table->mem->malloc_fcn(tsize);
-      if (!newV)
+      if (! newV)
         return NULL;
       memset(newV, 0, tsize);
       for (i = 0; i < table->size; i++)
@@ -6715,7 +6427,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
           size_t j = newHash & newMask;
           step = 0;
           while (newV[j]) {
-            if (!step)
+            if (! step)
               step = PROBE_STEP(newHash, newMask, newPower);
             j < step ? (j += newSize - step) : (j -= step);
           }
@@ -6728,14 +6440,14 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
       i = h & newMask;
       step = 0;
       while (table->v[i]) {
-        if (!step)
+        if (! step)
           step = PROBE_STEP(h, newMask, newPower);
         i < step ? (i += newSize - step) : (i -= step);
       }
     }
   }
   table->v[i] = (NAMED *)table->mem->malloc_fcn(createSize);
-  if (!table->v[i])
+  if (! table->v[i])
     return NULL;
   memset(table->v[i], 0, createSize);
   table->v[i]->name = name;
@@ -6744,8 +6456,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
 }
 
 static void FASTCALL
-hashTableClear(HASH_TABLE *table)
-{
+hashTableClear(HASH_TABLE *table) {
   size_t i;
   for (i = 0; i < table->size; i++) {
     table->mem->free_fcn(table->v[i]);
@@ -6755,8 +6466,7 @@ hashTableClear(HASH_TABLE *table)
 }
 
 static void FASTCALL
-hashTableDestroy(HASH_TABLE *table)
-{
+hashTableDestroy(HASH_TABLE *table) {
   size_t i;
   for (i = 0; i < table->size; i++)
     table->mem->free_fcn(table->v[i]);
@@ -6764,8 +6474,7 @@ hashTableDestroy(HASH_TABLE *table)
 }
 
 static void FASTCALL
-hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms)
-{
+hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) {
   p->power = 0;
   p->size = 0;
   p->used = 0;
@@ -6774,15 +6483,13 @@ hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms)
 }
 
 static void FASTCALL
-hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table)
-{
+hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) {
   iter->p = table->v;
   iter->end = iter->p + table->size;
 }
 
-static NAMED * FASTCALL
-hashTableIterNext(HASH_TABLE_ITER *iter)
-{
+static NAMED *FASTCALL
+hashTableIterNext(HASH_TABLE_ITER *iter) {
   while (iter->p != iter->end) {
     NAMED *tem = *(iter->p)++;
     if (tem)
@@ -6792,8 +6499,7 @@ hashTableIterNext(HASH_TABLE_ITER *iter)
 }
 
 static void FASTCALL
-poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms)
-{
+poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms) {
   pool->blocks = NULL;
   pool->freeBlocks = NULL;
   pool->start = NULL;
@@ -6803,9 +6509,8 @@ poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms)
 }
 
 static void FASTCALL
-poolClear(STRING_POOL *pool)
-{
-  if (!pool->freeBlocks)
+poolClear(STRING_POOL *pool) {
+  if (! pool->freeBlocks)
     pool->freeBlocks = pool->blocks;
   else {
     BLOCK *p = pool->blocks;
@@ -6823,8 +6528,7 @@ poolClear(STRING_POOL *pool)
 }
 
 static void FASTCALL
-poolDestroy(STRING_POOL *pool)
-{
+poolDestroy(STRING_POOL *pool) {
   BLOCK *p = pool->blocks;
   while (p) {
     BLOCK *tem = p->next;
@@ -6840,26 +6544,26 @@ poolDestroy(STRING_POOL *pool)
 }
 
 static XML_Char *
-poolAppend(STRING_POOL *pool, const ENCODING *enc,
-           const char *ptr, const char *end)
-{
-  if (!pool->ptr && !poolGrow(pool))
+poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr,
+           const char *end) {
+  if (! pool->ptr && ! poolGrow(pool))
     return NULL;
   for (;;) {
-    const enum XML_Convert_Result convert_res = XmlConvert(enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end);
-    if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+    const enum XML_Convert_Result convert_res = XmlConvert(
+        enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end);
+    if ((convert_res == XML_CONVERT_COMPLETED)
+        || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
       break;
-    if (!poolGrow(pool))
+    if (! poolGrow(pool))
       return NULL;
   }
   return pool->start;
 }
 
-static const XML_Char * FASTCALL
-poolCopyString(STRING_POOL *pool, const XML_Char *s)
-{
+static const XML_Char *FASTCALL
+poolCopyString(STRING_POOL *pool, const XML_Char *s) {
   do {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
   } while (*s++);
   s = pool->start;
@@ -6868,9 +6572,8 @@ poolCopyString(STRING_POOL *pool, const XML_Char *s)
 }
 
 static const XML_Char *
-poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
-{
-  if (!pool->ptr && !poolGrow(pool)) {
+poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) {
+  if (! pool->ptr && ! poolGrow(pool)) {
     /* The following line is unreachable given the current usage of
      * poolCopyStringN().  Currently it is called from exactly one
      * place to copy the text of a simple general entity.  By that
@@ -6885,7 +6588,7 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
     return NULL; /* LCOV_EXCL_LINE */
   }
   for (; n > 0; --n, s++) {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
   }
   s = pool->start;
@@ -6893,11 +6596,10 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
   return s;
 }
 
-static const XML_Char * FASTCALL
-poolAppendString(STRING_POOL *pool, const XML_Char *s)
-{
+static const XML_Char *FASTCALL
+poolAppendString(STRING_POOL *pool, const XML_Char *s) {
   while (*s) {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
     s++;
   }
@@ -6905,20 +6607,18 @@ poolAppendString(STRING_POOL *pool, const XML_Char *s)
 }
 
 static XML_Char *
-poolStoreString(STRING_POOL *pool, const ENCODING *enc,
-                const char *ptr, const char *end)
-{
-  if (!poolAppend(pool, enc, ptr, end))
+poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr,
+                const char *end) {
+  if (! poolAppend(pool, enc, ptr, end))
     return NULL;
-  if (pool->ptr == pool->end && !poolGrow(pool))
+  if (pool->ptr == pool->end && ! poolGrow(pool))
     return NULL;
   *(pool->ptr)++ = 0;
   return pool->start;
 }
 
 static size_t
-poolBytesToAllocateFor(int blockSize)
-{
+poolBytesToAllocateFor(int blockSize) {
   /* Unprotected math would be:
   ** return offsetof(BLOCK, s) + blockSize * sizeof(XML_Char);
   **
@@ -6926,7 +6626,7 @@ poolBytesToAllocateFor(int blockSize)
   ** For a + b * c we check b * c in isolation first, so that addition of a
   ** on top has no chance of making us accept a small non-negative number
   */
-  const size_t stretch = sizeof(XML_Char);  /* can be 4 bytes */
+  const size_t stretch = sizeof(XML_Char); /* can be 4 bytes */
 
   if (blockSize <= 0)
     return 0;
@@ -6936,8 +6636,8 @@ poolBytesToAllocateFor(int blockSize)
 
   {
     const int stretchedBlockSize = blockSize * (int)stretch;
-    const int bytesToAllocate = (int)(
-        offsetof(BLOCK, s) + (unsigned)stretchedBlockSize);
+    const int bytesToAllocate
+        = (int)(offsetof(BLOCK, s) + (unsigned)stretchedBlockSize);
     if (bytesToAllocate < 0)
       return 0;
 
@@ -6946,8 +6646,7 @@ poolBytesToAllocateFor(int blockSize)
 }
 
 static XML_Bool FASTCALL
-poolGrow(STRING_POOL *pool)
-{
+poolGrow(STRING_POOL *pool) {
   if (pool->freeBlocks) {
     if (pool->start == 0) {
       pool->blocks = pool->freeBlocks;
@@ -6973,7 +6672,7 @@ poolGrow(STRING_POOL *pool)
   }
   if (pool->blocks && pool->start == pool->blocks->s) {
     BLOCK *temp;
-    int blockSize = (int)((unsigned)(pool->end - pool->start)*2U);
+    int blockSize = (int)((unsigned)(pool->end - pool->start) * 2U);
     size_t bytesToAllocate;
 
     /* NOTE: Needs to be calculated prior to calling `realloc`
@@ -6994,8 +6693,8 @@ poolGrow(STRING_POOL *pool)
     if (bytesToAllocate == 0)
       return XML_FALSE;
 
-    temp = (BLOCK *)
-      pool->mem->realloc_fcn(pool->blocks, (unsigned)bytesToAllocate);
+    temp = (BLOCK *)pool->mem->realloc_fcn(pool->blocks,
+                                           (unsigned)bytesToAllocate);
     if (temp == NULL)
       return XML_FALSE;
     pool->blocks = temp;
@@ -7003,8 +6702,7 @@ poolGrow(STRING_POOL *pool)
     pool->ptr = pool->blocks->s + offsetInsideBlock;
     pool->start = pool->blocks->s;
     pool->end = pool->start + blockSize;
-  }
-  else {
+  } else {
     BLOCK *tem;
     int blockSize = (int)(pool->end - pool->start);
     size_t bytesToAllocate;
@@ -7019,7 +6717,7 @@ poolGrow(STRING_POOL *pool)
        * function).  Either way it isn't readily testable, so we
        * exclude it from the coverage statistics.
        */
-      return XML_FALSE;  /* LCOV_EXCL_LINE */
+      return XML_FALSE; /* LCOV_EXCL_LINE */
     }
 
     if (blockSize < INIT_BLOCK_SIZE)
@@ -7037,14 +6735,13 @@ poolGrow(STRING_POOL *pool)
       return XML_FALSE;
 
     tem = (BLOCK *)pool->mem->malloc_fcn(bytesToAllocate);
-    if (!tem)
+    if (! tem)
       return XML_FALSE;
     tem->size = blockSize;
     tem->next = pool->blocks;
     pool->blocks = tem;
     if (pool->ptr != pool->start)
-      memcpy(tem->s, pool->start,
-             (pool->ptr - pool->start) * sizeof(XML_Char));
+      memcpy(tem->s, pool->start, (pool->ptr - pool->start) * sizeof(XML_Char));
     pool->ptr = tem->s + (pool->ptr - pool->start);
     pool->start = tem->s;
     pool->end = tem->s + blockSize;
@@ -7053,15 +6750,14 @@ poolGrow(STRING_POOL *pool)
 }
 
 static int FASTCALL
-nextScaffoldPart(XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
-  CONTENT_SCAFFOLD * me;
+nextScaffoldPart(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
+  CONTENT_SCAFFOLD *me;
   int next;
 
-  if (!dtd->scaffIndex) {
+  if (! dtd->scaffIndex) {
     dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int));
-    if (!dtd->scaffIndex)
+    if (! dtd->scaffIndex)
       return -1;
     dtd->scaffIndex[0] = 0;
   }
@@ -7069,15 +6765,14 @@ nextScaffoldPart(XML_Parser parser)
   if (dtd->scaffCount >= dtd->scaffSize) {
     CONTENT_SCAFFOLD *temp;
     if (dtd->scaffold) {
-      temp = (CONTENT_SCAFFOLD *)
-        REALLOC(parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD));
+      temp = (CONTENT_SCAFFOLD *)REALLOC(
+          parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD));
       if (temp == NULL)
         return -1;
       dtd->scaffSize *= 2;
-    }
-    else {
+    } else {
       temp = (CONTENT_SCAFFOLD *)MALLOC(parser, INIT_SCAFFOLD_ELEMENTS
-                                        * sizeof(CONTENT_SCAFFOLD));
+                                                    * sizeof(CONTENT_SCAFFOLD));
       if (temp == NULL)
         return -1;
       dtd->scaffSize = INIT_SCAFFOLD_ELEMENTS;
@@ -7087,11 +6782,12 @@ nextScaffoldPart(XML_Parser parser)
   next = dtd->scaffCount++;
   me = &dtd->scaffold[next];
   if (dtd->scaffLevel) {
-    CONTENT_SCAFFOLD *parent = &dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel-1]];
+    CONTENT_SCAFFOLD *parent
+        = &dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]];
     if (parent->lastchild) {
       dtd->scaffold[parent->lastchild].nextsib = next;
     }
-    if (!parent->childcnt)
+    if (! parent->childcnt)
       parent->firstchild = next;
     parent->lastchild = next;
     parent->childcnt++;
@@ -7101,13 +6797,9 @@ nextScaffoldPart(XML_Parser parser)
 }
 
 static void
-build_node(XML_Parser parser,
-           int src_node,
-           XML_Content *dest,
-           XML_Content **contpos,
-           XML_Char **strpos)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+build_node(XML_Parser parser, int src_node, XML_Content *dest,
+           XML_Content **contpos, XML_Char **strpos) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   dest->type = dtd->scaffold[src_node].type;
   dest->quant = dtd->scaffold[src_node].quant;
   if (dest->type == XML_CTYPE_NAME) {
@@ -7116,21 +6808,19 @@ build_node(XML_Parser parser,
     src = dtd->scaffold[src_node].name;
     for (;;) {
       *(*strpos)++ = *src;
-      if (!*src)
+      if (! *src)
         break;
       src++;
     }
     dest->numchildren = 0;
     dest->children = NULL;
-  }
-  else {
+  } else {
     unsigned int i;
     int cn;
     dest->numchildren = dtd->scaffold[src_node].childcnt;
     dest->children = *contpos;
     *contpos += dest->numchildren;
-    for (i = 0, cn = dtd->scaffold[src_node].firstchild;
-         i < dest->numchildren;
+    for (i = 0, cn = dtd->scaffold[src_node].firstchild; i < dest->numchildren;
          i++, cn = dtd->scaffold[cn].nextsib) {
       build_node(parser, cn, &(dest->children[i]), contpos, strpos);
     }
@@ -7139,20 +6829,19 @@ build_node(XML_Parser parser,
 }
 
 static XML_Content *
-build_model (XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+build_model(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   XML_Content *ret;
   XML_Content *cpos;
-  XML_Char * str;
+  XML_Char *str;
   int allocsize = (dtd->scaffCount * sizeof(XML_Content)
                    + (dtd->contentStringLen * sizeof(XML_Char)));
 
   ret = (XML_Content *)MALLOC(parser, allocsize);
-  if (!ret)
+  if (! ret)
     return NULL;
 
-  str =  (XML_Char *) (&ret[dtd->scaffCount]);
+  str = (XML_Char *)(&ret[dtd->scaffCount]);
   cpos = &ret[1];
 
   build_node(parser, 0, ret, &cpos, &str);
@@ -7160,49 +6849,45 @@ build_model (XML_Parser parser)
 }
 
 static ELEMENT_TYPE *
-getElementType(XML_Parser parser,
-               const ENCODING *enc,
-               const char *ptr,
-               const char *end)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr,
+               const char *end) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *name = poolStoreString(&dtd->pool, enc, ptr, end);
   ELEMENT_TYPE *ret;
 
-  if (!name)
+  if (! name)
     return NULL;
-  ret = (ELEMENT_TYPE *) lookup(parser, &dtd->elementTypes, name, sizeof(ELEMENT_TYPE));
-  if (!ret)
+  ret = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, name,
+                               sizeof(ELEMENT_TYPE));
+  if (! ret)
     return NULL;
   if (ret->name != name)
     poolDiscard(&dtd->pool);
   else {
     poolFinish(&dtd->pool);
-    if (!setElementTypePrefix(parser, ret))
+    if (! setElementTypePrefix(parser, ret))
       return NULL;
   }
   return ret;
 }
 
 static XML_Char *
-copyString(const XML_Char *s,
-           const XML_Memory_Handling_Suite *memsuite)
-{
-    int charsRequired = 0;
-    XML_Char *result;
-
-    /* First determine how long the string is */
-    while (s[charsRequired] != 0) {
-      charsRequired++;
-    }
-    /* Include the terminator */
+copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) {
+  int charsRequired = 0;
+  XML_Char *result;
+
+  /* First determine how long the string is */
+  while (s[charsRequired] != 0) {
     charsRequired++;
+  }
+  /* Include the terminator */
+  charsRequired++;
 
-    /* Now allocate space for the copy */
-    result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char));
-    if (result == NULL)
-        return NULL;
-    /* Copy the original into place */
-    memcpy(result, s, charsRequired * sizeof(XML_Char));
-    return result;
+  /* Now allocate space for the copy */
+  result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char));
+  if (result == NULL)
+    return NULL;
+  /* Copy the original into place */
+  memcpy(result, s, charsRequired * sizeof(XML_Char));
+  return result;
 }
diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c
index 708507d575be..4d3e3e86e9e8 100644
--- a/Modules/expat/xmlrole.c
+++ b/Modules/expat/xmlrole.c
@@ -33,11 +33,11 @@
 #include <stddef.h>
 
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #else
-#ifdef HAVE_EXPAT_CONFIG_H
-#include <expat_config.h>
-#endif
+#  ifdef HAVE_EXPAT_CONFIG_H
+#    include <expat_config.h>
+#  endif
 #endif /* ndef _WIN32 */
 
 #include "expat_external.h"
@@ -52,107 +52,88 @@
 
 */
 
-static const char KW_ANY[] = {
-    ASCII_A, ASCII_N, ASCII_Y, '\0' };
-static const char KW_ATTLIST[] = {
-    ASCII_A, ASCII_T, ASCII_T, ASCII_L, ASCII_I, ASCII_S, ASCII_T, '\0' };
-static const char KW_CDATA[] = {
-    ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_DOCTYPE[] = {
-    ASCII_D, ASCII_O, ASCII_C, ASCII_T, ASCII_Y, ASCII_P, ASCII_E, '\0' };
-static const char KW_ELEMENT[] = {
-    ASCII_E, ASCII_L, ASCII_E, ASCII_M, ASCII_E, ASCII_N, ASCII_T, '\0' };
-static const char KW_EMPTY[] = {
-    ASCII_E, ASCII_M, ASCII_P, ASCII_T, ASCII_Y, '\0' };
-static const char KW_ENTITIES[] = {
-    ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_I, ASCII_E, ASCII_S,
-    '\0' };
-static const char KW_ENTITY[] = {
-    ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0' };
-static const char KW_FIXED[] = {
-    ASCII_F, ASCII_I, ASCII_X, ASCII_E, ASCII_D, '\0' };
-static const char KW_ID[] = {
-    ASCII_I, ASCII_D, '\0' };
-static const char KW_IDREF[] = {
-    ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0' };
-static const char KW_IDREFS[] = {
-    ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0' };
+static const char KW_ANY[] = {ASCII_A, ASCII_N, ASCII_Y, '\0'};
+static const char KW_ATTLIST[]
+    = {ASCII_A, ASCII_T, ASCII_T, ASCII_L, ASCII_I, ASCII_S, ASCII_T, '\0'};
+static const char KW_CDATA[]
+    = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_DOCTYPE[]
+    = {ASCII_D, ASCII_O, ASCII_C, ASCII_T, ASCII_Y, ASCII_P, ASCII_E, '\0'};
+static const char KW_ELEMENT[]
+    = {ASCII_E, ASCII_L, ASCII_E, ASCII_M, ASCII_E, ASCII_N, ASCII_T, '\0'};
+static const char KW_EMPTY[]
+    = {ASCII_E, ASCII_M, ASCII_P, ASCII_T, ASCII_Y, '\0'};
+static const char KW_ENTITIES[] = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T,
+                                   ASCII_I, ASCII_E, ASCII_S, '\0'};
+static const char KW_ENTITY[]
+    = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0'};
+static const char KW_FIXED[]
+    = {ASCII_F, ASCII_I, ASCII_X, ASCII_E, ASCII_D, '\0'};
+static const char KW_ID[] = {ASCII_I, ASCII_D, '\0'};
+static const char KW_IDREF[]
+    = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0'};
+static const char KW_IDREFS[]
+    = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0'};
 #ifdef XML_DTD
-static const char KW_IGNORE[] = {
-    ASCII_I, ASCII_G, ASCII_N, ASCII_O, ASCII_R, ASCII_E, '\0' };
+static const char KW_IGNORE[]
+    = {ASCII_I, ASCII_G, ASCII_N, ASCII_O, ASCII_R, ASCII_E, '\0'};
 #endif
-static const char KW_IMPLIED[] = {
-    ASCII_I, ASCII_M, ASCII_P, ASCII_L, ASCII_I, ASCII_E, ASCII_D, '\0' };
+static const char KW_IMPLIED[]
+    = {ASCII_I, ASCII_M, ASCII_P, ASCII_L, ASCII_I, ASCII_E, ASCII_D, '\0'};
 #ifdef XML_DTD
-static const char KW_INCLUDE[] = {
-    ASCII_I, ASCII_N, ASCII_C, ASCII_L, ASCII_U, ASCII_D, ASCII_E, '\0' };
+static const char KW_INCLUDE[]
+    = {ASCII_I, ASCII_N, ASCII_C, ASCII_L, ASCII_U, ASCII_D, ASCII_E, '\0'};
 #endif
-static const char KW_NDATA[] = {
-    ASCII_N, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_NMTOKEN[] = {
-    ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0' };
-static const char KW_NMTOKENS[] = {
-    ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, ASCII_S,
-    '\0' };
-static const char KW_NOTATION[] =
-    { ASCII_N, ASCII_O, ASCII_T, ASCII_A, ASCII_T, ASCII_I, ASCII_O, ASCII_N,
-      '\0' };
-static const char KW_PCDATA[] = {
-    ASCII_P, ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_PUBLIC[] = {
-    ASCII_P, ASCII_U, ASCII_B, ASCII_L, ASCII_I, ASCII_C, '\0' };
-static const char KW_REQUIRED[] = {
-    ASCII_R, ASCII_E, ASCII_Q, ASCII_U, ASCII_I, ASCII_R, ASCII_E, ASCII_D,
-    '\0' };
-static const char KW_SYSTEM[] = {
-    ASCII_S, ASCII_Y, ASCII_S, ASCII_T, ASCII_E, ASCII_M, '\0' };
+static const char KW_NDATA[]
+    = {ASCII_N, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_NMTOKEN[]
+    = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0'};
+static const char KW_NMTOKENS[] = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K,
+                                   ASCII_E, ASCII_N, ASCII_S, '\0'};
+static const char KW_NOTATION[] = {ASCII_N, ASCII_O, ASCII_T, ASCII_A, ASCII_T,
+                                   ASCII_I, ASCII_O, ASCII_N, '\0'};
+static const char KW_PCDATA[]
+    = {ASCII_P, ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_PUBLIC[]
+    = {ASCII_P, ASCII_U, ASCII_B, ASCII_L, ASCII_I, ASCII_C, '\0'};
+static const char KW_REQUIRED[] = {ASCII_R, ASCII_E, ASCII_Q, ASCII_U, ASCII_I,
+                                   ASCII_R, ASCII_E, ASCII_D, '\0'};
+static const char KW_SYSTEM[]
+    = {ASCII_S, ASCII_Y, ASCII_S, ASCII_T, ASCII_E, ASCII_M, '\0'};
 
 #ifndef MIN_BYTES_PER_CHAR
-#define MIN_BYTES_PER_CHAR(enc) ((enc)->minBytesPerChar)
+#  define MIN_BYTES_PER_CHAR(enc) ((enc)->minBytesPerChar)
 #endif
 
 #ifdef XML_DTD
-#define setTopLevel(state) \
-  ((state)->handler = ((state)->documentEntity \
-                       ? internalSubset \
-                       : externalSubset1))
+#  define setTopLevel(state)                                                   \
+    ((state)->handler                                                          \
+     = ((state)->documentEntity ? internalSubset : externalSubset1))
 #else /* not XML_DTD */
-#define setTopLevel(state) ((state)->handler = internalSubset)
+#  define setTopLevel(state) ((state)->handler = internalSubset)
 #endif /* not XML_DTD */
 
-typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state,
-                                   int tok,
-                                   const char *ptr,
-                                   const char *end,
+typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state, int tok,
+                                   const char *ptr, const char *end,
                                    const ENCODING *enc);
 
-static PROLOG_HANDLER
-  prolog0, prolog1, prolog2,
-  doctype0, doctype1, doctype2, doctype3, doctype4, doctype5,
-  internalSubset,
-  entity0, entity1, entity2, entity3, entity4, entity5, entity6,
-  entity7, entity8, entity9, entity10,
-  notation0, notation1, notation2, notation3, notation4,
-  attlist0, attlist1, attlist2, attlist3, attlist4, attlist5, attlist6,
-  attlist7, attlist8, attlist9,
-  element0, element1, element2, element3, element4, element5, element6,
-  element7,
+static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2,
+    doctype3, doctype4, doctype5, internalSubset, entity0, entity1, entity2,
+    entity3, entity4, entity5, entity6, entity7, entity8, entity9, entity10,
+    notation0, notation1, notation2, notation3, notation4, attlist0, attlist1,
+    attlist2, attlist3, attlist4, attlist5, attlist6, attlist7, attlist8,
+    attlist9, element0, element1, element2, element3, element4, element5,
+    element6, element7,
 #ifdef XML_DTD
-  externalSubset0, externalSubset1,
-  condSect0, condSect1, condSect2,
+    externalSubset0, externalSubset1, condSect0, condSect1, condSect2,
 #endif /* XML_DTD */
-  declClose,
-  error;
+    declClose, error;
 
 static int FASTCALL common(PROLOG_STATE *state, int tok);
 
 static int PTRCALL
-prolog0(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     state->handler = prolog1;
@@ -169,10 +150,8 @@ prolog0(PROLOG_STATE *state,
   case XML_TOK_BOM:
     return XML_ROLE_NONE;
   case XML_TOK_DECL_OPEN:
-    if (!XmlNameMatchesAscii(enc,
-                             ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                             end,
-                             KW_DOCTYPE))
+    if (! XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
+                              KW_DOCTYPE))
       break;
     state->handler = doctype0;
     return XML_ROLE_DOCTYPE_NONE;
@@ -184,12 +163,8 @@ prolog0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-prolog1(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -207,10 +182,8 @@ prolog1(PROLOG_STATE *state,
      */
     return XML_ROLE_NONE; /* LCOV_EXCL_LINE */
   case XML_TOK_DECL_OPEN:
-    if (!XmlNameMatchesAscii(enc,
-                             ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                             end,
-                             KW_DOCTYPE))
+    if (! XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
+                              KW_DOCTYPE))
       break;
     state->handler = doctype0;
     return XML_ROLE_DOCTYPE_NONE;
@@ -222,12 +195,11 @@ prolog1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-prolog2(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -243,12 +215,11 @@ prolog2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -261,12 +232,8 @@ doctype0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype1(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -291,12 +258,11 @@ doctype1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype2(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -308,12 +274,11 @@ doctype2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -325,12 +290,11 @@ doctype3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -345,12 +309,11 @@ doctype4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -362,40 +325,28 @@ doctype5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-internalSubset(PROLOG_STATE *state,
-               int tok,
-               const char *ptr,
-               const char *end,
-               const ENCODING *enc)
-{
+internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+               const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
   case XML_TOK_DECL_OPEN:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ENTITY)) {
       state->handler = entity0;
       return XML_ROLE_ENTITY_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ATTLIST)) {
       state->handler = attlist0;
       return XML_ROLE_ATTLIST_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ELEMENT)) {
       state->handler = element0;
       return XML_ROLE_ELEMENT_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_NOTATION)) {
       state->handler = notation0;
       return XML_ROLE_NOTATION_NONE;
@@ -419,12 +370,8 @@ internalSubset(PROLOG_STATE *state,
 #ifdef XML_DTD
 
 static int PTRCALL
-externalSubset0(PROLOG_STATE *state,
-                int tok,
-                const char *ptr,
-                const char *end,
-                const ENCODING *enc)
-{
+externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+                const ENCODING *enc) {
   state->handler = externalSubset1;
   if (tok == XML_TOK_XML_DECL)
     return XML_ROLE_TEXT_DECL;
@@ -432,12 +379,8 @@ externalSubset0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-externalSubset1(PROLOG_STATE *state,
-                int tok,
-                const char *ptr,
-                const char *end,
-                const ENCODING *enc)
-{
+externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+                const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_COND_SECT_OPEN:
     state->handler = condSect0;
@@ -464,12 +407,11 @@ externalSubset1(PROLOG_STATE *state,
 #endif /* XML_DTD */
 
 static int PTRCALL
-entity0(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -484,12 +426,11 @@ entity0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity1(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -501,12 +442,8 @@ entity1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity2(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -529,12 +466,11 @@ entity2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity3(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -546,12 +482,11 @@ entity3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity4(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -563,12 +498,8 @@ entity4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity5(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -586,12 +517,11 @@ entity5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity6(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -604,12 +534,8 @@ entity6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity7(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -632,12 +558,11 @@ entity7(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity8(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -649,12 +574,11 @@ entity8(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity9(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -666,12 +590,11 @@ entity9(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity10(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -683,12 +606,11 @@ entity10(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation0(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -700,12 +622,8 @@ notation0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation1(PROLOG_STATE *state,
-          int tok,
-          const char *ptr,
-          const char *end,
-          const ENCODING *enc)
-{
+notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -724,12 +642,11 @@ notation1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation2(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -741,12 +658,11 @@ notation2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation3(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -759,12 +675,11 @@ notation3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation4(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -780,12 +695,11 @@ notation4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -798,12 +712,11 @@ attlist0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist1(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -819,34 +732,23 @@ attlist1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist2(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
-  case XML_TOK_NAME:
-    {
-      static const char * const types[] = {
-        KW_CDATA,
-        KW_ID,
-        KW_IDREF,
-        KW_IDREFS,
-        KW_ENTITY,
-        KW_ENTITIES,
-        KW_NMTOKEN,
-        KW_NMTOKENS,
-      };
-      int i;
-      for (i = 0; i < (int)(sizeof(types)/sizeof(types[0])); i++)
-        if (XmlNameMatchesAscii(enc, ptr, end, types[i])) {
-          state->handler = attlist8;
-          return XML_ROLE_ATTRIBUTE_TYPE_CDATA + i;
-        }
-    }
+  case XML_TOK_NAME: {
+    static const char *const types[] = {
+        KW_CDATA,  KW_ID,       KW_IDREF,   KW_IDREFS,
+        KW_ENTITY, KW_ENTITIES, KW_NMTOKEN, KW_NMTOKENS,
+    };
+    int i;
+    for (i = 0; i < (int)(sizeof(types) / sizeof(types[0])); i++)
+      if (XmlNameMatchesAscii(enc, ptr, end, types[i])) {
+        state->handler = attlist8;
+        return XML_ROLE_ATTRIBUTE_TYPE_CDATA + i;
+      }
+  }
     if (XmlNameMatchesAscii(enc, ptr, end, KW_NOTATION)) {
       state->handler = attlist5;
       return XML_ROLE_ATTLIST_NONE;
@@ -860,12 +762,11 @@ attlist2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -879,12 +780,11 @@ attlist3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -899,12 +799,11 @@ attlist4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -916,12 +815,11 @@ attlist5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist6(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -933,12 +831,11 @@ attlist6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist7(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -954,33 +851,23 @@ attlist7(PROLOG_STATE *state,
 
 /* default value */
 static int PTRCALL
-attlist8(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
   case XML_TOK_POUND_NAME:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_IMPLIED)) {
       state->handler = attlist1;
       return XML_ROLE_IMPLIED_ATTRIBUTE_VALUE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_REQUIRED)) {
       state->handler = attlist1;
       return XML_ROLE_REQUIRED_ATTRIBUTE_VALUE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_FIXED)) {
       state->handler = attlist9;
       return XML_ROLE_ATTLIST_NONE;
@@ -994,12 +881,11 @@ attlist8(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist9(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -1011,12 +897,11 @@ attlist9(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1029,12 +914,8 @@ element0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element1(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1059,19 +940,13 @@ element1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element2(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
   case XML_TOK_POUND_NAME:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_PCDATA)) {
       state->handler = element3;
       return XML_ROLE_CONTENT_PCDATA;
@@ -1099,12 +974,11 @@ element2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1124,12 +998,11 @@ element3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1142,12 +1015,11 @@ element4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1163,12 +1035,11 @@ element5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element6(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1193,12 +1064,11 @@ element6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element7(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1243,12 +1113,8 @@ element7(PROLOG_STATE *state,
 #ifdef XML_DTD
 
 static int PTRCALL
-condSect0(PROLOG_STATE *state,
-          int tok,
-          const char *ptr,
-          const char *end,
-          const ENCODING *enc)
-{
+condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1267,12 +1133,11 @@ condSect0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-condSect1(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1285,12 +1150,11 @@ condSect1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-condSect2(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1304,12 +1168,11 @@ condSect2(PROLOG_STATE *state,
 #endif /* XML_DTD */
 
 static int PTRCALL
-declClose(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return state->role_none;
@@ -1341,21 +1204,21 @@ declClose(PROLOG_STATE *state,
  * LCOV_EXCL_START
  */
 static int PTRCALL
-error(PROLOG_STATE *UNUSED_P(state),
-      int UNUSED_P(tok),
-      const char *UNUSED_P(ptr),
-      const char *UNUSED_P(end),
-      const ENCODING *UNUSED_P(enc))
-{
+error(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+      const ENCODING *enc) {
+  UNUSED_P(state);
+  UNUSED_P(tok);
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   return XML_ROLE_NONE;
 }
 /* LCOV_EXCL_STOP */
 
 static int FASTCALL
-common(PROLOG_STATE *state, int tok)
-{
+common(PROLOG_STATE *state, int tok) {
 #ifdef XML_DTD
-  if (!state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF)
+  if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF)
     return XML_ROLE_INNER_PARAM_ENTITY_REF;
 #endif
   state->handler = error;
@@ -1363,8 +1226,7 @@ common(PROLOG_STATE *state, int tok)
 }
 
 void
-XmlPrologStateInit(PROLOG_STATE *state)
-{
+XmlPrologStateInit(PROLOG_STATE *state) {
   state->handler = prolog0;
 #ifdef XML_DTD
   state->documentEntity = 1;
@@ -1376,8 +1238,7 @@ XmlPrologStateInit(PROLOG_STATE *state)
 #ifdef XML_DTD
 
 void
-XmlPrologStateInitExternalEntity(PROLOG_STATE *state)
-{
+XmlPrologStateInitExternalEntity(PROLOG_STATE *state) {
   state->handler = externalSubset0;
   state->documentEntity = 0;
   state->includeLevel = 0;
diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h
index e5f048eab55c..036aba64fd29 100644
--- a/Modules/expat/xmlrole.h
+++ b/Modules/expat/xmlrole.h
@@ -36,7 +36,7 @@
 #ifdef __VMS
 /*      0        1         2         3      0        1         2         3
         1234567890123456789012345678901     1234567890123456789012345678901 */
-#define XmlPrologStateInitExternalEntity    XmlPrologStateInitExternalEnt
+#  define XmlPrologStateInitExternalEntity XmlPrologStateInitExternalEnt
 #endif
 
 #include "xmltok.h"
@@ -113,11 +113,8 @@ enum {
 };
 
 typedef struct prolog_state {
-  int (PTRCALL *handler) (struct prolog_state *state,
-                          int tok,
-                          const char *ptr,
-                          const char *end,
-                          const ENCODING *enc);
+  int(PTRCALL *handler)(struct prolog_state *state, int tok, const char *ptr,
+                        const char *end, const ENCODING *enc);
   unsigned level;
   int role_none;
 #ifdef XML_DTD
@@ -132,8 +129,8 @@ void XmlPrologStateInit(PROLOG_STATE *);
 void XmlPrologStateInitExternalEntity(PROLOG_STATE *);
 #endif /* XML_DTD */
 
-#define XmlTokenRole(state, tok, ptr, end, enc) \
- (((state)->handler)(state, tok, ptr, end, enc))
+#define XmlTokenRole(state, tok, ptr, end, enc)                                \
+  (((state)->handler)(state, tok, ptr, end, enc))
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c
index 6b415d83972c..11e9d1ccdad4 100644
--- a/Modules/expat/xmltok.c
+++ b/Modules/expat/xmltok.c
@@ -31,24 +31,23 @@
 */
 
 #include <stddef.h>
-#include <string.h>  /* memcpy */
+#include <string.h> /* memcpy */
 
 #if defined(_MSC_VER) && (_MSC_VER <= 1700)
-  /* for vs2012/11.0/1700 and earlier Visual Studio compilers */
-# define bool   int
-# define false  0
-# define true   1
+/* for vs2012/11.0/1700 and earlier Visual Studio compilers */
+#  define bool int
+#  define false 0
+#  define true 1
 #else
-# include <stdbool.h>
+#  include <stdbool.h>
 #endif
 
-
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #else
-#ifdef HAVE_EXPAT_CONFIG_H
-#include <expat_config.h>
-#endif
+#  ifdef HAVE_EXPAT_CONFIG_H
+#    include <expat_config.h>
+#  endif
 #endif /* ndef _WIN32 */
 
 #include "expat_external.h"
@@ -57,58 +56,49 @@
 #include "nametab.h"
 
 #ifdef XML_DTD
-#define IGNORE_SECTION_TOK_VTABLE , PREFIX(ignoreSectionTok)
+#  define IGNORE_SECTION_TOK_VTABLE , PREFIX(ignoreSectionTok)
 #else
-#define IGNORE_SECTION_TOK_VTABLE /* as nothing */
+#  define IGNORE_SECTION_TOK_VTABLE /* as nothing */
 #endif
 
-#define VTABLE1 \
-  { PREFIX(prologTok), PREFIX(contentTok), \
-    PREFIX(cdataSectionTok) IGNORE_SECTION_TOK_VTABLE }, \
-  { PREFIX(attributeValueTok), PREFIX(entityValueTok) }, \
-  PREFIX(nameMatchesAscii), \
-  PREFIX(nameLength), \
-  PREFIX(skipS), \
-  PREFIX(getAtts), \
-  PREFIX(charRefNumber), \
-  PREFIX(predefinedEntityName), \
-  PREFIX(updatePosition), \
-  PREFIX(isPublicId)
+#define VTABLE1                                                                \
+  {PREFIX(prologTok), PREFIX(contentTok),                                      \
+   PREFIX(cdataSectionTok) IGNORE_SECTION_TOK_VTABLE},                         \
+      {PREFIX(attributeValueTok), PREFIX(entityValueTok)},                     \
+      PREFIX(nameMatchesAscii), PREFIX(nameLength), PREFIX(skipS),             \
+      PREFIX(getAtts), PREFIX(charRefNumber), PREFIX(predefinedEntityName),    \
+      PREFIX(updatePosition), PREFIX(isPublicId)
 
 #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16)
 
-#define UCS2_GET_NAMING(pages, hi, lo) \
-   (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo) & 0x1F)))
+#define UCS2_GET_NAMING(pages, hi, lo)                                         \
+  (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo)&0x1F)))
 
 /* A 2 byte UTF-8 representation splits the characters 11 bits between
    the bottom 5 and 6 bits of the bytes.  We need 8 bits to index into
    pages, 3 bits to add to that index and 5 bits to generate the mask.
 */
-#define UTF8_GET_NAMING2(pages, byte) \
-    (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3) \
-                      + ((((byte)[0]) & 3) << 1) \
-                      + ((((byte)[1]) >> 5) & 1)] \
-         & (1u << (((byte)[1]) & 0x1F)))
+#define UTF8_GET_NAMING2(pages, byte)                                          \
+  (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3)                         \
+                + ((((byte)[0]) & 3) << 1) + ((((byte)[1]) >> 5) & 1)]         \
+   & (1u << (((byte)[1]) & 0x1F)))
 
 /* A 3 byte UTF-8 representation splits the characters 16 bits between
    the bottom 4, 6 and 6 bits of the bytes.  We need 8 bits to index
    into pages, 3 bits to add to that index and 5 bits to generate the
    mask.
 */
-#define UTF8_GET_NAMING3(pages, byte) \
-  (namingBitmap[((pages)[((((byte)[0]) & 0xF) << 4) \
-                             + ((((byte)[1]) >> 2) & 0xF)] \
-                       << 3) \
-                      + ((((byte)[1]) & 3) << 1) \
-                      + ((((byte)[2]) >> 5) & 1)] \
-         & (1u << (((byte)[2]) & 0x1F)))
-
-#define UTF8_GET_NAMING(pages, p, n) \
-  ((n) == 2 \
-  ? UTF8_GET_NAMING2(pages, (const unsigned char *)(p)) \
-  : ((n) == 3 \
-     ? UTF8_GET_NAMING3(pages, (const unsigned char *)(p)) \
-     : 0))
+#define UTF8_GET_NAMING3(pages, byte)                                          \
+  (namingBitmap                                                                \
+       [((pages)[((((byte)[0]) & 0xF) << 4) + ((((byte)[1]) >> 2) & 0xF)]      \
+         << 3)                                                                 \
+        + ((((byte)[1]) & 3) << 1) + ((((byte)[2]) >> 5) & 1)]                 \
+   & (1u << (((byte)[2]) & 0x1F)))
+
+#define UTF8_GET_NAMING(pages, p, n)                                           \
+  ((n) == 2                                                                    \
+       ? UTF8_GET_NAMING2(pages, (const unsigned char *)(p))                   \
+       : ((n) == 3 ? UTF8_GET_NAMING3(pages, (const unsigned char *)(p)) : 0))
 
 /* Detection of invalid UTF-8 sequences is based on Table 3.1B
    of Unicode 3.2: http://www.unicode.org/unicode/reports/tr28/
@@ -120,88 +110,76 @@
      (A & 0xC0) == 0xC0  means A > 0xBF
 */
 
-#define UTF8_INVALID2(p) \
+#define UTF8_INVALID2(p)                                                       \
   ((*p) < 0xC2 || ((p)[1] & 0x80) == 0 || ((p)[1] & 0xC0) == 0xC0)
 
-#define UTF8_INVALID3(p) \
-  (((p)[2] & 0x80) == 0 \
-  || \
-  ((*p) == 0xEF && (p)[1] == 0xBF \
-    ? \
-    (p)[2] > 0xBD \
-    : \
-    ((p)[2] & 0xC0) == 0xC0) \
-  || \
-  ((*p) == 0xE0 \
-    ? \
-    (p)[1] < 0xA0 || ((p)[1] & 0xC0) == 0xC0 \
-    : \
-    ((p)[1] & 0x80) == 0 \
-    || \
-    ((*p) == 0xED ? (p)[1] > 0x9F : ((p)[1] & 0xC0) == 0xC0)))
-
-#define UTF8_INVALID4(p) \
-  (((p)[3] & 0x80) == 0 || ((p)[3] & 0xC0) == 0xC0 \
-  || \
-  ((p)[2] & 0x80) == 0 || ((p)[2] & 0xC0) == 0xC0 \
-  || \
-  ((*p) == 0xF0 \
-    ? \
-    (p)[1] < 0x90 || ((p)[1] & 0xC0) == 0xC0 \
-    : \
-    ((p)[1] & 0x80) == 0 \
-    || \
-    ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0)))
+#define UTF8_INVALID3(p)                                                       \
+  (((p)[2] & 0x80) == 0                                                        \
+   || ((*p) == 0xEF && (p)[1] == 0xBF ? (p)[2] > 0xBD                          \
+                                      : ((p)[2] & 0xC0) == 0xC0)               \
+   || ((*p) == 0xE0                                                            \
+           ? (p)[1] < 0xA0 || ((p)[1] & 0xC0) == 0xC0                          \
+           : ((p)[1] & 0x80) == 0                                              \
+                 || ((*p) == 0xED ? (p)[1] > 0x9F : ((p)[1] & 0xC0) == 0xC0)))
+
+#define UTF8_INVALID4(p)                                                       \
+  (((p)[3] & 0x80) == 0 || ((p)[3] & 0xC0) == 0xC0 || ((p)[2] & 0x80) == 0     \
+   || ((p)[2] & 0xC0) == 0xC0                                                  \
+   || ((*p) == 0xF0                                                            \
+           ? (p)[1] < 0x90 || ((p)[1] & 0xC0) == 0xC0                          \
+           : ((p)[1] & 0x80) == 0                                              \
+                 || ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0)))
 
 static int PTRFASTCALL
-isNever(const ENCODING *UNUSED_P(enc), const char *UNUSED_P(p))
-{
+isNever(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  UNUSED_P(p);
   return 0;
 }
 
 static int PTRFASTCALL
-utf8_isName2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isName2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING2(namePages, (const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isName3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isName3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING3(namePages, (const unsigned char *)p);
 }
 
 #define utf8_isName4 isNever
 
 static int PTRFASTCALL
-utf8_isNmstrt2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isNmstrt2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isNmstrt3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isNmstrt3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p);
 }
 
 #define utf8_isNmstrt4 isNever
 
 static int PTRFASTCALL
-utf8_isInvalid2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID2((const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isInvalid3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID3((const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isInvalid4(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid4(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID4((const unsigned char *)p);
 }
 
@@ -209,61 +187,44 @@ struct normal_encoding {
   ENCODING enc;
   unsigned char type[256];
 #ifdef XML_MIN_SIZE
-  int (PTRFASTCALL *byteType)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNameMin)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *);
-  int (PTRFASTCALL *byteToAscii)(const ENCODING *, const char *);
-  int (PTRCALL *charMatches)(const ENCODING *, const char *, int);
+  int(PTRFASTCALL *byteType)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNameMin)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *);
+  int(PTRFASTCALL *byteToAscii)(const ENCODING *, const char *);
+  int(PTRCALL *charMatches)(const ENCODING *, const char *, int);
 #endif /* XML_MIN_SIZE */
-  int (PTRFASTCALL *isName2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isName3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isName4)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid4)(const ENCODING *, const char *);
 };
 
-#define AS_NORMAL_ENCODING(enc)   ((const struct normal_encoding *) (enc))
+#define AS_NORMAL_ENCODING(enc) ((const struct normal_encoding *)(enc))
 
 #ifdef XML_MIN_SIZE
 
-#define STANDARD_VTABLE(E) \
- E ## byteType, \
- E ## isNameMin, \
- E ## isNmstrtMin, \
- E ## byteToAscii, \
- E ## charMatches,
+#  define STANDARD_VTABLE(E)                                                   \
+    E##byteType, E##isNameMin, E##isNmstrtMin, E##byteToAscii, E##charMatches,
 
 #else
 
-#define STANDARD_VTABLE(E) /* as nothing */
+#  define STANDARD_VTABLE(E) /* as nothing */
 
 #endif
 
-#define NORMAL_VTABLE(E) \
- E ## isName2, \
- E ## isName3, \
- E ## isName4, \
- E ## isNmstrt2, \
- E ## isNmstrt3, \
- E ## isNmstrt4, \
- E ## isInvalid2, \
- E ## isInvalid3, \
- E ## isInvalid4
-
-#define NULL_VTABLE \
- /* isName2 */ NULL, \
- /* isName3 */ NULL, \
- /* isName4 */ NULL, \
- /* isNmstrt2 */ NULL, \
- /* isNmstrt3 */ NULL, \
- /* isNmstrt4 */ NULL, \
- /* isInvalid2 */ NULL, \
- /* isInvalid3 */ NULL, \
- /* isInvalid4 */ NULL
+#define NORMAL_VTABLE(E)                                                       \
+  E##isName2, E##isName3, E##isName4, E##isNmstrt2, E##isNmstrt3,              \
+      E##isNmstrt4, E##isInvalid2, E##isInvalid3, E##isInvalid4
+
+#define NULL_VTABLE                                                            \
+  /* isName2 */ NULL, /* isName3 */ NULL, /* isName4 */ NULL,                  \
+      /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL,        \
+      /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL
 
 static int FASTCALL checkCharRefNumber(int);
 
@@ -271,75 +232,70 @@ static int FASTCALL checkCharRefNumber(int);
 #include "ascii.h"
 
 #ifdef XML_MIN_SIZE
-#define sb_isNameMin isNever
-#define sb_isNmstrtMin isNever
+#  define sb_isNameMin isNever
+#  define sb_isNmstrtMin isNever
 #endif
 
 #ifdef XML_MIN_SIZE
-#define MINBPC(enc) ((enc)->minBytesPerChar)
+#  define MINBPC(enc) ((enc)->minBytesPerChar)
 #else
 /* minimum bytes per character */
-#define MINBPC(enc) 1
+#  define MINBPC(enc) 1
 #endif
 
-#define SB_BYTE_TYPE(enc, p) \
+#define SB_BYTE_TYPE(enc, p)                                                   \
   (((struct normal_encoding *)(enc))->type[(unsigned char)*(p)])
 
 #ifdef XML_MIN_SIZE
 static int PTRFASTCALL
-sb_byteType(const ENCODING *enc, const char *p)
-{
+sb_byteType(const ENCODING *enc, const char *p) {
   return SB_BYTE_TYPE(enc, p);
 }
-#define BYTE_TYPE(enc, p) \
- (AS_NORMAL_ENCODING(enc)->byteType(enc, p))
+#  define BYTE_TYPE(enc, p) (AS_NORMAL_ENCODING(enc)->byteType(enc, p))
 #else
-#define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
+#  define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
 #endif
 
 #ifdef XML_MIN_SIZE
-#define BYTE_TO_ASCII(enc, p) \
- (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p))
+#  define BYTE_TO_ASCII(enc, p) (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p))
 static int PTRFASTCALL
-sb_byteToAscii(const ENCODING *enc, const char *p)
-{
+sb_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return *p;
 }
 #else
-#define BYTE_TO_ASCII(enc, p) (*(p))
+#  define BYTE_TO_ASCII(enc, p) (*(p))
 #endif
 
-#define IS_NAME_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isName ## n(enc, p))
-#define IS_NMSTRT_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isNmstrt ## n(enc, p))
-#define IS_INVALID_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isInvalid ## n(enc, p))
+#define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p))
+#define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p))
+#define IS_INVALID_CHAR(enc, p, n)                                             \
+  (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p))
 
 #ifdef XML_MIN_SIZE
-#define IS_NAME_CHAR_MINBPC(enc, p) \
- (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p))
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) \
- (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p))
+#  define IS_NAME_CHAR_MINBPC(enc, p)                                          \
+    (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p))
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p)                                        \
+    (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p))
 #else
-#define IS_NAME_CHAR_MINBPC(enc, p) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
+#  define IS_NAME_CHAR_MINBPC(enc, p) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
 #endif
 
 #ifdef XML_MIN_SIZE
-#define CHAR_MATCHES(enc, p, c) \
- (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c))
+#  define CHAR_MATCHES(enc, p, c)                                              \
+    (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c))
 static int PTRCALL
-sb_charMatches(const ENCODING *enc, const char *p, int c)
-{
+sb_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
   return *p == c;
 }
 #else
 /* c is an ASCII character */
-#define CHAR_MATCHES(enc, p, c) (*(p) == c)
+#  define CHAR_MATCHES(enc, p, c) (*(p) == c)
 #endif
 
-#define PREFIX(ident) normal_ ## ident
+#define PREFIX(ident) normal_##ident
 #define XML_TOK_IMPL_C
 #include "xmltok_impl.c"
 #undef XML_TOK_IMPL_C
@@ -354,42 +310,46 @@ sb_charMatches(const ENCODING *enc, const char *p, int c)
 #undef IS_NMSTRT_CHAR_MINBPC
 #undef IS_INVALID_CHAR
 
-enum {  /* UTF8_cvalN is value of masked first byte of N byte sequence */
-  UTF8_cval1 = 0x00,
-  UTF8_cval2 = 0xc0,
-  UTF8_cval3 = 0xe0,
-  UTF8_cval4 = 0xf0
+enum { /* UTF8_cvalN is value of masked first byte of N byte sequence */
+       UTF8_cval1 = 0x00,
+       UTF8_cval2 = 0xc0,
+       UTF8_cval3 = 0xe0,
+       UTF8_cval4 = 0xf0
 };
 
 void
-_INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** fromLimRef)
-{
-  const char * fromLim = *fromLimRef;
+_INTERNAL_trim_to_complete_utf8_characters(const char *from,
+                                           const char **fromLimRef) {
+  const char *fromLim = *fromLimRef;
   size_t walked = 0;
   for (; fromLim > from; fromLim--, walked++) {
     const unsigned char prev = (unsigned char)fromLim[-1];
-    if ((prev & 0xf8u) == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */
+    if ((prev & 0xf8u)
+        == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */
       if (walked + 1 >= 4) {
         fromLim += 4 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0xf0u) == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */
+    } else if ((prev & 0xf0u)
+               == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */
       if (walked + 1 >= 3) {
         fromLim += 3 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0xe0u) == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */
+    } else if ((prev & 0xe0u)
+               == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */
       if (walked + 1 >= 2) {
         fromLim += 2 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0x80u) == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */
+    } else if ((prev & 0x80u)
+               == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */
       break;
     }
   }
@@ -397,16 +357,15 @@ _INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** from
 }
 
 static enum XML_Convert_Result PTRCALL
-utf8_toUtf8(const ENCODING *UNUSED_P(enc),
-            const char **fromP, const char *fromLim,
-            char **toP, const char *toLim)
-{
+utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+            char **toP, const char *toLim) {
   bool input_incomplete = false;
   bool output_exhausted = false;
 
   /* Avoid copying partial characters (due to limited space). */
   const ptrdiff_t bytesAvailable = fromLim - *fromP;
   const ptrdiff_t bytesStorable = toLim - *toP;
+  UNUSED_P(enc);
   if (bytesAvailable > bytesStorable) {
     fromLim = *fromP + bytesStorable;
     output_exhausted = true;
@@ -414,7 +373,7 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
 
   /* Avoid copying partial characters (from incomplete input). */
   {
-    const char * const fromLimBefore = fromLim;
+    const char *const fromLimBefore = fromLim;
     _INTERNAL_trim_to_complete_utf8_characters(*fromP, &fromLim);
     if (fromLim < fromLimBefore) {
       input_incomplete = true;
@@ -428,7 +387,7 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
     *toP += bytesToCopy;
   }
 
-  if (output_exhausted)  /* needs to go first */
+  if (output_exhausted) /* needs to go first */
     return XML_CONVERT_OUTPUT_EXHAUSTED;
   else if (input_incomplete)
     return XML_CONVERT_INPUT_INCOMPLETE;
@@ -437,10 +396,8 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
 }
 
 static enum XML_Convert_Result PTRCALL
-utf8_toUtf16(const ENCODING *enc,
-             const char **fromP, const char *fromLim,
-             unsigned short **toP, const unsigned short *toLim)
-{
+utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+             unsigned short **toP, const unsigned short *toLim) {
   enum XML_Convert_Result res = XML_CONVERT_COMPLETED;
   unsigned short *to = *toP;
   const char *from = *fromP;
@@ -459,30 +416,28 @@ utf8_toUtf16(const ENCODING *enc,
         res = XML_CONVERT_INPUT_INCOMPLETE;
         goto after;
       }
-      *to++ = (unsigned short)(((from[0] & 0xf) << 12)
-                               | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f));
+      *to++ = (unsigned short)(((from[0] & 0xf) << 12) | ((from[1] & 0x3f) << 6)
+                               | (from[2] & 0x3f));
       from += 3;
       break;
-    case BT_LEAD4:
-      {
-        unsigned long n;
-        if (toLim - to < 2) {
-          res = XML_CONVERT_OUTPUT_EXHAUSTED;
-          goto after;
-        }
-        if (fromLim - from < 4) {
-          res = XML_CONVERT_INPUT_INCOMPLETE;
-          goto after;
-        }
-        n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12)
-            | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f);
-        n -= 0x10000;
-        to[0] = (unsigned short)((n >> 10) | 0xD800);
-        to[1] = (unsigned short)((n & 0x3FF) | 0xDC00);
-        to += 2;
-        from += 4;
+    case BT_LEAD4: {
+      unsigned long n;
+      if (toLim - to < 2) {
+        res = XML_CONVERT_OUTPUT_EXHAUSTED;
+        goto after;
       }
-      break;
+      if (fromLim - from < 4) {
+        res = XML_CONVERT_INPUT_INCOMPLETE;
+        goto after;
+      }
+      n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12)
+          | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f);
+      n -= 0x10000;
+      to[0] = (unsigned short)((n >> 10) | 0xD800);
+      to[1] = (unsigned short)((n & 0x3FF) | 0xDC00);
+      to += 2;
+      from += 4;
+    } break;
     default:
       *to++ = *from++;
       break;
@@ -497,56 +452,51 @@ utf8_toUtf16(const ENCODING *enc,
 }
 
 #ifdef XML_NS
-static const struct normal_encoding utf8_encoding_ns = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
-#include "asciitab.h"
-#include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+static const struct normal_encoding utf8_encoding_ns
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
+#  include "asciitab.h"
+#  include "utf8tab.h"
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 #endif
 
-static const struct normal_encoding utf8_encoding = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding utf8_encoding
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 #ifdef XML_NS
 
-static const struct normal_encoding internal_utf8_encoding_ns = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
-#include "iasciitab.h"
-#include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+static const struct normal_encoding internal_utf8_encoding_ns
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
+#  include "iasciitab.h"
+#  include "utf8tab.h"
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 #endif
 
-static const struct normal_encoding internal_utf8_encoding = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding internal_utf8_encoding
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "iasciitab.h"
 #undef BT_COLON
 #include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 static enum XML_Convert_Result PTRCALL
-latin1_toUtf8(const ENCODING *UNUSED_P(enc),
-              const char **fromP, const char *fromLim,
-              char **toP, const char *toLim)
-{
+latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+              char **toP, const char *toLim) {
+  UNUSED_P(enc);
   for (;;) {
     unsigned char c;
     if (*fromP == fromLim)
@@ -558,8 +508,7 @@ latin1_toUtf8(const ENCODING *UNUSED_P(enc),
       *(*toP)++ = (char)((c >> 6) | UTF8_cval2);
       *(*toP)++ = (char)((c & 0x3f) | 0x80);
       (*fromP)++;
-    }
-    else {
+    } else {
       if (*toP == toLim)
         return XML_CONVERT_OUTPUT_EXHAUSTED;
       *(*toP)++ = *(*fromP)++;
@@ -568,10 +517,9 @@ latin1_toUtf8(const ENCODING *UNUSED_P(enc),
 }
 
 static enum XML_Convert_Result PTRCALL
-latin1_toUtf16(const ENCODING *UNUSED_P(enc),
-               const char **fromP, const char *fromLim,
-               unsigned short **toP, const unsigned short *toLim)
-{
+latin1_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+               unsigned short **toP, const unsigned short *toLim) {
+  UNUSED_P(enc);
   while (*fromP < fromLim && *toP < toLim)
     *(*toP)++ = (unsigned char)*(*fromP)++;
 
@@ -583,33 +531,30 @@ latin1_toUtf16(const ENCODING *UNUSED_P(enc),
 
 #ifdef XML_NS
 
-static const struct normal_encoding latin1_encoding_ns = {
-  { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+static const struct normal_encoding latin1_encoding_ns
+    = {{VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0},
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding latin1_encoding = {
-  { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
-  {
+static const struct normal_encoding latin1_encoding
+    = {{VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 static enum XML_Convert_Result PTRCALL
-ascii_toUtf8(const ENCODING *UNUSED_P(enc),
-             const char **fromP, const char *fromLim,
-             char **toP, const char *toLim)
-{
+ascii_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+             char **toP, const char *toLim) {
+  UNUSED_P(enc);
   while (*fromP < fromLim && *toP < toLim)
     *(*toP)++ = *(*fromP)++;
 
@@ -621,40 +566,45 @@ ascii_toUtf8(const ENCODING *UNUSED_P(enc),
 
 #ifdef XML_NS
 
-static const struct normal_encoding ascii_encoding_ns = {
-  { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
-  {
-#include "asciitab.h"
-/* BT_NONXML == 0 */
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+static const struct normal_encoding ascii_encoding_ns
+    = {{VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0},
+       {
+#  include "asciitab.h"
+           /* BT_NONXML == 0 */
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding ascii_encoding = {
-  { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding ascii_encoding
+    = {{VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
-/* BT_NONXML == 0 */
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+           /* BT_NONXML == 0 */
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 static int PTRFASTCALL
-unicode_byte_type(char hi, char lo)
-{
+unicode_byte_type(char hi, char lo) {
   switch ((unsigned char)hi) {
-  case 0xD8: case 0xD9: case 0xDA: case 0xDB:
+  /* 0xD800?0xDBFF first 16-bit code unit or high surrogate (W1) */
+  case 0xD8:
+  case 0xD9:
+  case 0xDA:
+  case 0xDB:
     return BT_LEAD4;
-  case 0xDC: case 0xDD: case 0xDE: case 0xDF:
+  /* 0xDC00?0xDFFF second 16-bit code unit or low surrogate (W2) */
+  case 0xDC:
+  case 0xDD:
+  case 0xDE:
+  case 0xDF:
     return BT_TRAIL;
   case 0xFF:
     switch ((unsigned char)lo) {
-    case 0xFF:
-    case 0xFE:
+    case 0xFF: /* noncharacter-FFFF */
+    case 0xFE: /* noncharacter-FFFE */
       return BT_NONXML;
     }
     break;
@@ -662,102 +612,105 @@ unicode_byte_type(char hi, char lo)
   return BT_NONASCII;
 }
 
-#define DEFINE_UTF16_TO_UTF8(E) \
-static enum XML_Convert_Result  PTRCALL \
-E ## toUtf8(const ENCODING *UNUSED_P(enc), \
-            const char **fromP, const char *fromLim, \
-            char **toP, const char *toLim) \
-{ \
-  const char *from = *fromP; \
-  fromLim = from + (((fromLim - from) >> 1) << 1);  /* shrink to even */ \
-  for (; from < fromLim; from += 2) { \
-    int plane; \
-    unsigned char lo2; \
-    unsigned char lo = GET_LO(from); \
-    unsigned char hi = GET_HI(from); \
-    switch (hi) { \
-    case 0: \
-      if (lo < 0x80) { \
-        if (*toP == toLim) { \
-          *fromP = from; \
-          return XML_CONVERT_OUTPUT_EXHAUSTED; \
-        } \
-        *(*toP)++ = lo; \
-        break; \
-      } \
-      /* fall through */ \
-    case 0x1: case 0x2: case 0x3: \
-    case 0x4: case 0x5: case 0x6: case 0x7: \
-      if (toLim -  *toP < 2) { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      *(*toP)++ = ((lo >> 6) | (hi << 2) |  UTF8_cval2); \
-      *(*toP)++ = ((lo & 0x3f) | 0x80); \
-      break; \
-    default: \
-      if (toLim -  *toP < 3)  { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      /* 16 bits divided 4, 6, 6 amongst 3 bytes */ \
-      *(*toP)++ = ((hi >> 4) | UTF8_cval3); \
-      *(*toP)++ = (((hi & 0xf) << 2) | (lo >> 6) | 0x80); \
-      *(*toP)++ = ((lo & 0x3f) | 0x80); \
-      break; \
-    case 0xD8: case 0xD9: case 0xDA: case 0xDB: \
-      if (toLim -  *toP < 4) { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      if (fromLim - from < 4) { \
-        *fromP = from; \
-        return XML_CONVERT_INPUT_INCOMPLETE; \
-      } \
-      plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1; \
-      *(*toP)++ = ((plane >> 2) | UTF8_cval4); \
-      *(*toP)++ = (((lo >> 2) & 0xF) | ((plane & 0x3) << 4) | 0x80); \
-      from += 2; \
-      lo2 = GET_LO(from); \
-      *(*toP)++ = (((lo & 0x3) << 4) \
-                   | ((GET_HI(from) & 0x3) << 2) \
-                   | (lo2 >> 6) \
-                   | 0x80); \
-      *(*toP)++ = ((lo2 & 0x3f) | 0x80); \
-      break; \
-    } \
-  } \
-  *fromP = from; \
-  if (from < fromLim) \
-    return XML_CONVERT_INPUT_INCOMPLETE; \
-  else \
-    return XML_CONVERT_COMPLETED; \
-}
+#define DEFINE_UTF16_TO_UTF8(E)                                                \
+  static enum XML_Convert_Result PTRCALL E##toUtf8(                            \
+      const ENCODING *enc, const char **fromP, const char *fromLim,            \
+      char **toP, const char *toLim) {                                         \
+    const char *from = *fromP;                                                 \
+    UNUSED_P(enc);                                                             \
+    fromLim = from + (((fromLim - from) >> 1) << 1); /* shrink to even */      \
+    for (; from < fromLim; from += 2) {                                        \
+      int plane;                                                               \
+      unsigned char lo2;                                                       \
+      unsigned char lo = GET_LO(from);                                         \
+      unsigned char hi = GET_HI(from);                                         \
+      switch (hi) {                                                            \
+      case 0:                                                                  \
+        if (lo < 0x80) {                                                       \
+          if (*toP == toLim) {                                                 \
+            *fromP = from;                                                     \
+            return XML_CONVERT_OUTPUT_EXHAUSTED;                               \
+          }                                                                    \
+          *(*toP)++ = lo;                                                      \
+          break;                                                               \
+        }                                                                      \
+        /* fall through */                                                     \
+      case 0x1:                                                                \
+      case 0x2:                                                                \
+      case 0x3:                                                                \
+      case 0x4:                                                                \
+      case 0x5:                                                                \
+      case 0x6:                                                                \
+      case 0x7:                                                                \
+        if (toLim - *toP < 2) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        *(*toP)++ = ((lo >> 6) | (hi << 2) | UTF8_cval2);                      \
+        *(*toP)++ = ((lo & 0x3f) | 0x80);                                      \
+        break;                                                                 \
+      default:                                                                 \
+        if (toLim - *toP < 3) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        /* 16 bits divided 4, 6, 6 amongst 3 bytes */                          \
+        *(*toP)++ = ((hi >> 4) | UTF8_cval3);                                  \
+        *(*toP)++ = (((hi & 0xf) << 2) | (lo >> 6) | 0x80);                    \
+        *(*toP)++ = ((lo & 0x3f) | 0x80);                                      \
+        break;                                                                 \
+      case 0xD8:                                                               \
+      case 0xD9:                                                               \
+      case 0xDA:                                                               \
+      case 0xDB:                                                               \
+        if (toLim - *toP < 4) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        if (fromLim - from < 4) {                                              \
+          *fromP = from;                                                       \
+          return XML_CONVERT_INPUT_INCOMPLETE;                                 \
+        }                                                                      \
+        plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1;                   \
+        *(*toP)++ = (char)((plane >> 2) | UTF8_cval4);                         \
+        *(*toP)++ = (((lo >> 2) & 0xF) | ((plane & 0x3) << 4) | 0x80);         \
+        from += 2;                                                             \
+        lo2 = GET_LO(from);                                                    \
+        *(*toP)++ = (((lo & 0x3) << 4) | ((GET_HI(from) & 0x3) << 2)           \
+                     | (lo2 >> 6) | 0x80);                                     \
+        *(*toP)++ = ((lo2 & 0x3f) | 0x80);                                     \
+        break;                                                                 \
+      }                                                                        \
+    }                                                                          \
+    *fromP = from;                                                             \
+    if (from < fromLim)                                                        \
+      return XML_CONVERT_INPUT_INCOMPLETE;                                     \
+    else                                                                       \
+      return XML_CONVERT_COMPLETED;                                            \
+  }
 
-#define DEFINE_UTF16_TO_UTF16(E) \
-static enum XML_Convert_Result  PTRCALL \
-E ## toUtf16(const ENCODING *UNUSED_P(enc), \
-             const char **fromP, const char *fromLim, \
-             unsigned short **toP, const unsigned short *toLim) \
-{ \
-  enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \
-  fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1);  /* shrink to even */ \
-  /* Avoid copying first half only of surrogate */ \
-  if (fromLim - *fromP > ((toLim - *toP) << 1) \
-      && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \
-    fromLim -= 2; \
-    res = XML_CONVERT_INPUT_INCOMPLETE; \
-  } \
-  for (; *fromP < fromLim && *toP < toLim; *fromP += 2) \
-    *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP); \
-  if ((*toP == toLim) && (*fromP < fromLim)) \
-    return XML_CONVERT_OUTPUT_EXHAUSTED; \
-  else \
-    return res; \
-}
+#define DEFINE_UTF16_TO_UTF16(E)                                               \
+  static enum XML_Convert_Result PTRCALL E##toUtf16(                           \
+      const ENCODING *enc, const char **fromP, const char *fromLim,            \
+      unsigned short **toP, const unsigned short *toLim) {                     \
+    enum XML_Convert_Result res = XML_CONVERT_COMPLETED;                       \
+    UNUSED_P(enc);                                                             \
+    fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */  \
+    /* Avoid copying first half only of surrogate */                           \
+    if (fromLim - *fromP > ((toLim - *toP) << 1)                               \
+        && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) {                             \
+      fromLim -= 2;                                                            \
+      res = XML_CONVERT_INPUT_INCOMPLETE;                                      \
+    }                                                                          \
+    for (; *fromP < fromLim && *toP < toLim; *fromP += 2)                      \
+      *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP);                      \
+    if ((*toP == toLim) && (*fromP < fromLim))                                 \
+      return XML_CONVERT_OUTPUT_EXHAUSTED;                                     \
+    else                                                                       \
+      return res;                                                              \
+  }
 
-#define SET2(ptr, ch) \
-  (((ptr)[0] = ((ch) & 0xff)), ((ptr)[1] = ((ch) >> 8)))
+#define SET2(ptr, ch) (((ptr)[0] = ((ch)&0xff)), ((ptr)[1] = ((ch) >> 8)))
 #define GET_LO(ptr) ((unsigned char)(ptr)[0])
 #define GET_HI(ptr) ((unsigned char)(ptr)[1])
 
@@ -768,8 +721,7 @@ DEFINE_UTF16_TO_UTF16(little2_)
 #undef GET_LO
 #undef GET_HI
 
-#define SET2(ptr, ch) \
-  (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch) & 0xFF)))
+#define SET2(ptr, ch) (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch)&0xFF)))
 #define GET_LO(ptr) ((unsigned char)(ptr)[1])
 #define GET_HI(ptr) ((unsigned char)(ptr)[0])
 
@@ -780,292 +732,279 @@ DEFINE_UTF16_TO_UTF16(big2_)
 #undef GET_LO
 #undef GET_HI
 
-#define LITTLE2_BYTE_TYPE(enc, p) \
- ((p)[1] == 0 \
-  ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)] \
-  : unicode_byte_type((p)[1], (p)[0]))
-#define LITTLE2_BYTE_TO_ASCII(enc, p) ((p)[1] == 0 ? (p)[0] : -1)
-#define LITTLE2_CHAR_MATCHES(enc, p, c) ((p)[1] == 0 && (p)[0] == c)
-#define LITTLE2_IS_NAME_CHAR_MINBPC(enc, p) \
+#define LITTLE2_BYTE_TYPE(enc, p)                                              \
+  ((p)[1] == 0 ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)]  \
+               : unicode_byte_type((p)[1], (p)[0]))
+#define LITTLE2_BYTE_TO_ASCII(p) ((p)[1] == 0 ? (p)[0] : -1)
+#define LITTLE2_CHAR_MATCHES(p, c) ((p)[1] == 0 && (p)[0] == c)
+#define LITTLE2_IS_NAME_CHAR_MINBPC(p)                                         \
   UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0])
-#define LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
+#define LITTLE2_IS_NMSTRT_CHAR_MINBPC(p)                                       \
   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[1], (unsigned char)p[0])
 
 #ifdef XML_MIN_SIZE
 
 static int PTRFASTCALL
-little2_byteType(const ENCODING *enc, const char *p)
-{
+little2_byteType(const ENCODING *enc, const char *p) {
   return LITTLE2_BYTE_TYPE(enc, p);
 }
 
 static int PTRFASTCALL
-little2_byteToAscii(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_BYTE_TO_ASCII(enc, p);
+little2_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_BYTE_TO_ASCII(p);
 }
 
 static int PTRCALL
-little2_charMatches(const ENCODING *enc, const char *p, int c)
-{
-  return LITTLE2_CHAR_MATCHES(enc, p, c);
+little2_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
+  return LITTLE2_CHAR_MATCHES(p, c);
 }
 
 static int PTRFASTCALL
-little2_isNameMin(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_IS_NAME_CHAR_MINBPC(enc, p);
+little2_isNameMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_IS_NAME_CHAR_MINBPC(p);
 }
 
 static int PTRFASTCALL
-little2_isNmstrtMin(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p);
+little2_isNmstrtMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_IS_NMSTRT_CHAR_MINBPC(p);
 }
 
-#undef VTABLE
-#define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
+#  undef VTABLE
+#  define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
 
 #else /* not XML_MIN_SIZE */
 
-#undef PREFIX
-#define PREFIX(ident) little2_ ## ident
-#define MINBPC(enc) 2
+#  undef PREFIX
+#  define PREFIX(ident) little2_##ident
+#  define MINBPC(enc) 2
 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
-#define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
-#define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(enc, p)
-#define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(enc, p, c)
-#define IS_NAME_CHAR(enc, p, n) 0
-#define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(enc, p)
-#define IS_NMSTRT_CHAR(enc, p, n) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p)
-
-#define XML_TOK_IMPL_C
-#include "xmltok_impl.c"
-#undef XML_TOK_IMPL_C
-
-#undef MINBPC
-#undef BYTE_TYPE
-#undef BYTE_TO_ASCII
-#undef CHAR_MATCHES
-#undef IS_NAME_CHAR
-#undef IS_NAME_CHAR_MINBPC
-#undef IS_NMSTRT_CHAR
-#undef IS_NMSTRT_CHAR_MINBPC
-#undef IS_INVALID_CHAR
+#  define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
+#  define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p)
+#  define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c)
+#  define IS_NAME_CHAR(enc, p, n) 0
+#  define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p)
+#  define IS_NMSTRT_CHAR(enc, p, n) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p)
+
+#  define XML_TOK_IMPL_C
+#  include "xmltok_impl.c"
+#  undef XML_TOK_IMPL_C
+
+#  undef MINBPC
+#  undef BYTE_TYPE
+#  undef BYTE_TO_ASCII
+#  undef CHAR_MATCHES
+#  undef IS_NAME_CHAR
+#  undef IS_NAME_CHAR_MINBPC
+#  undef IS_NMSTRT_CHAR
+#  undef IS_NMSTRT_CHAR_MINBPC
+#  undef IS_INVALID_CHAR
 
 #endif /* not XML_MIN_SIZE */
 
 #ifdef XML_NS
 
-static const struct normal_encoding little2_encoding_ns = {
-  { VTABLE, 2, 0,
-#if BYTEORDER == 1234
-    1
-#else
-    0
-#endif
-  },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding little2_encoding_ns
+    = {{VTABLE, 2, 0,
+#  if BYTEORDER == 1234
+        1
+#  else
+        0
+#  endif
+       },
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding little2_encoding = {
-  { VTABLE, 2, 0,
+static const struct normal_encoding little2_encoding
+    = {{VTABLE, 2, 0,
 #if BYTEORDER == 1234
-    1
+        1
 #else
-    0
+        0
 #endif
-  },
-  {
+       },
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #if BYTEORDER != 4321
 
-#ifdef XML_NS
+#  ifdef XML_NS
 
-static const struct normal_encoding internal_little2_encoding_ns = {
-  { VTABLE, 2, 0, 1 },
-  {
-#include "iasciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_little2_encoding_ns
+    = {{VTABLE, 2, 0, 1},
+       {
+#    include "iasciitab.h"
+#    include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
-#endif
+#  endif
 
-static const struct normal_encoding internal_little2_encoding = {
-  { VTABLE, 2, 0, 1 },
-  {
-#define BT_COLON BT_NMSTRT
-#include "iasciitab.h"
-#undef BT_COLON
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_little2_encoding
+    = {{VTABLE, 2, 0, 1},
+       {
+#  define BT_COLON BT_NMSTRT
+#  include "iasciitab.h"
+#  undef BT_COLON
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #endif
 
-
-#define BIG2_BYTE_TYPE(enc, p) \
- ((p)[0] == 0 \
-  ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]] \
-  : unicode_byte_type((p)[0], (p)[1]))
-#define BIG2_BYTE_TO_ASCII(enc, p) ((p)[0] == 0 ? (p)[1] : -1)
-#define BIG2_CHAR_MATCHES(enc, p, c) ((p)[0] == 0 && (p)[1] == c)
-#define BIG2_IS_NAME_CHAR_MINBPC(enc, p) \
+#define BIG2_BYTE_TYPE(enc, p)                                                 \
+  ((p)[0] == 0                                                                 \
+       ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]]        \
+       : unicode_byte_type((p)[0], (p)[1]))
+#define BIG2_BYTE_TO_ASCII(p) ((p)[0] == 0 ? (p)[1] : -1)
+#define BIG2_CHAR_MATCHES(p, c) ((p)[0] == 0 && (p)[1] == c)
+#define BIG2_IS_NAME_CHAR_MINBPC(p)                                            \
   UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1])
-#define BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
+#define BIG2_IS_NMSTRT_CHAR_MINBPC(p)                                          \
   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[0], (unsigned char)p[1])
 
 #ifdef XML_MIN_SIZE
 
 static int PTRFASTCALL
-big2_byteType(const ENCODING *enc, const char *p)
-{
+big2_byteType(const ENCODING *enc, const char *p) {
   return BIG2_BYTE_TYPE(enc, p);
 }
 
 static int PTRFASTCALL
-big2_byteToAscii(const ENCODING *enc, const char *p)
-{
-  return BIG2_BYTE_TO_ASCII(enc, p);
+big2_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_BYTE_TO_ASCII(p);
 }
 
 static int PTRCALL
-big2_charMatches(const ENCODING *enc, const char *p, int c)
-{
-  return BIG2_CHAR_MATCHES(enc, p, c);
+big2_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
+  return BIG2_CHAR_MATCHES(p, c);
 }
 
 static int PTRFASTCALL
-big2_isNameMin(const ENCODING *enc, const char *p)
-{
-  return BIG2_IS_NAME_CHAR_MINBPC(enc, p);
+big2_isNameMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_IS_NAME_CHAR_MINBPC(p);
 }
 
 static int PTRFASTCALL
-big2_isNmstrtMin(const ENCODING *enc, const char *p)
-{
-  return BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p);
+big2_isNmstrtMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_IS_NMSTRT_CHAR_MINBPC(p);
 }
 
-#undef VTABLE
-#define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
+#  undef VTABLE
+#  define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
 
 #else /* not XML_MIN_SIZE */
 
-#undef PREFIX
-#define PREFIX(ident) big2_ ## ident
-#define MINBPC(enc) 2
+#  undef PREFIX
+#  define PREFIX(ident) big2_##ident
+#  define MINBPC(enc) 2
 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
-#define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
-#define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(enc, p)
-#define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(enc, p, c)
-#define IS_NAME_CHAR(enc, p, n) 0
-#define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(enc, p)
-#define IS_NMSTRT_CHAR(enc, p, n) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p)
-
-#define XML_TOK_IMPL_C
-#include "xmltok_impl.c"
-#undef XML_TOK_IMPL_C
-
-#undef MINBPC
-#undef BYTE_TYPE
-#undef BYTE_TO_ASCII
-#undef CHAR_MATCHES
-#undef IS_NAME_CHAR
-#undef IS_NAME_CHAR_MINBPC
-#undef IS_NMSTRT_CHAR
-#undef IS_NMSTRT_CHAR_MINBPC
-#undef IS_INVALID_CHAR
+#  define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
+#  define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p)
+#  define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c)
+#  define IS_NAME_CHAR(enc, p, n) 0
+#  define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p)
+#  define IS_NMSTRT_CHAR(enc, p, n) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p)
+
+#  define XML_TOK_IMPL_C
+#  include "xmltok_impl.c"
+#  undef XML_TOK_IMPL_C
+
+#  undef MINBPC
+#  undef BYTE_TYPE
+#  undef BYTE_TO_ASCII
+#  undef CHAR_MATCHES
+#  undef IS_NAME_CHAR
+#  undef IS_NAME_CHAR_MINBPC
+#  undef IS_NMSTRT_CHAR
+#  undef IS_NMSTRT_CHAR_MINBPC
+#  undef IS_INVALID_CHAR
 
 #endif /* not XML_MIN_SIZE */
 
 #ifdef XML_NS
 
-static const struct normal_encoding big2_encoding_ns = {
-  { VTABLE, 2, 0,
-#if BYTEORDER == 4321
-  1
-#else
-  0
-#endif
-  },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding big2_encoding_ns
+    = {{VTABLE, 2, 0,
+#  if BYTEORDER == 4321
+        1
+#  else
+        0
+#  endif
+       },
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding big2_encoding = {
-  { VTABLE, 2, 0,
+static const struct normal_encoding big2_encoding
+    = {{VTABLE, 2, 0,
 #if BYTEORDER == 4321
-  1
+        1
 #else
-  0
+        0
 #endif
-  },
-  {
+       },
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #if BYTEORDER != 1234
 
-#ifdef XML_NS
+#  ifdef XML_NS
 
-static const struct normal_encoding internal_big2_encoding_ns = {
-  { VTABLE, 2, 0, 1 },
-  {
-#include "iasciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_big2_encoding_ns
+    = {{VTABLE, 2, 0, 1},
+       {
+#    include "iasciitab.h"
+#    include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
-#endif
+#  endif
 
-static const struct normal_encoding internal_big2_encoding = {
-  { VTABLE, 2, 0, 1 },
-  {
-#define BT_COLON BT_NMSTRT
-#include "iasciitab.h"
-#undef BT_COLON
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_big2_encoding
+    = {{VTABLE, 2, 0, 1},
+       {
+#  define BT_COLON BT_NMSTRT
+#  include "iasciitab.h"
+#  undef BT_COLON
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #endif
 
 #undef PREFIX
 
 static int FASTCALL
-streqci(const char *s1, const char *s2)
-{
+streqci(const char *s1, const char *s2) {
   for (;;) {
     char c1 = *s1++;
     char c2 = *s2++;
@@ -1079,22 +1018,21 @@ streqci(const char *s1, const char *s2)
       c2 += ASCII_A - ASCII_a; /* LCOV_EXCL_LINE */
     if (c1 != c2)
       return 0;
-    if (!c1)
+    if (! c1)
       break;
   }
   return 1;
 }
 
 static void PTRCALL
-initUpdatePosition(const ENCODING *UNUSED_P(enc), const char *ptr,
-                   const char *end, POSITION *pos)
-{
+initUpdatePosition(const ENCODING *enc, const char *ptr, const char *end,
+                   POSITION *pos) {
+  UNUSED_P(enc);
   normal_updatePosition(&utf8_encoding.enc, ptr, end, pos);
 }
 
 static int
-toAscii(const ENCODING *enc, const char *ptr, const char *end)
-{
+toAscii(const ENCODING *enc, const char *ptr, const char *end) {
   char buf[1];
   char *p = buf;
   XmlUtf8Convert(enc, &ptr, end, &p, p + 1);
@@ -1105,8 +1043,7 @@ toAscii(const ENCODING *enc, const char *ptr, const char *end)
 }
 
 static int FASTCALL
-isSpace(int c)
-{
+isSpace(int c) {
   switch (c) {
   case 0x20:
   case 0xD:
@@ -1121,21 +1058,16 @@ isSpace(int c)
    followed by name=val.
 */
 static int
-parsePseudoAttribute(const ENCODING *enc,
-                     const char *ptr,
-                     const char *end,
-                     const char **namePtr,
-                     const char **nameEndPtr,
-                     const char **valPtr,
-                     const char **nextTokPtr)
-{
+parsePseudoAttribute(const ENCODING *enc, const char *ptr, const char *end,
+                     const char **namePtr, const char **nameEndPtr,
+                     const char **valPtr, const char **nextTokPtr) {
   int c;
   char open;
   if (ptr == end) {
     *namePtr = NULL;
     return 1;
   }
-  if (!isSpace(toAscii(enc, ptr, end))) {
+  if (! isSpace(toAscii(enc, ptr, end))) {
     *nextTokPtr = ptr;
     return 0;
   }
@@ -1191,12 +1123,9 @@ parsePseudoAttribute(const ENCODING *enc,
     c = toAscii(enc, ptr, end);
     if (c == open)
       break;
-    if (!(ASCII_a <= c && c <= ASCII_z)
-        && !(ASCII_A <= c && c <= ASCII_Z)
-        && !(ASCII_0 <= c && c <= ASCII_9)
-        && c != ASCII_PERIOD
-        && c != ASCII_MINUS
-        && c != ASCII_UNDERSCORE) {
+    if (! (ASCII_a <= c && c <= ASCII_z) && ! (ASCII_A <= c && c <= ASCII_Z)
+        && ! (ASCII_0 <= c && c <= ASCII_9) && c != ASCII_PERIOD
+        && c != ASCII_MINUS && c != ASCII_UNDERSCORE) {
       *nextTokPtr = ptr;
       return 0;
     }
@@ -1205,68 +1134,52 @@ parsePseudoAttribute(const ENCODING *enc,
   return 1;
 }
 
-static const char KW_version[] = {
-  ASCII_v, ASCII_e, ASCII_r, ASCII_s, ASCII_i, ASCII_o, ASCII_n, '\0'
-};
+static const char KW_version[]
+    = {ASCII_v, ASCII_e, ASCII_r, ASCII_s, ASCII_i, ASCII_o, ASCII_n, '\0'};
 
-static const char KW_encoding[] = {
-  ASCII_e, ASCII_n, ASCII_c, ASCII_o, ASCII_d, ASCII_i, ASCII_n, ASCII_g, '\0'
-};
+static const char KW_encoding[] = {ASCII_e, ASCII_n, ASCII_c, ASCII_o, ASCII_d,
+                                   ASCII_i, ASCII_n, ASCII_g, '\0'};
 
-static const char KW_standalone[] = {
-  ASCII_s, ASCII_t, ASCII_a, ASCII_n, ASCII_d, ASCII_a, ASCII_l, ASCII_o,
-  ASCII_n, ASCII_e, '\0'
-};
+static const char KW_standalone[]
+    = {ASCII_s, ASCII_t, ASCII_a, ASCII_n, ASCII_d, ASCII_a,
+       ASCII_l, ASCII_o, ASCII_n, ASCII_e, '\0'};
 
-static const char KW_yes[] = {
-  ASCII_y, ASCII_e, ASCII_s,  '\0'
-};
+static const char KW_yes[] = {ASCII_y, ASCII_e, ASCII_s, '\0'};
 
-static const char KW_no[] = {
-  ASCII_n, ASCII_o,  '\0'
-};
+static const char KW_no[] = {ASCII_n, ASCII_o, '\0'};
 
 static int
-doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
-                                                 const char *,
+doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *,
                                                  const char *),
-               int isGeneralTextEntity,
-               const ENCODING *enc,
-               const char *ptr,
-               const char *end,
-               const char **badPtr,
-               const char **versionPtr,
-               const char **versionEndPtr,
-               const char **encodingName,
-               const ENCODING **encoding,
-               int *standalone)
-{
+               int isGeneralTextEntity, const ENCODING *enc, const char *ptr,
+               const char *end, const char **badPtr, const char **versionPtr,
+               const char **versionEndPtr, const char **encodingName,
+               const ENCODING **encoding, int *standalone) {
   const char *val = NULL;
   const char *name = NULL;
   const char *nameEnd = NULL;
   ptr += 5 * enc->minBytesPerChar;
   end -= 2 * enc->minBytesPerChar;
-  if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)
-      || !name) {
+  if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)
+      || ! name) {
     *badPtr = ptr;
     return 0;
   }
-  if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_version)) {
-    if (!isGeneralTextEntity) {
+  if (! XmlNameMatchesAscii(enc, name, nameEnd, KW_version)) {
+    if (! isGeneralTextEntity) {
       *badPtr = name;
       return 0;
     }
-  }
-  else {
+  } else {
     if (versionPtr)
       *versionPtr = val;
     if (versionEndPtr)
       *versionEndPtr = ptr;
-    if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
+    if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
       *badPtr = ptr;
       return 0;
     }
-    if (!name) {
+    if (! name) {
       if (isGeneralTextEntity) {
         /* a TextDecl must have an EncodingDecl */
         *badPtr = ptr;
@@ -1277,7 +1190,7 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
   }
   if (XmlNameMatchesAscii(enc, name, nameEnd, KW_encoding)) {
     int c = toAscii(enc, val, end);
-    if (!(ASCII_a <= c && c <= ASCII_z) && !(ASCII_A <= c && c <= ASCII_Z)) {
+    if (! (ASCII_a <= c && c <= ASCII_z) && ! (ASCII_A <= c && c <= ASCII_Z)) {
       *badPtr = val;
       return 0;
     }
@@ -1285,14 +1198,14 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
       *encodingName = val;
     if (encoding)
       *encoding = encodingFinder(enc, val, ptr - enc->minBytesPerChar);
-    if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
+    if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
       *badPtr = ptr;
       return 0;
     }
-    if (!name)
+    if (! name)
       return 1;
   }
-  if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_standalone)
+  if (! XmlNameMatchesAscii(enc, name, nameEnd, KW_standalone)
       || isGeneralTextEntity) {
     *badPtr = name;
     return 0;
@@ -1300,12 +1213,10 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
   if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_yes)) {
     if (standalone)
       *standalone = 1;
-  }
-  else if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_no)) {
+  } else if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_no)) {
     if (standalone)
       *standalone = 0;
-  }
-  else {
+  } else {
     *badPtr = val;
     return 0;
   }
@@ -1319,11 +1230,16 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
 }
 
 static int FASTCALL
-checkCharRefNumber(int result)
-{
+checkCharRefNumber(int result) {
   switch (result >> 8) {
-  case 0xD8: case 0xD9: case 0xDA: case 0xDB:
-  case 0xDC: case 0xDD: case 0xDE: case 0xDF:
+  case 0xD8:
+  case 0xD9:
+  case 0xDA:
+  case 0xDB:
+  case 0xDC:
+  case 0xDD:
+  case 0xDE:
+  case 0xDF:
     return -1;
   case 0:
     if (latin1_encoding.type[result] == BT_NONXML)
@@ -1338,8 +1254,7 @@ checkCharRefNumber(int result)
 }
 
 int FASTCALL
-XmlUtf8Encode(int c, char *buf)
-{
+XmlUtf8Encode(int c, char *buf) {
   enum {
     /* minN is minimum legal resulting value for N byte sequence */
     min2 = 0x80,
@@ -1375,8 +1290,7 @@ XmlUtf8Encode(int c, char *buf)
 }
 
 int FASTCALL
-XmlUtf16Encode(int charNum, unsigned short *buf)
-{
+XmlUtf16Encode(int charNum, unsigned short *buf) {
   if (charNum < 0)
     return 0;
   if (charNum < 0x10000) {
@@ -1400,17 +1314,15 @@ struct unknown_encoding {
   char utf8[256][4];
 };
 
-#define AS_UNKNOWN_ENCODING(enc)  ((const struct unknown_encoding *) (enc))
+#define AS_UNKNOWN_ENCODING(enc) ((const struct unknown_encoding *)(enc))
 
 int
-XmlSizeOfUnknownEncoding(void)
-{
+XmlSizeOfUnknownEncoding(void) {
   return sizeof(struct unknown_encoding);
 }
 
 static int PTRFASTCALL
-unknown_isName(const ENCODING *enc, const char *p)
-{
+unknown_isName(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   if (c & ~0xFFFF)
@@ -1419,8 +1331,7 @@ unknown_isName(const ENCODING *enc, const char *p)
 }
 
 static int PTRFASTCALL
-unknown_isNmstrt(const ENCODING *enc, const char *p)
-{
+unknown_isNmstrt(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   if (c & ~0xFFFF)
@@ -1429,18 +1340,15 @@ unknown_isNmstrt(const ENCODING *enc, const char *p)
 }
 
 static int PTRFASTCALL
-unknown_isInvalid(const ENCODING *enc, const char *p)
-{
+unknown_isInvalid(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   return (c & ~0xFFFF) || checkCharRefNumber(c) < 0;
 }
 
 static enum XML_Convert_Result PTRCALL
-unknown_toUtf8(const ENCODING *enc,
-               const char **fromP, const char *fromLim,
-               char **toP, const char *toLim)
-{
+unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+               char **toP, const char *toLim) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   char buf[XML_UTF8_ENCODE_MAX];
   for (;;) {
@@ -1458,8 +1366,7 @@ unknown_toUtf8(const ENCODING *enc,
       utf8 = buf;
       *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP]
                  - (BT_LEAD2 - 2));
-    }
-    else {
+    } else {
       if (n > toLim - *toP)
         return XML_CONVERT_OUTPUT_EXHAUSTED;
       (*fromP)++;
@@ -1470,20 +1377,16 @@ unknown_toUtf8(const ENCODING *enc,
 }
 
 static enum XML_Convert_Result PTRCALL
-unknown_toUtf16(const ENCODING *enc,
-                const char **fromP, const char *fromLim,
-                unsigned short **toP, const unsigned short *toLim)
-{
+unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+                unsigned short **toP, const unsigned short *toLim) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   while (*fromP < fromLim && *toP < toLim) {
     unsigned short c = uenc->utf16[(unsigned char)**fromP];
     if (c == 0) {
-      c = (unsigned short)
-          uenc->convert(uenc->userData, *fromP);
+      c = (unsigned short)uenc->convert(uenc->userData, *fromP);
       *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP]
                  - (BT_LEAD2 - 2));
-    }
-    else
+    } else
       (*fromP)++;
     *(*toP)++ = c;
   }
@@ -1495,19 +1398,14 @@ unknown_toUtf16(const ENCODING *enc,
 }
 
 ENCODING *
-XmlInitUnknownEncoding(void *mem,
-                       int *table,
-                       CONVERTER convert,
-                       void *userData)
-{
+XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert,
+                       void *userData) {
   int i;
   struct unknown_encoding *e = (struct unknown_encoding *)mem;
-  for (i = 0; i < (int)sizeof(struct normal_encoding); i++)
-    ((char *)mem)[i] = ((char *)&latin1_encoding)[i];
+  memcpy(mem, &latin1_encoding, sizeof(struct normal_encoding));
   for (i = 0; i < 128; i++)
     if (latin1_encoding.type[i] != BT_OTHER
-        && latin1_encoding.type[i] != BT_NONXML
-        && table[i] != i)
+        && latin1_encoding.type[i] != BT_NONXML && table[i] != i)
       return 0;
   for (i = 0; i < 256; i++) {
     int c = table[i];
@@ -1517,35 +1415,30 @@ XmlInitUnknownEncoding(void *mem,
       e->utf16[i] = 0xFFFF;
       e->utf8[i][0] = 1;
       e->utf8[i][1] = 0;
-    }
-    else if (c < 0) {
+    } else if (c < 0) {
       if (c < -4)
         return 0;
       /* Multi-byte sequences need a converter function */
-      if (!convert)
+      if (! convert)
         return 0;
       e->normal.type[i] = (unsigned char)(BT_LEAD2 - (c + 2));
       e->utf8[i][0] = 0;
       e->utf16[i] = 0;
-    }
-    else if (c < 0x80) {
+    } else if (c < 0x80) {
       if (latin1_encoding.type[c] != BT_OTHER
-          && latin1_encoding.type[c] != BT_NONXML
-          && c != i)
+          && latin1_encoding.type[c] != BT_NONXML && c != i)
         return 0;
       e->normal.type[i] = latin1_encoding.type[c];
       e->utf8[i][0] = 1;
       e->utf8[i][1] = (char)c;
       e->utf16[i] = (unsigned short)(c == 0 ? 0xFFFF : c);
-    }
-    else if (checkCharRefNumber(c) < 0) {
+    } else if (checkCharRefNumber(c) < 0) {
       e->normal.type[i] = BT_NONXML;
       /* This shouldn't really get used. */
       e->utf16[i] = 0xFFFF;
       e->utf8[i][0] = 1;
       e->utf8[i][1] = 0;
-    }
-    else {
+    } else {
       if (c > 0xFFFF)
         return 0;
       if (UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xff))
@@ -1590,44 +1483,32 @@ enum {
   NO_ENC
 };
 
-static const char KW_ISO_8859_1[] = {
-  ASCII_I, ASCII_S, ASCII_O, ASCII_MINUS, ASCII_8, ASCII_8, ASCII_5, ASCII_9,
-  ASCII_MINUS, ASCII_1, '\0'
-};
-static const char KW_US_ASCII[] = {
-  ASCII_U, ASCII_S, ASCII_MINUS, ASCII_A, ASCII_S, ASCII_C, ASCII_I, ASCII_I,
-  '\0'
-};
-static const char KW_UTF_8[] =  {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_8, '\0'
-};
-static const char KW_UTF_16[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, '\0'
-};
-static const char KW_UTF_16BE[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_B, ASCII_E,
-  '\0'
-};
-static const char KW_UTF_16LE[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_L, ASCII_E,
-  '\0'
-};
+static const char KW_ISO_8859_1[]
+    = {ASCII_I, ASCII_S, ASCII_O,     ASCII_MINUS, ASCII_8, ASCII_8,
+       ASCII_5, ASCII_9, ASCII_MINUS, ASCII_1,     '\0'};
+static const char KW_US_ASCII[]
+    = {ASCII_U, ASCII_S, ASCII_MINUS, ASCII_A, ASCII_S,
+       ASCII_C, ASCII_I, ASCII_I,     '\0'};
+static const char KW_UTF_8[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_8, '\0'};
+static const char KW_UTF_16[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, '\0'};
+static const char KW_UTF_16BE[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1,
+       ASCII_6, ASCII_B, ASCII_E, '\0'};
+static const char KW_UTF_16LE[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1,
+       ASCII_6, ASCII_L, ASCII_E, '\0'};
 
 static int FASTCALL
-getEncodingIndex(const char *name)
-{
-  static const char * const encodingNames[] = {
-    KW_ISO_8859_1,
-    KW_US_ASCII,
-    KW_UTF_8,
-    KW_UTF_16,
-    KW_UTF_16BE,
-    KW_UTF_16LE,
+getEncodingIndex(const char *name) {
+  static const char *const encodingNames[] = {
+      KW_ISO_8859_1, KW_US_ASCII, KW_UTF_8, KW_UTF_16, KW_UTF_16BE, KW_UTF_16LE,
   };
   int i;
   if (name == NULL)
     return NO_ENC;
-  for (i = 0; i < (int)(sizeof(encodingNames)/sizeof(encodingNames[0])); i++)
+  for (i = 0; i < (int)(sizeof(encodingNames) / sizeof(encodingNames[0])); i++)
     if (streqci(name, encodingNames[i]))
       return i;
   return UNKNOWN_ENC;
@@ -1647,15 +1528,9 @@ getEncodingIndex(const char *name)
    XML_PROLOG_STATE otherwise.
 */
 
-
 static int
-initScan(const ENCODING * const *encodingTable,
-         const INIT_ENCODING *enc,
-         int state,
-         const char *ptr,
-         const char *end,
-         const char **nextTokPtr)
-{
+initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc,
+         int state, const char *ptr, const char *end, const char **nextTokPtr) {
   const ENCODING **encPtr;
 
   if (ptr >= end)
@@ -1680,20 +1555,17 @@ initScan(const ENCODING * const *encodingTable,
     case 0xFE:
     case 0xFF:
     case 0xEF: /* possibly first byte of UTF-8 BOM */
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       /* fall through */
     case 0x00:
     case 0x3C:
       return XML_TOK_PARTIAL;
     }
-  }
-  else {
+  } else {
     switch (((unsigned char)ptr[0] << 8) | (unsigned char)ptr[1]) {
     case 0xFEFF:
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       *nextTokPtr = ptr + 2;
       *encPtr = encodingTable[UTF_16BE_ENC];
@@ -1707,8 +1579,7 @@ initScan(const ENCODING * const *encodingTable,
       *encPtr = encodingTable[UTF_16LE_ENC];
       return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
     case 0xFFFE:
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       *nextTokPtr = ptr + 2;
       *encPtr = encodingTable[UTF_16LE_ENC];
@@ -1723,8 +1594,8 @@ initScan(const ENCODING * const *encodingTable,
       */
       if (state == XML_CONTENT_STATE) {
         int e = INIT_ENC_INDEX(enc);
-        if (e == ISO_8859_1_ENC || e == UTF_16BE_ENC
-            || e == UTF_16LE_ENC || e == UTF_16_ENC)
+        if (e == ISO_8859_1_ENC || e == UTF_16BE_ENC || e == UTF_16LE_ENC
+            || e == UTF_16_ENC)
           break;
       }
       if (ptr + 2 == end)
@@ -1747,8 +1618,7 @@ initScan(const ENCODING * const *encodingTable,
           break;
         *encPtr = encodingTable[UTF_16BE_ENC];
         return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
-      }
-      else if (ptr[1] == '\0') {
+      } else if (ptr[1] == '\0') {
         /* We could recover here in the case:
             - parsing an external entity
             - second byte is 0
@@ -1770,7 +1640,6 @@ initScan(const ENCODING * const *encodingTable,
   return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
 }
 
-
 #define NS(x) x
 #define ns(x) x
 #define XML_TOK_NS_C
@@ -1781,22 +1650,19 @@ initScan(const ENCODING * const *encodingTable,
 
 #ifdef XML_NS
 
-#define NS(x) x ## NS
-#define ns(x) x ## _ns
+#  define NS(x) x##NS
+#  define ns(x) x##_ns
 
-#define XML_TOK_NS_C
-#include "xmltok_ns.c"
-#undef XML_TOK_NS_C
+#  define XML_TOK_NS_C
+#  include "xmltok_ns.c"
+#  undef XML_TOK_NS_C
 
-#undef NS
-#undef ns
+#  undef NS
+#  undef ns
 
 ENCODING *
-XmlInitUnknownEncodingNS(void *mem,
-                         int *table,
-                         CONVERTER convert,
-                         void *userData)
-{
+XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert,
+                         void *userData) {
   ENCODING *enc = XmlInitUnknownEncoding(mem, table, convert, userData);
   if (enc)
     ((struct normal_encoding *)enc)->type[ASCII_COLON] = BT_COLON;
diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h
index 50926f38ab32..2adbf5307bef 100644
--- a/Modules/expat/xmltok.h
+++ b/Modules/expat/xmltok.h
@@ -38,16 +38,18 @@ extern "C" {
 #endif
 
 /* The following token may be returned by XmlContentTok */
-#define XML_TOK_TRAILING_RSQB -5 /* ] or ]] at the end of the scan; might be
-                                    start of illegal ]]> sequence */
+#define XML_TOK_TRAILING_RSQB                                                  \
+  -5 /* ] or ]] at the end of the scan; might be                               \
+        start of illegal ]]> sequence */
 /* The following tokens may be returned by both XmlPrologTok and
    XmlContentTok.
 */
-#define XML_TOK_NONE -4          /* The string to be scanned is empty */
-#define XML_TOK_TRAILING_CR -3   /* A CR at the end of the scan;
-                                    might be part of CRLF sequence */
-#define XML_TOK_PARTIAL_CHAR -2  /* only part of a multibyte sequence */
-#define XML_TOK_PARTIAL -1       /* only part of a token */
+#define XML_TOK_NONE -4 /* The string to be scanned is empty */
+#define XML_TOK_TRAILING_CR                                                    \
+  -3                            /* A CR at the end of the scan;                \
+                                   might be part of CRLF sequence */
+#define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */
+#define XML_TOK_PARTIAL -1      /* only part of a token */
 #define XML_TOK_INVALID 0
 
 /* The following tokens are returned by XmlContentTok; some are also
@@ -62,24 +64,24 @@ extern "C" {
 #define XML_TOK_DATA_NEWLINE 7
 #define XML_TOK_CDATA_SECT_OPEN 8
 #define XML_TOK_ENTITY_REF 9
-#define XML_TOK_CHAR_REF 10               /* numeric character reference */
+#define XML_TOK_CHAR_REF 10 /* numeric character reference */
 
 /* The following tokens may be returned by both XmlPrologTok and
    XmlContentTok.
 */
-#define XML_TOK_PI 11                     /* processing instruction */
-#define XML_TOK_XML_DECL 12               /* XML decl or text decl */
+#define XML_TOK_PI 11       /* processing instruction */
+#define XML_TOK_XML_DECL 12 /* XML decl or text decl */
 #define XML_TOK_COMMENT 13
-#define XML_TOK_BOM 14                    /* Byte order mark */
+#define XML_TOK_BOM 14 /* Byte order mark */
 
 /* The following tokens are returned only by XmlPrologTok */
 #define XML_TOK_PROLOG_S 15
-#define XML_TOK_DECL_OPEN 16              /* <!foo */
-#define XML_TOK_DECL_CLOSE 17             /* > */
+#define XML_TOK_DECL_OPEN 16  /* <!foo */
+#define XML_TOK_DECL_CLOSE 17 /* > */
 #define XML_TOK_NAME 18
 #define XML_TOK_NMTOKEN 19
-#define XML_TOK_POUND_NAME 20             /* #name */
-#define XML_TOK_OR 21                     /* | */
+#define XML_TOK_POUND_NAME 20 /* #name */
+#define XML_TOK_OR 21         /* | */
 #define XML_TOK_PERCENT 22
 #define XML_TOK_OPEN_PAREN 23
 #define XML_TOK_CLOSE_PAREN 24
@@ -90,14 +92,14 @@ extern "C" {
 #define XML_TOK_INSTANCE_START 29
 
 /* The following occur only in element type declarations */
-#define XML_TOK_NAME_QUESTION 30          /* name? */
-#define XML_TOK_NAME_ASTERISK 31          /* name* */
-#define XML_TOK_NAME_PLUS 32              /* name+ */
-#define XML_TOK_COND_SECT_OPEN 33         /* <![ */
-#define XML_TOK_COND_SECT_CLOSE 34        /* ]]> */
-#define XML_TOK_CLOSE_PAREN_QUESTION 35   /* )? */
-#define XML_TOK_CLOSE_PAREN_ASTERISK 36   /* )* */
-#define XML_TOK_CLOSE_PAREN_PLUS 37       /* )+ */
+#define XML_TOK_NAME_QUESTION 30        /* name? */
+#define XML_TOK_NAME_ASTERISK 31        /* name* */
+#define XML_TOK_NAME_PLUS 32            /* name+ */
+#define XML_TOK_COND_SECT_OPEN 33       /* <![ */
+#define XML_TOK_COND_SECT_CLOSE 34      /* ]]> */
+#define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */
+#define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */
+#define XML_TOK_CLOSE_PAREN_PLUS 37     /* )+ */
 #define XML_TOK_COMMA 38
 
 /* The following token is returned only by XmlAttributeValueTok */
@@ -112,20 +114,20 @@ extern "C" {
 #define XML_TOK_PREFIXED_NAME 41
 
 #ifdef XML_DTD
-#define XML_TOK_IGNORE_SECT 42
+#  define XML_TOK_IGNORE_SECT 42
 #endif /* XML_DTD */
 
 #ifdef XML_DTD
-#define XML_N_STATES 4
+#  define XML_N_STATES 4
 #else /* not XML_DTD */
-#define XML_N_STATES 3
+#  define XML_N_STATES 3
 #endif /* not XML_DTD */
 
 #define XML_PROLOG_STATE 0
 #define XML_CONTENT_STATE 1
 #define XML_CDATA_SECTION_STATE 2
 #ifdef XML_DTD
-#define XML_IGNORE_SECTION_STATE 3
+#  define XML_IGNORE_SECTION_STATE 3
 #endif /* XML_DTD */
 
 #define XML_N_LITERAL_TYPES 2
@@ -153,52 +155,41 @@ typedef struct {
 struct encoding;
 typedef struct encoding ENCODING;
 
-typedef int (PTRCALL *SCANNER)(const ENCODING *,
-                               const char *,
-                               const char *,
-                               const char **);
+typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *,
+                              const char **);
 
 enum XML_Convert_Result {
   XML_CONVERT_COMPLETED = 0,
   XML_CONVERT_INPUT_INCOMPLETE = 1,
-  XML_CONVERT_OUTPUT_EXHAUSTED = 2  /* and therefore potentially input remaining as well */
+  XML_CONVERT_OUTPUT_EXHAUSTED
+  = 2 /* and therefore potentially input remaining as well */
 };
 
 struct encoding {
   SCANNER scanners[XML_N_STATES];
   SCANNER literalScanners[XML_N_LITERAL_TYPES];
-  int (PTRCALL *nameMatchesAscii)(const ENCODING *,
-                                  const char *,
-                                  const char *,
-                                  const char *);
-  int (PTRFASTCALL *nameLength)(const ENCODING *, const char *);
+  int(PTRCALL *nameMatchesAscii)(const ENCODING *, const char *, const char *,
+                                 const char *);
+  int(PTRFASTCALL *nameLength)(const ENCODING *, const char *);
   const char *(PTRFASTCALL *skipS)(const ENCODING *, const char *);
-  int (PTRCALL *getAtts)(const ENCODING *enc,
-                         const char *ptr,
-                         int attsMax,
-                         ATTRIBUTE *atts);
-  int (PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr);
-  int (PTRCALL *predefinedEntityName)(const ENCODING *,
-                                      const char *,
-                                      const char *);
-  void (PTRCALL *updatePosition)(const ENCODING *,
-                                 const char *ptr,
-                                 const char *end,
-                                 POSITION *);
-  int (PTRCALL *isPublicId)(const ENCODING *enc,
-                            const char *ptr,
-                            const char *end,
-                            const char **badPtr);
-  enum XML_Convert_Result (PTRCALL *utf8Convert)(const ENCODING *enc,
-                              const char **fromP,
-                              const char *fromLim,
-                              char **toP,
-                              const char *toLim);
-  enum XML_Convert_Result (PTRCALL *utf16Convert)(const ENCODING *enc,
-                               const char **fromP,
-                               const char *fromLim,
-                               unsigned short **toP,
-                               const unsigned short *toLim);
+  int(PTRCALL *getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
+                        ATTRIBUTE *atts);
+  int(PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr);
+  int(PTRCALL *predefinedEntityName)(const ENCODING *, const char *,
+                                     const char *);
+  void(PTRCALL *updatePosition)(const ENCODING *, const char *ptr,
+                                const char *end, POSITION *);
+  int(PTRCALL *isPublicId)(const ENCODING *enc, const char *ptr,
+                           const char *end, const char **badPtr);
+  enum XML_Convert_Result(PTRCALL *utf8Convert)(const ENCODING *enc,
+                                                const char **fromP,
+                                                const char *fromLim, char **toP,
+                                                const char *toLim);
+  enum XML_Convert_Result(PTRCALL *utf16Convert)(const ENCODING *enc,
+                                                 const char **fromP,
+                                                 const char *fromLim,
+                                                 unsigned short **toP,
+                                                 const unsigned short *toLim);
   int minBytesPerChar;
   char isUtf8;
   char isUtf16;
@@ -225,66 +216,62 @@ struct encoding {
    the prolog outside literals, comments and processing instructions.
 */
 
-
-#define XmlTok(enc, state, ptr, end, nextTokPtr) \
+#define XmlTok(enc, state, ptr, end, nextTokPtr)                               \
   (((enc)->scanners[state])(enc, ptr, end, nextTokPtr))
 
-#define XmlPrologTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr)
+#define XmlPrologTok(enc, ptr, end, nextTokPtr)                                \
+  XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr)
 
-#define XmlContentTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr)
+#define XmlContentTok(enc, ptr, end, nextTokPtr)                               \
+  XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr)
 
-#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr)
+#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr)                          \
+  XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr)
 
 #ifdef XML_DTD
 
-#define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr)
+#  define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr)                       \
+    XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr)
 
 #endif /* XML_DTD */
 
 /* This is used for performing a 2nd-level tokenization on the content
    of a literal that has already been returned by XmlTok.
 */
-#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \
+#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr)                  \
   (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr))
 
-#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \
-   XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr)
+#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr)                        \
+  XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr)
 
-#define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \
-   XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr)
+#define XmlEntityValueTok(enc, ptr, end, nextTokPtr)                           \
+  XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr)
 
-#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \
+#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2)                             \
   (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2))
 
-#define XmlNameLength(enc, ptr) \
-  (((enc)->nameLength)(enc, ptr))
+#define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr))
 
-#define XmlSkipS(enc, ptr) \
-  (((enc)->skipS)(enc, ptr))
+#define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr))
 
-#define XmlGetAttributes(enc, ptr, attsMax, atts) \
+#define XmlGetAttributes(enc, ptr, attsMax, atts)                              \
   (((enc)->getAtts)(enc, ptr, attsMax, atts))
 
-#define XmlCharRefNumber(enc, ptr) \
-  (((enc)->charRefNumber)(enc, ptr))
+#define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr))
 
-#define XmlPredefinedEntityName(enc, ptr, end) \
+#define XmlPredefinedEntityName(enc, ptr, end)                                 \
   (((enc)->predefinedEntityName)(enc, ptr, end))
 
-#define XmlUpdatePosition(enc, ptr, end, pos) \
+#define XmlUpdatePosition(enc, ptr, end, pos)                                  \
   (((enc)->updatePosition)(enc, ptr, end, pos))
 
-#define XmlIsPublicId(enc, ptr, end, badPtr) \
+#define XmlIsPublicId(enc, ptr, end, badPtr)                                   \
   (((enc)->isPublicId)(enc, ptr, end, badPtr))
 
-#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \
+#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim)                        \
   (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim))
 
-#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \
+#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim)                       \
   (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim))
 
 typedef struct {
@@ -292,16 +279,11 @@ typedef struct {
   const ENCODING **encPtr;
 } INIT_ENCODING;
 
-int XmlParseXmlDecl(int isGeneralTextEntity,
-                    const ENCODING *enc,
-                    const char *ptr,
-                    const char *end,
-                    const char **badPtr,
-                    const char **versionPtr,
-                    const char **versionEndPtr,
+int XmlParseXmlDecl(int isGeneralTextEntity, const ENCODING *enc,
+                    const char *ptr, const char *end, const char **badPtr,
+                    const char **versionPtr, const char **versionEndPtr,
                     const char **encodingNamePtr,
-                    const ENCODING **namedEncodingPtr,
-                    int *standalonePtr);
+                    const ENCODING **namedEncodingPtr, int *standalonePtr);
 
 int XmlInitEncoding(INIT_ENCODING *, const ENCODING **, const char *name);
 const ENCODING *XmlGetUtf8InternalEncoding(void);
@@ -310,34 +292,22 @@ int FASTCALL XmlUtf8Encode(int charNumber, char *buf);
 int FASTCALL XmlUtf16Encode(int charNumber, unsigned short *buf);
 int XmlSizeOfUnknownEncoding(void);
 
+typedef int(XMLCALL *CONVERTER)(void *userData, const char *p);
 
-typedef int (XMLCALL *CONVERTER) (void *userData, const char *p);
-
-ENCODING *
-XmlInitUnknownEncoding(void *mem,
-                       int *table,
-                       CONVERTER convert,
-                       void *userData);
+ENCODING *XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert,
+                                 void *userData);
 
-int XmlParseXmlDeclNS(int isGeneralTextEntity,
-                      const ENCODING *enc,
-                      const char *ptr,
-                      const char *end,
-                      const char **badPtr,
-                      const char **versionPtr,
-                      const char **versionEndPtr,
+int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc,
+                      const char *ptr, const char *end, const char **badPtr,
+                      const char **versionPtr, const char **versionEndPtr,
                       const char **encodingNamePtr,
-                      const ENCODING **namedEncodingPtr,
-                      int *standalonePtr);
+                      const ENCODING **namedEncodingPtr, int *standalonePtr);
 
 int XmlInitEncodingNS(INIT_ENCODING *, const ENCODING **, const char *name);
 const ENCODING *XmlGetUtf8InternalEncodingNS(void);
 const ENCODING *XmlGetUtf16InternalEncodingNS(void);
-ENCODING *
-XmlInitUnknownEncodingNS(void *mem,
-                         int *table,
-                         CONVERTER convert,
-                         void *userData);
+ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert,
+                                   void *userData);
 #ifdef __cplusplus
 }
 #endif
diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c
index 4d9ae7dc3896..c209221cd79d 100644
--- a/Modules/expat/xmltok_impl.c
+++ b/Modules/expat/xmltok_impl.c
@@ -32,130 +32,124 @@
 
 #ifdef XML_TOK_IMPL_C
 
-#ifndef IS_INVALID_CHAR
-#define IS_INVALID_CHAR(enc, ptr, n) (0)
-#endif
-
-#define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n) \
-        return XML_TOK_PARTIAL_CHAR; \
-      if (IS_INVALID_CHAR(enc, ptr, n)) { \
-        *(nextTokPtr) = (ptr); \
-        return XML_TOK_INVALID; \
-      } \
-      ptr += n; \
-      break;
+#  ifndef IS_INVALID_CHAR
+#    define IS_INVALID_CHAR(enc, ptr, n) (0)
+#  endif
+
+#  define INVALID_LEAD_CASE(n, ptr, nextTokPtr)                                \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (IS_INVALID_CHAR(enc, ptr, n)) {                                        \
+      *(nextTokPtr) = (ptr);                                                   \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define INVALID_CASES(ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(2, ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(3, ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(4, ptr, nextTokPtr) \
-  case BT_NONXML: \
-  case BT_MALFORM: \
-  case BT_TRAIL: \
-    *(nextTokPtr) = (ptr); \
+#  define INVALID_CASES(ptr, nextTokPtr)                                       \
+    INVALID_LEAD_CASE(2, ptr, nextTokPtr)                                      \
+    INVALID_LEAD_CASE(3, ptr, nextTokPtr)                                      \
+    INVALID_LEAD_CASE(4, ptr, nextTokPtr)                                      \
+  case BT_NONXML:                                                              \
+  case BT_MALFORM:                                                             \
+  case BT_TRAIL:                                                               \
+    *(nextTokPtr) = (ptr);                                                     \
     return XML_TOK_INVALID;
 
-#define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr) \
-   case BT_LEAD ## n: \
-     if (end - ptr < n) \
-       return XML_TOK_PARTIAL_CHAR; \
-     if (!IS_NAME_CHAR(enc, ptr, n)) { \
-       *nextTokPtr = ptr; \
-       return XML_TOK_INVALID; \
-     } \
-     ptr += n; \
-     break;
-
-#define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) \
-  case BT_NONASCII: \
-    if (!IS_NAME_CHAR_MINBPC(enc, ptr)) { \
-      *nextTokPtr = ptr; \
-      return XML_TOK_INVALID; \
-    } \
-    /* fall through */ \
-  case BT_NMSTRT: \
-  case BT_HEX: \
-  case BT_DIGIT: \
-  case BT_NAME: \
-  case BT_MINUS: \
-    ptr += MINBPC(enc); \
-    break; \
-  CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr) \
-  CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr) \
-  CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)
-
-#define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr) \
-   case BT_LEAD ## n: \
-     if (end - ptr < n) \
-       return XML_TOK_PARTIAL_CHAR; \
-     if (!IS_NMSTRT_CHAR(enc, ptr, n)) { \
-       *nextTokPtr = ptr; \
-       return XML_TOK_INVALID; \
-     } \
-     ptr += n; \
-     break;
-
-#define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) \
-  case BT_NONASCII: \
-    if (!IS_NMSTRT_CHAR_MINBPC(enc, ptr)) { \
-      *nextTokPtr = ptr; \
-      return XML_TOK_INVALID; \
-    } \
-    /* fall through */ \
-  case BT_NMSTRT: \
-  case BT_HEX: \
-    ptr += MINBPC(enc); \
-    break; \
-  CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr) \
-  CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr) \
-  CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)
-
-#ifndef PREFIX
-#define PREFIX(ident) ident
-#endif
-
-
-#define HAS_CHARS(enc, ptr, end, count) \
-    (end - ptr >= count * MINBPC(enc))
+#  define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr)                        \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (! IS_NAME_CHAR(enc, ptr, n)) {                                         \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define HAS_CHAR(enc, ptr, end) \
-    HAS_CHARS(enc, ptr, end, 1)
+#  define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)                          \
+  case BT_NONASCII:                                                            \
+    if (! IS_NAME_CHAR_MINBPC(enc, ptr)) {                                     \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    /* fall through */                                                         \
+  case BT_NMSTRT:                                                              \
+  case BT_HEX:                                                                 \
+  case BT_DIGIT:                                                               \
+  case BT_NAME:                                                                \
+  case BT_MINUS:                                                               \
+    ptr += MINBPC(enc);                                                        \
+    break;                                                                     \
+    CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr)                              \
+    CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr)                              \
+    CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)
+
+#  define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr)                      \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (! IS_NMSTRT_CHAR(enc, ptr, n)) {                                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define REQUIRE_CHARS(enc, ptr, end, count) \
-    { \
-      if (! HAS_CHARS(enc, ptr, end, count)) { \
-        return XML_TOK_PARTIAL; \
-      } \
+#  define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)                        \
+  case BT_NONASCII:                                                            \
+    if (! IS_NMSTRT_CHAR_MINBPC(enc, ptr)) {                                   \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    /* fall through */                                                         \
+  case BT_NMSTRT:                                                              \
+  case BT_HEX:                                                                 \
+    ptr += MINBPC(enc);                                                        \
+    break;                                                                     \
+    CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr)                            \
+    CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr)                            \
+    CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)
+
+#  ifndef PREFIX
+#    define PREFIX(ident) ident
+#  endif
+
+#  define HAS_CHARS(enc, ptr, end, count) (end - ptr >= count * MINBPC(enc))
+
+#  define HAS_CHAR(enc, ptr, end) HAS_CHARS(enc, ptr, end, 1)
+
+#  define REQUIRE_CHARS(enc, ptr, end, count)                                  \
+    {                                                                          \
+      if (! HAS_CHARS(enc, ptr, end, count)) {                                 \
+        return XML_TOK_PARTIAL;                                                \
+      }                                                                        \
     }
 
-#define REQUIRE_CHAR(enc, ptr, end) \
-    REQUIRE_CHARS(enc, ptr, end, 1)
-
+#  define REQUIRE_CHAR(enc, ptr, end) REQUIRE_CHARS(enc, ptr, end, 1)
 
 /* ptr points to character following "<!-" */
 
 static int PTRCALL
-PREFIX(scanComment)(const ENCODING *enc, const char *ptr,
-                    const char *end, const char **nextTokPtr)
-{
+PREFIX(scanComment)(const ENCODING *enc, const char *ptr, const char *end,
+                    const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
-    if (!CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
     }
     ptr += MINBPC(enc);
     while (HAS_CHAR(enc, ptr, end)) {
       switch (BYTE_TYPE(enc, ptr)) {
-      INVALID_CASES(ptr, nextTokPtr)
+        INVALID_CASES(ptr, nextTokPtr)
       case BT_MINUS:
         ptr += MINBPC(enc);
         REQUIRE_CHAR(enc, ptr, end);
         if (CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
-          if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+          if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
             *nextTokPtr = ptr;
             return XML_TOK_INVALID;
           }
@@ -175,9 +169,8 @@ PREFIX(scanComment)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "<!" */
 
 static int PTRCALL
-PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
-                 const char *end, const char **nextTokPtr)
-{
+PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, const char *end,
+                 const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
   case BT_MINUS:
@@ -199,12 +192,17 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
       REQUIRE_CHARS(enc, ptr, end, 2);
       /* don't allow <!ENTITY% foo "whatever"> */
       switch (BYTE_TYPE(enc, ptr + MINBPC(enc))) {
-      case BT_S: case BT_CR: case BT_LF: case BT_PERCNT:
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+      case BT_PERCNT:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       /* fall through */
-    case BT_S: case BT_CR: case BT_LF:
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       *nextTokPtr = ptr;
       return XML_TOK_DECL_OPEN;
     case BT_NMSTRT:
@@ -220,12 +218,12 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                      const char *end, int *tokPtr)
-{
+PREFIX(checkPiTarget)(const ENCODING *enc, const char *ptr, const char *end,
+                      int *tokPtr) {
   int upper = 0;
+  UNUSED_P(enc);
   *tokPtr = XML_TOK_PI;
-  if (end - ptr != MINBPC(enc)*3)
+  if (end - ptr != MINBPC(enc) * 3)
     return 1;
   switch (BYTE_TO_ASCII(enc, ptr)) {
   case ASCII_x:
@@ -265,30 +263,31 @@ PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr,
 /* ptr points to character following "<?" */
 
 static int PTRCALL
-PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
-               const char *end, const char **nextTokPtr)
-{
+PREFIX(scanPi)(const ENCODING *enc, const char *ptr, const char *end,
+               const char **nextTokPtr) {
   int tok;
   const char *target = ptr;
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_S: case BT_CR: case BT_LF:
-      if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
+      if (! PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       ptr += MINBPC(enc);
       while (HAS_CHAR(enc, ptr, end)) {
         switch (BYTE_TYPE(enc, ptr)) {
-        INVALID_CASES(ptr, nextTokPtr)
+          INVALID_CASES(ptr, nextTokPtr)
         case BT_QUEST:
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
@@ -304,7 +303,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
       }
       return XML_TOK_PARTIAL;
     case BT_QUEST:
-      if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
+      if (! PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -324,16 +323,16 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                         const char *end, const char **nextTokPtr)
-{
-  static const char CDATA_LSQB[] = { ASCII_C, ASCII_D, ASCII_A,
-                                     ASCII_T, ASCII_A, ASCII_LSQB };
+PREFIX(scanCdataSection)(const ENCODING *enc, const char *ptr, const char *end,
+                         const char **nextTokPtr) {
+  static const char CDATA_LSQB[]
+      = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, ASCII_LSQB};
   int i;
+  UNUSED_P(enc);
   /* CDATA[ */
   REQUIRE_CHARS(enc, ptr, end, 6);
   for (i = 0; i < 6; i++, ptr += MINBPC(enc)) {
-    if (!CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {
+    if (! CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
     }
@@ -343,9 +342,8 @@ PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
-                        const char *end, const char **nextTokPtr)
-{
+PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
+                        const char **nextTokPtr) {
   if (ptr >= end)
     return XML_TOK_NONE;
   if (MINBPC(enc) > 1) {
@@ -361,11 +359,11 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
   case BT_RSQB:
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
-    if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
+    if (! CHAR_MATCHES(enc, ptr, ASCII_RSQB))
       break;
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
-    if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
       ptr -= MINBPC(enc);
       break;
     }
@@ -381,23 +379,25 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
   case BT_LF:
     *nextTokPtr = ptr + MINBPC(enc);
     return XML_TOK_DATA_NEWLINE;
-  INVALID_CASES(ptr, nextTokPtr)
+    INVALID_CASES(ptr, nextTokPtr)
   default:
     ptr += MINBPC(enc);
     break;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \
-        *nextTokPtr = ptr; \
-        return XML_TOK_DATA_CHARS; \
-      } \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) {                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_DATA_CHARS;                                               \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONXML:
     case BT_MALFORM:
     case BT_TRAIL:
@@ -418,23 +418,26 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "</" */
 
 static int PTRCALL
-PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
-                   const char *end, const char **nextTokPtr)
-{
+PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr, const char *end,
+                   const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_S: case BT_CR: case BT_LF:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) {
         switch (BYTE_TYPE(enc, ptr)) {
-        case BT_S: case BT_CR: case BT_LF:
+        case BT_S:
+        case BT_CR:
+        case BT_LF:
           break;
         case BT_GT:
           *nextTokPtr = ptr + MINBPC(enc);
@@ -445,13 +448,13 @@ PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
         }
       }
       return XML_TOK_PARTIAL;
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
       /* no need to check qname syntax here,
          since end-tag must match exactly */
       ptr += MINBPC(enc);
       break;
-#endif
+#  endif
     case BT_GT:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_END_TAG;
@@ -466,9 +469,8 @@ PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "&#X" */
 
 static int PTRCALL
-PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr,
-                       const char *end, const char **nextTokPtr)
-{
+PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, const char *end,
+                       const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
     case BT_DIGIT:
@@ -498,9 +500,8 @@ PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "&#" */
 
 static int PTRCALL
-PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr,
-                    const char *end, const char **nextTokPtr)
-{
+PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, const char *end,
+                    const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
     if (CHAR_MATCHES(enc, ptr, ASCII_x))
       return PREFIX(scanHexCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -531,11 +532,10 @@ PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr,
 
 static int PTRCALL
 PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
-                const char **nextTokPtr)
-{
+                const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   case BT_NUM:
     return PREFIX(scanCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
   default:
@@ -544,7 +544,7 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
     case BT_SEMI:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_ENTITY_REF;
@@ -560,15 +560,14 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
-                 const char **nextTokPtr)
-{
-#ifdef XML_NS
+                 const char **nextTokPtr) {
+#  ifdef XML_NS
   int hadColon = 0;
-#endif
+#  endif
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-#ifdef XML_NS
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+#  ifdef XML_NS
     case BT_COLON:
       if (hadColon) {
         *nextTokPtr = ptr;
@@ -578,14 +577,16 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
       switch (BYTE_TYPE(enc, ptr)) {
-      CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+        CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
       default:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       break;
-#endif
-    case BT_S: case BT_CR: case BT_LF:
+#  endif
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       for (;;) {
         int t;
 
@@ -605,101 +606,101 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
         }
       }
       /* fall through */
-    case BT_EQUALS:
-      {
-        int open;
-#ifdef XML_NS
-        hadColon = 0;
-#endif
-        for (;;) {
-          ptr += MINBPC(enc);
-          REQUIRE_CHAR(enc, ptr, end);
-          open = BYTE_TYPE(enc, ptr);
-          if (open == BT_QUOT || open == BT_APOS)
-            break;
-          switch (open) {
-          case BT_S:
-          case BT_LF:
-          case BT_CR:
-            break;
-          default:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          }
-        }
+    case BT_EQUALS: {
+      int open;
+#  ifdef XML_NS
+      hadColon = 0;
+#  endif
+      for (;;) {
         ptr += MINBPC(enc);
-        /* in attribute value */
-        for (;;) {
-          int t;
-          REQUIRE_CHAR(enc, ptr, end);
-          t = BYTE_TYPE(enc, ptr);
-          if (t == open)
-            break;
-          switch (t) {
+        REQUIRE_CHAR(enc, ptr, end);
+        open = BYTE_TYPE(enc, ptr);
+        if (open == BT_QUOT || open == BT_APOS)
+          break;
+        switch (open) {
+        case BT_S:
+        case BT_LF:
+        case BT_CR:
+          break;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
+        }
+      }
+      ptr += MINBPC(enc);
+      /* in attribute value */
+      for (;;) {
+        int t;
+        REQUIRE_CHAR(enc, ptr, end);
+        t = BYTE_TYPE(enc, ptr);
+        if (t == open)
+          break;
+        switch (t) {
           INVALID_CASES(ptr, nextTokPtr)
-          case BT_AMP:
-            {
-              int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);
-              if (tok <= 0) {
-                if (tok == XML_TOK_INVALID)
-                  *nextTokPtr = ptr;
-                return tok;
-              }
-              break;
-            }
-          case BT_LT:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          default:
-            ptr += MINBPC(enc);
-            break;
+        case BT_AMP: {
+          int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);
+          if (tok <= 0) {
+            if (tok == XML_TOK_INVALID)
+              *nextTokPtr = ptr;
+            return tok;
           }
+          break;
+        }
+        case BT_LT:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
+        default:
+          ptr += MINBPC(enc);
+          break;
         }
+      }
+      ptr += MINBPC(enc);
+      REQUIRE_CHAR(enc, ptr, end);
+      switch (BYTE_TYPE(enc, ptr)) {
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+        break;
+      case BT_SOL:
+        goto sol;
+      case BT_GT:
+        goto gt;
+      default:
+        *nextTokPtr = ptr;
+        return XML_TOK_INVALID;
+      }
+      /* ptr points to closing quote */
+      for (;;) {
         ptr += MINBPC(enc);
         REQUIRE_CHAR(enc, ptr, end);
         switch (BYTE_TYPE(enc, ptr)) {
+          CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
         case BT_S:
         case BT_CR:
         case BT_LF:
-          break;
-        case BT_SOL:
-          goto sol;
+          continue;
         case BT_GT:
-          goto gt;
-        default:
-          *nextTokPtr = ptr;
-          return XML_TOK_INVALID;
-        }
-        /* ptr points to closing quote */
-        for (;;) {
+        gt:
+          *nextTokPtr = ptr + MINBPC(enc);
+          return XML_TOK_START_TAG_WITH_ATTS;
+        case BT_SOL:
+        sol:
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
-          switch (BYTE_TYPE(enc, ptr)) {
-          CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-          case BT_S: case BT_CR: case BT_LF:
-            continue;
-          case BT_GT:
-          gt:
-            *nextTokPtr = ptr + MINBPC(enc);
-            return XML_TOK_START_TAG_WITH_ATTS;
-          case BT_SOL:
-          sol:
-            ptr += MINBPC(enc);
-            REQUIRE_CHAR(enc, ptr, end);
-            if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
-              *nextTokPtr = ptr;
-              return XML_TOK_INVALID;
-            }
-            *nextTokPtr = ptr + MINBPC(enc);
-            return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;
-          default:
+          if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
             *nextTokPtr = ptr;
             return XML_TOK_INVALID;
           }
-          break;
+          *nextTokPtr = ptr + MINBPC(enc);
+          return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
         }
         break;
       }
+      break;
+    }
     default:
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
@@ -712,14 +713,13 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
-               const char **nextTokPtr)
-{
-#ifdef XML_NS
+               const char **nextTokPtr) {
+#  ifdef XML_NS
   int hadColon;
-#endif
+#  endif
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   case BT_EXCL:
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
@@ -727,8 +727,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_MINUS:
       return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);
     case BT_LSQB:
-      return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc),
-                                      end, nextTokPtr);
+      return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc), end, nextTokPtr);
     }
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
@@ -740,14 +739,14 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
-#ifdef XML_NS
+#  ifdef XML_NS
   hadColon = 0;
-#endif
+#  endif
   /* we have a start-tag */
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-#ifdef XML_NS
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+#  ifdef XML_NS
     case BT_COLON:
       if (hadColon) {
         *nextTokPtr = ptr;
@@ -757,34 +756,37 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
       switch (BYTE_TYPE(enc, ptr)) {
-      CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+        CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
       default:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       break;
-#endif
-    case BT_S: case BT_CR: case BT_LF:
-      {
-        ptr += MINBPC(enc);
-        while (HAS_CHAR(enc, ptr, end)) {
-          switch (BYTE_TYPE(enc, ptr)) {
+#  endif
+    case BT_S:
+    case BT_CR:
+    case BT_LF: {
+      ptr += MINBPC(enc);
+      while (HAS_CHAR(enc, ptr, end)) {
+        switch (BYTE_TYPE(enc, ptr)) {
           CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-          case BT_GT:
-            goto gt;
-          case BT_SOL:
-            goto sol;
-          case BT_S: case BT_CR: case BT_LF:
-            ptr += MINBPC(enc);
-            continue;
-          default:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          }
-          return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);
+        case BT_GT:
+          goto gt;
+        case BT_SOL:
+          goto sol;
+        case BT_S:
+        case BT_CR:
+        case BT_LF:
+          ptr += MINBPC(enc);
+          continue;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
         }
-        return XML_TOK_PARTIAL;
+        return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);
       }
+      return XML_TOK_PARTIAL;
+    }
     case BT_GT:
     gt:
       *nextTokPtr = ptr + MINBPC(enc);
@@ -793,7 +795,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     sol:
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
-      if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+      if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -809,8 +811,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **nextTokPtr)
-{
+                   const char **nextTokPtr) {
   if (ptr >= end)
     return XML_TOK_NONE;
   if (MINBPC(enc) > 1) {
@@ -842,48 +843,50 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
     ptr += MINBPC(enc);
     if (! HAS_CHAR(enc, ptr, end))
       return XML_TOK_TRAILING_RSQB;
-    if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
+    if (! CHAR_MATCHES(enc, ptr, ASCII_RSQB))
       break;
     ptr += MINBPC(enc);
     if (! HAS_CHAR(enc, ptr, end))
       return XML_TOK_TRAILING_RSQB;
-    if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
       ptr -= MINBPC(enc);
       break;
     }
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
-  INVALID_CASES(ptr, nextTokPtr)
+    INVALID_CASES(ptr, nextTokPtr)
   default:
     ptr += MINBPC(enc);
     break;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \
-        *nextTokPtr = ptr; \
-        return XML_TOK_DATA_CHARS; \
-      } \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) {                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_DATA_CHARS;                                               \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_RSQB:
       if (HAS_CHARS(enc, ptr, end, 2)) {
-         if (!CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {
-           ptr += MINBPC(enc);
-           break;
-         }
-         if (HAS_CHARS(enc, ptr, end, 3)) {
-           if (!CHAR_MATCHES(enc, ptr + 2*MINBPC(enc), ASCII_GT)) {
-             ptr += MINBPC(enc);
-             break;
-           }
-           *nextTokPtr = ptr + 2*MINBPC(enc);
-           return XML_TOK_INVALID;
-         }
+        if (! CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {
+          ptr += MINBPC(enc);
+          break;
+        }
+        if (HAS_CHARS(enc, ptr, end, 3)) {
+          if (! CHAR_MATCHES(enc, ptr + 2 * MINBPC(enc), ASCII_GT)) {
+            ptr += MINBPC(enc);
+            break;
+          }
+          *nextTokPtr = ptr + 2 * MINBPC(enc);
+          return XML_TOK_INVALID;
+        }
       }
       /* fall through */
     case BT_AMP:
@@ -908,12 +911,14 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
-                    const char **nextTokPtr)
-{
+                    const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-  case BT_S: case BT_LF: case BT_CR: case BT_PERCNT:
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+  case BT_S:
+  case BT_LF:
+  case BT_CR:
+  case BT_PERCNT:
     *nextTokPtr = ptr;
     return XML_TOK_PERCENT;
   default:
@@ -922,7 +927,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
     case BT_SEMI:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_PARAM_ENTITY_REF;
@@ -936,20 +941,24 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,
-                      const char **nextTokPtr)
-{
+                      const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_CR: case BT_LF: case BT_S:
-    case BT_RPAR: case BT_GT: case BT_PERCNT: case BT_VERBAR:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_CR:
+    case BT_LF:
+    case BT_S:
+    case BT_RPAR:
+    case BT_GT:
+    case BT_PERCNT:
+    case BT_VERBAR:
       *nextTokPtr = ptr;
       return XML_TOK_POUND_NAME;
     default:
@@ -961,14 +970,12 @@ PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,
 }
 
 static int PTRCALL
-PREFIX(scanLit)(int open, const ENCODING *enc,
-                const char *ptr, const char *end,
-                const char **nextTokPtr)
-{
+PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end,
+                const char **nextTokPtr) {
   while (HAS_CHAR(enc, ptr, end)) {
     int t = BYTE_TYPE(enc, ptr);
     switch (t) {
-    INVALID_CASES(ptr, nextTokPtr)
+      INVALID_CASES(ptr, nextTokPtr)
     case BT_QUOT:
     case BT_APOS:
       ptr += MINBPC(enc);
@@ -978,8 +985,12 @@ PREFIX(scanLit)(int open, const ENCODING *enc,
         return -XML_TOK_LITERAL;
       *nextTokPtr = ptr;
       switch (BYTE_TYPE(enc, ptr)) {
-      case BT_S: case BT_CR: case BT_LF:
-      case BT_GT: case BT_PERCNT: case BT_LSQB:
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+      case BT_GT:
+      case BT_PERCNT:
+      case BT_LSQB:
         return XML_TOK_LITERAL;
       default:
         return XML_TOK_INVALID;
@@ -994,8 +1005,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc,
 
 static int PTRCALL
 PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
-                  const char **nextTokPtr)
-{
+                  const char **nextTokPtr) {
   int tok;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1013,27 +1023,26 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     return PREFIX(scanLit)(BT_QUOT, enc, ptr + MINBPC(enc), end, nextTokPtr);
   case BT_APOS:
     return PREFIX(scanLit)(BT_APOS, enc, ptr + MINBPC(enc), end, nextTokPtr);
-  case BT_LT:
-    {
-      ptr += MINBPC(enc);
-      REQUIRE_CHAR(enc, ptr, end);
-      switch (BYTE_TYPE(enc, ptr)) {
-      case BT_EXCL:
-        return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-      case BT_QUEST:
-        return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-      case BT_NMSTRT:
-      case BT_HEX:
-      case BT_NONASCII:
-      case BT_LEAD2:
-      case BT_LEAD3:
-      case BT_LEAD4:
-        *nextTokPtr = ptr - MINBPC(enc);
-        return XML_TOK_INSTANCE_START;
-      }
-      *nextTokPtr = ptr;
-      return XML_TOK_INVALID;
+  case BT_LT: {
+    ptr += MINBPC(enc);
+    REQUIRE_CHAR(enc, ptr, end);
+    switch (BYTE_TYPE(enc, ptr)) {
+    case BT_EXCL:
+      return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);
+    case BT_QUEST:
+      return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
+    case BT_NMSTRT:
+    case BT_HEX:
+    case BT_NONASCII:
+    case BT_LEAD2:
+    case BT_LEAD3:
+    case BT_LEAD4:
+      *nextTokPtr = ptr - MINBPC(enc);
+      return XML_TOK_INSTANCE_START;
     }
+    *nextTokPtr = ptr;
+    return XML_TOK_INVALID;
+  }
   case BT_CR:
     if (ptr + MINBPC(enc) == end) {
       *nextTokPtr = end;
@@ -1041,13 +1050,15 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
       return -XML_TOK_PROLOG_S;
     }
     /* fall through */
-  case BT_S: case BT_LF:
+  case BT_S:
+  case BT_LF:
     for (;;) {
       ptr += MINBPC(enc);
       if (! HAS_CHAR(enc, ptr, end))
         break;
       switch (BYTE_TYPE(enc, ptr)) {
-      case BT_S: case BT_LF:
+      case BT_S:
+      case BT_LF:
         break;
       case BT_CR:
         /* don't split CR/LF pair */
@@ -1076,7 +1087,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) {
       REQUIRE_CHARS(enc, ptr, end, 2);
       if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_GT)) {
-        *nextTokPtr = ptr + 2*MINBPC(enc);
+        *nextTokPtr = ptr + 2 * MINBPC(enc);
         return XML_TOK_COND_SECT_CLOSE;
       }
     }
@@ -1099,8 +1110,12 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_PLUS:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_CLOSE_PAREN_PLUS;
-    case BT_CR: case BT_LF: case BT_S:
-    case BT_GT: case BT_COMMA: case BT_VERBAR:
+    case BT_CR:
+    case BT_LF:
+    case BT_S:
+    case BT_GT:
+    case BT_COMMA:
+    case BT_VERBAR:
     case BT_RPAR:
       *nextTokPtr = ptr;
       return XML_TOK_CLOSE_PAREN;
@@ -1115,24 +1130,26 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     return XML_TOK_DECL_CLOSE;
   case BT_NUM:
     return PREFIX(scanPoundName)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-#define LEAD_CASE(n) \
-  case BT_LEAD ## n: \
-    if (end - ptr < n) \
-      return XML_TOK_PARTIAL_CHAR; \
-    if (IS_NMSTRT_CHAR(enc, ptr, n)) { \
-      ptr += n; \
-      tok = XML_TOK_NAME; \
-      break; \
-    } \
-    if (IS_NAME_CHAR(enc, ptr, n)) { \
-      ptr += n; \
-      tok = XML_TOK_NMTOKEN; \
-      break; \
-    } \
-    *nextTokPtr = ptr; \
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (IS_NMSTRT_CHAR(enc, ptr, n)) {                                         \
+      ptr += n;                                                                \
+      tok = XML_TOK_NAME;                                                      \
+      break;                                                                   \
+    }                                                                          \
+    if (IS_NAME_CHAR(enc, ptr, n)) {                                           \
+      ptr += n;                                                                \
+      tok = XML_TOK_NMTOKEN;                                                   \
+      break;                                                                   \
+    }                                                                          \
+    *nextTokPtr = ptr;                                                         \
     return XML_TOK_INVALID;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+    LEAD_CASE(2)
+    LEAD_CASE(3)
+    LEAD_CASE(4)
+#  undef LEAD_CASE
   case BT_NMSTRT:
   case BT_HEX:
     tok = XML_TOK_NAME;
@@ -1141,9 +1158,9 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
   case BT_DIGIT:
   case BT_NAME:
   case BT_MINUS:
-#ifdef XML_NS
+#  ifdef XML_NS
   case BT_COLON:
-#endif
+#  endif
     tok = XML_TOK_NMTOKEN;
     ptr += MINBPC(enc);
     break;
@@ -1165,13 +1182,19 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_GT: case BT_RPAR: case BT_COMMA:
-    case BT_VERBAR: case BT_LSQB: case BT_PERCNT:
-    case BT_S: case BT_CR: case BT_LF:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_GT:
+    case BT_RPAR:
+    case BT_COMMA:
+    case BT_VERBAR:
+    case BT_LSQB:
+    case BT_PERCNT:
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       *nextTokPtr = ptr;
       return tok;
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
       ptr += MINBPC(enc);
       switch (tok) {
@@ -1179,7 +1202,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
         REQUIRE_CHAR(enc, ptr, end);
         tok = XML_TOK_PREFIXED_NAME;
         switch (BYTE_TYPE(enc, ptr)) {
-        CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+          CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
         default:
           tok = XML_TOK_NMTOKEN;
           break;
@@ -1190,23 +1213,23 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
         break;
       }
       break;
-#endif
+#  endif
     case BT_PLUS:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_NAME_PLUS;
     case BT_AST:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_NAME_ASTERISK;
     case BT_QUEST:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -1221,9 +1244,8 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
 }
 
 static int PTRCALL
-PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
-                          const char *end, const char **nextTokPtr)
-{
+PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end,
+                          const char **nextTokPtr) {
   const char *start;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1238,10 +1260,14 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
   start = ptr;
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_AMP:
       if (ptr == start)
         return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -1287,9 +1313,8 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
-                       const char *end, const char **nextTokPtr)
-{
+PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end,
+                       const char **nextTokPtr) {
   const char *start;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1304,10 +1329,14 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
   start = ptr;
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_AMP:
       if (ptr == start)
         return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -1315,8 +1344,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
       return XML_TOK_DATA_CHARS;
     case BT_PERCNT:
       if (ptr == start) {
-        int tok =  PREFIX(scanPercent)(enc, ptr + MINBPC(enc),
-                                       end, nextTokPtr);
+        int tok = PREFIX(scanPercent)(enc, ptr + MINBPC(enc), end, nextTokPtr);
         return (tok == XML_TOK_PERCENT) ? XML_TOK_INVALID : tok;
       }
       *nextTokPtr = ptr;
@@ -1349,12 +1377,11 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
   return XML_TOK_DATA_CHARS;
 }
 
-#ifdef XML_DTD
+#  ifdef XML_DTD
 
 static int PTRCALL
-PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
-                         const char *end, const char **nextTokPtr)
-{
+PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
+                         const char **nextTokPtr) {
   int level = 0;
   if (MINBPC(enc) > 1) {
     size_t n = end - ptr;
@@ -1365,7 +1392,7 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    INVALID_CASES(ptr, nextTokPtr)
+      INVALID_CASES(ptr, nextTokPtr)
     case BT_LT:
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
@@ -1402,12 +1429,11 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
   return XML_TOK_PARTIAL;
 }
 
-#endif /* XML_DTD */
+#  endif /* XML_DTD */
 
 static int PTRCALL
 PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **badPtr)
-{
+                   const char **badPtr) {
   ptr += MINBPC(enc);
   end -= MINBPC(enc);
   for (; HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) {
@@ -1430,9 +1456,9 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_AST:
     case BT_PERCNT:
     case BT_NUM:
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
-#endif
+#  endif
       break;
     case BT_S:
       if (CHAR_MATCHES(enc, ptr, ASCII_TAB)) {
@@ -1442,7 +1468,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
       break;
     case BT_NAME:
     case BT_NMSTRT:
-      if (!(BYTE_TO_ASCII(enc, ptr) & ~0x7f))
+      if (! (BYTE_TO_ASCII(enc, ptr) & ~0x7f))
         break;
       /* fall through */
     default:
@@ -1466,9 +1492,8 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
 */
 
 static int PTRCALL
-PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
-                int attsMax, ATTRIBUTE *atts)
-{
+PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
+                ATTRIBUTE *atts) {
   enum { other, inName, inValue } state = inName;
   int nAtts = 0;
   int open = 0; /* defined when state == inValue;
@@ -1476,32 +1501,35 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
 
   for (ptr += MINBPC(enc);; ptr += MINBPC(enc)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define START_NAME \
-      if (state == other) { \
-        if (nAtts < attsMax) { \
-          atts[nAtts].name = ptr; \
-          atts[nAtts].normalized = 1; \
-        } \
-        state = inName; \
-      }
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: START_NAME ptr += (n - MINBPC(enc)); break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define START_NAME                                                           \
+    if (state == other) {                                                      \
+      if (nAtts < attsMax) {                                                   \
+        atts[nAtts].name = ptr;                                                \
+        atts[nAtts].normalized = 1;                                            \
+      }                                                                        \
+      state = inName;                                                          \
+    }
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    START_NAME ptr += (n - MINBPC(enc));                                       \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONASCII:
     case BT_NMSTRT:
     case BT_HEX:
       START_NAME
       break;
-#undef START_NAME
+#  undef START_NAME
     case BT_QUOT:
       if (state != inValue) {
         if (nAtts < attsMax)
           atts[nAtts].valuePtr = ptr + MINBPC(enc);
         state = inValue;
         open = BT_QUOT;
-      }
-      else if (open == BT_QUOT) {
+      } else if (open == BT_QUOT) {
         state = other;
         if (nAtts < attsMax)
           atts[nAtts].valueEnd = ptr;
@@ -1514,8 +1542,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
           atts[nAtts].valuePtr = ptr + MINBPC(enc);
         state = inValue;
         open = BT_APOS;
-      }
-      else if (open == BT_APOS) {
+      } else if (open == BT_APOS) {
         state = other;
         if (nAtts < attsMax)
           atts[nAtts].valueEnd = ptr;
@@ -1529,16 +1556,15 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
     case BT_S:
       if (state == inName)
         state = other;
-      else if (state == inValue
-               && nAtts < attsMax
-               && atts[nAtts].normalized
+      else if (state == inValue && nAtts < attsMax && atts[nAtts].normalized
                && (ptr == atts[nAtts].valuePtr
                    || BYTE_TO_ASCII(enc, ptr) != ASCII_SPACE
                    || BYTE_TO_ASCII(enc, ptr + MINBPC(enc)) == ASCII_SPACE
                    || BYTE_TYPE(enc, ptr + MINBPC(enc)) == open))
         atts[nAtts].normalized = 0;
       break;
-    case BT_CR: case BT_LF:
+    case BT_CR:
+    case BT_LF:
       /* This case ensures that the first attribute name is counted
          Apart from that we could just change state on the quote. */
       if (state == inName)
@@ -1559,29 +1585,44 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRFASTCALL
-PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
-{
+PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) {
   int result = 0;
   /* skip &# */
-  ptr += 2*MINBPC(enc);
+  UNUSED_P(enc);
+  ptr += 2 * MINBPC(enc);
   if (CHAR_MATCHES(enc, ptr, ASCII_x)) {
-    for (ptr += MINBPC(enc);
-         !CHAR_MATCHES(enc, ptr, ASCII_SEMI);
+    for (ptr += MINBPC(enc); ! CHAR_MATCHES(enc, ptr, ASCII_SEMI);
          ptr += MINBPC(enc)) {
       int c = BYTE_TO_ASCII(enc, ptr);
       switch (c) {
-      case ASCII_0: case ASCII_1: case ASCII_2: case ASCII_3: case ASCII_4:
-      case ASCII_5: case ASCII_6: case ASCII_7: case ASCII_8: case ASCII_9:
+      case ASCII_0:
+      case ASCII_1:
+      case ASCII_2:
+      case ASCII_3:
+      case ASCII_4:
+      case ASCII_5:
+      case ASCII_6:
+      case ASCII_7:
+      case ASCII_8:
+      case ASCII_9:
         result <<= 4;
         result |= (c - ASCII_0);
         break;
-      case ASCII_A: case ASCII_B: case ASCII_C:
-      case ASCII_D: case ASCII_E: case ASCII_F:
+      case ASCII_A:
+      case ASCII_B:
+      case ASCII_C:
+      case ASCII_D:
+      case ASCII_E:
+      case ASCII_F:
         result <<= 4;
         result += 10 + (c - ASCII_A);
         break;
-      case ASCII_a: case ASCII_b: case ASCII_c:
-      case ASCII_d: case ASCII_e: case ASCII_f:
+      case ASCII_a:
+      case ASCII_b:
+      case ASCII_c:
+      case ASCII_d:
+      case ASCII_e:
+      case ASCII_f:
         result <<= 4;
         result += 10 + (c - ASCII_a);
         break;
@@ -1589,9 +1630,8 @@ PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
       if (result >= 0x110000)
         return -1;
     }
-  }
-  else {
-    for (; !CHAR_MATCHES(enc, ptr, ASCII_SEMI); ptr += MINBPC(enc)) {
+  } else {
+    for (; ! CHAR_MATCHES(enc, ptr, ASCII_SEMI); ptr += MINBPC(enc)) {
       int c = BYTE_TO_ASCII(enc, ptr);
       result *= 10;
       result += (c - ASCII_0);
@@ -1603,10 +1643,10 @@ PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
 }
 
 static int PTRCALL
-PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                             const char *end)
-{
-  switch ((end - ptr)/MINBPC(enc)) {
+PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr,
+                             const char *end) {
+  UNUSED_P(enc);
+  switch ((end - ptr) / MINBPC(enc)) {
   case 2:
     if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_t)) {
       switch (BYTE_TO_ASCII(enc, ptr)) {
@@ -1657,9 +1697,9 @@ PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1,
-                         const char *end1, const char *ptr2)
-{
+PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1,
+                         const char *end1, const char *ptr2) {
+  UNUSED_P(enc);
   for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) {
     if (end1 - ptr1 < MINBPC(enc)) {
       /* This line cannot be executed.  The incoming data has already
@@ -1669,27 +1709,30 @@ PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1,
        */
       return 0; /* LCOV_EXCL_LINE */
     }
-    if (!CHAR_MATCHES(enc, ptr1, *ptr2))
+    if (! CHAR_MATCHES(enc, ptr1, *ptr2))
       return 0;
   }
   return ptr1 == end1;
 }
 
 static int PTRFASTCALL
-PREFIX(nameLength)(const ENCODING *enc, const char *ptr)
-{
+PREFIX(nameLength)(const ENCODING *enc, const char *ptr) {
   const char *start = ptr;
   for (;;) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONASCII:
     case BT_NMSTRT:
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
-#endif
+#  endif
     case BT_HEX:
     case BT_DIGIT:
     case BT_NAME:
@@ -1702,9 +1745,8 @@ PREFIX(nameLength)(const ENCODING *enc, const char *ptr)
   }
 }
 
-static const char * PTRFASTCALL
-PREFIX(skipS)(const ENCODING *enc, const char *ptr)
-{
+static const char *PTRFASTCALL
+PREFIX(skipS)(const ENCODING *enc, const char *ptr) {
   for (;;) {
     switch (BYTE_TYPE(enc, ptr)) {
     case BT_LF:
@@ -1719,19 +1761,18 @@ PREFIX(skipS)(const ENCODING *enc, const char *ptr)
 }
 
 static void PTRCALL
-PREFIX(updatePosition)(const ENCODING *enc,
-                       const char *ptr,
-                       const char *end,
-                       POSITION *pos)
-{
+PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end,
+                       POSITION *pos) {
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_LF:
       pos->columnNumber = (XML_Size)-1;
       pos->lineNumber++;
@@ -1752,12 +1793,12 @@ PREFIX(updatePosition)(const ENCODING *enc,
   }
 }
 
-#undef DO_LEAD_CASE
-#undef MULTIBYTE_CASES
-#undef INVALID_CASES
-#undef CHECK_NAME_CASE
-#undef CHECK_NAME_CASES
-#undef CHECK_NMSTRT_CASE
-#undef CHECK_NMSTRT_CASES
+#  undef DO_LEAD_CASE
+#  undef MULTIBYTE_CASES
+#  undef INVALID_CASES
+#  undef CHECK_NAME_CASE
+#  undef CHECK_NAME_CASES
+#  undef CHECK_NMSTRT_CASE
+#  undef CHECK_NMSTRT_CASES
 
 #endif /* XML_TOK_IMPL_C */
diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h
index a6420f48eedc..e925dbc7e2c8 100644
--- a/Modules/expat/xmltok_impl.h
+++ b/Modules/expat/xmltok_impl.h
@@ -31,43 +31,43 @@
 */
 
 enum {
-  BT_NONXML,
-  BT_MALFORM,
-  BT_LT,
-  BT_AMP,
-  BT_RSQB,
-  BT_LEAD2,
-  BT_LEAD3,
-  BT_LEAD4,
-  BT_TRAIL,
-  BT_CR,
-  BT_LF,
-  BT_GT,
-  BT_QUOT,
-  BT_APOS,
-  BT_EQUALS,
-  BT_QUEST,
-  BT_EXCL,
-  BT_SOL,
-  BT_SEMI,
-  BT_NUM,
-  BT_LSQB,
-  BT_S,
-  BT_NMSTRT,
-  BT_COLON,
-  BT_HEX,
-  BT_DIGIT,
-  BT_NAME,
-  BT_MINUS,
-  BT_OTHER, /* known not to be a name or name start character */
+  BT_NONXML,   /* e.g. noncharacter-FFFF */
+  BT_MALFORM,  /* illegal, with regard to encoding */
+  BT_LT,       /* less than = "<" */
+  BT_AMP,      /* ampersand = "&" */
+  BT_RSQB,     /* right square bracket = "[" */
+  BT_LEAD2,    /* lead byte of a 2-byte UTF-8 character */
+  BT_LEAD3,    /* lead byte of a 3-byte UTF-8 character */
+  BT_LEAD4,    /* lead byte of a 4-byte UTF-8 character */
+  BT_TRAIL,    /* trailing unit, e.g. second 16-bit unit of a 4-byte char. */
+  BT_CR,       /* carriage return = "\r" */
+  BT_LF,       /* line feed = "\n" */
+  BT_GT,       /* greater than = ">" */
+  BT_QUOT,     /* quotation character = "\"" */
+  BT_APOS,     /* aposthrophe = "'" */
+  BT_EQUALS,   /* equal sign = "=" */
+  BT_QUEST,    /* question mark = "?" */
+  BT_EXCL,     /* exclamation mark = "!" */
+  BT_SOL,      /* solidus, slash = "/" */
+  BT_SEMI,     /* semicolon = ";" */
+  BT_NUM,      /* number sign = "#" */
+  BT_LSQB,     /* left square bracket = "[" */
+  BT_S,        /* white space, e.g. "\t", " "[, "\r"] */
+  BT_NMSTRT,   /* non-hex name start letter = "G".."Z" + "g".."z" + "_" */
+  BT_COLON,    /* colon = ":" */
+  BT_HEX,      /* hex letter = "A".."F" + "a".."f" */
+  BT_DIGIT,    /* digit = "0".."9" */
+  BT_NAME,     /* dot and middle dot = "." + chr(0xb7) */
+  BT_MINUS,    /* minus = "-" */
+  BT_OTHER,    /* known not to be a name or name start character */
   BT_NONASCII, /* might be a name or name start character */
-  BT_PERCNT,
-  BT_LPAR,
-  BT_RPAR,
-  BT_AST,
-  BT_PLUS,
-  BT_COMMA,
-  BT_VERBAR
+  BT_PERCNT,   /* percent sign = "%" */
+  BT_LPAR,     /* left parenthesis = "(" */
+  BT_RPAR,     /* right parenthesis = "(" */
+  BT_AST,      /* asterisk = "*" */
+  BT_PLUS,     /* plus sign = "+" */
+  BT_COMMA,    /* comma = "," */
+  BT_VERBAR    /* vertical bar = "|" */
 };
 
 #include <stddef.h>
diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c
index 23d31e8e4249..919c74e9f97f 100644
--- a/Modules/expat/xmltok_ns.c
+++ b/Modules/expat/xmltok_ns.c
@@ -33,56 +33,47 @@
 #ifdef XML_TOK_NS_C
 
 const ENCODING *
-NS(XmlGetUtf8InternalEncoding)(void)
-{
+NS(XmlGetUtf8InternalEncoding)(void) {
   return &ns(internal_utf8_encoding).enc;
 }
 
 const ENCODING *
-NS(XmlGetUtf16InternalEncoding)(void)
-{
-#if BYTEORDER == 1234
+NS(XmlGetUtf16InternalEncoding)(void) {
+#  if BYTEORDER == 1234
   return &ns(internal_little2_encoding).enc;
-#elif BYTEORDER == 4321
+#  elif BYTEORDER == 4321
   return &ns(internal_big2_encoding).enc;
-#else
+#  else
   const short n = 1;
-  return (*(const char *)&n
-          ? &ns(internal_little2_encoding).enc
-          : &ns(internal_big2_encoding).enc);
-#endif
+  return (*(const char *)&n ? &ns(internal_little2_encoding).enc
+                            : &ns(internal_big2_encoding).enc);
+#  endif
 }
 
-static const ENCODING * const NS(encodings)[] = {
-  &ns(latin1_encoding).enc,
-  &ns(ascii_encoding).enc,
-  &ns(utf8_encoding).enc,
-  &ns(big2_encoding).enc,
-  &ns(big2_encoding).enc,
-  &ns(little2_encoding).enc,
-  &ns(utf8_encoding).enc /* NO_ENC */
+static const ENCODING *const NS(encodings)[] = {
+    &ns(latin1_encoding).enc, &ns(ascii_encoding).enc,
+    &ns(utf8_encoding).enc,   &ns(big2_encoding).enc,
+    &ns(big2_encoding).enc,   &ns(little2_encoding).enc,
+    &ns(utf8_encoding).enc /* NO_ENC */
 };
 
 static int PTRCALL
 NS(initScanProlog)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **nextTokPtr)
-{
-  return initScan(NS(encodings), (const INIT_ENCODING *)enc,
-                  XML_PROLOG_STATE, ptr, end, nextTokPtr);
+                   const char **nextTokPtr) {
+  return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_PROLOG_STATE,
+                  ptr, end, nextTokPtr);
 }
 
 static int PTRCALL
 NS(initScanContent)(const ENCODING *enc, const char *ptr, const char *end,
-                    const char **nextTokPtr)
-{
-  return initScan(NS(encodings), (const INIT_ENCODING *)enc,
-                  XML_CONTENT_STATE, ptr, end, nextTokPtr);
+                    const char **nextTokPtr) {
+  return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_CONTENT_STATE,
+                  ptr, end, nextTokPtr);
 }
 
 int
 NS(XmlInitEncoding)(INIT_ENCODING *p, const ENCODING **encPtr,
-                    const char *name)
-{
+                    const char *name) {
   int i = getEncodingIndex(name);
   if (i == UNKNOWN_ENC)
     return 0;
@@ -96,9 +87,8 @@ NS(XmlInitEncoding)(INIT_ENCODING *p, const ENCODING **encPtr,
 }
 
 static const ENCODING *
-NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end)
-{
-#define ENCODING_MAX 128
+NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end) {
+#  define ENCODING_MAX 128
   char buf[ENCODING_MAX];
   char *p = buf;
   int i;
@@ -115,28 +105,14 @@ NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end)
 }
 
 int
-NS(XmlParseXmlDecl)(int isGeneralTextEntity,
-                    const ENCODING *enc,
-                    const char *ptr,
-                    const char *end,
-                    const char **badPtr,
-                    const char **versionPtr,
-                    const char **versionEndPtr,
-                    const char **encodingName,
-                    const ENCODING **encoding,
-                    int *standalone)
-{
-  return doParseXmlDecl(NS(findEncoding),
-                        isGeneralTextEntity,
-                        enc,
-                        ptr,
-                        end,
-                        badPtr,
-                        versionPtr,
-                        versionEndPtr,
-                        encodingName,
-                        encoding,
-                        standalone);
+NS(XmlParseXmlDecl)(int isGeneralTextEntity, const ENCODING *enc,
+                    const char *ptr, const char *end, const char **badPtr,
+                    const char **versionPtr, const char **versionEndPtr,
+                    const char **encodingName, const ENCODING **encoding,
+                    int *standalone) {
+  return doParseXmlDecl(NS(findEncoding), isGeneralTextEntity, enc, ptr, end,
+                        badPtr, versionPtr, versionEndPtr, encodingName,
+                        encoding, standalone);
 }
 
 #endif /* XML_TOK_NS_C */
diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj
index 2338af4505b4..e430a520a2a1 100644
--- a/PCbuild/_elementtree.vcxproj
+++ b/PCbuild/_elementtree.vcxproj
@@ -84,7 +84,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\_elementtree.c" />
-    <ClCompile Include="..\Modules\expat\loadlibrary.c" />
     <ClCompile Include="..\Modules\expat\xmlparse.c" />
     <ClCompile Include="..\Modules\expat\xmlrole.c" />
     <ClCompile Include="..\Modules\expat\xmltok.c" />
diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters
index 4597ee521b33..6acdf35846ab 100644
--- a/PCbuild/_elementtree.vcxproj.filters
+++ b/PCbuild/_elementtree.vcxproj.filters
@@ -33,9 +33,6 @@
     <ClInclude Include="..\Modules\expat\latin1tab.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\expat\loadlibrary.c">
-      <Filter>Header Files</Filter>
-    </ClInclude>
     <ClInclude Include="..\Modules\expat\macconfig.h">
       <Filter>Header Files</Filter>
     </ClInclude>
diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj
index 51ca69e95d70..9a765ff07bde 100644
--- a/PCbuild/pyexpat.vcxproj
+++ b/PCbuild/pyexpat.vcxproj
@@ -68,7 +68,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\pyexpat.c" />
-    <ClCompile Include="..\Modules\expat\loadlibrary.c" />
     <ClCompile Include="..\Modules\expat\xmlparse.c" />
     <ClCompile Include="..\Modules\expat\xmlrole.c" />
     <ClCompile Include="..\Modules\expat\xmltok.c" />
diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters
index cb02847980c6..f8d46026c9c2 100644
--- a/PCbuild/pyexpat.vcxproj.filters
+++ b/PCbuild/pyexpat.vcxproj.filters
@@ -20,9 +20,6 @@
     <ClCompile Include="..\Modules\pyexpat.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\expat\loadlibrary.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Modules\expat\xmlparse.c">
       <Filter>Source Files</Filter>
     </ClCompile>


From webhook-mailer at python.org  Thu Sep 26 00:57:20 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 26 Sep 2019 04:57:20 -0000
Subject: [Python-checkins] [3.8] closes bpo-38174: Update vendored expat
 library to 2.2.8. (GH-16409)
Message-ID: <mailman.97.1569473841.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/d75bf44a2812a249663f979dc8379d27c1633ba5
commit: d75bf44a2812a249663f979dc8379d27c1633ba5
branch: 3.8
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-25T21:57:14-07:00
summary:

[3.8] closes bpo-38174: Update vendored expat library to 2.2.8. (GH-16409)

Fixes CVE-2019-15903. See full changelog at https://github.com/libexpat/libexpat/blob/R_2_2_8/expat/Changes.
(cherry picked from commit 52b940803860e37bcc3f6096b2d24e7c20a0e807)

files:
A Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
D Modules/expat/loadlibrary.c
M Modules/expat/asciitab.h
M Modules/expat/expat.h
M Modules/expat/expat_external.h
M Modules/expat/iasciitab.h
M Modules/expat/internal.h
M Modules/expat/latin1tab.h
M Modules/expat/nametab.h
M Modules/expat/siphash.h
M Modules/expat/utf8tab.h
M Modules/expat/winconfig.h
M Modules/expat/xmlparse.c
M Modules/expat/xmlrole.c
M Modules/expat/xmlrole.h
M Modules/expat/xmltok.c
M Modules/expat/xmltok.h
M Modules/expat/xmltok_impl.c
M Modules/expat/xmltok_impl.h
M Modules/expat/xmltok_ns.c
M PCbuild/_elementtree.vcxproj
M PCbuild/_elementtree.vcxproj.filters
M PCbuild/pyexpat.vcxproj
M PCbuild/pyexpat.vcxproj.filters

diff --git a/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst b/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
new file mode 100644
index 000000000000..e3da6fcdd139
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
@@ -0,0 +1,2 @@
+Update vendorized expat library version to 2.2.8, which resolves
+CVE-2019-15903.
diff --git a/Modules/expat/asciitab.h b/Modules/expat/asciitab.h
index 2f59fd929065..63b1d1b4482e 100644
--- a/Modules/expat/asciitab.h
+++ b/Modules/expat/asciitab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
-/* 0x0C */ BT_NONXML, BT_CR, BT_NONXML, BT_NONXML,
-/* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
-/* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
-/* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
-/* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
-/* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
-/* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
-/* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
-/* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
-/* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
+    /* 0x0C */ BT_NONXML, BT_CR, BT_NONXML, BT_NONXML,
+    /* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
+    /* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
+    /* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
+    /* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
+    /* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
+    /* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
+    /* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
+    /* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
+    /* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h
index c050f1d918ce..6c8eb1fda4a3 100644
--- a/Modules/expat/expat.h
+++ b/Modules/expat/expat.h
@@ -33,15 +33,6 @@
 #ifndef Expat_INCLUDED
 #define Expat_INCLUDED 1
 
-#ifdef __VMS
-/*      0        1         2         3      0        1         2         3
-        1234567890123456789012345678901     1234567890123456789012345678901 */
-#define XML_SetProcessingInstructionHandler XML_SetProcessingInstrHandler
-#define XML_SetUnparsedEntityDeclHandler    XML_SetUnparsedEntDeclHandler
-#define XML_SetStartNamespaceDeclHandler    XML_SetStartNamespcDeclHandler
-#define XML_SetExternalEntityRefHandlerArg  XML_SetExternalEntRefHandlerArg
-#endif
-
 #include <stdlib.h>
 #include "expat_external.h"
 
@@ -53,8 +44,8 @@ struct XML_ParserStruct;
 typedef struct XML_ParserStruct *XML_Parser;
 
 typedef unsigned char XML_Bool;
-#define XML_TRUE   ((XML_Bool) 1)
-#define XML_FALSE  ((XML_Bool) 0)
+#define XML_TRUE ((XML_Bool)1)
+#define XML_FALSE ((XML_Bool)0)
 
 /* The XML_Status enum gives the possible return values for several
    API functions.  The preprocessor #defines are included so this
@@ -164,25 +155,23 @@ enum XML_Content_Quant {
 typedef struct XML_cp XML_Content;
 
 struct XML_cp {
-  enum XML_Content_Type         type;
-  enum XML_Content_Quant        quant;
-  XML_Char *                    name;
-  unsigned int                  numchildren;
-  XML_Content *                 children;
+  enum XML_Content_Type type;
+  enum XML_Content_Quant quant;
+  XML_Char *name;
+  unsigned int numchildren;
+  XML_Content *children;
 };
 
-
 /* This is called for an element declaration. See above for
    description of the model argument. It's the caller's responsibility
    to free model when finished with it.
 */
-typedef void (XMLCALL *XML_ElementDeclHandler) (void *userData,
-                                                const XML_Char *name,
-                                                XML_Content *model);
+typedef void(XMLCALL *XML_ElementDeclHandler)(void *userData,
+                                              const XML_Char *name,
+                                              XML_Content *model);
 
 XMLPARSEAPI(void)
-XML_SetElementDeclHandler(XML_Parser parser,
-                          XML_ElementDeclHandler eldecl);
+XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl);
 
 /* The Attlist declaration handler is called for *each* attribute. So
    a single Attlist declaration with multiple attributes declared will
@@ -192,17 +181,12 @@ XML_SetElementDeclHandler(XML_Parser parser,
    value will be NULL in the case of "#REQUIRED". If "isrequired" is
    true and default is non-NULL, then this is a "#FIXED" default.
 */
-typedef void (XMLCALL *XML_AttlistDeclHandler) (
-                                    void            *userData,
-                                    const XML_Char  *elname,
-                                    const XML_Char  *attname,
-                                    const XML_Char  *att_type,
-                                    const XML_Char  *dflt,
-                                    int              isrequired);
+typedef void(XMLCALL *XML_AttlistDeclHandler)(
+    void *userData, const XML_Char *elname, const XML_Char *attname,
+    const XML_Char *att_type, const XML_Char *dflt, int isrequired);
 
 XMLPARSEAPI(void)
-XML_SetAttlistDeclHandler(XML_Parser parser,
-                          XML_AttlistDeclHandler attdecl);
+XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl);
 
 /* The XML declaration handler is called for *both* XML declarations
    and text declarations. The way to distinguish is that the version
@@ -212,15 +196,13 @@ XML_SetAttlistDeclHandler(XML_Parser parser,
    was no standalone parameter in the declaration, that it was given
    as no, or that it was given as yes.
 */
-typedef void (XMLCALL *XML_XmlDeclHandler) (void           *userData,
-                                            const XML_Char *version,
-                                            const XML_Char *encoding,
-                                            int             standalone);
+typedef void(XMLCALL *XML_XmlDeclHandler)(void *userData,
+                                          const XML_Char *version,
+                                          const XML_Char *encoding,
+                                          int standalone);
 
 XMLPARSEAPI(void)
-XML_SetXmlDeclHandler(XML_Parser parser,
-                      XML_XmlDeclHandler xmldecl);
-
+XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler xmldecl);
 
 typedef struct {
   void *(*malloc_fcn)(size_t size);
@@ -248,7 +230,6 @@ XML_ParserCreate(const XML_Char *encoding);
 XMLPARSEAPI(XML_Parser)
 XML_ParserCreateNS(const XML_Char *encoding, XML_Char namespaceSeparator);
 
-
 /* Constructs a new parser using the memory management suite referred to
    by memsuite. If memsuite is NULL, then use the standard library memory
    suite. If namespaceSeparator is non-NULL it creates a parser with
@@ -278,31 +259,27 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encoding);
 /* atts is array of name/value pairs, terminated by 0;
    names and values are 0 terminated.
 */
-typedef void (XMLCALL *XML_StartElementHandler) (void *userData,
-                                                 const XML_Char *name,
-                                                 const XML_Char **atts);
-
-typedef void (XMLCALL *XML_EndElementHandler) (void *userData,
-                                               const XML_Char *name);
+typedef void(XMLCALL *XML_StartElementHandler)(void *userData,
+                                               const XML_Char *name,
+                                               const XML_Char **atts);
 
+typedef void(XMLCALL *XML_EndElementHandler)(void *userData,
+                                             const XML_Char *name);
 
 /* s is not 0 terminated. */
-typedef void (XMLCALL *XML_CharacterDataHandler) (void *userData,
-                                                  const XML_Char *s,
-                                                  int len);
+typedef void(XMLCALL *XML_CharacterDataHandler)(void *userData,
+                                                const XML_Char *s, int len);
 
 /* target and data are 0 terminated */
-typedef void (XMLCALL *XML_ProcessingInstructionHandler) (
-                                                void *userData,
-                                                const XML_Char *target,
-                                                const XML_Char *data);
+typedef void(XMLCALL *XML_ProcessingInstructionHandler)(void *userData,
+                                                        const XML_Char *target,
+                                                        const XML_Char *data);
 
 /* data is 0 terminated */
-typedef void (XMLCALL *XML_CommentHandler) (void *userData,
-                                            const XML_Char *data);
+typedef void(XMLCALL *XML_CommentHandler)(void *userData, const XML_Char *data);
 
-typedef void (XMLCALL *XML_StartCdataSectionHandler) (void *userData);
-typedef void (XMLCALL *XML_EndCdataSectionHandler) (void *userData);
+typedef void(XMLCALL *XML_StartCdataSectionHandler)(void *userData);
+typedef void(XMLCALL *XML_EndCdataSectionHandler)(void *userData);
 
 /* This is called for any characters in the XML document for which
    there is no applicable handler.  This includes both characters that
@@ -317,25 +294,23 @@ typedef void (XMLCALL *XML_EndCdataSectionHandler) (void *userData);
    default handler: for example, a comment might be split between
    multiple calls.
 */
-typedef void (XMLCALL *XML_DefaultHandler) (void *userData,
-                                            const XML_Char *s,
-                                            int len);
+typedef void(XMLCALL *XML_DefaultHandler)(void *userData, const XML_Char *s,
+                                          int len);
 
 /* This is called for the start of the DOCTYPE declaration, before
    any DTD or internal subset is parsed.
 */
-typedef void (XMLCALL *XML_StartDoctypeDeclHandler) (
-                                            void *userData,
-                                            const XML_Char *doctypeName,
-                                            const XML_Char *sysid,
-                                            const XML_Char *pubid,
-                                            int has_internal_subset);
+typedef void(XMLCALL *XML_StartDoctypeDeclHandler)(void *userData,
+                                                   const XML_Char *doctypeName,
+                                                   const XML_Char *sysid,
+                                                   const XML_Char *pubid,
+                                                   int has_internal_subset);
 
 /* This is called for the start of the DOCTYPE declaration when the
    closing > is encountered, but after processing any external
    subset.
 */
-typedef void (XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
+typedef void(XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
 
 /* This is called for entity declarations. The is_parameter_entity
    argument will be non-zero if the entity is a parameter entity, zero
@@ -355,20 +330,14 @@ typedef void (XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
    Note that is_parameter_entity can't be changed to XML_Bool, since
    that would break binary compatibility.
 */
-typedef void (XMLCALL *XML_EntityDeclHandler) (
-                              void *userData,
-                              const XML_Char *entityName,
-                              int is_parameter_entity,
-                              const XML_Char *value,
-                              int value_length,
-                              const XML_Char *base,
-                              const XML_Char *systemId,
-                              const XML_Char *publicId,
-                              const XML_Char *notationName);
+typedef void(XMLCALL *XML_EntityDeclHandler)(
+    void *userData, const XML_Char *entityName, int is_parameter_entity,
+    const XML_Char *value, int value_length, const XML_Char *base,
+    const XML_Char *systemId, const XML_Char *publicId,
+    const XML_Char *notationName);
 
 XMLPARSEAPI(void)
-XML_SetEntityDeclHandler(XML_Parser parser,
-                         XML_EntityDeclHandler handler);
+XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler);
 
 /* OBSOLETE -- OBSOLETE -- OBSOLETE
    This handler has been superseded by the EntityDeclHandler above.
@@ -379,24 +348,20 @@ XML_SetEntityDeclHandler(XML_Parser parser,
    entityName, systemId and notationName arguments will never be
    NULL. The other arguments may be.
 */
-typedef void (XMLCALL *XML_UnparsedEntityDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *entityName,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId,
-                                    const XML_Char *notationName);
+typedef void(XMLCALL *XML_UnparsedEntityDeclHandler)(
+    void *userData, const XML_Char *entityName, const XML_Char *base,
+    const XML_Char *systemId, const XML_Char *publicId,
+    const XML_Char *notationName);
 
 /* This is called for a declaration of notation.  The base argument is
    whatever was set by XML_SetBase. The notationName will never be
    NULL.  The other arguments can be.
 */
-typedef void (XMLCALL *XML_NotationDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *notationName,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId);
+typedef void(XMLCALL *XML_NotationDeclHandler)(void *userData,
+                                               const XML_Char *notationName,
+                                               const XML_Char *base,
+                                               const XML_Char *systemId,
+                                               const XML_Char *publicId);
 
 /* When namespace processing is enabled, these are called once for
    each namespace declaration. The call to the start and end element
@@ -404,14 +369,12 @@ typedef void (XMLCALL *XML_NotationDeclHandler) (
    declaration handlers. For an xmlns attribute, prefix will be
    NULL.  For an xmlns="" attribute, uri will be NULL.
 */
-typedef void (XMLCALL *XML_StartNamespaceDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *prefix,
-                                    const XML_Char *uri);
+typedef void(XMLCALL *XML_StartNamespaceDeclHandler)(void *userData,
+                                                     const XML_Char *prefix,
+                                                     const XML_Char *uri);
 
-typedef void (XMLCALL *XML_EndNamespaceDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *prefix);
+typedef void(XMLCALL *XML_EndNamespaceDeclHandler)(void *userData,
+                                                   const XML_Char *prefix);
 
 /* This is called if the document is not standalone, that is, it has an
    external subset or a reference to a parameter entity, but does not
@@ -422,7 +385,7 @@ typedef void (XMLCALL *XML_EndNamespaceDeclHandler) (
    conditions above this handler will only be called if the referenced
    entity was actually read.
 */
-typedef int (XMLCALL *XML_NotStandaloneHandler) (void *userData);
+typedef int(XMLCALL *XML_NotStandaloneHandler)(void *userData);
 
 /* This is called for a reference to an external parsed general
    entity.  The referenced entity is not automatically parsed.  The
@@ -458,12 +421,11 @@ typedef int (XMLCALL *XML_NotStandaloneHandler) (void *userData);
    Note that unlike other handlers the first argument is the parser,
    not userData.
 */
-typedef int (XMLCALL *XML_ExternalEntityRefHandler) (
-                                    XML_Parser parser,
-                                    const XML_Char *context,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId);
+typedef int(XMLCALL *XML_ExternalEntityRefHandler)(XML_Parser parser,
+                                                   const XML_Char *context,
+                                                   const XML_Char *base,
+                                                   const XML_Char *systemId,
+                                                   const XML_Char *publicId);
 
 /* This is called in two situations:
    1) An entity reference is encountered for which no declaration
@@ -475,10 +437,9 @@ typedef int (XMLCALL *XML_ExternalEntityRefHandler) (
          the event would be out of sync with the reporting of the
          declarations or attribute values
 */
-typedef void (XMLCALL *XML_SkippedEntityHandler) (
-                                    void *userData,
-                                    const XML_Char *entityName,
-                                    int is_parameter_entity);
+typedef void(XMLCALL *XML_SkippedEntityHandler)(void *userData,
+                                                const XML_Char *entityName,
+                                                int is_parameter_entity);
 
 /* This structure is filled in by the XML_UnknownEncodingHandler to
    provide information to the parser about encodings that are unknown
@@ -535,8 +496,8 @@ typedef void (XMLCALL *XML_SkippedEntityHandler) (
 typedef struct {
   int map[256];
   void *data;
-  int (XMLCALL *convert)(void *data, const char *s);
-  void (XMLCALL *release)(void *data);
+  int(XMLCALL *convert)(void *data, const char *s);
+  void(XMLCALL *release)(void *data);
 } XML_Encoding;
 
 /* This is called for an encoding that is unknown to the parser.
@@ -554,23 +515,19 @@ typedef struct {
    If info does not describe a suitable encoding, then the parser will
    return an XML_UNKNOWN_ENCODING error.
 */
-typedef int (XMLCALL *XML_UnknownEncodingHandler) (
-                                    void *encodingHandlerData,
-                                    const XML_Char *name,
-                                    XML_Encoding *info);
+typedef int(XMLCALL *XML_UnknownEncodingHandler)(void *encodingHandlerData,
+                                                 const XML_Char *name,
+                                                 XML_Encoding *info);
 
 XMLPARSEAPI(void)
-XML_SetElementHandler(XML_Parser parser,
-                      XML_StartElementHandler start,
+XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start,
                       XML_EndElementHandler end);
 
 XMLPARSEAPI(void)
-XML_SetStartElementHandler(XML_Parser parser,
-                           XML_StartElementHandler handler);
+XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetEndElementHandler(XML_Parser parser,
-                         XML_EndElementHandler handler);
+XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetCharacterDataHandler(XML_Parser parser,
@@ -580,8 +537,7 @@ XMLPARSEAPI(void)
 XML_SetProcessingInstructionHandler(XML_Parser parser,
                                     XML_ProcessingInstructionHandler handler);
 XMLPARSEAPI(void)
-XML_SetCommentHandler(XML_Parser parser,
-                      XML_CommentHandler handler);
+XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetCdataSectionHandler(XML_Parser parser,
@@ -601,20 +557,17 @@ XML_SetEndCdataSectionHandler(XML_Parser parser,
    default handler, or to the skipped entity handler, if one is set.
 */
 XMLPARSEAPI(void)
-XML_SetDefaultHandler(XML_Parser parser,
-                      XML_DefaultHandler handler);
+XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler);
 
 /* This sets the default handler but does not inhibit expansion of
    internal entities.  The entity reference will not be passed to the
    default handler.
 */
 XMLPARSEAPI(void)
-XML_SetDefaultHandlerExpand(XML_Parser parser,
-                            XML_DefaultHandler handler);
+XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetDoctypeDeclHandler(XML_Parser parser,
-                          XML_StartDoctypeDeclHandler start,
+XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start,
                           XML_EndDoctypeDeclHandler end);
 
 XMLPARSEAPI(void)
@@ -622,16 +575,14 @@ XML_SetStartDoctypeDeclHandler(XML_Parser parser,
                                XML_StartDoctypeDeclHandler start);
 
 XMLPARSEAPI(void)
-XML_SetEndDoctypeDeclHandler(XML_Parser parser,
-                             XML_EndDoctypeDeclHandler end);
+XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end);
 
 XMLPARSEAPI(void)
 XML_SetUnparsedEntityDeclHandler(XML_Parser parser,
                                  XML_UnparsedEntityDeclHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetNotationDeclHandler(XML_Parser parser,
-                           XML_NotationDeclHandler handler);
+XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetNamespaceDeclHandler(XML_Parser parser,
@@ -659,8 +610,7 @@ XML_SetExternalEntityRefHandler(XML_Parser parser,
    instead of the parser object.
 */
 XMLPARSEAPI(void)
-XML_SetExternalEntityRefHandlerArg(XML_Parser parser,
-                                   void *arg);
+XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg);
 
 XMLPARSEAPI(void)
 XML_SetSkippedEntityHandler(XML_Parser parser,
@@ -740,7 +690,6 @@ XML_UseParserAsHandlerArg(XML_Parser parser);
 XMLPARSEAPI(enum XML_Error)
 XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD);
 
-
 /* Sets the base to be used for resolving relative URIs in system
    identifiers in declarations.  Resolving relative identifiers is
    left to the application: this value will be passed through as the
@@ -780,10 +729,10 @@ XML_GetIdAttributeIndex(XML_Parser parser);
    info->valueEnd - info->valueStart = 4 bytes.
 */
 typedef struct {
-  XML_Index  nameStart;  /* Offset to beginning of the attribute name. */
-  XML_Index  nameEnd;    /* Offset after the attribute name's last byte. */
-  XML_Index  valueStart; /* Offset to beginning of the attribute value. */
-  XML_Index  valueEnd;   /* Offset after the attribute value's last byte. */
+  XML_Index nameStart;  /* Offset to beginning of the attribute name. */
+  XML_Index nameEnd;    /* Offset after the attribute name's last byte. */
+  XML_Index valueStart; /* Offset to beginning of the attribute value. */
+  XML_Index valueEnd;   /* Offset after the attribute value's last byte. */
 } XML_AttrInfo;
 
 /* Returns an array of XML_AttrInfo structures for the attribute/value pairs
@@ -819,20 +768,20 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal);
    (resumable = 0) an already suspended parser. Some call-backs may
    still follow because they would otherwise get lost. Examples:
    - endElementHandler() for empty elements when stopped in
-     startElementHandler(), 
-   - endNameSpaceDeclHandler() when stopped in endElementHandler(), 
+     startElementHandler(),
+   - endNameSpaceDeclHandler() when stopped in endElementHandler(),
    and possibly others.
 
    Can be called from most handlers, including DTD related call-backs,
    except when parsing an external parameter entity and resumable != 0.
    Returns XML_STATUS_OK when successful, XML_STATUS_ERROR otherwise.
-   Possible error codes: 
+   Possible error codes:
    - XML_ERROR_SUSPENDED: when suspending an already suspended parser.
    - XML_ERROR_FINISHED: when the parser has already finished.
    - XML_ERROR_SUSPEND_PE: when suspending while parsing an external PE.
 
-   When resumable != 0 (true) then parsing is suspended, that is, 
-   XML_Parse() and XML_ParseBuffer() return XML_STATUS_SUSPENDED. 
+   When resumable != 0 (true) then parsing is suspended, that is,
+   XML_Parse() and XML_ParseBuffer() return XML_STATUS_SUSPENDED.
    Otherwise, parsing is aborted, that is, XML_Parse() and XML_ParseBuffer()
    return XML_STATUS_ERROR with error code XML_ERROR_ABORTED.
 
@@ -843,7 +792,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal);
    the externalEntityRefHandler() to call XML_StopParser() on the parent
    parser (recursively), if one wants to stop parsing altogether.
 
-   When suspended, parsing can be resumed by calling XML_ResumeParser(). 
+   When suspended, parsing can be resumed by calling XML_ResumeParser().
 */
 XMLPARSEAPI(enum XML_Status)
 XML_StopParser(XML_Parser parser, XML_Bool resumable);
@@ -851,7 +800,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable);
 /* Resumes parsing after it has been suspended with XML_StopParser().
    Must not be called from within a handler call-back. Returns same
    status codes as XML_Parse() or XML_ParseBuffer().
-   Additional error code XML_ERROR_NOT_SUSPENDED possible.   
+   Additional error code XML_ERROR_NOT_SUSPENDED possible.
 
    *Note*:
    This must be called on the most deeply nested child parser instance
@@ -863,12 +812,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable);
 XMLPARSEAPI(enum XML_Status)
 XML_ResumeParser(XML_Parser parser);
 
-enum XML_Parsing {
-  XML_INITIALIZED,
-  XML_PARSING,
-  XML_FINISHED,
-  XML_SUSPENDED
-};
+enum XML_Parsing { XML_INITIALIZED, XML_PARSING, XML_FINISHED, XML_SUSPENDED };
 
 typedef struct {
   enum XML_Parsing parsing;
@@ -900,8 +844,7 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status);
    Otherwise returns a new XML_Parser object.
 */
 XMLPARSEAPI(XML_Parser)
-XML_ExternalEntityParserCreate(XML_Parser parser,
-                               const XML_Char *context,
+XML_ExternalEntityParserCreate(XML_Parser parser, const XML_Char *context,
                                const XML_Char *encoding);
 
 enum XML_ParamEntityParsing {
@@ -945,8 +888,7 @@ XML_SetParamEntityParsing(XML_Parser parser,
    Note: If parser == NULL, the function will do nothing and return 0.
 */
 XMLPARSEAPI(int)
-XML_SetHashSalt(XML_Parser parser,
-                unsigned long hash_salt);
+XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt);
 
 /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then
    XML_GetErrorCode returns information about the error.
@@ -963,7 +905,7 @@ XML_GetErrorCode(XML_Parser parser);
    be within the relevant markup.  When called outside of the callback
    functions, the position indicated will be just past the last parse
    event (regardless of whether there was an associated callback).
-   
+
    They may also be called after returning from a call to XML_Parse
    or XML_ParseBuffer.  If the return value is XML_STATUS_ERROR then
    the location is the location of the character at which the error
@@ -995,14 +937,12 @@ XML_GetCurrentByteCount(XML_Parser parser);
    the handler that makes the call.
 */
 XMLPARSEAPI(const char *)
-XML_GetInputContext(XML_Parser parser,
-                    int *offset,
-                    int *size);
+XML_GetInputContext(XML_Parser parser, int *offset, int *size);
 
 /* For backwards compatibility with previous versions. */
-#define XML_GetErrorLineNumber   XML_GetCurrentLineNumber
+#define XML_GetErrorLineNumber XML_GetCurrentLineNumber
 #define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber
-#define XML_GetErrorByteIndex    XML_GetCurrentByteIndex
+#define XML_GetErrorByteIndex XML_GetCurrentByteIndex
 
 /* Frees the content model passed to the element declaration handler */
 XMLPARSEAPI(void)
@@ -1062,21 +1002,20 @@ enum XML_FeatureEnum {
 };
 
 typedef struct {
-  enum XML_FeatureEnum  feature;
-  const XML_LChar       *name;
-  long int              value;
+  enum XML_FeatureEnum feature;
+  const XML_LChar *name;
+  long int value;
 } XML_Feature;
 
 XMLPARSEAPI(const XML_Feature *)
 XML_GetFeatureList(void);
 
-
 /* Expat follows the semantic versioning convention.
    See http://semver.org.
 */
 #define XML_MAJOR_VERSION 2
 #define XML_MINOR_VERSION 2
-#define XML_MICRO_VERSION 7
+#define XML_MICRO_VERSION 8
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h
index fc63c2228d3e..f2b75dda8e27 100644
--- a/Modules/expat/expat_external.h
+++ b/Modules/expat/expat_external.h
@@ -35,10 +35,6 @@
 
 /* External API definitions */
 
-#if defined(_MSC_EXTENSIONS) && !defined(__BEOS__) && !defined(__CYGWIN__)
-# define XML_USE_MSC_EXTENSIONS 1
-#endif
-
 /* Expat tries very hard to make the API boundary very specifically
    defined.  There are two macros defined to control this boundary;
    each of these can be defined before including this header to
@@ -61,12 +57,17 @@
    compiled with the cdecl calling convention as the default since
    system headers may assume the cdecl convention.
 */
+
+/* Namespace external symbols to allow multiple libexpat version to
+   co-exist. */
+#include "pyexpatns.h"
+
 #ifndef XMLCALL
-# if defined(_MSC_VER)
-#  define XMLCALL __cdecl
-# elif defined(__GNUC__) && defined(__i386) && !defined(__INTEL_COMPILER)
-#  define XMLCALL __attribute__((cdecl))
-# else
+#  if defined(_MSC_VER)
+#    define XMLCALL __cdecl
+#  elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER)
+#    define XMLCALL __attribute__((cdecl))
+#  else
 /* For any platform which uses this definition and supports more than
    one calling convention, we need to extend this definition to
    declare the convention used on that platform, if it's possible to
@@ -77,49 +78,46 @@
    pre-processor and how to specify the same calling convention as the
    platform's malloc() implementation.
 */
-#  define XMLCALL
-# endif
-#endif  /* not defined XMLCALL */
-
-/* Namespace external symbols to allow multiple libexpat version to
-   co-exist. */
-#include "pyexpatns.h"
-
+#    define XMLCALL
+#  endif
+#endif /* not defined XMLCALL */
 
-#if !defined(XML_STATIC) && !defined(XMLIMPORT)
-# ifndef XML_BUILDING_EXPAT
+#if ! defined(XML_STATIC) && ! defined(XMLIMPORT)
+#  ifndef XML_BUILDING_EXPAT
 /* using Expat from an application */
 
-#  ifdef XML_USE_MSC_EXTENSIONS
-#   define XMLIMPORT __declspec(dllimport)
-#  endif
+#    if defined(_MSC_EXTENSIONS) && ! defined(__BEOS__) && ! defined(__CYGWIN__)
+#      define XMLIMPORT __declspec(dllimport)
+#    endif
 
-# endif
-#endif  /* not defined XML_STATIC */
+#  endif
+#endif /* not defined XML_STATIC */
 
 #ifndef XML_ENABLE_VISIBILITY
-# define XML_ENABLE_VISIBILITY 0
+#  define XML_ENABLE_VISIBILITY 0
 #endif
 
-#if !defined(XMLIMPORT) && XML_ENABLE_VISIBILITY
-# define XMLIMPORT __attribute__ ((visibility ("default")))
+#if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY
+#  define XMLIMPORT __attribute__((visibility("default")))
 #endif
 
 /* If we didn't define it above, define it away: */
 #ifndef XMLIMPORT
-# define XMLIMPORT
+#  define XMLIMPORT
 #endif
 
-#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
-# define XML_ATTR_MALLOC __attribute__((__malloc__))
+#if defined(__GNUC__)                                                          \
+    && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
+#  define XML_ATTR_MALLOC __attribute__((__malloc__))
 #else
-# define XML_ATTR_MALLOC
+#  define XML_ATTR_MALLOC
 #endif
 
-#if defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
-# define XML_ATTR_ALLOC_SIZE(x)  __attribute__((__alloc_size__(x)))
+#if defined(__GNUC__)                                                          \
+    && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
+#  define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x)))
 #else
-# define XML_ATTR_ALLOC_SIZE(x)
+#  define XML_ATTR_ALLOC_SIZE(x)
 #endif
 
 #define XMLPARSEAPI(type) XMLIMPORT type XMLCALL
@@ -129,35 +127,30 @@ extern "C" {
 #endif
 
 #ifdef XML_UNICODE_WCHAR_T
-# ifndef XML_UNICODE
-#  define XML_UNICODE
-# endif
-# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
-#  error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
-# endif
+#  ifndef XML_UNICODE
+#    define XML_UNICODE
+#  endif
+#  if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
+#    error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
+#  endif
 #endif
 
-#ifdef XML_UNICODE     /* Information is UTF-16 encoded. */
-# ifdef XML_UNICODE_WCHAR_T
+#ifdef XML_UNICODE /* Information is UTF-16 encoded. */
+#  ifdef XML_UNICODE_WCHAR_T
 typedef wchar_t XML_Char;
 typedef wchar_t XML_LChar;
-# else
+#  else
 typedef unsigned short XML_Char;
 typedef char XML_LChar;
-# endif /* XML_UNICODE_WCHAR_T */
-#else                  /* Information is UTF-8 encoded. */
+#  endif /* XML_UNICODE_WCHAR_T */
+#else    /* Information is UTF-8 encoded. */
 typedef char XML_Char;
 typedef char XML_LChar;
-#endif /* XML_UNICODE */
+#endif   /* XML_UNICODE */
 
-#ifdef XML_LARGE_SIZE  /* Use large integers for file/stream positions. */
-# if defined(XML_USE_MSC_EXTENSIONS) && _MSC_VER < 1400
-typedef __int64 XML_Index; 
-typedef unsigned __int64 XML_Size;
-# else
+#ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */
 typedef long long XML_Index;
 typedef unsigned long long XML_Size;
-# endif
 #else
 typedef long XML_Index;
 typedef unsigned long XML_Size;
diff --git a/Modules/expat/iasciitab.h b/Modules/expat/iasciitab.h
index ce4a4bf7edef..ea97cfcf678e 100644
--- a/Modules/expat/iasciitab.h
+++ b/Modules/expat/iasciitab.h
@@ -32,34 +32,34 @@
 
 /* Like asciitab.h, except that 0xD has code BT_S rather than BT_CR */
 /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
-/* 0x0C */ BT_NONXML, BT_S, BT_NONXML, BT_NONXML,
-/* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
-/* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
-/* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
-/* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
-/* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
-/* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
-/* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
-/* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
-/* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
+    /* 0x0C */ BT_NONXML, BT_S, BT_NONXML, BT_NONXML,
+    /* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
+    /* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
+    /* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
+    /* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
+    /* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
+    /* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
+    /* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
+    /* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
+    /* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h
index dc4ef0c7e258..60913dab762f 100644
--- a/Modules/expat/internal.h
+++ b/Modules/expat/internal.h
@@ -49,7 +49,7 @@
    USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
-#if defined(__GNUC__) && defined(__i386__) && !defined(__MINGW32__)
+#if defined(__GNUC__) && defined(__i386__) && ! defined(__MINGW32__)
 /* We'll use this version by default only where we know it helps.
 
    regparm() generates warnings on Solaris boxes.   See SF bug #692878.
@@ -59,8 +59,8 @@
    #define FASTCALL __attribute__((stdcall, regparm(3)))
    and let's try this:
 */
-#define FASTCALL __attribute__((regparm(3)))
-#define PTRFASTCALL __attribute__((regparm(3)))
+#  define FASTCALL __attribute__((regparm(3)))
+#  define PTRFASTCALL __attribute__((regparm(3)))
 #endif
 
 /* Using __fastcall seems to have an unexpected negative effect under
@@ -74,55 +74,49 @@
 /* Make sure all of these are defined if they aren't already. */
 
 #ifndef FASTCALL
-#define FASTCALL
+#  define FASTCALL
 #endif
 
 #ifndef PTRCALL
-#define PTRCALL
+#  define PTRCALL
 #endif
 
 #ifndef PTRFASTCALL
-#define PTRFASTCALL
+#  define PTRFASTCALL
 #endif
 
 #ifndef XML_MIN_SIZE
-#if !defined(__cplusplus) && !defined(inline)
-#ifdef __GNUC__
-#define inline __inline
-#endif /* __GNUC__ */
-#endif
+#  if ! defined(__cplusplus) && ! defined(inline)
+#    ifdef __GNUC__
+#      define inline __inline
+#    endif /* __GNUC__ */
+#  endif
 #endif /* XML_MIN_SIZE */
 
 #ifdef __cplusplus
-#define inline inline
+#  define inline inline
 #else
-#ifndef inline
-#define inline
-#endif
+#  ifndef inline
+#    define inline
+#  endif
 #endif
 
 #ifndef UNUSED_P
-# ifdef __GNUC__
-#  define UNUSED_P(p) UNUSED_ ## p __attribute__((__unused__))
-# else
-#  define UNUSED_P(p) UNUSED_ ## p
-# endif
+#  define UNUSED_P(p) (void)p
 #endif
 
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-
 #ifdef XML_ENABLE_VISIBILITY
-#if XML_ENABLE_VISIBILITY
-__attribute__ ((visibility ("default")))
-#endif
+#  if XML_ENABLE_VISIBILITY
+__attribute__((visibility("default")))
+#  endif
 #endif
 void
-_INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** fromLimRef);
-
+_INTERNAL_trim_to_complete_utf8_characters(const char *from,
+                                           const char **fromLimRef);
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/latin1tab.h b/Modules/expat/latin1tab.h
index 95dfa52b1fbe..6f916041355a 100644
--- a/Modules/expat/latin1tab.h
+++ b/Modules/expat/latin1tab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x80 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x84 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x88 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x8C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x90 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x94 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x98 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x9C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA4 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
-/* 0xAC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xB0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xB4 */ BT_OTHER, BT_NMSTRT, BT_OTHER, BT_NAME,
-/* 0xB8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
-/* 0xBC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xC0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xC4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xC8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xCC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xD0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xD4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0xD8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xDC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xEC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xF0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xF4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0xF8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xFC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x84 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x88 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x8C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x90 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x94 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x98 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x9C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA4 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
+    /* 0xAC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xB0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xB4 */ BT_OTHER, BT_NMSTRT, BT_OTHER, BT_NAME,
+    /* 0xB8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
+    /* 0xBC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xC0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xC4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xC8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xCC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xD0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xD4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0xD8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xDC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xEC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xF0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xF4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0xF8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xFC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
diff --git a/Modules/expat/loadlibrary.c b/Modules/expat/loadlibrary.c
deleted file mode 100644
index 2f80b3a379fd..000000000000
--- a/Modules/expat/loadlibrary.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/***************************************************************************
- *                                  _   _ ____  _
- *  Project                     ___| | | |  _ \| |
- *                             / __| | | | |_) | |
- *                            | (__| |_| |  _ <| |___
- *                             \___|\___/|_| \_\_____|
- *
- * Copyright (C) 2016 - 2017, Steve Holme, <steve_holme at hotmail.com>.
- * Copyright (C) 2017, Expat development team
- *
- * All rights reserved.
- * Licensed under the MIT license:
- *
- * Permission to  use, copy,  modify, and distribute  this software  for any
- * purpose with  or without fee is  hereby granted, provided that  the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE  SOFTWARE  IS  PROVIDED  "AS  IS",  WITHOUT  WARRANTY  OF  ANY  KIND,
- * EXPRESS  OR IMPLIED,  INCLUDING  BUT  NOT LIMITED  TO  THE WARRANTIES  OF
- * MERCHANTABILITY, FITNESS FOR A  PARTICULAR PURPOSE AND NONINFRINGEMENT OF
- * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
- * CONTRACT, TORT OR  OTHERWISE, ARISING FROM, OUT OF OR  IN CONNECTION WITH
- * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * Except as contained in this notice,  the name of a copyright holder shall
- * not be used in advertising or otherwise to promote the sale, use or other
- * dealings  in this  Software without  prior written  authorization of  the
- * copyright holder.
- *
- ***************************************************************************/
-
-#if defined(_WIN32)
-
-#include <windows.h>
-#include <tchar.h>
-
-
-HMODULE _Expat_LoadLibrary(LPCTSTR filename);
-
-
-#if !defined(LOAD_WITH_ALTERED_SEARCH_PATH)
-#define LOAD_WITH_ALTERED_SEARCH_PATH  0x00000008
-#endif
-
-#if !defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
-#define LOAD_LIBRARY_SEARCH_SYSTEM32   0x00000800
-#endif
-
-/* We use our own typedef here since some headers might lack these */
-typedef HMODULE (APIENTRY *LOADLIBRARYEX_FN)(LPCTSTR, HANDLE, DWORD);
-
-/* See function definitions in winbase.h */
-#ifdef UNICODE
-#  ifdef _WIN32_WCE
-#    define LOADLIBRARYEX  L"LoadLibraryExW"
-#  else
-#    define LOADLIBRARYEX  "LoadLibraryExW"
-#  endif
-#else
-#  define LOADLIBRARYEX    "LoadLibraryExA"
-#endif
-
-
-/*
- * _Expat_LoadLibrary()
- *
- * This is used to dynamically load DLLs using the most secure method available
- * for the version of Windows that we are running on.
- *
- * Parameters:
- *
- * filename  [in] - The filename or full path of the DLL to load. If only the
- *                  filename is passed then the DLL will be loaded from the
- *                  Windows system directory.
- *
- * Returns the handle of the module on success; otherwise NULL.
- */
-HMODULE _Expat_LoadLibrary(LPCTSTR filename)
-{
-  HMODULE hModule = NULL;
-  LOADLIBRARYEX_FN pLoadLibraryEx = NULL;
-
-  /* Get a handle to kernel32 so we can access it's functions at runtime */
-  HMODULE hKernel32 = GetModuleHandle(TEXT("kernel32"));
-  if(!hKernel32)
-    return NULL;  /* LCOV_EXCL_LINE */
-
-  /* Attempt to find LoadLibraryEx() which is only available on Windows 2000
-     and above */
-  pLoadLibraryEx = (LOADLIBRARYEX_FN) GetProcAddress(hKernel32, LOADLIBRARYEX);
-
-  /* Detect if there's already a path in the filename and load the library if
-     there is. Note: Both back slashes and forward slashes have been supported
-     since the earlier days of DOS at an API level although they are not
-     supported by command prompt */
-  if(_tcspbrk(filename, TEXT("\\/"))) {
-    /** !checksrc! disable BANNEDFUNC 1 **/
-    hModule = pLoadLibraryEx ?
-      pLoadLibraryEx(filename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) :
-      LoadLibrary(filename);
-  }
-  /* Detect if KB2533623 is installed, as LOAD_LIBARY_SEARCH_SYSTEM32 is only
-     supported on Windows Vista, Windows Server 2008, Windows 7 and Windows
-     Server 2008 R2 with this patch or natively on Windows 8 and above */
-  else if(pLoadLibraryEx && GetProcAddress(hKernel32, "AddDllDirectory")) {
-    /* Load the DLL from the Windows system directory */
-    hModule = pLoadLibraryEx(filename, NULL, LOAD_LIBRARY_SEARCH_SYSTEM32);
-  }
-  else {
-    /* Attempt to get the Windows system path */
-    UINT systemdirlen = GetSystemDirectory(NULL, 0);
-    if(systemdirlen) {
-      /* Allocate space for the full DLL path (Room for the null terminator
-         is included in systemdirlen) */
-      size_t filenamelen = _tcslen(filename);
-      TCHAR *path = malloc(sizeof(TCHAR) * (systemdirlen + 1 + filenamelen));
-      if(path && GetSystemDirectory(path, systemdirlen)) {
-        /* Calculate the full DLL path */
-        _tcscpy(path + _tcslen(path), TEXT("\\"));
-        _tcscpy(path + _tcslen(path), filename);
-
-        /* Load the DLL from the Windows system directory */
-        /** !checksrc! disable BANNEDFUNC 1 **/
-        hModule = pLoadLibraryEx ?
-          pLoadLibraryEx(path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) :
-          LoadLibrary(path);
-
-      }
-      free(path);
-    }
-  }
-
-  return hModule;
-}
-
-#else /* defined(_WIN32) */
-
-/* ISO C requires a translation unit to contain at least one declaration
-   [-Wempty-translation-unit] */
-typedef int _TRANSLATION_UNIT_LOAD_LIBRARY_C_NOT_EMTPY;
-
-#endif /* defined(_WIN32) */
diff --git a/Modules/expat/nametab.h b/Modules/expat/nametab.h
index bfa2bd38cd95..3681df348eeb 100644
--- a/Modules/expat/nametab.h
+++ b/Modules/expat/nametab.h
@@ -31,152 +31,106 @@
 */
 
 static const unsigned namingBitmap[] = {
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0x00000000, 0x04000000, 0x87FFFFFE, 0x07FFFFFE,
-0x00000000, 0x00000000, 0xFF7FFFFF, 0xFF7FFFFF,
-0xFFFFFFFF, 0x7FF3FFFF, 0xFFFFFDFE, 0x7FFFFFFF,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFE00F, 0xFC31FFFF,
-0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
-0xFFFFFFFF, 0xF80001FF, 0x00000003, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFD740, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
-0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
-0xFFFF0003, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF,
-0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
-0x0000007F, 0x00000000, 0xFFFF0000, 0x000707FF,
-0x00000000, 0x07FFFFFE, 0x000007FE, 0xFFFE0000,
-0xFFFFFFFF, 0x7CFFFFFF, 0x002F7FFF, 0x00000060,
-0xFFFFFFE0, 0x23FFFFFF, 0xFF000000, 0x00000003,
-0xFFF99FE0, 0x03C5FDFF, 0xB0000000, 0x00030003,
-0xFFF987E0, 0x036DFDFF, 0x5E000000, 0x001C0000,
-0xFFFBAFE0, 0x23EDFDFF, 0x00000000, 0x00000001,
-0xFFF99FE0, 0x23CDFDFF, 0xB0000000, 0x00000003,
-0xD63DC7E0, 0x03BFC718, 0x00000000, 0x00000000,
-0xFFFDDFE0, 0x03EFFDFF, 0x00000000, 0x00000003,
-0xFFFDDFE0, 0x03EFFDFF, 0x40000000, 0x00000003,
-0xFFFDDFE0, 0x03FFFDFF, 0x00000000, 0x00000003,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFE, 0x000D7FFF, 0x0000003F, 0x00000000,
-0xFEF02596, 0x200D6CAE, 0x0000001F, 0x00000000,
-0x00000000, 0x00000000, 0xFFFFFEFF, 0x000003FF,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0xFFFFFFFF, 0xFFFF003F, 0x007FFFFF,
-0x0007DAED, 0x50000000, 0x82315001, 0x002C62AB,
-0x40000000, 0xF580C900, 0x00000007, 0x02010800,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0x0FFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x03FFFFFF,
-0x3F3FFFFF, 0xFFFFFFFF, 0xAAFF3F3F, 0x3FFFFFFF,
-0xFFFFFFFF, 0x5FDFFFFF, 0x0FCF1FDC, 0x1FDC1FFF,
-0x00000000, 0x00004C40, 0x00000000, 0x00000000,
-0x00000007, 0x00000000, 0x00000000, 0x00000000,
-0x00000080, 0x000003FE, 0xFFFFFFFE, 0xFFFFFFFF,
-0x001FFFFF, 0xFFFFFFFE, 0xFFFFFFFF, 0x07FFFFFF,
-0xFFFFFFE0, 0x00001FFF, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0x0000003F, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0x0000000F, 0x00000000, 0x00000000,
-0x00000000, 0x07FF6000, 0x87FFFFFE, 0x07FFFFFE,
-0x00000000, 0x00800000, 0xFF7FFFFF, 0xFF7FFFFF,
-0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
-0xFFFFFFFF, 0xF80001FF, 0x00030003, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000003,
-0xFFFFD7C0, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
-0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
-0xFFFF007B, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF,
-0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
-0xFFFE007F, 0xBBFFFFFB, 0xFFFF0016, 0x000707FF,
-0x00000000, 0x07FFFFFE, 0x0007FFFF, 0xFFFF03FF,
-0xFFFFFFFF, 0x7CFFFFFF, 0xFFEF7FFF, 0x03FF3DFF,
-0xFFFFFFEE, 0xF3FFFFFF, 0xFF1E3FFF, 0x0000FFCF,
-0xFFF99FEE, 0xD3C5FDFF, 0xB080399F, 0x0003FFCF,
-0xFFF987E4, 0xD36DFDFF, 0x5E003987, 0x001FFFC0,
-0xFFFBAFEE, 0xF3EDFDFF, 0x00003BBF, 0x0000FFC1,
-0xFFF99FEE, 0xF3CDFDFF, 0xB0C0398F, 0x0000FFC3,
-0xD63DC7EC, 0xC3BFC718, 0x00803DC7, 0x0000FF80,
-0xFFFDDFEE, 0xC3EFFDFF, 0x00603DDF, 0x0000FFC3,
-0xFFFDDFEC, 0xC3EFFDFF, 0x40603DDF, 0x0000FFC3,
-0xFFFDDFEC, 0xC3FFFDFF, 0x00803DCF, 0x0000FFC3,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFE, 0x07FF7FFF, 0x03FF7FFF, 0x00000000,
-0xFEF02596, 0x3BFF6CAE, 0x03FF3F5F, 0x00000000,
-0x03000000, 0xC2A003FF, 0xFFFFFEFF, 0xFFFE03FF,
-0xFEBF0FDF, 0x02FE3FFF, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x1FFF0000, 0x00000002,
-0x000000A0, 0x003EFFFE, 0xFFFFFFFE, 0xFFFFFFFF,
-0x661FFFFF, 0xFFFFFFFE, 0xFFFFFFFF, 0x77FFFFFF,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x04000000,
+    0x87FFFFFE, 0x07FFFFFE, 0x00000000, 0x00000000, 0xFF7FFFFF, 0xFF7FFFFF,
+    0xFFFFFFFF, 0x7FF3FFFF, 0xFFFFFDFE, 0x7FFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFE00F, 0xFC31FFFF, 0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xF80001FF, 0x00000003, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFD740, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
+    0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF, 0xFFFF0003, 0xFFFFFFFF,
+    0xFFFF199F, 0x033FCFFF, 0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
+    0x0000007F, 0x00000000, 0xFFFF0000, 0x000707FF, 0x00000000, 0x07FFFFFE,
+    0x000007FE, 0xFFFE0000, 0xFFFFFFFF, 0x7CFFFFFF, 0x002F7FFF, 0x00000060,
+    0xFFFFFFE0, 0x23FFFFFF, 0xFF000000, 0x00000003, 0xFFF99FE0, 0x03C5FDFF,
+    0xB0000000, 0x00030003, 0xFFF987E0, 0x036DFDFF, 0x5E000000, 0x001C0000,
+    0xFFFBAFE0, 0x23EDFDFF, 0x00000000, 0x00000001, 0xFFF99FE0, 0x23CDFDFF,
+    0xB0000000, 0x00000003, 0xD63DC7E0, 0x03BFC718, 0x00000000, 0x00000000,
+    0xFFFDDFE0, 0x03EFFDFF, 0x00000000, 0x00000003, 0xFFFDDFE0, 0x03EFFDFF,
+    0x40000000, 0x00000003, 0xFFFDDFE0, 0x03FFFDFF, 0x00000000, 0x00000003,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFE, 0x000D7FFF,
+    0x0000003F, 0x00000000, 0xFEF02596, 0x200D6CAE, 0x0000001F, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFFEFF, 0x000003FF, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0xFFFFFFFF, 0xFFFF003F, 0x007FFFFF, 0x0007DAED, 0x50000000,
+    0x82315001, 0x002C62AB, 0x40000000, 0xF580C900, 0x00000007, 0x02010800,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0FFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0x03FFFFFF, 0x3F3FFFFF, 0xFFFFFFFF, 0xAAFF3F3F, 0x3FFFFFFF,
+    0xFFFFFFFF, 0x5FDFFFFF, 0x0FCF1FDC, 0x1FDC1FFF, 0x00000000, 0x00004C40,
+    0x00000000, 0x00000000, 0x00000007, 0x00000000, 0x00000000, 0x00000000,
+    0x00000080, 0x000003FE, 0xFFFFFFFE, 0xFFFFFFFF, 0x001FFFFF, 0xFFFFFFFE,
+    0xFFFFFFFF, 0x07FFFFFF, 0xFFFFFFE0, 0x00001FFF, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000000, 0x00000000,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000000F,
+    0x00000000, 0x00000000, 0x00000000, 0x07FF6000, 0x87FFFFFE, 0x07FFFFFE,
+    0x00000000, 0x00800000, 0xFF7FFFFF, 0xFF7FFFFF, 0x00FFFFFF, 0x00000000,
+    0xFFFF0000, 0xFFFFFFFF, 0xFFFFFFFF, 0xF80001FF, 0x00030003, 0x00000000,
+    0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000003, 0xFFFFD7C0, 0xFFFFFFFB,
+    0x547F7FFF, 0x000FFFFD, 0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
+    0xFFFF007B, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF, 0x00000000, 0xFFFE0000,
+    0x027FFFFF, 0xFFFFFFFE, 0xFFFE007F, 0xBBFFFFFB, 0xFFFF0016, 0x000707FF,
+    0x00000000, 0x07FFFFFE, 0x0007FFFF, 0xFFFF03FF, 0xFFFFFFFF, 0x7CFFFFFF,
+    0xFFEF7FFF, 0x03FF3DFF, 0xFFFFFFEE, 0xF3FFFFFF, 0xFF1E3FFF, 0x0000FFCF,
+    0xFFF99FEE, 0xD3C5FDFF, 0xB080399F, 0x0003FFCF, 0xFFF987E4, 0xD36DFDFF,
+    0x5E003987, 0x001FFFC0, 0xFFFBAFEE, 0xF3EDFDFF, 0x00003BBF, 0x0000FFC1,
+    0xFFF99FEE, 0xF3CDFDFF, 0xB0C0398F, 0x0000FFC3, 0xD63DC7EC, 0xC3BFC718,
+    0x00803DC7, 0x0000FF80, 0xFFFDDFEE, 0xC3EFFDFF, 0x00603DDF, 0x0000FFC3,
+    0xFFFDDFEC, 0xC3EFFDFF, 0x40603DDF, 0x0000FFC3, 0xFFFDDFEC, 0xC3FFFDFF,
+    0x00803DCF, 0x0000FFC3, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0xFFFFFFFE, 0x07FF7FFF, 0x03FF7FFF, 0x00000000, 0xFEF02596, 0x3BFF6CAE,
+    0x03FF3F5F, 0x00000000, 0x03000000, 0xC2A003FF, 0xFFFFFEFF, 0xFFFE03FF,
+    0xFEBF0FDF, 0x02FE3FFF, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x1FFF0000, 0x00000002,
+    0x000000A0, 0x003EFFFE, 0xFFFFFFFE, 0xFFFFFFFF, 0x661FFFFF, 0xFFFFFFFE,
+    0xFFFFFFFF, 0x77FFFFFF,
 };
 static const unsigned char nmstrtPages[] = {
-0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00,
-0x00, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
-0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13,
-0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x15, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x17,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x09, 0x0A, 0x0B,
+    0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13, 0x00, 0x14, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x15, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00,
 };
 static const unsigned char namePages[] = {
-0x19, 0x03, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x00,
-0x00, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25,
-0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13,
-0x26, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x27, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x17,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x19, 0x03, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x00, 0x00, 0x1F, 0x20, 0x21,
+    0x22, 0x23, 0x24, 0x25, 0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13, 0x26, 0x14, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x27, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00,
 };
diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h
index 4d6786d7839a..bfee65a332f1 100644
--- a/Modules/expat/siphash.h
+++ b/Modules/expat/siphash.h
@@ -11,6 +11,10 @@
  * --------------------------------------------------------------------------
  * HISTORY:
  *
+ * 2019-08-03  (Sebastian Pipping)
+ *   - Mark part of sip24_valid as to be excluded from clang-format
+ *   - Re-format code using clang-format 9
+ *
  * 2018-07-08  (Anton Maklakov)
  *   - Add "fall through" markers for GCC's -Wimplicit-fallthrough
  *
@@ -94,186 +98,186 @@
 #include <stddef.h> /* size_t */
 
 #if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER < 1600)
-  /* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */
-  typedef unsigned __int8   uint8_t;
-  typedef unsigned __int32 uint32_t;
-  typedef unsigned __int64 uint64_t;
+/* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */
+typedef unsigned __int8 uint8_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
 #else
- #include <stdint.h> /* uint64_t uint32_t uint8_t */
+#  include <stdint.h> /* uint64_t uint32_t uint8_t */
 #endif
 
-
 /*
  * Workaround to not require a C++11 compiler for using ULL suffix
  * if this code is included and compiled as C++; related GCC warning is:
  * warning: use of C++11 long long integer constant [-Wlong-long]
  */
-#define _SIP_ULL(high, low)  (((uint64_t)high << 32) | low)
-
-
-#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b))))
+#define _SIP_ULL(high, low) (((uint64_t)high << 32) | low)
 
-#define SIP_U32TO8_LE(p, v) \
-	(p)[0] = (uint8_t)((v) >>  0); (p)[1] = (uint8_t)((v) >>  8); \
-	(p)[2] = (uint8_t)((v) >> 16); (p)[3] = (uint8_t)((v) >> 24);
+#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
 
-#define SIP_U64TO8_LE(p, v) \
-	SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >>  0)); \
-	SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
+#define SIP_U32TO8_LE(p, v)                                                    \
+  (p)[0] = (uint8_t)((v) >> 0);                                                \
+  (p)[1] = (uint8_t)((v) >> 8);                                                \
+  (p)[2] = (uint8_t)((v) >> 16);                                               \
+  (p)[3] = (uint8_t)((v) >> 24);
 
-#define SIP_U8TO64_LE(p) \
-	(((uint64_t)((p)[0]) <<  0) | \
-	 ((uint64_t)((p)[1]) <<  8) | \
-	 ((uint64_t)((p)[2]) << 16) | \
-	 ((uint64_t)((p)[3]) << 24) | \
-	 ((uint64_t)((p)[4]) << 32) | \
-	 ((uint64_t)((p)[5]) << 40) | \
-	 ((uint64_t)((p)[6]) << 48) | \
-	 ((uint64_t)((p)[7]) << 56))
+#define SIP_U64TO8_LE(p, v)                                                    \
+  SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >> 0));                                \
+  SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
 
+#define SIP_U8TO64_LE(p)                                                       \
+  (((uint64_t)((p)[0]) << 0) | ((uint64_t)((p)[1]) << 8)                       \
+   | ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24)                   \
+   | ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40)                   \
+   | ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))
 
-#define SIPHASH_INITIALIZER { 0, 0, 0, 0, { 0 }, 0, 0 }
+#define SIPHASH_INITIALIZER                                                    \
+  { 0, 0, 0, 0, {0}, 0, 0 }
 
 struct siphash {
-	uint64_t v0, v1, v2, v3;
+  uint64_t v0, v1, v2, v3;
 
-	unsigned char buf[8], *p;
-	uint64_t c;
+  unsigned char buf[8], *p;
+  uint64_t c;
 }; /* struct siphash */
 
-
 #define SIP_KEYLEN 16
 
 struct sipkey {
-	uint64_t k[2];
+  uint64_t k[2];
 }; /* struct sipkey */
 
-#define sip_keyof(k) sip_tokey(&(struct sipkey){ { 0 } }, (k))
+#define sip_keyof(k) sip_tokey(&(struct sipkey){{0}}, (k))
 
-static struct sipkey *sip_tokey(struct sipkey *key, const void *src) {
-	key->k[0] = SIP_U8TO64_LE((const unsigned char *)src);
-	key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8);
-	return key;
+static struct sipkey *
+sip_tokey(struct sipkey *key, const void *src) {
+  key->k[0] = SIP_U8TO64_LE((const unsigned char *)src);
+  key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8);
+  return key;
 } /* sip_tokey() */
 
-
 #ifdef SIPHASH_TOBIN
 
-#define sip_binof(v) sip_tobin((unsigned char[8]){ 0 }, (v))
+#  define sip_binof(v) sip_tobin((unsigned char[8]){0}, (v))
 
-static void *sip_tobin(void *dst, uint64_t u64) {
-	SIP_U64TO8_LE((unsigned char *)dst, u64);
-	return dst;
+static void *
+sip_tobin(void *dst, uint64_t u64) {
+  SIP_U64TO8_LE((unsigned char *)dst, u64);
+  return dst;
 } /* sip_tobin() */
 
-#endif  /* SIPHASH_TOBIN */
-
+#endif /* SIPHASH_TOBIN */
 
-static void sip_round(struct siphash *H, const int rounds) {
-	int i;
+static void
+sip_round(struct siphash *H, const int rounds) {
+  int i;
 
-	for (i = 0; i < rounds; i++) {
-		H->v0 += H->v1;
-		H->v1 = SIP_ROTL(H->v1, 13);
-		H->v1 ^= H->v0;
-		H->v0 = SIP_ROTL(H->v0, 32);
+  for (i = 0; i < rounds; i++) {
+    H->v0 += H->v1;
+    H->v1 = SIP_ROTL(H->v1, 13);
+    H->v1 ^= H->v0;
+    H->v0 = SIP_ROTL(H->v0, 32);
 
-		H->v2 += H->v3;
-		H->v3 = SIP_ROTL(H->v3, 16);
-		H->v3 ^= H->v2;
+    H->v2 += H->v3;
+    H->v3 = SIP_ROTL(H->v3, 16);
+    H->v3 ^= H->v2;
 
-		H->v0 += H->v3;
-		H->v3 = SIP_ROTL(H->v3, 21);
-		H->v3 ^= H->v0;
+    H->v0 += H->v3;
+    H->v3 = SIP_ROTL(H->v3, 21);
+    H->v3 ^= H->v0;
 
-		H->v2 += H->v1;
-		H->v1 = SIP_ROTL(H->v1, 17);
-		H->v1 ^= H->v2;
-		H->v2 = SIP_ROTL(H->v2, 32);
-	}
+    H->v2 += H->v1;
+    H->v1 = SIP_ROTL(H->v1, 17);
+    H->v1 ^= H->v2;
+    H->v2 = SIP_ROTL(H->v2, 32);
+  }
 } /* sip_round() */
 
+static struct siphash *
+sip24_init(struct siphash *H, const struct sipkey *key) {
+  H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0];
+  H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1];
+  H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0];
+  H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1];
 
-static struct siphash *sip24_init(struct siphash *H,
-		const struct sipkey *key) {
-	H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0];
-	H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1];
-	H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0];
-	H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1];
+  H->p = H->buf;
+  H->c = 0;
 
-	H->p = H->buf;
-	H->c = 0;
-
-	return H;
+  return H;
 } /* sip24_init() */
 
+#define sip_endof(a) (&(a)[sizeof(a) / sizeof *(a)])
 
-#define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)])
-
-static struct siphash *sip24_update(struct siphash *H, const void *src,
-		size_t len) {
-	const unsigned char *p = (const unsigned char *)src, *pe = p + len;
-	uint64_t m;
+static struct siphash *
+sip24_update(struct siphash *H, const void *src, size_t len) {
+  const unsigned char *p = (const unsigned char *)src, *pe = p + len;
+  uint64_t m;
 
-	do {
-		while (p < pe && H->p < sip_endof(H->buf))
-			*H->p++ = *p++;
+  do {
+    while (p < pe && H->p < sip_endof(H->buf))
+      *H->p++ = *p++;
 
-		if (H->p < sip_endof(H->buf))
-			break;
+    if (H->p < sip_endof(H->buf))
+      break;
 
-		m = SIP_U8TO64_LE(H->buf);
-		H->v3 ^= m;
-		sip_round(H, 2);
-		H->v0 ^= m;
+    m = SIP_U8TO64_LE(H->buf);
+    H->v3 ^= m;
+    sip_round(H, 2);
+    H->v0 ^= m;
 
-		H->p = H->buf;
-		H->c += 8;
-	} while (p < pe);
+    H->p = H->buf;
+    H->c += 8;
+  } while (p < pe);
 
-	return H;
+  return H;
 } /* sip24_update() */
 
-
-static uint64_t sip24_final(struct siphash *H) {
-	const char left = (char)(H->p - H->buf);
-	uint64_t b = (H->c + left) << 56;
-
-	switch (left) {
-	case 7: b |= (uint64_t)H->buf[6] << 48;
-		/* fall through */
-	case 6: b |= (uint64_t)H->buf[5] << 40;
-		/* fall through */
-	case 5: b |= (uint64_t)H->buf[4] << 32;
-		/* fall through */
-	case 4: b |= (uint64_t)H->buf[3] << 24;
-		/* fall through */
-	case 3: b |= (uint64_t)H->buf[2] << 16;
-		/* fall through */
-	case 2: b |= (uint64_t)H->buf[1] << 8;
-		/* fall through */
-	case 1: b |= (uint64_t)H->buf[0] << 0;
-		/* fall through */
-	case 0: break;
-	}
-
-	H->v3 ^= b;
-	sip_round(H, 2);
-	H->v0 ^= b;
-	H->v2 ^= 0xff;
-	sip_round(H, 4);
-
-	return H->v0 ^ H->v1 ^ H->v2  ^ H->v3;
+static uint64_t
+sip24_final(struct siphash *H) {
+  const char left = (char)(H->p - H->buf);
+  uint64_t b = (H->c + left) << 56;
+
+  switch (left) {
+  case 7:
+    b |= (uint64_t)H->buf[6] << 48;
+    /* fall through */
+  case 6:
+    b |= (uint64_t)H->buf[5] << 40;
+    /* fall through */
+  case 5:
+    b |= (uint64_t)H->buf[4] << 32;
+    /* fall through */
+  case 4:
+    b |= (uint64_t)H->buf[3] << 24;
+    /* fall through */
+  case 3:
+    b |= (uint64_t)H->buf[2] << 16;
+    /* fall through */
+  case 2:
+    b |= (uint64_t)H->buf[1] << 8;
+    /* fall through */
+  case 1:
+    b |= (uint64_t)H->buf[0] << 0;
+    /* fall through */
+  case 0:
+    break;
+  }
+
+  H->v3 ^= b;
+  sip_round(H, 2);
+  H->v0 ^= b;
+  H->v2 ^= 0xff;
+  sip_round(H, 4);
+
+  return H->v0 ^ H->v1 ^ H->v2 ^ H->v3;
 } /* sip24_final() */
 
-
-static uint64_t siphash24(const void *src, size_t len,
-		const struct sipkey *key) {
-	struct siphash state = SIPHASH_INITIALIZER;
-	return sip24_final(sip24_update(sip24_init(&state, key), src, len));
+static uint64_t
+siphash24(const void *src, size_t len, const struct sipkey *key) {
+  struct siphash state = SIPHASH_INITIALIZER;
+  return sip24_final(sip24_update(sip24_init(&state, key), src, len));
 } /* siphash24() */
 
-
 /*
  * SipHash-2-4 output with
  * k = 00 01 02 ...
@@ -285,107 +289,110 @@ static uint64_t siphash24(const void *src, size_t len,
  * ...
  * in = 00 01 02 ... 3e (63 bytes)
  */
-static int sip24_valid(void) {
-	static const unsigned char vectors[64][8] = {
-		{ 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, },
-		{ 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, },
-		{ 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, },
-		{ 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, },
-		{ 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, },
-		{ 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, },
-		{ 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, },
-		{ 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, },
-		{ 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, },
-		{ 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, },
-		{ 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, },
-		{ 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, },
-		{ 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, },
-		{ 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, },
-		{ 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, },
-		{ 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, },
-		{ 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, },
-		{ 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, },
-		{ 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, },
-		{ 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, },
-		{ 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, },
-		{ 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, },
-		{ 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, },
-		{ 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, },
-		{ 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, },
-		{ 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, },
-		{ 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, },
-		{ 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, },
-		{ 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, },
-		{ 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, },
-		{ 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, },
-		{ 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, },
-		{ 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, },
-		{ 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, },
-		{ 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, },
-		{ 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, },
-		{ 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, },
-		{ 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, },
-		{ 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, },
-		{ 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, },
-		{ 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, },
-		{ 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, },
-		{ 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, },
-		{ 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, },
-		{ 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, },
-		{ 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, },
-		{ 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, },
-		{ 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, },
-		{ 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, },
-		{ 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, },
-		{ 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, },
-		{ 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, },
-		{ 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, },
-		{ 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, },
-		{ 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, },
-		{ 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, },
-		{ 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, },
-		{ 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, },
-		{ 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, },
-		{ 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, },
-		{ 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, },
-		{ 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, },
-		{ 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, },
-		{ 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, }
-	};
-	unsigned char in[64];
-	struct sipkey k;
-	size_t i;
-
-	sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011"
-			"\012\013\014\015\016\017");
-
-	for (i = 0; i < sizeof in; ++i) {
-		in[i] = (unsigned char)i;
-
-		if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i]))
-			return 0;
-	}
-
-	return 1;
+static int
+sip24_valid(void) {
+  /* clang-format off */
+  static const unsigned char vectors[64][8] = {
+    { 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, },
+    { 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, },
+    { 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, },
+    { 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, },
+    { 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, },
+    { 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, },
+    { 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, },
+    { 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, },
+    { 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, },
+    { 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, },
+    { 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, },
+    { 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, },
+    { 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, },
+    { 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, },
+    { 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, },
+    { 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, },
+    { 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, },
+    { 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, },
+    { 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, },
+    { 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, },
+    { 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, },
+    { 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, },
+    { 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, },
+    { 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, },
+    { 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, },
+    { 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, },
+    { 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, },
+    { 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, },
+    { 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, },
+    { 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, },
+    { 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, },
+    { 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, },
+    { 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, },
+    { 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, },
+    { 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, },
+    { 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, },
+    { 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, },
+    { 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, },
+    { 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, },
+    { 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, },
+    { 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, },
+    { 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, },
+    { 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, },
+    { 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, },
+    { 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, },
+    { 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, },
+    { 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, },
+    { 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, },
+    { 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, },
+    { 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, },
+    { 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, },
+    { 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, },
+    { 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, },
+    { 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, },
+    { 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, },
+    { 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, },
+    { 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, },
+    { 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, },
+    { 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, },
+    { 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, },
+    { 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, },
+    { 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, },
+    { 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, },
+    { 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, }
+  };
+  /* clang-format on */
+
+  unsigned char in[64];
+  struct sipkey k;
+  size_t i;
+
+  sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011"
+                "\012\013\014\015\016\017");
+
+  for (i = 0; i < sizeof in; ++i) {
+    in[i] = (unsigned char)i;
+
+    if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i]))
+      return 0;
+  }
+
+  return 1;
 } /* sip24_valid() */
 
-
 #ifdef SIPHASH_MAIN
 
-#include <stdio.h>
+#  include <stdio.h>
 
-int main(void) {
-	const int ok = sip24_valid();
+int
+main(void) {
+  const int ok = sip24_valid();
 
-	if (ok)
-		puts("OK");
-	else
-		puts("FAIL");
+  if (ok)
+    puts("OK");
+  else
+    puts("FAIL");
 
-	return !ok;
+  return ! ok;
 } /* main() */
 
 #endif /* SIPHASH_MAIN */
 
-
 #endif /* SIPHASH_H */
diff --git a/Modules/expat/utf8tab.h b/Modules/expat/utf8tab.h
index fa0bed6f5d75..a22986acbb95 100644
--- a/Modules/expat/utf8tab.h
+++ b/Modules/expat/utf8tab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x80 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x84 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x88 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x8C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x90 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x94 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x98 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x9C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xAC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xBC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xC0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xC4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xC8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xCC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xDC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xE0 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xE4 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xE8 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xEC */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xF0 */ BT_LEAD4, BT_LEAD4, BT_LEAD4, BT_LEAD4,
-/* 0xF4 */ BT_LEAD4, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0xF8 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0xFC */ BT_NONXML, BT_NONXML, BT_MALFORM, BT_MALFORM,
+    /* 0x84 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x88 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x8C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x90 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x94 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x98 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x9C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xAC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xBC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xC0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xC4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xC8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xCC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xDC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xE0 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xE4 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xE8 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xEC */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xF0 */ BT_LEAD4, BT_LEAD4, BT_LEAD4, BT_LEAD4,
+    /* 0xF4 */ BT_LEAD4, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0xF8 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0xFC */ BT_NONXML, BT_NONXML, BT_MALFORM, BT_MALFORM,
diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h
index 28a043c6044c..562a4a82dc1d 100644
--- a/Modules/expat/winconfig.h
+++ b/Modules/expat/winconfig.h
@@ -40,20 +40,17 @@
 #include <memory.h>
 #include <string.h>
 
+#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */
+#  include <expat_config.h>
+#else /* !defined(HAVE_EXPAT_CONFIG_H) */
 
-#if defined(HAVE_EXPAT_CONFIG_H)  /* e.g. MinGW */
-# include <expat_config.h>
-#else  /* !defined(HAVE_EXPAT_CONFIG_H) */
-
-
-#define XML_NS 1
-#define XML_DTD 1
-#define XML_CONTEXT_BYTES 1024
+#  define XML_NS 1
+#  define XML_DTD 1
+#  define XML_CONTEXT_BYTES 1024
 
 /* we will assume all Windows platforms are little endian */
-#define BYTEORDER 1234
+#  define BYTEORDER 1234
 
 #endif /* !defined(HAVE_EXPAT_CONFIG_H) */
 
-
 #endif /* ndef WINCONFIG_H */
diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c
index 311dbc4dfb9c..e740f0e19c7d 100644
--- a/Modules/expat/xmlparse.c
+++ b/Modules/expat/xmlparse.c
@@ -1,4 +1,4 @@
-/* 69df5be70289a11fb834869ce4a91c23c1d9dd04baffcbd10e86742d149a080c (2.2.7+)
+/* f2d0ab6d1d4422a08cf1cf3bbdfba96b49dea42fb5ff4615e03a2a25c306e769 (2.2.8+)
                             __  __            _
                          ___\ \/ /_ __   __ _| |_
                         / _ \\  /| '_ \ / _` | __|
@@ -30,33 +30,38 @@
    USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
-#if !defined(_GNU_SOURCE)
-# define _GNU_SOURCE 1                  /* syscall prototype */
+#if ! defined(_GNU_SOURCE)
+#  define _GNU_SOURCE 1 /* syscall prototype */
+#endif
+
+#ifdef _WIN32
+/* force stdlib to define rand_s() */
+#  define _CRT_RAND_S
 #endif
 
 #include <stddef.h>
-#include <string.h>                     /* memset(), memcpy() */
+#include <string.h> /* memset(), memcpy() */
 #include <assert.h>
-#include <limits.h>                     /* UINT_MAX */
-#include <stdio.h>                      /* fprintf */
-#include <stdlib.h>                     /* getenv */
+#include <limits.h> /* UINT_MAX */
+#include <stdio.h>  /* fprintf */
+#include <stdlib.h> /* getenv, rand_s */
 
 #ifdef _WIN32
-#define getpid GetCurrentProcessId
+#  define getpid GetCurrentProcessId
 #else
-#include <sys/time.h>                   /* gettimeofday() */
-#include <sys/types.h>                  /* getpid() */
-#include <unistd.h>                     /* getpid() */
-#include <fcntl.h>                      /* O_RDONLY */
-#include <errno.h>
+#  include <sys/time.h>  /* gettimeofday() */
+#  include <sys/types.h> /* getpid() */
+#  include <unistd.h>    /* getpid() */
+#  include <fcntl.h>     /* O_RDONLY */
+#  include <errno.h>
 #endif
 
 #define XML_BUILDING_EXPAT 1
 
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #elif defined(HAVE_EXPAT_CONFIG_H)
-#include <expat_config.h>
+#  include <expat_config.h>
 #endif /* ndef _WIN32 */
 
 #include "ascii.h"
@@ -64,33 +69,31 @@
 #include "siphash.h"
 
 #if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
-# if defined(HAVE_GETRANDOM)
-#  include <sys/random.h>    /* getrandom */
-# else
-#  include <unistd.h>        /* syscall */
-#  include <sys/syscall.h>   /* SYS_getrandom */
-# endif
-# if ! defined(GRND_NONBLOCK)
-#  define GRND_NONBLOCK  0x0001
-# endif  /* defined(GRND_NONBLOCK) */
-#endif  /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
-
-#if defined(HAVE_LIBBSD) \
+#  if defined(HAVE_GETRANDOM)
+#    include <sys/random.h> /* getrandom */
+#  else
+#    include <unistd.h>      /* syscall */
+#    include <sys/syscall.h> /* SYS_getrandom */
+#  endif
+#  if ! defined(GRND_NONBLOCK)
+#    define GRND_NONBLOCK 0x0001
+#  endif /* defined(GRND_NONBLOCK) */
+#endif   /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
+
+#if defined(HAVE_LIBBSD)                                                       \
     && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM))
-# include <bsd/stdlib.h>
+#  include <bsd/stdlib.h>
 #endif
 
-#if defined(_WIN32) && !defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
-# define LOAD_LIBRARY_SEARCH_SYSTEM32  0x00000800
+#if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
+#  define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800
 #endif
 
-#if !defined(HAVE_GETRANDOM) && !defined(HAVE_SYSCALL_GETRANDOM) \
-    && !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_ARC4RANDOM) \
-    && !defined(XML_DEV_URANDOM) \
-    && !defined(_WIN32) \
-    && !defined(XML_POOR_ENTROPY)
-# error  \
-    You do not have support for any sources of high quality entropy \
+#if ! defined(HAVE_GETRANDOM) && ! defined(HAVE_SYSCALL_GETRANDOM)             \
+    && ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)            \
+    && ! defined(XML_DEV_URANDOM) && ! defined(_WIN32)                         \
+    && ! defined(XML_POOR_ENTROPY)
+#  error You do not have support for any sources of high quality entropy \
     enabled.  For end user security, that is probably not what you want. \
     \
     Your options include: \
@@ -101,7 +104,7 @@
       * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \
       * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \
       * Linux / BSD / macOS (/dev/urandom): XML_DEV_URANDOM \
-      * Windows (RtlGenRandom): _WIN32. \
+      * Windows (rand_s): _WIN32. \
     \
     If insist on not using any of these, bypass this error by defining \
     XML_POOR_ENTROPY; you have been warned. \
@@ -110,56 +113,55 @@
     to the build system, please open a bug.  Thank you!
 #endif
 
-
 #ifdef XML_UNICODE
-#define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX
-#define XmlConvert XmlUtf16Convert
-#define XmlGetInternalEncoding XmlGetUtf16InternalEncoding
-#define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS
-#define XmlEncode XmlUtf16Encode
+#  define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX
+#  define XmlConvert XmlUtf16Convert
+#  define XmlGetInternalEncoding XmlGetUtf16InternalEncoding
+#  define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS
+#  define XmlEncode XmlUtf16Encode
 /* Using pointer subtraction to convert to integer type. */
-#define MUST_CONVERT(enc, s) (!(enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1))
+#  define MUST_CONVERT(enc, s)                                                 \
+    (! (enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1))
 typedef unsigned short ICHAR;
 #else
-#define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX
-#define XmlConvert XmlUtf8Convert
-#define XmlGetInternalEncoding XmlGetUtf8InternalEncoding
-#define XmlGetInternalEncodingNS XmlGetUtf8InternalEncodingNS
-#define XmlEncode XmlUtf8Encode
-#define MUST_CONVERT(enc, s) (!(enc)->isUtf8)
+#  define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX
+#  define XmlConvert XmlUtf8Convert
+#  define XmlGetInternalEncoding XmlGetUtf8InternalEncoding
+#  define XmlGetInternalEncodingNS XmlGetUtf8InternalEncodingNS
+#  define XmlEncode XmlUtf8Encode
+#  define MUST_CONVERT(enc, s) (! (enc)->isUtf8)
 typedef char ICHAR;
 #endif
 
-
 #ifndef XML_NS
 
-#define XmlInitEncodingNS XmlInitEncoding
-#define XmlInitUnknownEncodingNS XmlInitUnknownEncoding
-#undef XmlGetInternalEncodingNS
-#define XmlGetInternalEncodingNS XmlGetInternalEncoding
-#define XmlParseXmlDeclNS XmlParseXmlDecl
+#  define XmlInitEncodingNS XmlInitEncoding
+#  define XmlInitUnknownEncodingNS XmlInitUnknownEncoding
+#  undef XmlGetInternalEncodingNS
+#  define XmlGetInternalEncodingNS XmlGetInternalEncoding
+#  define XmlParseXmlDeclNS XmlParseXmlDecl
 
 #endif
 
 #ifdef XML_UNICODE
 
-#ifdef XML_UNICODE_WCHAR_T
-#define XML_T(x) (const wchar_t)x
-#define XML_L(x) L ## x
-#else
-#define XML_T(x) (const unsigned short)x
-#define XML_L(x) x
-#endif
+#  ifdef XML_UNICODE_WCHAR_T
+#    define XML_T(x) (const wchar_t) x
+#    define XML_L(x) L##x
+#  else
+#    define XML_T(x) (const unsigned short)x
+#    define XML_L(x) x
+#  endif
 
 #else
 
-#define XML_T(x) x
-#define XML_L(x) x
+#  define XML_T(x) x
+#  define XML_L(x) x
 
 #endif
 
 /* Round up n to be a multiple of sz, where sz is a power of 2. */
-#define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1))
+#define ROUND_UP(n, sz) (((n) + ((sz)-1)) & ~((sz)-1))
 
 /* Do safe (NULL-aware) pointer arithmetic */
 #define EXPAT_SAFE_PTR_DIFF(p, q) (((p) && (q)) ? ((p) - (q)) : 0)
@@ -182,11 +184,9 @@ typedef struct {
   const XML_Memory_Handling_Suite *mem;
 } HASH_TABLE;
 
-static size_t
-keylen(KEY s);
+static size_t keylen(KEY s);
 
-static void
-copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key);
+static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key);
 
 /* For probing (after a collision) we need a step size relative prime
    to the hash table size, which is a power of 2. We use double-hashing,
@@ -196,9 +196,9 @@ copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key);
    We limit the maximum step size to table->size / 4 (mask >> 2) and make
    it odd, since odd numbers are always relative prime to a power of 2.
 */
-#define SECOND_HASH(hash, mask, power) \
-  ((((hash) & ~(mask)) >> ((power) - 1)) & ((mask) >> 2))
-#define PROBE_STEP(hash, mask, power) \
+#define SECOND_HASH(hash, mask, power)                                         \
+  ((((hash) & ~(mask)) >> ((power)-1)) & ((mask) >> 2))
+#define PROBE_STEP(hash, mask, power)                                          \
   ((unsigned char)((SECOND_HASH(hash, mask, power)) | 1))
 
 typedef struct {
@@ -206,7 +206,7 @@ typedef struct {
   NAMED **end;
 } HASH_TABLE_ITER;
 
-#define INIT_TAG_BUF_SIZE 32  /* must be a multiple of sizeof(XML_Char) */
+#define INIT_TAG_BUF_SIZE 32 /* must be a multiple of sizeof(XML_Char) */
 #define INIT_DATA_BUF_SIZE 1024
 #define INIT_ATTS_SIZE 16
 #define INIT_ATTS_VERSION 0xFFFFFFFF
@@ -253,20 +253,20 @@ typedef struct {
    TAG objects in a free list.
 */
 typedef struct tag {
-  struct tag *parent;           /* parent of this element */
-  const char *rawName;          /* tagName in the original encoding */
+  struct tag *parent;  /* parent of this element */
+  const char *rawName; /* tagName in the original encoding */
   int rawNameLength;
-  TAG_NAME name;                /* tagName in the API encoding */
-  char *buf;                    /* buffer for name components */
-  char *bufEnd;                 /* end of the buffer */
+  TAG_NAME name; /* tagName in the API encoding */
+  char *buf;     /* buffer for name components */
+  char *bufEnd;  /* end of the buffer */
   BINDING *bindings;
 } TAG;
 
 typedef struct {
   const XML_Char *name;
   const XML_Char *textPtr;
-  int textLen;                  /* length in XML_Chars */
-  int processed;                /* # of processed bytes - when suspended */
+  int textLen;   /* length in XML_Chars */
+  int processed; /* # of processed bytes - when suspended */
   const XML_Char *systemId;
   const XML_Char *base;
   const XML_Char *publicId;
@@ -277,13 +277,13 @@ typedef struct {
 } ENTITY;
 
 typedef struct {
-  enum XML_Content_Type         type;
-  enum XML_Content_Quant        quant;
-  const XML_Char *              name;
-  int                           firstchild;
-  int                           lastchild;
-  int                           childcnt;
-  int                           nextsib;
+  enum XML_Content_Type type;
+  enum XML_Content_Quant quant;
+  const XML_Char *name;
+  int firstchild;
+  int lastchild;
+  int childcnt;
+  int nextsib;
 } CONTENT_SCAFFOLD;
 
 #define INIT_SCAFFOLD_ELEMENTS 32
@@ -371,10 +371,8 @@ typedef struct open_internal_entity {
   XML_Bool betweenDecl; /* WFC: PE Between Declarations */
 } OPEN_INTERNAL_ENTITY;
 
-typedef enum XML_Error PTRCALL Processor(XML_Parser parser,
-                                         const char *start,
-                                         const char *end,
-                                         const char **endPtr);
+typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start,
+                                         const char *end, const char **endPtr);
 
 static Processor prologProcessor;
 static Processor prologInitProcessor;
@@ -395,118 +393,101 @@ static Processor externalEntityInitProcessor3;
 static Processor externalEntityContentProcessor;
 static Processor internalEntityProcessor;
 
-static enum XML_Error
-handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName);
-static enum XML_Error
-processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
-               const char *s, const char *next);
-static enum XML_Error
-initializeEncoding(XML_Parser parser);
-static enum XML_Error
-doProlog(XML_Parser parser, const ENCODING *enc, const char *s,
-         const char *end, int tok, const char *next, const char **nextPtr,
-         XML_Bool haveMore);
-static enum XML_Error
-processInternalEntity(XML_Parser parser, ENTITY *entity,
-                      XML_Bool betweenDecl);
-static enum XML_Error
-doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
-          const char *start, const char *end, const char **endPtr,
-          XML_Bool haveMore);
-static enum XML_Error
-doCdataSection(XML_Parser parser, const ENCODING *, const char **startPtr,
-               const char *end, const char **nextPtr, XML_Bool haveMore);
+static enum XML_Error handleUnknownEncoding(XML_Parser parser,
+                                            const XML_Char *encodingName);
+static enum XML_Error processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
+                                     const char *s, const char *next);
+static enum XML_Error initializeEncoding(XML_Parser parser);
+static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc,
+                               const char *s, const char *end, int tok,
+                               const char *next, const char **nextPtr,
+                               XML_Bool haveMore, XML_Bool allowClosingDoctype);
+static enum XML_Error processInternalEntity(XML_Parser parser, ENTITY *entity,
+                                            XML_Bool betweenDecl);
+static enum XML_Error doContent(XML_Parser parser, int startTagLevel,
+                                const ENCODING *enc, const char *start,
+                                const char *end, const char **endPtr,
+                                XML_Bool haveMore);
+static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *,
+                                     const char **startPtr, const char *end,
+                                     const char **nextPtr, XML_Bool haveMore);
 #ifdef XML_DTD
-static enum XML_Error
-doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr,
-                const char *end, const char **nextPtr, XML_Bool haveMore);
+static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *,
+                                      const char **startPtr, const char *end,
+                                      const char **nextPtr, XML_Bool haveMore);
 #endif /* XML_DTD */
 
-static void
-freeBindings(XML_Parser parser, BINDING *bindings);
-static enum XML_Error
-storeAtts(XML_Parser parser, const ENCODING *, const char *s,
-          TAG_NAME *tagNamePtr, BINDING **bindingsPtr);
-static enum XML_Error
-addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
-           const XML_Char *uri, BINDING **bindingsPtr);
-static int
-defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata,
-                XML_Bool isId, const XML_Char *dfltValue, XML_Parser parser);
-static enum XML_Error
-storeAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata,
-                    const char *, const char *, STRING_POOL *);
-static enum XML_Error
-appendAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata,
-                     const char *, const char *, STRING_POOL *);
-static ATTRIBUTE_ID *
-getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start,
-               const char *end);
-static int
-setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *);
-static enum XML_Error
-storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *start,
-                 const char *end);
-static int
-reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
-                            const char *start, const char *end);
-static int
-reportComment(XML_Parser parser, const ENCODING *enc, const char *start,
-              const char *end);
-static void
-reportDefault(XML_Parser parser, const ENCODING *enc, const char *start,
-              const char *end);
-
-static const XML_Char * getContext(XML_Parser parser);
-static XML_Bool
-setContext(XML_Parser parser, const XML_Char *context);
+static void freeBindings(XML_Parser parser, BINDING *bindings);
+static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *,
+                                const char *s, TAG_NAME *tagNamePtr,
+                                BINDING **bindingsPtr);
+static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix,
+                                 const ATTRIBUTE_ID *attId, const XML_Char *uri,
+                                 BINDING **bindingsPtr);
+static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata,
+                           XML_Bool isId, const XML_Char *dfltValue,
+                           XML_Parser parser);
+static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *,
+                                          XML_Bool isCdata, const char *,
+                                          const char *, STRING_POOL *);
+static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *,
+                                           XML_Bool isCdata, const char *,
+                                           const char *, STRING_POOL *);
+static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc,
+                                    const char *start, const char *end);
+static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *);
+static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc,
+                                       const char *start, const char *end);
+static int reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
+                                       const char *start, const char *end);
+static int reportComment(XML_Parser parser, const ENCODING *enc,
+                         const char *start, const char *end);
+static void reportDefault(XML_Parser parser, const ENCODING *enc,
+                          const char *start, const char *end);
+
+static const XML_Char *getContext(XML_Parser parser);
+static XML_Bool setContext(XML_Parser parser, const XML_Char *context);
 
 static void FASTCALL normalizePublicId(XML_Char *s);
 
-static DTD * dtdCreate(const XML_Memory_Handling_Suite *ms);
+static DTD *dtdCreate(const XML_Memory_Handling_Suite *ms);
 /* do not call if m_parentParser != NULL */
 static void dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms);
-static void
-dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms);
-static int
-dtdCopy(XML_Parser oldParser,
-        DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms);
-static int
-copyEntityTable(XML_Parser oldParser,
-                HASH_TABLE *, STRING_POOL *, const HASH_TABLE *);
-static NAMED *
-lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize);
-static void FASTCALL
-hashTableInit(HASH_TABLE *, const XML_Memory_Handling_Suite *ms);
+static void dtdDestroy(DTD *p, XML_Bool isDocEntity,
+                       const XML_Memory_Handling_Suite *ms);
+static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
+                   const XML_Memory_Handling_Suite *ms);
+static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *, STRING_POOL *,
+                           const HASH_TABLE *);
+static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name,
+                     size_t createSize);
+static void FASTCALL hashTableInit(HASH_TABLE *,
+                                   const XML_Memory_Handling_Suite *ms);
 static void FASTCALL hashTableClear(HASH_TABLE *);
 static void FASTCALL hashTableDestroy(HASH_TABLE *);
-static void FASTCALL
-hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *);
-static NAMED * FASTCALL hashTableIterNext(HASH_TABLE_ITER *);
+static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *);
+static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *);
 
-static void FASTCALL
-poolInit(STRING_POOL *, const XML_Memory_Handling_Suite *ms);
+static void FASTCALL poolInit(STRING_POOL *,
+                              const XML_Memory_Handling_Suite *ms);
 static void FASTCALL poolClear(STRING_POOL *);
 static void FASTCALL poolDestroy(STRING_POOL *);
-static XML_Char *
-poolAppend(STRING_POOL *pool, const ENCODING *enc,
-           const char *ptr, const char *end);
-static XML_Char *
-poolStoreString(STRING_POOL *pool, const ENCODING *enc,
-                const char *ptr, const char *end);
+static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc,
+                            const char *ptr, const char *end);
+static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc,
+                                 const char *ptr, const char *end);
 static XML_Bool FASTCALL poolGrow(STRING_POOL *pool);
-static const XML_Char * FASTCALL
-poolCopyString(STRING_POOL *pool, const XML_Char *s);
-static const XML_Char *
-poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n);
-static const XML_Char * FASTCALL
-poolAppendString(STRING_POOL *pool, const XML_Char *s);
+static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool,
+                                               const XML_Char *s);
+static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s,
+                                       int n);
+static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool,
+                                                 const XML_Char *s);
 
 static int FASTCALL nextScaffoldPart(XML_Parser parser);
-static XML_Content * build_model(XML_Parser parser);
-static ELEMENT_TYPE *
-getElementType(XML_Parser parser, const ENCODING *enc,
-               const char *ptr, const char *end);
+static XML_Content *build_model(XML_Parser parser);
+static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc,
+                                    const char *ptr, const char *end);
 
 static XML_Char *copyString(const XML_Char *s,
                             const XML_Memory_Handling_Suite *memsuite);
@@ -514,14 +495,11 @@ static XML_Char *copyString(const XML_Char *s,
 static unsigned long generate_hash_secret_salt(XML_Parser parser);
 static XML_Bool startParsing(XML_Parser parser);
 
-static XML_Parser
-parserCreate(const XML_Char *encodingName,
-             const XML_Memory_Handling_Suite *memsuite,
-             const XML_Char *nameSep,
-             DTD *dtd);
+static XML_Parser parserCreate(const XML_Char *encodingName,
+                               const XML_Memory_Handling_Suite *memsuite,
+                               const XML_Char *nameSep, DTD *dtd);
 
-static void
-parserInit(XML_Parser parser, const XML_Char *encodingName);
+static void parserInit(XML_Parser parser, const XML_Char *encodingName);
 
 #define poolStart(pool) ((pool)->start)
 #define poolEnd(pool) ((pool)->ptr)
@@ -530,10 +508,10 @@ parserInit(XML_Parser parser, const XML_Char *encodingName);
 #define poolLastChar(pool) (((pool)->ptr)[-1])
 #define poolDiscard(pool) ((pool)->ptr = (pool)->start)
 #define poolFinish(pool) ((pool)->start = (pool)->ptr)
-#define poolAppendChar(pool, c) \
-  (((pool)->ptr == (pool)->end && !poolGrow(pool)) \
-   ? 0 \
-   : ((*((pool)->ptr)++ = c), 1))
+#define poolAppendChar(pool, c)                                                \
+  (((pool)->ptr == (pool)->end && ! poolGrow(pool))                            \
+       ? 0                                                                     \
+       : ((*((pool)->ptr)++ = c), 1))
 
 struct XML_ParserStruct {
   /* The first member must be m_userData so that the XML_GetUserData
@@ -584,7 +562,7 @@ struct XML_ParserStruct {
   void *m_unknownEncodingMem;
   void *m_unknownEncodingData;
   void *m_unknownEncodingHandlerData;
-  void (XMLCALL *m_unknownEncodingRelease)(void *);
+  void(XMLCALL *m_unknownEncodingRelease)(void *);
   PROLOG_STATE m_prologState;
   Processor *m_processor;
   enum XML_Error m_errorCode;
@@ -638,57 +616,55 @@ struct XML_ParserStruct {
   unsigned long m_hash_secret_salt;
 };
 
-#define MALLOC(parser, s)      (parser->m_mem.malloc_fcn((s)))
-#define REALLOC(parser, p, s)  (parser->m_mem.realloc_fcn((p),(s)))
-#define FREE(parser, p)        (parser->m_mem.free_fcn((p)))
-
+#define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s)))
+#define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s)))
+#define FREE(parser, p) (parser->m_mem.free_fcn((p)))
 
 XML_Parser XMLCALL
-XML_ParserCreate(const XML_Char *encodingName)
-{
+XML_ParserCreate(const XML_Char *encodingName) {
   return XML_ParserCreate_MM(encodingName, NULL, NULL);
 }
 
 XML_Parser XMLCALL
-XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep)
-{
+XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) {
   XML_Char tmp[2];
   *tmp = nsSep;
   return XML_ParserCreate_MM(encodingName, NULL, tmp);
 }
 
-static const XML_Char implicitContext[] = {
-  ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h, ASCII_t, ASCII_t, ASCII_p,
-  ASCII_COLON, ASCII_SLASH, ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w,
-  ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g,
-  ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L, ASCII_SLASH, ASCII_1, ASCII_9,
-  ASCII_9, ASCII_8, ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e,
-  ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'
-};
-
+static const XML_Char implicitContext[]
+    = {ASCII_x,     ASCII_m,     ASCII_l,      ASCII_EQUALS, ASCII_h,
+       ASCII_t,     ASCII_t,     ASCII_p,      ASCII_COLON,  ASCII_SLASH,
+       ASCII_SLASH, ASCII_w,     ASCII_w,      ASCII_w,      ASCII_PERIOD,
+       ASCII_w,     ASCII_3,     ASCII_PERIOD, ASCII_o,      ASCII_r,
+       ASCII_g,     ASCII_SLASH, ASCII_X,      ASCII_M,      ASCII_L,
+       ASCII_SLASH, ASCII_1,     ASCII_9,      ASCII_9,      ASCII_8,
+       ASCII_SLASH, ASCII_n,     ASCII_a,      ASCII_m,      ASCII_e,
+       ASCII_s,     ASCII_p,     ASCII_a,      ASCII_c,      ASCII_e,
+       '\0'};
 
 /* To avoid warnings about unused functions: */
 #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)
 
-#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
+#  if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
 
 /* Obtain entropy on Linux 3.17+ */
 static int
-writeRandomBytes_getrandom_nonblock(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
+writeRandomBytes_getrandom_nonblock(void *target, size_t count) {
+  int success = 0; /* full count bytes written? */
   size_t bytesWrittenTotal = 0;
   const unsigned int getrandomFlags = GRND_NONBLOCK;
 
   do {
-    void * const currentTarget = (void*)((char*)target + bytesWrittenTotal);
+    void *const currentTarget = (void *)((char *)target + bytesWrittenTotal);
     const size_t bytesToWrite = count - bytesWrittenTotal;
 
     const int bytesWrittenMore =
-#if defined(HAVE_GETRANDOM)
+#    if defined(HAVE_GETRANDOM)
         getrandom(currentTarget, bytesToWrite, getrandomFlags);
-#else
+#    else
         syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags);
-#endif
+#    endif
 
     if (bytesWrittenMore > 0) {
       bytesWrittenTotal += bytesWrittenMore;
@@ -700,15 +676,14 @@ writeRandomBytes_getrandom_nonblock(void * target, size_t count) {
   return success;
 }
 
-#endif  /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
-
+#  endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
 
-#if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
+#  if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
 
 /* Extract entropy from /dev/urandom */
 static int
-writeRandomBytes_dev_urandom(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
+writeRandomBytes_dev_urandom(void *target, size_t count) {
+  int success = 0; /* full count bytes written? */
   size_t bytesWrittenTotal = 0;
 
   const int fd = open("/dev/urandom", O_RDONLY);
@@ -717,7 +692,7 @@ writeRandomBytes_dev_urandom(void * target, size_t count) {
   }
 
   do {
-    void * const currentTarget = (void*)((char*)target + bytesWrittenTotal);
+    void *const currentTarget = (void *)((char *)target + bytesWrittenTotal);
     const size_t bytesToWrite = count - bytesWrittenTotal;
 
     const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite);
@@ -733,15 +708,14 @@ writeRandomBytes_dev_urandom(void * target, size_t count) {
   return success;
 }
 
-#endif  /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
-
-#endif  /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
+#  endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
 
+#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
 
 #if defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF)
 
 static void
-writeRandomBytes_arc4random(void * target, size_t count) {
+writeRandomBytes_arc4random(void *target, size_t count) {
   size_t bytesWrittenTotal = 0;
 
   while (bytesWrittenTotal < count) {
@@ -749,93 +723,82 @@ writeRandomBytes_arc4random(void * target, size_t count) {
     size_t i = 0;
 
     for (; (i < sizeof(random32)) && (bytesWrittenTotal < count);
-        i++, bytesWrittenTotal++) {
+         i++, bytesWrittenTotal++) {
       const uint8_t random8 = (uint8_t)(random32 >> (i * 8));
       ((uint8_t *)target)[bytesWrittenTotal] = random8;
     }
   }
 }
 
-#endif  /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */
-
+#endif /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */
 
 #ifdef _WIN32
 
-typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG);
-HMODULE _Expat_LoadLibrary(LPCTSTR filename);  /* see loadlibrary.c */
-
-/* Obtain entropy on Windows XP / Windows Server 2003 and later.
- * Hint on RtlGenRandom and the following article from libsodium.
- *
- * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI
- * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographically-secure-random-number-on-windows-without-using-cryptoapi/
+/* Obtain entropy on Windows using the rand_s() function which
+ * generates cryptographically secure random numbers.  Internally it
+ * uses RtlGenRandom API which is present in Windows XP and later.
  */
 static int
-writeRandomBytes_RtlGenRandom(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
-  const HMODULE advapi32 = _Expat_LoadLibrary(TEXT("ADVAPI32.DLL"));
-
-  if (advapi32) {
-    const RTLGENRANDOM_FUNC RtlGenRandom
-        = (RTLGENRANDOM_FUNC)GetProcAddress(advapi32, "SystemFunction036");
-    if (RtlGenRandom) {
-      if (RtlGenRandom((PVOID)target, (ULONG)count) == TRUE) {
-        success = 1;
-      }
+writeRandomBytes_rand_s(void *target, size_t count) {
+  size_t bytesWrittenTotal = 0;
+
+  while (bytesWrittenTotal < count) {
+    unsigned int random32 = 0;
+    size_t i = 0;
+
+    if (rand_s(&random32))
+      return 0; /* failure */
+
+    for (; (i < sizeof(random32)) && (bytesWrittenTotal < count);
+         i++, bytesWrittenTotal++) {
+      const uint8_t random8 = (uint8_t)(random32 >> (i * 8));
+      ((uint8_t *)target)[bytesWrittenTotal] = random8;
     }
-    FreeLibrary(advapi32);
   }
-
-  return success;
+  return 1; /* success */
 }
 
 #endif /* _WIN32 */
 
-
 #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)
 
 static unsigned long
-gather_time_entropy(void)
-{
-#ifdef _WIN32
+gather_time_entropy(void) {
+#  ifdef _WIN32
   FILETIME ft;
   GetSystemTimeAsFileTime(&ft); /* never fails */
   return ft.dwHighDateTime ^ ft.dwLowDateTime;
-#else
+#  else
   struct timeval tv;
   int gettimeofday_res;
 
   gettimeofday_res = gettimeofday(&tv, NULL);
 
-#if defined(NDEBUG)
+#    if defined(NDEBUG)
   (void)gettimeofday_res;
-#else
-  assert (gettimeofday_res == 0);
-#endif  /* defined(NDEBUG) */
+#    else
+  assert(gettimeofday_res == 0);
+#    endif /* defined(NDEBUG) */
 
   /* Microseconds time is <20 bits entropy */
   return tv.tv_usec;
-#endif
+#  endif
 }
 
-#endif  /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
-
+#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
 
 static unsigned long
-ENTROPY_DEBUG(const char * label, unsigned long entropy) {
-  const char * const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG");
+ENTROPY_DEBUG(const char *label, unsigned long entropy) {
+  const char *const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG");
   if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) {
-    fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n",
-        label,
-        (int)sizeof(entropy) * 2, entropy,
-        (unsigned long)sizeof(entropy));
+    fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", label,
+            (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy));
   }
   return entropy;
 }
 
 static unsigned long
-generate_hash_secret_salt(XML_Parser parser)
-{
+generate_hash_secret_salt(XML_Parser parser) {
   unsigned long entropy;
   (void)parser;
 
@@ -848,20 +811,20 @@ generate_hash_secret_salt(XML_Parser parser)
   return ENTROPY_DEBUG("arc4random", entropy);
 #else
   /* Try high quality providers first .. */
-#ifdef _WIN32
-  if (writeRandomBytes_RtlGenRandom((void *)&entropy, sizeof(entropy))) {
-    return ENTROPY_DEBUG("RtlGenRandom", entropy);
+#  ifdef _WIN32
+  if (writeRandomBytes_rand_s((void *)&entropy, sizeof(entropy))) {
+    return ENTROPY_DEBUG("rand_s", entropy);
   }
-#elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
+#  elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
   if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) {
     return ENTROPY_DEBUG("getrandom", entropy);
   }
-#endif
-#if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
+#  endif
+#  if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
   if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) {
     return ENTROPY_DEBUG("/dev/urandom", entropy);
   }
-#endif  /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
+#  endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
   /* .. and self-made low quality for backup: */
 
   /* Process ID is 0 bits entropy if attacker has local access */
@@ -872,7 +835,7 @@ generate_hash_secret_salt(XML_Parser parser)
     return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647);
   } else {
     return ENTROPY_DEBUG("fallback(8)",
-        entropy * (unsigned long)2305843009213693951ULL);
+                         entropy * (unsigned long)2305843009213693951ULL);
   }
 #endif
 }
@@ -884,49 +847,43 @@ get_hash_secret_salt(XML_Parser parser) {
   return parser->m_hash_secret_salt;
 }
 
-static XML_Bool  /* only valid for root parser */
-startParsing(XML_Parser parser)
-{
-    /* hash functions must be initialized before setContext() is called */
-    if (parser->m_hash_secret_salt == 0)
-      parser->m_hash_secret_salt = generate_hash_secret_salt(parser);
-    if (parser->m_ns) {
-      /* implicit context only set for root parser, since child
-         parsers (i.e. external entity parsers) will inherit it
-      */
-      return setContext(parser, implicitContext);
-    }
-    return XML_TRUE;
+static XML_Bool /* only valid for root parser */
+startParsing(XML_Parser parser) {
+  /* hash functions must be initialized before setContext() is called */
+  if (parser->m_hash_secret_salt == 0)
+    parser->m_hash_secret_salt = generate_hash_secret_salt(parser);
+  if (parser->m_ns) {
+    /* implicit context only set for root parser, since child
+       parsers (i.e. external entity parsers) will inherit it
+    */
+    return setContext(parser, implicitContext);
+  }
+  return XML_TRUE;
 }
 
 XML_Parser XMLCALL
 XML_ParserCreate_MM(const XML_Char *encodingName,
                     const XML_Memory_Handling_Suite *memsuite,
-                    const XML_Char *nameSep)
-{
+                    const XML_Char *nameSep) {
   return parserCreate(encodingName, memsuite, nameSep, NULL);
 }
 
 static XML_Parser
 parserCreate(const XML_Char *encodingName,
-             const XML_Memory_Handling_Suite *memsuite,
-             const XML_Char *nameSep,
-             DTD *dtd)
-{
+             const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep,
+             DTD *dtd) {
   XML_Parser parser;
 
   if (memsuite) {
     XML_Memory_Handling_Suite *mtemp;
-    parser = (XML_Parser)
-      memsuite->malloc_fcn(sizeof(struct XML_ParserStruct));
+    parser = (XML_Parser)memsuite->malloc_fcn(sizeof(struct XML_ParserStruct));
     if (parser != NULL) {
       mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem);
       mtemp->malloc_fcn = memsuite->malloc_fcn;
       mtemp->realloc_fcn = memsuite->realloc_fcn;
       mtemp->free_fcn = memsuite->free_fcn;
     }
-  }
-  else {
+  } else {
     XML_Memory_Handling_Suite *mtemp;
     parser = (XML_Parser)malloc(sizeof(struct XML_ParserStruct));
     if (parser != NULL) {
@@ -937,27 +894,30 @@ parserCreate(const XML_Char *encodingName,
     }
   }
 
-  if (!parser)
+  if (! parser)
     return parser;
 
   parser->m_buffer = NULL;
   parser->m_bufferLim = NULL;
 
   parser->m_attsSize = INIT_ATTS_SIZE;
-  parser->m_atts = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE));
+  parser->m_atts
+      = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE));
   if (parser->m_atts == NULL) {
     FREE(parser, parser);
     return NULL;
   }
 #ifdef XML_ATTR_INFO
-  parser->m_attInfo = (XML_AttrInfo*)MALLOC(parser, parser->m_attsSize * sizeof(XML_AttrInfo));
+  parser->m_attInfo = (XML_AttrInfo *)MALLOC(
+      parser, parser->m_attsSize * sizeof(XML_AttrInfo));
   if (parser->m_attInfo == NULL) {
     FREE(parser, parser->m_atts);
     FREE(parser, parser);
     return NULL;
   }
 #endif
-  parser->m_dataBuf = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char));
+  parser->m_dataBuf
+      = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char));
   if (parser->m_dataBuf == NULL) {
     FREE(parser, parser->m_atts);
 #ifdef XML_ATTR_INFO
@@ -1007,7 +967,7 @@ parserCreate(const XML_Char *encodingName,
   poolInit(&parser->m_temp2Pool, &(parser->m_mem));
   parserInit(parser, encodingName);
 
-  if (encodingName && !parser->m_protocolEncodingName) {
+  if (encodingName && ! parser->m_protocolEncodingName) {
     XML_ParserFree(parser);
     return NULL;
   }
@@ -1016,8 +976,7 @@ parserCreate(const XML_Char *encodingName,
     parser->m_ns = XML_TRUE;
     parser->m_internalEncoding = XmlGetInternalEncodingNS();
     parser->m_namespaceSeparator = *nameSep;
-  }
-  else {
+  } else {
     parser->m_internalEncoding = XmlGetInternalEncoding();
   }
 
@@ -1025,8 +984,7 @@ parserCreate(const XML_Char *encodingName,
 }
 
 static void
-parserInit(XML_Parser parser, const XML_Char *encodingName)
-{
+parserInit(XML_Parser parser, const XML_Char *encodingName) {
   parser->m_processor = prologInitProcessor;
   XmlPrologStateInit(&parser->m_prologState);
   if (encodingName != NULL) {
@@ -1099,8 +1057,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName)
 
 /* moves list of bindings to m_freeBindingList */
 static void FASTCALL
-moveToFreeBindingList(XML_Parser parser, BINDING *bindings)
-{
+moveToFreeBindingList(XML_Parser parser, BINDING *bindings) {
   while (bindings) {
     BINDING *b = bindings;
     bindings = bindings->nextTagBinding;
@@ -1110,13 +1067,12 @@ moveToFreeBindingList(XML_Parser parser, BINDING *bindings)
 }
 
 XML_Bool XMLCALL
-XML_ParserReset(XML_Parser parser, const XML_Char *encodingName)
-{
+XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) {
   TAG *tStk;
   OPEN_INTERNAL_ENTITY *openEntityList;
 
   if (parser == NULL)
-      return XML_FALSE;
+    return XML_FALSE;
 
   if (parser->m_parentParser)
     return XML_FALSE;
@@ -1152,15 +1108,15 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName)
 }
 
 enum XML_Status XMLCALL
-XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName)
-{
+XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) {
   if (parser == NULL)
-      return XML_STATUS_ERROR;
+    return XML_STATUS_ERROR;
   /* Block after XML_Parse()/XML_ParseBuffer() has been called.
      XXX There's no way for the caller to determine which of the
      XXX possible error cases caused the XML_STATUS_ERROR return.
   */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return XML_STATUS_ERROR;
 
   /* Get rid of any previous encoding name */
@@ -1172,17 +1128,15 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName)
   else {
     /* Copy the new encoding name into allocated memory */
     parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem));
-    if (!parser->m_protocolEncodingName)
+    if (! parser->m_protocolEncodingName)
       return XML_STATUS_ERROR;
   }
   return XML_STATUS_OK;
 }
 
 XML_Parser XMLCALL
-XML_ExternalEntityParserCreate(XML_Parser oldParser,
-                               const XML_Char *context,
-                               const XML_Char *encodingName)
-{
+XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
+                               const XML_Char *encodingName) {
   XML_Parser parser = oldParser;
   DTD *newDtd = NULL;
   DTD *oldDtd;
@@ -1206,7 +1160,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   XML_AttlistDeclHandler oldAttlistDeclHandler;
   XML_EntityDeclHandler oldEntityDeclHandler;
   XML_XmlDeclHandler oldXmlDeclHandler;
-  ELEMENT_TYPE * oldDeclElementType;
+  ELEMENT_TYPE *oldDeclElementType;
 
   void *oldUserData;
   void *oldHandlerArg;
@@ -1269,7 +1223,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   oldhash_secret_salt = parser->m_hash_secret_salt;
 
 #ifdef XML_DTD
-  if (!context)
+  if (! context)
     newDtd = oldDtd;
 #endif /* XML_DTD */
 
@@ -1282,12 +1236,11 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
     XML_Char tmp[2];
     *tmp = parser->m_namespaceSeparator;
     parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd);
-  }
-  else {
+  } else {
     parser = parserCreate(encodingName, &parser->m_mem, NULL, newDtd);
   }
 
-  if (!parser)
+  if (! parser)
     return NULL;
 
   parser->m_startElementHandler = oldStartElementHandler;
@@ -1327,21 +1280,20 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   parser->m_prologState.inEntityValue = oldInEntityValue;
   if (context) {
 #endif /* XML_DTD */
-    if (!dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem)
-      || !setContext(parser, context)) {
+    if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem)
+        || ! setContext(parser, context)) {
       XML_ParserFree(parser);
       return NULL;
     }
     parser->m_processor = externalEntityInitProcessor;
 #ifdef XML_DTD
-  }
-  else {
-    /* The DTD instance referenced by parser->m_dtd is shared between the document's
-       root parser and external PE parsers, therefore one does not need to
-       call setContext. In addition, one also *must* not call setContext,
-       because this would overwrite existing prefix->binding pointers in
-       parser->m_dtd with ones that get destroyed with the external PE parser.
-       This would leave those prefixes with dangling pointers.
+  } else {
+    /* The DTD instance referenced by parser->m_dtd is shared between the
+       document's root parser and external PE parsers, therefore one does not
+       need to call setContext. In addition, one also *must* not call
+       setContext, because this would overwrite existing prefix->binding
+       pointers in parser->m_dtd with ones that get destroyed with the external
+       PE parser. This would leave those prefixes with dangling pointers.
     */
     parser->m_isParamEntity = XML_TRUE;
     XmlPrologStateInitExternalEntity(&parser->m_prologState);
@@ -1352,11 +1304,10 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
 }
 
 static void FASTCALL
-destroyBindings(BINDING *bindings, XML_Parser parser)
-{
+destroyBindings(BINDING *bindings, XML_Parser parser) {
   for (;;) {
     BINDING *b = bindings;
-    if (!b)
+    if (! b)
       break;
     bindings = b->nextTagBinding;
     FREE(parser, b->uri);
@@ -1365,8 +1316,7 @@ destroyBindings(BINDING *bindings, XML_Parser parser)
 }
 
 void XMLCALL
-XML_ParserFree(XML_Parser parser)
-{
+XML_ParserFree(XML_Parser parser) {
   TAG *tagList;
   OPEN_INTERNAL_ENTITY *entityList;
   if (parser == NULL)
@@ -1411,11 +1361,12 @@ XML_ParserFree(XML_Parser parser)
   /* external parameter entity parsers share the DTD structure
      parser->m_dtd with the root parser, so we must not destroy it
   */
-  if (!parser->m_isParamEntity && parser->m_dtd)
+  if (! parser->m_isParamEntity && parser->m_dtd)
 #else
   if (parser->m_dtd)
 #endif /* XML_DTD */
-    dtdDestroy(parser->m_dtd, (XML_Bool)!parser->m_parentParser, &parser->m_mem);
+    dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser,
+               &parser->m_mem);
   FREE(parser, (void *)parser->m_atts);
 #ifdef XML_ATTR_INFO
   FREE(parser, (void *)parser->m_attInfo);
@@ -1431,20 +1382,19 @@ XML_ParserFree(XML_Parser parser)
 }
 
 void XMLCALL
-XML_UseParserAsHandlerArg(XML_Parser parser)
-{
+XML_UseParserAsHandlerArg(XML_Parser parser) {
   if (parser != NULL)
     parser->m_handlerArg = parser;
 }
 
 enum XML_Error XMLCALL
-XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD)
-{
+XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) {
   if (parser == NULL)
     return XML_ERROR_INVALID_ARGUMENT;
 #ifdef XML_DTD
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING;
   parser->m_useForeignDTD = useDTD;
   return XML_ERROR_NONE;
@@ -1454,19 +1404,18 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD)
 }
 
 void XMLCALL
-XML_SetReturnNSTriplet(XML_Parser parser, int do_nst)
-{
+XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) {
   if (parser == NULL)
     return;
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return;
   parser->m_ns_triplets = do_nst ? XML_TRUE : XML_FALSE;
 }
 
 void XMLCALL
-XML_SetUserData(XML_Parser parser, void *p)
-{
+XML_SetUserData(XML_Parser parser, void *p) {
   if (parser == NULL)
     return;
   if (parser->m_handlerArg == parser->m_userData)
@@ -1476,49 +1425,43 @@ XML_SetUserData(XML_Parser parser, void *p)
 }
 
 enum XML_Status XMLCALL
-XML_SetBase(XML_Parser parser, const XML_Char *p)
-{
+XML_SetBase(XML_Parser parser, const XML_Char *p) {
   if (parser == NULL)
     return XML_STATUS_ERROR;
   if (p) {
     p = poolCopyString(&parser->m_dtd->pool, p);
-    if (!p)
+    if (! p)
       return XML_STATUS_ERROR;
     parser->m_curBase = p;
-  }
-  else
+  } else
     parser->m_curBase = NULL;
   return XML_STATUS_OK;
 }
 
-const XML_Char * XMLCALL
-XML_GetBase(XML_Parser parser)
-{
+const XML_Char *XMLCALL
+XML_GetBase(XML_Parser parser) {
   if (parser == NULL)
     return NULL;
   return parser->m_curBase;
 }
 
 int XMLCALL
-XML_GetSpecifiedAttributeCount(XML_Parser parser)
-{
+XML_GetSpecifiedAttributeCount(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   return parser->m_nSpecifiedAtts;
 }
 
 int XMLCALL
-XML_GetIdAttributeIndex(XML_Parser parser)
-{
+XML_GetIdAttributeIndex(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   return parser->m_idAttIndex;
 }
 
 #ifdef XML_ATTR_INFO
-const XML_AttrInfo * XMLCALL
-XML_GetAttributeInfo(XML_Parser parser)
-{
+const XML_AttrInfo *XMLCALL
+XML_GetAttributeInfo(XML_Parser parser) {
   if (parser == NULL)
     return NULL;
   return parser->m_attInfo;
@@ -1526,10 +1469,8 @@ XML_GetAttributeInfo(XML_Parser parser)
 #endif
 
 void XMLCALL
-XML_SetElementHandler(XML_Parser parser,
-                      XML_StartElementHandler start,
-                      XML_EndElementHandler end)
-{
+XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start,
+                      XML_EndElementHandler end) {
   if (parser == NULL)
     return;
   parser->m_startElementHandler = start;
@@ -1537,39 +1478,33 @@ XML_SetElementHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetStartElementHandler(XML_Parser parser,
-                           XML_StartElementHandler start) {
+XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler start) {
   if (parser != NULL)
     parser->m_startElementHandler = start;
 }
 
 void XMLCALL
-XML_SetEndElementHandler(XML_Parser parser,
-                         XML_EndElementHandler end) {
+XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler end) {
   if (parser != NULL)
     parser->m_endElementHandler = end;
 }
 
 void XMLCALL
 XML_SetCharacterDataHandler(XML_Parser parser,
-                            XML_CharacterDataHandler handler)
-{
+                            XML_CharacterDataHandler handler) {
   if (parser != NULL)
     parser->m_characterDataHandler = handler;
 }
 
 void XMLCALL
 XML_SetProcessingInstructionHandler(XML_Parser parser,
-                                    XML_ProcessingInstructionHandler handler)
-{
+                                    XML_ProcessingInstructionHandler handler) {
   if (parser != NULL)
     parser->m_processingInstructionHandler = handler;
 }
 
 void XMLCALL
-XML_SetCommentHandler(XML_Parser parser,
-                      XML_CommentHandler handler)
-{
+XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler) {
   if (parser != NULL)
     parser->m_commentHandler = handler;
 }
@@ -1577,8 +1512,7 @@ XML_SetCommentHandler(XML_Parser parser,
 void XMLCALL
 XML_SetCdataSectionHandler(XML_Parser parser,
                            XML_StartCdataSectionHandler start,
-                           XML_EndCdataSectionHandler end)
-{
+                           XML_EndCdataSectionHandler end) {
   if (parser == NULL)
     return;
   parser->m_startCdataSectionHandler = start;
@@ -1600,9 +1534,7 @@ XML_SetEndCdataSectionHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDefaultHandler(XML_Parser parser,
-                      XML_DefaultHandler handler)
-{
+XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler) {
   if (parser == NULL)
     return;
   parser->m_defaultHandler = handler;
@@ -1610,9 +1542,7 @@ XML_SetDefaultHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDefaultHandlerExpand(XML_Parser parser,
-                            XML_DefaultHandler handler)
-{
+XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler) {
   if (parser == NULL)
     return;
   parser->m_defaultHandler = handler;
@@ -1620,10 +1550,8 @@ XML_SetDefaultHandlerExpand(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDoctypeDeclHandler(XML_Parser parser,
-                          XML_StartDoctypeDeclHandler start,
-                          XML_EndDoctypeDeclHandler end)
-{
+XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start,
+                          XML_EndDoctypeDeclHandler end) {
   if (parser == NULL)
     return;
   parser->m_startDoctypeDeclHandler = start;
@@ -1638,24 +1566,20 @@ XML_SetStartDoctypeDeclHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetEndDoctypeDeclHandler(XML_Parser parser,
-                             XML_EndDoctypeDeclHandler end) {
+XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end) {
   if (parser != NULL)
     parser->m_endDoctypeDeclHandler = end;
 }
 
 void XMLCALL
 XML_SetUnparsedEntityDeclHandler(XML_Parser parser,
-                                 XML_UnparsedEntityDeclHandler handler)
-{
+                                 XML_UnparsedEntityDeclHandler handler) {
   if (parser != NULL)
     parser->m_unparsedEntityDeclHandler = handler;
 }
 
 void XMLCALL
-XML_SetNotationDeclHandler(XML_Parser parser,
-                           XML_NotationDeclHandler handler)
-{
+XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler) {
   if (parser != NULL)
     parser->m_notationDeclHandler = handler;
 }
@@ -1663,8 +1587,7 @@ XML_SetNotationDeclHandler(XML_Parser parser,
 void XMLCALL
 XML_SetNamespaceDeclHandler(XML_Parser parser,
                             XML_StartNamespaceDeclHandler start,
-                            XML_EndNamespaceDeclHandler end)
-{
+                            XML_EndNamespaceDeclHandler end) {
   if (parser == NULL)
     return;
   parser->m_startNamespaceDeclHandler = start;
@@ -1687,23 +1610,20 @@ XML_SetEndNamespaceDeclHandler(XML_Parser parser,
 
 void XMLCALL
 XML_SetNotStandaloneHandler(XML_Parser parser,
-                            XML_NotStandaloneHandler handler)
-{
+                            XML_NotStandaloneHandler handler) {
   if (parser != NULL)
     parser->m_notStandaloneHandler = handler;
 }
 
 void XMLCALL
 XML_SetExternalEntityRefHandler(XML_Parser parser,
-                                XML_ExternalEntityRefHandler handler)
-{
+                                XML_ExternalEntityRefHandler handler) {
   if (parser != NULL)
     parser->m_externalEntityRefHandler = handler;
 }
 
 void XMLCALL
-XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg)
-{
+XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg) {
   if (parser == NULL)
     return;
   if (arg)
@@ -1714,17 +1634,14 @@ XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg)
 
 void XMLCALL
 XML_SetSkippedEntityHandler(XML_Parser parser,
-                            XML_SkippedEntityHandler handler)
-{
+                            XML_SkippedEntityHandler handler) {
   if (parser != NULL)
     parser->m_skippedEntityHandler = handler;
 }
 
 void XMLCALL
 XML_SetUnknownEncodingHandler(XML_Parser parser,
-                              XML_UnknownEncodingHandler handler,
-                              void *data)
-{
+                              XML_UnknownEncodingHandler handler, void *data) {
   if (parser == NULL)
     return;
   parser->m_unknownEncodingHandler = handler;
@@ -1732,44 +1649,37 @@ XML_SetUnknownEncodingHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetElementDeclHandler(XML_Parser parser,
-                          XML_ElementDeclHandler eldecl)
-{
+XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl) {
   if (parser != NULL)
     parser->m_elementDeclHandler = eldecl;
 }
 
 void XMLCALL
-XML_SetAttlistDeclHandler(XML_Parser parser,
-                          XML_AttlistDeclHandler attdecl)
-{
+XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl) {
   if (parser != NULL)
     parser->m_attlistDeclHandler = attdecl;
 }
 
 void XMLCALL
-XML_SetEntityDeclHandler(XML_Parser parser,
-                         XML_EntityDeclHandler handler)
-{
+XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler) {
   if (parser != NULL)
     parser->m_entityDeclHandler = handler;
 }
 
 void XMLCALL
-XML_SetXmlDeclHandler(XML_Parser parser,
-                      XML_XmlDeclHandler handler) {
+XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler handler) {
   if (parser != NULL)
     parser->m_xmlDeclHandler = handler;
 }
 
 int XMLCALL
 XML_SetParamEntityParsing(XML_Parser parser,
-                          enum XML_ParamEntityParsing peParsing)
-{
+                          enum XML_ParamEntityParsing peParsing) {
   if (parser == NULL)
     return 0;
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return 0;
 #ifdef XML_DTD
   parser->m_paramEntityParsing = peParsing;
@@ -1780,23 +1690,21 @@ XML_SetParamEntityParsing(XML_Parser parser,
 }
 
 int XMLCALL
-XML_SetHashSalt(XML_Parser parser,
-                unsigned long hash_salt)
-{
+XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) {
   if (parser == NULL)
     return 0;
   if (parser->m_parentParser)
     return XML_SetHashSalt(parser->m_parentParser, hash_salt);
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return 0;
   parser->m_hash_secret_salt = hash_salt;
   return 1;
 }
 
 enum XML_Status XMLCALL
-XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
-{
+XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) {
   if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) {
     if (parser != NULL)
       parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT;
@@ -1810,7 +1718,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     parser->m_errorCode = XML_ERROR_FINISHED;
     return XML_STATUS_ERROR;
   case XML_INITIALIZED:
-    if (parser->m_parentParser == NULL && !startParsing(parser)) {
+    if (parser->m_parentParser == NULL && ! startParsing(parser)) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return XML_STATUS_ERROR;
     }
@@ -1821,7 +1729,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
 
   if (len == 0) {
     parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
-    if (!isFinal)
+    if (! isFinal)
       return XML_STATUS_OK;
     parser->m_positionPtr = parser->m_bufferPtr;
     parser->m_parseEndPtr = parser->m_bufferEnd;
@@ -1830,7 +1738,9 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
        data are the final chunk of input, then we have to check them again
        to detect errors based on that fact.
     */
-    parser->m_errorCode = parser->m_processor(parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
+    parser->m_errorCode
+        = parser->m_processor(parser, parser->m_bufferPtr,
+                              parser->m_parseEndPtr, &parser->m_bufferPtr);
 
     if (parser->m_errorCode == XML_ERROR_NONE) {
       switch (parser->m_parsingStatus.parsing) {
@@ -1847,7 +1757,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
          *
          * LCOV_EXCL_START
          */
-        XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+        XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                          parser->m_bufferPtr, &parser->m_position);
         parser->m_positionPtr = parser->m_bufferPtr;
         return XML_STATUS_SUSPENDED;
         /* LCOV_EXCL_STOP */
@@ -1870,23 +1781,23 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     enum XML_Status result;
     /* Detect overflow (a+b > MAX <==> b > MAX-a) */
     if (len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) {
-       parser->m_errorCode = XML_ERROR_NO_MEMORY;
-       parser->m_eventPtr = parser->m_eventEndPtr = NULL;
-       parser->m_processor = errorProcessor;
-       return XML_STATUS_ERROR;
+      parser->m_errorCode = XML_ERROR_NO_MEMORY;
+      parser->m_eventPtr = parser->m_eventEndPtr = NULL;
+      parser->m_processor = errorProcessor;
+      return XML_STATUS_ERROR;
     }
     parser->m_parseEndByteIndex += len;
     parser->m_positionPtr = s;
     parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
 
-    parser->m_errorCode = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end);
+    parser->m_errorCode
+        = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end);
 
     if (parser->m_errorCode != XML_ERROR_NONE) {
       parser->m_eventEndPtr = parser->m_eventPtr;
       parser->m_processor = errorProcessor;
       return XML_STATUS_ERROR;
-    }
-    else {
+    } else {
       switch (parser->m_parsingStatus.parsing) {
       case XML_SUSPENDED:
         result = XML_STATUS_SUSPENDED;
@@ -1903,10 +1814,12 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
       }
     }
 
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, end, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, end,
+                      &parser->m_position);
     nLeftOver = s + len - end;
     if (nLeftOver) {
-      if (parser->m_buffer == NULL || nLeftOver > parser->m_bufferLim - parser->m_buffer) {
+      if (parser->m_buffer == NULL
+          || nLeftOver > parser->m_bufferLim - parser->m_buffer) {
         /* avoid _signed_ integer overflow */
         char *temp = NULL;
         const int bytesToAllocate = (int)((unsigned)len * 2U);
@@ -1932,7 +1845,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     parser->m_eventEndPtr = parser->m_bufferPtr;
     return result;
   }
-#endif  /* not defined XML_CONTEXT_BYTES */
+#endif /* not defined XML_CONTEXT_BYTES */
   else {
     void *buff = XML_GetBuffer(parser, len);
     if (buff == NULL)
@@ -1945,8 +1858,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
 }
 
 enum XML_Status XMLCALL
-XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
-{
+XML_ParseBuffer(XML_Parser parser, int len, int isFinal) {
   const char *start;
   enum XML_Status result = XML_STATUS_OK;
 
@@ -1960,7 +1872,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
     parser->m_errorCode = XML_ERROR_FINISHED;
     return XML_STATUS_ERROR;
   case XML_INITIALIZED:
-    if (parser->m_parentParser == NULL && !startParsing(parser)) {
+    if (parser->m_parentParser == NULL && ! startParsing(parser)) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return XML_STATUS_ERROR;
     }
@@ -1976,14 +1888,14 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
   parser->m_parseEndByteIndex += len;
   parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
 
-  parser->m_errorCode = parser->m_processor(parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr);
+  parser->m_errorCode = parser->m_processor(
+      parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr);
 
   if (parser->m_errorCode != XML_ERROR_NONE) {
     parser->m_eventEndPtr = parser->m_eventPtr;
     parser->m_processor = errorProcessor;
     return XML_STATUS_ERROR;
-  }
-  else {
+  } else {
     switch (parser->m_parsingStatus.parsing) {
     case XML_SUSPENDED:
       result = XML_STATUS_SUSPENDED;
@@ -1994,18 +1906,18 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
         parser->m_parsingStatus.parsing = XML_FINISHED;
         return result;
       }
-    default: ;  /* should not happen */
+    default:; /* should not happen */
     }
   }
 
-  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                    parser->m_bufferPtr, &parser->m_position);
   parser->m_positionPtr = parser->m_bufferPtr;
   return result;
 }
 
-void * XMLCALL
-XML_GetBuffer(XML_Parser parser, int len)
-{
+void *XMLCALL
+XML_GetBuffer(XML_Parser parser, int len) {
   if (parser == NULL)
     return NULL;
   if (len < 0) {
@@ -2019,17 +1931,17 @@ XML_GetBuffer(XML_Parser parser, int len)
   case XML_FINISHED:
     parser->m_errorCode = XML_ERROR_FINISHED;
     return NULL;
-  default: ;
+  default:;
   }
 
   if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) {
 #ifdef XML_CONTEXT_BYTES
     int keep;
-#endif  /* defined XML_CONTEXT_BYTES */
+#endif /* defined XML_CONTEXT_BYTES */
     /* Do not invoke signed arithmetic overflow: */
-    int neededSize = (int) ((unsigned)len +
-                            (unsigned)EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd,
-                                                          parser->m_bufferPtr));
+    int neededSize = (int)((unsigned)len
+                           + (unsigned)EXPAT_SAFE_PTR_DIFF(
+                               parser->m_bufferEnd, parser->m_bufferPtr));
     if (neededSize < 0) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return NULL;
@@ -2039,13 +1951,18 @@ XML_GetBuffer(XML_Parser parser, int len)
     if (keep > XML_CONTEXT_BYTES)
       keep = XML_CONTEXT_BYTES;
     neededSize += keep;
-#endif  /* defined XML_CONTEXT_BYTES */
-    if (neededSize <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) {
+#endif /* defined XML_CONTEXT_BYTES */
+    if (neededSize
+        <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) {
 #ifdef XML_CONTEXT_BYTES
       if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) {
-          int offset = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) - keep;
-        /* The buffer pointers cannot be NULL here; we have at least some bytes in the buffer */
-        memmove(parser->m_buffer, &parser->m_buffer[offset], parser->m_bufferEnd - parser->m_bufferPtr + keep);
+        int offset
+            = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)
+              - keep;
+        /* The buffer pointers cannot be NULL here; we have at least some bytes
+         * in the buffer */
+        memmove(parser->m_buffer, &parser->m_buffer[offset],
+                parser->m_bufferEnd - parser->m_bufferPtr + keep);
         parser->m_bufferEnd -= offset;
         parser->m_bufferPtr -= offset;
       }
@@ -2053,20 +1970,21 @@ XML_GetBuffer(XML_Parser parser, int len)
       if (parser->m_buffer && parser->m_bufferPtr) {
         memmove(parser->m_buffer, parser->m_bufferPtr,
                 EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr));
-        parser->m_bufferEnd = parser->m_buffer +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
+        parser->m_bufferEnd
+            = parser->m_buffer
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
         parser->m_bufferPtr = parser->m_buffer;
       }
-#endif  /* not defined XML_CONTEXT_BYTES */
-    }
-    else {
+#endif /* not defined XML_CONTEXT_BYTES */
+    } else {
       char *newBuf;
-      int bufferSize = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr);
+      int bufferSize
+          = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr);
       if (bufferSize == 0)
         bufferSize = INIT_BUFFER_SIZE;
       do {
         /* Do not invoke signed arithmetic overflow: */
-        bufferSize = (int) (2U * (unsigned) bufferSize);
+        bufferSize = (int)(2U * (unsigned)bufferSize);
       } while (bufferSize < neededSize && bufferSize > 0);
       if (bufferSize <= 0) {
         parser->m_errorCode = XML_ERROR_NO_MEMORY;
@@ -2080,18 +1998,17 @@ XML_GetBuffer(XML_Parser parser, int len)
       parser->m_bufferLim = newBuf + bufferSize;
 #ifdef XML_CONTEXT_BYTES
       if (parser->m_bufferPtr) {
-        int keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer);
-        if (keep > XML_CONTEXT_BYTES)
-          keep = XML_CONTEXT_BYTES;
         memcpy(newBuf, &parser->m_bufferPtr[-keep],
-               EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) + keep);
+               EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)
+                   + keep);
         FREE(parser, parser->m_buffer);
         parser->m_buffer = newBuf;
-        parser->m_bufferEnd = parser->m_buffer +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) + keep;
+        parser->m_bufferEnd
+            = parser->m_buffer
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)
+              + keep;
         parser->m_bufferPtr = parser->m_buffer + keep;
-      }
-      else {
+      } else {
         /* This must be a brand new buffer with no data in it yet */
         parser->m_bufferEnd = newBuf;
         parser->m_bufferPtr = parser->m_buffer = newBuf;
@@ -2101,15 +2018,15 @@ XML_GetBuffer(XML_Parser parser, int len)
         memcpy(newBuf, parser->m_bufferPtr,
                EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr));
         FREE(parser, parser->m_buffer);
-        parser->m_bufferEnd = newBuf +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
-      }
-      else {
+        parser->m_bufferEnd
+            = newBuf
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
+      } else {
         /* This must be a brand new buffer with no data in it yet */
         parser->m_bufferEnd = newBuf;
       }
       parser->m_bufferPtr = parser->m_buffer = newBuf;
-#endif  /* not defined XML_CONTEXT_BYTES */
+#endif /* not defined XML_CONTEXT_BYTES */
     }
     parser->m_eventPtr = parser->m_eventEndPtr = NULL;
     parser->m_positionPtr = NULL;
@@ -2118,8 +2035,7 @@ XML_GetBuffer(XML_Parser parser, int len)
 }
 
 enum XML_Status XMLCALL
-XML_StopParser(XML_Parser parser, XML_Bool resumable)
-{
+XML_StopParser(XML_Parser parser, XML_Bool resumable) {
   if (parser == NULL)
     return XML_STATUS_ERROR;
   switch (parser->m_parsingStatus.parsing) {
@@ -2142,16 +2058,14 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable)
       }
 #endif
       parser->m_parsingStatus.parsing = XML_SUSPENDED;
-    }
-    else
+    } else
       parser->m_parsingStatus.parsing = XML_FINISHED;
   }
   return XML_STATUS_OK;
 }
 
 enum XML_Status XMLCALL
-XML_ResumeParser(XML_Parser parser)
-{
+XML_ResumeParser(XML_Parser parser) {
   enum XML_Status result = XML_STATUS_OK;
 
   if (parser == NULL)
@@ -2162,14 +2076,14 @@ XML_ResumeParser(XML_Parser parser)
   }
   parser->m_parsingStatus.parsing = XML_PARSING;
 
-  parser->m_errorCode = parser->m_processor(parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
+  parser->m_errorCode = parser->m_processor(
+      parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
 
   if (parser->m_errorCode != XML_ERROR_NONE) {
     parser->m_eventEndPtr = parser->m_eventPtr;
     parser->m_processor = errorProcessor;
     return XML_STATUS_ERROR;
-  }
-  else {
+  } else {
     switch (parser->m_parsingStatus.parsing) {
     case XML_SUSPENDED:
       result = XML_STATUS_SUSPENDED;
@@ -2180,18 +2094,18 @@ XML_ResumeParser(XML_Parser parser)
         parser->m_parsingStatus.parsing = XML_FINISHED;
         return result;
       }
-    default: ;
+    default:;
     }
   }
 
-  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                    parser->m_bufferPtr, &parser->m_position);
   parser->m_positionPtr = parser->m_bufferPtr;
   return result;
 }
 
 void XMLCALL
-XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status)
-{
+XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) {
   if (parser == NULL)
     return;
   assert(status != NULL);
@@ -2199,26 +2113,24 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status)
 }
 
 enum XML_Error XMLCALL
-XML_GetErrorCode(XML_Parser parser)
-{
+XML_GetErrorCode(XML_Parser parser) {
   if (parser == NULL)
     return XML_ERROR_INVALID_ARGUMENT;
   return parser->m_errorCode;
 }
 
 XML_Index XMLCALL
-XML_GetCurrentByteIndex(XML_Parser parser)
-{
+XML_GetCurrentByteIndex(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   if (parser->m_eventPtr)
-    return (XML_Index)(parser->m_parseEndByteIndex - (parser->m_parseEndPtr - parser->m_eventPtr));
+    return (XML_Index)(parser->m_parseEndByteIndex
+                       - (parser->m_parseEndPtr - parser->m_eventPtr));
   return -1;
 }
 
 int XMLCALL
-XML_GetCurrentByteCount(XML_Parser parser)
-{
+XML_GetCurrentByteCount(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventEndPtr && parser->m_eventPtr)
@@ -2226,9 +2138,8 @@ XML_GetCurrentByteCount(XML_Parser parser)
   return 0;
 }
 
-const char * XMLCALL
-XML_GetInputContext(XML_Parser parser, int *offset, int *size)
-{
+const char *XMLCALL
+XML_GetInputContext(XML_Parser parser, int *offset, int *size) {
 #ifdef XML_CONTEXT_BYTES
   if (parser == NULL)
     return NULL;
@@ -2236,7 +2147,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size)
     if (offset != NULL)
       *offset = (int)(parser->m_eventPtr - parser->m_buffer);
     if (size != NULL)
-      *size   = (int)(parser->m_bufferEnd - parser->m_buffer);
+      *size = (int)(parser->m_bufferEnd - parser->m_buffer);
     return parser->m_buffer;
   }
 #else
@@ -2244,82 +2155,76 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size)
   (void)offset;
   (void)size;
 #endif /* defined XML_CONTEXT_BYTES */
-  return (char *) 0;
+  return (char *)0;
 }
 
 XML_Size XMLCALL
-XML_GetCurrentLineNumber(XML_Parser parser)
-{
+XML_GetCurrentLineNumber(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) {
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_eventPtr, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                      parser->m_eventPtr, &parser->m_position);
     parser->m_positionPtr = parser->m_eventPtr;
   }
   return parser->m_position.lineNumber + 1;
 }
 
 XML_Size XMLCALL
-XML_GetCurrentColumnNumber(XML_Parser parser)
-{
+XML_GetCurrentColumnNumber(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) {
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_eventPtr, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                      parser->m_eventPtr, &parser->m_position);
     parser->m_positionPtr = parser->m_eventPtr;
   }
   return parser->m_position.columnNumber;
 }
 
 void XMLCALL
-XML_FreeContentModel(XML_Parser parser, XML_Content *model)
-{
+XML_FreeContentModel(XML_Parser parser, XML_Content *model) {
   if (parser != NULL)
     FREE(parser, model);
 }
 
-void * XMLCALL
-XML_MemMalloc(XML_Parser parser, size_t size)
-{
+void *XMLCALL
+XML_MemMalloc(XML_Parser parser, size_t size) {
   if (parser == NULL)
     return NULL;
   return MALLOC(parser, size);
 }
 
-void * XMLCALL
-XML_MemRealloc(XML_Parser parser, void *ptr, size_t size)
-{
+void *XMLCALL
+XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) {
   if (parser == NULL)
     return NULL;
   return REALLOC(parser, ptr, size);
 }
 
 void XMLCALL
-XML_MemFree(XML_Parser parser, void *ptr)
-{
+XML_MemFree(XML_Parser parser, void *ptr) {
   if (parser != NULL)
     FREE(parser, ptr);
 }
 
 void XMLCALL
-XML_DefaultCurrent(XML_Parser parser)
-{
+XML_DefaultCurrent(XML_Parser parser) {
   if (parser == NULL)
     return;
   if (parser->m_defaultHandler) {
     if (parser->m_openInternalEntities)
-      reportDefault(parser,
-                    parser->m_internalEncoding,
+      reportDefault(parser, parser->m_internalEncoding,
                     parser->m_openInternalEntities->internalEventPtr,
                     parser->m_openInternalEntities->internalEventEndPtr);
     else
-      reportDefault(parser, parser->m_encoding, parser->m_eventPtr, parser->m_eventEndPtr);
+      reportDefault(parser, parser->m_encoding, parser->m_eventPtr,
+                    parser->m_eventEndPtr);
   }
 }
 
-const XML_LChar * XMLCALL
-XML_ErrorString(enum XML_Error code)
-{
+const XML_LChar *XMLCALL
+XML_ErrorString(enum XML_Error code) {
   switch (code) {
   case XML_ERROR_NONE:
     return NULL;
@@ -2401,21 +2306,22 @@ XML_ErrorString(enum XML_Error code)
     return XML_L("cannot suspend in external parameter entity");
   /* Added in 2.0.0. */
   case XML_ERROR_RESERVED_PREFIX_XML:
-    return XML_L("reserved prefix (xml) must not be undeclared or bound to another namespace name");
+    return XML_L(
+        "reserved prefix (xml) must not be undeclared or bound to another namespace name");
   case XML_ERROR_RESERVED_PREFIX_XMLNS:
     return XML_L("reserved prefix (xmlns) must not be declared or undeclared");
   case XML_ERROR_RESERVED_NAMESPACE_URI:
-    return XML_L("prefix must not be bound to one of the reserved namespace names");
+    return XML_L(
+        "prefix must not be bound to one of the reserved namespace names");
   /* Added in 2.2.5. */
-  case XML_ERROR_INVALID_ARGUMENT:  /* Constant added in 2.2.1, already */
+  case XML_ERROR_INVALID_ARGUMENT: /* Constant added in 2.2.1, already */
     return XML_L("invalid argument");
   }
   return NULL;
 }
 
-const XML_LChar * XMLCALL
+const XML_LChar *XMLCALL
 XML_ExpatVersion(void) {
-
   /* V1 is used to string-ize the version number. However, it would
      string-ize the actual version macro *names* unless we get them
      substituted before being passed to V1. CPP is defined to expand
@@ -2424,8 +2330,8 @@ XML_ExpatVersion(void) {
      with the correct numerals. */
   /* ### I'm assuming cpp is portable in this respect... */
 
-#define V1(a,b,c) XML_L(#a)XML_L(".")XML_L(#b)XML_L(".")XML_L(#c)
-#define V2(a,b,c) XML_L("expat_")V1(a,b,c)
+#define V1(a, b, c) XML_L(#a) XML_L(".") XML_L(#b) XML_L(".") XML_L(#c)
+#define V2(a, b, c) XML_L("expat_") V1(a, b, c)
 
   return V2(XML_MAJOR_VERSION, XML_MINOR_VERSION, XML_MICRO_VERSION);
 
@@ -2434,8 +2340,7 @@ XML_ExpatVersion(void) {
 }
 
 XML_Expat_Version XMLCALL
-XML_ExpatVersionInfo(void)
-{
+XML_ExpatVersionInfo(void) {
   XML_Expat_Version version;
 
   version.major = XML_MAJOR_VERSION;
@@ -2445,41 +2350,39 @@ XML_ExpatVersionInfo(void)
   return version;
 }
 
-const XML_Feature * XMLCALL
-XML_GetFeatureList(void)
-{
-  static const XML_Feature features[] = {
-    {XML_FEATURE_SIZEOF_XML_CHAR,  XML_L("sizeof(XML_Char)"),
-     sizeof(XML_Char)},
-    {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"),
-     sizeof(XML_LChar)},
+const XML_Feature *XMLCALL
+XML_GetFeatureList(void) {
+  static const XML_Feature features[]
+      = {{XML_FEATURE_SIZEOF_XML_CHAR, XML_L("sizeof(XML_Char)"),
+          sizeof(XML_Char)},
+         {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"),
+          sizeof(XML_LChar)},
 #ifdef XML_UNICODE
-    {XML_FEATURE_UNICODE,          XML_L("XML_UNICODE"), 0},
+         {XML_FEATURE_UNICODE, XML_L("XML_UNICODE"), 0},
 #endif
 #ifdef XML_UNICODE_WCHAR_T
-    {XML_FEATURE_UNICODE_WCHAR_T,  XML_L("XML_UNICODE_WCHAR_T"), 0},
+         {XML_FEATURE_UNICODE_WCHAR_T, XML_L("XML_UNICODE_WCHAR_T"), 0},
 #endif
 #ifdef XML_DTD
-    {XML_FEATURE_DTD,              XML_L("XML_DTD"), 0},
+         {XML_FEATURE_DTD, XML_L("XML_DTD"), 0},
 #endif
 #ifdef XML_CONTEXT_BYTES
-    {XML_FEATURE_CONTEXT_BYTES,    XML_L("XML_CONTEXT_BYTES"),
-     XML_CONTEXT_BYTES},
+         {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"),
+          XML_CONTEXT_BYTES},
 #endif
 #ifdef XML_MIN_SIZE
-    {XML_FEATURE_MIN_SIZE,         XML_L("XML_MIN_SIZE"), 0},
+         {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0},
 #endif
 #ifdef XML_NS
-    {XML_FEATURE_NS,               XML_L("XML_NS"), 0},
+         {XML_FEATURE_NS, XML_L("XML_NS"), 0},
 #endif
 #ifdef XML_LARGE_SIZE
-    {XML_FEATURE_LARGE_SIZE,       XML_L("XML_LARGE_SIZE"), 0},
+         {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0},
 #endif
 #ifdef XML_ATTR_INFO
-    {XML_FEATURE_ATTR_INFO,        XML_L("XML_ATTR_INFO"), 0},
+         {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0},
 #endif
-    {XML_FEATURE_END,              NULL, 0}
-  };
+         {XML_FEATURE_END, NULL, 0}};
 
   return features;
 }
@@ -2490,8 +2393,7 @@ XML_GetFeatureList(void)
    permanent location, since the parse buffer is about to be discarded.
 */
 static XML_Bool
-storeRawNames(XML_Parser parser)
-{
+storeRawNames(XML_Parser parser) {
   TAG *tag = parser->m_tagStack;
   while (tag) {
     int bufSize;
@@ -2521,8 +2423,8 @@ storeRawNames(XML_Parser parser)
          then update it as well, since it will always point into tag->buf
       */
       if (tag->name.localPart)
-        tag->name.localPart = (XML_Char *)temp + (tag->name.localPart -
-                                                  (XML_Char *)tag->buf);
+        tag->name.localPart
+            = (XML_Char *)temp + (tag->name.localPart - (XML_Char *)tag->buf);
       tag->buf = temp;
       tag->bufEnd = temp + bufSize;
       rawNameBuf = temp + nameLen;
@@ -2535,26 +2437,21 @@ storeRawNames(XML_Parser parser)
 }
 
 static enum XML_Error PTRCALL
-contentProcessor(XML_Parser parser,
-                 const char *start,
-                 const char *end,
-                 const char **endPtr)
-{
-  enum XML_Error result = doContent(parser, 0, parser->m_encoding, start, end,
-                                    endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+contentProcessor(XML_Parser parser, const char *start, const char *end,
+                 const char **endPtr) {
+  enum XML_Error result
+      = doContent(parser, 0, parser->m_encoding, start, end, endPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result == XML_ERROR_NONE) {
-    if (!storeRawNames(parser))
+    if (! storeRawNames(parser))
       return XML_ERROR_NO_MEMORY;
   }
   return result;
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor(XML_Parser parser,
-                            const char *start,
-                            const char *end,
-                            const char **endPtr)
-{
+externalEntityInitProcessor(XML_Parser parser, const char *start,
+                            const char *end, const char **endPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -2563,11 +2460,8 @@ externalEntityInitProcessor(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor2(XML_Parser parser,
-                             const char *start,
-                             const char *end,
-                             const char **endPtr)
-{
+externalEntityInitProcessor2(XML_Parser parser, const char *start,
+                             const char *end, const char **endPtr) {
   const char *next = start; /* XmlContentTok doesn't always set the last arg */
   int tok = XmlContentTok(parser->m_encoding, start, end, &next);
   switch (tok) {
@@ -2577,21 +2471,21 @@ externalEntityInitProcessor2(XML_Parser parser,
        doContent (by detecting XML_TOK_NONE) without processing any xml text
        declaration - causing the error XML_ERROR_MISPLACED_XML_PI in doContent.
     */
-    if (next == end && !parser->m_parsingStatus.finalBuffer) {
+    if (next == end && ! parser->m_parsingStatus.finalBuffer) {
       *endPtr = next;
       return XML_ERROR_NONE;
     }
     start = next;
     break;
   case XML_TOK_PARTIAL:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
     parser->m_eventPtr = start;
     return XML_ERROR_UNCLOSED_TOKEN;
   case XML_TOK_PARTIAL_CHAR:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
@@ -2603,11 +2497,8 @@ externalEntityInitProcessor2(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor3(XML_Parser parser,
-                             const char *start,
-                             const char *end,
-                             const char **endPtr)
-{
+externalEntityInitProcessor3(XML_Parser parser, const char *start,
+                             const char *end, const char **endPtr) {
   int tok;
   const char *next = start; /* XmlContentTok doesn't always set the last arg */
   parser->m_eventPtr = start;
@@ -2615,31 +2506,29 @@ externalEntityInitProcessor3(XML_Parser parser,
   parser->m_eventEndPtr = next;
 
   switch (tok) {
-  case XML_TOK_XML_DECL:
-    {
-      enum XML_Error result;
-      result = processXmlDecl(parser, 1, start, next);
-      if (result != XML_ERROR_NONE)
-        return result;
-      switch (parser->m_parsingStatus.parsing) {
-      case XML_SUSPENDED:
-        *endPtr = next;
-        return XML_ERROR_NONE;
-      case XML_FINISHED:
-        return XML_ERROR_ABORTED;
-      default:
-        start = next;
-      }
+  case XML_TOK_XML_DECL: {
+    enum XML_Error result;
+    result = processXmlDecl(parser, 1, start, next);
+    if (result != XML_ERROR_NONE)
+      return result;
+    switch (parser->m_parsingStatus.parsing) {
+    case XML_SUSPENDED:
+      *endPtr = next;
+      return XML_ERROR_NONE;
+    case XML_FINISHED:
+      return XML_ERROR_ABORTED;
+    default:
+      start = next;
     }
-    break;
+  } break;
   case XML_TOK_PARTIAL:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
     return XML_ERROR_UNCLOSED_TOKEN;
   case XML_TOK_PARTIAL_CHAR:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
@@ -2651,39 +2540,31 @@ externalEntityInitProcessor3(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityContentProcessor(XML_Parser parser,
-                               const char *start,
-                               const char *end,
-                               const char **endPtr)
-{
-  enum XML_Error result = doContent(parser, 1, parser->m_encoding, start, end,
-                                    endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+externalEntityContentProcessor(XML_Parser parser, const char *start,
+                               const char *end, const char **endPtr) {
+  enum XML_Error result
+      = doContent(parser, 1, parser->m_encoding, start, end, endPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result == XML_ERROR_NONE) {
-    if (!storeRawNames(parser))
+    if (! storeRawNames(parser))
       return XML_ERROR_NO_MEMORY;
   }
   return result;
 }
 
 static enum XML_Error
-doContent(XML_Parser parser,
-          int startTagLevel,
-          const ENCODING *enc,
-          const char *s,
-          const char *end,
-          const char **nextPtr,
-          XML_Bool haveMore)
-{
+doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
+          const char *s, const char *end, const char **nextPtr,
+          XML_Bool haveMore) {
   /* save one level of indirection */
-  DTD * const dtd = parser->m_dtd;
+  DTD *const dtd = parser->m_dtd;
 
   const char **eventPP;
   const char **eventEndPP;
   if (enc == parser->m_encoding) {
     eventPP = &parser->m_eventPtr;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -2703,8 +2584,7 @@ doContent(XML_Parser parser,
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, end);
       /* We are at the end of the final buffer, should we check for
          XML_SUSPENDED, XML_FINISHED?
@@ -2742,185 +2622,178 @@ doContent(XML_Parser parser,
         return XML_ERROR_NONE;
       }
       return XML_ERROR_PARTIAL_CHAR;
-    case XML_TOK_ENTITY_REF:
-      {
-        const XML_Char *name;
-        ENTITY *entity;
-        XML_Char ch = (XML_Char) XmlPredefinedEntityName(enc,
-                                              s + enc->minBytesPerChar,
-                                              next - enc->minBytesPerChar);
-        if (ch) {
-          if (parser->m_characterDataHandler)
-            parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1);
-          else if (parser->m_defaultHandler)
-            reportDefault(parser, enc, s, next);
-          break;
-        }
-        name = poolStoreString(&dtd->pool, enc,
-                                s + enc->minBytesPerChar,
-                                next - enc->minBytesPerChar);
-        if (!name)
-          return XML_ERROR_NO_MEMORY;
-        entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
-        poolDiscard(&dtd->pool);
-        /* First, determine if a check for an existing declaration is needed;
-           if yes, check that the entity exists, and that it is internal,
-           otherwise call the skipped entity or default handler.
-        */
-        if (!dtd->hasParamEntityRefs || dtd->standalone) {
-          if (!entity)
-            return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal)
-            return XML_ERROR_ENTITY_DECLARED_IN_PE;
-        }
-        else if (!entity) {
+    case XML_TOK_ENTITY_REF: {
+      const XML_Char *name;
+      ENTITY *entity;
+      XML_Char ch = (XML_Char)XmlPredefinedEntityName(
+          enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar);
+      if (ch) {
+        if (parser->m_characterDataHandler)
+          parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1);
+        else if (parser->m_defaultHandler)
+          reportDefault(parser, enc, s, next);
+        break;
+      }
+      name = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                             next - enc->minBytesPerChar);
+      if (! name)
+        return XML_ERROR_NO_MEMORY;
+      entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
+      poolDiscard(&dtd->pool);
+      /* First, determine if a check for an existing declaration is needed;
+         if yes, check that the entity exists, and that it is internal,
+         otherwise call the skipped entity or default handler.
+      */
+      if (! dtd->hasParamEntityRefs || dtd->standalone) {
+        if (! entity)
+          return XML_ERROR_UNDEFINED_ENTITY;
+        else if (! entity->is_internal)
+          return XML_ERROR_ENTITY_DECLARED_IN_PE;
+      } else if (! entity) {
+        if (parser->m_skippedEntityHandler)
+          parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+        else if (parser->m_defaultHandler)
+          reportDefault(parser, enc, s, next);
+        break;
+      }
+      if (entity->open)
+        return XML_ERROR_RECURSIVE_ENTITY_REF;
+      if (entity->notation)
+        return XML_ERROR_BINARY_ENTITY_REF;
+      if (entity->textPtr) {
+        enum XML_Error result;
+        if (! parser->m_defaultExpandInternalEntities) {
           if (parser->m_skippedEntityHandler)
-            parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+            parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name,
+                                           0);
           else if (parser->m_defaultHandler)
             reportDefault(parser, enc, s, next);
           break;
         }
-        if (entity->open)
-          return XML_ERROR_RECURSIVE_ENTITY_REF;
-        if (entity->notation)
-          return XML_ERROR_BINARY_ENTITY_REF;
-        if (entity->textPtr) {
-          enum XML_Error result;
-          if (!parser->m_defaultExpandInternalEntities) {
-            if (parser->m_skippedEntityHandler)
-              parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name, 0);
-            else if (parser->m_defaultHandler)
-              reportDefault(parser, enc, s, next);
-            break;
-          }
-          result = processInternalEntity(parser, entity, XML_FALSE);
-          if (result != XML_ERROR_NONE)
-            return result;
-        }
-        else if (parser->m_externalEntityRefHandler) {
-          const XML_Char *context;
-          entity->open = XML_TRUE;
-          context = getContext(parser);
-          entity->open = XML_FALSE;
-          if (!context)
-            return XML_ERROR_NO_MEMORY;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        context,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
-            return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
-          poolDiscard(&parser->m_tempPool);
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        break;
-      }
+        result = processInternalEntity(parser, entity, XML_FALSE);
+        if (result != XML_ERROR_NONE)
+          return result;
+      } else if (parser->m_externalEntityRefHandler) {
+        const XML_Char *context;
+        entity->open = XML_TRUE;
+        context = getContext(parser);
+        entity->open = XML_FALSE;
+        if (! context)
+          return XML_ERROR_NO_MEMORY;
+        if (! parser->m_externalEntityRefHandler(
+                parser->m_externalEntityRefHandlerArg, context, entity->base,
+                entity->systemId, entity->publicId))
+          return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
+        poolDiscard(&parser->m_tempPool);
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      break;
+    }
     case XML_TOK_START_TAG_NO_ATTS:
       /* fall through */
-    case XML_TOK_START_TAG_WITH_ATTS:
-      {
-        TAG *tag;
-        enum XML_Error result;
-        XML_Char *toPtr;
-        if (parser->m_freeTagList) {
-          tag = parser->m_freeTagList;
-          parser->m_freeTagList = parser->m_freeTagList->parent;
+    case XML_TOK_START_TAG_WITH_ATTS: {
+      TAG *tag;
+      enum XML_Error result;
+      XML_Char *toPtr;
+      if (parser->m_freeTagList) {
+        tag = parser->m_freeTagList;
+        parser->m_freeTagList = parser->m_freeTagList->parent;
+      } else {
+        tag = (TAG *)MALLOC(parser, sizeof(TAG));
+        if (! tag)
+          return XML_ERROR_NO_MEMORY;
+        tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE);
+        if (! tag->buf) {
+          FREE(parser, tag);
+          return XML_ERROR_NO_MEMORY;
         }
-        else {
-          tag = (TAG *)MALLOC(parser, sizeof(TAG));
-          if (!tag)
-            return XML_ERROR_NO_MEMORY;
-          tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE);
-          if (!tag->buf) {
-            FREE(parser, tag);
-            return XML_ERROR_NO_MEMORY;
+        tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE;
+      }
+      tag->bindings = NULL;
+      tag->parent = parser->m_tagStack;
+      parser->m_tagStack = tag;
+      tag->name.localPart = NULL;
+      tag->name.prefix = NULL;
+      tag->rawName = s + enc->minBytesPerChar;
+      tag->rawNameLength = XmlNameLength(enc, tag->rawName);
+      ++parser->m_tagLevel;
+      {
+        const char *rawNameEnd = tag->rawName + tag->rawNameLength;
+        const char *fromPtr = tag->rawName;
+        toPtr = (XML_Char *)tag->buf;
+        for (;;) {
+          int bufSize;
+          int convLen;
+          const enum XML_Convert_Result convert_res
+              = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr,
+                           (ICHAR *)tag->bufEnd - 1);
+          convLen = (int)(toPtr - (XML_Char *)tag->buf);
+          if ((fromPtr >= rawNameEnd)
+              || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) {
+            tag->name.strLen = convLen;
+            break;
           }
-          tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE;
-        }
-        tag->bindings = NULL;
-        tag->parent = parser->m_tagStack;
-        parser->m_tagStack = tag;
-        tag->name.localPart = NULL;
-        tag->name.prefix = NULL;
-        tag->rawName = s + enc->minBytesPerChar;
-        tag->rawNameLength = XmlNameLength(enc, tag->rawName);
-        ++parser->m_tagLevel;
-        {
-          const char *rawNameEnd = tag->rawName + tag->rawNameLength;
-          const char *fromPtr = tag->rawName;
-          toPtr = (XML_Char *)tag->buf;
-          for (;;) {
-            int bufSize;
-            int convLen;
-            const enum XML_Convert_Result convert_res = XmlConvert(enc,
-                       &fromPtr, rawNameEnd,
-                       (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1);
-            convLen = (int)(toPtr - (XML_Char *)tag->buf);
-            if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) {
-              tag->name.strLen = convLen;
-              break;
-            }
-            bufSize = (int)(tag->bufEnd - tag->buf) << 1;
-            {
-              char *temp = (char *)REALLOC(parser, tag->buf, bufSize);
-              if (temp == NULL)
-                return XML_ERROR_NO_MEMORY;
-              tag->buf = temp;
-              tag->bufEnd = temp + bufSize;
-              toPtr = (XML_Char *)temp + convLen;
-            }
+          bufSize = (int)(tag->bufEnd - tag->buf) << 1;
+          {
+            char *temp = (char *)REALLOC(parser, tag->buf, bufSize);
+            if (temp == NULL)
+              return XML_ERROR_NO_MEMORY;
+            tag->buf = temp;
+            tag->bufEnd = temp + bufSize;
+            toPtr = (XML_Char *)temp + convLen;
           }
         }
-        tag->name.str = (XML_Char *)tag->buf;
-        *toPtr = XML_T('\0');
-        result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings));
-        if (result)
-          return result;
-        if (parser->m_startElementHandler)
-          parser->m_startElementHandler(parser->m_handlerArg, tag->name.str,
-                              (const XML_Char **)parser->m_atts);
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        poolClear(&parser->m_tempPool);
-        break;
       }
+      tag->name.str = (XML_Char *)tag->buf;
+      *toPtr = XML_T('\0');
+      result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings));
+      if (result)
+        return result;
+      if (parser->m_startElementHandler)
+        parser->m_startElementHandler(parser->m_handlerArg, tag->name.str,
+                                      (const XML_Char **)parser->m_atts);
+      else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      poolClear(&parser->m_tempPool);
+      break;
+    }
     case XML_TOK_EMPTY_ELEMENT_NO_ATTS:
       /* fall through */
-    case XML_TOK_EMPTY_ELEMENT_WITH_ATTS:
-      {
-        const char *rawName = s + enc->minBytesPerChar;
-        enum XML_Error result;
-        BINDING *bindings = NULL;
-        XML_Bool noElmHandlers = XML_TRUE;
-        TAG_NAME name;
-        name.str = poolStoreString(&parser->m_tempPool, enc, rawName,
-                                   rawName + XmlNameLength(enc, rawName));
-        if (!name.str)
-          return XML_ERROR_NO_MEMORY;
-        poolFinish(&parser->m_tempPool);
-        result = storeAtts(parser, enc, s, &name, &bindings);
-        if (result != XML_ERROR_NONE) {
-          freeBindings(parser, bindings);
-          return result;
-        }
-        poolFinish(&parser->m_tempPool);
-        if (parser->m_startElementHandler) {
-          parser->m_startElementHandler(parser->m_handlerArg, name.str, (const XML_Char **)parser->m_atts);
-          noElmHandlers = XML_FALSE;
-        }
-        if (parser->m_endElementHandler) {
-          if (parser->m_startElementHandler)
-            *eventPP = *eventEndPP;
-          parser->m_endElementHandler(parser->m_handlerArg, name.str);
-          noElmHandlers = XML_FALSE;
-        }
-        if (noElmHandlers && parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        poolClear(&parser->m_tempPool);
+    case XML_TOK_EMPTY_ELEMENT_WITH_ATTS: {
+      const char *rawName = s + enc->minBytesPerChar;
+      enum XML_Error result;
+      BINDING *bindings = NULL;
+      XML_Bool noElmHandlers = XML_TRUE;
+      TAG_NAME name;
+      name.str = poolStoreString(&parser->m_tempPool, enc, rawName,
+                                 rawName + XmlNameLength(enc, rawName));
+      if (! name.str)
+        return XML_ERROR_NO_MEMORY;
+      poolFinish(&parser->m_tempPool);
+      result = storeAtts(parser, enc, s, &name, &bindings);
+      if (result != XML_ERROR_NONE) {
         freeBindings(parser, bindings);
+        return result;
+      }
+      poolFinish(&parser->m_tempPool);
+      if (parser->m_startElementHandler) {
+        parser->m_startElementHandler(parser->m_handlerArg, name.str,
+                                      (const XML_Char **)parser->m_atts);
+        noElmHandlers = XML_FALSE;
+      }
+      if (parser->m_endElementHandler) {
+        if (parser->m_startElementHandler)
+          *eventPP = *eventEndPP;
+        parser->m_endElementHandler(parser->m_handlerArg, name.str);
+        noElmHandlers = XML_FALSE;
       }
-      if ((parser->m_tagLevel == 0) && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
+      if (noElmHandlers && parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      poolClear(&parser->m_tempPool);
+      freeBindings(parser, bindings);
+    }
+      if ((parser->m_tagLevel == 0)
+          && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
         if (parser->m_parsingStatus.parsing == XML_SUSPENDED)
           parser->m_processor = epilogProcessor;
         else
@@ -2937,7 +2810,7 @@ doContent(XML_Parser parser,
         parser->m_tagStack = tag->parent;
         tag->parent = parser->m_freeTagList;
         parser->m_freeTagList = tag;
-        rawName = s + enc->minBytesPerChar*2;
+        rawName = s + enc->minBytesPerChar * 2;
         len = XmlNameLength(enc, rawName);
         if (len != tag->rawNameLength
             || memcmp(tag->rawName, rawName, len) != 0) {
@@ -2957,86 +2830,89 @@ doContent(XML_Parser parser,
             */
             uri = (XML_Char *)tag->name.str + tag->name.uriLen;
             /* don't need to check for space - already done in storeAtts() */
-            while (*localPart) *uri++ = *localPart++;
+            while (*localPart)
+              *uri++ = *localPart++;
             prefix = (XML_Char *)tag->name.prefix;
             if (parser->m_ns_triplets && prefix) {
               *uri++ = parser->m_namespaceSeparator;
-              while (*prefix) *uri++ = *prefix++;
-             }
+              while (*prefix)
+                *uri++ = *prefix++;
+            }
             *uri = XML_T('\0');
           }
           parser->m_endElementHandler(parser->m_handlerArg, tag->name.str);
-        }
-        else if (parser->m_defaultHandler)
+        } else if (parser->m_defaultHandler)
           reportDefault(parser, enc, s, next);
         while (tag->bindings) {
           BINDING *b = tag->bindings;
           if (parser->m_endNamespaceDeclHandler)
-            parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
+            parser->m_endNamespaceDeclHandler(parser->m_handlerArg,
+                                              b->prefix->name);
           tag->bindings = tag->bindings->nextTagBinding;
           b->nextTagBinding = parser->m_freeBindingList;
           parser->m_freeBindingList = b;
           b->prefix->binding = b->prevPrefixBinding;
         }
-        if (parser->m_tagLevel == 0)
-          return epilogProcessor(parser, next, end, nextPtr);
-      }
-      break;
-    case XML_TOK_CHAR_REF:
-      {
-        int n = XmlCharRefNumber(enc, s);
-        if (n < 0)
-          return XML_ERROR_BAD_CHAR_REF;
-        if (parser->m_characterDataHandler) {
-          XML_Char buf[XML_ENCODE_MAX];
-          parser->m_characterDataHandler(parser->m_handlerArg, buf, XmlEncode(n, (ICHAR *)buf));
+        if ((parser->m_tagLevel == 0)
+            && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
+          if (parser->m_parsingStatus.parsing == XML_SUSPENDED)
+            parser->m_processor = epilogProcessor;
+          else
+            return epilogProcessor(parser, next, end, nextPtr);
         }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
       }
       break;
+    case XML_TOK_CHAR_REF: {
+      int n = XmlCharRefNumber(enc, s);
+      if (n < 0)
+        return XML_ERROR_BAD_CHAR_REF;
+      if (parser->m_characterDataHandler) {
+        XML_Char buf[XML_ENCODE_MAX];
+        parser->m_characterDataHandler(parser->m_handlerArg, buf,
+                                       XmlEncode(n, (ICHAR *)buf));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_XML_DECL:
       return XML_ERROR_MISPLACED_XML_PI;
     case XML_TOK_DATA_NEWLINE:
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       break;
-    case XML_TOK_CDATA_SECT_OPEN:
-      {
-        enum XML_Error result;
-        if (parser->m_startCdataSectionHandler)
-          parser->m_startCdataSectionHandler(parser->m_handlerArg);
-/* BEGIN disabled code */
-        /* Suppose you doing a transformation on a document that involves
-           changing only the character data.  You set up a defaultHandler
-           and a characterDataHandler.  The defaultHandler simply copies
-           characters through.  The characterDataHandler does the
-           transformation and writes the characters out escaping them as
-           necessary.  This case will fail to work if we leave out the
-           following two lines (because & and < inside CDATA sections will
-           be incorrectly escaped).
-
-           However, now we have a start/endCdataSectionHandler, so it seems
-           easier to let the user deal with this.
-        */
-        else if (0 && parser->m_characterDataHandler)
-          parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0);
-/* END disabled code */
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore);
-        if (result != XML_ERROR_NONE)
-          return result;
-        else if (!next) {
-          parser->m_processor = cdataSectionProcessor;
-          return result;
-        }
+    case XML_TOK_CDATA_SECT_OPEN: {
+      enum XML_Error result;
+      if (parser->m_startCdataSectionHandler)
+        parser->m_startCdataSectionHandler(parser->m_handlerArg);
+      /* BEGIN disabled code */
+      /* Suppose you doing a transformation on a document that involves
+         changing only the character data.  You set up a defaultHandler
+         and a characterDataHandler.  The defaultHandler simply copies
+         characters through.  The characterDataHandler does the
+         transformation and writes the characters out escaping them as
+         necessary.  This case will fail to work if we leave out the
+         following two lines (because & and < inside CDATA sections will
+         be incorrectly escaped).
+
+         However, now we have a start/endCdataSectionHandler, so it seems
+         easier to let the user deal with this.
+      */
+      else if (0 && parser->m_characterDataHandler)
+        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                                       0);
+      /* END disabled code */
+      else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore);
+      if (result != XML_ERROR_NONE)
+        return result;
+      else if (! next) {
+        parser->m_processor = cdataSectionProcessor;
+        return result;
       }
-      break;
+    } break;
     case XML_TOK_TRAILING_RSQB:
       if (haveMore) {
         *nextPtr = s;
@@ -3046,15 +2922,14 @@ doContent(XML_Parser parser,
         if (MUST_CONVERT(enc, s)) {
           ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
           XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-          parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                               (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-        }
-        else
-          parser->m_characterDataHandler(parser->m_handlerArg,
-                               (XML_Char *)s,
-                               (int)((XML_Char *)end - (XML_Char *)s));
-      }
-      else if (parser->m_defaultHandler)
+          parser->m_characterDataHandler(
+              parser->m_handlerArg, parser->m_dataBuf,
+              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+        } else
+          parser->m_characterDataHandler(
+              parser->m_handlerArg, (XML_Char *)s,
+              (int)((XML_Char *)end - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, end);
       /* We are at the end of the final buffer, should we check for
          XML_SUSPENDED, XML_FINISHED?
@@ -3069,37 +2944,34 @@ doContent(XML_Parser parser,
       }
       *nextPtr = end;
       return XML_ERROR_NONE;
-    case XML_TOK_DATA_CHARS:
-      {
-        XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
-        if (charDataHandler) {
-          if (MUST_CONVERT(enc, s)) {
-            for (;;) {
-              ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-              const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-              *eventEndPP = s;
-              charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-              if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
-                break;
-              *eventPP = s;
-            }
+    case XML_TOK_DATA_CHARS: {
+      XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
+      if (charDataHandler) {
+        if (MUST_CONVERT(enc, s)) {
+          for (;;) {
+            ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
+            const enum XML_Convert_Result convert_res = XmlConvert(
+                enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+            *eventEndPP = s;
+            charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                            (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+            if ((convert_res == XML_CONVERT_COMPLETED)
+                || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+              break;
+            *eventPP = s;
           }
-          else
-            charDataHandler(parser->m_handlerArg,
-                            (XML_Char *)s,
-                            (int)((XML_Char *)next - (XML_Char *)s));
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-      }
-      break;
+        } else
+          charDataHandler(parser->m_handlerArg, (XML_Char *)s,
+                          (int)((XML_Char *)next - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_PI:
-      if (!reportProcessingInstruction(parser, enc, s, next))
+      if (! reportProcessingInstruction(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_COMMENT:
-      if (!reportComment(parser, enc, s, next))
+      if (! reportComment(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     default:
@@ -3122,7 +2994,7 @@ doContent(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
   /* not reached */
@@ -3133,8 +3005,7 @@ doContent(XML_Parser parser,
  * reused as appropriate.
  */
 static void
-freeBindings(XML_Parser parser, BINDING *bindings)
-{
+freeBindings(XML_Parser parser, BINDING *bindings) {
   while (bindings) {
     BINDING *b = bindings;
 
@@ -3142,7 +3013,7 @@ freeBindings(XML_Parser parser, BINDING *bindings)
      * binding in addBindings(), so call the end handler now.
      */
     if (parser->m_endNamespaceDeclHandler)
-        parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
+      parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
 
     bindings = bindings->nextTagBinding;
     b->nextTagBinding = parser->m_freeBindingList;
@@ -3162,14 +3033,12 @@ freeBindings(XML_Parser parser, BINDING *bindings)
    - generate namespace aware element name (URI, prefix)
 */
 static enum XML_Error
-storeAtts(XML_Parser parser, const ENCODING *enc,
-          const char *attStr, TAG_NAME *tagNamePtr,
-          BINDING **bindingsPtr)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
+          TAG_NAME *tagNamePtr, BINDING **bindingsPtr) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   ELEMENT_TYPE *elementType;
   int nDefaultAtts;
-  const XML_Char **appAtts;   /* the attribute list for the application */
+  const XML_Char **appAtts; /* the attribute list for the application */
   int attIndex = 0;
   int prefixLen;
   int i;
@@ -3180,16 +3049,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   const XML_Char *localPart;
 
   /* lookup the element type name */
-  elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str,0);
-  if (!elementType) {
+  elementType
+      = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str, 0);
+  if (! elementType) {
     const XML_Char *name = poolCopyString(&dtd->pool, tagNamePtr->str);
-    if (!name)
+    if (! name)
       return XML_ERROR_NO_MEMORY;
     elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, name,
                                          sizeof(ELEMENT_TYPE));
-    if (!elementType)
+    if (! elementType)
       return XML_ERROR_NO_MEMORY;
-    if (parser->m_ns && !setElementTypePrefix(parser, elementType))
+    if (parser->m_ns && ! setElementTypePrefix(parser, elementType))
       return XML_ERROR_NO_MEMORY;
   }
   nDefaultAtts = elementType->nDefaultAtts;
@@ -3203,14 +3073,16 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     XML_AttrInfo *temp2;
 #endif
     parser->m_attsSize = n + nDefaultAtts + INIT_ATTS_SIZE;
-    temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts, parser->m_attsSize * sizeof(ATTRIBUTE));
+    temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts,
+                                parser->m_attsSize * sizeof(ATTRIBUTE));
     if (temp == NULL) {
       parser->m_attsSize = oldAttsSize;
       return XML_ERROR_NO_MEMORY;
     }
     parser->m_atts = temp;
 #ifdef XML_ATTR_INFO
-    temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo, parser->m_attsSize * sizeof(XML_AttrInfo));
+    temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo,
+                                    parser->m_attsSize * sizeof(XML_AttrInfo));
     if (temp2 == NULL) {
       parser->m_attsSize = oldAttsSize;
       return XML_ERROR_NO_MEMORY;
@@ -3228,18 +3100,20 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     XML_AttrInfo *currAttInfo = &parser->m_attInfo[i];
 #endif
     /* add the name and value to the attribute list */
-    ATTRIBUTE_ID *attId = getAttributeId(parser, enc, currAtt->name,
-                                         currAtt->name
-                                         + XmlNameLength(enc, currAtt->name));
-    if (!attId)
+    ATTRIBUTE_ID *attId
+        = getAttributeId(parser, enc, currAtt->name,
+                         currAtt->name + XmlNameLength(enc, currAtt->name));
+    if (! attId)
       return XML_ERROR_NO_MEMORY;
 #ifdef XML_ATTR_INFO
-    currAttInfo->nameStart = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name);
-    currAttInfo->nameEnd = currAttInfo->nameStart +
-                           XmlNameLength(enc, currAtt->name);
-    currAttInfo->valueStart = parser->m_parseEndByteIndex -
-                            (parser->m_parseEndPtr - currAtt->valuePtr);
-    currAttInfo->valueEnd = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->valueEnd);
+    currAttInfo->nameStart
+        = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name);
+    currAttInfo->nameEnd
+        = currAttInfo->nameStart + XmlNameLength(enc, currAtt->name);
+    currAttInfo->valueStart = parser->m_parseEndByteIndex
+                              - (parser->m_parseEndPtr - currAtt->valuePtr);
+    currAttInfo->valueEnd = parser->m_parseEndByteIndex
+                            - (parser->m_parseEndPtr - currAtt->valueEnd);
 #endif
     /* Detect duplicate attributes by their QNames. This does not work when
        namespace processing is turned on and different prefixes for the same
@@ -3252,7 +3126,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     }
     (attId->name)[-1] = 1;
     appAtts[attIndex++] = attId->name;
-    if (!parser->m_atts[i].normalized) {
+    if (! parser->m_atts[i].normalized) {
       enum XML_Error result;
       XML_Bool isCdata = XML_TRUE;
 
@@ -3268,17 +3142,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
       }
 
       /* normalize the attribute value */
-      result = storeAttributeValue(parser, enc, isCdata,
-                                   parser->m_atts[i].valuePtr, parser->m_atts[i].valueEnd,
-                                   &parser->m_tempPool);
+      result = storeAttributeValue(
+          parser, enc, isCdata, parser->m_atts[i].valuePtr,
+          parser->m_atts[i].valueEnd, &parser->m_tempPool);
       if (result)
         return result;
       appAtts[attIndex] = poolStart(&parser->m_tempPool);
       poolFinish(&parser->m_tempPool);
-    }
-    else {
+    } else {
       /* the value did not need normalizing */
-      appAtts[attIndex] = poolStoreString(&parser->m_tempPool, enc, parser->m_atts[i].valuePtr,
+      appAtts[attIndex] = poolStoreString(&parser->m_tempPool, enc,
+                                          parser->m_atts[i].valuePtr,
                                           parser->m_atts[i].valueEnd);
       if (appAtts[attIndex] == 0)
         return XML_ERROR_NO_MEMORY;
@@ -3293,15 +3167,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         if (result)
           return result;
         --attIndex;
-      }
-      else {
+      } else {
         /* deal with other prefixed names later */
         attIndex++;
         nPrefixes++;
         (attId->name)[-1] = 2;
       }
-    }
-    else
+    } else
       attIndex++;
   }
 
@@ -3313,29 +3185,26 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         parser->m_idAttIndex = i;
         break;
       }
-  }
-  else
+  } else
     parser->m_idAttIndex = -1;
 
   /* do attribute defaulting */
   for (i = 0; i < nDefaultAtts; i++) {
     const DEFAULT_ATTRIBUTE *da = elementType->defaultAtts + i;
-    if (!(da->id->name)[-1] && da->value) {
+    if (! (da->id->name)[-1] && da->value) {
       if (da->id->prefix) {
         if (da->id->xmlns) {
           enum XML_Error result = addBinding(parser, da->id->prefix, da->id,
                                              da->value, bindingsPtr);
           if (result)
             return result;
-        }
-        else {
+        } else {
           (da->id->name)[-1] = 2;
           nPrefixes++;
           appAtts[attIndex++] = da->id->name;
           appAtts[attIndex++] = da->value;
         }
-      }
-      else {
+      } else {
         (da->id->name)[-1] = 1;
         appAtts[attIndex++] = da->id->name;
         appAtts[attIndex++] = da->value;
@@ -3348,31 +3217,34 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
      and clear flags that say whether attributes were specified */
   i = 0;
   if (nPrefixes) {
-    int j;  /* hash table index */
+    int j; /* hash table index */
     unsigned long version = parser->m_nsAttsVersion;
     int nsAttsSize = (int)1 << parser->m_nsAttsPower;
     unsigned char oldNsAttsPower = parser->m_nsAttsPower;
     /* size of hash table must be at least 2 * (# of prefixed attributes) */
-    if ((nPrefixes << 1) >> parser->m_nsAttsPower) {  /* true for m_nsAttsPower = 0 */
+    if ((nPrefixes << 1)
+        >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */
       NS_ATT *temp;
       /* hash table size must also be a power of 2 and >= 8 */
-      while (nPrefixes >> parser->m_nsAttsPower++);
+      while (nPrefixes >> parser->m_nsAttsPower++)
+        ;
       if (parser->m_nsAttsPower < 3)
         parser->m_nsAttsPower = 3;
       nsAttsSize = (int)1 << parser->m_nsAttsPower;
-      temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT));
-      if (!temp) {
+      temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts,
+                               nsAttsSize * sizeof(NS_ATT));
+      if (! temp) {
         /* Restore actual size of memory in m_nsAtts */
         parser->m_nsAttsPower = oldNsAttsPower;
         return XML_ERROR_NO_MEMORY;
       }
       parser->m_nsAtts = temp;
-      version = 0;  /* force re-initialization of m_nsAtts hash table */
+      version = 0; /* force re-initialization of m_nsAtts hash table */
     }
     /* using a version flag saves us from initializing m_nsAtts every time */
-    if (!version) {  /* initialize version flags when version wraps around */
+    if (! version) { /* initialize version flags when version wraps around */
       version = INIT_ATTS_VERSION;
-      for (j = nsAttsSize; j != 0; )
+      for (j = nsAttsSize; j != 0;)
         parser->m_nsAtts[--j].version = version;
     }
     parser->m_nsAttsVersion = --version;
@@ -3380,7 +3252,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     /* expand prefixed names and check for duplicates */
     for (; i < attIndex; i += 2) {
       const XML_Char *s = appAtts[i];
-      if (s[-1] == 2) {  /* prefixed */
+      if (s[-1] == 2) { /* prefixed */
         ATTRIBUTE_ID *id;
         const BINDING *b;
         unsigned long uriHash;
@@ -3390,16 +3262,16 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         copy_salt_to_sipkey(parser, &sip_key);
         sip24_init(&sip_state, &sip_key);
 
-        ((XML_Char *)s)[-1] = 0;  /* clear flag */
+        ((XML_Char *)s)[-1] = 0; /* clear flag */
         id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0);
-        if (!id || !id->prefix) {
+        if (! id || ! id->prefix) {
           /* This code is walking through the appAtts array, dealing
            * with (in this case) a prefixed attribute name.  To be in
            * the array, the attribute must have already been bound, so
            * has to have passed through the hash table lookup once
            * already.  That implies that an entry for it already
            * exists, so the lookup above will return a pointer to
-           * already allocated memory.  There is no opportunity for
+           * already allocated memory.  There is no opportunaity for
            * the allocator to fail, so the condition above cannot be
            * fulfilled.
            *
@@ -3410,12 +3282,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
           return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */
         }
         b = id->prefix->binding;
-        if (!b)
+        if (! b)
           return XML_ERROR_UNBOUND_PREFIX;
 
         for (j = 0; j < b->uriLen; j++) {
           const XML_Char c = b->uri[j];
-          if (!poolAppendChar(&parser->m_tempPool, c))
+          if (! poolAppendChar(&parser->m_tempPool, c))
             return XML_ERROR_NO_MEMORY;
         }
 
@@ -3426,8 +3298,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
 
         sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char));
 
-        do {  /* copies null terminator */
-          if (!poolAppendChar(&parser->m_tempPool, *s))
+        do { /* copies null terminator */
+          if (! poolAppendChar(&parser->m_tempPool, *s))
             return XML_ERROR_NO_MEMORY;
         } while (*s++);
 
@@ -3438,28 +3310,29 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
           */
           unsigned char step = 0;
           unsigned long mask = nsAttsSize - 1;
-          j = uriHash & mask;  /* index into hash table */
+          j = uriHash & mask; /* index into hash table */
           while (parser->m_nsAtts[j].version == version) {
             /* for speed we compare stored hash values first */
             if (uriHash == parser->m_nsAtts[j].hash) {
               const XML_Char *s1 = poolStart(&parser->m_tempPool);
               const XML_Char *s2 = parser->m_nsAtts[j].uriName;
               /* s1 is null terminated, but not s2 */
-              for (; *s1 == *s2 && *s1 != 0; s1++, s2++);
+              for (; *s1 == *s2 && *s1 != 0; s1++, s2++)
+                ;
               if (*s1 == 0)
                 return XML_ERROR_DUPLICATE_ATTRIBUTE;
             }
-            if (!step)
+            if (! step)
               step = PROBE_STEP(uriHash, mask, parser->m_nsAttsPower);
             j < step ? (j += nsAttsSize - step) : (j -= step);
           }
         }
 
-        if (parser->m_ns_triplets) {  /* append namespace separator and prefix */
+        if (parser->m_ns_triplets) { /* append namespace separator and prefix */
           parser->m_tempPool.ptr[-1] = parser->m_namespaceSeparator;
           s = b->prefix->name;
           do {
-            if (!poolAppendChar(&parser->m_tempPool, *s))
+            if (! poolAppendChar(&parser->m_tempPool, *s))
               return XML_ERROR_NO_MEMORY;
           } while (*s++);
         }
@@ -3474,13 +3347,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         parser->m_nsAtts[j].hash = uriHash;
         parser->m_nsAtts[j].uriName = s;
 
-        if (!--nPrefixes) {
+        if (! --nPrefixes) {
           i += 2;
           break;
         }
-      }
-      else  /* not prefixed */
-        ((XML_Char *)s)[-1] = 0;  /* clear flag */
+      } else                     /* not prefixed */
+        ((XML_Char *)s)[-1] = 0; /* clear flag */
     }
   }
   /* clear flags for the remaining attributes */
@@ -3489,40 +3361,38 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   for (binding = *bindingsPtr; binding; binding = binding->nextTagBinding)
     binding->attId->name[-1] = 0;
 
-  if (!parser->m_ns)
+  if (! parser->m_ns)
     return XML_ERROR_NONE;
 
   /* expand the element type name */
   if (elementType->prefix) {
     binding = elementType->prefix->binding;
-    if (!binding)
+    if (! binding)
       return XML_ERROR_UNBOUND_PREFIX;
     localPart = tagNamePtr->str;
     while (*localPart++ != XML_T(ASCII_COLON))
       ;
-  }
-  else if (dtd->defaultPrefix.binding) {
+  } else if (dtd->defaultPrefix.binding) {
     binding = dtd->defaultPrefix.binding;
     localPart = tagNamePtr->str;
-  }
-  else
+  } else
     return XML_ERROR_NONE;
   prefixLen = 0;
   if (parser->m_ns_triplets && binding->prefix->name) {
     for (; binding->prefix->name[prefixLen++];)
-      ;  /* prefixLen includes null terminator */
+      ; /* prefixLen includes null terminator */
   }
   tagNamePtr->localPart = localPart;
   tagNamePtr->uriLen = binding->uriLen;
   tagNamePtr->prefix = binding->prefix->name;
   tagNamePtr->prefixLen = prefixLen;
   for (i = 0; localPart[i++];)
-    ;  /* i includes null terminator */
+    ; /* i includes null terminator */
   n = i + binding->uriLen + prefixLen;
   if (n > binding->uriAlloc) {
     TAG *p;
     uri = (XML_Char *)MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char));
-    if (!uri)
+    if (! uri)
       return XML_ERROR_NO_MEMORY;
     binding->uriAlloc = n + EXPAND_SPARE;
     memcpy(uri, binding->uri, binding->uriLen * sizeof(XML_Char));
@@ -3538,7 +3408,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   /* we always have a namespace separator between localPart and prefix */
   if (prefixLen) {
     uri += i - 1;
-    *uri = parser->m_namespaceSeparator;  /* replace null terminator */
+    *uri = parser->m_namespaceSeparator; /* replace null terminator */
     memcpy(uri + 1, binding->prefix->name, prefixLen * sizeof(XML_Char));
   }
   tagNamePtr->str = binding->uri;
@@ -3550,27 +3420,25 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
 */
 static enum XML_Error
 addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
-           const XML_Char *uri, BINDING **bindingsPtr)
-{
-  static const XML_Char xmlNamespace[] = {
-    ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, ASCII_SLASH,
-    ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD,
-    ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L,
-    ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8, ASCII_SLASH,
-    ASCII_n, ASCII_a, ASCII_m, ASCII_e, ASCII_s, ASCII_p, ASCII_a, ASCII_c,
-    ASCII_e, '\0'
-  };
-  static const int xmlLen =
-    (int)sizeof(xmlNamespace)/sizeof(XML_Char) - 1;
-  static const XML_Char xmlnsNamespace[] = {
-    ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, ASCII_SLASH,
-    ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD,
-    ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_2, ASCII_0, ASCII_0,
-    ASCII_0, ASCII_SLASH, ASCII_x, ASCII_m, ASCII_l, ASCII_n, ASCII_s,
-    ASCII_SLASH, '\0'
-  };
-  static const int xmlnsLen =
-    (int)sizeof(xmlnsNamespace)/sizeof(XML_Char) - 1;
+           const XML_Char *uri, BINDING **bindingsPtr) {
+  static const XML_Char xmlNamespace[]
+      = {ASCII_h,      ASCII_t,     ASCII_t,     ASCII_p,      ASCII_COLON,
+         ASCII_SLASH,  ASCII_SLASH, ASCII_w,     ASCII_w,      ASCII_w,
+         ASCII_PERIOD, ASCII_w,     ASCII_3,     ASCII_PERIOD, ASCII_o,
+         ASCII_r,      ASCII_g,     ASCII_SLASH, ASCII_X,      ASCII_M,
+         ASCII_L,      ASCII_SLASH, ASCII_1,     ASCII_9,      ASCII_9,
+         ASCII_8,      ASCII_SLASH, ASCII_n,     ASCII_a,      ASCII_m,
+         ASCII_e,      ASCII_s,     ASCII_p,     ASCII_a,      ASCII_c,
+         ASCII_e,      '\0'};
+  static const int xmlLen = (int)sizeof(xmlNamespace) / sizeof(XML_Char) - 1;
+  static const XML_Char xmlnsNamespace[]
+      = {ASCII_h,     ASCII_t,      ASCII_t, ASCII_p, ASCII_COLON,  ASCII_SLASH,
+         ASCII_SLASH, ASCII_w,      ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w,
+         ASCII_3,     ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g,      ASCII_SLASH,
+         ASCII_2,     ASCII_0,      ASCII_0, ASCII_0, ASCII_SLASH,  ASCII_x,
+         ASCII_m,     ASCII_l,      ASCII_n, ASCII_s, ASCII_SLASH,  '\0'};
+  static const int xmlnsLen
+      = (int)sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1;
 
   XML_Bool mustBeXML = XML_FALSE;
   XML_Bool isXML = XML_TRUE;
@@ -3583,14 +3451,11 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   if (*uri == XML_T('\0') && prefix->name)
     return XML_ERROR_UNDECLARING_PREFIX;
 
-  if (prefix->name
-      && prefix->name[0] == XML_T(ASCII_x)
+  if (prefix->name && prefix->name[0] == XML_T(ASCII_x)
       && prefix->name[1] == XML_T(ASCII_m)
       && prefix->name[2] == XML_T(ASCII_l)) {
-
     /* Not allowed to bind xmlns */
-    if (prefix->name[3] == XML_T(ASCII_n)
-        && prefix->name[4] == XML_T(ASCII_s)
+    if (prefix->name[3] == XML_T(ASCII_n) && prefix->name[4] == XML_T(ASCII_s)
         && prefix->name[5] == XML_T('\0'))
       return XML_ERROR_RESERVED_PREFIX_XMLNS;
 
@@ -3602,7 +3467,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
     if (isXML && (len > xmlLen || uri[len] != xmlNamespace[len]))
       isXML = XML_FALSE;
 
-    if (!mustBeXML && isXMLNS
+    if (! mustBeXML && isXMLNS
         && (len > xmlnsLen || uri[len] != xmlnsNamespace[len]))
       isXMLNS = XML_FALSE;
   }
@@ -3621,21 +3486,21 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   if (parser->m_freeBindingList) {
     b = parser->m_freeBindingList;
     if (len > b->uriAlloc) {
-      XML_Char *temp = (XML_Char *)REALLOC(parser, b->uri,
-                          sizeof(XML_Char) * (len + EXPAND_SPARE));
+      XML_Char *temp = (XML_Char *)REALLOC(
+          parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE));
       if (temp == NULL)
         return XML_ERROR_NO_MEMORY;
       b->uri = temp;
       b->uriAlloc = len + EXPAND_SPARE;
     }
     parser->m_freeBindingList = b->nextTagBinding;
-  }
-  else {
+  } else {
     b = (BINDING *)MALLOC(parser, sizeof(BINDING));
-    if (!b)
+    if (! b)
       return XML_ERROR_NO_MEMORY;
-    b->uri = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE));
-    if (!b->uri) {
+    b->uri
+        = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE));
+    if (! b->uri) {
       FREE(parser, b);
       return XML_ERROR_NO_MEMORY;
     }
@@ -3658,7 +3523,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   /* if attId == NULL then we are not starting a namespace scope */
   if (attId && parser->m_startNamespaceDeclHandler)
     parser->m_startNamespaceDeclHandler(parser->m_handlerArg, prefix->name,
-                              prefix->binding ? uri : 0);
+                                        prefix->binding ? uri : 0);
   return XML_ERROR_NONE;
 }
 
@@ -3666,21 +3531,18 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
    the whole file is parsed with one call.
 */
 static enum XML_Error PTRCALL
-cdataSectionProcessor(XML_Parser parser,
-                      const char *start,
-                      const char *end,
-                      const char **endPtr)
-{
-  enum XML_Error result = doCdataSection(parser, parser->m_encoding, &start, end,
-                                         endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+cdataSectionProcessor(XML_Parser parser, const char *start, const char *end,
+                      const char **endPtr) {
+  enum XML_Error result
+      = doCdataSection(parser, parser->m_encoding, &start, end, endPtr,
+                       (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result != XML_ERROR_NONE)
     return result;
   if (start) {
-    if (parser->m_parentParser) {  /* we are parsing an external entity */
+    if (parser->m_parentParser) { /* we are parsing an external entity */
       parser->m_processor = externalEntityContentProcessor;
       return externalEntityContentProcessor(parser, start, end, endPtr);
-    }
-    else {
+    } else {
       parser->m_processor = contentProcessor;
       return contentProcessor(parser, start, end, endPtr);
     }
@@ -3692,13 +3554,8 @@ cdataSectionProcessor(XML_Parser parser,
    the section is not yet closed.
 */
 static enum XML_Error
-doCdataSection(XML_Parser parser,
-               const ENCODING *enc,
-               const char **startPtr,
-               const char *end,
-               const char **nextPtr,
-               XML_Bool haveMore)
-{
+doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
+               const char *end, const char **nextPtr, XML_Bool haveMore) {
   const char *s = *startPtr;
   const char **eventPP;
   const char **eventEndPP;
@@ -3706,8 +3563,7 @@ doCdataSection(XML_Parser parser,
     eventPP = &parser->m_eventPtr;
     *eventPP = s;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -3722,11 +3578,12 @@ doCdataSection(XML_Parser parser,
     case XML_TOK_CDATA_SECT_CLOSE:
       if (parser->m_endCdataSectionHandler)
         parser->m_endCdataSectionHandler(parser->m_handlerArg);
-/* BEGIN disabled code */
+      /* BEGIN disabled code */
       /* see comment under XML_TOK_CDATA_SECT_OPEN */
       else if (0 && parser->m_characterDataHandler)
-        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0);
-/* END disabled code */
+        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                                       0);
+      /* END disabled code */
       else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       *startPtr = next;
@@ -3739,35 +3596,31 @@ doCdataSection(XML_Parser parser,
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       break;
-    case XML_TOK_DATA_CHARS:
-      {
-        XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
-        if (charDataHandler) {
-          if (MUST_CONVERT(enc, s)) {
-            for (;;) {
-              ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-              const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-              *eventEndPP = next;
-              charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-              if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
-                break;
-              *eventPP = s;
-            }
+    case XML_TOK_DATA_CHARS: {
+      XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
+      if (charDataHandler) {
+        if (MUST_CONVERT(enc, s)) {
+          for (;;) {
+            ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
+            const enum XML_Convert_Result convert_res = XmlConvert(
+                enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+            *eventEndPP = next;
+            charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                            (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+            if ((convert_res == XML_CONVERT_COMPLETED)
+                || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+              break;
+            *eventPP = s;
           }
-          else
-            charDataHandler(parser->m_handlerArg,
-                            (XML_Char *)s,
-                            (int)((XML_Char *)next - (XML_Char *)s));
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-      }
-      break;
+        } else
+          charDataHandler(parser->m_handlerArg, (XML_Char *)s,
+                          (int)((XML_Char *)next - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_INVALID:
       *eventPP = next;
       return XML_ERROR_INVALID_TOKEN;
@@ -3791,7 +3644,7 @@ doCdataSection(XML_Parser parser,
        * statistics.
        *
        * LCOV_EXCL_START
-      */
+       */
       *eventPP = next;
       return XML_ERROR_UNEXPECTED_STATE;
       /* LCOV_EXCL_STOP */
@@ -3804,7 +3657,7 @@ doCdataSection(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
   /* not reached */
@@ -3816,13 +3669,11 @@ doCdataSection(XML_Parser parser,
    the whole file is parsed with one call.
 */
 static enum XML_Error PTRCALL
-ignoreSectionProcessor(XML_Parser parser,
-                       const char *start,
-                       const char *end,
-                       const char **endPtr)
-{
-  enum XML_Error result = doIgnoreSection(parser, parser->m_encoding, &start, end,
-                                          endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end,
+                       const char **endPtr) {
+  enum XML_Error result
+      = doIgnoreSection(parser, parser->m_encoding, &start, end, endPtr,
+                        (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result != XML_ERROR_NONE)
     return result;
   if (start) {
@@ -3836,13 +3687,8 @@ ignoreSectionProcessor(XML_Parser parser,
    if the section is not yet closed.
 */
 static enum XML_Error
-doIgnoreSection(XML_Parser parser,
-                const ENCODING *enc,
-                const char **startPtr,
-                const char *end,
-                const char **nextPtr,
-                XML_Bool haveMore)
-{
+doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
+                const char *end, const char **nextPtr, XML_Bool haveMore) {
   const char *next;
   int tok;
   const char *s = *startPtr;
@@ -3852,8 +3698,7 @@ doIgnoreSection(XML_Parser parser,
     eventPP = &parser->m_eventPtr;
     *eventPP = s;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     /* It's not entirely clear, but it seems the following two lines
      * of code cannot be executed.  The only occasions on which 'enc'
      * is not 'encoding' are when this function is called
@@ -3917,13 +3762,12 @@ doIgnoreSection(XML_Parser parser,
 #endif /* XML_DTD */
 
 static enum XML_Error
-initializeEncoding(XML_Parser parser)
-{
+initializeEncoding(XML_Parser parser) {
   const char *s;
 #ifdef XML_UNICODE
   char encodingBuf[128];
   /* See comments abount `protoclEncodingName` in parserInit() */
-  if (!parser->m_protocolEncodingName)
+  if (! parser->m_protocolEncodingName)
     s = NULL;
   else {
     int i;
@@ -3941,15 +3785,15 @@ initializeEncoding(XML_Parser parser)
 #else
   s = parser->m_protocolEncodingName;
 #endif
-  if ((parser->m_ns ? XmlInitEncodingNS : XmlInitEncoding)(&parser->m_initEncoding, &parser->m_encoding, s))
+  if ((parser->m_ns ? XmlInitEncodingNS : XmlInitEncoding)(
+          &parser->m_initEncoding, &parser->m_encoding, s))
     return XML_ERROR_NONE;
   return handleUnknownEncoding(parser, parser->m_protocolEncodingName);
 }
 
 static enum XML_Error
-processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
-               const char *s, const char *next)
-{
+processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s,
+               const char *next) {
   const char *encodingName = NULL;
   const XML_Char *storedEncName = NULL;
   const ENCODING *newEncoding = NULL;
@@ -3957,52 +3801,41 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
   const char *versionend;
   const XML_Char *storedversion = NULL;
   int standalone = -1;
-  if (!(parser->m_ns
-        ? XmlParseXmlDeclNS
-        : XmlParseXmlDecl)(isGeneralTextEntity,
-                           parser->m_encoding,
-                           s,
-                           next,
-                           &parser->m_eventPtr,
-                           &version,
-                           &versionend,
-                           &encodingName,
-                           &newEncoding,
-                           &standalone)) {
+  if (! (parser->m_ns ? XmlParseXmlDeclNS : XmlParseXmlDecl)(
+          isGeneralTextEntity, parser->m_encoding, s, next, &parser->m_eventPtr,
+          &version, &versionend, &encodingName, &newEncoding, &standalone)) {
     if (isGeneralTextEntity)
       return XML_ERROR_TEXT_DECL;
     else
       return XML_ERROR_XML_DECL;
   }
-  if (!isGeneralTextEntity && standalone == 1) {
+  if (! isGeneralTextEntity && standalone == 1) {
     parser->m_dtd->standalone = XML_TRUE;
 #ifdef XML_DTD
-    if (parser->m_paramEntityParsing == XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE)
+    if (parser->m_paramEntityParsing
+        == XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE)
       parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER;
 #endif /* XML_DTD */
   }
   if (parser->m_xmlDeclHandler) {
     if (encodingName != NULL) {
-      storedEncName = poolStoreString(&parser->m_temp2Pool,
-                                      parser->m_encoding,
-                                      encodingName,
-                                      encodingName
-                                      + XmlNameLength(parser->m_encoding, encodingName));
-      if (!storedEncName)
-              return XML_ERROR_NO_MEMORY;
+      storedEncName = poolStoreString(
+          &parser->m_temp2Pool, parser->m_encoding, encodingName,
+          encodingName + XmlNameLength(parser->m_encoding, encodingName));
+      if (! storedEncName)
+        return XML_ERROR_NO_MEMORY;
       poolFinish(&parser->m_temp2Pool);
     }
     if (version) {
-      storedversion = poolStoreString(&parser->m_temp2Pool,
-                                      parser->m_encoding,
-                                      version,
-                                      versionend - parser->m_encoding->minBytesPerChar);
-      if (!storedversion)
+      storedversion
+          = poolStoreString(&parser->m_temp2Pool, parser->m_encoding, version,
+                            versionend - parser->m_encoding->minBytesPerChar);
+      if (! storedversion)
         return XML_ERROR_NO_MEMORY;
     }
-    parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName, standalone);
-  }
-  else if (parser->m_defaultHandler)
+    parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName,
+                             standalone);
+  } else if (parser->m_defaultHandler)
     reportDefault(parser, parser->m_encoding, s, next);
   if (parser->m_protocolEncodingName == NULL) {
     if (newEncoding) {
@@ -4012,20 +3845,19 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
        * this is UTF-16, is it the same endianness?
        */
       if (newEncoding->minBytesPerChar != parser->m_encoding->minBytesPerChar
-          || (newEncoding->minBytesPerChar == 2 &&
-              newEncoding != parser->m_encoding)) {
+          || (newEncoding->minBytesPerChar == 2
+              && newEncoding != parser->m_encoding)) {
         parser->m_eventPtr = encodingName;
         return XML_ERROR_INCORRECT_ENCODING;
       }
       parser->m_encoding = newEncoding;
-    }
-    else if (encodingName) {
+    } else if (encodingName) {
       enum XML_Error result;
-      if (!storedEncName) {
+      if (! storedEncName) {
         storedEncName = poolStoreString(
-          &parser->m_temp2Pool, parser->m_encoding, encodingName,
-          encodingName + XmlNameLength(parser->m_encoding, encodingName));
-        if (!storedEncName)
+            &parser->m_temp2Pool, parser->m_encoding, encodingName,
+            encodingName + XmlNameLength(parser->m_encoding, encodingName));
+        if (! storedEncName)
           return XML_ERROR_NO_MEMORY;
       }
       result = handleUnknownEncoding(parser, storedEncName);
@@ -4043,8 +3875,7 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
 }
 
 static enum XML_Error
-handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
-{
+handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) {
   if (parser->m_unknownEncodingHandler) {
     XML_Encoding info;
     int i;
@@ -4053,21 +3884,17 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
     info.convert = NULL;
     info.data = NULL;
     info.release = NULL;
-    if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData, encodingName,
-                               &info)) {
+    if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData,
+                                         encodingName, &info)) {
       ENCODING *enc;
       parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding());
-      if (!parser->m_unknownEncodingMem) {
+      if (! parser->m_unknownEncodingMem) {
         if (info.release)
           info.release(info.data);
         return XML_ERROR_NO_MEMORY;
       }
-      enc = (parser->m_ns
-             ? XmlInitUnknownEncodingNS
-             : XmlInitUnknownEncoding)(parser->m_unknownEncodingMem,
-                                       info.map,
-                                       info.convert,
-                                       info.data);
+      enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)(
+          parser->m_unknownEncodingMem, info.map, info.convert, info.data);
       if (enc) {
         parser->m_unknownEncodingData = info.data;
         parser->m_unknownEncodingRelease = info.release;
@@ -4082,11 +3909,8 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
 }
 
 static enum XML_Error PTRCALL
-prologInitProcessor(XML_Parser parser,
-                    const char *s,
-                    const char *end,
-                    const char **nextPtr)
-{
+prologInitProcessor(XML_Parser parser, const char *s, const char *end,
+                    const char **nextPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -4097,11 +3921,8 @@ prologInitProcessor(XML_Parser parser,
 #ifdef XML_DTD
 
 static enum XML_Error PTRCALL
-externalParEntInitProcessor(XML_Parser parser,
-                            const char *s,
-                            const char *end,
-                            const char **nextPtr)
-{
+externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end,
+                            const char **nextPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -4113,19 +3934,15 @@ externalParEntInitProcessor(XML_Parser parser,
   if (parser->m_prologState.inEntityValue) {
     parser->m_processor = entityValueInitProcessor;
     return entityValueInitProcessor(parser, s, end, nextPtr);
-  }
-  else {
+  } else {
     parser->m_processor = externalParEntProcessor;
     return externalParEntProcessor(parser, s, end, nextPtr);
   }
 }
 
 static enum XML_Error PTRCALL
-entityValueInitProcessor(XML_Parser parser,
-                         const char *s,
-                         const char *end,
-                         const char **nextPtr)
-{
+entityValueInitProcessor(XML_Parser parser, const char *s, const char *end,
+                         const char **nextPtr) {
   int tok;
   const char *start = s;
   const char *next = start;
@@ -4135,7 +3952,7 @@ entityValueInitProcessor(XML_Parser parser,
     tok = XmlPrologTok(parser->m_encoding, start, end, &next);
     parser->m_eventEndPtr = next;
     if (tok <= 0) {
-      if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+      if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -4146,22 +3963,21 @@ entityValueInitProcessor(XML_Parser parser,
         return XML_ERROR_UNCLOSED_TOKEN;
       case XML_TOK_PARTIAL_CHAR:
         return XML_ERROR_PARTIAL_CHAR;
-      case XML_TOK_NONE:   /* start == end */
+      case XML_TOK_NONE: /* start == end */
       default:
         break;
       }
       /* found end of entity value - can store it now */
       return storeEntityValue(parser, parser->m_encoding, s, end);
-    }
-    else if (tok == XML_TOK_XML_DECL) {
+    } else if (tok == XML_TOK_XML_DECL) {
       enum XML_Error result;
       result = processXmlDecl(parser, 0, start, next);
       if (result != XML_ERROR_NONE)
         return result;
-      /* At this point, m_parsingStatus.parsing cannot be XML_SUSPENDED.  For that
-       * to happen, a parameter entity parsing handler must have
-       * attempted to suspend the parser, which fails and raises an
-       * error.  The parser can be aborted, but can't be suspended.
+      /* At this point, m_parsingStatus.parsing cannot be XML_SUSPENDED.  For
+       * that to happen, a parameter entity parsing handler must have attempted
+       * to suspend the parser, which fails and raises an error.  The parser can
+       * be aborted, but can't be suspended.
        */
       if (parser->m_parsingStatus.parsing == XML_FINISHED)
         return XML_ERROR_ABORTED;
@@ -4177,7 +3993,8 @@ entityValueInitProcessor(XML_Parser parser,
        then, when this routine is entered the next time, XmlPrologTok will
        return XML_TOK_INVALID, since the BOM is still in the buffer
     */
-    else if (tok == XML_TOK_BOM && next == end && !parser->m_parsingStatus.finalBuffer) {
+    else if (tok == XML_TOK_BOM && next == end
+             && ! parser->m_parsingStatus.finalBuffer) {
       *nextPtr = next;
       return XML_ERROR_NONE;
     }
@@ -4195,17 +4012,14 @@ entityValueInitProcessor(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalParEntProcessor(XML_Parser parser,
-                        const char *s,
-                        const char *end,
-                        const char **nextPtr)
-{
+externalParEntProcessor(XML_Parser parser, const char *s, const char *end,
+                        const char **nextPtr) {
   const char *next = s;
   int tok;
 
   tok = XmlPrologTok(parser->m_encoding, s, end, &next);
   if (tok <= 0) {
-    if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+    if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
       *nextPtr = s;
       return XML_ERROR_NONE;
     }
@@ -4216,7 +4030,7 @@ externalParEntProcessor(XML_Parser parser,
       return XML_ERROR_UNCLOSED_TOKEN;
     case XML_TOK_PARTIAL_CHAR:
       return XML_ERROR_PARTIAL_CHAR;
-    case XML_TOK_NONE:   /* start == end */
+    case XML_TOK_NONE: /* start == end */
     default:
       break;
     }
@@ -4231,16 +4045,13 @@ externalParEntProcessor(XML_Parser parser,
   }
 
   parser->m_processor = prologProcessor;
-  return doProlog(parser, parser->m_encoding, s, end, tok, next,
-                  nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+  return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
 }
 
 static enum XML_Error PTRCALL
-entityValueProcessor(XML_Parser parser,
-                     const char *s,
-                     const char *end,
-                     const char **nextPtr)
-{
+entityValueProcessor(XML_Parser parser, const char *s, const char *end,
+                     const char **nextPtr) {
   const char *start = s;
   const char *next = s;
   const ENCODING *enc = parser->m_encoding;
@@ -4249,7 +4060,7 @@ entityValueProcessor(XML_Parser parser,
   for (;;) {
     tok = XmlPrologTok(enc, start, end, &next);
     if (tok <= 0) {
-      if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+      if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -4260,7 +4071,7 @@ entityValueProcessor(XML_Parser parser,
         return XML_ERROR_UNCLOSED_TOKEN;
       case XML_TOK_PARTIAL_CHAR:
         return XML_ERROR_PARTIAL_CHAR;
-      case XML_TOK_NONE:   /* start == end */
+      case XML_TOK_NONE: /* start == end */
       default:
         break;
       }
@@ -4274,52 +4085,46 @@ entityValueProcessor(XML_Parser parser,
 #endif /* XML_DTD */
 
 static enum XML_Error PTRCALL
-prologProcessor(XML_Parser parser,
-                const char *s,
-                const char *end,
-                const char **nextPtr)
-{
+prologProcessor(XML_Parser parser, const char *s, const char *end,
+                const char **nextPtr) {
   const char *next = s;
   int tok = XmlPrologTok(parser->m_encoding, s, end, &next);
-  return doProlog(parser, parser->m_encoding, s, end, tok, next,
-                  nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+  return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
 }
 
 static enum XML_Error
-doProlog(XML_Parser parser,
-         const ENCODING *enc,
-         const char *s,
-         const char *end,
-         int tok,
-         const char *next,
-         const char **nextPtr,
-         XML_Bool haveMore)
-{
+doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
+         int tok, const char *next, const char **nextPtr, XML_Bool haveMore,
+         XML_Bool allowClosingDoctype) {
 #ifdef XML_DTD
-  static const XML_Char externalSubsetName[] = { ASCII_HASH , '\0' };
+  static const XML_Char externalSubsetName[] = {ASCII_HASH, '\0'};
 #endif /* XML_DTD */
-  static const XML_Char atypeCDATA[] =
-      { ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-  static const XML_Char atypeID[] = { ASCII_I, ASCII_D, '\0' };
-  static const XML_Char atypeIDREF[] =
-      { ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0' };
-  static const XML_Char atypeIDREFS[] =
-      { ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0' };
-  static const XML_Char atypeENTITY[] =
-      { ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0' };
-  static const XML_Char atypeENTITIES[] = { ASCII_E, ASCII_N,
-      ASCII_T, ASCII_I, ASCII_T, ASCII_I, ASCII_E, ASCII_S, '\0' };
-  static const XML_Char atypeNMTOKEN[] = {
-      ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0' };
-  static const XML_Char atypeNMTOKENS[] = { ASCII_N, ASCII_M, ASCII_T,
-      ASCII_O, ASCII_K, ASCII_E, ASCII_N, ASCII_S, '\0' };
-  static const XML_Char notationPrefix[] = { ASCII_N, ASCII_O, ASCII_T,
-      ASCII_A, ASCII_T, ASCII_I, ASCII_O, ASCII_N, ASCII_LPAREN, '\0' };
-  static const XML_Char enumValueSep[] = { ASCII_PIPE, '\0' };
-  static const XML_Char enumValueStart[] = { ASCII_LPAREN, '\0' };
+  static const XML_Char atypeCDATA[]
+      = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+  static const XML_Char atypeID[] = {ASCII_I, ASCII_D, '\0'};
+  static const XML_Char atypeIDREF[]
+      = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0'};
+  static const XML_Char atypeIDREFS[]
+      = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0'};
+  static const XML_Char atypeENTITY[]
+      = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0'};
+  static const XML_Char atypeENTITIES[]
+      = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T,
+         ASCII_I, ASCII_E, ASCII_S, '\0'};
+  static const XML_Char atypeNMTOKEN[]
+      = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0'};
+  static const XML_Char atypeNMTOKENS[]
+      = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K,
+         ASCII_E, ASCII_N, ASCII_S, '\0'};
+  static const XML_Char notationPrefix[]
+      = {ASCII_N, ASCII_O, ASCII_T, ASCII_A,      ASCII_T,
+         ASCII_I, ASCII_O, ASCII_N, ASCII_LPAREN, '\0'};
+  static const XML_Char enumValueSep[] = {ASCII_PIPE, '\0'};
+  static const XML_Char enumValueStart[] = {ASCII_LPAREN, '\0'};
 
   /* save one level of indirection */
-  DTD * const dtd = parser->m_dtd;
+  DTD *const dtd = parser->m_dtd;
 
   const char **eventPP;
   const char **eventEndPP;
@@ -4328,8 +4133,7 @@ doProlog(XML_Parser parser,
   if (enc == parser->m_encoding) {
     eventPP = &parser->m_eventPtr;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -4358,7 +4162,8 @@ doProlog(XML_Parser parser,
       case XML_TOK_NONE:
 #ifdef XML_DTD
         /* for internal PE NOT referenced between declarations */
-        if (enc != parser->m_encoding && !parser->m_openInternalEntities->betweenDecl) {
+        if (enc != parser->m_encoding
+            && ! parser->m_openInternalEntities->betweenDecl) {
           *nextPtr = s;
           return XML_ERROR_NONE;
         }
@@ -4383,19 +4188,18 @@ doProlog(XML_Parser parser,
     }
     role = XmlTokenRole(&parser->m_prologState, tok, s, next, enc);
     switch (role) {
-    case XML_ROLE_XML_DECL:
-      {
-        enum XML_Error result = processXmlDecl(parser, 0, s, next);
-        if (result != XML_ERROR_NONE)
-          return result;
-        enc = parser->m_encoding;
-        handleDefault = XML_FALSE;
-      }
-      break;
+    case XML_ROLE_XML_DECL: {
+      enum XML_Error result = processXmlDecl(parser, 0, s, next);
+      if (result != XML_ERROR_NONE)
+        return result;
+      enc = parser->m_encoding;
+      handleDefault = XML_FALSE;
+    } break;
     case XML_ROLE_DOCTYPE_NAME:
       if (parser->m_startDoctypeDeclHandler) {
-        parser->m_doctypeName = poolStoreString(&parser->m_tempPool, enc, s, next);
-        if (!parser->m_doctypeName)
+        parser->m_doctypeName
+            = poolStoreString(&parser->m_tempPool, enc, s, next);
+        if (! parser->m_doctypeName)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
         parser->m_doctypePubid = NULL;
@@ -4405,43 +4209,40 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_DOCTYPE_INTERNAL_SUBSET:
       if (parser->m_startDoctypeDeclHandler) {
-        parser->m_startDoctypeDeclHandler(parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
-                                parser->m_doctypePubid, 1);
+        parser->m_startDoctypeDeclHandler(
+            parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
+            parser->m_doctypePubid, 1);
         parser->m_doctypeName = NULL;
         poolClear(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
       break;
 #ifdef XML_DTD
-    case XML_ROLE_TEXT_DECL:
-      {
-        enum XML_Error result = processXmlDecl(parser, 1, s, next);
-        if (result != XML_ERROR_NONE)
-          return result;
-        enc = parser->m_encoding;
-        handleDefault = XML_FALSE;
-      }
-      break;
+    case XML_ROLE_TEXT_DECL: {
+      enum XML_Error result = processXmlDecl(parser, 1, s, next);
+      if (result != XML_ERROR_NONE)
+        return result;
+      enc = parser->m_encoding;
+      handleDefault = XML_FALSE;
+    } break;
 #endif /* XML_DTD */
     case XML_ROLE_DOCTYPE_PUBLIC_ID:
 #ifdef XML_DTD
       parser->m_useForeignDTD = XML_FALSE;
-      parser->m_declEntity = (ENTITY *)lookup(parser,
-                                    &dtd->paramEntities,
-                                    externalSubsetName,
-                                    sizeof(ENTITY));
-      if (!parser->m_declEntity)
+      parser->m_declEntity = (ENTITY *)lookup(
+          parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY));
+      if (! parser->m_declEntity)
         return XML_ERROR_NO_MEMORY;
 #endif /* XML_DTD */
       dtd->hasParamEntityRefs = XML_TRUE;
       if (parser->m_startDoctypeDeclHandler) {
         XML_Char *pubId;
-        if (!XmlIsPublicId(enc, s, next, eventPP))
+        if (! XmlIsPublicId(enc, s, next, eventPP))
           return XML_ERROR_PUBLICID;
         pubId = poolStoreString(&parser->m_tempPool, enc,
                                 s + enc->minBytesPerChar,
                                 next - enc->minBytesPerChar);
-        if (!pubId)
+        if (! pubId)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(pubId);
         poolFinish(&parser->m_tempPool);
@@ -4451,15 +4252,14 @@ doProlog(XML_Parser parser,
       }
       /* fall through */
     case XML_ROLE_ENTITY_PUBLIC_ID:
-      if (!XmlIsPublicId(enc, s, next, eventPP))
+      if (! XmlIsPublicId(enc, s, next, eventPP))
         return XML_ERROR_PUBLICID;
     alreadyChecked:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        XML_Char *tem = poolStoreString(&dtd->pool,
-                                        enc,
-                                        s + enc->minBytesPerChar,
-                                        next - enc->minBytesPerChar);
-        if (!tem)
+        XML_Char *tem
+            = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                              next - enc->minBytesPerChar);
+        if (! tem)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(tem);
         parser->m_declEntity->publicId = tem;
@@ -4472,9 +4272,15 @@ doProlog(XML_Parser parser,
       }
       break;
     case XML_ROLE_DOCTYPE_CLOSE:
+      if (allowClosingDoctype != XML_TRUE) {
+        /* Must not close doctype from within expanded parameter entities */
+        return XML_ERROR_INVALID_TOKEN;
+      }
+
       if (parser->m_doctypeName) {
-        parser->m_startDoctypeDeclHandler(parser->m_handlerArg, parser->m_doctypeName,
-                                parser->m_doctypeSysid, parser->m_doctypePubid, 0);
+        parser->m_startDoctypeDeclHandler(
+            parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
+            parser->m_doctypePubid, 0);
         poolClear(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
@@ -4486,12 +4292,11 @@ doProlog(XML_Parser parser,
       if (parser->m_doctypeSysid || parser->m_useForeignDTD) {
         XML_Bool hadParamEntityRefs = dtd->hasParamEntityRefs;
         dtd->hasParamEntityRefs = XML_TRUE;
-        if (parser->m_paramEntityParsing && parser->m_externalEntityRefHandler) {
-          ENTITY *entity = (ENTITY *)lookup(parser,
-                                            &dtd->paramEntities,
-                                            externalSubsetName,
-                                            sizeof(ENTITY));
-          if (!entity) {
+        if (parser->m_paramEntityParsing
+            && parser->m_externalEntityRefHandler) {
+          ENTITY *entity = (ENTITY *)lookup(parser, &dtd->paramEntities,
+                                            externalSubsetName, sizeof(ENTITY));
+          if (! entity) {
             /* The external subset name "#" will have already been
              * inserted into the hash table at the start of the
              * external entity parsing, so no allocation will happen
@@ -4502,22 +4307,19 @@ doProlog(XML_Parser parser,
           if (parser->m_useForeignDTD)
             entity->base = parser->m_curBase;
           dtd->paramEntityRead = XML_FALSE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId))
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           if (dtd->paramEntityRead) {
-            if (!dtd->standalone &&
-                parser->m_notStandaloneHandler &&
-                !parser->m_notStandaloneHandler(parser->m_handlerArg))
+            if (! dtd->standalone && parser->m_notStandaloneHandler
+                && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
               return XML_ERROR_NOT_STANDALONE;
           }
           /* if we didn't read the foreign DTD then this means that there
              is no external subset and we must reset dtd->hasParamEntityRefs
           */
-          else if (!parser->m_doctypeSysid)
+          else if (! parser->m_doctypeSysid)
             dtd->hasParamEntityRefs = hadParamEntityRefs;
           /* end of DTD - no need to update dtd->keepProcessing */
         }
@@ -4537,24 +4339,21 @@ doProlog(XML_Parser parser,
       if (parser->m_useForeignDTD) {
         XML_Bool hadParamEntityRefs = dtd->hasParamEntityRefs;
         dtd->hasParamEntityRefs = XML_TRUE;
-        if (parser->m_paramEntityParsing && parser->m_externalEntityRefHandler) {
+        if (parser->m_paramEntityParsing
+            && parser->m_externalEntityRefHandler) {
           ENTITY *entity = (ENTITY *)lookup(parser, &dtd->paramEntities,
-                                            externalSubsetName,
-                                            sizeof(ENTITY));
-          if (!entity)
+                                            externalSubsetName, sizeof(ENTITY));
+          if (! entity)
             return XML_ERROR_NO_MEMORY;
           entity->base = parser->m_curBase;
           dtd->paramEntityRead = XML_FALSE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId))
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           if (dtd->paramEntityRead) {
-            if (!dtd->standalone &&
-                parser->m_notStandaloneHandler &&
-                !parser->m_notStandaloneHandler(parser->m_handlerArg))
+            if (! dtd->standalone && parser->m_notStandaloneHandler
+                && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
               return XML_ERROR_NOT_STANDALONE;
           }
           /* if we didn't read the foreign DTD then this means that there
@@ -4570,12 +4369,12 @@ doProlog(XML_Parser parser,
       return contentProcessor(parser, s, end, nextPtr);
     case XML_ROLE_ATTLIST_ELEMENT_NAME:
       parser->m_declElementType = getElementType(parser, enc, s, next);
-      if (!parser->m_declElementType)
+      if (! parser->m_declElementType)
         return XML_ERROR_NO_MEMORY;
       goto checkAttListDeclHandler;
     case XML_ROLE_ATTRIBUTE_NAME:
       parser->m_declAttributeId = getAttributeId(parser, enc, s, next);
-      if (!parser->m_declAttributeId)
+      if (! parser->m_declAttributeId)
         return XML_ERROR_NO_MEMORY;
       parser->m_declAttributeIsCdata = XML_FALSE;
       parser->m_declAttributeType = NULL;
@@ -4616,15 +4415,13 @@ doProlog(XML_Parser parser,
         const XML_Char *prefix;
         if (parser->m_declAttributeType) {
           prefix = enumValueSep;
+        } else {
+          prefix = (role == XML_ROLE_ATTRIBUTE_NOTATION_VALUE ? notationPrefix
+                                                              : enumValueStart);
         }
-        else {
-          prefix = (role == XML_ROLE_ATTRIBUTE_NOTATION_VALUE
-                    ? notationPrefix
-                    : enumValueStart);
-        }
-        if (!poolAppendString(&parser->m_tempPool, prefix))
+        if (! poolAppendString(&parser->m_tempPool, prefix))
           return XML_ERROR_NO_MEMORY;
-        if (!poolAppend(&parser->m_tempPool, enc, s, next))
+        if (! poolAppend(&parser->m_tempPool, enc, s, next))
           return XML_ERROR_NO_MEMORY;
         parser->m_declAttributeType = parser->m_tempPool.start;
         handleDefault = XML_FALSE;
@@ -4633,25 +4430,27 @@ doProlog(XML_Parser parser,
     case XML_ROLE_IMPLIED_ATTRIBUTE_VALUE:
     case XML_ROLE_REQUIRED_ATTRIBUTE_VALUE:
       if (dtd->keepProcessing) {
-        if (!defineAttribute(parser->m_declElementType, parser->m_declAttributeId,
-                             parser->m_declAttributeIsCdata, parser->m_declAttributeIsId,
-                             0, parser))
+        if (! defineAttribute(parser->m_declElementType,
+                              parser->m_declAttributeId,
+                              parser->m_declAttributeIsCdata,
+                              parser->m_declAttributeIsId, 0, parser))
           return XML_ERROR_NO_MEMORY;
         if (parser->m_attlistDeclHandler && parser->m_declAttributeType) {
           if (*parser->m_declAttributeType == XML_T(ASCII_LPAREN)
               || (*parser->m_declAttributeType == XML_T(ASCII_N)
                   && parser->m_declAttributeType[1] == XML_T(ASCII_O))) {
             /* Enumerated or Notation type */
-            if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
-                || !poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+            if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
+                || ! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
               return XML_ERROR_NO_MEMORY;
             parser->m_declAttributeType = parser->m_tempPool.start;
             poolFinish(&parser->m_tempPool);
           }
           *eventEndPP = s;
-          parser->m_attlistDeclHandler(parser->m_handlerArg, parser->m_declElementType->name,
-                             parser->m_declAttributeId->name, parser->m_declAttributeType,
-                             0, role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE);
+          parser->m_attlistDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name,
+              parser->m_declAttributeId->name, parser->m_declAttributeType, 0,
+              role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE);
           poolClear(&parser->m_tempPool);
           handleDefault = XML_FALSE;
         }
@@ -4661,35 +4460,34 @@ doProlog(XML_Parser parser,
     case XML_ROLE_FIXED_ATTRIBUTE_VALUE:
       if (dtd->keepProcessing) {
         const XML_Char *attVal;
-        enum XML_Error result =
-          storeAttributeValue(parser, enc, parser->m_declAttributeIsCdata,
-                              s + enc->minBytesPerChar,
-                              next - enc->minBytesPerChar,
-                              &dtd->pool);
+        enum XML_Error result = storeAttributeValue(
+            parser, enc, parser->m_declAttributeIsCdata,
+            s + enc->minBytesPerChar, next - enc->minBytesPerChar, &dtd->pool);
         if (result)
           return result;
         attVal = poolStart(&dtd->pool);
         poolFinish(&dtd->pool);
         /* ID attributes aren't allowed to have a default */
-        if (!defineAttribute(parser->m_declElementType, parser->m_declAttributeId,
-                             parser->m_declAttributeIsCdata, XML_FALSE, attVal, parser))
+        if (! defineAttribute(
+                parser->m_declElementType, parser->m_declAttributeId,
+                parser->m_declAttributeIsCdata, XML_FALSE, attVal, parser))
           return XML_ERROR_NO_MEMORY;
         if (parser->m_attlistDeclHandler && parser->m_declAttributeType) {
           if (*parser->m_declAttributeType == XML_T(ASCII_LPAREN)
               || (*parser->m_declAttributeType == XML_T(ASCII_N)
                   && parser->m_declAttributeType[1] == XML_T(ASCII_O))) {
             /* Enumerated or Notation type */
-            if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
-                || !poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+            if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
+                || ! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
               return XML_ERROR_NO_MEMORY;
             parser->m_declAttributeType = parser->m_tempPool.start;
             poolFinish(&parser->m_tempPool);
           }
           *eventEndPP = s;
-          parser->m_attlistDeclHandler(parser->m_handlerArg, parser->m_declElementType->name,
-                             parser->m_declAttributeId->name, parser->m_declAttributeType,
-                             attVal,
-                             role == XML_ROLE_FIXED_ATTRIBUTE_VALUE);
+          parser->m_attlistDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name,
+              parser->m_declAttributeId->name, parser->m_declAttributeType,
+              attVal, role == XML_ROLE_FIXED_ATTRIBUTE_VALUE);
           poolClear(&parser->m_tempPool);
           handleDefault = XML_FALSE;
         }
@@ -4697,25 +4495,22 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_ENTITY_VALUE:
       if (dtd->keepProcessing) {
-        enum XML_Error result = storeEntityValue(parser, enc,
-                                            s + enc->minBytesPerChar,
-                                            next - enc->minBytesPerChar);
+        enum XML_Error result = storeEntityValue(
+            parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar);
         if (parser->m_declEntity) {
           parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool);
-          parser->m_declEntity->textLen = (int)(poolLength(&dtd->entityValuePool));
+          parser->m_declEntity->textLen
+              = (int)(poolLength(&dtd->entityValuePool));
           poolFinish(&dtd->entityValuePool);
           if (parser->m_entityDeclHandler) {
             *eventEndPP = s;
-            parser->m_entityDeclHandler(parser->m_handlerArg,
-                              parser->m_declEntity->name,
-                              parser->m_declEntity->is_param,
-                              parser->m_declEntity->textPtr,
-                              parser->m_declEntity->textLen,
-                              parser->m_curBase, 0, 0, 0);
+            parser->m_entityDeclHandler(
+                parser->m_handlerArg, parser->m_declEntity->name,
+                parser->m_declEntity->is_param, parser->m_declEntity->textPtr,
+                parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0);
             handleDefault = XML_FALSE;
           }
-        }
-        else
+        } else
           poolDiscard(&dtd->entityValuePool);
         if (result != XML_ERROR_NONE)
           return result;
@@ -4728,8 +4523,8 @@ doProlog(XML_Parser parser,
       dtd->hasParamEntityRefs = XML_TRUE;
       if (parser->m_startDoctypeDeclHandler) {
         parser->m_doctypeSysid = poolStoreString(&parser->m_tempPool, enc,
-                                       s + enc->minBytesPerChar,
-                                       next - enc->minBytesPerChar);
+                                                 s + enc->minBytesPerChar,
+                                                 next - enc->minBytesPerChar);
         if (parser->m_doctypeSysid == NULL)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
@@ -4741,22 +4536,20 @@ doProlog(XML_Parser parser,
            for the case where no parser->m_startDoctypeDeclHandler is set */
         parser->m_doctypeSysid = externalSubsetName;
 #endif /* XML_DTD */
-      if (!dtd->standalone
+      if (! dtd->standalone
 #ifdef XML_DTD
-          && !parser->m_paramEntityParsing
+          && ! parser->m_paramEntityParsing
 #endif /* XML_DTD */
           && parser->m_notStandaloneHandler
-          && !parser->m_notStandaloneHandler(parser->m_handlerArg))
+          && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
         return XML_ERROR_NOT_STANDALONE;
 #ifndef XML_DTD
       break;
-#else /* XML_DTD */
-      if (!parser->m_declEntity) {
-        parser->m_declEntity = (ENTITY *)lookup(parser,
-                                      &dtd->paramEntities,
-                                      externalSubsetName,
-                                      sizeof(ENTITY));
-        if (!parser->m_declEntity)
+#else  /* XML_DTD */
+      if (! parser->m_declEntity) {
+        parser->m_declEntity = (ENTITY *)lookup(
+            parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY));
+        if (! parser->m_declEntity)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity->publicId = NULL;
       }
@@ -4764,10 +4557,10 @@ doProlog(XML_Parser parser,
       /* fall through */
     case XML_ROLE_ENTITY_SYSTEM_ID:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        parser->m_declEntity->systemId = poolStoreString(&dtd->pool, enc,
-                                               s + enc->minBytesPerChar,
-                                               next - enc->minBytesPerChar);
-        if (!parser->m_declEntity->systemId)
+        parser->m_declEntity->systemId
+            = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                              next - enc->minBytesPerChar);
+        if (! parser->m_declEntity->systemId)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity->base = parser->m_curBase;
         poolFinish(&dtd->pool);
@@ -4779,115 +4572,103 @@ doProlog(XML_Parser parser,
       }
       break;
     case XML_ROLE_ENTITY_COMPLETE:
-      if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) {
+      if (dtd->keepProcessing && parser->m_declEntity
+          && parser->m_entityDeclHandler) {
         *eventEndPP = s;
-        parser->m_entityDeclHandler(parser->m_handlerArg,
-                          parser->m_declEntity->name,
-                          parser->m_declEntity->is_param,
-                          0,0,
-                          parser->m_declEntity->base,
-                          parser->m_declEntity->systemId,
-                          parser->m_declEntity->publicId,
-                          0);
+        parser->m_entityDeclHandler(
+            parser->m_handlerArg, parser->m_declEntity->name,
+            parser->m_declEntity->is_param, 0, 0, parser->m_declEntity->base,
+            parser->m_declEntity->systemId, parser->m_declEntity->publicId, 0);
         handleDefault = XML_FALSE;
       }
       break;
     case XML_ROLE_ENTITY_NOTATION_NAME:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        parser->m_declEntity->notation = poolStoreString(&dtd->pool, enc, s, next);
-        if (!parser->m_declEntity->notation)
+        parser->m_declEntity->notation
+            = poolStoreString(&dtd->pool, enc, s, next);
+        if (! parser->m_declEntity->notation)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&dtd->pool);
         if (parser->m_unparsedEntityDeclHandler) {
           *eventEndPP = s;
-          parser->m_unparsedEntityDeclHandler(parser->m_handlerArg,
-                                    parser->m_declEntity->name,
-                                    parser->m_declEntity->base,
-                                    parser->m_declEntity->systemId,
-                                    parser->m_declEntity->publicId,
-                                    parser->m_declEntity->notation);
+          parser->m_unparsedEntityDeclHandler(
+              parser->m_handlerArg, parser->m_declEntity->name,
+              parser->m_declEntity->base, parser->m_declEntity->systemId,
+              parser->m_declEntity->publicId, parser->m_declEntity->notation);
           handleDefault = XML_FALSE;
-        }
-        else if (parser->m_entityDeclHandler) {
+        } else if (parser->m_entityDeclHandler) {
           *eventEndPP = s;
-          parser->m_entityDeclHandler(parser->m_handlerArg,
-                            parser->m_declEntity->name,
-                            0,0,0,
-                            parser->m_declEntity->base,
-                            parser->m_declEntity->systemId,
-                            parser->m_declEntity->publicId,
-                            parser->m_declEntity->notation);
+          parser->m_entityDeclHandler(
+              parser->m_handlerArg, parser->m_declEntity->name, 0, 0, 0,
+              parser->m_declEntity->base, parser->m_declEntity->systemId,
+              parser->m_declEntity->publicId, parser->m_declEntity->notation);
           handleDefault = XML_FALSE;
         }
       }
       break;
-    case XML_ROLE_GENERAL_ENTITY_NAME:
-      {
-        if (XmlPredefinedEntityName(enc, s, next)) {
-          parser->m_declEntity = NULL;
-          break;
-        }
-        if (dtd->keepProcessing) {
-          const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
-          if (!name)
-            return XML_ERROR_NO_MEMORY;
-          parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->generalEntities, name,
-                                        sizeof(ENTITY));
-          if (!parser->m_declEntity)
-            return XML_ERROR_NO_MEMORY;
-          if (parser->m_declEntity->name != name) {
-            poolDiscard(&dtd->pool);
-            parser->m_declEntity = NULL;
-          }
-          else {
-            poolFinish(&dtd->pool);
-            parser->m_declEntity->publicId = NULL;
-            parser->m_declEntity->is_param = XML_FALSE;
-            /* if we have a parent parser or are reading an internal parameter
-               entity, then the entity declaration is not considered "internal"
-            */
-            parser->m_declEntity->is_internal = !(parser->m_parentParser || parser->m_openInternalEntities);
-            if (parser->m_entityDeclHandler)
-              handleDefault = XML_FALSE;
-          }
-        }
-        else {
+    case XML_ROLE_GENERAL_ENTITY_NAME: {
+      if (XmlPredefinedEntityName(enc, s, next)) {
+        parser->m_declEntity = NULL;
+        break;
+      }
+      if (dtd->keepProcessing) {
+        const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
+        if (! name)
+          return XML_ERROR_NO_MEMORY;
+        parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->generalEntities,
+                                                name, sizeof(ENTITY));
+        if (! parser->m_declEntity)
+          return XML_ERROR_NO_MEMORY;
+        if (parser->m_declEntity->name != name) {
           poolDiscard(&dtd->pool);
           parser->m_declEntity = NULL;
+        } else {
+          poolFinish(&dtd->pool);
+          parser->m_declEntity->publicId = NULL;
+          parser->m_declEntity->is_param = XML_FALSE;
+          /* if we have a parent parser or are reading an internal parameter
+             entity, then the entity declaration is not considered "internal"
+          */
+          parser->m_declEntity->is_internal
+              = ! (parser->m_parentParser || parser->m_openInternalEntities);
+          if (parser->m_entityDeclHandler)
+            handleDefault = XML_FALSE;
         }
+      } else {
+        poolDiscard(&dtd->pool);
+        parser->m_declEntity = NULL;
       }
-      break;
+    } break;
     case XML_ROLE_PARAM_ENTITY_NAME:
 #ifdef XML_DTD
       if (dtd->keepProcessing) {
         const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
-        if (!name)
+        if (! name)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->paramEntities,
-                                           name, sizeof(ENTITY));
-        if (!parser->m_declEntity)
+                                                name, sizeof(ENTITY));
+        if (! parser->m_declEntity)
           return XML_ERROR_NO_MEMORY;
         if (parser->m_declEntity->name != name) {
           poolDiscard(&dtd->pool);
           parser->m_declEntity = NULL;
-        }
-        else {
+        } else {
           poolFinish(&dtd->pool);
           parser->m_declEntity->publicId = NULL;
           parser->m_declEntity->is_param = XML_TRUE;
           /* if we have a parent parser or are reading an internal parameter
              entity, then the entity declaration is not considered "internal"
           */
-          parser->m_declEntity->is_internal = !(parser->m_parentParser || parser->m_openInternalEntities);
+          parser->m_declEntity->is_internal
+              = ! (parser->m_parentParser || parser->m_openInternalEntities);
           if (parser->m_entityDeclHandler)
             handleDefault = XML_FALSE;
         }
-      }
-      else {
+      } else {
         poolDiscard(&dtd->pool);
         parser->m_declEntity = NULL;
       }
-#else /* not XML_DTD */
+#else  /* not XML_DTD */
       parser->m_declEntity = NULL;
 #endif /* XML_DTD */
       break;
@@ -4895,22 +4676,23 @@ doProlog(XML_Parser parser,
       parser->m_declNotationPublicId = NULL;
       parser->m_declNotationName = NULL;
       if (parser->m_notationDeclHandler) {
-        parser->m_declNotationName = poolStoreString(&parser->m_tempPool, enc, s, next);
-        if (!parser->m_declNotationName)
+        parser->m_declNotationName
+            = poolStoreString(&parser->m_tempPool, enc, s, next);
+        if (! parser->m_declNotationName)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
       break;
     case XML_ROLE_NOTATION_PUBLIC_ID:
-      if (!XmlIsPublicId(enc, s, next, eventPP))
+      if (! XmlIsPublicId(enc, s, next, eventPP))
         return XML_ERROR_PUBLICID;
-      if (parser->m_declNotationName) {  /* means m_notationDeclHandler != NULL */
-        XML_Char *tem = poolStoreString(&parser->m_tempPool,
-                                        enc,
+      if (parser
+              ->m_declNotationName) { /* means m_notationDeclHandler != NULL */
+        XML_Char *tem = poolStoreString(&parser->m_tempPool, enc,
                                         s + enc->minBytesPerChar,
                                         next - enc->minBytesPerChar);
-        if (!tem)
+        if (! tem)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(tem);
         parser->m_declNotationPublicId = tem;
@@ -4920,18 +4702,15 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_NOTATION_SYSTEM_ID:
       if (parser->m_declNotationName && parser->m_notationDeclHandler) {
-        const XML_Char *systemId
-          = poolStoreString(&parser->m_tempPool, enc,
-                            s + enc->minBytesPerChar,
-                            next - enc->minBytesPerChar);
-        if (!systemId)
+        const XML_Char *systemId = poolStoreString(&parser->m_tempPool, enc,
+                                                   s + enc->minBytesPerChar,
+                                                   next - enc->minBytesPerChar);
+        if (! systemId)
           return XML_ERROR_NO_MEMORY;
         *eventEndPP = s;
-        parser->m_notationDeclHandler(parser->m_handlerArg,
-                            parser->m_declNotationName,
-                            parser->m_curBase,
-                            systemId,
-                            parser->m_declNotationPublicId);
+        parser->m_notationDeclHandler(
+            parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase,
+            systemId, parser->m_declNotationPublicId);
         handleDefault = XML_FALSE;
       }
       poolClear(&parser->m_tempPool);
@@ -4939,11 +4718,9 @@ doProlog(XML_Parser parser,
     case XML_ROLE_NOTATION_NO_SYSTEM_ID:
       if (parser->m_declNotationPublicId && parser->m_notationDeclHandler) {
         *eventEndPP = s;
-        parser->m_notationDeclHandler(parser->m_handlerArg,
-                            parser->m_declNotationName,
-                            parser->m_curBase,
-                            0,
-                            parser->m_declNotationPublicId);
+        parser->m_notationDeclHandler(
+            parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase,
+            0, parser->m_declNotationPublicId);
         handleDefault = XML_FALSE;
       }
       poolClear(&parser->m_tempPool);
@@ -4960,42 +4737,44 @@ doProlog(XML_Parser parser,
         return XML_ERROR_SYNTAX;
       }
 #ifdef XML_DTD
-    case XML_ROLE_IGNORE_SECT:
-      {
-        enum XML_Error result;
-        if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        handleDefault = XML_FALSE;
-        result = doIgnoreSection(parser, enc, &next, end, nextPtr, haveMore);
-        if (result != XML_ERROR_NONE)
-          return result;
-        else if (!next) {
-          parser->m_processor = ignoreSectionProcessor;
-          return result;
-        }
+    case XML_ROLE_IGNORE_SECT: {
+      enum XML_Error result;
+      if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      handleDefault = XML_FALSE;
+      result = doIgnoreSection(parser, enc, &next, end, nextPtr, haveMore);
+      if (result != XML_ERROR_NONE)
+        return result;
+      else if (! next) {
+        parser->m_processor = ignoreSectionProcessor;
+        return result;
       }
-      break;
+    } break;
 #endif /* XML_DTD */
     case XML_ROLE_GROUP_OPEN:
       if (parser->m_prologState.level >= parser->m_groupSize) {
         if (parser->m_groupSize) {
-          char *temp = (char *)REALLOC(parser, parser->m_groupConnector, parser->m_groupSize *= 2);
-          if (temp == NULL) {
-            parser->m_groupSize /= 2;
-            return XML_ERROR_NO_MEMORY;
+          {
+            char *const new_connector = (char *)REALLOC(
+                parser, parser->m_groupConnector, parser->m_groupSize *= 2);
+            if (new_connector == NULL) {
+              parser->m_groupSize /= 2;
+              return XML_ERROR_NO_MEMORY;
+            }
+            parser->m_groupConnector = new_connector;
           }
-          parser->m_groupConnector = temp;
+
           if (dtd->scaffIndex) {
-            int *temp = (int *)REALLOC(parser, dtd->scaffIndex,
-                          parser->m_groupSize * sizeof(int));
-            if (temp == NULL)
+            int *const new_scaff_index = (int *)REALLOC(
+                parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int));
+            if (new_scaff_index == NULL)
               return XML_ERROR_NO_MEMORY;
-            dtd->scaffIndex = temp;
+            dtd->scaffIndex = new_scaff_index;
           }
-        }
-        else {
-          parser->m_groupConnector = (char *)MALLOC(parser, parser->m_groupSize = 32);
-          if (!parser->m_groupConnector) {
+        } else {
+          parser->m_groupConnector
+              = (char *)MALLOC(parser, parser->m_groupSize = 32);
+          if (! parser->m_groupConnector) {
             parser->m_groupSize = 0;
             return XML_ERROR_NO_MEMORY;
           }
@@ -5006,6 +4785,7 @@ doProlog(XML_Parser parser,
         int myindex = nextScaffoldPart(parser);
         if (myindex < 0)
           return XML_ERROR_NO_MEMORY;
+        assert(dtd->scaffIndex != NULL);
         dtd->scaffIndex[dtd->scaffLevel] = myindex;
         dtd->scaffLevel++;
         dtd->scaffold[myindex].type = XML_CTYPE_SEQ;
@@ -5024,10 +4804,9 @@ doProlog(XML_Parser parser,
       if (parser->m_groupConnector[parser->m_prologState.level] == ASCII_COMMA)
         return XML_ERROR_SYNTAX;
       if (dtd->in_eldecl
-          && !parser->m_groupConnector[parser->m_prologState.level]
+          && ! parser->m_groupConnector[parser->m_prologState.level]
           && (dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]].type
-              != XML_CTYPE_MIXED)
-          ) {
+              != XML_CTYPE_MIXED)) {
         dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]].type
             = XML_CTYPE_CHOICE;
         if (parser->m_elementDeclHandler)
@@ -5039,15 +4818,14 @@ doProlog(XML_Parser parser,
 #ifdef XML_DTD
     case XML_ROLE_INNER_PARAM_ENTITY_REF:
       dtd->hasParamEntityRefs = XML_TRUE;
-      if (!parser->m_paramEntityParsing)
+      if (! parser->m_paramEntityParsing)
         dtd->keepProcessing = dtd->standalone;
       else {
         const XML_Char *name;
         ENTITY *entity;
-        name = poolStoreString(&dtd->pool, enc,
-                                s + enc->minBytesPerChar,
-                                next - enc->minBytesPerChar);
-        if (!name)
+        name = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                               next - enc->minBytesPerChar);
+        if (! name)
           return XML_ERROR_NO_MEMORY;
         entity = (ENTITY *)lookup(parser, &dtd->paramEntities, name, 0);
         poolDiscard(&dtd->pool);
@@ -5055,13 +4833,12 @@ doProlog(XML_Parser parser,
            if yes, check that the entity exists, and that it is internal,
            otherwise call the skipped entity handler
         */
-        if (parser->m_prologState.documentEntity &&
-            (dtd->standalone
-             ? !parser->m_openInternalEntities
-             : !dtd->hasParamEntityRefs)) {
-          if (!entity)
+        if (parser->m_prologState.documentEntity
+            && (dtd->standalone ? ! parser->m_openInternalEntities
+                                : ! dtd->hasParamEntityRefs)) {
+          if (! entity)
             return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal) {
+          else if (! entity->is_internal) {
             /* It's hard to exhaustively search the code to be sure,
              * but there doesn't seem to be a way of executing the
              * following line.  There are two cases:
@@ -5084,11 +4861,11 @@ doProlog(XML_Parser parser,
              */
             return XML_ERROR_ENTITY_DECLARED_IN_PE; /* LCOV_EXCL_LINE */
           }
-        }
-        else if (!entity) {
+        } else if (! entity) {
           dtd->keepProcessing = dtd->standalone;
           /* cannot report skipped entities in declarations */
-          if ((role == XML_ROLE_PARAM_ENTITY_REF) && parser->m_skippedEntityHandler) {
+          if ((role == XML_ROLE_PARAM_ENTITY_REF)
+              && parser->m_skippedEntityHandler) {
             parser->m_skippedEntityHandler(parser->m_handlerArg, name, 1);
             handleDefault = XML_FALSE;
           }
@@ -5098,8 +4875,8 @@ doProlog(XML_Parser parser,
           return XML_ERROR_RECURSIVE_ENTITY_REF;
         if (entity->textPtr) {
           enum XML_Error result;
-          XML_Bool betweenDecl =
-            (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE);
+          XML_Bool betweenDecl
+              = (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE);
           result = processInternalEntity(parser, entity, betweenDecl);
           if (result != XML_ERROR_NONE)
             return result;
@@ -5109,39 +4886,35 @@ doProlog(XML_Parser parser,
         if (parser->m_externalEntityRefHandler) {
           dtd->paramEntityRead = XML_FALSE;
           entity->open = XML_TRUE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId)) {
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId)) {
             entity->open = XML_FALSE;
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           }
           entity->open = XML_FALSE;
           handleDefault = XML_FALSE;
-          if (!dtd->paramEntityRead) {
+          if (! dtd->paramEntityRead) {
             dtd->keepProcessing = dtd->standalone;
             break;
           }
-        }
-        else {
+        } else {
           dtd->keepProcessing = dtd->standalone;
           break;
         }
       }
 #endif /* XML_DTD */
-      if (!dtd->standalone &&
-          parser->m_notStandaloneHandler &&
-          !parser->m_notStandaloneHandler(parser->m_handlerArg))
+      if (! dtd->standalone && parser->m_notStandaloneHandler
+          && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
         return XML_ERROR_NOT_STANDALONE;
       break;
 
-    /* Element declaration stuff */
+      /* Element declaration stuff */
 
     case XML_ROLE_ELEMENT_NAME:
       if (parser->m_elementDeclHandler) {
         parser->m_declElementType = getElementType(parser, enc, s, next);
-        if (!parser->m_declElementType)
+        if (! parser->m_declElementType)
           return XML_ERROR_NO_MEMORY;
         dtd->scaffLevel = 0;
         dtd->scaffCount = 0;
@@ -5154,18 +4927,19 @@ doProlog(XML_Parser parser,
     case XML_ROLE_CONTENT_EMPTY:
       if (dtd->in_eldecl) {
         if (parser->m_elementDeclHandler) {
-          XML_Content * content = (XML_Content *) MALLOC(parser, sizeof(XML_Content));
-          if (!content)
+          XML_Content *content
+              = (XML_Content *)MALLOC(parser, sizeof(XML_Content));
+          if (! content)
             return XML_ERROR_NO_MEMORY;
           content->quant = XML_CQUANT_NONE;
           content->name = NULL;
           content->numchildren = 0;
           content->children = NULL;
-          content->type = ((role == XML_ROLE_CONTENT_ANY) ?
-                           XML_CTYPE_ANY :
-                           XML_CTYPE_EMPTY);
+          content->type = ((role == XML_ROLE_CONTENT_ANY) ? XML_CTYPE_ANY
+                                                          : XML_CTYPE_EMPTY);
           *eventEndPP = s;
-          parser->m_elementDeclHandler(parser->m_handlerArg, parser->m_declElementType->name, content);
+          parser->m_elementDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name, content);
           handleDefault = XML_FALSE;
         }
         dtd->in_eldecl = XML_FALSE;
@@ -5197,22 +4971,22 @@ doProlog(XML_Parser parser,
         ELEMENT_TYPE *el;
         const XML_Char *name;
         int nameLen;
-        const char *nxt = (quant == XML_CQUANT_NONE
-                           ? next
-                           : next - enc->minBytesPerChar);
+        const char *nxt
+            = (quant == XML_CQUANT_NONE ? next : next - enc->minBytesPerChar);
         int myindex = nextScaffoldPart(parser);
         if (myindex < 0)
           return XML_ERROR_NO_MEMORY;
         dtd->scaffold[myindex].type = XML_CTYPE_NAME;
         dtd->scaffold[myindex].quant = quant;
         el = getElementType(parser, enc, s, nxt);
-        if (!el)
+        if (! el)
           return XML_ERROR_NO_MEMORY;
         name = el->name;
         dtd->scaffold[myindex].name = name;
         nameLen = 0;
-        for (; name[nameLen++]; );
-        dtd->contentStringLen +=  nameLen;
+        for (; name[nameLen++];)
+          ;
+        dtd->contentStringLen += nameLen;
         if (parser->m_elementDeclHandler)
           handleDefault = XML_FALSE;
       }
@@ -5236,12 +5010,13 @@ doProlog(XML_Parser parser,
         dtd->scaffLevel--;
         dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel]].quant = quant;
         if (dtd->scaffLevel == 0) {
-          if (!handleDefault) {
+          if (! handleDefault) {
             XML_Content *model = build_model(parser);
-            if (!model)
+            if (! model)
               return XML_ERROR_NO_MEMORY;
             *eventEndPP = s;
-            parser->m_elementDeclHandler(parser->m_handlerArg, parser->m_declElementType->name, model);
+            parser->m_elementDeclHandler(
+                parser->m_handlerArg, parser->m_declElementType->name, model);
           }
           dtd->in_eldecl = XML_FALSE;
           dtd->contentStringLen = 0;
@@ -5251,12 +5026,12 @@ doProlog(XML_Parser parser,
       /* End element declaration stuff */
 
     case XML_ROLE_PI:
-      if (!reportProcessingInstruction(parser, enc, s, next))
+      if (! reportProcessingInstruction(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       handleDefault = XML_FALSE;
       break;
     case XML_ROLE_COMMENT:
-      if (!reportComment(parser, enc, s, next))
+      if (! reportComment(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       handleDefault = XML_FALSE;
       break;
@@ -5307,11 +5082,8 @@ doProlog(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-epilogProcessor(XML_Parser parser,
-                const char *s,
-                const char *end,
-                const char **nextPtr)
-{
+epilogProcessor(XML_Parser parser, const char *s, const char *end,
+                const char **nextPtr) {
   parser->m_processor = epilogProcessor;
   parser->m_eventPtr = s;
   for (;;) {
@@ -5336,24 +5108,24 @@ epilogProcessor(XML_Parser parser,
         reportDefault(parser, parser->m_encoding, s, next);
       break;
     case XML_TOK_PI:
-      if (!reportProcessingInstruction(parser, parser->m_encoding, s, next))
+      if (! reportProcessingInstruction(parser, parser->m_encoding, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_COMMENT:
-      if (!reportComment(parser, parser->m_encoding, s, next))
+      if (! reportComment(parser, parser->m_encoding, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_INVALID:
       parser->m_eventPtr = next;
       return XML_ERROR_INVALID_TOKEN;
     case XML_TOK_PARTIAL:
-      if (!parser->m_parsingStatus.finalBuffer) {
+      if (! parser->m_parsingStatus.finalBuffer) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
       return XML_ERROR_UNCLOSED_TOKEN;
     case XML_TOK_PARTIAL_CHAR:
-      if (!parser->m_parsingStatus.finalBuffer) {
+      if (! parser->m_parsingStatus.finalBuffer) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -5368,15 +5140,13 @@ epilogProcessor(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
 }
 
 static enum XML_Error
-processInternalEntity(XML_Parser parser, ENTITY *entity,
-                      XML_Bool betweenDecl)
-{
+processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) {
   const char *textStart, *textEnd;
   const char *next;
   enum XML_Error result;
@@ -5385,10 +5155,10 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
   if (parser->m_freeInternalEntities) {
     openEntity = parser->m_freeInternalEntities;
     parser->m_freeInternalEntities = openEntity->next;
-  }
-  else {
-    openEntity = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY));
-    if (!openEntity)
+  } else {
+    openEntity
+        = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY));
+    if (! openEntity)
       return XML_ERROR_NO_MEMORY;
   }
   entity->open = XML_TRUE;
@@ -5407,21 +5177,20 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
 
 #ifdef XML_DTD
   if (entity->is_param) {
-    int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
-    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok,
-                      next, &next, XML_FALSE);
-  }
-  else
+    int tok
+        = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
+    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
+                      tok, next, &next, XML_FALSE, XML_FALSE);
+  } else
 #endif /* XML_DTD */
-    result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, textStart,
-                       textEnd, &next, XML_FALSE);
+    result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding,
+                       textStart, textEnd, &next, XML_FALSE);
 
   if (result == XML_ERROR_NONE) {
     if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
       entity->processed = (int)(next - textStart);
       parser->m_processor = internalEntityProcessor;
-    }
-    else {
+    } else {
       entity->open = XML_FALSE;
       parser->m_openInternalEntities = openEntity->next;
       /* put openEntity back in list of free instances */
@@ -5433,17 +5202,14 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
 }
 
 static enum XML_Error PTRCALL
-internalEntityProcessor(XML_Parser parser,
-                        const char *s,
-                        const char *end,
-                        const char **nextPtr)
-{
+internalEntityProcessor(XML_Parser parser, const char *s, const char *end,
+                        const char **nextPtr) {
   ENTITY *entity;
   const char *textStart, *textEnd;
   const char *next;
   enum XML_Error result;
   OPEN_INTERNAL_ENTITY *openEntity = parser->m_openInternalEntities;
-  if (!openEntity)
+  if (! openEntity)
     return XML_ERROR_UNEXPECTED_STATE;
 
   entity = openEntity->entity;
@@ -5454,22 +5220,23 @@ internalEntityProcessor(XML_Parser parser,
 
 #ifdef XML_DTD
   if (entity->is_param) {
-    int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
-    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok,
-                      next, &next, XML_FALSE);
-  }
-  else
+    int tok
+        = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
+    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
+                      tok, next, &next, XML_FALSE, XML_TRUE);
+  } else
 #endif /* XML_DTD */
-    result = doContent(parser, openEntity->startTagLevel, parser->m_internalEncoding,
-                       textStart, textEnd, &next, XML_FALSE);
+    result = doContent(parser, openEntity->startTagLevel,
+                       parser->m_internalEncoding, textStart, textEnd, &next,
+                       XML_FALSE);
 
   if (result != XML_ERROR_NONE)
     return result;
-  else if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
+  else if (textEnd != next
+           && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
     entity->processed = (int)(next - (char *)entity->textPtr);
     return result;
-  }
-  else {
+  } else {
     entity->open = XML_FALSE;
     parser->m_openInternalEntities = openEntity->next;
     /* put openEntity back in list of free instances */
@@ -5483,49 +5250,45 @@ internalEntityProcessor(XML_Parser parser,
     parser->m_processor = prologProcessor;
     tok = XmlPrologTok(parser->m_encoding, s, end, &next);
     return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
-                    (XML_Bool)!parser->m_parsingStatus.finalBuffer);
-  }
-  else
+                    (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
+  } else
 #endif /* XML_DTD */
   {
     parser->m_processor = contentProcessor;
     /* see externalEntityContentProcessor vs contentProcessor */
-    return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding, s, end,
-                     nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+    return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding,
+                     s, end, nextPtr,
+                     (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   }
 }
 
 static enum XML_Error PTRCALL
-errorProcessor(XML_Parser parser,
-               const char *UNUSED_P(s),
-               const char *UNUSED_P(end),
-               const char **UNUSED_P(nextPtr))
-{
+errorProcessor(XML_Parser parser, const char *s, const char *end,
+               const char **nextPtr) {
+  UNUSED_P(s);
+  UNUSED_P(end);
+  UNUSED_P(nextPtr);
   return parser->m_errorCode;
 }
 
 static enum XML_Error
 storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
-                    const char *ptr, const char *end,
-                    STRING_POOL *pool)
-{
-  enum XML_Error result = appendAttributeValue(parser, enc, isCdata, ptr,
-                                               end, pool);
+                    const char *ptr, const char *end, STRING_POOL *pool) {
+  enum XML_Error result
+      = appendAttributeValue(parser, enc, isCdata, ptr, end, pool);
   if (result)
     return result;
-  if (!isCdata && poolLength(pool) && poolLastChar(pool) == 0x20)
+  if (! isCdata && poolLength(pool) && poolLastChar(pool) == 0x20)
     poolChop(pool);
-  if (!poolAppendChar(pool, XML_T('\0')))
+  if (! poolAppendChar(pool, XML_T('\0')))
     return XML_ERROR_NO_MEMORY;
   return XML_ERROR_NONE;
 }
 
 static enum XML_Error
 appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
-                     const char *ptr, const char *end,
-                     STRING_POOL *pool)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+                     const char *ptr, const char *end, STRING_POOL *pool) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   for (;;) {
     const char *next;
     int tok = XmlAttributeValueTok(enc, ptr, end, &next);
@@ -5540,38 +5303,35 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
       if (enc == parser->m_encoding)
         parser->m_eventPtr = ptr;
       return XML_ERROR_INVALID_TOKEN;
-    case XML_TOK_CHAR_REF:
-      {
-        XML_Char buf[XML_ENCODE_MAX];
-        int i;
-        int n = XmlCharRefNumber(enc, ptr);
-        if (n < 0) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_BAD_CHAR_REF;
-        }
-        if (!isCdata
-            && n == 0x20 /* space */
-            && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
-          break;
-        n = XmlEncode(n, (ICHAR *)buf);
-        /* The XmlEncode() functions can never return 0 here.  That
-         * error return happens if the code point passed in is either
-         * negative or greater than or equal to 0x110000.  The
-         * XmlCharRefNumber() functions will all return a number
-         * strictly less than 0x110000 or a negative value if an error
-         * occurred.  The negative value is intercepted above, so
-         * XmlEncode() is never passed a value it might return an
-         * error for.
-         */
-        for (i = 0; i < n; i++) {
-          if (!poolAppendChar(pool, buf[i]))
-            return XML_ERROR_NO_MEMORY;
-        }
+    case XML_TOK_CHAR_REF: {
+      XML_Char buf[XML_ENCODE_MAX];
+      int i;
+      int n = XmlCharRefNumber(enc, ptr);
+      if (n < 0) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_BAD_CHAR_REF;
       }
-      break;
+      if (! isCdata && n == 0x20 /* space */
+          && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
+        break;
+      n = XmlEncode(n, (ICHAR *)buf);
+      /* The XmlEncode() functions can never return 0 here.  That
+       * error return happens if the code point passed in is either
+       * negative or greater than or equal to 0x110000.  The
+       * XmlCharRefNumber() functions will all return a number
+       * strictly less than 0x110000 or a negative value if an error
+       * occurred.  The negative value is intercepted above, so
+       * XmlEncode() is never passed a value it might return an
+       * error for.
+       */
+      for (i = 0; i < n; i++) {
+        if (! poolAppendChar(pool, buf[i]))
+          return XML_ERROR_NO_MEMORY;
+      }
+    } break;
     case XML_TOK_DATA_CHARS:
-      if (!poolAppend(pool, enc, ptr, next))
+      if (! poolAppend(pool, enc, ptr, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_TRAILING_CR:
@@ -5579,109 +5339,103 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
       /* fall through */
     case XML_TOK_ATTRIBUTE_VALUE_S:
     case XML_TOK_DATA_NEWLINE:
-      if (!isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
+      if (! isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
         break;
-      if (!poolAppendChar(pool, 0x20))
+      if (! poolAppendChar(pool, 0x20))
         return XML_ERROR_NO_MEMORY;
       break;
-    case XML_TOK_ENTITY_REF:
-      {
-        const XML_Char *name;
-        ENTITY *entity;
-        char checkEntityDecl;
-        XML_Char ch = (XML_Char) XmlPredefinedEntityName(enc,
-                                              ptr + enc->minBytesPerChar,
-                                              next - enc->minBytesPerChar);
-        if (ch) {
-          if (!poolAppendChar(pool, ch))
-                return XML_ERROR_NO_MEMORY;
-          break;
-        }
-        name = poolStoreString(&parser->m_temp2Pool, enc,
-                               ptr + enc->minBytesPerChar,
-                               next - enc->minBytesPerChar);
-        if (!name)
+    case XML_TOK_ENTITY_REF: {
+      const XML_Char *name;
+      ENTITY *entity;
+      char checkEntityDecl;
+      XML_Char ch = (XML_Char)XmlPredefinedEntityName(
+          enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar);
+      if (ch) {
+        if (! poolAppendChar(pool, ch))
           return XML_ERROR_NO_MEMORY;
-        entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
-        poolDiscard(&parser->m_temp2Pool);
-        /* First, determine if a check for an existing declaration is needed;
-           if yes, check that the entity exists, and that it is internal.
-        */
-        if (pool == &dtd->pool)  /* are we called from prolog? */
-          checkEntityDecl =
+        break;
+      }
+      name = poolStoreString(&parser->m_temp2Pool, enc,
+                             ptr + enc->minBytesPerChar,
+                             next - enc->minBytesPerChar);
+      if (! name)
+        return XML_ERROR_NO_MEMORY;
+      entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
+      poolDiscard(&parser->m_temp2Pool);
+      /* First, determine if a check for an existing declaration is needed;
+         if yes, check that the entity exists, and that it is internal.
+      */
+      if (pool == &dtd->pool) /* are we called from prolog? */
+        checkEntityDecl =
 #ifdef XML_DTD
-              parser->m_prologState.documentEntity &&
+            parser->m_prologState.documentEntity &&
 #endif /* XML_DTD */
-              (dtd->standalone
-               ? !parser->m_openInternalEntities
-               : !dtd->hasParamEntityRefs);
-        else /* if (pool == &parser->m_tempPool): we are called from content */
-          checkEntityDecl = !dtd->hasParamEntityRefs || dtd->standalone;
-        if (checkEntityDecl) {
-          if (!entity)
-            return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal)
-            return XML_ERROR_ENTITY_DECLARED_IN_PE;
-        }
-        else if (!entity) {
-          /* Cannot report skipped entity here - see comments on
-             parser->m_skippedEntityHandler.
-          if (parser->m_skippedEntityHandler)
-            parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
-          */
-          /* Cannot call the default handler because this would be
-             out of sync with the call to the startElementHandler.
-          if ((pool == &parser->m_tempPool) && parser->m_defaultHandler)
-            reportDefault(parser, enc, ptr, next);
-          */
-          break;
-        }
-        if (entity->open) {
-          if (enc == parser->m_encoding) {
-            /* It does not appear that this line can be executed.
-             *
-             * The "if (entity->open)" check catches recursive entity
-             * definitions.  In order to be called with an open
-             * entity, it must have gone through this code before and
-             * been through the recursive call to
-             * appendAttributeValue() some lines below.  That call
-             * sets the local encoding ("enc") to the parser's
-             * internal encoding (internal_utf8 or internal_utf16),
-             * which can never be the same as the principle encoding.
-             * It doesn't appear there is another code path that gets
-             * here with entity->open being TRUE.
-             *
-             * Since it is not certain that this logic is watertight,
-             * we keep the line and merely exclude it from coverage
-             * tests.
-             */
-            parser->m_eventPtr = ptr; /* LCOV_EXCL_LINE */
-          }
-          return XML_ERROR_RECURSIVE_ENTITY_REF;
-        }
-        if (entity->notation) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_BINARY_ENTITY_REF;
-        }
-        if (!entity->textPtr) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF;
-        }
-        else {
-          enum XML_Error result;
-          const XML_Char *textEnd = entity->textPtr + entity->textLen;
-          entity->open = XML_TRUE;
-          result = appendAttributeValue(parser, parser->m_internalEncoding, isCdata,
-                                        (char *)entity->textPtr,
-                                        (char *)textEnd, pool);
-          entity->open = XML_FALSE;
-          if (result)
-            return result;
+            (dtd->standalone ? ! parser->m_openInternalEntities
+                             : ! dtd->hasParamEntityRefs);
+      else /* if (pool == &parser->m_tempPool): we are called from content */
+        checkEntityDecl = ! dtd->hasParamEntityRefs || dtd->standalone;
+      if (checkEntityDecl) {
+        if (! entity)
+          return XML_ERROR_UNDEFINED_ENTITY;
+        else if (! entity->is_internal)
+          return XML_ERROR_ENTITY_DECLARED_IN_PE;
+      } else if (! entity) {
+        /* Cannot report skipped entity here - see comments on
+           parser->m_skippedEntityHandler.
+        if (parser->m_skippedEntityHandler)
+          parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+        */
+        /* Cannot call the default handler because this would be
+           out of sync with the call to the startElementHandler.
+        if ((pool == &parser->m_tempPool) && parser->m_defaultHandler)
+          reportDefault(parser, enc, ptr, next);
+        */
+        break;
+      }
+      if (entity->open) {
+        if (enc == parser->m_encoding) {
+          /* It does not appear that this line can be executed.
+           *
+           * The "if (entity->open)" check catches recursive entity
+           * definitions.  In order to be called with an open
+           * entity, it must have gone through this code before and
+           * been through the recursive call to
+           * appendAttributeValue() some lines below.  That call
+           * sets the local encoding ("enc") to the parser's
+           * internal encoding (internal_utf8 or internal_utf16),
+           * which can never be the same as the principle encoding.
+           * It doesn't appear there is another code path that gets
+           * here with entity->open being TRUE.
+           *
+           * Since it is not certain that this logic is watertight,
+           * we keep the line and merely exclude it from coverage
+           * tests.
+           */
+          parser->m_eventPtr = ptr; /* LCOV_EXCL_LINE */
         }
+        return XML_ERROR_RECURSIVE_ENTITY_REF;
       }
-      break;
+      if (entity->notation) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_BINARY_ENTITY_REF;
+      }
+      if (! entity->textPtr) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF;
+      } else {
+        enum XML_Error result;
+        const XML_Char *textEnd = entity->textPtr + entity->textLen;
+        entity->open = XML_TRUE;
+        result = appendAttributeValue(parser, parser->m_internalEncoding,
+                                      isCdata, (char *)entity->textPtr,
+                                      (char *)textEnd, pool);
+        entity->open = XML_FALSE;
+        if (result)
+          return result;
+      }
+    } break;
     default:
       /* The only token returned by XmlAttributeValueTok() that does
        * not have an explicit case here is XML_TOK_PARTIAL_CHAR.
@@ -5705,12 +5459,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
 }
 
 static enum XML_Error
-storeEntityValue(XML_Parser parser,
-                 const ENCODING *enc,
-                 const char *entityTextPtr,
-                 const char *entityTextEnd)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+storeEntityValue(XML_Parser parser, const ENCODING *enc,
+                 const char *entityTextPtr, const char *entityTextEnd) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   STRING_POOL *pool = &(dtd->entityValuePool);
   enum XML_Error result = XML_ERROR_NONE;
 #ifdef XML_DTD
@@ -5720,8 +5471,8 @@ storeEntityValue(XML_Parser parser,
   /* never return Null for the value argument in EntityDeclHandler,
      since this would indicate an external entity; therefore we
      have to make sure that entityValuePool.start is not null */
-  if (!pool->blocks) {
-    if (!poolGrow(pool))
+  if (! pool->blocks) {
+    if (! poolGrow(pool))
       return XML_ERROR_NO_MEMORY;
   }
 
@@ -5737,13 +5488,13 @@ storeEntityValue(XML_Parser parser,
         name = poolStoreString(&parser->m_tempPool, enc,
                                entityTextPtr + enc->minBytesPerChar,
                                next - enc->minBytesPerChar);
-        if (!name) {
+        if (! name) {
           result = XML_ERROR_NO_MEMORY;
           goto endEntityValue;
         }
         entity = (ENTITY *)lookup(parser, &dtd->paramEntities, name, 0);
         poolDiscard(&parser->m_tempPool);
-        if (!entity) {
+        if (! entity) {
           /* not a well-formedness error - see XML 1.0: WFC Entity Declared */
           /* cannot report skipped entity here - see comments on
              parser->m_skippedEntityHandler
@@ -5763,29 +5514,23 @@ storeEntityValue(XML_Parser parser,
           if (parser->m_externalEntityRefHandler) {
             dtd->paramEntityRead = XML_FALSE;
             entity->open = XML_TRUE;
-            if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                          0,
-                                          entity->base,
-                                          entity->systemId,
-                                          entity->publicId)) {
+            if (! parser->m_externalEntityRefHandler(
+                    parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                    entity->systemId, entity->publicId)) {
               entity->open = XML_FALSE;
               result = XML_ERROR_EXTERNAL_ENTITY_HANDLING;
               goto endEntityValue;
             }
             entity->open = XML_FALSE;
-            if (!dtd->paramEntityRead)
+            if (! dtd->paramEntityRead)
               dtd->keepProcessing = dtd->standalone;
-          }
-          else
+          } else
             dtd->keepProcessing = dtd->standalone;
-        }
-        else {
+        } else {
           entity->open = XML_TRUE;
-          result = storeEntityValue(parser,
-                                    parser->m_internalEncoding,
-                                    (char *)entity->textPtr,
-                                    (char *)(entity->textPtr
-                                             + entity->textLen));
+          result = storeEntityValue(
+              parser, parser->m_internalEncoding, (char *)entity->textPtr,
+              (char *)(entity->textPtr + entity->textLen));
           entity->open = XML_FALSE;
           if (result)
             goto endEntityValue;
@@ -5803,7 +5548,7 @@ storeEntityValue(XML_Parser parser,
       goto endEntityValue;
     case XML_TOK_ENTITY_REF:
     case XML_TOK_DATA_CHARS:
-      if (!poolAppend(pool, enc, entityTextPtr, next)) {
+      if (! poolAppend(pool, enc, entityTextPtr, next)) {
         result = XML_ERROR_NO_MEMORY;
         goto endEntityValue;
       }
@@ -5812,42 +5557,40 @@ storeEntityValue(XML_Parser parser,
       next = entityTextPtr + enc->minBytesPerChar;
       /* fall through */
     case XML_TOK_DATA_NEWLINE:
-      if (pool->end == pool->ptr && !poolGrow(pool)) {
-              result = XML_ERROR_NO_MEMORY;
+      if (pool->end == pool->ptr && ! poolGrow(pool)) {
+        result = XML_ERROR_NO_MEMORY;
         goto endEntityValue;
       }
       *(pool->ptr)++ = 0xA;
       break;
-    case XML_TOK_CHAR_REF:
-      {
-        XML_Char buf[XML_ENCODE_MAX];
-        int i;
-        int n = XmlCharRefNumber(enc, entityTextPtr);
-        if (n < 0) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = entityTextPtr;
-          result = XML_ERROR_BAD_CHAR_REF;
+    case XML_TOK_CHAR_REF: {
+      XML_Char buf[XML_ENCODE_MAX];
+      int i;
+      int n = XmlCharRefNumber(enc, entityTextPtr);
+      if (n < 0) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = entityTextPtr;
+        result = XML_ERROR_BAD_CHAR_REF;
+        goto endEntityValue;
+      }
+      n = XmlEncode(n, (ICHAR *)buf);
+      /* The XmlEncode() functions can never return 0 here.  That
+       * error return happens if the code point passed in is either
+       * negative or greater than or equal to 0x110000.  The
+       * XmlCharRefNumber() functions will all return a number
+       * strictly less than 0x110000 or a negative value if an error
+       * occurred.  The negative value is intercepted above, so
+       * XmlEncode() is never passed a value it might return an
+       * error for.
+       */
+      for (i = 0; i < n; i++) {
+        if (pool->end == pool->ptr && ! poolGrow(pool)) {
+          result = XML_ERROR_NO_MEMORY;
           goto endEntityValue;
         }
-        n = XmlEncode(n, (ICHAR *)buf);
-        /* The XmlEncode() functions can never return 0 here.  That
-         * error return happens if the code point passed in is either
-         * negative or greater than or equal to 0x110000.  The
-         * XmlCharRefNumber() functions will all return a number
-         * strictly less than 0x110000 or a negative value if an error
-         * occurred.  The negative value is intercepted above, so
-         * XmlEncode() is never passed a value it might return an
-         * error for.
-         */
-        for (i = 0; i < n; i++) {
-          if (pool->end == pool->ptr && !poolGrow(pool)) {
-            result = XML_ERROR_NO_MEMORY;
-            goto endEntityValue;
-          }
-          *(pool->ptr)++ = buf[i];
-        }
+        *(pool->ptr)++ = buf[i];
       }
-      break;
+    } break;
     case XML_TOK_PARTIAL:
       if (enc == parser->m_encoding)
         parser->m_eventPtr = entityTextPtr;
@@ -5882,8 +5625,7 @@ storeEntityValue(XML_Parser parser,
 }
 
 static void FASTCALL
-normalizeLines(XML_Char *s)
-{
+normalizeLines(XML_Char *s) {
   XML_Char *p;
   for (;; s++) {
     if (*s == XML_T('\0'))
@@ -5897,8 +5639,7 @@ normalizeLines(XML_Char *s)
       *p++ = 0xA;
       if (*++s == 0xA)
         s++;
-    }
-    else
+    } else
       *p++ = *s++;
   } while (*s);
   *p = XML_T('\0');
@@ -5906,12 +5647,11 @@ normalizeLines(XML_Char *s)
 
 static int
 reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
-                            const char *start, const char *end)
-{
+                            const char *start, const char *end) {
   const XML_Char *target;
   XML_Char *data;
   const char *tem;
-  if (!parser->m_processingInstructionHandler) {
+  if (! parser->m_processingInstructionHandler) {
     if (parser->m_defaultHandler)
       reportDefault(parser, enc, start, end);
     return 1;
@@ -5919,13 +5659,12 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
   start += enc->minBytesPerChar * 2;
   tem = start + XmlNameLength(enc, start);
   target = poolStoreString(&parser->m_tempPool, enc, start, tem);
-  if (!target)
+  if (! target)
     return 0;
   poolFinish(&parser->m_tempPool);
-  data = poolStoreString(&parser->m_tempPool, enc,
-                        XmlSkipS(enc, tem),
-                        end - enc->minBytesPerChar*2);
-  if (!data)
+  data = poolStoreString(&parser->m_tempPool, enc, XmlSkipS(enc, tem),
+                         end - enc->minBytesPerChar * 2);
+  if (! data)
     return 0;
   normalizeLines(data);
   parser->m_processingInstructionHandler(parser->m_handlerArg, target, data);
@@ -5934,20 +5673,18 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
 }
 
 static int
-reportComment(XML_Parser parser, const ENCODING *enc,
-              const char *start, const char *end)
-{
+reportComment(XML_Parser parser, const ENCODING *enc, const char *start,
+              const char *end) {
   XML_Char *data;
-  if (!parser->m_commentHandler) {
+  if (! parser->m_commentHandler) {
     if (parser->m_defaultHandler)
       reportDefault(parser, enc, start, end);
     return 1;
   }
-  data = poolStoreString(&parser->m_tempPool,
-                         enc,
+  data = poolStoreString(&parser->m_tempPool, enc,
                          start + enc->minBytesPerChar * 4,
                          end - enc->minBytesPerChar * 3);
-  if (!data)
+  if (! data)
     return 0;
   normalizeLines(data);
   parser->m_commentHandler(parser->m_handlerArg, data);
@@ -5956,9 +5693,8 @@ reportComment(XML_Parser parser, const ENCODING *enc,
 }
 
 static void
-reportDefault(XML_Parser parser, const ENCODING *enc,
-              const char *s, const char *end)
-{
+reportDefault(XML_Parser parser, const ENCODING *enc, const char *s,
+              const char *end) {
   if (MUST_CONVERT(enc, s)) {
     enum XML_Convert_Result convert_res;
     const char **eventPP;
@@ -5966,8 +5702,7 @@ reportDefault(XML_Parser parser, const ENCODING *enc,
     if (enc == parser->m_encoding) {
       eventPP = &parser->m_eventPtr;
       eventEndPP = &parser->m_eventEndPtr;
-    }
-    else {
+    } else {
       /* To get here, two things must be true; the parser must be
        * using a character encoding that is not the same as the
        * encoding passed in, and the encoding passed in must need
@@ -5990,21 +5725,22 @@ reportDefault(XML_Parser parser, const ENCODING *enc,
     }
     do {
       ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-      convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+      convert_res
+          = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
       *eventEndPP = s;
-      parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+      parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf,
+                               (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
       *eventPP = s;
-    } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE));
-  }
-  else
-    parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s, (int)((XML_Char *)end - (XML_Char *)s));
+    } while ((convert_res != XML_CONVERT_COMPLETED)
+             && (convert_res != XML_CONVERT_INPUT_INCOMPLETE));
+  } else
+    parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s,
+                             (int)((XML_Char *)end - (XML_Char *)s));
 }
 
-
 static int
 defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
-                XML_Bool isId, const XML_Char *value, XML_Parser parser)
-{
+                XML_Bool isId, const XML_Char *value, XML_Parser parser) {
   DEFAULT_ATTRIBUTE *att;
   if (value || isId) {
     /* The handling of default attributes gets messed up if we have
@@ -6013,24 +5749,23 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
     for (i = 0; i < type->nDefaultAtts; i++)
       if (attId == type->defaultAtts[i].id)
         return 1;
-    if (isId && !type->idAtt && !attId->xmlns)
+    if (isId && ! type->idAtt && ! attId->xmlns)
       type->idAtt = attId;
   }
   if (type->nDefaultAtts == type->allocDefaultAtts) {
     if (type->allocDefaultAtts == 0) {
       type->allocDefaultAtts = 8;
-      type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC(parser, type->allocDefaultAtts
-                            * sizeof(DEFAULT_ATTRIBUTE));
-      if (!type->defaultAtts) {
+      type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC(
+          parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
+      if (! type->defaultAtts) {
         type->allocDefaultAtts = 0;
         return 0;
       }
-    }
-    else {
+    } else {
       DEFAULT_ATTRIBUTE *temp;
       int count = type->allocDefaultAtts * 2;
-      temp = (DEFAULT_ATTRIBUTE *)
-        REALLOC(parser, type->defaultAtts, (count * sizeof(DEFAULT_ATTRIBUTE)));
+      temp = (DEFAULT_ATTRIBUTE *)REALLOC(parser, type->defaultAtts,
+                                          (count * sizeof(DEFAULT_ATTRIBUTE)));
       if (temp == NULL)
         return 0;
       type->allocDefaultAtts = count;
@@ -6041,30 +5776,29 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
   att->id = attId;
   att->value = value;
   att->isCdata = isCdata;
-  if (!isCdata)
+  if (! isCdata)
     attId->maybeTokenized = XML_TRUE;
   type->nDefaultAtts += 1;
   return 1;
 }
 
 static int
-setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *name;
   for (name = elementType->name; *name; name++) {
     if (*name == XML_T(ASCII_COLON)) {
       PREFIX *prefix;
       const XML_Char *s;
       for (s = elementType->name; s != name; s++) {
-        if (!poolAppendChar(&dtd->pool, *s))
+        if (! poolAppendChar(&dtd->pool, *s))
           return 0;
       }
-      if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+      if (! poolAppendChar(&dtd->pool, XML_T('\0')))
         return 0;
       prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&dtd->pool),
                                 sizeof(PREFIX));
-      if (!prefix)
+      if (! prefix)
         return 0;
       if (prefix->name == poolStart(&dtd->pool))
         poolFinish(&dtd->pool);
@@ -6078,55 +5812,53 @@ setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType)
 }
 
 static ATTRIBUTE_ID *
-getAttributeId(XML_Parser parser, const ENCODING *enc,
-               const char *start, const char *end)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start,
+               const char *end) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   ATTRIBUTE_ID *id;
   const XML_Char *name;
-  if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+  if (! poolAppendChar(&dtd->pool, XML_T('\0')))
     return NULL;
   name = poolStoreString(&dtd->pool, enc, start, end);
-  if (!name)
+  if (! name)
     return NULL;
   /* skip quotation mark - its storage will be re-used (like in name[-1]) */
   ++name;
-  id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name, sizeof(ATTRIBUTE_ID));
-  if (!id)
+  id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name,
+                              sizeof(ATTRIBUTE_ID));
+  if (! id)
     return NULL;
   if (id->name != name)
     poolDiscard(&dtd->pool);
   else {
     poolFinish(&dtd->pool);
-    if (!parser->m_ns)
+    if (! parser->m_ns)
       ;
-    else if (name[0] == XML_T(ASCII_x)
-        && name[1] == XML_T(ASCII_m)
-        && name[2] == XML_T(ASCII_l)
-        && name[3] == XML_T(ASCII_n)
-        && name[4] == XML_T(ASCII_s)
-        && (name[5] == XML_T('\0') || name[5] == XML_T(ASCII_COLON))) {
+    else if (name[0] == XML_T(ASCII_x) && name[1] == XML_T(ASCII_m)
+             && name[2] == XML_T(ASCII_l) && name[3] == XML_T(ASCII_n)
+             && name[4] == XML_T(ASCII_s)
+             && (name[5] == XML_T('\0') || name[5] == XML_T(ASCII_COLON))) {
       if (name[5] == XML_T('\0'))
         id->prefix = &dtd->defaultPrefix;
       else
-        id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6, sizeof(PREFIX));
+        id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6,
+                                      sizeof(PREFIX));
       id->xmlns = XML_TRUE;
-    }
-    else {
+    } else {
       int i;
       for (i = 0; name[i]; i++) {
         /* attributes without prefix are *not* in the default namespace */
         if (name[i] == XML_T(ASCII_COLON)) {
           int j;
           for (j = 0; j < i; j++) {
-            if (!poolAppendChar(&dtd->pool, name[j]))
+            if (! poolAppendChar(&dtd->pool, name[j]))
               return NULL;
           }
-          if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+          if (! poolAppendChar(&dtd->pool, XML_T('\0')))
             return NULL;
-          id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&dtd->pool),
-                                        sizeof(PREFIX));
-          if (!id->prefix)
+          id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes,
+                                        poolStart(&dtd->pool), sizeof(PREFIX));
+          if (! id->prefix)
             return NULL;
           if (id->prefix->name == poolStart(&dtd->pool))
             poolFinish(&dtd->pool);
@@ -6143,22 +5875,22 @@ getAttributeId(XML_Parser parser, const ENCODING *enc,
 #define CONTEXT_SEP XML_T(ASCII_FF)
 
 static const XML_Char *
-getContext(XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getContext(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   HASH_TABLE_ITER iter;
   XML_Bool needSep = XML_FALSE;
 
   if (dtd->defaultPrefix.binding) {
     int i;
     int len;
-    if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
+    if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
       return NULL;
     len = dtd->defaultPrefix.binding->uriLen;
     if (parser->m_namespaceSeparator)
       len--;
     for (i = 0; i < len; i++) {
-      if (!poolAppendChar(&parser->m_tempPool, dtd->defaultPrefix.binding->uri[i])) {
+      if (! poolAppendChar(&parser->m_tempPool,
+                           dtd->defaultPrefix.binding->uri[i])) {
         /* Because of memory caching, I don't believe this line can be
          * executed.
          *
@@ -6190,9 +5922,9 @@ getContext(XML_Parser parser)
     int len;
     const XML_Char *s;
     PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter);
-    if (!prefix)
+    if (! prefix)
       break;
-    if (!prefix->binding) {
+    if (! prefix->binding) {
       /* This test appears to be (justifiable) paranoia.  There does
        * not seem to be a way of injecting a prefix without a binding
        * that doesn't get errored long before this function is called.
@@ -6201,98 +5933,96 @@ getContext(XML_Parser parser)
        */
       continue; /* LCOV_EXCL_LINE */
     }
-    if (needSep && !poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
+    if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
       return NULL;
     for (s = prefix->name; *s; s++)
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return NULL;
-    if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
+    if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
       return NULL;
     len = prefix->binding->uriLen;
     if (parser->m_namespaceSeparator)
       len--;
     for (i = 0; i < len; i++)
-      if (!poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i]))
+      if (! poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i]))
         return NULL;
     needSep = XML_TRUE;
   }
 
-
   hashTableIterInit(&iter, &(dtd->generalEntities));
   for (;;) {
     const XML_Char *s;
     ENTITY *e = (ENTITY *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
-    if (!e->open)
+    if (! e->open)
       continue;
-    if (needSep && !poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
+    if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
       return NULL;
     for (s = e->name; *s; s++)
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return 0;
     needSep = XML_TRUE;
   }
 
-  if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+  if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
     return NULL;
   return parser->m_tempPool.start;
 }
 
 static XML_Bool
-setContext(XML_Parser parser, const XML_Char *context)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+setContext(XML_Parser parser, const XML_Char *context) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *s = context;
 
   while (*context != XML_T('\0')) {
     if (*s == CONTEXT_SEP || *s == XML_T('\0')) {
       ENTITY *e;
-      if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+      if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
         return XML_FALSE;
-      e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0);
+      e = (ENTITY *)lookup(parser, &dtd->generalEntities,
+                           poolStart(&parser->m_tempPool), 0);
       if (e)
         e->open = XML_TRUE;
       if (*s != XML_T('\0'))
         s++;
       context = s;
       poolDiscard(&parser->m_tempPool);
-    }
-    else if (*s == XML_T(ASCII_EQUALS)) {
+    } else if (*s == XML_T(ASCII_EQUALS)) {
       PREFIX *prefix;
       if (poolLength(&parser->m_tempPool) == 0)
         prefix = &dtd->defaultPrefix;
       else {
-        if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+        if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
           return XML_FALSE;
-        prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&parser->m_tempPool),
-                                  sizeof(PREFIX));
-        if (!prefix)
+        prefix
+            = (PREFIX *)lookup(parser, &dtd->prefixes,
+                               poolStart(&parser->m_tempPool), sizeof(PREFIX));
+        if (! prefix)
           return XML_FALSE;
         if (prefix->name == poolStart(&parser->m_tempPool)) {
           prefix->name = poolCopyString(&dtd->pool, prefix->name);
-          if (!prefix->name)
+          if (! prefix->name)
             return XML_FALSE;
         }
         poolDiscard(&parser->m_tempPool);
       }
-      for (context = s + 1;
-           *context != CONTEXT_SEP && *context != XML_T('\0');
+      for (context = s + 1; *context != CONTEXT_SEP && *context != XML_T('\0');
            context++)
-        if (!poolAppendChar(&parser->m_tempPool, *context))
+        if (! poolAppendChar(&parser->m_tempPool, *context))
           return XML_FALSE;
-      if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+      if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
         return XML_FALSE;
       if (addBinding(parser, prefix, NULL, poolStart(&parser->m_tempPool),
-                     &parser->m_inheritedBindings) != XML_ERROR_NONE)
+                     &parser->m_inheritedBindings)
+          != XML_ERROR_NONE)
         return XML_FALSE;
       poolDiscard(&parser->m_tempPool);
       if (*context != XML_T('\0'))
         ++context;
       s = context;
-    }
-    else {
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+    } else {
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return XML_FALSE;
       s++;
     }
@@ -6301,8 +6031,7 @@ setContext(XML_Parser parser, const XML_Char *context)
 }
 
 static void FASTCALL
-normalizePublicId(XML_Char *publicId)
-{
+normalizePublicId(XML_Char *publicId) {
   XML_Char *p = publicId;
   XML_Char *s;
   for (s = publicId; *s; s++) {
@@ -6323,8 +6052,7 @@ normalizePublicId(XML_Char *publicId)
 }
 
 static DTD *
-dtdCreate(const XML_Memory_Handling_Suite *ms)
-{
+dtdCreate(const XML_Memory_Handling_Suite *ms) {
   DTD *p = (DTD *)ms->malloc_fcn(sizeof(DTD));
   if (p == NULL)
     return p;
@@ -6356,13 +6084,12 @@ dtdCreate(const XML_Memory_Handling_Suite *ms)
 }
 
 static void
-dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms)
-{
+dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
   hashTableIterInit(&iter, &(p->elementTypes));
   for (;;) {
     ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
     if (e->allocDefaultAtts != 0)
       ms->free_fcn(e->defaultAtts);
@@ -6398,13 +6125,12 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms)
 }
 
 static void
-dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms)
-{
+dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
   hashTableIterInit(&iter, &(p->elementTypes));
   for (;;) {
     ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
     if (e->allocDefaultAtts != 0)
       ms->free_fcn(e->defaultAtts);
@@ -6429,8 +6155,8 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms)
    The new DTD has already been initialized.
 */
 static int
-dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms)
-{
+dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
+        const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
 
   /* Copy the prefix table. */
@@ -6439,12 +6165,12 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
   for (;;) {
     const XML_Char *name;
     const PREFIX *oldP = (PREFIX *)hashTableIterNext(&iter);
-    if (!oldP)
+    if (! oldP)
       break;
     name = poolCopyString(&(newDtd->pool), oldP->name);
-    if (!name)
+    if (! name)
       return 0;
-    if (!lookup(oldParser, &(newDtd->prefixes), name, sizeof(PREFIX)))
+    if (! lookup(oldParser, &(newDtd->prefixes), name, sizeof(PREFIX)))
       return 0;
   }
 
@@ -6457,18 +6183,18 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
     const XML_Char *name;
     const ATTRIBUTE_ID *oldA = (ATTRIBUTE_ID *)hashTableIterNext(&iter);
 
-    if (!oldA)
+    if (! oldA)
       break;
     /* Remember to allocate the scratch byte before the name. */
-    if (!poolAppendChar(&(newDtd->pool), XML_T('\0')))
+    if (! poolAppendChar(&(newDtd->pool), XML_T('\0')))
       return 0;
     name = poolCopyString(&(newDtd->pool), oldA->name);
-    if (!name)
+    if (! name)
       return 0;
     ++name;
     newA = (ATTRIBUTE_ID *)lookup(oldParser, &(newDtd->attributeIds), name,
                                   sizeof(ATTRIBUTE_ID));
-    if (!newA)
+    if (! newA)
       return 0;
     newA->maybeTokenized = oldA->maybeTokenized;
     if (oldA->prefix) {
@@ -6490,57 +6216,52 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
     ELEMENT_TYPE *newE;
     const XML_Char *name;
     const ELEMENT_TYPE *oldE = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!oldE)
+    if (! oldE)
       break;
     name = poolCopyString(&(newDtd->pool), oldE->name);
-    if (!name)
+    if (! name)
       return 0;
     newE = (ELEMENT_TYPE *)lookup(oldParser, &(newDtd->elementTypes), name,
                                   sizeof(ELEMENT_TYPE));
-    if (!newE)
+    if (! newE)
       return 0;
     if (oldE->nDefaultAtts) {
-      newE->defaultAtts = (DEFAULT_ATTRIBUTE *)
-          ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
-      if (!newE->defaultAtts) {
+      newE->defaultAtts = (DEFAULT_ATTRIBUTE *)ms->malloc_fcn(
+          oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
+      if (! newE->defaultAtts) {
         return 0;
       }
     }
     if (oldE->idAtt)
-      newE->idAtt = (ATTRIBUTE_ID *)
-          lookup(oldParser, &(newDtd->attributeIds), oldE->idAtt->name, 0);
+      newE->idAtt = (ATTRIBUTE_ID *)lookup(oldParser, &(newDtd->attributeIds),
+                                           oldE->idAtt->name, 0);
     newE->allocDefaultAtts = newE->nDefaultAtts = oldE->nDefaultAtts;
     if (oldE->prefix)
       newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes),
                                       oldE->prefix->name, 0);
     for (i = 0; i < newE->nDefaultAtts; i++) {
-      newE->defaultAtts[i].id = (ATTRIBUTE_ID *)
-          lookup(oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0);
+      newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup(
+          oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0);
       newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata;
       if (oldE->defaultAtts[i].value) {
         newE->defaultAtts[i].value
             = poolCopyString(&(newDtd->pool), oldE->defaultAtts[i].value);
-        if (!newE->defaultAtts[i].value)
+        if (! newE->defaultAtts[i].value)
           return 0;
-      }
-      else
+      } else
         newE->defaultAtts[i].value = NULL;
     }
   }
 
   /* Copy the entity tables. */
-  if (!copyEntityTable(oldParser,
-                       &(newDtd->generalEntities),
-                       &(newDtd->pool),
-                       &(oldDtd->generalEntities)))
-      return 0;
+  if (! copyEntityTable(oldParser, &(newDtd->generalEntities), &(newDtd->pool),
+                        &(oldDtd->generalEntities)))
+    return 0;
 
 #ifdef XML_DTD
-  if (!copyEntityTable(oldParser,
-                       &(newDtd->paramEntities),
-                       &(newDtd->pool),
-                       &(oldDtd->paramEntities)))
-      return 0;
+  if (! copyEntityTable(oldParser, &(newDtd->paramEntities), &(newDtd->pool),
+                        &(oldDtd->paramEntities)))
+    return 0;
   newDtd->paramEntityRead = oldDtd->paramEntityRead;
 #endif /* XML_DTD */
 
@@ -6557,14 +6278,11 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
   newDtd->scaffIndex = oldDtd->scaffIndex;
 
   return 1;
-}  /* End dtdCopy */
+} /* End dtdCopy */
 
 static int
-copyEntityTable(XML_Parser oldParser,
-                HASH_TABLE *newTable,
-                STRING_POOL *newPool,
-                const HASH_TABLE *oldTable)
-{
+copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable,
+                STRING_POOL *newPool, const HASH_TABLE *oldTable) {
   HASH_TABLE_ITER iter;
   const XML_Char *cachedOldBase = NULL;
   const XML_Char *cachedNewBase = NULL;
@@ -6575,17 +6293,17 @@ copyEntityTable(XML_Parser oldParser,
     ENTITY *newE;
     const XML_Char *name;
     const ENTITY *oldE = (ENTITY *)hashTableIterNext(&iter);
-    if (!oldE)
+    if (! oldE)
       break;
     name = poolCopyString(newPool, oldE->name);
-    if (!name)
+    if (! name)
       return 0;
     newE = (ENTITY *)lookup(oldParser, newTable, name, sizeof(ENTITY));
-    if (!newE)
+    if (! newE)
       return 0;
     if (oldE->systemId) {
       const XML_Char *tem = poolCopyString(newPool, oldE->systemId);
-      if (!tem)
+      if (! tem)
         return 0;
       newE->systemId = tem;
       if (oldE->base) {
@@ -6594,29 +6312,28 @@ copyEntityTable(XML_Parser oldParser,
         else {
           cachedOldBase = oldE->base;
           tem = poolCopyString(newPool, cachedOldBase);
-          if (!tem)
+          if (! tem)
             return 0;
           cachedNewBase = newE->base = tem;
         }
       }
       if (oldE->publicId) {
         tem = poolCopyString(newPool, oldE->publicId);
-        if (!tem)
+        if (! tem)
           return 0;
         newE->publicId = tem;
       }
-    }
-    else {
-      const XML_Char *tem = poolCopyStringN(newPool, oldE->textPtr,
-                                            oldE->textLen);
-      if (!tem)
+    } else {
+      const XML_Char *tem
+          = poolCopyStringN(newPool, oldE->textPtr, oldE->textLen);
+      if (! tem)
         return 0;
       newE->textPtr = tem;
       newE->textLen = oldE->textLen;
     }
     if (oldE->notation) {
       const XML_Char *tem = poolCopyString(newPool, oldE->notation);
-      if (!tem)
+      if (! tem)
         return 0;
       newE->notation = tem;
     }
@@ -6629,8 +6346,7 @@ copyEntityTable(XML_Parser oldParser,
 #define INIT_POWER 6
 
 static XML_Bool FASTCALL
-keyeq(KEY s1, KEY s2)
-{
+keyeq(KEY s1, KEY s2) {
   for (; *s1 == *s2; s1++, s2++)
     if (*s1 == 0)
       return XML_TRUE;
@@ -6638,23 +6354,21 @@ keyeq(KEY s1, KEY s2)
 }
 
 static size_t
-keylen(KEY s)
-{
+keylen(KEY s) {
   size_t len = 0;
-  for (; *s; s++, len++);
+  for (; *s; s++, len++)
+    ;
   return len;
 }
 
 static void
-copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key)
-{
+copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) {
   key->k[0] = 0;
   key->k[1] = get_hash_secret_salt(parser);
 }
 
 static unsigned long FASTCALL
-hash(XML_Parser parser, KEY s)
-{
+hash(XML_Parser parser, KEY s) {
   struct siphash state;
   struct sipkey key;
   (void)sip24_valid;
@@ -6665,26 +6379,24 @@ hash(XML_Parser parser, KEY s)
 }
 
 static NAMED *
-lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
-{
+lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) {
   size_t i;
   if (table->size == 0) {
     size_t tsize;
-    if (!createSize)
+    if (! createSize)
       return NULL;
     table->power = INIT_POWER;
     /* table->size is a power of 2 */
     table->size = (size_t)1 << INIT_POWER;
     tsize = table->size * sizeof(NAMED *);
     table->v = (NAMED **)table->mem->malloc_fcn(tsize);
-    if (!table->v) {
+    if (! table->v) {
       table->size = 0;
       return NULL;
     }
     memset(table->v, 0, tsize);
     i = hash(parser, name) & ((unsigned long)table->size - 1);
-  }
-  else {
+  } else {
     unsigned long h = hash(parser, name);
     unsigned long mask = (unsigned long)table->size - 1;
     unsigned char step = 0;
@@ -6692,11 +6404,11 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
     while (table->v[i]) {
       if (keyeq(name, table->v[i]->name))
         return table->v[i];
-      if (!step)
+      if (! step)
         step = PROBE_STEP(h, mask, table->power);
       i < step ? (i += table->size - step) : (i -= step);
     }
-    if (!createSize)
+    if (! createSize)
       return NULL;
 
     /* check for overflow (table is half full) */
@@ -6706,7 +6418,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
       unsigned long newMask = (unsigned long)newSize - 1;
       size_t tsize = newSize * sizeof(NAMED *);
       NAMED **newV = (NAMED **)table->mem->malloc_fcn(tsize);
-      if (!newV)
+      if (! newV)
         return NULL;
       memset(newV, 0, tsize);
       for (i = 0; i < table->size; i++)
@@ -6715,7 +6427,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
           size_t j = newHash & newMask;
           step = 0;
           while (newV[j]) {
-            if (!step)
+            if (! step)
               step = PROBE_STEP(newHash, newMask, newPower);
             j < step ? (j += newSize - step) : (j -= step);
           }
@@ -6728,14 +6440,14 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
       i = h & newMask;
       step = 0;
       while (table->v[i]) {
-        if (!step)
+        if (! step)
           step = PROBE_STEP(h, newMask, newPower);
         i < step ? (i += newSize - step) : (i -= step);
       }
     }
   }
   table->v[i] = (NAMED *)table->mem->malloc_fcn(createSize);
-  if (!table->v[i])
+  if (! table->v[i])
     return NULL;
   memset(table->v[i], 0, createSize);
   table->v[i]->name = name;
@@ -6744,8 +6456,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
 }
 
 static void FASTCALL
-hashTableClear(HASH_TABLE *table)
-{
+hashTableClear(HASH_TABLE *table) {
   size_t i;
   for (i = 0; i < table->size; i++) {
     table->mem->free_fcn(table->v[i]);
@@ -6755,8 +6466,7 @@ hashTableClear(HASH_TABLE *table)
 }
 
 static void FASTCALL
-hashTableDestroy(HASH_TABLE *table)
-{
+hashTableDestroy(HASH_TABLE *table) {
   size_t i;
   for (i = 0; i < table->size; i++)
     table->mem->free_fcn(table->v[i]);
@@ -6764,8 +6474,7 @@ hashTableDestroy(HASH_TABLE *table)
 }
 
 static void FASTCALL
-hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms)
-{
+hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) {
   p->power = 0;
   p->size = 0;
   p->used = 0;
@@ -6774,15 +6483,13 @@ hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms)
 }
 
 static void FASTCALL
-hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table)
-{
+hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) {
   iter->p = table->v;
   iter->end = iter->p + table->size;
 }
 
-static NAMED * FASTCALL
-hashTableIterNext(HASH_TABLE_ITER *iter)
-{
+static NAMED *FASTCALL
+hashTableIterNext(HASH_TABLE_ITER *iter) {
   while (iter->p != iter->end) {
     NAMED *tem = *(iter->p)++;
     if (tem)
@@ -6792,8 +6499,7 @@ hashTableIterNext(HASH_TABLE_ITER *iter)
 }
 
 static void FASTCALL
-poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms)
-{
+poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms) {
   pool->blocks = NULL;
   pool->freeBlocks = NULL;
   pool->start = NULL;
@@ -6803,9 +6509,8 @@ poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms)
 }
 
 static void FASTCALL
-poolClear(STRING_POOL *pool)
-{
-  if (!pool->freeBlocks)
+poolClear(STRING_POOL *pool) {
+  if (! pool->freeBlocks)
     pool->freeBlocks = pool->blocks;
   else {
     BLOCK *p = pool->blocks;
@@ -6823,8 +6528,7 @@ poolClear(STRING_POOL *pool)
 }
 
 static void FASTCALL
-poolDestroy(STRING_POOL *pool)
-{
+poolDestroy(STRING_POOL *pool) {
   BLOCK *p = pool->blocks;
   while (p) {
     BLOCK *tem = p->next;
@@ -6840,26 +6544,26 @@ poolDestroy(STRING_POOL *pool)
 }
 
 static XML_Char *
-poolAppend(STRING_POOL *pool, const ENCODING *enc,
-           const char *ptr, const char *end)
-{
-  if (!pool->ptr && !poolGrow(pool))
+poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr,
+           const char *end) {
+  if (! pool->ptr && ! poolGrow(pool))
     return NULL;
   for (;;) {
-    const enum XML_Convert_Result convert_res = XmlConvert(enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end);
-    if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+    const enum XML_Convert_Result convert_res = XmlConvert(
+        enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end);
+    if ((convert_res == XML_CONVERT_COMPLETED)
+        || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
       break;
-    if (!poolGrow(pool))
+    if (! poolGrow(pool))
       return NULL;
   }
   return pool->start;
 }
 
-static const XML_Char * FASTCALL
-poolCopyString(STRING_POOL *pool, const XML_Char *s)
-{
+static const XML_Char *FASTCALL
+poolCopyString(STRING_POOL *pool, const XML_Char *s) {
   do {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
   } while (*s++);
   s = pool->start;
@@ -6868,9 +6572,8 @@ poolCopyString(STRING_POOL *pool, const XML_Char *s)
 }
 
 static const XML_Char *
-poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
-{
-  if (!pool->ptr && !poolGrow(pool)) {
+poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) {
+  if (! pool->ptr && ! poolGrow(pool)) {
     /* The following line is unreachable given the current usage of
      * poolCopyStringN().  Currently it is called from exactly one
      * place to copy the text of a simple general entity.  By that
@@ -6885,7 +6588,7 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
     return NULL; /* LCOV_EXCL_LINE */
   }
   for (; n > 0; --n, s++) {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
   }
   s = pool->start;
@@ -6893,11 +6596,10 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
   return s;
 }
 
-static const XML_Char * FASTCALL
-poolAppendString(STRING_POOL *pool, const XML_Char *s)
-{
+static const XML_Char *FASTCALL
+poolAppendString(STRING_POOL *pool, const XML_Char *s) {
   while (*s) {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
     s++;
   }
@@ -6905,20 +6607,18 @@ poolAppendString(STRING_POOL *pool, const XML_Char *s)
 }
 
 static XML_Char *
-poolStoreString(STRING_POOL *pool, const ENCODING *enc,
-                const char *ptr, const char *end)
-{
-  if (!poolAppend(pool, enc, ptr, end))
+poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr,
+                const char *end) {
+  if (! poolAppend(pool, enc, ptr, end))
     return NULL;
-  if (pool->ptr == pool->end && !poolGrow(pool))
+  if (pool->ptr == pool->end && ! poolGrow(pool))
     return NULL;
   *(pool->ptr)++ = 0;
   return pool->start;
 }
 
 static size_t
-poolBytesToAllocateFor(int blockSize)
-{
+poolBytesToAllocateFor(int blockSize) {
   /* Unprotected math would be:
   ** return offsetof(BLOCK, s) + blockSize * sizeof(XML_Char);
   **
@@ -6926,7 +6626,7 @@ poolBytesToAllocateFor(int blockSize)
   ** For a + b * c we check b * c in isolation first, so that addition of a
   ** on top has no chance of making us accept a small non-negative number
   */
-  const size_t stretch = sizeof(XML_Char);  /* can be 4 bytes */
+  const size_t stretch = sizeof(XML_Char); /* can be 4 bytes */
 
   if (blockSize <= 0)
     return 0;
@@ -6936,8 +6636,8 @@ poolBytesToAllocateFor(int blockSize)
 
   {
     const int stretchedBlockSize = blockSize * (int)stretch;
-    const int bytesToAllocate = (int)(
-        offsetof(BLOCK, s) + (unsigned)stretchedBlockSize);
+    const int bytesToAllocate
+        = (int)(offsetof(BLOCK, s) + (unsigned)stretchedBlockSize);
     if (bytesToAllocate < 0)
       return 0;
 
@@ -6946,8 +6646,7 @@ poolBytesToAllocateFor(int blockSize)
 }
 
 static XML_Bool FASTCALL
-poolGrow(STRING_POOL *pool)
-{
+poolGrow(STRING_POOL *pool) {
   if (pool->freeBlocks) {
     if (pool->start == 0) {
       pool->blocks = pool->freeBlocks;
@@ -6973,7 +6672,7 @@ poolGrow(STRING_POOL *pool)
   }
   if (pool->blocks && pool->start == pool->blocks->s) {
     BLOCK *temp;
-    int blockSize = (int)((unsigned)(pool->end - pool->start)*2U);
+    int blockSize = (int)((unsigned)(pool->end - pool->start) * 2U);
     size_t bytesToAllocate;
 
     /* NOTE: Needs to be calculated prior to calling `realloc`
@@ -6994,8 +6693,8 @@ poolGrow(STRING_POOL *pool)
     if (bytesToAllocate == 0)
       return XML_FALSE;
 
-    temp = (BLOCK *)
-      pool->mem->realloc_fcn(pool->blocks, (unsigned)bytesToAllocate);
+    temp = (BLOCK *)pool->mem->realloc_fcn(pool->blocks,
+                                           (unsigned)bytesToAllocate);
     if (temp == NULL)
       return XML_FALSE;
     pool->blocks = temp;
@@ -7003,8 +6702,7 @@ poolGrow(STRING_POOL *pool)
     pool->ptr = pool->blocks->s + offsetInsideBlock;
     pool->start = pool->blocks->s;
     pool->end = pool->start + blockSize;
-  }
-  else {
+  } else {
     BLOCK *tem;
     int blockSize = (int)(pool->end - pool->start);
     size_t bytesToAllocate;
@@ -7019,7 +6717,7 @@ poolGrow(STRING_POOL *pool)
        * function).  Either way it isn't readily testable, so we
        * exclude it from the coverage statistics.
        */
-      return XML_FALSE;  /* LCOV_EXCL_LINE */
+      return XML_FALSE; /* LCOV_EXCL_LINE */
     }
 
     if (blockSize < INIT_BLOCK_SIZE)
@@ -7037,14 +6735,13 @@ poolGrow(STRING_POOL *pool)
       return XML_FALSE;
 
     tem = (BLOCK *)pool->mem->malloc_fcn(bytesToAllocate);
-    if (!tem)
+    if (! tem)
       return XML_FALSE;
     tem->size = blockSize;
     tem->next = pool->blocks;
     pool->blocks = tem;
     if (pool->ptr != pool->start)
-      memcpy(tem->s, pool->start,
-             (pool->ptr - pool->start) * sizeof(XML_Char));
+      memcpy(tem->s, pool->start, (pool->ptr - pool->start) * sizeof(XML_Char));
     pool->ptr = tem->s + (pool->ptr - pool->start);
     pool->start = tem->s;
     pool->end = tem->s + blockSize;
@@ -7053,15 +6750,14 @@ poolGrow(STRING_POOL *pool)
 }
 
 static int FASTCALL
-nextScaffoldPart(XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
-  CONTENT_SCAFFOLD * me;
+nextScaffoldPart(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
+  CONTENT_SCAFFOLD *me;
   int next;
 
-  if (!dtd->scaffIndex) {
+  if (! dtd->scaffIndex) {
     dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int));
-    if (!dtd->scaffIndex)
+    if (! dtd->scaffIndex)
       return -1;
     dtd->scaffIndex[0] = 0;
   }
@@ -7069,15 +6765,14 @@ nextScaffoldPart(XML_Parser parser)
   if (dtd->scaffCount >= dtd->scaffSize) {
     CONTENT_SCAFFOLD *temp;
     if (dtd->scaffold) {
-      temp = (CONTENT_SCAFFOLD *)
-        REALLOC(parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD));
+      temp = (CONTENT_SCAFFOLD *)REALLOC(
+          parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD));
       if (temp == NULL)
         return -1;
       dtd->scaffSize *= 2;
-    }
-    else {
+    } else {
       temp = (CONTENT_SCAFFOLD *)MALLOC(parser, INIT_SCAFFOLD_ELEMENTS
-                                        * sizeof(CONTENT_SCAFFOLD));
+                                                    * sizeof(CONTENT_SCAFFOLD));
       if (temp == NULL)
         return -1;
       dtd->scaffSize = INIT_SCAFFOLD_ELEMENTS;
@@ -7087,11 +6782,12 @@ nextScaffoldPart(XML_Parser parser)
   next = dtd->scaffCount++;
   me = &dtd->scaffold[next];
   if (dtd->scaffLevel) {
-    CONTENT_SCAFFOLD *parent = &dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel-1]];
+    CONTENT_SCAFFOLD *parent
+        = &dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]];
     if (parent->lastchild) {
       dtd->scaffold[parent->lastchild].nextsib = next;
     }
-    if (!parent->childcnt)
+    if (! parent->childcnt)
       parent->firstchild = next;
     parent->lastchild = next;
     parent->childcnt++;
@@ -7101,13 +6797,9 @@ nextScaffoldPart(XML_Parser parser)
 }
 
 static void
-build_node(XML_Parser parser,
-           int src_node,
-           XML_Content *dest,
-           XML_Content **contpos,
-           XML_Char **strpos)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+build_node(XML_Parser parser, int src_node, XML_Content *dest,
+           XML_Content **contpos, XML_Char **strpos) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   dest->type = dtd->scaffold[src_node].type;
   dest->quant = dtd->scaffold[src_node].quant;
   if (dest->type == XML_CTYPE_NAME) {
@@ -7116,21 +6808,19 @@ build_node(XML_Parser parser,
     src = dtd->scaffold[src_node].name;
     for (;;) {
       *(*strpos)++ = *src;
-      if (!*src)
+      if (! *src)
         break;
       src++;
     }
     dest->numchildren = 0;
     dest->children = NULL;
-  }
-  else {
+  } else {
     unsigned int i;
     int cn;
     dest->numchildren = dtd->scaffold[src_node].childcnt;
     dest->children = *contpos;
     *contpos += dest->numchildren;
-    for (i = 0, cn = dtd->scaffold[src_node].firstchild;
-         i < dest->numchildren;
+    for (i = 0, cn = dtd->scaffold[src_node].firstchild; i < dest->numchildren;
          i++, cn = dtd->scaffold[cn].nextsib) {
       build_node(parser, cn, &(dest->children[i]), contpos, strpos);
     }
@@ -7139,20 +6829,19 @@ build_node(XML_Parser parser,
 }
 
 static XML_Content *
-build_model (XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+build_model(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   XML_Content *ret;
   XML_Content *cpos;
-  XML_Char * str;
+  XML_Char *str;
   int allocsize = (dtd->scaffCount * sizeof(XML_Content)
                    + (dtd->contentStringLen * sizeof(XML_Char)));
 
   ret = (XML_Content *)MALLOC(parser, allocsize);
-  if (!ret)
+  if (! ret)
     return NULL;
 
-  str =  (XML_Char *) (&ret[dtd->scaffCount]);
+  str = (XML_Char *)(&ret[dtd->scaffCount]);
   cpos = &ret[1];
 
   build_node(parser, 0, ret, &cpos, &str);
@@ -7160,49 +6849,45 @@ build_model (XML_Parser parser)
 }
 
 static ELEMENT_TYPE *
-getElementType(XML_Parser parser,
-               const ENCODING *enc,
-               const char *ptr,
-               const char *end)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr,
+               const char *end) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *name = poolStoreString(&dtd->pool, enc, ptr, end);
   ELEMENT_TYPE *ret;
 
-  if (!name)
+  if (! name)
     return NULL;
-  ret = (ELEMENT_TYPE *) lookup(parser, &dtd->elementTypes, name, sizeof(ELEMENT_TYPE));
-  if (!ret)
+  ret = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, name,
+                               sizeof(ELEMENT_TYPE));
+  if (! ret)
     return NULL;
   if (ret->name != name)
     poolDiscard(&dtd->pool);
   else {
     poolFinish(&dtd->pool);
-    if (!setElementTypePrefix(parser, ret))
+    if (! setElementTypePrefix(parser, ret))
       return NULL;
   }
   return ret;
 }
 
 static XML_Char *
-copyString(const XML_Char *s,
-           const XML_Memory_Handling_Suite *memsuite)
-{
-    int charsRequired = 0;
-    XML_Char *result;
-
-    /* First determine how long the string is */
-    while (s[charsRequired] != 0) {
-      charsRequired++;
-    }
-    /* Include the terminator */
+copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) {
+  int charsRequired = 0;
+  XML_Char *result;
+
+  /* First determine how long the string is */
+  while (s[charsRequired] != 0) {
     charsRequired++;
+  }
+  /* Include the terminator */
+  charsRequired++;
 
-    /* Now allocate space for the copy */
-    result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char));
-    if (result == NULL)
-        return NULL;
-    /* Copy the original into place */
-    memcpy(result, s, charsRequired * sizeof(XML_Char));
-    return result;
+  /* Now allocate space for the copy */
+  result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char));
+  if (result == NULL)
+    return NULL;
+  /* Copy the original into place */
+  memcpy(result, s, charsRequired * sizeof(XML_Char));
+  return result;
 }
diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c
index 708507d575be..4d3e3e86e9e8 100644
--- a/Modules/expat/xmlrole.c
+++ b/Modules/expat/xmlrole.c
@@ -33,11 +33,11 @@
 #include <stddef.h>
 
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #else
-#ifdef HAVE_EXPAT_CONFIG_H
-#include <expat_config.h>
-#endif
+#  ifdef HAVE_EXPAT_CONFIG_H
+#    include <expat_config.h>
+#  endif
 #endif /* ndef _WIN32 */
 
 #include "expat_external.h"
@@ -52,107 +52,88 @@
 
 */
 
-static const char KW_ANY[] = {
-    ASCII_A, ASCII_N, ASCII_Y, '\0' };
-static const char KW_ATTLIST[] = {
-    ASCII_A, ASCII_T, ASCII_T, ASCII_L, ASCII_I, ASCII_S, ASCII_T, '\0' };
-static const char KW_CDATA[] = {
-    ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_DOCTYPE[] = {
-    ASCII_D, ASCII_O, ASCII_C, ASCII_T, ASCII_Y, ASCII_P, ASCII_E, '\0' };
-static const char KW_ELEMENT[] = {
-    ASCII_E, ASCII_L, ASCII_E, ASCII_M, ASCII_E, ASCII_N, ASCII_T, '\0' };
-static const char KW_EMPTY[] = {
-    ASCII_E, ASCII_M, ASCII_P, ASCII_T, ASCII_Y, '\0' };
-static const char KW_ENTITIES[] = {
-    ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_I, ASCII_E, ASCII_S,
-    '\0' };
-static const char KW_ENTITY[] = {
-    ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0' };
-static const char KW_FIXED[] = {
-    ASCII_F, ASCII_I, ASCII_X, ASCII_E, ASCII_D, '\0' };
-static const char KW_ID[] = {
-    ASCII_I, ASCII_D, '\0' };
-static const char KW_IDREF[] = {
-    ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0' };
-static const char KW_IDREFS[] = {
-    ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0' };
+static const char KW_ANY[] = {ASCII_A, ASCII_N, ASCII_Y, '\0'};
+static const char KW_ATTLIST[]
+    = {ASCII_A, ASCII_T, ASCII_T, ASCII_L, ASCII_I, ASCII_S, ASCII_T, '\0'};
+static const char KW_CDATA[]
+    = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_DOCTYPE[]
+    = {ASCII_D, ASCII_O, ASCII_C, ASCII_T, ASCII_Y, ASCII_P, ASCII_E, '\0'};
+static const char KW_ELEMENT[]
+    = {ASCII_E, ASCII_L, ASCII_E, ASCII_M, ASCII_E, ASCII_N, ASCII_T, '\0'};
+static const char KW_EMPTY[]
+    = {ASCII_E, ASCII_M, ASCII_P, ASCII_T, ASCII_Y, '\0'};
+static const char KW_ENTITIES[] = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T,
+                                   ASCII_I, ASCII_E, ASCII_S, '\0'};
+static const char KW_ENTITY[]
+    = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0'};
+static const char KW_FIXED[]
+    = {ASCII_F, ASCII_I, ASCII_X, ASCII_E, ASCII_D, '\0'};
+static const char KW_ID[] = {ASCII_I, ASCII_D, '\0'};
+static const char KW_IDREF[]
+    = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0'};
+static const char KW_IDREFS[]
+    = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0'};
 #ifdef XML_DTD
-static const char KW_IGNORE[] = {
-    ASCII_I, ASCII_G, ASCII_N, ASCII_O, ASCII_R, ASCII_E, '\0' };
+static const char KW_IGNORE[]
+    = {ASCII_I, ASCII_G, ASCII_N, ASCII_O, ASCII_R, ASCII_E, '\0'};
 #endif
-static const char KW_IMPLIED[] = {
-    ASCII_I, ASCII_M, ASCII_P, ASCII_L, ASCII_I, ASCII_E, ASCII_D, '\0' };
+static const char KW_IMPLIED[]
+    = {ASCII_I, ASCII_M, ASCII_P, ASCII_L, ASCII_I, ASCII_E, ASCII_D, '\0'};
 #ifdef XML_DTD
-static const char KW_INCLUDE[] = {
-    ASCII_I, ASCII_N, ASCII_C, ASCII_L, ASCII_U, ASCII_D, ASCII_E, '\0' };
+static const char KW_INCLUDE[]
+    = {ASCII_I, ASCII_N, ASCII_C, ASCII_L, ASCII_U, ASCII_D, ASCII_E, '\0'};
 #endif
-static const char KW_NDATA[] = {
-    ASCII_N, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_NMTOKEN[] = {
-    ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0' };
-static const char KW_NMTOKENS[] = {
-    ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, ASCII_S,
-    '\0' };
-static const char KW_NOTATION[] =
-    { ASCII_N, ASCII_O, ASCII_T, ASCII_A, ASCII_T, ASCII_I, ASCII_O, ASCII_N,
-      '\0' };
-static const char KW_PCDATA[] = {
-    ASCII_P, ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_PUBLIC[] = {
-    ASCII_P, ASCII_U, ASCII_B, ASCII_L, ASCII_I, ASCII_C, '\0' };
-static const char KW_REQUIRED[] = {
-    ASCII_R, ASCII_E, ASCII_Q, ASCII_U, ASCII_I, ASCII_R, ASCII_E, ASCII_D,
-    '\0' };
-static const char KW_SYSTEM[] = {
-    ASCII_S, ASCII_Y, ASCII_S, ASCII_T, ASCII_E, ASCII_M, '\0' };
+static const char KW_NDATA[]
+    = {ASCII_N, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_NMTOKEN[]
+    = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0'};
+static const char KW_NMTOKENS[] = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K,
+                                   ASCII_E, ASCII_N, ASCII_S, '\0'};
+static const char KW_NOTATION[] = {ASCII_N, ASCII_O, ASCII_T, ASCII_A, ASCII_T,
+                                   ASCII_I, ASCII_O, ASCII_N, '\0'};
+static const char KW_PCDATA[]
+    = {ASCII_P, ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_PUBLIC[]
+    = {ASCII_P, ASCII_U, ASCII_B, ASCII_L, ASCII_I, ASCII_C, '\0'};
+static const char KW_REQUIRED[] = {ASCII_R, ASCII_E, ASCII_Q, ASCII_U, ASCII_I,
+                                   ASCII_R, ASCII_E, ASCII_D, '\0'};
+static const char KW_SYSTEM[]
+    = {ASCII_S, ASCII_Y, ASCII_S, ASCII_T, ASCII_E, ASCII_M, '\0'};
 
 #ifndef MIN_BYTES_PER_CHAR
-#define MIN_BYTES_PER_CHAR(enc) ((enc)->minBytesPerChar)
+#  define MIN_BYTES_PER_CHAR(enc) ((enc)->minBytesPerChar)
 #endif
 
 #ifdef XML_DTD
-#define setTopLevel(state) \
-  ((state)->handler = ((state)->documentEntity \
-                       ? internalSubset \
-                       : externalSubset1))
+#  define setTopLevel(state)                                                   \
+    ((state)->handler                                                          \
+     = ((state)->documentEntity ? internalSubset : externalSubset1))
 #else /* not XML_DTD */
-#define setTopLevel(state) ((state)->handler = internalSubset)
+#  define setTopLevel(state) ((state)->handler = internalSubset)
 #endif /* not XML_DTD */
 
-typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state,
-                                   int tok,
-                                   const char *ptr,
-                                   const char *end,
+typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state, int tok,
+                                   const char *ptr, const char *end,
                                    const ENCODING *enc);
 
-static PROLOG_HANDLER
-  prolog0, prolog1, prolog2,
-  doctype0, doctype1, doctype2, doctype3, doctype4, doctype5,
-  internalSubset,
-  entity0, entity1, entity2, entity3, entity4, entity5, entity6,
-  entity7, entity8, entity9, entity10,
-  notation0, notation1, notation2, notation3, notation4,
-  attlist0, attlist1, attlist2, attlist3, attlist4, attlist5, attlist6,
-  attlist7, attlist8, attlist9,
-  element0, element1, element2, element3, element4, element5, element6,
-  element7,
+static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2,
+    doctype3, doctype4, doctype5, internalSubset, entity0, entity1, entity2,
+    entity3, entity4, entity5, entity6, entity7, entity8, entity9, entity10,
+    notation0, notation1, notation2, notation3, notation4, attlist0, attlist1,
+    attlist2, attlist3, attlist4, attlist5, attlist6, attlist7, attlist8,
+    attlist9, element0, element1, element2, element3, element4, element5,
+    element6, element7,
 #ifdef XML_DTD
-  externalSubset0, externalSubset1,
-  condSect0, condSect1, condSect2,
+    externalSubset0, externalSubset1, condSect0, condSect1, condSect2,
 #endif /* XML_DTD */
-  declClose,
-  error;
+    declClose, error;
 
 static int FASTCALL common(PROLOG_STATE *state, int tok);
 
 static int PTRCALL
-prolog0(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     state->handler = prolog1;
@@ -169,10 +150,8 @@ prolog0(PROLOG_STATE *state,
   case XML_TOK_BOM:
     return XML_ROLE_NONE;
   case XML_TOK_DECL_OPEN:
-    if (!XmlNameMatchesAscii(enc,
-                             ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                             end,
-                             KW_DOCTYPE))
+    if (! XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
+                              KW_DOCTYPE))
       break;
     state->handler = doctype0;
     return XML_ROLE_DOCTYPE_NONE;
@@ -184,12 +163,8 @@ prolog0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-prolog1(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -207,10 +182,8 @@ prolog1(PROLOG_STATE *state,
      */
     return XML_ROLE_NONE; /* LCOV_EXCL_LINE */
   case XML_TOK_DECL_OPEN:
-    if (!XmlNameMatchesAscii(enc,
-                             ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                             end,
-                             KW_DOCTYPE))
+    if (! XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
+                              KW_DOCTYPE))
       break;
     state->handler = doctype0;
     return XML_ROLE_DOCTYPE_NONE;
@@ -222,12 +195,11 @@ prolog1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-prolog2(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -243,12 +215,11 @@ prolog2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -261,12 +232,8 @@ doctype0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype1(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -291,12 +258,11 @@ doctype1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype2(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -308,12 +274,11 @@ doctype2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -325,12 +290,11 @@ doctype3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -345,12 +309,11 @@ doctype4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -362,40 +325,28 @@ doctype5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-internalSubset(PROLOG_STATE *state,
-               int tok,
-               const char *ptr,
-               const char *end,
-               const ENCODING *enc)
-{
+internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+               const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
   case XML_TOK_DECL_OPEN:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ENTITY)) {
       state->handler = entity0;
       return XML_ROLE_ENTITY_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ATTLIST)) {
       state->handler = attlist0;
       return XML_ROLE_ATTLIST_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ELEMENT)) {
       state->handler = element0;
       return XML_ROLE_ELEMENT_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_NOTATION)) {
       state->handler = notation0;
       return XML_ROLE_NOTATION_NONE;
@@ -419,12 +370,8 @@ internalSubset(PROLOG_STATE *state,
 #ifdef XML_DTD
 
 static int PTRCALL
-externalSubset0(PROLOG_STATE *state,
-                int tok,
-                const char *ptr,
-                const char *end,
-                const ENCODING *enc)
-{
+externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+                const ENCODING *enc) {
   state->handler = externalSubset1;
   if (tok == XML_TOK_XML_DECL)
     return XML_ROLE_TEXT_DECL;
@@ -432,12 +379,8 @@ externalSubset0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-externalSubset1(PROLOG_STATE *state,
-                int tok,
-                const char *ptr,
-                const char *end,
-                const ENCODING *enc)
-{
+externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+                const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_COND_SECT_OPEN:
     state->handler = condSect0;
@@ -464,12 +407,11 @@ externalSubset1(PROLOG_STATE *state,
 #endif /* XML_DTD */
 
 static int PTRCALL
-entity0(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -484,12 +426,11 @@ entity0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity1(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -501,12 +442,8 @@ entity1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity2(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -529,12 +466,11 @@ entity2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity3(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -546,12 +482,11 @@ entity3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity4(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -563,12 +498,8 @@ entity4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity5(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -586,12 +517,11 @@ entity5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity6(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -604,12 +534,8 @@ entity6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity7(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -632,12 +558,11 @@ entity7(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity8(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -649,12 +574,11 @@ entity8(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity9(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -666,12 +590,11 @@ entity9(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity10(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -683,12 +606,11 @@ entity10(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation0(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -700,12 +622,8 @@ notation0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation1(PROLOG_STATE *state,
-          int tok,
-          const char *ptr,
-          const char *end,
-          const ENCODING *enc)
-{
+notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -724,12 +642,11 @@ notation1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation2(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -741,12 +658,11 @@ notation2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation3(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -759,12 +675,11 @@ notation3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation4(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -780,12 +695,11 @@ notation4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -798,12 +712,11 @@ attlist0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist1(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -819,34 +732,23 @@ attlist1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist2(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
-  case XML_TOK_NAME:
-    {
-      static const char * const types[] = {
-        KW_CDATA,
-        KW_ID,
-        KW_IDREF,
-        KW_IDREFS,
-        KW_ENTITY,
-        KW_ENTITIES,
-        KW_NMTOKEN,
-        KW_NMTOKENS,
-      };
-      int i;
-      for (i = 0; i < (int)(sizeof(types)/sizeof(types[0])); i++)
-        if (XmlNameMatchesAscii(enc, ptr, end, types[i])) {
-          state->handler = attlist8;
-          return XML_ROLE_ATTRIBUTE_TYPE_CDATA + i;
-        }
-    }
+  case XML_TOK_NAME: {
+    static const char *const types[] = {
+        KW_CDATA,  KW_ID,       KW_IDREF,   KW_IDREFS,
+        KW_ENTITY, KW_ENTITIES, KW_NMTOKEN, KW_NMTOKENS,
+    };
+    int i;
+    for (i = 0; i < (int)(sizeof(types) / sizeof(types[0])); i++)
+      if (XmlNameMatchesAscii(enc, ptr, end, types[i])) {
+        state->handler = attlist8;
+        return XML_ROLE_ATTRIBUTE_TYPE_CDATA + i;
+      }
+  }
     if (XmlNameMatchesAscii(enc, ptr, end, KW_NOTATION)) {
       state->handler = attlist5;
       return XML_ROLE_ATTLIST_NONE;
@@ -860,12 +762,11 @@ attlist2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -879,12 +780,11 @@ attlist3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -899,12 +799,11 @@ attlist4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -916,12 +815,11 @@ attlist5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist6(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -933,12 +831,11 @@ attlist6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist7(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -954,33 +851,23 @@ attlist7(PROLOG_STATE *state,
 
 /* default value */
 static int PTRCALL
-attlist8(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
   case XML_TOK_POUND_NAME:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_IMPLIED)) {
       state->handler = attlist1;
       return XML_ROLE_IMPLIED_ATTRIBUTE_VALUE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_REQUIRED)) {
       state->handler = attlist1;
       return XML_ROLE_REQUIRED_ATTRIBUTE_VALUE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_FIXED)) {
       state->handler = attlist9;
       return XML_ROLE_ATTLIST_NONE;
@@ -994,12 +881,11 @@ attlist8(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist9(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -1011,12 +897,11 @@ attlist9(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1029,12 +914,8 @@ element0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element1(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1059,19 +940,13 @@ element1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element2(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
   case XML_TOK_POUND_NAME:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_PCDATA)) {
       state->handler = element3;
       return XML_ROLE_CONTENT_PCDATA;
@@ -1099,12 +974,11 @@ element2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1124,12 +998,11 @@ element3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1142,12 +1015,11 @@ element4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1163,12 +1035,11 @@ element5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element6(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1193,12 +1064,11 @@ element6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element7(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1243,12 +1113,8 @@ element7(PROLOG_STATE *state,
 #ifdef XML_DTD
 
 static int PTRCALL
-condSect0(PROLOG_STATE *state,
-          int tok,
-          const char *ptr,
-          const char *end,
-          const ENCODING *enc)
-{
+condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1267,12 +1133,11 @@ condSect0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-condSect1(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1285,12 +1150,11 @@ condSect1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-condSect2(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1304,12 +1168,11 @@ condSect2(PROLOG_STATE *state,
 #endif /* XML_DTD */
 
 static int PTRCALL
-declClose(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return state->role_none;
@@ -1341,21 +1204,21 @@ declClose(PROLOG_STATE *state,
  * LCOV_EXCL_START
  */
 static int PTRCALL
-error(PROLOG_STATE *UNUSED_P(state),
-      int UNUSED_P(tok),
-      const char *UNUSED_P(ptr),
-      const char *UNUSED_P(end),
-      const ENCODING *UNUSED_P(enc))
-{
+error(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+      const ENCODING *enc) {
+  UNUSED_P(state);
+  UNUSED_P(tok);
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   return XML_ROLE_NONE;
 }
 /* LCOV_EXCL_STOP */
 
 static int FASTCALL
-common(PROLOG_STATE *state, int tok)
-{
+common(PROLOG_STATE *state, int tok) {
 #ifdef XML_DTD
-  if (!state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF)
+  if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF)
     return XML_ROLE_INNER_PARAM_ENTITY_REF;
 #endif
   state->handler = error;
@@ -1363,8 +1226,7 @@ common(PROLOG_STATE *state, int tok)
 }
 
 void
-XmlPrologStateInit(PROLOG_STATE *state)
-{
+XmlPrologStateInit(PROLOG_STATE *state) {
   state->handler = prolog0;
 #ifdef XML_DTD
   state->documentEntity = 1;
@@ -1376,8 +1238,7 @@ XmlPrologStateInit(PROLOG_STATE *state)
 #ifdef XML_DTD
 
 void
-XmlPrologStateInitExternalEntity(PROLOG_STATE *state)
-{
+XmlPrologStateInitExternalEntity(PROLOG_STATE *state) {
   state->handler = externalSubset0;
   state->documentEntity = 0;
   state->includeLevel = 0;
diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h
index e5f048eab55c..036aba64fd29 100644
--- a/Modules/expat/xmlrole.h
+++ b/Modules/expat/xmlrole.h
@@ -36,7 +36,7 @@
 #ifdef __VMS
 /*      0        1         2         3      0        1         2         3
         1234567890123456789012345678901     1234567890123456789012345678901 */
-#define XmlPrologStateInitExternalEntity    XmlPrologStateInitExternalEnt
+#  define XmlPrologStateInitExternalEntity XmlPrologStateInitExternalEnt
 #endif
 
 #include "xmltok.h"
@@ -113,11 +113,8 @@ enum {
 };
 
 typedef struct prolog_state {
-  int (PTRCALL *handler) (struct prolog_state *state,
-                          int tok,
-                          const char *ptr,
-                          const char *end,
-                          const ENCODING *enc);
+  int(PTRCALL *handler)(struct prolog_state *state, int tok, const char *ptr,
+                        const char *end, const ENCODING *enc);
   unsigned level;
   int role_none;
 #ifdef XML_DTD
@@ -132,8 +129,8 @@ void XmlPrologStateInit(PROLOG_STATE *);
 void XmlPrologStateInitExternalEntity(PROLOG_STATE *);
 #endif /* XML_DTD */
 
-#define XmlTokenRole(state, tok, ptr, end, enc) \
- (((state)->handler)(state, tok, ptr, end, enc))
+#define XmlTokenRole(state, tok, ptr, end, enc)                                \
+  (((state)->handler)(state, tok, ptr, end, enc))
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c
index 6b415d83972c..11e9d1ccdad4 100644
--- a/Modules/expat/xmltok.c
+++ b/Modules/expat/xmltok.c
@@ -31,24 +31,23 @@
 */
 
 #include <stddef.h>
-#include <string.h>  /* memcpy */
+#include <string.h> /* memcpy */
 
 #if defined(_MSC_VER) && (_MSC_VER <= 1700)
-  /* for vs2012/11.0/1700 and earlier Visual Studio compilers */
-# define bool   int
-# define false  0
-# define true   1
+/* for vs2012/11.0/1700 and earlier Visual Studio compilers */
+#  define bool int
+#  define false 0
+#  define true 1
 #else
-# include <stdbool.h>
+#  include <stdbool.h>
 #endif
 
-
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #else
-#ifdef HAVE_EXPAT_CONFIG_H
-#include <expat_config.h>
-#endif
+#  ifdef HAVE_EXPAT_CONFIG_H
+#    include <expat_config.h>
+#  endif
 #endif /* ndef _WIN32 */
 
 #include "expat_external.h"
@@ -57,58 +56,49 @@
 #include "nametab.h"
 
 #ifdef XML_DTD
-#define IGNORE_SECTION_TOK_VTABLE , PREFIX(ignoreSectionTok)
+#  define IGNORE_SECTION_TOK_VTABLE , PREFIX(ignoreSectionTok)
 #else
-#define IGNORE_SECTION_TOK_VTABLE /* as nothing */
+#  define IGNORE_SECTION_TOK_VTABLE /* as nothing */
 #endif
 
-#define VTABLE1 \
-  { PREFIX(prologTok), PREFIX(contentTok), \
-    PREFIX(cdataSectionTok) IGNORE_SECTION_TOK_VTABLE }, \
-  { PREFIX(attributeValueTok), PREFIX(entityValueTok) }, \
-  PREFIX(nameMatchesAscii), \
-  PREFIX(nameLength), \
-  PREFIX(skipS), \
-  PREFIX(getAtts), \
-  PREFIX(charRefNumber), \
-  PREFIX(predefinedEntityName), \
-  PREFIX(updatePosition), \
-  PREFIX(isPublicId)
+#define VTABLE1                                                                \
+  {PREFIX(prologTok), PREFIX(contentTok),                                      \
+   PREFIX(cdataSectionTok) IGNORE_SECTION_TOK_VTABLE},                         \
+      {PREFIX(attributeValueTok), PREFIX(entityValueTok)},                     \
+      PREFIX(nameMatchesAscii), PREFIX(nameLength), PREFIX(skipS),             \
+      PREFIX(getAtts), PREFIX(charRefNumber), PREFIX(predefinedEntityName),    \
+      PREFIX(updatePosition), PREFIX(isPublicId)
 
 #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16)
 
-#define UCS2_GET_NAMING(pages, hi, lo) \
-   (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo) & 0x1F)))
+#define UCS2_GET_NAMING(pages, hi, lo)                                         \
+  (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo)&0x1F)))
 
 /* A 2 byte UTF-8 representation splits the characters 11 bits between
    the bottom 5 and 6 bits of the bytes.  We need 8 bits to index into
    pages, 3 bits to add to that index and 5 bits to generate the mask.
 */
-#define UTF8_GET_NAMING2(pages, byte) \
-    (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3) \
-                      + ((((byte)[0]) & 3) << 1) \
-                      + ((((byte)[1]) >> 5) & 1)] \
-         & (1u << (((byte)[1]) & 0x1F)))
+#define UTF8_GET_NAMING2(pages, byte)                                          \
+  (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3)                         \
+                + ((((byte)[0]) & 3) << 1) + ((((byte)[1]) >> 5) & 1)]         \
+   & (1u << (((byte)[1]) & 0x1F)))
 
 /* A 3 byte UTF-8 representation splits the characters 16 bits between
    the bottom 4, 6 and 6 bits of the bytes.  We need 8 bits to index
    into pages, 3 bits to add to that index and 5 bits to generate the
    mask.
 */
-#define UTF8_GET_NAMING3(pages, byte) \
-  (namingBitmap[((pages)[((((byte)[0]) & 0xF) << 4) \
-                             + ((((byte)[1]) >> 2) & 0xF)] \
-                       << 3) \
-                      + ((((byte)[1]) & 3) << 1) \
-                      + ((((byte)[2]) >> 5) & 1)] \
-         & (1u << (((byte)[2]) & 0x1F)))
-
-#define UTF8_GET_NAMING(pages, p, n) \
-  ((n) == 2 \
-  ? UTF8_GET_NAMING2(pages, (const unsigned char *)(p)) \
-  : ((n) == 3 \
-     ? UTF8_GET_NAMING3(pages, (const unsigned char *)(p)) \
-     : 0))
+#define UTF8_GET_NAMING3(pages, byte)                                          \
+  (namingBitmap                                                                \
+       [((pages)[((((byte)[0]) & 0xF) << 4) + ((((byte)[1]) >> 2) & 0xF)]      \
+         << 3)                                                                 \
+        + ((((byte)[1]) & 3) << 1) + ((((byte)[2]) >> 5) & 1)]                 \
+   & (1u << (((byte)[2]) & 0x1F)))
+
+#define UTF8_GET_NAMING(pages, p, n)                                           \
+  ((n) == 2                                                                    \
+       ? UTF8_GET_NAMING2(pages, (const unsigned char *)(p))                   \
+       : ((n) == 3 ? UTF8_GET_NAMING3(pages, (const unsigned char *)(p)) : 0))
 
 /* Detection of invalid UTF-8 sequences is based on Table 3.1B
    of Unicode 3.2: http://www.unicode.org/unicode/reports/tr28/
@@ -120,88 +110,76 @@
      (A & 0xC0) == 0xC0  means A > 0xBF
 */
 
-#define UTF8_INVALID2(p) \
+#define UTF8_INVALID2(p)                                                       \
   ((*p) < 0xC2 || ((p)[1] & 0x80) == 0 || ((p)[1] & 0xC0) == 0xC0)
 
-#define UTF8_INVALID3(p) \
-  (((p)[2] & 0x80) == 0 \
-  || \
-  ((*p) == 0xEF && (p)[1] == 0xBF \
-    ? \
-    (p)[2] > 0xBD \
-    : \
-    ((p)[2] & 0xC0) == 0xC0) \
-  || \
-  ((*p) == 0xE0 \
-    ? \
-    (p)[1] < 0xA0 || ((p)[1] & 0xC0) == 0xC0 \
-    : \
-    ((p)[1] & 0x80) == 0 \
-    || \
-    ((*p) == 0xED ? (p)[1] > 0x9F : ((p)[1] & 0xC0) == 0xC0)))
-
-#define UTF8_INVALID4(p) \
-  (((p)[3] & 0x80) == 0 || ((p)[3] & 0xC0) == 0xC0 \
-  || \
-  ((p)[2] & 0x80) == 0 || ((p)[2] & 0xC0) == 0xC0 \
-  || \
-  ((*p) == 0xF0 \
-    ? \
-    (p)[1] < 0x90 || ((p)[1] & 0xC0) == 0xC0 \
-    : \
-    ((p)[1] & 0x80) == 0 \
-    || \
-    ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0)))
+#define UTF8_INVALID3(p)                                                       \
+  (((p)[2] & 0x80) == 0                                                        \
+   || ((*p) == 0xEF && (p)[1] == 0xBF ? (p)[2] > 0xBD                          \
+                                      : ((p)[2] & 0xC0) == 0xC0)               \
+   || ((*p) == 0xE0                                                            \
+           ? (p)[1] < 0xA0 || ((p)[1] & 0xC0) == 0xC0                          \
+           : ((p)[1] & 0x80) == 0                                              \
+                 || ((*p) == 0xED ? (p)[1] > 0x9F : ((p)[1] & 0xC0) == 0xC0)))
+
+#define UTF8_INVALID4(p)                                                       \
+  (((p)[3] & 0x80) == 0 || ((p)[3] & 0xC0) == 0xC0 || ((p)[2] & 0x80) == 0     \
+   || ((p)[2] & 0xC0) == 0xC0                                                  \
+   || ((*p) == 0xF0                                                            \
+           ? (p)[1] < 0x90 || ((p)[1] & 0xC0) == 0xC0                          \
+           : ((p)[1] & 0x80) == 0                                              \
+                 || ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0)))
 
 static int PTRFASTCALL
-isNever(const ENCODING *UNUSED_P(enc), const char *UNUSED_P(p))
-{
+isNever(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  UNUSED_P(p);
   return 0;
 }
 
 static int PTRFASTCALL
-utf8_isName2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isName2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING2(namePages, (const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isName3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isName3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING3(namePages, (const unsigned char *)p);
 }
 
 #define utf8_isName4 isNever
 
 static int PTRFASTCALL
-utf8_isNmstrt2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isNmstrt2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isNmstrt3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isNmstrt3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p);
 }
 
 #define utf8_isNmstrt4 isNever
 
 static int PTRFASTCALL
-utf8_isInvalid2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID2((const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isInvalid3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID3((const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isInvalid4(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid4(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID4((const unsigned char *)p);
 }
 
@@ -209,61 +187,44 @@ struct normal_encoding {
   ENCODING enc;
   unsigned char type[256];
 #ifdef XML_MIN_SIZE
-  int (PTRFASTCALL *byteType)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNameMin)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *);
-  int (PTRFASTCALL *byteToAscii)(const ENCODING *, const char *);
-  int (PTRCALL *charMatches)(const ENCODING *, const char *, int);
+  int(PTRFASTCALL *byteType)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNameMin)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *);
+  int(PTRFASTCALL *byteToAscii)(const ENCODING *, const char *);
+  int(PTRCALL *charMatches)(const ENCODING *, const char *, int);
 #endif /* XML_MIN_SIZE */
-  int (PTRFASTCALL *isName2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isName3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isName4)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid4)(const ENCODING *, const char *);
 };
 
-#define AS_NORMAL_ENCODING(enc)   ((const struct normal_encoding *) (enc))
+#define AS_NORMAL_ENCODING(enc) ((const struct normal_encoding *)(enc))
 
 #ifdef XML_MIN_SIZE
 
-#define STANDARD_VTABLE(E) \
- E ## byteType, \
- E ## isNameMin, \
- E ## isNmstrtMin, \
- E ## byteToAscii, \
- E ## charMatches,
+#  define STANDARD_VTABLE(E)                                                   \
+    E##byteType, E##isNameMin, E##isNmstrtMin, E##byteToAscii, E##charMatches,
 
 #else
 
-#define STANDARD_VTABLE(E) /* as nothing */
+#  define STANDARD_VTABLE(E) /* as nothing */
 
 #endif
 
-#define NORMAL_VTABLE(E) \
- E ## isName2, \
- E ## isName3, \
- E ## isName4, \
- E ## isNmstrt2, \
- E ## isNmstrt3, \
- E ## isNmstrt4, \
- E ## isInvalid2, \
- E ## isInvalid3, \
- E ## isInvalid4
-
-#define NULL_VTABLE \
- /* isName2 */ NULL, \
- /* isName3 */ NULL, \
- /* isName4 */ NULL, \
- /* isNmstrt2 */ NULL, \
- /* isNmstrt3 */ NULL, \
- /* isNmstrt4 */ NULL, \
- /* isInvalid2 */ NULL, \
- /* isInvalid3 */ NULL, \
- /* isInvalid4 */ NULL
+#define NORMAL_VTABLE(E)                                                       \
+  E##isName2, E##isName3, E##isName4, E##isNmstrt2, E##isNmstrt3,              \
+      E##isNmstrt4, E##isInvalid2, E##isInvalid3, E##isInvalid4
+
+#define NULL_VTABLE                                                            \
+  /* isName2 */ NULL, /* isName3 */ NULL, /* isName4 */ NULL,                  \
+      /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL,        \
+      /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL
 
 static int FASTCALL checkCharRefNumber(int);
 
@@ -271,75 +232,70 @@ static int FASTCALL checkCharRefNumber(int);
 #include "ascii.h"
 
 #ifdef XML_MIN_SIZE
-#define sb_isNameMin isNever
-#define sb_isNmstrtMin isNever
+#  define sb_isNameMin isNever
+#  define sb_isNmstrtMin isNever
 #endif
 
 #ifdef XML_MIN_SIZE
-#define MINBPC(enc) ((enc)->minBytesPerChar)
+#  define MINBPC(enc) ((enc)->minBytesPerChar)
 #else
 /* minimum bytes per character */
-#define MINBPC(enc) 1
+#  define MINBPC(enc) 1
 #endif
 
-#define SB_BYTE_TYPE(enc, p) \
+#define SB_BYTE_TYPE(enc, p)                                                   \
   (((struct normal_encoding *)(enc))->type[(unsigned char)*(p)])
 
 #ifdef XML_MIN_SIZE
 static int PTRFASTCALL
-sb_byteType(const ENCODING *enc, const char *p)
-{
+sb_byteType(const ENCODING *enc, const char *p) {
   return SB_BYTE_TYPE(enc, p);
 }
-#define BYTE_TYPE(enc, p) \
- (AS_NORMAL_ENCODING(enc)->byteType(enc, p))
+#  define BYTE_TYPE(enc, p) (AS_NORMAL_ENCODING(enc)->byteType(enc, p))
 #else
-#define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
+#  define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
 #endif
 
 #ifdef XML_MIN_SIZE
-#define BYTE_TO_ASCII(enc, p) \
- (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p))
+#  define BYTE_TO_ASCII(enc, p) (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p))
 static int PTRFASTCALL
-sb_byteToAscii(const ENCODING *enc, const char *p)
-{
+sb_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return *p;
 }
 #else
-#define BYTE_TO_ASCII(enc, p) (*(p))
+#  define BYTE_TO_ASCII(enc, p) (*(p))
 #endif
 
-#define IS_NAME_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isName ## n(enc, p))
-#define IS_NMSTRT_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isNmstrt ## n(enc, p))
-#define IS_INVALID_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isInvalid ## n(enc, p))
+#define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p))
+#define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p))
+#define IS_INVALID_CHAR(enc, p, n)                                             \
+  (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p))
 
 #ifdef XML_MIN_SIZE
-#define IS_NAME_CHAR_MINBPC(enc, p) \
- (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p))
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) \
- (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p))
+#  define IS_NAME_CHAR_MINBPC(enc, p)                                          \
+    (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p))
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p)                                        \
+    (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p))
 #else
-#define IS_NAME_CHAR_MINBPC(enc, p) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
+#  define IS_NAME_CHAR_MINBPC(enc, p) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
 #endif
 
 #ifdef XML_MIN_SIZE
-#define CHAR_MATCHES(enc, p, c) \
- (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c))
+#  define CHAR_MATCHES(enc, p, c)                                              \
+    (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c))
 static int PTRCALL
-sb_charMatches(const ENCODING *enc, const char *p, int c)
-{
+sb_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
   return *p == c;
 }
 #else
 /* c is an ASCII character */
-#define CHAR_MATCHES(enc, p, c) (*(p) == c)
+#  define CHAR_MATCHES(enc, p, c) (*(p) == c)
 #endif
 
-#define PREFIX(ident) normal_ ## ident
+#define PREFIX(ident) normal_##ident
 #define XML_TOK_IMPL_C
 #include "xmltok_impl.c"
 #undef XML_TOK_IMPL_C
@@ -354,42 +310,46 @@ sb_charMatches(const ENCODING *enc, const char *p, int c)
 #undef IS_NMSTRT_CHAR_MINBPC
 #undef IS_INVALID_CHAR
 
-enum {  /* UTF8_cvalN is value of masked first byte of N byte sequence */
-  UTF8_cval1 = 0x00,
-  UTF8_cval2 = 0xc0,
-  UTF8_cval3 = 0xe0,
-  UTF8_cval4 = 0xf0
+enum { /* UTF8_cvalN is value of masked first byte of N byte sequence */
+       UTF8_cval1 = 0x00,
+       UTF8_cval2 = 0xc0,
+       UTF8_cval3 = 0xe0,
+       UTF8_cval4 = 0xf0
 };
 
 void
-_INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** fromLimRef)
-{
-  const char * fromLim = *fromLimRef;
+_INTERNAL_trim_to_complete_utf8_characters(const char *from,
+                                           const char **fromLimRef) {
+  const char *fromLim = *fromLimRef;
   size_t walked = 0;
   for (; fromLim > from; fromLim--, walked++) {
     const unsigned char prev = (unsigned char)fromLim[-1];
-    if ((prev & 0xf8u) == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */
+    if ((prev & 0xf8u)
+        == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */
       if (walked + 1 >= 4) {
         fromLim += 4 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0xf0u) == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */
+    } else if ((prev & 0xf0u)
+               == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */
       if (walked + 1 >= 3) {
         fromLim += 3 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0xe0u) == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */
+    } else if ((prev & 0xe0u)
+               == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */
       if (walked + 1 >= 2) {
         fromLim += 2 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0x80u) == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */
+    } else if ((prev & 0x80u)
+               == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */
       break;
     }
   }
@@ -397,16 +357,15 @@ _INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** from
 }
 
 static enum XML_Convert_Result PTRCALL
-utf8_toUtf8(const ENCODING *UNUSED_P(enc),
-            const char **fromP, const char *fromLim,
-            char **toP, const char *toLim)
-{
+utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+            char **toP, const char *toLim) {
   bool input_incomplete = false;
   bool output_exhausted = false;
 
   /* Avoid copying partial characters (due to limited space). */
   const ptrdiff_t bytesAvailable = fromLim - *fromP;
   const ptrdiff_t bytesStorable = toLim - *toP;
+  UNUSED_P(enc);
   if (bytesAvailable > bytesStorable) {
     fromLim = *fromP + bytesStorable;
     output_exhausted = true;
@@ -414,7 +373,7 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
 
   /* Avoid copying partial characters (from incomplete input). */
   {
-    const char * const fromLimBefore = fromLim;
+    const char *const fromLimBefore = fromLim;
     _INTERNAL_trim_to_complete_utf8_characters(*fromP, &fromLim);
     if (fromLim < fromLimBefore) {
       input_incomplete = true;
@@ -428,7 +387,7 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
     *toP += bytesToCopy;
   }
 
-  if (output_exhausted)  /* needs to go first */
+  if (output_exhausted) /* needs to go first */
     return XML_CONVERT_OUTPUT_EXHAUSTED;
   else if (input_incomplete)
     return XML_CONVERT_INPUT_INCOMPLETE;
@@ -437,10 +396,8 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
 }
 
 static enum XML_Convert_Result PTRCALL
-utf8_toUtf16(const ENCODING *enc,
-             const char **fromP, const char *fromLim,
-             unsigned short **toP, const unsigned short *toLim)
-{
+utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+             unsigned short **toP, const unsigned short *toLim) {
   enum XML_Convert_Result res = XML_CONVERT_COMPLETED;
   unsigned short *to = *toP;
   const char *from = *fromP;
@@ -459,30 +416,28 @@ utf8_toUtf16(const ENCODING *enc,
         res = XML_CONVERT_INPUT_INCOMPLETE;
         goto after;
       }
-      *to++ = (unsigned short)(((from[0] & 0xf) << 12)
-                               | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f));
+      *to++ = (unsigned short)(((from[0] & 0xf) << 12) | ((from[1] & 0x3f) << 6)
+                               | (from[2] & 0x3f));
       from += 3;
       break;
-    case BT_LEAD4:
-      {
-        unsigned long n;
-        if (toLim - to < 2) {
-          res = XML_CONVERT_OUTPUT_EXHAUSTED;
-          goto after;
-        }
-        if (fromLim - from < 4) {
-          res = XML_CONVERT_INPUT_INCOMPLETE;
-          goto after;
-        }
-        n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12)
-            | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f);
-        n -= 0x10000;
-        to[0] = (unsigned short)((n >> 10) | 0xD800);
-        to[1] = (unsigned short)((n & 0x3FF) | 0xDC00);
-        to += 2;
-        from += 4;
+    case BT_LEAD4: {
+      unsigned long n;
+      if (toLim - to < 2) {
+        res = XML_CONVERT_OUTPUT_EXHAUSTED;
+        goto after;
       }
-      break;
+      if (fromLim - from < 4) {
+        res = XML_CONVERT_INPUT_INCOMPLETE;
+        goto after;
+      }
+      n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12)
+          | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f);
+      n -= 0x10000;
+      to[0] = (unsigned short)((n >> 10) | 0xD800);
+      to[1] = (unsigned short)((n & 0x3FF) | 0xDC00);
+      to += 2;
+      from += 4;
+    } break;
     default:
       *to++ = *from++;
       break;
@@ -497,56 +452,51 @@ utf8_toUtf16(const ENCODING *enc,
 }
 
 #ifdef XML_NS
-static const struct normal_encoding utf8_encoding_ns = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
-#include "asciitab.h"
-#include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+static const struct normal_encoding utf8_encoding_ns
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
+#  include "asciitab.h"
+#  include "utf8tab.h"
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 #endif
 
-static const struct normal_encoding utf8_encoding = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding utf8_encoding
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 #ifdef XML_NS
 
-static const struct normal_encoding internal_utf8_encoding_ns = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
-#include "iasciitab.h"
-#include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+static const struct normal_encoding internal_utf8_encoding_ns
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
+#  include "iasciitab.h"
+#  include "utf8tab.h"
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 #endif
 
-static const struct normal_encoding internal_utf8_encoding = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding internal_utf8_encoding
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "iasciitab.h"
 #undef BT_COLON
 #include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 static enum XML_Convert_Result PTRCALL
-latin1_toUtf8(const ENCODING *UNUSED_P(enc),
-              const char **fromP, const char *fromLim,
-              char **toP, const char *toLim)
-{
+latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+              char **toP, const char *toLim) {
+  UNUSED_P(enc);
   for (;;) {
     unsigned char c;
     if (*fromP == fromLim)
@@ -558,8 +508,7 @@ latin1_toUtf8(const ENCODING *UNUSED_P(enc),
       *(*toP)++ = (char)((c >> 6) | UTF8_cval2);
       *(*toP)++ = (char)((c & 0x3f) | 0x80);
       (*fromP)++;
-    }
-    else {
+    } else {
       if (*toP == toLim)
         return XML_CONVERT_OUTPUT_EXHAUSTED;
       *(*toP)++ = *(*fromP)++;
@@ -568,10 +517,9 @@ latin1_toUtf8(const ENCODING *UNUSED_P(enc),
 }
 
 static enum XML_Convert_Result PTRCALL
-latin1_toUtf16(const ENCODING *UNUSED_P(enc),
-               const char **fromP, const char *fromLim,
-               unsigned short **toP, const unsigned short *toLim)
-{
+latin1_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+               unsigned short **toP, const unsigned short *toLim) {
+  UNUSED_P(enc);
   while (*fromP < fromLim && *toP < toLim)
     *(*toP)++ = (unsigned char)*(*fromP)++;
 
@@ -583,33 +531,30 @@ latin1_toUtf16(const ENCODING *UNUSED_P(enc),
 
 #ifdef XML_NS
 
-static const struct normal_encoding latin1_encoding_ns = {
-  { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+static const struct normal_encoding latin1_encoding_ns
+    = {{VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0},
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding latin1_encoding = {
-  { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
-  {
+static const struct normal_encoding latin1_encoding
+    = {{VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 static enum XML_Convert_Result PTRCALL
-ascii_toUtf8(const ENCODING *UNUSED_P(enc),
-             const char **fromP, const char *fromLim,
-             char **toP, const char *toLim)
-{
+ascii_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+             char **toP, const char *toLim) {
+  UNUSED_P(enc);
   while (*fromP < fromLim && *toP < toLim)
     *(*toP)++ = *(*fromP)++;
 
@@ -621,40 +566,45 @@ ascii_toUtf8(const ENCODING *UNUSED_P(enc),
 
 #ifdef XML_NS
 
-static const struct normal_encoding ascii_encoding_ns = {
-  { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
-  {
-#include "asciitab.h"
-/* BT_NONXML == 0 */
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+static const struct normal_encoding ascii_encoding_ns
+    = {{VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0},
+       {
+#  include "asciitab.h"
+           /* BT_NONXML == 0 */
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding ascii_encoding = {
-  { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding ascii_encoding
+    = {{VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
-/* BT_NONXML == 0 */
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+           /* BT_NONXML == 0 */
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 static int PTRFASTCALL
-unicode_byte_type(char hi, char lo)
-{
+unicode_byte_type(char hi, char lo) {
   switch ((unsigned char)hi) {
-  case 0xD8: case 0xD9: case 0xDA: case 0xDB:
+  /* 0xD800?0xDBFF first 16-bit code unit or high surrogate (W1) */
+  case 0xD8:
+  case 0xD9:
+  case 0xDA:
+  case 0xDB:
     return BT_LEAD4;
-  case 0xDC: case 0xDD: case 0xDE: case 0xDF:
+  /* 0xDC00?0xDFFF second 16-bit code unit or low surrogate (W2) */
+  case 0xDC:
+  case 0xDD:
+  case 0xDE:
+  case 0xDF:
     return BT_TRAIL;
   case 0xFF:
     switch ((unsigned char)lo) {
-    case 0xFF:
-    case 0xFE:
+    case 0xFF: /* noncharacter-FFFF */
+    case 0xFE: /* noncharacter-FFFE */
       return BT_NONXML;
     }
     break;
@@ -662,102 +612,105 @@ unicode_byte_type(char hi, char lo)
   return BT_NONASCII;
 }
 
-#define DEFINE_UTF16_TO_UTF8(E) \
-static enum XML_Convert_Result  PTRCALL \
-E ## toUtf8(const ENCODING *UNUSED_P(enc), \
-            const char **fromP, const char *fromLim, \
-            char **toP, const char *toLim) \
-{ \
-  const char *from = *fromP; \
-  fromLim = from + (((fromLim - from) >> 1) << 1);  /* shrink to even */ \
-  for (; from < fromLim; from += 2) { \
-    int plane; \
-    unsigned char lo2; \
-    unsigned char lo = GET_LO(from); \
-    unsigned char hi = GET_HI(from); \
-    switch (hi) { \
-    case 0: \
-      if (lo < 0x80) { \
-        if (*toP == toLim) { \
-          *fromP = from; \
-          return XML_CONVERT_OUTPUT_EXHAUSTED; \
-        } \
-        *(*toP)++ = lo; \
-        break; \
-      } \
-      /* fall through */ \
-    case 0x1: case 0x2: case 0x3: \
-    case 0x4: case 0x5: case 0x6: case 0x7: \
-      if (toLim -  *toP < 2) { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      *(*toP)++ = ((lo >> 6) | (hi << 2) |  UTF8_cval2); \
-      *(*toP)++ = ((lo & 0x3f) | 0x80); \
-      break; \
-    default: \
-      if (toLim -  *toP < 3)  { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      /* 16 bits divided 4, 6, 6 amongst 3 bytes */ \
-      *(*toP)++ = ((hi >> 4) | UTF8_cval3); \
-      *(*toP)++ = (((hi & 0xf) << 2) | (lo >> 6) | 0x80); \
-      *(*toP)++ = ((lo & 0x3f) | 0x80); \
-      break; \
-    case 0xD8: case 0xD9: case 0xDA: case 0xDB: \
-      if (toLim -  *toP < 4) { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      if (fromLim - from < 4) { \
-        *fromP = from; \
-        return XML_CONVERT_INPUT_INCOMPLETE; \
-      } \
-      plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1; \
-      *(*toP)++ = ((plane >> 2) | UTF8_cval4); \
-      *(*toP)++ = (((lo >> 2) & 0xF) | ((plane & 0x3) << 4) | 0x80); \
-      from += 2; \
-      lo2 = GET_LO(from); \
-      *(*toP)++ = (((lo & 0x3) << 4) \
-                   | ((GET_HI(from) & 0x3) << 2) \
-                   | (lo2 >> 6) \
-                   | 0x80); \
-      *(*toP)++ = ((lo2 & 0x3f) | 0x80); \
-      break; \
-    } \
-  } \
-  *fromP = from; \
-  if (from < fromLim) \
-    return XML_CONVERT_INPUT_INCOMPLETE; \
-  else \
-    return XML_CONVERT_COMPLETED; \
-}
+#define DEFINE_UTF16_TO_UTF8(E)                                                \
+  static enum XML_Convert_Result PTRCALL E##toUtf8(                            \
+      const ENCODING *enc, const char **fromP, const char *fromLim,            \
+      char **toP, const char *toLim) {                                         \
+    const char *from = *fromP;                                                 \
+    UNUSED_P(enc);                                                             \
+    fromLim = from + (((fromLim - from) >> 1) << 1); /* shrink to even */      \
+    for (; from < fromLim; from += 2) {                                        \
+      int plane;                                                               \
+      unsigned char lo2;                                                       \
+      unsigned char lo = GET_LO(from);                                         \
+      unsigned char hi = GET_HI(from);                                         \
+      switch (hi) {                                                            \
+      case 0:                                                                  \
+        if (lo < 0x80) {                                                       \
+          if (*toP == toLim) {                                                 \
+            *fromP = from;                                                     \
+            return XML_CONVERT_OUTPUT_EXHAUSTED;                               \
+          }                                                                    \
+          *(*toP)++ = lo;                                                      \
+          break;                                                               \
+        }                                                                      \
+        /* fall through */                                                     \
+      case 0x1:                                                                \
+      case 0x2:                                                                \
+      case 0x3:                                                                \
+      case 0x4:                                                                \
+      case 0x5:                                                                \
+      case 0x6:                                                                \
+      case 0x7:                                                                \
+        if (toLim - *toP < 2) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        *(*toP)++ = ((lo >> 6) | (hi << 2) | UTF8_cval2);                      \
+        *(*toP)++ = ((lo & 0x3f) | 0x80);                                      \
+        break;                                                                 \
+      default:                                                                 \
+        if (toLim - *toP < 3) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        /* 16 bits divided 4, 6, 6 amongst 3 bytes */                          \
+        *(*toP)++ = ((hi >> 4) | UTF8_cval3);                                  \
+        *(*toP)++ = (((hi & 0xf) << 2) | (lo >> 6) | 0x80);                    \
+        *(*toP)++ = ((lo & 0x3f) | 0x80);                                      \
+        break;                                                                 \
+      case 0xD8:                                                               \
+      case 0xD9:                                                               \
+      case 0xDA:                                                               \
+      case 0xDB:                                                               \
+        if (toLim - *toP < 4) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        if (fromLim - from < 4) {                                              \
+          *fromP = from;                                                       \
+          return XML_CONVERT_INPUT_INCOMPLETE;                                 \
+        }                                                                      \
+        plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1;                   \
+        *(*toP)++ = (char)((plane >> 2) | UTF8_cval4);                         \
+        *(*toP)++ = (((lo >> 2) & 0xF) | ((plane & 0x3) << 4) | 0x80);         \
+        from += 2;                                                             \
+        lo2 = GET_LO(from);                                                    \
+        *(*toP)++ = (((lo & 0x3) << 4) | ((GET_HI(from) & 0x3) << 2)           \
+                     | (lo2 >> 6) | 0x80);                                     \
+        *(*toP)++ = ((lo2 & 0x3f) | 0x80);                                     \
+        break;                                                                 \
+      }                                                                        \
+    }                                                                          \
+    *fromP = from;                                                             \
+    if (from < fromLim)                                                        \
+      return XML_CONVERT_INPUT_INCOMPLETE;                                     \
+    else                                                                       \
+      return XML_CONVERT_COMPLETED;                                            \
+  }
 
-#define DEFINE_UTF16_TO_UTF16(E) \
-static enum XML_Convert_Result  PTRCALL \
-E ## toUtf16(const ENCODING *UNUSED_P(enc), \
-             const char **fromP, const char *fromLim, \
-             unsigned short **toP, const unsigned short *toLim) \
-{ \
-  enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \
-  fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1);  /* shrink to even */ \
-  /* Avoid copying first half only of surrogate */ \
-  if (fromLim - *fromP > ((toLim - *toP) << 1) \
-      && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \
-    fromLim -= 2; \
-    res = XML_CONVERT_INPUT_INCOMPLETE; \
-  } \
-  for (; *fromP < fromLim && *toP < toLim; *fromP += 2) \
-    *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP); \
-  if ((*toP == toLim) && (*fromP < fromLim)) \
-    return XML_CONVERT_OUTPUT_EXHAUSTED; \
-  else \
-    return res; \
-}
+#define DEFINE_UTF16_TO_UTF16(E)                                               \
+  static enum XML_Convert_Result PTRCALL E##toUtf16(                           \
+      const ENCODING *enc, const char **fromP, const char *fromLim,            \
+      unsigned short **toP, const unsigned short *toLim) {                     \
+    enum XML_Convert_Result res = XML_CONVERT_COMPLETED;                       \
+    UNUSED_P(enc);                                                             \
+    fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */  \
+    /* Avoid copying first half only of surrogate */                           \
+    if (fromLim - *fromP > ((toLim - *toP) << 1)                               \
+        && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) {                             \
+      fromLim -= 2;                                                            \
+      res = XML_CONVERT_INPUT_INCOMPLETE;                                      \
+    }                                                                          \
+    for (; *fromP < fromLim && *toP < toLim; *fromP += 2)                      \
+      *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP);                      \
+    if ((*toP == toLim) && (*fromP < fromLim))                                 \
+      return XML_CONVERT_OUTPUT_EXHAUSTED;                                     \
+    else                                                                       \
+      return res;                                                              \
+  }
 
-#define SET2(ptr, ch) \
-  (((ptr)[0] = ((ch) & 0xff)), ((ptr)[1] = ((ch) >> 8)))
+#define SET2(ptr, ch) (((ptr)[0] = ((ch)&0xff)), ((ptr)[1] = ((ch) >> 8)))
 #define GET_LO(ptr) ((unsigned char)(ptr)[0])
 #define GET_HI(ptr) ((unsigned char)(ptr)[1])
 
@@ -768,8 +721,7 @@ DEFINE_UTF16_TO_UTF16(little2_)
 #undef GET_LO
 #undef GET_HI
 
-#define SET2(ptr, ch) \
-  (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch) & 0xFF)))
+#define SET2(ptr, ch) (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch)&0xFF)))
 #define GET_LO(ptr) ((unsigned char)(ptr)[1])
 #define GET_HI(ptr) ((unsigned char)(ptr)[0])
 
@@ -780,292 +732,279 @@ DEFINE_UTF16_TO_UTF16(big2_)
 #undef GET_LO
 #undef GET_HI
 
-#define LITTLE2_BYTE_TYPE(enc, p) \
- ((p)[1] == 0 \
-  ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)] \
-  : unicode_byte_type((p)[1], (p)[0]))
-#define LITTLE2_BYTE_TO_ASCII(enc, p) ((p)[1] == 0 ? (p)[0] : -1)
-#define LITTLE2_CHAR_MATCHES(enc, p, c) ((p)[1] == 0 && (p)[0] == c)
-#define LITTLE2_IS_NAME_CHAR_MINBPC(enc, p) \
+#define LITTLE2_BYTE_TYPE(enc, p)                                              \
+  ((p)[1] == 0 ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)]  \
+               : unicode_byte_type((p)[1], (p)[0]))
+#define LITTLE2_BYTE_TO_ASCII(p) ((p)[1] == 0 ? (p)[0] : -1)
+#define LITTLE2_CHAR_MATCHES(p, c) ((p)[1] == 0 && (p)[0] == c)
+#define LITTLE2_IS_NAME_CHAR_MINBPC(p)                                         \
   UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0])
-#define LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
+#define LITTLE2_IS_NMSTRT_CHAR_MINBPC(p)                                       \
   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[1], (unsigned char)p[0])
 
 #ifdef XML_MIN_SIZE
 
 static int PTRFASTCALL
-little2_byteType(const ENCODING *enc, const char *p)
-{
+little2_byteType(const ENCODING *enc, const char *p) {
   return LITTLE2_BYTE_TYPE(enc, p);
 }
 
 static int PTRFASTCALL
-little2_byteToAscii(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_BYTE_TO_ASCII(enc, p);
+little2_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_BYTE_TO_ASCII(p);
 }
 
 static int PTRCALL
-little2_charMatches(const ENCODING *enc, const char *p, int c)
-{
-  return LITTLE2_CHAR_MATCHES(enc, p, c);
+little2_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
+  return LITTLE2_CHAR_MATCHES(p, c);
 }
 
 static int PTRFASTCALL
-little2_isNameMin(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_IS_NAME_CHAR_MINBPC(enc, p);
+little2_isNameMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_IS_NAME_CHAR_MINBPC(p);
 }
 
 static int PTRFASTCALL
-little2_isNmstrtMin(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p);
+little2_isNmstrtMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_IS_NMSTRT_CHAR_MINBPC(p);
 }
 
-#undef VTABLE
-#define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
+#  undef VTABLE
+#  define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
 
 #else /* not XML_MIN_SIZE */
 
-#undef PREFIX
-#define PREFIX(ident) little2_ ## ident
-#define MINBPC(enc) 2
+#  undef PREFIX
+#  define PREFIX(ident) little2_##ident
+#  define MINBPC(enc) 2
 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
-#define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
-#define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(enc, p)
-#define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(enc, p, c)
-#define IS_NAME_CHAR(enc, p, n) 0
-#define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(enc, p)
-#define IS_NMSTRT_CHAR(enc, p, n) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p)
-
-#define XML_TOK_IMPL_C
-#include "xmltok_impl.c"
-#undef XML_TOK_IMPL_C
-
-#undef MINBPC
-#undef BYTE_TYPE
-#undef BYTE_TO_ASCII
-#undef CHAR_MATCHES
-#undef IS_NAME_CHAR
-#undef IS_NAME_CHAR_MINBPC
-#undef IS_NMSTRT_CHAR
-#undef IS_NMSTRT_CHAR_MINBPC
-#undef IS_INVALID_CHAR
+#  define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
+#  define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p)
+#  define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c)
+#  define IS_NAME_CHAR(enc, p, n) 0
+#  define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p)
+#  define IS_NMSTRT_CHAR(enc, p, n) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p)
+
+#  define XML_TOK_IMPL_C
+#  include "xmltok_impl.c"
+#  undef XML_TOK_IMPL_C
+
+#  undef MINBPC
+#  undef BYTE_TYPE
+#  undef BYTE_TO_ASCII
+#  undef CHAR_MATCHES
+#  undef IS_NAME_CHAR
+#  undef IS_NAME_CHAR_MINBPC
+#  undef IS_NMSTRT_CHAR
+#  undef IS_NMSTRT_CHAR_MINBPC
+#  undef IS_INVALID_CHAR
 
 #endif /* not XML_MIN_SIZE */
 
 #ifdef XML_NS
 
-static const struct normal_encoding little2_encoding_ns = {
-  { VTABLE, 2, 0,
-#if BYTEORDER == 1234
-    1
-#else
-    0
-#endif
-  },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding little2_encoding_ns
+    = {{VTABLE, 2, 0,
+#  if BYTEORDER == 1234
+        1
+#  else
+        0
+#  endif
+       },
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding little2_encoding = {
-  { VTABLE, 2, 0,
+static const struct normal_encoding little2_encoding
+    = {{VTABLE, 2, 0,
 #if BYTEORDER == 1234
-    1
+        1
 #else
-    0
+        0
 #endif
-  },
-  {
+       },
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #if BYTEORDER != 4321
 
-#ifdef XML_NS
+#  ifdef XML_NS
 
-static const struct normal_encoding internal_little2_encoding_ns = {
-  { VTABLE, 2, 0, 1 },
-  {
-#include "iasciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_little2_encoding_ns
+    = {{VTABLE, 2, 0, 1},
+       {
+#    include "iasciitab.h"
+#    include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
-#endif
+#  endif
 
-static const struct normal_encoding internal_little2_encoding = {
-  { VTABLE, 2, 0, 1 },
-  {
-#define BT_COLON BT_NMSTRT
-#include "iasciitab.h"
-#undef BT_COLON
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_little2_encoding
+    = {{VTABLE, 2, 0, 1},
+       {
+#  define BT_COLON BT_NMSTRT
+#  include "iasciitab.h"
+#  undef BT_COLON
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #endif
 
-
-#define BIG2_BYTE_TYPE(enc, p) \
- ((p)[0] == 0 \
-  ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]] \
-  : unicode_byte_type((p)[0], (p)[1]))
-#define BIG2_BYTE_TO_ASCII(enc, p) ((p)[0] == 0 ? (p)[1] : -1)
-#define BIG2_CHAR_MATCHES(enc, p, c) ((p)[0] == 0 && (p)[1] == c)
-#define BIG2_IS_NAME_CHAR_MINBPC(enc, p) \
+#define BIG2_BYTE_TYPE(enc, p)                                                 \
+  ((p)[0] == 0                                                                 \
+       ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]]        \
+       : unicode_byte_type((p)[0], (p)[1]))
+#define BIG2_BYTE_TO_ASCII(p) ((p)[0] == 0 ? (p)[1] : -1)
+#define BIG2_CHAR_MATCHES(p, c) ((p)[0] == 0 && (p)[1] == c)
+#define BIG2_IS_NAME_CHAR_MINBPC(p)                                            \
   UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1])
-#define BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
+#define BIG2_IS_NMSTRT_CHAR_MINBPC(p)                                          \
   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[0], (unsigned char)p[1])
 
 #ifdef XML_MIN_SIZE
 
 static int PTRFASTCALL
-big2_byteType(const ENCODING *enc, const char *p)
-{
+big2_byteType(const ENCODING *enc, const char *p) {
   return BIG2_BYTE_TYPE(enc, p);
 }
 
 static int PTRFASTCALL
-big2_byteToAscii(const ENCODING *enc, const char *p)
-{
-  return BIG2_BYTE_TO_ASCII(enc, p);
+big2_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_BYTE_TO_ASCII(p);
 }
 
 static int PTRCALL
-big2_charMatches(const ENCODING *enc, const char *p, int c)
-{
-  return BIG2_CHAR_MATCHES(enc, p, c);
+big2_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
+  return BIG2_CHAR_MATCHES(p, c);
 }
 
 static int PTRFASTCALL
-big2_isNameMin(const ENCODING *enc, const char *p)
-{
-  return BIG2_IS_NAME_CHAR_MINBPC(enc, p);
+big2_isNameMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_IS_NAME_CHAR_MINBPC(p);
 }
 
 static int PTRFASTCALL
-big2_isNmstrtMin(const ENCODING *enc, const char *p)
-{
-  return BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p);
+big2_isNmstrtMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_IS_NMSTRT_CHAR_MINBPC(p);
 }
 
-#undef VTABLE
-#define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
+#  undef VTABLE
+#  define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
 
 #else /* not XML_MIN_SIZE */
 
-#undef PREFIX
-#define PREFIX(ident) big2_ ## ident
-#define MINBPC(enc) 2
+#  undef PREFIX
+#  define PREFIX(ident) big2_##ident
+#  define MINBPC(enc) 2
 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
-#define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
-#define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(enc, p)
-#define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(enc, p, c)
-#define IS_NAME_CHAR(enc, p, n) 0
-#define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(enc, p)
-#define IS_NMSTRT_CHAR(enc, p, n) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p)
-
-#define XML_TOK_IMPL_C
-#include "xmltok_impl.c"
-#undef XML_TOK_IMPL_C
-
-#undef MINBPC
-#undef BYTE_TYPE
-#undef BYTE_TO_ASCII
-#undef CHAR_MATCHES
-#undef IS_NAME_CHAR
-#undef IS_NAME_CHAR_MINBPC
-#undef IS_NMSTRT_CHAR
-#undef IS_NMSTRT_CHAR_MINBPC
-#undef IS_INVALID_CHAR
+#  define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
+#  define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p)
+#  define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c)
+#  define IS_NAME_CHAR(enc, p, n) 0
+#  define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p)
+#  define IS_NMSTRT_CHAR(enc, p, n) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p)
+
+#  define XML_TOK_IMPL_C
+#  include "xmltok_impl.c"
+#  undef XML_TOK_IMPL_C
+
+#  undef MINBPC
+#  undef BYTE_TYPE
+#  undef BYTE_TO_ASCII
+#  undef CHAR_MATCHES
+#  undef IS_NAME_CHAR
+#  undef IS_NAME_CHAR_MINBPC
+#  undef IS_NMSTRT_CHAR
+#  undef IS_NMSTRT_CHAR_MINBPC
+#  undef IS_INVALID_CHAR
 
 #endif /* not XML_MIN_SIZE */
 
 #ifdef XML_NS
 
-static const struct normal_encoding big2_encoding_ns = {
-  { VTABLE, 2, 0,
-#if BYTEORDER == 4321
-  1
-#else
-  0
-#endif
-  },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding big2_encoding_ns
+    = {{VTABLE, 2, 0,
+#  if BYTEORDER == 4321
+        1
+#  else
+        0
+#  endif
+       },
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding big2_encoding = {
-  { VTABLE, 2, 0,
+static const struct normal_encoding big2_encoding
+    = {{VTABLE, 2, 0,
 #if BYTEORDER == 4321
-  1
+        1
 #else
-  0
+        0
 #endif
-  },
-  {
+       },
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #if BYTEORDER != 1234
 
-#ifdef XML_NS
+#  ifdef XML_NS
 
-static const struct normal_encoding internal_big2_encoding_ns = {
-  { VTABLE, 2, 0, 1 },
-  {
-#include "iasciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_big2_encoding_ns
+    = {{VTABLE, 2, 0, 1},
+       {
+#    include "iasciitab.h"
+#    include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
-#endif
+#  endif
 
-static const struct normal_encoding internal_big2_encoding = {
-  { VTABLE, 2, 0, 1 },
-  {
-#define BT_COLON BT_NMSTRT
-#include "iasciitab.h"
-#undef BT_COLON
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_big2_encoding
+    = {{VTABLE, 2, 0, 1},
+       {
+#  define BT_COLON BT_NMSTRT
+#  include "iasciitab.h"
+#  undef BT_COLON
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #endif
 
 #undef PREFIX
 
 static int FASTCALL
-streqci(const char *s1, const char *s2)
-{
+streqci(const char *s1, const char *s2) {
   for (;;) {
     char c1 = *s1++;
     char c2 = *s2++;
@@ -1079,22 +1018,21 @@ streqci(const char *s1, const char *s2)
       c2 += ASCII_A - ASCII_a; /* LCOV_EXCL_LINE */
     if (c1 != c2)
       return 0;
-    if (!c1)
+    if (! c1)
       break;
   }
   return 1;
 }
 
 static void PTRCALL
-initUpdatePosition(const ENCODING *UNUSED_P(enc), const char *ptr,
-                   const char *end, POSITION *pos)
-{
+initUpdatePosition(const ENCODING *enc, const char *ptr, const char *end,
+                   POSITION *pos) {
+  UNUSED_P(enc);
   normal_updatePosition(&utf8_encoding.enc, ptr, end, pos);
 }
 
 static int
-toAscii(const ENCODING *enc, const char *ptr, const char *end)
-{
+toAscii(const ENCODING *enc, const char *ptr, const char *end) {
   char buf[1];
   char *p = buf;
   XmlUtf8Convert(enc, &ptr, end, &p, p + 1);
@@ -1105,8 +1043,7 @@ toAscii(const ENCODING *enc, const char *ptr, const char *end)
 }
 
 static int FASTCALL
-isSpace(int c)
-{
+isSpace(int c) {
   switch (c) {
   case 0x20:
   case 0xD:
@@ -1121,21 +1058,16 @@ isSpace(int c)
    followed by name=val.
 */
 static int
-parsePseudoAttribute(const ENCODING *enc,
-                     const char *ptr,
-                     const char *end,
-                     const char **namePtr,
-                     const char **nameEndPtr,
-                     const char **valPtr,
-                     const char **nextTokPtr)
-{
+parsePseudoAttribute(const ENCODING *enc, const char *ptr, const char *end,
+                     const char **namePtr, const char **nameEndPtr,
+                     const char **valPtr, const char **nextTokPtr) {
   int c;
   char open;
   if (ptr == end) {
     *namePtr = NULL;
     return 1;
   }
-  if (!isSpace(toAscii(enc, ptr, end))) {
+  if (! isSpace(toAscii(enc, ptr, end))) {
     *nextTokPtr = ptr;
     return 0;
   }
@@ -1191,12 +1123,9 @@ parsePseudoAttribute(const ENCODING *enc,
     c = toAscii(enc, ptr, end);
     if (c == open)
       break;
-    if (!(ASCII_a <= c && c <= ASCII_z)
-        && !(ASCII_A <= c && c <= ASCII_Z)
-        && !(ASCII_0 <= c && c <= ASCII_9)
-        && c != ASCII_PERIOD
-        && c != ASCII_MINUS
-        && c != ASCII_UNDERSCORE) {
+    if (! (ASCII_a <= c && c <= ASCII_z) && ! (ASCII_A <= c && c <= ASCII_Z)
+        && ! (ASCII_0 <= c && c <= ASCII_9) && c != ASCII_PERIOD
+        && c != ASCII_MINUS && c != ASCII_UNDERSCORE) {
       *nextTokPtr = ptr;
       return 0;
     }
@@ -1205,68 +1134,52 @@ parsePseudoAttribute(const ENCODING *enc,
   return 1;
 }
 
-static const char KW_version[] = {
-  ASCII_v, ASCII_e, ASCII_r, ASCII_s, ASCII_i, ASCII_o, ASCII_n, '\0'
-};
+static const char KW_version[]
+    = {ASCII_v, ASCII_e, ASCII_r, ASCII_s, ASCII_i, ASCII_o, ASCII_n, '\0'};
 
-static const char KW_encoding[] = {
-  ASCII_e, ASCII_n, ASCII_c, ASCII_o, ASCII_d, ASCII_i, ASCII_n, ASCII_g, '\0'
-};
+static const char KW_encoding[] = {ASCII_e, ASCII_n, ASCII_c, ASCII_o, ASCII_d,
+                                   ASCII_i, ASCII_n, ASCII_g, '\0'};
 
-static const char KW_standalone[] = {
-  ASCII_s, ASCII_t, ASCII_a, ASCII_n, ASCII_d, ASCII_a, ASCII_l, ASCII_o,
-  ASCII_n, ASCII_e, '\0'
-};
+static const char KW_standalone[]
+    = {ASCII_s, ASCII_t, ASCII_a, ASCII_n, ASCII_d, ASCII_a,
+       ASCII_l, ASCII_o, ASCII_n, ASCII_e, '\0'};
 
-static const char KW_yes[] = {
-  ASCII_y, ASCII_e, ASCII_s,  '\0'
-};
+static const char KW_yes[] = {ASCII_y, ASCII_e, ASCII_s, '\0'};
 
-static const char KW_no[] = {
-  ASCII_n, ASCII_o,  '\0'
-};
+static const char KW_no[] = {ASCII_n, ASCII_o, '\0'};
 
 static int
-doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
-                                                 const char *,
+doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *,
                                                  const char *),
-               int isGeneralTextEntity,
-               const ENCODING *enc,
-               const char *ptr,
-               const char *end,
-               const char **badPtr,
-               const char **versionPtr,
-               const char **versionEndPtr,
-               const char **encodingName,
-               const ENCODING **encoding,
-               int *standalone)
-{
+               int isGeneralTextEntity, const ENCODING *enc, const char *ptr,
+               const char *end, const char **badPtr, const char **versionPtr,
+               const char **versionEndPtr, const char **encodingName,
+               const ENCODING **encoding, int *standalone) {
   const char *val = NULL;
   const char *name = NULL;
   const char *nameEnd = NULL;
   ptr += 5 * enc->minBytesPerChar;
   end -= 2 * enc->minBytesPerChar;
-  if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)
-      || !name) {
+  if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)
+      || ! name) {
     *badPtr = ptr;
     return 0;
   }
-  if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_version)) {
-    if (!isGeneralTextEntity) {
+  if (! XmlNameMatchesAscii(enc, name, nameEnd, KW_version)) {
+    if (! isGeneralTextEntity) {
       *badPtr = name;
       return 0;
     }
-  }
-  else {
+  } else {
     if (versionPtr)
       *versionPtr = val;
     if (versionEndPtr)
       *versionEndPtr = ptr;
-    if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
+    if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
       *badPtr = ptr;
       return 0;
     }
-    if (!name) {
+    if (! name) {
       if (isGeneralTextEntity) {
         /* a TextDecl must have an EncodingDecl */
         *badPtr = ptr;
@@ -1277,7 +1190,7 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
   }
   if (XmlNameMatchesAscii(enc, name, nameEnd, KW_encoding)) {
     int c = toAscii(enc, val, end);
-    if (!(ASCII_a <= c && c <= ASCII_z) && !(ASCII_A <= c && c <= ASCII_Z)) {
+    if (! (ASCII_a <= c && c <= ASCII_z) && ! (ASCII_A <= c && c <= ASCII_Z)) {
       *badPtr = val;
       return 0;
     }
@@ -1285,14 +1198,14 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
       *encodingName = val;
     if (encoding)
       *encoding = encodingFinder(enc, val, ptr - enc->minBytesPerChar);
-    if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
+    if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
       *badPtr = ptr;
       return 0;
     }
-    if (!name)
+    if (! name)
       return 1;
   }
-  if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_standalone)
+  if (! XmlNameMatchesAscii(enc, name, nameEnd, KW_standalone)
       || isGeneralTextEntity) {
     *badPtr = name;
     return 0;
@@ -1300,12 +1213,10 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
   if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_yes)) {
     if (standalone)
       *standalone = 1;
-  }
-  else if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_no)) {
+  } else if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_no)) {
     if (standalone)
       *standalone = 0;
-  }
-  else {
+  } else {
     *badPtr = val;
     return 0;
   }
@@ -1319,11 +1230,16 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
 }
 
 static int FASTCALL
-checkCharRefNumber(int result)
-{
+checkCharRefNumber(int result) {
   switch (result >> 8) {
-  case 0xD8: case 0xD9: case 0xDA: case 0xDB:
-  case 0xDC: case 0xDD: case 0xDE: case 0xDF:
+  case 0xD8:
+  case 0xD9:
+  case 0xDA:
+  case 0xDB:
+  case 0xDC:
+  case 0xDD:
+  case 0xDE:
+  case 0xDF:
     return -1;
   case 0:
     if (latin1_encoding.type[result] == BT_NONXML)
@@ -1338,8 +1254,7 @@ checkCharRefNumber(int result)
 }
 
 int FASTCALL
-XmlUtf8Encode(int c, char *buf)
-{
+XmlUtf8Encode(int c, char *buf) {
   enum {
     /* minN is minimum legal resulting value for N byte sequence */
     min2 = 0x80,
@@ -1375,8 +1290,7 @@ XmlUtf8Encode(int c, char *buf)
 }
 
 int FASTCALL
-XmlUtf16Encode(int charNum, unsigned short *buf)
-{
+XmlUtf16Encode(int charNum, unsigned short *buf) {
   if (charNum < 0)
     return 0;
   if (charNum < 0x10000) {
@@ -1400,17 +1314,15 @@ struct unknown_encoding {
   char utf8[256][4];
 };
 
-#define AS_UNKNOWN_ENCODING(enc)  ((const struct unknown_encoding *) (enc))
+#define AS_UNKNOWN_ENCODING(enc) ((const struct unknown_encoding *)(enc))
 
 int
-XmlSizeOfUnknownEncoding(void)
-{
+XmlSizeOfUnknownEncoding(void) {
   return sizeof(struct unknown_encoding);
 }
 
 static int PTRFASTCALL
-unknown_isName(const ENCODING *enc, const char *p)
-{
+unknown_isName(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   if (c & ~0xFFFF)
@@ -1419,8 +1331,7 @@ unknown_isName(const ENCODING *enc, const char *p)
 }
 
 static int PTRFASTCALL
-unknown_isNmstrt(const ENCODING *enc, const char *p)
-{
+unknown_isNmstrt(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   if (c & ~0xFFFF)
@@ -1429,18 +1340,15 @@ unknown_isNmstrt(const ENCODING *enc, const char *p)
 }
 
 static int PTRFASTCALL
-unknown_isInvalid(const ENCODING *enc, const char *p)
-{
+unknown_isInvalid(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   return (c & ~0xFFFF) || checkCharRefNumber(c) < 0;
 }
 
 static enum XML_Convert_Result PTRCALL
-unknown_toUtf8(const ENCODING *enc,
-               const char **fromP, const char *fromLim,
-               char **toP, const char *toLim)
-{
+unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+               char **toP, const char *toLim) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   char buf[XML_UTF8_ENCODE_MAX];
   for (;;) {
@@ -1458,8 +1366,7 @@ unknown_toUtf8(const ENCODING *enc,
       utf8 = buf;
       *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP]
                  - (BT_LEAD2 - 2));
-    }
-    else {
+    } else {
       if (n > toLim - *toP)
         return XML_CONVERT_OUTPUT_EXHAUSTED;
       (*fromP)++;
@@ -1470,20 +1377,16 @@ unknown_toUtf8(const ENCODING *enc,
 }
 
 static enum XML_Convert_Result PTRCALL
-unknown_toUtf16(const ENCODING *enc,
-                const char **fromP, const char *fromLim,
-                unsigned short **toP, const unsigned short *toLim)
-{
+unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+                unsigned short **toP, const unsigned short *toLim) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   while (*fromP < fromLim && *toP < toLim) {
     unsigned short c = uenc->utf16[(unsigned char)**fromP];
     if (c == 0) {
-      c = (unsigned short)
-          uenc->convert(uenc->userData, *fromP);
+      c = (unsigned short)uenc->convert(uenc->userData, *fromP);
       *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP]
                  - (BT_LEAD2 - 2));
-    }
-    else
+    } else
       (*fromP)++;
     *(*toP)++ = c;
   }
@@ -1495,19 +1398,14 @@ unknown_toUtf16(const ENCODING *enc,
 }
 
 ENCODING *
-XmlInitUnknownEncoding(void *mem,
-                       int *table,
-                       CONVERTER convert,
-                       void *userData)
-{
+XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert,
+                       void *userData) {
   int i;
   struct unknown_encoding *e = (struct unknown_encoding *)mem;
-  for (i = 0; i < (int)sizeof(struct normal_encoding); i++)
-    ((char *)mem)[i] = ((char *)&latin1_encoding)[i];
+  memcpy(mem, &latin1_encoding, sizeof(struct normal_encoding));
   for (i = 0; i < 128; i++)
     if (latin1_encoding.type[i] != BT_OTHER
-        && latin1_encoding.type[i] != BT_NONXML
-        && table[i] != i)
+        && latin1_encoding.type[i] != BT_NONXML && table[i] != i)
       return 0;
   for (i = 0; i < 256; i++) {
     int c = table[i];
@@ -1517,35 +1415,30 @@ XmlInitUnknownEncoding(void *mem,
       e->utf16[i] = 0xFFFF;
       e->utf8[i][0] = 1;
       e->utf8[i][1] = 0;
-    }
-    else if (c < 0) {
+    } else if (c < 0) {
       if (c < -4)
         return 0;
       /* Multi-byte sequences need a converter function */
-      if (!convert)
+      if (! convert)
         return 0;
       e->normal.type[i] = (unsigned char)(BT_LEAD2 - (c + 2));
       e->utf8[i][0] = 0;
       e->utf16[i] = 0;
-    }
-    else if (c < 0x80) {
+    } else if (c < 0x80) {
       if (latin1_encoding.type[c] != BT_OTHER
-          && latin1_encoding.type[c] != BT_NONXML
-          && c != i)
+          && latin1_encoding.type[c] != BT_NONXML && c != i)
         return 0;
       e->normal.type[i] = latin1_encoding.type[c];
       e->utf8[i][0] = 1;
       e->utf8[i][1] = (char)c;
       e->utf16[i] = (unsigned short)(c == 0 ? 0xFFFF : c);
-    }
-    else if (checkCharRefNumber(c) < 0) {
+    } else if (checkCharRefNumber(c) < 0) {
       e->normal.type[i] = BT_NONXML;
       /* This shouldn't really get used. */
       e->utf16[i] = 0xFFFF;
       e->utf8[i][0] = 1;
       e->utf8[i][1] = 0;
-    }
-    else {
+    } else {
       if (c > 0xFFFF)
         return 0;
       if (UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xff))
@@ -1590,44 +1483,32 @@ enum {
   NO_ENC
 };
 
-static const char KW_ISO_8859_1[] = {
-  ASCII_I, ASCII_S, ASCII_O, ASCII_MINUS, ASCII_8, ASCII_8, ASCII_5, ASCII_9,
-  ASCII_MINUS, ASCII_1, '\0'
-};
-static const char KW_US_ASCII[] = {
-  ASCII_U, ASCII_S, ASCII_MINUS, ASCII_A, ASCII_S, ASCII_C, ASCII_I, ASCII_I,
-  '\0'
-};
-static const char KW_UTF_8[] =  {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_8, '\0'
-};
-static const char KW_UTF_16[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, '\0'
-};
-static const char KW_UTF_16BE[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_B, ASCII_E,
-  '\0'
-};
-static const char KW_UTF_16LE[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_L, ASCII_E,
-  '\0'
-};
+static const char KW_ISO_8859_1[]
+    = {ASCII_I, ASCII_S, ASCII_O,     ASCII_MINUS, ASCII_8, ASCII_8,
+       ASCII_5, ASCII_9, ASCII_MINUS, ASCII_1,     '\0'};
+static const char KW_US_ASCII[]
+    = {ASCII_U, ASCII_S, ASCII_MINUS, ASCII_A, ASCII_S,
+       ASCII_C, ASCII_I, ASCII_I,     '\0'};
+static const char KW_UTF_8[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_8, '\0'};
+static const char KW_UTF_16[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, '\0'};
+static const char KW_UTF_16BE[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1,
+       ASCII_6, ASCII_B, ASCII_E, '\0'};
+static const char KW_UTF_16LE[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1,
+       ASCII_6, ASCII_L, ASCII_E, '\0'};
 
 static int FASTCALL
-getEncodingIndex(const char *name)
-{
-  static const char * const encodingNames[] = {
-    KW_ISO_8859_1,
-    KW_US_ASCII,
-    KW_UTF_8,
-    KW_UTF_16,
-    KW_UTF_16BE,
-    KW_UTF_16LE,
+getEncodingIndex(const char *name) {
+  static const char *const encodingNames[] = {
+      KW_ISO_8859_1, KW_US_ASCII, KW_UTF_8, KW_UTF_16, KW_UTF_16BE, KW_UTF_16LE,
   };
   int i;
   if (name == NULL)
     return NO_ENC;
-  for (i = 0; i < (int)(sizeof(encodingNames)/sizeof(encodingNames[0])); i++)
+  for (i = 0; i < (int)(sizeof(encodingNames) / sizeof(encodingNames[0])); i++)
     if (streqci(name, encodingNames[i]))
       return i;
   return UNKNOWN_ENC;
@@ -1647,15 +1528,9 @@ getEncodingIndex(const char *name)
    XML_PROLOG_STATE otherwise.
 */
 
-
 static int
-initScan(const ENCODING * const *encodingTable,
-         const INIT_ENCODING *enc,
-         int state,
-         const char *ptr,
-         const char *end,
-         const char **nextTokPtr)
-{
+initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc,
+         int state, const char *ptr, const char *end, const char **nextTokPtr) {
   const ENCODING **encPtr;
 
   if (ptr >= end)
@@ -1680,20 +1555,17 @@ initScan(const ENCODING * const *encodingTable,
     case 0xFE:
     case 0xFF:
     case 0xEF: /* possibly first byte of UTF-8 BOM */
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       /* fall through */
     case 0x00:
     case 0x3C:
       return XML_TOK_PARTIAL;
     }
-  }
-  else {
+  } else {
     switch (((unsigned char)ptr[0] << 8) | (unsigned char)ptr[1]) {
     case 0xFEFF:
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       *nextTokPtr = ptr + 2;
       *encPtr = encodingTable[UTF_16BE_ENC];
@@ -1707,8 +1579,7 @@ initScan(const ENCODING * const *encodingTable,
       *encPtr = encodingTable[UTF_16LE_ENC];
       return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
     case 0xFFFE:
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       *nextTokPtr = ptr + 2;
       *encPtr = encodingTable[UTF_16LE_ENC];
@@ -1723,8 +1594,8 @@ initScan(const ENCODING * const *encodingTable,
       */
       if (state == XML_CONTENT_STATE) {
         int e = INIT_ENC_INDEX(enc);
-        if (e == ISO_8859_1_ENC || e == UTF_16BE_ENC
-            || e == UTF_16LE_ENC || e == UTF_16_ENC)
+        if (e == ISO_8859_1_ENC || e == UTF_16BE_ENC || e == UTF_16LE_ENC
+            || e == UTF_16_ENC)
           break;
       }
       if (ptr + 2 == end)
@@ -1747,8 +1618,7 @@ initScan(const ENCODING * const *encodingTable,
           break;
         *encPtr = encodingTable[UTF_16BE_ENC];
         return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
-      }
-      else if (ptr[1] == '\0') {
+      } else if (ptr[1] == '\0') {
         /* We could recover here in the case:
             - parsing an external entity
             - second byte is 0
@@ -1770,7 +1640,6 @@ initScan(const ENCODING * const *encodingTable,
   return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
 }
 
-
 #define NS(x) x
 #define ns(x) x
 #define XML_TOK_NS_C
@@ -1781,22 +1650,19 @@ initScan(const ENCODING * const *encodingTable,
 
 #ifdef XML_NS
 
-#define NS(x) x ## NS
-#define ns(x) x ## _ns
+#  define NS(x) x##NS
+#  define ns(x) x##_ns
 
-#define XML_TOK_NS_C
-#include "xmltok_ns.c"
-#undef XML_TOK_NS_C
+#  define XML_TOK_NS_C
+#  include "xmltok_ns.c"
+#  undef XML_TOK_NS_C
 
-#undef NS
-#undef ns
+#  undef NS
+#  undef ns
 
 ENCODING *
-XmlInitUnknownEncodingNS(void *mem,
-                         int *table,
-                         CONVERTER convert,
-                         void *userData)
-{
+XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert,
+                         void *userData) {
   ENCODING *enc = XmlInitUnknownEncoding(mem, table, convert, userData);
   if (enc)
     ((struct normal_encoding *)enc)->type[ASCII_COLON] = BT_COLON;
diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h
index 50926f38ab32..2adbf5307bef 100644
--- a/Modules/expat/xmltok.h
+++ b/Modules/expat/xmltok.h
@@ -38,16 +38,18 @@ extern "C" {
 #endif
 
 /* The following token may be returned by XmlContentTok */
-#define XML_TOK_TRAILING_RSQB -5 /* ] or ]] at the end of the scan; might be
-                                    start of illegal ]]> sequence */
+#define XML_TOK_TRAILING_RSQB                                                  \
+  -5 /* ] or ]] at the end of the scan; might be                               \
+        start of illegal ]]> sequence */
 /* The following tokens may be returned by both XmlPrologTok and
    XmlContentTok.
 */
-#define XML_TOK_NONE -4          /* The string to be scanned is empty */
-#define XML_TOK_TRAILING_CR -3   /* A CR at the end of the scan;
-                                    might be part of CRLF sequence */
-#define XML_TOK_PARTIAL_CHAR -2  /* only part of a multibyte sequence */
-#define XML_TOK_PARTIAL -1       /* only part of a token */
+#define XML_TOK_NONE -4 /* The string to be scanned is empty */
+#define XML_TOK_TRAILING_CR                                                    \
+  -3                            /* A CR at the end of the scan;                \
+                                   might be part of CRLF sequence */
+#define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */
+#define XML_TOK_PARTIAL -1      /* only part of a token */
 #define XML_TOK_INVALID 0
 
 /* The following tokens are returned by XmlContentTok; some are also
@@ -62,24 +64,24 @@ extern "C" {
 #define XML_TOK_DATA_NEWLINE 7
 #define XML_TOK_CDATA_SECT_OPEN 8
 #define XML_TOK_ENTITY_REF 9
-#define XML_TOK_CHAR_REF 10               /* numeric character reference */
+#define XML_TOK_CHAR_REF 10 /* numeric character reference */
 
 /* The following tokens may be returned by both XmlPrologTok and
    XmlContentTok.
 */
-#define XML_TOK_PI 11                     /* processing instruction */
-#define XML_TOK_XML_DECL 12               /* XML decl or text decl */
+#define XML_TOK_PI 11       /* processing instruction */
+#define XML_TOK_XML_DECL 12 /* XML decl or text decl */
 #define XML_TOK_COMMENT 13
-#define XML_TOK_BOM 14                    /* Byte order mark */
+#define XML_TOK_BOM 14 /* Byte order mark */
 
 /* The following tokens are returned only by XmlPrologTok */
 #define XML_TOK_PROLOG_S 15
-#define XML_TOK_DECL_OPEN 16              /* <!foo */
-#define XML_TOK_DECL_CLOSE 17             /* > */
+#define XML_TOK_DECL_OPEN 16  /* <!foo */
+#define XML_TOK_DECL_CLOSE 17 /* > */
 #define XML_TOK_NAME 18
 #define XML_TOK_NMTOKEN 19
-#define XML_TOK_POUND_NAME 20             /* #name */
-#define XML_TOK_OR 21                     /* | */
+#define XML_TOK_POUND_NAME 20 /* #name */
+#define XML_TOK_OR 21         /* | */
 #define XML_TOK_PERCENT 22
 #define XML_TOK_OPEN_PAREN 23
 #define XML_TOK_CLOSE_PAREN 24
@@ -90,14 +92,14 @@ extern "C" {
 #define XML_TOK_INSTANCE_START 29
 
 /* The following occur only in element type declarations */
-#define XML_TOK_NAME_QUESTION 30          /* name? */
-#define XML_TOK_NAME_ASTERISK 31          /* name* */
-#define XML_TOK_NAME_PLUS 32              /* name+ */
-#define XML_TOK_COND_SECT_OPEN 33         /* <![ */
-#define XML_TOK_COND_SECT_CLOSE 34        /* ]]> */
-#define XML_TOK_CLOSE_PAREN_QUESTION 35   /* )? */
-#define XML_TOK_CLOSE_PAREN_ASTERISK 36   /* )* */
-#define XML_TOK_CLOSE_PAREN_PLUS 37       /* )+ */
+#define XML_TOK_NAME_QUESTION 30        /* name? */
+#define XML_TOK_NAME_ASTERISK 31        /* name* */
+#define XML_TOK_NAME_PLUS 32            /* name+ */
+#define XML_TOK_COND_SECT_OPEN 33       /* <![ */
+#define XML_TOK_COND_SECT_CLOSE 34      /* ]]> */
+#define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */
+#define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */
+#define XML_TOK_CLOSE_PAREN_PLUS 37     /* )+ */
 #define XML_TOK_COMMA 38
 
 /* The following token is returned only by XmlAttributeValueTok */
@@ -112,20 +114,20 @@ extern "C" {
 #define XML_TOK_PREFIXED_NAME 41
 
 #ifdef XML_DTD
-#define XML_TOK_IGNORE_SECT 42
+#  define XML_TOK_IGNORE_SECT 42
 #endif /* XML_DTD */
 
 #ifdef XML_DTD
-#define XML_N_STATES 4
+#  define XML_N_STATES 4
 #else /* not XML_DTD */
-#define XML_N_STATES 3
+#  define XML_N_STATES 3
 #endif /* not XML_DTD */
 
 #define XML_PROLOG_STATE 0
 #define XML_CONTENT_STATE 1
 #define XML_CDATA_SECTION_STATE 2
 #ifdef XML_DTD
-#define XML_IGNORE_SECTION_STATE 3
+#  define XML_IGNORE_SECTION_STATE 3
 #endif /* XML_DTD */
 
 #define XML_N_LITERAL_TYPES 2
@@ -153,52 +155,41 @@ typedef struct {
 struct encoding;
 typedef struct encoding ENCODING;
 
-typedef int (PTRCALL *SCANNER)(const ENCODING *,
-                               const char *,
-                               const char *,
-                               const char **);
+typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *,
+                              const char **);
 
 enum XML_Convert_Result {
   XML_CONVERT_COMPLETED = 0,
   XML_CONVERT_INPUT_INCOMPLETE = 1,
-  XML_CONVERT_OUTPUT_EXHAUSTED = 2  /* and therefore potentially input remaining as well */
+  XML_CONVERT_OUTPUT_EXHAUSTED
+  = 2 /* and therefore potentially input remaining as well */
 };
 
 struct encoding {
   SCANNER scanners[XML_N_STATES];
   SCANNER literalScanners[XML_N_LITERAL_TYPES];
-  int (PTRCALL *nameMatchesAscii)(const ENCODING *,
-                                  const char *,
-                                  const char *,
-                                  const char *);
-  int (PTRFASTCALL *nameLength)(const ENCODING *, const char *);
+  int(PTRCALL *nameMatchesAscii)(const ENCODING *, const char *, const char *,
+                                 const char *);
+  int(PTRFASTCALL *nameLength)(const ENCODING *, const char *);
   const char *(PTRFASTCALL *skipS)(const ENCODING *, const char *);
-  int (PTRCALL *getAtts)(const ENCODING *enc,
-                         const char *ptr,
-                         int attsMax,
-                         ATTRIBUTE *atts);
-  int (PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr);
-  int (PTRCALL *predefinedEntityName)(const ENCODING *,
-                                      const char *,
-                                      const char *);
-  void (PTRCALL *updatePosition)(const ENCODING *,
-                                 const char *ptr,
-                                 const char *end,
-                                 POSITION *);
-  int (PTRCALL *isPublicId)(const ENCODING *enc,
-                            const char *ptr,
-                            const char *end,
-                            const char **badPtr);
-  enum XML_Convert_Result (PTRCALL *utf8Convert)(const ENCODING *enc,
-                              const char **fromP,
-                              const char *fromLim,
-                              char **toP,
-                              const char *toLim);
-  enum XML_Convert_Result (PTRCALL *utf16Convert)(const ENCODING *enc,
-                               const char **fromP,
-                               const char *fromLim,
-                               unsigned short **toP,
-                               const unsigned short *toLim);
+  int(PTRCALL *getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
+                        ATTRIBUTE *atts);
+  int(PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr);
+  int(PTRCALL *predefinedEntityName)(const ENCODING *, const char *,
+                                     const char *);
+  void(PTRCALL *updatePosition)(const ENCODING *, const char *ptr,
+                                const char *end, POSITION *);
+  int(PTRCALL *isPublicId)(const ENCODING *enc, const char *ptr,
+                           const char *end, const char **badPtr);
+  enum XML_Convert_Result(PTRCALL *utf8Convert)(const ENCODING *enc,
+                                                const char **fromP,
+                                                const char *fromLim, char **toP,
+                                                const char *toLim);
+  enum XML_Convert_Result(PTRCALL *utf16Convert)(const ENCODING *enc,
+                                                 const char **fromP,
+                                                 const char *fromLim,
+                                                 unsigned short **toP,
+                                                 const unsigned short *toLim);
   int minBytesPerChar;
   char isUtf8;
   char isUtf16;
@@ -225,66 +216,62 @@ struct encoding {
    the prolog outside literals, comments and processing instructions.
 */
 
-
-#define XmlTok(enc, state, ptr, end, nextTokPtr) \
+#define XmlTok(enc, state, ptr, end, nextTokPtr)                               \
   (((enc)->scanners[state])(enc, ptr, end, nextTokPtr))
 
-#define XmlPrologTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr)
+#define XmlPrologTok(enc, ptr, end, nextTokPtr)                                \
+  XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr)
 
-#define XmlContentTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr)
+#define XmlContentTok(enc, ptr, end, nextTokPtr)                               \
+  XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr)
 
-#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr)
+#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr)                          \
+  XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr)
 
 #ifdef XML_DTD
 
-#define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr)
+#  define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr)                       \
+    XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr)
 
 #endif /* XML_DTD */
 
 /* This is used for performing a 2nd-level tokenization on the content
    of a literal that has already been returned by XmlTok.
 */
-#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \
+#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr)                  \
   (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr))
 
-#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \
-   XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr)
+#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr)                        \
+  XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr)
 
-#define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \
-   XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr)
+#define XmlEntityValueTok(enc, ptr, end, nextTokPtr)                           \
+  XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr)
 
-#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \
+#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2)                             \
   (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2))
 
-#define XmlNameLength(enc, ptr) \
-  (((enc)->nameLength)(enc, ptr))
+#define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr))
 
-#define XmlSkipS(enc, ptr) \
-  (((enc)->skipS)(enc, ptr))
+#define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr))
 
-#define XmlGetAttributes(enc, ptr, attsMax, atts) \
+#define XmlGetAttributes(enc, ptr, attsMax, atts)                              \
   (((enc)->getAtts)(enc, ptr, attsMax, atts))
 
-#define XmlCharRefNumber(enc, ptr) \
-  (((enc)->charRefNumber)(enc, ptr))
+#define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr))
 
-#define XmlPredefinedEntityName(enc, ptr, end) \
+#define XmlPredefinedEntityName(enc, ptr, end)                                 \
   (((enc)->predefinedEntityName)(enc, ptr, end))
 
-#define XmlUpdatePosition(enc, ptr, end, pos) \
+#define XmlUpdatePosition(enc, ptr, end, pos)                                  \
   (((enc)->updatePosition)(enc, ptr, end, pos))
 
-#define XmlIsPublicId(enc, ptr, end, badPtr) \
+#define XmlIsPublicId(enc, ptr, end, badPtr)                                   \
   (((enc)->isPublicId)(enc, ptr, end, badPtr))
 
-#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \
+#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim)                        \
   (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim))
 
-#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \
+#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim)                       \
   (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim))
 
 typedef struct {
@@ -292,16 +279,11 @@ typedef struct {
   const ENCODING **encPtr;
 } INIT_ENCODING;
 
-int XmlParseXmlDecl(int isGeneralTextEntity,
-                    const ENCODING *enc,
-                    const char *ptr,
-                    const char *end,
-                    const char **badPtr,
-                    const char **versionPtr,
-                    const char **versionEndPtr,
+int XmlParseXmlDecl(int isGeneralTextEntity, const ENCODING *enc,
+                    const char *ptr, const char *end, const char **badPtr,
+                    const char **versionPtr, const char **versionEndPtr,
                     const char **encodingNamePtr,
-                    const ENCODING **namedEncodingPtr,
-                    int *standalonePtr);
+                    const ENCODING **namedEncodingPtr, int *standalonePtr);
 
 int XmlInitEncoding(INIT_ENCODING *, const ENCODING **, const char *name);
 const ENCODING *XmlGetUtf8InternalEncoding(void);
@@ -310,34 +292,22 @@ int FASTCALL XmlUtf8Encode(int charNumber, char *buf);
 int FASTCALL XmlUtf16Encode(int charNumber, unsigned short *buf);
 int XmlSizeOfUnknownEncoding(void);
 
+typedef int(XMLCALL *CONVERTER)(void *userData, const char *p);
 
-typedef int (XMLCALL *CONVERTER) (void *userData, const char *p);
-
-ENCODING *
-XmlInitUnknownEncoding(void *mem,
-                       int *table,
-                       CONVERTER convert,
-                       void *userData);
+ENCODING *XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert,
+                                 void *userData);
 
-int XmlParseXmlDeclNS(int isGeneralTextEntity,
-                      const ENCODING *enc,
-                      const char *ptr,
-                      const char *end,
-                      const char **badPtr,
-                      const char **versionPtr,
-                      const char **versionEndPtr,
+int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc,
+                      const char *ptr, const char *end, const char **badPtr,
+                      const char **versionPtr, const char **versionEndPtr,
                       const char **encodingNamePtr,
-                      const ENCODING **namedEncodingPtr,
-                      int *standalonePtr);
+                      const ENCODING **namedEncodingPtr, int *standalonePtr);
 
 int XmlInitEncodingNS(INIT_ENCODING *, const ENCODING **, const char *name);
 const ENCODING *XmlGetUtf8InternalEncodingNS(void);
 const ENCODING *XmlGetUtf16InternalEncodingNS(void);
-ENCODING *
-XmlInitUnknownEncodingNS(void *mem,
-                         int *table,
-                         CONVERTER convert,
-                         void *userData);
+ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert,
+                                   void *userData);
 #ifdef __cplusplus
 }
 #endif
diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c
index 4d9ae7dc3896..c209221cd79d 100644
--- a/Modules/expat/xmltok_impl.c
+++ b/Modules/expat/xmltok_impl.c
@@ -32,130 +32,124 @@
 
 #ifdef XML_TOK_IMPL_C
 
-#ifndef IS_INVALID_CHAR
-#define IS_INVALID_CHAR(enc, ptr, n) (0)
-#endif
-
-#define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n) \
-        return XML_TOK_PARTIAL_CHAR; \
-      if (IS_INVALID_CHAR(enc, ptr, n)) { \
-        *(nextTokPtr) = (ptr); \
-        return XML_TOK_INVALID; \
-      } \
-      ptr += n; \
-      break;
+#  ifndef IS_INVALID_CHAR
+#    define IS_INVALID_CHAR(enc, ptr, n) (0)
+#  endif
+
+#  define INVALID_LEAD_CASE(n, ptr, nextTokPtr)                                \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (IS_INVALID_CHAR(enc, ptr, n)) {                                        \
+      *(nextTokPtr) = (ptr);                                                   \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define INVALID_CASES(ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(2, ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(3, ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(4, ptr, nextTokPtr) \
-  case BT_NONXML: \
-  case BT_MALFORM: \
-  case BT_TRAIL: \
-    *(nextTokPtr) = (ptr); \
+#  define INVALID_CASES(ptr, nextTokPtr)                                       \
+    INVALID_LEAD_CASE(2, ptr, nextTokPtr)                                      \
+    INVALID_LEAD_CASE(3, ptr, nextTokPtr)                                      \
+    INVALID_LEAD_CASE(4, ptr, nextTokPtr)                                      \
+  case BT_NONXML:                                                              \
+  case BT_MALFORM:                                                             \
+  case BT_TRAIL:                                                               \
+    *(nextTokPtr) = (ptr);                                                     \
     return XML_TOK_INVALID;
 
-#define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr) \
-   case BT_LEAD ## n: \
-     if (end - ptr < n) \
-       return XML_TOK_PARTIAL_CHAR; \
-     if (!IS_NAME_CHAR(enc, ptr, n)) { \
-       *nextTokPtr = ptr; \
-       return XML_TOK_INVALID; \
-     } \
-     ptr += n; \
-     break;
-
-#define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) \
-  case BT_NONASCII: \
-    if (!IS_NAME_CHAR_MINBPC(enc, ptr)) { \
-      *nextTokPtr = ptr; \
-      return XML_TOK_INVALID; \
-    } \
-    /* fall through */ \
-  case BT_NMSTRT: \
-  case BT_HEX: \
-  case BT_DIGIT: \
-  case BT_NAME: \
-  case BT_MINUS: \
-    ptr += MINBPC(enc); \
-    break; \
-  CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr) \
-  CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr) \
-  CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)
-
-#define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr) \
-   case BT_LEAD ## n: \
-     if (end - ptr < n) \
-       return XML_TOK_PARTIAL_CHAR; \
-     if (!IS_NMSTRT_CHAR(enc, ptr, n)) { \
-       *nextTokPtr = ptr; \
-       return XML_TOK_INVALID; \
-     } \
-     ptr += n; \
-     break;
-
-#define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) \
-  case BT_NONASCII: \
-    if (!IS_NMSTRT_CHAR_MINBPC(enc, ptr)) { \
-      *nextTokPtr = ptr; \
-      return XML_TOK_INVALID; \
-    } \
-    /* fall through */ \
-  case BT_NMSTRT: \
-  case BT_HEX: \
-    ptr += MINBPC(enc); \
-    break; \
-  CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr) \
-  CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr) \
-  CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)
-
-#ifndef PREFIX
-#define PREFIX(ident) ident
-#endif
-
-
-#define HAS_CHARS(enc, ptr, end, count) \
-    (end - ptr >= count * MINBPC(enc))
+#  define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr)                        \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (! IS_NAME_CHAR(enc, ptr, n)) {                                         \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define HAS_CHAR(enc, ptr, end) \
-    HAS_CHARS(enc, ptr, end, 1)
+#  define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)                          \
+  case BT_NONASCII:                                                            \
+    if (! IS_NAME_CHAR_MINBPC(enc, ptr)) {                                     \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    /* fall through */                                                         \
+  case BT_NMSTRT:                                                              \
+  case BT_HEX:                                                                 \
+  case BT_DIGIT:                                                               \
+  case BT_NAME:                                                                \
+  case BT_MINUS:                                                               \
+    ptr += MINBPC(enc);                                                        \
+    break;                                                                     \
+    CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr)                              \
+    CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr)                              \
+    CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)
+
+#  define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr)                      \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (! IS_NMSTRT_CHAR(enc, ptr, n)) {                                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define REQUIRE_CHARS(enc, ptr, end, count) \
-    { \
-      if (! HAS_CHARS(enc, ptr, end, count)) { \
-        return XML_TOK_PARTIAL; \
-      } \
+#  define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)                        \
+  case BT_NONASCII:                                                            \
+    if (! IS_NMSTRT_CHAR_MINBPC(enc, ptr)) {                                   \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    /* fall through */                                                         \
+  case BT_NMSTRT:                                                              \
+  case BT_HEX:                                                                 \
+    ptr += MINBPC(enc);                                                        \
+    break;                                                                     \
+    CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr)                            \
+    CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr)                            \
+    CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)
+
+#  ifndef PREFIX
+#    define PREFIX(ident) ident
+#  endif
+
+#  define HAS_CHARS(enc, ptr, end, count) (end - ptr >= count * MINBPC(enc))
+
+#  define HAS_CHAR(enc, ptr, end) HAS_CHARS(enc, ptr, end, 1)
+
+#  define REQUIRE_CHARS(enc, ptr, end, count)                                  \
+    {                                                                          \
+      if (! HAS_CHARS(enc, ptr, end, count)) {                                 \
+        return XML_TOK_PARTIAL;                                                \
+      }                                                                        \
     }
 
-#define REQUIRE_CHAR(enc, ptr, end) \
-    REQUIRE_CHARS(enc, ptr, end, 1)
-
+#  define REQUIRE_CHAR(enc, ptr, end) REQUIRE_CHARS(enc, ptr, end, 1)
 
 /* ptr points to character following "<!-" */
 
 static int PTRCALL
-PREFIX(scanComment)(const ENCODING *enc, const char *ptr,
-                    const char *end, const char **nextTokPtr)
-{
+PREFIX(scanComment)(const ENCODING *enc, const char *ptr, const char *end,
+                    const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
-    if (!CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
     }
     ptr += MINBPC(enc);
     while (HAS_CHAR(enc, ptr, end)) {
       switch (BYTE_TYPE(enc, ptr)) {
-      INVALID_CASES(ptr, nextTokPtr)
+        INVALID_CASES(ptr, nextTokPtr)
       case BT_MINUS:
         ptr += MINBPC(enc);
         REQUIRE_CHAR(enc, ptr, end);
         if (CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
-          if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+          if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
             *nextTokPtr = ptr;
             return XML_TOK_INVALID;
           }
@@ -175,9 +169,8 @@ PREFIX(scanComment)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "<!" */
 
 static int PTRCALL
-PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
-                 const char *end, const char **nextTokPtr)
-{
+PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, const char *end,
+                 const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
   case BT_MINUS:
@@ -199,12 +192,17 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
       REQUIRE_CHARS(enc, ptr, end, 2);
       /* don't allow <!ENTITY% foo "whatever"> */
       switch (BYTE_TYPE(enc, ptr + MINBPC(enc))) {
-      case BT_S: case BT_CR: case BT_LF: case BT_PERCNT:
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+      case BT_PERCNT:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       /* fall through */
-    case BT_S: case BT_CR: case BT_LF:
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       *nextTokPtr = ptr;
       return XML_TOK_DECL_OPEN;
     case BT_NMSTRT:
@@ -220,12 +218,12 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                      const char *end, int *tokPtr)
-{
+PREFIX(checkPiTarget)(const ENCODING *enc, const char *ptr, const char *end,
+                      int *tokPtr) {
   int upper = 0;
+  UNUSED_P(enc);
   *tokPtr = XML_TOK_PI;
-  if (end - ptr != MINBPC(enc)*3)
+  if (end - ptr != MINBPC(enc) * 3)
     return 1;
   switch (BYTE_TO_ASCII(enc, ptr)) {
   case ASCII_x:
@@ -265,30 +263,31 @@ PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr,
 /* ptr points to character following "<?" */
 
 static int PTRCALL
-PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
-               const char *end, const char **nextTokPtr)
-{
+PREFIX(scanPi)(const ENCODING *enc, const char *ptr, const char *end,
+               const char **nextTokPtr) {
   int tok;
   const char *target = ptr;
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_S: case BT_CR: case BT_LF:
-      if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
+      if (! PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       ptr += MINBPC(enc);
       while (HAS_CHAR(enc, ptr, end)) {
         switch (BYTE_TYPE(enc, ptr)) {
-        INVALID_CASES(ptr, nextTokPtr)
+          INVALID_CASES(ptr, nextTokPtr)
         case BT_QUEST:
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
@@ -304,7 +303,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
       }
       return XML_TOK_PARTIAL;
     case BT_QUEST:
-      if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
+      if (! PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -324,16 +323,16 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                         const char *end, const char **nextTokPtr)
-{
-  static const char CDATA_LSQB[] = { ASCII_C, ASCII_D, ASCII_A,
-                                     ASCII_T, ASCII_A, ASCII_LSQB };
+PREFIX(scanCdataSection)(const ENCODING *enc, const char *ptr, const char *end,
+                         const char **nextTokPtr) {
+  static const char CDATA_LSQB[]
+      = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, ASCII_LSQB};
   int i;
+  UNUSED_P(enc);
   /* CDATA[ */
   REQUIRE_CHARS(enc, ptr, end, 6);
   for (i = 0; i < 6; i++, ptr += MINBPC(enc)) {
-    if (!CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {
+    if (! CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
     }
@@ -343,9 +342,8 @@ PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
-                        const char *end, const char **nextTokPtr)
-{
+PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
+                        const char **nextTokPtr) {
   if (ptr >= end)
     return XML_TOK_NONE;
   if (MINBPC(enc) > 1) {
@@ -361,11 +359,11 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
   case BT_RSQB:
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
-    if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
+    if (! CHAR_MATCHES(enc, ptr, ASCII_RSQB))
       break;
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
-    if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
       ptr -= MINBPC(enc);
       break;
     }
@@ -381,23 +379,25 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
   case BT_LF:
     *nextTokPtr = ptr + MINBPC(enc);
     return XML_TOK_DATA_NEWLINE;
-  INVALID_CASES(ptr, nextTokPtr)
+    INVALID_CASES(ptr, nextTokPtr)
   default:
     ptr += MINBPC(enc);
     break;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \
-        *nextTokPtr = ptr; \
-        return XML_TOK_DATA_CHARS; \
-      } \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) {                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_DATA_CHARS;                                               \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONXML:
     case BT_MALFORM:
     case BT_TRAIL:
@@ -418,23 +418,26 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "</" */
 
 static int PTRCALL
-PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
-                   const char *end, const char **nextTokPtr)
-{
+PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr, const char *end,
+                   const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_S: case BT_CR: case BT_LF:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) {
         switch (BYTE_TYPE(enc, ptr)) {
-        case BT_S: case BT_CR: case BT_LF:
+        case BT_S:
+        case BT_CR:
+        case BT_LF:
           break;
         case BT_GT:
           *nextTokPtr = ptr + MINBPC(enc);
@@ -445,13 +448,13 @@ PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
         }
       }
       return XML_TOK_PARTIAL;
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
       /* no need to check qname syntax here,
          since end-tag must match exactly */
       ptr += MINBPC(enc);
       break;
-#endif
+#  endif
     case BT_GT:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_END_TAG;
@@ -466,9 +469,8 @@ PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "&#X" */
 
 static int PTRCALL
-PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr,
-                       const char *end, const char **nextTokPtr)
-{
+PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, const char *end,
+                       const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
     case BT_DIGIT:
@@ -498,9 +500,8 @@ PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "&#" */
 
 static int PTRCALL
-PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr,
-                    const char *end, const char **nextTokPtr)
-{
+PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, const char *end,
+                    const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
     if (CHAR_MATCHES(enc, ptr, ASCII_x))
       return PREFIX(scanHexCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -531,11 +532,10 @@ PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr,
 
 static int PTRCALL
 PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
-                const char **nextTokPtr)
-{
+                const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   case BT_NUM:
     return PREFIX(scanCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
   default:
@@ -544,7 +544,7 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
     case BT_SEMI:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_ENTITY_REF;
@@ -560,15 +560,14 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
-                 const char **nextTokPtr)
-{
-#ifdef XML_NS
+                 const char **nextTokPtr) {
+#  ifdef XML_NS
   int hadColon = 0;
-#endif
+#  endif
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-#ifdef XML_NS
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+#  ifdef XML_NS
     case BT_COLON:
       if (hadColon) {
         *nextTokPtr = ptr;
@@ -578,14 +577,16 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
       switch (BYTE_TYPE(enc, ptr)) {
-      CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+        CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
       default:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       break;
-#endif
-    case BT_S: case BT_CR: case BT_LF:
+#  endif
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       for (;;) {
         int t;
 
@@ -605,101 +606,101 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
         }
       }
       /* fall through */
-    case BT_EQUALS:
-      {
-        int open;
-#ifdef XML_NS
-        hadColon = 0;
-#endif
-        for (;;) {
-          ptr += MINBPC(enc);
-          REQUIRE_CHAR(enc, ptr, end);
-          open = BYTE_TYPE(enc, ptr);
-          if (open == BT_QUOT || open == BT_APOS)
-            break;
-          switch (open) {
-          case BT_S:
-          case BT_LF:
-          case BT_CR:
-            break;
-          default:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          }
-        }
+    case BT_EQUALS: {
+      int open;
+#  ifdef XML_NS
+      hadColon = 0;
+#  endif
+      for (;;) {
         ptr += MINBPC(enc);
-        /* in attribute value */
-        for (;;) {
-          int t;
-          REQUIRE_CHAR(enc, ptr, end);
-          t = BYTE_TYPE(enc, ptr);
-          if (t == open)
-            break;
-          switch (t) {
+        REQUIRE_CHAR(enc, ptr, end);
+        open = BYTE_TYPE(enc, ptr);
+        if (open == BT_QUOT || open == BT_APOS)
+          break;
+        switch (open) {
+        case BT_S:
+        case BT_LF:
+        case BT_CR:
+          break;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
+        }
+      }
+      ptr += MINBPC(enc);
+      /* in attribute value */
+      for (;;) {
+        int t;
+        REQUIRE_CHAR(enc, ptr, end);
+        t = BYTE_TYPE(enc, ptr);
+        if (t == open)
+          break;
+        switch (t) {
           INVALID_CASES(ptr, nextTokPtr)
-          case BT_AMP:
-            {
-              int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);
-              if (tok <= 0) {
-                if (tok == XML_TOK_INVALID)
-                  *nextTokPtr = ptr;
-                return tok;
-              }
-              break;
-            }
-          case BT_LT:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          default:
-            ptr += MINBPC(enc);
-            break;
+        case BT_AMP: {
+          int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);
+          if (tok <= 0) {
+            if (tok == XML_TOK_INVALID)
+              *nextTokPtr = ptr;
+            return tok;
           }
+          break;
+        }
+        case BT_LT:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
+        default:
+          ptr += MINBPC(enc);
+          break;
         }
+      }
+      ptr += MINBPC(enc);
+      REQUIRE_CHAR(enc, ptr, end);
+      switch (BYTE_TYPE(enc, ptr)) {
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+        break;
+      case BT_SOL:
+        goto sol;
+      case BT_GT:
+        goto gt;
+      default:
+        *nextTokPtr = ptr;
+        return XML_TOK_INVALID;
+      }
+      /* ptr points to closing quote */
+      for (;;) {
         ptr += MINBPC(enc);
         REQUIRE_CHAR(enc, ptr, end);
         switch (BYTE_TYPE(enc, ptr)) {
+          CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
         case BT_S:
         case BT_CR:
         case BT_LF:
-          break;
-        case BT_SOL:
-          goto sol;
+          continue;
         case BT_GT:
-          goto gt;
-        default:
-          *nextTokPtr = ptr;
-          return XML_TOK_INVALID;
-        }
-        /* ptr points to closing quote */
-        for (;;) {
+        gt:
+          *nextTokPtr = ptr + MINBPC(enc);
+          return XML_TOK_START_TAG_WITH_ATTS;
+        case BT_SOL:
+        sol:
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
-          switch (BYTE_TYPE(enc, ptr)) {
-          CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-          case BT_S: case BT_CR: case BT_LF:
-            continue;
-          case BT_GT:
-          gt:
-            *nextTokPtr = ptr + MINBPC(enc);
-            return XML_TOK_START_TAG_WITH_ATTS;
-          case BT_SOL:
-          sol:
-            ptr += MINBPC(enc);
-            REQUIRE_CHAR(enc, ptr, end);
-            if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
-              *nextTokPtr = ptr;
-              return XML_TOK_INVALID;
-            }
-            *nextTokPtr = ptr + MINBPC(enc);
-            return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;
-          default:
+          if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
             *nextTokPtr = ptr;
             return XML_TOK_INVALID;
           }
-          break;
+          *nextTokPtr = ptr + MINBPC(enc);
+          return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
         }
         break;
       }
+      break;
+    }
     default:
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
@@ -712,14 +713,13 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
-               const char **nextTokPtr)
-{
-#ifdef XML_NS
+               const char **nextTokPtr) {
+#  ifdef XML_NS
   int hadColon;
-#endif
+#  endif
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   case BT_EXCL:
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
@@ -727,8 +727,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_MINUS:
       return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);
     case BT_LSQB:
-      return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc),
-                                      end, nextTokPtr);
+      return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc), end, nextTokPtr);
     }
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
@@ -740,14 +739,14 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
-#ifdef XML_NS
+#  ifdef XML_NS
   hadColon = 0;
-#endif
+#  endif
   /* we have a start-tag */
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-#ifdef XML_NS
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+#  ifdef XML_NS
     case BT_COLON:
       if (hadColon) {
         *nextTokPtr = ptr;
@@ -757,34 +756,37 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
       switch (BYTE_TYPE(enc, ptr)) {
-      CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+        CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
       default:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       break;
-#endif
-    case BT_S: case BT_CR: case BT_LF:
-      {
-        ptr += MINBPC(enc);
-        while (HAS_CHAR(enc, ptr, end)) {
-          switch (BYTE_TYPE(enc, ptr)) {
+#  endif
+    case BT_S:
+    case BT_CR:
+    case BT_LF: {
+      ptr += MINBPC(enc);
+      while (HAS_CHAR(enc, ptr, end)) {
+        switch (BYTE_TYPE(enc, ptr)) {
           CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-          case BT_GT:
-            goto gt;
-          case BT_SOL:
-            goto sol;
-          case BT_S: case BT_CR: case BT_LF:
-            ptr += MINBPC(enc);
-            continue;
-          default:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          }
-          return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);
+        case BT_GT:
+          goto gt;
+        case BT_SOL:
+          goto sol;
+        case BT_S:
+        case BT_CR:
+        case BT_LF:
+          ptr += MINBPC(enc);
+          continue;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
         }
-        return XML_TOK_PARTIAL;
+        return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);
       }
+      return XML_TOK_PARTIAL;
+    }
     case BT_GT:
     gt:
       *nextTokPtr = ptr + MINBPC(enc);
@@ -793,7 +795,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     sol:
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
-      if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+      if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -809,8 +811,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **nextTokPtr)
-{
+                   const char **nextTokPtr) {
   if (ptr >= end)
     return XML_TOK_NONE;
   if (MINBPC(enc) > 1) {
@@ -842,48 +843,50 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
     ptr += MINBPC(enc);
     if (! HAS_CHAR(enc, ptr, end))
       return XML_TOK_TRAILING_RSQB;
-    if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
+    if (! CHAR_MATCHES(enc, ptr, ASCII_RSQB))
       break;
     ptr += MINBPC(enc);
     if (! HAS_CHAR(enc, ptr, end))
       return XML_TOK_TRAILING_RSQB;
-    if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
       ptr -= MINBPC(enc);
       break;
     }
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
-  INVALID_CASES(ptr, nextTokPtr)
+    INVALID_CASES(ptr, nextTokPtr)
   default:
     ptr += MINBPC(enc);
     break;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \
-        *nextTokPtr = ptr; \
-        return XML_TOK_DATA_CHARS; \
-      } \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) {                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_DATA_CHARS;                                               \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_RSQB:
       if (HAS_CHARS(enc, ptr, end, 2)) {
-         if (!CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {
-           ptr += MINBPC(enc);
-           break;
-         }
-         if (HAS_CHARS(enc, ptr, end, 3)) {
-           if (!CHAR_MATCHES(enc, ptr + 2*MINBPC(enc), ASCII_GT)) {
-             ptr += MINBPC(enc);
-             break;
-           }
-           *nextTokPtr = ptr + 2*MINBPC(enc);
-           return XML_TOK_INVALID;
-         }
+        if (! CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {
+          ptr += MINBPC(enc);
+          break;
+        }
+        if (HAS_CHARS(enc, ptr, end, 3)) {
+          if (! CHAR_MATCHES(enc, ptr + 2 * MINBPC(enc), ASCII_GT)) {
+            ptr += MINBPC(enc);
+            break;
+          }
+          *nextTokPtr = ptr + 2 * MINBPC(enc);
+          return XML_TOK_INVALID;
+        }
       }
       /* fall through */
     case BT_AMP:
@@ -908,12 +911,14 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
-                    const char **nextTokPtr)
-{
+                    const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-  case BT_S: case BT_LF: case BT_CR: case BT_PERCNT:
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+  case BT_S:
+  case BT_LF:
+  case BT_CR:
+  case BT_PERCNT:
     *nextTokPtr = ptr;
     return XML_TOK_PERCENT;
   default:
@@ -922,7 +927,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
     case BT_SEMI:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_PARAM_ENTITY_REF;
@@ -936,20 +941,24 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,
-                      const char **nextTokPtr)
-{
+                      const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_CR: case BT_LF: case BT_S:
-    case BT_RPAR: case BT_GT: case BT_PERCNT: case BT_VERBAR:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_CR:
+    case BT_LF:
+    case BT_S:
+    case BT_RPAR:
+    case BT_GT:
+    case BT_PERCNT:
+    case BT_VERBAR:
       *nextTokPtr = ptr;
       return XML_TOK_POUND_NAME;
     default:
@@ -961,14 +970,12 @@ PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,
 }
 
 static int PTRCALL
-PREFIX(scanLit)(int open, const ENCODING *enc,
-                const char *ptr, const char *end,
-                const char **nextTokPtr)
-{
+PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end,
+                const char **nextTokPtr) {
   while (HAS_CHAR(enc, ptr, end)) {
     int t = BYTE_TYPE(enc, ptr);
     switch (t) {
-    INVALID_CASES(ptr, nextTokPtr)
+      INVALID_CASES(ptr, nextTokPtr)
     case BT_QUOT:
     case BT_APOS:
       ptr += MINBPC(enc);
@@ -978,8 +985,12 @@ PREFIX(scanLit)(int open, const ENCODING *enc,
         return -XML_TOK_LITERAL;
       *nextTokPtr = ptr;
       switch (BYTE_TYPE(enc, ptr)) {
-      case BT_S: case BT_CR: case BT_LF:
-      case BT_GT: case BT_PERCNT: case BT_LSQB:
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+      case BT_GT:
+      case BT_PERCNT:
+      case BT_LSQB:
         return XML_TOK_LITERAL;
       default:
         return XML_TOK_INVALID;
@@ -994,8 +1005,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc,
 
 static int PTRCALL
 PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
-                  const char **nextTokPtr)
-{
+                  const char **nextTokPtr) {
   int tok;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1013,27 +1023,26 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     return PREFIX(scanLit)(BT_QUOT, enc, ptr + MINBPC(enc), end, nextTokPtr);
   case BT_APOS:
     return PREFIX(scanLit)(BT_APOS, enc, ptr + MINBPC(enc), end, nextTokPtr);
-  case BT_LT:
-    {
-      ptr += MINBPC(enc);
-      REQUIRE_CHAR(enc, ptr, end);
-      switch (BYTE_TYPE(enc, ptr)) {
-      case BT_EXCL:
-        return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-      case BT_QUEST:
-        return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-      case BT_NMSTRT:
-      case BT_HEX:
-      case BT_NONASCII:
-      case BT_LEAD2:
-      case BT_LEAD3:
-      case BT_LEAD4:
-        *nextTokPtr = ptr - MINBPC(enc);
-        return XML_TOK_INSTANCE_START;
-      }
-      *nextTokPtr = ptr;
-      return XML_TOK_INVALID;
+  case BT_LT: {
+    ptr += MINBPC(enc);
+    REQUIRE_CHAR(enc, ptr, end);
+    switch (BYTE_TYPE(enc, ptr)) {
+    case BT_EXCL:
+      return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);
+    case BT_QUEST:
+      return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
+    case BT_NMSTRT:
+    case BT_HEX:
+    case BT_NONASCII:
+    case BT_LEAD2:
+    case BT_LEAD3:
+    case BT_LEAD4:
+      *nextTokPtr = ptr - MINBPC(enc);
+      return XML_TOK_INSTANCE_START;
     }
+    *nextTokPtr = ptr;
+    return XML_TOK_INVALID;
+  }
   case BT_CR:
     if (ptr + MINBPC(enc) == end) {
       *nextTokPtr = end;
@@ -1041,13 +1050,15 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
       return -XML_TOK_PROLOG_S;
     }
     /* fall through */
-  case BT_S: case BT_LF:
+  case BT_S:
+  case BT_LF:
     for (;;) {
       ptr += MINBPC(enc);
       if (! HAS_CHAR(enc, ptr, end))
         break;
       switch (BYTE_TYPE(enc, ptr)) {
-      case BT_S: case BT_LF:
+      case BT_S:
+      case BT_LF:
         break;
       case BT_CR:
         /* don't split CR/LF pair */
@@ -1076,7 +1087,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) {
       REQUIRE_CHARS(enc, ptr, end, 2);
       if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_GT)) {
-        *nextTokPtr = ptr + 2*MINBPC(enc);
+        *nextTokPtr = ptr + 2 * MINBPC(enc);
         return XML_TOK_COND_SECT_CLOSE;
       }
     }
@@ -1099,8 +1110,12 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_PLUS:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_CLOSE_PAREN_PLUS;
-    case BT_CR: case BT_LF: case BT_S:
-    case BT_GT: case BT_COMMA: case BT_VERBAR:
+    case BT_CR:
+    case BT_LF:
+    case BT_S:
+    case BT_GT:
+    case BT_COMMA:
+    case BT_VERBAR:
     case BT_RPAR:
       *nextTokPtr = ptr;
       return XML_TOK_CLOSE_PAREN;
@@ -1115,24 +1130,26 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     return XML_TOK_DECL_CLOSE;
   case BT_NUM:
     return PREFIX(scanPoundName)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-#define LEAD_CASE(n) \
-  case BT_LEAD ## n: \
-    if (end - ptr < n) \
-      return XML_TOK_PARTIAL_CHAR; \
-    if (IS_NMSTRT_CHAR(enc, ptr, n)) { \
-      ptr += n; \
-      tok = XML_TOK_NAME; \
-      break; \
-    } \
-    if (IS_NAME_CHAR(enc, ptr, n)) { \
-      ptr += n; \
-      tok = XML_TOK_NMTOKEN; \
-      break; \
-    } \
-    *nextTokPtr = ptr; \
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (IS_NMSTRT_CHAR(enc, ptr, n)) {                                         \
+      ptr += n;                                                                \
+      tok = XML_TOK_NAME;                                                      \
+      break;                                                                   \
+    }                                                                          \
+    if (IS_NAME_CHAR(enc, ptr, n)) {                                           \
+      ptr += n;                                                                \
+      tok = XML_TOK_NMTOKEN;                                                   \
+      break;                                                                   \
+    }                                                                          \
+    *nextTokPtr = ptr;                                                         \
     return XML_TOK_INVALID;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+    LEAD_CASE(2)
+    LEAD_CASE(3)
+    LEAD_CASE(4)
+#  undef LEAD_CASE
   case BT_NMSTRT:
   case BT_HEX:
     tok = XML_TOK_NAME;
@@ -1141,9 +1158,9 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
   case BT_DIGIT:
   case BT_NAME:
   case BT_MINUS:
-#ifdef XML_NS
+#  ifdef XML_NS
   case BT_COLON:
-#endif
+#  endif
     tok = XML_TOK_NMTOKEN;
     ptr += MINBPC(enc);
     break;
@@ -1165,13 +1182,19 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_GT: case BT_RPAR: case BT_COMMA:
-    case BT_VERBAR: case BT_LSQB: case BT_PERCNT:
-    case BT_S: case BT_CR: case BT_LF:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_GT:
+    case BT_RPAR:
+    case BT_COMMA:
+    case BT_VERBAR:
+    case BT_LSQB:
+    case BT_PERCNT:
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       *nextTokPtr = ptr;
       return tok;
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
       ptr += MINBPC(enc);
       switch (tok) {
@@ -1179,7 +1202,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
         REQUIRE_CHAR(enc, ptr, end);
         tok = XML_TOK_PREFIXED_NAME;
         switch (BYTE_TYPE(enc, ptr)) {
-        CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+          CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
         default:
           tok = XML_TOK_NMTOKEN;
           break;
@@ -1190,23 +1213,23 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
         break;
       }
       break;
-#endif
+#  endif
     case BT_PLUS:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_NAME_PLUS;
     case BT_AST:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_NAME_ASTERISK;
     case BT_QUEST:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -1221,9 +1244,8 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
 }
 
 static int PTRCALL
-PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
-                          const char *end, const char **nextTokPtr)
-{
+PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end,
+                          const char **nextTokPtr) {
   const char *start;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1238,10 +1260,14 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
   start = ptr;
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_AMP:
       if (ptr == start)
         return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -1287,9 +1313,8 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
-                       const char *end, const char **nextTokPtr)
-{
+PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end,
+                       const char **nextTokPtr) {
   const char *start;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1304,10 +1329,14 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
   start = ptr;
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_AMP:
       if (ptr == start)
         return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -1315,8 +1344,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
       return XML_TOK_DATA_CHARS;
     case BT_PERCNT:
       if (ptr == start) {
-        int tok =  PREFIX(scanPercent)(enc, ptr + MINBPC(enc),
-                                       end, nextTokPtr);
+        int tok = PREFIX(scanPercent)(enc, ptr + MINBPC(enc), end, nextTokPtr);
         return (tok == XML_TOK_PERCENT) ? XML_TOK_INVALID : tok;
       }
       *nextTokPtr = ptr;
@@ -1349,12 +1377,11 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
   return XML_TOK_DATA_CHARS;
 }
 
-#ifdef XML_DTD
+#  ifdef XML_DTD
 
 static int PTRCALL
-PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
-                         const char *end, const char **nextTokPtr)
-{
+PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
+                         const char **nextTokPtr) {
   int level = 0;
   if (MINBPC(enc) > 1) {
     size_t n = end - ptr;
@@ -1365,7 +1392,7 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    INVALID_CASES(ptr, nextTokPtr)
+      INVALID_CASES(ptr, nextTokPtr)
     case BT_LT:
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
@@ -1402,12 +1429,11 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
   return XML_TOK_PARTIAL;
 }
 
-#endif /* XML_DTD */
+#  endif /* XML_DTD */
 
 static int PTRCALL
 PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **badPtr)
-{
+                   const char **badPtr) {
   ptr += MINBPC(enc);
   end -= MINBPC(enc);
   for (; HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) {
@@ -1430,9 +1456,9 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_AST:
     case BT_PERCNT:
     case BT_NUM:
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
-#endif
+#  endif
       break;
     case BT_S:
       if (CHAR_MATCHES(enc, ptr, ASCII_TAB)) {
@@ -1442,7 +1468,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
       break;
     case BT_NAME:
     case BT_NMSTRT:
-      if (!(BYTE_TO_ASCII(enc, ptr) & ~0x7f))
+      if (! (BYTE_TO_ASCII(enc, ptr) & ~0x7f))
         break;
       /* fall through */
     default:
@@ -1466,9 +1492,8 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
 */
 
 static int PTRCALL
-PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
-                int attsMax, ATTRIBUTE *atts)
-{
+PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
+                ATTRIBUTE *atts) {
   enum { other, inName, inValue } state = inName;
   int nAtts = 0;
   int open = 0; /* defined when state == inValue;
@@ -1476,32 +1501,35 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
 
   for (ptr += MINBPC(enc);; ptr += MINBPC(enc)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define START_NAME \
-      if (state == other) { \
-        if (nAtts < attsMax) { \
-          atts[nAtts].name = ptr; \
-          atts[nAtts].normalized = 1; \
-        } \
-        state = inName; \
-      }
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: START_NAME ptr += (n - MINBPC(enc)); break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define START_NAME                                                           \
+    if (state == other) {                                                      \
+      if (nAtts < attsMax) {                                                   \
+        atts[nAtts].name = ptr;                                                \
+        atts[nAtts].normalized = 1;                                            \
+      }                                                                        \
+      state = inName;                                                          \
+    }
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    START_NAME ptr += (n - MINBPC(enc));                                       \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONASCII:
     case BT_NMSTRT:
     case BT_HEX:
       START_NAME
       break;
-#undef START_NAME
+#  undef START_NAME
     case BT_QUOT:
       if (state != inValue) {
         if (nAtts < attsMax)
           atts[nAtts].valuePtr = ptr + MINBPC(enc);
         state = inValue;
         open = BT_QUOT;
-      }
-      else if (open == BT_QUOT) {
+      } else if (open == BT_QUOT) {
         state = other;
         if (nAtts < attsMax)
           atts[nAtts].valueEnd = ptr;
@@ -1514,8 +1542,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
           atts[nAtts].valuePtr = ptr + MINBPC(enc);
         state = inValue;
         open = BT_APOS;
-      }
-      else if (open == BT_APOS) {
+      } else if (open == BT_APOS) {
         state = other;
         if (nAtts < attsMax)
           atts[nAtts].valueEnd = ptr;
@@ -1529,16 +1556,15 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
     case BT_S:
       if (state == inName)
         state = other;
-      else if (state == inValue
-               && nAtts < attsMax
-               && atts[nAtts].normalized
+      else if (state == inValue && nAtts < attsMax && atts[nAtts].normalized
                && (ptr == atts[nAtts].valuePtr
                    || BYTE_TO_ASCII(enc, ptr) != ASCII_SPACE
                    || BYTE_TO_ASCII(enc, ptr + MINBPC(enc)) == ASCII_SPACE
                    || BYTE_TYPE(enc, ptr + MINBPC(enc)) == open))
         atts[nAtts].normalized = 0;
       break;
-    case BT_CR: case BT_LF:
+    case BT_CR:
+    case BT_LF:
       /* This case ensures that the first attribute name is counted
          Apart from that we could just change state on the quote. */
       if (state == inName)
@@ -1559,29 +1585,44 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRFASTCALL
-PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
-{
+PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) {
   int result = 0;
   /* skip &# */
-  ptr += 2*MINBPC(enc);
+  UNUSED_P(enc);
+  ptr += 2 * MINBPC(enc);
   if (CHAR_MATCHES(enc, ptr, ASCII_x)) {
-    for (ptr += MINBPC(enc);
-         !CHAR_MATCHES(enc, ptr, ASCII_SEMI);
+    for (ptr += MINBPC(enc); ! CHAR_MATCHES(enc, ptr, ASCII_SEMI);
          ptr += MINBPC(enc)) {
       int c = BYTE_TO_ASCII(enc, ptr);
       switch (c) {
-      case ASCII_0: case ASCII_1: case ASCII_2: case ASCII_3: case ASCII_4:
-      case ASCII_5: case ASCII_6: case ASCII_7: case ASCII_8: case ASCII_9:
+      case ASCII_0:
+      case ASCII_1:
+      case ASCII_2:
+      case ASCII_3:
+      case ASCII_4:
+      case ASCII_5:
+      case ASCII_6:
+      case ASCII_7:
+      case ASCII_8:
+      case ASCII_9:
         result <<= 4;
         result |= (c - ASCII_0);
         break;
-      case ASCII_A: case ASCII_B: case ASCII_C:
-      case ASCII_D: case ASCII_E: case ASCII_F:
+      case ASCII_A:
+      case ASCII_B:
+      case ASCII_C:
+      case ASCII_D:
+      case ASCII_E:
+      case ASCII_F:
         result <<= 4;
         result += 10 + (c - ASCII_A);
         break;
-      case ASCII_a: case ASCII_b: case ASCII_c:
-      case ASCII_d: case ASCII_e: case ASCII_f:
+      case ASCII_a:
+      case ASCII_b:
+      case ASCII_c:
+      case ASCII_d:
+      case ASCII_e:
+      case ASCII_f:
         result <<= 4;
         result += 10 + (c - ASCII_a);
         break;
@@ -1589,9 +1630,8 @@ PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
       if (result >= 0x110000)
         return -1;
     }
-  }
-  else {
-    for (; !CHAR_MATCHES(enc, ptr, ASCII_SEMI); ptr += MINBPC(enc)) {
+  } else {
+    for (; ! CHAR_MATCHES(enc, ptr, ASCII_SEMI); ptr += MINBPC(enc)) {
       int c = BYTE_TO_ASCII(enc, ptr);
       result *= 10;
       result += (c - ASCII_0);
@@ -1603,10 +1643,10 @@ PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
 }
 
 static int PTRCALL
-PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                             const char *end)
-{
-  switch ((end - ptr)/MINBPC(enc)) {
+PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr,
+                             const char *end) {
+  UNUSED_P(enc);
+  switch ((end - ptr) / MINBPC(enc)) {
   case 2:
     if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_t)) {
       switch (BYTE_TO_ASCII(enc, ptr)) {
@@ -1657,9 +1697,9 @@ PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1,
-                         const char *end1, const char *ptr2)
-{
+PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1,
+                         const char *end1, const char *ptr2) {
+  UNUSED_P(enc);
   for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) {
     if (end1 - ptr1 < MINBPC(enc)) {
       /* This line cannot be executed.  The incoming data has already
@@ -1669,27 +1709,30 @@ PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1,
        */
       return 0; /* LCOV_EXCL_LINE */
     }
-    if (!CHAR_MATCHES(enc, ptr1, *ptr2))
+    if (! CHAR_MATCHES(enc, ptr1, *ptr2))
       return 0;
   }
   return ptr1 == end1;
 }
 
 static int PTRFASTCALL
-PREFIX(nameLength)(const ENCODING *enc, const char *ptr)
-{
+PREFIX(nameLength)(const ENCODING *enc, const char *ptr) {
   const char *start = ptr;
   for (;;) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONASCII:
     case BT_NMSTRT:
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
-#endif
+#  endif
     case BT_HEX:
     case BT_DIGIT:
     case BT_NAME:
@@ -1702,9 +1745,8 @@ PREFIX(nameLength)(const ENCODING *enc, const char *ptr)
   }
 }
 
-static const char * PTRFASTCALL
-PREFIX(skipS)(const ENCODING *enc, const char *ptr)
-{
+static const char *PTRFASTCALL
+PREFIX(skipS)(const ENCODING *enc, const char *ptr) {
   for (;;) {
     switch (BYTE_TYPE(enc, ptr)) {
     case BT_LF:
@@ -1719,19 +1761,18 @@ PREFIX(skipS)(const ENCODING *enc, const char *ptr)
 }
 
 static void PTRCALL
-PREFIX(updatePosition)(const ENCODING *enc,
-                       const char *ptr,
-                       const char *end,
-                       POSITION *pos)
-{
+PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end,
+                       POSITION *pos) {
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_LF:
       pos->columnNumber = (XML_Size)-1;
       pos->lineNumber++;
@@ -1752,12 +1793,12 @@ PREFIX(updatePosition)(const ENCODING *enc,
   }
 }
 
-#undef DO_LEAD_CASE
-#undef MULTIBYTE_CASES
-#undef INVALID_CASES
-#undef CHECK_NAME_CASE
-#undef CHECK_NAME_CASES
-#undef CHECK_NMSTRT_CASE
-#undef CHECK_NMSTRT_CASES
+#  undef DO_LEAD_CASE
+#  undef MULTIBYTE_CASES
+#  undef INVALID_CASES
+#  undef CHECK_NAME_CASE
+#  undef CHECK_NAME_CASES
+#  undef CHECK_NMSTRT_CASE
+#  undef CHECK_NMSTRT_CASES
 
 #endif /* XML_TOK_IMPL_C */
diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h
index a6420f48eedc..e925dbc7e2c8 100644
--- a/Modules/expat/xmltok_impl.h
+++ b/Modules/expat/xmltok_impl.h
@@ -31,43 +31,43 @@
 */
 
 enum {
-  BT_NONXML,
-  BT_MALFORM,
-  BT_LT,
-  BT_AMP,
-  BT_RSQB,
-  BT_LEAD2,
-  BT_LEAD3,
-  BT_LEAD4,
-  BT_TRAIL,
-  BT_CR,
-  BT_LF,
-  BT_GT,
-  BT_QUOT,
-  BT_APOS,
-  BT_EQUALS,
-  BT_QUEST,
-  BT_EXCL,
-  BT_SOL,
-  BT_SEMI,
-  BT_NUM,
-  BT_LSQB,
-  BT_S,
-  BT_NMSTRT,
-  BT_COLON,
-  BT_HEX,
-  BT_DIGIT,
-  BT_NAME,
-  BT_MINUS,
-  BT_OTHER, /* known not to be a name or name start character */
+  BT_NONXML,   /* e.g. noncharacter-FFFF */
+  BT_MALFORM,  /* illegal, with regard to encoding */
+  BT_LT,       /* less than = "<" */
+  BT_AMP,      /* ampersand = "&" */
+  BT_RSQB,     /* right square bracket = "[" */
+  BT_LEAD2,    /* lead byte of a 2-byte UTF-8 character */
+  BT_LEAD3,    /* lead byte of a 3-byte UTF-8 character */
+  BT_LEAD4,    /* lead byte of a 4-byte UTF-8 character */
+  BT_TRAIL,    /* trailing unit, e.g. second 16-bit unit of a 4-byte char. */
+  BT_CR,       /* carriage return = "\r" */
+  BT_LF,       /* line feed = "\n" */
+  BT_GT,       /* greater than = ">" */
+  BT_QUOT,     /* quotation character = "\"" */
+  BT_APOS,     /* aposthrophe = "'" */
+  BT_EQUALS,   /* equal sign = "=" */
+  BT_QUEST,    /* question mark = "?" */
+  BT_EXCL,     /* exclamation mark = "!" */
+  BT_SOL,      /* solidus, slash = "/" */
+  BT_SEMI,     /* semicolon = ";" */
+  BT_NUM,      /* number sign = "#" */
+  BT_LSQB,     /* left square bracket = "[" */
+  BT_S,        /* white space, e.g. "\t", " "[, "\r"] */
+  BT_NMSTRT,   /* non-hex name start letter = "G".."Z" + "g".."z" + "_" */
+  BT_COLON,    /* colon = ":" */
+  BT_HEX,      /* hex letter = "A".."F" + "a".."f" */
+  BT_DIGIT,    /* digit = "0".."9" */
+  BT_NAME,     /* dot and middle dot = "." + chr(0xb7) */
+  BT_MINUS,    /* minus = "-" */
+  BT_OTHER,    /* known not to be a name or name start character */
   BT_NONASCII, /* might be a name or name start character */
-  BT_PERCNT,
-  BT_LPAR,
-  BT_RPAR,
-  BT_AST,
-  BT_PLUS,
-  BT_COMMA,
-  BT_VERBAR
+  BT_PERCNT,   /* percent sign = "%" */
+  BT_LPAR,     /* left parenthesis = "(" */
+  BT_RPAR,     /* right parenthesis = "(" */
+  BT_AST,      /* asterisk = "*" */
+  BT_PLUS,     /* plus sign = "+" */
+  BT_COMMA,    /* comma = "," */
+  BT_VERBAR    /* vertical bar = "|" */
 };
 
 #include <stddef.h>
diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c
index 23d31e8e4249..919c74e9f97f 100644
--- a/Modules/expat/xmltok_ns.c
+++ b/Modules/expat/xmltok_ns.c
@@ -33,56 +33,47 @@
 #ifdef XML_TOK_NS_C
 
 const ENCODING *
-NS(XmlGetUtf8InternalEncoding)(void)
-{
+NS(XmlGetUtf8InternalEncoding)(void) {
   return &ns(internal_utf8_encoding).enc;
 }
 
 const ENCODING *
-NS(XmlGetUtf16InternalEncoding)(void)
-{
-#if BYTEORDER == 1234
+NS(XmlGetUtf16InternalEncoding)(void) {
+#  if BYTEORDER == 1234
   return &ns(internal_little2_encoding).enc;
-#elif BYTEORDER == 4321
+#  elif BYTEORDER == 4321
   return &ns(internal_big2_encoding).enc;
-#else
+#  else
   const short n = 1;
-  return (*(const char *)&n
-          ? &ns(internal_little2_encoding).enc
-          : &ns(internal_big2_encoding).enc);
-#endif
+  return (*(const char *)&n ? &ns(internal_little2_encoding).enc
+                            : &ns(internal_big2_encoding).enc);
+#  endif
 }
 
-static const ENCODING * const NS(encodings)[] = {
-  &ns(latin1_encoding).enc,
-  &ns(ascii_encoding).enc,
-  &ns(utf8_encoding).enc,
-  &ns(big2_encoding).enc,
-  &ns(big2_encoding).enc,
-  &ns(little2_encoding).enc,
-  &ns(utf8_encoding).enc /* NO_ENC */
+static const ENCODING *const NS(encodings)[] = {
+    &ns(latin1_encoding).enc, &ns(ascii_encoding).enc,
+    &ns(utf8_encoding).enc,   &ns(big2_encoding).enc,
+    &ns(big2_encoding).enc,   &ns(little2_encoding).enc,
+    &ns(utf8_encoding).enc /* NO_ENC */
 };
 
 static int PTRCALL
 NS(initScanProlog)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **nextTokPtr)
-{
-  return initScan(NS(encodings), (const INIT_ENCODING *)enc,
-                  XML_PROLOG_STATE, ptr, end, nextTokPtr);
+                   const char **nextTokPtr) {
+  return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_PROLOG_STATE,
+                  ptr, end, nextTokPtr);
 }
 
 static int PTRCALL
 NS(initScanContent)(const ENCODING *enc, const char *ptr, const char *end,
-                    const char **nextTokPtr)
-{
-  return initScan(NS(encodings), (const INIT_ENCODING *)enc,
-                  XML_CONTENT_STATE, ptr, end, nextTokPtr);
+                    const char **nextTokPtr) {
+  return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_CONTENT_STATE,
+                  ptr, end, nextTokPtr);
 }
 
 int
 NS(XmlInitEncoding)(INIT_ENCODING *p, const ENCODING **encPtr,
-                    const char *name)
-{
+                    const char *name) {
   int i = getEncodingIndex(name);
   if (i == UNKNOWN_ENC)
     return 0;
@@ -96,9 +87,8 @@ NS(XmlInitEncoding)(INIT_ENCODING *p, const ENCODING **encPtr,
 }
 
 static const ENCODING *
-NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end)
-{
-#define ENCODING_MAX 128
+NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end) {
+#  define ENCODING_MAX 128
   char buf[ENCODING_MAX];
   char *p = buf;
   int i;
@@ -115,28 +105,14 @@ NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end)
 }
 
 int
-NS(XmlParseXmlDecl)(int isGeneralTextEntity,
-                    const ENCODING *enc,
-                    const char *ptr,
-                    const char *end,
-                    const char **badPtr,
-                    const char **versionPtr,
-                    const char **versionEndPtr,
-                    const char **encodingName,
-                    const ENCODING **encoding,
-                    int *standalone)
-{
-  return doParseXmlDecl(NS(findEncoding),
-                        isGeneralTextEntity,
-                        enc,
-                        ptr,
-                        end,
-                        badPtr,
-                        versionPtr,
-                        versionEndPtr,
-                        encodingName,
-                        encoding,
-                        standalone);
+NS(XmlParseXmlDecl)(int isGeneralTextEntity, const ENCODING *enc,
+                    const char *ptr, const char *end, const char **badPtr,
+                    const char **versionPtr, const char **versionEndPtr,
+                    const char **encodingName, const ENCODING **encoding,
+                    int *standalone) {
+  return doParseXmlDecl(NS(findEncoding), isGeneralTextEntity, enc, ptr, end,
+                        badPtr, versionPtr, versionEndPtr, encodingName,
+                        encoding, standalone);
 }
 
 #endif /* XML_TOK_NS_C */
diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj
index 33a017327189..4a125b243b78 100644
--- a/PCbuild/_elementtree.vcxproj
+++ b/PCbuild/_elementtree.vcxproj
@@ -116,7 +116,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\_elementtree.c" />
-    <ClCompile Include="..\Modules\expat\loadlibrary.c" />
     <ClCompile Include="..\Modules\expat\xmlparse.c" />
     <ClCompile Include="..\Modules\expat\xmlrole.c" />
     <ClCompile Include="..\Modules\expat\xmltok.c" />
diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters
index 4597ee521b33..6acdf35846ab 100644
--- a/PCbuild/_elementtree.vcxproj.filters
+++ b/PCbuild/_elementtree.vcxproj.filters
@@ -33,9 +33,6 @@
     <ClInclude Include="..\Modules\expat\latin1tab.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\expat\loadlibrary.c">
-      <Filter>Header Files</Filter>
-    </ClInclude>
     <ClInclude Include="..\Modules\expat\macconfig.h">
       <Filter>Header Files</Filter>
     </ClInclude>
diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj
index 28a11a82936e..b2d9f5d57d49 100644
--- a/PCbuild/pyexpat.vcxproj
+++ b/PCbuild/pyexpat.vcxproj
@@ -100,7 +100,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\pyexpat.c" />
-    <ClCompile Include="..\Modules\expat\loadlibrary.c" />
     <ClCompile Include="..\Modules\expat\xmlparse.c" />
     <ClCompile Include="..\Modules\expat\xmlrole.c" />
     <ClCompile Include="..\Modules\expat\xmltok.c" />
diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters
index cb02847980c6..f8d46026c9c2 100644
--- a/PCbuild/pyexpat.vcxproj.filters
+++ b/PCbuild/pyexpat.vcxproj.filters
@@ -20,9 +20,6 @@
     <ClCompile Include="..\Modules\pyexpat.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\expat\loadlibrary.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Modules\expat\xmlparse.c">
       <Filter>Source Files</Filter>
     </ClCompile>


From webhook-mailer at python.org  Thu Sep 26 01:00:32 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 26 Sep 2019 05:00:32 -0000
Subject: [Python-checkins] [3.6] closes bpo-38174: Update vendored expat
 library to 2.2.8. (GH-16410)
Message-ID: <mailman.98.1569474033.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/f0501630b0ba31448c230c756b1027647f4ef100
commit: f0501630b0ba31448c230c756b1027647f4ef100
branch: 3.6
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-25T22:00:26-07:00
summary:

[3.6] closes bpo-38174: Update vendored expat library to 2.2.8. (GH-16410)

Fixes CVE-2019-15903. See full changelog at https://github.com/libexpat/libexpat/blob/R_2_2_8/expat/Changes..
(cherry picked from commit 52b940803860e37bcc3f6096b2d24e7c20a0e807)

files:
A Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
D Modules/expat/loadlibrary.c
M Modules/expat/asciitab.h
M Modules/expat/expat.h
M Modules/expat/expat_external.h
M Modules/expat/iasciitab.h
M Modules/expat/internal.h
M Modules/expat/latin1tab.h
M Modules/expat/nametab.h
M Modules/expat/siphash.h
M Modules/expat/utf8tab.h
M Modules/expat/winconfig.h
M Modules/expat/xmlparse.c
M Modules/expat/xmlrole.c
M Modules/expat/xmlrole.h
M Modules/expat/xmltok.c
M Modules/expat/xmltok.h
M Modules/expat/xmltok_impl.c
M Modules/expat/xmltok_impl.h
M Modules/expat/xmltok_ns.c
M PCbuild/_elementtree.vcxproj
M PCbuild/_elementtree.vcxproj.filters
M PCbuild/pyexpat.vcxproj
M PCbuild/pyexpat.vcxproj.filters

diff --git a/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst b/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
new file mode 100644
index 000000000000..e3da6fcdd139
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-09-23-21-02-46.bpo-38174.MeWuJd.rst
@@ -0,0 +1,2 @@
+Update vendorized expat library version to 2.2.8, which resolves
+CVE-2019-15903.
diff --git a/Modules/expat/asciitab.h b/Modules/expat/asciitab.h
index 2f59fd929065..63b1d1b4482e 100644
--- a/Modules/expat/asciitab.h
+++ b/Modules/expat/asciitab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
-/* 0x0C */ BT_NONXML, BT_CR, BT_NONXML, BT_NONXML,
-/* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
-/* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
-/* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
-/* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
-/* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
-/* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
-/* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
-/* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
-/* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
+    /* 0x0C */ BT_NONXML, BT_CR, BT_NONXML, BT_NONXML,
+    /* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
+    /* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
+    /* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
+    /* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
+    /* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
+    /* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
+    /* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
+    /* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
+    /* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h
index c050f1d918ce..6c8eb1fda4a3 100644
--- a/Modules/expat/expat.h
+++ b/Modules/expat/expat.h
@@ -33,15 +33,6 @@
 #ifndef Expat_INCLUDED
 #define Expat_INCLUDED 1
 
-#ifdef __VMS
-/*      0        1         2         3      0        1         2         3
-        1234567890123456789012345678901     1234567890123456789012345678901 */
-#define XML_SetProcessingInstructionHandler XML_SetProcessingInstrHandler
-#define XML_SetUnparsedEntityDeclHandler    XML_SetUnparsedEntDeclHandler
-#define XML_SetStartNamespaceDeclHandler    XML_SetStartNamespcDeclHandler
-#define XML_SetExternalEntityRefHandlerArg  XML_SetExternalEntRefHandlerArg
-#endif
-
 #include <stdlib.h>
 #include "expat_external.h"
 
@@ -53,8 +44,8 @@ struct XML_ParserStruct;
 typedef struct XML_ParserStruct *XML_Parser;
 
 typedef unsigned char XML_Bool;
-#define XML_TRUE   ((XML_Bool) 1)
-#define XML_FALSE  ((XML_Bool) 0)
+#define XML_TRUE ((XML_Bool)1)
+#define XML_FALSE ((XML_Bool)0)
 
 /* The XML_Status enum gives the possible return values for several
    API functions.  The preprocessor #defines are included so this
@@ -164,25 +155,23 @@ enum XML_Content_Quant {
 typedef struct XML_cp XML_Content;
 
 struct XML_cp {
-  enum XML_Content_Type         type;
-  enum XML_Content_Quant        quant;
-  XML_Char *                    name;
-  unsigned int                  numchildren;
-  XML_Content *                 children;
+  enum XML_Content_Type type;
+  enum XML_Content_Quant quant;
+  XML_Char *name;
+  unsigned int numchildren;
+  XML_Content *children;
 };
 
-
 /* This is called for an element declaration. See above for
    description of the model argument. It's the caller's responsibility
    to free model when finished with it.
 */
-typedef void (XMLCALL *XML_ElementDeclHandler) (void *userData,
-                                                const XML_Char *name,
-                                                XML_Content *model);
+typedef void(XMLCALL *XML_ElementDeclHandler)(void *userData,
+                                              const XML_Char *name,
+                                              XML_Content *model);
 
 XMLPARSEAPI(void)
-XML_SetElementDeclHandler(XML_Parser parser,
-                          XML_ElementDeclHandler eldecl);
+XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl);
 
 /* The Attlist declaration handler is called for *each* attribute. So
    a single Attlist declaration with multiple attributes declared will
@@ -192,17 +181,12 @@ XML_SetElementDeclHandler(XML_Parser parser,
    value will be NULL in the case of "#REQUIRED". If "isrequired" is
    true and default is non-NULL, then this is a "#FIXED" default.
 */
-typedef void (XMLCALL *XML_AttlistDeclHandler) (
-                                    void            *userData,
-                                    const XML_Char  *elname,
-                                    const XML_Char  *attname,
-                                    const XML_Char  *att_type,
-                                    const XML_Char  *dflt,
-                                    int              isrequired);
+typedef void(XMLCALL *XML_AttlistDeclHandler)(
+    void *userData, const XML_Char *elname, const XML_Char *attname,
+    const XML_Char *att_type, const XML_Char *dflt, int isrequired);
 
 XMLPARSEAPI(void)
-XML_SetAttlistDeclHandler(XML_Parser parser,
-                          XML_AttlistDeclHandler attdecl);
+XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl);
 
 /* The XML declaration handler is called for *both* XML declarations
    and text declarations. The way to distinguish is that the version
@@ -212,15 +196,13 @@ XML_SetAttlistDeclHandler(XML_Parser parser,
    was no standalone parameter in the declaration, that it was given
    as no, or that it was given as yes.
 */
-typedef void (XMLCALL *XML_XmlDeclHandler) (void           *userData,
-                                            const XML_Char *version,
-                                            const XML_Char *encoding,
-                                            int             standalone);
+typedef void(XMLCALL *XML_XmlDeclHandler)(void *userData,
+                                          const XML_Char *version,
+                                          const XML_Char *encoding,
+                                          int standalone);
 
 XMLPARSEAPI(void)
-XML_SetXmlDeclHandler(XML_Parser parser,
-                      XML_XmlDeclHandler xmldecl);
-
+XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler xmldecl);
 
 typedef struct {
   void *(*malloc_fcn)(size_t size);
@@ -248,7 +230,6 @@ XML_ParserCreate(const XML_Char *encoding);
 XMLPARSEAPI(XML_Parser)
 XML_ParserCreateNS(const XML_Char *encoding, XML_Char namespaceSeparator);
 
-
 /* Constructs a new parser using the memory management suite referred to
    by memsuite. If memsuite is NULL, then use the standard library memory
    suite. If namespaceSeparator is non-NULL it creates a parser with
@@ -278,31 +259,27 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encoding);
 /* atts is array of name/value pairs, terminated by 0;
    names and values are 0 terminated.
 */
-typedef void (XMLCALL *XML_StartElementHandler) (void *userData,
-                                                 const XML_Char *name,
-                                                 const XML_Char **atts);
-
-typedef void (XMLCALL *XML_EndElementHandler) (void *userData,
-                                               const XML_Char *name);
+typedef void(XMLCALL *XML_StartElementHandler)(void *userData,
+                                               const XML_Char *name,
+                                               const XML_Char **atts);
 
+typedef void(XMLCALL *XML_EndElementHandler)(void *userData,
+                                             const XML_Char *name);
 
 /* s is not 0 terminated. */
-typedef void (XMLCALL *XML_CharacterDataHandler) (void *userData,
-                                                  const XML_Char *s,
-                                                  int len);
+typedef void(XMLCALL *XML_CharacterDataHandler)(void *userData,
+                                                const XML_Char *s, int len);
 
 /* target and data are 0 terminated */
-typedef void (XMLCALL *XML_ProcessingInstructionHandler) (
-                                                void *userData,
-                                                const XML_Char *target,
-                                                const XML_Char *data);
+typedef void(XMLCALL *XML_ProcessingInstructionHandler)(void *userData,
+                                                        const XML_Char *target,
+                                                        const XML_Char *data);
 
 /* data is 0 terminated */
-typedef void (XMLCALL *XML_CommentHandler) (void *userData,
-                                            const XML_Char *data);
+typedef void(XMLCALL *XML_CommentHandler)(void *userData, const XML_Char *data);
 
-typedef void (XMLCALL *XML_StartCdataSectionHandler) (void *userData);
-typedef void (XMLCALL *XML_EndCdataSectionHandler) (void *userData);
+typedef void(XMLCALL *XML_StartCdataSectionHandler)(void *userData);
+typedef void(XMLCALL *XML_EndCdataSectionHandler)(void *userData);
 
 /* This is called for any characters in the XML document for which
    there is no applicable handler.  This includes both characters that
@@ -317,25 +294,23 @@ typedef void (XMLCALL *XML_EndCdataSectionHandler) (void *userData);
    default handler: for example, a comment might be split between
    multiple calls.
 */
-typedef void (XMLCALL *XML_DefaultHandler) (void *userData,
-                                            const XML_Char *s,
-                                            int len);
+typedef void(XMLCALL *XML_DefaultHandler)(void *userData, const XML_Char *s,
+                                          int len);
 
 /* This is called for the start of the DOCTYPE declaration, before
    any DTD or internal subset is parsed.
 */
-typedef void (XMLCALL *XML_StartDoctypeDeclHandler) (
-                                            void *userData,
-                                            const XML_Char *doctypeName,
-                                            const XML_Char *sysid,
-                                            const XML_Char *pubid,
-                                            int has_internal_subset);
+typedef void(XMLCALL *XML_StartDoctypeDeclHandler)(void *userData,
+                                                   const XML_Char *doctypeName,
+                                                   const XML_Char *sysid,
+                                                   const XML_Char *pubid,
+                                                   int has_internal_subset);
 
 /* This is called for the start of the DOCTYPE declaration when the
    closing > is encountered, but after processing any external
    subset.
 */
-typedef void (XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
+typedef void(XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
 
 /* This is called for entity declarations. The is_parameter_entity
    argument will be non-zero if the entity is a parameter entity, zero
@@ -355,20 +330,14 @@ typedef void (XMLCALL *XML_EndDoctypeDeclHandler)(void *userData);
    Note that is_parameter_entity can't be changed to XML_Bool, since
    that would break binary compatibility.
 */
-typedef void (XMLCALL *XML_EntityDeclHandler) (
-                              void *userData,
-                              const XML_Char *entityName,
-                              int is_parameter_entity,
-                              const XML_Char *value,
-                              int value_length,
-                              const XML_Char *base,
-                              const XML_Char *systemId,
-                              const XML_Char *publicId,
-                              const XML_Char *notationName);
+typedef void(XMLCALL *XML_EntityDeclHandler)(
+    void *userData, const XML_Char *entityName, int is_parameter_entity,
+    const XML_Char *value, int value_length, const XML_Char *base,
+    const XML_Char *systemId, const XML_Char *publicId,
+    const XML_Char *notationName);
 
 XMLPARSEAPI(void)
-XML_SetEntityDeclHandler(XML_Parser parser,
-                         XML_EntityDeclHandler handler);
+XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler);
 
 /* OBSOLETE -- OBSOLETE -- OBSOLETE
    This handler has been superseded by the EntityDeclHandler above.
@@ -379,24 +348,20 @@ XML_SetEntityDeclHandler(XML_Parser parser,
    entityName, systemId and notationName arguments will never be
    NULL. The other arguments may be.
 */
-typedef void (XMLCALL *XML_UnparsedEntityDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *entityName,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId,
-                                    const XML_Char *notationName);
+typedef void(XMLCALL *XML_UnparsedEntityDeclHandler)(
+    void *userData, const XML_Char *entityName, const XML_Char *base,
+    const XML_Char *systemId, const XML_Char *publicId,
+    const XML_Char *notationName);
 
 /* This is called for a declaration of notation.  The base argument is
    whatever was set by XML_SetBase. The notationName will never be
    NULL.  The other arguments can be.
 */
-typedef void (XMLCALL *XML_NotationDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *notationName,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId);
+typedef void(XMLCALL *XML_NotationDeclHandler)(void *userData,
+                                               const XML_Char *notationName,
+                                               const XML_Char *base,
+                                               const XML_Char *systemId,
+                                               const XML_Char *publicId);
 
 /* When namespace processing is enabled, these are called once for
    each namespace declaration. The call to the start and end element
@@ -404,14 +369,12 @@ typedef void (XMLCALL *XML_NotationDeclHandler) (
    declaration handlers. For an xmlns attribute, prefix will be
    NULL.  For an xmlns="" attribute, uri will be NULL.
 */
-typedef void (XMLCALL *XML_StartNamespaceDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *prefix,
-                                    const XML_Char *uri);
+typedef void(XMLCALL *XML_StartNamespaceDeclHandler)(void *userData,
+                                                     const XML_Char *prefix,
+                                                     const XML_Char *uri);
 
-typedef void (XMLCALL *XML_EndNamespaceDeclHandler) (
-                                    void *userData,
-                                    const XML_Char *prefix);
+typedef void(XMLCALL *XML_EndNamespaceDeclHandler)(void *userData,
+                                                   const XML_Char *prefix);
 
 /* This is called if the document is not standalone, that is, it has an
    external subset or a reference to a parameter entity, but does not
@@ -422,7 +385,7 @@ typedef void (XMLCALL *XML_EndNamespaceDeclHandler) (
    conditions above this handler will only be called if the referenced
    entity was actually read.
 */
-typedef int (XMLCALL *XML_NotStandaloneHandler) (void *userData);
+typedef int(XMLCALL *XML_NotStandaloneHandler)(void *userData);
 
 /* This is called for a reference to an external parsed general
    entity.  The referenced entity is not automatically parsed.  The
@@ -458,12 +421,11 @@ typedef int (XMLCALL *XML_NotStandaloneHandler) (void *userData);
    Note that unlike other handlers the first argument is the parser,
    not userData.
 */
-typedef int (XMLCALL *XML_ExternalEntityRefHandler) (
-                                    XML_Parser parser,
-                                    const XML_Char *context,
-                                    const XML_Char *base,
-                                    const XML_Char *systemId,
-                                    const XML_Char *publicId);
+typedef int(XMLCALL *XML_ExternalEntityRefHandler)(XML_Parser parser,
+                                                   const XML_Char *context,
+                                                   const XML_Char *base,
+                                                   const XML_Char *systemId,
+                                                   const XML_Char *publicId);
 
 /* This is called in two situations:
    1) An entity reference is encountered for which no declaration
@@ -475,10 +437,9 @@ typedef int (XMLCALL *XML_ExternalEntityRefHandler) (
          the event would be out of sync with the reporting of the
          declarations or attribute values
 */
-typedef void (XMLCALL *XML_SkippedEntityHandler) (
-                                    void *userData,
-                                    const XML_Char *entityName,
-                                    int is_parameter_entity);
+typedef void(XMLCALL *XML_SkippedEntityHandler)(void *userData,
+                                                const XML_Char *entityName,
+                                                int is_parameter_entity);
 
 /* This structure is filled in by the XML_UnknownEncodingHandler to
    provide information to the parser about encodings that are unknown
@@ -535,8 +496,8 @@ typedef void (XMLCALL *XML_SkippedEntityHandler) (
 typedef struct {
   int map[256];
   void *data;
-  int (XMLCALL *convert)(void *data, const char *s);
-  void (XMLCALL *release)(void *data);
+  int(XMLCALL *convert)(void *data, const char *s);
+  void(XMLCALL *release)(void *data);
 } XML_Encoding;
 
 /* This is called for an encoding that is unknown to the parser.
@@ -554,23 +515,19 @@ typedef struct {
    If info does not describe a suitable encoding, then the parser will
    return an XML_UNKNOWN_ENCODING error.
 */
-typedef int (XMLCALL *XML_UnknownEncodingHandler) (
-                                    void *encodingHandlerData,
-                                    const XML_Char *name,
-                                    XML_Encoding *info);
+typedef int(XMLCALL *XML_UnknownEncodingHandler)(void *encodingHandlerData,
+                                                 const XML_Char *name,
+                                                 XML_Encoding *info);
 
 XMLPARSEAPI(void)
-XML_SetElementHandler(XML_Parser parser,
-                      XML_StartElementHandler start,
+XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start,
                       XML_EndElementHandler end);
 
 XMLPARSEAPI(void)
-XML_SetStartElementHandler(XML_Parser parser,
-                           XML_StartElementHandler handler);
+XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetEndElementHandler(XML_Parser parser,
-                         XML_EndElementHandler handler);
+XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetCharacterDataHandler(XML_Parser parser,
@@ -580,8 +537,7 @@ XMLPARSEAPI(void)
 XML_SetProcessingInstructionHandler(XML_Parser parser,
                                     XML_ProcessingInstructionHandler handler);
 XMLPARSEAPI(void)
-XML_SetCommentHandler(XML_Parser parser,
-                      XML_CommentHandler handler);
+XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetCdataSectionHandler(XML_Parser parser,
@@ -601,20 +557,17 @@ XML_SetEndCdataSectionHandler(XML_Parser parser,
    default handler, or to the skipped entity handler, if one is set.
 */
 XMLPARSEAPI(void)
-XML_SetDefaultHandler(XML_Parser parser,
-                      XML_DefaultHandler handler);
+XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler);
 
 /* This sets the default handler but does not inhibit expansion of
    internal entities.  The entity reference will not be passed to the
    default handler.
 */
 XMLPARSEAPI(void)
-XML_SetDefaultHandlerExpand(XML_Parser parser,
-                            XML_DefaultHandler handler);
+XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetDoctypeDeclHandler(XML_Parser parser,
-                          XML_StartDoctypeDeclHandler start,
+XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start,
                           XML_EndDoctypeDeclHandler end);
 
 XMLPARSEAPI(void)
@@ -622,16 +575,14 @@ XML_SetStartDoctypeDeclHandler(XML_Parser parser,
                                XML_StartDoctypeDeclHandler start);
 
 XMLPARSEAPI(void)
-XML_SetEndDoctypeDeclHandler(XML_Parser parser,
-                             XML_EndDoctypeDeclHandler end);
+XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end);
 
 XMLPARSEAPI(void)
 XML_SetUnparsedEntityDeclHandler(XML_Parser parser,
                                  XML_UnparsedEntityDeclHandler handler);
 
 XMLPARSEAPI(void)
-XML_SetNotationDeclHandler(XML_Parser parser,
-                           XML_NotationDeclHandler handler);
+XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler);
 
 XMLPARSEAPI(void)
 XML_SetNamespaceDeclHandler(XML_Parser parser,
@@ -659,8 +610,7 @@ XML_SetExternalEntityRefHandler(XML_Parser parser,
    instead of the parser object.
 */
 XMLPARSEAPI(void)
-XML_SetExternalEntityRefHandlerArg(XML_Parser parser,
-                                   void *arg);
+XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg);
 
 XMLPARSEAPI(void)
 XML_SetSkippedEntityHandler(XML_Parser parser,
@@ -740,7 +690,6 @@ XML_UseParserAsHandlerArg(XML_Parser parser);
 XMLPARSEAPI(enum XML_Error)
 XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD);
 
-
 /* Sets the base to be used for resolving relative URIs in system
    identifiers in declarations.  Resolving relative identifiers is
    left to the application: this value will be passed through as the
@@ -780,10 +729,10 @@ XML_GetIdAttributeIndex(XML_Parser parser);
    info->valueEnd - info->valueStart = 4 bytes.
 */
 typedef struct {
-  XML_Index  nameStart;  /* Offset to beginning of the attribute name. */
-  XML_Index  nameEnd;    /* Offset after the attribute name's last byte. */
-  XML_Index  valueStart; /* Offset to beginning of the attribute value. */
-  XML_Index  valueEnd;   /* Offset after the attribute value's last byte. */
+  XML_Index nameStart;  /* Offset to beginning of the attribute name. */
+  XML_Index nameEnd;    /* Offset after the attribute name's last byte. */
+  XML_Index valueStart; /* Offset to beginning of the attribute value. */
+  XML_Index valueEnd;   /* Offset after the attribute value's last byte. */
 } XML_AttrInfo;
 
 /* Returns an array of XML_AttrInfo structures for the attribute/value pairs
@@ -819,20 +768,20 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal);
    (resumable = 0) an already suspended parser. Some call-backs may
    still follow because they would otherwise get lost. Examples:
    - endElementHandler() for empty elements when stopped in
-     startElementHandler(), 
-   - endNameSpaceDeclHandler() when stopped in endElementHandler(), 
+     startElementHandler(),
+   - endNameSpaceDeclHandler() when stopped in endElementHandler(),
    and possibly others.
 
    Can be called from most handlers, including DTD related call-backs,
    except when parsing an external parameter entity and resumable != 0.
    Returns XML_STATUS_OK when successful, XML_STATUS_ERROR otherwise.
-   Possible error codes: 
+   Possible error codes:
    - XML_ERROR_SUSPENDED: when suspending an already suspended parser.
    - XML_ERROR_FINISHED: when the parser has already finished.
    - XML_ERROR_SUSPEND_PE: when suspending while parsing an external PE.
 
-   When resumable != 0 (true) then parsing is suspended, that is, 
-   XML_Parse() and XML_ParseBuffer() return XML_STATUS_SUSPENDED. 
+   When resumable != 0 (true) then parsing is suspended, that is,
+   XML_Parse() and XML_ParseBuffer() return XML_STATUS_SUSPENDED.
    Otherwise, parsing is aborted, that is, XML_Parse() and XML_ParseBuffer()
    return XML_STATUS_ERROR with error code XML_ERROR_ABORTED.
 
@@ -843,7 +792,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal);
    the externalEntityRefHandler() to call XML_StopParser() on the parent
    parser (recursively), if one wants to stop parsing altogether.
 
-   When suspended, parsing can be resumed by calling XML_ResumeParser(). 
+   When suspended, parsing can be resumed by calling XML_ResumeParser().
 */
 XMLPARSEAPI(enum XML_Status)
 XML_StopParser(XML_Parser parser, XML_Bool resumable);
@@ -851,7 +800,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable);
 /* Resumes parsing after it has been suspended with XML_StopParser().
    Must not be called from within a handler call-back. Returns same
    status codes as XML_Parse() or XML_ParseBuffer().
-   Additional error code XML_ERROR_NOT_SUSPENDED possible.   
+   Additional error code XML_ERROR_NOT_SUSPENDED possible.
 
    *Note*:
    This must be called on the most deeply nested child parser instance
@@ -863,12 +812,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable);
 XMLPARSEAPI(enum XML_Status)
 XML_ResumeParser(XML_Parser parser);
 
-enum XML_Parsing {
-  XML_INITIALIZED,
-  XML_PARSING,
-  XML_FINISHED,
-  XML_SUSPENDED
-};
+enum XML_Parsing { XML_INITIALIZED, XML_PARSING, XML_FINISHED, XML_SUSPENDED };
 
 typedef struct {
   enum XML_Parsing parsing;
@@ -900,8 +844,7 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status);
    Otherwise returns a new XML_Parser object.
 */
 XMLPARSEAPI(XML_Parser)
-XML_ExternalEntityParserCreate(XML_Parser parser,
-                               const XML_Char *context,
+XML_ExternalEntityParserCreate(XML_Parser parser, const XML_Char *context,
                                const XML_Char *encoding);
 
 enum XML_ParamEntityParsing {
@@ -945,8 +888,7 @@ XML_SetParamEntityParsing(XML_Parser parser,
    Note: If parser == NULL, the function will do nothing and return 0.
 */
 XMLPARSEAPI(int)
-XML_SetHashSalt(XML_Parser parser,
-                unsigned long hash_salt);
+XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt);
 
 /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then
    XML_GetErrorCode returns information about the error.
@@ -963,7 +905,7 @@ XML_GetErrorCode(XML_Parser parser);
    be within the relevant markup.  When called outside of the callback
    functions, the position indicated will be just past the last parse
    event (regardless of whether there was an associated callback).
-   
+
    They may also be called after returning from a call to XML_Parse
    or XML_ParseBuffer.  If the return value is XML_STATUS_ERROR then
    the location is the location of the character at which the error
@@ -995,14 +937,12 @@ XML_GetCurrentByteCount(XML_Parser parser);
    the handler that makes the call.
 */
 XMLPARSEAPI(const char *)
-XML_GetInputContext(XML_Parser parser,
-                    int *offset,
-                    int *size);
+XML_GetInputContext(XML_Parser parser, int *offset, int *size);
 
 /* For backwards compatibility with previous versions. */
-#define XML_GetErrorLineNumber   XML_GetCurrentLineNumber
+#define XML_GetErrorLineNumber XML_GetCurrentLineNumber
 #define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber
-#define XML_GetErrorByteIndex    XML_GetCurrentByteIndex
+#define XML_GetErrorByteIndex XML_GetCurrentByteIndex
 
 /* Frees the content model passed to the element declaration handler */
 XMLPARSEAPI(void)
@@ -1062,21 +1002,20 @@ enum XML_FeatureEnum {
 };
 
 typedef struct {
-  enum XML_FeatureEnum  feature;
-  const XML_LChar       *name;
-  long int              value;
+  enum XML_FeatureEnum feature;
+  const XML_LChar *name;
+  long int value;
 } XML_Feature;
 
 XMLPARSEAPI(const XML_Feature *)
 XML_GetFeatureList(void);
 
-
 /* Expat follows the semantic versioning convention.
    See http://semver.org.
 */
 #define XML_MAJOR_VERSION 2
 #define XML_MINOR_VERSION 2
-#define XML_MICRO_VERSION 7
+#define XML_MICRO_VERSION 8
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h
index fc63c2228d3e..f2b75dda8e27 100644
--- a/Modules/expat/expat_external.h
+++ b/Modules/expat/expat_external.h
@@ -35,10 +35,6 @@
 
 /* External API definitions */
 
-#if defined(_MSC_EXTENSIONS) && !defined(__BEOS__) && !defined(__CYGWIN__)
-# define XML_USE_MSC_EXTENSIONS 1
-#endif
-
 /* Expat tries very hard to make the API boundary very specifically
    defined.  There are two macros defined to control this boundary;
    each of these can be defined before including this header to
@@ -61,12 +57,17 @@
    compiled with the cdecl calling convention as the default since
    system headers may assume the cdecl convention.
 */
+
+/* Namespace external symbols to allow multiple libexpat version to
+   co-exist. */
+#include "pyexpatns.h"
+
 #ifndef XMLCALL
-# if defined(_MSC_VER)
-#  define XMLCALL __cdecl
-# elif defined(__GNUC__) && defined(__i386) && !defined(__INTEL_COMPILER)
-#  define XMLCALL __attribute__((cdecl))
-# else
+#  if defined(_MSC_VER)
+#    define XMLCALL __cdecl
+#  elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER)
+#    define XMLCALL __attribute__((cdecl))
+#  else
 /* For any platform which uses this definition and supports more than
    one calling convention, we need to extend this definition to
    declare the convention used on that platform, if it's possible to
@@ -77,49 +78,46 @@
    pre-processor and how to specify the same calling convention as the
    platform's malloc() implementation.
 */
-#  define XMLCALL
-# endif
-#endif  /* not defined XMLCALL */
-
-/* Namespace external symbols to allow multiple libexpat version to
-   co-exist. */
-#include "pyexpatns.h"
-
+#    define XMLCALL
+#  endif
+#endif /* not defined XMLCALL */
 
-#if !defined(XML_STATIC) && !defined(XMLIMPORT)
-# ifndef XML_BUILDING_EXPAT
+#if ! defined(XML_STATIC) && ! defined(XMLIMPORT)
+#  ifndef XML_BUILDING_EXPAT
 /* using Expat from an application */
 
-#  ifdef XML_USE_MSC_EXTENSIONS
-#   define XMLIMPORT __declspec(dllimport)
-#  endif
+#    if defined(_MSC_EXTENSIONS) && ! defined(__BEOS__) && ! defined(__CYGWIN__)
+#      define XMLIMPORT __declspec(dllimport)
+#    endif
 
-# endif
-#endif  /* not defined XML_STATIC */
+#  endif
+#endif /* not defined XML_STATIC */
 
 #ifndef XML_ENABLE_VISIBILITY
-# define XML_ENABLE_VISIBILITY 0
+#  define XML_ENABLE_VISIBILITY 0
 #endif
 
-#if !defined(XMLIMPORT) && XML_ENABLE_VISIBILITY
-# define XMLIMPORT __attribute__ ((visibility ("default")))
+#if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY
+#  define XMLIMPORT __attribute__((visibility("default")))
 #endif
 
 /* If we didn't define it above, define it away: */
 #ifndef XMLIMPORT
-# define XMLIMPORT
+#  define XMLIMPORT
 #endif
 
-#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
-# define XML_ATTR_MALLOC __attribute__((__malloc__))
+#if defined(__GNUC__)                                                          \
+    && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
+#  define XML_ATTR_MALLOC __attribute__((__malloc__))
 #else
-# define XML_ATTR_MALLOC
+#  define XML_ATTR_MALLOC
 #endif
 
-#if defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
-# define XML_ATTR_ALLOC_SIZE(x)  __attribute__((__alloc_size__(x)))
+#if defined(__GNUC__)                                                          \
+    && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
+#  define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x)))
 #else
-# define XML_ATTR_ALLOC_SIZE(x)
+#  define XML_ATTR_ALLOC_SIZE(x)
 #endif
 
 #define XMLPARSEAPI(type) XMLIMPORT type XMLCALL
@@ -129,35 +127,30 @@ extern "C" {
 #endif
 
 #ifdef XML_UNICODE_WCHAR_T
-# ifndef XML_UNICODE
-#  define XML_UNICODE
-# endif
-# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
-#  error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
-# endif
+#  ifndef XML_UNICODE
+#    define XML_UNICODE
+#  endif
+#  if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
+#    error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
+#  endif
 #endif
 
-#ifdef XML_UNICODE     /* Information is UTF-16 encoded. */
-# ifdef XML_UNICODE_WCHAR_T
+#ifdef XML_UNICODE /* Information is UTF-16 encoded. */
+#  ifdef XML_UNICODE_WCHAR_T
 typedef wchar_t XML_Char;
 typedef wchar_t XML_LChar;
-# else
+#  else
 typedef unsigned short XML_Char;
 typedef char XML_LChar;
-# endif /* XML_UNICODE_WCHAR_T */
-#else                  /* Information is UTF-8 encoded. */
+#  endif /* XML_UNICODE_WCHAR_T */
+#else    /* Information is UTF-8 encoded. */
 typedef char XML_Char;
 typedef char XML_LChar;
-#endif /* XML_UNICODE */
+#endif   /* XML_UNICODE */
 
-#ifdef XML_LARGE_SIZE  /* Use large integers for file/stream positions. */
-# if defined(XML_USE_MSC_EXTENSIONS) && _MSC_VER < 1400
-typedef __int64 XML_Index; 
-typedef unsigned __int64 XML_Size;
-# else
+#ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */
 typedef long long XML_Index;
 typedef unsigned long long XML_Size;
-# endif
 #else
 typedef long XML_Index;
 typedef unsigned long XML_Size;
diff --git a/Modules/expat/iasciitab.h b/Modules/expat/iasciitab.h
index ce4a4bf7edef..ea97cfcf678e 100644
--- a/Modules/expat/iasciitab.h
+++ b/Modules/expat/iasciitab.h
@@ -32,34 +32,34 @@
 
 /* Like asciitab.h, except that 0xD has code BT_S rather than BT_CR */
 /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
-/* 0x0C */ BT_NONXML, BT_S, BT_NONXML, BT_NONXML,
-/* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
-/* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
-/* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
-/* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
-/* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
-/* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
-/* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
-/* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
-/* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
-/* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
-/* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
-/* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
+    /* 0x0C */ BT_NONXML, BT_S, BT_NONXML, BT_NONXML,
+    /* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
+    /* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
+    /* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
+    /* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
+    /* 0x30 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x34 */ BT_DIGIT, BT_DIGIT, BT_DIGIT, BT_DIGIT,
+    /* 0x38 */ BT_DIGIT, BT_DIGIT, BT_COLON, BT_SEMI,
+    /* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
+    /* 0x40 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x44 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
+    /* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
+    /* 0x60 */ BT_OTHER, BT_HEX, BT_HEX, BT_HEX,
+    /* 0x64 */ BT_HEX, BT_HEX, BT_HEX, BT_NMSTRT,
+    /* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h
index dc4ef0c7e258..60913dab762f 100644
--- a/Modules/expat/internal.h
+++ b/Modules/expat/internal.h
@@ -49,7 +49,7 @@
    USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
-#if defined(__GNUC__) && defined(__i386__) && !defined(__MINGW32__)
+#if defined(__GNUC__) && defined(__i386__) && ! defined(__MINGW32__)
 /* We'll use this version by default only where we know it helps.
 
    regparm() generates warnings on Solaris boxes.   See SF bug #692878.
@@ -59,8 +59,8 @@
    #define FASTCALL __attribute__((stdcall, regparm(3)))
    and let's try this:
 */
-#define FASTCALL __attribute__((regparm(3)))
-#define PTRFASTCALL __attribute__((regparm(3)))
+#  define FASTCALL __attribute__((regparm(3)))
+#  define PTRFASTCALL __attribute__((regparm(3)))
 #endif
 
 /* Using __fastcall seems to have an unexpected negative effect under
@@ -74,55 +74,49 @@
 /* Make sure all of these are defined if they aren't already. */
 
 #ifndef FASTCALL
-#define FASTCALL
+#  define FASTCALL
 #endif
 
 #ifndef PTRCALL
-#define PTRCALL
+#  define PTRCALL
 #endif
 
 #ifndef PTRFASTCALL
-#define PTRFASTCALL
+#  define PTRFASTCALL
 #endif
 
 #ifndef XML_MIN_SIZE
-#if !defined(__cplusplus) && !defined(inline)
-#ifdef __GNUC__
-#define inline __inline
-#endif /* __GNUC__ */
-#endif
+#  if ! defined(__cplusplus) && ! defined(inline)
+#    ifdef __GNUC__
+#      define inline __inline
+#    endif /* __GNUC__ */
+#  endif
 #endif /* XML_MIN_SIZE */
 
 #ifdef __cplusplus
-#define inline inline
+#  define inline inline
 #else
-#ifndef inline
-#define inline
-#endif
+#  ifndef inline
+#    define inline
+#  endif
 #endif
 
 #ifndef UNUSED_P
-# ifdef __GNUC__
-#  define UNUSED_P(p) UNUSED_ ## p __attribute__((__unused__))
-# else
-#  define UNUSED_P(p) UNUSED_ ## p
-# endif
+#  define UNUSED_P(p) (void)p
 #endif
 
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-
 #ifdef XML_ENABLE_VISIBILITY
-#if XML_ENABLE_VISIBILITY
-__attribute__ ((visibility ("default")))
-#endif
+#  if XML_ENABLE_VISIBILITY
+__attribute__((visibility("default")))
+#  endif
 #endif
 void
-_INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** fromLimRef);
-
+_INTERNAL_trim_to_complete_utf8_characters(const char *from,
+                                           const char **fromLimRef);
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/latin1tab.h b/Modules/expat/latin1tab.h
index 95dfa52b1fbe..6f916041355a 100644
--- a/Modules/expat/latin1tab.h
+++ b/Modules/expat/latin1tab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x80 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x84 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x88 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x8C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x90 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x94 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x98 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0x9C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA4 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xA8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
-/* 0xAC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xB0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xB4 */ BT_OTHER, BT_NMSTRT, BT_OTHER, BT_NAME,
-/* 0xB8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
-/* 0xBC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
-/* 0xC0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xC4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xC8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xCC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xD0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xD4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0xD8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xDC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xE8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xEC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xF0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xF4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
-/* 0xF8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
-/* 0xFC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0x84 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x88 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x8C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x90 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x94 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x98 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0x9C */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA4 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xA8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
+    /* 0xAC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xB0 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xB4 */ BT_OTHER, BT_NMSTRT, BT_OTHER, BT_NAME,
+    /* 0xB8 */ BT_OTHER, BT_OTHER, BT_NMSTRT, BT_OTHER,
+    /* 0xBC */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER,
+    /* 0xC0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xC4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xC8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xCC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xD0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xD4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0xD8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xDC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xE8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xEC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xF0 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xF4 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
+    /* 0xF8 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
+    /* 0xFC */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
diff --git a/Modules/expat/loadlibrary.c b/Modules/expat/loadlibrary.c
deleted file mode 100644
index 35fdf98bce6c..000000000000
--- a/Modules/expat/loadlibrary.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/***************************************************************************
- *                                  _   _ ____  _
- *  Project                     ___| | | |  _ \| |
- *                             / __| | | | |_) | |
- *                            | (__| |_| |  _ <| |___
- *                             \___|\___/|_| \_\_____|
- *
- * Copyright (C) 2016 - 2017, Steve Holme, <steve_holme at hotmail.com>.
- * Copyright (C) 2017, Expat development team
- *
- * All rights reserved.
- * Licensed under the MIT license:
- *
- * Permission to  use, copy,  modify, and distribute  this software  for any
- * purpose with  or without fee is  hereby granted, provided that  the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE  SOFTWARE  IS  PROVIDED  "AS  IS",  WITHOUT  WARRANTY  OF  ANY  KIND,
- * EXPRESS  OR IMPLIED,  INCLUDING  BUT  NOT LIMITED  TO  THE WARRANTIES  OF
- * MERCHANTABILITY, FITNESS FOR A  PARTICULAR PURPOSE AND NONINFRINGEMENT OF
- * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
- * CONTRACT, TORT OR  OTHERWISE, ARISING FROM, OUT OF OR  IN CONNECTION WITH
- * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * Except as contained in this notice,  the name of a copyright holder shall
- * not be used in advertising or otherwise to promote the sale, use or other
- * dealings  in this  Software without  prior written  authorization of  the
- * copyright holder.
- *
- ***************************************************************************/
-
-#if defined(_WIN32)
-
-#include <windows.h>
-#include <tchar.h>
-
-
-HMODULE _Expat_LoadLibrary(LPCTSTR filename);
-
-
-#if !defined(LOAD_WITH_ALTERED_SEARCH_PATH)
-#define LOAD_WITH_ALTERED_SEARCH_PATH  0x00000008
-#endif
-
-#if !defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
-#define LOAD_LIBRARY_SEARCH_SYSTEM32   0x00000800
-#endif
-
-/* We use our own typedef here since some headers might lack these */
-typedef HMODULE (APIENTRY *LOADLIBRARYEX_FN)(LPCTSTR, HANDLE, DWORD);
-
-/* See function definitions in winbase.h */
-#ifdef UNICODE
-#  ifdef _WIN32_WCE
-#    define LOADLIBARYEX  L"LoadLibraryExW"
-#  else
-#    define LOADLIBARYEX  "LoadLibraryExW"
-#  endif
-#else
-#  define LOADLIBARYEX    "LoadLibraryExA"
-#endif
-
-
-/*
- * _Expat_LoadLibrary()
- *
- * This is used to dynamically load DLLs using the most secure method available
- * for the version of Windows that we are running on.
- *
- * Parameters:
- *
- * filename  [in] - The filename or full path of the DLL to load. If only the
- *                  filename is passed then the DLL will be loaded from the
- *                  Windows system directory.
- *
- * Returns the handle of the module on success; otherwise NULL.
- */
-HMODULE _Expat_LoadLibrary(LPCTSTR filename)
-{
-  HMODULE hModule = NULL;
-  LOADLIBRARYEX_FN pLoadLibraryEx = NULL;
-
-  /* Get a handle to kernel32 so we can access it's functions at runtime */
-  HMODULE hKernel32 = GetModuleHandle(TEXT("kernel32"));
-  if(!hKernel32)
-    return NULL;  /* LCOV_EXCL_LINE */
-
-  /* Attempt to find LoadLibraryEx() which is only available on Windows 2000
-     and above */
-  pLoadLibraryEx = (LOADLIBRARYEX_FN) GetProcAddress(hKernel32, LOADLIBARYEX);
-
-  /* Detect if there's already a path in the filename and load the library if
-     there is. Note: Both back slashes and forward slashes have been supported
-     since the earlier days of DOS at an API level although they are not
-     supported by command prompt */
-  if(_tcspbrk(filename, TEXT("\\/"))) {
-    /** !checksrc! disable BANNEDFUNC 1 **/
-    hModule = pLoadLibraryEx ?
-      pLoadLibraryEx(filename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) :
-      LoadLibrary(filename);
-  }
-  /* Detect if KB2533623 is installed, as LOAD_LIBARY_SEARCH_SYSTEM32 is only
-     supported on Windows Vista, Windows Server 2008, Windows 7 and Windows
-     Server 2008 R2 with this patch or natively on Windows 8 and above */
-  else if(pLoadLibraryEx && GetProcAddress(hKernel32, "AddDllDirectory")) {
-    /* Load the DLL from the Windows system directory */
-    hModule = pLoadLibraryEx(filename, NULL, LOAD_LIBRARY_SEARCH_SYSTEM32);
-  }
-  else {
-    /* Attempt to get the Windows system path */
-    UINT systemdirlen = GetSystemDirectory(NULL, 0);
-    if(systemdirlen) {
-      /* Allocate space for the full DLL path (Room for the null terminator
-         is included in systemdirlen) */
-      size_t filenamelen = _tcslen(filename);
-      TCHAR *path = malloc(sizeof(TCHAR) * (systemdirlen + 1 + filenamelen));
-      if(path && GetSystemDirectory(path, systemdirlen)) {
-        /* Calculate the full DLL path */
-        _tcscpy(path + _tcslen(path), TEXT("\\"));
-        _tcscpy(path + _tcslen(path), filename);
-
-        /* Load the DLL from the Windows system directory */
-        /** !checksrc! disable BANNEDFUNC 1 **/
-        hModule = pLoadLibraryEx ?
-          pLoadLibraryEx(path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) :
-          LoadLibrary(path);
-
-      }
-      free(path);
-    }
-  }
-
-  return hModule;
-}
-
-#else /* defined(_WIN32) */
-
-/* ISO C requires a translation unit to contain at least one declaration
-   [-Wempty-translation-unit] */
-typedef int _TRANSLATION_UNIT_LOAD_LIBRARY_C_NOT_EMTPY;
-
-#endif /* defined(_WIN32) */
diff --git a/Modules/expat/nametab.h b/Modules/expat/nametab.h
index bfa2bd38cd95..3681df348eeb 100644
--- a/Modules/expat/nametab.h
+++ b/Modules/expat/nametab.h
@@ -31,152 +31,106 @@
 */
 
 static const unsigned namingBitmap[] = {
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0x00000000, 0x04000000, 0x87FFFFFE, 0x07FFFFFE,
-0x00000000, 0x00000000, 0xFF7FFFFF, 0xFF7FFFFF,
-0xFFFFFFFF, 0x7FF3FFFF, 0xFFFFFDFE, 0x7FFFFFFF,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFE00F, 0xFC31FFFF,
-0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
-0xFFFFFFFF, 0xF80001FF, 0x00000003, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFD740, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
-0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
-0xFFFF0003, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF,
-0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
-0x0000007F, 0x00000000, 0xFFFF0000, 0x000707FF,
-0x00000000, 0x07FFFFFE, 0x000007FE, 0xFFFE0000,
-0xFFFFFFFF, 0x7CFFFFFF, 0x002F7FFF, 0x00000060,
-0xFFFFFFE0, 0x23FFFFFF, 0xFF000000, 0x00000003,
-0xFFF99FE0, 0x03C5FDFF, 0xB0000000, 0x00030003,
-0xFFF987E0, 0x036DFDFF, 0x5E000000, 0x001C0000,
-0xFFFBAFE0, 0x23EDFDFF, 0x00000000, 0x00000001,
-0xFFF99FE0, 0x23CDFDFF, 0xB0000000, 0x00000003,
-0xD63DC7E0, 0x03BFC718, 0x00000000, 0x00000000,
-0xFFFDDFE0, 0x03EFFDFF, 0x00000000, 0x00000003,
-0xFFFDDFE0, 0x03EFFDFF, 0x40000000, 0x00000003,
-0xFFFDDFE0, 0x03FFFDFF, 0x00000000, 0x00000003,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFE, 0x000D7FFF, 0x0000003F, 0x00000000,
-0xFEF02596, 0x200D6CAE, 0x0000001F, 0x00000000,
-0x00000000, 0x00000000, 0xFFFFFEFF, 0x000003FF,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0xFFFFFFFF, 0xFFFF003F, 0x007FFFFF,
-0x0007DAED, 0x50000000, 0x82315001, 0x002C62AB,
-0x40000000, 0xF580C900, 0x00000007, 0x02010800,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0x0FFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x03FFFFFF,
-0x3F3FFFFF, 0xFFFFFFFF, 0xAAFF3F3F, 0x3FFFFFFF,
-0xFFFFFFFF, 0x5FDFFFFF, 0x0FCF1FDC, 0x1FDC1FFF,
-0x00000000, 0x00004C40, 0x00000000, 0x00000000,
-0x00000007, 0x00000000, 0x00000000, 0x00000000,
-0x00000080, 0x000003FE, 0xFFFFFFFE, 0xFFFFFFFF,
-0x001FFFFF, 0xFFFFFFFE, 0xFFFFFFFF, 0x07FFFFFF,
-0xFFFFFFE0, 0x00001FFF, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0x0000003F, 0x00000000, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
-0xFFFFFFFF, 0x0000000F, 0x00000000, 0x00000000,
-0x00000000, 0x07FF6000, 0x87FFFFFE, 0x07FFFFFE,
-0x00000000, 0x00800000, 0xFF7FFFFF, 0xFF7FFFFF,
-0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
-0xFFFFFFFF, 0xF80001FF, 0x00030003, 0x00000000,
-0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000003,
-0xFFFFD7C0, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
-0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
-0xFFFF007B, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF,
-0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
-0xFFFE007F, 0xBBFFFFFB, 0xFFFF0016, 0x000707FF,
-0x00000000, 0x07FFFFFE, 0x0007FFFF, 0xFFFF03FF,
-0xFFFFFFFF, 0x7CFFFFFF, 0xFFEF7FFF, 0x03FF3DFF,
-0xFFFFFFEE, 0xF3FFFFFF, 0xFF1E3FFF, 0x0000FFCF,
-0xFFF99FEE, 0xD3C5FDFF, 0xB080399F, 0x0003FFCF,
-0xFFF987E4, 0xD36DFDFF, 0x5E003987, 0x001FFFC0,
-0xFFFBAFEE, 0xF3EDFDFF, 0x00003BBF, 0x0000FFC1,
-0xFFF99FEE, 0xF3CDFDFF, 0xB0C0398F, 0x0000FFC3,
-0xD63DC7EC, 0xC3BFC718, 0x00803DC7, 0x0000FF80,
-0xFFFDDFEE, 0xC3EFFDFF, 0x00603DDF, 0x0000FFC3,
-0xFFFDDFEC, 0xC3EFFDFF, 0x40603DDF, 0x0000FFC3,
-0xFFFDDFEC, 0xC3FFFDFF, 0x00803DCF, 0x0000FFC3,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0xFFFFFFFE, 0x07FF7FFF, 0x03FF7FFF, 0x00000000,
-0xFEF02596, 0x3BFF6CAE, 0x03FF3F5F, 0x00000000,
-0x03000000, 0xC2A003FF, 0xFFFFFEFF, 0xFFFE03FF,
-0xFEBF0FDF, 0x02FE3FFF, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x00000000, 0x00000000,
-0x00000000, 0x00000000, 0x1FFF0000, 0x00000002,
-0x000000A0, 0x003EFFFE, 0xFFFFFFFE, 0xFFFFFFFF,
-0x661FFFFF, 0xFFFFFFFE, 0xFFFFFFFF, 0x77FFFFFF,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x04000000,
+    0x87FFFFFE, 0x07FFFFFE, 0x00000000, 0x00000000, 0xFF7FFFFF, 0xFF7FFFFF,
+    0xFFFFFFFF, 0x7FF3FFFF, 0xFFFFFDFE, 0x7FFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFE00F, 0xFC31FFFF, 0x00FFFFFF, 0x00000000, 0xFFFF0000, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xF80001FF, 0x00000003, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFD740, 0xFFFFFFFB, 0x547F7FFF, 0x000FFFFD,
+    0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF, 0xFFFF0003, 0xFFFFFFFF,
+    0xFFFF199F, 0x033FCFFF, 0x00000000, 0xFFFE0000, 0x027FFFFF, 0xFFFFFFFE,
+    0x0000007F, 0x00000000, 0xFFFF0000, 0x000707FF, 0x00000000, 0x07FFFFFE,
+    0x000007FE, 0xFFFE0000, 0xFFFFFFFF, 0x7CFFFFFF, 0x002F7FFF, 0x00000060,
+    0xFFFFFFE0, 0x23FFFFFF, 0xFF000000, 0x00000003, 0xFFF99FE0, 0x03C5FDFF,
+    0xB0000000, 0x00030003, 0xFFF987E0, 0x036DFDFF, 0x5E000000, 0x001C0000,
+    0xFFFBAFE0, 0x23EDFDFF, 0x00000000, 0x00000001, 0xFFF99FE0, 0x23CDFDFF,
+    0xB0000000, 0x00000003, 0xD63DC7E0, 0x03BFC718, 0x00000000, 0x00000000,
+    0xFFFDDFE0, 0x03EFFDFF, 0x00000000, 0x00000003, 0xFFFDDFE0, 0x03EFFDFF,
+    0x40000000, 0x00000003, 0xFFFDDFE0, 0x03FFFDFF, 0x00000000, 0x00000003,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFE, 0x000D7FFF,
+    0x0000003F, 0x00000000, 0xFEF02596, 0x200D6CAE, 0x0000001F, 0x00000000,
+    0x00000000, 0x00000000, 0xFFFFFEFF, 0x000003FF, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0xFFFFFFFF, 0xFFFF003F, 0x007FFFFF, 0x0007DAED, 0x50000000,
+    0x82315001, 0x002C62AB, 0x40000000, 0xF580C900, 0x00000007, 0x02010800,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0FFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0x03FFFFFF, 0x3F3FFFFF, 0xFFFFFFFF, 0xAAFF3F3F, 0x3FFFFFFF,
+    0xFFFFFFFF, 0x5FDFFFFF, 0x0FCF1FDC, 0x1FDC1FFF, 0x00000000, 0x00004C40,
+    0x00000000, 0x00000000, 0x00000007, 0x00000000, 0x00000000, 0x00000000,
+    0x00000080, 0x000003FE, 0xFFFFFFFE, 0xFFFFFFFF, 0x001FFFFF, 0xFFFFFFFE,
+    0xFFFFFFFF, 0x07FFFFFF, 0xFFFFFFE0, 0x00001FFF, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000000, 0x00000000,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000000F,
+    0x00000000, 0x00000000, 0x00000000, 0x07FF6000, 0x87FFFFFE, 0x07FFFFFE,
+    0x00000000, 0x00800000, 0xFF7FFFFF, 0xFF7FFFFF, 0x00FFFFFF, 0x00000000,
+    0xFFFF0000, 0xFFFFFFFF, 0xFFFFFFFF, 0xF80001FF, 0x00030003, 0x00000000,
+    0xFFFFFFFF, 0xFFFFFFFF, 0x0000003F, 0x00000003, 0xFFFFD7C0, 0xFFFFFFFB,
+    0x547F7FFF, 0x000FFFFD, 0xFFFFDFFE, 0xFFFFFFFF, 0xDFFEFFFF, 0xFFFFFFFF,
+    0xFFFF007B, 0xFFFFFFFF, 0xFFFF199F, 0x033FCFFF, 0x00000000, 0xFFFE0000,
+    0x027FFFFF, 0xFFFFFFFE, 0xFFFE007F, 0xBBFFFFFB, 0xFFFF0016, 0x000707FF,
+    0x00000000, 0x07FFFFFE, 0x0007FFFF, 0xFFFF03FF, 0xFFFFFFFF, 0x7CFFFFFF,
+    0xFFEF7FFF, 0x03FF3DFF, 0xFFFFFFEE, 0xF3FFFFFF, 0xFF1E3FFF, 0x0000FFCF,
+    0xFFF99FEE, 0xD3C5FDFF, 0xB080399F, 0x0003FFCF, 0xFFF987E4, 0xD36DFDFF,
+    0x5E003987, 0x001FFFC0, 0xFFFBAFEE, 0xF3EDFDFF, 0x00003BBF, 0x0000FFC1,
+    0xFFF99FEE, 0xF3CDFDFF, 0xB0C0398F, 0x0000FFC3, 0xD63DC7EC, 0xC3BFC718,
+    0x00803DC7, 0x0000FF80, 0xFFFDDFEE, 0xC3EFFDFF, 0x00603DDF, 0x0000FFC3,
+    0xFFFDDFEC, 0xC3EFFDFF, 0x40603DDF, 0x0000FFC3, 0xFFFDDFEC, 0xC3FFFDFF,
+    0x00803DCF, 0x0000FFC3, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0xFFFFFFFE, 0x07FF7FFF, 0x03FF7FFF, 0x00000000, 0xFEF02596, 0x3BFF6CAE,
+    0x03FF3F5F, 0x00000000, 0x03000000, 0xC2A003FF, 0xFFFFFEFF, 0xFFFE03FF,
+    0xFEBF0FDF, 0x02FE3FFF, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x1FFF0000, 0x00000002,
+    0x000000A0, 0x003EFFFE, 0xFFFFFFFE, 0xFFFFFFFF, 0x661FFFFF, 0xFFFFFFFE,
+    0xFFFFFFFF, 0x77FFFFFF,
 };
 static const unsigned char nmstrtPages[] = {
-0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00,
-0x00, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
-0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13,
-0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x15, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x17,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x09, 0x0A, 0x0B,
+    0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13, 0x00, 0x14, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x15, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00,
 };
 static const unsigned char namePages[] = {
-0x19, 0x03, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x00,
-0x00, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25,
-0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13,
-0x26, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x27, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x17,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x19, 0x03, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x00, 0x00, 0x1F, 0x20, 0x21,
+    0x22, 0x23, 0x24, 0x25, 0x10, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x13, 0x26, 0x14, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x27, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x18,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00,
 };
diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h
index 4d6786d7839a..bfee65a332f1 100644
--- a/Modules/expat/siphash.h
+++ b/Modules/expat/siphash.h
@@ -11,6 +11,10 @@
  * --------------------------------------------------------------------------
  * HISTORY:
  *
+ * 2019-08-03  (Sebastian Pipping)
+ *   - Mark part of sip24_valid as to be excluded from clang-format
+ *   - Re-format code using clang-format 9
+ *
  * 2018-07-08  (Anton Maklakov)
  *   - Add "fall through" markers for GCC's -Wimplicit-fallthrough
  *
@@ -94,186 +98,186 @@
 #include <stddef.h> /* size_t */
 
 #if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER < 1600)
-  /* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */
-  typedef unsigned __int8   uint8_t;
-  typedef unsigned __int32 uint32_t;
-  typedef unsigned __int64 uint64_t;
+/* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */
+typedef unsigned __int8 uint8_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
 #else
- #include <stdint.h> /* uint64_t uint32_t uint8_t */
+#  include <stdint.h> /* uint64_t uint32_t uint8_t */
 #endif
 
-
 /*
  * Workaround to not require a C++11 compiler for using ULL suffix
  * if this code is included and compiled as C++; related GCC warning is:
  * warning: use of C++11 long long integer constant [-Wlong-long]
  */
-#define _SIP_ULL(high, low)  (((uint64_t)high << 32) | low)
-
-
-#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b))))
+#define _SIP_ULL(high, low) (((uint64_t)high << 32) | low)
 
-#define SIP_U32TO8_LE(p, v) \
-	(p)[0] = (uint8_t)((v) >>  0); (p)[1] = (uint8_t)((v) >>  8); \
-	(p)[2] = (uint8_t)((v) >> 16); (p)[3] = (uint8_t)((v) >> 24);
+#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
 
-#define SIP_U64TO8_LE(p, v) \
-	SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >>  0)); \
-	SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
+#define SIP_U32TO8_LE(p, v)                                                    \
+  (p)[0] = (uint8_t)((v) >> 0);                                                \
+  (p)[1] = (uint8_t)((v) >> 8);                                                \
+  (p)[2] = (uint8_t)((v) >> 16);                                               \
+  (p)[3] = (uint8_t)((v) >> 24);
 
-#define SIP_U8TO64_LE(p) \
-	(((uint64_t)((p)[0]) <<  0) | \
-	 ((uint64_t)((p)[1]) <<  8) | \
-	 ((uint64_t)((p)[2]) << 16) | \
-	 ((uint64_t)((p)[3]) << 24) | \
-	 ((uint64_t)((p)[4]) << 32) | \
-	 ((uint64_t)((p)[5]) << 40) | \
-	 ((uint64_t)((p)[6]) << 48) | \
-	 ((uint64_t)((p)[7]) << 56))
+#define SIP_U64TO8_LE(p, v)                                                    \
+  SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >> 0));                                \
+  SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
 
+#define SIP_U8TO64_LE(p)                                                       \
+  (((uint64_t)((p)[0]) << 0) | ((uint64_t)((p)[1]) << 8)                       \
+   | ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24)                   \
+   | ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40)                   \
+   | ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))
 
-#define SIPHASH_INITIALIZER { 0, 0, 0, 0, { 0 }, 0, 0 }
+#define SIPHASH_INITIALIZER                                                    \
+  { 0, 0, 0, 0, {0}, 0, 0 }
 
 struct siphash {
-	uint64_t v0, v1, v2, v3;
+  uint64_t v0, v1, v2, v3;
 
-	unsigned char buf[8], *p;
-	uint64_t c;
+  unsigned char buf[8], *p;
+  uint64_t c;
 }; /* struct siphash */
 
-
 #define SIP_KEYLEN 16
 
 struct sipkey {
-	uint64_t k[2];
+  uint64_t k[2];
 }; /* struct sipkey */
 
-#define sip_keyof(k) sip_tokey(&(struct sipkey){ { 0 } }, (k))
+#define sip_keyof(k) sip_tokey(&(struct sipkey){{0}}, (k))
 
-static struct sipkey *sip_tokey(struct sipkey *key, const void *src) {
-	key->k[0] = SIP_U8TO64_LE((const unsigned char *)src);
-	key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8);
-	return key;
+static struct sipkey *
+sip_tokey(struct sipkey *key, const void *src) {
+  key->k[0] = SIP_U8TO64_LE((const unsigned char *)src);
+  key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8);
+  return key;
 } /* sip_tokey() */
 
-
 #ifdef SIPHASH_TOBIN
 
-#define sip_binof(v) sip_tobin((unsigned char[8]){ 0 }, (v))
+#  define sip_binof(v) sip_tobin((unsigned char[8]){0}, (v))
 
-static void *sip_tobin(void *dst, uint64_t u64) {
-	SIP_U64TO8_LE((unsigned char *)dst, u64);
-	return dst;
+static void *
+sip_tobin(void *dst, uint64_t u64) {
+  SIP_U64TO8_LE((unsigned char *)dst, u64);
+  return dst;
 } /* sip_tobin() */
 
-#endif  /* SIPHASH_TOBIN */
-
+#endif /* SIPHASH_TOBIN */
 
-static void sip_round(struct siphash *H, const int rounds) {
-	int i;
+static void
+sip_round(struct siphash *H, const int rounds) {
+  int i;
 
-	for (i = 0; i < rounds; i++) {
-		H->v0 += H->v1;
-		H->v1 = SIP_ROTL(H->v1, 13);
-		H->v1 ^= H->v0;
-		H->v0 = SIP_ROTL(H->v0, 32);
+  for (i = 0; i < rounds; i++) {
+    H->v0 += H->v1;
+    H->v1 = SIP_ROTL(H->v1, 13);
+    H->v1 ^= H->v0;
+    H->v0 = SIP_ROTL(H->v0, 32);
 
-		H->v2 += H->v3;
-		H->v3 = SIP_ROTL(H->v3, 16);
-		H->v3 ^= H->v2;
+    H->v2 += H->v3;
+    H->v3 = SIP_ROTL(H->v3, 16);
+    H->v3 ^= H->v2;
 
-		H->v0 += H->v3;
-		H->v3 = SIP_ROTL(H->v3, 21);
-		H->v3 ^= H->v0;
+    H->v0 += H->v3;
+    H->v3 = SIP_ROTL(H->v3, 21);
+    H->v3 ^= H->v0;
 
-		H->v2 += H->v1;
-		H->v1 = SIP_ROTL(H->v1, 17);
-		H->v1 ^= H->v2;
-		H->v2 = SIP_ROTL(H->v2, 32);
-	}
+    H->v2 += H->v1;
+    H->v1 = SIP_ROTL(H->v1, 17);
+    H->v1 ^= H->v2;
+    H->v2 = SIP_ROTL(H->v2, 32);
+  }
 } /* sip_round() */
 
+static struct siphash *
+sip24_init(struct siphash *H, const struct sipkey *key) {
+  H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0];
+  H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1];
+  H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0];
+  H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1];
 
-static struct siphash *sip24_init(struct siphash *H,
-		const struct sipkey *key) {
-	H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0];
-	H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1];
-	H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0];
-	H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1];
+  H->p = H->buf;
+  H->c = 0;
 
-	H->p = H->buf;
-	H->c = 0;
-
-	return H;
+  return H;
 } /* sip24_init() */
 
+#define sip_endof(a) (&(a)[sizeof(a) / sizeof *(a)])
 
-#define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)])
-
-static struct siphash *sip24_update(struct siphash *H, const void *src,
-		size_t len) {
-	const unsigned char *p = (const unsigned char *)src, *pe = p + len;
-	uint64_t m;
+static struct siphash *
+sip24_update(struct siphash *H, const void *src, size_t len) {
+  const unsigned char *p = (const unsigned char *)src, *pe = p + len;
+  uint64_t m;
 
-	do {
-		while (p < pe && H->p < sip_endof(H->buf))
-			*H->p++ = *p++;
+  do {
+    while (p < pe && H->p < sip_endof(H->buf))
+      *H->p++ = *p++;
 
-		if (H->p < sip_endof(H->buf))
-			break;
+    if (H->p < sip_endof(H->buf))
+      break;
 
-		m = SIP_U8TO64_LE(H->buf);
-		H->v3 ^= m;
-		sip_round(H, 2);
-		H->v0 ^= m;
+    m = SIP_U8TO64_LE(H->buf);
+    H->v3 ^= m;
+    sip_round(H, 2);
+    H->v0 ^= m;
 
-		H->p = H->buf;
-		H->c += 8;
-	} while (p < pe);
+    H->p = H->buf;
+    H->c += 8;
+  } while (p < pe);
 
-	return H;
+  return H;
 } /* sip24_update() */
 
-
-static uint64_t sip24_final(struct siphash *H) {
-	const char left = (char)(H->p - H->buf);
-	uint64_t b = (H->c + left) << 56;
-
-	switch (left) {
-	case 7: b |= (uint64_t)H->buf[6] << 48;
-		/* fall through */
-	case 6: b |= (uint64_t)H->buf[5] << 40;
-		/* fall through */
-	case 5: b |= (uint64_t)H->buf[4] << 32;
-		/* fall through */
-	case 4: b |= (uint64_t)H->buf[3] << 24;
-		/* fall through */
-	case 3: b |= (uint64_t)H->buf[2] << 16;
-		/* fall through */
-	case 2: b |= (uint64_t)H->buf[1] << 8;
-		/* fall through */
-	case 1: b |= (uint64_t)H->buf[0] << 0;
-		/* fall through */
-	case 0: break;
-	}
-
-	H->v3 ^= b;
-	sip_round(H, 2);
-	H->v0 ^= b;
-	H->v2 ^= 0xff;
-	sip_round(H, 4);
-
-	return H->v0 ^ H->v1 ^ H->v2  ^ H->v3;
+static uint64_t
+sip24_final(struct siphash *H) {
+  const char left = (char)(H->p - H->buf);
+  uint64_t b = (H->c + left) << 56;
+
+  switch (left) {
+  case 7:
+    b |= (uint64_t)H->buf[6] << 48;
+    /* fall through */
+  case 6:
+    b |= (uint64_t)H->buf[5] << 40;
+    /* fall through */
+  case 5:
+    b |= (uint64_t)H->buf[4] << 32;
+    /* fall through */
+  case 4:
+    b |= (uint64_t)H->buf[3] << 24;
+    /* fall through */
+  case 3:
+    b |= (uint64_t)H->buf[2] << 16;
+    /* fall through */
+  case 2:
+    b |= (uint64_t)H->buf[1] << 8;
+    /* fall through */
+  case 1:
+    b |= (uint64_t)H->buf[0] << 0;
+    /* fall through */
+  case 0:
+    break;
+  }
+
+  H->v3 ^= b;
+  sip_round(H, 2);
+  H->v0 ^= b;
+  H->v2 ^= 0xff;
+  sip_round(H, 4);
+
+  return H->v0 ^ H->v1 ^ H->v2 ^ H->v3;
 } /* sip24_final() */
 
-
-static uint64_t siphash24(const void *src, size_t len,
-		const struct sipkey *key) {
-	struct siphash state = SIPHASH_INITIALIZER;
-	return sip24_final(sip24_update(sip24_init(&state, key), src, len));
+static uint64_t
+siphash24(const void *src, size_t len, const struct sipkey *key) {
+  struct siphash state = SIPHASH_INITIALIZER;
+  return sip24_final(sip24_update(sip24_init(&state, key), src, len));
 } /* siphash24() */
 
-
 /*
  * SipHash-2-4 output with
  * k = 00 01 02 ...
@@ -285,107 +289,110 @@ static uint64_t siphash24(const void *src, size_t len,
  * ...
  * in = 00 01 02 ... 3e (63 bytes)
  */
-static int sip24_valid(void) {
-	static const unsigned char vectors[64][8] = {
-		{ 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, },
-		{ 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, },
-		{ 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, },
-		{ 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, },
-		{ 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, },
-		{ 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, },
-		{ 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, },
-		{ 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, },
-		{ 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, },
-		{ 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, },
-		{ 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, },
-		{ 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, },
-		{ 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, },
-		{ 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, },
-		{ 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, },
-		{ 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, },
-		{ 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, },
-		{ 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, },
-		{ 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, },
-		{ 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, },
-		{ 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, },
-		{ 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, },
-		{ 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, },
-		{ 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, },
-		{ 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, },
-		{ 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, },
-		{ 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, },
-		{ 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, },
-		{ 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, },
-		{ 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, },
-		{ 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, },
-		{ 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, },
-		{ 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, },
-		{ 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, },
-		{ 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, },
-		{ 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, },
-		{ 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, },
-		{ 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, },
-		{ 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, },
-		{ 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, },
-		{ 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, },
-		{ 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, },
-		{ 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, },
-		{ 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, },
-		{ 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, },
-		{ 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, },
-		{ 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, },
-		{ 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, },
-		{ 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, },
-		{ 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, },
-		{ 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, },
-		{ 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, },
-		{ 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, },
-		{ 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, },
-		{ 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, },
-		{ 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, },
-		{ 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, },
-		{ 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, },
-		{ 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, },
-		{ 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, },
-		{ 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, },
-		{ 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, },
-		{ 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, },
-		{ 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, }
-	};
-	unsigned char in[64];
-	struct sipkey k;
-	size_t i;
-
-	sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011"
-			"\012\013\014\015\016\017");
-
-	for (i = 0; i < sizeof in; ++i) {
-		in[i] = (unsigned char)i;
-
-		if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i]))
-			return 0;
-	}
-
-	return 1;
+static int
+sip24_valid(void) {
+  /* clang-format off */
+  static const unsigned char vectors[64][8] = {
+    { 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, },
+    { 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, },
+    { 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, },
+    { 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, },
+    { 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, },
+    { 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, },
+    { 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, },
+    { 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, },
+    { 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, },
+    { 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, },
+    { 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, },
+    { 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, },
+    { 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, },
+    { 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, },
+    { 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, },
+    { 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, },
+    { 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, },
+    { 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, },
+    { 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, },
+    { 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, },
+    { 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, },
+    { 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, },
+    { 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, },
+    { 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, },
+    { 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, },
+    { 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, },
+    { 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, },
+    { 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, },
+    { 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, },
+    { 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, },
+    { 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, },
+    { 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, },
+    { 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, },
+    { 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, },
+    { 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, },
+    { 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, },
+    { 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, },
+    { 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, },
+    { 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, },
+    { 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, },
+    { 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, },
+    { 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, },
+    { 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, },
+    { 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, },
+    { 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, },
+    { 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, },
+    { 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, },
+    { 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, },
+    { 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, },
+    { 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, },
+    { 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, },
+    { 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, },
+    { 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, },
+    { 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, },
+    { 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, },
+    { 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, },
+    { 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, },
+    { 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, },
+    { 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, },
+    { 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, },
+    { 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, },
+    { 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, },
+    { 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, },
+    { 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, }
+  };
+  /* clang-format on */
+
+  unsigned char in[64];
+  struct sipkey k;
+  size_t i;
+
+  sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011"
+                "\012\013\014\015\016\017");
+
+  for (i = 0; i < sizeof in; ++i) {
+    in[i] = (unsigned char)i;
+
+    if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i]))
+      return 0;
+  }
+
+  return 1;
 } /* sip24_valid() */
 
-
 #ifdef SIPHASH_MAIN
 
-#include <stdio.h>
+#  include <stdio.h>
 
-int main(void) {
-	const int ok = sip24_valid();
+int
+main(void) {
+  const int ok = sip24_valid();
 
-	if (ok)
-		puts("OK");
-	else
-		puts("FAIL");
+  if (ok)
+    puts("OK");
+  else
+    puts("FAIL");
 
-	return !ok;
+  return ! ok;
 } /* main() */
 
 #endif /* SIPHASH_MAIN */
 
-
 #endif /* SIPHASH_H */
diff --git a/Modules/expat/utf8tab.h b/Modules/expat/utf8tab.h
index fa0bed6f5d75..a22986acbb95 100644
--- a/Modules/expat/utf8tab.h
+++ b/Modules/expat/utf8tab.h
@@ -31,34 +31,34 @@
 */
 
 /* 0x80 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x84 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x88 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x8C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x90 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x94 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x98 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0x9C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xA8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xAC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xB8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xBC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
-/* 0xC0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xC4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xC8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xCC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xD8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xDC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
-/* 0xE0 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xE4 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xE8 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xEC */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
-/* 0xF0 */ BT_LEAD4, BT_LEAD4, BT_LEAD4, BT_LEAD4,
-/* 0xF4 */ BT_LEAD4, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0xF8 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
-/* 0xFC */ BT_NONXML, BT_NONXML, BT_MALFORM, BT_MALFORM,
+    /* 0x84 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x88 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x8C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x90 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x94 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x98 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0x9C */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xA8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xAC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB0 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB4 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xB8 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xBC */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL,
+    /* 0xC0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xC4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xC8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xCC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD0 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD4 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xD8 */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xDC */ BT_LEAD2, BT_LEAD2, BT_LEAD2, BT_LEAD2,
+    /* 0xE0 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xE4 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xE8 */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xEC */ BT_LEAD3, BT_LEAD3, BT_LEAD3, BT_LEAD3,
+    /* 0xF0 */ BT_LEAD4, BT_LEAD4, BT_LEAD4, BT_LEAD4,
+    /* 0xF4 */ BT_LEAD4, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0xF8 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
+    /* 0xFC */ BT_NONXML, BT_NONXML, BT_MALFORM, BT_MALFORM,
diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h
index 28a043c6044c..562a4a82dc1d 100644
--- a/Modules/expat/winconfig.h
+++ b/Modules/expat/winconfig.h
@@ -40,20 +40,17 @@
 #include <memory.h>
 #include <string.h>
 
+#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */
+#  include <expat_config.h>
+#else /* !defined(HAVE_EXPAT_CONFIG_H) */
 
-#if defined(HAVE_EXPAT_CONFIG_H)  /* e.g. MinGW */
-# include <expat_config.h>
-#else  /* !defined(HAVE_EXPAT_CONFIG_H) */
-
-
-#define XML_NS 1
-#define XML_DTD 1
-#define XML_CONTEXT_BYTES 1024
+#  define XML_NS 1
+#  define XML_DTD 1
+#  define XML_CONTEXT_BYTES 1024
 
 /* we will assume all Windows platforms are little endian */
-#define BYTEORDER 1234
+#  define BYTEORDER 1234
 
 #endif /* !defined(HAVE_EXPAT_CONFIG_H) */
 
-
 #endif /* ndef WINCONFIG_H */
diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c
index 9c0987f4f6d8..e740f0e19c7d 100644
--- a/Modules/expat/xmlparse.c
+++ b/Modules/expat/xmlparse.c
@@ -1,4 +1,4 @@
-/* 69df5be70289a11fb834869ce4a91c23c1d9dd04baffcbd10e86742d149a080c (2.2.7+)
+/* f2d0ab6d1d4422a08cf1cf3bbdfba96b49dea42fb5ff4615e03a2a25c306e769 (2.2.8+)
                             __  __            _
                          ___\ \/ /_ __   __ _| |_
                         / _ \\  /| '_ \ / _` | __|
@@ -30,33 +30,38 @@
    USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
-#if !defined(_GNU_SOURCE)
-# define _GNU_SOURCE 1                  /* syscall prototype */
+#if ! defined(_GNU_SOURCE)
+#  define _GNU_SOURCE 1 /* syscall prototype */
+#endif
+
+#ifdef _WIN32
+/* force stdlib to define rand_s() */
+#  define _CRT_RAND_S
 #endif
 
 #include <stddef.h>
-#include <string.h>                     /* memset(), memcpy() */
+#include <string.h> /* memset(), memcpy() */
 #include <assert.h>
-#include <limits.h>                     /* UINT_MAX */
-#include <stdio.h>                      /* fprintf */
-#include <stdlib.h>                     /* getenv */
+#include <limits.h> /* UINT_MAX */
+#include <stdio.h>  /* fprintf */
+#include <stdlib.h> /* getenv, rand_s */
 
 #ifdef _WIN32
-#define getpid GetCurrentProcessId
+#  define getpid GetCurrentProcessId
 #else
-#include <sys/time.h>                   /* gettimeofday() */
-#include <sys/types.h>                  /* getpid() */
-#include <unistd.h>                     /* getpid() */
-#include <fcntl.h>                      /* O_RDONLY */
-#include <errno.h>
+#  include <sys/time.h>  /* gettimeofday() */
+#  include <sys/types.h> /* getpid() */
+#  include <unistd.h>    /* getpid() */
+#  include <fcntl.h>     /* O_RDONLY */
+#  include <errno.h>
 #endif
 
 #define XML_BUILDING_EXPAT 1
 
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #elif defined(HAVE_EXPAT_CONFIG_H)
-#include <expat_config.h>
+#  include <expat_config.h>
 #endif /* ndef _WIN32 */
 
 #include "ascii.h"
@@ -64,33 +69,31 @@
 #include "siphash.h"
 
 #if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
-# if defined(HAVE_GETRANDOM)
-#  include <sys/random.h>    /* getrandom */
-# else
-#  include <unistd.h>        /* syscall */
-#  include <sys/syscall.h>   /* SYS_getrandom */
-# endif
-# if ! defined(GRND_NONBLOCK)
-#  define GRND_NONBLOCK  0x0001
-# endif  /* defined(GRND_NONBLOCK) */
-#endif  /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
-
-#if defined(HAVE_LIBBSD) \
+#  if defined(HAVE_GETRANDOM)
+#    include <sys/random.h> /* getrandom */
+#  else
+#    include <unistd.h>      /* syscall */
+#    include <sys/syscall.h> /* SYS_getrandom */
+#  endif
+#  if ! defined(GRND_NONBLOCK)
+#    define GRND_NONBLOCK 0x0001
+#  endif /* defined(GRND_NONBLOCK) */
+#endif   /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
+
+#if defined(HAVE_LIBBSD)                                                       \
     && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM))
-# include <bsd/stdlib.h>
+#  include <bsd/stdlib.h>
 #endif
 
-#if defined(_WIN32) && !defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
-# define LOAD_LIBRARY_SEARCH_SYSTEM32  0x00000800
+#if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
+#  define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800
 #endif
 
-#if !defined(HAVE_GETRANDOM) && !defined(HAVE_SYSCALL_GETRANDOM) \
-    && !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_ARC4RANDOM) \
-    && !defined(XML_DEV_URANDOM) \
-    && !defined(_WIN32) \
-    && !defined(XML_POOR_ENTROPY)
-# error  \
-    You do not have support for any sources of high quality entropy \
+#if ! defined(HAVE_GETRANDOM) && ! defined(HAVE_SYSCALL_GETRANDOM)             \
+    && ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)            \
+    && ! defined(XML_DEV_URANDOM) && ! defined(_WIN32)                         \
+    && ! defined(XML_POOR_ENTROPY)
+#  error You do not have support for any sources of high quality entropy \
     enabled.  For end user security, that is probably not what you want. \
     \
     Your options include: \
@@ -101,7 +104,7 @@
       * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \
       * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \
       * Linux / BSD / macOS (/dev/urandom): XML_DEV_URANDOM \
-      * Windows (RtlGenRandom): _WIN32. \
+      * Windows (rand_s): _WIN32. \
     \
     If insist on not using any of these, bypass this error by defining \
     XML_POOR_ENTROPY; you have been warned. \
@@ -110,56 +113,55 @@
     to the build system, please open a bug.  Thank you!
 #endif
 
-
 #ifdef XML_UNICODE
-#define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX
-#define XmlConvert XmlUtf16Convert
-#define XmlGetInternalEncoding XmlGetUtf16InternalEncoding
-#define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS
-#define XmlEncode XmlUtf16Encode
+#  define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX
+#  define XmlConvert XmlUtf16Convert
+#  define XmlGetInternalEncoding XmlGetUtf16InternalEncoding
+#  define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS
+#  define XmlEncode XmlUtf16Encode
 /* Using pointer subtraction to convert to integer type. */
-#define MUST_CONVERT(enc, s) (!(enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1))
+#  define MUST_CONVERT(enc, s)                                                 \
+    (! (enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1))
 typedef unsigned short ICHAR;
 #else
-#define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX
-#define XmlConvert XmlUtf8Convert
-#define XmlGetInternalEncoding XmlGetUtf8InternalEncoding
-#define XmlGetInternalEncodingNS XmlGetUtf8InternalEncodingNS
-#define XmlEncode XmlUtf8Encode
-#define MUST_CONVERT(enc, s) (!(enc)->isUtf8)
+#  define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX
+#  define XmlConvert XmlUtf8Convert
+#  define XmlGetInternalEncoding XmlGetUtf8InternalEncoding
+#  define XmlGetInternalEncodingNS XmlGetUtf8InternalEncodingNS
+#  define XmlEncode XmlUtf8Encode
+#  define MUST_CONVERT(enc, s) (! (enc)->isUtf8)
 typedef char ICHAR;
 #endif
 
-
 #ifndef XML_NS
 
-#define XmlInitEncodingNS XmlInitEncoding
-#define XmlInitUnknownEncodingNS XmlInitUnknownEncoding
-#undef XmlGetInternalEncodingNS
-#define XmlGetInternalEncodingNS XmlGetInternalEncoding
-#define XmlParseXmlDeclNS XmlParseXmlDecl
+#  define XmlInitEncodingNS XmlInitEncoding
+#  define XmlInitUnknownEncodingNS XmlInitUnknownEncoding
+#  undef XmlGetInternalEncodingNS
+#  define XmlGetInternalEncodingNS XmlGetInternalEncoding
+#  define XmlParseXmlDeclNS XmlParseXmlDecl
 
 #endif
 
 #ifdef XML_UNICODE
 
-#ifdef XML_UNICODE_WCHAR_T
-#define XML_T(x) (const wchar_t)x
-#define XML_L(x) L ## x
-#else
-#define XML_T(x) (const unsigned short)x
-#define XML_L(x) x
-#endif
+#  ifdef XML_UNICODE_WCHAR_T
+#    define XML_T(x) (const wchar_t) x
+#    define XML_L(x) L##x
+#  else
+#    define XML_T(x) (const unsigned short)x
+#    define XML_L(x) x
+#  endif
 
 #else
 
-#define XML_T(x) x
-#define XML_L(x) x
+#  define XML_T(x) x
+#  define XML_L(x) x
 
 #endif
 
 /* Round up n to be a multiple of sz, where sz is a power of 2. */
-#define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1))
+#define ROUND_UP(n, sz) (((n) + ((sz)-1)) & ~((sz)-1))
 
 /* Do safe (NULL-aware) pointer arithmetic */
 #define EXPAT_SAFE_PTR_DIFF(p, q) (((p) && (q)) ? ((p) - (q)) : 0)
@@ -182,11 +184,9 @@ typedef struct {
   const XML_Memory_Handling_Suite *mem;
 } HASH_TABLE;
 
-static size_t
-keylen(KEY s);
+static size_t keylen(KEY s);
 
-static void
-copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key);
+static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key);
 
 /* For probing (after a collision) we need a step size relative prime
    to the hash table size, which is a power of 2. We use double-hashing,
@@ -196,9 +196,9 @@ copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key);
    We limit the maximum step size to table->size / 4 (mask >> 2) and make
    it odd, since odd numbers are always relative prime to a power of 2.
 */
-#define SECOND_HASH(hash, mask, power) \
-  ((((hash) & ~(mask)) >> ((power) - 1)) & ((mask) >> 2))
-#define PROBE_STEP(hash, mask, power) \
+#define SECOND_HASH(hash, mask, power)                                         \
+  ((((hash) & ~(mask)) >> ((power)-1)) & ((mask) >> 2))
+#define PROBE_STEP(hash, mask, power)                                          \
   ((unsigned char)((SECOND_HASH(hash, mask, power)) | 1))
 
 typedef struct {
@@ -206,7 +206,7 @@ typedef struct {
   NAMED **end;
 } HASH_TABLE_ITER;
 
-#define INIT_TAG_BUF_SIZE 32  /* must be a multiple of sizeof(XML_Char) */
+#define INIT_TAG_BUF_SIZE 32 /* must be a multiple of sizeof(XML_Char) */
 #define INIT_DATA_BUF_SIZE 1024
 #define INIT_ATTS_SIZE 16
 #define INIT_ATTS_VERSION 0xFFFFFFFF
@@ -253,20 +253,20 @@ typedef struct {
    TAG objects in a free list.
 */
 typedef struct tag {
-  struct tag *parent;           /* parent of this element */
-  const char *rawName;          /* tagName in the original encoding */
+  struct tag *parent;  /* parent of this element */
+  const char *rawName; /* tagName in the original encoding */
   int rawNameLength;
-  TAG_NAME name;                /* tagName in the API encoding */
-  char *buf;                    /* buffer for name components */
-  char *bufEnd;                 /* end of the buffer */
+  TAG_NAME name; /* tagName in the API encoding */
+  char *buf;     /* buffer for name components */
+  char *bufEnd;  /* end of the buffer */
   BINDING *bindings;
 } TAG;
 
 typedef struct {
   const XML_Char *name;
   const XML_Char *textPtr;
-  int textLen;                  /* length in XML_Chars */
-  int processed;                /* # of processed bytes - when suspended */
+  int textLen;   /* length in XML_Chars */
+  int processed; /* # of processed bytes - when suspended */
   const XML_Char *systemId;
   const XML_Char *base;
   const XML_Char *publicId;
@@ -277,13 +277,13 @@ typedef struct {
 } ENTITY;
 
 typedef struct {
-  enum XML_Content_Type         type;
-  enum XML_Content_Quant        quant;
-  const XML_Char *              name;
-  int                           firstchild;
-  int                           lastchild;
-  int                           childcnt;
-  int                           nextsib;
+  enum XML_Content_Type type;
+  enum XML_Content_Quant quant;
+  const XML_Char *name;
+  int firstchild;
+  int lastchild;
+  int childcnt;
+  int nextsib;
 } CONTENT_SCAFFOLD;
 
 #define INIT_SCAFFOLD_ELEMENTS 32
@@ -371,10 +371,8 @@ typedef struct open_internal_entity {
   XML_Bool betweenDecl; /* WFC: PE Between Declarations */
 } OPEN_INTERNAL_ENTITY;
 
-typedef enum XML_Error PTRCALL Processor(XML_Parser parser,
-                                         const char *start,
-                                         const char *end,
-                                         const char **endPtr);
+typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start,
+                                         const char *end, const char **endPtr);
 
 static Processor prologProcessor;
 static Processor prologInitProcessor;
@@ -395,118 +393,101 @@ static Processor externalEntityInitProcessor3;
 static Processor externalEntityContentProcessor;
 static Processor internalEntityProcessor;
 
-static enum XML_Error
-handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName);
-static enum XML_Error
-processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
-               const char *s, const char *next);
-static enum XML_Error
-initializeEncoding(XML_Parser parser);
-static enum XML_Error
-doProlog(XML_Parser parser, const ENCODING *enc, const char *s,
-         const char *end, int tok, const char *next, const char **nextPtr,
-         XML_Bool haveMore);
-static enum XML_Error
-processInternalEntity(XML_Parser parser, ENTITY *entity,
-                      XML_Bool betweenDecl);
-static enum XML_Error
-doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
-          const char *start, const char *end, const char **endPtr,
-          XML_Bool haveMore);
-static enum XML_Error
-doCdataSection(XML_Parser parser, const ENCODING *, const char **startPtr,
-               const char *end, const char **nextPtr, XML_Bool haveMore);
+static enum XML_Error handleUnknownEncoding(XML_Parser parser,
+                                            const XML_Char *encodingName);
+static enum XML_Error processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
+                                     const char *s, const char *next);
+static enum XML_Error initializeEncoding(XML_Parser parser);
+static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc,
+                               const char *s, const char *end, int tok,
+                               const char *next, const char **nextPtr,
+                               XML_Bool haveMore, XML_Bool allowClosingDoctype);
+static enum XML_Error processInternalEntity(XML_Parser parser, ENTITY *entity,
+                                            XML_Bool betweenDecl);
+static enum XML_Error doContent(XML_Parser parser, int startTagLevel,
+                                const ENCODING *enc, const char *start,
+                                const char *end, const char **endPtr,
+                                XML_Bool haveMore);
+static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *,
+                                     const char **startPtr, const char *end,
+                                     const char **nextPtr, XML_Bool haveMore);
 #ifdef XML_DTD
-static enum XML_Error
-doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr,
-                const char *end, const char **nextPtr, XML_Bool haveMore);
+static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *,
+                                      const char **startPtr, const char *end,
+                                      const char **nextPtr, XML_Bool haveMore);
 #endif /* XML_DTD */
 
-static void
-freeBindings(XML_Parser parser, BINDING *bindings);
-static enum XML_Error
-storeAtts(XML_Parser parser, const ENCODING *, const char *s,
-          TAG_NAME *tagNamePtr, BINDING **bindingsPtr);
-static enum XML_Error
-addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
-           const XML_Char *uri, BINDING **bindingsPtr);
-static int
-defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata,
-                XML_Bool isId, const XML_Char *dfltValue, XML_Parser parser);
-static enum XML_Error
-storeAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata,
-                    const char *, const char *, STRING_POOL *);
-static enum XML_Error
-appendAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata,
-                     const char *, const char *, STRING_POOL *);
-static ATTRIBUTE_ID *
-getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start,
-               const char *end);
-static int
-setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *);
-static enum XML_Error
-storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *start,
-                 const char *end);
-static int
-reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
-                            const char *start, const char *end);
-static int
-reportComment(XML_Parser parser, const ENCODING *enc, const char *start,
-              const char *end);
-static void
-reportDefault(XML_Parser parser, const ENCODING *enc, const char *start,
-              const char *end);
-
-static const XML_Char * getContext(XML_Parser parser);
-static XML_Bool
-setContext(XML_Parser parser, const XML_Char *context);
+static void freeBindings(XML_Parser parser, BINDING *bindings);
+static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *,
+                                const char *s, TAG_NAME *tagNamePtr,
+                                BINDING **bindingsPtr);
+static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix,
+                                 const ATTRIBUTE_ID *attId, const XML_Char *uri,
+                                 BINDING **bindingsPtr);
+static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata,
+                           XML_Bool isId, const XML_Char *dfltValue,
+                           XML_Parser parser);
+static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *,
+                                          XML_Bool isCdata, const char *,
+                                          const char *, STRING_POOL *);
+static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *,
+                                           XML_Bool isCdata, const char *,
+                                           const char *, STRING_POOL *);
+static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc,
+                                    const char *start, const char *end);
+static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *);
+static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc,
+                                       const char *start, const char *end);
+static int reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
+                                       const char *start, const char *end);
+static int reportComment(XML_Parser parser, const ENCODING *enc,
+                         const char *start, const char *end);
+static void reportDefault(XML_Parser parser, const ENCODING *enc,
+                          const char *start, const char *end);
+
+static const XML_Char *getContext(XML_Parser parser);
+static XML_Bool setContext(XML_Parser parser, const XML_Char *context);
 
 static void FASTCALL normalizePublicId(XML_Char *s);
 
-static DTD * dtdCreate(const XML_Memory_Handling_Suite *ms);
+static DTD *dtdCreate(const XML_Memory_Handling_Suite *ms);
 /* do not call if m_parentParser != NULL */
 static void dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms);
-static void
-dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms);
-static int
-dtdCopy(XML_Parser oldParser,
-        DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms);
-static int
-copyEntityTable(XML_Parser oldParser,
-                HASH_TABLE *, STRING_POOL *, const HASH_TABLE *);
-static NAMED *
-lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize);
-static void FASTCALL
-hashTableInit(HASH_TABLE *, const XML_Memory_Handling_Suite *ms);
+static void dtdDestroy(DTD *p, XML_Bool isDocEntity,
+                       const XML_Memory_Handling_Suite *ms);
+static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
+                   const XML_Memory_Handling_Suite *ms);
+static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *, STRING_POOL *,
+                           const HASH_TABLE *);
+static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name,
+                     size_t createSize);
+static void FASTCALL hashTableInit(HASH_TABLE *,
+                                   const XML_Memory_Handling_Suite *ms);
 static void FASTCALL hashTableClear(HASH_TABLE *);
 static void FASTCALL hashTableDestroy(HASH_TABLE *);
-static void FASTCALL
-hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *);
-static NAMED * FASTCALL hashTableIterNext(HASH_TABLE_ITER *);
+static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *);
+static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *);
 
-static void FASTCALL
-poolInit(STRING_POOL *, const XML_Memory_Handling_Suite *ms);
+static void FASTCALL poolInit(STRING_POOL *,
+                              const XML_Memory_Handling_Suite *ms);
 static void FASTCALL poolClear(STRING_POOL *);
 static void FASTCALL poolDestroy(STRING_POOL *);
-static XML_Char *
-poolAppend(STRING_POOL *pool, const ENCODING *enc,
-           const char *ptr, const char *end);
-static XML_Char *
-poolStoreString(STRING_POOL *pool, const ENCODING *enc,
-                const char *ptr, const char *end);
+static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc,
+                            const char *ptr, const char *end);
+static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc,
+                                 const char *ptr, const char *end);
 static XML_Bool FASTCALL poolGrow(STRING_POOL *pool);
-static const XML_Char * FASTCALL
-poolCopyString(STRING_POOL *pool, const XML_Char *s);
-static const XML_Char *
-poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n);
-static const XML_Char * FASTCALL
-poolAppendString(STRING_POOL *pool, const XML_Char *s);
+static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool,
+                                               const XML_Char *s);
+static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s,
+                                       int n);
+static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool,
+                                                 const XML_Char *s);
 
 static int FASTCALL nextScaffoldPart(XML_Parser parser);
-static XML_Content * build_model(XML_Parser parser);
-static ELEMENT_TYPE *
-getElementType(XML_Parser parser, const ENCODING *enc,
-               const char *ptr, const char *end);
+static XML_Content *build_model(XML_Parser parser);
+static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc,
+                                    const char *ptr, const char *end);
 
 static XML_Char *copyString(const XML_Char *s,
                             const XML_Memory_Handling_Suite *memsuite);
@@ -514,14 +495,11 @@ static XML_Char *copyString(const XML_Char *s,
 static unsigned long generate_hash_secret_salt(XML_Parser parser);
 static XML_Bool startParsing(XML_Parser parser);
 
-static XML_Parser
-parserCreate(const XML_Char *encodingName,
-             const XML_Memory_Handling_Suite *memsuite,
-             const XML_Char *nameSep,
-             DTD *dtd);
+static XML_Parser parserCreate(const XML_Char *encodingName,
+                               const XML_Memory_Handling_Suite *memsuite,
+                               const XML_Char *nameSep, DTD *dtd);
 
-static void
-parserInit(XML_Parser parser, const XML_Char *encodingName);
+static void parserInit(XML_Parser parser, const XML_Char *encodingName);
 
 #define poolStart(pool) ((pool)->start)
 #define poolEnd(pool) ((pool)->ptr)
@@ -530,10 +508,10 @@ parserInit(XML_Parser parser, const XML_Char *encodingName);
 #define poolLastChar(pool) (((pool)->ptr)[-1])
 #define poolDiscard(pool) ((pool)->ptr = (pool)->start)
 #define poolFinish(pool) ((pool)->start = (pool)->ptr)
-#define poolAppendChar(pool, c) \
-  (((pool)->ptr == (pool)->end && !poolGrow(pool)) \
-   ? 0 \
-   : ((*((pool)->ptr)++ = c), 1))
+#define poolAppendChar(pool, c)                                                \
+  (((pool)->ptr == (pool)->end && ! poolGrow(pool))                            \
+       ? 0                                                                     \
+       : ((*((pool)->ptr)++ = c), 1))
 
 struct XML_ParserStruct {
   /* The first member must be m_userData so that the XML_GetUserData
@@ -584,7 +562,7 @@ struct XML_ParserStruct {
   void *m_unknownEncodingMem;
   void *m_unknownEncodingData;
   void *m_unknownEncodingHandlerData;
-  void (XMLCALL *m_unknownEncodingRelease)(void *);
+  void(XMLCALL *m_unknownEncodingRelease)(void *);
   PROLOG_STATE m_prologState;
   Processor *m_processor;
   enum XML_Error m_errorCode;
@@ -638,57 +616,55 @@ struct XML_ParserStruct {
   unsigned long m_hash_secret_salt;
 };
 
-#define MALLOC(parser, s)      (parser->m_mem.malloc_fcn((s)))
-#define REALLOC(parser, p, s)  (parser->m_mem.realloc_fcn((p),(s)))
-#define FREE(parser, p)        (parser->m_mem.free_fcn((p)))
-
+#define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s)))
+#define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s)))
+#define FREE(parser, p) (parser->m_mem.free_fcn((p)))
 
 XML_Parser XMLCALL
-XML_ParserCreate(const XML_Char *encodingName)
-{
+XML_ParserCreate(const XML_Char *encodingName) {
   return XML_ParserCreate_MM(encodingName, NULL, NULL);
 }
 
 XML_Parser XMLCALL
-XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep)
-{
+XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) {
   XML_Char tmp[2];
   *tmp = nsSep;
   return XML_ParserCreate_MM(encodingName, NULL, tmp);
 }
 
-static const XML_Char implicitContext[] = {
-  ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h, ASCII_t, ASCII_t, ASCII_p,
-  ASCII_COLON, ASCII_SLASH, ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w,
-  ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g,
-  ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L, ASCII_SLASH, ASCII_1, ASCII_9,
-  ASCII_9, ASCII_8, ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e,
-  ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'
-};
-
+static const XML_Char implicitContext[]
+    = {ASCII_x,     ASCII_m,     ASCII_l,      ASCII_EQUALS, ASCII_h,
+       ASCII_t,     ASCII_t,     ASCII_p,      ASCII_COLON,  ASCII_SLASH,
+       ASCII_SLASH, ASCII_w,     ASCII_w,      ASCII_w,      ASCII_PERIOD,
+       ASCII_w,     ASCII_3,     ASCII_PERIOD, ASCII_o,      ASCII_r,
+       ASCII_g,     ASCII_SLASH, ASCII_X,      ASCII_M,      ASCII_L,
+       ASCII_SLASH, ASCII_1,     ASCII_9,      ASCII_9,      ASCII_8,
+       ASCII_SLASH, ASCII_n,     ASCII_a,      ASCII_m,      ASCII_e,
+       ASCII_s,     ASCII_p,     ASCII_a,      ASCII_c,      ASCII_e,
+       '\0'};
 
 /* To avoid warnings about unused functions: */
 #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)
 
-#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
+#  if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
 
 /* Obtain entropy on Linux 3.17+ */
 static int
-writeRandomBytes_getrandom_nonblock(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
+writeRandomBytes_getrandom_nonblock(void *target, size_t count) {
+  int success = 0; /* full count bytes written? */
   size_t bytesWrittenTotal = 0;
   const unsigned int getrandomFlags = GRND_NONBLOCK;
 
   do {
-    void * const currentTarget = (void*)((char*)target + bytesWrittenTotal);
+    void *const currentTarget = (void *)((char *)target + bytesWrittenTotal);
     const size_t bytesToWrite = count - bytesWrittenTotal;
 
     const int bytesWrittenMore =
-#if defined(HAVE_GETRANDOM)
+#    if defined(HAVE_GETRANDOM)
         getrandom(currentTarget, bytesToWrite, getrandomFlags);
-#else
+#    else
         syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags);
-#endif
+#    endif
 
     if (bytesWrittenMore > 0) {
       bytesWrittenTotal += bytesWrittenMore;
@@ -700,15 +676,14 @@ writeRandomBytes_getrandom_nonblock(void * target, size_t count) {
   return success;
 }
 
-#endif  /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
-
+#  endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */
 
-#if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
+#  if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
 
 /* Extract entropy from /dev/urandom */
 static int
-writeRandomBytes_dev_urandom(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
+writeRandomBytes_dev_urandom(void *target, size_t count) {
+  int success = 0; /* full count bytes written? */
   size_t bytesWrittenTotal = 0;
 
   const int fd = open("/dev/urandom", O_RDONLY);
@@ -717,7 +692,7 @@ writeRandomBytes_dev_urandom(void * target, size_t count) {
   }
 
   do {
-    void * const currentTarget = (void*)((char*)target + bytesWrittenTotal);
+    void *const currentTarget = (void *)((char *)target + bytesWrittenTotal);
     const size_t bytesToWrite = count - bytesWrittenTotal;
 
     const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite);
@@ -733,15 +708,14 @@ writeRandomBytes_dev_urandom(void * target, size_t count) {
   return success;
 }
 
-#endif  /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
-
-#endif  /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
+#  endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
 
+#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
 
 #if defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF)
 
 static void
-writeRandomBytes_arc4random(void * target, size_t count) {
+writeRandomBytes_arc4random(void *target, size_t count) {
   size_t bytesWrittenTotal = 0;
 
   while (bytesWrittenTotal < count) {
@@ -749,93 +723,82 @@ writeRandomBytes_arc4random(void * target, size_t count) {
     size_t i = 0;
 
     for (; (i < sizeof(random32)) && (bytesWrittenTotal < count);
-        i++, bytesWrittenTotal++) {
+         i++, bytesWrittenTotal++) {
       const uint8_t random8 = (uint8_t)(random32 >> (i * 8));
       ((uint8_t *)target)[bytesWrittenTotal] = random8;
     }
   }
 }
 
-#endif  /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */
-
+#endif /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */
 
 #ifdef _WIN32
 
-typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG);
-HMODULE _Expat_LoadLibrary(LPCTSTR filename);  /* see loadlibrary.c */
-
-/* Obtain entropy on Windows XP / Windows Server 2003 and later.
- * Hint on RtlGenRandom and the following article from libsodium.
- *
- * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI
- * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographically-secure-random-number-on-windows-without-using-cryptoapi/
+/* Obtain entropy on Windows using the rand_s() function which
+ * generates cryptographically secure random numbers.  Internally it
+ * uses RtlGenRandom API which is present in Windows XP and later.
  */
 static int
-writeRandomBytes_RtlGenRandom(void * target, size_t count) {
-  int success = 0;  /* full count bytes written? */
-  const HMODULE advapi32 = _Expat_LoadLibrary(TEXT("ADVAPI32.DLL"));
-
-  if (advapi32) {
-    const RTLGENRANDOM_FUNC RtlGenRandom
-        = (RTLGENRANDOM_FUNC)GetProcAddress(advapi32, "SystemFunction036");
-    if (RtlGenRandom) {
-      if (RtlGenRandom((PVOID)target, (ULONG)count) == TRUE) {
-        success = 1;
-      }
+writeRandomBytes_rand_s(void *target, size_t count) {
+  size_t bytesWrittenTotal = 0;
+
+  while (bytesWrittenTotal < count) {
+    unsigned int random32 = 0;
+    size_t i = 0;
+
+    if (rand_s(&random32))
+      return 0; /* failure */
+
+    for (; (i < sizeof(random32)) && (bytesWrittenTotal < count);
+         i++, bytesWrittenTotal++) {
+      const uint8_t random8 = (uint8_t)(random32 >> (i * 8));
+      ((uint8_t *)target)[bytesWrittenTotal] = random8;
     }
-    FreeLibrary(advapi32);
   }
-
-  return success;
+  return 1; /* success */
 }
 
 #endif /* _WIN32 */
 
-
 #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM)
 
 static unsigned long
-gather_time_entropy(void)
-{
-#ifdef _WIN32
+gather_time_entropy(void) {
+#  ifdef _WIN32
   FILETIME ft;
   GetSystemTimeAsFileTime(&ft); /* never fails */
   return ft.dwHighDateTime ^ ft.dwLowDateTime;
-#else
+#  else
   struct timeval tv;
   int gettimeofday_res;
 
   gettimeofday_res = gettimeofday(&tv, NULL);
 
-#if defined(NDEBUG)
+#    if defined(NDEBUG)
   (void)gettimeofday_res;
-#else
-  assert (gettimeofday_res == 0);
-#endif  /* defined(NDEBUG) */
+#    else
+  assert(gettimeofday_res == 0);
+#    endif /* defined(NDEBUG) */
 
   /* Microseconds time is <20 bits entropy */
   return tv.tv_usec;
-#endif
+#  endif
 }
 
-#endif  /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
-
+#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */
 
 static unsigned long
-ENTROPY_DEBUG(const char * label, unsigned long entropy) {
-  const char * const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG");
+ENTROPY_DEBUG(const char *label, unsigned long entropy) {
+  const char *const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG");
   if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) {
-    fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n",
-        label,
-        (int)sizeof(entropy) * 2, entropy,
-        (unsigned long)sizeof(entropy));
+    fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", label,
+            (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy));
   }
   return entropy;
 }
 
 static unsigned long
-generate_hash_secret_salt(XML_Parser parser)
-{
+generate_hash_secret_salt(XML_Parser parser) {
   unsigned long entropy;
   (void)parser;
 
@@ -848,20 +811,20 @@ generate_hash_secret_salt(XML_Parser parser)
   return ENTROPY_DEBUG("arc4random", entropy);
 #else
   /* Try high quality providers first .. */
-#ifdef _WIN32
-  if (writeRandomBytes_RtlGenRandom((void *)&entropy, sizeof(entropy))) {
-    return ENTROPY_DEBUG("RtlGenRandom", entropy);
+#  ifdef _WIN32
+  if (writeRandomBytes_rand_s((void *)&entropy, sizeof(entropy))) {
+    return ENTROPY_DEBUG("rand_s", entropy);
   }
-#elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
+#  elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)
   if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) {
     return ENTROPY_DEBUG("getrandom", entropy);
   }
-#endif
-#if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
+#  endif
+#  if ! defined(_WIN32) && defined(XML_DEV_URANDOM)
   if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) {
     return ENTROPY_DEBUG("/dev/urandom", entropy);
   }
-#endif  /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
+#  endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */
   /* .. and self-made low quality for backup: */
 
   /* Process ID is 0 bits entropy if attacker has local access */
@@ -872,7 +835,7 @@ generate_hash_secret_salt(XML_Parser parser)
     return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647);
   } else {
     return ENTROPY_DEBUG("fallback(8)",
-        entropy * (unsigned long)2305843009213693951ULL);
+                         entropy * (unsigned long)2305843009213693951ULL);
   }
 #endif
 }
@@ -884,49 +847,43 @@ get_hash_secret_salt(XML_Parser parser) {
   return parser->m_hash_secret_salt;
 }
 
-static XML_Bool  /* only valid for root parser */
-startParsing(XML_Parser parser)
-{
-    /* hash functions must be initialized before setContext() is called */
-    if (parser->m_hash_secret_salt == 0)
-      parser->m_hash_secret_salt = generate_hash_secret_salt(parser);
-    if (parser->m_ns) {
-      /* implicit context only set for root parser, since child
-         parsers (i.e. external entity parsers) will inherit it
-      */
-      return setContext(parser, implicitContext);
-    }
-    return XML_TRUE;
+static XML_Bool /* only valid for root parser */
+startParsing(XML_Parser parser) {
+  /* hash functions must be initialized before setContext() is called */
+  if (parser->m_hash_secret_salt == 0)
+    parser->m_hash_secret_salt = generate_hash_secret_salt(parser);
+  if (parser->m_ns) {
+    /* implicit context only set for root parser, since child
+       parsers (i.e. external entity parsers) will inherit it
+    */
+    return setContext(parser, implicitContext);
+  }
+  return XML_TRUE;
 }
 
 XML_Parser XMLCALL
 XML_ParserCreate_MM(const XML_Char *encodingName,
                     const XML_Memory_Handling_Suite *memsuite,
-                    const XML_Char *nameSep)
-{
+                    const XML_Char *nameSep) {
   return parserCreate(encodingName, memsuite, nameSep, NULL);
 }
 
 static XML_Parser
 parserCreate(const XML_Char *encodingName,
-             const XML_Memory_Handling_Suite *memsuite,
-             const XML_Char *nameSep,
-             DTD *dtd)
-{
+             const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep,
+             DTD *dtd) {
   XML_Parser parser;
 
   if (memsuite) {
     XML_Memory_Handling_Suite *mtemp;
-    parser = (XML_Parser)
-      memsuite->malloc_fcn(sizeof(struct XML_ParserStruct));
+    parser = (XML_Parser)memsuite->malloc_fcn(sizeof(struct XML_ParserStruct));
     if (parser != NULL) {
       mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem);
       mtemp->malloc_fcn = memsuite->malloc_fcn;
       mtemp->realloc_fcn = memsuite->realloc_fcn;
       mtemp->free_fcn = memsuite->free_fcn;
     }
-  }
-  else {
+  } else {
     XML_Memory_Handling_Suite *mtemp;
     parser = (XML_Parser)malloc(sizeof(struct XML_ParserStruct));
     if (parser != NULL) {
@@ -937,27 +894,30 @@ parserCreate(const XML_Char *encodingName,
     }
   }
 
-  if (!parser)
+  if (! parser)
     return parser;
 
   parser->m_buffer = NULL;
   parser->m_bufferLim = NULL;
 
   parser->m_attsSize = INIT_ATTS_SIZE;
-  parser->m_atts = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE));
+  parser->m_atts
+      = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE));
   if (parser->m_atts == NULL) {
     FREE(parser, parser);
     return NULL;
   }
 #ifdef XML_ATTR_INFO
-  parser->m_attInfo = (XML_AttrInfo*)MALLOC(parser, parser->m_attsSize * sizeof(XML_AttrInfo));
+  parser->m_attInfo = (XML_AttrInfo *)MALLOC(
+      parser, parser->m_attsSize * sizeof(XML_AttrInfo));
   if (parser->m_attInfo == NULL) {
     FREE(parser, parser->m_atts);
     FREE(parser, parser);
     return NULL;
   }
 #endif
-  parser->m_dataBuf = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char));
+  parser->m_dataBuf
+      = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char));
   if (parser->m_dataBuf == NULL) {
     FREE(parser, parser->m_atts);
 #ifdef XML_ATTR_INFO
@@ -1007,7 +967,7 @@ parserCreate(const XML_Char *encodingName,
   poolInit(&parser->m_temp2Pool, &(parser->m_mem));
   parserInit(parser, encodingName);
 
-  if (encodingName && !parser->m_protocolEncodingName) {
+  if (encodingName && ! parser->m_protocolEncodingName) {
     XML_ParserFree(parser);
     return NULL;
   }
@@ -1016,8 +976,7 @@ parserCreate(const XML_Char *encodingName,
     parser->m_ns = XML_TRUE;
     parser->m_internalEncoding = XmlGetInternalEncodingNS();
     parser->m_namespaceSeparator = *nameSep;
-  }
-  else {
+  } else {
     parser->m_internalEncoding = XmlGetInternalEncoding();
   }
 
@@ -1025,8 +984,7 @@ parserCreate(const XML_Char *encodingName,
 }
 
 static void
-parserInit(XML_Parser parser, const XML_Char *encodingName)
-{
+parserInit(XML_Parser parser, const XML_Char *encodingName) {
   parser->m_processor = prologInitProcessor;
   XmlPrologStateInit(&parser->m_prologState);
   if (encodingName != NULL) {
@@ -1099,8 +1057,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName)
 
 /* moves list of bindings to m_freeBindingList */
 static void FASTCALL
-moveToFreeBindingList(XML_Parser parser, BINDING *bindings)
-{
+moveToFreeBindingList(XML_Parser parser, BINDING *bindings) {
   while (bindings) {
     BINDING *b = bindings;
     bindings = bindings->nextTagBinding;
@@ -1110,13 +1067,12 @@ moveToFreeBindingList(XML_Parser parser, BINDING *bindings)
 }
 
 XML_Bool XMLCALL
-XML_ParserReset(XML_Parser parser, const XML_Char *encodingName)
-{
+XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) {
   TAG *tStk;
   OPEN_INTERNAL_ENTITY *openEntityList;
 
   if (parser == NULL)
-      return XML_FALSE;
+    return XML_FALSE;
 
   if (parser->m_parentParser)
     return XML_FALSE;
@@ -1152,15 +1108,15 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName)
 }
 
 enum XML_Status XMLCALL
-XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName)
-{
+XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) {
   if (parser == NULL)
-      return XML_STATUS_ERROR;
+    return XML_STATUS_ERROR;
   /* Block after XML_Parse()/XML_ParseBuffer() has been called.
      XXX There's no way for the caller to determine which of the
      XXX possible error cases caused the XML_STATUS_ERROR return.
   */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return XML_STATUS_ERROR;
 
   /* Get rid of any previous encoding name */
@@ -1172,17 +1128,15 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName)
   else {
     /* Copy the new encoding name into allocated memory */
     parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem));
-    if (!parser->m_protocolEncodingName)
+    if (! parser->m_protocolEncodingName)
       return XML_STATUS_ERROR;
   }
   return XML_STATUS_OK;
 }
 
 XML_Parser XMLCALL
-XML_ExternalEntityParserCreate(XML_Parser oldParser,
-                               const XML_Char *context,
-                               const XML_Char *encodingName)
-{
+XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context,
+                               const XML_Char *encodingName) {
   XML_Parser parser = oldParser;
   DTD *newDtd = NULL;
   DTD *oldDtd;
@@ -1206,7 +1160,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   XML_AttlistDeclHandler oldAttlistDeclHandler;
   XML_EntityDeclHandler oldEntityDeclHandler;
   XML_XmlDeclHandler oldXmlDeclHandler;
-  ELEMENT_TYPE * oldDeclElementType;
+  ELEMENT_TYPE *oldDeclElementType;
 
   void *oldUserData;
   void *oldHandlerArg;
@@ -1269,7 +1223,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   oldhash_secret_salt = parser->m_hash_secret_salt;
 
 #ifdef XML_DTD
-  if (!context)
+  if (! context)
     newDtd = oldDtd;
 #endif /* XML_DTD */
 
@@ -1282,12 +1236,11 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
     XML_Char tmp[2];
     *tmp = parser->m_namespaceSeparator;
     parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd);
-  }
-  else {
+  } else {
     parser = parserCreate(encodingName, &parser->m_mem, NULL, newDtd);
   }
 
-  if (!parser)
+  if (! parser)
     return NULL;
 
   parser->m_startElementHandler = oldStartElementHandler;
@@ -1327,21 +1280,20 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
   parser->m_prologState.inEntityValue = oldInEntityValue;
   if (context) {
 #endif /* XML_DTD */
-    if (!dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem)
-      || !setContext(parser, context)) {
+    if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem)
+        || ! setContext(parser, context)) {
       XML_ParserFree(parser);
       return NULL;
     }
     parser->m_processor = externalEntityInitProcessor;
 #ifdef XML_DTD
-  }
-  else {
-    /* The DTD instance referenced by parser->m_dtd is shared between the document's
-       root parser and external PE parsers, therefore one does not need to
-       call setContext. In addition, one also *must* not call setContext,
-       because this would overwrite existing prefix->binding pointers in
-       parser->m_dtd with ones that get destroyed with the external PE parser.
-       This would leave those prefixes with dangling pointers.
+  } else {
+    /* The DTD instance referenced by parser->m_dtd is shared between the
+       document's root parser and external PE parsers, therefore one does not
+       need to call setContext. In addition, one also *must* not call
+       setContext, because this would overwrite existing prefix->binding
+       pointers in parser->m_dtd with ones that get destroyed with the external
+       PE parser. This would leave those prefixes with dangling pointers.
     */
     parser->m_isParamEntity = XML_TRUE;
     XmlPrologStateInitExternalEntity(&parser->m_prologState);
@@ -1352,11 +1304,10 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser,
 }
 
 static void FASTCALL
-destroyBindings(BINDING *bindings, XML_Parser parser)
-{
+destroyBindings(BINDING *bindings, XML_Parser parser) {
   for (;;) {
     BINDING *b = bindings;
-    if (!b)
+    if (! b)
       break;
     bindings = b->nextTagBinding;
     FREE(parser, b->uri);
@@ -1365,8 +1316,7 @@ destroyBindings(BINDING *bindings, XML_Parser parser)
 }
 
 void XMLCALL
-XML_ParserFree(XML_Parser parser)
-{
+XML_ParserFree(XML_Parser parser) {
   TAG *tagList;
   OPEN_INTERNAL_ENTITY *entityList;
   if (parser == NULL)
@@ -1411,11 +1361,12 @@ XML_ParserFree(XML_Parser parser)
   /* external parameter entity parsers share the DTD structure
      parser->m_dtd with the root parser, so we must not destroy it
   */
-  if (!parser->m_isParamEntity && parser->m_dtd)
+  if (! parser->m_isParamEntity && parser->m_dtd)
 #else
   if (parser->m_dtd)
 #endif /* XML_DTD */
-    dtdDestroy(parser->m_dtd, (XML_Bool)!parser->m_parentParser, &parser->m_mem);
+    dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser,
+               &parser->m_mem);
   FREE(parser, (void *)parser->m_atts);
 #ifdef XML_ATTR_INFO
   FREE(parser, (void *)parser->m_attInfo);
@@ -1431,20 +1382,19 @@ XML_ParserFree(XML_Parser parser)
 }
 
 void XMLCALL
-XML_UseParserAsHandlerArg(XML_Parser parser)
-{
+XML_UseParserAsHandlerArg(XML_Parser parser) {
   if (parser != NULL)
     parser->m_handlerArg = parser;
 }
 
 enum XML_Error XMLCALL
-XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD)
-{
+XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) {
   if (parser == NULL)
     return XML_ERROR_INVALID_ARGUMENT;
 #ifdef XML_DTD
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING;
   parser->m_useForeignDTD = useDTD;
   return XML_ERROR_NONE;
@@ -1454,19 +1404,18 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD)
 }
 
 void XMLCALL
-XML_SetReturnNSTriplet(XML_Parser parser, int do_nst)
-{
+XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) {
   if (parser == NULL)
     return;
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return;
   parser->m_ns_triplets = do_nst ? XML_TRUE : XML_FALSE;
 }
 
 void XMLCALL
-XML_SetUserData(XML_Parser parser, void *p)
-{
+XML_SetUserData(XML_Parser parser, void *p) {
   if (parser == NULL)
     return;
   if (parser->m_handlerArg == parser->m_userData)
@@ -1476,49 +1425,43 @@ XML_SetUserData(XML_Parser parser, void *p)
 }
 
 enum XML_Status XMLCALL
-XML_SetBase(XML_Parser parser, const XML_Char *p)
-{
+XML_SetBase(XML_Parser parser, const XML_Char *p) {
   if (parser == NULL)
     return XML_STATUS_ERROR;
   if (p) {
     p = poolCopyString(&parser->m_dtd->pool, p);
-    if (!p)
+    if (! p)
       return XML_STATUS_ERROR;
     parser->m_curBase = p;
-  }
-  else
+  } else
     parser->m_curBase = NULL;
   return XML_STATUS_OK;
 }
 
-const XML_Char * XMLCALL
-XML_GetBase(XML_Parser parser)
-{
+const XML_Char *XMLCALL
+XML_GetBase(XML_Parser parser) {
   if (parser == NULL)
     return NULL;
   return parser->m_curBase;
 }
 
 int XMLCALL
-XML_GetSpecifiedAttributeCount(XML_Parser parser)
-{
+XML_GetSpecifiedAttributeCount(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   return parser->m_nSpecifiedAtts;
 }
 
 int XMLCALL
-XML_GetIdAttributeIndex(XML_Parser parser)
-{
+XML_GetIdAttributeIndex(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   return parser->m_idAttIndex;
 }
 
 #ifdef XML_ATTR_INFO
-const XML_AttrInfo * XMLCALL
-XML_GetAttributeInfo(XML_Parser parser)
-{
+const XML_AttrInfo *XMLCALL
+XML_GetAttributeInfo(XML_Parser parser) {
   if (parser == NULL)
     return NULL;
   return parser->m_attInfo;
@@ -1526,10 +1469,8 @@ XML_GetAttributeInfo(XML_Parser parser)
 #endif
 
 void XMLCALL
-XML_SetElementHandler(XML_Parser parser,
-                      XML_StartElementHandler start,
-                      XML_EndElementHandler end)
-{
+XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start,
+                      XML_EndElementHandler end) {
   if (parser == NULL)
     return;
   parser->m_startElementHandler = start;
@@ -1537,39 +1478,33 @@ XML_SetElementHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetStartElementHandler(XML_Parser parser,
-                           XML_StartElementHandler start) {
+XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler start) {
   if (parser != NULL)
     parser->m_startElementHandler = start;
 }
 
 void XMLCALL
-XML_SetEndElementHandler(XML_Parser parser,
-                         XML_EndElementHandler end) {
+XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler end) {
   if (parser != NULL)
     parser->m_endElementHandler = end;
 }
 
 void XMLCALL
 XML_SetCharacterDataHandler(XML_Parser parser,
-                            XML_CharacterDataHandler handler)
-{
+                            XML_CharacterDataHandler handler) {
   if (parser != NULL)
     parser->m_characterDataHandler = handler;
 }
 
 void XMLCALL
 XML_SetProcessingInstructionHandler(XML_Parser parser,
-                                    XML_ProcessingInstructionHandler handler)
-{
+                                    XML_ProcessingInstructionHandler handler) {
   if (parser != NULL)
     parser->m_processingInstructionHandler = handler;
 }
 
 void XMLCALL
-XML_SetCommentHandler(XML_Parser parser,
-                      XML_CommentHandler handler)
-{
+XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler) {
   if (parser != NULL)
     parser->m_commentHandler = handler;
 }
@@ -1577,8 +1512,7 @@ XML_SetCommentHandler(XML_Parser parser,
 void XMLCALL
 XML_SetCdataSectionHandler(XML_Parser parser,
                            XML_StartCdataSectionHandler start,
-                           XML_EndCdataSectionHandler end)
-{
+                           XML_EndCdataSectionHandler end) {
   if (parser == NULL)
     return;
   parser->m_startCdataSectionHandler = start;
@@ -1600,9 +1534,7 @@ XML_SetEndCdataSectionHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDefaultHandler(XML_Parser parser,
-                      XML_DefaultHandler handler)
-{
+XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler) {
   if (parser == NULL)
     return;
   parser->m_defaultHandler = handler;
@@ -1610,9 +1542,7 @@ XML_SetDefaultHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDefaultHandlerExpand(XML_Parser parser,
-                            XML_DefaultHandler handler)
-{
+XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler) {
   if (parser == NULL)
     return;
   parser->m_defaultHandler = handler;
@@ -1620,10 +1550,8 @@ XML_SetDefaultHandlerExpand(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetDoctypeDeclHandler(XML_Parser parser,
-                          XML_StartDoctypeDeclHandler start,
-                          XML_EndDoctypeDeclHandler end)
-{
+XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start,
+                          XML_EndDoctypeDeclHandler end) {
   if (parser == NULL)
     return;
   parser->m_startDoctypeDeclHandler = start;
@@ -1638,24 +1566,20 @@ XML_SetStartDoctypeDeclHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetEndDoctypeDeclHandler(XML_Parser parser,
-                             XML_EndDoctypeDeclHandler end) {
+XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end) {
   if (parser != NULL)
     parser->m_endDoctypeDeclHandler = end;
 }
 
 void XMLCALL
 XML_SetUnparsedEntityDeclHandler(XML_Parser parser,
-                                 XML_UnparsedEntityDeclHandler handler)
-{
+                                 XML_UnparsedEntityDeclHandler handler) {
   if (parser != NULL)
     parser->m_unparsedEntityDeclHandler = handler;
 }
 
 void XMLCALL
-XML_SetNotationDeclHandler(XML_Parser parser,
-                           XML_NotationDeclHandler handler)
-{
+XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler) {
   if (parser != NULL)
     parser->m_notationDeclHandler = handler;
 }
@@ -1663,8 +1587,7 @@ XML_SetNotationDeclHandler(XML_Parser parser,
 void XMLCALL
 XML_SetNamespaceDeclHandler(XML_Parser parser,
                             XML_StartNamespaceDeclHandler start,
-                            XML_EndNamespaceDeclHandler end)
-{
+                            XML_EndNamespaceDeclHandler end) {
   if (parser == NULL)
     return;
   parser->m_startNamespaceDeclHandler = start;
@@ -1687,23 +1610,20 @@ XML_SetEndNamespaceDeclHandler(XML_Parser parser,
 
 void XMLCALL
 XML_SetNotStandaloneHandler(XML_Parser parser,
-                            XML_NotStandaloneHandler handler)
-{
+                            XML_NotStandaloneHandler handler) {
   if (parser != NULL)
     parser->m_notStandaloneHandler = handler;
 }
 
 void XMLCALL
 XML_SetExternalEntityRefHandler(XML_Parser parser,
-                                XML_ExternalEntityRefHandler handler)
-{
+                                XML_ExternalEntityRefHandler handler) {
   if (parser != NULL)
     parser->m_externalEntityRefHandler = handler;
 }
 
 void XMLCALL
-XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg)
-{
+XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg) {
   if (parser == NULL)
     return;
   if (arg)
@@ -1714,17 +1634,14 @@ XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg)
 
 void XMLCALL
 XML_SetSkippedEntityHandler(XML_Parser parser,
-                            XML_SkippedEntityHandler handler)
-{
+                            XML_SkippedEntityHandler handler) {
   if (parser != NULL)
     parser->m_skippedEntityHandler = handler;
 }
 
 void XMLCALL
 XML_SetUnknownEncodingHandler(XML_Parser parser,
-                              XML_UnknownEncodingHandler handler,
-                              void *data)
-{
+                              XML_UnknownEncodingHandler handler, void *data) {
   if (parser == NULL)
     return;
   parser->m_unknownEncodingHandler = handler;
@@ -1732,44 +1649,37 @@ XML_SetUnknownEncodingHandler(XML_Parser parser,
 }
 
 void XMLCALL
-XML_SetElementDeclHandler(XML_Parser parser,
-                          XML_ElementDeclHandler eldecl)
-{
+XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl) {
   if (parser != NULL)
     parser->m_elementDeclHandler = eldecl;
 }
 
 void XMLCALL
-XML_SetAttlistDeclHandler(XML_Parser parser,
-                          XML_AttlistDeclHandler attdecl)
-{
+XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl) {
   if (parser != NULL)
     parser->m_attlistDeclHandler = attdecl;
 }
 
 void XMLCALL
-XML_SetEntityDeclHandler(XML_Parser parser,
-                         XML_EntityDeclHandler handler)
-{
+XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler) {
   if (parser != NULL)
     parser->m_entityDeclHandler = handler;
 }
 
 void XMLCALL
-XML_SetXmlDeclHandler(XML_Parser parser,
-                      XML_XmlDeclHandler handler) {
+XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler handler) {
   if (parser != NULL)
     parser->m_xmlDeclHandler = handler;
 }
 
 int XMLCALL
 XML_SetParamEntityParsing(XML_Parser parser,
-                          enum XML_ParamEntityParsing peParsing)
-{
+                          enum XML_ParamEntityParsing peParsing) {
   if (parser == NULL)
     return 0;
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return 0;
 #ifdef XML_DTD
   parser->m_paramEntityParsing = peParsing;
@@ -1780,23 +1690,21 @@ XML_SetParamEntityParsing(XML_Parser parser,
 }
 
 int XMLCALL
-XML_SetHashSalt(XML_Parser parser,
-                unsigned long hash_salt)
-{
+XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) {
   if (parser == NULL)
     return 0;
   if (parser->m_parentParser)
     return XML_SetHashSalt(parser->m_parentParser, hash_salt);
   /* block after XML_Parse()/XML_ParseBuffer() has been called */
-  if (parser->m_parsingStatus.parsing == XML_PARSING || parser->m_parsingStatus.parsing == XML_SUSPENDED)
+  if (parser->m_parsingStatus.parsing == XML_PARSING
+      || parser->m_parsingStatus.parsing == XML_SUSPENDED)
     return 0;
   parser->m_hash_secret_salt = hash_salt;
   return 1;
 }
 
 enum XML_Status XMLCALL
-XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
-{
+XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) {
   if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) {
     if (parser != NULL)
       parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT;
@@ -1810,7 +1718,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     parser->m_errorCode = XML_ERROR_FINISHED;
     return XML_STATUS_ERROR;
   case XML_INITIALIZED:
-    if (parser->m_parentParser == NULL && !startParsing(parser)) {
+    if (parser->m_parentParser == NULL && ! startParsing(parser)) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return XML_STATUS_ERROR;
     }
@@ -1821,7 +1729,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
 
   if (len == 0) {
     parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
-    if (!isFinal)
+    if (! isFinal)
       return XML_STATUS_OK;
     parser->m_positionPtr = parser->m_bufferPtr;
     parser->m_parseEndPtr = parser->m_bufferEnd;
@@ -1830,7 +1738,9 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
        data are the final chunk of input, then we have to check them again
        to detect errors based on that fact.
     */
-    parser->m_errorCode = parser->m_processor(parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
+    parser->m_errorCode
+        = parser->m_processor(parser, parser->m_bufferPtr,
+                              parser->m_parseEndPtr, &parser->m_bufferPtr);
 
     if (parser->m_errorCode == XML_ERROR_NONE) {
       switch (parser->m_parsingStatus.parsing) {
@@ -1847,7 +1757,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
          *
          * LCOV_EXCL_START
          */
-        XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+        XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                          parser->m_bufferPtr, &parser->m_position);
         parser->m_positionPtr = parser->m_bufferPtr;
         return XML_STATUS_SUSPENDED;
         /* LCOV_EXCL_STOP */
@@ -1870,23 +1781,23 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     enum XML_Status result;
     /* Detect overflow (a+b > MAX <==> b > MAX-a) */
     if (len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) {
-       parser->m_errorCode = XML_ERROR_NO_MEMORY;
-       parser->m_eventPtr = parser->m_eventEndPtr = NULL;
-       parser->m_processor = errorProcessor;
-       return XML_STATUS_ERROR;
+      parser->m_errorCode = XML_ERROR_NO_MEMORY;
+      parser->m_eventPtr = parser->m_eventEndPtr = NULL;
+      parser->m_processor = errorProcessor;
+      return XML_STATUS_ERROR;
     }
     parser->m_parseEndByteIndex += len;
     parser->m_positionPtr = s;
     parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
 
-    parser->m_errorCode = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end);
+    parser->m_errorCode
+        = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end);
 
     if (parser->m_errorCode != XML_ERROR_NONE) {
       parser->m_eventEndPtr = parser->m_eventPtr;
       parser->m_processor = errorProcessor;
       return XML_STATUS_ERROR;
-    }
-    else {
+    } else {
       switch (parser->m_parsingStatus.parsing) {
       case XML_SUSPENDED:
         result = XML_STATUS_SUSPENDED;
@@ -1903,10 +1814,12 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
       }
     }
 
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, end, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, end,
+                      &parser->m_position);
     nLeftOver = s + len - end;
     if (nLeftOver) {
-      if (parser->m_buffer == NULL || nLeftOver > parser->m_bufferLim - parser->m_buffer) {
+      if (parser->m_buffer == NULL
+          || nLeftOver > parser->m_bufferLim - parser->m_buffer) {
         /* avoid _signed_ integer overflow */
         char *temp = NULL;
         const int bytesToAllocate = (int)((unsigned)len * 2U);
@@ -1932,7 +1845,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
     parser->m_eventEndPtr = parser->m_bufferPtr;
     return result;
   }
-#endif  /* not defined XML_CONTEXT_BYTES */
+#endif /* not defined XML_CONTEXT_BYTES */
   else {
     void *buff = XML_GetBuffer(parser, len);
     if (buff == NULL)
@@ -1945,8 +1858,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal)
 }
 
 enum XML_Status XMLCALL
-XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
-{
+XML_ParseBuffer(XML_Parser parser, int len, int isFinal) {
   const char *start;
   enum XML_Status result = XML_STATUS_OK;
 
@@ -1960,7 +1872,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
     parser->m_errorCode = XML_ERROR_FINISHED;
     return XML_STATUS_ERROR;
   case XML_INITIALIZED:
-    if (parser->m_parentParser == NULL && !startParsing(parser)) {
+    if (parser->m_parentParser == NULL && ! startParsing(parser)) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return XML_STATUS_ERROR;
     }
@@ -1976,14 +1888,14 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
   parser->m_parseEndByteIndex += len;
   parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal;
 
-  parser->m_errorCode = parser->m_processor(parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr);
+  parser->m_errorCode = parser->m_processor(
+      parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr);
 
   if (parser->m_errorCode != XML_ERROR_NONE) {
     parser->m_eventEndPtr = parser->m_eventPtr;
     parser->m_processor = errorProcessor;
     return XML_STATUS_ERROR;
-  }
-  else {
+  } else {
     switch (parser->m_parsingStatus.parsing) {
     case XML_SUSPENDED:
       result = XML_STATUS_SUSPENDED;
@@ -1994,18 +1906,18 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal)
         parser->m_parsingStatus.parsing = XML_FINISHED;
         return result;
       }
-    default: ;  /* should not happen */
+    default:; /* should not happen */
     }
   }
 
-  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                    parser->m_bufferPtr, &parser->m_position);
   parser->m_positionPtr = parser->m_bufferPtr;
   return result;
 }
 
-void * XMLCALL
-XML_GetBuffer(XML_Parser parser, int len)
-{
+void *XMLCALL
+XML_GetBuffer(XML_Parser parser, int len) {
   if (parser == NULL)
     return NULL;
   if (len < 0) {
@@ -2019,17 +1931,17 @@ XML_GetBuffer(XML_Parser parser, int len)
   case XML_FINISHED:
     parser->m_errorCode = XML_ERROR_FINISHED;
     return NULL;
-  default: ;
+  default:;
   }
 
   if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) {
 #ifdef XML_CONTEXT_BYTES
     int keep;
-#endif  /* defined XML_CONTEXT_BYTES */
+#endif /* defined XML_CONTEXT_BYTES */
     /* Do not invoke signed arithmetic overflow: */
-    int neededSize = (int) ((unsigned)len +
-                            (unsigned)EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd,
-                                                          parser->m_bufferPtr));
+    int neededSize = (int)((unsigned)len
+                           + (unsigned)EXPAT_SAFE_PTR_DIFF(
+                               parser->m_bufferEnd, parser->m_bufferPtr));
     if (neededSize < 0) {
       parser->m_errorCode = XML_ERROR_NO_MEMORY;
       return NULL;
@@ -2039,13 +1951,18 @@ XML_GetBuffer(XML_Parser parser, int len)
     if (keep > XML_CONTEXT_BYTES)
       keep = XML_CONTEXT_BYTES;
     neededSize += keep;
-#endif  /* defined XML_CONTEXT_BYTES */
-    if (neededSize <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) {
+#endif /* defined XML_CONTEXT_BYTES */
+    if (neededSize
+        <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) {
 #ifdef XML_CONTEXT_BYTES
       if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) {
-          int offset = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) - keep;
-        /* The buffer pointers cannot be NULL here; we have at least some bytes in the buffer */
-        memmove(parser->m_buffer, &parser->m_buffer[offset], parser->m_bufferEnd - parser->m_bufferPtr + keep);
+        int offset
+            = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)
+              - keep;
+        /* The buffer pointers cannot be NULL here; we have at least some bytes
+         * in the buffer */
+        memmove(parser->m_buffer, &parser->m_buffer[offset],
+                parser->m_bufferEnd - parser->m_bufferPtr + keep);
         parser->m_bufferEnd -= offset;
         parser->m_bufferPtr -= offset;
       }
@@ -2053,20 +1970,21 @@ XML_GetBuffer(XML_Parser parser, int len)
       if (parser->m_buffer && parser->m_bufferPtr) {
         memmove(parser->m_buffer, parser->m_bufferPtr,
                 EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr));
-        parser->m_bufferEnd = parser->m_buffer +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
+        parser->m_bufferEnd
+            = parser->m_buffer
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
         parser->m_bufferPtr = parser->m_buffer;
       }
-#endif  /* not defined XML_CONTEXT_BYTES */
-    }
-    else {
+#endif /* not defined XML_CONTEXT_BYTES */
+    } else {
       char *newBuf;
-      int bufferSize = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr);
+      int bufferSize
+          = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr);
       if (bufferSize == 0)
         bufferSize = INIT_BUFFER_SIZE;
       do {
         /* Do not invoke signed arithmetic overflow: */
-        bufferSize = (int) (2U * (unsigned) bufferSize);
+        bufferSize = (int)(2U * (unsigned)bufferSize);
       } while (bufferSize < neededSize && bufferSize > 0);
       if (bufferSize <= 0) {
         parser->m_errorCode = XML_ERROR_NO_MEMORY;
@@ -2080,18 +1998,17 @@ XML_GetBuffer(XML_Parser parser, int len)
       parser->m_bufferLim = newBuf + bufferSize;
 #ifdef XML_CONTEXT_BYTES
       if (parser->m_bufferPtr) {
-        int keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer);
-        if (keep > XML_CONTEXT_BYTES)
-          keep = XML_CONTEXT_BYTES;
         memcpy(newBuf, &parser->m_bufferPtr[-keep],
-               EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) + keep);
+               EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)
+                   + keep);
         FREE(parser, parser->m_buffer);
         parser->m_buffer = newBuf;
-        parser->m_bufferEnd = parser->m_buffer +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) + keep;
+        parser->m_bufferEnd
+            = parser->m_buffer
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)
+              + keep;
         parser->m_bufferPtr = parser->m_buffer + keep;
-      }
-      else {
+      } else {
         /* This must be a brand new buffer with no data in it yet */
         parser->m_bufferEnd = newBuf;
         parser->m_bufferPtr = parser->m_buffer = newBuf;
@@ -2101,15 +2018,15 @@ XML_GetBuffer(XML_Parser parser, int len)
         memcpy(newBuf, parser->m_bufferPtr,
                EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr));
         FREE(parser, parser->m_buffer);
-        parser->m_bufferEnd = newBuf +
-            EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
-      }
-      else {
+        parser->m_bufferEnd
+            = newBuf
+              + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr);
+      } else {
         /* This must be a brand new buffer with no data in it yet */
         parser->m_bufferEnd = newBuf;
       }
       parser->m_bufferPtr = parser->m_buffer = newBuf;
-#endif  /* not defined XML_CONTEXT_BYTES */
+#endif /* not defined XML_CONTEXT_BYTES */
     }
     parser->m_eventPtr = parser->m_eventEndPtr = NULL;
     parser->m_positionPtr = NULL;
@@ -2118,8 +2035,7 @@ XML_GetBuffer(XML_Parser parser, int len)
 }
 
 enum XML_Status XMLCALL
-XML_StopParser(XML_Parser parser, XML_Bool resumable)
-{
+XML_StopParser(XML_Parser parser, XML_Bool resumable) {
   if (parser == NULL)
     return XML_STATUS_ERROR;
   switch (parser->m_parsingStatus.parsing) {
@@ -2142,16 +2058,14 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable)
       }
 #endif
       parser->m_parsingStatus.parsing = XML_SUSPENDED;
-    }
-    else
+    } else
       parser->m_parsingStatus.parsing = XML_FINISHED;
   }
   return XML_STATUS_OK;
 }
 
 enum XML_Status XMLCALL
-XML_ResumeParser(XML_Parser parser)
-{
+XML_ResumeParser(XML_Parser parser) {
   enum XML_Status result = XML_STATUS_OK;
 
   if (parser == NULL)
@@ -2162,14 +2076,14 @@ XML_ResumeParser(XML_Parser parser)
   }
   parser->m_parsingStatus.parsing = XML_PARSING;
 
-  parser->m_errorCode = parser->m_processor(parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
+  parser->m_errorCode = parser->m_processor(
+      parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr);
 
   if (parser->m_errorCode != XML_ERROR_NONE) {
     parser->m_eventEndPtr = parser->m_eventPtr;
     parser->m_processor = errorProcessor;
     return XML_STATUS_ERROR;
-  }
-  else {
+  } else {
     switch (parser->m_parsingStatus.parsing) {
     case XML_SUSPENDED:
       result = XML_STATUS_SUSPENDED;
@@ -2180,18 +2094,18 @@ XML_ResumeParser(XML_Parser parser)
         parser->m_parsingStatus.parsing = XML_FINISHED;
         return result;
       }
-    default: ;
+    default:;
     }
   }
 
-  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_bufferPtr, &parser->m_position);
+  XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                    parser->m_bufferPtr, &parser->m_position);
   parser->m_positionPtr = parser->m_bufferPtr;
   return result;
 }
 
 void XMLCALL
-XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status)
-{
+XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) {
   if (parser == NULL)
     return;
   assert(status != NULL);
@@ -2199,26 +2113,24 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status)
 }
 
 enum XML_Error XMLCALL
-XML_GetErrorCode(XML_Parser parser)
-{
+XML_GetErrorCode(XML_Parser parser) {
   if (parser == NULL)
     return XML_ERROR_INVALID_ARGUMENT;
   return parser->m_errorCode;
 }
 
 XML_Index XMLCALL
-XML_GetCurrentByteIndex(XML_Parser parser)
-{
+XML_GetCurrentByteIndex(XML_Parser parser) {
   if (parser == NULL)
     return -1;
   if (parser->m_eventPtr)
-    return (XML_Index)(parser->m_parseEndByteIndex - (parser->m_parseEndPtr - parser->m_eventPtr));
+    return (XML_Index)(parser->m_parseEndByteIndex
+                       - (parser->m_parseEndPtr - parser->m_eventPtr));
   return -1;
 }
 
 int XMLCALL
-XML_GetCurrentByteCount(XML_Parser parser)
-{
+XML_GetCurrentByteCount(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventEndPtr && parser->m_eventPtr)
@@ -2226,9 +2138,8 @@ XML_GetCurrentByteCount(XML_Parser parser)
   return 0;
 }
 
-const char * XMLCALL
-XML_GetInputContext(XML_Parser parser, int *offset, int *size)
-{
+const char *XMLCALL
+XML_GetInputContext(XML_Parser parser, int *offset, int *size) {
 #ifdef XML_CONTEXT_BYTES
   if (parser == NULL)
     return NULL;
@@ -2236,7 +2147,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size)
     if (offset != NULL)
       *offset = (int)(parser->m_eventPtr - parser->m_buffer);
     if (size != NULL)
-      *size   = (int)(parser->m_bufferEnd - parser->m_buffer);
+      *size = (int)(parser->m_bufferEnd - parser->m_buffer);
     return parser->m_buffer;
   }
 #else
@@ -2244,82 +2155,76 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size)
   (void)offset;
   (void)size;
 #endif /* defined XML_CONTEXT_BYTES */
-  return (char *) 0;
+  return (char *)0;
 }
 
 XML_Size XMLCALL
-XML_GetCurrentLineNumber(XML_Parser parser)
-{
+XML_GetCurrentLineNumber(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) {
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_eventPtr, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                      parser->m_eventPtr, &parser->m_position);
     parser->m_positionPtr = parser->m_eventPtr;
   }
   return parser->m_position.lineNumber + 1;
 }
 
 XML_Size XMLCALL
-XML_GetCurrentColumnNumber(XML_Parser parser)
-{
+XML_GetCurrentColumnNumber(XML_Parser parser) {
   if (parser == NULL)
     return 0;
   if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) {
-    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, parser->m_eventPtr, &parser->m_position);
+    XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr,
+                      parser->m_eventPtr, &parser->m_position);
     parser->m_positionPtr = parser->m_eventPtr;
   }
   return parser->m_position.columnNumber;
 }
 
 void XMLCALL
-XML_FreeContentModel(XML_Parser parser, XML_Content *model)
-{
+XML_FreeContentModel(XML_Parser parser, XML_Content *model) {
   if (parser != NULL)
     FREE(parser, model);
 }
 
-void * XMLCALL
-XML_MemMalloc(XML_Parser parser, size_t size)
-{
+void *XMLCALL
+XML_MemMalloc(XML_Parser parser, size_t size) {
   if (parser == NULL)
     return NULL;
   return MALLOC(parser, size);
 }
 
-void * XMLCALL
-XML_MemRealloc(XML_Parser parser, void *ptr, size_t size)
-{
+void *XMLCALL
+XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) {
   if (parser == NULL)
     return NULL;
   return REALLOC(parser, ptr, size);
 }
 
 void XMLCALL
-XML_MemFree(XML_Parser parser, void *ptr)
-{
+XML_MemFree(XML_Parser parser, void *ptr) {
   if (parser != NULL)
     FREE(parser, ptr);
 }
 
 void XMLCALL
-XML_DefaultCurrent(XML_Parser parser)
-{
+XML_DefaultCurrent(XML_Parser parser) {
   if (parser == NULL)
     return;
   if (parser->m_defaultHandler) {
     if (parser->m_openInternalEntities)
-      reportDefault(parser,
-                    parser->m_internalEncoding,
+      reportDefault(parser, parser->m_internalEncoding,
                     parser->m_openInternalEntities->internalEventPtr,
                     parser->m_openInternalEntities->internalEventEndPtr);
     else
-      reportDefault(parser, parser->m_encoding, parser->m_eventPtr, parser->m_eventEndPtr);
+      reportDefault(parser, parser->m_encoding, parser->m_eventPtr,
+                    parser->m_eventEndPtr);
   }
 }
 
-const XML_LChar * XMLCALL
-XML_ErrorString(enum XML_Error code)
-{
+const XML_LChar *XMLCALL
+XML_ErrorString(enum XML_Error code) {
   switch (code) {
   case XML_ERROR_NONE:
     return NULL;
@@ -2401,21 +2306,22 @@ XML_ErrorString(enum XML_Error code)
     return XML_L("cannot suspend in external parameter entity");
   /* Added in 2.0.0. */
   case XML_ERROR_RESERVED_PREFIX_XML:
-    return XML_L("reserved prefix (xml) must not be undeclared or bound to another namespace name");
+    return XML_L(
+        "reserved prefix (xml) must not be undeclared or bound to another namespace name");
   case XML_ERROR_RESERVED_PREFIX_XMLNS:
     return XML_L("reserved prefix (xmlns) must not be declared or undeclared");
   case XML_ERROR_RESERVED_NAMESPACE_URI:
-    return XML_L("prefix must not be bound to one of the reserved namespace names");
+    return XML_L(
+        "prefix must not be bound to one of the reserved namespace names");
   /* Added in 2.2.5. */
-  case XML_ERROR_INVALID_ARGUMENT:  /* Constant added in 2.2.1, already */
+  case XML_ERROR_INVALID_ARGUMENT: /* Constant added in 2.2.1, already */
     return XML_L("invalid argument");
   }
   return NULL;
 }
 
-const XML_LChar * XMLCALL
+const XML_LChar *XMLCALL
 XML_ExpatVersion(void) {
-
   /* V1 is used to string-ize the version number. However, it would
      string-ize the actual version macro *names* unless we get them
      substituted before being passed to V1. CPP is defined to expand
@@ -2424,8 +2330,8 @@ XML_ExpatVersion(void) {
      with the correct numerals. */
   /* ### I'm assuming cpp is portable in this respect... */
 
-#define V1(a,b,c) XML_L(#a)XML_L(".")XML_L(#b)XML_L(".")XML_L(#c)
-#define V2(a,b,c) XML_L("expat_")V1(a,b,c)
+#define V1(a, b, c) XML_L(#a) XML_L(".") XML_L(#b) XML_L(".") XML_L(#c)
+#define V2(a, b, c) XML_L("expat_") V1(a, b, c)
 
   return V2(XML_MAJOR_VERSION, XML_MINOR_VERSION, XML_MICRO_VERSION);
 
@@ -2434,8 +2340,7 @@ XML_ExpatVersion(void) {
 }
 
 XML_Expat_Version XMLCALL
-XML_ExpatVersionInfo(void)
-{
+XML_ExpatVersionInfo(void) {
   XML_Expat_Version version;
 
   version.major = XML_MAJOR_VERSION;
@@ -2445,41 +2350,39 @@ XML_ExpatVersionInfo(void)
   return version;
 }
 
-const XML_Feature * XMLCALL
-XML_GetFeatureList(void)
-{
-  static const XML_Feature features[] = {
-    {XML_FEATURE_SIZEOF_XML_CHAR,  XML_L("sizeof(XML_Char)"),
-     sizeof(XML_Char)},
-    {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"),
-     sizeof(XML_LChar)},
+const XML_Feature *XMLCALL
+XML_GetFeatureList(void) {
+  static const XML_Feature features[]
+      = {{XML_FEATURE_SIZEOF_XML_CHAR, XML_L("sizeof(XML_Char)"),
+          sizeof(XML_Char)},
+         {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"),
+          sizeof(XML_LChar)},
 #ifdef XML_UNICODE
-    {XML_FEATURE_UNICODE,          XML_L("XML_UNICODE"), 0},
+         {XML_FEATURE_UNICODE, XML_L("XML_UNICODE"), 0},
 #endif
 #ifdef XML_UNICODE_WCHAR_T
-    {XML_FEATURE_UNICODE_WCHAR_T,  XML_L("XML_UNICODE_WCHAR_T"), 0},
+         {XML_FEATURE_UNICODE_WCHAR_T, XML_L("XML_UNICODE_WCHAR_T"), 0},
 #endif
 #ifdef XML_DTD
-    {XML_FEATURE_DTD,              XML_L("XML_DTD"), 0},
+         {XML_FEATURE_DTD, XML_L("XML_DTD"), 0},
 #endif
 #ifdef XML_CONTEXT_BYTES
-    {XML_FEATURE_CONTEXT_BYTES,    XML_L("XML_CONTEXT_BYTES"),
-     XML_CONTEXT_BYTES},
+         {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"),
+          XML_CONTEXT_BYTES},
 #endif
 #ifdef XML_MIN_SIZE
-    {XML_FEATURE_MIN_SIZE,         XML_L("XML_MIN_SIZE"), 0},
+         {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0},
 #endif
 #ifdef XML_NS
-    {XML_FEATURE_NS,               XML_L("XML_NS"), 0},
+         {XML_FEATURE_NS, XML_L("XML_NS"), 0},
 #endif
 #ifdef XML_LARGE_SIZE
-    {XML_FEATURE_LARGE_SIZE,       XML_L("XML_LARGE_SIZE"), 0},
+         {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0},
 #endif
 #ifdef XML_ATTR_INFO
-    {XML_FEATURE_ATTR_INFO,        XML_L("XML_ATTR_INFO"), 0},
+         {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0},
 #endif
-    {XML_FEATURE_END,              NULL, 0}
-  };
+         {XML_FEATURE_END, NULL, 0}};
 
   return features;
 }
@@ -2490,8 +2393,7 @@ XML_GetFeatureList(void)
    permanent location, since the parse buffer is about to be discarded.
 */
 static XML_Bool
-storeRawNames(XML_Parser parser)
-{
+storeRawNames(XML_Parser parser) {
   TAG *tag = parser->m_tagStack;
   while (tag) {
     int bufSize;
@@ -2521,8 +2423,8 @@ storeRawNames(XML_Parser parser)
          then update it as well, since it will always point into tag->buf
       */
       if (tag->name.localPart)
-        tag->name.localPart = (XML_Char *)temp + (tag->name.localPart -
-                                                  (XML_Char *)tag->buf);
+        tag->name.localPart
+            = (XML_Char *)temp + (tag->name.localPart - (XML_Char *)tag->buf);
       tag->buf = temp;
       tag->bufEnd = temp + bufSize;
       rawNameBuf = temp + nameLen;
@@ -2535,26 +2437,21 @@ storeRawNames(XML_Parser parser)
 }
 
 static enum XML_Error PTRCALL
-contentProcessor(XML_Parser parser,
-                 const char *start,
-                 const char *end,
-                 const char **endPtr)
-{
-  enum XML_Error result = doContent(parser, 0, parser->m_encoding, start, end,
-                                    endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+contentProcessor(XML_Parser parser, const char *start, const char *end,
+                 const char **endPtr) {
+  enum XML_Error result
+      = doContent(parser, 0, parser->m_encoding, start, end, endPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result == XML_ERROR_NONE) {
-    if (!storeRawNames(parser))
+    if (! storeRawNames(parser))
       return XML_ERROR_NO_MEMORY;
   }
   return result;
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor(XML_Parser parser,
-                            const char *start,
-                            const char *end,
-                            const char **endPtr)
-{
+externalEntityInitProcessor(XML_Parser parser, const char *start,
+                            const char *end, const char **endPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -2563,11 +2460,8 @@ externalEntityInitProcessor(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor2(XML_Parser parser,
-                             const char *start,
-                             const char *end,
-                             const char **endPtr)
-{
+externalEntityInitProcessor2(XML_Parser parser, const char *start,
+                             const char *end, const char **endPtr) {
   const char *next = start; /* XmlContentTok doesn't always set the last arg */
   int tok = XmlContentTok(parser->m_encoding, start, end, &next);
   switch (tok) {
@@ -2577,21 +2471,21 @@ externalEntityInitProcessor2(XML_Parser parser,
        doContent (by detecting XML_TOK_NONE) without processing any xml text
        declaration - causing the error XML_ERROR_MISPLACED_XML_PI in doContent.
     */
-    if (next == end && !parser->m_parsingStatus.finalBuffer) {
+    if (next == end && ! parser->m_parsingStatus.finalBuffer) {
       *endPtr = next;
       return XML_ERROR_NONE;
     }
     start = next;
     break;
   case XML_TOK_PARTIAL:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
     parser->m_eventPtr = start;
     return XML_ERROR_UNCLOSED_TOKEN;
   case XML_TOK_PARTIAL_CHAR:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
@@ -2603,11 +2497,8 @@ externalEntityInitProcessor2(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityInitProcessor3(XML_Parser parser,
-                             const char *start,
-                             const char *end,
-                             const char **endPtr)
-{
+externalEntityInitProcessor3(XML_Parser parser, const char *start,
+                             const char *end, const char **endPtr) {
   int tok;
   const char *next = start; /* XmlContentTok doesn't always set the last arg */
   parser->m_eventPtr = start;
@@ -2615,31 +2506,29 @@ externalEntityInitProcessor3(XML_Parser parser,
   parser->m_eventEndPtr = next;
 
   switch (tok) {
-  case XML_TOK_XML_DECL:
-    {
-      enum XML_Error result;
-      result = processXmlDecl(parser, 1, start, next);
-      if (result != XML_ERROR_NONE)
-        return result;
-      switch (parser->m_parsingStatus.parsing) {
-      case XML_SUSPENDED:
-        *endPtr = next;
-        return XML_ERROR_NONE;
-      case XML_FINISHED:
-        return XML_ERROR_ABORTED;
-      default:
-        start = next;
-      }
+  case XML_TOK_XML_DECL: {
+    enum XML_Error result;
+    result = processXmlDecl(parser, 1, start, next);
+    if (result != XML_ERROR_NONE)
+      return result;
+    switch (parser->m_parsingStatus.parsing) {
+    case XML_SUSPENDED:
+      *endPtr = next;
+      return XML_ERROR_NONE;
+    case XML_FINISHED:
+      return XML_ERROR_ABORTED;
+    default:
+      start = next;
     }
-    break;
+  } break;
   case XML_TOK_PARTIAL:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
     return XML_ERROR_UNCLOSED_TOKEN;
   case XML_TOK_PARTIAL_CHAR:
-    if (!parser->m_parsingStatus.finalBuffer) {
+    if (! parser->m_parsingStatus.finalBuffer) {
       *endPtr = start;
       return XML_ERROR_NONE;
     }
@@ -2651,39 +2540,31 @@ externalEntityInitProcessor3(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalEntityContentProcessor(XML_Parser parser,
-                               const char *start,
-                               const char *end,
-                               const char **endPtr)
-{
-  enum XML_Error result = doContent(parser, 1, parser->m_encoding, start, end,
-                                    endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+externalEntityContentProcessor(XML_Parser parser, const char *start,
+                               const char *end, const char **endPtr) {
+  enum XML_Error result
+      = doContent(parser, 1, parser->m_encoding, start, end, endPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result == XML_ERROR_NONE) {
-    if (!storeRawNames(parser))
+    if (! storeRawNames(parser))
       return XML_ERROR_NO_MEMORY;
   }
   return result;
 }
 
 static enum XML_Error
-doContent(XML_Parser parser,
-          int startTagLevel,
-          const ENCODING *enc,
-          const char *s,
-          const char *end,
-          const char **nextPtr,
-          XML_Bool haveMore)
-{
+doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc,
+          const char *s, const char *end, const char **nextPtr,
+          XML_Bool haveMore) {
   /* save one level of indirection */
-  DTD * const dtd = parser->m_dtd;
+  DTD *const dtd = parser->m_dtd;
 
   const char **eventPP;
   const char **eventEndPP;
   if (enc == parser->m_encoding) {
     eventPP = &parser->m_eventPtr;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -2703,8 +2584,7 @@ doContent(XML_Parser parser,
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, end);
       /* We are at the end of the final buffer, should we check for
          XML_SUSPENDED, XML_FINISHED?
@@ -2742,185 +2622,178 @@ doContent(XML_Parser parser,
         return XML_ERROR_NONE;
       }
       return XML_ERROR_PARTIAL_CHAR;
-    case XML_TOK_ENTITY_REF:
-      {
-        const XML_Char *name;
-        ENTITY *entity;
-        XML_Char ch = (XML_Char) XmlPredefinedEntityName(enc,
-                                              s + enc->minBytesPerChar,
-                                              next - enc->minBytesPerChar);
-        if (ch) {
-          if (parser->m_characterDataHandler)
-            parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1);
-          else if (parser->m_defaultHandler)
-            reportDefault(parser, enc, s, next);
-          break;
-        }
-        name = poolStoreString(&dtd->pool, enc,
-                                s + enc->minBytesPerChar,
-                                next - enc->minBytesPerChar);
-        if (!name)
-          return XML_ERROR_NO_MEMORY;
-        entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
-        poolDiscard(&dtd->pool);
-        /* First, determine if a check for an existing declaration is needed;
-           if yes, check that the entity exists, and that it is internal,
-           otherwise call the skipped entity or default handler.
-        */
-        if (!dtd->hasParamEntityRefs || dtd->standalone) {
-          if (!entity)
-            return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal)
-            return XML_ERROR_ENTITY_DECLARED_IN_PE;
-        }
-        else if (!entity) {
+    case XML_TOK_ENTITY_REF: {
+      const XML_Char *name;
+      ENTITY *entity;
+      XML_Char ch = (XML_Char)XmlPredefinedEntityName(
+          enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar);
+      if (ch) {
+        if (parser->m_characterDataHandler)
+          parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1);
+        else if (parser->m_defaultHandler)
+          reportDefault(parser, enc, s, next);
+        break;
+      }
+      name = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                             next - enc->minBytesPerChar);
+      if (! name)
+        return XML_ERROR_NO_MEMORY;
+      entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
+      poolDiscard(&dtd->pool);
+      /* First, determine if a check for an existing declaration is needed;
+         if yes, check that the entity exists, and that it is internal,
+         otherwise call the skipped entity or default handler.
+      */
+      if (! dtd->hasParamEntityRefs || dtd->standalone) {
+        if (! entity)
+          return XML_ERROR_UNDEFINED_ENTITY;
+        else if (! entity->is_internal)
+          return XML_ERROR_ENTITY_DECLARED_IN_PE;
+      } else if (! entity) {
+        if (parser->m_skippedEntityHandler)
+          parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+        else if (parser->m_defaultHandler)
+          reportDefault(parser, enc, s, next);
+        break;
+      }
+      if (entity->open)
+        return XML_ERROR_RECURSIVE_ENTITY_REF;
+      if (entity->notation)
+        return XML_ERROR_BINARY_ENTITY_REF;
+      if (entity->textPtr) {
+        enum XML_Error result;
+        if (! parser->m_defaultExpandInternalEntities) {
           if (parser->m_skippedEntityHandler)
-            parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+            parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name,
+                                           0);
           else if (parser->m_defaultHandler)
             reportDefault(parser, enc, s, next);
           break;
         }
-        if (entity->open)
-          return XML_ERROR_RECURSIVE_ENTITY_REF;
-        if (entity->notation)
-          return XML_ERROR_BINARY_ENTITY_REF;
-        if (entity->textPtr) {
-          enum XML_Error result;
-          if (!parser->m_defaultExpandInternalEntities) {
-            if (parser->m_skippedEntityHandler)
-              parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name, 0);
-            else if (parser->m_defaultHandler)
-              reportDefault(parser, enc, s, next);
-            break;
-          }
-          result = processInternalEntity(parser, entity, XML_FALSE);
-          if (result != XML_ERROR_NONE)
-            return result;
-        }
-        else if (parser->m_externalEntityRefHandler) {
-          const XML_Char *context;
-          entity->open = XML_TRUE;
-          context = getContext(parser);
-          entity->open = XML_FALSE;
-          if (!context)
-            return XML_ERROR_NO_MEMORY;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        context,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
-            return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
-          poolDiscard(&parser->m_tempPool);
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        break;
-      }
+        result = processInternalEntity(parser, entity, XML_FALSE);
+        if (result != XML_ERROR_NONE)
+          return result;
+      } else if (parser->m_externalEntityRefHandler) {
+        const XML_Char *context;
+        entity->open = XML_TRUE;
+        context = getContext(parser);
+        entity->open = XML_FALSE;
+        if (! context)
+          return XML_ERROR_NO_MEMORY;
+        if (! parser->m_externalEntityRefHandler(
+                parser->m_externalEntityRefHandlerArg, context, entity->base,
+                entity->systemId, entity->publicId))
+          return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
+        poolDiscard(&parser->m_tempPool);
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      break;
+    }
     case XML_TOK_START_TAG_NO_ATTS:
       /* fall through */
-    case XML_TOK_START_TAG_WITH_ATTS:
-      {
-        TAG *tag;
-        enum XML_Error result;
-        XML_Char *toPtr;
-        if (parser->m_freeTagList) {
-          tag = parser->m_freeTagList;
-          parser->m_freeTagList = parser->m_freeTagList->parent;
+    case XML_TOK_START_TAG_WITH_ATTS: {
+      TAG *tag;
+      enum XML_Error result;
+      XML_Char *toPtr;
+      if (parser->m_freeTagList) {
+        tag = parser->m_freeTagList;
+        parser->m_freeTagList = parser->m_freeTagList->parent;
+      } else {
+        tag = (TAG *)MALLOC(parser, sizeof(TAG));
+        if (! tag)
+          return XML_ERROR_NO_MEMORY;
+        tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE);
+        if (! tag->buf) {
+          FREE(parser, tag);
+          return XML_ERROR_NO_MEMORY;
         }
-        else {
-          tag = (TAG *)MALLOC(parser, sizeof(TAG));
-          if (!tag)
-            return XML_ERROR_NO_MEMORY;
-          tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE);
-          if (!tag->buf) {
-            FREE(parser, tag);
-            return XML_ERROR_NO_MEMORY;
+        tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE;
+      }
+      tag->bindings = NULL;
+      tag->parent = parser->m_tagStack;
+      parser->m_tagStack = tag;
+      tag->name.localPart = NULL;
+      tag->name.prefix = NULL;
+      tag->rawName = s + enc->minBytesPerChar;
+      tag->rawNameLength = XmlNameLength(enc, tag->rawName);
+      ++parser->m_tagLevel;
+      {
+        const char *rawNameEnd = tag->rawName + tag->rawNameLength;
+        const char *fromPtr = tag->rawName;
+        toPtr = (XML_Char *)tag->buf;
+        for (;;) {
+          int bufSize;
+          int convLen;
+          const enum XML_Convert_Result convert_res
+              = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr,
+                           (ICHAR *)tag->bufEnd - 1);
+          convLen = (int)(toPtr - (XML_Char *)tag->buf);
+          if ((fromPtr >= rawNameEnd)
+              || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) {
+            tag->name.strLen = convLen;
+            break;
           }
-          tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE;
-        }
-        tag->bindings = NULL;
-        tag->parent = parser->m_tagStack;
-        parser->m_tagStack = tag;
-        tag->name.localPart = NULL;
-        tag->name.prefix = NULL;
-        tag->rawName = s + enc->minBytesPerChar;
-        tag->rawNameLength = XmlNameLength(enc, tag->rawName);
-        ++parser->m_tagLevel;
-        {
-          const char *rawNameEnd = tag->rawName + tag->rawNameLength;
-          const char *fromPtr = tag->rawName;
-          toPtr = (XML_Char *)tag->buf;
-          for (;;) {
-            int bufSize;
-            int convLen;
-            const enum XML_Convert_Result convert_res = XmlConvert(enc,
-                       &fromPtr, rawNameEnd,
-                       (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1);
-            convLen = (int)(toPtr - (XML_Char *)tag->buf);
-            if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) {
-              tag->name.strLen = convLen;
-              break;
-            }
-            bufSize = (int)(tag->bufEnd - tag->buf) << 1;
-            {
-              char *temp = (char *)REALLOC(parser, tag->buf, bufSize);
-              if (temp == NULL)
-                return XML_ERROR_NO_MEMORY;
-              tag->buf = temp;
-              tag->bufEnd = temp + bufSize;
-              toPtr = (XML_Char *)temp + convLen;
-            }
+          bufSize = (int)(tag->bufEnd - tag->buf) << 1;
+          {
+            char *temp = (char *)REALLOC(parser, tag->buf, bufSize);
+            if (temp == NULL)
+              return XML_ERROR_NO_MEMORY;
+            tag->buf = temp;
+            tag->bufEnd = temp + bufSize;
+            toPtr = (XML_Char *)temp + convLen;
           }
         }
-        tag->name.str = (XML_Char *)tag->buf;
-        *toPtr = XML_T('\0');
-        result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings));
-        if (result)
-          return result;
-        if (parser->m_startElementHandler)
-          parser->m_startElementHandler(parser->m_handlerArg, tag->name.str,
-                              (const XML_Char **)parser->m_atts);
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        poolClear(&parser->m_tempPool);
-        break;
       }
+      tag->name.str = (XML_Char *)tag->buf;
+      *toPtr = XML_T('\0');
+      result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings));
+      if (result)
+        return result;
+      if (parser->m_startElementHandler)
+        parser->m_startElementHandler(parser->m_handlerArg, tag->name.str,
+                                      (const XML_Char **)parser->m_atts);
+      else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      poolClear(&parser->m_tempPool);
+      break;
+    }
     case XML_TOK_EMPTY_ELEMENT_NO_ATTS:
       /* fall through */
-    case XML_TOK_EMPTY_ELEMENT_WITH_ATTS:
-      {
-        const char *rawName = s + enc->minBytesPerChar;
-        enum XML_Error result;
-        BINDING *bindings = NULL;
-        XML_Bool noElmHandlers = XML_TRUE;
-        TAG_NAME name;
-        name.str = poolStoreString(&parser->m_tempPool, enc, rawName,
-                                   rawName + XmlNameLength(enc, rawName));
-        if (!name.str)
-          return XML_ERROR_NO_MEMORY;
-        poolFinish(&parser->m_tempPool);
-        result = storeAtts(parser, enc, s, &name, &bindings);
-        if (result != XML_ERROR_NONE) {
-          freeBindings(parser, bindings);
-          return result;
-        }
-        poolFinish(&parser->m_tempPool);
-        if (parser->m_startElementHandler) {
-          parser->m_startElementHandler(parser->m_handlerArg, name.str, (const XML_Char **)parser->m_atts);
-          noElmHandlers = XML_FALSE;
-        }
-        if (parser->m_endElementHandler) {
-          if (parser->m_startElementHandler)
-            *eventPP = *eventEndPP;
-          parser->m_endElementHandler(parser->m_handlerArg, name.str);
-          noElmHandlers = XML_FALSE;
-        }
-        if (noElmHandlers && parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        poolClear(&parser->m_tempPool);
+    case XML_TOK_EMPTY_ELEMENT_WITH_ATTS: {
+      const char *rawName = s + enc->minBytesPerChar;
+      enum XML_Error result;
+      BINDING *bindings = NULL;
+      XML_Bool noElmHandlers = XML_TRUE;
+      TAG_NAME name;
+      name.str = poolStoreString(&parser->m_tempPool, enc, rawName,
+                                 rawName + XmlNameLength(enc, rawName));
+      if (! name.str)
+        return XML_ERROR_NO_MEMORY;
+      poolFinish(&parser->m_tempPool);
+      result = storeAtts(parser, enc, s, &name, &bindings);
+      if (result != XML_ERROR_NONE) {
         freeBindings(parser, bindings);
+        return result;
+      }
+      poolFinish(&parser->m_tempPool);
+      if (parser->m_startElementHandler) {
+        parser->m_startElementHandler(parser->m_handlerArg, name.str,
+                                      (const XML_Char **)parser->m_atts);
+        noElmHandlers = XML_FALSE;
+      }
+      if (parser->m_endElementHandler) {
+        if (parser->m_startElementHandler)
+          *eventPP = *eventEndPP;
+        parser->m_endElementHandler(parser->m_handlerArg, name.str);
+        noElmHandlers = XML_FALSE;
       }
-      if ((parser->m_tagLevel == 0) && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
+      if (noElmHandlers && parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      poolClear(&parser->m_tempPool);
+      freeBindings(parser, bindings);
+    }
+      if ((parser->m_tagLevel == 0)
+          && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
         if (parser->m_parsingStatus.parsing == XML_SUSPENDED)
           parser->m_processor = epilogProcessor;
         else
@@ -2937,7 +2810,7 @@ doContent(XML_Parser parser,
         parser->m_tagStack = tag->parent;
         tag->parent = parser->m_freeTagList;
         parser->m_freeTagList = tag;
-        rawName = s + enc->minBytesPerChar*2;
+        rawName = s + enc->minBytesPerChar * 2;
         len = XmlNameLength(enc, rawName);
         if (len != tag->rawNameLength
             || memcmp(tag->rawName, rawName, len) != 0) {
@@ -2957,86 +2830,89 @@ doContent(XML_Parser parser,
             */
             uri = (XML_Char *)tag->name.str + tag->name.uriLen;
             /* don't need to check for space - already done in storeAtts() */
-            while (*localPart) *uri++ = *localPart++;
+            while (*localPart)
+              *uri++ = *localPart++;
             prefix = (XML_Char *)tag->name.prefix;
             if (parser->m_ns_triplets && prefix) {
               *uri++ = parser->m_namespaceSeparator;
-              while (*prefix) *uri++ = *prefix++;
-             }
+              while (*prefix)
+                *uri++ = *prefix++;
+            }
             *uri = XML_T('\0');
           }
           parser->m_endElementHandler(parser->m_handlerArg, tag->name.str);
-        }
-        else if (parser->m_defaultHandler)
+        } else if (parser->m_defaultHandler)
           reportDefault(parser, enc, s, next);
         while (tag->bindings) {
           BINDING *b = tag->bindings;
           if (parser->m_endNamespaceDeclHandler)
-            parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
+            parser->m_endNamespaceDeclHandler(parser->m_handlerArg,
+                                              b->prefix->name);
           tag->bindings = tag->bindings->nextTagBinding;
           b->nextTagBinding = parser->m_freeBindingList;
           parser->m_freeBindingList = b;
           b->prefix->binding = b->prevPrefixBinding;
         }
-        if (parser->m_tagLevel == 0)
-          return epilogProcessor(parser, next, end, nextPtr);
-      }
-      break;
-    case XML_TOK_CHAR_REF:
-      {
-        int n = XmlCharRefNumber(enc, s);
-        if (n < 0)
-          return XML_ERROR_BAD_CHAR_REF;
-        if (parser->m_characterDataHandler) {
-          XML_Char buf[XML_ENCODE_MAX];
-          parser->m_characterDataHandler(parser->m_handlerArg, buf, XmlEncode(n, (ICHAR *)buf));
+        if ((parser->m_tagLevel == 0)
+            && (parser->m_parsingStatus.parsing != XML_FINISHED)) {
+          if (parser->m_parsingStatus.parsing == XML_SUSPENDED)
+            parser->m_processor = epilogProcessor;
+          else
+            return epilogProcessor(parser, next, end, nextPtr);
         }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
       }
       break;
+    case XML_TOK_CHAR_REF: {
+      int n = XmlCharRefNumber(enc, s);
+      if (n < 0)
+        return XML_ERROR_BAD_CHAR_REF;
+      if (parser->m_characterDataHandler) {
+        XML_Char buf[XML_ENCODE_MAX];
+        parser->m_characterDataHandler(parser->m_handlerArg, buf,
+                                       XmlEncode(n, (ICHAR *)buf));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_XML_DECL:
       return XML_ERROR_MISPLACED_XML_PI;
     case XML_TOK_DATA_NEWLINE:
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       break;
-    case XML_TOK_CDATA_SECT_OPEN:
-      {
-        enum XML_Error result;
-        if (parser->m_startCdataSectionHandler)
-          parser->m_startCdataSectionHandler(parser->m_handlerArg);
-/* BEGIN disabled code */
-        /* Suppose you doing a transformation on a document that involves
-           changing only the character data.  You set up a defaultHandler
-           and a characterDataHandler.  The defaultHandler simply copies
-           characters through.  The characterDataHandler does the
-           transformation and writes the characters out escaping them as
-           necessary.  This case will fail to work if we leave out the
-           following two lines (because & and < inside CDATA sections will
-           be incorrectly escaped).
-
-           However, now we have a start/endCdataSectionHandler, so it seems
-           easier to let the user deal with this.
-        */
-        else if (0 && parser->m_characterDataHandler)
-          parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0);
-/* END disabled code */
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore);
-        if (result != XML_ERROR_NONE)
-          return result;
-        else if (!next) {
-          parser->m_processor = cdataSectionProcessor;
-          return result;
-        }
+    case XML_TOK_CDATA_SECT_OPEN: {
+      enum XML_Error result;
+      if (parser->m_startCdataSectionHandler)
+        parser->m_startCdataSectionHandler(parser->m_handlerArg);
+      /* BEGIN disabled code */
+      /* Suppose you doing a transformation on a document that involves
+         changing only the character data.  You set up a defaultHandler
+         and a characterDataHandler.  The defaultHandler simply copies
+         characters through.  The characterDataHandler does the
+         transformation and writes the characters out escaping them as
+         necessary.  This case will fail to work if we leave out the
+         following two lines (because & and < inside CDATA sections will
+         be incorrectly escaped).
+
+         However, now we have a start/endCdataSectionHandler, so it seems
+         easier to let the user deal with this.
+      */
+      else if (0 && parser->m_characterDataHandler)
+        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                                       0);
+      /* END disabled code */
+      else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore);
+      if (result != XML_ERROR_NONE)
+        return result;
+      else if (! next) {
+        parser->m_processor = cdataSectionProcessor;
+        return result;
       }
-      break;
+    } break;
     case XML_TOK_TRAILING_RSQB:
       if (haveMore) {
         *nextPtr = s;
@@ -3046,15 +2922,14 @@ doContent(XML_Parser parser,
         if (MUST_CONVERT(enc, s)) {
           ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
           XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-          parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                               (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-        }
-        else
-          parser->m_characterDataHandler(parser->m_handlerArg,
-                               (XML_Char *)s,
-                               (int)((XML_Char *)end - (XML_Char *)s));
-      }
-      else if (parser->m_defaultHandler)
+          parser->m_characterDataHandler(
+              parser->m_handlerArg, parser->m_dataBuf,
+              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+        } else
+          parser->m_characterDataHandler(
+              parser->m_handlerArg, (XML_Char *)s,
+              (int)((XML_Char *)end - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, end);
       /* We are at the end of the final buffer, should we check for
          XML_SUSPENDED, XML_FINISHED?
@@ -3069,37 +2944,34 @@ doContent(XML_Parser parser,
       }
       *nextPtr = end;
       return XML_ERROR_NONE;
-    case XML_TOK_DATA_CHARS:
-      {
-        XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
-        if (charDataHandler) {
-          if (MUST_CONVERT(enc, s)) {
-            for (;;) {
-              ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-              const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-              *eventEndPP = s;
-              charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-              if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
-                break;
-              *eventPP = s;
-            }
+    case XML_TOK_DATA_CHARS: {
+      XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
+      if (charDataHandler) {
+        if (MUST_CONVERT(enc, s)) {
+          for (;;) {
+            ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
+            const enum XML_Convert_Result convert_res = XmlConvert(
+                enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+            *eventEndPP = s;
+            charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                            (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+            if ((convert_res == XML_CONVERT_COMPLETED)
+                || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+              break;
+            *eventPP = s;
           }
-          else
-            charDataHandler(parser->m_handlerArg,
-                            (XML_Char *)s,
-                            (int)((XML_Char *)next - (XML_Char *)s));
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-      }
-      break;
+        } else
+          charDataHandler(parser->m_handlerArg, (XML_Char *)s,
+                          (int)((XML_Char *)next - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_PI:
-      if (!reportProcessingInstruction(parser, enc, s, next))
+      if (! reportProcessingInstruction(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_COMMENT:
-      if (!reportComment(parser, enc, s, next))
+      if (! reportComment(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     default:
@@ -3122,7 +2994,7 @@ doContent(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
   /* not reached */
@@ -3133,8 +3005,7 @@ doContent(XML_Parser parser,
  * reused as appropriate.
  */
 static void
-freeBindings(XML_Parser parser, BINDING *bindings)
-{
+freeBindings(XML_Parser parser, BINDING *bindings) {
   while (bindings) {
     BINDING *b = bindings;
 
@@ -3142,7 +3013,7 @@ freeBindings(XML_Parser parser, BINDING *bindings)
      * binding in addBindings(), so call the end handler now.
      */
     if (parser->m_endNamespaceDeclHandler)
-        parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
+      parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name);
 
     bindings = bindings->nextTagBinding;
     b->nextTagBinding = parser->m_freeBindingList;
@@ -3162,14 +3033,12 @@ freeBindings(XML_Parser parser, BINDING *bindings)
    - generate namespace aware element name (URI, prefix)
 */
 static enum XML_Error
-storeAtts(XML_Parser parser, const ENCODING *enc,
-          const char *attStr, TAG_NAME *tagNamePtr,
-          BINDING **bindingsPtr)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr,
+          TAG_NAME *tagNamePtr, BINDING **bindingsPtr) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   ELEMENT_TYPE *elementType;
   int nDefaultAtts;
-  const XML_Char **appAtts;   /* the attribute list for the application */
+  const XML_Char **appAtts; /* the attribute list for the application */
   int attIndex = 0;
   int prefixLen;
   int i;
@@ -3180,16 +3049,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   const XML_Char *localPart;
 
   /* lookup the element type name */
-  elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str,0);
-  if (!elementType) {
+  elementType
+      = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str, 0);
+  if (! elementType) {
     const XML_Char *name = poolCopyString(&dtd->pool, tagNamePtr->str);
-    if (!name)
+    if (! name)
       return XML_ERROR_NO_MEMORY;
     elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, name,
                                          sizeof(ELEMENT_TYPE));
-    if (!elementType)
+    if (! elementType)
       return XML_ERROR_NO_MEMORY;
-    if (parser->m_ns && !setElementTypePrefix(parser, elementType))
+    if (parser->m_ns && ! setElementTypePrefix(parser, elementType))
       return XML_ERROR_NO_MEMORY;
   }
   nDefaultAtts = elementType->nDefaultAtts;
@@ -3203,14 +3073,16 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     XML_AttrInfo *temp2;
 #endif
     parser->m_attsSize = n + nDefaultAtts + INIT_ATTS_SIZE;
-    temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts, parser->m_attsSize * sizeof(ATTRIBUTE));
+    temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts,
+                                parser->m_attsSize * sizeof(ATTRIBUTE));
     if (temp == NULL) {
       parser->m_attsSize = oldAttsSize;
       return XML_ERROR_NO_MEMORY;
     }
     parser->m_atts = temp;
 #ifdef XML_ATTR_INFO
-    temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo, parser->m_attsSize * sizeof(XML_AttrInfo));
+    temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo,
+                                    parser->m_attsSize * sizeof(XML_AttrInfo));
     if (temp2 == NULL) {
       parser->m_attsSize = oldAttsSize;
       return XML_ERROR_NO_MEMORY;
@@ -3228,18 +3100,20 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     XML_AttrInfo *currAttInfo = &parser->m_attInfo[i];
 #endif
     /* add the name and value to the attribute list */
-    ATTRIBUTE_ID *attId = getAttributeId(parser, enc, currAtt->name,
-                                         currAtt->name
-                                         + XmlNameLength(enc, currAtt->name));
-    if (!attId)
+    ATTRIBUTE_ID *attId
+        = getAttributeId(parser, enc, currAtt->name,
+                         currAtt->name + XmlNameLength(enc, currAtt->name));
+    if (! attId)
       return XML_ERROR_NO_MEMORY;
 #ifdef XML_ATTR_INFO
-    currAttInfo->nameStart = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name);
-    currAttInfo->nameEnd = currAttInfo->nameStart +
-                           XmlNameLength(enc, currAtt->name);
-    currAttInfo->valueStart = parser->m_parseEndByteIndex -
-                            (parser->m_parseEndPtr - currAtt->valuePtr);
-    currAttInfo->valueEnd = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->valueEnd);
+    currAttInfo->nameStart
+        = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name);
+    currAttInfo->nameEnd
+        = currAttInfo->nameStart + XmlNameLength(enc, currAtt->name);
+    currAttInfo->valueStart = parser->m_parseEndByteIndex
+                              - (parser->m_parseEndPtr - currAtt->valuePtr);
+    currAttInfo->valueEnd = parser->m_parseEndByteIndex
+                            - (parser->m_parseEndPtr - currAtt->valueEnd);
 #endif
     /* Detect duplicate attributes by their QNames. This does not work when
        namespace processing is turned on and different prefixes for the same
@@ -3252,7 +3126,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     }
     (attId->name)[-1] = 1;
     appAtts[attIndex++] = attId->name;
-    if (!parser->m_atts[i].normalized) {
+    if (! parser->m_atts[i].normalized) {
       enum XML_Error result;
       XML_Bool isCdata = XML_TRUE;
 
@@ -3268,17 +3142,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
       }
 
       /* normalize the attribute value */
-      result = storeAttributeValue(parser, enc, isCdata,
-                                   parser->m_atts[i].valuePtr, parser->m_atts[i].valueEnd,
-                                   &parser->m_tempPool);
+      result = storeAttributeValue(
+          parser, enc, isCdata, parser->m_atts[i].valuePtr,
+          parser->m_atts[i].valueEnd, &parser->m_tempPool);
       if (result)
         return result;
       appAtts[attIndex] = poolStart(&parser->m_tempPool);
       poolFinish(&parser->m_tempPool);
-    }
-    else {
+    } else {
       /* the value did not need normalizing */
-      appAtts[attIndex] = poolStoreString(&parser->m_tempPool, enc, parser->m_atts[i].valuePtr,
+      appAtts[attIndex] = poolStoreString(&parser->m_tempPool, enc,
+                                          parser->m_atts[i].valuePtr,
                                           parser->m_atts[i].valueEnd);
       if (appAtts[attIndex] == 0)
         return XML_ERROR_NO_MEMORY;
@@ -3293,15 +3167,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         if (result)
           return result;
         --attIndex;
-      }
-      else {
+      } else {
         /* deal with other prefixed names later */
         attIndex++;
         nPrefixes++;
         (attId->name)[-1] = 2;
       }
-    }
-    else
+    } else
       attIndex++;
   }
 
@@ -3313,29 +3185,26 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         parser->m_idAttIndex = i;
         break;
       }
-  }
-  else
+  } else
     parser->m_idAttIndex = -1;
 
   /* do attribute defaulting */
   for (i = 0; i < nDefaultAtts; i++) {
     const DEFAULT_ATTRIBUTE *da = elementType->defaultAtts + i;
-    if (!(da->id->name)[-1] && da->value) {
+    if (! (da->id->name)[-1] && da->value) {
       if (da->id->prefix) {
         if (da->id->xmlns) {
           enum XML_Error result = addBinding(parser, da->id->prefix, da->id,
                                              da->value, bindingsPtr);
           if (result)
             return result;
-        }
-        else {
+        } else {
           (da->id->name)[-1] = 2;
           nPrefixes++;
           appAtts[attIndex++] = da->id->name;
           appAtts[attIndex++] = da->value;
         }
-      }
-      else {
+      } else {
         (da->id->name)[-1] = 1;
         appAtts[attIndex++] = da->id->name;
         appAtts[attIndex++] = da->value;
@@ -3348,31 +3217,34 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
      and clear flags that say whether attributes were specified */
   i = 0;
   if (nPrefixes) {
-    int j;  /* hash table index */
+    int j; /* hash table index */
     unsigned long version = parser->m_nsAttsVersion;
     int nsAttsSize = (int)1 << parser->m_nsAttsPower;
     unsigned char oldNsAttsPower = parser->m_nsAttsPower;
     /* size of hash table must be at least 2 * (# of prefixed attributes) */
-    if ((nPrefixes << 1) >> parser->m_nsAttsPower) {  /* true for m_nsAttsPower = 0 */
+    if ((nPrefixes << 1)
+        >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */
       NS_ATT *temp;
       /* hash table size must also be a power of 2 and >= 8 */
-      while (nPrefixes >> parser->m_nsAttsPower++);
+      while (nPrefixes >> parser->m_nsAttsPower++)
+        ;
       if (parser->m_nsAttsPower < 3)
         parser->m_nsAttsPower = 3;
       nsAttsSize = (int)1 << parser->m_nsAttsPower;
-      temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT));
-      if (!temp) {
+      temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts,
+                               nsAttsSize * sizeof(NS_ATT));
+      if (! temp) {
         /* Restore actual size of memory in m_nsAtts */
         parser->m_nsAttsPower = oldNsAttsPower;
         return XML_ERROR_NO_MEMORY;
       }
       parser->m_nsAtts = temp;
-      version = 0;  /* force re-initialization of m_nsAtts hash table */
+      version = 0; /* force re-initialization of m_nsAtts hash table */
     }
     /* using a version flag saves us from initializing m_nsAtts every time */
-    if (!version) {  /* initialize version flags when version wraps around */
+    if (! version) { /* initialize version flags when version wraps around */
       version = INIT_ATTS_VERSION;
-      for (j = nsAttsSize; j != 0; )
+      for (j = nsAttsSize; j != 0;)
         parser->m_nsAtts[--j].version = version;
     }
     parser->m_nsAttsVersion = --version;
@@ -3380,7 +3252,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
     /* expand prefixed names and check for duplicates */
     for (; i < attIndex; i += 2) {
       const XML_Char *s = appAtts[i];
-      if (s[-1] == 2) {  /* prefixed */
+      if (s[-1] == 2) { /* prefixed */
         ATTRIBUTE_ID *id;
         const BINDING *b;
         unsigned long uriHash;
@@ -3390,9 +3262,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         copy_salt_to_sipkey(parser, &sip_key);
         sip24_init(&sip_state, &sip_key);
 
-        ((XML_Char *)s)[-1] = 0;  /* clear flag */
+        ((XML_Char *)s)[-1] = 0; /* clear flag */
         id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0);
-        if (!id || !id->prefix) {
+        if (! id || ! id->prefix) {
           /* This code is walking through the appAtts array, dealing
            * with (in this case) a prefixed attribute name.  To be in
            * the array, the attribute must have already been bound, so
@@ -3410,12 +3282,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
           return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */
         }
         b = id->prefix->binding;
-        if (!b)
+        if (! b)
           return XML_ERROR_UNBOUND_PREFIX;
 
         for (j = 0; j < b->uriLen; j++) {
           const XML_Char c = b->uri[j];
-          if (!poolAppendChar(&parser->m_tempPool, c))
+          if (! poolAppendChar(&parser->m_tempPool, c))
             return XML_ERROR_NO_MEMORY;
         }
 
@@ -3426,8 +3298,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
 
         sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char));
 
-        do {  /* copies null terminator */
-          if (!poolAppendChar(&parser->m_tempPool, *s))
+        do { /* copies null terminator */
+          if (! poolAppendChar(&parser->m_tempPool, *s))
             return XML_ERROR_NO_MEMORY;
         } while (*s++);
 
@@ -3438,28 +3310,29 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
           */
           unsigned char step = 0;
           unsigned long mask = nsAttsSize - 1;
-          j = uriHash & mask;  /* index into hash table */
+          j = uriHash & mask; /* index into hash table */
           while (parser->m_nsAtts[j].version == version) {
             /* for speed we compare stored hash values first */
             if (uriHash == parser->m_nsAtts[j].hash) {
               const XML_Char *s1 = poolStart(&parser->m_tempPool);
               const XML_Char *s2 = parser->m_nsAtts[j].uriName;
               /* s1 is null terminated, but not s2 */
-              for (; *s1 == *s2 && *s1 != 0; s1++, s2++);
+              for (; *s1 == *s2 && *s1 != 0; s1++, s2++)
+                ;
               if (*s1 == 0)
                 return XML_ERROR_DUPLICATE_ATTRIBUTE;
             }
-            if (!step)
+            if (! step)
               step = PROBE_STEP(uriHash, mask, parser->m_nsAttsPower);
             j < step ? (j += nsAttsSize - step) : (j -= step);
           }
         }
 
-        if (parser->m_ns_triplets) {  /* append namespace separator and prefix */
+        if (parser->m_ns_triplets) { /* append namespace separator and prefix */
           parser->m_tempPool.ptr[-1] = parser->m_namespaceSeparator;
           s = b->prefix->name;
           do {
-            if (!poolAppendChar(&parser->m_tempPool, *s))
+            if (! poolAppendChar(&parser->m_tempPool, *s))
               return XML_ERROR_NO_MEMORY;
           } while (*s++);
         }
@@ -3474,13 +3347,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
         parser->m_nsAtts[j].hash = uriHash;
         parser->m_nsAtts[j].uriName = s;
 
-        if (!--nPrefixes) {
+        if (! --nPrefixes) {
           i += 2;
           break;
         }
-      }
-      else  /* not prefixed */
-        ((XML_Char *)s)[-1] = 0;  /* clear flag */
+      } else                     /* not prefixed */
+        ((XML_Char *)s)[-1] = 0; /* clear flag */
     }
   }
   /* clear flags for the remaining attributes */
@@ -3489,40 +3361,38 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   for (binding = *bindingsPtr; binding; binding = binding->nextTagBinding)
     binding->attId->name[-1] = 0;
 
-  if (!parser->m_ns)
+  if (! parser->m_ns)
     return XML_ERROR_NONE;
 
   /* expand the element type name */
   if (elementType->prefix) {
     binding = elementType->prefix->binding;
-    if (!binding)
+    if (! binding)
       return XML_ERROR_UNBOUND_PREFIX;
     localPart = tagNamePtr->str;
     while (*localPart++ != XML_T(ASCII_COLON))
       ;
-  }
-  else if (dtd->defaultPrefix.binding) {
+  } else if (dtd->defaultPrefix.binding) {
     binding = dtd->defaultPrefix.binding;
     localPart = tagNamePtr->str;
-  }
-  else
+  } else
     return XML_ERROR_NONE;
   prefixLen = 0;
   if (parser->m_ns_triplets && binding->prefix->name) {
     for (; binding->prefix->name[prefixLen++];)
-      ;  /* prefixLen includes null terminator */
+      ; /* prefixLen includes null terminator */
   }
   tagNamePtr->localPart = localPart;
   tagNamePtr->uriLen = binding->uriLen;
   tagNamePtr->prefix = binding->prefix->name;
   tagNamePtr->prefixLen = prefixLen;
   for (i = 0; localPart[i++];)
-    ;  /* i includes null terminator */
+    ; /* i includes null terminator */
   n = i + binding->uriLen + prefixLen;
   if (n > binding->uriAlloc) {
     TAG *p;
     uri = (XML_Char *)MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char));
-    if (!uri)
+    if (! uri)
       return XML_ERROR_NO_MEMORY;
     binding->uriAlloc = n + EXPAND_SPARE;
     memcpy(uri, binding->uri, binding->uriLen * sizeof(XML_Char));
@@ -3538,7 +3408,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
   /* we always have a namespace separator between localPart and prefix */
   if (prefixLen) {
     uri += i - 1;
-    *uri = parser->m_namespaceSeparator;  /* replace null terminator */
+    *uri = parser->m_namespaceSeparator; /* replace null terminator */
     memcpy(uri + 1, binding->prefix->name, prefixLen * sizeof(XML_Char));
   }
   tagNamePtr->str = binding->uri;
@@ -3550,27 +3420,25 @@ storeAtts(XML_Parser parser, const ENCODING *enc,
 */
 static enum XML_Error
 addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
-           const XML_Char *uri, BINDING **bindingsPtr)
-{
-  static const XML_Char xmlNamespace[] = {
-    ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, ASCII_SLASH,
-    ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD,
-    ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L,
-    ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8, ASCII_SLASH,
-    ASCII_n, ASCII_a, ASCII_m, ASCII_e, ASCII_s, ASCII_p, ASCII_a, ASCII_c,
-    ASCII_e, '\0'
-  };
-  static const int xmlLen =
-    (int)sizeof(xmlNamespace)/sizeof(XML_Char) - 1;
-  static const XML_Char xmlnsNamespace[] = {
-    ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, ASCII_SLASH,
-    ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w, ASCII_3, ASCII_PERIOD,
-    ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_2, ASCII_0, ASCII_0,
-    ASCII_0, ASCII_SLASH, ASCII_x, ASCII_m, ASCII_l, ASCII_n, ASCII_s,
-    ASCII_SLASH, '\0'
-  };
-  static const int xmlnsLen =
-    (int)sizeof(xmlnsNamespace)/sizeof(XML_Char) - 1;
+           const XML_Char *uri, BINDING **bindingsPtr) {
+  static const XML_Char xmlNamespace[]
+      = {ASCII_h,      ASCII_t,     ASCII_t,     ASCII_p,      ASCII_COLON,
+         ASCII_SLASH,  ASCII_SLASH, ASCII_w,     ASCII_w,      ASCII_w,
+         ASCII_PERIOD, ASCII_w,     ASCII_3,     ASCII_PERIOD, ASCII_o,
+         ASCII_r,      ASCII_g,     ASCII_SLASH, ASCII_X,      ASCII_M,
+         ASCII_L,      ASCII_SLASH, ASCII_1,     ASCII_9,      ASCII_9,
+         ASCII_8,      ASCII_SLASH, ASCII_n,     ASCII_a,      ASCII_m,
+         ASCII_e,      ASCII_s,     ASCII_p,     ASCII_a,      ASCII_c,
+         ASCII_e,      '\0'};
+  static const int xmlLen = (int)sizeof(xmlNamespace) / sizeof(XML_Char) - 1;
+  static const XML_Char xmlnsNamespace[]
+      = {ASCII_h,     ASCII_t,      ASCII_t, ASCII_p, ASCII_COLON,  ASCII_SLASH,
+         ASCII_SLASH, ASCII_w,      ASCII_w, ASCII_w, ASCII_PERIOD, ASCII_w,
+         ASCII_3,     ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g,      ASCII_SLASH,
+         ASCII_2,     ASCII_0,      ASCII_0, ASCII_0, ASCII_SLASH,  ASCII_x,
+         ASCII_m,     ASCII_l,      ASCII_n, ASCII_s, ASCII_SLASH,  '\0'};
+  static const int xmlnsLen
+      = (int)sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1;
 
   XML_Bool mustBeXML = XML_FALSE;
   XML_Bool isXML = XML_TRUE;
@@ -3583,14 +3451,11 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   if (*uri == XML_T('\0') && prefix->name)
     return XML_ERROR_UNDECLARING_PREFIX;
 
-  if (prefix->name
-      && prefix->name[0] == XML_T(ASCII_x)
+  if (prefix->name && prefix->name[0] == XML_T(ASCII_x)
       && prefix->name[1] == XML_T(ASCII_m)
       && prefix->name[2] == XML_T(ASCII_l)) {
-
     /* Not allowed to bind xmlns */
-    if (prefix->name[3] == XML_T(ASCII_n)
-        && prefix->name[4] == XML_T(ASCII_s)
+    if (prefix->name[3] == XML_T(ASCII_n) && prefix->name[4] == XML_T(ASCII_s)
         && prefix->name[5] == XML_T('\0'))
       return XML_ERROR_RESERVED_PREFIX_XMLNS;
 
@@ -3602,7 +3467,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
     if (isXML && (len > xmlLen || uri[len] != xmlNamespace[len]))
       isXML = XML_FALSE;
 
-    if (!mustBeXML && isXMLNS
+    if (! mustBeXML && isXMLNS
         && (len > xmlnsLen || uri[len] != xmlnsNamespace[len]))
       isXMLNS = XML_FALSE;
   }
@@ -3621,21 +3486,21 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   if (parser->m_freeBindingList) {
     b = parser->m_freeBindingList;
     if (len > b->uriAlloc) {
-      XML_Char *temp = (XML_Char *)REALLOC(parser, b->uri,
-                          sizeof(XML_Char) * (len + EXPAND_SPARE));
+      XML_Char *temp = (XML_Char *)REALLOC(
+          parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE));
       if (temp == NULL)
         return XML_ERROR_NO_MEMORY;
       b->uri = temp;
       b->uriAlloc = len + EXPAND_SPARE;
     }
     parser->m_freeBindingList = b->nextTagBinding;
-  }
-  else {
+  } else {
     b = (BINDING *)MALLOC(parser, sizeof(BINDING));
-    if (!b)
+    if (! b)
       return XML_ERROR_NO_MEMORY;
-    b->uri = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE));
-    if (!b->uri) {
+    b->uri
+        = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE));
+    if (! b->uri) {
       FREE(parser, b);
       return XML_ERROR_NO_MEMORY;
     }
@@ -3658,7 +3523,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
   /* if attId == NULL then we are not starting a namespace scope */
   if (attId && parser->m_startNamespaceDeclHandler)
     parser->m_startNamespaceDeclHandler(parser->m_handlerArg, prefix->name,
-                              prefix->binding ? uri : 0);
+                                        prefix->binding ? uri : 0);
   return XML_ERROR_NONE;
 }
 
@@ -3666,21 +3531,18 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId,
    the whole file is parsed with one call.
 */
 static enum XML_Error PTRCALL
-cdataSectionProcessor(XML_Parser parser,
-                      const char *start,
-                      const char *end,
-                      const char **endPtr)
-{
-  enum XML_Error result = doCdataSection(parser, parser->m_encoding, &start, end,
-                                         endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+cdataSectionProcessor(XML_Parser parser, const char *start, const char *end,
+                      const char **endPtr) {
+  enum XML_Error result
+      = doCdataSection(parser, parser->m_encoding, &start, end, endPtr,
+                       (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result != XML_ERROR_NONE)
     return result;
   if (start) {
-    if (parser->m_parentParser) {  /* we are parsing an external entity */
+    if (parser->m_parentParser) { /* we are parsing an external entity */
       parser->m_processor = externalEntityContentProcessor;
       return externalEntityContentProcessor(parser, start, end, endPtr);
-    }
-    else {
+    } else {
       parser->m_processor = contentProcessor;
       return contentProcessor(parser, start, end, endPtr);
     }
@@ -3692,13 +3554,8 @@ cdataSectionProcessor(XML_Parser parser,
    the section is not yet closed.
 */
 static enum XML_Error
-doCdataSection(XML_Parser parser,
-               const ENCODING *enc,
-               const char **startPtr,
-               const char *end,
-               const char **nextPtr,
-               XML_Bool haveMore)
-{
+doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
+               const char *end, const char **nextPtr, XML_Bool haveMore) {
   const char *s = *startPtr;
   const char **eventPP;
   const char **eventEndPP;
@@ -3706,8 +3563,7 @@ doCdataSection(XML_Parser parser,
     eventPP = &parser->m_eventPtr;
     *eventPP = s;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -3722,11 +3578,12 @@ doCdataSection(XML_Parser parser,
     case XML_TOK_CDATA_SECT_CLOSE:
       if (parser->m_endCdataSectionHandler)
         parser->m_endCdataSectionHandler(parser->m_handlerArg);
-/* BEGIN disabled code */
+      /* BEGIN disabled code */
       /* see comment under XML_TOK_CDATA_SECT_OPEN */
       else if (0 && parser->m_characterDataHandler)
-        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0);
-/* END disabled code */
+        parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                                       0);
+      /* END disabled code */
       else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       *startPtr = next;
@@ -3739,35 +3596,31 @@ doCdataSection(XML_Parser parser,
       if (parser->m_characterDataHandler) {
         XML_Char c = 0xA;
         parser->m_characterDataHandler(parser->m_handlerArg, &c, 1);
-      }
-      else if (parser->m_defaultHandler)
+      } else if (parser->m_defaultHandler)
         reportDefault(parser, enc, s, next);
       break;
-    case XML_TOK_DATA_CHARS:
-      {
-        XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
-        if (charDataHandler) {
-          if (MUST_CONVERT(enc, s)) {
-            for (;;) {
-              ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-              const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
-              *eventEndPP = next;
-              charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
-                              (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
-              if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
-                break;
-              *eventPP = s;
-            }
+    case XML_TOK_DATA_CHARS: {
+      XML_CharacterDataHandler charDataHandler = parser->m_characterDataHandler;
+      if (charDataHandler) {
+        if (MUST_CONVERT(enc, s)) {
+          for (;;) {
+            ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
+            const enum XML_Convert_Result convert_res = XmlConvert(
+                enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+            *eventEndPP = next;
+            charDataHandler(parser->m_handlerArg, parser->m_dataBuf,
+                            (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+            if ((convert_res == XML_CONVERT_COMPLETED)
+                || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+              break;
+            *eventPP = s;
           }
-          else
-            charDataHandler(parser->m_handlerArg,
-                            (XML_Char *)s,
-                            (int)((XML_Char *)next - (XML_Char *)s));
-        }
-        else if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-      }
-      break;
+        } else
+          charDataHandler(parser->m_handlerArg, (XML_Char *)s,
+                          (int)((XML_Char *)next - (XML_Char *)s));
+      } else if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+    } break;
     case XML_TOK_INVALID:
       *eventPP = next;
       return XML_ERROR_INVALID_TOKEN;
@@ -3791,7 +3644,7 @@ doCdataSection(XML_Parser parser,
        * statistics.
        *
        * LCOV_EXCL_START
-      */
+       */
       *eventPP = next;
       return XML_ERROR_UNEXPECTED_STATE;
       /* LCOV_EXCL_STOP */
@@ -3804,7 +3657,7 @@ doCdataSection(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
   /* not reached */
@@ -3816,13 +3669,11 @@ doCdataSection(XML_Parser parser,
    the whole file is parsed with one call.
 */
 static enum XML_Error PTRCALL
-ignoreSectionProcessor(XML_Parser parser,
-                       const char *start,
-                       const char *end,
-                       const char **endPtr)
-{
-  enum XML_Error result = doIgnoreSection(parser, parser->m_encoding, &start, end,
-                                          endPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end,
+                       const char **endPtr) {
+  enum XML_Error result
+      = doIgnoreSection(parser, parser->m_encoding, &start, end, endPtr,
+                        (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   if (result != XML_ERROR_NONE)
     return result;
   if (start) {
@@ -3836,13 +3687,8 @@ ignoreSectionProcessor(XML_Parser parser,
    if the section is not yet closed.
 */
 static enum XML_Error
-doIgnoreSection(XML_Parser parser,
-                const ENCODING *enc,
-                const char **startPtr,
-                const char *end,
-                const char **nextPtr,
-                XML_Bool haveMore)
-{
+doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr,
+                const char *end, const char **nextPtr, XML_Bool haveMore) {
   const char *next;
   int tok;
   const char *s = *startPtr;
@@ -3852,8 +3698,7 @@ doIgnoreSection(XML_Parser parser,
     eventPP = &parser->m_eventPtr;
     *eventPP = s;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     /* It's not entirely clear, but it seems the following two lines
      * of code cannot be executed.  The only occasions on which 'enc'
      * is not 'encoding' are when this function is called
@@ -3917,13 +3762,12 @@ doIgnoreSection(XML_Parser parser,
 #endif /* XML_DTD */
 
 static enum XML_Error
-initializeEncoding(XML_Parser parser)
-{
+initializeEncoding(XML_Parser parser) {
   const char *s;
 #ifdef XML_UNICODE
   char encodingBuf[128];
   /* See comments abount `protoclEncodingName` in parserInit() */
-  if (!parser->m_protocolEncodingName)
+  if (! parser->m_protocolEncodingName)
     s = NULL;
   else {
     int i;
@@ -3941,15 +3785,15 @@ initializeEncoding(XML_Parser parser)
 #else
   s = parser->m_protocolEncodingName;
 #endif
-  if ((parser->m_ns ? XmlInitEncodingNS : XmlInitEncoding)(&parser->m_initEncoding, &parser->m_encoding, s))
+  if ((parser->m_ns ? XmlInitEncodingNS : XmlInitEncoding)(
+          &parser->m_initEncoding, &parser->m_encoding, s))
     return XML_ERROR_NONE;
   return handleUnknownEncoding(parser, parser->m_protocolEncodingName);
 }
 
 static enum XML_Error
-processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
-               const char *s, const char *next)
-{
+processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s,
+               const char *next) {
   const char *encodingName = NULL;
   const XML_Char *storedEncName = NULL;
   const ENCODING *newEncoding = NULL;
@@ -3957,52 +3801,41 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
   const char *versionend;
   const XML_Char *storedversion = NULL;
   int standalone = -1;
-  if (!(parser->m_ns
-        ? XmlParseXmlDeclNS
-        : XmlParseXmlDecl)(isGeneralTextEntity,
-                           parser->m_encoding,
-                           s,
-                           next,
-                           &parser->m_eventPtr,
-                           &version,
-                           &versionend,
-                           &encodingName,
-                           &newEncoding,
-                           &standalone)) {
+  if (! (parser->m_ns ? XmlParseXmlDeclNS : XmlParseXmlDecl)(
+          isGeneralTextEntity, parser->m_encoding, s, next, &parser->m_eventPtr,
+          &version, &versionend, &encodingName, &newEncoding, &standalone)) {
     if (isGeneralTextEntity)
       return XML_ERROR_TEXT_DECL;
     else
       return XML_ERROR_XML_DECL;
   }
-  if (!isGeneralTextEntity && standalone == 1) {
+  if (! isGeneralTextEntity && standalone == 1) {
     parser->m_dtd->standalone = XML_TRUE;
 #ifdef XML_DTD
-    if (parser->m_paramEntityParsing == XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE)
+    if (parser->m_paramEntityParsing
+        == XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE)
       parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER;
 #endif /* XML_DTD */
   }
   if (parser->m_xmlDeclHandler) {
     if (encodingName != NULL) {
-      storedEncName = poolStoreString(&parser->m_temp2Pool,
-                                      parser->m_encoding,
-                                      encodingName,
-                                      encodingName
-                                      + XmlNameLength(parser->m_encoding, encodingName));
-      if (!storedEncName)
-              return XML_ERROR_NO_MEMORY;
+      storedEncName = poolStoreString(
+          &parser->m_temp2Pool, parser->m_encoding, encodingName,
+          encodingName + XmlNameLength(parser->m_encoding, encodingName));
+      if (! storedEncName)
+        return XML_ERROR_NO_MEMORY;
       poolFinish(&parser->m_temp2Pool);
     }
     if (version) {
-      storedversion = poolStoreString(&parser->m_temp2Pool,
-                                      parser->m_encoding,
-                                      version,
-                                      versionend - parser->m_encoding->minBytesPerChar);
-      if (!storedversion)
+      storedversion
+          = poolStoreString(&parser->m_temp2Pool, parser->m_encoding, version,
+                            versionend - parser->m_encoding->minBytesPerChar);
+      if (! storedversion)
         return XML_ERROR_NO_MEMORY;
     }
-    parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName, standalone);
-  }
-  else if (parser->m_defaultHandler)
+    parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName,
+                             standalone);
+  } else if (parser->m_defaultHandler)
     reportDefault(parser, parser->m_encoding, s, next);
   if (parser->m_protocolEncodingName == NULL) {
     if (newEncoding) {
@@ -4012,20 +3845,19 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
        * this is UTF-16, is it the same endianness?
        */
       if (newEncoding->minBytesPerChar != parser->m_encoding->minBytesPerChar
-          || (newEncoding->minBytesPerChar == 2 &&
-              newEncoding != parser->m_encoding)) {
+          || (newEncoding->minBytesPerChar == 2
+              && newEncoding != parser->m_encoding)) {
         parser->m_eventPtr = encodingName;
         return XML_ERROR_INCORRECT_ENCODING;
       }
       parser->m_encoding = newEncoding;
-    }
-    else if (encodingName) {
+    } else if (encodingName) {
       enum XML_Error result;
-      if (!storedEncName) {
+      if (! storedEncName) {
         storedEncName = poolStoreString(
-          &parser->m_temp2Pool, parser->m_encoding, encodingName,
-          encodingName + XmlNameLength(parser->m_encoding, encodingName));
-        if (!storedEncName)
+            &parser->m_temp2Pool, parser->m_encoding, encodingName,
+            encodingName + XmlNameLength(parser->m_encoding, encodingName));
+        if (! storedEncName)
           return XML_ERROR_NO_MEMORY;
       }
       result = handleUnknownEncoding(parser, storedEncName);
@@ -4043,8 +3875,7 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity,
 }
 
 static enum XML_Error
-handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
-{
+handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) {
   if (parser->m_unknownEncodingHandler) {
     XML_Encoding info;
     int i;
@@ -4053,21 +3884,17 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
     info.convert = NULL;
     info.data = NULL;
     info.release = NULL;
-    if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData, encodingName,
-                               &info)) {
+    if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData,
+                                         encodingName, &info)) {
       ENCODING *enc;
       parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding());
-      if (!parser->m_unknownEncodingMem) {
+      if (! parser->m_unknownEncodingMem) {
         if (info.release)
           info.release(info.data);
         return XML_ERROR_NO_MEMORY;
       }
-      enc = (parser->m_ns
-             ? XmlInitUnknownEncodingNS
-             : XmlInitUnknownEncoding)(parser->m_unknownEncodingMem,
-                                       info.map,
-                                       info.convert,
-                                       info.data);
+      enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)(
+          parser->m_unknownEncodingMem, info.map, info.convert, info.data);
       if (enc) {
         parser->m_unknownEncodingData = info.data;
         parser->m_unknownEncodingRelease = info.release;
@@ -4082,11 +3909,8 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName)
 }
 
 static enum XML_Error PTRCALL
-prologInitProcessor(XML_Parser parser,
-                    const char *s,
-                    const char *end,
-                    const char **nextPtr)
-{
+prologInitProcessor(XML_Parser parser, const char *s, const char *end,
+                    const char **nextPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -4097,11 +3921,8 @@ prologInitProcessor(XML_Parser parser,
 #ifdef XML_DTD
 
 static enum XML_Error PTRCALL
-externalParEntInitProcessor(XML_Parser parser,
-                            const char *s,
-                            const char *end,
-                            const char **nextPtr)
-{
+externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end,
+                            const char **nextPtr) {
   enum XML_Error result = initializeEncoding(parser);
   if (result != XML_ERROR_NONE)
     return result;
@@ -4113,19 +3934,15 @@ externalParEntInitProcessor(XML_Parser parser,
   if (parser->m_prologState.inEntityValue) {
     parser->m_processor = entityValueInitProcessor;
     return entityValueInitProcessor(parser, s, end, nextPtr);
-  }
-  else {
+  } else {
     parser->m_processor = externalParEntProcessor;
     return externalParEntProcessor(parser, s, end, nextPtr);
   }
 }
 
 static enum XML_Error PTRCALL
-entityValueInitProcessor(XML_Parser parser,
-                         const char *s,
-                         const char *end,
-                         const char **nextPtr)
-{
+entityValueInitProcessor(XML_Parser parser, const char *s, const char *end,
+                         const char **nextPtr) {
   int tok;
   const char *start = s;
   const char *next = start;
@@ -4135,7 +3952,7 @@ entityValueInitProcessor(XML_Parser parser,
     tok = XmlPrologTok(parser->m_encoding, start, end, &next);
     parser->m_eventEndPtr = next;
     if (tok <= 0) {
-      if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+      if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -4146,22 +3963,21 @@ entityValueInitProcessor(XML_Parser parser,
         return XML_ERROR_UNCLOSED_TOKEN;
       case XML_TOK_PARTIAL_CHAR:
         return XML_ERROR_PARTIAL_CHAR;
-      case XML_TOK_NONE:   /* start == end */
+      case XML_TOK_NONE: /* start == end */
       default:
         break;
       }
       /* found end of entity value - can store it now */
       return storeEntityValue(parser, parser->m_encoding, s, end);
-    }
-    else if (tok == XML_TOK_XML_DECL) {
+    } else if (tok == XML_TOK_XML_DECL) {
       enum XML_Error result;
       result = processXmlDecl(parser, 0, start, next);
       if (result != XML_ERROR_NONE)
         return result;
-      /* At this point, m_parsingStatus.parsing cannot be XML_SUSPENDED.  For that
-       * to happen, a parameter entity parsing handler must have
-       * attempted to suspend the parser, which fails and raises an
-       * error.  The parser can be aborted, but can't be suspended.
+      /* At this point, m_parsingStatus.parsing cannot be XML_SUSPENDED.  For
+       * that to happen, a parameter entity parsing handler must have attempted
+       * to suspend the parser, which fails and raises an error.  The parser can
+       * be aborted, but can't be suspended.
        */
       if (parser->m_parsingStatus.parsing == XML_FINISHED)
         return XML_ERROR_ABORTED;
@@ -4177,7 +3993,8 @@ entityValueInitProcessor(XML_Parser parser,
        then, when this routine is entered the next time, XmlPrologTok will
        return XML_TOK_INVALID, since the BOM is still in the buffer
     */
-    else if (tok == XML_TOK_BOM && next == end && !parser->m_parsingStatus.finalBuffer) {
+    else if (tok == XML_TOK_BOM && next == end
+             && ! parser->m_parsingStatus.finalBuffer) {
       *nextPtr = next;
       return XML_ERROR_NONE;
     }
@@ -4195,17 +4012,14 @@ entityValueInitProcessor(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-externalParEntProcessor(XML_Parser parser,
-                        const char *s,
-                        const char *end,
-                        const char **nextPtr)
-{
+externalParEntProcessor(XML_Parser parser, const char *s, const char *end,
+                        const char **nextPtr) {
   const char *next = s;
   int tok;
 
   tok = XmlPrologTok(parser->m_encoding, s, end, &next);
   if (tok <= 0) {
-    if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+    if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
       *nextPtr = s;
       return XML_ERROR_NONE;
     }
@@ -4216,7 +4030,7 @@ externalParEntProcessor(XML_Parser parser,
       return XML_ERROR_UNCLOSED_TOKEN;
     case XML_TOK_PARTIAL_CHAR:
       return XML_ERROR_PARTIAL_CHAR;
-    case XML_TOK_NONE:   /* start == end */
+    case XML_TOK_NONE: /* start == end */
     default:
       break;
     }
@@ -4231,16 +4045,13 @@ externalParEntProcessor(XML_Parser parser,
   }
 
   parser->m_processor = prologProcessor;
-  return doProlog(parser, parser->m_encoding, s, end, tok, next,
-                  nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+  return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
 }
 
 static enum XML_Error PTRCALL
-entityValueProcessor(XML_Parser parser,
-                     const char *s,
-                     const char *end,
-                     const char **nextPtr)
-{
+entityValueProcessor(XML_Parser parser, const char *s, const char *end,
+                     const char **nextPtr) {
   const char *start = s;
   const char *next = s;
   const ENCODING *enc = parser->m_encoding;
@@ -4249,7 +4060,7 @@ entityValueProcessor(XML_Parser parser,
   for (;;) {
     tok = XmlPrologTok(enc, start, end, &next);
     if (tok <= 0) {
-      if (!parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
+      if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -4260,7 +4071,7 @@ entityValueProcessor(XML_Parser parser,
         return XML_ERROR_UNCLOSED_TOKEN;
       case XML_TOK_PARTIAL_CHAR:
         return XML_ERROR_PARTIAL_CHAR;
-      case XML_TOK_NONE:   /* start == end */
+      case XML_TOK_NONE: /* start == end */
       default:
         break;
       }
@@ -4274,52 +4085,46 @@ entityValueProcessor(XML_Parser parser,
 #endif /* XML_DTD */
 
 static enum XML_Error PTRCALL
-prologProcessor(XML_Parser parser,
-                const char *s,
-                const char *end,
-                const char **nextPtr)
-{
+prologProcessor(XML_Parser parser, const char *s, const char *end,
+                const char **nextPtr) {
   const char *next = s;
   int tok = XmlPrologTok(parser->m_encoding, s, end, &next);
-  return doProlog(parser, parser->m_encoding, s, end, tok, next,
-                  nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+  return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
+                  (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
 }
 
 static enum XML_Error
-doProlog(XML_Parser parser,
-         const ENCODING *enc,
-         const char *s,
-         const char *end,
-         int tok,
-         const char *next,
-         const char **nextPtr,
-         XML_Bool haveMore)
-{
+doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end,
+         int tok, const char *next, const char **nextPtr, XML_Bool haveMore,
+         XML_Bool allowClosingDoctype) {
 #ifdef XML_DTD
-  static const XML_Char externalSubsetName[] = { ASCII_HASH , '\0' };
+  static const XML_Char externalSubsetName[] = {ASCII_HASH, '\0'};
 #endif /* XML_DTD */
-  static const XML_Char atypeCDATA[] =
-      { ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-  static const XML_Char atypeID[] = { ASCII_I, ASCII_D, '\0' };
-  static const XML_Char atypeIDREF[] =
-      { ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0' };
-  static const XML_Char atypeIDREFS[] =
-      { ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0' };
-  static const XML_Char atypeENTITY[] =
-      { ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0' };
-  static const XML_Char atypeENTITIES[] = { ASCII_E, ASCII_N,
-      ASCII_T, ASCII_I, ASCII_T, ASCII_I, ASCII_E, ASCII_S, '\0' };
-  static const XML_Char atypeNMTOKEN[] = {
-      ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0' };
-  static const XML_Char atypeNMTOKENS[] = { ASCII_N, ASCII_M, ASCII_T,
-      ASCII_O, ASCII_K, ASCII_E, ASCII_N, ASCII_S, '\0' };
-  static const XML_Char notationPrefix[] = { ASCII_N, ASCII_O, ASCII_T,
-      ASCII_A, ASCII_T, ASCII_I, ASCII_O, ASCII_N, ASCII_LPAREN, '\0' };
-  static const XML_Char enumValueSep[] = { ASCII_PIPE, '\0' };
-  static const XML_Char enumValueStart[] = { ASCII_LPAREN, '\0' };
+  static const XML_Char atypeCDATA[]
+      = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+  static const XML_Char atypeID[] = {ASCII_I, ASCII_D, '\0'};
+  static const XML_Char atypeIDREF[]
+      = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0'};
+  static const XML_Char atypeIDREFS[]
+      = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0'};
+  static const XML_Char atypeENTITY[]
+      = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0'};
+  static const XML_Char atypeENTITIES[]
+      = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T,
+         ASCII_I, ASCII_E, ASCII_S, '\0'};
+  static const XML_Char atypeNMTOKEN[]
+      = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0'};
+  static const XML_Char atypeNMTOKENS[]
+      = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K,
+         ASCII_E, ASCII_N, ASCII_S, '\0'};
+  static const XML_Char notationPrefix[]
+      = {ASCII_N, ASCII_O, ASCII_T, ASCII_A,      ASCII_T,
+         ASCII_I, ASCII_O, ASCII_N, ASCII_LPAREN, '\0'};
+  static const XML_Char enumValueSep[] = {ASCII_PIPE, '\0'};
+  static const XML_Char enumValueStart[] = {ASCII_LPAREN, '\0'};
 
   /* save one level of indirection */
-  DTD * const dtd = parser->m_dtd;
+  DTD *const dtd = parser->m_dtd;
 
   const char **eventPP;
   const char **eventEndPP;
@@ -4328,8 +4133,7 @@ doProlog(XML_Parser parser,
   if (enc == parser->m_encoding) {
     eventPP = &parser->m_eventPtr;
     eventEndPP = &parser->m_eventEndPtr;
-  }
-  else {
+  } else {
     eventPP = &(parser->m_openInternalEntities->internalEventPtr);
     eventEndPP = &(parser->m_openInternalEntities->internalEventEndPtr);
   }
@@ -4358,7 +4162,8 @@ doProlog(XML_Parser parser,
       case XML_TOK_NONE:
 #ifdef XML_DTD
         /* for internal PE NOT referenced between declarations */
-        if (enc != parser->m_encoding && !parser->m_openInternalEntities->betweenDecl) {
+        if (enc != parser->m_encoding
+            && ! parser->m_openInternalEntities->betweenDecl) {
           *nextPtr = s;
           return XML_ERROR_NONE;
         }
@@ -4383,19 +4188,18 @@ doProlog(XML_Parser parser,
     }
     role = XmlTokenRole(&parser->m_prologState, tok, s, next, enc);
     switch (role) {
-    case XML_ROLE_XML_DECL:
-      {
-        enum XML_Error result = processXmlDecl(parser, 0, s, next);
-        if (result != XML_ERROR_NONE)
-          return result;
-        enc = parser->m_encoding;
-        handleDefault = XML_FALSE;
-      }
-      break;
+    case XML_ROLE_XML_DECL: {
+      enum XML_Error result = processXmlDecl(parser, 0, s, next);
+      if (result != XML_ERROR_NONE)
+        return result;
+      enc = parser->m_encoding;
+      handleDefault = XML_FALSE;
+    } break;
     case XML_ROLE_DOCTYPE_NAME:
       if (parser->m_startDoctypeDeclHandler) {
-        parser->m_doctypeName = poolStoreString(&parser->m_tempPool, enc, s, next);
-        if (!parser->m_doctypeName)
+        parser->m_doctypeName
+            = poolStoreString(&parser->m_tempPool, enc, s, next);
+        if (! parser->m_doctypeName)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
         parser->m_doctypePubid = NULL;
@@ -4405,43 +4209,40 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_DOCTYPE_INTERNAL_SUBSET:
       if (parser->m_startDoctypeDeclHandler) {
-        parser->m_startDoctypeDeclHandler(parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
-                                parser->m_doctypePubid, 1);
+        parser->m_startDoctypeDeclHandler(
+            parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
+            parser->m_doctypePubid, 1);
         parser->m_doctypeName = NULL;
         poolClear(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
       break;
 #ifdef XML_DTD
-    case XML_ROLE_TEXT_DECL:
-      {
-        enum XML_Error result = processXmlDecl(parser, 1, s, next);
-        if (result != XML_ERROR_NONE)
-          return result;
-        enc = parser->m_encoding;
-        handleDefault = XML_FALSE;
-      }
-      break;
+    case XML_ROLE_TEXT_DECL: {
+      enum XML_Error result = processXmlDecl(parser, 1, s, next);
+      if (result != XML_ERROR_NONE)
+        return result;
+      enc = parser->m_encoding;
+      handleDefault = XML_FALSE;
+    } break;
 #endif /* XML_DTD */
     case XML_ROLE_DOCTYPE_PUBLIC_ID:
 #ifdef XML_DTD
       parser->m_useForeignDTD = XML_FALSE;
-      parser->m_declEntity = (ENTITY *)lookup(parser,
-                                    &dtd->paramEntities,
-                                    externalSubsetName,
-                                    sizeof(ENTITY));
-      if (!parser->m_declEntity)
+      parser->m_declEntity = (ENTITY *)lookup(
+          parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY));
+      if (! parser->m_declEntity)
         return XML_ERROR_NO_MEMORY;
 #endif /* XML_DTD */
       dtd->hasParamEntityRefs = XML_TRUE;
       if (parser->m_startDoctypeDeclHandler) {
         XML_Char *pubId;
-        if (!XmlIsPublicId(enc, s, next, eventPP))
+        if (! XmlIsPublicId(enc, s, next, eventPP))
           return XML_ERROR_PUBLICID;
         pubId = poolStoreString(&parser->m_tempPool, enc,
                                 s + enc->minBytesPerChar,
                                 next - enc->minBytesPerChar);
-        if (!pubId)
+        if (! pubId)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(pubId);
         poolFinish(&parser->m_tempPool);
@@ -4451,15 +4252,14 @@ doProlog(XML_Parser parser,
       }
       /* fall through */
     case XML_ROLE_ENTITY_PUBLIC_ID:
-      if (!XmlIsPublicId(enc, s, next, eventPP))
+      if (! XmlIsPublicId(enc, s, next, eventPP))
         return XML_ERROR_PUBLICID;
     alreadyChecked:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        XML_Char *tem = poolStoreString(&dtd->pool,
-                                        enc,
-                                        s + enc->minBytesPerChar,
-                                        next - enc->minBytesPerChar);
-        if (!tem)
+        XML_Char *tem
+            = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                              next - enc->minBytesPerChar);
+        if (! tem)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(tem);
         parser->m_declEntity->publicId = tem;
@@ -4472,9 +4272,15 @@ doProlog(XML_Parser parser,
       }
       break;
     case XML_ROLE_DOCTYPE_CLOSE:
+      if (allowClosingDoctype != XML_TRUE) {
+        /* Must not close doctype from within expanded parameter entities */
+        return XML_ERROR_INVALID_TOKEN;
+      }
+
       if (parser->m_doctypeName) {
-        parser->m_startDoctypeDeclHandler(parser->m_handlerArg, parser->m_doctypeName,
-                                parser->m_doctypeSysid, parser->m_doctypePubid, 0);
+        parser->m_startDoctypeDeclHandler(
+            parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid,
+            parser->m_doctypePubid, 0);
         poolClear(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
@@ -4486,12 +4292,11 @@ doProlog(XML_Parser parser,
       if (parser->m_doctypeSysid || parser->m_useForeignDTD) {
         XML_Bool hadParamEntityRefs = dtd->hasParamEntityRefs;
         dtd->hasParamEntityRefs = XML_TRUE;
-        if (parser->m_paramEntityParsing && parser->m_externalEntityRefHandler) {
-          ENTITY *entity = (ENTITY *)lookup(parser,
-                                            &dtd->paramEntities,
-                                            externalSubsetName,
-                                            sizeof(ENTITY));
-          if (!entity) {
+        if (parser->m_paramEntityParsing
+            && parser->m_externalEntityRefHandler) {
+          ENTITY *entity = (ENTITY *)lookup(parser, &dtd->paramEntities,
+                                            externalSubsetName, sizeof(ENTITY));
+          if (! entity) {
             /* The external subset name "#" will have already been
              * inserted into the hash table at the start of the
              * external entity parsing, so no allocation will happen
@@ -4502,22 +4307,19 @@ doProlog(XML_Parser parser,
           if (parser->m_useForeignDTD)
             entity->base = parser->m_curBase;
           dtd->paramEntityRead = XML_FALSE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId))
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           if (dtd->paramEntityRead) {
-            if (!dtd->standalone &&
-                parser->m_notStandaloneHandler &&
-                !parser->m_notStandaloneHandler(parser->m_handlerArg))
+            if (! dtd->standalone && parser->m_notStandaloneHandler
+                && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
               return XML_ERROR_NOT_STANDALONE;
           }
           /* if we didn't read the foreign DTD then this means that there
              is no external subset and we must reset dtd->hasParamEntityRefs
           */
-          else if (!parser->m_doctypeSysid)
+          else if (! parser->m_doctypeSysid)
             dtd->hasParamEntityRefs = hadParamEntityRefs;
           /* end of DTD - no need to update dtd->keepProcessing */
         }
@@ -4537,24 +4339,21 @@ doProlog(XML_Parser parser,
       if (parser->m_useForeignDTD) {
         XML_Bool hadParamEntityRefs = dtd->hasParamEntityRefs;
         dtd->hasParamEntityRefs = XML_TRUE;
-        if (parser->m_paramEntityParsing && parser->m_externalEntityRefHandler) {
+        if (parser->m_paramEntityParsing
+            && parser->m_externalEntityRefHandler) {
           ENTITY *entity = (ENTITY *)lookup(parser, &dtd->paramEntities,
-                                            externalSubsetName,
-                                            sizeof(ENTITY));
-          if (!entity)
+                                            externalSubsetName, sizeof(ENTITY));
+          if (! entity)
             return XML_ERROR_NO_MEMORY;
           entity->base = parser->m_curBase;
           dtd->paramEntityRead = XML_FALSE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId))
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId))
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           if (dtd->paramEntityRead) {
-            if (!dtd->standalone &&
-                parser->m_notStandaloneHandler &&
-                !parser->m_notStandaloneHandler(parser->m_handlerArg))
+            if (! dtd->standalone && parser->m_notStandaloneHandler
+                && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
               return XML_ERROR_NOT_STANDALONE;
           }
           /* if we didn't read the foreign DTD then this means that there
@@ -4570,12 +4369,12 @@ doProlog(XML_Parser parser,
       return contentProcessor(parser, s, end, nextPtr);
     case XML_ROLE_ATTLIST_ELEMENT_NAME:
       parser->m_declElementType = getElementType(parser, enc, s, next);
-      if (!parser->m_declElementType)
+      if (! parser->m_declElementType)
         return XML_ERROR_NO_MEMORY;
       goto checkAttListDeclHandler;
     case XML_ROLE_ATTRIBUTE_NAME:
       parser->m_declAttributeId = getAttributeId(parser, enc, s, next);
-      if (!parser->m_declAttributeId)
+      if (! parser->m_declAttributeId)
         return XML_ERROR_NO_MEMORY;
       parser->m_declAttributeIsCdata = XML_FALSE;
       parser->m_declAttributeType = NULL;
@@ -4616,15 +4415,13 @@ doProlog(XML_Parser parser,
         const XML_Char *prefix;
         if (parser->m_declAttributeType) {
           prefix = enumValueSep;
+        } else {
+          prefix = (role == XML_ROLE_ATTRIBUTE_NOTATION_VALUE ? notationPrefix
+                                                              : enumValueStart);
         }
-        else {
-          prefix = (role == XML_ROLE_ATTRIBUTE_NOTATION_VALUE
-                    ? notationPrefix
-                    : enumValueStart);
-        }
-        if (!poolAppendString(&parser->m_tempPool, prefix))
+        if (! poolAppendString(&parser->m_tempPool, prefix))
           return XML_ERROR_NO_MEMORY;
-        if (!poolAppend(&parser->m_tempPool, enc, s, next))
+        if (! poolAppend(&parser->m_tempPool, enc, s, next))
           return XML_ERROR_NO_MEMORY;
         parser->m_declAttributeType = parser->m_tempPool.start;
         handleDefault = XML_FALSE;
@@ -4633,25 +4430,27 @@ doProlog(XML_Parser parser,
     case XML_ROLE_IMPLIED_ATTRIBUTE_VALUE:
     case XML_ROLE_REQUIRED_ATTRIBUTE_VALUE:
       if (dtd->keepProcessing) {
-        if (!defineAttribute(parser->m_declElementType, parser->m_declAttributeId,
-                             parser->m_declAttributeIsCdata, parser->m_declAttributeIsId,
-                             0, parser))
+        if (! defineAttribute(parser->m_declElementType,
+                              parser->m_declAttributeId,
+                              parser->m_declAttributeIsCdata,
+                              parser->m_declAttributeIsId, 0, parser))
           return XML_ERROR_NO_MEMORY;
         if (parser->m_attlistDeclHandler && parser->m_declAttributeType) {
           if (*parser->m_declAttributeType == XML_T(ASCII_LPAREN)
               || (*parser->m_declAttributeType == XML_T(ASCII_N)
                   && parser->m_declAttributeType[1] == XML_T(ASCII_O))) {
             /* Enumerated or Notation type */
-            if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
-                || !poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+            if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
+                || ! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
               return XML_ERROR_NO_MEMORY;
             parser->m_declAttributeType = parser->m_tempPool.start;
             poolFinish(&parser->m_tempPool);
           }
           *eventEndPP = s;
-          parser->m_attlistDeclHandler(parser->m_handlerArg, parser->m_declElementType->name,
-                             parser->m_declAttributeId->name, parser->m_declAttributeType,
-                             0, role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE);
+          parser->m_attlistDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name,
+              parser->m_declAttributeId->name, parser->m_declAttributeType, 0,
+              role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE);
           poolClear(&parser->m_tempPool);
           handleDefault = XML_FALSE;
         }
@@ -4661,35 +4460,34 @@ doProlog(XML_Parser parser,
     case XML_ROLE_FIXED_ATTRIBUTE_VALUE:
       if (dtd->keepProcessing) {
         const XML_Char *attVal;
-        enum XML_Error result =
-          storeAttributeValue(parser, enc, parser->m_declAttributeIsCdata,
-                              s + enc->minBytesPerChar,
-                              next - enc->minBytesPerChar,
-                              &dtd->pool);
+        enum XML_Error result = storeAttributeValue(
+            parser, enc, parser->m_declAttributeIsCdata,
+            s + enc->minBytesPerChar, next - enc->minBytesPerChar, &dtd->pool);
         if (result)
           return result;
         attVal = poolStart(&dtd->pool);
         poolFinish(&dtd->pool);
         /* ID attributes aren't allowed to have a default */
-        if (!defineAttribute(parser->m_declElementType, parser->m_declAttributeId,
-                             parser->m_declAttributeIsCdata, XML_FALSE, attVal, parser))
+        if (! defineAttribute(
+                parser->m_declElementType, parser->m_declAttributeId,
+                parser->m_declAttributeIsCdata, XML_FALSE, attVal, parser))
           return XML_ERROR_NO_MEMORY;
         if (parser->m_attlistDeclHandler && parser->m_declAttributeType) {
           if (*parser->m_declAttributeType == XML_T(ASCII_LPAREN)
               || (*parser->m_declAttributeType == XML_T(ASCII_N)
                   && parser->m_declAttributeType[1] == XML_T(ASCII_O))) {
             /* Enumerated or Notation type */
-            if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
-                || !poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+            if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_RPAREN))
+                || ! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
               return XML_ERROR_NO_MEMORY;
             parser->m_declAttributeType = parser->m_tempPool.start;
             poolFinish(&parser->m_tempPool);
           }
           *eventEndPP = s;
-          parser->m_attlistDeclHandler(parser->m_handlerArg, parser->m_declElementType->name,
-                             parser->m_declAttributeId->name, parser->m_declAttributeType,
-                             attVal,
-                             role == XML_ROLE_FIXED_ATTRIBUTE_VALUE);
+          parser->m_attlistDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name,
+              parser->m_declAttributeId->name, parser->m_declAttributeType,
+              attVal, role == XML_ROLE_FIXED_ATTRIBUTE_VALUE);
           poolClear(&parser->m_tempPool);
           handleDefault = XML_FALSE;
         }
@@ -4697,25 +4495,22 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_ENTITY_VALUE:
       if (dtd->keepProcessing) {
-        enum XML_Error result = storeEntityValue(parser, enc,
-                                            s + enc->minBytesPerChar,
-                                            next - enc->minBytesPerChar);
+        enum XML_Error result = storeEntityValue(
+            parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar);
         if (parser->m_declEntity) {
           parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool);
-          parser->m_declEntity->textLen = (int)(poolLength(&dtd->entityValuePool));
+          parser->m_declEntity->textLen
+              = (int)(poolLength(&dtd->entityValuePool));
           poolFinish(&dtd->entityValuePool);
           if (parser->m_entityDeclHandler) {
             *eventEndPP = s;
-            parser->m_entityDeclHandler(parser->m_handlerArg,
-                              parser->m_declEntity->name,
-                              parser->m_declEntity->is_param,
-                              parser->m_declEntity->textPtr,
-                              parser->m_declEntity->textLen,
-                              parser->m_curBase, 0, 0, 0);
+            parser->m_entityDeclHandler(
+                parser->m_handlerArg, parser->m_declEntity->name,
+                parser->m_declEntity->is_param, parser->m_declEntity->textPtr,
+                parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0);
             handleDefault = XML_FALSE;
           }
-        }
-        else
+        } else
           poolDiscard(&dtd->entityValuePool);
         if (result != XML_ERROR_NONE)
           return result;
@@ -4728,8 +4523,8 @@ doProlog(XML_Parser parser,
       dtd->hasParamEntityRefs = XML_TRUE;
       if (parser->m_startDoctypeDeclHandler) {
         parser->m_doctypeSysid = poolStoreString(&parser->m_tempPool, enc,
-                                       s + enc->minBytesPerChar,
-                                       next - enc->minBytesPerChar);
+                                                 s + enc->minBytesPerChar,
+                                                 next - enc->minBytesPerChar);
         if (parser->m_doctypeSysid == NULL)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
@@ -4741,22 +4536,20 @@ doProlog(XML_Parser parser,
            for the case where no parser->m_startDoctypeDeclHandler is set */
         parser->m_doctypeSysid = externalSubsetName;
 #endif /* XML_DTD */
-      if (!dtd->standalone
+      if (! dtd->standalone
 #ifdef XML_DTD
-          && !parser->m_paramEntityParsing
+          && ! parser->m_paramEntityParsing
 #endif /* XML_DTD */
           && parser->m_notStandaloneHandler
-          && !parser->m_notStandaloneHandler(parser->m_handlerArg))
+          && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
         return XML_ERROR_NOT_STANDALONE;
 #ifndef XML_DTD
       break;
-#else /* XML_DTD */
-      if (!parser->m_declEntity) {
-        parser->m_declEntity = (ENTITY *)lookup(parser,
-                                      &dtd->paramEntities,
-                                      externalSubsetName,
-                                      sizeof(ENTITY));
-        if (!parser->m_declEntity)
+#else  /* XML_DTD */
+      if (! parser->m_declEntity) {
+        parser->m_declEntity = (ENTITY *)lookup(
+            parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY));
+        if (! parser->m_declEntity)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity->publicId = NULL;
       }
@@ -4764,10 +4557,10 @@ doProlog(XML_Parser parser,
       /* fall through */
     case XML_ROLE_ENTITY_SYSTEM_ID:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        parser->m_declEntity->systemId = poolStoreString(&dtd->pool, enc,
-                                               s + enc->minBytesPerChar,
-                                               next - enc->minBytesPerChar);
-        if (!parser->m_declEntity->systemId)
+        parser->m_declEntity->systemId
+            = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                              next - enc->minBytesPerChar);
+        if (! parser->m_declEntity->systemId)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity->base = parser->m_curBase;
         poolFinish(&dtd->pool);
@@ -4779,115 +4572,103 @@ doProlog(XML_Parser parser,
       }
       break;
     case XML_ROLE_ENTITY_COMPLETE:
-      if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) {
+      if (dtd->keepProcessing && parser->m_declEntity
+          && parser->m_entityDeclHandler) {
         *eventEndPP = s;
-        parser->m_entityDeclHandler(parser->m_handlerArg,
-                          parser->m_declEntity->name,
-                          parser->m_declEntity->is_param,
-                          0,0,
-                          parser->m_declEntity->base,
-                          parser->m_declEntity->systemId,
-                          parser->m_declEntity->publicId,
-                          0);
+        parser->m_entityDeclHandler(
+            parser->m_handlerArg, parser->m_declEntity->name,
+            parser->m_declEntity->is_param, 0, 0, parser->m_declEntity->base,
+            parser->m_declEntity->systemId, parser->m_declEntity->publicId, 0);
         handleDefault = XML_FALSE;
       }
       break;
     case XML_ROLE_ENTITY_NOTATION_NAME:
       if (dtd->keepProcessing && parser->m_declEntity) {
-        parser->m_declEntity->notation = poolStoreString(&dtd->pool, enc, s, next);
-        if (!parser->m_declEntity->notation)
+        parser->m_declEntity->notation
+            = poolStoreString(&dtd->pool, enc, s, next);
+        if (! parser->m_declEntity->notation)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&dtd->pool);
         if (parser->m_unparsedEntityDeclHandler) {
           *eventEndPP = s;
-          parser->m_unparsedEntityDeclHandler(parser->m_handlerArg,
-                                    parser->m_declEntity->name,
-                                    parser->m_declEntity->base,
-                                    parser->m_declEntity->systemId,
-                                    parser->m_declEntity->publicId,
-                                    parser->m_declEntity->notation);
+          parser->m_unparsedEntityDeclHandler(
+              parser->m_handlerArg, parser->m_declEntity->name,
+              parser->m_declEntity->base, parser->m_declEntity->systemId,
+              parser->m_declEntity->publicId, parser->m_declEntity->notation);
           handleDefault = XML_FALSE;
-        }
-        else if (parser->m_entityDeclHandler) {
+        } else if (parser->m_entityDeclHandler) {
           *eventEndPP = s;
-          parser->m_entityDeclHandler(parser->m_handlerArg,
-                            parser->m_declEntity->name,
-                            0,0,0,
-                            parser->m_declEntity->base,
-                            parser->m_declEntity->systemId,
-                            parser->m_declEntity->publicId,
-                            parser->m_declEntity->notation);
+          parser->m_entityDeclHandler(
+              parser->m_handlerArg, parser->m_declEntity->name, 0, 0, 0,
+              parser->m_declEntity->base, parser->m_declEntity->systemId,
+              parser->m_declEntity->publicId, parser->m_declEntity->notation);
           handleDefault = XML_FALSE;
         }
       }
       break;
-    case XML_ROLE_GENERAL_ENTITY_NAME:
-      {
-        if (XmlPredefinedEntityName(enc, s, next)) {
-          parser->m_declEntity = NULL;
-          break;
-        }
-        if (dtd->keepProcessing) {
-          const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
-          if (!name)
-            return XML_ERROR_NO_MEMORY;
-          parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->generalEntities, name,
-                                        sizeof(ENTITY));
-          if (!parser->m_declEntity)
-            return XML_ERROR_NO_MEMORY;
-          if (parser->m_declEntity->name != name) {
-            poolDiscard(&dtd->pool);
-            parser->m_declEntity = NULL;
-          }
-          else {
-            poolFinish(&dtd->pool);
-            parser->m_declEntity->publicId = NULL;
-            parser->m_declEntity->is_param = XML_FALSE;
-            /* if we have a parent parser or are reading an internal parameter
-               entity, then the entity declaration is not considered "internal"
-            */
-            parser->m_declEntity->is_internal = !(parser->m_parentParser || parser->m_openInternalEntities);
-            if (parser->m_entityDeclHandler)
-              handleDefault = XML_FALSE;
-          }
-        }
-        else {
+    case XML_ROLE_GENERAL_ENTITY_NAME: {
+      if (XmlPredefinedEntityName(enc, s, next)) {
+        parser->m_declEntity = NULL;
+        break;
+      }
+      if (dtd->keepProcessing) {
+        const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
+        if (! name)
+          return XML_ERROR_NO_MEMORY;
+        parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->generalEntities,
+                                                name, sizeof(ENTITY));
+        if (! parser->m_declEntity)
+          return XML_ERROR_NO_MEMORY;
+        if (parser->m_declEntity->name != name) {
           poolDiscard(&dtd->pool);
           parser->m_declEntity = NULL;
+        } else {
+          poolFinish(&dtd->pool);
+          parser->m_declEntity->publicId = NULL;
+          parser->m_declEntity->is_param = XML_FALSE;
+          /* if we have a parent parser or are reading an internal parameter
+             entity, then the entity declaration is not considered "internal"
+          */
+          parser->m_declEntity->is_internal
+              = ! (parser->m_parentParser || parser->m_openInternalEntities);
+          if (parser->m_entityDeclHandler)
+            handleDefault = XML_FALSE;
         }
+      } else {
+        poolDiscard(&dtd->pool);
+        parser->m_declEntity = NULL;
       }
-      break;
+    } break;
     case XML_ROLE_PARAM_ENTITY_NAME:
 #ifdef XML_DTD
       if (dtd->keepProcessing) {
         const XML_Char *name = poolStoreString(&dtd->pool, enc, s, next);
-        if (!name)
+        if (! name)
           return XML_ERROR_NO_MEMORY;
         parser->m_declEntity = (ENTITY *)lookup(parser, &dtd->paramEntities,
-                                           name, sizeof(ENTITY));
-        if (!parser->m_declEntity)
+                                                name, sizeof(ENTITY));
+        if (! parser->m_declEntity)
           return XML_ERROR_NO_MEMORY;
         if (parser->m_declEntity->name != name) {
           poolDiscard(&dtd->pool);
           parser->m_declEntity = NULL;
-        }
-        else {
+        } else {
           poolFinish(&dtd->pool);
           parser->m_declEntity->publicId = NULL;
           parser->m_declEntity->is_param = XML_TRUE;
           /* if we have a parent parser or are reading an internal parameter
              entity, then the entity declaration is not considered "internal"
           */
-          parser->m_declEntity->is_internal = !(parser->m_parentParser || parser->m_openInternalEntities);
+          parser->m_declEntity->is_internal
+              = ! (parser->m_parentParser || parser->m_openInternalEntities);
           if (parser->m_entityDeclHandler)
             handleDefault = XML_FALSE;
         }
-      }
-      else {
+      } else {
         poolDiscard(&dtd->pool);
         parser->m_declEntity = NULL;
       }
-#else /* not XML_DTD */
+#else  /* not XML_DTD */
       parser->m_declEntity = NULL;
 #endif /* XML_DTD */
       break;
@@ -4895,22 +4676,23 @@ doProlog(XML_Parser parser,
       parser->m_declNotationPublicId = NULL;
       parser->m_declNotationName = NULL;
       if (parser->m_notationDeclHandler) {
-        parser->m_declNotationName = poolStoreString(&parser->m_tempPool, enc, s, next);
-        if (!parser->m_declNotationName)
+        parser->m_declNotationName
+            = poolStoreString(&parser->m_tempPool, enc, s, next);
+        if (! parser->m_declNotationName)
           return XML_ERROR_NO_MEMORY;
         poolFinish(&parser->m_tempPool);
         handleDefault = XML_FALSE;
       }
       break;
     case XML_ROLE_NOTATION_PUBLIC_ID:
-      if (!XmlIsPublicId(enc, s, next, eventPP))
+      if (! XmlIsPublicId(enc, s, next, eventPP))
         return XML_ERROR_PUBLICID;
-      if (parser->m_declNotationName) {  /* means m_notationDeclHandler != NULL */
-        XML_Char *tem = poolStoreString(&parser->m_tempPool,
-                                        enc,
+      if (parser
+              ->m_declNotationName) { /* means m_notationDeclHandler != NULL */
+        XML_Char *tem = poolStoreString(&parser->m_tempPool, enc,
                                         s + enc->minBytesPerChar,
                                         next - enc->minBytesPerChar);
-        if (!tem)
+        if (! tem)
           return XML_ERROR_NO_MEMORY;
         normalizePublicId(tem);
         parser->m_declNotationPublicId = tem;
@@ -4920,18 +4702,15 @@ doProlog(XML_Parser parser,
       break;
     case XML_ROLE_NOTATION_SYSTEM_ID:
       if (parser->m_declNotationName && parser->m_notationDeclHandler) {
-        const XML_Char *systemId
-          = poolStoreString(&parser->m_tempPool, enc,
-                            s + enc->minBytesPerChar,
-                            next - enc->minBytesPerChar);
-        if (!systemId)
+        const XML_Char *systemId = poolStoreString(&parser->m_tempPool, enc,
+                                                   s + enc->minBytesPerChar,
+                                                   next - enc->minBytesPerChar);
+        if (! systemId)
           return XML_ERROR_NO_MEMORY;
         *eventEndPP = s;
-        parser->m_notationDeclHandler(parser->m_handlerArg,
-                            parser->m_declNotationName,
-                            parser->m_curBase,
-                            systemId,
-                            parser->m_declNotationPublicId);
+        parser->m_notationDeclHandler(
+            parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase,
+            systemId, parser->m_declNotationPublicId);
         handleDefault = XML_FALSE;
       }
       poolClear(&parser->m_tempPool);
@@ -4939,11 +4718,9 @@ doProlog(XML_Parser parser,
     case XML_ROLE_NOTATION_NO_SYSTEM_ID:
       if (parser->m_declNotationPublicId && parser->m_notationDeclHandler) {
         *eventEndPP = s;
-        parser->m_notationDeclHandler(parser->m_handlerArg,
-                            parser->m_declNotationName,
-                            parser->m_curBase,
-                            0,
-                            parser->m_declNotationPublicId);
+        parser->m_notationDeclHandler(
+            parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase,
+            0, parser->m_declNotationPublicId);
         handleDefault = XML_FALSE;
       }
       poolClear(&parser->m_tempPool);
@@ -4960,42 +4737,44 @@ doProlog(XML_Parser parser,
         return XML_ERROR_SYNTAX;
       }
 #ifdef XML_DTD
-    case XML_ROLE_IGNORE_SECT:
-      {
-        enum XML_Error result;
-        if (parser->m_defaultHandler)
-          reportDefault(parser, enc, s, next);
-        handleDefault = XML_FALSE;
-        result = doIgnoreSection(parser, enc, &next, end, nextPtr, haveMore);
-        if (result != XML_ERROR_NONE)
-          return result;
-        else if (!next) {
-          parser->m_processor = ignoreSectionProcessor;
-          return result;
-        }
+    case XML_ROLE_IGNORE_SECT: {
+      enum XML_Error result;
+      if (parser->m_defaultHandler)
+        reportDefault(parser, enc, s, next);
+      handleDefault = XML_FALSE;
+      result = doIgnoreSection(parser, enc, &next, end, nextPtr, haveMore);
+      if (result != XML_ERROR_NONE)
+        return result;
+      else if (! next) {
+        parser->m_processor = ignoreSectionProcessor;
+        return result;
       }
-      break;
+    } break;
 #endif /* XML_DTD */
     case XML_ROLE_GROUP_OPEN:
       if (parser->m_prologState.level >= parser->m_groupSize) {
         if (parser->m_groupSize) {
-          char *temp = (char *)REALLOC(parser, parser->m_groupConnector, parser->m_groupSize *= 2);
-          if (temp == NULL) {
-            parser->m_groupSize /= 2;
-            return XML_ERROR_NO_MEMORY;
+          {
+            char *const new_connector = (char *)REALLOC(
+                parser, parser->m_groupConnector, parser->m_groupSize *= 2);
+            if (new_connector == NULL) {
+              parser->m_groupSize /= 2;
+              return XML_ERROR_NO_MEMORY;
+            }
+            parser->m_groupConnector = new_connector;
           }
-          parser->m_groupConnector = temp;
+
           if (dtd->scaffIndex) {
-            int *temp = (int *)REALLOC(parser, dtd->scaffIndex,
-                          parser->m_groupSize * sizeof(int));
-            if (temp == NULL)
+            int *const new_scaff_index = (int *)REALLOC(
+                parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int));
+            if (new_scaff_index == NULL)
               return XML_ERROR_NO_MEMORY;
-            dtd->scaffIndex = temp;
+            dtd->scaffIndex = new_scaff_index;
           }
-        }
-        else {
-          parser->m_groupConnector = (char *)MALLOC(parser, parser->m_groupSize = 32);
-          if (!parser->m_groupConnector) {
+        } else {
+          parser->m_groupConnector
+              = (char *)MALLOC(parser, parser->m_groupSize = 32);
+          if (! parser->m_groupConnector) {
             parser->m_groupSize = 0;
             return XML_ERROR_NO_MEMORY;
           }
@@ -5006,6 +4785,7 @@ doProlog(XML_Parser parser,
         int myindex = nextScaffoldPart(parser);
         if (myindex < 0)
           return XML_ERROR_NO_MEMORY;
+        assert(dtd->scaffIndex != NULL);
         dtd->scaffIndex[dtd->scaffLevel] = myindex;
         dtd->scaffLevel++;
         dtd->scaffold[myindex].type = XML_CTYPE_SEQ;
@@ -5024,10 +4804,9 @@ doProlog(XML_Parser parser,
       if (parser->m_groupConnector[parser->m_prologState.level] == ASCII_COMMA)
         return XML_ERROR_SYNTAX;
       if (dtd->in_eldecl
-          && !parser->m_groupConnector[parser->m_prologState.level]
+          && ! parser->m_groupConnector[parser->m_prologState.level]
           && (dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]].type
-              != XML_CTYPE_MIXED)
-          ) {
+              != XML_CTYPE_MIXED)) {
         dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]].type
             = XML_CTYPE_CHOICE;
         if (parser->m_elementDeclHandler)
@@ -5039,15 +4818,14 @@ doProlog(XML_Parser parser,
 #ifdef XML_DTD
     case XML_ROLE_INNER_PARAM_ENTITY_REF:
       dtd->hasParamEntityRefs = XML_TRUE;
-      if (!parser->m_paramEntityParsing)
+      if (! parser->m_paramEntityParsing)
         dtd->keepProcessing = dtd->standalone;
       else {
         const XML_Char *name;
         ENTITY *entity;
-        name = poolStoreString(&dtd->pool, enc,
-                                s + enc->minBytesPerChar,
-                                next - enc->minBytesPerChar);
-        if (!name)
+        name = poolStoreString(&dtd->pool, enc, s + enc->minBytesPerChar,
+                               next - enc->minBytesPerChar);
+        if (! name)
           return XML_ERROR_NO_MEMORY;
         entity = (ENTITY *)lookup(parser, &dtd->paramEntities, name, 0);
         poolDiscard(&dtd->pool);
@@ -5055,13 +4833,12 @@ doProlog(XML_Parser parser,
            if yes, check that the entity exists, and that it is internal,
            otherwise call the skipped entity handler
         */
-        if (parser->m_prologState.documentEntity &&
-            (dtd->standalone
-             ? !parser->m_openInternalEntities
-             : !dtd->hasParamEntityRefs)) {
-          if (!entity)
+        if (parser->m_prologState.documentEntity
+            && (dtd->standalone ? ! parser->m_openInternalEntities
+                                : ! dtd->hasParamEntityRefs)) {
+          if (! entity)
             return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal) {
+          else if (! entity->is_internal) {
             /* It's hard to exhaustively search the code to be sure,
              * but there doesn't seem to be a way of executing the
              * following line.  There are two cases:
@@ -5084,11 +4861,11 @@ doProlog(XML_Parser parser,
              */
             return XML_ERROR_ENTITY_DECLARED_IN_PE; /* LCOV_EXCL_LINE */
           }
-        }
-        else if (!entity) {
+        } else if (! entity) {
           dtd->keepProcessing = dtd->standalone;
           /* cannot report skipped entities in declarations */
-          if ((role == XML_ROLE_PARAM_ENTITY_REF) && parser->m_skippedEntityHandler) {
+          if ((role == XML_ROLE_PARAM_ENTITY_REF)
+              && parser->m_skippedEntityHandler) {
             parser->m_skippedEntityHandler(parser->m_handlerArg, name, 1);
             handleDefault = XML_FALSE;
           }
@@ -5098,8 +4875,8 @@ doProlog(XML_Parser parser,
           return XML_ERROR_RECURSIVE_ENTITY_REF;
         if (entity->textPtr) {
           enum XML_Error result;
-          XML_Bool betweenDecl =
-            (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE);
+          XML_Bool betweenDecl
+              = (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE);
           result = processInternalEntity(parser, entity, betweenDecl);
           if (result != XML_ERROR_NONE)
             return result;
@@ -5109,39 +4886,35 @@ doProlog(XML_Parser parser,
         if (parser->m_externalEntityRefHandler) {
           dtd->paramEntityRead = XML_FALSE;
           entity->open = XML_TRUE;
-          if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                        0,
-                                        entity->base,
-                                        entity->systemId,
-                                        entity->publicId)) {
+          if (! parser->m_externalEntityRefHandler(
+                  parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                  entity->systemId, entity->publicId)) {
             entity->open = XML_FALSE;
             return XML_ERROR_EXTERNAL_ENTITY_HANDLING;
           }
           entity->open = XML_FALSE;
           handleDefault = XML_FALSE;
-          if (!dtd->paramEntityRead) {
+          if (! dtd->paramEntityRead) {
             dtd->keepProcessing = dtd->standalone;
             break;
           }
-        }
-        else {
+        } else {
           dtd->keepProcessing = dtd->standalone;
           break;
         }
       }
 #endif /* XML_DTD */
-      if (!dtd->standalone &&
-          parser->m_notStandaloneHandler &&
-          !parser->m_notStandaloneHandler(parser->m_handlerArg))
+      if (! dtd->standalone && parser->m_notStandaloneHandler
+          && ! parser->m_notStandaloneHandler(parser->m_handlerArg))
         return XML_ERROR_NOT_STANDALONE;
       break;
 
-    /* Element declaration stuff */
+      /* Element declaration stuff */
 
     case XML_ROLE_ELEMENT_NAME:
       if (parser->m_elementDeclHandler) {
         parser->m_declElementType = getElementType(parser, enc, s, next);
-        if (!parser->m_declElementType)
+        if (! parser->m_declElementType)
           return XML_ERROR_NO_MEMORY;
         dtd->scaffLevel = 0;
         dtd->scaffCount = 0;
@@ -5154,18 +4927,19 @@ doProlog(XML_Parser parser,
     case XML_ROLE_CONTENT_EMPTY:
       if (dtd->in_eldecl) {
         if (parser->m_elementDeclHandler) {
-          XML_Content * content = (XML_Content *) MALLOC(parser, sizeof(XML_Content));
-          if (!content)
+          XML_Content *content
+              = (XML_Content *)MALLOC(parser, sizeof(XML_Content));
+          if (! content)
             return XML_ERROR_NO_MEMORY;
           content->quant = XML_CQUANT_NONE;
           content->name = NULL;
           content->numchildren = 0;
           content->children = NULL;
-          content->type = ((role == XML_ROLE_CONTENT_ANY) ?
-                           XML_CTYPE_ANY :
-                           XML_CTYPE_EMPTY);
+          content->type = ((role == XML_ROLE_CONTENT_ANY) ? XML_CTYPE_ANY
+                                                          : XML_CTYPE_EMPTY);
           *eventEndPP = s;
-          parser->m_elementDeclHandler(parser->m_handlerArg, parser->m_declElementType->name, content);
+          parser->m_elementDeclHandler(
+              parser->m_handlerArg, parser->m_declElementType->name, content);
           handleDefault = XML_FALSE;
         }
         dtd->in_eldecl = XML_FALSE;
@@ -5197,22 +4971,22 @@ doProlog(XML_Parser parser,
         ELEMENT_TYPE *el;
         const XML_Char *name;
         int nameLen;
-        const char *nxt = (quant == XML_CQUANT_NONE
-                           ? next
-                           : next - enc->minBytesPerChar);
+        const char *nxt
+            = (quant == XML_CQUANT_NONE ? next : next - enc->minBytesPerChar);
         int myindex = nextScaffoldPart(parser);
         if (myindex < 0)
           return XML_ERROR_NO_MEMORY;
         dtd->scaffold[myindex].type = XML_CTYPE_NAME;
         dtd->scaffold[myindex].quant = quant;
         el = getElementType(parser, enc, s, nxt);
-        if (!el)
+        if (! el)
           return XML_ERROR_NO_MEMORY;
         name = el->name;
         dtd->scaffold[myindex].name = name;
         nameLen = 0;
-        for (; name[nameLen++]; );
-        dtd->contentStringLen +=  nameLen;
+        for (; name[nameLen++];)
+          ;
+        dtd->contentStringLen += nameLen;
         if (parser->m_elementDeclHandler)
           handleDefault = XML_FALSE;
       }
@@ -5236,12 +5010,13 @@ doProlog(XML_Parser parser,
         dtd->scaffLevel--;
         dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel]].quant = quant;
         if (dtd->scaffLevel == 0) {
-          if (!handleDefault) {
+          if (! handleDefault) {
             XML_Content *model = build_model(parser);
-            if (!model)
+            if (! model)
               return XML_ERROR_NO_MEMORY;
             *eventEndPP = s;
-            parser->m_elementDeclHandler(parser->m_handlerArg, parser->m_declElementType->name, model);
+            parser->m_elementDeclHandler(
+                parser->m_handlerArg, parser->m_declElementType->name, model);
           }
           dtd->in_eldecl = XML_FALSE;
           dtd->contentStringLen = 0;
@@ -5251,12 +5026,12 @@ doProlog(XML_Parser parser,
       /* End element declaration stuff */
 
     case XML_ROLE_PI:
-      if (!reportProcessingInstruction(parser, enc, s, next))
+      if (! reportProcessingInstruction(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       handleDefault = XML_FALSE;
       break;
     case XML_ROLE_COMMENT:
-      if (!reportComment(parser, enc, s, next))
+      if (! reportComment(parser, enc, s, next))
         return XML_ERROR_NO_MEMORY;
       handleDefault = XML_FALSE;
       break;
@@ -5307,11 +5082,8 @@ doProlog(XML_Parser parser,
 }
 
 static enum XML_Error PTRCALL
-epilogProcessor(XML_Parser parser,
-                const char *s,
-                const char *end,
-                const char **nextPtr)
-{
+epilogProcessor(XML_Parser parser, const char *s, const char *end,
+                const char **nextPtr) {
   parser->m_processor = epilogProcessor;
   parser->m_eventPtr = s;
   for (;;) {
@@ -5336,24 +5108,24 @@ epilogProcessor(XML_Parser parser,
         reportDefault(parser, parser->m_encoding, s, next);
       break;
     case XML_TOK_PI:
-      if (!reportProcessingInstruction(parser, parser->m_encoding, s, next))
+      if (! reportProcessingInstruction(parser, parser->m_encoding, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_COMMENT:
-      if (!reportComment(parser, parser->m_encoding, s, next))
+      if (! reportComment(parser, parser->m_encoding, s, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_INVALID:
       parser->m_eventPtr = next;
       return XML_ERROR_INVALID_TOKEN;
     case XML_TOK_PARTIAL:
-      if (!parser->m_parsingStatus.finalBuffer) {
+      if (! parser->m_parsingStatus.finalBuffer) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
       return XML_ERROR_UNCLOSED_TOKEN;
     case XML_TOK_PARTIAL_CHAR:
-      if (!parser->m_parsingStatus.finalBuffer) {
+      if (! parser->m_parsingStatus.finalBuffer) {
         *nextPtr = s;
         return XML_ERROR_NONE;
       }
@@ -5368,15 +5140,13 @@ epilogProcessor(XML_Parser parser,
       return XML_ERROR_NONE;
     case XML_FINISHED:
       return XML_ERROR_ABORTED;
-    default: ;
+    default:;
     }
   }
 }
 
 static enum XML_Error
-processInternalEntity(XML_Parser parser, ENTITY *entity,
-                      XML_Bool betweenDecl)
-{
+processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) {
   const char *textStart, *textEnd;
   const char *next;
   enum XML_Error result;
@@ -5385,10 +5155,10 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
   if (parser->m_freeInternalEntities) {
     openEntity = parser->m_freeInternalEntities;
     parser->m_freeInternalEntities = openEntity->next;
-  }
-  else {
-    openEntity = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY));
-    if (!openEntity)
+  } else {
+    openEntity
+        = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY));
+    if (! openEntity)
       return XML_ERROR_NO_MEMORY;
   }
   entity->open = XML_TRUE;
@@ -5407,21 +5177,20 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
 
 #ifdef XML_DTD
   if (entity->is_param) {
-    int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
-    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok,
-                      next, &next, XML_FALSE);
-  }
-  else
+    int tok
+        = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
+    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
+                      tok, next, &next, XML_FALSE, XML_FALSE);
+  } else
 #endif /* XML_DTD */
-    result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, textStart,
-                       textEnd, &next, XML_FALSE);
+    result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding,
+                       textStart, textEnd, &next, XML_FALSE);
 
   if (result == XML_ERROR_NONE) {
     if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
       entity->processed = (int)(next - textStart);
       parser->m_processor = internalEntityProcessor;
-    }
-    else {
+    } else {
       entity->open = XML_FALSE;
       parser->m_openInternalEntities = openEntity->next;
       /* put openEntity back in list of free instances */
@@ -5433,17 +5202,14 @@ processInternalEntity(XML_Parser parser, ENTITY *entity,
 }
 
 static enum XML_Error PTRCALL
-internalEntityProcessor(XML_Parser parser,
-                        const char *s,
-                        const char *end,
-                        const char **nextPtr)
-{
+internalEntityProcessor(XML_Parser parser, const char *s, const char *end,
+                        const char **nextPtr) {
   ENTITY *entity;
   const char *textStart, *textEnd;
   const char *next;
   enum XML_Error result;
   OPEN_INTERNAL_ENTITY *openEntity = parser->m_openInternalEntities;
-  if (!openEntity)
+  if (! openEntity)
     return XML_ERROR_UNEXPECTED_STATE;
 
   entity = openEntity->entity;
@@ -5454,22 +5220,23 @@ internalEntityProcessor(XML_Parser parser,
 
 #ifdef XML_DTD
   if (entity->is_param) {
-    int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
-    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok,
-                      next, &next, XML_FALSE);
-  }
-  else
+    int tok
+        = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next);
+    result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd,
+                      tok, next, &next, XML_FALSE, XML_TRUE);
+  } else
 #endif /* XML_DTD */
-    result = doContent(parser, openEntity->startTagLevel, parser->m_internalEncoding,
-                       textStart, textEnd, &next, XML_FALSE);
+    result = doContent(parser, openEntity->startTagLevel,
+                       parser->m_internalEncoding, textStart, textEnd, &next,
+                       XML_FALSE);
 
   if (result != XML_ERROR_NONE)
     return result;
-  else if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
+  else if (textEnd != next
+           && parser->m_parsingStatus.parsing == XML_SUSPENDED) {
     entity->processed = (int)(next - (char *)entity->textPtr);
     return result;
-  }
-  else {
+  } else {
     entity->open = XML_FALSE;
     parser->m_openInternalEntities = openEntity->next;
     /* put openEntity back in list of free instances */
@@ -5483,49 +5250,45 @@ internalEntityProcessor(XML_Parser parser,
     parser->m_processor = prologProcessor;
     tok = XmlPrologTok(parser->m_encoding, s, end, &next);
     return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr,
-                    (XML_Bool)!parser->m_parsingStatus.finalBuffer);
-  }
-  else
+                    (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE);
+  } else
 #endif /* XML_DTD */
   {
     parser->m_processor = contentProcessor;
     /* see externalEntityContentProcessor vs contentProcessor */
-    return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding, s, end,
-                     nextPtr, (XML_Bool)!parser->m_parsingStatus.finalBuffer);
+    return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding,
+                     s, end, nextPtr,
+                     (XML_Bool)! parser->m_parsingStatus.finalBuffer);
   }
 }
 
 static enum XML_Error PTRCALL
-errorProcessor(XML_Parser parser,
-               const char *UNUSED_P(s),
-               const char *UNUSED_P(end),
-               const char **UNUSED_P(nextPtr))
-{
+errorProcessor(XML_Parser parser, const char *s, const char *end,
+               const char **nextPtr) {
+  UNUSED_P(s);
+  UNUSED_P(end);
+  UNUSED_P(nextPtr);
   return parser->m_errorCode;
 }
 
 static enum XML_Error
 storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
-                    const char *ptr, const char *end,
-                    STRING_POOL *pool)
-{
-  enum XML_Error result = appendAttributeValue(parser, enc, isCdata, ptr,
-                                               end, pool);
+                    const char *ptr, const char *end, STRING_POOL *pool) {
+  enum XML_Error result
+      = appendAttributeValue(parser, enc, isCdata, ptr, end, pool);
   if (result)
     return result;
-  if (!isCdata && poolLength(pool) && poolLastChar(pool) == 0x20)
+  if (! isCdata && poolLength(pool) && poolLastChar(pool) == 0x20)
     poolChop(pool);
-  if (!poolAppendChar(pool, XML_T('\0')))
+  if (! poolAppendChar(pool, XML_T('\0')))
     return XML_ERROR_NO_MEMORY;
   return XML_ERROR_NONE;
 }
 
 static enum XML_Error
 appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
-                     const char *ptr, const char *end,
-                     STRING_POOL *pool)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+                     const char *ptr, const char *end, STRING_POOL *pool) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   for (;;) {
     const char *next;
     int tok = XmlAttributeValueTok(enc, ptr, end, &next);
@@ -5540,38 +5303,35 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
       if (enc == parser->m_encoding)
         parser->m_eventPtr = ptr;
       return XML_ERROR_INVALID_TOKEN;
-    case XML_TOK_CHAR_REF:
-      {
-        XML_Char buf[XML_ENCODE_MAX];
-        int i;
-        int n = XmlCharRefNumber(enc, ptr);
-        if (n < 0) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_BAD_CHAR_REF;
-        }
-        if (!isCdata
-            && n == 0x20 /* space */
-            && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
-          break;
-        n = XmlEncode(n, (ICHAR *)buf);
-        /* The XmlEncode() functions can never return 0 here.  That
-         * error return happens if the code point passed in is either
-         * negative or greater than or equal to 0x110000.  The
-         * XmlCharRefNumber() functions will all return a number
-         * strictly less than 0x110000 or a negative value if an error
-         * occurred.  The negative value is intercepted above, so
-         * XmlEncode() is never passed a value it might return an
-         * error for.
-         */
-        for (i = 0; i < n; i++) {
-          if (!poolAppendChar(pool, buf[i]))
-            return XML_ERROR_NO_MEMORY;
-        }
+    case XML_TOK_CHAR_REF: {
+      XML_Char buf[XML_ENCODE_MAX];
+      int i;
+      int n = XmlCharRefNumber(enc, ptr);
+      if (n < 0) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_BAD_CHAR_REF;
       }
-      break;
+      if (! isCdata && n == 0x20 /* space */
+          && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
+        break;
+      n = XmlEncode(n, (ICHAR *)buf);
+      /* The XmlEncode() functions can never return 0 here.  That
+       * error return happens if the code point passed in is either
+       * negative or greater than or equal to 0x110000.  The
+       * XmlCharRefNumber() functions will all return a number
+       * strictly less than 0x110000 or a negative value if an error
+       * occurred.  The negative value is intercepted above, so
+       * XmlEncode() is never passed a value it might return an
+       * error for.
+       */
+      for (i = 0; i < n; i++) {
+        if (! poolAppendChar(pool, buf[i]))
+          return XML_ERROR_NO_MEMORY;
+      }
+    } break;
     case XML_TOK_DATA_CHARS:
-      if (!poolAppend(pool, enc, ptr, next))
+      if (! poolAppend(pool, enc, ptr, next))
         return XML_ERROR_NO_MEMORY;
       break;
     case XML_TOK_TRAILING_CR:
@@ -5579,109 +5339,103 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
       /* fall through */
     case XML_TOK_ATTRIBUTE_VALUE_S:
     case XML_TOK_DATA_NEWLINE:
-      if (!isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
+      if (! isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20))
         break;
-      if (!poolAppendChar(pool, 0x20))
+      if (! poolAppendChar(pool, 0x20))
         return XML_ERROR_NO_MEMORY;
       break;
-    case XML_TOK_ENTITY_REF:
-      {
-        const XML_Char *name;
-        ENTITY *entity;
-        char checkEntityDecl;
-        XML_Char ch = (XML_Char) XmlPredefinedEntityName(enc,
-                                              ptr + enc->minBytesPerChar,
-                                              next - enc->minBytesPerChar);
-        if (ch) {
-          if (!poolAppendChar(pool, ch))
-                return XML_ERROR_NO_MEMORY;
-          break;
-        }
-        name = poolStoreString(&parser->m_temp2Pool, enc,
-                               ptr + enc->minBytesPerChar,
-                               next - enc->minBytesPerChar);
-        if (!name)
+    case XML_TOK_ENTITY_REF: {
+      const XML_Char *name;
+      ENTITY *entity;
+      char checkEntityDecl;
+      XML_Char ch = (XML_Char)XmlPredefinedEntityName(
+          enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar);
+      if (ch) {
+        if (! poolAppendChar(pool, ch))
           return XML_ERROR_NO_MEMORY;
-        entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
-        poolDiscard(&parser->m_temp2Pool);
-        /* First, determine if a check for an existing declaration is needed;
-           if yes, check that the entity exists, and that it is internal.
-        */
-        if (pool == &dtd->pool)  /* are we called from prolog? */
-          checkEntityDecl =
+        break;
+      }
+      name = poolStoreString(&parser->m_temp2Pool, enc,
+                             ptr + enc->minBytesPerChar,
+                             next - enc->minBytesPerChar);
+      if (! name)
+        return XML_ERROR_NO_MEMORY;
+      entity = (ENTITY *)lookup(parser, &dtd->generalEntities, name, 0);
+      poolDiscard(&parser->m_temp2Pool);
+      /* First, determine if a check for an existing declaration is needed;
+         if yes, check that the entity exists, and that it is internal.
+      */
+      if (pool == &dtd->pool) /* are we called from prolog? */
+        checkEntityDecl =
 #ifdef XML_DTD
-              parser->m_prologState.documentEntity &&
+            parser->m_prologState.documentEntity &&
 #endif /* XML_DTD */
-              (dtd->standalone
-               ? !parser->m_openInternalEntities
-               : !dtd->hasParamEntityRefs);
-        else /* if (pool == &parser->m_tempPool): we are called from content */
-          checkEntityDecl = !dtd->hasParamEntityRefs || dtd->standalone;
-        if (checkEntityDecl) {
-          if (!entity)
-            return XML_ERROR_UNDEFINED_ENTITY;
-          else if (!entity->is_internal)
-            return XML_ERROR_ENTITY_DECLARED_IN_PE;
-        }
-        else if (!entity) {
-          /* Cannot report skipped entity here - see comments on
-             parser->m_skippedEntityHandler.
-          if (parser->m_skippedEntityHandler)
-            parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
-          */
-          /* Cannot call the default handler because this would be
-             out of sync with the call to the startElementHandler.
-          if ((pool == &parser->m_tempPool) && parser->m_defaultHandler)
-            reportDefault(parser, enc, ptr, next);
-          */
-          break;
-        }
-        if (entity->open) {
-          if (enc == parser->m_encoding) {
-            /* It does not appear that this line can be executed.
-             *
-             * The "if (entity->open)" check catches recursive entity
-             * definitions.  In order to be called with an open
-             * entity, it must have gone through this code before and
-             * been through the recursive call to
-             * appendAttributeValue() some lines below.  That call
-             * sets the local encoding ("enc") to the parser's
-             * internal encoding (internal_utf8 or internal_utf16),
-             * which can never be the same as the principle encoding.
-             * It doesn't appear there is another code path that gets
-             * here with entity->open being TRUE.
-             *
-             * Since it is not certain that this logic is watertight,
-             * we keep the line and merely exclude it from coverage
-             * tests.
-             */
-            parser->m_eventPtr = ptr; /* LCOV_EXCL_LINE */
-          }
-          return XML_ERROR_RECURSIVE_ENTITY_REF;
-        }
-        if (entity->notation) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_BINARY_ENTITY_REF;
-        }
-        if (!entity->textPtr) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = ptr;
-          return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF;
-        }
-        else {
-          enum XML_Error result;
-          const XML_Char *textEnd = entity->textPtr + entity->textLen;
-          entity->open = XML_TRUE;
-          result = appendAttributeValue(parser, parser->m_internalEncoding, isCdata,
-                                        (char *)entity->textPtr,
-                                        (char *)textEnd, pool);
-          entity->open = XML_FALSE;
-          if (result)
-            return result;
+            (dtd->standalone ? ! parser->m_openInternalEntities
+                             : ! dtd->hasParamEntityRefs);
+      else /* if (pool == &parser->m_tempPool): we are called from content */
+        checkEntityDecl = ! dtd->hasParamEntityRefs || dtd->standalone;
+      if (checkEntityDecl) {
+        if (! entity)
+          return XML_ERROR_UNDEFINED_ENTITY;
+        else if (! entity->is_internal)
+          return XML_ERROR_ENTITY_DECLARED_IN_PE;
+      } else if (! entity) {
+        /* Cannot report skipped entity here - see comments on
+           parser->m_skippedEntityHandler.
+        if (parser->m_skippedEntityHandler)
+          parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0);
+        */
+        /* Cannot call the default handler because this would be
+           out of sync with the call to the startElementHandler.
+        if ((pool == &parser->m_tempPool) && parser->m_defaultHandler)
+          reportDefault(parser, enc, ptr, next);
+        */
+        break;
+      }
+      if (entity->open) {
+        if (enc == parser->m_encoding) {
+          /* It does not appear that this line can be executed.
+           *
+           * The "if (entity->open)" check catches recursive entity
+           * definitions.  In order to be called with an open
+           * entity, it must have gone through this code before and
+           * been through the recursive call to
+           * appendAttributeValue() some lines below.  That call
+           * sets the local encoding ("enc") to the parser's
+           * internal encoding (internal_utf8 or internal_utf16),
+           * which can never be the same as the principle encoding.
+           * It doesn't appear there is another code path that gets
+           * here with entity->open being TRUE.
+           *
+           * Since it is not certain that this logic is watertight,
+           * we keep the line and merely exclude it from coverage
+           * tests.
+           */
+          parser->m_eventPtr = ptr; /* LCOV_EXCL_LINE */
         }
+        return XML_ERROR_RECURSIVE_ENTITY_REF;
       }
-      break;
+      if (entity->notation) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_BINARY_ENTITY_REF;
+      }
+      if (! entity->textPtr) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = ptr;
+        return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF;
+      } else {
+        enum XML_Error result;
+        const XML_Char *textEnd = entity->textPtr + entity->textLen;
+        entity->open = XML_TRUE;
+        result = appendAttributeValue(parser, parser->m_internalEncoding,
+                                      isCdata, (char *)entity->textPtr,
+                                      (char *)textEnd, pool);
+        entity->open = XML_FALSE;
+        if (result)
+          return result;
+      }
+    } break;
     default:
       /* The only token returned by XmlAttributeValueTok() that does
        * not have an explicit case here is XML_TOK_PARTIAL_CHAR.
@@ -5705,12 +5459,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata,
 }
 
 static enum XML_Error
-storeEntityValue(XML_Parser parser,
-                 const ENCODING *enc,
-                 const char *entityTextPtr,
-                 const char *entityTextEnd)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+storeEntityValue(XML_Parser parser, const ENCODING *enc,
+                 const char *entityTextPtr, const char *entityTextEnd) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   STRING_POOL *pool = &(dtd->entityValuePool);
   enum XML_Error result = XML_ERROR_NONE;
 #ifdef XML_DTD
@@ -5720,8 +5471,8 @@ storeEntityValue(XML_Parser parser,
   /* never return Null for the value argument in EntityDeclHandler,
      since this would indicate an external entity; therefore we
      have to make sure that entityValuePool.start is not null */
-  if (!pool->blocks) {
-    if (!poolGrow(pool))
+  if (! pool->blocks) {
+    if (! poolGrow(pool))
       return XML_ERROR_NO_MEMORY;
   }
 
@@ -5737,13 +5488,13 @@ storeEntityValue(XML_Parser parser,
         name = poolStoreString(&parser->m_tempPool, enc,
                                entityTextPtr + enc->minBytesPerChar,
                                next - enc->minBytesPerChar);
-        if (!name) {
+        if (! name) {
           result = XML_ERROR_NO_MEMORY;
           goto endEntityValue;
         }
         entity = (ENTITY *)lookup(parser, &dtd->paramEntities, name, 0);
         poolDiscard(&parser->m_tempPool);
-        if (!entity) {
+        if (! entity) {
           /* not a well-formedness error - see XML 1.0: WFC Entity Declared */
           /* cannot report skipped entity here - see comments on
              parser->m_skippedEntityHandler
@@ -5763,29 +5514,23 @@ storeEntityValue(XML_Parser parser,
           if (parser->m_externalEntityRefHandler) {
             dtd->paramEntityRead = XML_FALSE;
             entity->open = XML_TRUE;
-            if (!parser->m_externalEntityRefHandler(parser->m_externalEntityRefHandlerArg,
-                                          0,
-                                          entity->base,
-                                          entity->systemId,
-                                          entity->publicId)) {
+            if (! parser->m_externalEntityRefHandler(
+                    parser->m_externalEntityRefHandlerArg, 0, entity->base,
+                    entity->systemId, entity->publicId)) {
               entity->open = XML_FALSE;
               result = XML_ERROR_EXTERNAL_ENTITY_HANDLING;
               goto endEntityValue;
             }
             entity->open = XML_FALSE;
-            if (!dtd->paramEntityRead)
+            if (! dtd->paramEntityRead)
               dtd->keepProcessing = dtd->standalone;
-          }
-          else
+          } else
             dtd->keepProcessing = dtd->standalone;
-        }
-        else {
+        } else {
           entity->open = XML_TRUE;
-          result = storeEntityValue(parser,
-                                    parser->m_internalEncoding,
-                                    (char *)entity->textPtr,
-                                    (char *)(entity->textPtr
-                                             + entity->textLen));
+          result = storeEntityValue(
+              parser, parser->m_internalEncoding, (char *)entity->textPtr,
+              (char *)(entity->textPtr + entity->textLen));
           entity->open = XML_FALSE;
           if (result)
             goto endEntityValue;
@@ -5803,7 +5548,7 @@ storeEntityValue(XML_Parser parser,
       goto endEntityValue;
     case XML_TOK_ENTITY_REF:
     case XML_TOK_DATA_CHARS:
-      if (!poolAppend(pool, enc, entityTextPtr, next)) {
+      if (! poolAppend(pool, enc, entityTextPtr, next)) {
         result = XML_ERROR_NO_MEMORY;
         goto endEntityValue;
       }
@@ -5812,42 +5557,40 @@ storeEntityValue(XML_Parser parser,
       next = entityTextPtr + enc->minBytesPerChar;
       /* fall through */
     case XML_TOK_DATA_NEWLINE:
-      if (pool->end == pool->ptr && !poolGrow(pool)) {
-              result = XML_ERROR_NO_MEMORY;
+      if (pool->end == pool->ptr && ! poolGrow(pool)) {
+        result = XML_ERROR_NO_MEMORY;
         goto endEntityValue;
       }
       *(pool->ptr)++ = 0xA;
       break;
-    case XML_TOK_CHAR_REF:
-      {
-        XML_Char buf[XML_ENCODE_MAX];
-        int i;
-        int n = XmlCharRefNumber(enc, entityTextPtr);
-        if (n < 0) {
-          if (enc == parser->m_encoding)
-            parser->m_eventPtr = entityTextPtr;
-          result = XML_ERROR_BAD_CHAR_REF;
+    case XML_TOK_CHAR_REF: {
+      XML_Char buf[XML_ENCODE_MAX];
+      int i;
+      int n = XmlCharRefNumber(enc, entityTextPtr);
+      if (n < 0) {
+        if (enc == parser->m_encoding)
+          parser->m_eventPtr = entityTextPtr;
+        result = XML_ERROR_BAD_CHAR_REF;
+        goto endEntityValue;
+      }
+      n = XmlEncode(n, (ICHAR *)buf);
+      /* The XmlEncode() functions can never return 0 here.  That
+       * error return happens if the code point passed in is either
+       * negative or greater than or equal to 0x110000.  The
+       * XmlCharRefNumber() functions will all return a number
+       * strictly less than 0x110000 or a negative value if an error
+       * occurred.  The negative value is intercepted above, so
+       * XmlEncode() is never passed a value it might return an
+       * error for.
+       */
+      for (i = 0; i < n; i++) {
+        if (pool->end == pool->ptr && ! poolGrow(pool)) {
+          result = XML_ERROR_NO_MEMORY;
           goto endEntityValue;
         }
-        n = XmlEncode(n, (ICHAR *)buf);
-        /* The XmlEncode() functions can never return 0 here.  That
-         * error return happens if the code point passed in is either
-         * negative or greater than or equal to 0x110000.  The
-         * XmlCharRefNumber() functions will all return a number
-         * strictly less than 0x110000 or a negative value if an error
-         * occurred.  The negative value is intercepted above, so
-         * XmlEncode() is never passed a value it might return an
-         * error for.
-         */
-        for (i = 0; i < n; i++) {
-          if (pool->end == pool->ptr && !poolGrow(pool)) {
-            result = XML_ERROR_NO_MEMORY;
-            goto endEntityValue;
-          }
-          *(pool->ptr)++ = buf[i];
-        }
+        *(pool->ptr)++ = buf[i];
       }
-      break;
+    } break;
     case XML_TOK_PARTIAL:
       if (enc == parser->m_encoding)
         parser->m_eventPtr = entityTextPtr;
@@ -5882,8 +5625,7 @@ storeEntityValue(XML_Parser parser,
 }
 
 static void FASTCALL
-normalizeLines(XML_Char *s)
-{
+normalizeLines(XML_Char *s) {
   XML_Char *p;
   for (;; s++) {
     if (*s == XML_T('\0'))
@@ -5897,8 +5639,7 @@ normalizeLines(XML_Char *s)
       *p++ = 0xA;
       if (*++s == 0xA)
         s++;
-    }
-    else
+    } else
       *p++ = *s++;
   } while (*s);
   *p = XML_T('\0');
@@ -5906,12 +5647,11 @@ normalizeLines(XML_Char *s)
 
 static int
 reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
-                            const char *start, const char *end)
-{
+                            const char *start, const char *end) {
   const XML_Char *target;
   XML_Char *data;
   const char *tem;
-  if (!parser->m_processingInstructionHandler) {
+  if (! parser->m_processingInstructionHandler) {
     if (parser->m_defaultHandler)
       reportDefault(parser, enc, start, end);
     return 1;
@@ -5919,13 +5659,12 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
   start += enc->minBytesPerChar * 2;
   tem = start + XmlNameLength(enc, start);
   target = poolStoreString(&parser->m_tempPool, enc, start, tem);
-  if (!target)
+  if (! target)
     return 0;
   poolFinish(&parser->m_tempPool);
-  data = poolStoreString(&parser->m_tempPool, enc,
-                        XmlSkipS(enc, tem),
-                        end - enc->minBytesPerChar*2);
-  if (!data)
+  data = poolStoreString(&parser->m_tempPool, enc, XmlSkipS(enc, tem),
+                         end - enc->minBytesPerChar * 2);
+  if (! data)
     return 0;
   normalizeLines(data);
   parser->m_processingInstructionHandler(parser->m_handlerArg, target, data);
@@ -5934,20 +5673,18 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc,
 }
 
 static int
-reportComment(XML_Parser parser, const ENCODING *enc,
-              const char *start, const char *end)
-{
+reportComment(XML_Parser parser, const ENCODING *enc, const char *start,
+              const char *end) {
   XML_Char *data;
-  if (!parser->m_commentHandler) {
+  if (! parser->m_commentHandler) {
     if (parser->m_defaultHandler)
       reportDefault(parser, enc, start, end);
     return 1;
   }
-  data = poolStoreString(&parser->m_tempPool,
-                         enc,
+  data = poolStoreString(&parser->m_tempPool, enc,
                          start + enc->minBytesPerChar * 4,
                          end - enc->minBytesPerChar * 3);
-  if (!data)
+  if (! data)
     return 0;
   normalizeLines(data);
   parser->m_commentHandler(parser->m_handlerArg, data);
@@ -5956,9 +5693,8 @@ reportComment(XML_Parser parser, const ENCODING *enc,
 }
 
 static void
-reportDefault(XML_Parser parser, const ENCODING *enc,
-              const char *s, const char *end)
-{
+reportDefault(XML_Parser parser, const ENCODING *enc, const char *s,
+              const char *end) {
   if (MUST_CONVERT(enc, s)) {
     enum XML_Convert_Result convert_res;
     const char **eventPP;
@@ -5966,8 +5702,7 @@ reportDefault(XML_Parser parser, const ENCODING *enc,
     if (enc == parser->m_encoding) {
       eventPP = &parser->m_eventPtr;
       eventEndPP = &parser->m_eventEndPtr;
-    }
-    else {
+    } else {
       /* To get here, two things must be true; the parser must be
        * using a character encoding that is not the same as the
        * encoding passed in, and the encoding passed in must need
@@ -5990,21 +5725,22 @@ reportDefault(XML_Parser parser, const ENCODING *enc,
     }
     do {
       ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf;
-      convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
+      convert_res
+          = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd);
       *eventEndPP = s;
-      parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
+      parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf,
+                               (int)(dataPtr - (ICHAR *)parser->m_dataBuf));
       *eventPP = s;
-    } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE));
-  }
-  else
-    parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s, (int)((XML_Char *)end - (XML_Char *)s));
+    } while ((convert_res != XML_CONVERT_COMPLETED)
+             && (convert_res != XML_CONVERT_INPUT_INCOMPLETE));
+  } else
+    parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s,
+                             (int)((XML_Char *)end - (XML_Char *)s));
 }
 
-
 static int
 defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
-                XML_Bool isId, const XML_Char *value, XML_Parser parser)
-{
+                XML_Bool isId, const XML_Char *value, XML_Parser parser) {
   DEFAULT_ATTRIBUTE *att;
   if (value || isId) {
     /* The handling of default attributes gets messed up if we have
@@ -6013,24 +5749,23 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
     for (i = 0; i < type->nDefaultAtts; i++)
       if (attId == type->defaultAtts[i].id)
         return 1;
-    if (isId && !type->idAtt && !attId->xmlns)
+    if (isId && ! type->idAtt && ! attId->xmlns)
       type->idAtt = attId;
   }
   if (type->nDefaultAtts == type->allocDefaultAtts) {
     if (type->allocDefaultAtts == 0) {
       type->allocDefaultAtts = 8;
-      type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC(parser, type->allocDefaultAtts
-                            * sizeof(DEFAULT_ATTRIBUTE));
-      if (!type->defaultAtts) {
+      type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC(
+          parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
+      if (! type->defaultAtts) {
         type->allocDefaultAtts = 0;
         return 0;
       }
-    }
-    else {
+    } else {
       DEFAULT_ATTRIBUTE *temp;
       int count = type->allocDefaultAtts * 2;
-      temp = (DEFAULT_ATTRIBUTE *)
-        REALLOC(parser, type->defaultAtts, (count * sizeof(DEFAULT_ATTRIBUTE)));
+      temp = (DEFAULT_ATTRIBUTE *)REALLOC(parser, type->defaultAtts,
+                                          (count * sizeof(DEFAULT_ATTRIBUTE)));
       if (temp == NULL)
         return 0;
       type->allocDefaultAtts = count;
@@ -6041,30 +5776,29 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata,
   att->id = attId;
   att->value = value;
   att->isCdata = isCdata;
-  if (!isCdata)
+  if (! isCdata)
     attId->maybeTokenized = XML_TRUE;
   type->nDefaultAtts += 1;
   return 1;
 }
 
 static int
-setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *name;
   for (name = elementType->name; *name; name++) {
     if (*name == XML_T(ASCII_COLON)) {
       PREFIX *prefix;
       const XML_Char *s;
       for (s = elementType->name; s != name; s++) {
-        if (!poolAppendChar(&dtd->pool, *s))
+        if (! poolAppendChar(&dtd->pool, *s))
           return 0;
       }
-      if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+      if (! poolAppendChar(&dtd->pool, XML_T('\0')))
         return 0;
       prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&dtd->pool),
                                 sizeof(PREFIX));
-      if (!prefix)
+      if (! prefix)
         return 0;
       if (prefix->name == poolStart(&dtd->pool))
         poolFinish(&dtd->pool);
@@ -6078,55 +5812,53 @@ setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType)
 }
 
 static ATTRIBUTE_ID *
-getAttributeId(XML_Parser parser, const ENCODING *enc,
-               const char *start, const char *end)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start,
+               const char *end) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   ATTRIBUTE_ID *id;
   const XML_Char *name;
-  if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+  if (! poolAppendChar(&dtd->pool, XML_T('\0')))
     return NULL;
   name = poolStoreString(&dtd->pool, enc, start, end);
-  if (!name)
+  if (! name)
     return NULL;
   /* skip quotation mark - its storage will be re-used (like in name[-1]) */
   ++name;
-  id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name, sizeof(ATTRIBUTE_ID));
-  if (!id)
+  id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name,
+                              sizeof(ATTRIBUTE_ID));
+  if (! id)
     return NULL;
   if (id->name != name)
     poolDiscard(&dtd->pool);
   else {
     poolFinish(&dtd->pool);
-    if (!parser->m_ns)
+    if (! parser->m_ns)
       ;
-    else if (name[0] == XML_T(ASCII_x)
-        && name[1] == XML_T(ASCII_m)
-        && name[2] == XML_T(ASCII_l)
-        && name[3] == XML_T(ASCII_n)
-        && name[4] == XML_T(ASCII_s)
-        && (name[5] == XML_T('\0') || name[5] == XML_T(ASCII_COLON))) {
+    else if (name[0] == XML_T(ASCII_x) && name[1] == XML_T(ASCII_m)
+             && name[2] == XML_T(ASCII_l) && name[3] == XML_T(ASCII_n)
+             && name[4] == XML_T(ASCII_s)
+             && (name[5] == XML_T('\0') || name[5] == XML_T(ASCII_COLON))) {
       if (name[5] == XML_T('\0'))
         id->prefix = &dtd->defaultPrefix;
       else
-        id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6, sizeof(PREFIX));
+        id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6,
+                                      sizeof(PREFIX));
       id->xmlns = XML_TRUE;
-    }
-    else {
+    } else {
       int i;
       for (i = 0; name[i]; i++) {
         /* attributes without prefix are *not* in the default namespace */
         if (name[i] == XML_T(ASCII_COLON)) {
           int j;
           for (j = 0; j < i; j++) {
-            if (!poolAppendChar(&dtd->pool, name[j]))
+            if (! poolAppendChar(&dtd->pool, name[j]))
               return NULL;
           }
-          if (!poolAppendChar(&dtd->pool, XML_T('\0')))
+          if (! poolAppendChar(&dtd->pool, XML_T('\0')))
             return NULL;
-          id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&dtd->pool),
-                                        sizeof(PREFIX));
-          if (!id->prefix)
+          id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes,
+                                        poolStart(&dtd->pool), sizeof(PREFIX));
+          if (! id->prefix)
             return NULL;
           if (id->prefix->name == poolStart(&dtd->pool))
             poolFinish(&dtd->pool);
@@ -6143,22 +5875,22 @@ getAttributeId(XML_Parser parser, const ENCODING *enc,
 #define CONTEXT_SEP XML_T(ASCII_FF)
 
 static const XML_Char *
-getContext(XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getContext(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   HASH_TABLE_ITER iter;
   XML_Bool needSep = XML_FALSE;
 
   if (dtd->defaultPrefix.binding) {
     int i;
     int len;
-    if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
+    if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
       return NULL;
     len = dtd->defaultPrefix.binding->uriLen;
     if (parser->m_namespaceSeparator)
       len--;
     for (i = 0; i < len; i++) {
-      if (!poolAppendChar(&parser->m_tempPool, dtd->defaultPrefix.binding->uri[i])) {
+      if (! poolAppendChar(&parser->m_tempPool,
+                           dtd->defaultPrefix.binding->uri[i])) {
         /* Because of memory caching, I don't believe this line can be
          * executed.
          *
@@ -6190,9 +5922,9 @@ getContext(XML_Parser parser)
     int len;
     const XML_Char *s;
     PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter);
-    if (!prefix)
+    if (! prefix)
       break;
-    if (!prefix->binding) {
+    if (! prefix->binding) {
       /* This test appears to be (justifiable) paranoia.  There does
        * not seem to be a way of injecting a prefix without a binding
        * that doesn't get errored long before this function is called.
@@ -6201,98 +5933,96 @@ getContext(XML_Parser parser)
        */
       continue; /* LCOV_EXCL_LINE */
     }
-    if (needSep && !poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
+    if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
       return NULL;
     for (s = prefix->name; *s; s++)
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return NULL;
-    if (!poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
+    if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS)))
       return NULL;
     len = prefix->binding->uriLen;
     if (parser->m_namespaceSeparator)
       len--;
     for (i = 0; i < len; i++)
-      if (!poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i]))
+      if (! poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i]))
         return NULL;
     needSep = XML_TRUE;
   }
 
-
   hashTableIterInit(&iter, &(dtd->generalEntities));
   for (;;) {
     const XML_Char *s;
     ENTITY *e = (ENTITY *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
-    if (!e->open)
+    if (! e->open)
       continue;
-    if (needSep && !poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
+    if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP))
       return NULL;
     for (s = e->name; *s; s++)
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return 0;
     needSep = XML_TRUE;
   }
 
-  if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+  if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
     return NULL;
   return parser->m_tempPool.start;
 }
 
 static XML_Bool
-setContext(XML_Parser parser, const XML_Char *context)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+setContext(XML_Parser parser, const XML_Char *context) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *s = context;
 
   while (*context != XML_T('\0')) {
     if (*s == CONTEXT_SEP || *s == XML_T('\0')) {
       ENTITY *e;
-      if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+      if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
         return XML_FALSE;
-      e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0);
+      e = (ENTITY *)lookup(parser, &dtd->generalEntities,
+                           poolStart(&parser->m_tempPool), 0);
       if (e)
         e->open = XML_TRUE;
       if (*s != XML_T('\0'))
         s++;
       context = s;
       poolDiscard(&parser->m_tempPool);
-    }
-    else if (*s == XML_T(ASCII_EQUALS)) {
+    } else if (*s == XML_T(ASCII_EQUALS)) {
       PREFIX *prefix;
       if (poolLength(&parser->m_tempPool) == 0)
         prefix = &dtd->defaultPrefix;
       else {
-        if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+        if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
           return XML_FALSE;
-        prefix = (PREFIX *)lookup(parser, &dtd->prefixes, poolStart(&parser->m_tempPool),
-                                  sizeof(PREFIX));
-        if (!prefix)
+        prefix
+            = (PREFIX *)lookup(parser, &dtd->prefixes,
+                               poolStart(&parser->m_tempPool), sizeof(PREFIX));
+        if (! prefix)
           return XML_FALSE;
         if (prefix->name == poolStart(&parser->m_tempPool)) {
           prefix->name = poolCopyString(&dtd->pool, prefix->name);
-          if (!prefix->name)
+          if (! prefix->name)
             return XML_FALSE;
         }
         poolDiscard(&parser->m_tempPool);
       }
-      for (context = s + 1;
-           *context != CONTEXT_SEP && *context != XML_T('\0');
+      for (context = s + 1; *context != CONTEXT_SEP && *context != XML_T('\0');
            context++)
-        if (!poolAppendChar(&parser->m_tempPool, *context))
+        if (! poolAppendChar(&parser->m_tempPool, *context))
           return XML_FALSE;
-      if (!poolAppendChar(&parser->m_tempPool, XML_T('\0')))
+      if (! poolAppendChar(&parser->m_tempPool, XML_T('\0')))
         return XML_FALSE;
       if (addBinding(parser, prefix, NULL, poolStart(&parser->m_tempPool),
-                     &parser->m_inheritedBindings) != XML_ERROR_NONE)
+                     &parser->m_inheritedBindings)
+          != XML_ERROR_NONE)
         return XML_FALSE;
       poolDiscard(&parser->m_tempPool);
       if (*context != XML_T('\0'))
         ++context;
       s = context;
-    }
-    else {
-      if (!poolAppendChar(&parser->m_tempPool, *s))
+    } else {
+      if (! poolAppendChar(&parser->m_tempPool, *s))
         return XML_FALSE;
       s++;
     }
@@ -6301,8 +6031,7 @@ setContext(XML_Parser parser, const XML_Char *context)
 }
 
 static void FASTCALL
-normalizePublicId(XML_Char *publicId)
-{
+normalizePublicId(XML_Char *publicId) {
   XML_Char *p = publicId;
   XML_Char *s;
   for (s = publicId; *s; s++) {
@@ -6323,8 +6052,7 @@ normalizePublicId(XML_Char *publicId)
 }
 
 static DTD *
-dtdCreate(const XML_Memory_Handling_Suite *ms)
-{
+dtdCreate(const XML_Memory_Handling_Suite *ms) {
   DTD *p = (DTD *)ms->malloc_fcn(sizeof(DTD));
   if (p == NULL)
     return p;
@@ -6356,13 +6084,12 @@ dtdCreate(const XML_Memory_Handling_Suite *ms)
 }
 
 static void
-dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms)
-{
+dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
   hashTableIterInit(&iter, &(p->elementTypes));
   for (;;) {
     ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
     if (e->allocDefaultAtts != 0)
       ms->free_fcn(e->defaultAtts);
@@ -6398,13 +6125,12 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms)
 }
 
 static void
-dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms)
-{
+dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
   hashTableIterInit(&iter, &(p->elementTypes));
   for (;;) {
     ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!e)
+    if (! e)
       break;
     if (e->allocDefaultAtts != 0)
       ms->free_fcn(e->defaultAtts);
@@ -6429,8 +6155,8 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms)
    The new DTD has already been initialized.
 */
 static int
-dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms)
-{
+dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd,
+        const XML_Memory_Handling_Suite *ms) {
   HASH_TABLE_ITER iter;
 
   /* Copy the prefix table. */
@@ -6439,12 +6165,12 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
   for (;;) {
     const XML_Char *name;
     const PREFIX *oldP = (PREFIX *)hashTableIterNext(&iter);
-    if (!oldP)
+    if (! oldP)
       break;
     name = poolCopyString(&(newDtd->pool), oldP->name);
-    if (!name)
+    if (! name)
       return 0;
-    if (!lookup(oldParser, &(newDtd->prefixes), name, sizeof(PREFIX)))
+    if (! lookup(oldParser, &(newDtd->prefixes), name, sizeof(PREFIX)))
       return 0;
   }
 
@@ -6457,18 +6183,18 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
     const XML_Char *name;
     const ATTRIBUTE_ID *oldA = (ATTRIBUTE_ID *)hashTableIterNext(&iter);
 
-    if (!oldA)
+    if (! oldA)
       break;
     /* Remember to allocate the scratch byte before the name. */
-    if (!poolAppendChar(&(newDtd->pool), XML_T('\0')))
+    if (! poolAppendChar(&(newDtd->pool), XML_T('\0')))
       return 0;
     name = poolCopyString(&(newDtd->pool), oldA->name);
-    if (!name)
+    if (! name)
       return 0;
     ++name;
     newA = (ATTRIBUTE_ID *)lookup(oldParser, &(newDtd->attributeIds), name,
                                   sizeof(ATTRIBUTE_ID));
-    if (!newA)
+    if (! newA)
       return 0;
     newA->maybeTokenized = oldA->maybeTokenized;
     if (oldA->prefix) {
@@ -6490,57 +6216,52 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
     ELEMENT_TYPE *newE;
     const XML_Char *name;
     const ELEMENT_TYPE *oldE = (ELEMENT_TYPE *)hashTableIterNext(&iter);
-    if (!oldE)
+    if (! oldE)
       break;
     name = poolCopyString(&(newDtd->pool), oldE->name);
-    if (!name)
+    if (! name)
       return 0;
     newE = (ELEMENT_TYPE *)lookup(oldParser, &(newDtd->elementTypes), name,
                                   sizeof(ELEMENT_TYPE));
-    if (!newE)
+    if (! newE)
       return 0;
     if (oldE->nDefaultAtts) {
-      newE->defaultAtts = (DEFAULT_ATTRIBUTE *)
-          ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
-      if (!newE->defaultAtts) {
+      newE->defaultAtts = (DEFAULT_ATTRIBUTE *)ms->malloc_fcn(
+          oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE));
+      if (! newE->defaultAtts) {
         return 0;
       }
     }
     if (oldE->idAtt)
-      newE->idAtt = (ATTRIBUTE_ID *)
-          lookup(oldParser, &(newDtd->attributeIds), oldE->idAtt->name, 0);
+      newE->idAtt = (ATTRIBUTE_ID *)lookup(oldParser, &(newDtd->attributeIds),
+                                           oldE->idAtt->name, 0);
     newE->allocDefaultAtts = newE->nDefaultAtts = oldE->nDefaultAtts;
     if (oldE->prefix)
       newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes),
                                       oldE->prefix->name, 0);
     for (i = 0; i < newE->nDefaultAtts; i++) {
-      newE->defaultAtts[i].id = (ATTRIBUTE_ID *)
-          lookup(oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0);
+      newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup(
+          oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0);
       newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata;
       if (oldE->defaultAtts[i].value) {
         newE->defaultAtts[i].value
             = poolCopyString(&(newDtd->pool), oldE->defaultAtts[i].value);
-        if (!newE->defaultAtts[i].value)
+        if (! newE->defaultAtts[i].value)
           return 0;
-      }
-      else
+      } else
         newE->defaultAtts[i].value = NULL;
     }
   }
 
   /* Copy the entity tables. */
-  if (!copyEntityTable(oldParser,
-                       &(newDtd->generalEntities),
-                       &(newDtd->pool),
-                       &(oldDtd->generalEntities)))
-      return 0;
+  if (! copyEntityTable(oldParser, &(newDtd->generalEntities), &(newDtd->pool),
+                        &(oldDtd->generalEntities)))
+    return 0;
 
 #ifdef XML_DTD
-  if (!copyEntityTable(oldParser,
-                       &(newDtd->paramEntities),
-                       &(newDtd->pool),
-                       &(oldDtd->paramEntities)))
-      return 0;
+  if (! copyEntityTable(oldParser, &(newDtd->paramEntities), &(newDtd->pool),
+                        &(oldDtd->paramEntities)))
+    return 0;
   newDtd->paramEntityRead = oldDtd->paramEntityRead;
 #endif /* XML_DTD */
 
@@ -6557,14 +6278,11 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H
   newDtd->scaffIndex = oldDtd->scaffIndex;
 
   return 1;
-}  /* End dtdCopy */
+} /* End dtdCopy */
 
 static int
-copyEntityTable(XML_Parser oldParser,
-                HASH_TABLE *newTable,
-                STRING_POOL *newPool,
-                const HASH_TABLE *oldTable)
-{
+copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable,
+                STRING_POOL *newPool, const HASH_TABLE *oldTable) {
   HASH_TABLE_ITER iter;
   const XML_Char *cachedOldBase = NULL;
   const XML_Char *cachedNewBase = NULL;
@@ -6575,17 +6293,17 @@ copyEntityTable(XML_Parser oldParser,
     ENTITY *newE;
     const XML_Char *name;
     const ENTITY *oldE = (ENTITY *)hashTableIterNext(&iter);
-    if (!oldE)
+    if (! oldE)
       break;
     name = poolCopyString(newPool, oldE->name);
-    if (!name)
+    if (! name)
       return 0;
     newE = (ENTITY *)lookup(oldParser, newTable, name, sizeof(ENTITY));
-    if (!newE)
+    if (! newE)
       return 0;
     if (oldE->systemId) {
       const XML_Char *tem = poolCopyString(newPool, oldE->systemId);
-      if (!tem)
+      if (! tem)
         return 0;
       newE->systemId = tem;
       if (oldE->base) {
@@ -6594,29 +6312,28 @@ copyEntityTable(XML_Parser oldParser,
         else {
           cachedOldBase = oldE->base;
           tem = poolCopyString(newPool, cachedOldBase);
-          if (!tem)
+          if (! tem)
             return 0;
           cachedNewBase = newE->base = tem;
         }
       }
       if (oldE->publicId) {
         tem = poolCopyString(newPool, oldE->publicId);
-        if (!tem)
+        if (! tem)
           return 0;
         newE->publicId = tem;
       }
-    }
-    else {
-      const XML_Char *tem = poolCopyStringN(newPool, oldE->textPtr,
-                                            oldE->textLen);
-      if (!tem)
+    } else {
+      const XML_Char *tem
+          = poolCopyStringN(newPool, oldE->textPtr, oldE->textLen);
+      if (! tem)
         return 0;
       newE->textPtr = tem;
       newE->textLen = oldE->textLen;
     }
     if (oldE->notation) {
       const XML_Char *tem = poolCopyString(newPool, oldE->notation);
-      if (!tem)
+      if (! tem)
         return 0;
       newE->notation = tem;
     }
@@ -6629,8 +6346,7 @@ copyEntityTable(XML_Parser oldParser,
 #define INIT_POWER 6
 
 static XML_Bool FASTCALL
-keyeq(KEY s1, KEY s2)
-{
+keyeq(KEY s1, KEY s2) {
   for (; *s1 == *s2; s1++, s2++)
     if (*s1 == 0)
       return XML_TRUE;
@@ -6638,23 +6354,21 @@ keyeq(KEY s1, KEY s2)
 }
 
 static size_t
-keylen(KEY s)
-{
+keylen(KEY s) {
   size_t len = 0;
-  for (; *s; s++, len++);
+  for (; *s; s++, len++)
+    ;
   return len;
 }
 
 static void
-copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key)
-{
+copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) {
   key->k[0] = 0;
   key->k[1] = get_hash_secret_salt(parser);
 }
 
 static unsigned long FASTCALL
-hash(XML_Parser parser, KEY s)
-{
+hash(XML_Parser parser, KEY s) {
   struct siphash state;
   struct sipkey key;
   (void)sip24_valid;
@@ -6665,26 +6379,24 @@ hash(XML_Parser parser, KEY s)
 }
 
 static NAMED *
-lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
-{
+lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) {
   size_t i;
   if (table->size == 0) {
     size_t tsize;
-    if (!createSize)
+    if (! createSize)
       return NULL;
     table->power = INIT_POWER;
     /* table->size is a power of 2 */
     table->size = (size_t)1 << INIT_POWER;
     tsize = table->size * sizeof(NAMED *);
     table->v = (NAMED **)table->mem->malloc_fcn(tsize);
-    if (!table->v) {
+    if (! table->v) {
       table->size = 0;
       return NULL;
     }
     memset(table->v, 0, tsize);
     i = hash(parser, name) & ((unsigned long)table->size - 1);
-  }
-  else {
+  } else {
     unsigned long h = hash(parser, name);
     unsigned long mask = (unsigned long)table->size - 1;
     unsigned char step = 0;
@@ -6692,11 +6404,11 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
     while (table->v[i]) {
       if (keyeq(name, table->v[i]->name))
         return table->v[i];
-      if (!step)
+      if (! step)
         step = PROBE_STEP(h, mask, table->power);
       i < step ? (i += table->size - step) : (i -= step);
     }
-    if (!createSize)
+    if (! createSize)
       return NULL;
 
     /* check for overflow (table is half full) */
@@ -6706,7 +6418,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
       unsigned long newMask = (unsigned long)newSize - 1;
       size_t tsize = newSize * sizeof(NAMED *);
       NAMED **newV = (NAMED **)table->mem->malloc_fcn(tsize);
-      if (!newV)
+      if (! newV)
         return NULL;
       memset(newV, 0, tsize);
       for (i = 0; i < table->size; i++)
@@ -6715,7 +6427,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
           size_t j = newHash & newMask;
           step = 0;
           while (newV[j]) {
-            if (!step)
+            if (! step)
               step = PROBE_STEP(newHash, newMask, newPower);
             j < step ? (j += newSize - step) : (j -= step);
           }
@@ -6728,14 +6440,14 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
       i = h & newMask;
       step = 0;
       while (table->v[i]) {
-        if (!step)
+        if (! step)
           step = PROBE_STEP(h, newMask, newPower);
         i < step ? (i += newSize - step) : (i -= step);
       }
     }
   }
   table->v[i] = (NAMED *)table->mem->malloc_fcn(createSize);
-  if (!table->v[i])
+  if (! table->v[i])
     return NULL;
   memset(table->v[i], 0, createSize);
   table->v[i]->name = name;
@@ -6744,8 +6456,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize)
 }
 
 static void FASTCALL
-hashTableClear(HASH_TABLE *table)
-{
+hashTableClear(HASH_TABLE *table) {
   size_t i;
   for (i = 0; i < table->size; i++) {
     table->mem->free_fcn(table->v[i]);
@@ -6755,8 +6466,7 @@ hashTableClear(HASH_TABLE *table)
 }
 
 static void FASTCALL
-hashTableDestroy(HASH_TABLE *table)
-{
+hashTableDestroy(HASH_TABLE *table) {
   size_t i;
   for (i = 0; i < table->size; i++)
     table->mem->free_fcn(table->v[i]);
@@ -6764,8 +6474,7 @@ hashTableDestroy(HASH_TABLE *table)
 }
 
 static void FASTCALL
-hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms)
-{
+hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) {
   p->power = 0;
   p->size = 0;
   p->used = 0;
@@ -6774,15 +6483,13 @@ hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms)
 }
 
 static void FASTCALL
-hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table)
-{
+hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) {
   iter->p = table->v;
   iter->end = iter->p + table->size;
 }
 
-static NAMED * FASTCALL
-hashTableIterNext(HASH_TABLE_ITER *iter)
-{
+static NAMED *FASTCALL
+hashTableIterNext(HASH_TABLE_ITER *iter) {
   while (iter->p != iter->end) {
     NAMED *tem = *(iter->p)++;
     if (tem)
@@ -6792,8 +6499,7 @@ hashTableIterNext(HASH_TABLE_ITER *iter)
 }
 
 static void FASTCALL
-poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms)
-{
+poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms) {
   pool->blocks = NULL;
   pool->freeBlocks = NULL;
   pool->start = NULL;
@@ -6803,9 +6509,8 @@ poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms)
 }
 
 static void FASTCALL
-poolClear(STRING_POOL *pool)
-{
-  if (!pool->freeBlocks)
+poolClear(STRING_POOL *pool) {
+  if (! pool->freeBlocks)
     pool->freeBlocks = pool->blocks;
   else {
     BLOCK *p = pool->blocks;
@@ -6823,8 +6528,7 @@ poolClear(STRING_POOL *pool)
 }
 
 static void FASTCALL
-poolDestroy(STRING_POOL *pool)
-{
+poolDestroy(STRING_POOL *pool) {
   BLOCK *p = pool->blocks;
   while (p) {
     BLOCK *tem = p->next;
@@ -6840,26 +6544,26 @@ poolDestroy(STRING_POOL *pool)
 }
 
 static XML_Char *
-poolAppend(STRING_POOL *pool, const ENCODING *enc,
-           const char *ptr, const char *end)
-{
-  if (!pool->ptr && !poolGrow(pool))
+poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr,
+           const char *end) {
+  if (! pool->ptr && ! poolGrow(pool))
     return NULL;
   for (;;) {
-    const enum XML_Convert_Result convert_res = XmlConvert(enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end);
-    if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
+    const enum XML_Convert_Result convert_res = XmlConvert(
+        enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end);
+    if ((convert_res == XML_CONVERT_COMPLETED)
+        || (convert_res == XML_CONVERT_INPUT_INCOMPLETE))
       break;
-    if (!poolGrow(pool))
+    if (! poolGrow(pool))
       return NULL;
   }
   return pool->start;
 }
 
-static const XML_Char * FASTCALL
-poolCopyString(STRING_POOL *pool, const XML_Char *s)
-{
+static const XML_Char *FASTCALL
+poolCopyString(STRING_POOL *pool, const XML_Char *s) {
   do {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
   } while (*s++);
   s = pool->start;
@@ -6868,9 +6572,8 @@ poolCopyString(STRING_POOL *pool, const XML_Char *s)
 }
 
 static const XML_Char *
-poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
-{
-  if (!pool->ptr && !poolGrow(pool)) {
+poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) {
+  if (! pool->ptr && ! poolGrow(pool)) {
     /* The following line is unreachable given the current usage of
      * poolCopyStringN().  Currently it is called from exactly one
      * place to copy the text of a simple general entity.  By that
@@ -6885,7 +6588,7 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
     return NULL; /* LCOV_EXCL_LINE */
   }
   for (; n > 0; --n, s++) {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
   }
   s = pool->start;
@@ -6893,11 +6596,10 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n)
   return s;
 }
 
-static const XML_Char * FASTCALL
-poolAppendString(STRING_POOL *pool, const XML_Char *s)
-{
+static const XML_Char *FASTCALL
+poolAppendString(STRING_POOL *pool, const XML_Char *s) {
   while (*s) {
-    if (!poolAppendChar(pool, *s))
+    if (! poolAppendChar(pool, *s))
       return NULL;
     s++;
   }
@@ -6905,20 +6607,18 @@ poolAppendString(STRING_POOL *pool, const XML_Char *s)
 }
 
 static XML_Char *
-poolStoreString(STRING_POOL *pool, const ENCODING *enc,
-                const char *ptr, const char *end)
-{
-  if (!poolAppend(pool, enc, ptr, end))
+poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr,
+                const char *end) {
+  if (! poolAppend(pool, enc, ptr, end))
     return NULL;
-  if (pool->ptr == pool->end && !poolGrow(pool))
+  if (pool->ptr == pool->end && ! poolGrow(pool))
     return NULL;
   *(pool->ptr)++ = 0;
   return pool->start;
 }
 
 static size_t
-poolBytesToAllocateFor(int blockSize)
-{
+poolBytesToAllocateFor(int blockSize) {
   /* Unprotected math would be:
   ** return offsetof(BLOCK, s) + blockSize * sizeof(XML_Char);
   **
@@ -6926,7 +6626,7 @@ poolBytesToAllocateFor(int blockSize)
   ** For a + b * c we check b * c in isolation first, so that addition of a
   ** on top has no chance of making us accept a small non-negative number
   */
-  const size_t stretch = sizeof(XML_Char);  /* can be 4 bytes */
+  const size_t stretch = sizeof(XML_Char); /* can be 4 bytes */
 
   if (blockSize <= 0)
     return 0;
@@ -6936,8 +6636,8 @@ poolBytesToAllocateFor(int blockSize)
 
   {
     const int stretchedBlockSize = blockSize * (int)stretch;
-    const int bytesToAllocate = (int)(
-        offsetof(BLOCK, s) + (unsigned)stretchedBlockSize);
+    const int bytesToAllocate
+        = (int)(offsetof(BLOCK, s) + (unsigned)stretchedBlockSize);
     if (bytesToAllocate < 0)
       return 0;
 
@@ -6946,8 +6646,7 @@ poolBytesToAllocateFor(int blockSize)
 }
 
 static XML_Bool FASTCALL
-poolGrow(STRING_POOL *pool)
-{
+poolGrow(STRING_POOL *pool) {
   if (pool->freeBlocks) {
     if (pool->start == 0) {
       pool->blocks = pool->freeBlocks;
@@ -6973,7 +6672,7 @@ poolGrow(STRING_POOL *pool)
   }
   if (pool->blocks && pool->start == pool->blocks->s) {
     BLOCK *temp;
-    int blockSize = (int)((unsigned)(pool->end - pool->start)*2U);
+    int blockSize = (int)((unsigned)(pool->end - pool->start) * 2U);
     size_t bytesToAllocate;
 
     /* NOTE: Needs to be calculated prior to calling `realloc`
@@ -6994,8 +6693,8 @@ poolGrow(STRING_POOL *pool)
     if (bytesToAllocate == 0)
       return XML_FALSE;
 
-    temp = (BLOCK *)
-      pool->mem->realloc_fcn(pool->blocks, (unsigned)bytesToAllocate);
+    temp = (BLOCK *)pool->mem->realloc_fcn(pool->blocks,
+                                           (unsigned)bytesToAllocate);
     if (temp == NULL)
       return XML_FALSE;
     pool->blocks = temp;
@@ -7003,8 +6702,7 @@ poolGrow(STRING_POOL *pool)
     pool->ptr = pool->blocks->s + offsetInsideBlock;
     pool->start = pool->blocks->s;
     pool->end = pool->start + blockSize;
-  }
-  else {
+  } else {
     BLOCK *tem;
     int blockSize = (int)(pool->end - pool->start);
     size_t bytesToAllocate;
@@ -7019,7 +6717,7 @@ poolGrow(STRING_POOL *pool)
        * function).  Either way it isn't readily testable, so we
        * exclude it from the coverage statistics.
        */
-      return XML_FALSE;  /* LCOV_EXCL_LINE */
+      return XML_FALSE; /* LCOV_EXCL_LINE */
     }
 
     if (blockSize < INIT_BLOCK_SIZE)
@@ -7037,14 +6735,13 @@ poolGrow(STRING_POOL *pool)
       return XML_FALSE;
 
     tem = (BLOCK *)pool->mem->malloc_fcn(bytesToAllocate);
-    if (!tem)
+    if (! tem)
       return XML_FALSE;
     tem->size = blockSize;
     tem->next = pool->blocks;
     pool->blocks = tem;
     if (pool->ptr != pool->start)
-      memcpy(tem->s, pool->start,
-             (pool->ptr - pool->start) * sizeof(XML_Char));
+      memcpy(tem->s, pool->start, (pool->ptr - pool->start) * sizeof(XML_Char));
     pool->ptr = tem->s + (pool->ptr - pool->start);
     pool->start = tem->s;
     pool->end = tem->s + blockSize;
@@ -7053,15 +6750,14 @@ poolGrow(STRING_POOL *pool)
 }
 
 static int FASTCALL
-nextScaffoldPart(XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
-  CONTENT_SCAFFOLD * me;
+nextScaffoldPart(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
+  CONTENT_SCAFFOLD *me;
   int next;
 
-  if (!dtd->scaffIndex) {
+  if (! dtd->scaffIndex) {
     dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int));
-    if (!dtd->scaffIndex)
+    if (! dtd->scaffIndex)
       return -1;
     dtd->scaffIndex[0] = 0;
   }
@@ -7069,15 +6765,14 @@ nextScaffoldPart(XML_Parser parser)
   if (dtd->scaffCount >= dtd->scaffSize) {
     CONTENT_SCAFFOLD *temp;
     if (dtd->scaffold) {
-      temp = (CONTENT_SCAFFOLD *)
-        REALLOC(parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD));
+      temp = (CONTENT_SCAFFOLD *)REALLOC(
+          parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD));
       if (temp == NULL)
         return -1;
       dtd->scaffSize *= 2;
-    }
-    else {
+    } else {
       temp = (CONTENT_SCAFFOLD *)MALLOC(parser, INIT_SCAFFOLD_ELEMENTS
-                                        * sizeof(CONTENT_SCAFFOLD));
+                                                    * sizeof(CONTENT_SCAFFOLD));
       if (temp == NULL)
         return -1;
       dtd->scaffSize = INIT_SCAFFOLD_ELEMENTS;
@@ -7087,11 +6782,12 @@ nextScaffoldPart(XML_Parser parser)
   next = dtd->scaffCount++;
   me = &dtd->scaffold[next];
   if (dtd->scaffLevel) {
-    CONTENT_SCAFFOLD *parent = &dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel-1]];
+    CONTENT_SCAFFOLD *parent
+        = &dtd->scaffold[dtd->scaffIndex[dtd->scaffLevel - 1]];
     if (parent->lastchild) {
       dtd->scaffold[parent->lastchild].nextsib = next;
     }
-    if (!parent->childcnt)
+    if (! parent->childcnt)
       parent->firstchild = next;
     parent->lastchild = next;
     parent->childcnt++;
@@ -7101,13 +6797,9 @@ nextScaffoldPart(XML_Parser parser)
 }
 
 static void
-build_node(XML_Parser parser,
-           int src_node,
-           XML_Content *dest,
-           XML_Content **contpos,
-           XML_Char **strpos)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+build_node(XML_Parser parser, int src_node, XML_Content *dest,
+           XML_Content **contpos, XML_Char **strpos) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   dest->type = dtd->scaffold[src_node].type;
   dest->quant = dtd->scaffold[src_node].quant;
   if (dest->type == XML_CTYPE_NAME) {
@@ -7116,21 +6808,19 @@ build_node(XML_Parser parser,
     src = dtd->scaffold[src_node].name;
     for (;;) {
       *(*strpos)++ = *src;
-      if (!*src)
+      if (! *src)
         break;
       src++;
     }
     dest->numchildren = 0;
     dest->children = NULL;
-  }
-  else {
+  } else {
     unsigned int i;
     int cn;
     dest->numchildren = dtd->scaffold[src_node].childcnt;
     dest->children = *contpos;
     *contpos += dest->numchildren;
-    for (i = 0, cn = dtd->scaffold[src_node].firstchild;
-         i < dest->numchildren;
+    for (i = 0, cn = dtd->scaffold[src_node].firstchild; i < dest->numchildren;
          i++, cn = dtd->scaffold[cn].nextsib) {
       build_node(parser, cn, &(dest->children[i]), contpos, strpos);
     }
@@ -7139,20 +6829,19 @@ build_node(XML_Parser parser,
 }
 
 static XML_Content *
-build_model (XML_Parser parser)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+build_model(XML_Parser parser) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   XML_Content *ret;
   XML_Content *cpos;
-  XML_Char * str;
+  XML_Char *str;
   int allocsize = (dtd->scaffCount * sizeof(XML_Content)
                    + (dtd->contentStringLen * sizeof(XML_Char)));
 
   ret = (XML_Content *)MALLOC(parser, allocsize);
-  if (!ret)
+  if (! ret)
     return NULL;
 
-  str =  (XML_Char *) (&ret[dtd->scaffCount]);
+  str = (XML_Char *)(&ret[dtd->scaffCount]);
   cpos = &ret[1];
 
   build_node(parser, 0, ret, &cpos, &str);
@@ -7160,49 +6849,45 @@ build_model (XML_Parser parser)
 }
 
 static ELEMENT_TYPE *
-getElementType(XML_Parser parser,
-               const ENCODING *enc,
-               const char *ptr,
-               const char *end)
-{
-  DTD * const dtd = parser->m_dtd;  /* save one level of indirection */
+getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr,
+               const char *end) {
+  DTD *const dtd = parser->m_dtd; /* save one level of indirection */
   const XML_Char *name = poolStoreString(&dtd->pool, enc, ptr, end);
   ELEMENT_TYPE *ret;
 
-  if (!name)
+  if (! name)
     return NULL;
-  ret = (ELEMENT_TYPE *) lookup(parser, &dtd->elementTypes, name, sizeof(ELEMENT_TYPE));
-  if (!ret)
+  ret = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, name,
+                               sizeof(ELEMENT_TYPE));
+  if (! ret)
     return NULL;
   if (ret->name != name)
     poolDiscard(&dtd->pool);
   else {
     poolFinish(&dtd->pool);
-    if (!setElementTypePrefix(parser, ret))
+    if (! setElementTypePrefix(parser, ret))
       return NULL;
   }
   return ret;
 }
 
 static XML_Char *
-copyString(const XML_Char *s,
-           const XML_Memory_Handling_Suite *memsuite)
-{
-    int charsRequired = 0;
-    XML_Char *result;
-
-    /* First determine how long the string is */
-    while (s[charsRequired] != 0) {
-      charsRequired++;
-    }
-    /* Include the terminator */
+copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) {
+  int charsRequired = 0;
+  XML_Char *result;
+
+  /* First determine how long the string is */
+  while (s[charsRequired] != 0) {
     charsRequired++;
+  }
+  /* Include the terminator */
+  charsRequired++;
 
-    /* Now allocate space for the copy */
-    result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char));
-    if (result == NULL)
-        return NULL;
-    /* Copy the original into place */
-    memcpy(result, s, charsRequired * sizeof(XML_Char));
-    return result;
+  /* Now allocate space for the copy */
+  result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char));
+  if (result == NULL)
+    return NULL;
+  /* Copy the original into place */
+  memcpy(result, s, charsRequired * sizeof(XML_Char));
+  return result;
 }
diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c
index 708507d575be..4d3e3e86e9e8 100644
--- a/Modules/expat/xmlrole.c
+++ b/Modules/expat/xmlrole.c
@@ -33,11 +33,11 @@
 #include <stddef.h>
 
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #else
-#ifdef HAVE_EXPAT_CONFIG_H
-#include <expat_config.h>
-#endif
+#  ifdef HAVE_EXPAT_CONFIG_H
+#    include <expat_config.h>
+#  endif
 #endif /* ndef _WIN32 */
 
 #include "expat_external.h"
@@ -52,107 +52,88 @@
 
 */
 
-static const char KW_ANY[] = {
-    ASCII_A, ASCII_N, ASCII_Y, '\0' };
-static const char KW_ATTLIST[] = {
-    ASCII_A, ASCII_T, ASCII_T, ASCII_L, ASCII_I, ASCII_S, ASCII_T, '\0' };
-static const char KW_CDATA[] = {
-    ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_DOCTYPE[] = {
-    ASCII_D, ASCII_O, ASCII_C, ASCII_T, ASCII_Y, ASCII_P, ASCII_E, '\0' };
-static const char KW_ELEMENT[] = {
-    ASCII_E, ASCII_L, ASCII_E, ASCII_M, ASCII_E, ASCII_N, ASCII_T, '\0' };
-static const char KW_EMPTY[] = {
-    ASCII_E, ASCII_M, ASCII_P, ASCII_T, ASCII_Y, '\0' };
-static const char KW_ENTITIES[] = {
-    ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_I, ASCII_E, ASCII_S,
-    '\0' };
-static const char KW_ENTITY[] = {
-    ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0' };
-static const char KW_FIXED[] = {
-    ASCII_F, ASCII_I, ASCII_X, ASCII_E, ASCII_D, '\0' };
-static const char KW_ID[] = {
-    ASCII_I, ASCII_D, '\0' };
-static const char KW_IDREF[] = {
-    ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0' };
-static const char KW_IDREFS[] = {
-    ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0' };
+static const char KW_ANY[] = {ASCII_A, ASCII_N, ASCII_Y, '\0'};
+static const char KW_ATTLIST[]
+    = {ASCII_A, ASCII_T, ASCII_T, ASCII_L, ASCII_I, ASCII_S, ASCII_T, '\0'};
+static const char KW_CDATA[]
+    = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_DOCTYPE[]
+    = {ASCII_D, ASCII_O, ASCII_C, ASCII_T, ASCII_Y, ASCII_P, ASCII_E, '\0'};
+static const char KW_ELEMENT[]
+    = {ASCII_E, ASCII_L, ASCII_E, ASCII_M, ASCII_E, ASCII_N, ASCII_T, '\0'};
+static const char KW_EMPTY[]
+    = {ASCII_E, ASCII_M, ASCII_P, ASCII_T, ASCII_Y, '\0'};
+static const char KW_ENTITIES[] = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T,
+                                   ASCII_I, ASCII_E, ASCII_S, '\0'};
+static const char KW_ENTITY[]
+    = {ASCII_E, ASCII_N, ASCII_T, ASCII_I, ASCII_T, ASCII_Y, '\0'};
+static const char KW_FIXED[]
+    = {ASCII_F, ASCII_I, ASCII_X, ASCII_E, ASCII_D, '\0'};
+static const char KW_ID[] = {ASCII_I, ASCII_D, '\0'};
+static const char KW_IDREF[]
+    = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, '\0'};
+static const char KW_IDREFS[]
+    = {ASCII_I, ASCII_D, ASCII_R, ASCII_E, ASCII_F, ASCII_S, '\0'};
 #ifdef XML_DTD
-static const char KW_IGNORE[] = {
-    ASCII_I, ASCII_G, ASCII_N, ASCII_O, ASCII_R, ASCII_E, '\0' };
+static const char KW_IGNORE[]
+    = {ASCII_I, ASCII_G, ASCII_N, ASCII_O, ASCII_R, ASCII_E, '\0'};
 #endif
-static const char KW_IMPLIED[] = {
-    ASCII_I, ASCII_M, ASCII_P, ASCII_L, ASCII_I, ASCII_E, ASCII_D, '\0' };
+static const char KW_IMPLIED[]
+    = {ASCII_I, ASCII_M, ASCII_P, ASCII_L, ASCII_I, ASCII_E, ASCII_D, '\0'};
 #ifdef XML_DTD
-static const char KW_INCLUDE[] = {
-    ASCII_I, ASCII_N, ASCII_C, ASCII_L, ASCII_U, ASCII_D, ASCII_E, '\0' };
+static const char KW_INCLUDE[]
+    = {ASCII_I, ASCII_N, ASCII_C, ASCII_L, ASCII_U, ASCII_D, ASCII_E, '\0'};
 #endif
-static const char KW_NDATA[] = {
-    ASCII_N, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_NMTOKEN[] = {
-    ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0' };
-static const char KW_NMTOKENS[] = {
-    ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, ASCII_S,
-    '\0' };
-static const char KW_NOTATION[] =
-    { ASCII_N, ASCII_O, ASCII_T, ASCII_A, ASCII_T, ASCII_I, ASCII_O, ASCII_N,
-      '\0' };
-static const char KW_PCDATA[] = {
-    ASCII_P, ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0' };
-static const char KW_PUBLIC[] = {
-    ASCII_P, ASCII_U, ASCII_B, ASCII_L, ASCII_I, ASCII_C, '\0' };
-static const char KW_REQUIRED[] = {
-    ASCII_R, ASCII_E, ASCII_Q, ASCII_U, ASCII_I, ASCII_R, ASCII_E, ASCII_D,
-    '\0' };
-static const char KW_SYSTEM[] = {
-    ASCII_S, ASCII_Y, ASCII_S, ASCII_T, ASCII_E, ASCII_M, '\0' };
+static const char KW_NDATA[]
+    = {ASCII_N, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_NMTOKEN[]
+    = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K, ASCII_E, ASCII_N, '\0'};
+static const char KW_NMTOKENS[] = {ASCII_N, ASCII_M, ASCII_T, ASCII_O, ASCII_K,
+                                   ASCII_E, ASCII_N, ASCII_S, '\0'};
+static const char KW_NOTATION[] = {ASCII_N, ASCII_O, ASCII_T, ASCII_A, ASCII_T,
+                                   ASCII_I, ASCII_O, ASCII_N, '\0'};
+static const char KW_PCDATA[]
+    = {ASCII_P, ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, '\0'};
+static const char KW_PUBLIC[]
+    = {ASCII_P, ASCII_U, ASCII_B, ASCII_L, ASCII_I, ASCII_C, '\0'};
+static const char KW_REQUIRED[] = {ASCII_R, ASCII_E, ASCII_Q, ASCII_U, ASCII_I,
+                                   ASCII_R, ASCII_E, ASCII_D, '\0'};
+static const char KW_SYSTEM[]
+    = {ASCII_S, ASCII_Y, ASCII_S, ASCII_T, ASCII_E, ASCII_M, '\0'};
 
 #ifndef MIN_BYTES_PER_CHAR
-#define MIN_BYTES_PER_CHAR(enc) ((enc)->minBytesPerChar)
+#  define MIN_BYTES_PER_CHAR(enc) ((enc)->minBytesPerChar)
 #endif
 
 #ifdef XML_DTD
-#define setTopLevel(state) \
-  ((state)->handler = ((state)->documentEntity \
-                       ? internalSubset \
-                       : externalSubset1))
+#  define setTopLevel(state)                                                   \
+    ((state)->handler                                                          \
+     = ((state)->documentEntity ? internalSubset : externalSubset1))
 #else /* not XML_DTD */
-#define setTopLevel(state) ((state)->handler = internalSubset)
+#  define setTopLevel(state) ((state)->handler = internalSubset)
 #endif /* not XML_DTD */
 
-typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state,
-                                   int tok,
-                                   const char *ptr,
-                                   const char *end,
+typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state, int tok,
+                                   const char *ptr, const char *end,
                                    const ENCODING *enc);
 
-static PROLOG_HANDLER
-  prolog0, prolog1, prolog2,
-  doctype0, doctype1, doctype2, doctype3, doctype4, doctype5,
-  internalSubset,
-  entity0, entity1, entity2, entity3, entity4, entity5, entity6,
-  entity7, entity8, entity9, entity10,
-  notation0, notation1, notation2, notation3, notation4,
-  attlist0, attlist1, attlist2, attlist3, attlist4, attlist5, attlist6,
-  attlist7, attlist8, attlist9,
-  element0, element1, element2, element3, element4, element5, element6,
-  element7,
+static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2,
+    doctype3, doctype4, doctype5, internalSubset, entity0, entity1, entity2,
+    entity3, entity4, entity5, entity6, entity7, entity8, entity9, entity10,
+    notation0, notation1, notation2, notation3, notation4, attlist0, attlist1,
+    attlist2, attlist3, attlist4, attlist5, attlist6, attlist7, attlist8,
+    attlist9, element0, element1, element2, element3, element4, element5,
+    element6, element7,
 #ifdef XML_DTD
-  externalSubset0, externalSubset1,
-  condSect0, condSect1, condSect2,
+    externalSubset0, externalSubset1, condSect0, condSect1, condSect2,
 #endif /* XML_DTD */
-  declClose,
-  error;
+    declClose, error;
 
 static int FASTCALL common(PROLOG_STATE *state, int tok);
 
 static int PTRCALL
-prolog0(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     state->handler = prolog1;
@@ -169,10 +150,8 @@ prolog0(PROLOG_STATE *state,
   case XML_TOK_BOM:
     return XML_ROLE_NONE;
   case XML_TOK_DECL_OPEN:
-    if (!XmlNameMatchesAscii(enc,
-                             ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                             end,
-                             KW_DOCTYPE))
+    if (! XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
+                              KW_DOCTYPE))
       break;
     state->handler = doctype0;
     return XML_ROLE_DOCTYPE_NONE;
@@ -184,12 +163,8 @@ prolog0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-prolog1(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -207,10 +182,8 @@ prolog1(PROLOG_STATE *state,
      */
     return XML_ROLE_NONE; /* LCOV_EXCL_LINE */
   case XML_TOK_DECL_OPEN:
-    if (!XmlNameMatchesAscii(enc,
-                             ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                             end,
-                             KW_DOCTYPE))
+    if (! XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
+                              KW_DOCTYPE))
       break;
     state->handler = doctype0;
     return XML_ROLE_DOCTYPE_NONE;
@@ -222,12 +195,11 @@ prolog1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-prolog2(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -243,12 +215,11 @@ prolog2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -261,12 +232,8 @@ doctype0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype1(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -291,12 +258,11 @@ doctype1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype2(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -308,12 +274,11 @@ doctype2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -325,12 +290,11 @@ doctype3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -345,12 +309,11 @@ doctype4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-doctype5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_DOCTYPE_NONE;
@@ -362,40 +325,28 @@ doctype5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-internalSubset(PROLOG_STATE *state,
-               int tok,
-               const char *ptr,
-               const char *end,
-               const ENCODING *enc)
-{
+internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+               const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
   case XML_TOK_DECL_OPEN:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ENTITY)) {
       state->handler = entity0;
       return XML_ROLE_ENTITY_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ATTLIST)) {
       state->handler = attlist0;
       return XML_ROLE_ATTLIST_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_ELEMENT)) {
       state->handler = element0;
       return XML_ROLE_ELEMENT_NONE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + 2 * MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), end,
                             KW_NOTATION)) {
       state->handler = notation0;
       return XML_ROLE_NOTATION_NONE;
@@ -419,12 +370,8 @@ internalSubset(PROLOG_STATE *state,
 #ifdef XML_DTD
 
 static int PTRCALL
-externalSubset0(PROLOG_STATE *state,
-                int tok,
-                const char *ptr,
-                const char *end,
-                const ENCODING *enc)
-{
+externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+                const ENCODING *enc) {
   state->handler = externalSubset1;
   if (tok == XML_TOK_XML_DECL)
     return XML_ROLE_TEXT_DECL;
@@ -432,12 +379,8 @@ externalSubset0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-externalSubset1(PROLOG_STATE *state,
-                int tok,
-                const char *ptr,
-                const char *end,
-                const ENCODING *enc)
-{
+externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+                const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_COND_SECT_OPEN:
     state->handler = condSect0;
@@ -464,12 +407,11 @@ externalSubset1(PROLOG_STATE *state,
 #endif /* XML_DTD */
 
 static int PTRCALL
-entity0(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -484,12 +426,11 @@ entity0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity1(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -501,12 +442,8 @@ entity1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity2(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -529,12 +466,11 @@ entity2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity3(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -546,12 +482,11 @@ entity3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity4(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -563,12 +498,8 @@ entity4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity5(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -586,12 +517,11 @@ entity5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity6(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -604,12 +534,8 @@ entity6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity7(PROLOG_STATE *state,
-        int tok,
-        const char *ptr,
-        const char *end,
-        const ENCODING *enc)
-{
+entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -632,12 +558,11 @@ entity7(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity8(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -649,12 +574,11 @@ entity8(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity9(PROLOG_STATE *state,
-        int tok,
-        const char *UNUSED_P(ptr),
-        const char *UNUSED_P(end),
-        const ENCODING *UNUSED_P(enc))
-{
+entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+        const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -666,12 +590,11 @@ entity9(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-entity10(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ENTITY_NONE;
@@ -683,12 +606,11 @@ entity10(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation0(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -700,12 +622,8 @@ notation0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation1(PROLOG_STATE *state,
-          int tok,
-          const char *ptr,
-          const char *end,
-          const ENCODING *enc)
-{
+notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -724,12 +642,11 @@ notation1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation2(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -741,12 +658,11 @@ notation2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation3(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -759,12 +675,11 @@ notation3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-notation4(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NOTATION_NONE;
@@ -780,12 +695,11 @@ notation4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -798,12 +712,11 @@ attlist0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist1(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -819,34 +732,23 @@ attlist1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist2(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
-  case XML_TOK_NAME:
-    {
-      static const char * const types[] = {
-        KW_CDATA,
-        KW_ID,
-        KW_IDREF,
-        KW_IDREFS,
-        KW_ENTITY,
-        KW_ENTITIES,
-        KW_NMTOKEN,
-        KW_NMTOKENS,
-      };
-      int i;
-      for (i = 0; i < (int)(sizeof(types)/sizeof(types[0])); i++)
-        if (XmlNameMatchesAscii(enc, ptr, end, types[i])) {
-          state->handler = attlist8;
-          return XML_ROLE_ATTRIBUTE_TYPE_CDATA + i;
-        }
-    }
+  case XML_TOK_NAME: {
+    static const char *const types[] = {
+        KW_CDATA,  KW_ID,       KW_IDREF,   KW_IDREFS,
+        KW_ENTITY, KW_ENTITIES, KW_NMTOKEN, KW_NMTOKENS,
+    };
+    int i;
+    for (i = 0; i < (int)(sizeof(types) / sizeof(types[0])); i++)
+      if (XmlNameMatchesAscii(enc, ptr, end, types[i])) {
+        state->handler = attlist8;
+        return XML_ROLE_ATTRIBUTE_TYPE_CDATA + i;
+      }
+  }
     if (XmlNameMatchesAscii(enc, ptr, end, KW_NOTATION)) {
       state->handler = attlist5;
       return XML_ROLE_ATTLIST_NONE;
@@ -860,12 +762,11 @@ attlist2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -879,12 +780,11 @@ attlist3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -899,12 +799,11 @@ attlist4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -916,12 +815,11 @@ attlist5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist6(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -933,12 +831,11 @@ attlist6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist7(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -954,33 +851,23 @@ attlist7(PROLOG_STATE *state,
 
 /* default value */
 static int PTRCALL
-attlist8(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
   case XML_TOK_POUND_NAME:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_IMPLIED)) {
       state->handler = attlist1;
       return XML_ROLE_IMPLIED_ATTRIBUTE_VALUE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_REQUIRED)) {
       state->handler = attlist1;
       return XML_ROLE_REQUIRED_ATTRIBUTE_VALUE;
     }
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_FIXED)) {
       state->handler = attlist9;
       return XML_ROLE_ATTLIST_NONE;
@@ -994,12 +881,11 @@ attlist8(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-attlist9(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ATTLIST_NONE;
@@ -1011,12 +897,11 @@ attlist9(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element0(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1029,12 +914,8 @@ element0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element1(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1059,19 +940,13 @@ element1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element2(PROLOG_STATE *state,
-         int tok,
-         const char *ptr,
-         const char *end,
-         const ENCODING *enc)
-{
+element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
   case XML_TOK_POUND_NAME:
-    if (XmlNameMatchesAscii(enc,
-                            ptr + MIN_BYTES_PER_CHAR(enc),
-                            end,
+    if (XmlNameMatchesAscii(enc, ptr + MIN_BYTES_PER_CHAR(enc), end,
                             KW_PCDATA)) {
       state->handler = element3;
       return XML_ROLE_CONTENT_PCDATA;
@@ -1099,12 +974,11 @@ element2(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element3(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1124,12 +998,11 @@ element3(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element4(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1142,12 +1015,11 @@ element4(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element5(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1163,12 +1035,11 @@ element5(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element6(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1193,12 +1064,11 @@ element6(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-element7(PROLOG_STATE *state,
-         int tok,
-         const char *UNUSED_P(ptr),
-         const char *UNUSED_P(end),
-         const ENCODING *UNUSED_P(enc))
-{
+element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+         const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_ELEMENT_NONE;
@@ -1243,12 +1113,8 @@ element7(PROLOG_STATE *state,
 #ifdef XML_DTD
 
 static int PTRCALL
-condSect0(PROLOG_STATE *state,
-          int tok,
-          const char *ptr,
-          const char *end,
-          const ENCODING *enc)
-{
+condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1267,12 +1133,11 @@ condSect0(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-condSect1(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1285,12 +1150,11 @@ condSect1(PROLOG_STATE *state,
 }
 
 static int PTRCALL
-condSect2(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return XML_ROLE_NONE;
@@ -1304,12 +1168,11 @@ condSect2(PROLOG_STATE *state,
 #endif /* XML_DTD */
 
 static int PTRCALL
-declClose(PROLOG_STATE *state,
-          int tok,
-          const char *UNUSED_P(ptr),
-          const char *UNUSED_P(end),
-          const ENCODING *UNUSED_P(enc))
-{
+declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+          const ENCODING *enc) {
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   switch (tok) {
   case XML_TOK_PROLOG_S:
     return state->role_none;
@@ -1341,21 +1204,21 @@ declClose(PROLOG_STATE *state,
  * LCOV_EXCL_START
  */
 static int PTRCALL
-error(PROLOG_STATE *UNUSED_P(state),
-      int UNUSED_P(tok),
-      const char *UNUSED_P(ptr),
-      const char *UNUSED_P(end),
-      const ENCODING *UNUSED_P(enc))
-{
+error(PROLOG_STATE *state, int tok, const char *ptr, const char *end,
+      const ENCODING *enc) {
+  UNUSED_P(state);
+  UNUSED_P(tok);
+  UNUSED_P(ptr);
+  UNUSED_P(end);
+  UNUSED_P(enc);
   return XML_ROLE_NONE;
 }
 /* LCOV_EXCL_STOP */
 
 static int FASTCALL
-common(PROLOG_STATE *state, int tok)
-{
+common(PROLOG_STATE *state, int tok) {
 #ifdef XML_DTD
-  if (!state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF)
+  if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF)
     return XML_ROLE_INNER_PARAM_ENTITY_REF;
 #endif
   state->handler = error;
@@ -1363,8 +1226,7 @@ common(PROLOG_STATE *state, int tok)
 }
 
 void
-XmlPrologStateInit(PROLOG_STATE *state)
-{
+XmlPrologStateInit(PROLOG_STATE *state) {
   state->handler = prolog0;
 #ifdef XML_DTD
   state->documentEntity = 1;
@@ -1376,8 +1238,7 @@ XmlPrologStateInit(PROLOG_STATE *state)
 #ifdef XML_DTD
 
 void
-XmlPrologStateInitExternalEntity(PROLOG_STATE *state)
-{
+XmlPrologStateInitExternalEntity(PROLOG_STATE *state) {
   state->handler = externalSubset0;
   state->documentEntity = 0;
   state->includeLevel = 0;
diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h
index e5f048eab55c..036aba64fd29 100644
--- a/Modules/expat/xmlrole.h
+++ b/Modules/expat/xmlrole.h
@@ -36,7 +36,7 @@
 #ifdef __VMS
 /*      0        1         2         3      0        1         2         3
         1234567890123456789012345678901     1234567890123456789012345678901 */
-#define XmlPrologStateInitExternalEntity    XmlPrologStateInitExternalEnt
+#  define XmlPrologStateInitExternalEntity XmlPrologStateInitExternalEnt
 #endif
 
 #include "xmltok.h"
@@ -113,11 +113,8 @@ enum {
 };
 
 typedef struct prolog_state {
-  int (PTRCALL *handler) (struct prolog_state *state,
-                          int tok,
-                          const char *ptr,
-                          const char *end,
-                          const ENCODING *enc);
+  int(PTRCALL *handler)(struct prolog_state *state, int tok, const char *ptr,
+                        const char *end, const ENCODING *enc);
   unsigned level;
   int role_none;
 #ifdef XML_DTD
@@ -132,8 +129,8 @@ void XmlPrologStateInit(PROLOG_STATE *);
 void XmlPrologStateInitExternalEntity(PROLOG_STATE *);
 #endif /* XML_DTD */
 
-#define XmlTokenRole(state, tok, ptr, end, enc) \
- (((state)->handler)(state, tok, ptr, end, enc))
+#define XmlTokenRole(state, tok, ptr, end, enc)                                \
+  (((state)->handler)(state, tok, ptr, end, enc))
 
 #ifdef __cplusplus
 }
diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c
index 6b415d83972c..11e9d1ccdad4 100644
--- a/Modules/expat/xmltok.c
+++ b/Modules/expat/xmltok.c
@@ -31,24 +31,23 @@
 */
 
 #include <stddef.h>
-#include <string.h>  /* memcpy */
+#include <string.h> /* memcpy */
 
 #if defined(_MSC_VER) && (_MSC_VER <= 1700)
-  /* for vs2012/11.0/1700 and earlier Visual Studio compilers */
-# define bool   int
-# define false  0
-# define true   1
+/* for vs2012/11.0/1700 and earlier Visual Studio compilers */
+#  define bool int
+#  define false 0
+#  define true 1
 #else
-# include <stdbool.h>
+#  include <stdbool.h>
 #endif
 
-
 #ifdef _WIN32
-#include "winconfig.h"
+#  include "winconfig.h"
 #else
-#ifdef HAVE_EXPAT_CONFIG_H
-#include <expat_config.h>
-#endif
+#  ifdef HAVE_EXPAT_CONFIG_H
+#    include <expat_config.h>
+#  endif
 #endif /* ndef _WIN32 */
 
 #include "expat_external.h"
@@ -57,58 +56,49 @@
 #include "nametab.h"
 
 #ifdef XML_DTD
-#define IGNORE_SECTION_TOK_VTABLE , PREFIX(ignoreSectionTok)
+#  define IGNORE_SECTION_TOK_VTABLE , PREFIX(ignoreSectionTok)
 #else
-#define IGNORE_SECTION_TOK_VTABLE /* as nothing */
+#  define IGNORE_SECTION_TOK_VTABLE /* as nothing */
 #endif
 
-#define VTABLE1 \
-  { PREFIX(prologTok), PREFIX(contentTok), \
-    PREFIX(cdataSectionTok) IGNORE_SECTION_TOK_VTABLE }, \
-  { PREFIX(attributeValueTok), PREFIX(entityValueTok) }, \
-  PREFIX(nameMatchesAscii), \
-  PREFIX(nameLength), \
-  PREFIX(skipS), \
-  PREFIX(getAtts), \
-  PREFIX(charRefNumber), \
-  PREFIX(predefinedEntityName), \
-  PREFIX(updatePosition), \
-  PREFIX(isPublicId)
+#define VTABLE1                                                                \
+  {PREFIX(prologTok), PREFIX(contentTok),                                      \
+   PREFIX(cdataSectionTok) IGNORE_SECTION_TOK_VTABLE},                         \
+      {PREFIX(attributeValueTok), PREFIX(entityValueTok)},                     \
+      PREFIX(nameMatchesAscii), PREFIX(nameLength), PREFIX(skipS),             \
+      PREFIX(getAtts), PREFIX(charRefNumber), PREFIX(predefinedEntityName),    \
+      PREFIX(updatePosition), PREFIX(isPublicId)
 
 #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16)
 
-#define UCS2_GET_NAMING(pages, hi, lo) \
-   (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo) & 0x1F)))
+#define UCS2_GET_NAMING(pages, hi, lo)                                         \
+  (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo)&0x1F)))
 
 /* A 2 byte UTF-8 representation splits the characters 11 bits between
    the bottom 5 and 6 bits of the bytes.  We need 8 bits to index into
    pages, 3 bits to add to that index and 5 bits to generate the mask.
 */
-#define UTF8_GET_NAMING2(pages, byte) \
-    (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3) \
-                      + ((((byte)[0]) & 3) << 1) \
-                      + ((((byte)[1]) >> 5) & 1)] \
-         & (1u << (((byte)[1]) & 0x1F)))
+#define UTF8_GET_NAMING2(pages, byte)                                          \
+  (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3)                         \
+                + ((((byte)[0]) & 3) << 1) + ((((byte)[1]) >> 5) & 1)]         \
+   & (1u << (((byte)[1]) & 0x1F)))
 
 /* A 3 byte UTF-8 representation splits the characters 16 bits between
    the bottom 4, 6 and 6 bits of the bytes.  We need 8 bits to index
    into pages, 3 bits to add to that index and 5 bits to generate the
    mask.
 */
-#define UTF8_GET_NAMING3(pages, byte) \
-  (namingBitmap[((pages)[((((byte)[0]) & 0xF) << 4) \
-                             + ((((byte)[1]) >> 2) & 0xF)] \
-                       << 3) \
-                      + ((((byte)[1]) & 3) << 1) \
-                      + ((((byte)[2]) >> 5) & 1)] \
-         & (1u << (((byte)[2]) & 0x1F)))
-
-#define UTF8_GET_NAMING(pages, p, n) \
-  ((n) == 2 \
-  ? UTF8_GET_NAMING2(pages, (const unsigned char *)(p)) \
-  : ((n) == 3 \
-     ? UTF8_GET_NAMING3(pages, (const unsigned char *)(p)) \
-     : 0))
+#define UTF8_GET_NAMING3(pages, byte)                                          \
+  (namingBitmap                                                                \
+       [((pages)[((((byte)[0]) & 0xF) << 4) + ((((byte)[1]) >> 2) & 0xF)]      \
+         << 3)                                                                 \
+        + ((((byte)[1]) & 3) << 1) + ((((byte)[2]) >> 5) & 1)]                 \
+   & (1u << (((byte)[2]) & 0x1F)))
+
+#define UTF8_GET_NAMING(pages, p, n)                                           \
+  ((n) == 2                                                                    \
+       ? UTF8_GET_NAMING2(pages, (const unsigned char *)(p))                   \
+       : ((n) == 3 ? UTF8_GET_NAMING3(pages, (const unsigned char *)(p)) : 0))
 
 /* Detection of invalid UTF-8 sequences is based on Table 3.1B
    of Unicode 3.2: http://www.unicode.org/unicode/reports/tr28/
@@ -120,88 +110,76 @@
      (A & 0xC0) == 0xC0  means A > 0xBF
 */
 
-#define UTF8_INVALID2(p) \
+#define UTF8_INVALID2(p)                                                       \
   ((*p) < 0xC2 || ((p)[1] & 0x80) == 0 || ((p)[1] & 0xC0) == 0xC0)
 
-#define UTF8_INVALID3(p) \
-  (((p)[2] & 0x80) == 0 \
-  || \
-  ((*p) == 0xEF && (p)[1] == 0xBF \
-    ? \
-    (p)[2] > 0xBD \
-    : \
-    ((p)[2] & 0xC0) == 0xC0) \
-  || \
-  ((*p) == 0xE0 \
-    ? \
-    (p)[1] < 0xA0 || ((p)[1] & 0xC0) == 0xC0 \
-    : \
-    ((p)[1] & 0x80) == 0 \
-    || \
-    ((*p) == 0xED ? (p)[1] > 0x9F : ((p)[1] & 0xC0) == 0xC0)))
-
-#define UTF8_INVALID4(p) \
-  (((p)[3] & 0x80) == 0 || ((p)[3] & 0xC0) == 0xC0 \
-  || \
-  ((p)[2] & 0x80) == 0 || ((p)[2] & 0xC0) == 0xC0 \
-  || \
-  ((*p) == 0xF0 \
-    ? \
-    (p)[1] < 0x90 || ((p)[1] & 0xC0) == 0xC0 \
-    : \
-    ((p)[1] & 0x80) == 0 \
-    || \
-    ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0)))
+#define UTF8_INVALID3(p)                                                       \
+  (((p)[2] & 0x80) == 0                                                        \
+   || ((*p) == 0xEF && (p)[1] == 0xBF ? (p)[2] > 0xBD                          \
+                                      : ((p)[2] & 0xC0) == 0xC0)               \
+   || ((*p) == 0xE0                                                            \
+           ? (p)[1] < 0xA0 || ((p)[1] & 0xC0) == 0xC0                          \
+           : ((p)[1] & 0x80) == 0                                              \
+                 || ((*p) == 0xED ? (p)[1] > 0x9F : ((p)[1] & 0xC0) == 0xC0)))
+
+#define UTF8_INVALID4(p)                                                       \
+  (((p)[3] & 0x80) == 0 || ((p)[3] & 0xC0) == 0xC0 || ((p)[2] & 0x80) == 0     \
+   || ((p)[2] & 0xC0) == 0xC0                                                  \
+   || ((*p) == 0xF0                                                            \
+           ? (p)[1] < 0x90 || ((p)[1] & 0xC0) == 0xC0                          \
+           : ((p)[1] & 0x80) == 0                                              \
+                 || ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0)))
 
 static int PTRFASTCALL
-isNever(const ENCODING *UNUSED_P(enc), const char *UNUSED_P(p))
-{
+isNever(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  UNUSED_P(p);
   return 0;
 }
 
 static int PTRFASTCALL
-utf8_isName2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isName2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING2(namePages, (const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isName3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isName3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING3(namePages, (const unsigned char *)p);
 }
 
 #define utf8_isName4 isNever
 
 static int PTRFASTCALL
-utf8_isNmstrt2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isNmstrt2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isNmstrt3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isNmstrt3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p);
 }
 
 #define utf8_isNmstrt4 isNever
 
 static int PTRFASTCALL
-utf8_isInvalid2(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid2(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID2((const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isInvalid3(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid3(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID3((const unsigned char *)p);
 }
 
 static int PTRFASTCALL
-utf8_isInvalid4(const ENCODING *UNUSED_P(enc), const char *p)
-{
+utf8_isInvalid4(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return UTF8_INVALID4((const unsigned char *)p);
 }
 
@@ -209,61 +187,44 @@ struct normal_encoding {
   ENCODING enc;
   unsigned char type[256];
 #ifdef XML_MIN_SIZE
-  int (PTRFASTCALL *byteType)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNameMin)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *);
-  int (PTRFASTCALL *byteToAscii)(const ENCODING *, const char *);
-  int (PTRCALL *charMatches)(const ENCODING *, const char *, int);
+  int(PTRFASTCALL *byteType)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNameMin)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *);
+  int(PTRFASTCALL *byteToAscii)(const ENCODING *, const char *);
+  int(PTRCALL *charMatches)(const ENCODING *, const char *, int);
 #endif /* XML_MIN_SIZE */
-  int (PTRFASTCALL *isName2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isName3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isName4)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid2)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid3)(const ENCODING *, const char *);
-  int (PTRFASTCALL *isInvalid4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isName4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid2)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid3)(const ENCODING *, const char *);
+  int(PTRFASTCALL *isInvalid4)(const ENCODING *, const char *);
 };
 
-#define AS_NORMAL_ENCODING(enc)   ((const struct normal_encoding *) (enc))
+#define AS_NORMAL_ENCODING(enc) ((const struct normal_encoding *)(enc))
 
 #ifdef XML_MIN_SIZE
 
-#define STANDARD_VTABLE(E) \
- E ## byteType, \
- E ## isNameMin, \
- E ## isNmstrtMin, \
- E ## byteToAscii, \
- E ## charMatches,
+#  define STANDARD_VTABLE(E)                                                   \
+    E##byteType, E##isNameMin, E##isNmstrtMin, E##byteToAscii, E##charMatches,
 
 #else
 
-#define STANDARD_VTABLE(E) /* as nothing */
+#  define STANDARD_VTABLE(E) /* as nothing */
 
 #endif
 
-#define NORMAL_VTABLE(E) \
- E ## isName2, \
- E ## isName3, \
- E ## isName4, \
- E ## isNmstrt2, \
- E ## isNmstrt3, \
- E ## isNmstrt4, \
- E ## isInvalid2, \
- E ## isInvalid3, \
- E ## isInvalid4
-
-#define NULL_VTABLE \
- /* isName2 */ NULL, \
- /* isName3 */ NULL, \
- /* isName4 */ NULL, \
- /* isNmstrt2 */ NULL, \
- /* isNmstrt3 */ NULL, \
- /* isNmstrt4 */ NULL, \
- /* isInvalid2 */ NULL, \
- /* isInvalid3 */ NULL, \
- /* isInvalid4 */ NULL
+#define NORMAL_VTABLE(E)                                                       \
+  E##isName2, E##isName3, E##isName4, E##isNmstrt2, E##isNmstrt3,              \
+      E##isNmstrt4, E##isInvalid2, E##isInvalid3, E##isInvalid4
+
+#define NULL_VTABLE                                                            \
+  /* isName2 */ NULL, /* isName3 */ NULL, /* isName4 */ NULL,                  \
+      /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL,        \
+      /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL
 
 static int FASTCALL checkCharRefNumber(int);
 
@@ -271,75 +232,70 @@ static int FASTCALL checkCharRefNumber(int);
 #include "ascii.h"
 
 #ifdef XML_MIN_SIZE
-#define sb_isNameMin isNever
-#define sb_isNmstrtMin isNever
+#  define sb_isNameMin isNever
+#  define sb_isNmstrtMin isNever
 #endif
 
 #ifdef XML_MIN_SIZE
-#define MINBPC(enc) ((enc)->minBytesPerChar)
+#  define MINBPC(enc) ((enc)->minBytesPerChar)
 #else
 /* minimum bytes per character */
-#define MINBPC(enc) 1
+#  define MINBPC(enc) 1
 #endif
 
-#define SB_BYTE_TYPE(enc, p) \
+#define SB_BYTE_TYPE(enc, p)                                                   \
   (((struct normal_encoding *)(enc))->type[(unsigned char)*(p)])
 
 #ifdef XML_MIN_SIZE
 static int PTRFASTCALL
-sb_byteType(const ENCODING *enc, const char *p)
-{
+sb_byteType(const ENCODING *enc, const char *p) {
   return SB_BYTE_TYPE(enc, p);
 }
-#define BYTE_TYPE(enc, p) \
- (AS_NORMAL_ENCODING(enc)->byteType(enc, p))
+#  define BYTE_TYPE(enc, p) (AS_NORMAL_ENCODING(enc)->byteType(enc, p))
 #else
-#define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
+#  define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
 #endif
 
 #ifdef XML_MIN_SIZE
-#define BYTE_TO_ASCII(enc, p) \
- (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p))
+#  define BYTE_TO_ASCII(enc, p) (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p))
 static int PTRFASTCALL
-sb_byteToAscii(const ENCODING *enc, const char *p)
-{
+sb_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
   return *p;
 }
 #else
-#define BYTE_TO_ASCII(enc, p) (*(p))
+#  define BYTE_TO_ASCII(enc, p) (*(p))
 #endif
 
-#define IS_NAME_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isName ## n(enc, p))
-#define IS_NMSTRT_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isNmstrt ## n(enc, p))
-#define IS_INVALID_CHAR(enc, p, n) \
- (AS_NORMAL_ENCODING(enc)->isInvalid ## n(enc, p))
+#define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p))
+#define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p))
+#define IS_INVALID_CHAR(enc, p, n)                                             \
+  (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p))
 
 #ifdef XML_MIN_SIZE
-#define IS_NAME_CHAR_MINBPC(enc, p) \
- (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p))
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) \
- (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p))
+#  define IS_NAME_CHAR_MINBPC(enc, p)                                          \
+    (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p))
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p)                                        \
+    (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p))
 #else
-#define IS_NAME_CHAR_MINBPC(enc, p) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
+#  define IS_NAME_CHAR_MINBPC(enc, p) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
 #endif
 
 #ifdef XML_MIN_SIZE
-#define CHAR_MATCHES(enc, p, c) \
- (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c))
+#  define CHAR_MATCHES(enc, p, c)                                              \
+    (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c))
 static int PTRCALL
-sb_charMatches(const ENCODING *enc, const char *p, int c)
-{
+sb_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
   return *p == c;
 }
 #else
 /* c is an ASCII character */
-#define CHAR_MATCHES(enc, p, c) (*(p) == c)
+#  define CHAR_MATCHES(enc, p, c) (*(p) == c)
 #endif
 
-#define PREFIX(ident) normal_ ## ident
+#define PREFIX(ident) normal_##ident
 #define XML_TOK_IMPL_C
 #include "xmltok_impl.c"
 #undef XML_TOK_IMPL_C
@@ -354,42 +310,46 @@ sb_charMatches(const ENCODING *enc, const char *p, int c)
 #undef IS_NMSTRT_CHAR_MINBPC
 #undef IS_INVALID_CHAR
 
-enum {  /* UTF8_cvalN is value of masked first byte of N byte sequence */
-  UTF8_cval1 = 0x00,
-  UTF8_cval2 = 0xc0,
-  UTF8_cval3 = 0xe0,
-  UTF8_cval4 = 0xf0
+enum { /* UTF8_cvalN is value of masked first byte of N byte sequence */
+       UTF8_cval1 = 0x00,
+       UTF8_cval2 = 0xc0,
+       UTF8_cval3 = 0xe0,
+       UTF8_cval4 = 0xf0
 };
 
 void
-_INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** fromLimRef)
-{
-  const char * fromLim = *fromLimRef;
+_INTERNAL_trim_to_complete_utf8_characters(const char *from,
+                                           const char **fromLimRef) {
+  const char *fromLim = *fromLimRef;
   size_t walked = 0;
   for (; fromLim > from; fromLim--, walked++) {
     const unsigned char prev = (unsigned char)fromLim[-1];
-    if ((prev & 0xf8u) == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */
+    if ((prev & 0xf8u)
+        == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */
       if (walked + 1 >= 4) {
         fromLim += 4 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0xf0u) == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */
+    } else if ((prev & 0xf0u)
+               == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */
       if (walked + 1 >= 3) {
         fromLim += 3 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0xe0u) == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */
+    } else if ((prev & 0xe0u)
+               == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */
       if (walked + 1 >= 2) {
         fromLim += 2 - 1;
         break;
       } else {
         walked = 0;
       }
-    } else if ((prev & 0x80u) == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */
+    } else if ((prev & 0x80u)
+               == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */
       break;
     }
   }
@@ -397,16 +357,15 @@ _INTERNAL_trim_to_complete_utf8_characters(const char * from, const char ** from
 }
 
 static enum XML_Convert_Result PTRCALL
-utf8_toUtf8(const ENCODING *UNUSED_P(enc),
-            const char **fromP, const char *fromLim,
-            char **toP, const char *toLim)
-{
+utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+            char **toP, const char *toLim) {
   bool input_incomplete = false;
   bool output_exhausted = false;
 
   /* Avoid copying partial characters (due to limited space). */
   const ptrdiff_t bytesAvailable = fromLim - *fromP;
   const ptrdiff_t bytesStorable = toLim - *toP;
+  UNUSED_P(enc);
   if (bytesAvailable > bytesStorable) {
     fromLim = *fromP + bytesStorable;
     output_exhausted = true;
@@ -414,7 +373,7 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
 
   /* Avoid copying partial characters (from incomplete input). */
   {
-    const char * const fromLimBefore = fromLim;
+    const char *const fromLimBefore = fromLim;
     _INTERNAL_trim_to_complete_utf8_characters(*fromP, &fromLim);
     if (fromLim < fromLimBefore) {
       input_incomplete = true;
@@ -428,7 +387,7 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
     *toP += bytesToCopy;
   }
 
-  if (output_exhausted)  /* needs to go first */
+  if (output_exhausted) /* needs to go first */
     return XML_CONVERT_OUTPUT_EXHAUSTED;
   else if (input_incomplete)
     return XML_CONVERT_INPUT_INCOMPLETE;
@@ -437,10 +396,8 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
 }
 
 static enum XML_Convert_Result PTRCALL
-utf8_toUtf16(const ENCODING *enc,
-             const char **fromP, const char *fromLim,
-             unsigned short **toP, const unsigned short *toLim)
-{
+utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+             unsigned short **toP, const unsigned short *toLim) {
   enum XML_Convert_Result res = XML_CONVERT_COMPLETED;
   unsigned short *to = *toP;
   const char *from = *fromP;
@@ -459,30 +416,28 @@ utf8_toUtf16(const ENCODING *enc,
         res = XML_CONVERT_INPUT_INCOMPLETE;
         goto after;
       }
-      *to++ = (unsigned short)(((from[0] & 0xf) << 12)
-                               | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f));
+      *to++ = (unsigned short)(((from[0] & 0xf) << 12) | ((from[1] & 0x3f) << 6)
+                               | (from[2] & 0x3f));
       from += 3;
       break;
-    case BT_LEAD4:
-      {
-        unsigned long n;
-        if (toLim - to < 2) {
-          res = XML_CONVERT_OUTPUT_EXHAUSTED;
-          goto after;
-        }
-        if (fromLim - from < 4) {
-          res = XML_CONVERT_INPUT_INCOMPLETE;
-          goto after;
-        }
-        n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12)
-            | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f);
-        n -= 0x10000;
-        to[0] = (unsigned short)((n >> 10) | 0xD800);
-        to[1] = (unsigned short)((n & 0x3FF) | 0xDC00);
-        to += 2;
-        from += 4;
+    case BT_LEAD4: {
+      unsigned long n;
+      if (toLim - to < 2) {
+        res = XML_CONVERT_OUTPUT_EXHAUSTED;
+        goto after;
       }
-      break;
+      if (fromLim - from < 4) {
+        res = XML_CONVERT_INPUT_INCOMPLETE;
+        goto after;
+      }
+      n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12)
+          | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f);
+      n -= 0x10000;
+      to[0] = (unsigned short)((n >> 10) | 0xD800);
+      to[1] = (unsigned short)((n & 0x3FF) | 0xDC00);
+      to += 2;
+      from += 4;
+    } break;
     default:
       *to++ = *from++;
       break;
@@ -497,56 +452,51 @@ utf8_toUtf16(const ENCODING *enc,
 }
 
 #ifdef XML_NS
-static const struct normal_encoding utf8_encoding_ns = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
-#include "asciitab.h"
-#include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+static const struct normal_encoding utf8_encoding_ns
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
+#  include "asciitab.h"
+#  include "utf8tab.h"
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 #endif
 
-static const struct normal_encoding utf8_encoding = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding utf8_encoding
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 #ifdef XML_NS
 
-static const struct normal_encoding internal_utf8_encoding_ns = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
-#include "iasciitab.h"
-#include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+static const struct normal_encoding internal_utf8_encoding_ns
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
+#  include "iasciitab.h"
+#  include "utf8tab.h"
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 #endif
 
-static const struct normal_encoding internal_utf8_encoding = {
-  { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding internal_utf8_encoding
+    = {{VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "iasciitab.h"
 #undef BT_COLON
 #include "utf8tab.h"
-  },
-  STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
-};
+       },
+       STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)};
 
 static enum XML_Convert_Result PTRCALL
-latin1_toUtf8(const ENCODING *UNUSED_P(enc),
-              const char **fromP, const char *fromLim,
-              char **toP, const char *toLim)
-{
+latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+              char **toP, const char *toLim) {
+  UNUSED_P(enc);
   for (;;) {
     unsigned char c;
     if (*fromP == fromLim)
@@ -558,8 +508,7 @@ latin1_toUtf8(const ENCODING *UNUSED_P(enc),
       *(*toP)++ = (char)((c >> 6) | UTF8_cval2);
       *(*toP)++ = (char)((c & 0x3f) | 0x80);
       (*fromP)++;
-    }
-    else {
+    } else {
       if (*toP == toLim)
         return XML_CONVERT_OUTPUT_EXHAUSTED;
       *(*toP)++ = *(*fromP)++;
@@ -568,10 +517,9 @@ latin1_toUtf8(const ENCODING *UNUSED_P(enc),
 }
 
 static enum XML_Convert_Result PTRCALL
-latin1_toUtf16(const ENCODING *UNUSED_P(enc),
-               const char **fromP, const char *fromLim,
-               unsigned short **toP, const unsigned short *toLim)
-{
+latin1_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+               unsigned short **toP, const unsigned short *toLim) {
+  UNUSED_P(enc);
   while (*fromP < fromLim && *toP < toLim)
     *(*toP)++ = (unsigned char)*(*fromP)++;
 
@@ -583,33 +531,30 @@ latin1_toUtf16(const ENCODING *UNUSED_P(enc),
 
 #ifdef XML_NS
 
-static const struct normal_encoding latin1_encoding_ns = {
-  { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+static const struct normal_encoding latin1_encoding_ns
+    = {{VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0},
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding latin1_encoding = {
-  { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
-  {
+static const struct normal_encoding latin1_encoding
+    = {{VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 static enum XML_Convert_Result PTRCALL
-ascii_toUtf8(const ENCODING *UNUSED_P(enc),
-             const char **fromP, const char *fromLim,
-             char **toP, const char *toLim)
-{
+ascii_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+             char **toP, const char *toLim) {
+  UNUSED_P(enc);
   while (*fromP < fromLim && *toP < toLim)
     *(*toP)++ = *(*fromP)++;
 
@@ -621,40 +566,45 @@ ascii_toUtf8(const ENCODING *UNUSED_P(enc),
 
 #ifdef XML_NS
 
-static const struct normal_encoding ascii_encoding_ns = {
-  { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
-  {
-#include "asciitab.h"
-/* BT_NONXML == 0 */
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+static const struct normal_encoding ascii_encoding_ns
+    = {{VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0},
+       {
+#  include "asciitab.h"
+           /* BT_NONXML == 0 */
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding ascii_encoding = {
-  { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
-  {
+static const struct normal_encoding ascii_encoding
+    = {{VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0},
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
-/* BT_NONXML == 0 */
-  },
-  STANDARD_VTABLE(sb_) NULL_VTABLE
-};
+           /* BT_NONXML == 0 */
+       },
+       STANDARD_VTABLE(sb_) NULL_VTABLE};
 
 static int PTRFASTCALL
-unicode_byte_type(char hi, char lo)
-{
+unicode_byte_type(char hi, char lo) {
   switch ((unsigned char)hi) {
-  case 0xD8: case 0xD9: case 0xDA: case 0xDB:
+  /* 0xD800?0xDBFF first 16-bit code unit or high surrogate (W1) */
+  case 0xD8:
+  case 0xD9:
+  case 0xDA:
+  case 0xDB:
     return BT_LEAD4;
-  case 0xDC: case 0xDD: case 0xDE: case 0xDF:
+  /* 0xDC00?0xDFFF second 16-bit code unit or low surrogate (W2) */
+  case 0xDC:
+  case 0xDD:
+  case 0xDE:
+  case 0xDF:
     return BT_TRAIL;
   case 0xFF:
     switch ((unsigned char)lo) {
-    case 0xFF:
-    case 0xFE:
+    case 0xFF: /* noncharacter-FFFF */
+    case 0xFE: /* noncharacter-FFFE */
       return BT_NONXML;
     }
     break;
@@ -662,102 +612,105 @@ unicode_byte_type(char hi, char lo)
   return BT_NONASCII;
 }
 
-#define DEFINE_UTF16_TO_UTF8(E) \
-static enum XML_Convert_Result  PTRCALL \
-E ## toUtf8(const ENCODING *UNUSED_P(enc), \
-            const char **fromP, const char *fromLim, \
-            char **toP, const char *toLim) \
-{ \
-  const char *from = *fromP; \
-  fromLim = from + (((fromLim - from) >> 1) << 1);  /* shrink to even */ \
-  for (; from < fromLim; from += 2) { \
-    int plane; \
-    unsigned char lo2; \
-    unsigned char lo = GET_LO(from); \
-    unsigned char hi = GET_HI(from); \
-    switch (hi) { \
-    case 0: \
-      if (lo < 0x80) { \
-        if (*toP == toLim) { \
-          *fromP = from; \
-          return XML_CONVERT_OUTPUT_EXHAUSTED; \
-        } \
-        *(*toP)++ = lo; \
-        break; \
-      } \
-      /* fall through */ \
-    case 0x1: case 0x2: case 0x3: \
-    case 0x4: case 0x5: case 0x6: case 0x7: \
-      if (toLim -  *toP < 2) { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      *(*toP)++ = ((lo >> 6) | (hi << 2) |  UTF8_cval2); \
-      *(*toP)++ = ((lo & 0x3f) | 0x80); \
-      break; \
-    default: \
-      if (toLim -  *toP < 3)  { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      /* 16 bits divided 4, 6, 6 amongst 3 bytes */ \
-      *(*toP)++ = ((hi >> 4) | UTF8_cval3); \
-      *(*toP)++ = (((hi & 0xf) << 2) | (lo >> 6) | 0x80); \
-      *(*toP)++ = ((lo & 0x3f) | 0x80); \
-      break; \
-    case 0xD8: case 0xD9: case 0xDA: case 0xDB: \
-      if (toLim -  *toP < 4) { \
-        *fromP = from; \
-        return XML_CONVERT_OUTPUT_EXHAUSTED; \
-      } \
-      if (fromLim - from < 4) { \
-        *fromP = from; \
-        return XML_CONVERT_INPUT_INCOMPLETE; \
-      } \
-      plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1; \
-      *(*toP)++ = ((plane >> 2) | UTF8_cval4); \
-      *(*toP)++ = (((lo >> 2) & 0xF) | ((plane & 0x3) << 4) | 0x80); \
-      from += 2; \
-      lo2 = GET_LO(from); \
-      *(*toP)++ = (((lo & 0x3) << 4) \
-                   | ((GET_HI(from) & 0x3) << 2) \
-                   | (lo2 >> 6) \
-                   | 0x80); \
-      *(*toP)++ = ((lo2 & 0x3f) | 0x80); \
-      break; \
-    } \
-  } \
-  *fromP = from; \
-  if (from < fromLim) \
-    return XML_CONVERT_INPUT_INCOMPLETE; \
-  else \
-    return XML_CONVERT_COMPLETED; \
-}
+#define DEFINE_UTF16_TO_UTF8(E)                                                \
+  static enum XML_Convert_Result PTRCALL E##toUtf8(                            \
+      const ENCODING *enc, const char **fromP, const char *fromLim,            \
+      char **toP, const char *toLim) {                                         \
+    const char *from = *fromP;                                                 \
+    UNUSED_P(enc);                                                             \
+    fromLim = from + (((fromLim - from) >> 1) << 1); /* shrink to even */      \
+    for (; from < fromLim; from += 2) {                                        \
+      int plane;                                                               \
+      unsigned char lo2;                                                       \
+      unsigned char lo = GET_LO(from);                                         \
+      unsigned char hi = GET_HI(from);                                         \
+      switch (hi) {                                                            \
+      case 0:                                                                  \
+        if (lo < 0x80) {                                                       \
+          if (*toP == toLim) {                                                 \
+            *fromP = from;                                                     \
+            return XML_CONVERT_OUTPUT_EXHAUSTED;                               \
+          }                                                                    \
+          *(*toP)++ = lo;                                                      \
+          break;                                                               \
+        }                                                                      \
+        /* fall through */                                                     \
+      case 0x1:                                                                \
+      case 0x2:                                                                \
+      case 0x3:                                                                \
+      case 0x4:                                                                \
+      case 0x5:                                                                \
+      case 0x6:                                                                \
+      case 0x7:                                                                \
+        if (toLim - *toP < 2) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        *(*toP)++ = ((lo >> 6) | (hi << 2) | UTF8_cval2);                      \
+        *(*toP)++ = ((lo & 0x3f) | 0x80);                                      \
+        break;                                                                 \
+      default:                                                                 \
+        if (toLim - *toP < 3) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        /* 16 bits divided 4, 6, 6 amongst 3 bytes */                          \
+        *(*toP)++ = ((hi >> 4) | UTF8_cval3);                                  \
+        *(*toP)++ = (((hi & 0xf) << 2) | (lo >> 6) | 0x80);                    \
+        *(*toP)++ = ((lo & 0x3f) | 0x80);                                      \
+        break;                                                                 \
+      case 0xD8:                                                               \
+      case 0xD9:                                                               \
+      case 0xDA:                                                               \
+      case 0xDB:                                                               \
+        if (toLim - *toP < 4) {                                                \
+          *fromP = from;                                                       \
+          return XML_CONVERT_OUTPUT_EXHAUSTED;                                 \
+        }                                                                      \
+        if (fromLim - from < 4) {                                              \
+          *fromP = from;                                                       \
+          return XML_CONVERT_INPUT_INCOMPLETE;                                 \
+        }                                                                      \
+        plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1;                   \
+        *(*toP)++ = (char)((plane >> 2) | UTF8_cval4);                         \
+        *(*toP)++ = (((lo >> 2) & 0xF) | ((plane & 0x3) << 4) | 0x80);         \
+        from += 2;                                                             \
+        lo2 = GET_LO(from);                                                    \
+        *(*toP)++ = (((lo & 0x3) << 4) | ((GET_HI(from) & 0x3) << 2)           \
+                     | (lo2 >> 6) | 0x80);                                     \
+        *(*toP)++ = ((lo2 & 0x3f) | 0x80);                                     \
+        break;                                                                 \
+      }                                                                        \
+    }                                                                          \
+    *fromP = from;                                                             \
+    if (from < fromLim)                                                        \
+      return XML_CONVERT_INPUT_INCOMPLETE;                                     \
+    else                                                                       \
+      return XML_CONVERT_COMPLETED;                                            \
+  }
 
-#define DEFINE_UTF16_TO_UTF16(E) \
-static enum XML_Convert_Result  PTRCALL \
-E ## toUtf16(const ENCODING *UNUSED_P(enc), \
-             const char **fromP, const char *fromLim, \
-             unsigned short **toP, const unsigned short *toLim) \
-{ \
-  enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \
-  fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1);  /* shrink to even */ \
-  /* Avoid copying first half only of surrogate */ \
-  if (fromLim - *fromP > ((toLim - *toP) << 1) \
-      && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \
-    fromLim -= 2; \
-    res = XML_CONVERT_INPUT_INCOMPLETE; \
-  } \
-  for (; *fromP < fromLim && *toP < toLim; *fromP += 2) \
-    *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP); \
-  if ((*toP == toLim) && (*fromP < fromLim)) \
-    return XML_CONVERT_OUTPUT_EXHAUSTED; \
-  else \
-    return res; \
-}
+#define DEFINE_UTF16_TO_UTF16(E)                                               \
+  static enum XML_Convert_Result PTRCALL E##toUtf16(                           \
+      const ENCODING *enc, const char **fromP, const char *fromLim,            \
+      unsigned short **toP, const unsigned short *toLim) {                     \
+    enum XML_Convert_Result res = XML_CONVERT_COMPLETED;                       \
+    UNUSED_P(enc);                                                             \
+    fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */  \
+    /* Avoid copying first half only of surrogate */                           \
+    if (fromLim - *fromP > ((toLim - *toP) << 1)                               \
+        && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) {                             \
+      fromLim -= 2;                                                            \
+      res = XML_CONVERT_INPUT_INCOMPLETE;                                      \
+    }                                                                          \
+    for (; *fromP < fromLim && *toP < toLim; *fromP += 2)                      \
+      *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP);                      \
+    if ((*toP == toLim) && (*fromP < fromLim))                                 \
+      return XML_CONVERT_OUTPUT_EXHAUSTED;                                     \
+    else                                                                       \
+      return res;                                                              \
+  }
 
-#define SET2(ptr, ch) \
-  (((ptr)[0] = ((ch) & 0xff)), ((ptr)[1] = ((ch) >> 8)))
+#define SET2(ptr, ch) (((ptr)[0] = ((ch)&0xff)), ((ptr)[1] = ((ch) >> 8)))
 #define GET_LO(ptr) ((unsigned char)(ptr)[0])
 #define GET_HI(ptr) ((unsigned char)(ptr)[1])
 
@@ -768,8 +721,7 @@ DEFINE_UTF16_TO_UTF16(little2_)
 #undef GET_LO
 #undef GET_HI
 
-#define SET2(ptr, ch) \
-  (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch) & 0xFF)))
+#define SET2(ptr, ch) (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch)&0xFF)))
 #define GET_LO(ptr) ((unsigned char)(ptr)[1])
 #define GET_HI(ptr) ((unsigned char)(ptr)[0])
 
@@ -780,292 +732,279 @@ DEFINE_UTF16_TO_UTF16(big2_)
 #undef GET_LO
 #undef GET_HI
 
-#define LITTLE2_BYTE_TYPE(enc, p) \
- ((p)[1] == 0 \
-  ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)] \
-  : unicode_byte_type((p)[1], (p)[0]))
-#define LITTLE2_BYTE_TO_ASCII(enc, p) ((p)[1] == 0 ? (p)[0] : -1)
-#define LITTLE2_CHAR_MATCHES(enc, p, c) ((p)[1] == 0 && (p)[0] == c)
-#define LITTLE2_IS_NAME_CHAR_MINBPC(enc, p) \
+#define LITTLE2_BYTE_TYPE(enc, p)                                              \
+  ((p)[1] == 0 ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)]  \
+               : unicode_byte_type((p)[1], (p)[0]))
+#define LITTLE2_BYTE_TO_ASCII(p) ((p)[1] == 0 ? (p)[0] : -1)
+#define LITTLE2_CHAR_MATCHES(p, c) ((p)[1] == 0 && (p)[0] == c)
+#define LITTLE2_IS_NAME_CHAR_MINBPC(p)                                         \
   UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0])
-#define LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
+#define LITTLE2_IS_NMSTRT_CHAR_MINBPC(p)                                       \
   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[1], (unsigned char)p[0])
 
 #ifdef XML_MIN_SIZE
 
 static int PTRFASTCALL
-little2_byteType(const ENCODING *enc, const char *p)
-{
+little2_byteType(const ENCODING *enc, const char *p) {
   return LITTLE2_BYTE_TYPE(enc, p);
 }
 
 static int PTRFASTCALL
-little2_byteToAscii(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_BYTE_TO_ASCII(enc, p);
+little2_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_BYTE_TO_ASCII(p);
 }
 
 static int PTRCALL
-little2_charMatches(const ENCODING *enc, const char *p, int c)
-{
-  return LITTLE2_CHAR_MATCHES(enc, p, c);
+little2_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
+  return LITTLE2_CHAR_MATCHES(p, c);
 }
 
 static int PTRFASTCALL
-little2_isNameMin(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_IS_NAME_CHAR_MINBPC(enc, p);
+little2_isNameMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_IS_NAME_CHAR_MINBPC(p);
 }
 
 static int PTRFASTCALL
-little2_isNmstrtMin(const ENCODING *enc, const char *p)
-{
-  return LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p);
+little2_isNmstrtMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return LITTLE2_IS_NMSTRT_CHAR_MINBPC(p);
 }
 
-#undef VTABLE
-#define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
+#  undef VTABLE
+#  define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
 
 #else /* not XML_MIN_SIZE */
 
-#undef PREFIX
-#define PREFIX(ident) little2_ ## ident
-#define MINBPC(enc) 2
+#  undef PREFIX
+#  define PREFIX(ident) little2_##ident
+#  define MINBPC(enc) 2
 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
-#define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
-#define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(enc, p)
-#define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(enc, p, c)
-#define IS_NAME_CHAR(enc, p, n) 0
-#define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(enc, p)
-#define IS_NMSTRT_CHAR(enc, p, n) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p)
-
-#define XML_TOK_IMPL_C
-#include "xmltok_impl.c"
-#undef XML_TOK_IMPL_C
-
-#undef MINBPC
-#undef BYTE_TYPE
-#undef BYTE_TO_ASCII
-#undef CHAR_MATCHES
-#undef IS_NAME_CHAR
-#undef IS_NAME_CHAR_MINBPC
-#undef IS_NMSTRT_CHAR
-#undef IS_NMSTRT_CHAR_MINBPC
-#undef IS_INVALID_CHAR
+#  define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
+#  define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p)
+#  define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c)
+#  define IS_NAME_CHAR(enc, p, n) 0
+#  define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p)
+#  define IS_NMSTRT_CHAR(enc, p, n) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p)
+
+#  define XML_TOK_IMPL_C
+#  include "xmltok_impl.c"
+#  undef XML_TOK_IMPL_C
+
+#  undef MINBPC
+#  undef BYTE_TYPE
+#  undef BYTE_TO_ASCII
+#  undef CHAR_MATCHES
+#  undef IS_NAME_CHAR
+#  undef IS_NAME_CHAR_MINBPC
+#  undef IS_NMSTRT_CHAR
+#  undef IS_NMSTRT_CHAR_MINBPC
+#  undef IS_INVALID_CHAR
 
 #endif /* not XML_MIN_SIZE */
 
 #ifdef XML_NS
 
-static const struct normal_encoding little2_encoding_ns = {
-  { VTABLE, 2, 0,
-#if BYTEORDER == 1234
-    1
-#else
-    0
-#endif
-  },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding little2_encoding_ns
+    = {{VTABLE, 2, 0,
+#  if BYTEORDER == 1234
+        1
+#  else
+        0
+#  endif
+       },
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding little2_encoding = {
-  { VTABLE, 2, 0,
+static const struct normal_encoding little2_encoding
+    = {{VTABLE, 2, 0,
 #if BYTEORDER == 1234
-    1
+        1
 #else
-    0
+        0
 #endif
-  },
-  {
+       },
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #if BYTEORDER != 4321
 
-#ifdef XML_NS
+#  ifdef XML_NS
 
-static const struct normal_encoding internal_little2_encoding_ns = {
-  { VTABLE, 2, 0, 1 },
-  {
-#include "iasciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_little2_encoding_ns
+    = {{VTABLE, 2, 0, 1},
+       {
+#    include "iasciitab.h"
+#    include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
-#endif
+#  endif
 
-static const struct normal_encoding internal_little2_encoding = {
-  { VTABLE, 2, 0, 1 },
-  {
-#define BT_COLON BT_NMSTRT
-#include "iasciitab.h"
-#undef BT_COLON
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(little2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_little2_encoding
+    = {{VTABLE, 2, 0, 1},
+       {
+#  define BT_COLON BT_NMSTRT
+#  include "iasciitab.h"
+#  undef BT_COLON
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(little2_) NULL_VTABLE};
 
 #endif
 
-
-#define BIG2_BYTE_TYPE(enc, p) \
- ((p)[0] == 0 \
-  ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]] \
-  : unicode_byte_type((p)[0], (p)[1]))
-#define BIG2_BYTE_TO_ASCII(enc, p) ((p)[0] == 0 ? (p)[1] : -1)
-#define BIG2_CHAR_MATCHES(enc, p, c) ((p)[0] == 0 && (p)[1] == c)
-#define BIG2_IS_NAME_CHAR_MINBPC(enc, p) \
+#define BIG2_BYTE_TYPE(enc, p)                                                 \
+  ((p)[0] == 0                                                                 \
+       ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]]        \
+       : unicode_byte_type((p)[0], (p)[1]))
+#define BIG2_BYTE_TO_ASCII(p) ((p)[0] == 0 ? (p)[1] : -1)
+#define BIG2_CHAR_MATCHES(p, c) ((p)[0] == 0 && (p)[1] == c)
+#define BIG2_IS_NAME_CHAR_MINBPC(p)                                            \
   UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1])
-#define BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
+#define BIG2_IS_NMSTRT_CHAR_MINBPC(p)                                          \
   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[0], (unsigned char)p[1])
 
 #ifdef XML_MIN_SIZE
 
 static int PTRFASTCALL
-big2_byteType(const ENCODING *enc, const char *p)
-{
+big2_byteType(const ENCODING *enc, const char *p) {
   return BIG2_BYTE_TYPE(enc, p);
 }
 
 static int PTRFASTCALL
-big2_byteToAscii(const ENCODING *enc, const char *p)
-{
-  return BIG2_BYTE_TO_ASCII(enc, p);
+big2_byteToAscii(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_BYTE_TO_ASCII(p);
 }
 
 static int PTRCALL
-big2_charMatches(const ENCODING *enc, const char *p, int c)
-{
-  return BIG2_CHAR_MATCHES(enc, p, c);
+big2_charMatches(const ENCODING *enc, const char *p, int c) {
+  UNUSED_P(enc);
+  return BIG2_CHAR_MATCHES(p, c);
 }
 
 static int PTRFASTCALL
-big2_isNameMin(const ENCODING *enc, const char *p)
-{
-  return BIG2_IS_NAME_CHAR_MINBPC(enc, p);
+big2_isNameMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_IS_NAME_CHAR_MINBPC(p);
 }
 
 static int PTRFASTCALL
-big2_isNmstrtMin(const ENCODING *enc, const char *p)
-{
-  return BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p);
+big2_isNmstrtMin(const ENCODING *enc, const char *p) {
+  UNUSED_P(enc);
+  return BIG2_IS_NMSTRT_CHAR_MINBPC(p);
 }
 
-#undef VTABLE
-#define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
+#  undef VTABLE
+#  define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
 
 #else /* not XML_MIN_SIZE */
 
-#undef PREFIX
-#define PREFIX(ident) big2_ ## ident
-#define MINBPC(enc) 2
+#  undef PREFIX
+#  define PREFIX(ident) big2_##ident
+#  define MINBPC(enc) 2
 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
-#define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
-#define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(enc, p)
-#define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(enc, p, c)
-#define IS_NAME_CHAR(enc, p, n) 0
-#define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(enc, p)
-#define IS_NMSTRT_CHAR(enc, p, n) (0)
-#define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p)
-
-#define XML_TOK_IMPL_C
-#include "xmltok_impl.c"
-#undef XML_TOK_IMPL_C
-
-#undef MINBPC
-#undef BYTE_TYPE
-#undef BYTE_TO_ASCII
-#undef CHAR_MATCHES
-#undef IS_NAME_CHAR
-#undef IS_NAME_CHAR_MINBPC
-#undef IS_NMSTRT_CHAR
-#undef IS_NMSTRT_CHAR_MINBPC
-#undef IS_INVALID_CHAR
+#  define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
+#  define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p)
+#  define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c)
+#  define IS_NAME_CHAR(enc, p, n) 0
+#  define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p)
+#  define IS_NMSTRT_CHAR(enc, p, n) (0)
+#  define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p)
+
+#  define XML_TOK_IMPL_C
+#  include "xmltok_impl.c"
+#  undef XML_TOK_IMPL_C
+
+#  undef MINBPC
+#  undef BYTE_TYPE
+#  undef BYTE_TO_ASCII
+#  undef CHAR_MATCHES
+#  undef IS_NAME_CHAR
+#  undef IS_NAME_CHAR_MINBPC
+#  undef IS_NMSTRT_CHAR
+#  undef IS_NMSTRT_CHAR_MINBPC
+#  undef IS_INVALID_CHAR
 
 #endif /* not XML_MIN_SIZE */
 
 #ifdef XML_NS
 
-static const struct normal_encoding big2_encoding_ns = {
-  { VTABLE, 2, 0,
-#if BYTEORDER == 4321
-  1
-#else
-  0
-#endif
-  },
-  {
-#include "asciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding big2_encoding_ns
+    = {{VTABLE, 2, 0,
+#  if BYTEORDER == 4321
+        1
+#  else
+        0
+#  endif
+       },
+       {
+#  include "asciitab.h"
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #endif
 
-static const struct normal_encoding big2_encoding = {
-  { VTABLE, 2, 0,
+static const struct normal_encoding big2_encoding
+    = {{VTABLE, 2, 0,
 #if BYTEORDER == 4321
-  1
+        1
 #else
-  0
+        0
 #endif
-  },
-  {
+       },
+       {
 #define BT_COLON BT_NMSTRT
 #include "asciitab.h"
 #undef BT_COLON
 #include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #if BYTEORDER != 1234
 
-#ifdef XML_NS
+#  ifdef XML_NS
 
-static const struct normal_encoding internal_big2_encoding_ns = {
-  { VTABLE, 2, 0, 1 },
-  {
-#include "iasciitab.h"
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_big2_encoding_ns
+    = {{VTABLE, 2, 0, 1},
+       {
+#    include "iasciitab.h"
+#    include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
-#endif
+#  endif
 
-static const struct normal_encoding internal_big2_encoding = {
-  { VTABLE, 2, 0, 1 },
-  {
-#define BT_COLON BT_NMSTRT
-#include "iasciitab.h"
-#undef BT_COLON
-#include "latin1tab.h"
-  },
-  STANDARD_VTABLE(big2_) NULL_VTABLE
-};
+static const struct normal_encoding internal_big2_encoding
+    = {{VTABLE, 2, 0, 1},
+       {
+#  define BT_COLON BT_NMSTRT
+#  include "iasciitab.h"
+#  undef BT_COLON
+#  include "latin1tab.h"
+       },
+       STANDARD_VTABLE(big2_) NULL_VTABLE};
 
 #endif
 
 #undef PREFIX
 
 static int FASTCALL
-streqci(const char *s1, const char *s2)
-{
+streqci(const char *s1, const char *s2) {
   for (;;) {
     char c1 = *s1++;
     char c2 = *s2++;
@@ -1079,22 +1018,21 @@ streqci(const char *s1, const char *s2)
       c2 += ASCII_A - ASCII_a; /* LCOV_EXCL_LINE */
     if (c1 != c2)
       return 0;
-    if (!c1)
+    if (! c1)
       break;
   }
   return 1;
 }
 
 static void PTRCALL
-initUpdatePosition(const ENCODING *UNUSED_P(enc), const char *ptr,
-                   const char *end, POSITION *pos)
-{
+initUpdatePosition(const ENCODING *enc, const char *ptr, const char *end,
+                   POSITION *pos) {
+  UNUSED_P(enc);
   normal_updatePosition(&utf8_encoding.enc, ptr, end, pos);
 }
 
 static int
-toAscii(const ENCODING *enc, const char *ptr, const char *end)
-{
+toAscii(const ENCODING *enc, const char *ptr, const char *end) {
   char buf[1];
   char *p = buf;
   XmlUtf8Convert(enc, &ptr, end, &p, p + 1);
@@ -1105,8 +1043,7 @@ toAscii(const ENCODING *enc, const char *ptr, const char *end)
 }
 
 static int FASTCALL
-isSpace(int c)
-{
+isSpace(int c) {
   switch (c) {
   case 0x20:
   case 0xD:
@@ -1121,21 +1058,16 @@ isSpace(int c)
    followed by name=val.
 */
 static int
-parsePseudoAttribute(const ENCODING *enc,
-                     const char *ptr,
-                     const char *end,
-                     const char **namePtr,
-                     const char **nameEndPtr,
-                     const char **valPtr,
-                     const char **nextTokPtr)
-{
+parsePseudoAttribute(const ENCODING *enc, const char *ptr, const char *end,
+                     const char **namePtr, const char **nameEndPtr,
+                     const char **valPtr, const char **nextTokPtr) {
   int c;
   char open;
   if (ptr == end) {
     *namePtr = NULL;
     return 1;
   }
-  if (!isSpace(toAscii(enc, ptr, end))) {
+  if (! isSpace(toAscii(enc, ptr, end))) {
     *nextTokPtr = ptr;
     return 0;
   }
@@ -1191,12 +1123,9 @@ parsePseudoAttribute(const ENCODING *enc,
     c = toAscii(enc, ptr, end);
     if (c == open)
       break;
-    if (!(ASCII_a <= c && c <= ASCII_z)
-        && !(ASCII_A <= c && c <= ASCII_Z)
-        && !(ASCII_0 <= c && c <= ASCII_9)
-        && c != ASCII_PERIOD
-        && c != ASCII_MINUS
-        && c != ASCII_UNDERSCORE) {
+    if (! (ASCII_a <= c && c <= ASCII_z) && ! (ASCII_A <= c && c <= ASCII_Z)
+        && ! (ASCII_0 <= c && c <= ASCII_9) && c != ASCII_PERIOD
+        && c != ASCII_MINUS && c != ASCII_UNDERSCORE) {
       *nextTokPtr = ptr;
       return 0;
     }
@@ -1205,68 +1134,52 @@ parsePseudoAttribute(const ENCODING *enc,
   return 1;
 }
 
-static const char KW_version[] = {
-  ASCII_v, ASCII_e, ASCII_r, ASCII_s, ASCII_i, ASCII_o, ASCII_n, '\0'
-};
+static const char KW_version[]
+    = {ASCII_v, ASCII_e, ASCII_r, ASCII_s, ASCII_i, ASCII_o, ASCII_n, '\0'};
 
-static const char KW_encoding[] = {
-  ASCII_e, ASCII_n, ASCII_c, ASCII_o, ASCII_d, ASCII_i, ASCII_n, ASCII_g, '\0'
-};
+static const char KW_encoding[] = {ASCII_e, ASCII_n, ASCII_c, ASCII_o, ASCII_d,
+                                   ASCII_i, ASCII_n, ASCII_g, '\0'};
 
-static const char KW_standalone[] = {
-  ASCII_s, ASCII_t, ASCII_a, ASCII_n, ASCII_d, ASCII_a, ASCII_l, ASCII_o,
-  ASCII_n, ASCII_e, '\0'
-};
+static const char KW_standalone[]
+    = {ASCII_s, ASCII_t, ASCII_a, ASCII_n, ASCII_d, ASCII_a,
+       ASCII_l, ASCII_o, ASCII_n, ASCII_e, '\0'};
 
-static const char KW_yes[] = {
-  ASCII_y, ASCII_e, ASCII_s,  '\0'
-};
+static const char KW_yes[] = {ASCII_y, ASCII_e, ASCII_s, '\0'};
 
-static const char KW_no[] = {
-  ASCII_n, ASCII_o,  '\0'
-};
+static const char KW_no[] = {ASCII_n, ASCII_o, '\0'};
 
 static int
-doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
-                                                 const char *,
+doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *,
                                                  const char *),
-               int isGeneralTextEntity,
-               const ENCODING *enc,
-               const char *ptr,
-               const char *end,
-               const char **badPtr,
-               const char **versionPtr,
-               const char **versionEndPtr,
-               const char **encodingName,
-               const ENCODING **encoding,
-               int *standalone)
-{
+               int isGeneralTextEntity, const ENCODING *enc, const char *ptr,
+               const char *end, const char **badPtr, const char **versionPtr,
+               const char **versionEndPtr, const char **encodingName,
+               const ENCODING **encoding, int *standalone) {
   const char *val = NULL;
   const char *name = NULL;
   const char *nameEnd = NULL;
   ptr += 5 * enc->minBytesPerChar;
   end -= 2 * enc->minBytesPerChar;
-  if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)
-      || !name) {
+  if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)
+      || ! name) {
     *badPtr = ptr;
     return 0;
   }
-  if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_version)) {
-    if (!isGeneralTextEntity) {
+  if (! XmlNameMatchesAscii(enc, name, nameEnd, KW_version)) {
+    if (! isGeneralTextEntity) {
       *badPtr = name;
       return 0;
     }
-  }
-  else {
+  } else {
     if (versionPtr)
       *versionPtr = val;
     if (versionEndPtr)
       *versionEndPtr = ptr;
-    if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
+    if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
       *badPtr = ptr;
       return 0;
     }
-    if (!name) {
+    if (! name) {
       if (isGeneralTextEntity) {
         /* a TextDecl must have an EncodingDecl */
         *badPtr = ptr;
@@ -1277,7 +1190,7 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
   }
   if (XmlNameMatchesAscii(enc, name, nameEnd, KW_encoding)) {
     int c = toAscii(enc, val, end);
-    if (!(ASCII_a <= c && c <= ASCII_z) && !(ASCII_A <= c && c <= ASCII_Z)) {
+    if (! (ASCII_a <= c && c <= ASCII_z) && ! (ASCII_A <= c && c <= ASCII_Z)) {
       *badPtr = val;
       return 0;
     }
@@ -1285,14 +1198,14 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
       *encodingName = val;
     if (encoding)
       *encoding = encodingFinder(enc, val, ptr - enc->minBytesPerChar);
-    if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
+    if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
       *badPtr = ptr;
       return 0;
     }
-    if (!name)
+    if (! name)
       return 1;
   }
-  if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_standalone)
+  if (! XmlNameMatchesAscii(enc, name, nameEnd, KW_standalone)
       || isGeneralTextEntity) {
     *badPtr = name;
     return 0;
@@ -1300,12 +1213,10 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
   if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_yes)) {
     if (standalone)
       *standalone = 1;
-  }
-  else if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_no)) {
+  } else if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_no)) {
     if (standalone)
       *standalone = 0;
-  }
-  else {
+  } else {
     *badPtr = val;
     return 0;
   }
@@ -1319,11 +1230,16 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
 }
 
 static int FASTCALL
-checkCharRefNumber(int result)
-{
+checkCharRefNumber(int result) {
   switch (result >> 8) {
-  case 0xD8: case 0xD9: case 0xDA: case 0xDB:
-  case 0xDC: case 0xDD: case 0xDE: case 0xDF:
+  case 0xD8:
+  case 0xD9:
+  case 0xDA:
+  case 0xDB:
+  case 0xDC:
+  case 0xDD:
+  case 0xDE:
+  case 0xDF:
     return -1;
   case 0:
     if (latin1_encoding.type[result] == BT_NONXML)
@@ -1338,8 +1254,7 @@ checkCharRefNumber(int result)
 }
 
 int FASTCALL
-XmlUtf8Encode(int c, char *buf)
-{
+XmlUtf8Encode(int c, char *buf) {
   enum {
     /* minN is minimum legal resulting value for N byte sequence */
     min2 = 0x80,
@@ -1375,8 +1290,7 @@ XmlUtf8Encode(int c, char *buf)
 }
 
 int FASTCALL
-XmlUtf16Encode(int charNum, unsigned short *buf)
-{
+XmlUtf16Encode(int charNum, unsigned short *buf) {
   if (charNum < 0)
     return 0;
   if (charNum < 0x10000) {
@@ -1400,17 +1314,15 @@ struct unknown_encoding {
   char utf8[256][4];
 };
 
-#define AS_UNKNOWN_ENCODING(enc)  ((const struct unknown_encoding *) (enc))
+#define AS_UNKNOWN_ENCODING(enc) ((const struct unknown_encoding *)(enc))
 
 int
-XmlSizeOfUnknownEncoding(void)
-{
+XmlSizeOfUnknownEncoding(void) {
   return sizeof(struct unknown_encoding);
 }
 
 static int PTRFASTCALL
-unknown_isName(const ENCODING *enc, const char *p)
-{
+unknown_isName(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   if (c & ~0xFFFF)
@@ -1419,8 +1331,7 @@ unknown_isName(const ENCODING *enc, const char *p)
 }
 
 static int PTRFASTCALL
-unknown_isNmstrt(const ENCODING *enc, const char *p)
-{
+unknown_isNmstrt(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   if (c & ~0xFFFF)
@@ -1429,18 +1340,15 @@ unknown_isNmstrt(const ENCODING *enc, const char *p)
 }
 
 static int PTRFASTCALL
-unknown_isInvalid(const ENCODING *enc, const char *p)
-{
+unknown_isInvalid(const ENCODING *enc, const char *p) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   int c = uenc->convert(uenc->userData, p);
   return (c & ~0xFFFF) || checkCharRefNumber(c) < 0;
 }
 
 static enum XML_Convert_Result PTRCALL
-unknown_toUtf8(const ENCODING *enc,
-               const char **fromP, const char *fromLim,
-               char **toP, const char *toLim)
-{
+unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim,
+               char **toP, const char *toLim) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   char buf[XML_UTF8_ENCODE_MAX];
   for (;;) {
@@ -1458,8 +1366,7 @@ unknown_toUtf8(const ENCODING *enc,
       utf8 = buf;
       *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP]
                  - (BT_LEAD2 - 2));
-    }
-    else {
+    } else {
       if (n > toLim - *toP)
         return XML_CONVERT_OUTPUT_EXHAUSTED;
       (*fromP)++;
@@ -1470,20 +1377,16 @@ unknown_toUtf8(const ENCODING *enc,
 }
 
 static enum XML_Convert_Result PTRCALL
-unknown_toUtf16(const ENCODING *enc,
-                const char **fromP, const char *fromLim,
-                unsigned short **toP, const unsigned short *toLim)
-{
+unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim,
+                unsigned short **toP, const unsigned short *toLim) {
   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
   while (*fromP < fromLim && *toP < toLim) {
     unsigned short c = uenc->utf16[(unsigned char)**fromP];
     if (c == 0) {
-      c = (unsigned short)
-          uenc->convert(uenc->userData, *fromP);
+      c = (unsigned short)uenc->convert(uenc->userData, *fromP);
       *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP]
                  - (BT_LEAD2 - 2));
-    }
-    else
+    } else
       (*fromP)++;
     *(*toP)++ = c;
   }
@@ -1495,19 +1398,14 @@ unknown_toUtf16(const ENCODING *enc,
 }
 
 ENCODING *
-XmlInitUnknownEncoding(void *mem,
-                       int *table,
-                       CONVERTER convert,
-                       void *userData)
-{
+XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert,
+                       void *userData) {
   int i;
   struct unknown_encoding *e = (struct unknown_encoding *)mem;
-  for (i = 0; i < (int)sizeof(struct normal_encoding); i++)
-    ((char *)mem)[i] = ((char *)&latin1_encoding)[i];
+  memcpy(mem, &latin1_encoding, sizeof(struct normal_encoding));
   for (i = 0; i < 128; i++)
     if (latin1_encoding.type[i] != BT_OTHER
-        && latin1_encoding.type[i] != BT_NONXML
-        && table[i] != i)
+        && latin1_encoding.type[i] != BT_NONXML && table[i] != i)
       return 0;
   for (i = 0; i < 256; i++) {
     int c = table[i];
@@ -1517,35 +1415,30 @@ XmlInitUnknownEncoding(void *mem,
       e->utf16[i] = 0xFFFF;
       e->utf8[i][0] = 1;
       e->utf8[i][1] = 0;
-    }
-    else if (c < 0) {
+    } else if (c < 0) {
       if (c < -4)
         return 0;
       /* Multi-byte sequences need a converter function */
-      if (!convert)
+      if (! convert)
         return 0;
       e->normal.type[i] = (unsigned char)(BT_LEAD2 - (c + 2));
       e->utf8[i][0] = 0;
       e->utf16[i] = 0;
-    }
-    else if (c < 0x80) {
+    } else if (c < 0x80) {
       if (latin1_encoding.type[c] != BT_OTHER
-          && latin1_encoding.type[c] != BT_NONXML
-          && c != i)
+          && latin1_encoding.type[c] != BT_NONXML && c != i)
         return 0;
       e->normal.type[i] = latin1_encoding.type[c];
       e->utf8[i][0] = 1;
       e->utf8[i][1] = (char)c;
       e->utf16[i] = (unsigned short)(c == 0 ? 0xFFFF : c);
-    }
-    else if (checkCharRefNumber(c) < 0) {
+    } else if (checkCharRefNumber(c) < 0) {
       e->normal.type[i] = BT_NONXML;
       /* This shouldn't really get used. */
       e->utf16[i] = 0xFFFF;
       e->utf8[i][0] = 1;
       e->utf8[i][1] = 0;
-    }
-    else {
+    } else {
       if (c > 0xFFFF)
         return 0;
       if (UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xff))
@@ -1590,44 +1483,32 @@ enum {
   NO_ENC
 };
 
-static const char KW_ISO_8859_1[] = {
-  ASCII_I, ASCII_S, ASCII_O, ASCII_MINUS, ASCII_8, ASCII_8, ASCII_5, ASCII_9,
-  ASCII_MINUS, ASCII_1, '\0'
-};
-static const char KW_US_ASCII[] = {
-  ASCII_U, ASCII_S, ASCII_MINUS, ASCII_A, ASCII_S, ASCII_C, ASCII_I, ASCII_I,
-  '\0'
-};
-static const char KW_UTF_8[] =  {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_8, '\0'
-};
-static const char KW_UTF_16[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, '\0'
-};
-static const char KW_UTF_16BE[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_B, ASCII_E,
-  '\0'
-};
-static const char KW_UTF_16LE[] = {
-  ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_L, ASCII_E,
-  '\0'
-};
+static const char KW_ISO_8859_1[]
+    = {ASCII_I, ASCII_S, ASCII_O,     ASCII_MINUS, ASCII_8, ASCII_8,
+       ASCII_5, ASCII_9, ASCII_MINUS, ASCII_1,     '\0'};
+static const char KW_US_ASCII[]
+    = {ASCII_U, ASCII_S, ASCII_MINUS, ASCII_A, ASCII_S,
+       ASCII_C, ASCII_I, ASCII_I,     '\0'};
+static const char KW_UTF_8[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_8, '\0'};
+static const char KW_UTF_16[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, '\0'};
+static const char KW_UTF_16BE[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1,
+       ASCII_6, ASCII_B, ASCII_E, '\0'};
+static const char KW_UTF_16LE[]
+    = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1,
+       ASCII_6, ASCII_L, ASCII_E, '\0'};
 
 static int FASTCALL
-getEncodingIndex(const char *name)
-{
-  static const char * const encodingNames[] = {
-    KW_ISO_8859_1,
-    KW_US_ASCII,
-    KW_UTF_8,
-    KW_UTF_16,
-    KW_UTF_16BE,
-    KW_UTF_16LE,
+getEncodingIndex(const char *name) {
+  static const char *const encodingNames[] = {
+      KW_ISO_8859_1, KW_US_ASCII, KW_UTF_8, KW_UTF_16, KW_UTF_16BE, KW_UTF_16LE,
   };
   int i;
   if (name == NULL)
     return NO_ENC;
-  for (i = 0; i < (int)(sizeof(encodingNames)/sizeof(encodingNames[0])); i++)
+  for (i = 0; i < (int)(sizeof(encodingNames) / sizeof(encodingNames[0])); i++)
     if (streqci(name, encodingNames[i]))
       return i;
   return UNKNOWN_ENC;
@@ -1647,15 +1528,9 @@ getEncodingIndex(const char *name)
    XML_PROLOG_STATE otherwise.
 */
 
-
 static int
-initScan(const ENCODING * const *encodingTable,
-         const INIT_ENCODING *enc,
-         int state,
-         const char *ptr,
-         const char *end,
-         const char **nextTokPtr)
-{
+initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc,
+         int state, const char *ptr, const char *end, const char **nextTokPtr) {
   const ENCODING **encPtr;
 
   if (ptr >= end)
@@ -1680,20 +1555,17 @@ initScan(const ENCODING * const *encodingTable,
     case 0xFE:
     case 0xFF:
     case 0xEF: /* possibly first byte of UTF-8 BOM */
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       /* fall through */
     case 0x00:
     case 0x3C:
       return XML_TOK_PARTIAL;
     }
-  }
-  else {
+  } else {
     switch (((unsigned char)ptr[0] << 8) | (unsigned char)ptr[1]) {
     case 0xFEFF:
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       *nextTokPtr = ptr + 2;
       *encPtr = encodingTable[UTF_16BE_ENC];
@@ -1707,8 +1579,7 @@ initScan(const ENCODING * const *encodingTable,
       *encPtr = encodingTable[UTF_16LE_ENC];
       return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
     case 0xFFFE:
-      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
-          && state == XML_CONTENT_STATE)
+      if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE)
         break;
       *nextTokPtr = ptr + 2;
       *encPtr = encodingTable[UTF_16LE_ENC];
@@ -1723,8 +1594,8 @@ initScan(const ENCODING * const *encodingTable,
       */
       if (state == XML_CONTENT_STATE) {
         int e = INIT_ENC_INDEX(enc);
-        if (e == ISO_8859_1_ENC || e == UTF_16BE_ENC
-            || e == UTF_16LE_ENC || e == UTF_16_ENC)
+        if (e == ISO_8859_1_ENC || e == UTF_16BE_ENC || e == UTF_16LE_ENC
+            || e == UTF_16_ENC)
           break;
       }
       if (ptr + 2 == end)
@@ -1747,8 +1618,7 @@ initScan(const ENCODING * const *encodingTable,
           break;
         *encPtr = encodingTable[UTF_16BE_ENC];
         return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
-      }
-      else if (ptr[1] == '\0') {
+      } else if (ptr[1] == '\0') {
         /* We could recover here in the case:
             - parsing an external entity
             - second byte is 0
@@ -1770,7 +1640,6 @@ initScan(const ENCODING * const *encodingTable,
   return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
 }
 
-
 #define NS(x) x
 #define ns(x) x
 #define XML_TOK_NS_C
@@ -1781,22 +1650,19 @@ initScan(const ENCODING * const *encodingTable,
 
 #ifdef XML_NS
 
-#define NS(x) x ## NS
-#define ns(x) x ## _ns
+#  define NS(x) x##NS
+#  define ns(x) x##_ns
 
-#define XML_TOK_NS_C
-#include "xmltok_ns.c"
-#undef XML_TOK_NS_C
+#  define XML_TOK_NS_C
+#  include "xmltok_ns.c"
+#  undef XML_TOK_NS_C
 
-#undef NS
-#undef ns
+#  undef NS
+#  undef ns
 
 ENCODING *
-XmlInitUnknownEncodingNS(void *mem,
-                         int *table,
-                         CONVERTER convert,
-                         void *userData)
-{
+XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert,
+                         void *userData) {
   ENCODING *enc = XmlInitUnknownEncoding(mem, table, convert, userData);
   if (enc)
     ((struct normal_encoding *)enc)->type[ASCII_COLON] = BT_COLON;
diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h
index 50926f38ab32..2adbf5307bef 100644
--- a/Modules/expat/xmltok.h
+++ b/Modules/expat/xmltok.h
@@ -38,16 +38,18 @@ extern "C" {
 #endif
 
 /* The following token may be returned by XmlContentTok */
-#define XML_TOK_TRAILING_RSQB -5 /* ] or ]] at the end of the scan; might be
-                                    start of illegal ]]> sequence */
+#define XML_TOK_TRAILING_RSQB                                                  \
+  -5 /* ] or ]] at the end of the scan; might be                               \
+        start of illegal ]]> sequence */
 /* The following tokens may be returned by both XmlPrologTok and
    XmlContentTok.
 */
-#define XML_TOK_NONE -4          /* The string to be scanned is empty */
-#define XML_TOK_TRAILING_CR -3   /* A CR at the end of the scan;
-                                    might be part of CRLF sequence */
-#define XML_TOK_PARTIAL_CHAR -2  /* only part of a multibyte sequence */
-#define XML_TOK_PARTIAL -1       /* only part of a token */
+#define XML_TOK_NONE -4 /* The string to be scanned is empty */
+#define XML_TOK_TRAILING_CR                                                    \
+  -3                            /* A CR at the end of the scan;                \
+                                   might be part of CRLF sequence */
+#define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */
+#define XML_TOK_PARTIAL -1      /* only part of a token */
 #define XML_TOK_INVALID 0
 
 /* The following tokens are returned by XmlContentTok; some are also
@@ -62,24 +64,24 @@ extern "C" {
 #define XML_TOK_DATA_NEWLINE 7
 #define XML_TOK_CDATA_SECT_OPEN 8
 #define XML_TOK_ENTITY_REF 9
-#define XML_TOK_CHAR_REF 10               /* numeric character reference */
+#define XML_TOK_CHAR_REF 10 /* numeric character reference */
 
 /* The following tokens may be returned by both XmlPrologTok and
    XmlContentTok.
 */
-#define XML_TOK_PI 11                     /* processing instruction */
-#define XML_TOK_XML_DECL 12               /* XML decl or text decl */
+#define XML_TOK_PI 11       /* processing instruction */
+#define XML_TOK_XML_DECL 12 /* XML decl or text decl */
 #define XML_TOK_COMMENT 13
-#define XML_TOK_BOM 14                    /* Byte order mark */
+#define XML_TOK_BOM 14 /* Byte order mark */
 
 /* The following tokens are returned only by XmlPrologTok */
 #define XML_TOK_PROLOG_S 15
-#define XML_TOK_DECL_OPEN 16              /* <!foo */
-#define XML_TOK_DECL_CLOSE 17             /* > */
+#define XML_TOK_DECL_OPEN 16  /* <!foo */
+#define XML_TOK_DECL_CLOSE 17 /* > */
 #define XML_TOK_NAME 18
 #define XML_TOK_NMTOKEN 19
-#define XML_TOK_POUND_NAME 20             /* #name */
-#define XML_TOK_OR 21                     /* | */
+#define XML_TOK_POUND_NAME 20 /* #name */
+#define XML_TOK_OR 21         /* | */
 #define XML_TOK_PERCENT 22
 #define XML_TOK_OPEN_PAREN 23
 #define XML_TOK_CLOSE_PAREN 24
@@ -90,14 +92,14 @@ extern "C" {
 #define XML_TOK_INSTANCE_START 29
 
 /* The following occur only in element type declarations */
-#define XML_TOK_NAME_QUESTION 30          /* name? */
-#define XML_TOK_NAME_ASTERISK 31          /* name* */
-#define XML_TOK_NAME_PLUS 32              /* name+ */
-#define XML_TOK_COND_SECT_OPEN 33         /* <![ */
-#define XML_TOK_COND_SECT_CLOSE 34        /* ]]> */
-#define XML_TOK_CLOSE_PAREN_QUESTION 35   /* )? */
-#define XML_TOK_CLOSE_PAREN_ASTERISK 36   /* )* */
-#define XML_TOK_CLOSE_PAREN_PLUS 37       /* )+ */
+#define XML_TOK_NAME_QUESTION 30        /* name? */
+#define XML_TOK_NAME_ASTERISK 31        /* name* */
+#define XML_TOK_NAME_PLUS 32            /* name+ */
+#define XML_TOK_COND_SECT_OPEN 33       /* <![ */
+#define XML_TOK_COND_SECT_CLOSE 34      /* ]]> */
+#define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */
+#define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */
+#define XML_TOK_CLOSE_PAREN_PLUS 37     /* )+ */
 #define XML_TOK_COMMA 38
 
 /* The following token is returned only by XmlAttributeValueTok */
@@ -112,20 +114,20 @@ extern "C" {
 #define XML_TOK_PREFIXED_NAME 41
 
 #ifdef XML_DTD
-#define XML_TOK_IGNORE_SECT 42
+#  define XML_TOK_IGNORE_SECT 42
 #endif /* XML_DTD */
 
 #ifdef XML_DTD
-#define XML_N_STATES 4
+#  define XML_N_STATES 4
 #else /* not XML_DTD */
-#define XML_N_STATES 3
+#  define XML_N_STATES 3
 #endif /* not XML_DTD */
 
 #define XML_PROLOG_STATE 0
 #define XML_CONTENT_STATE 1
 #define XML_CDATA_SECTION_STATE 2
 #ifdef XML_DTD
-#define XML_IGNORE_SECTION_STATE 3
+#  define XML_IGNORE_SECTION_STATE 3
 #endif /* XML_DTD */
 
 #define XML_N_LITERAL_TYPES 2
@@ -153,52 +155,41 @@ typedef struct {
 struct encoding;
 typedef struct encoding ENCODING;
 
-typedef int (PTRCALL *SCANNER)(const ENCODING *,
-                               const char *,
-                               const char *,
-                               const char **);
+typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *,
+                              const char **);
 
 enum XML_Convert_Result {
   XML_CONVERT_COMPLETED = 0,
   XML_CONVERT_INPUT_INCOMPLETE = 1,
-  XML_CONVERT_OUTPUT_EXHAUSTED = 2  /* and therefore potentially input remaining as well */
+  XML_CONVERT_OUTPUT_EXHAUSTED
+  = 2 /* and therefore potentially input remaining as well */
 };
 
 struct encoding {
   SCANNER scanners[XML_N_STATES];
   SCANNER literalScanners[XML_N_LITERAL_TYPES];
-  int (PTRCALL *nameMatchesAscii)(const ENCODING *,
-                                  const char *,
-                                  const char *,
-                                  const char *);
-  int (PTRFASTCALL *nameLength)(const ENCODING *, const char *);
+  int(PTRCALL *nameMatchesAscii)(const ENCODING *, const char *, const char *,
+                                 const char *);
+  int(PTRFASTCALL *nameLength)(const ENCODING *, const char *);
   const char *(PTRFASTCALL *skipS)(const ENCODING *, const char *);
-  int (PTRCALL *getAtts)(const ENCODING *enc,
-                         const char *ptr,
-                         int attsMax,
-                         ATTRIBUTE *atts);
-  int (PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr);
-  int (PTRCALL *predefinedEntityName)(const ENCODING *,
-                                      const char *,
-                                      const char *);
-  void (PTRCALL *updatePosition)(const ENCODING *,
-                                 const char *ptr,
-                                 const char *end,
-                                 POSITION *);
-  int (PTRCALL *isPublicId)(const ENCODING *enc,
-                            const char *ptr,
-                            const char *end,
-                            const char **badPtr);
-  enum XML_Convert_Result (PTRCALL *utf8Convert)(const ENCODING *enc,
-                              const char **fromP,
-                              const char *fromLim,
-                              char **toP,
-                              const char *toLim);
-  enum XML_Convert_Result (PTRCALL *utf16Convert)(const ENCODING *enc,
-                               const char **fromP,
-                               const char *fromLim,
-                               unsigned short **toP,
-                               const unsigned short *toLim);
+  int(PTRCALL *getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
+                        ATTRIBUTE *atts);
+  int(PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr);
+  int(PTRCALL *predefinedEntityName)(const ENCODING *, const char *,
+                                     const char *);
+  void(PTRCALL *updatePosition)(const ENCODING *, const char *ptr,
+                                const char *end, POSITION *);
+  int(PTRCALL *isPublicId)(const ENCODING *enc, const char *ptr,
+                           const char *end, const char **badPtr);
+  enum XML_Convert_Result(PTRCALL *utf8Convert)(const ENCODING *enc,
+                                                const char **fromP,
+                                                const char *fromLim, char **toP,
+                                                const char *toLim);
+  enum XML_Convert_Result(PTRCALL *utf16Convert)(const ENCODING *enc,
+                                                 const char **fromP,
+                                                 const char *fromLim,
+                                                 unsigned short **toP,
+                                                 const unsigned short *toLim);
   int minBytesPerChar;
   char isUtf8;
   char isUtf16;
@@ -225,66 +216,62 @@ struct encoding {
    the prolog outside literals, comments and processing instructions.
 */
 
-
-#define XmlTok(enc, state, ptr, end, nextTokPtr) \
+#define XmlTok(enc, state, ptr, end, nextTokPtr)                               \
   (((enc)->scanners[state])(enc, ptr, end, nextTokPtr))
 
-#define XmlPrologTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr)
+#define XmlPrologTok(enc, ptr, end, nextTokPtr)                                \
+  XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr)
 
-#define XmlContentTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr)
+#define XmlContentTok(enc, ptr, end, nextTokPtr)                               \
+  XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr)
 
-#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr)
+#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr)                          \
+  XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr)
 
 #ifdef XML_DTD
 
-#define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \
-   XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr)
+#  define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr)                       \
+    XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr)
 
 #endif /* XML_DTD */
 
 /* This is used for performing a 2nd-level tokenization on the content
    of a literal that has already been returned by XmlTok.
 */
-#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \
+#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr)                  \
   (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr))
 
-#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \
-   XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr)
+#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr)                        \
+  XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr)
 
-#define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \
-   XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr)
+#define XmlEntityValueTok(enc, ptr, end, nextTokPtr)                           \
+  XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr)
 
-#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \
+#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2)                             \
   (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2))
 
-#define XmlNameLength(enc, ptr) \
-  (((enc)->nameLength)(enc, ptr))
+#define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr))
 
-#define XmlSkipS(enc, ptr) \
-  (((enc)->skipS)(enc, ptr))
+#define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr))
 
-#define XmlGetAttributes(enc, ptr, attsMax, atts) \
+#define XmlGetAttributes(enc, ptr, attsMax, atts)                              \
   (((enc)->getAtts)(enc, ptr, attsMax, atts))
 
-#define XmlCharRefNumber(enc, ptr) \
-  (((enc)->charRefNumber)(enc, ptr))
+#define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr))
 
-#define XmlPredefinedEntityName(enc, ptr, end) \
+#define XmlPredefinedEntityName(enc, ptr, end)                                 \
   (((enc)->predefinedEntityName)(enc, ptr, end))
 
-#define XmlUpdatePosition(enc, ptr, end, pos) \
+#define XmlUpdatePosition(enc, ptr, end, pos)                                  \
   (((enc)->updatePosition)(enc, ptr, end, pos))
 
-#define XmlIsPublicId(enc, ptr, end, badPtr) \
+#define XmlIsPublicId(enc, ptr, end, badPtr)                                   \
   (((enc)->isPublicId)(enc, ptr, end, badPtr))
 
-#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \
+#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim)                        \
   (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim))
 
-#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \
+#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim)                       \
   (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim))
 
 typedef struct {
@@ -292,16 +279,11 @@ typedef struct {
   const ENCODING **encPtr;
 } INIT_ENCODING;
 
-int XmlParseXmlDecl(int isGeneralTextEntity,
-                    const ENCODING *enc,
-                    const char *ptr,
-                    const char *end,
-                    const char **badPtr,
-                    const char **versionPtr,
-                    const char **versionEndPtr,
+int XmlParseXmlDecl(int isGeneralTextEntity, const ENCODING *enc,
+                    const char *ptr, const char *end, const char **badPtr,
+                    const char **versionPtr, const char **versionEndPtr,
                     const char **encodingNamePtr,
-                    const ENCODING **namedEncodingPtr,
-                    int *standalonePtr);
+                    const ENCODING **namedEncodingPtr, int *standalonePtr);
 
 int XmlInitEncoding(INIT_ENCODING *, const ENCODING **, const char *name);
 const ENCODING *XmlGetUtf8InternalEncoding(void);
@@ -310,34 +292,22 @@ int FASTCALL XmlUtf8Encode(int charNumber, char *buf);
 int FASTCALL XmlUtf16Encode(int charNumber, unsigned short *buf);
 int XmlSizeOfUnknownEncoding(void);
 
+typedef int(XMLCALL *CONVERTER)(void *userData, const char *p);
 
-typedef int (XMLCALL *CONVERTER) (void *userData, const char *p);
-
-ENCODING *
-XmlInitUnknownEncoding(void *mem,
-                       int *table,
-                       CONVERTER convert,
-                       void *userData);
+ENCODING *XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert,
+                                 void *userData);
 
-int XmlParseXmlDeclNS(int isGeneralTextEntity,
-                      const ENCODING *enc,
-                      const char *ptr,
-                      const char *end,
-                      const char **badPtr,
-                      const char **versionPtr,
-                      const char **versionEndPtr,
+int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc,
+                      const char *ptr, const char *end, const char **badPtr,
+                      const char **versionPtr, const char **versionEndPtr,
                       const char **encodingNamePtr,
-                      const ENCODING **namedEncodingPtr,
-                      int *standalonePtr);
+                      const ENCODING **namedEncodingPtr, int *standalonePtr);
 
 int XmlInitEncodingNS(INIT_ENCODING *, const ENCODING **, const char *name);
 const ENCODING *XmlGetUtf8InternalEncodingNS(void);
 const ENCODING *XmlGetUtf16InternalEncodingNS(void);
-ENCODING *
-XmlInitUnknownEncodingNS(void *mem,
-                         int *table,
-                         CONVERTER convert,
-                         void *userData);
+ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert,
+                                   void *userData);
 #ifdef __cplusplus
 }
 #endif
diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c
index 4d9ae7dc3896..c209221cd79d 100644
--- a/Modules/expat/xmltok_impl.c
+++ b/Modules/expat/xmltok_impl.c
@@ -32,130 +32,124 @@
 
 #ifdef XML_TOK_IMPL_C
 
-#ifndef IS_INVALID_CHAR
-#define IS_INVALID_CHAR(enc, ptr, n) (0)
-#endif
-
-#define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n) \
-        return XML_TOK_PARTIAL_CHAR; \
-      if (IS_INVALID_CHAR(enc, ptr, n)) { \
-        *(nextTokPtr) = (ptr); \
-        return XML_TOK_INVALID; \
-      } \
-      ptr += n; \
-      break;
+#  ifndef IS_INVALID_CHAR
+#    define IS_INVALID_CHAR(enc, ptr, n) (0)
+#  endif
+
+#  define INVALID_LEAD_CASE(n, ptr, nextTokPtr)                                \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (IS_INVALID_CHAR(enc, ptr, n)) {                                        \
+      *(nextTokPtr) = (ptr);                                                   \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define INVALID_CASES(ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(2, ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(3, ptr, nextTokPtr) \
-  INVALID_LEAD_CASE(4, ptr, nextTokPtr) \
-  case BT_NONXML: \
-  case BT_MALFORM: \
-  case BT_TRAIL: \
-    *(nextTokPtr) = (ptr); \
+#  define INVALID_CASES(ptr, nextTokPtr)                                       \
+    INVALID_LEAD_CASE(2, ptr, nextTokPtr)                                      \
+    INVALID_LEAD_CASE(3, ptr, nextTokPtr)                                      \
+    INVALID_LEAD_CASE(4, ptr, nextTokPtr)                                      \
+  case BT_NONXML:                                                              \
+  case BT_MALFORM:                                                             \
+  case BT_TRAIL:                                                               \
+    *(nextTokPtr) = (ptr);                                                     \
     return XML_TOK_INVALID;
 
-#define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr) \
-   case BT_LEAD ## n: \
-     if (end - ptr < n) \
-       return XML_TOK_PARTIAL_CHAR; \
-     if (!IS_NAME_CHAR(enc, ptr, n)) { \
-       *nextTokPtr = ptr; \
-       return XML_TOK_INVALID; \
-     } \
-     ptr += n; \
-     break;
-
-#define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) \
-  case BT_NONASCII: \
-    if (!IS_NAME_CHAR_MINBPC(enc, ptr)) { \
-      *nextTokPtr = ptr; \
-      return XML_TOK_INVALID; \
-    } \
-    /* fall through */ \
-  case BT_NMSTRT: \
-  case BT_HEX: \
-  case BT_DIGIT: \
-  case BT_NAME: \
-  case BT_MINUS: \
-    ptr += MINBPC(enc); \
-    break; \
-  CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr) \
-  CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr) \
-  CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)
-
-#define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr) \
-   case BT_LEAD ## n: \
-     if (end - ptr < n) \
-       return XML_TOK_PARTIAL_CHAR; \
-     if (!IS_NMSTRT_CHAR(enc, ptr, n)) { \
-       *nextTokPtr = ptr; \
-       return XML_TOK_INVALID; \
-     } \
-     ptr += n; \
-     break;
-
-#define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) \
-  case BT_NONASCII: \
-    if (!IS_NMSTRT_CHAR_MINBPC(enc, ptr)) { \
-      *nextTokPtr = ptr; \
-      return XML_TOK_INVALID; \
-    } \
-    /* fall through */ \
-  case BT_NMSTRT: \
-  case BT_HEX: \
-    ptr += MINBPC(enc); \
-    break; \
-  CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr) \
-  CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr) \
-  CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)
-
-#ifndef PREFIX
-#define PREFIX(ident) ident
-#endif
-
-
-#define HAS_CHARS(enc, ptr, end, count) \
-    (end - ptr >= count * MINBPC(enc))
+#  define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr)                        \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (! IS_NAME_CHAR(enc, ptr, n)) {                                         \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define HAS_CHAR(enc, ptr, end) \
-    HAS_CHARS(enc, ptr, end, 1)
+#  define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)                          \
+  case BT_NONASCII:                                                            \
+    if (! IS_NAME_CHAR_MINBPC(enc, ptr)) {                                     \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    /* fall through */                                                         \
+  case BT_NMSTRT:                                                              \
+  case BT_HEX:                                                                 \
+  case BT_DIGIT:                                                               \
+  case BT_NAME:                                                                \
+  case BT_MINUS:                                                               \
+    ptr += MINBPC(enc);                                                        \
+    break;                                                                     \
+    CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr)                              \
+    CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr)                              \
+    CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)
+
+#  define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr)                      \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (! IS_NMSTRT_CHAR(enc, ptr, n)) {                                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
 
-#define REQUIRE_CHARS(enc, ptr, end, count) \
-    { \
-      if (! HAS_CHARS(enc, ptr, end, count)) { \
-        return XML_TOK_PARTIAL; \
-      } \
+#  define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)                        \
+  case BT_NONASCII:                                                            \
+    if (! IS_NMSTRT_CHAR_MINBPC(enc, ptr)) {                                   \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_INVALID;                                                  \
+    }                                                                          \
+    /* fall through */                                                         \
+  case BT_NMSTRT:                                                              \
+  case BT_HEX:                                                                 \
+    ptr += MINBPC(enc);                                                        \
+    break;                                                                     \
+    CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr)                            \
+    CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr)                            \
+    CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)
+
+#  ifndef PREFIX
+#    define PREFIX(ident) ident
+#  endif
+
+#  define HAS_CHARS(enc, ptr, end, count) (end - ptr >= count * MINBPC(enc))
+
+#  define HAS_CHAR(enc, ptr, end) HAS_CHARS(enc, ptr, end, 1)
+
+#  define REQUIRE_CHARS(enc, ptr, end, count)                                  \
+    {                                                                          \
+      if (! HAS_CHARS(enc, ptr, end, count)) {                                 \
+        return XML_TOK_PARTIAL;                                                \
+      }                                                                        \
     }
 
-#define REQUIRE_CHAR(enc, ptr, end) \
-    REQUIRE_CHARS(enc, ptr, end, 1)
-
+#  define REQUIRE_CHAR(enc, ptr, end) REQUIRE_CHARS(enc, ptr, end, 1)
 
 /* ptr points to character following "<!-" */
 
 static int PTRCALL
-PREFIX(scanComment)(const ENCODING *enc, const char *ptr,
-                    const char *end, const char **nextTokPtr)
-{
+PREFIX(scanComment)(const ENCODING *enc, const char *ptr, const char *end,
+                    const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
-    if (!CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
     }
     ptr += MINBPC(enc);
     while (HAS_CHAR(enc, ptr, end)) {
       switch (BYTE_TYPE(enc, ptr)) {
-      INVALID_CASES(ptr, nextTokPtr)
+        INVALID_CASES(ptr, nextTokPtr)
       case BT_MINUS:
         ptr += MINBPC(enc);
         REQUIRE_CHAR(enc, ptr, end);
         if (CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
-          if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+          if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
             *nextTokPtr = ptr;
             return XML_TOK_INVALID;
           }
@@ -175,9 +169,8 @@ PREFIX(scanComment)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "<!" */
 
 static int PTRCALL
-PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
-                 const char *end, const char **nextTokPtr)
-{
+PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, const char *end,
+                 const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
   case BT_MINUS:
@@ -199,12 +192,17 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
       REQUIRE_CHARS(enc, ptr, end, 2);
       /* don't allow <!ENTITY% foo "whatever"> */
       switch (BYTE_TYPE(enc, ptr + MINBPC(enc))) {
-      case BT_S: case BT_CR: case BT_LF: case BT_PERCNT:
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+      case BT_PERCNT:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       /* fall through */
-    case BT_S: case BT_CR: case BT_LF:
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       *nextTokPtr = ptr;
       return XML_TOK_DECL_OPEN;
     case BT_NMSTRT:
@@ -220,12 +218,12 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                      const char *end, int *tokPtr)
-{
+PREFIX(checkPiTarget)(const ENCODING *enc, const char *ptr, const char *end,
+                      int *tokPtr) {
   int upper = 0;
+  UNUSED_P(enc);
   *tokPtr = XML_TOK_PI;
-  if (end - ptr != MINBPC(enc)*3)
+  if (end - ptr != MINBPC(enc) * 3)
     return 1;
   switch (BYTE_TO_ASCII(enc, ptr)) {
   case ASCII_x:
@@ -265,30 +263,31 @@ PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr,
 /* ptr points to character following "<?" */
 
 static int PTRCALL
-PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
-               const char *end, const char **nextTokPtr)
-{
+PREFIX(scanPi)(const ENCODING *enc, const char *ptr, const char *end,
+               const char **nextTokPtr) {
   int tok;
   const char *target = ptr;
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_S: case BT_CR: case BT_LF:
-      if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
+      if (! PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       ptr += MINBPC(enc);
       while (HAS_CHAR(enc, ptr, end)) {
         switch (BYTE_TYPE(enc, ptr)) {
-        INVALID_CASES(ptr, nextTokPtr)
+          INVALID_CASES(ptr, nextTokPtr)
         case BT_QUEST:
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
@@ -304,7 +303,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
       }
       return XML_TOK_PARTIAL;
     case BT_QUEST:
-      if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
+      if (! PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -324,16 +323,16 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                         const char *end, const char **nextTokPtr)
-{
-  static const char CDATA_LSQB[] = { ASCII_C, ASCII_D, ASCII_A,
-                                     ASCII_T, ASCII_A, ASCII_LSQB };
+PREFIX(scanCdataSection)(const ENCODING *enc, const char *ptr, const char *end,
+                         const char **nextTokPtr) {
+  static const char CDATA_LSQB[]
+      = {ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, ASCII_LSQB};
   int i;
+  UNUSED_P(enc);
   /* CDATA[ */
   REQUIRE_CHARS(enc, ptr, end, 6);
   for (i = 0; i < 6; i++, ptr += MINBPC(enc)) {
-    if (!CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {
+    if (! CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
     }
@@ -343,9 +342,8 @@ PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
-                        const char *end, const char **nextTokPtr)
-{
+PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
+                        const char **nextTokPtr) {
   if (ptr >= end)
     return XML_TOK_NONE;
   if (MINBPC(enc) > 1) {
@@ -361,11 +359,11 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
   case BT_RSQB:
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
-    if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
+    if (! CHAR_MATCHES(enc, ptr, ASCII_RSQB))
       break;
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
-    if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
       ptr -= MINBPC(enc);
       break;
     }
@@ -381,23 +379,25 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
   case BT_LF:
     *nextTokPtr = ptr + MINBPC(enc);
     return XML_TOK_DATA_NEWLINE;
-  INVALID_CASES(ptr, nextTokPtr)
+    INVALID_CASES(ptr, nextTokPtr)
   default:
     ptr += MINBPC(enc);
     break;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \
-        *nextTokPtr = ptr; \
-        return XML_TOK_DATA_CHARS; \
-      } \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) {                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_DATA_CHARS;                                               \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONXML:
     case BT_MALFORM:
     case BT_TRAIL:
@@ -418,23 +418,26 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "</" */
 
 static int PTRCALL
-PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
-                   const char *end, const char **nextTokPtr)
-{
+PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr, const char *end,
+                   const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_S: case BT_CR: case BT_LF:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) {
         switch (BYTE_TYPE(enc, ptr)) {
-        case BT_S: case BT_CR: case BT_LF:
+        case BT_S:
+        case BT_CR:
+        case BT_LF:
           break;
         case BT_GT:
           *nextTokPtr = ptr + MINBPC(enc);
@@ -445,13 +448,13 @@ PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
         }
       }
       return XML_TOK_PARTIAL;
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
       /* no need to check qname syntax here,
          since end-tag must match exactly */
       ptr += MINBPC(enc);
       break;
-#endif
+#  endif
     case BT_GT:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_END_TAG;
@@ -466,9 +469,8 @@ PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "&#X" */
 
 static int PTRCALL
-PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr,
-                       const char *end, const char **nextTokPtr)
-{
+PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, const char *end,
+                       const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
     case BT_DIGIT:
@@ -498,9 +500,8 @@ PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr,
 /* ptr points to character following "&#" */
 
 static int PTRCALL
-PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr,
-                    const char *end, const char **nextTokPtr)
-{
+PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, const char *end,
+                    const char **nextTokPtr) {
   if (HAS_CHAR(enc, ptr, end)) {
     if (CHAR_MATCHES(enc, ptr, ASCII_x))
       return PREFIX(scanHexCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -531,11 +532,10 @@ PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr,
 
 static int PTRCALL
 PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
-                const char **nextTokPtr)
-{
+                const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   case BT_NUM:
     return PREFIX(scanCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
   default:
@@ -544,7 +544,7 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
     case BT_SEMI:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_ENTITY_REF;
@@ -560,15 +560,14 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
-                 const char **nextTokPtr)
-{
-#ifdef XML_NS
+                 const char **nextTokPtr) {
+#  ifdef XML_NS
   int hadColon = 0;
-#endif
+#  endif
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-#ifdef XML_NS
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+#  ifdef XML_NS
     case BT_COLON:
       if (hadColon) {
         *nextTokPtr = ptr;
@@ -578,14 +577,16 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
       switch (BYTE_TYPE(enc, ptr)) {
-      CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+        CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
       default:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       break;
-#endif
-    case BT_S: case BT_CR: case BT_LF:
+#  endif
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       for (;;) {
         int t;
 
@@ -605,101 +606,101 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
         }
       }
       /* fall through */
-    case BT_EQUALS:
-      {
-        int open;
-#ifdef XML_NS
-        hadColon = 0;
-#endif
-        for (;;) {
-          ptr += MINBPC(enc);
-          REQUIRE_CHAR(enc, ptr, end);
-          open = BYTE_TYPE(enc, ptr);
-          if (open == BT_QUOT || open == BT_APOS)
-            break;
-          switch (open) {
-          case BT_S:
-          case BT_LF:
-          case BT_CR:
-            break;
-          default:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          }
-        }
+    case BT_EQUALS: {
+      int open;
+#  ifdef XML_NS
+      hadColon = 0;
+#  endif
+      for (;;) {
         ptr += MINBPC(enc);
-        /* in attribute value */
-        for (;;) {
-          int t;
-          REQUIRE_CHAR(enc, ptr, end);
-          t = BYTE_TYPE(enc, ptr);
-          if (t == open)
-            break;
-          switch (t) {
+        REQUIRE_CHAR(enc, ptr, end);
+        open = BYTE_TYPE(enc, ptr);
+        if (open == BT_QUOT || open == BT_APOS)
+          break;
+        switch (open) {
+        case BT_S:
+        case BT_LF:
+        case BT_CR:
+          break;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
+        }
+      }
+      ptr += MINBPC(enc);
+      /* in attribute value */
+      for (;;) {
+        int t;
+        REQUIRE_CHAR(enc, ptr, end);
+        t = BYTE_TYPE(enc, ptr);
+        if (t == open)
+          break;
+        switch (t) {
           INVALID_CASES(ptr, nextTokPtr)
-          case BT_AMP:
-            {
-              int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);
-              if (tok <= 0) {
-                if (tok == XML_TOK_INVALID)
-                  *nextTokPtr = ptr;
-                return tok;
-              }
-              break;
-            }
-          case BT_LT:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          default:
-            ptr += MINBPC(enc);
-            break;
+        case BT_AMP: {
+          int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);
+          if (tok <= 0) {
+            if (tok == XML_TOK_INVALID)
+              *nextTokPtr = ptr;
+            return tok;
           }
+          break;
+        }
+        case BT_LT:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
+        default:
+          ptr += MINBPC(enc);
+          break;
         }
+      }
+      ptr += MINBPC(enc);
+      REQUIRE_CHAR(enc, ptr, end);
+      switch (BYTE_TYPE(enc, ptr)) {
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+        break;
+      case BT_SOL:
+        goto sol;
+      case BT_GT:
+        goto gt;
+      default:
+        *nextTokPtr = ptr;
+        return XML_TOK_INVALID;
+      }
+      /* ptr points to closing quote */
+      for (;;) {
         ptr += MINBPC(enc);
         REQUIRE_CHAR(enc, ptr, end);
         switch (BYTE_TYPE(enc, ptr)) {
+          CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
         case BT_S:
         case BT_CR:
         case BT_LF:
-          break;
-        case BT_SOL:
-          goto sol;
+          continue;
         case BT_GT:
-          goto gt;
-        default:
-          *nextTokPtr = ptr;
-          return XML_TOK_INVALID;
-        }
-        /* ptr points to closing quote */
-        for (;;) {
+        gt:
+          *nextTokPtr = ptr + MINBPC(enc);
+          return XML_TOK_START_TAG_WITH_ATTS;
+        case BT_SOL:
+        sol:
           ptr += MINBPC(enc);
           REQUIRE_CHAR(enc, ptr, end);
-          switch (BYTE_TYPE(enc, ptr)) {
-          CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-          case BT_S: case BT_CR: case BT_LF:
-            continue;
-          case BT_GT:
-          gt:
-            *nextTokPtr = ptr + MINBPC(enc);
-            return XML_TOK_START_TAG_WITH_ATTS;
-          case BT_SOL:
-          sol:
-            ptr += MINBPC(enc);
-            REQUIRE_CHAR(enc, ptr, end);
-            if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
-              *nextTokPtr = ptr;
-              return XML_TOK_INVALID;
-            }
-            *nextTokPtr = ptr + MINBPC(enc);
-            return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;
-          default:
+          if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
             *nextTokPtr = ptr;
             return XML_TOK_INVALID;
           }
-          break;
+          *nextTokPtr = ptr + MINBPC(enc);
+          return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
         }
         break;
       }
+      break;
+    }
     default:
       *nextTokPtr = ptr;
       return XML_TOK_INVALID;
@@ -712,14 +713,13 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
-               const char **nextTokPtr)
-{
-#ifdef XML_NS
+               const char **nextTokPtr) {
+#  ifdef XML_NS
   int hadColon;
-#endif
+#  endif
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   case BT_EXCL:
     ptr += MINBPC(enc);
     REQUIRE_CHAR(enc, ptr, end);
@@ -727,8 +727,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_MINUS:
       return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);
     case BT_LSQB:
-      return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc),
-                                      end, nextTokPtr);
+      return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc), end, nextTokPtr);
     }
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
@@ -740,14 +739,14 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
-#ifdef XML_NS
+#  ifdef XML_NS
   hadColon = 0;
-#endif
+#  endif
   /* we have a start-tag */
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-#ifdef XML_NS
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+#  ifdef XML_NS
     case BT_COLON:
       if (hadColon) {
         *nextTokPtr = ptr;
@@ -757,34 +756,37 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
       switch (BYTE_TYPE(enc, ptr)) {
-      CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+        CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
       default:
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       break;
-#endif
-    case BT_S: case BT_CR: case BT_LF:
-      {
-        ptr += MINBPC(enc);
-        while (HAS_CHAR(enc, ptr, end)) {
-          switch (BYTE_TYPE(enc, ptr)) {
+#  endif
+    case BT_S:
+    case BT_CR:
+    case BT_LF: {
+      ptr += MINBPC(enc);
+      while (HAS_CHAR(enc, ptr, end)) {
+        switch (BYTE_TYPE(enc, ptr)) {
           CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-          case BT_GT:
-            goto gt;
-          case BT_SOL:
-            goto sol;
-          case BT_S: case BT_CR: case BT_LF:
-            ptr += MINBPC(enc);
-            continue;
-          default:
-            *nextTokPtr = ptr;
-            return XML_TOK_INVALID;
-          }
-          return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);
+        case BT_GT:
+          goto gt;
+        case BT_SOL:
+          goto sol;
+        case BT_S:
+        case BT_CR:
+        case BT_LF:
+          ptr += MINBPC(enc);
+          continue;
+        default:
+          *nextTokPtr = ptr;
+          return XML_TOK_INVALID;
         }
-        return XML_TOK_PARTIAL;
+        return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);
       }
+      return XML_TOK_PARTIAL;
+    }
     case BT_GT:
     gt:
       *nextTokPtr = ptr + MINBPC(enc);
@@ -793,7 +795,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
     sol:
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
-      if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+      if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -809,8 +811,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **nextTokPtr)
-{
+                   const char **nextTokPtr) {
   if (ptr >= end)
     return XML_TOK_NONE;
   if (MINBPC(enc) > 1) {
@@ -842,48 +843,50 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
     ptr += MINBPC(enc);
     if (! HAS_CHAR(enc, ptr, end))
       return XML_TOK_TRAILING_RSQB;
-    if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
+    if (! CHAR_MATCHES(enc, ptr, ASCII_RSQB))
       break;
     ptr += MINBPC(enc);
     if (! HAS_CHAR(enc, ptr, end))
       return XML_TOK_TRAILING_RSQB;
-    if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
+    if (! CHAR_MATCHES(enc, ptr, ASCII_GT)) {
       ptr -= MINBPC(enc);
       break;
     }
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
-  INVALID_CASES(ptr, nextTokPtr)
+    INVALID_CASES(ptr, nextTokPtr)
   default:
     ptr += MINBPC(enc);
     break;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \
-        *nextTokPtr = ptr; \
-        return XML_TOK_DATA_CHARS; \
-      } \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) {                       \
+      *nextTokPtr = ptr;                                                       \
+      return XML_TOK_DATA_CHARS;                                               \
+    }                                                                          \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_RSQB:
       if (HAS_CHARS(enc, ptr, end, 2)) {
-         if (!CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {
-           ptr += MINBPC(enc);
-           break;
-         }
-         if (HAS_CHARS(enc, ptr, end, 3)) {
-           if (!CHAR_MATCHES(enc, ptr + 2*MINBPC(enc), ASCII_GT)) {
-             ptr += MINBPC(enc);
-             break;
-           }
-           *nextTokPtr = ptr + 2*MINBPC(enc);
-           return XML_TOK_INVALID;
-         }
+        if (! CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {
+          ptr += MINBPC(enc);
+          break;
+        }
+        if (HAS_CHARS(enc, ptr, end, 3)) {
+          if (! CHAR_MATCHES(enc, ptr + 2 * MINBPC(enc), ASCII_GT)) {
+            ptr += MINBPC(enc);
+            break;
+          }
+          *nextTokPtr = ptr + 2 * MINBPC(enc);
+          return XML_TOK_INVALID;
+        }
       }
       /* fall through */
     case BT_AMP:
@@ -908,12 +911,14 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
-                    const char **nextTokPtr)
-{
+                    const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
-  case BT_S: case BT_LF: case BT_CR: case BT_PERCNT:
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+  case BT_S:
+  case BT_LF:
+  case BT_CR:
+  case BT_PERCNT:
     *nextTokPtr = ptr;
     return XML_TOK_PERCENT;
   default:
@@ -922,7 +927,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
     case BT_SEMI:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_PARAM_ENTITY_REF;
@@ -936,20 +941,24 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,
 
 static int PTRCALL
 PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,
-                      const char **nextTokPtr)
-{
+                      const char **nextTokPtr) {
   REQUIRE_CHAR(enc, ptr, end);
   switch (BYTE_TYPE(enc, ptr)) {
-  CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
+    CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
   default:
     *nextTokPtr = ptr;
     return XML_TOK_INVALID;
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_CR: case BT_LF: case BT_S:
-    case BT_RPAR: case BT_GT: case BT_PERCNT: case BT_VERBAR:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_CR:
+    case BT_LF:
+    case BT_S:
+    case BT_RPAR:
+    case BT_GT:
+    case BT_PERCNT:
+    case BT_VERBAR:
       *nextTokPtr = ptr;
       return XML_TOK_POUND_NAME;
     default:
@@ -961,14 +970,12 @@ PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,
 }
 
 static int PTRCALL
-PREFIX(scanLit)(int open, const ENCODING *enc,
-                const char *ptr, const char *end,
-                const char **nextTokPtr)
-{
+PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end,
+                const char **nextTokPtr) {
   while (HAS_CHAR(enc, ptr, end)) {
     int t = BYTE_TYPE(enc, ptr);
     switch (t) {
-    INVALID_CASES(ptr, nextTokPtr)
+      INVALID_CASES(ptr, nextTokPtr)
     case BT_QUOT:
     case BT_APOS:
       ptr += MINBPC(enc);
@@ -978,8 +985,12 @@ PREFIX(scanLit)(int open, const ENCODING *enc,
         return -XML_TOK_LITERAL;
       *nextTokPtr = ptr;
       switch (BYTE_TYPE(enc, ptr)) {
-      case BT_S: case BT_CR: case BT_LF:
-      case BT_GT: case BT_PERCNT: case BT_LSQB:
+      case BT_S:
+      case BT_CR:
+      case BT_LF:
+      case BT_GT:
+      case BT_PERCNT:
+      case BT_LSQB:
         return XML_TOK_LITERAL;
       default:
         return XML_TOK_INVALID;
@@ -994,8 +1005,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc,
 
 static int PTRCALL
 PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
-                  const char **nextTokPtr)
-{
+                  const char **nextTokPtr) {
   int tok;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1013,27 +1023,26 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     return PREFIX(scanLit)(BT_QUOT, enc, ptr + MINBPC(enc), end, nextTokPtr);
   case BT_APOS:
     return PREFIX(scanLit)(BT_APOS, enc, ptr + MINBPC(enc), end, nextTokPtr);
-  case BT_LT:
-    {
-      ptr += MINBPC(enc);
-      REQUIRE_CHAR(enc, ptr, end);
-      switch (BYTE_TYPE(enc, ptr)) {
-      case BT_EXCL:
-        return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-      case BT_QUEST:
-        return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-      case BT_NMSTRT:
-      case BT_HEX:
-      case BT_NONASCII:
-      case BT_LEAD2:
-      case BT_LEAD3:
-      case BT_LEAD4:
-        *nextTokPtr = ptr - MINBPC(enc);
-        return XML_TOK_INSTANCE_START;
-      }
-      *nextTokPtr = ptr;
-      return XML_TOK_INVALID;
+  case BT_LT: {
+    ptr += MINBPC(enc);
+    REQUIRE_CHAR(enc, ptr, end);
+    switch (BYTE_TYPE(enc, ptr)) {
+    case BT_EXCL:
+      return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);
+    case BT_QUEST:
+      return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
+    case BT_NMSTRT:
+    case BT_HEX:
+    case BT_NONASCII:
+    case BT_LEAD2:
+    case BT_LEAD3:
+    case BT_LEAD4:
+      *nextTokPtr = ptr - MINBPC(enc);
+      return XML_TOK_INSTANCE_START;
     }
+    *nextTokPtr = ptr;
+    return XML_TOK_INVALID;
+  }
   case BT_CR:
     if (ptr + MINBPC(enc) == end) {
       *nextTokPtr = end;
@@ -1041,13 +1050,15 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
       return -XML_TOK_PROLOG_S;
     }
     /* fall through */
-  case BT_S: case BT_LF:
+  case BT_S:
+  case BT_LF:
     for (;;) {
       ptr += MINBPC(enc);
       if (! HAS_CHAR(enc, ptr, end))
         break;
       switch (BYTE_TYPE(enc, ptr)) {
-      case BT_S: case BT_LF:
+      case BT_S:
+      case BT_LF:
         break;
       case BT_CR:
         /* don't split CR/LF pair */
@@ -1076,7 +1087,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) {
       REQUIRE_CHARS(enc, ptr, end, 2);
       if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_GT)) {
-        *nextTokPtr = ptr + 2*MINBPC(enc);
+        *nextTokPtr = ptr + 2 * MINBPC(enc);
         return XML_TOK_COND_SECT_CLOSE;
       }
     }
@@ -1099,8 +1110,12 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_PLUS:
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_CLOSE_PAREN_PLUS;
-    case BT_CR: case BT_LF: case BT_S:
-    case BT_GT: case BT_COMMA: case BT_VERBAR:
+    case BT_CR:
+    case BT_LF:
+    case BT_S:
+    case BT_GT:
+    case BT_COMMA:
+    case BT_VERBAR:
     case BT_RPAR:
       *nextTokPtr = ptr;
       return XML_TOK_CLOSE_PAREN;
@@ -1115,24 +1130,26 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
     return XML_TOK_DECL_CLOSE;
   case BT_NUM:
     return PREFIX(scanPoundName)(enc, ptr + MINBPC(enc), end, nextTokPtr);
-#define LEAD_CASE(n) \
-  case BT_LEAD ## n: \
-    if (end - ptr < n) \
-      return XML_TOK_PARTIAL_CHAR; \
-    if (IS_NMSTRT_CHAR(enc, ptr, n)) { \
-      ptr += n; \
-      tok = XML_TOK_NAME; \
-      break; \
-    } \
-    if (IS_NAME_CHAR(enc, ptr, n)) { \
-      ptr += n; \
-      tok = XML_TOK_NMTOKEN; \
-      break; \
-    } \
-    *nextTokPtr = ptr; \
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    if (end - ptr < n)                                                         \
+      return XML_TOK_PARTIAL_CHAR;                                             \
+    if (IS_NMSTRT_CHAR(enc, ptr, n)) {                                         \
+      ptr += n;                                                                \
+      tok = XML_TOK_NAME;                                                      \
+      break;                                                                   \
+    }                                                                          \
+    if (IS_NAME_CHAR(enc, ptr, n)) {                                           \
+      ptr += n;                                                                \
+      tok = XML_TOK_NMTOKEN;                                                   \
+      break;                                                                   \
+    }                                                                          \
+    *nextTokPtr = ptr;                                                         \
     return XML_TOK_INVALID;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+    LEAD_CASE(2)
+    LEAD_CASE(3)
+    LEAD_CASE(4)
+#  undef LEAD_CASE
   case BT_NMSTRT:
   case BT_HEX:
     tok = XML_TOK_NAME;
@@ -1141,9 +1158,9 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
   case BT_DIGIT:
   case BT_NAME:
   case BT_MINUS:
-#ifdef XML_NS
+#  ifdef XML_NS
   case BT_COLON:
-#endif
+#  endif
     tok = XML_TOK_NMTOKEN;
     ptr += MINBPC(enc);
     break;
@@ -1165,13 +1182,19 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
-    case BT_GT: case BT_RPAR: case BT_COMMA:
-    case BT_VERBAR: case BT_LSQB: case BT_PERCNT:
-    case BT_S: case BT_CR: case BT_LF:
+      CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+    case BT_GT:
+    case BT_RPAR:
+    case BT_COMMA:
+    case BT_VERBAR:
+    case BT_LSQB:
+    case BT_PERCNT:
+    case BT_S:
+    case BT_CR:
+    case BT_LF:
       *nextTokPtr = ptr;
       return tok;
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
       ptr += MINBPC(enc);
       switch (tok) {
@@ -1179,7 +1202,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
         REQUIRE_CHAR(enc, ptr, end);
         tok = XML_TOK_PREFIXED_NAME;
         switch (BYTE_TYPE(enc, ptr)) {
-        CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
+          CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
         default:
           tok = XML_TOK_NMTOKEN;
           break;
@@ -1190,23 +1213,23 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
         break;
       }
       break;
-#endif
+#  endif
     case BT_PLUS:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_NAME_PLUS;
     case BT_AST:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
       *nextTokPtr = ptr + MINBPC(enc);
       return XML_TOK_NAME_ASTERISK;
     case BT_QUEST:
-      if (tok == XML_TOK_NMTOKEN)  {
+      if (tok == XML_TOK_NMTOKEN) {
         *nextTokPtr = ptr;
         return XML_TOK_INVALID;
       }
@@ -1221,9 +1244,8 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,
 }
 
 static int PTRCALL
-PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
-                          const char *end, const char **nextTokPtr)
-{
+PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end,
+                          const char **nextTokPtr) {
   const char *start;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1238,10 +1260,14 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
   start = ptr;
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_AMP:
       if (ptr == start)
         return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -1287,9 +1313,8 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
-                       const char *end, const char **nextTokPtr)
-{
+PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end,
+                       const char **nextTokPtr) {
   const char *start;
   if (ptr >= end)
     return XML_TOK_NONE;
@@ -1304,10 +1329,14 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
   start = ptr;
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_AMP:
       if (ptr == start)
         return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
@@ -1315,8 +1344,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
       return XML_TOK_DATA_CHARS;
     case BT_PERCNT:
       if (ptr == start) {
-        int tok =  PREFIX(scanPercent)(enc, ptr + MINBPC(enc),
-                                       end, nextTokPtr);
+        int tok = PREFIX(scanPercent)(enc, ptr + MINBPC(enc), end, nextTokPtr);
         return (tok == XML_TOK_PERCENT) ? XML_TOK_INVALID : tok;
       }
       *nextTokPtr = ptr;
@@ -1349,12 +1377,11 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
   return XML_TOK_DATA_CHARS;
 }
 
-#ifdef XML_DTD
+#  ifdef XML_DTD
 
 static int PTRCALL
-PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
-                         const char *end, const char **nextTokPtr)
-{
+PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end,
+                         const char **nextTokPtr) {
   int level = 0;
   if (MINBPC(enc) > 1) {
     size_t n = end - ptr;
@@ -1365,7 +1392,7 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
   }
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-    INVALID_CASES(ptr, nextTokPtr)
+      INVALID_CASES(ptr, nextTokPtr)
     case BT_LT:
       ptr += MINBPC(enc);
       REQUIRE_CHAR(enc, ptr, end);
@@ -1402,12 +1429,11 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr,
   return XML_TOK_PARTIAL;
 }
 
-#endif /* XML_DTD */
+#  endif /* XML_DTD */
 
 static int PTRCALL
 PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **badPtr)
-{
+                   const char **badPtr) {
   ptr += MINBPC(enc);
   end -= MINBPC(enc);
   for (; HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) {
@@ -1430,9 +1456,9 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
     case BT_AST:
     case BT_PERCNT:
     case BT_NUM:
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
-#endif
+#  endif
       break;
     case BT_S:
       if (CHAR_MATCHES(enc, ptr, ASCII_TAB)) {
@@ -1442,7 +1468,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
       break;
     case BT_NAME:
     case BT_NMSTRT:
-      if (!(BYTE_TO_ASCII(enc, ptr) & ~0x7f))
+      if (! (BYTE_TO_ASCII(enc, ptr) & ~0x7f))
         break;
       /* fall through */
     default:
@@ -1466,9 +1492,8 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end,
 */
 
 static int PTRCALL
-PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
-                int attsMax, ATTRIBUTE *atts)
-{
+PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax,
+                ATTRIBUTE *atts) {
   enum { other, inName, inValue } state = inName;
   int nAtts = 0;
   int open = 0; /* defined when state == inValue;
@@ -1476,32 +1501,35 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
 
   for (ptr += MINBPC(enc);; ptr += MINBPC(enc)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define START_NAME \
-      if (state == other) { \
-        if (nAtts < attsMax) { \
-          atts[nAtts].name = ptr; \
-          atts[nAtts].normalized = 1; \
-        } \
-        state = inName; \
-      }
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: START_NAME ptr += (n - MINBPC(enc)); break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define START_NAME                                                           \
+    if (state == other) {                                                      \
+      if (nAtts < attsMax) {                                                   \
+        atts[nAtts].name = ptr;                                                \
+        atts[nAtts].normalized = 1;                                            \
+      }                                                                        \
+      state = inName;                                                          \
+    }
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    START_NAME ptr += (n - MINBPC(enc));                                       \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONASCII:
     case BT_NMSTRT:
     case BT_HEX:
       START_NAME
       break;
-#undef START_NAME
+#  undef START_NAME
     case BT_QUOT:
       if (state != inValue) {
         if (nAtts < attsMax)
           atts[nAtts].valuePtr = ptr + MINBPC(enc);
         state = inValue;
         open = BT_QUOT;
-      }
-      else if (open == BT_QUOT) {
+      } else if (open == BT_QUOT) {
         state = other;
         if (nAtts < attsMax)
           atts[nAtts].valueEnd = ptr;
@@ -1514,8 +1542,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
           atts[nAtts].valuePtr = ptr + MINBPC(enc);
         state = inValue;
         open = BT_APOS;
-      }
-      else if (open == BT_APOS) {
+      } else if (open == BT_APOS) {
         state = other;
         if (nAtts < attsMax)
           atts[nAtts].valueEnd = ptr;
@@ -1529,16 +1556,15 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
     case BT_S:
       if (state == inName)
         state = other;
-      else if (state == inValue
-               && nAtts < attsMax
-               && atts[nAtts].normalized
+      else if (state == inValue && nAtts < attsMax && atts[nAtts].normalized
                && (ptr == atts[nAtts].valuePtr
                    || BYTE_TO_ASCII(enc, ptr) != ASCII_SPACE
                    || BYTE_TO_ASCII(enc, ptr + MINBPC(enc)) == ASCII_SPACE
                    || BYTE_TYPE(enc, ptr + MINBPC(enc)) == open))
         atts[nAtts].normalized = 0;
       break;
-    case BT_CR: case BT_LF:
+    case BT_CR:
+    case BT_LF:
       /* This case ensures that the first attribute name is counted
          Apart from that we could just change state on the quote. */
       if (state == inName)
@@ -1559,29 +1585,44 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr,
 }
 
 static int PTRFASTCALL
-PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
-{
+PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) {
   int result = 0;
   /* skip &# */
-  ptr += 2*MINBPC(enc);
+  UNUSED_P(enc);
+  ptr += 2 * MINBPC(enc);
   if (CHAR_MATCHES(enc, ptr, ASCII_x)) {
-    for (ptr += MINBPC(enc);
-         !CHAR_MATCHES(enc, ptr, ASCII_SEMI);
+    for (ptr += MINBPC(enc); ! CHAR_MATCHES(enc, ptr, ASCII_SEMI);
          ptr += MINBPC(enc)) {
       int c = BYTE_TO_ASCII(enc, ptr);
       switch (c) {
-      case ASCII_0: case ASCII_1: case ASCII_2: case ASCII_3: case ASCII_4:
-      case ASCII_5: case ASCII_6: case ASCII_7: case ASCII_8: case ASCII_9:
+      case ASCII_0:
+      case ASCII_1:
+      case ASCII_2:
+      case ASCII_3:
+      case ASCII_4:
+      case ASCII_5:
+      case ASCII_6:
+      case ASCII_7:
+      case ASCII_8:
+      case ASCII_9:
         result <<= 4;
         result |= (c - ASCII_0);
         break;
-      case ASCII_A: case ASCII_B: case ASCII_C:
-      case ASCII_D: case ASCII_E: case ASCII_F:
+      case ASCII_A:
+      case ASCII_B:
+      case ASCII_C:
+      case ASCII_D:
+      case ASCII_E:
+      case ASCII_F:
         result <<= 4;
         result += 10 + (c - ASCII_A);
         break;
-      case ASCII_a: case ASCII_b: case ASCII_c:
-      case ASCII_d: case ASCII_e: case ASCII_f:
+      case ASCII_a:
+      case ASCII_b:
+      case ASCII_c:
+      case ASCII_d:
+      case ASCII_e:
+      case ASCII_f:
         result <<= 4;
         result += 10 + (c - ASCII_a);
         break;
@@ -1589,9 +1630,8 @@ PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
       if (result >= 0x110000)
         return -1;
     }
-  }
-  else {
-    for (; !CHAR_MATCHES(enc, ptr, ASCII_SEMI); ptr += MINBPC(enc)) {
+  } else {
+    for (; ! CHAR_MATCHES(enc, ptr, ASCII_SEMI); ptr += MINBPC(enc)) {
       int c = BYTE_TO_ASCII(enc, ptr);
       result *= 10;
       result += (c - ASCII_0);
@@ -1603,10 +1643,10 @@ PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr)
 }
 
 static int PTRCALL
-PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr,
-                             const char *end)
-{
-  switch ((end - ptr)/MINBPC(enc)) {
+PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr,
+                             const char *end) {
+  UNUSED_P(enc);
+  switch ((end - ptr) / MINBPC(enc)) {
   case 2:
     if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_t)) {
       switch (BYTE_TO_ASCII(enc, ptr)) {
@@ -1657,9 +1697,9 @@ PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr,
 }
 
 static int PTRCALL
-PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1,
-                         const char *end1, const char *ptr2)
-{
+PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1,
+                         const char *end1, const char *ptr2) {
+  UNUSED_P(enc);
   for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) {
     if (end1 - ptr1 < MINBPC(enc)) {
       /* This line cannot be executed.  The incoming data has already
@@ -1669,27 +1709,30 @@ PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1,
        */
       return 0; /* LCOV_EXCL_LINE */
     }
-    if (!CHAR_MATCHES(enc, ptr1, *ptr2))
+    if (! CHAR_MATCHES(enc, ptr1, *ptr2))
       return 0;
   }
   return ptr1 == end1;
 }
 
 static int PTRFASTCALL
-PREFIX(nameLength)(const ENCODING *enc, const char *ptr)
-{
+PREFIX(nameLength)(const ENCODING *enc, const char *ptr) {
   const char *start = ptr;
   for (;;) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: ptr += n; break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_NONASCII:
     case BT_NMSTRT:
-#ifdef XML_NS
+#  ifdef XML_NS
     case BT_COLON:
-#endif
+#  endif
     case BT_HEX:
     case BT_DIGIT:
     case BT_NAME:
@@ -1702,9 +1745,8 @@ PREFIX(nameLength)(const ENCODING *enc, const char *ptr)
   }
 }
 
-static const char * PTRFASTCALL
-PREFIX(skipS)(const ENCODING *enc, const char *ptr)
-{
+static const char *PTRFASTCALL
+PREFIX(skipS)(const ENCODING *enc, const char *ptr) {
   for (;;) {
     switch (BYTE_TYPE(enc, ptr)) {
     case BT_LF:
@@ -1719,19 +1761,18 @@ PREFIX(skipS)(const ENCODING *enc, const char *ptr)
 }
 
 static void PTRCALL
-PREFIX(updatePosition)(const ENCODING *enc,
-                       const char *ptr,
-                       const char *end,
-                       POSITION *pos)
-{
+PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end,
+                       POSITION *pos) {
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
-#define LEAD_CASE(n) \
-    case BT_LEAD ## n: \
-      ptr += n; \
-      break;
-    LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
-#undef LEAD_CASE
+#  define LEAD_CASE(n)                                                         \
+  case BT_LEAD##n:                                                             \
+    ptr += n;                                                                  \
+    break;
+      LEAD_CASE(2)
+      LEAD_CASE(3)
+      LEAD_CASE(4)
+#  undef LEAD_CASE
     case BT_LF:
       pos->columnNumber = (XML_Size)-1;
       pos->lineNumber++;
@@ -1752,12 +1793,12 @@ PREFIX(updatePosition)(const ENCODING *enc,
   }
 }
 
-#undef DO_LEAD_CASE
-#undef MULTIBYTE_CASES
-#undef INVALID_CASES
-#undef CHECK_NAME_CASE
-#undef CHECK_NAME_CASES
-#undef CHECK_NMSTRT_CASE
-#undef CHECK_NMSTRT_CASES
+#  undef DO_LEAD_CASE
+#  undef MULTIBYTE_CASES
+#  undef INVALID_CASES
+#  undef CHECK_NAME_CASE
+#  undef CHECK_NAME_CASES
+#  undef CHECK_NMSTRT_CASE
+#  undef CHECK_NMSTRT_CASES
 
 #endif /* XML_TOK_IMPL_C */
diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h
index a6420f48eedc..e925dbc7e2c8 100644
--- a/Modules/expat/xmltok_impl.h
+++ b/Modules/expat/xmltok_impl.h
@@ -31,43 +31,43 @@
 */
 
 enum {
-  BT_NONXML,
-  BT_MALFORM,
-  BT_LT,
-  BT_AMP,
-  BT_RSQB,
-  BT_LEAD2,
-  BT_LEAD3,
-  BT_LEAD4,
-  BT_TRAIL,
-  BT_CR,
-  BT_LF,
-  BT_GT,
-  BT_QUOT,
-  BT_APOS,
-  BT_EQUALS,
-  BT_QUEST,
-  BT_EXCL,
-  BT_SOL,
-  BT_SEMI,
-  BT_NUM,
-  BT_LSQB,
-  BT_S,
-  BT_NMSTRT,
-  BT_COLON,
-  BT_HEX,
-  BT_DIGIT,
-  BT_NAME,
-  BT_MINUS,
-  BT_OTHER, /* known not to be a name or name start character */
+  BT_NONXML,   /* e.g. noncharacter-FFFF */
+  BT_MALFORM,  /* illegal, with regard to encoding */
+  BT_LT,       /* less than = "<" */
+  BT_AMP,      /* ampersand = "&" */
+  BT_RSQB,     /* right square bracket = "[" */
+  BT_LEAD2,    /* lead byte of a 2-byte UTF-8 character */
+  BT_LEAD3,    /* lead byte of a 3-byte UTF-8 character */
+  BT_LEAD4,    /* lead byte of a 4-byte UTF-8 character */
+  BT_TRAIL,    /* trailing unit, e.g. second 16-bit unit of a 4-byte char. */
+  BT_CR,       /* carriage return = "\r" */
+  BT_LF,       /* line feed = "\n" */
+  BT_GT,       /* greater than = ">" */
+  BT_QUOT,     /* quotation character = "\"" */
+  BT_APOS,     /* aposthrophe = "'" */
+  BT_EQUALS,   /* equal sign = "=" */
+  BT_QUEST,    /* question mark = "?" */
+  BT_EXCL,     /* exclamation mark = "!" */
+  BT_SOL,      /* solidus, slash = "/" */
+  BT_SEMI,     /* semicolon = ";" */
+  BT_NUM,      /* number sign = "#" */
+  BT_LSQB,     /* left square bracket = "[" */
+  BT_S,        /* white space, e.g. "\t", " "[, "\r"] */
+  BT_NMSTRT,   /* non-hex name start letter = "G".."Z" + "g".."z" + "_" */
+  BT_COLON,    /* colon = ":" */
+  BT_HEX,      /* hex letter = "A".."F" + "a".."f" */
+  BT_DIGIT,    /* digit = "0".."9" */
+  BT_NAME,     /* dot and middle dot = "." + chr(0xb7) */
+  BT_MINUS,    /* minus = "-" */
+  BT_OTHER,    /* known not to be a name or name start character */
   BT_NONASCII, /* might be a name or name start character */
-  BT_PERCNT,
-  BT_LPAR,
-  BT_RPAR,
-  BT_AST,
-  BT_PLUS,
-  BT_COMMA,
-  BT_VERBAR
+  BT_PERCNT,   /* percent sign = "%" */
+  BT_LPAR,     /* left parenthesis = "(" */
+  BT_RPAR,     /* right parenthesis = "(" */
+  BT_AST,      /* asterisk = "*" */
+  BT_PLUS,     /* plus sign = "+" */
+  BT_COMMA,    /* comma = "," */
+  BT_VERBAR    /* vertical bar = "|" */
 };
 
 #include <stddef.h>
diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c
index 23d31e8e4249..919c74e9f97f 100644
--- a/Modules/expat/xmltok_ns.c
+++ b/Modules/expat/xmltok_ns.c
@@ -33,56 +33,47 @@
 #ifdef XML_TOK_NS_C
 
 const ENCODING *
-NS(XmlGetUtf8InternalEncoding)(void)
-{
+NS(XmlGetUtf8InternalEncoding)(void) {
   return &ns(internal_utf8_encoding).enc;
 }
 
 const ENCODING *
-NS(XmlGetUtf16InternalEncoding)(void)
-{
-#if BYTEORDER == 1234
+NS(XmlGetUtf16InternalEncoding)(void) {
+#  if BYTEORDER == 1234
   return &ns(internal_little2_encoding).enc;
-#elif BYTEORDER == 4321
+#  elif BYTEORDER == 4321
   return &ns(internal_big2_encoding).enc;
-#else
+#  else
   const short n = 1;
-  return (*(const char *)&n
-          ? &ns(internal_little2_encoding).enc
-          : &ns(internal_big2_encoding).enc);
-#endif
+  return (*(const char *)&n ? &ns(internal_little2_encoding).enc
+                            : &ns(internal_big2_encoding).enc);
+#  endif
 }
 
-static const ENCODING * const NS(encodings)[] = {
-  &ns(latin1_encoding).enc,
-  &ns(ascii_encoding).enc,
-  &ns(utf8_encoding).enc,
-  &ns(big2_encoding).enc,
-  &ns(big2_encoding).enc,
-  &ns(little2_encoding).enc,
-  &ns(utf8_encoding).enc /* NO_ENC */
+static const ENCODING *const NS(encodings)[] = {
+    &ns(latin1_encoding).enc, &ns(ascii_encoding).enc,
+    &ns(utf8_encoding).enc,   &ns(big2_encoding).enc,
+    &ns(big2_encoding).enc,   &ns(little2_encoding).enc,
+    &ns(utf8_encoding).enc /* NO_ENC */
 };
 
 static int PTRCALL
 NS(initScanProlog)(const ENCODING *enc, const char *ptr, const char *end,
-                   const char **nextTokPtr)
-{
-  return initScan(NS(encodings), (const INIT_ENCODING *)enc,
-                  XML_PROLOG_STATE, ptr, end, nextTokPtr);
+                   const char **nextTokPtr) {
+  return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_PROLOG_STATE,
+                  ptr, end, nextTokPtr);
 }
 
 static int PTRCALL
 NS(initScanContent)(const ENCODING *enc, const char *ptr, const char *end,
-                    const char **nextTokPtr)
-{
-  return initScan(NS(encodings), (const INIT_ENCODING *)enc,
-                  XML_CONTENT_STATE, ptr, end, nextTokPtr);
+                    const char **nextTokPtr) {
+  return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_CONTENT_STATE,
+                  ptr, end, nextTokPtr);
 }
 
 int
 NS(XmlInitEncoding)(INIT_ENCODING *p, const ENCODING **encPtr,
-                    const char *name)
-{
+                    const char *name) {
   int i = getEncodingIndex(name);
   if (i == UNKNOWN_ENC)
     return 0;
@@ -96,9 +87,8 @@ NS(XmlInitEncoding)(INIT_ENCODING *p, const ENCODING **encPtr,
 }
 
 static const ENCODING *
-NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end)
-{
-#define ENCODING_MAX 128
+NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end) {
+#  define ENCODING_MAX 128
   char buf[ENCODING_MAX];
   char *p = buf;
   int i;
@@ -115,28 +105,14 @@ NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end)
 }
 
 int
-NS(XmlParseXmlDecl)(int isGeneralTextEntity,
-                    const ENCODING *enc,
-                    const char *ptr,
-                    const char *end,
-                    const char **badPtr,
-                    const char **versionPtr,
-                    const char **versionEndPtr,
-                    const char **encodingName,
-                    const ENCODING **encoding,
-                    int *standalone)
-{
-  return doParseXmlDecl(NS(findEncoding),
-                        isGeneralTextEntity,
-                        enc,
-                        ptr,
-                        end,
-                        badPtr,
-                        versionPtr,
-                        versionEndPtr,
-                        encodingName,
-                        encoding,
-                        standalone);
+NS(XmlParseXmlDecl)(int isGeneralTextEntity, const ENCODING *enc,
+                    const char *ptr, const char *end, const char **badPtr,
+                    const char **versionPtr, const char **versionEndPtr,
+                    const char **encodingName, const ENCODING **encoding,
+                    int *standalone) {
+  return doParseXmlDecl(NS(findEncoding), isGeneralTextEntity, enc, ptr, end,
+                        badPtr, versionPtr, versionEndPtr, encodingName,
+                        encoding, standalone);
 }
 
 #endif /* XML_TOK_NS_C */
diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj
index 0db61f208183..639ba479db57 100644
--- a/PCbuild/_elementtree.vcxproj
+++ b/PCbuild/_elementtree.vcxproj
@@ -87,7 +87,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\_elementtree.c" />
-    <ClCompile Include="..\Modules\expat\loadlibrary.c" />
     <ClCompile Include="..\Modules\expat\xmlparse.c" />
     <ClCompile Include="..\Modules\expat\xmlrole.c" />
     <ClCompile Include="..\Modules\expat\xmltok.c" />
diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters
index 4597ee521b33..6acdf35846ab 100644
--- a/PCbuild/_elementtree.vcxproj.filters
+++ b/PCbuild/_elementtree.vcxproj.filters
@@ -33,9 +33,6 @@
     <ClInclude Include="..\Modules\expat\latin1tab.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\expat\loadlibrary.c">
-      <Filter>Header Files</Filter>
-    </ClInclude>
     <ClInclude Include="..\Modules\expat\macconfig.h">
       <Filter>Header Files</Filter>
     </ClInclude>
diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj
index 51ca69e95d70..9a765ff07bde 100644
--- a/PCbuild/pyexpat.vcxproj
+++ b/PCbuild/pyexpat.vcxproj
@@ -68,7 +68,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\pyexpat.c" />
-    <ClCompile Include="..\Modules\expat\loadlibrary.c" />
     <ClCompile Include="..\Modules\expat\xmlparse.c" />
     <ClCompile Include="..\Modules\expat\xmlrole.c" />
     <ClCompile Include="..\Modules\expat\xmltok.c" />
diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters
index cb02847980c6..f8d46026c9c2 100644
--- a/PCbuild/pyexpat.vcxproj.filters
+++ b/PCbuild/pyexpat.vcxproj.filters
@@ -20,9 +20,6 @@
     <ClCompile Include="..\Modules\pyexpat.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\expat\loadlibrary.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Modules\expat\xmlparse.c">
       <Filter>Source Files</Filter>
     </ClCompile>


From webhook-mailer at python.org  Thu Sep 26 01:07:13 2019
From: webhook-mailer at python.org (Benjamin Peterson)
Date: Thu, 26 Sep 2019 05:07:13 -0000
Subject: [Python-checkins] bpo-38174 follow up: Remove loadlibrary.c from
 VS9.0. (GH-16411)
Message-ID: <mailman.99.1569474434.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/90b4e49c98dd0ff948e13dd0618240c2294eb23d
commit: 90b4e49c98dd0ff948e13dd0618240c2294eb23d
branch: 2.7
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-25T22:07:09-07:00
summary:

bpo-38174 follow up: Remove loadlibrary.c from VS9.0. (GH-16411)

files:
M PC/VS9.0/_elementtree.vcproj
M PC/VS9.0/pyexpat.vcproj

diff --git a/PC/VS9.0/_elementtree.vcproj b/PC/VS9.0/_elementtree.vcproj
index 2915eb3eef0e..f89409cc3ed1 100644
--- a/PC/VS9.0/_elementtree.vcproj
+++ b/PC/VS9.0/_elementtree.vcproj
@@ -594,10 +594,6 @@
 				RelativePath="..\..\Modules\_elementtree.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\Modules\expat\loadlibrary.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\Modules\expat\xmlparse.c"
 				>
diff --git a/PC/VS9.0/pyexpat.vcproj b/PC/VS9.0/pyexpat.vcproj
index 7a2474e67cb2..76b698f74028 100644
--- a/PC/VS9.0/pyexpat.vcproj
+++ b/PC/VS9.0/pyexpat.vcproj
@@ -534,10 +534,6 @@
 				RelativePath="..\..\Modules\pyexpat.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\Modules\expat\loadlibrary.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\Modules\expat\xmlparse.c"
 				>


From webhook-mailer at python.org  Thu Sep 26 02:28:44 2019
From: webhook-mailer at python.org (Petr Viktorin)
Date: Thu, 26 Sep 2019 06:28:44 -0000
Subject: [Python-checkins] bpo-38112: Compileall improvements (GH-16012)
Message-ID: <mailman.100.1569479326.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/8e7bb991de7c88583bc6663d8bbc541054ca8dc4
commit: 8e7bb991de7c88583bc6663d8bbc541054ca8dc4
branch: master
author: Lum?r 'Frenzy' Balhar <frenzy.madness at gmail.com>
committer: Petr Viktorin <encukou at gmail.com>
date: 2019-09-26T08:28:26+02:00
summary:

bpo-38112: Compileall improvements (GH-16012)

* Raise the limit of maximum path depth to actual  recursion limit

* Add posibilities to adjust a path compiled in .pyc  file.

Now, you can:
- Strip a part of path from a beggining of path into compiled file
   example "-s /test /test/build/real/test.py" ? "build/real/test.py"
- Append some new path to a beggining of path into compiled file
   example "-p /boo real/test.py" ? "/boo/real/test.py"

You can also use both options in the same time. In that case,
striping is done before appending.

* Add a possibility to specify multiple optimization levels

Each optimization level then leads to separated compiled file.
Use `action='append'` instead of `nargs='+'` for the -o option.
Instead of `-o 0 1 2`, specify `-o 0 -o 1 -o 2`. It's more to type,
but much more explicit.

* Add a symlinks limitation feature

This feature allows us to limit byte-compilation of symbolic
links if they are pointing outside specified dir (build root
for example).

files:
A Misc/NEWS.d/next/Library/2019-09-24-10-55-01.bpo-38112.2EinX9.rst
M Doc/library/compileall.rst
M Lib/compileall.py
M Lib/test/test_compileall.py

diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst
index 9ce5ca819c68..394d60634f1e 100644
--- a/Doc/library/compileall.rst
+++ b/Doc/library/compileall.rst
@@ -52,6 +52,13 @@ compile Python sources.
    cases where the source file does not exist at the time the byte-code file is
    executed.
 
+.. cmdoption:: -s strip_prefix
+.. cmdoption:: -p prepend_prefix
+
+   Remove (``-s``) or append (``-p``) the given prefix of paths
+   recorded in the ``.pyc`` files.
+   Cannot be combined with ``-d``.
+
 .. cmdoption:: -x regex
 
    regex is used to search the full path to each file considered for
@@ -96,6 +103,16 @@ compile Python sources.
    variable is not set, and ``checked-hash`` if the ``SOURCE_DATE_EPOCH``
    environment variable is set.
 
+.. cmdoption:: -o level
+
+   Compile with the given optimization level. May be used multiple times
+   to compile for multiple levels at a time (for example,
+   ``compileall -o 1 -o 2``).
+
+.. cmdoption:: -e dir
+
+   Ignore symlinks pointing outside the given directory.
+
 .. versionchanged:: 3.2
    Added the ``-i``, ``-b`` and ``-h`` options.
 
@@ -107,6 +124,12 @@ compile Python sources.
 .. versionchanged:: 3.7
    Added the ``--invalidation-mode`` option.
 
+.. versionchanged:: 3.9
+   Added the ``-s``, ``-p``, ``-e`` options.
+   Raised the default recursion limit from 10 to
+   :py:func:`sys.getrecursionlimit()`.
+   Added the possibility to specify the ``-o`` option multiple times.
+
 
 There is no command-line option to control the optimization level used by the
 :func:`compile` function, because the Python interpreter itself already
@@ -120,7 +143,7 @@ runtime.
 Public functions
 ----------------
 
-.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None)
+.. function:: compile_dir(dir, maxlevels=sys.getrecursionlimit(), ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, stripdir=None, prependdir=None, limit_sl_dest=None)
 
    Recursively descend the directory tree named by *dir*, compiling all :file:`.py`
    files along the way. Return a true value if all the files compiled successfully,
@@ -166,6 +189,10 @@ Public functions
    :class:`py_compile.PycInvalidationMode` enum and controls how the generated
    pycs are invalidated at runtime.
 
+   The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to
+   the ``-s``, ``-p`` and ``-e`` options described above.
+   They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`.
+
    .. versionchanged:: 3.2
       Added the *legacy* and *optimize* parameter.
 
@@ -191,6 +218,9 @@ Public functions
    .. versionchanged:: 3.8
       Setting *workers* to 0 now chooses the optimal number of cores.
 
+   .. versionchanged:: 3.9
+      Added *stripdir*, *prependdir* and *limit_sl_dest* arguments.
+
 .. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None)
 
    Compile the file with path *fullname*. Return a true value if the file
@@ -223,6 +253,10 @@ Public functions
    :class:`py_compile.PycInvalidationMode` enum and controls how the generated
    pycs are invalidated at runtime.
 
+   The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to
+   the ``-s``, ``-p`` and ``-e`` options described above.
+   They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`.
+
    .. versionadded:: 3.2
 
    .. versionchanged:: 3.5
@@ -238,6 +272,9 @@ Public functions
    .. versionchanged:: 3.7.2
       The *invalidation_mode* parameter's default value is updated to None.
 
+   .. versionchanged:: 3.9
+      Added *stripdir*, *prependdir* and *limit_sl_dest* arguments.
+
 .. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1, invalidation_mode=None)
 
    Byte-compile all the :file:`.py` files found along ``sys.path``. Return a
diff --git a/Lib/compileall.py b/Lib/compileall.py
index 49306d9dabbc..26caf34baec0 100644
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -17,10 +17,13 @@
 import struct
 
 from functools import partial
+from pathlib import Path
+
+RECURSION_LIMIT = sys.getrecursionlimit()
 
 __all__ = ["compile_dir","compile_file","compile_path"]
 
-def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
+def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0):
     if quiet < 2 and isinstance(dir, os.PathLike):
         dir = os.fspath(dir)
     if not quiet:
@@ -36,35 +39,39 @@ def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
         if name == '__pycache__':
             continue
         fullname = os.path.join(dir, name)
-        if ddir is not None:
-            dfile = os.path.join(ddir, name)
-        else:
-            dfile = None
         if not os.path.isdir(fullname):
             yield fullname
         elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
               os.path.isdir(fullname) and not os.path.islink(fullname)):
-            yield from _walk_dir(fullname, ddir=dfile,
-                                 maxlevels=maxlevels - 1, quiet=quiet)
+            yield from _walk_dir(fullname, maxlevels=maxlevels - 1,
+                                 quiet=quiet)
 
-def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
-                quiet=0, legacy=False, optimize=-1, workers=1,
-                invalidation_mode=None):
+def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False,
+                rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
+                invalidation_mode=None, stripdir=None,
+                prependdir=None, limit_sl_dest=None):
     """Byte-compile all modules in the given directory tree.
 
     Arguments (only dir is required):
 
     dir:       the directory to byte-compile
-    maxlevels: maximum recursion level (default 10)
+    maxlevels: maximum recursion level (default `sys.getrecursionlimit()`)
     ddir:      the directory that will be prepended to the path to the
                file as it is compiled into each byte-code file.
     force:     if True, force compilation, even if timestamps are up-to-date
     quiet:     full output with False or 0, errors only with 1,
                no output with 2
     legacy:    if True, produce legacy pyc paths instead of PEP 3147 paths
-    optimize:  optimization level or -1 for level of the interpreter
+    optimize:  int or list of optimization levels or -1 for level of
+               the interpreter. Multiple levels leads to multiple compiled
+               files each with one optimization level.
     workers:   maximum number of parallel workers
     invalidation_mode: how the up-to-dateness of the pyc will be checked
+    stripdir:  part of path to left-strip from source file path
+    prependdir: path to prepend to beggining of original file path, applied
+               after stripdir
+    limit_sl_dest: ignore symlinks if they are pointing outside of
+                   the defined path
     """
     ProcessPoolExecutor = None
     if workers < 0:
@@ -76,8 +83,7 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
             from concurrent.futures import ProcessPoolExecutor
         except ImportError:
             workers = 1
-    files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
-                      ddir=ddir)
+    files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
     success = True
     if workers != 1 and ProcessPoolExecutor is not None:
         # If workers == 0, let ProcessPoolExecutor choose
@@ -88,19 +94,25 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
                                            rx=rx, quiet=quiet,
                                            legacy=legacy,
                                            optimize=optimize,
-                                           invalidation_mode=invalidation_mode),
+                                           invalidation_mode=invalidation_mode,
+                                           stripdir=stripdir,
+                                           prependdir=prependdir,
+                                           limit_sl_dest=limit_sl_dest),
                                    files)
             success = min(results, default=True)
     else:
         for file in files:
             if not compile_file(file, ddir, force, rx, quiet,
-                                legacy, optimize, invalidation_mode):
+                                legacy, optimize, invalidation_mode,
+                                stripdir=stripdir, prependdir=prependdir,
+                                limit_sl_dest=limit_sl_dest):
                 success = False
     return success
 
 def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
                  legacy=False, optimize=-1,
-                 invalidation_mode=None):
+                 invalidation_mode=None, stripdir=None, prependdir=None,
+                 limit_sl_dest=None):
     """Byte-compile one file.
 
     Arguments (only fullname is required):
@@ -112,32 +124,76 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
     quiet:     full output with False or 0, errors only with 1,
                no output with 2
     legacy:    if True, produce legacy pyc paths instead of PEP 3147 paths
-    optimize:  optimization level or -1 for level of the interpreter
+    optimize:  int or list of optimization levels or -1 for level of
+               the interpreter. Multiple levels leads to multiple compiled
+               files each with one optimization level.
     invalidation_mode: how the up-to-dateness of the pyc will be checked
+    stripdir:  part of path to left-strip from source file path
+    prependdir: path to prepend to beggining of original file path, applied
+               after stripdir
+    limit_sl_dest: ignore symlinks if they are pointing outside of
+                   the defined path.
     """
+
+    if ddir is not None and (stripdir is not None or prependdir is not None):
+        raise ValueError(("Destination dir (ddir) cannot be used "
+                          "in combination with stripdir or prependdir"))
+
     success = True
     if quiet < 2 and isinstance(fullname, os.PathLike):
         fullname = os.fspath(fullname)
     name = os.path.basename(fullname)
+
+    dfile = None
+
     if ddir is not None:
         dfile = os.path.join(ddir, name)
-    else:
-        dfile = None
+
+    if stripdir is not None:
+        fullname_parts = fullname.split(os.path.sep)
+        stripdir_parts = stripdir.split(os.path.sep)
+        ddir_parts = list(fullname_parts)
+
+        for spart, opart in zip(stripdir_parts, fullname_parts):
+            if spart == opart:
+                ddir_parts.remove(spart)
+
+        dfile = os.path.join(*ddir_parts)
+
+    if prependdir is not None:
+        if dfile is None:
+            dfile = os.path.join(prependdir, fullname)
+        else:
+            dfile = os.path.join(prependdir, dfile)
+
+    if isinstance(optimize, int):
+        optimize = [optimize]
+
     if rx is not None:
         mo = rx.search(fullname)
         if mo:
             return success
+
+    if limit_sl_dest is not None and os.path.islink(fullname):
+        if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents:
+            return success
+
+    opt_cfiles = {}
+
     if os.path.isfile(fullname):
-        if legacy:
-            cfile = fullname + 'c'
-        else:
-            if optimize >= 0:
-                opt = optimize if optimize >= 1 else ''
-                cfile = importlib.util.cache_from_source(
-                                fullname, optimization=opt)
+        for opt_level in optimize:
+            if legacy:
+                opt_cfiles[opt_level] = fullname + 'c'
             else:
-                cfile = importlib.util.cache_from_source(fullname)
-            cache_dir = os.path.dirname(cfile)
+                if opt_level >= 0:
+                    opt = opt_level if opt_level >= 1 else ''
+                    cfile = (importlib.util.cache_from_source(
+                             fullname, optimization=opt))
+                    opt_cfiles[opt_level] = cfile
+                else:
+                    cfile = importlib.util.cache_from_source(fullname)
+                    opt_cfiles[opt_level] = cfile
+
         head, tail = name[:-3], name[-3:]
         if tail == '.py':
             if not force:
@@ -145,18 +201,22 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
                     mtime = int(os.stat(fullname).st_mtime)
                     expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER,
                                          0, mtime)
-                    with open(cfile, 'rb') as chandle:
-                        actual = chandle.read(12)
-                    if expect == actual:
+                    for cfile in opt_cfiles.values():
+                        with open(cfile, 'rb') as chandle:
+                            actual = chandle.read(12)
+                        if expect != actual:
+                            break
+                    else:
                         return success
                 except OSError:
                     pass
             if not quiet:
                 print('Compiling {!r}...'.format(fullname))
             try:
-                ok = py_compile.compile(fullname, cfile, dfile, True,
-                                        optimize=optimize,
-                                        invalidation_mode=invalidation_mode)
+                for opt_level, cfile in opt_cfiles.items():
+                    ok = py_compile.compile(fullname, cfile, dfile, True,
+                                            optimize=opt_level,
+                                            invalidation_mode=invalidation_mode)
             except py_compile.PyCompileError as err:
                 success = False
                 if quiet >= 2:
@@ -225,7 +285,7 @@ def main():
     parser = argparse.ArgumentParser(
         description='Utilities to support installing Python libraries.')
     parser.add_argument('-l', action='store_const', const=0,
-                        default=10, dest='maxlevels',
+                        default=RECURSION_LIMIT, dest='maxlevels',
                         help="don't recurse into subdirectories")
     parser.add_argument('-r', type=int, dest='recursion',
                         help=('control the maximum recursion level. '
@@ -243,6 +303,20 @@ def main():
                               'compile-time tracebacks and in runtime '
                               'tracebacks in cases where the source file is '
                               'unavailable'))
+    parser.add_argument('-s', metavar='STRIPDIR',  dest='stripdir',
+                        default=None,
+                        help=('part of path to left-strip from path '
+                              'to source file - for example buildroot. '
+                              '`-d` and `-s` options cannot be '
+                              'specified together.'))
+    parser.add_argument('-p', metavar='PREPENDDIR',  dest='prependdir',
+                        default=None,
+                        help=('path to add as prefix to path '
+                              'to source file - for example / to make '
+                              'it absolute when some part is removed '
+                              'by `-s` option. '
+                              '`-d` and `-p` options cannot be '
+                              'specified together.'))
     parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
                         help=('skip files matching the regular expression; '
                               'the regexp is searched for in the full path '
@@ -265,6 +339,12 @@ def main():
                               '"checked-hash" if the SOURCE_DATE_EPOCH '
                               'environment variable is set, and '
                               '"timestamp" otherwise.'))
+    parser.add_argument('-o', action='append', type=int, dest='opt_levels',
+                        help=('Optimization levels to run compilation with.'
+                              'Default is -1 which uses optimization level of'
+                              'Python interpreter itself (specified by -O).'))
+    parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest',
+                        help='Ignore symlinks pointing outsite of the DIR')
 
     args = parser.parse_args()
     compile_dests = args.compile_dest
@@ -273,12 +353,22 @@ def main():
         import re
         args.rx = re.compile(args.rx)
 
+    if args.limit_sl_dest == "":
+        args.limit_sl_dest = None
 
     if args.recursion is not None:
         maxlevels = args.recursion
     else:
         maxlevels = args.maxlevels
 
+    if args.opt_levels is None:
+        args.opt_levels = [-1]
+
+    if args.ddir is not None and (
+        args.stripdir is not None or args.prependdir is not None
+    ):
+        parser.error("-d cannot be used in combination with -s or -p")
+
     # if flist is provided then load it
     if args.flist:
         try:
@@ -303,13 +393,21 @@ def main():
                 if os.path.isfile(dest):
                     if not compile_file(dest, args.ddir, args.force, args.rx,
                                         args.quiet, args.legacy,
-                                        invalidation_mode=invalidation_mode):
+                                        invalidation_mode=invalidation_mode,
+                                        stripdir=args.stripdir,
+                                        prependdir=args.prependdir,
+                                        optimize=args.opt_levels,
+                                        limit_sl_dest=args.limit_sl_dest):
                         success = False
                 else:
                     if not compile_dir(dest, maxlevels, args.ddir,
                                        args.force, args.rx, args.quiet,
                                        args.legacy, workers=args.workers,
-                                       invalidation_mode=invalidation_mode):
+                                       invalidation_mode=invalidation_mode,
+                                       stripdir=args.stripdir,
+                                       prependdir=args.prependdir,
+                                       optimize=args.opt_levels,
+                                       limit_sl_dest=args.limit_sl_dest):
                         success = False
             return success
         else:
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 99d843704fc7..af885c512240 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -41,6 +41,16 @@ def setUp(self):
         os.mkdir(self.subdirectory)
         self.source_path3 = os.path.join(self.subdirectory, '_test3.py')
         shutil.copyfile(self.source_path, self.source_path3)
+        many_directories = [str(number) for number in range(1, 100)]
+        self.long_path = os.path.join(self.directory,
+                                      "long",
+                                      *many_directories)
+        os.makedirs(self.long_path)
+        self.source_path_long = os.path.join(self.long_path, '_test4.py')
+        shutil.copyfile(self.source_path, self.source_path_long)
+        self.bc_path_long = importlib.util.cache_from_source(
+            self.source_path_long
+        )
 
     def tearDown(self):
         shutil.rmtree(self.directory)
@@ -194,6 +204,123 @@ def test_compile_missing_multiprocessing(self, compile_file_mock):
         compileall.compile_dir(self.directory, quiet=True, workers=5)
         self.assertTrue(compile_file_mock.called)
 
+    def text_compile_dir_maxlevels(self):
+        # Test the actual impact of maxlevels attr
+        compileall.compile_dir(os.path.join(self.directory, "long"),
+                               maxlevels=10, quiet=True)
+        self.assertFalse(os.path.isfile(self.bc_path_long))
+        compileall.compile_dir(os.path.join(self.directory, "long"),
+                               maxlevels=110, quiet=True)
+        self.assertTrue(os.path.isfile(self.bc_path_long))
+
+    def test_strip_only(self):
+        fullpath = ["test", "build", "real", "path"]
+        path = os.path.join(self.directory, *fullpath)
+        os.makedirs(path)
+        script = script_helper.make_script(path, "test", "1 / 0")
+        bc = importlib.util.cache_from_source(script)
+        stripdir = os.path.join(self.directory, *fullpath[:2])
+        compileall.compile_dir(path, quiet=True, stripdir=stripdir)
+        rc, out, err = script_helper.assert_python_failure(bc)
+        expected_in = os.path.join(*fullpath[2:])
+        self.assertIn(
+            expected_in,
+            str(err, encoding=sys.getdefaultencoding())
+        )
+        self.assertNotIn(
+            stripdir,
+            str(err, encoding=sys.getdefaultencoding())
+        )
+
+    def test_prepend_only(self):
+        fullpath = ["test", "build", "real", "path"]
+        path = os.path.join(self.directory, *fullpath)
+        os.makedirs(path)
+        script = script_helper.make_script(path, "test", "1 / 0")
+        bc = importlib.util.cache_from_source(script)
+        prependdir = "/foo"
+        compileall.compile_dir(path, quiet=True, prependdir=prependdir)
+        rc, out, err = script_helper.assert_python_failure(bc)
+        expected_in = os.path.join(prependdir, self.directory, *fullpath)
+        self.assertIn(
+            expected_in,
+            str(err, encoding=sys.getdefaultencoding())
+        )
+
+    def test_strip_and_prepend(self):
+        fullpath = ["test", "build", "real", "path"]
+        path = os.path.join(self.directory, *fullpath)
+        os.makedirs(path)
+        script = script_helper.make_script(path, "test", "1 / 0")
+        bc = importlib.util.cache_from_source(script)
+        stripdir = os.path.join(self.directory, *fullpath[:2])
+        prependdir = "/foo"
+        compileall.compile_dir(path, quiet=True,
+                               stripdir=stripdir, prependdir=prependdir)
+        rc, out, err = script_helper.assert_python_failure(bc)
+        expected_in = os.path.join(prependdir, *fullpath[2:])
+        self.assertIn(
+            expected_in,
+            str(err, encoding=sys.getdefaultencoding())
+        )
+        self.assertNotIn(
+            stripdir,
+            str(err, encoding=sys.getdefaultencoding())
+        )
+
+    def test_strip_prepend_and_ddir(self):
+        fullpath = ["test", "build", "real", "path", "ddir"]
+        path = os.path.join(self.directory, *fullpath)
+        os.makedirs(path)
+        script_helper.make_script(path, "test", "1 / 0")
+        with self.assertRaises(ValueError):
+            compileall.compile_dir(path, quiet=True, ddir="/bar",
+                                   stripdir="/foo", prependdir="/bar")
+
+    def test_multiple_optimization_levels(self):
+        script = script_helper.make_script(self.directory,
+                                           "test_optimization",
+                                           "a = 0")
+        bc = []
+        for opt_level in "", 1, 2, 3:
+            bc.append(importlib.util.cache_from_source(script,
+                                                       optimization=opt_level))
+        test_combinations = [[0, 1], [1, 2], [0, 2], [0, 1, 2]]
+        for opt_combination in test_combinations:
+            compileall.compile_file(script, quiet=True,
+                                    optimize=opt_combination)
+            for opt_level in opt_combination:
+                self.assertTrue(os.path.isfile(bc[opt_level]))
+                try:
+                    os.unlink(bc[opt_level])
+                except Exception:
+                    pass
+
+    @support.skip_unless_symlink
+    def test_ignore_symlink_destination(self):
+        # Create folders for allowed files, symlinks and prohibited area
+        allowed_path = os.path.join(self.directory, "test", "dir", "allowed")
+        symlinks_path = os.path.join(self.directory, "test", "dir", "symlinks")
+        prohibited_path = os.path.join(self.directory, "test", "dir", "prohibited")
+        os.makedirs(allowed_path)
+        os.makedirs(symlinks_path)
+        os.makedirs(prohibited_path)
+
+        # Create scripts and symlinks and remember their byte-compiled versions
+        allowed_script = script_helper.make_script(allowed_path, "test_allowed", "a = 0")
+        prohibited_script = script_helper.make_script(prohibited_path, "test_prohibited", "a = 0")
+        allowed_symlink = os.path.join(symlinks_path, "test_allowed.py")
+        prohibited_symlink = os.path.join(symlinks_path, "test_prohibited.py")
+        os.symlink(allowed_script, allowed_symlink)
+        os.symlink(prohibited_script, prohibited_symlink)
+        allowed_bc = importlib.util.cache_from_source(allowed_symlink)
+        prohibited_bc = importlib.util.cache_from_source(prohibited_symlink)
+
+        compileall.compile_dir(symlinks_path, quiet=True, limit_sl_dest=allowed_path)
+
+        self.assertTrue(os.path.isfile(allowed_bc))
+        self.assertFalse(os.path.isfile(prohibited_bc))
+
 
 class CompileallTestsWithSourceEpoch(CompileallTestsBase,
                                      unittest.TestCase,
@@ -436,6 +563,20 @@ def test_recursion_limit(self):
         self.assertCompiled(spamfn)
         self.assertCompiled(eggfn)
 
+    @support.skip_unless_symlink
+    def test_symlink_loop(self):
+        # Currently, compileall ignores symlinks to directories.
+        # If that limitation is ever lifted, it should protect against
+        # recursion in symlink loops.
+        pkg = os.path.join(self.pkgdir, 'spam')
+        script_helper.make_pkg(pkg)
+        os.symlink('.', os.path.join(pkg, 'evil'))
+        os.symlink('.', os.path.join(pkg, 'evil2'))
+        self.assertRunOK('-q', self.pkgdir)
+        self.assertCompiled(os.path.join(
+            self.pkgdir, 'spam', 'evil', 'evil2', '__init__.py'
+        ))
+
     def test_quiet(self):
         noisy = self.assertRunOK(self.pkgdir)
         quiet = self.assertRunOK('-q', self.pkgdir)
@@ -577,6 +718,74 @@ def test_workers_available_cores(self, compile_dir):
             self.assertTrue(compile_dir.called)
             self.assertEqual(compile_dir.call_args[-1]['workers'], 0)
 
+    def test_strip_and_prepend(self):
+        fullpath = ["test", "build", "real", "path"]
+        path = os.path.join(self.directory, *fullpath)
+        os.makedirs(path)
+        script = script_helper.make_script(path, "test", "1 / 0")
+        bc = importlib.util.cache_from_source(script)
+        stripdir = os.path.join(self.directory, *fullpath[:2])
+        prependdir = "/foo"
+        self.assertRunOK("-s", stripdir, "-p", prependdir, path)
+        rc, out, err = script_helper.assert_python_failure(bc)
+        expected_in = os.path.join(prependdir, *fullpath[2:])
+        self.assertIn(
+            expected_in,
+            str(err, encoding=sys.getdefaultencoding())
+        )
+        self.assertNotIn(
+            stripdir,
+            str(err, encoding=sys.getdefaultencoding())
+        )
+
+    def test_multiple_optimization_levels(self):
+        path = os.path.join(self.directory, "optimizations")
+        os.makedirs(path)
+        script = script_helper.make_script(path,
+                                           "test_optimization",
+                                           "a = 0")
+        bc = []
+        for opt_level in "", 1, 2, 3:
+            bc.append(importlib.util.cache_from_source(script,
+                                                       optimization=opt_level))
+        test_combinations = [["0", "1"],
+                             ["1", "2"],
+                             ["0", "2"],
+                             ["0", "1", "2"]]
+        for opt_combination in test_combinations:
+            self.assertRunOK(path, *("-o" + str(n) for n in opt_combination))
+            for opt_level in opt_combination:
+                self.assertTrue(os.path.isfile(bc[int(opt_level)]))
+                try:
+                    os.unlink(bc[opt_level])
+                except Exception:
+                    pass
+
+    @support.skip_unless_symlink
+    def test_ignore_symlink_destination(self):
+        # Create folders for allowed files, symlinks and prohibited area
+        allowed_path = os.path.join(self.directory, "test", "dir", "allowed")
+        symlinks_path = os.path.join(self.directory, "test", "dir", "symlinks")
+        prohibited_path = os.path.join(self.directory, "test", "dir", "prohibited")
+        os.makedirs(allowed_path)
+        os.makedirs(symlinks_path)
+        os.makedirs(prohibited_path)
+
+        # Create scripts and symlinks and remember their byte-compiled versions
+        allowed_script = script_helper.make_script(allowed_path, "test_allowed", "a = 0")
+        prohibited_script = script_helper.make_script(prohibited_path, "test_prohibited", "a = 0")
+        allowed_symlink = os.path.join(symlinks_path, "test_allowed.py")
+        prohibited_symlink = os.path.join(symlinks_path, "test_prohibited.py")
+        os.symlink(allowed_script, allowed_symlink)
+        os.symlink(prohibited_script, prohibited_symlink)
+        allowed_bc = importlib.util.cache_from_source(allowed_symlink)
+        prohibited_bc = importlib.util.cache_from_source(prohibited_symlink)
+
+        self.assertRunOK(symlinks_path, "-e", allowed_path)
+
+        self.assertTrue(os.path.isfile(allowed_bc))
+        self.assertFalse(os.path.isfile(prohibited_bc))
+
 
 class CommandLineTestsWithSourceEpoch(CommandLineTestsBase,
                                        unittest.TestCase,
diff --git a/Misc/NEWS.d/next/Library/2019-09-24-10-55-01.bpo-38112.2EinX9.rst b/Misc/NEWS.d/next/Library/2019-09-24-10-55-01.bpo-38112.2EinX9.rst
new file mode 100644
index 000000000000..ea49898de8d8
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-24-10-55-01.bpo-38112.2EinX9.rst
@@ -0,0 +1,3 @@
+:mod:`compileall` has a higher default recursion limit and new command-line
+arguments for path manipulation, symlinks handling, and multiple
+optimization levels.


From webhook-mailer at python.org  Thu Sep 26 05:22:37 2019
From: webhook-mailer at python.org (Giampaolo Rodola)
Date: Thu, 26 Sep 2019 09:22:37 -0000
Subject: [Python-checkins] bpo-38244: [Documentation] Fix unreliable link to
 sever in ftplib.rst (#16319)
Message-ID: <mailman.101.1569489758.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/a7414571a113815033fdc9f7abb3f5a24e44c472
commit: a7414571a113815033fdc9f7abb3f5a24e44c472
branch: master
author: Prateek Nayak <45075669+Kriyszig at users.noreply.github.com>
committer: Giampaolo Rodola <g.rodola at gmail.com>
date: 2019-09-26T17:22:31+08:00
summary:

bpo-38244: [Documentation] Fix unreliable link to sever in ftplib.rst (#16319)

files:
M Doc/library/ftplib.rst

diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst
index db0212367340..79a0286fb544 100644
--- a/Doc/library/ftplib.rst
+++ b/Doc/library/ftplib.rst
@@ -22,7 +22,7 @@ as mirroring other FTP servers.  It is also used by the module
 Here's a sample session using the :mod:`ftplib` module::
 
    >>> from ftplib import FTP
-   >>> ftp = FTP('ftp.debian.org')     # connect to host, default port
+   >>> ftp = FTP('ftp.us.debian.org')  # connect to host, default port
    >>> ftp.login()                     # user anonymous, passwd anonymous@
    '230 Login successful.'
    >>> ftp.cwd('debian')               # change into "debian" directory


From webhook-mailer at python.org  Thu Sep 26 05:54:06 2019
From: webhook-mailer at python.org (Petr Viktorin)
Date: Thu, 26 Sep 2019 09:54:06 -0000
Subject: [Python-checkins] bpo-38112: compileall: Skip long path path on
 Windows if the path can't be created (GH-16414)
Message-ID: <mailman.102.1569491648.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/4267c989e7fc6cd528e8a1d04a07fac5cca85ec7
commit: 4267c989e7fc6cd528e8a1d04a07fac5cca85ec7
branch: master
author: Petr Viktorin <encukou at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-26T11:53:51+02:00
summary:

bpo-38112: compileall: Skip long path path on Windows if the path can't be created (GH-16414)

This avoids the buildbot failure on Windows:
```
FileNotFoundError: [WinError 206] The filename or extension is too long: 'd:\\temp\\tmp5r3z438t\\long\\1\\2\\3\\4\\5\\6\\7\\8\\9\\10\\11\\12\\13\\14\\15\\16\\17\\18\\19\\20\\21\\22\\23\\24\\25\\26\\27\\28\\29\\30\\31\\32\\33\\34\\35\\36\\37\\38\\39\\40\\41\\42\\43\\44\\45\\46\\47\\48\\49\\50\\51\\52\\53\\54\\55\\56\\57\\58\\59\\60\\61\\62\\63\\64\\65\\66\\67\\68\\69\\70\\71\\72\\73\\74\\75\\76\\77\\78'
```
Creates a path that's long but avoids OS restrictions.

https://bugs.python.org/issue38112

files:
M Lib/test/test_compileall.py

diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index af885c512240..205457f3b480 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -11,6 +11,7 @@
 import time
 import unittest
 import io
+import errno
 
 from unittest import mock, skipUnless
 try:
@@ -41,20 +42,57 @@ def setUp(self):
         os.mkdir(self.subdirectory)
         self.source_path3 = os.path.join(self.subdirectory, '_test3.py')
         shutil.copyfile(self.source_path, self.source_path3)
-        many_directories = [str(number) for number in range(1, 100)]
-        self.long_path = os.path.join(self.directory,
-                                      "long",
-                                      *many_directories)
-        os.makedirs(self.long_path)
-        self.source_path_long = os.path.join(self.long_path, '_test4.py')
-        shutil.copyfile(self.source_path, self.source_path_long)
-        self.bc_path_long = importlib.util.cache_from_source(
-            self.source_path_long
-        )
 
     def tearDown(self):
         shutil.rmtree(self.directory)
 
+    def create_long_path(self):
+        long_path = os.path.join(self.directory, "long")
+
+        # Create a long path, 10 directories at a time.
+        # It will be 100 directories deep, or shorter if the OS limits it.
+        for i in range(10):
+            longer_path = os.path.join(
+                long_path, *(f"long_directory_{i}_{j}" for j in range(10))
+            )
+
+            # Check if we can open __pycache__/*.pyc.
+            # Also, put in the source file that we want to compile
+            longer_source = os.path.join(longer_path, '_test_long.py')
+            longer_cache = importlib.util.cache_from_source(longer_source)
+            try:
+                os.makedirs(longer_path)
+                shutil.copyfile(self.source_path, longer_source)
+                os.makedirs(os.path.dirname(longer_cache))
+                # Make sure we can write to the cache
+                with open(longer_cache, 'w'):
+                    pass
+            except FileNotFoundError:
+                # On Windows, a  FileNotFoundError("The filename or extension
+                # is too long") is raised for long paths
+                if sys.platform == "win32":
+                    break
+                else:
+                    raise
+            except OSError as exc:
+                if exc.errno == errno.ENAMETOOLONG:
+                    break
+                else:
+                    raise
+
+            # Remove the __pycache__
+            shutil.rmtree(os.path.dirname(longer_cache))
+
+            long_path = longer_path
+            long_source = longer_source
+            long_cache = longer_cache
+
+        if i < 2:
+            raise ValueError('Path limit is too short')
+
+        self.source_path_long = long_source
+        self.bc_path_long = long_cache
+
     def add_bad_source_file(self):
         self.bad_source_path = os.path.join(self.directory, '_test_bad.py')
         with open(self.bad_source_path, 'w') as file:
@@ -204,13 +242,14 @@ def test_compile_missing_multiprocessing(self, compile_file_mock):
         compileall.compile_dir(self.directory, quiet=True, workers=5)
         self.assertTrue(compile_file_mock.called)
 
-    def text_compile_dir_maxlevels(self):
+    def test_compile_dir_maxlevels(self):
         # Test the actual impact of maxlevels attr
+        self.create_long_path()
         compileall.compile_dir(os.path.join(self.directory, "long"),
                                maxlevels=10, quiet=True)
         self.assertFalse(os.path.isfile(self.bc_path_long))
         compileall.compile_dir(os.path.join(self.directory, "long"),
-                               maxlevels=110, quiet=True)
+                               quiet=True)
         self.assertTrue(os.path.isfile(self.bc_path_long))
 
     def test_strip_only(self):


From webhook-mailer at python.org  Thu Sep 26 06:15:13 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Thu, 26 Sep 2019 10:15:13 -0000
Subject: [Python-checkins] bpo-38223: Reorganize test_shutil. (GH-16281)
Message-ID: <mailman.103.1569492914.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/4f2eac04e4fee4d7d188ad2eeb0e610e3d7254bb
commit: 4f2eac04e4fee4d7d188ad2eeb0e610e3d7254bb
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-26T13:15:08+03:00
summary:

bpo-38223: Reorganize test_shutil. (GH-16281)

* Group tests for specific functions and groups of related functions
into separate classes.
* Clean up creating and cleaning up temporary directories.
* Simplify and make more robust monkey patching of shutil.open.

files:
M Lib/test/test_shutil.py

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 636e3bd97955..ab0f96d25135 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -159,18 +159,8 @@ def _maxdataOK():
     else:
         return True
 
-class TestShutil(unittest.TestCase):
-
-    def setUp(self):
-        super(TestShutil, self).setUp()
-        self.tempdirs = []
-
-    def tearDown(self):
-        super(TestShutil, self).tearDown()
-        while self.tempdirs:
-            d = self.tempdirs.pop()
-            shutil.rmtree(d, os.name in ('nt', 'cygwin'))
 
+class BaseTest:
 
     def mkdtemp(self):
         """Create a temporary directory that will be cleaned up.
@@ -178,9 +168,12 @@ def mkdtemp(self):
         Returns the path of the directory.
         """
         d = tempfile.mkdtemp()
-        self.tempdirs.append(d)
+        self.addCleanup(support.rmtree, d)
         return d
 
+
+class TestRmTree(BaseTest, unittest.TestCase):
+
     def test_rmtree_works_on_bytes(self):
         tmp = self.mkdtemp()
         victim = os.path.join(tmp, 'killme')
@@ -382,141 +375,500 @@ def raiser(fn, *args, **kwargs):
         finally:
             os.lstat = orig_lstat
 
-    @support.skip_unless_symlink
-    def test_copymode_follow_symlinks(self):
-        tmp_dir = self.mkdtemp()
-        src = os.path.join(tmp_dir, 'foo')
-        dst = os.path.join(tmp_dir, 'bar')
-        src_link = os.path.join(tmp_dir, 'baz')
-        dst_link = os.path.join(tmp_dir, 'quux')
-        write_file(src, 'foo')
-        write_file(dst, 'foo')
-        os.symlink(src, src_link)
-        os.symlink(dst, dst_link)
-        os.chmod(src, stat.S_IRWXU|stat.S_IRWXG)
-        # file to file
-        os.chmod(dst, stat.S_IRWXO)
-        self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
-        shutil.copymode(src, dst)
-        self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
-        # On Windows, os.chmod does not follow symlinks (issue #15411)
-        if os.name != 'nt':
-            # follow src link
-            os.chmod(dst, stat.S_IRWXO)
-            shutil.copymode(src_link, dst)
-            self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
-            # follow dst link
-            os.chmod(dst, stat.S_IRWXO)
-            shutil.copymode(src, dst_link)
-            self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
-            # follow both links
-            os.chmod(dst, stat.S_IRWXO)
-            shutil.copymode(src_link, dst_link)
-            self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+    def test_rmtree_uses_safe_fd_version_if_available(self):
+        _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <=
+                             os.supports_dir_fd and
+                             os.listdir in os.supports_fd and
+                             os.stat in os.supports_follow_symlinks)
+        if _use_fd_functions:
+            self.assertTrue(shutil._use_fd_functions)
+            self.assertTrue(shutil.rmtree.avoids_symlink_attacks)
+            tmp_dir = self.mkdtemp()
+            d = os.path.join(tmp_dir, 'a')
+            os.mkdir(d)
+            try:
+                real_rmtree = shutil._rmtree_safe_fd
+                class Called(Exception): pass
+                def _raiser(*args, **kwargs):
+                    raise Called
+                shutil._rmtree_safe_fd = _raiser
+                self.assertRaises(Called, shutil.rmtree, d)
+            finally:
+                shutil._rmtree_safe_fd = real_rmtree
+        else:
+            self.assertFalse(shutil._use_fd_functions)
+            self.assertFalse(shutil.rmtree.avoids_symlink_attacks)
 
-    @unittest.skipUnless(hasattr(os, 'lchmod'), 'requires os.lchmod')
-    @support.skip_unless_symlink
-    def test_copymode_symlink_to_symlink(self):
-        tmp_dir = self.mkdtemp()
-        src = os.path.join(tmp_dir, 'foo')
-        dst = os.path.join(tmp_dir, 'bar')
-        src_link = os.path.join(tmp_dir, 'baz')
-        dst_link = os.path.join(tmp_dir, 'quux')
-        write_file(src, 'foo')
-        write_file(dst, 'foo')
-        os.symlink(src, src_link)
-        os.symlink(dst, dst_link)
-        os.chmod(src, stat.S_IRWXU|stat.S_IRWXG)
-        os.chmod(dst, stat.S_IRWXU)
-        os.lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG)
-        # link to link
-        os.lchmod(dst_link, stat.S_IRWXO)
-        shutil.copymode(src_link, dst_link, follow_symlinks=False)
-        self.assertEqual(os.lstat(src_link).st_mode,
-                         os.lstat(dst_link).st_mode)
-        self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
-        # src link - use chmod
-        os.lchmod(dst_link, stat.S_IRWXO)
-        shutil.copymode(src_link, dst, follow_symlinks=False)
-        self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
-        # dst link - use chmod
-        os.lchmod(dst_link, stat.S_IRWXO)
-        shutil.copymode(src, dst_link, follow_symlinks=False)
-        self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+    def test_rmtree_dont_delete_file(self):
+        # When called on a file instead of a directory, don't delete it.
+        handle, path = tempfile.mkstemp()
+        os.close(handle)
+        self.assertRaises(NotADirectoryError, shutil.rmtree, path)
+        os.remove(path)
 
-    @unittest.skipIf(hasattr(os, 'lchmod'), 'requires os.lchmod to be missing')
     @support.skip_unless_symlink
-    def test_copymode_symlink_to_symlink_wo_lchmod(self):
-        tmp_dir = self.mkdtemp()
-        src = os.path.join(tmp_dir, 'foo')
-        dst = os.path.join(tmp_dir, 'bar')
-        src_link = os.path.join(tmp_dir, 'baz')
-        dst_link = os.path.join(tmp_dir, 'quux')
-        write_file(src, 'foo')
-        write_file(dst, 'foo')
-        os.symlink(src, src_link)
-        os.symlink(dst, dst_link)
-        shutil.copymode(src_link, dst_link, follow_symlinks=False)  # silent fail
+    def test_rmtree_on_symlink(self):
+        # bug 1669.
+        os.mkdir(TESTFN)
+        try:
+            src = os.path.join(TESTFN, 'cheese')
+            dst = os.path.join(TESTFN, 'shop')
+            os.mkdir(src)
+            os.symlink(src, dst)
+            self.assertRaises(OSError, shutil.rmtree, dst)
+            shutil.rmtree(dst, ignore_errors=True)
+        finally:
+            shutil.rmtree(TESTFN, ignore_errors=True)
+
+    @unittest.skipUnless(_winapi, 'only relevant on Windows')
+    def test_rmtree_on_junction(self):
+        os.mkdir(TESTFN)
+        try:
+            src = os.path.join(TESTFN, 'cheese')
+            dst = os.path.join(TESTFN, 'shop')
+            os.mkdir(src)
+            open(os.path.join(src, 'spam'), 'wb').close()
+            _winapi.CreateJunction(src, dst)
+            self.assertRaises(OSError, shutil.rmtree, dst)
+            shutil.rmtree(dst, ignore_errors=True)
+        finally:
+            shutil.rmtree(TESTFN, ignore_errors=True)
+
+
+class TestCopyTree(BaseTest, unittest.TestCase):
+
+    def test_copytree_simple(self):
+        src_dir = tempfile.mkdtemp()
+        dst_dir = os.path.join(tempfile.mkdtemp(), 'destination')
+        self.addCleanup(shutil.rmtree, src_dir)
+        self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir))
+        write_file((src_dir, 'test.txt'), '123')
+        os.mkdir(os.path.join(src_dir, 'test_dir'))
+        write_file((src_dir, 'test_dir', 'test.txt'), '456')
+
+        shutil.copytree(src_dir, dst_dir)
+        self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt')))
+        self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir')))
+        self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir',
+                                                    'test.txt')))
+        actual = read_file((dst_dir, 'test.txt'))
+        self.assertEqual(actual, '123')
+        actual = read_file((dst_dir, 'test_dir', 'test.txt'))
+        self.assertEqual(actual, '456')
+
+    def test_copytree_dirs_exist_ok(self):
+        src_dir = tempfile.mkdtemp()
+        dst_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, src_dir)
+        self.addCleanup(shutil.rmtree, dst_dir)
+
+        write_file((src_dir, 'nonexisting.txt'), '123')
+        os.mkdir(os.path.join(src_dir, 'existing_dir'))
+        os.mkdir(os.path.join(dst_dir, 'existing_dir'))
+        write_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced')
+        write_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced')
+
+        shutil.copytree(src_dir, dst_dir, dirs_exist_ok=True)
+        self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'nonexisting.txt')))
+        self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'existing_dir')))
+        self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'existing_dir',
+                                                    'existing.txt')))
+        actual = read_file((dst_dir, 'nonexisting.txt'))
+        self.assertEqual(actual, '123')
+        actual = read_file((dst_dir, 'existing_dir', 'existing.txt'))
+        self.assertEqual(actual, 'has been replaced')
+
+        with self.assertRaises(FileExistsError):
+            shutil.copytree(src_dir, dst_dir, dirs_exist_ok=False)
 
     @support.skip_unless_symlink
-    def test_copystat_symlinks(self):
+    def test_copytree_symlinks(self):
         tmp_dir = self.mkdtemp()
-        src = os.path.join(tmp_dir, 'foo')
-        dst = os.path.join(tmp_dir, 'bar')
-        src_link = os.path.join(tmp_dir, 'baz')
-        dst_link = os.path.join(tmp_dir, 'qux')
-        write_file(src, 'foo')
-        src_stat = os.stat(src)
-        os.utime(src, (src_stat.st_atime,
-                       src_stat.st_mtime - 42.0))  # ensure different mtimes
-        write_file(dst, 'bar')
-        self.assertNotEqual(os.stat(src).st_mtime, os.stat(dst).st_mtime)
-        os.symlink(src, src_link)
-        os.symlink(dst, dst_link)
+        src_dir = os.path.join(tmp_dir, 'src')
+        dst_dir = os.path.join(tmp_dir, 'dst')
+        sub_dir = os.path.join(src_dir, 'sub')
+        os.mkdir(src_dir)
+        os.mkdir(sub_dir)
+        write_file((src_dir, 'file.txt'), 'foo')
+        src_link = os.path.join(sub_dir, 'link')
+        dst_link = os.path.join(dst_dir, 'sub/link')
+        os.symlink(os.path.join(src_dir, 'file.txt'),
+                   src_link)
         if hasattr(os, 'lchmod'):
-            os.lchmod(src_link, stat.S_IRWXO)
+            os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO)
         if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'):
             os.lchflags(src_link, stat.UF_NODUMP)
-        src_link_stat = os.lstat(src_link)
-        # follow
-        if hasattr(os, 'lchmod'):
-            shutil.copystat(src_link, dst_link, follow_symlinks=True)
-            self.assertNotEqual(src_link_stat.st_mode, os.stat(dst).st_mode)
-        # don't follow
-        shutil.copystat(src_link, dst_link, follow_symlinks=False)
-        dst_link_stat = os.lstat(dst_link)
-        if os.utime in os.supports_follow_symlinks:
-            for attr in 'st_atime', 'st_mtime':
-                # The modification times may be truncated in the new file.
-                self.assertLessEqual(getattr(src_link_stat, attr),
-                                     getattr(dst_link_stat, attr) + 1)
+        src_stat = os.lstat(src_link)
+        shutil.copytree(src_dir, dst_dir, symlinks=True)
+        self.assertTrue(os.path.islink(os.path.join(dst_dir, 'sub', 'link')))
+        actual = os.readlink(os.path.join(dst_dir, 'sub', 'link'))
+        # Bad practice to blindly strip the prefix as it may be required to
+        # correctly refer to the file, but we're only comparing paths here.
+        if os.name == 'nt' and actual.startswith('\\\\?\\'):
+            actual = actual[4:]
+        self.assertEqual(actual, os.path.join(src_dir, 'file.txt'))
+        dst_stat = os.lstat(dst_link)
         if hasattr(os, 'lchmod'):
-            self.assertEqual(src_link_stat.st_mode, dst_link_stat.st_mode)
-        if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'):
-            self.assertEqual(src_link_stat.st_flags, dst_link_stat.st_flags)
-        # tell to follow but dst is not a link
-        shutil.copystat(src_link, dst, follow_symlinks=False)
-        self.assertTrue(abs(os.stat(src).st_mtime - os.stat(dst).st_mtime) <
-                        00000.1)
-
-    @unittest.skipUnless(hasattr(os, 'chflags') and
-                         hasattr(errno, 'EOPNOTSUPP') and
-                         hasattr(errno, 'ENOTSUP'),
-                         "requires os.chflags, EOPNOTSUPP & ENOTSUP")
-    def test_copystat_handles_harmless_chflags_errors(self):
-        tmpdir = self.mkdtemp()
-        file1 = os.path.join(tmpdir, 'file1')
-        file2 = os.path.join(tmpdir, 'file2')
-        write_file(file1, 'xxx')
-        write_file(file2, 'xxx')
-
-        def make_chflags_raiser(err):
-            ex = OSError()
+            self.assertEqual(dst_stat.st_mode, src_stat.st_mode)
+        if hasattr(os, 'lchflags'):
+            self.assertEqual(dst_stat.st_flags, src_stat.st_flags)
 
-            def _chflags_raiser(path, flags, *, follow_symlinks=True):
-                ex.errno = err
-                raise ex
-            return _chflags_raiser
+    def test_copytree_with_exclude(self):
+        # creating data
+        join = os.path.join
+        exists = os.path.exists
+        src_dir = tempfile.mkdtemp()
+        try:
+            dst_dir = join(tempfile.mkdtemp(), 'destination')
+            write_file((src_dir, 'test.txt'), '123')
+            write_file((src_dir, 'test.tmp'), '123')
+            os.mkdir(join(src_dir, 'test_dir'))
+            write_file((src_dir, 'test_dir', 'test.txt'), '456')
+            os.mkdir(join(src_dir, 'test_dir2'))
+            write_file((src_dir, 'test_dir2', 'test.txt'), '456')
+            os.mkdir(join(src_dir, 'test_dir2', 'subdir'))
+            os.mkdir(join(src_dir, 'test_dir2', 'subdir2'))
+            write_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456')
+            write_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456')
+
+            # testing glob-like patterns
+            try:
+                patterns = shutil.ignore_patterns('*.tmp', 'test_dir2')
+                shutil.copytree(src_dir, dst_dir, ignore=patterns)
+                # checking the result: some elements should not be copied
+                self.assertTrue(exists(join(dst_dir, 'test.txt')))
+                self.assertFalse(exists(join(dst_dir, 'test.tmp')))
+                self.assertFalse(exists(join(dst_dir, 'test_dir2')))
+            finally:
+                shutil.rmtree(dst_dir)
+            try:
+                patterns = shutil.ignore_patterns('*.tmp', 'subdir*')
+                shutil.copytree(src_dir, dst_dir, ignore=patterns)
+                # checking the result: some elements should not be copied
+                self.assertFalse(exists(join(dst_dir, 'test.tmp')))
+                self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2')))
+                self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir')))
+            finally:
+                shutil.rmtree(dst_dir)
+
+            # testing callable-style
+            try:
+                def _filter(src, names):
+                    res = []
+                    for name in names:
+                        path = os.path.join(src, name)
+
+                        if (os.path.isdir(path) and
+                            path.split()[-1] == 'subdir'):
+                            res.append(name)
+                        elif os.path.splitext(path)[-1] in ('.py'):
+                            res.append(name)
+                    return res
+
+                shutil.copytree(src_dir, dst_dir, ignore=_filter)
+
+                # checking the result: some elements should not be copied
+                self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2',
+                                             'test.py')))
+                self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir')))
+
+            finally:
+                shutil.rmtree(dst_dir)
+        finally:
+            shutil.rmtree(src_dir)
+            shutil.rmtree(os.path.dirname(dst_dir))
+
+    def test_copytree_retains_permissions(self):
+        tmp_dir = tempfile.mkdtemp()
+        src_dir = os.path.join(tmp_dir, 'source')
+        os.mkdir(src_dir)
+        dst_dir = os.path.join(tmp_dir, 'destination')
+        self.addCleanup(shutil.rmtree, tmp_dir)
+
+        os.chmod(src_dir, 0o777)
+        write_file((src_dir, 'permissive.txt'), '123')
+        os.chmod(os.path.join(src_dir, 'permissive.txt'), 0o777)
+        write_file((src_dir, 'restrictive.txt'), '456')
+        os.chmod(os.path.join(src_dir, 'restrictive.txt'), 0o600)
+        restrictive_subdir = tempfile.mkdtemp(dir=src_dir)
+        os.chmod(restrictive_subdir, 0o600)
+
+        shutil.copytree(src_dir, dst_dir)
+        self.assertEqual(os.stat(src_dir).st_mode, os.stat(dst_dir).st_mode)
+        self.assertEqual(os.stat(os.path.join(src_dir, 'permissive.txt')).st_mode,
+                          os.stat(os.path.join(dst_dir, 'permissive.txt')).st_mode)
+        self.assertEqual(os.stat(os.path.join(src_dir, 'restrictive.txt')).st_mode,
+                          os.stat(os.path.join(dst_dir, 'restrictive.txt')).st_mode)
+        restrictive_subdir_dst = os.path.join(dst_dir,
+                                              os.path.split(restrictive_subdir)[1])
+        self.assertEqual(os.stat(restrictive_subdir).st_mode,
+                          os.stat(restrictive_subdir_dst).st_mode)
+
+    @unittest.mock.patch('os.chmod')
+    def test_copytree_winerror(self, mock_patch):
+        # When copying to VFAT, copystat() raises OSError. On Windows, the
+        # exception object has a meaningful 'winerror' attribute, but not
+        # on other operating systems. Do not assume 'winerror' is set.
+        src_dir = tempfile.mkdtemp()
+        dst_dir = os.path.join(tempfile.mkdtemp(), 'destination')
+        self.addCleanup(shutil.rmtree, src_dir)
+        self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir))
+
+        mock_patch.side_effect = PermissionError('ka-boom')
+        with self.assertRaises(shutil.Error):
+            shutil.copytree(src_dir, dst_dir)
+
+    def test_copytree_custom_copy_function(self):
+        # See: https://bugs.python.org/issue35648
+        def custom_cpfun(a, b):
+            flag.append(None)
+            self.assertIsInstance(a, str)
+            self.assertIsInstance(b, str)
+            self.assertEqual(a, os.path.join(src, 'foo'))
+            self.assertEqual(b, os.path.join(dst, 'foo'))
+
+        flag = []
+        src = tempfile.mkdtemp()
+        self.addCleanup(support.rmtree, src)
+        dst = tempfile.mktemp()
+        self.addCleanup(support.rmtree, dst)
+        with open(os.path.join(src, 'foo'), 'w') as f:
+            f.close()
+        shutil.copytree(src, dst, copy_function=custom_cpfun)
+        self.assertEqual(len(flag), 1)
+
+    # Issue #3002: copyfile and copytree block indefinitely on named pipes
+    @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()')
+    @support.skip_unless_symlink
+    def test_copytree_named_pipe(self):
+        os.mkdir(TESTFN)
+        try:
+            subdir = os.path.join(TESTFN, "subdir")
+            os.mkdir(subdir)
+            pipe = os.path.join(subdir, "mypipe")
+            try:
+                os.mkfifo(pipe)
+            except PermissionError as e:
+                self.skipTest('os.mkfifo(): %s' % e)
+            try:
+                shutil.copytree(TESTFN, TESTFN2)
+            except shutil.Error as e:
+                errors = e.args[0]
+                self.assertEqual(len(errors), 1)
+                src, dst, error_msg = errors[0]
+                self.assertEqual("`%s` is a named pipe" % pipe, error_msg)
+            else:
+                self.fail("shutil.Error should have been raised")
+        finally:
+            shutil.rmtree(TESTFN, ignore_errors=True)
+            shutil.rmtree(TESTFN2, ignore_errors=True)
+
+    def test_copytree_special_func(self):
+        src_dir = self.mkdtemp()
+        dst_dir = os.path.join(self.mkdtemp(), 'destination')
+        write_file((src_dir, 'test.txt'), '123')
+        os.mkdir(os.path.join(src_dir, 'test_dir'))
+        write_file((src_dir, 'test_dir', 'test.txt'), '456')
+
+        copied = []
+        def _copy(src, dst):
+            copied.append((src, dst))
+
+        shutil.copytree(src_dir, dst_dir, copy_function=_copy)
+        self.assertEqual(len(copied), 2)
+
+    @support.skip_unless_symlink
+    def test_copytree_dangling_symlinks(self):
+        # a dangling symlink raises an error at the end
+        src_dir = self.mkdtemp()
+        dst_dir = os.path.join(self.mkdtemp(), 'destination')
+        os.symlink('IDONTEXIST', os.path.join(src_dir, 'test.txt'))
+        os.mkdir(os.path.join(src_dir, 'test_dir'))
+        write_file((src_dir, 'test_dir', 'test.txt'), '456')
+        self.assertRaises(Error, shutil.copytree, src_dir, dst_dir)
+
+        # a dangling symlink is ignored with the proper flag
+        dst_dir = os.path.join(self.mkdtemp(), 'destination2')
+        shutil.copytree(src_dir, dst_dir, ignore_dangling_symlinks=True)
+        self.assertNotIn('test.txt', os.listdir(dst_dir))
+
+        # a dangling symlink is copied if symlinks=True
+        dst_dir = os.path.join(self.mkdtemp(), 'destination3')
+        shutil.copytree(src_dir, dst_dir, symlinks=True)
+        self.assertIn('test.txt', os.listdir(dst_dir))
+
+    @support.skip_unless_symlink
+    def test_copytree_symlink_dir(self):
+        src_dir = self.mkdtemp()
+        dst_dir = os.path.join(self.mkdtemp(), 'destination')
+        os.mkdir(os.path.join(src_dir, 'real_dir'))
+        with open(os.path.join(src_dir, 'real_dir', 'test.txt'), 'w'):
+            pass
+        os.symlink(os.path.join(src_dir, 'real_dir'),
+                   os.path.join(src_dir, 'link_to_dir'),
+                   target_is_directory=True)
+
+        shutil.copytree(src_dir, dst_dir, symlinks=False)
+        self.assertFalse(os.path.islink(os.path.join(dst_dir, 'link_to_dir')))
+        self.assertIn('test.txt', os.listdir(os.path.join(dst_dir, 'link_to_dir')))
+
+        dst_dir = os.path.join(self.mkdtemp(), 'destination2')
+        shutil.copytree(src_dir, dst_dir, symlinks=True)
+        self.assertTrue(os.path.islink(os.path.join(dst_dir, 'link_to_dir')))
+        self.assertIn('test.txt', os.listdir(os.path.join(dst_dir, 'link_to_dir')))
+
+    def test_copytree_return_value(self):
+        # copytree returns its destination path.
+        src_dir = self.mkdtemp()
+        dst_dir = src_dir + "dest"
+        self.addCleanup(shutil.rmtree, dst_dir, True)
+        src = os.path.join(src_dir, 'foo')
+        write_file(src, 'foo')
+        rv = shutil.copytree(src_dir, dst_dir)
+        self.assertEqual(['foo'], os.listdir(rv))
+
+
+class TestCopy(BaseTest, unittest.TestCase):
+
+    ### shutil.copymode
+
+    @support.skip_unless_symlink
+    def test_copymode_follow_symlinks(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        dst = os.path.join(tmp_dir, 'bar')
+        src_link = os.path.join(tmp_dir, 'baz')
+        dst_link = os.path.join(tmp_dir, 'quux')
+        write_file(src, 'foo')
+        write_file(dst, 'foo')
+        os.symlink(src, src_link)
+        os.symlink(dst, dst_link)
+        os.chmod(src, stat.S_IRWXU|stat.S_IRWXG)
+        # file to file
+        os.chmod(dst, stat.S_IRWXO)
+        self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+        shutil.copymode(src, dst)
+        self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+        # On Windows, os.chmod does not follow symlinks (issue #15411)
+        if os.name != 'nt':
+            # follow src link
+            os.chmod(dst, stat.S_IRWXO)
+            shutil.copymode(src_link, dst)
+            self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+            # follow dst link
+            os.chmod(dst, stat.S_IRWXO)
+            shutil.copymode(src, dst_link)
+            self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+            # follow both links
+            os.chmod(dst, stat.S_IRWXO)
+            shutil.copymode(src_link, dst_link)
+            self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+
+    @unittest.skipUnless(hasattr(os, 'lchmod'), 'requires os.lchmod')
+    @support.skip_unless_symlink
+    def test_copymode_symlink_to_symlink(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        dst = os.path.join(tmp_dir, 'bar')
+        src_link = os.path.join(tmp_dir, 'baz')
+        dst_link = os.path.join(tmp_dir, 'quux')
+        write_file(src, 'foo')
+        write_file(dst, 'foo')
+        os.symlink(src, src_link)
+        os.symlink(dst, dst_link)
+        os.chmod(src, stat.S_IRWXU|stat.S_IRWXG)
+        os.chmod(dst, stat.S_IRWXU)
+        os.lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG)
+        # link to link
+        os.lchmod(dst_link, stat.S_IRWXO)
+        shutil.copymode(src_link, dst_link, follow_symlinks=False)
+        self.assertEqual(os.lstat(src_link).st_mode,
+                         os.lstat(dst_link).st_mode)
+        self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+        # src link - use chmod
+        os.lchmod(dst_link, stat.S_IRWXO)
+        shutil.copymode(src_link, dst, follow_symlinks=False)
+        self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+        # dst link - use chmod
+        os.lchmod(dst_link, stat.S_IRWXO)
+        shutil.copymode(src, dst_link, follow_symlinks=False)
+        self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+
+    @unittest.skipIf(hasattr(os, 'lchmod'), 'requires os.lchmod to be missing')
+    @support.skip_unless_symlink
+    def test_copymode_symlink_to_symlink_wo_lchmod(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        dst = os.path.join(tmp_dir, 'bar')
+        src_link = os.path.join(tmp_dir, 'baz')
+        dst_link = os.path.join(tmp_dir, 'quux')
+        write_file(src, 'foo')
+        write_file(dst, 'foo')
+        os.symlink(src, src_link)
+        os.symlink(dst, dst_link)
+        shutil.copymode(src_link, dst_link, follow_symlinks=False)  # silent fail
+
+    ### shutil.copystat
+
+    @support.skip_unless_symlink
+    def test_copystat_symlinks(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        dst = os.path.join(tmp_dir, 'bar')
+        src_link = os.path.join(tmp_dir, 'baz')
+        dst_link = os.path.join(tmp_dir, 'qux')
+        write_file(src, 'foo')
+        src_stat = os.stat(src)
+        os.utime(src, (src_stat.st_atime,
+                       src_stat.st_mtime - 42.0))  # ensure different mtimes
+        write_file(dst, 'bar')
+        self.assertNotEqual(os.stat(src).st_mtime, os.stat(dst).st_mtime)
+        os.symlink(src, src_link)
+        os.symlink(dst, dst_link)
+        if hasattr(os, 'lchmod'):
+            os.lchmod(src_link, stat.S_IRWXO)
+        if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'):
+            os.lchflags(src_link, stat.UF_NODUMP)
+        src_link_stat = os.lstat(src_link)
+        # follow
+        if hasattr(os, 'lchmod'):
+            shutil.copystat(src_link, dst_link, follow_symlinks=True)
+            self.assertNotEqual(src_link_stat.st_mode, os.stat(dst).st_mode)
+        # don't follow
+        shutil.copystat(src_link, dst_link, follow_symlinks=False)
+        dst_link_stat = os.lstat(dst_link)
+        if os.utime in os.supports_follow_symlinks:
+            for attr in 'st_atime', 'st_mtime':
+                # The modification times may be truncated in the new file.
+                self.assertLessEqual(getattr(src_link_stat, attr),
+                                     getattr(dst_link_stat, attr) + 1)
+        if hasattr(os, 'lchmod'):
+            self.assertEqual(src_link_stat.st_mode, dst_link_stat.st_mode)
+        if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'):
+            self.assertEqual(src_link_stat.st_flags, dst_link_stat.st_flags)
+        # tell to follow but dst is not a link
+        shutil.copystat(src_link, dst, follow_symlinks=False)
+        self.assertTrue(abs(os.stat(src).st_mtime - os.stat(dst).st_mtime) <
+                        00000.1)
+
+    @unittest.skipUnless(hasattr(os, 'chflags') and
+                         hasattr(errno, 'EOPNOTSUPP') and
+                         hasattr(errno, 'ENOTSUP'),
+                         "requires os.chflags, EOPNOTSUPP & ENOTSUP")
+    def test_copystat_handles_harmless_chflags_errors(self):
+        tmpdir = self.mkdtemp()
+        file1 = os.path.join(tmpdir, 'file1')
+        file2 = os.path.join(tmpdir, 'file2')
+        write_file(file1, 'xxx')
+        write_file(file2, 'xxx')
+
+        def make_chflags_raiser(err):
+            ex = OSError()
+
+            def _chflags_raiser(path, flags, *, follow_symlinks=True):
+                ex.errno = err
+                raise ex
+            return _chflags_raiser
         old_chflags = os.chflags
         try:
             for err in errno.EOPNOTSUPP, errno.ENOTSUP:
@@ -528,6 +880,8 @@ def _chflags_raiser(path, flags, *, follow_symlinks=True):
         finally:
             os.chflags = old_chflags
 
+    ### shutil.copyxattr
+
     @support.skip_unless_xattr
     def test_copyxattr(self):
         tmp_dir = self.mkdtemp()
@@ -617,6 +971,24 @@ def test_copyxattr_symlinks(self):
         shutil._copyxattr(src_link, dst, follow_symlinks=False)
         self.assertEqual(os.getxattr(dst, 'trusted.foo'), b'43')
 
+    ### shutil.copy
+
+    def _copy_file(self, method):
+        fname = 'test.txt'
+        tmpdir = self.mkdtemp()
+        write_file((tmpdir, fname), 'xxx')
+        file1 = os.path.join(tmpdir, fname)
+        tmpdir2 = self.mkdtemp()
+        method(file1, tmpdir2)
+        file2 = os.path.join(tmpdir2, fname)
+        return (file1, file2)
+
+    def test_copy(self):
+        # Ensure that the copied file exists and has the same mode bits.
+        file1, file2 = self._copy_file(shutil.copy)
+        self.assertTrue(os.path.exists(file2))
+        self.assertEqual(os.stat(file1).st_mode, os.stat(file2).st_mode)
+
     @support.skip_unless_symlink
     def test_copy_symlinks(self):
         tmp_dir = self.mkdtemp()
@@ -640,6 +1012,25 @@ def test_copy_symlinks(self):
             self.assertEqual(os.lstat(src_link).st_mode,
                              os.lstat(dst).st_mode)
 
+    ### shutil.copy2
+
+    @unittest.skipUnless(hasattr(os, 'utime'), 'requires os.utime')
+    def test_copy2(self):
+        # Ensure that the copied file exists and has the same mode and
+        # modification time bits.
+        file1, file2 = self._copy_file(shutil.copy2)
+        self.assertTrue(os.path.exists(file2))
+        file1_stat = os.stat(file1)
+        file2_stat = os.stat(file2)
+        self.assertEqual(file1_stat.st_mode, file2_stat.st_mode)
+        for attr in 'st_atime', 'st_mtime':
+            # The modification times may be truncated in the new file.
+            self.assertLessEqual(getattr(file1_stat, attr),
+                                 getattr(file2_stat, attr) + 1)
+        if hasattr(os, 'chflags') and hasattr(file1_stat, 'st_flags'):
+            self.assertEqual(getattr(file1_stat, 'st_flags'),
+                             getattr(file2_stat, 'st_flags'))
+
     @support.skip_unless_symlink
     def test_copy2_symlinks(self):
         tmp_dir = self.mkdtemp()
@@ -688,6 +1079,20 @@ def test_copy2_xattr(self):
                 os.getxattr(dst, 'user.foo'))
         os.remove(dst)
 
+    def test_copy_return_value(self):
+        # copy and copy2 both return their destination path.
+        for fn in (shutil.copy, shutil.copy2):
+            src_dir = self.mkdtemp()
+            dst_dir = self.mkdtemp()
+            src = os.path.join(src_dir, 'foo')
+            write_file(src, 'foo')
+            rv = fn(src, dst_dir)
+            self.assertEqual(rv, os.path.join(dst_dir, 'foo'))
+            rv = fn(src, os.path.join(dst_dir, 'bar'))
+            self.assertEqual(rv, os.path.join(dst_dir, 'bar'))
+
+    ### shutil.copyfile
+
     @support.skip_unless_symlink
     def test_copyfile_symlinks(self):
         tmp_dir = self.mkdtemp()
@@ -700,241 +1105,10 @@ def test_copyfile_symlinks(self):
         # don't follow
         shutil.copyfile(link, dst_link, follow_symlinks=False)
         self.assertTrue(os.path.islink(dst_link))
-        self.assertEqual(os.readlink(link), os.readlink(dst_link))
-        # follow
-        shutil.copyfile(link, dst)
-        self.assertFalse(os.path.islink(dst))
-
-    def test_rmtree_uses_safe_fd_version_if_available(self):
-        _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <=
-                             os.supports_dir_fd and
-                             os.listdir in os.supports_fd and
-                             os.stat in os.supports_follow_symlinks)
-        if _use_fd_functions:
-            self.assertTrue(shutil._use_fd_functions)
-            self.assertTrue(shutil.rmtree.avoids_symlink_attacks)
-            tmp_dir = self.mkdtemp()
-            d = os.path.join(tmp_dir, 'a')
-            os.mkdir(d)
-            try:
-                real_rmtree = shutil._rmtree_safe_fd
-                class Called(Exception): pass
-                def _raiser(*args, **kwargs):
-                    raise Called
-                shutil._rmtree_safe_fd = _raiser
-                self.assertRaises(Called, shutil.rmtree, d)
-            finally:
-                shutil._rmtree_safe_fd = real_rmtree
-        else:
-            self.assertFalse(shutil._use_fd_functions)
-            self.assertFalse(shutil.rmtree.avoids_symlink_attacks)
-
-    def test_rmtree_dont_delete_file(self):
-        # When called on a file instead of a directory, don't delete it.
-        handle, path = tempfile.mkstemp()
-        os.close(handle)
-        self.assertRaises(NotADirectoryError, shutil.rmtree, path)
-        os.remove(path)
-
-    def test_copytree_simple(self):
-        src_dir = tempfile.mkdtemp()
-        dst_dir = os.path.join(tempfile.mkdtemp(), 'destination')
-        self.addCleanup(shutil.rmtree, src_dir)
-        self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir))
-        write_file((src_dir, 'test.txt'), '123')
-        os.mkdir(os.path.join(src_dir, 'test_dir'))
-        write_file((src_dir, 'test_dir', 'test.txt'), '456')
-
-        shutil.copytree(src_dir, dst_dir)
-        self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt')))
-        self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir')))
-        self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir',
-                                                    'test.txt')))
-        actual = read_file((dst_dir, 'test.txt'))
-        self.assertEqual(actual, '123')
-        actual = read_file((dst_dir, 'test_dir', 'test.txt'))
-        self.assertEqual(actual, '456')
-
-    def test_copytree_dirs_exist_ok(self):
-        src_dir = tempfile.mkdtemp()
-        dst_dir = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, src_dir)
-        self.addCleanup(shutil.rmtree, dst_dir)
-
-        write_file((src_dir, 'nonexisting.txt'), '123')
-        os.mkdir(os.path.join(src_dir, 'existing_dir'))
-        os.mkdir(os.path.join(dst_dir, 'existing_dir'))
-        write_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced')
-        write_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced')
-
-        shutil.copytree(src_dir, dst_dir, dirs_exist_ok=True)
-        self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'nonexisting.txt')))
-        self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'existing_dir')))
-        self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'existing_dir',
-                                                    'existing.txt')))
-        actual = read_file((dst_dir, 'nonexisting.txt'))
-        self.assertEqual(actual, '123')
-        actual = read_file((dst_dir, 'existing_dir', 'existing.txt'))
-        self.assertEqual(actual, 'has been replaced')
-
-        with self.assertRaises(FileExistsError):
-            shutil.copytree(src_dir, dst_dir, dirs_exist_ok=False)
-
-    @support.skip_unless_symlink
-    def test_copytree_symlinks(self):
-        tmp_dir = self.mkdtemp()
-        src_dir = os.path.join(tmp_dir, 'src')
-        dst_dir = os.path.join(tmp_dir, 'dst')
-        sub_dir = os.path.join(src_dir, 'sub')
-        os.mkdir(src_dir)
-        os.mkdir(sub_dir)
-        write_file((src_dir, 'file.txt'), 'foo')
-        src_link = os.path.join(sub_dir, 'link')
-        dst_link = os.path.join(dst_dir, 'sub/link')
-        os.symlink(os.path.join(src_dir, 'file.txt'),
-                   src_link)
-        if hasattr(os, 'lchmod'):
-            os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO)
-        if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'):
-            os.lchflags(src_link, stat.UF_NODUMP)
-        src_stat = os.lstat(src_link)
-        shutil.copytree(src_dir, dst_dir, symlinks=True)
-        self.assertTrue(os.path.islink(os.path.join(dst_dir, 'sub', 'link')))
-        actual = os.readlink(os.path.join(dst_dir, 'sub', 'link'))
-        # Bad practice to blindly strip the prefix as it may be required to
-        # correctly refer to the file, but we're only comparing paths here.
-        if os.name == 'nt' and actual.startswith('\\\\?\\'):
-            actual = actual[4:]
-        self.assertEqual(actual, os.path.join(src_dir, 'file.txt'))
-        dst_stat = os.lstat(dst_link)
-        if hasattr(os, 'lchmod'):
-            self.assertEqual(dst_stat.st_mode, src_stat.st_mode)
-        if hasattr(os, 'lchflags'):
-            self.assertEqual(dst_stat.st_flags, src_stat.st_flags)
-
-    def test_copytree_with_exclude(self):
-        # creating data
-        join = os.path.join
-        exists = os.path.exists
-        src_dir = tempfile.mkdtemp()
-        try:
-            dst_dir = join(tempfile.mkdtemp(), 'destination')
-            write_file((src_dir, 'test.txt'), '123')
-            write_file((src_dir, 'test.tmp'), '123')
-            os.mkdir(join(src_dir, 'test_dir'))
-            write_file((src_dir, 'test_dir', 'test.txt'), '456')
-            os.mkdir(join(src_dir, 'test_dir2'))
-            write_file((src_dir, 'test_dir2', 'test.txt'), '456')
-            os.mkdir(join(src_dir, 'test_dir2', 'subdir'))
-            os.mkdir(join(src_dir, 'test_dir2', 'subdir2'))
-            write_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456')
-            write_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456')
-
-            # testing glob-like patterns
-            try:
-                patterns = shutil.ignore_patterns('*.tmp', 'test_dir2')
-                shutil.copytree(src_dir, dst_dir, ignore=patterns)
-                # checking the result: some elements should not be copied
-                self.assertTrue(exists(join(dst_dir, 'test.txt')))
-                self.assertFalse(exists(join(dst_dir, 'test.tmp')))
-                self.assertFalse(exists(join(dst_dir, 'test_dir2')))
-            finally:
-                shutil.rmtree(dst_dir)
-            try:
-                patterns = shutil.ignore_patterns('*.tmp', 'subdir*')
-                shutil.copytree(src_dir, dst_dir, ignore=patterns)
-                # checking the result: some elements should not be copied
-                self.assertFalse(exists(join(dst_dir, 'test.tmp')))
-                self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2')))
-                self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir')))
-            finally:
-                shutil.rmtree(dst_dir)
-
-            # testing callable-style
-            try:
-                def _filter(src, names):
-                    res = []
-                    for name in names:
-                        path = os.path.join(src, name)
-
-                        if (os.path.isdir(path) and
-                            path.split()[-1] == 'subdir'):
-                            res.append(name)
-                        elif os.path.splitext(path)[-1] in ('.py'):
-                            res.append(name)
-                    return res
-
-                shutil.copytree(src_dir, dst_dir, ignore=_filter)
-
-                # checking the result: some elements should not be copied
-                self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2',
-                                             'test.py')))
-                self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir')))
-
-            finally:
-                shutil.rmtree(dst_dir)
-        finally:
-            shutil.rmtree(src_dir)
-            shutil.rmtree(os.path.dirname(dst_dir))
-
-    def test_copytree_retains_permissions(self):
-        tmp_dir = tempfile.mkdtemp()
-        src_dir = os.path.join(tmp_dir, 'source')
-        os.mkdir(src_dir)
-        dst_dir = os.path.join(tmp_dir, 'destination')
-        self.addCleanup(shutil.rmtree, tmp_dir)
-
-        os.chmod(src_dir, 0o777)
-        write_file((src_dir, 'permissive.txt'), '123')
-        os.chmod(os.path.join(src_dir, 'permissive.txt'), 0o777)
-        write_file((src_dir, 'restrictive.txt'), '456')
-        os.chmod(os.path.join(src_dir, 'restrictive.txt'), 0o600)
-        restrictive_subdir = tempfile.mkdtemp(dir=src_dir)
-        os.chmod(restrictive_subdir, 0o600)
-
-        shutil.copytree(src_dir, dst_dir)
-        self.assertEqual(os.stat(src_dir).st_mode, os.stat(dst_dir).st_mode)
-        self.assertEqual(os.stat(os.path.join(src_dir, 'permissive.txt')).st_mode,
-                          os.stat(os.path.join(dst_dir, 'permissive.txt')).st_mode)
-        self.assertEqual(os.stat(os.path.join(src_dir, 'restrictive.txt')).st_mode,
-                          os.stat(os.path.join(dst_dir, 'restrictive.txt')).st_mode)
-        restrictive_subdir_dst = os.path.join(dst_dir,
-                                              os.path.split(restrictive_subdir)[1])
-        self.assertEqual(os.stat(restrictive_subdir).st_mode,
-                          os.stat(restrictive_subdir_dst).st_mode)
-
-    @unittest.mock.patch('os.chmod')
-    def test_copytree_winerror(self, mock_patch):
-        # When copying to VFAT, copystat() raises OSError. On Windows, the
-        # exception object has a meaningful 'winerror' attribute, but not
-        # on other operating systems. Do not assume 'winerror' is set.
-        src_dir = tempfile.mkdtemp()
-        dst_dir = os.path.join(tempfile.mkdtemp(), 'destination')
-        self.addCleanup(shutil.rmtree, src_dir)
-        self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir))
-
-        mock_patch.side_effect = PermissionError('ka-boom')
-        with self.assertRaises(shutil.Error):
-            shutil.copytree(src_dir, dst_dir)
-
-    def test_copytree_custom_copy_function(self):
-        # See: https://bugs.python.org/issue35648
-        def custom_cpfun(a, b):
-            flag.append(None)
-            self.assertIsInstance(a, str)
-            self.assertIsInstance(b, str)
-            self.assertEqual(a, os.path.join(src, 'foo'))
-            self.assertEqual(b, os.path.join(dst, 'foo'))
-
-        flag = []
-        src = tempfile.mkdtemp()
-        self.addCleanup(support.rmtree, src)
-        dst = tempfile.mktemp()
-        self.addCleanup(support.rmtree, dst)
-        with open(os.path.join(src, 'foo'), 'w') as f:
-            f.close()
-        shutil.copytree(src, dst, copy_function=custom_cpfun)
-        self.assertEqual(len(flag), 1)
+        self.assertEqual(os.readlink(link), os.readlink(dst_link))
+        # follow
+        shutil.copyfile(link, dst)
+        self.assertFalse(os.path.islink(dst))
 
     @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link')
     def test_dont_copy_file_onto_link_to_itself(self):
@@ -976,34 +1150,6 @@ def test_dont_copy_file_onto_symlink_to_itself(self):
         finally:
             shutil.rmtree(TESTFN, ignore_errors=True)
 
-    @support.skip_unless_symlink
-    def test_rmtree_on_symlink(self):
-        # bug 1669.
-        os.mkdir(TESTFN)
-        try:
-            src = os.path.join(TESTFN, 'cheese')
-            dst = os.path.join(TESTFN, 'shop')
-            os.mkdir(src)
-            os.symlink(src, dst)
-            self.assertRaises(OSError, shutil.rmtree, dst)
-            shutil.rmtree(dst, ignore_errors=True)
-        finally:
-            shutil.rmtree(TESTFN, ignore_errors=True)
-
-    @unittest.skipUnless(_winapi, 'only relevant on Windows')
-    def test_rmtree_on_junction(self):
-        os.mkdir(TESTFN)
-        try:
-            src = os.path.join(TESTFN, 'cheese')
-            dst = os.path.join(TESTFN, 'shop')
-            os.mkdir(src)
-            open(os.path.join(src, 'spam'), 'wb').close()
-            _winapi.CreateJunction(src, dst)
-            self.assertRaises(OSError, shutil.rmtree, dst)
-            shutil.rmtree(dst, ignore_errors=True)
-        finally:
-            shutil.rmtree(TESTFN, ignore_errors=True)
-
     # Issue #3002: copyfile and copytree block indefinitely on named pipes
     @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()')
     def test_copyfile_named_pipe(self):
@@ -1019,119 +1165,33 @@ def test_copyfile_named_pipe(self):
         finally:
             os.remove(TESTFN)
 
-    @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()')
-    @support.skip_unless_symlink
-    def test_copytree_named_pipe(self):
-        os.mkdir(TESTFN)
-        try:
-            subdir = os.path.join(TESTFN, "subdir")
-            os.mkdir(subdir)
-            pipe = os.path.join(subdir, "mypipe")
-            try:
-                os.mkfifo(pipe)
-            except PermissionError as e:
-                self.skipTest('os.mkfifo(): %s' % e)
-            try:
-                shutil.copytree(TESTFN, TESTFN2)
-            except shutil.Error as e:
-                errors = e.args[0]
-                self.assertEqual(len(errors), 1)
-                src, dst, error_msg = errors[0]
-                self.assertEqual("`%s` is a named pipe" % pipe, error_msg)
-            else:
-                self.fail("shutil.Error should have been raised")
-        finally:
-            shutil.rmtree(TESTFN, ignore_errors=True)
-            shutil.rmtree(TESTFN2, ignore_errors=True)
-
-    def test_copytree_special_func(self):
-
-        src_dir = self.mkdtemp()
-        dst_dir = os.path.join(self.mkdtemp(), 'destination')
-        write_file((src_dir, 'test.txt'), '123')
-        os.mkdir(os.path.join(src_dir, 'test_dir'))
-        write_file((src_dir, 'test_dir', 'test.txt'), '456')
-
-        copied = []
-        def _copy(src, dst):
-            copied.append((src, dst))
-
-        shutil.copytree(src_dir, dst_dir, copy_function=_copy)
-        self.assertEqual(len(copied), 2)
-
-    @support.skip_unless_symlink
-    def test_copytree_dangling_symlinks(self):
-
-        # a dangling symlink raises an error at the end
+    def test_copyfile_return_value(self):
+        # copytree returns its destination path.
         src_dir = self.mkdtemp()
-        dst_dir = os.path.join(self.mkdtemp(), 'destination')
-        os.symlink('IDONTEXIST', os.path.join(src_dir, 'test.txt'))
-        os.mkdir(os.path.join(src_dir, 'test_dir'))
-        write_file((src_dir, 'test_dir', 'test.txt'), '456')
-        self.assertRaises(Error, shutil.copytree, src_dir, dst_dir)
-
-        # a dangling symlink is ignored with the proper flag
-        dst_dir = os.path.join(self.mkdtemp(), 'destination2')
-        shutil.copytree(src_dir, dst_dir, ignore_dangling_symlinks=True)
-        self.assertNotIn('test.txt', os.listdir(dst_dir))
-
-        # a dangling symlink is copied if symlinks=True
-        dst_dir = os.path.join(self.mkdtemp(), 'destination3')
-        shutil.copytree(src_dir, dst_dir, symlinks=True)
-        self.assertIn('test.txt', os.listdir(dst_dir))
+        dst_dir = self.mkdtemp()
+        dst_file = os.path.join(dst_dir, 'bar')
+        src_file = os.path.join(src_dir, 'foo')
+        write_file(src_file, 'foo')
+        rv = shutil.copyfile(src_file, dst_file)
+        self.assertTrue(os.path.exists(rv))
+        self.assertEqual(read_file(src_file), read_file(dst_file))
 
-    @support.skip_unless_symlink
-    def test_copytree_symlink_dir(self):
+    def test_copyfile_same_file(self):
+        # copyfile() should raise SameFileError if the source and destination
+        # are the same.
         src_dir = self.mkdtemp()
-        dst_dir = os.path.join(self.mkdtemp(), 'destination')
-        os.mkdir(os.path.join(src_dir, 'real_dir'))
-        with open(os.path.join(src_dir, 'real_dir', 'test.txt'), 'w'):
-            pass
-        os.symlink(os.path.join(src_dir, 'real_dir'),
-                   os.path.join(src_dir, 'link_to_dir'),
-                   target_is_directory=True)
-
-        shutil.copytree(src_dir, dst_dir, symlinks=False)
-        self.assertFalse(os.path.islink(os.path.join(dst_dir, 'link_to_dir')))
-        self.assertIn('test.txt', os.listdir(os.path.join(dst_dir, 'link_to_dir')))
-
-        dst_dir = os.path.join(self.mkdtemp(), 'destination2')
-        shutil.copytree(src_dir, dst_dir, symlinks=True)
-        self.assertTrue(os.path.islink(os.path.join(dst_dir, 'link_to_dir')))
-        self.assertIn('test.txt', os.listdir(os.path.join(dst_dir, 'link_to_dir')))
+        src_file = os.path.join(src_dir, 'foo')
+        write_file(src_file, 'foo')
+        self.assertRaises(SameFileError, shutil.copyfile, src_file, src_file)
+        # But Error should work too, to stay backward compatible.
+        self.assertRaises(Error, shutil.copyfile, src_file, src_file)
+        # Make sure file is not corrupted.
+        self.assertEqual(read_file(src_file), 'foo')
 
-    def _copy_file(self, method):
-        fname = 'test.txt'
-        tmpdir = self.mkdtemp()
-        write_file((tmpdir, fname), 'xxx')
-        file1 = os.path.join(tmpdir, fname)
-        tmpdir2 = self.mkdtemp()
-        method(file1, tmpdir2)
-        file2 = os.path.join(tmpdir2, fname)
-        return (file1, file2)
 
-    def test_copy(self):
-        # Ensure that the copied file exists and has the same mode bits.
-        file1, file2 = self._copy_file(shutil.copy)
-        self.assertTrue(os.path.exists(file2))
-        self.assertEqual(os.stat(file1).st_mode, os.stat(file2).st_mode)
+class TestArchives(BaseTest, unittest.TestCase):
 
-    @unittest.skipUnless(hasattr(os, 'utime'), 'requires os.utime')
-    def test_copy2(self):
-        # Ensure that the copied file exists and has the same mode and
-        # modification time bits.
-        file1, file2 = self._copy_file(shutil.copy2)
-        self.assertTrue(os.path.exists(file2))
-        file1_stat = os.stat(file1)
-        file2_stat = os.stat(file2)
-        self.assertEqual(file1_stat.st_mode, file2_stat.st_mode)
-        for attr in 'st_atime', 'st_mtime':
-            # The modification times may be truncated in the new file.
-            self.assertLessEqual(getattr(file1_stat, attr),
-                                 getattr(file2_stat, attr) + 1)
-        if hasattr(os, 'chflags') and hasattr(file1_stat, 'st_flags'):
-            self.assertEqual(getattr(file1_stat, 'st_flags'),
-                             getattr(file2_stat, 'st_flags'))
+    ### shutil.make_archive
 
     @support.requires_zlib
     def test_make_tarball(self):
@@ -1411,6 +1471,8 @@ def test_register_archive_format(self):
         formats = [name for name, params in get_archive_formats()]
         self.assertNotIn('xxx', formats)
 
+    ### shutil.unpack_archive
+
     def check_unpack_archive(self, format):
         self.check_unpack_archive_with_converter(format, lambda path: path)
         self.check_unpack_archive_with_converter(format, pathlib.Path)
@@ -1483,6 +1545,9 @@ def _boo(filename, extract_dir, extra):
         unregister_unpack_format('Boo2')
         self.assertEqual(get_unpack_formats(), formats)
 
+
+class TestMisc(BaseTest, unittest.TestCase):
+
     @unittest.skipUnless(hasattr(shutil, 'disk_usage'),
                          "disk_usage not available on this platform")
     def test_disk_usage(self):
@@ -1501,8 +1566,6 @@ def test_disk_usage(self):
     @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support")
     @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown')
     def test_chown(self):
-
-        # cleaned-up automatically by TestShutil.tearDown method
         dirname = self.mkdtemp()
         filename = tempfile.mktemp(dir=dirname)
         write_file(filename, 'testing chown function')
@@ -1557,57 +1620,12 @@ def check_chown(path, uid=None, gid=None):
         shutil.chown(dirname, user, group)
         check_chown(dirname, uid, gid)
 
-    def test_copy_return_value(self):
-        # copy and copy2 both return their destination path.
-        for fn in (shutil.copy, shutil.copy2):
-            src_dir = self.mkdtemp()
-            dst_dir = self.mkdtemp()
-            src = os.path.join(src_dir, 'foo')
-            write_file(src, 'foo')
-            rv = fn(src, dst_dir)
-            self.assertEqual(rv, os.path.join(dst_dir, 'foo'))
-            rv = fn(src, os.path.join(dst_dir, 'bar'))
-            self.assertEqual(rv, os.path.join(dst_dir, 'bar'))
-
-    def test_copyfile_return_value(self):
-        # copytree returns its destination path.
-        src_dir = self.mkdtemp()
-        dst_dir = self.mkdtemp()
-        dst_file = os.path.join(dst_dir, 'bar')
-        src_file = os.path.join(src_dir, 'foo')
-        write_file(src_file, 'foo')
-        rv = shutil.copyfile(src_file, dst_file)
-        self.assertTrue(os.path.exists(rv))
-        self.assertEqual(read_file(src_file), read_file(dst_file))
-
-    def test_copyfile_same_file(self):
-        # copyfile() should raise SameFileError if the source and destination
-        # are the same.
-        src_dir = self.mkdtemp()
-        src_file = os.path.join(src_dir, 'foo')
-        write_file(src_file, 'foo')
-        self.assertRaises(SameFileError, shutil.copyfile, src_file, src_file)
-        # But Error should work too, to stay backward compatible.
-        self.assertRaises(Error, shutil.copyfile, src_file, src_file)
-        # Make sure file is not corrupted.
-        self.assertEqual(read_file(src_file), 'foo')
-
-    def test_copytree_return_value(self):
-        # copytree returns its destination path.
-        src_dir = self.mkdtemp()
-        dst_dir = src_dir + "dest"
-        self.addCleanup(shutil.rmtree, dst_dir, True)
-        src = os.path.join(src_dir, 'foo')
-        write_file(src, 'foo')
-        rv = shutil.copytree(src_dir, dst_dir)
-        self.assertEqual(['foo'], os.listdir(rv))
-
 
-class TestWhich(unittest.TestCase):
+class TestWhich(BaseTest, unittest.TestCase):
 
     def setUp(self):
         self.temp_dir = tempfile.mkdtemp(prefix="Tmp")
-        self.addCleanup(shutil.rmtree, self.temp_dir, True)
+        self.addCleanup(support.rmtree, self.temp_dir)
         # Give the temp_file an ".exe" suffix for all.
         # It's needed on Windows and not harmful on other platforms.
         self.temp_file = tempfile.NamedTemporaryFile(dir=self.temp_dir,
@@ -1784,25 +1802,17 @@ def setUp(self):
         self.ext = os.fsencode(self.ext)
 
 
-class TestMove(unittest.TestCase):
+class TestMove(BaseTest, unittest.TestCase):
 
     def setUp(self):
         filename = "foo"
-        self.src_dir = tempfile.mkdtemp()
-        self.dst_dir = tempfile.mkdtemp()
+        self.src_dir = self.mkdtemp()
+        self.dst_dir = self.mkdtemp()
         self.src_file = os.path.join(self.src_dir, filename)
         self.dst_file = os.path.join(self.dst_dir, filename)
         with open(self.src_file, "wb") as f:
             f.write(b"spam")
 
-    def tearDown(self):
-        for d in (self.src_dir, self.dst_dir):
-            try:
-                if d:
-                    shutil.rmtree(d)
-            except:
-                pass
-
     def _check_move_file(self, src, dst, real_dst):
         with open(src, "rb") as f:
             contents = f.read()
@@ -1841,10 +1851,7 @@ def test_move_dir(self):
         try:
             self._check_move_dir(self.src_dir, dst_dir, dst_dir)
         finally:
-            try:
-                shutil.rmtree(dst_dir)
-            except:
-                pass
+            support.rmtree(dst_dir)
 
     @mock_rename
     def test_move_dir_other_fs(self):
@@ -1891,7 +1898,7 @@ def test_destinsrc_false_negative(self):
                              msg='_destinsrc() wrongly concluded that '
                              'dst (%s) is not in src (%s)' % (dst, src))
         finally:
-            shutil.rmtree(TESTFN, ignore_errors=True)
+            support.rmtree(TESTFN)
 
     def test_destinsrc_false_positive(self):
         os.mkdir(TESTFN)
@@ -1903,7 +1910,7 @@ def test_destinsrc_false_positive(self):
                             msg='_destinsrc() wrongly concluded that '
                             'dst (%s) is in src (%s)' % (dst, src))
         finally:
-            shutil.rmtree(TESTFN, ignore_errors=True)
+            support.rmtree(TESTFN)
 
     @support.skip_unless_symlink
     @mock_rename
@@ -1975,10 +1982,24 @@ def _copy(src, dst):
         shutil.move(self.src_dir, self.dst_dir, copy_function=_copy)
         self.assertEqual(len(moved), 3)
 
+    def test_move_dir_caseinsensitive(self):
+        # Renames a folder to the same name
+        # but a different case.
 
-class TestCopyFile(unittest.TestCase):
+        self.src_dir = self.mkdtemp()
+        dst_dir = os.path.join(
+                os.path.dirname(self.src_dir),
+                os.path.basename(self.src_dir).upper())
+        self.assertNotEqual(self.src_dir, dst_dir)
 
-    _delete = False
+        try:
+            shutil.move(self.src_dir, dst_dir)
+            self.assertTrue(os.path.isdir(dst_dir))
+        finally:
+            os.rmdir(dst_dir)
+
+
+class TestCopyFile(unittest.TestCase):
 
     class Faux(object):
         _entered = False
@@ -1998,27 +2019,18 @@ def __exit__(self, exc_type, exc_val, exc_tb):
                 raise OSError("Cannot close")
             return self._suppress_at_exit
 
-    def tearDown(self):
-        if self._delete:
-            del shutil.open
-
-    def _set_shutil_open(self, func):
-        shutil.open = func
-        self._delete = True
-
     def test_w_source_open_fails(self):
         def _open(filename, mode='r'):
             if filename == 'srcfile':
                 raise OSError('Cannot open "srcfile"')
             assert 0  # shouldn't reach here.
 
-        self._set_shutil_open(_open)
-
-        self.assertRaises(OSError, shutil.copyfile, 'srcfile', 'destfile')
+        with support.swap_attr(shutil, 'open', _open):
+            with self.assertRaises(OSError):
+                shutil.copyfile('srcfile', 'destfile')
 
     @unittest.skipIf(MACOS, "skipped on macOS")
     def test_w_dest_open_fails(self):
-
         srcfile = self.Faux()
 
         def _open(filename, mode='r'):
@@ -2028,9 +2040,8 @@ def _open(filename, mode='r'):
                 raise OSError('Cannot open "destfile"')
             assert 0  # shouldn't reach here.
 
-        self._set_shutil_open(_open)
-
-        shutil.copyfile('srcfile', 'destfile')
+        with support.swap_attr(shutil, 'open', _open):
+            shutil.copyfile('srcfile', 'destfile')
         self.assertTrue(srcfile._entered)
         self.assertTrue(srcfile._exited_with[0] is OSError)
         self.assertEqual(srcfile._exited_with[1].args,
@@ -2038,7 +2049,6 @@ def _open(filename, mode='r'):
 
     @unittest.skipIf(MACOS, "skipped on macOS")
     def test_w_dest_close_fails(self):
-
         srcfile = self.Faux()
         destfile = self.Faux(True)
 
@@ -2049,9 +2059,8 @@ def _open(filename, mode='r'):
                 return destfile
             assert 0  # shouldn't reach here.
 
-        self._set_shutil_open(_open)
-
-        shutil.copyfile('srcfile', 'destfile')
+        with support.swap_attr(shutil, 'open', _open):
+            shutil.copyfile('srcfile', 'destfile')
         self.assertTrue(srcfile._entered)
         self.assertTrue(destfile._entered)
         self.assertTrue(destfile._raised)
@@ -2072,33 +2081,15 @@ def _open(filename, mode='r'):
                 return destfile
             assert 0  # shouldn't reach here.
 
-        self._set_shutil_open(_open)
-
-        self.assertRaises(OSError,
-                          shutil.copyfile, 'srcfile', 'destfile')
+        with support.swap_attr(shutil, 'open', _open):
+            with self.assertRaises(OSError):
+                shutil.copyfile('srcfile', 'destfile')
         self.assertTrue(srcfile._entered)
         self.assertTrue(destfile._entered)
         self.assertFalse(destfile._raised)
         self.assertTrue(srcfile._exited_with[0] is None)
         self.assertTrue(srcfile._raised)
 
-    def test_move_dir_caseinsensitive(self):
-        # Renames a folder to the same name
-        # but a different case.
-
-        self.src_dir = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.src_dir, True)
-        dst_dir = os.path.join(
-                os.path.dirname(self.src_dir),
-                os.path.basename(self.src_dir).upper())
-        self.assertNotEqual(self.src_dir, dst_dir)
-
-        try:
-            shutil.move(self.src_dir, dst_dir)
-            self.assertTrue(os.path.isdir(dst_dir))
-        finally:
-            os.rmdir(dst_dir)
-
 
 class TestCopyFileObj(unittest.TestCase):
     FILESIZE = 2 * 1024 * 1024
@@ -2401,7 +2392,7 @@ def zerocopy_fun(self, src, dst):
         return shutil._fastcopy_fcopyfile(src, dst, posix._COPYFILE_DATA)
 
 
-class TermsizeTests(unittest.TestCase):
+class TestGetTerminalSize(unittest.TestCase):
     def test_does_not_crash(self):
         """Check if get_terminal_size() returns a meaningful value.
 


From webhook-mailer at python.org  Thu Sep 26 09:16:36 2019
From: webhook-mailer at python.org (Petr Viktorin)
Date: Thu, 26 Sep 2019 13:16:36 -0000
Subject: [Python-checkins] compileall tests: Use shorter name for long_path
 test (GH-16419)
Message-ID: <mailman.104.1569503798.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/3d984a1fd0c05903268542a216fc496074b2e6da
commit: 3d984a1fd0c05903268542a216fc496074b2e6da
branch: master
author: Petr Viktorin <encukou at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-26T15:16:32+02:00
summary:

compileall tests: Use shorter name for long_path test (GH-16419)

Apparently, the path needs to be limited to 260 characters on
(some versions of) Windows.

files:
M Lib/test/test_compileall.py

diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 205457f3b480..3bb03b3dd384 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -53,7 +53,7 @@ def create_long_path(self):
         # It will be 100 directories deep, or shorter if the OS limits it.
         for i in range(10):
             longer_path = os.path.join(
-                long_path, *(f"long_directory_{i}_{j}" for j in range(10))
+                long_path, *(f"dir_{i}_{j}" for j in range(10))
             )
 
             # Check if we can open __pycache__/*.pyc.
@@ -87,8 +87,12 @@ def create_long_path(self):
             long_source = longer_source
             long_cache = longer_cache
 
+        # On Windows, MAX_PATH is 260 characters, our path with the 20
+        # directories is 160 characters long, leaving something for the
+        # root (self.directory) as well.
+        # Tests assume long_path contains at least 10 directories.
         if i < 2:
-            raise ValueError('Path limit is too short')
+            raise ValueError(f'"Long path" is too short: {long_path}')
 
         self.source_path_long = long_source
         self.bc_path_long = long_cache


From webhook-mailer at python.org  Thu Sep 26 09:52:04 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 26 Sep 2019 13:52:04 -0000
Subject: [Python-checkins] bpo-38234: Py_Initialize() sets global path
 configuration (GH-16421)
Message-ID: <mailman.105.1569505925.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/12f2f177fc483723406d7917194e7f655a20631b
commit: 12f2f177fc483723406d7917194e7f655a20631b
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-26T15:51:50+02:00
summary:

bpo-38234: Py_Initialize() sets global path configuration (GH-16421)

* Py_InitializeFromConfig() now writes PyConfig path configuration to
  the global path configuration (_Py_path_config).
* Add test_embed.test_get_pathconfig().
* Fix typo in _PyWideStringList_Join().

files:
M Include/internal/pycore_pathconfig.h
M Lib/test/test_embed.py
M Python/pathconfig.c
M Python/pylifecycle.c

diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h
index b5d114473654..ce75ccee835a 100644
--- a/Include/internal/pycore_pathconfig.h
+++ b/Include/internal/pycore_pathconfig.h
@@ -66,7 +66,7 @@ extern int _Py_FindEnvConfigValue(
 extern wchar_t* _Py_GetDLLPath(void);
 #endif
 
-extern PyStatus _PyPathConfig_Init(void);
+extern PyStatus _PyConfig_WritePathConfig(const PyConfig *config);
 extern void _Py_DumpPathConfig(PyThreadState *tstate);
 
 #ifdef __cplusplus
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 92b5136db7ea..ab086e2332da 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -1230,6 +1230,44 @@ def test_init_pyvenv_cfg(self):
                                    api=API_COMPAT, env=env,
                                    ignore_stderr=True, cwd=tmpdir)
 
+    def test_global_pathconfig(self):
+        # Test C API functions getting the path configuration:
+        #
+        # - Py_GetExecPrefix()
+        # - Py_GetPath()
+        # - Py_GetPrefix()
+        # - Py_GetProgramFullPath()
+        # - Py_GetProgramName()
+        # - Py_GetPythonHome()
+        #
+        # The global path configuration (_Py_path_config) must be a copy
+        # of the path configuration of PyInterpreter.config (PyConfig).
+        ctypes = support.import_module('ctypes')
+        _testinternalcapi = support.import_module('_testinternalcapi')
+
+        def get_func(name):
+            func = getattr(ctypes.pythonapi, name)
+            func.argtypes = ()
+            func.restype = ctypes.c_wchar_p
+            return func
+
+        Py_GetPath = get_func('Py_GetPath')
+        Py_GetPrefix = get_func('Py_GetPrefix')
+        Py_GetExecPrefix = get_func('Py_GetExecPrefix')
+        Py_GetProgramName = get_func('Py_GetProgramName')
+        Py_GetProgramFullPath = get_func('Py_GetProgramFullPath')
+        Py_GetPythonHome = get_func('Py_GetPythonHome')
+
+        config = _testinternalcapi.get_configs()['config']
+
+        self.assertEqual(Py_GetPath().split(os.path.pathsep),
+                         config['module_search_paths'])
+        self.assertEqual(Py_GetPrefix(), config['prefix'])
+        self.assertEqual(Py_GetExecPrefix(), config['exec_prefix'])
+        self.assertEqual(Py_GetProgramName(), config['program_name'])
+        self.assertEqual(Py_GetProgramFullPath(), config['executable'])
+        self.assertEqual(Py_GetPythonHome(), config['home'])
+
 
 class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
     def test_open_code_hook(self):
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index f4e149866878..812614532800 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -133,7 +133,7 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep)
     for (Py_ssize_t i=0; i < list->length; i++) {
         wchar_t *path = list->items[i];
         if (i != 0) {
-            *str++ = SEP;
+            *str++ = sep;
         }
         len = wcslen(path);
         memcpy(str, path, len * sizeof(wchar_t));
@@ -145,11 +145,11 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep)
 }
 
 
+#ifdef MS_WINDOWS
 /* Initialize _Py_dll_path on Windows. Do nothing on other platforms. */
-PyStatus
-_PyPathConfig_Init(void)
+static PyStatus
+_PyPathConfig_InitDLLPath(void)
 {
-#ifdef MS_WINDOWS
     if (_Py_dll_path == NULL) {
         /* Already set: nothing to do */
         return _PyStatus_OK();
@@ -165,9 +165,9 @@ _PyPathConfig_Init(void)
     if (_Py_dll_path == NULL) {
         return _PyStatus_NO_MEMORY();
     }
-#endif
     return _PyStatus_OK();
 }
+#endif
 
 
 static PyStatus
@@ -217,6 +217,20 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
 }
 
 
+PyStatus
+_PyConfig_WritePathConfig(const PyConfig *config)
+{
+#ifdef MS_WINDOWS
+    PyStatus status = _PyPathConfig_InitDLLPath();
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+#endif
+
+    return pathconfig_set_from_config(&_Py_path_config, config);
+}
+
+
 static PyStatus
 config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig)
 {
@@ -441,11 +455,12 @@ pathconfig_global_init(void)
 {
     PyStatus status;
 
-    /* Initialize _Py_dll_path if needed */
-    status = _PyPathConfig_Init();
+#ifdef MS_WINDOWS
+    status = _PyPathConfig_InitDLLPath();
     if (_PyStatus_EXCEPTION(status)) {
         Py_ExitStatusException(status);
     }
+#endif
 
     if (_Py_path_config.module_search_path == NULL) {
         status = pathconfig_global_read(&_Py_path_config);
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index e5b6283d8db0..eed583a4fdbc 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -475,7 +475,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime,
     config = &interp->config;
 
     if (config->_install_importlib) {
-        status = _PyPathConfig_Init();
+        status = _PyConfig_WritePathConfig(config);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
@@ -646,7 +646,7 @@ pycore_init_import_warnings(PyThreadState *tstate, PyObject *sysmod)
     }
 
     if (config->_install_importlib) {
-        status = _PyPathConfig_Init();
+        status = _PyConfig_WritePathConfig(config);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }


From webhook-mailer at python.org  Thu Sep 26 10:17:39 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 26 Sep 2019 14:17:39 -0000
Subject: [Python-checkins] [3.8] bpo-38234: Backport init path config
 changes from master (GH-16423)
Message-ID: <mailman.106.1569507459.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/96c8475362acb41decd1d7db9243f328973e5de7
commit: 96c8475362acb41decd1d7db9243f328973e5de7
branch: 3.8
author: Victor Stinner <vstinner at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-26T16:17:34+02:00
summary:

[3.8] bpo-38234: Backport init path config changes from master (GH-16423)

* bpo-38234: Py_SetPath() uses the program full path (GH-16357)

Py_SetPath() now sets sys.executable to the program full path
(Py_GetProgramFullPath()), rather than to the program name
(Py_GetProgramName()).

Fix also memory leaks in pathconfig_set_from_config().

(cherry picked from commit 1ce152a42eaa917d7763bce93f1e1ca72530d7ca)

* bpo-38234: Add tests for Python init path config (GH-16358)


(cherry picked from commit bb6bf7d342b4503a6227fd209fac934905b6a1aa)

* bpo-38234: test_embed: test pyvenv.cfg and pybuilddir.txt (GH-16366)

Add test_init_pybuilddir() and test_init_pyvenv_cfg() to test_embed
to test pyvenv.cfg and pybuilddir.txt configuration files.

Fix sysconfig._generate_posix_vars(): pybuilddir.txt uses UTF-8
encoding, not ASCII.

(cherry picked from commit 52ad33abbfb6637d74932617c7013bae0ccf6e32)

* bpo-38234: Cleanup getpath.c (GH-16367)

* search_for_prefix() directly calls reduce() if found is greater
  than 0.
* Add calculate_pybuilddir() subfunction.
* search_for_prefix(): add path string buffer for readability.
* Fix some error handling code paths: release resources on error.
* calculate_read_pyenv(): rename tmpbuffer to filename.
* test.pythoninfo now also logs windows.dll_path

(cherry picked from commit 221fd84703c545408bbb4a6e0b58459651331f5c)

* bpo-38234: Fix test_embed pathconfig tests (GH-16390)

bpo-38234: On macOS and FreeBSD, the temporary directory can be
symbolic link. For example, /tmp can be a symbolic link to /var/tmp.
Call realpath() to resolve all symbolic links.

(cherry picked from commit 00508a7407d7d300b487532e2271534b20e378a7)

* bpo-38234: Add test_init_setpath_config() to test_embed (GH-16402)

* Add test_embed.test_init_setpath_config(): test Py_SetPath()
  with PyConfig.
* test_init_setpath() and test_init_setpythonhome() no longer call
  Py_SetProgramName(), but use the default program name.
* _PyPathConfig: isolated, site_import  and base_executable
  fields are now only available on Windows.
* If executable is set explicitly in the configuration, ignore
  calculated base_executable: _PyConfig_InitPathConfig() copies
  executable to base_executable.
* Complete path config documentation.

(cherry picked from commit 8bf39b606ef7b02c0279a80789f3c4824b0da5e9)

* bpo-38234: Complete init config documentation (GH-16404)


(cherry picked from commit 88feaecd46a8f427e30ef7ad8cfcddfe392a2402)

* bpo-38234: Fix test_embed.test_init_setpath_config() on FreeBSD (GH-16406)

Explicitly preinitializes with a Python preconfiguration to avoid
Py_SetPath() implicit preinitialization with a compat
preconfiguration.

Fix also test_init_setpath() and test_init_setpythonhome() on macOS:
use self.test_exe as the executable (and base_executable), rather
than shutil.which('python3').

(cherry picked from commit 49d99f01e6e51acec5ca57a02e857f0796bc418b)

* bpo-38234: Py_Initialize() sets global path configuration (GH-16421)

* Py_InitializeFromConfig() now writes PyConfig path configuration to
  the global path configuration (_Py_path_config).
* Add test_embed.test_get_pathconfig().
* Fix typo in _PyWideStringList_Join().

(cherry picked from commit 12f2f177fc483723406d7917194e7f655a20631b)

files:
A Misc/NEWS.d/next/C API/2019-09-24-17-09-48.bpo-38234.d0bhEA.rst
M Doc/c-api/init.rst
M Doc/c-api/init_config.rst
M Doc/whatsnew/3.8.rst
M Include/internal/pycore_pathconfig.h
M Lib/sysconfig.py
M Lib/test/pythoninfo.py
M Lib/test/test_embed.py
M Modules/getpath.c
M PC/getpathp.c
M Programs/_testembed.c
M Python/pathconfig.c
M Python/pylifecycle.c

diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index 0f8ff3b0dde8..0b7a84d03153 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -472,8 +472,8 @@ Process-wide parameters
    dependent delimiter character, which is ``':'`` on Unix and Mac OS X, ``';'``
    on Windows.
 
-   This also causes :data:`sys.executable` to be set only to the raw program
-   name (see :c:func:`Py_SetProgramName`) and for :data:`sys.prefix` and
+   This also causes :data:`sys.executable` to be set to the program
+   full path (see :c:func:`Py_GetProgramFullPath`) and for :data:`sys.prefix` and
    :data:`sys.exec_prefix` to be empty.  It is up to the caller to modify these
    if required after calling :c:func:`Py_Initialize`.
 
@@ -483,6 +483,10 @@ Process-wide parameters
    The path argument is copied internally, so the caller may free it after the
    call completes.
 
+   .. versionchanged:: 3.8
+      The program full path is now used for :data:`sys.executable`, instead
+      of the program name.
+
 
 .. c:function:: const char* Py_GetVersion()
 
diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index bc24fa081317..0c3c725c841a 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -241,6 +241,7 @@ PyPreConfig
       locale to decide if it should be coerced.
 
    .. c:member:: int coerce_c_locale_warn
+
       If non-zero, emit a warning if the C locale is coerced.
 
    .. c:member:: int dev_mode
@@ -300,7 +301,7 @@ For :ref:`Python Configuration <init-python-config>`
 (:c:func:`PyPreConfig_InitPythonConfig`), if Python is initialized with
 command line arguments, the command line arguments must also be passed to
 preinitialize Python, since they have an effect on the pre-configuration
-like encodings. For example, the :option:`-X` ``utf8`` command line option
+like encodings. For example, the :option:`-X utf8 <-X>` command line option
 enables the UTF-8 Mode.
 
 ``PyMem_SetAllocator()`` can be called after :c:func:`Py_PreInitialize` and
@@ -464,7 +465,7 @@ PyConfig
 
    .. c:member:: int dev_mode
 
-      Development mode: see :option:`-X` ``dev``.
+      Development mode: see :option:`-X dev <-X>`.
 
    .. c:member:: int dump_refs
 
@@ -482,7 +483,7 @@ PyConfig
 
    .. c:member:: int faulthandler
 
-      If non-zero, call :func:`faulthandler.enable`.
+      If non-zero, call :func:`faulthandler.enable` at startup.
 
    .. c:member:: wchar_t* filesystem_encoding
 
@@ -504,6 +505,9 @@ PyConfig
 
       Python home directory.
 
+      Initialized from :envvar:`PYTHONHOME` environment variable value by
+      default.
+
    .. c:member:: int import_time
 
       If non-zero, profile import time.
@@ -561,7 +565,7 @@ PyConfig
 
       :data:`sys.path`. If :c:member:`~PyConfig.module_search_paths_set` is
       equal to 0, the :c:member:`~PyConfig.module_search_paths` is overridden
-      by the function computing the :ref:`Path Configuration
+      by the function calculating the :ref:`Path Configuration
       <init-path-config>`.
 
    .. c:member:: int optimization_level
@@ -586,9 +590,9 @@ PyConfig
 
    .. c:member:: int pathconfig_warnings
 
-      If equal to 0, suppress warnings when computing the path configuration
-      (Unix only, Windows does not log any warning). Otherwise, warnings are
-      written into ``stderr``.
+      If equal to 0, suppress warnings when calculating the :ref:`Path
+      Configuration <init-path-config>` (Unix only, Windows does not log any
+      warning). Otherwise, warnings are written into ``stderr``.
 
    .. c:member:: wchar_t* prefix
 
@@ -596,39 +600,46 @@ PyConfig
 
    .. c:member:: wchar_t* program_name
 
-      Program name.
+      Program name. Used to initialize :c:member:`~PyConfig.executable`, and in
+      early error messages.
 
    .. c:member:: wchar_t* pycache_prefix
 
-      ``.pyc`` cache prefix.
+      :data:`sys.pycache_prefix`: ``.pyc`` cache prefix.
+
+      If NULL, :data:`sys.pycache_prefix` is set to ``None``.
 
    .. c:member:: int quiet
 
       Quiet mode. For example, don't display the copyright and version messages
-      even in interactive mode.
+      in interactive mode.
 
    .. c:member:: wchar_t* run_command
 
-      ``python3 -c COMMAND`` argument.
+      ``python3 -c COMMAND`` argument. Used by :c:func:`Py_RunMain`.
 
    .. c:member:: wchar_t* run_filename
 
-      ``python3 FILENAME`` argument.
+      ``python3 FILENAME`` argument. Used by :c:func:`Py_RunMain`.
 
    .. c:member:: wchar_t* run_module
 
-      ``python3 -m MODULE`` argument.
+      ``python3 -m MODULE`` argument. Used by :c:func:`Py_RunMain`.
 
    .. c:member:: int show_alloc_count
 
       Show allocation counts at exit?
 
+      Set to 1 by :option:`-X showalloccount <-X>` command line option.
+
       Need a special Python build with ``COUNT_ALLOCS`` macro defined.
 
    .. c:member:: int show_ref_count
 
       Show total reference count at exit?
 
+      Set to 1 by :option:`-X showrefcount <-X>` command line option.
+
       Need a debug build of Python (``Py_REF_DEBUG`` macro must be defined).
 
    .. c:member:: int site_import
@@ -647,7 +658,7 @@ PyConfig
 
    .. c:member:: int tracemalloc
 
-      If non-zero, call :func:`tracemalloc.start`.
+      If non-zero, call :func:`tracemalloc.start` at startup.
 
    .. c:member:: int use_environment
 
@@ -669,6 +680,9 @@ PyConfig
 
       If non-zero, write ``.pyc`` files.
 
+      :data:`sys.dont_write_bytecode` is initialized to the inverted value of
+      :c:member:`~PyConfig.write_bytecode`.
+
    .. c:member:: PyWideStringList xoptions
 
       :data:`sys._xoptions`.
@@ -694,8 +708,8 @@ Function to initialize Python:
 The caller is responsible to handle exceptions (error or exit) using
 :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`.
 
-``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or
-``PyImport_ExtendInittab()`` is used: they must be set or called after Python
+If ``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or
+``PyImport_ExtendInittab()`` are used, they must be set or called after Python
 preinitialization and before the Python initialization.
 
 Example setting the program name::
@@ -760,7 +774,7 @@ configuration, and then override some parameters::
 
         /* Append our custom search path to sys.path */
         status = PyWideStringList_Append(&config.module_search_paths,
-                                      L"/path/to/more/modules");
+                                         L"/path/to/more/modules");
         if (PyStatus_Exception(status)) {
             goto done;
         }
@@ -791,9 +805,9 @@ isolate Python from the system. For example, to embed Python into an
 application.
 
 This configuration ignores global configuration variables, environments
-variables and command line arguments (:c:member:`PyConfig.argv` is not parsed).
-The C standard streams (ex: ``stdout``) and the LC_CTYPE locale are left
-unchanged by default.
+variables, command line arguments (:c:member:`PyConfig.argv` is not parsed)
+and user site directory. The C standard streams (ex: ``stdout``) and the
+LC_CTYPE locale are left unchanged. Signal handlers are not installed.
 
 Configuration files are still used with this configuration. Set the
 :ref:`Path Configuration <init-path-config>` ("output fields") to ignore these
@@ -864,29 +878,38 @@ Path Configuration
 
 :c:type:`PyConfig` contains multiple fields for the path configuration:
 
-* Path configuration input fields:
+* Path configuration inputs:
 
   * :c:member:`PyConfig.home`
   * :c:member:`PyConfig.pathconfig_warnings`
   * :c:member:`PyConfig.program_name`
   * :c:member:`PyConfig.pythonpath_env`
+  * current working directory: to get absolute paths
+  * ``PATH`` environment variable to get the program full path
+    (from :c:member:`PyConfig.program_name`)
+  * ``__PYVENV_LAUNCHER__`` environment variable
+  * (Windows only) Application paths in the registry under
+    "Software\Python\PythonCore\X.Y\PythonPath" of HKEY_CURRENT_USER and
+    HKEY_LOCAL_MACHINE (where X.Y is the Python version).
 
 * Path configuration output fields:
 
+  * :c:member:`PyConfig.base_exec_prefix`
   * :c:member:`PyConfig.base_executable`
+  * :c:member:`PyConfig.base_prefix`
   * :c:member:`PyConfig.exec_prefix`
   * :c:member:`PyConfig.executable`
-  * :c:member:`PyConfig.prefix`
   * :c:member:`PyConfig.module_search_paths_set`,
     :c:member:`PyConfig.module_search_paths`
+  * :c:member:`PyConfig.prefix`
 
-If at least one "output field" is not set, Python computes the path
+If at least one "output field" is not set, Python calculates the path
 configuration to fill unset fields. If
 :c:member:`~PyConfig.module_search_paths_set` is equal to 0,
 :c:member:`~PyConfig.module_search_paths` is overridden and
 :c:member:`~PyConfig.module_search_paths_set` is set to 1.
 
-It is possible to completely ignore the function computing the default
+It is possible to completely ignore the function calculating the default
 path configuration by setting explicitly all path configuration output
 fields listed above. A string is considered as set even if it is non-empty.
 ``module_search_paths`` is considered as set if
@@ -894,7 +917,7 @@ fields listed above. A string is considered as set even if it is non-empty.
 configuration input fields are ignored as well.
 
 Set :c:member:`~PyConfig.pathconfig_warnings` to 0 to suppress warnings when
-computing the path configuration (Unix only, Windows does not log any warning).
+calculating the path configuration (Unix only, Windows does not log any warning).
 
 If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix`
 fields are not set, they inherit their value from :c:member:`~PyConfig.prefix`
@@ -961,7 +984,7 @@ initialization, the core feature of the :pep:`432`:
   * Builtin exceptions;
   * Builtin and frozen modules;
   * The :mod:`sys` module is only partially initialized
-    (ex: :data:`sys.path` doesn't exist yet);
+    (ex: :data:`sys.path` doesn't exist yet).
 
 * "Main" initialization phase, Python is fully initialized:
 
@@ -987,9 +1010,9 @@ No module is imported during the "Core" phase and the ``importlib`` module is
 not configured: the :ref:`Path Configuration <init-path-config>` is only
 applied during the "Main" phase. It may allow to customize Python in Python to
 override or tune the :ref:`Path Configuration <init-path-config>`, maybe
-install a custom sys.meta_path importer or an import hook, etc.
+install a custom :data:`sys.meta_path` importer or an import hook, etc.
 
-It may become possible to compute the :ref:`Path Configuration
+It may become possible to calculatin the :ref:`Path Configuration
 <init-path-config>` in Python, after the Core phase and before the Main phase,
 which is one of the :pep:`432` motivation.
 
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index c2455f487b14..0995cb3b9119 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -1347,6 +1347,11 @@ Build and C API Changes
   parameter for indicating the number of positional-only arguments.
   (Contributed by Pablo Galindo in :issue:`37221`.)
 
+* :c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full
+  path (:c:func:`Py_GetProgramFullPath`) rather than to the program name
+  (:c:func:`Py_GetProgramName`).
+  (Contributed by Victor Stinner in :issue:`38234`.)
+
 
 Deprecated
 ==========
diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h
index 61b3790fe1f4..ce75ccee835a 100644
--- a/Include/internal/pycore_pathconfig.h
+++ b/Include/internal/pycore_pathconfig.h
@@ -19,6 +19,7 @@ typedef struct _PyPathConfig {
     wchar_t *program_name;
     /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */
     wchar_t *home;
+#ifdef MS_WINDOWS
     /* isolated and site_import are used to set Py_IsolatedFlag and
        Py_NoSiteFlag flags on Windows in read_pth_file(). These fields
        are ignored when their value are equal to -1 (unset). */
@@ -26,12 +27,18 @@ typedef struct _PyPathConfig {
     int site_import;
     /* Set when a venv is detected */
     wchar_t *base_executable;
+#endif
 } _PyPathConfig;
 
-#define _PyPathConfig_INIT \
-    {.module_search_path = NULL, \
-     .isolated = -1, \
-     .site_import = -1}
+#ifdef MS_WINDOWS
+#  define _PyPathConfig_INIT \
+      {.module_search_path = NULL, \
+       .isolated = -1, \
+       .site_import = -1}
+#else
+#  define _PyPathConfig_INIT \
+      {.module_search_path = NULL}
+#endif
 /* Note: _PyPathConfig_INIT sets other fields to 0/NULL */
 
 PyAPI_DATA(_PyPathConfig) _Py_path_config;
@@ -59,7 +66,7 @@ extern int _Py_FindEnvConfigValue(
 extern wchar_t* _Py_GetDLLPath(void);
 #endif
 
-extern PyStatus _PyPathConfig_Init(void);
+extern PyStatus _PyConfig_WritePathConfig(const PyConfig *config);
 extern void _Py_DumpPathConfig(PyThreadState *tstate);
 
 #ifdef __cplusplus
diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py
index e76e6927cb1f..b9e2fafbc084 100644
--- a/Lib/sysconfig.py
+++ b/Lib/sysconfig.py
@@ -412,7 +412,7 @@ def _generate_posix_vars():
         pprint.pprint(vars, stream=f)
 
     # Create file used for sys.path fixup -- see Modules/getpath.c
-    with open('pybuilddir.txt', 'w', encoding='ascii') as f:
+    with open('pybuilddir.txt', 'w', encoding='utf8') as f:
         f.write(pybuilddir)
 
 def _init_posix(vars):
diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py
index e9edf675b910..d2fa6c593659 100644
--- a/Lib/test/pythoninfo.py
+++ b/Lib/test/pythoninfo.py
@@ -161,6 +161,25 @@ def collect_builtins(info_add):
     info_add('builtins.float.double_format', float.__getformat__("double"))
 
 
+def collect_urandom(info_add):
+    import os
+
+    if hasattr(os, 'getrandom'):
+        # PEP 524: Check if system urandom is initialized
+        try:
+            try:
+                os.getrandom(1, os.GRND_NONBLOCK)
+                state = 'ready (initialized)'
+            except BlockingIOError as exc:
+                state = 'not seeded yet (%s)' % exc
+            info_add('os.getrandom', state)
+        except OSError as exc:
+            # Python was compiled on a more recent Linux version
+            # than the current Linux kernel: ignore OSError(ENOSYS)
+            if exc.errno != errno.ENOSYS:
+                raise
+
+
 def collect_os(info_add):
     import os
 
@@ -180,16 +199,16 @@ def format_attr(attr, value):
     )
     copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr)
 
-    call_func(info_add, 'os.cwd', os, 'getcwd')
+    call_func(info_add, 'os.getcwd', os, 'getcwd')
 
-    call_func(info_add, 'os.uid', os, 'getuid')
-    call_func(info_add, 'os.gid', os, 'getgid')
+    call_func(info_add, 'os.getuid', os, 'getuid')
+    call_func(info_add, 'os.getgid', os, 'getgid')
     call_func(info_add, 'os.uname', os, 'uname')
 
     def format_groups(groups):
         return ', '.join(map(str, groups))
 
-    call_func(info_add, 'os.groups', os, 'getgroups', formatter=format_groups)
+    call_func(info_add, 'os.getgroups', os, 'getgroups', formatter=format_groups)
 
     if hasattr(os, 'getlogin'):
         try:
@@ -202,7 +221,7 @@ def format_groups(groups):
             info_add("os.login", login)
 
     call_func(info_add, 'os.cpu_count', os, 'cpu_count')
-    call_func(info_add, 'os.loadavg', os, 'getloadavg')
+    call_func(info_add, 'os.getloadavg', os, 'getloadavg')
 
     # Environment variables used by the stdlib and tests. Don't log the full
     # environment: filter to list to not leak sensitive information.
@@ -286,20 +305,32 @@ def format_groups(groups):
         os.umask(mask)
         info_add("os.umask", '%03o' % mask)
 
-    if hasattr(os, 'getrandom'):
-        # PEP 524: Check if system urandom is initialized
-        try:
-            try:
-                os.getrandom(1, os.GRND_NONBLOCK)
-                state = 'ready (initialized)'
-            except BlockingIOError as exc:
-                state = 'not seeded yet (%s)' % exc
-            info_add('os.getrandom', state)
-        except OSError as exc:
-            # Python was compiled on a more recent Linux version
-            # than the current Linux kernel: ignore OSError(ENOSYS)
-            if exc.errno != errno.ENOSYS:
-                raise
+
+def collect_pwd(info_add):
+    try:
+        import pwd
+    except ImportError:
+        return
+    import os
+
+    uid = os.getuid()
+    try:
+        entry = pwd.getpwuid(uid)
+    except KeyError:
+        entry = None
+
+    info_add('pwd.getpwuid(%s)'% uid,
+             entry if entry is not None else '<KeyError>')
+
+    if entry is None:
+        # there is nothing interesting to read if the current user identifier
+        # is not the password database
+        return
+
+    if hasattr(os, 'getgrouplist'):
+        groups = os.getgrouplist(entry.pw_name, entry.pw_gid)
+        groups = ', '.join(map(str, groups))
+        info_add('os.getgrouplist', groups)
 
 
 def collect_readline(info_add):
@@ -620,36 +651,71 @@ def collect_subprocess(info_add):
     copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',))
 
 
+def collect_windows(info_add):
+    try:
+        import ctypes
+    except ImportError:
+        return
+
+    if not hasattr(ctypes, 'WinDLL'):
+        return
+
+    ntdll = ctypes.WinDLL('ntdll')
+    BOOLEAN = ctypes.c_ubyte
+
+    try:
+        RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled
+    except AttributeError:
+        res = '<function not available>'
+    else:
+        RtlAreLongPathsEnabled.restype = BOOLEAN
+        RtlAreLongPathsEnabled.argtypes = ()
+        res = bool(RtlAreLongPathsEnabled())
+    info_add('windows.RtlAreLongPathsEnabled', res)
+
+    try:
+        import _winapi
+        dll_path = _winapi.GetModuleFileName(sys.dllhandle)
+        info_add('windows.dll_path', dll_path)
+    except (ImportError, AttributeError):
+        pass
+
+
 def collect_info(info):
     error = False
     info_add = info.add
 
     for collect_func in (
-        # collect_os() should be the first, to check the getrandom() status
-        collect_os,
+        # collect_urandom() must be the first, to check the getrandom() status.
+        # Other functions may block on os.urandom() indirectly and so change
+        # its state.
+        collect_urandom,
 
         collect_builtins,
+        collect_cc,
+        collect_datetime,
+        collect_decimal,
+        collect_expat,
         collect_gdb,
+        collect_gdbm,
+        collect_get_config,
         collect_locale,
+        collect_os,
         collect_platform,
+        collect_pwd,
         collect_readline,
+        collect_resource,
         collect_socket,
         collect_sqlite,
         collect_ssl,
+        collect_subprocess,
         collect_sys,
         collect_sysconfig,
+        collect_testcapi,
         collect_time,
-        collect_datetime,
         collect_tkinter,
+        collect_windows,
         collect_zlib,
-        collect_expat,
-        collect_decimal,
-        collect_testcapi,
-        collect_resource,
-        collect_cc,
-        collect_gdbm,
-        collect_get_config,
-        collect_subprocess,
 
         # Collecting from tests should be last as they have side effects.
         collect_test_socket,
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index e02acbc6befe..ed2b96fbfc1f 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -3,15 +3,19 @@
 import unittest
 
 from collections import namedtuple
+import contextlib
 import json
 import os
 import re
+import shutil
 import subprocess
 import sys
+import tempfile
 import textwrap
 
 
 MS_WINDOWS = (os.name == 'nt')
+MACOS = (sys.platform == 'darwin')
 
 PYMEM_ALLOCATOR_NOT_SET = 0
 PYMEM_ALLOCATOR_DEBUG = 2
@@ -25,6 +29,12 @@
 API_ISOLATED = 3
 
 
+def debug_build(program):
+    program = os.path.basename(program)
+    name = os.path.splitext(program)[0]
+    return name.endswith("_d")
+
+
 def remove_python_envvars():
     env = dict(os.environ)
     # Remove PYTHON* environment variables to get deterministic environment
@@ -40,7 +50,7 @@ def setUp(self):
         basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
         exename = "_testembed"
         if MS_WINDOWS:
-            ext = ("_d" if "_d" in sys.executable else "") + ".exe"
+            ext = ("_d" if debug_build(sys.executable) else "") + ".exe"
             exename += ext
             exepath = os.path.dirname(sys.executable)
         else:
@@ -58,7 +68,8 @@ def tearDown(self):
         os.chdir(self.oldcwd)
 
     def run_embedded_interpreter(self, *args, env=None,
-                                 timeout=None, returncode=0, input=None):
+                                 timeout=None, returncode=0, input=None,
+                                 cwd=None):
         """Runs a test in the embedded interpreter"""
         cmd = [self.test_exe]
         cmd.extend(args)
@@ -72,7 +83,8 @@ def run_embedded_interpreter(self, *args, env=None,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE,
                              universal_newlines=True,
-                             env=env)
+                             env=env,
+                             cwd=cwd)
         try:
             (out, err) = p.communicate(input=input, timeout=timeout)
         except:
@@ -460,6 +472,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
     EXPECTED_CONFIG = None
 
+    @classmethod
+    def tearDownClass(cls):
+        # clear cache
+        cls.EXPECTED_CONFIG = None
+
     def main_xoptions(self, xoptions_list):
         xoptions = {}
         for opt in xoptions_list:
@@ -470,7 +487,8 @@ def main_xoptions(self, xoptions_list):
                 xoptions[opt] = True
         return xoptions
 
-    def _get_expected_config(self, env):
+    def _get_expected_config_impl(self):
+        env = remove_python_envvars()
         code = textwrap.dedent('''
             import json
             import sys
@@ -489,23 +507,37 @@ def _get_expected_config(self, env):
         args = [sys.executable, '-S', '-c', code]
         proc = subprocess.run(args, env=env,
                               stdout=subprocess.PIPE,
-                              stderr=subprocess.STDOUT)
+                              stderr=subprocess.PIPE)
         if proc.returncode:
             raise Exception(f"failed to get the default config: "
                             f"stdout={proc.stdout!r} stderr={proc.stderr!r}")
         stdout = proc.stdout.decode('utf-8')
+        # ignore stderr
         try:
             return json.loads(stdout)
         except json.JSONDecodeError:
             self.fail(f"fail to decode stdout: {stdout!r}")
 
+    def _get_expected_config(self):
+        cls = InitConfigTests
+        if cls.EXPECTED_CONFIG is None:
+            cls.EXPECTED_CONFIG = self._get_expected_config_impl()
+
+        # get a copy
+        configs = {}
+        for config_key, config_value in cls.EXPECTED_CONFIG.items():
+            config = {}
+            for key, value in config_value.items():
+                if isinstance(value, list):
+                    value = value.copy()
+                config[key] = value
+            configs[config_key] = config
+        return configs
+
     def get_expected_config(self, expected_preconfig, expected, env, api,
                             modify_path_cb=None):
         cls = self.__class__
-        if cls.EXPECTED_CONFIG is None:
-            cls.EXPECTED_CONFIG = self._get_expected_config(env)
-        configs = {key: dict(value)
-                   for key, value in self.EXPECTED_CONFIG.items()}
+        configs = self._get_expected_config()
 
         pre_config = configs['pre_config']
         for key, value in expected_preconfig.items():
@@ -553,9 +585,10 @@ def get_expected_config(self, expected_preconfig, expected, env, api,
             if value is self.GET_DEFAULT_CONFIG:
                 expected[key] = config[key]
 
-        prepend_path = expected['pythonpath_env']
-        if prepend_path is not None:
-            expected['module_search_paths'] = [prepend_path, *expected['module_search_paths']]
+        pythonpath_env = expected['pythonpath_env']
+        if pythonpath_env is not None:
+            paths = pythonpath_env.split(os.path.pathsep)
+            expected['module_search_paths'] = [*paths, *expected['module_search_paths']]
         if modify_path_cb is not None:
             expected['module_search_paths'] = expected['module_search_paths'].copy()
             modify_path_cb(expected['module_search_paths'])
@@ -603,13 +636,19 @@ def check_global_config(self, configs):
         self.assertEqual(configs['global_config'], expected)
 
     def check_all_configs(self, testname, expected_config=None,
-                     expected_preconfig=None, modify_path_cb=None, stderr=None,
-                     *, api):
-        env = remove_python_envvars()
-
-        if api == API_ISOLATED:
+                          expected_preconfig=None, modify_path_cb=None,
+                          stderr=None, *, api, preconfig_api=None,
+                          env=None, ignore_stderr=False, cwd=None):
+        new_env = remove_python_envvars()
+        if env is not None:
+            new_env.update(env)
+        env = new_env
+
+        if preconfig_api is None:
+            preconfig_api = api
+        if preconfig_api == API_ISOLATED:
             default_preconfig = self.PRE_CONFIG_ISOLATED
-        elif api == API_PYTHON:
+        elif preconfig_api == API_PYTHON:
             default_preconfig = self.PRE_CONFIG_PYTHON
         else:
             default_preconfig = self.PRE_CONFIG_COMPAT
@@ -631,10 +670,11 @@ def check_all_configs(self, testname, expected_config=None,
                                  expected_config, env,
                                  api, modify_path_cb)
 
-        out, err = self.run_embedded_interpreter(testname, env=env)
+        out, err = self.run_embedded_interpreter(testname,
+                                                 env=env, cwd=cwd)
         if stderr is None and not expected_config['verbose']:
             stderr = ""
-        if stderr is not None:
+        if stderr is not None and not ignore_stderr:
             self.assertEqual(err.rstrip(), stderr)
         try:
             configs = json.loads(out)
@@ -965,6 +1005,268 @@ def test_init_dont_parse_argv(self):
         self.check_all_configs("test_init_dont_parse_argv", config, pre_config,
                                api=API_PYTHON)
 
+    def default_program_name(self, config):
+        if MS_WINDOWS:
+            program_name = 'python'
+            executable = self.test_exe
+        else:
+            program_name = 'python3'
+            if MACOS:
+                executable = self.test_exe
+            else:
+                executable = shutil.which(program_name) or ''
+        config.update({
+            'program_name': program_name,
+            'base_executable': executable,
+            'executable': executable,
+        })
+
+    def test_init_setpath(self):
+        # Test Py_SetPath()
+        config = self._get_expected_config()
+        paths = config['config']['module_search_paths']
+
+        config = {
+            'module_search_paths': paths,
+            'prefix': '',
+            'base_prefix': '',
+            'exec_prefix': '',
+            'base_exec_prefix': '',
+        }
+        self.default_program_name(config)
+        env = {'TESTPATH': os.path.pathsep.join(paths)}
+        self.check_all_configs("test_init_setpath", config,
+                               api=API_COMPAT, env=env,
+                               ignore_stderr=True)
+
+    def test_init_setpath_config(self):
+        # Test Py_SetPath() with PyConfig
+        config = self._get_expected_config()
+        paths = config['config']['module_search_paths']
+
+        config = {
+            # set by Py_SetPath()
+            'module_search_paths': paths,
+            'prefix': '',
+            'base_prefix': '',
+            'exec_prefix': '',
+            'base_exec_prefix': '',
+            # overriden by PyConfig
+            'program_name': 'conf_program_name',
+            'base_executable': 'conf_executable',
+            'executable': 'conf_executable',
+        }
+        env = {'TESTPATH': os.path.pathsep.join(paths)}
+        self.check_all_configs("test_init_setpath_config", config,
+                               api=API_PYTHON, env=env, ignore_stderr=True)
+
+    def module_search_paths(self, prefix=None, exec_prefix=None):
+        config = self._get_expected_config()
+        if prefix is None:
+            prefix = config['config']['prefix']
+        if exec_prefix is None:
+            exec_prefix = config['config']['prefix']
+        if MS_WINDOWS:
+            return config['config']['module_search_paths']
+        else:
+            ver = sys.version_info
+            return [
+                os.path.join(prefix, 'lib',
+                             f'python{ver.major}{ver.minor}.zip'),
+                os.path.join(prefix, 'lib',
+                             f'python{ver.major}.{ver.minor}'),
+                os.path.join(exec_prefix, 'lib',
+                             f'python{ver.major}.{ver.minor}', 'lib-dynload'),
+            ]
+
+    @contextlib.contextmanager
+    def tmpdir_with_python(self):
+        # Temporary directory with a copy of the Python program
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # bpo-38234: On macOS and FreeBSD, the temporary directory
+            # can be symbolic link. For example, /tmp can be a symbolic link
+            # to /var/tmp. Call realpath() to resolve all symbolic links.
+            tmpdir = os.path.realpath(tmpdir)
+
+            if MS_WINDOWS:
+                # Copy pythonXY.dll (or pythonXY_d.dll)
+                ver = sys.version_info
+                dll = f'python{ver.major}{ver.minor}'
+                if debug_build(sys.executable):
+                    dll += '_d'
+                dll += '.dll'
+                dll = os.path.join(os.path.dirname(self.test_exe), dll)
+                dll_copy = os.path.join(tmpdir, os.path.basename(dll))
+                shutil.copyfile(dll, dll_copy)
+
+            # Copy Python program
+            exec_copy = os.path.join(tmpdir, os.path.basename(self.test_exe))
+            shutil.copyfile(self.test_exe, exec_copy)
+            shutil.copystat(self.test_exe, exec_copy)
+            self.test_exe = exec_copy
+
+            yield tmpdir
+
+    def test_init_setpythonhome(self):
+        # Test Py_SetPythonHome(home) with PYTHONPATH env var
+        config = self._get_expected_config()
+        paths = config['config']['module_search_paths']
+        paths_str = os.path.pathsep.join(paths)
+
+        for path in paths:
+            if not os.path.isdir(path):
+                continue
+            if os.path.exists(os.path.join(path, 'os.py')):
+                home = os.path.dirname(path)
+                break
+        else:
+            self.fail(f"Unable to find home in {paths!r}")
+
+        prefix = exec_prefix = home
+        ver = sys.version_info
+        expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)
+
+        config = {
+            'home': home,
+            'module_search_paths': expected_paths,
+            'prefix': prefix,
+            'base_prefix': prefix,
+            'exec_prefix': exec_prefix,
+            'base_exec_prefix': exec_prefix,
+            'pythonpath_env': paths_str,
+        }
+        self.default_program_name(config)
+        env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
+        self.check_all_configs("test_init_setpythonhome", config,
+                               api=API_COMPAT, env=env)
+
+    def copy_paths_by_env(self, config):
+        all_configs = self._get_expected_config()
+        paths = all_configs['config']['module_search_paths']
+        paths_str = os.path.pathsep.join(paths)
+        config['pythonpath_env'] = paths_str
+        env = {'PYTHONPATH': paths_str}
+        return env
+
+    @unittest.skipIf(MS_WINDOWS, 'Windows does not use pybuilddir.txt')
+    def test_init_pybuilddir(self):
+        # Test path configuration with pybuilddir.txt configuration file
+
+        with self.tmpdir_with_python() as tmpdir:
+            # pybuilddir.txt is a sub-directory relative to the current
+            # directory (tmpdir)
+            subdir = 'libdir'
+            libdir = os.path.join(tmpdir, subdir)
+            os.mkdir(libdir)
+
+            filename = os.path.join(tmpdir, 'pybuilddir.txt')
+            with open(filename, "w", encoding="utf8") as fp:
+                fp.write(subdir)
+
+            module_search_paths = self.module_search_paths()
+            module_search_paths[-1] = libdir
+
+            executable = self.test_exe
+            config = {
+                'base_executable': executable,
+                'executable': executable,
+                'module_search_paths': module_search_paths,
+            }
+            env = self.copy_paths_by_env(config)
+            self.check_all_configs("test_init_compat_config", config,
+                                   api=API_COMPAT, env=env,
+                                   ignore_stderr=True, cwd=tmpdir)
+
+    def test_init_pyvenv_cfg(self):
+        # Test path configuration with pyvenv.cfg configuration file
+
+        with self.tmpdir_with_python() as tmpdir, \
+             tempfile.TemporaryDirectory() as pyvenv_home:
+            ver = sys.version_info
+
+            if not MS_WINDOWS:
+                lib_dynload = os.path.join(pyvenv_home,
+                                           'lib',
+                                           f'python{ver.major}.{ver.minor}',
+                                           'lib-dynload')
+                os.makedirs(lib_dynload)
+            else:
+                lib_dynload = os.path.join(pyvenv_home, 'lib')
+                os.makedirs(lib_dynload)
+                # getpathp.c uses Lib\os.py as the LANDMARK
+                shutil.copyfile(os.__file__, os.path.join(lib_dynload, 'os.py'))
+
+            filename = os.path.join(tmpdir, 'pyvenv.cfg')
+            with open(filename, "w", encoding="utf8") as fp:
+                print("home = %s" % pyvenv_home, file=fp)
+                print("include-system-site-packages = false", file=fp)
+
+            paths = self.module_search_paths()
+            if not MS_WINDOWS:
+                paths[-1] = lib_dynload
+            else:
+                for index, path in enumerate(paths):
+                    if index == 0:
+                        paths[index] = os.path.join(tmpdir, os.path.basename(path))
+                    else:
+                        paths[index] = os.path.join(pyvenv_home, os.path.basename(path))
+                paths[-1] = pyvenv_home
+
+            executable = self.test_exe
+            exec_prefix = pyvenv_home
+            config = {
+                'base_exec_prefix': exec_prefix,
+                'exec_prefix': exec_prefix,
+                'base_executable': executable,
+                'executable': executable,
+                'module_search_paths': paths,
+            }
+            if MS_WINDOWS:
+                config['base_prefix'] = pyvenv_home
+                config['prefix'] = pyvenv_home
+            env = self.copy_paths_by_env(config)
+            self.check_all_configs("test_init_compat_config", config,
+                                   api=API_COMPAT, env=env,
+                                   ignore_stderr=True, cwd=tmpdir)
+
+    def test_global_pathconfig(self):
+        # Test C API functions getting the path configuration:
+        #
+        # - Py_GetExecPrefix()
+        # - Py_GetPath()
+        # - Py_GetPrefix()
+        # - Py_GetProgramFullPath()
+        # - Py_GetProgramName()
+        # - Py_GetPythonHome()
+        #
+        # The global path configuration (_Py_path_config) must be a copy
+        # of the path configuration of PyInterpreter.config (PyConfig).
+        ctypes = support.import_module('ctypes')
+        _testinternalcapi = support.import_module('_testinternalcapi')
+
+        def get_func(name):
+            func = getattr(ctypes.pythonapi, name)
+            func.argtypes = ()
+            func.restype = ctypes.c_wchar_p
+            return func
+
+        Py_GetPath = get_func('Py_GetPath')
+        Py_GetPrefix = get_func('Py_GetPrefix')
+        Py_GetExecPrefix = get_func('Py_GetExecPrefix')
+        Py_GetProgramName = get_func('Py_GetProgramName')
+        Py_GetProgramFullPath = get_func('Py_GetProgramFullPath')
+        Py_GetPythonHome = get_func('Py_GetPythonHome')
+
+        config = _testinternalcapi.get_configs()['config']
+
+        self.assertEqual(Py_GetPath().split(os.path.pathsep),
+                         config['module_search_paths'])
+        self.assertEqual(Py_GetPrefix(), config['prefix'])
+        self.assertEqual(Py_GetExecPrefix(), config['exec_prefix'])
+        self.assertEqual(Py_GetProgramName(), config['program_name'])
+        self.assertEqual(Py_GetProgramFullPath(), config['executable'])
+        self.assertEqual(Py_GetPythonHome(), config['home'])
+
 
 class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
     def test_open_code_hook(self):
diff --git a/Misc/NEWS.d/next/C API/2019-09-24-17-09-48.bpo-38234.d0bhEA.rst b/Misc/NEWS.d/next/C API/2019-09-24-17-09-48.bpo-38234.d0bhEA.rst
new file mode 100644
index 000000000000..ba4cc312e692
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-09-24-17-09-48.bpo-38234.d0bhEA.rst	
@@ -0,0 +1,3 @@
+:c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full
+path (:c:func:`Py_GetProgramFullPath`) rather than to the program name
+(:c:func:`Py_GetProgramName`).
diff --git a/Modules/getpath.c b/Modules/getpath.c
index de32c3d51702..b727f66953b4 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -370,12 +370,15 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
                   const wchar_t *argv0_path,
                   wchar_t *prefix, size_t prefix_len, int *found)
 {
+    wchar_t path[MAXPATHLEN+1];
+    memset(path, 0, sizeof(path));
+    size_t path_len = Py_ARRAY_LENGTH(path);
+
     PyStatus status;
-    size_t n;
-    wchar_t *vpath;
 
     /* If PYTHONHOME is set, we believe it unconditionally */
     if (pathconfig->home) {
+        /* Path: <home> / <lib_python> */
         if (safe_wcscpy(prefix, pathconfig->home, prefix_len) < 0) {
             return PATHLEN_ERR();
         }
@@ -387,27 +390,25 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
-        status = joinpath(prefix, LANDMARK, prefix_len);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
         *found = 1;
         return _PyStatus_OK();
     }
 
     /* Check to see if argv[0] is in the build directory */
-    if (safe_wcscpy(prefix, argv0_path, prefix_len) < 0) {
+    if (safe_wcscpy(path, argv0_path, path_len) < 0) {
         return PATHLEN_ERR();
     }
-    status = joinpath(prefix, L"Modules/Setup.local", prefix_len);
+    status = joinpath(path, L"Modules/Setup.local", path_len);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
-    if (isfile(prefix)) {
-        /* Check VPATH to see if argv0_path is in the build directory. */
-        vpath = Py_DecodeLocale(VPATH, NULL);
+    if (isfile(path)) {
+        /* Check VPATH to see if argv0_path is in the build directory.
+           VPATH can be empty. */
+        wchar_t *vpath = Py_DecodeLocale(VPATH, NULL);
         if (vpath != NULL) {
+            /* Path: <argv0_path> / <vpath> / Lib / LANDMARK */
             if (safe_wcscpy(prefix, argv0_path, prefix_len) < 0) {
                 return PATHLEN_ERR();
             }
@@ -428,6 +429,7 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
             if (ismodule(prefix, prefix_len)) {
                 *found = -1;
+                reduce(prefix);
                 return _PyStatus_OK();
             }
         }
@@ -440,7 +442,8 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
     }
 
     do {
-        n = wcslen(prefix);
+        /* Path: <argv0_path or substring> / <lib_python> / LANDMARK */
+        size_t n = wcslen(prefix);
         status = joinpath(prefix, calculate->lib_python, prefix_len);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
@@ -452,13 +455,15 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
         if (ismodule(prefix, prefix_len)) {
             *found = 1;
+            reduce(prefix);
             return _PyStatus_OK();
         }
         prefix[n] = L'\0';
         reduce(prefix);
     } while (prefix[0]);
 
-    /* Look at configure's PREFIX */
+    /* Look at configure's PREFIX.
+       Path: <PREFIX macro> / <lib_python> / LANDMARK */
     if (safe_wcscpy(prefix, calculate->prefix, prefix_len) < 0) {
         return PATHLEN_ERR();
     }
@@ -473,6 +478,7 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 
     if (ismodule(prefix, prefix_len)) {
         *found = 1;
+        reduce(prefix);
         return _PyStatus_OK();
     }
 
@@ -509,9 +515,6 @@ calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
             return status;
         }
     }
-    else {
-        reduce(prefix);
-    }
     return _PyStatus_OK();
 }
 
@@ -546,6 +549,67 @@ calculate_set_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
 }
 
 
+static PyStatus
+calculate_pybuilddir(const wchar_t *argv0_path,
+                     wchar_t *exec_prefix, size_t exec_prefix_len,
+                     int *found)
+{
+    PyStatus status;
+
+    wchar_t filename[MAXPATHLEN+1];
+    memset(filename, 0, sizeof(filename));
+    size_t filename_len = Py_ARRAY_LENGTH(filename);
+
+    /* Check to see if argv[0] is in the build directory. "pybuilddir.txt"
+       is written by setup.py and contains the relative path to the location
+       of shared library modules.
+
+       Filename: <argv0_path> / "pybuilddir.txt" */
+    if (safe_wcscpy(filename, argv0_path, filename_len) < 0) {
+        return PATHLEN_ERR();
+    }
+    status = joinpath(filename, L"pybuilddir.txt", filename_len);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
+    if (!isfile(filename)) {
+        return _PyStatus_OK();
+    }
+
+    FILE *fp = _Py_wfopen(filename, L"rb");
+    if (fp == NULL) {
+        errno = 0;
+        return _PyStatus_OK();
+    }
+
+    char buf[MAXPATHLEN + 1];
+    size_t n = fread(buf, 1, Py_ARRAY_LENGTH(buf) - 1, fp);
+    buf[n] = '\0';
+    fclose(fp);
+
+    size_t dec_len;
+    wchar_t *pybuilddir = _Py_DecodeUTF8_surrogateescape(buf, n, &dec_len);
+    if (!pybuilddir) {
+        return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len);
+    }
+
+    /* Path: <argv0_path> / <pybuilddir content> */
+    if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
+        PyMem_RawFree(pybuilddir);
+        return PATHLEN_ERR();
+    }
+    status = joinpath(exec_prefix, pybuilddir, exec_prefix_len);
+    PyMem_RawFree(pybuilddir);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
+    *found = -1;
+    return _PyStatus_OK();
+}
+
+
 /* search_for_exec_prefix requires that argv0_path be no more than
    MAXPATHLEN bytes long.
 */
@@ -556,10 +620,10 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
                        int *found)
 {
     PyStatus status;
-    size_t n;
 
     /* If PYTHONHOME is set, we believe it unconditionally */
     if (pathconfig->home) {
+        /* Path: <home> / <lib_python> / "lib-dynload" */
         wchar_t *delim = wcschr(pathconfig->home, DELIM);
         if (delim) {
             if (safe_wcscpy(exec_prefix, delim+1, exec_prefix_len) < 0) {
@@ -583,47 +647,15 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
         return _PyStatus_OK();
     }
 
-    /* Check to see if argv[0] is in the build directory. "pybuilddir.txt"
-       is written by setup.py and contains the relative path to the location
-       of shared library modules. */
-    if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
-        return PATHLEN_ERR();
-    }
-    status = joinpath(exec_prefix, L"pybuilddir.txt", exec_prefix_len);
+    /* Check for pybuilddir.txt */
+    assert(*found == 0);
+    status = calculate_pybuilddir(argv0_path, exec_prefix, exec_prefix_len,
+                                  found);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
-
-    if (isfile(exec_prefix)) {
-        FILE *f = _Py_wfopen(exec_prefix, L"rb");
-        if (f == NULL) {
-            errno = 0;
-        }
-        else {
-            char buf[MAXPATHLEN + 1];
-            n = fread(buf, 1, Py_ARRAY_LENGTH(buf) - 1, f);
-            buf[n] = '\0';
-            fclose(f);
-
-            wchar_t *pybuilddir;
-            size_t dec_len;
-            pybuilddir = _Py_DecodeUTF8_surrogateescape(buf, n, &dec_len);
-            if (!pybuilddir) {
-                return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len);
-            }
-
-            if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
-                return PATHLEN_ERR();
-            }
-            status = joinpath(exec_prefix, pybuilddir, exec_prefix_len);
-            PyMem_RawFree(pybuilddir );
-            if (_PyStatus_EXCEPTION(status)) {
-                return status;
-            }
-
-            *found = -1;
-            return _PyStatus_OK();
-        }
+    if (*found) {
+        return _PyStatus_OK();
     }
 
     /* Search from argv0_path, until root is found */
@@ -633,7 +665,8 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
     }
 
     do {
-        n = wcslen(exec_prefix);
+        /* Path: <argv0_path or substring> / <lib_python> / "lib-dynload" */
+        size_t n = wcslen(exec_prefix);
         status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
@@ -650,7 +683,9 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
         reduce(exec_prefix);
     } while (exec_prefix[0]);
 
-    /* Look at configure's EXEC_PREFIX */
+    /* Look at configure's EXEC_PREFIX.
+
+       Path: <EXEC_PREFIX macro> / <lib_python> / "lib-dynload" */
     if (safe_wcscpy(exec_prefix, calculate->exec_prefix, exec_prefix_len) < 0) {
         return PATHLEN_ERR();
     }
@@ -962,43 +997,49 @@ calculate_read_pyenv(PyCalculatePath *calculate,
                      wchar_t *argv0_path, size_t argv0_path_len)
 {
     PyStatus status;
-    wchar_t tmpbuffer[MAXPATHLEN+1];
-    const size_t buflen = Py_ARRAY_LENGTH(tmpbuffer);
-    wchar_t *env_cfg = L"pyvenv.cfg";
+    const wchar_t *env_cfg = L"pyvenv.cfg";
     FILE *env_file;
 
-    if (safe_wcscpy(tmpbuffer, argv0_path, buflen) < 0) {
+    wchar_t filename[MAXPATHLEN+1];
+    const size_t filename_len = Py_ARRAY_LENGTH(filename);
+    memset(filename, 0, sizeof(filename));
+
+    /* Filename: <argv0_path_len> / "pyvenv.cfg" */
+    if (safe_wcscpy(filename, argv0_path, filename_len) < 0) {
         return PATHLEN_ERR();
     }
 
-    status = joinpath(tmpbuffer, env_cfg, buflen);
+    status = joinpath(filename, env_cfg, filename_len);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
-    env_file = _Py_wfopen(tmpbuffer, L"r");
+    env_file = _Py_wfopen(filename, L"r");
     if (env_file == NULL) {
         errno = 0;
 
-        reduce(tmpbuffer);
-        reduce(tmpbuffer);
-        status = joinpath(tmpbuffer, env_cfg, buflen);
+        /* Filename: <basename(basename(argv0_path_len))> / "pyvenv.cfg" */
+        reduce(filename);
+        reduce(filename);
+        status = joinpath(filename, env_cfg, filename_len);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
 
-        env_file = _Py_wfopen(tmpbuffer, L"r");
+        env_file = _Py_wfopen(filename, L"r");
         if (env_file == NULL) {
             errno = 0;
+            return _PyStatus_OK();
         }
     }
 
-    if (env_file == NULL) {
-        return _PyStatus_OK();
-    }
-
     /* Look for a 'home' variable and set argv0_path to it, if found */
-    if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, buflen)) {
-        if (safe_wcscpy(argv0_path, tmpbuffer, argv0_path_len) < 0) {
+    wchar_t home[MAXPATHLEN+1];
+    memset(home, 0, sizeof(home));
+
+    if (_Py_FindEnvConfigValue(env_file, L"home",
+                               home, Py_ARRAY_LENGTH(home))) {
+        if (safe_wcscpy(argv0_path, home, argv0_path_len) < 0) {
+            fclose(env_file);
             return PATHLEN_ERR();
         }
     }
@@ -1012,12 +1053,12 @@ calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix,
                    wchar_t *zip_path, size_t zip_path_len)
 {
     PyStatus status;
-    if (safe_wcscpy(zip_path, prefix, zip_path_len) < 0) {
-        return PATHLEN_ERR();
-    }
 
     if (calculate->prefix_found > 0) {
         /* Use the reduced prefix returned by Py_GetPrefix() */
+        if (safe_wcscpy(zip_path, prefix, zip_path_len) < 0) {
+            return PATHLEN_ERR();
+        }
         reduce(zip_path);
         reduce(zip_path);
     }
@@ -1200,6 +1241,8 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
         return status;
     }
 
+    /* If a pyvenv.cfg configure file is found,
+       argv0_path is overriden with its 'home' variable. */
     status = calculate_read_pyenv(calculate,
                                   argv0_path, Py_ARRAY_LENGTH(argv0_path));
     if (_PyStatus_EXCEPTION(status)) {
diff --git a/PC/getpathp.c b/PC/getpathp.c
index c4c0636dddeb..8bac592aefdb 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -757,34 +757,34 @@ static void
 calculate_pyvenv_file(PyCalculatePath *calculate,
                       wchar_t *argv0_path, size_t argv0_path_len)
 {
-    wchar_t envbuffer[MAXPATHLEN+1];
+    wchar_t filename[MAXPATHLEN+1];
     const wchar_t *env_cfg = L"pyvenv.cfg";
 
-    wcscpy_s(envbuffer, MAXPATHLEN+1, argv0_path);
-    join(envbuffer, env_cfg);
+    /* Filename: <argv0_path_len> / "pyvenv.cfg" */
+    wcscpy_s(filename, MAXPATHLEN+1, argv0_path);
+    join(filename, env_cfg);
 
-    FILE *env_file = _Py_wfopen(envbuffer, L"r");
+    FILE *env_file = _Py_wfopen(filename, L"r");
     if (env_file == NULL) {
         errno = 0;
 
-        reduce(envbuffer);
-        reduce(envbuffer);
-        join(envbuffer, env_cfg);
+        /* Filename: <basename(basename(argv0_path_len))> / "pyvenv.cfg" */
+        reduce(filename);
+        reduce(filename);
+        join(filename, env_cfg);
 
-        env_file = _Py_wfopen(envbuffer, L"r");
+        env_file = _Py_wfopen(filename, L"r");
         if (env_file == NULL) {
             errno = 0;
+            return;
         }
     }
 
-    if (env_file == NULL) {
-        return;
-    }
-
     /* Look for a 'home' variable and set argv0_path to it, if found */
-    wchar_t tmpbuffer[MAXPATHLEN+1];
-    if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, MAXPATHLEN)) {
-        wcscpy_s(argv0_path, argv0_path_len, tmpbuffer);
+    wchar_t home[MAXPATHLEN+1];
+    if (_Py_FindEnvConfigValue(env_file, L"home",
+                               home, Py_ARRAY_LENGTH(home))) {
+        wcscpy_s(argv0_path, argv0_path_len, home);
     }
     fclose(env_file);
 }
@@ -1099,11 +1099,11 @@ calculate_free(PyCalculatePath *calculate)
    - __PYVENV_LAUNCHER__ environment variable
    - GetModuleFileNameW(NULL): fully qualified path of the executable file of
      the current process
-   - .pth configuration file
+   - ._pth configuration file
    - pyvenv.cfg configuration file
    - Registry key "Software\Python\PythonCore\X.Y\PythonPath"
-     of HKEY_LOCAL_MACHINE and HKEY_CURRENT_USER where X.Y is the Python
-     version (major.minor).
+     of HKEY_CURRENT_USER and HKEY_LOCAL_MACHINE where X.Y is the Python
+     version.
 
    Outputs, 'pathconfig' fields:
 
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index c3ccc0ec325b..83c266b885af 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -20,11 +20,12 @@
  * Executed via 'EmbeddingTests' in Lib/test/test_capi.py
  *********************************************************/
 
+/* Use path starting with "./" avoids a search along the PATH */
+#define PROGRAM_NAME L"./_testembed"
+
 static void _testembed_Py_Initialize(void)
 {
-    /* HACK: the "./" at front avoids a search along the PATH in
-       Modules/getpath.c */
-    Py_SetProgramName(L"./_testembed");
+    Py_SetProgramName(PROGRAM_NAME);
     Py_Initialize();
 }
 
@@ -363,8 +364,7 @@ config_set_wide_string_list(PyConfig *config, PyWideStringList *list,
 
 static void config_set_program_name(PyConfig *config)
 {
-    /* Use path starting with "./" avoids a search along the PATH */
-    const wchar_t *program_name = L"./_testembed";
+    const wchar_t *program_name = PROGRAM_NAME;
     config_set_string(config, &config->program_name, program_name);
 }
 
@@ -1263,7 +1263,7 @@ static int _audit_hook_run(const char *eventName, PyObject *args, void *userData
 static int test_audit_run_command(void)
 {
     AuditRunCommandTest test = {"cpython.run_command"};
-    wchar_t *argv[] = {L"./_testembed", L"-c", L"pass"};
+    wchar_t *argv[] = {PROGRAM_NAME, L"-c", L"pass"};
 
     Py_IgnoreEnvironmentFlag = 0;
     PySys_AddAuditHook(_audit_hook_run, (void*)&test);
@@ -1274,7 +1274,7 @@ static int test_audit_run_command(void)
 static int test_audit_run_file(void)
 {
     AuditRunCommandTest test = {"cpython.run_file"};
-    wchar_t *argv[] = {L"./_testembed", L"filename.py"};
+    wchar_t *argv[] = {PROGRAM_NAME, L"filename.py"};
 
     Py_IgnoreEnvironmentFlag = 0;
     PySys_AddAuditHook(_audit_hook_run, (void*)&test);
@@ -1312,21 +1312,21 @@ static int run_audit_run_test(int argc, wchar_t **argv, void *test)
 static int test_audit_run_interactivehook(void)
 {
     AuditRunCommandTest test = {"cpython.run_interactivehook", 10};
-    wchar_t *argv[] = {L"./_testembed"};
+    wchar_t *argv[] = {PROGRAM_NAME};
     return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test);
 }
 
 static int test_audit_run_startup(void)
 {
     AuditRunCommandTest test = {"cpython.run_startup", 10};
-    wchar_t *argv[] = {L"./_testembed"};
+    wchar_t *argv[] = {PROGRAM_NAME};
     return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test);
 }
 
 static int test_audit_run_stdin(void)
 {
     AuditRunCommandTest test = {"cpython.run_stdin"};
-    wchar_t *argv[] = {L"./_testembed"};
+    wchar_t *argv[] = {PROGRAM_NAME};
     return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test);
 }
 
@@ -1423,6 +1423,95 @@ static int test_init_sys_add(void)
 }
 
 
+static int test_init_setpath(void)
+{
+    char *env = getenv("TESTPATH");
+    if (!env) {
+        fprintf(stderr, "missing TESTPATH env var\n");
+        return 1;
+    }
+    wchar_t *path = Py_DecodeLocale(env, NULL);
+    if (path == NULL) {
+        fprintf(stderr, "failed to decode TESTPATH\n");
+        return 1;
+    }
+    Py_SetPath(path);
+    PyMem_RawFree(path);
+    putenv("TESTPATH=");
+
+    Py_Initialize();
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
+static int test_init_setpath_config(void)
+{
+    PyStatus status;
+    PyPreConfig preconfig;
+    PyPreConfig_InitPythonConfig(&preconfig);
+
+    /* Explicitly preinitializes with Python preconfiguration to avoid
+      Py_SetPath() implicit preinitialization with compat preconfiguration. */
+    status = Py_PreInitialize(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    char *env = getenv("TESTPATH");
+    if (!env) {
+        fprintf(stderr, "missing TESTPATH env var\n");
+        return 1;
+    }
+    wchar_t *path = Py_DecodeLocale(env, NULL);
+    if (path == NULL) {
+        fprintf(stderr, "failed to decode TESTPATH\n");
+        return 1;
+    }
+    Py_SetPath(path);
+    PyMem_RawFree(path);
+    putenv("TESTPATH=");
+
+    PyConfig config;
+
+    status = PyConfig_InitPythonConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+    config_set_string(&config, &config.program_name, L"conf_program_name");
+    config_set_string(&config, &config.executable, L"conf_executable");
+    init_from_config_clear(&config);
+
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
+static int test_init_setpythonhome(void)
+{
+    char *env = getenv("TESTHOME");
+    if (!env) {
+        fprintf(stderr, "missing TESTHOME env var\n");
+        return 1;
+    }
+    wchar_t *home = Py_DecodeLocale(env, NULL);
+    if (home == NULL) {
+        fprintf(stderr, "failed to decode TESTHOME\n");
+        return 1;
+    }
+    Py_SetPythonHome(home);
+    PyMem_RawFree(home);
+    putenv("TESTHOME=");
+
+    Py_Initialize();
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
 static void configure_init_main(PyConfig *config)
 {
     wchar_t* argv[] = {
@@ -1559,7 +1648,11 @@ static struct TestCase TestCases[] = {
     {"test_init_run_main", test_init_run_main},
     {"test_init_main", test_init_main},
     {"test_init_sys_add", test_init_sys_add},
+    {"test_init_setpath", test_init_setpath},
+    {"test_init_setpath_config", test_init_setpath_config},
+    {"test_init_setpythonhome", test_init_setpythonhome},
     {"test_run_main", test_run_main},
+
     {"test_open_code_hook", test_open_code_hook},
     {"test_audit", test_audit},
     {"test_audit_subinterpreter", test_audit_subinterpreter},
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index 09533100b463..d5b8b1acff8a 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -23,6 +23,7 @@ wchar_t *_Py_dll_path = NULL;
 static int
 copy_wstr(wchar_t **dst, const wchar_t *src)
 {
+    assert(*dst == NULL);
     if (src != NULL) {
         *dst = _PyMem_RawWcsdup(src);
         if (*dst == NULL) {
@@ -57,7 +58,10 @@ pathconfig_clear(_PyPathConfig *config)
     CLEAR(config->module_search_path);
     CLEAR(config->program_name);
     CLEAR(config->home);
+#ifdef MS_WINDOWS
     CLEAR(config->base_executable);
+#endif
+
 #undef CLEAR
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
@@ -82,9 +86,11 @@ pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2)
     COPY_ATTR(module_search_path);
     COPY_ATTR(program_name);
     COPY_ATTR(home);
+#ifdef MS_WINDOWS
     config->isolated = config2->isolated;
     config->site_import = config2->site_import;
     COPY_ATTR(base_executable);
+#endif
 
 #undef COPY_ATTR
 
@@ -127,7 +133,7 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep)
     for (Py_ssize_t i=0; i < list->length; i++) {
         wchar_t *path = list->items[i];
         if (i != 0) {
-            *str++ = SEP;
+            *str++ = sep;
         }
         len = wcslen(path);
         memcpy(str, path, len * sizeof(wchar_t));
@@ -139,11 +145,11 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep)
 }
 
 
+#ifdef MS_WINDOWS
 /* Initialize _Py_dll_path on Windows. Do nothing on other platforms. */
-PyStatus
-_PyPathConfig_Init(void)
+static PyStatus
+_PyPathConfig_InitDLLPath(void)
 {
-#ifdef MS_WINDOWS
     if (_Py_dll_path == NULL) {
         /* Already set: nothing to do */
         return _PyStatus_OK();
@@ -159,9 +165,9 @@ _PyPathConfig_Init(void)
     if (_Py_dll_path == NULL) {
         return _PyStatus_NO_MEMORY();
     }
-#endif
     return _PyStatus_OK();
 }
+#endif
 
 
 static PyStatus
@@ -172,6 +178,7 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
     if (config->module_search_paths_set) {
+        PyMem_RawFree(pathconfig->module_search_path);
         pathconfig->module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM);
         if (pathconfig->module_search_path == NULL) {
             goto no_memory;
@@ -180,17 +187,21 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
 
 #define COPY_CONFIG(PATH_ATTR, CONFIG_ATTR) \
         if (config->CONFIG_ATTR) { \
+            PyMem_RawFree(pathconfig->PATH_ATTR); \
+            pathconfig->PATH_ATTR = NULL; \
             if (copy_wstr(&pathconfig->PATH_ATTR, config->CONFIG_ATTR) < 0) { \
                 goto no_memory; \
             } \
         }
 
-    COPY_CONFIG(base_executable, base_executable);
     COPY_CONFIG(program_full_path, executable);
     COPY_CONFIG(prefix, prefix);
     COPY_CONFIG(exec_prefix, exec_prefix);
     COPY_CONFIG(program_name, program_name);
     COPY_CONFIG(home, home);
+#ifdef MS_WINDOWS
+    COPY_CONFIG(base_executable, base_executable);
+#endif
 
 #undef COPY_CONFIG
 
@@ -206,6 +217,20 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
 }
 
 
+PyStatus
+_PyConfig_WritePathConfig(const PyConfig *config)
+{
+#ifdef MS_WINDOWS
+    PyStatus status = _PyPathConfig_InitDLLPath();
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+#endif
+
+    return pathconfig_set_from_config(&_Py_path_config, config);
+}
+
+
 static PyStatus
 config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig)
 {
@@ -326,18 +351,32 @@ config_calculate_pathconfig(PyConfig *config)
             } \
         }
 
+#ifdef MS_WINDOWS
+    if (config->executable != NULL && config->base_executable == NULL) {
+        /* If executable is set explicitly in the configuration,
+           ignore calculated base_executable: _PyConfig_InitPathConfig()
+           will copy executable to base_executable */
+    }
+    else {
+        COPY_ATTR(base_executable, base_executable);
+    }
+#endif
+
     COPY_ATTR(program_full_path, executable);
     COPY_ATTR(prefix, prefix);
     COPY_ATTR(exec_prefix, exec_prefix);
-    COPY_ATTR(base_executable, base_executable);
+
 #undef COPY_ATTR
 
+#ifdef MS_WINDOWS
+    /* If a ._pth file is found: isolated and site_import are overriden */
     if (pathconfig.isolated != -1) {
         config->isolated = pathconfig.isolated;
     }
     if (pathconfig.site_import != -1) {
         config->site_import = pathconfig.site_import;
     }
+#endif
 
     status = _PyStatus_OK();
     goto done;
@@ -356,9 +395,9 @@ _PyConfig_InitPathConfig(PyConfig *config)
 {
     /* Do we need to calculate the path? */
     if (!config->module_search_paths_set
-        || (config->executable == NULL)
-        || (config->prefix == NULL)
-        || (config->exec_prefix == NULL))
+        || config->executable == NULL
+        || config->prefix == NULL
+        || config->exec_prefix == NULL)
     {
         PyStatus status = config_calculate_pathconfig(config);
         if (_PyStatus_EXCEPTION(status)) {
@@ -416,11 +455,12 @@ pathconfig_global_init(void)
 {
     PyStatus status;
 
-    /* Initialize _Py_dll_path if needed */
-    status = _PyPathConfig_Init();
+#ifdef MS_WINDOWS
+    status = _PyPathConfig_InitDLLPath();
     if (_PyStatus_EXCEPTION(status)) {
         Py_ExitStatusException(status);
     }
+#endif
 
     if (_Py_path_config.module_search_path == NULL) {
         status = pathconfig_global_read(&_Py_path_config);
@@ -438,7 +478,9 @@ pathconfig_global_init(void)
     assert(_Py_path_config.module_search_path != NULL);
     assert(_Py_path_config.program_name != NULL);
     /* home can be NULL */
+#ifdef MS_WINDOWS
     assert(_Py_path_config.base_executable != NULL);
+#endif
 }
 
 
@@ -455,16 +497,15 @@ Py_SetPath(const wchar_t *path)
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    /* Getting the program name calls pathconfig_global_init() */
-    wchar_t *program_name = _PyMem_RawWcsdup(Py_GetProgramName());
+    /* Getting the program full path calls pathconfig_global_init() */
+    wchar_t *program_full_path = _PyMem_RawWcsdup(Py_GetProgramFullPath());
 
     PyMem_RawFree(_Py_path_config.program_full_path);
     PyMem_RawFree(_Py_path_config.prefix);
     PyMem_RawFree(_Py_path_config.exec_prefix);
     PyMem_RawFree(_Py_path_config.module_search_path);
 
-    /* Copy program_name to program_full_path  */
-    _Py_path_config.program_full_path = program_name;
+    _Py_path_config.program_full_path = program_full_path;
     _Py_path_config.prefix = _PyMem_RawWcsdup(L"");
     _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L"");
     _Py_path_config.module_search_path = _PyMem_RawWcsdup(path);
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 4d7873fd5224..42a062e10865 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -472,7 +472,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime,
     config = &interp->config;
 
     if (config->_install_importlib) {
-        status = _PyPathConfig_Init();
+        status = _PyConfig_WritePathConfig(config);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
@@ -641,7 +641,7 @@ pycore_init_import_warnings(PyInterpreterState *interp, PyObject *sysmod)
     }
 
     if (config->_install_importlib) {
-        status = _PyPathConfig_Init();
+        status = _PyConfig_WritePathConfig(config);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }


From webhook-mailer at python.org  Thu Sep 26 10:54:29 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 26 Sep 2019 14:54:29 -0000
Subject: [Python-checkins] bpo-38239: Fix test_gdb for Link Time
 Optimization (LTO) (GH-16422)
Message-ID: <mailman.107.1569509670.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/64b4a3a2deabcd4103fac2759a311fe94159b4d1
commit: 64b4a3a2deabcd4103fac2759a311fe94159b4d1
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-26T16:54:13+02:00
summary:

bpo-38239: Fix test_gdb for Link Time Optimization (LTO) (GH-16422)

files:
A Misc/NEWS.d/next/Tests/2019-09-26-15-48-36.bpo-38239.MfoVzY.rst
M Lib/test/test_gdb.py

diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py
index e515d0d3fb51..eb94f92961f6 100644
--- a/Lib/test/test_gdb.py
+++ b/Lib/test/test_gdb.py
@@ -255,8 +255,15 @@ def get_gdb_repr(self, source,
         # gdb can insert additional '\n' and space characters in various places
         # in its output, depending on the width of the terminal it's connected
         # to (using its "wrap_here" function)
-        m = re.match(r'.*#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)\s+at\s+\S*Python/bltinmodule.c.*',
-                     gdb_output, re.DOTALL)
+        m = re.search(
+            # Match '#0 builtin_id(self=..., v=...)'
+            r'#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)'
+            # Match ' at Python/bltinmodule.c'.
+            # bpo-38239: builtin_id() is defined in Python/bltinmodule.c,
+            # but accept any "Directory\file.c" to support Link Time
+            # Optimization (LTO).
+            r'\s+at\s+\S*[A-Za-z]+/[A-Za-z0-9_-]+\.c',
+            gdb_output, re.DOTALL)
         if not m:
             self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output))
         return m.group(1), gdb_output
diff --git a/Misc/NEWS.d/next/Tests/2019-09-26-15-48-36.bpo-38239.MfoVzY.rst b/Misc/NEWS.d/next/Tests/2019-09-26-15-48-36.bpo-38239.MfoVzY.rst
new file mode 100644
index 000000000000..f79da29fa182
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-26-15-48-36.bpo-38239.MfoVzY.rst
@@ -0,0 +1 @@
+Fix test_gdb for Link Time Optimization (LTO) builds.


From webhook-mailer at python.org  Thu Sep 26 11:03:04 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 26 Sep 2019 15:03:04 -0000
Subject: [Python-checkins] bpo-38275: Skip ssl tests for disabled versions
 (GH-16386)
Message-ID: <mailman.108.1569510185.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/df6ac7e2b82d921a6e9ff5571b40c6dbcf635581
commit: df6ac7e2b82d921a6e9ff5571b40c6dbcf635581
branch: master
author: Christian Heimes <christian at python.org>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-26T08:02:59-07:00
summary:

bpo-38275: Skip ssl tests for disabled versions (GH-16386)



test_ssl now handles disabled TLS/SSL versions better. OpenSSL's crypto
policy and run-time settings are recognized and tests for disabled versions
are skipped.

Signed-off-by: Christian Heimes <christian at python.org>



https://bugs.python.org/issue38275

files:
A Misc/NEWS.d/next/Tests/2019-09-25-14-40-57.bpo-38275.-kdveI.rst
M Lib/test/test_ssl.py

diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 737a96989968..8bb74b438895 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -19,6 +19,7 @@
 import weakref
 import platform
 import sysconfig
+import functools
 try:
     import ctypes
 except ImportError:
@@ -146,6 +147,85 @@ def data_file(*name):
 OP_ENABLE_MIDDLEBOX_COMPAT = getattr(ssl, "OP_ENABLE_MIDDLEBOX_COMPAT", 0)
 
 
+def has_tls_protocol(protocol):
+    """Check if a TLS protocol is available and enabled
+
+    :param protocol: enum ssl._SSLMethod member or name
+    :return: bool
+    """
+    if isinstance(protocol, str):
+        assert protocol.startswith('PROTOCOL_')
+        protocol = getattr(ssl, protocol, None)
+        if protocol is None:
+            return False
+    if protocol in {
+        ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS_SERVER,
+        ssl.PROTOCOL_TLS_CLIENT
+    }:
+        # auto-negotiate protocols are always available
+        return True
+    name = protocol.name
+    return has_tls_version(name[len('PROTOCOL_'):])
+
+
+ at functools.lru_cache
+def has_tls_version(version):
+    """Check if a TLS/SSL version is enabled
+
+    :param version: TLS version name or ssl.TLSVersion member
+    :return: bool
+    """
+    if version == "SSLv2":
+        # never supported and not even in TLSVersion enum
+        return False
+
+    if isinstance(version, str):
+        version = ssl.TLSVersion.__members__[version]
+
+    # check compile time flags like ssl.HAS_TLSv1_2
+    if not getattr(ssl, f'HAS_{version.name}'):
+        return False
+
+    # check runtime and dynamic crypto policy settings. A TLS version may
+    # be compiled in but disabled by a policy or config option.
+    ctx = ssl.SSLContext()
+    if (
+            ctx.minimum_version != ssl.TLSVersion.MINIMUM_SUPPORTED and
+            version < ctx.minimum_version
+    ):
+        return False
+    if (
+        ctx.maximum_version != ssl.TLSVersion.MAXIMUM_SUPPORTED and
+        version > ctx.maximum_version
+    ):
+        return False
+
+    return True
+
+
+def requires_tls_version(version):
+    """Decorator to skip tests when a required TLS version is not available
+
+    :param version: TLS version name or ssl.TLSVersion member
+    :return:
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kw):
+            if not has_tls_version(version):
+                raise unittest.SkipTest(f"{version} is not available.")
+            else:
+                return func(*args, **kw)
+        return wrapper
+    return decorator
+
+
+requires_minimum_version = unittest.skipUnless(
+    hasattr(ssl.SSLContext, 'minimum_version'),
+    "required OpenSSL >= 1.1.0g"
+)
+
+
 def handle_error(prefix):
     exc_format = ' '.join(traceback.format_exception(*sys.exc_info()))
     if support.verbose:
@@ -1107,20 +1187,23 @@ def test_hostname_checks_common_name(self):
             with self.assertRaises(AttributeError):
                 ctx.hostname_checks_common_name = True
 
-    @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'),
-                         "required OpenSSL 1.1.0g")
+    @requires_minimum_version
     @unittest.skipIf(IS_LIBRESSL, "see bpo-34001")
     def test_min_max_version(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         # OpenSSL default is MINIMUM_SUPPORTED, however some vendors like
         # Fedora override the setting to TLS 1.0.
+        minimum_range = {
+            # stock OpenSSL
+            ssl.TLSVersion.MINIMUM_SUPPORTED,
+            # Fedora 29 uses TLS 1.0 by default
+            ssl.TLSVersion.TLSv1,
+            # RHEL 8 uses TLS 1.2 by default
+            ssl.TLSVersion.TLSv1_2
+        }
+
         self.assertIn(
-            ctx.minimum_version,
-            {ssl.TLSVersion.MINIMUM_SUPPORTED,
-             # Fedora 29 uses TLS 1.0 by default
-             ssl.TLSVersion.TLSv1,
-             # RHEL 8 uses TLS 1.2 by default
-             ssl.TLSVersion.TLSv1_2}
+            ctx.minimum_version, minimum_range
         )
         self.assertEqual(
             ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED
@@ -1166,8 +1249,8 @@ def test_min_max_version(self):
 
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1)
 
-        self.assertEqual(
-            ctx.minimum_version, ssl.TLSVersion.MINIMUM_SUPPORTED
+        self.assertIn(
+            ctx.minimum_version, minimum_range
         )
         self.assertEqual(
             ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED
@@ -2722,6 +2805,8 @@ def test_echo(self):
         for protocol in PROTOCOLS:
             if protocol in {ssl.PROTOCOL_TLS_CLIENT, ssl.PROTOCOL_TLS_SERVER}:
                 continue
+            if not has_tls_protocol(protocol):
+                continue
             with self.subTest(protocol=ssl._PROTOCOL_NAMES[protocol]):
                 context = ssl.SSLContext(protocol)
                 context.load_cert_chain(CERTFILE)
@@ -3013,7 +3098,7 @@ def test_wrong_cert_tls12(self):
             else:
                 self.fail("Use of invalid cert should have failed!")
 
-    @unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3")
+    @requires_tls_version('TLSv1_3')
     def test_wrong_cert_tls13(self):
         client_context, server_context, hostname = testing_context()
         # load client cert that is not signed by trusted CA
@@ -3108,8 +3193,7 @@ def test_ssl_cert_verify_error(self):
                     self.assertIn(msg, repr(e))
                     self.assertIn('certificate verify failed', repr(e))
 
-    @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'),
-                         "OpenSSL is compiled without SSLv2 support")
+    @requires_tls_version('SSLv2')
     def test_protocol_sslv2(self):
         """Connecting to an SSLv2 server with various client options"""
         if support.verbose:
@@ -3118,7 +3202,7 @@ def test_protocol_sslv2(self):
         try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL)
         try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED)
         try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLS, False)
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False)
         # SSLv23 client with specific SSL options
@@ -3135,7 +3219,7 @@ def test_PROTOCOL_TLS(self):
         """Connecting to an SSLv23 server with various client options"""
         if support.verbose:
             sys.stdout.write("\n")
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try:
                 try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv2, True)
             except OSError as x:
@@ -3144,34 +3228,36 @@ def test_PROTOCOL_TLS(self):
                     sys.stdout.write(
                         " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n"
                         % str(x))
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True)
-        try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1')
+        if has_tls_version('TLSv1'):
+            try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1')
 
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL)
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_OPTIONAL)
-        try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL)
+        if has_tls_version('TLSv1'):
+            try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL)
 
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED)
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_REQUIRED)
-        try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED)
+        if has_tls_version('TLSv1'):
+            try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED)
 
         # Server with specific SSL options
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False,
                            server_options=ssl.OP_NO_SSLv3)
         # Will choose TLSv1
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True,
                            server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
-        try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, False,
-                           server_options=ssl.OP_NO_TLSv1)
-
+        if has_tls_version('TLSv1'):
+            try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, False,
+                               server_options=ssl.OP_NO_TLSv1)
 
-    @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv3'),
-                         "OpenSSL is compiled without SSLv3 support")
+    @requires_tls_version('SSLv3')
     def test_protocol_sslv3(self):
         """Connecting to an SSLv3 server with various client options"""
         if support.verbose:
@@ -3179,7 +3265,7 @@ def test_protocol_sslv3(self):
         try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3')
         try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL)
         try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED)
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
         try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLS, False,
                            client_options=ssl.OP_NO_SSLv3)
@@ -3189,6 +3275,7 @@ def test_protocol_sslv3(self):
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLS,
                                False, client_options=ssl.OP_NO_SSLv2)
 
+    @requires_tls_version('TLSv1')
     def test_protocol_tlsv1(self):
         """Connecting to a TLSv1 server with various client options"""
         if support.verbose:
@@ -3196,34 +3283,32 @@ def test_protocol_tlsv1(self):
         try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1')
         try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL)
         try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED)
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLS, False,
                            client_options=ssl.OP_NO_TLSv1)
 
-    @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_1"),
-                         "TLS version 1.1 not supported.")
+    @requires_tls_version('TLSv1_1')
     def test_protocol_tlsv1_1(self):
         """Connecting to a TLSv1.1 server with various client options.
            Testing against older TLS versions."""
         if support.verbose:
             sys.stdout.write("\n")
         try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1')
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False)
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLS, False,
                            client_options=ssl.OP_NO_TLSv1_1)
 
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1')
-        try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False)
-        try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False)
+        try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False)
+        try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False)
 
-    @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"),
-                         "TLS version 1.2 not supported.")
+    @requires_tls_version('TLSv1_2')
     def test_protocol_tlsv1_2(self):
         """Connecting to a TLSv1.2 server with various client options.
            Testing against older TLS versions."""
@@ -3232,9 +3317,9 @@ def test_protocol_tlsv1_2(self):
         try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2',
                            server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,
                            client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,)
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False)
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLS, False,
                            client_options=ssl.OP_NO_TLSv1_2)
@@ -3677,7 +3762,7 @@ def test_version_basic(self):
                 self.assertIs(s.version(), None)
                 self.assertIs(s._sslobj, None)
                 s.connect((HOST, server.port))
-                if IS_OPENSSL_1_1_1 and ssl.HAS_TLSv1_3:
+                if IS_OPENSSL_1_1_1 and has_tls_version('TLSv1_3'):
                     self.assertEqual(s.version(), 'TLSv1.3')
                 elif ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
                     self.assertEqual(s.version(), 'TLSv1.2')
@@ -3686,8 +3771,7 @@ def test_version_basic(self):
             self.assertIs(s._sslobj, None)
             self.assertIs(s.version(), None)
 
-    @unittest.skipUnless(ssl.HAS_TLSv1_3,
-                         "test requires TLSv1.3 enabled OpenSSL")
+    @requires_tls_version('TLSv1_3')
     def test_tls1_3(self):
         context = ssl.SSLContext(ssl.PROTOCOL_TLS)
         context.load_cert_chain(CERTFILE)
@@ -3704,9 +3788,9 @@ def test_tls1_3(self):
                 })
                 self.assertEqual(s.version(), 'TLSv1.3')
 
-    @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'),
-                         "required OpenSSL 1.1.0g")
-    def test_min_max_version(self):
+    @requires_minimum_version
+    @requires_tls_version('TLSv1_2')
+    def test_min_max_version_tlsv1_2(self):
         client_context, server_context, hostname = testing_context()
         # client TLSv1.0 to 1.2
         client_context.minimum_version = ssl.TLSVersion.TLSv1
@@ -3721,7 +3805,13 @@ def test_min_max_version(self):
                 s.connect((HOST, server.port))
                 self.assertEqual(s.version(), 'TLSv1.2')
 
+    @requires_minimum_version
+    @requires_tls_version('TLSv1_1')
+    def test_min_max_version_tlsv1_1(self):
+        client_context, server_context, hostname = testing_context()
         # client 1.0 to 1.2, server 1.0 to 1.1
+        client_context.minimum_version = ssl.TLSVersion.TLSv1
+        client_context.maximum_version = ssl.TLSVersion.TLSv1_2
         server_context.minimum_version = ssl.TLSVersion.TLSv1
         server_context.maximum_version = ssl.TLSVersion.TLSv1_1
 
@@ -3731,6 +3821,10 @@ def test_min_max_version(self):
                 s.connect((HOST, server.port))
                 self.assertEqual(s.version(), 'TLSv1.1')
 
+    @requires_minimum_version
+    @requires_tls_version('TLSv1_2')
+    def test_min_max_version_mismatch(self):
+        client_context, server_context, hostname = testing_context()
         # client 1.0, server 1.2 (mismatch)
         server_context.maximum_version = ssl.TLSVersion.TLSv1_2
         server_context.minimum_version = ssl.TLSVersion.TLSv1_2
@@ -3743,10 +3837,8 @@ def test_min_max_version(self):
                     s.connect((HOST, server.port))
                 self.assertIn("alert", str(e.exception))
 
-
-    @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'),
-                         "required OpenSSL 1.1.0g")
-    @unittest.skipUnless(ssl.HAS_SSLv3, "requires SSLv3 support")
+    @requires_minimum_version
+    @requires_tls_version('SSLv3')
     def test_min_max_version_sslv3(self):
         client_context, server_context, hostname = testing_context()
         server_context.minimum_version = ssl.TLSVersion.SSLv3
@@ -4270,7 +4362,7 @@ def test_session_handling(self):
                                  'Session refers to a different SSLContext.')
 
 
- at unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3")
+ at unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3")
 class TestPostHandshakeAuth(unittest.TestCase):
     def test_pha_setter(self):
         protocols = [
diff --git a/Misc/NEWS.d/next/Tests/2019-09-25-14-40-57.bpo-38275.-kdveI.rst b/Misc/NEWS.d/next/Tests/2019-09-25-14-40-57.bpo-38275.-kdveI.rst
new file mode 100644
index 000000000000..893c0f137aea
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-25-14-40-57.bpo-38275.-kdveI.rst
@@ -0,0 +1,4 @@
+test_ssl now handles disabled TLS/SSL versions better. OpenSSL's crypto
+policy and run-time settings are recognized and tests for disabled versions
+are skipped. Tests also accept more TLS minimum_versions for platforms that
+override OpenSSL's default with strict settings.


From webhook-mailer at python.org  Thu Sep 26 11:13:44 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Thu, 26 Sep 2019 15:13:44 -0000
Subject: [Python-checkins] bpo-38239: Fix test_gdb for Link Time
 Optimization (LTO) (GH-16422)
Message-ID: <mailman.109.1569510824.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/c9893400652f38804aed0be8d8f70feda9465c47
commit: c9893400652f38804aed0be8d8f70feda9465c47
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-26T08:13:39-07:00
summary:

bpo-38239: Fix test_gdb for Link Time Optimization (LTO) (GH-16422)

(cherry picked from commit 64b4a3a2deabcd4103fac2759a311fe94159b4d1)

Co-authored-by: Victor Stinner <vstinner at redhat.com>

files:
A Misc/NEWS.d/next/Tests/2019-09-26-15-48-36.bpo-38239.MfoVzY.rst
M Lib/test/test_gdb.py

diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py
index e07d3273a455..e1060330550e 100644
--- a/Lib/test/test_gdb.py
+++ b/Lib/test/test_gdb.py
@@ -255,8 +255,15 @@ def get_gdb_repr(self, source,
         # gdb can insert additional '\n' and space characters in various places
         # in its output, depending on the width of the terminal it's connected
         # to (using its "wrap_here" function)
-        m = re.match(r'.*#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)\s+at\s+\S*Python/bltinmodule.c.*',
-                     gdb_output, re.DOTALL)
+        m = re.search(
+            # Match '#0 builtin_id(self=..., v=...)'
+            r'#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)'
+            # Match ' at Python/bltinmodule.c'.
+            # bpo-38239: builtin_id() is defined in Python/bltinmodule.c,
+            # but accept any "Directory\file.c" to support Link Time
+            # Optimization (LTO).
+            r'\s+at\s+\S*[A-Za-z]+/[A-Za-z0-9_-]+\.c',
+            gdb_output, re.DOTALL)
         if not m:
             self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output))
         return m.group(1), gdb_output
diff --git a/Misc/NEWS.d/next/Tests/2019-09-26-15-48-36.bpo-38239.MfoVzY.rst b/Misc/NEWS.d/next/Tests/2019-09-26-15-48-36.bpo-38239.MfoVzY.rst
new file mode 100644
index 000000000000..f79da29fa182
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-26-15-48-36.bpo-38239.MfoVzY.rst
@@ -0,0 +1 @@
+Fix test_gdb for Link Time Optimization (LTO) builds.


From webhook-mailer at python.org  Thu Sep 26 11:30:20 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Thu, 26 Sep 2019 15:30:20 -0000
Subject: [Python-checkins] bpo-38239: Fix test_gdb for Link Time
 Optimization (LTO) (GH-16422) (GH-16426)
Message-ID: <mailman.110.1569511820.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/e6b5ed1fbdbb7b44c7ab2a353fa0bc726b073740
commit: e6b5ed1fbdbb7b44c7ab2a353fa0bc726b073740
branch: 3.7
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-26T17:30:14+02:00
summary:

bpo-38239: Fix test_gdb for Link Time Optimization (LTO) (GH-16422) (GH-16426)

(cherry picked from commit 64b4a3a2deabcd4103fac2759a311fe94159b4d1)

files:
A Misc/NEWS.d/next/Tests/2019-09-26-15-48-36.bpo-38239.MfoVzY.rst
M Lib/test/test_gdb.py

diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py
index 7758e1fcd86c..1a9975fb84d1 100644
--- a/Lib/test/test_gdb.py
+++ b/Lib/test/test_gdb.py
@@ -252,8 +252,15 @@ def get_gdb_repr(self, source,
         # gdb can insert additional '\n' and space characters in various places
         # in its output, depending on the width of the terminal it's connected
         # to (using its "wrap_here" function)
-        m = re.match(r'.*#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)\)\s+at\s+\S*Python/bltinmodule.c.*',
-                     gdb_output, re.DOTALL)
+        m = re.search(
+            # Match '#0 builtin_id(self=..., v=...)'
+            r'#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)'
+            # Match ' at Python/bltinmodule.c'.
+            # bpo-38239: builtin_id() is defined in Python/bltinmodule.c,
+            # but accept any "Directory\file.c" to support Link Time
+            # Optimization (LTO).
+            r'\s+at\s+\S*[A-Za-z]+/[A-Za-z0-9_-]+\.c',
+            gdb_output, re.DOTALL)
         if not m:
             self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output))
         return m.group(1), gdb_output
diff --git a/Misc/NEWS.d/next/Tests/2019-09-26-15-48-36.bpo-38239.MfoVzY.rst b/Misc/NEWS.d/next/Tests/2019-09-26-15-48-36.bpo-38239.MfoVzY.rst
new file mode 100644
index 000000000000..f79da29fa182
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-26-15-48-36.bpo-38239.MfoVzY.rst
@@ -0,0 +1 @@
+Fix test_gdb for Link Time Optimization (LTO) builds.


From webhook-mailer at python.org  Thu Sep 26 12:23:33 2019
From: webhook-mailer at python.org (Christian Heimes)
Date: Thu, 26 Sep 2019 16:23:33 -0000
Subject: [Python-checkins] bpo-38275: Fix test_ssl issue caused by GH-16386
 (#16428)
Message-ID: <mailman.111.1569515014.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/9f77268f901cc3e8874e5042361520a0e482476a
commit: 9f77268f901cc3e8874e5042361520a0e482476a
branch: master
author: Christian Heimes <christian at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-26T18:23:17+02:00
summary:

bpo-38275: Fix test_ssl issue caused by GH-16386 (#16428)

Check presence of SSLContext.minimum_version to make tests pass with
old versions of OpenSSL.

Signed-off-by: Christian Heimes <christian at python.org>

files:
M Lib/test/test_ssl.py

diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 8bb74b438895..3cd6e927a469 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -190,11 +190,13 @@ def has_tls_version(version):
     # be compiled in but disabled by a policy or config option.
     ctx = ssl.SSLContext()
     if (
+            hasattr(ctx, 'minimum_version') and
             ctx.minimum_version != ssl.TLSVersion.MINIMUM_SUPPORTED and
             version < ctx.minimum_version
     ):
         return False
     if (
+        hasattr(ctx, 'maximum_version') and
         ctx.maximum_version != ssl.TLSVersion.MAXIMUM_SUPPORTED and
         version > ctx.maximum_version
     ):


From webhook-mailer at python.org  Thu Sep 26 15:43:23 2019
From: webhook-mailer at python.org (Tal Einat)
Date: Thu, 26 Sep 2019 19:43:23 -0000
Subject: [Python-checkins] bpo-28009: Fix uuid.uuid1() and uuid.get_node()
 on AIX (GH-8672)
Message-ID: <mailman.112.1569527005.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/0bcbfa43d55d9558cdcb256d8998366281322080
commit: 0bcbfa43d55d9558cdcb256d8998366281322080
branch: master
author: Michael Felt <aixtools at users.noreply.github.com>
committer: Tal Einat <taleinat+github at gmail.com>
date: 2019-09-26T22:43:15+03:00
summary:

bpo-28009: Fix uuid.uuid1() and uuid.get_node() on AIX (GH-8672)

files:
A Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst
M Lib/test/test_uuid.py
M Lib/uuid.py

diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py
index 92642d239b92..ddf7e6dc1b8e 100644
--- a/Lib/test/test_uuid.py
+++ b/Lib/test/test_uuid.py
@@ -1,4 +1,4 @@
-import unittest.mock
+import unittest
 from test import support
 import builtins
 import contextlib
@@ -15,7 +15,6 @@
 py_uuid = support.import_fresh_module('uuid', blocked=['_uuid'])
 c_uuid = support.import_fresh_module('uuid', fresh=['_uuid'])
 
-
 def importable(name):
     try:
         __import__(name)
@@ -459,7 +458,7 @@ def test_uuid1_eui64(self):
         # uuid.getnode to fall back on uuid._random_getnode, which will
         # generate a valid value.
         too_large_getter = lambda: 1 << 48
-        with unittest.mock.patch.multiple(
+        with mock.patch.multiple(
             self.uuid,
             _node=None,  # Ignore any cached node value.
             _GETTERS=[too_large_getter],
@@ -538,8 +537,8 @@ def mock_generate_time_safe(self, safe_value):
         f = self.uuid._generate_time_safe
         if f is None:
             self.skipTest('need uuid._generate_time_safe')
-        with unittest.mock.patch.object(self.uuid, '_generate_time_safe',
-                                        lambda: (f()[0], safe_value)):
+        with mock.patch.object(self.uuid, '_generate_time_safe',
+                               lambda: (f()[0], safe_value)):
             yield
 
     @unittest.skipUnless(os.name == 'posix', 'POSIX-only test')
@@ -674,27 +673,57 @@ class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase):
 class BaseTestInternals:
     _uuid = py_uuid
 
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    def test_find_mac(self):
+
+    def test_find_under_heading(self):
+        data = '''\
+Name  Mtu   Network     Address           Ipkts Ierrs    Opkts Oerrs  Coll
+en0   1500  link#2      fe.ad.c.1.23.4   1714807956     0 711348489     0     0
+                        01:00:5e:00:00:01
+en0   1500  192.168.129 x071             1714807956     0 711348489     0     0
+                        224.0.0.1
+en0   1500  192.168.90  x071             1714807956     0 711348489     0     0
+                        224.0.0.1
+'''
+
+        def mock_get_command_stdout(command, args):
+            return io.BytesIO(data.encode())
+
+        # The above data is from AIX - with '.' as _MAC_DELIM and strings
+        # shorter than 17 bytes (no leading 0). (_MAC_OMITS_LEADING_ZEROES=True)
+        with mock.patch.multiple(self.uuid,
+                                 _MAC_DELIM=b'.',
+                                 _MAC_OMITS_LEADING_ZEROES=True,
+                                 _get_command_stdout=mock_get_command_stdout):
+            mac = self.uuid._find_mac_under_heading(
+                command='netstat',
+                args='-ian',
+                heading=b'Address',
+            )
+
+        self.assertEqual(mac, 0xfead0c012304)
+
+    def test_find_mac_near_keyword(self):
+        # key and value are on the same line
         data = '''
-fake hwaddr
+fake      Link encap:UNSPEC  hwaddr 00-00
 cscotun0  Link encap:UNSPEC  HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00
 eth0      Link encap:Ethernet  HWaddr 12:34:56:78:90:ab
 '''
 
-        popen = unittest.mock.MagicMock()
-        popen.stdout = io.BytesIO(data.encode())
-
-        with unittest.mock.patch.object(shutil, 'which',
-                                        return_value='/sbin/ifconfig'):
-            with unittest.mock.patch.object(subprocess, 'Popen',
-                                            return_value=popen):
-                mac = self.uuid._find_mac(
-                    command='ifconfig',
-                    args='',
-                    hw_identifiers=[b'hwaddr'],
-                    get_index=lambda x: x + 1,
-                )
+        def mock_get_command_stdout(command, args):
+            return io.BytesIO(data.encode())
+
+        # The above data will only be parsed properly on non-AIX unixes.
+        with mock.patch.multiple(self.uuid,
+                                 _MAC_DELIM=b':',
+                                 _MAC_OMITS_LEADING_ZEROES=False,
+                                 _get_command_stdout=mock_get_command_stdout):
+            mac = self.uuid._find_mac_near_keyword(
+                command='ifconfig',
+                args='',
+                keywords=[b'hwaddr'],
+                get_word_index=lambda x: x + 1,
+            )
 
         self.assertEqual(mac, 0x1234567890ab)
 
diff --git a/Lib/uuid.py b/Lib/uuid.py
index 5f3bc9e8de44..6a436d371a3f 100644
--- a/Lib/uuid.py
+++ b/Lib/uuid.py
@@ -59,6 +59,12 @@
 _LINUX   = platform.system() == 'Linux'
 _WINDOWS = platform.system() == 'Windows'
 
+_MAC_DELIM = b':'
+_MAC_OMITS_LEADING_ZEROES = False
+if _AIX:
+    _MAC_DELIM = b'.'
+    _MAC_OMITS_LEADING_ZEROES = True
+
 RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, RESERVED_FUTURE = [
     'reserved for NCS compatibility', 'specified in RFC 4122',
     'reserved for Microsoft compatibility', 'reserved for future definition']
@@ -347,24 +353,32 @@ def version(self):
         if self.variant == RFC_4122:
             return int((self.int >> 76) & 0xf)
 
-def _popen(command, *args):
-    import os, shutil, subprocess
-    executable = shutil.which(command)
-    if executable is None:
-        path = os.pathsep.join(('/sbin', '/usr/sbin'))
-        executable = shutil.which(command, path=path)
+
+def _get_command_stdout(command, *args):
+    import io, os, shutil, subprocess
+
+    try:
+        path_dirs = os.environ.get('PATH', os.defpath).split(os.pathsep)
+        path_dirs.extend(['/sbin', '/usr/sbin'])
+        executable = shutil.which(command, path=os.pathsep.join(path_dirs))
         if executable is None:
             return None
-    # LC_ALL=C to ensure English output, stderr=DEVNULL to prevent output
-    # on stderr (Note: we don't have an example where the words we search
-    # for are actually localized, but in theory some system could do so.)
-    env = dict(os.environ)
-    env['LC_ALL'] = 'C'
-    proc = subprocess.Popen((executable,) + args,
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.DEVNULL,
-                            env=env)
-    return proc
+        # LC_ALL=C to ensure English output, stderr=DEVNULL to prevent output
+        # on stderr (Note: we don't have an example where the words we search
+        # for are actually localized, but in theory some system could do so.)
+        env = dict(os.environ)
+        env['LC_ALL'] = 'C'
+        proc = subprocess.Popen((executable,) + args,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.DEVNULL,
+                                env=env)
+        if not proc:
+            return None
+        stdout, stderr = proc.communicate()
+        return io.BytesIO(stdout)
+    except (OSError, subprocess.SubprocessError):
+        return None
+
 
 # For MAC (a.k.a. IEEE 802, or EUI-48) addresses, the second least significant
 # bit of the first octet signifies whether the MAC address is universally (0)
@@ -384,40 +398,101 @@ def _popen(command, *args):
 def _is_universal(mac):
     return not (mac & (1 << 41))
 
-def _find_mac(command, args, hw_identifiers, get_index):
+
+def _find_mac_near_keyword(command, args, keywords, get_word_index):
+    """Searches a command's output for a MAC address near a keyword.
+
+    Each line of words in the output is case-insensitively searched for
+    any of the given keywords.  Upon a match, get_word_index is invoked
+    to pick a word from the line, given the index of the match.  For
+    example, lambda i: 0 would get the first word on the line, while
+    lambda i: i - 1 would get the word preceding the keyword.
+    """
+    stdout = _get_command_stdout(command, args)
+    if stdout is None:
+        return None
+
     first_local_mac = None
+    for line in stdout:
+        words = line.lower().rstrip().split()
+        for i in range(len(words)):
+            if words[i] in keywords:
+                try:
+                    word = words[get_word_index(i)]
+                    mac = int(word.replace(_MAC_DELIM, b''), 16)
+                except (ValueError, IndexError):
+                    # Virtual interfaces, such as those provided by
+                    # VPNs, do not have a colon-delimited MAC address
+                    # as expected, but a 16-byte HWAddr separated by
+                    # dashes. These should be ignored in favor of a
+                    # real MAC address
+                    pass
+                else:
+                    if _is_universal(mac):
+                        return mac
+                    first_local_mac = first_local_mac or mac
+    return first_local_mac or None
+
+
+def _find_mac_under_heading(command, args, heading):
+    """Looks for a MAC address under a heading in a command's output.
+
+    The first line of words in the output is searched for the given
+    heading. Words at the same word index as the heading in subsequent
+    lines are then examined to see if they look like MAC addresses.
+    """
+    stdout = _get_command_stdout(command, args)
+    if stdout is None:
+        return None
+
+    keywords = stdout.readline().rstrip().split()
     try:
-        proc = _popen(command, *args.split())
-        if not proc:
-            return None
-        with proc:
-            for line in proc.stdout:
-                words = line.lower().rstrip().split()
-                for i in range(len(words)):
-                    if words[i] in hw_identifiers:
-                        try:
-                            word = words[get_index(i)]
-                            mac = int(word.replace(b':', b''), 16)
-                            if _is_universal(mac):
-                                return mac
-                            first_local_mac = first_local_mac or mac
-                        except (ValueError, IndexError):
-                            # Virtual interfaces, such as those provided by
-                            # VPNs, do not have a colon-delimited MAC address
-                            # as expected, but a 16-byte HWAddr separated by
-                            # dashes. These should be ignored in favor of a
-                            # real MAC address
-                            pass
-    except OSError:
-        pass
+        column_index = keywords.index(heading)
+    except ValueError:
+        return None
+
+    first_local_mac = None
+    for line in stdout:
+        try:
+            words = line.rstrip().split()
+            word = words[column_index]
+            if len(word) == 17:
+                mac = int(word.replace(_MAC_DELIM, b''), 16)
+            elif _MAC_OMITS_LEADING_ZEROES:
+                # (Only) on AIX the macaddr value given is not prefixed by 0, e.g.
+                # en0   1500  link#2      fa.bc.de.f7.62.4 110854824     0 160133733     0     0
+                # not
+                # en0   1500  link#2      fa.bc.de.f7.62.04 110854824     0 160133733     0     0
+                parts = word.split(_MAC_DELIM)
+                if len(parts) == 6 and all(0 < len(p) <= 2 for p in parts):
+                    hexstr = b''.join(p.rjust(2, b'0') for p in parts)
+                    mac = int(hexstr, 16)
+                else:
+                    continue
+            else:
+                continue
+        except (ValueError, IndexError):
+            # Virtual interfaces, such as those provided by
+            # VPNs, do not have a colon-delimited MAC address
+            # as expected, but a 16-byte HWAddr separated by
+            # dashes. These should be ignored in favor of a
+            # real MAC address
+            pass
+        else:
+            if _is_universal(mac):
+                return mac
+            first_local_mac = first_local_mac or mac
     return first_local_mac or None
 
+
+# The following functions call external programs to 'get' a macaddr value to
+# be used as basis for an uuid
 def _ifconfig_getnode():
     """Get the hardware address on Unix by running ifconfig."""
     # This works on Linux ('' or '-a'), Tru64 ('-av'), but not all Unixes.
     keywords = (b'hwaddr', b'ether', b'address:', b'lladdr')
     for args in ('', '-a', '-av'):
-        mac = _find_mac('ifconfig', args, keywords, lambda i: i+1)
+        mac = _find_mac_near_keyword('ifconfig', args, keywords, lambda i: i+1)
         if mac:
             return mac
         return None
@@ -425,7 +500,7 @@ def _ifconfig_getnode():
 def _ip_getnode():
     """Get the hardware address on Unix by running ip."""
     # This works on Linux with iproute2.
-    mac = _find_mac('ip', 'link', [b'link/ether'], lambda i: i+1)
+    mac = _find_mac_near_keyword('ip', 'link', [b'link/ether'], lambda i: i+1)
     if mac:
         return mac
     return None
@@ -439,17 +514,17 @@ def _arp_getnode():
         return None
 
     # Try getting the MAC addr from arp based on our IP address (Solaris).
-    mac = _find_mac('arp', '-an', [os.fsencode(ip_addr)], lambda i: -1)
+    mac = _find_mac_near_keyword('arp', '-an', [os.fsencode(ip_addr)], lambda i: -1)
     if mac:
         return mac
 
     # This works on OpenBSD
-    mac = _find_mac('arp', '-an', [os.fsencode(ip_addr)], lambda i: i+1)
+    mac = _find_mac_near_keyword('arp', '-an', [os.fsencode(ip_addr)], lambda i: i+1)
     if mac:
         return mac
 
     # This works on Linux, FreeBSD and NetBSD
-    mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)],
+    mac = _find_mac_near_keyword('arp', '-an', [os.fsencode('(%s)' % ip_addr)],
                     lambda i: i+2)
     # Return None instead of 0.
     if mac:
@@ -459,36 +534,12 @@ def _arp_getnode():
 def _lanscan_getnode():
     """Get the hardware address on Unix by running lanscan."""
     # This might work on HP-UX.
-    return _find_mac('lanscan', '-ai', [b'lan0'], lambda i: 0)
+    return _find_mac_near_keyword('lanscan', '-ai', [b'lan0'], lambda i: 0)
 
 def _netstat_getnode():
     """Get the hardware address on Unix by running netstat."""
-    # This might work on AIX, Tru64 UNIX.
-    first_local_mac = None
-    try:
-        proc = _popen('netstat', '-ia')
-        if not proc:
-            return None
-        with proc:
-            words = proc.stdout.readline().rstrip().split()
-            try:
-                i = words.index(b'Address')
-            except ValueError:
-                return None
-            for line in proc.stdout:
-                try:
-                    words = line.rstrip().split()
-                    word = words[i]
-                    if len(word) == 17 and word.count(b':') == 5:
-                        mac = int(word.replace(b':', b''), 16)
-                        if _is_universal(mac):
-                            return mac
-                        first_local_mac = first_local_mac or mac
-                except (ValueError, IndexError):
-                    pass
-    except OSError:
-        pass
-    return first_local_mac or None
+    # This works on AIX and might work on Tru64 UNIX.
+    return _find_mac_under_heading('netstat', '-ian', b'Address')
 
 def _ipconfig_getnode():
     """Get the hardware address on Windows by running ipconfig.exe."""
diff --git a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst
new file mode 100644
index 000000000000..40f638234a45
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst
@@ -0,0 +1,4 @@
+Fix uuid.getnode() on platforms with '.' as MAC Addr delimiter as well
+fix for MAC Addr format that omits a leading 0 in MAC Addr values.
+Currently, AIX is the only know platform with these settings.
+Patch by Michael Felt.


From webhook-mailer at python.org  Thu Sep 26 16:53:17 2019
From: webhook-mailer at python.org (Christian Heimes)
Date: Thu, 26 Sep 2019 20:53:17 -0000
Subject: [Python-checkins] [3.8] bpo-38275: Skip ssl tests for disabled
 versions (GH-16386) (GH-16425)
Message-ID: <mailman.113.1569531198.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/1931132db33dd002cef0b19b8eaa219c1757797e
commit: 1931132db33dd002cef0b19b8eaa219c1757797e
branch: 3.8
author: Christian Heimes <christian at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-26T22:53:09+02:00
summary:

[3.8] bpo-38275: Skip ssl tests for disabled versions (GH-16386) (GH-16425)

test_ssl now handles disabled TLS/SSL versions better. OpenSSL's crypto
policy and run-time settings are recognized and tests for disabled versions
are skipped.

Signed-off-by: Christian Heimes <christian at python.org>

https://bugs.python.org/issue38275
(cherry picked from commit df6ac7e2b82d921a6e9ff5571b40c6dbcf635581)

files:
A Misc/NEWS.d/next/Tests/2019-09-25-14-40-57.bpo-38275.-kdveI.rst
M Lib/test/test_ssl.py

diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 9f3365aa864c..419506f4d3c0 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -19,6 +19,7 @@
 import weakref
 import platform
 import sysconfig
+import functools
 try:
     import ctypes
 except ImportError:
@@ -143,6 +144,87 @@ def data_file(*name):
 OP_ENABLE_MIDDLEBOX_COMPAT = getattr(ssl, "OP_ENABLE_MIDDLEBOX_COMPAT", 0)
 
 
+def has_tls_protocol(protocol):
+    """Check if a TLS protocol is available and enabled
+
+    :param protocol: enum ssl._SSLMethod member or name
+    :return: bool
+    """
+    if isinstance(protocol, str):
+        assert protocol.startswith('PROTOCOL_')
+        protocol = getattr(ssl, protocol, None)
+        if protocol is None:
+            return False
+    if protocol in {
+        ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS_SERVER,
+        ssl.PROTOCOL_TLS_CLIENT
+    }:
+        # auto-negotiate protocols are always available
+        return True
+    name = protocol.name
+    return has_tls_version(name[len('PROTOCOL_'):])
+
+
+ at functools.lru_cache
+def has_tls_version(version):
+    """Check if a TLS/SSL version is enabled
+
+    :param version: TLS version name or ssl.TLSVersion member
+    :return: bool
+    """
+    if version == "SSLv2":
+        # never supported and not even in TLSVersion enum
+        return False
+
+    if isinstance(version, str):
+        version = ssl.TLSVersion.__members__[version]
+
+    # check compile time flags like ssl.HAS_TLSv1_2
+    if not getattr(ssl, f'HAS_{version.name}'):
+        return False
+
+    # check runtime and dynamic crypto policy settings. A TLS version may
+    # be compiled in but disabled by a policy or config option.
+    ctx = ssl.SSLContext()
+    if (
+            hasattr(ctx, 'minimum_version') and
+            ctx.minimum_version != ssl.TLSVersion.MINIMUM_SUPPORTED and
+            version < ctx.minimum_version
+    ):
+        return False
+    if (
+        hasattr(ctx, 'maximum_version') and
+        ctx.maximum_version != ssl.TLSVersion.MAXIMUM_SUPPORTED and
+        version > ctx.maximum_version
+    ):
+        return False
+
+    return True
+
+
+def requires_tls_version(version):
+    """Decorator to skip tests when a required TLS version is not available
+
+    :param version: TLS version name or ssl.TLSVersion member
+    :return:
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kw):
+            if not has_tls_version(version):
+                raise unittest.SkipTest(f"{version} is not available.")
+            else:
+                return func(*args, **kw)
+        return wrapper
+    return decorator
+
+
+requires_minimum_version = unittest.skipUnless(
+    hasattr(ssl.SSLContext, 'minimum_version'),
+    "required OpenSSL >= 1.1.0g"
+)
+
+
 def handle_error(prefix):
     exc_format = ' '.join(traceback.format_exception(*sys.exc_info()))
     if support.verbose:
@@ -1104,20 +1186,23 @@ def test_hostname_checks_common_name(self):
             with self.assertRaises(AttributeError):
                 ctx.hostname_checks_common_name = True
 
-    @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'),
-                         "required OpenSSL 1.1.0g")
+    @requires_minimum_version
     @unittest.skipIf(IS_LIBRESSL, "see bpo-34001")
     def test_min_max_version(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         # OpenSSL default is MINIMUM_SUPPORTED, however some vendors like
         # Fedora override the setting to TLS 1.0.
+        minimum_range = {
+            # stock OpenSSL
+            ssl.TLSVersion.MINIMUM_SUPPORTED,
+            # Fedora 29 uses TLS 1.0 by default
+            ssl.TLSVersion.TLSv1,
+            # RHEL 8 uses TLS 1.2 by default
+            ssl.TLSVersion.TLSv1_2
+        }
+
         self.assertIn(
-            ctx.minimum_version,
-            {ssl.TLSVersion.MINIMUM_SUPPORTED,
-             # Fedora 29 uses TLS 1.0 by default
-             ssl.TLSVersion.TLSv1,
-             # RHEL 8 uses TLS 1.2 by default
-             ssl.TLSVersion.TLSv1_2}
+            ctx.minimum_version, minimum_range
         )
         self.assertEqual(
             ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED
@@ -1163,8 +1248,8 @@ def test_min_max_version(self):
 
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1)
 
-        self.assertEqual(
-            ctx.minimum_version, ssl.TLSVersion.MINIMUM_SUPPORTED
+        self.assertIn(
+            ctx.minimum_version, minimum_range
         )
         self.assertEqual(
             ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED
@@ -2717,6 +2802,8 @@ def test_echo(self):
         for protocol in PROTOCOLS:
             if protocol in {ssl.PROTOCOL_TLS_CLIENT, ssl.PROTOCOL_TLS_SERVER}:
                 continue
+            if not has_tls_protocol(protocol):
+                continue
             with self.subTest(protocol=ssl._PROTOCOL_NAMES[protocol]):
                 context = ssl.SSLContext(protocol)
                 context.load_cert_chain(CERTFILE)
@@ -3008,7 +3095,7 @@ def test_wrong_cert_tls12(self):
             else:
                 self.fail("Use of invalid cert should have failed!")
 
-    @unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3")
+    @requires_tls_version('TLSv1_3')
     def test_wrong_cert_tls13(self):
         client_context, server_context, hostname = testing_context()
         # load client cert that is not signed by trusted CA
@@ -3103,8 +3190,7 @@ def test_ssl_cert_verify_error(self):
                     self.assertIn(msg, repr(e))
                     self.assertIn('certificate verify failed', repr(e))
 
-    @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'),
-                         "OpenSSL is compiled without SSLv2 support")
+    @requires_tls_version('SSLv2')
     def test_protocol_sslv2(self):
         """Connecting to an SSLv2 server with various client options"""
         if support.verbose:
@@ -3113,7 +3199,7 @@ def test_protocol_sslv2(self):
         try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL)
         try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED)
         try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLS, False)
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False)
         # SSLv23 client with specific SSL options
@@ -3130,7 +3216,7 @@ def test_PROTOCOL_TLS(self):
         """Connecting to an SSLv23 server with various client options"""
         if support.verbose:
             sys.stdout.write("\n")
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try:
                 try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv2, True)
             except OSError as x:
@@ -3139,34 +3225,36 @@ def test_PROTOCOL_TLS(self):
                     sys.stdout.write(
                         " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n"
                         % str(x))
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True)
-        try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1')
+        if has_tls_version('TLSv1'):
+            try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1')
 
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL)
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_OPTIONAL)
-        try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL)
+        if has_tls_version('TLSv1'):
+            try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL)
 
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED)
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_REQUIRED)
-        try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED)
+        if has_tls_version('TLSv1'):
+            try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED)
 
         # Server with specific SSL options
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False,
                            server_options=ssl.OP_NO_SSLv3)
         # Will choose TLSv1
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True,
                            server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
-        try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, False,
-                           server_options=ssl.OP_NO_TLSv1)
-
+        if has_tls_version('TLSv1'):
+            try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, False,
+                               server_options=ssl.OP_NO_TLSv1)
 
-    @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv3'),
-                         "OpenSSL is compiled without SSLv3 support")
+    @requires_tls_version('SSLv3')
     def test_protocol_sslv3(self):
         """Connecting to an SSLv3 server with various client options"""
         if support.verbose:
@@ -3174,7 +3262,7 @@ def test_protocol_sslv3(self):
         try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3')
         try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL)
         try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED)
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
         try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLS, False,
                            client_options=ssl.OP_NO_SSLv3)
@@ -3184,6 +3272,7 @@ def test_protocol_sslv3(self):
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLS,
                                False, client_options=ssl.OP_NO_SSLv2)
 
+    @requires_tls_version('TLSv1')
     def test_protocol_tlsv1(self):
         """Connecting to a TLSv1 server with various client options"""
         if support.verbose:
@@ -3191,34 +3280,32 @@ def test_protocol_tlsv1(self):
         try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1')
         try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL)
         try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED)
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLS, False,
                            client_options=ssl.OP_NO_TLSv1)
 
-    @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_1"),
-                         "TLS version 1.1 not supported.")
+    @requires_tls_version('TLSv1_1')
     def test_protocol_tlsv1_1(self):
         """Connecting to a TLSv1.1 server with various client options.
            Testing against older TLS versions."""
         if support.verbose:
             sys.stdout.write("\n")
         try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1')
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False)
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLS, False,
                            client_options=ssl.OP_NO_TLSv1_1)
 
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1')
-        try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False)
-        try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False)
+        try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False)
+        try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False)
 
-    @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"),
-                         "TLS version 1.2 not supported.")
+    @requires_tls_version('TLSv1_2')
     def test_protocol_tlsv1_2(self):
         """Connecting to a TLSv1.2 server with various client options.
            Testing against older TLS versions."""
@@ -3227,9 +3314,9 @@ def test_protocol_tlsv1_2(self):
         try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2',
                            server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,
                            client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,)
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False)
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLS, False,
                            client_options=ssl.OP_NO_TLSv1_2)
@@ -3672,7 +3759,7 @@ def test_version_basic(self):
                 self.assertIs(s.version(), None)
                 self.assertIs(s._sslobj, None)
                 s.connect((HOST, server.port))
-                if IS_OPENSSL_1_1_1 and ssl.HAS_TLSv1_3:
+                if IS_OPENSSL_1_1_1 and has_tls_version('TLSv1_3'):
                     self.assertEqual(s.version(), 'TLSv1.3')
                 elif ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
                     self.assertEqual(s.version(), 'TLSv1.2')
@@ -3681,8 +3768,7 @@ def test_version_basic(self):
             self.assertIs(s._sslobj, None)
             self.assertIs(s.version(), None)
 
-    @unittest.skipUnless(ssl.HAS_TLSv1_3,
-                         "test requires TLSv1.3 enabled OpenSSL")
+    @requires_tls_version('TLSv1_3')
     def test_tls1_3(self):
         context = ssl.SSLContext(ssl.PROTOCOL_TLS)
         context.load_cert_chain(CERTFILE)
@@ -3699,9 +3785,9 @@ def test_tls1_3(self):
                 })
                 self.assertEqual(s.version(), 'TLSv1.3')
 
-    @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'),
-                         "required OpenSSL 1.1.0g")
-    def test_min_max_version(self):
+    @requires_minimum_version
+    @requires_tls_version('TLSv1_2')
+    def test_min_max_version_tlsv1_2(self):
         client_context, server_context, hostname = testing_context()
         # client TLSv1.0 to 1.2
         client_context.minimum_version = ssl.TLSVersion.TLSv1
@@ -3716,7 +3802,13 @@ def test_min_max_version(self):
                 s.connect((HOST, server.port))
                 self.assertEqual(s.version(), 'TLSv1.2')
 
+    @requires_minimum_version
+    @requires_tls_version('TLSv1_1')
+    def test_min_max_version_tlsv1_1(self):
+        client_context, server_context, hostname = testing_context()
         # client 1.0 to 1.2, server 1.0 to 1.1
+        client_context.minimum_version = ssl.TLSVersion.TLSv1
+        client_context.maximum_version = ssl.TLSVersion.TLSv1_2
         server_context.minimum_version = ssl.TLSVersion.TLSv1
         server_context.maximum_version = ssl.TLSVersion.TLSv1_1
 
@@ -3726,6 +3818,10 @@ def test_min_max_version(self):
                 s.connect((HOST, server.port))
                 self.assertEqual(s.version(), 'TLSv1.1')
 
+    @requires_minimum_version
+    @requires_tls_version('TLSv1_2')
+    def test_min_max_version_mismatch(self):
+        client_context, server_context, hostname = testing_context()
         # client 1.0, server 1.2 (mismatch)
         server_context.maximum_version = ssl.TLSVersion.TLSv1_2
         server_context.minimum_version = ssl.TLSVersion.TLSv1_2
@@ -3738,10 +3834,8 @@ def test_min_max_version(self):
                     s.connect((HOST, server.port))
                 self.assertIn("alert", str(e.exception))
 
-
-    @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'),
-                         "required OpenSSL 1.1.0g")
-    @unittest.skipUnless(ssl.HAS_SSLv3, "requires SSLv3 support")
+    @requires_minimum_version
+    @requires_tls_version('SSLv3')
     def test_min_max_version_sslv3(self):
         client_context, server_context, hostname = testing_context()
         server_context.minimum_version = ssl.TLSVersion.SSLv3
@@ -4264,7 +4358,7 @@ def test_session_handling(self):
                                  'Session refers to a different SSLContext.')
 
 
- at unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3")
+ at unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3")
 class TestPostHandshakeAuth(unittest.TestCase):
     def test_pha_setter(self):
         protocols = [
diff --git a/Misc/NEWS.d/next/Tests/2019-09-25-14-40-57.bpo-38275.-kdveI.rst b/Misc/NEWS.d/next/Tests/2019-09-25-14-40-57.bpo-38275.-kdveI.rst
new file mode 100644
index 000000000000..893c0f137aea
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-25-14-40-57.bpo-38275.-kdveI.rst
@@ -0,0 +1,4 @@
+test_ssl now handles disabled TLS/SSL versions better. OpenSSL's crypto
+policy and run-time settings are recognized and tests for disabled versions
+are skipped. Tests also accept more TLS minimum_versions for platforms that
+override OpenSSL's default with strict settings.


From webhook-mailer at python.org  Thu Sep 26 16:54:38 2019
From: webhook-mailer at python.org (Christian Heimes)
Date: Thu, 26 Sep 2019 20:54:38 -0000
Subject: [Python-checkins] [3.7] bpo-38275: Skip ssl tests for disabled
 versions (GH-16427)
Message-ID: <mailman.114.1569531279.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/2c24f2cae00bbfe9af1e59778f6516d4c56ffc4e
commit: 2c24f2cae00bbfe9af1e59778f6516d4c56ffc4e
branch: 3.7
author: Christian Heimes <christian at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-26T22:54:33+02:00
summary:

[3.7] bpo-38275: Skip ssl tests for disabled versions (GH-16427)

test_ssl now handles disabled TLS/SSL versions better. OpenSSL's crypto
policy and run-time settings are recognized and tests for disabled versions
are skipped.

Signed-off-by: Christian Heimes <christian at python.org>

https://bugs.python.org/issue38275
(cherry picked from commit df6ac7e2b82d921a6e9ff5571b40c6dbcf635581)

files:
A Misc/NEWS.d/next/Tests/2019-09-25-14-40-57.bpo-38275.-kdveI.rst
M Lib/test/test_ssl.py

diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 04ede28ab7c0..e21e7e07455c 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -19,6 +19,7 @@
 import platform
 import functools
 import sysconfig
+import functools
 try:
     import ctypes
 except ImportError:
@@ -142,6 +143,87 @@ def data_file(*name):
 OP_ENABLE_MIDDLEBOX_COMPAT = getattr(ssl, "OP_ENABLE_MIDDLEBOX_COMPAT", 0)
 
 
+def has_tls_protocol(protocol):
+    """Check if a TLS protocol is available and enabled
+
+    :param protocol: enum ssl._SSLMethod member or name
+    :return: bool
+    """
+    if isinstance(protocol, str):
+        assert protocol.startswith('PROTOCOL_')
+        protocol = getattr(ssl, protocol, None)
+        if protocol is None:
+            return False
+    if protocol in {
+        ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS_SERVER,
+        ssl.PROTOCOL_TLS_CLIENT
+    }:
+        # auto-negotiate protocols are always available
+        return True
+    name = protocol.name
+    return has_tls_version(name[len('PROTOCOL_'):])
+
+
+ at functools.lru_cache()
+def has_tls_version(version):
+    """Check if a TLS/SSL version is enabled
+
+    :param version: TLS version name or ssl.TLSVersion member
+    :return: bool
+    """
+    if version == "SSLv2":
+        # never supported and not even in TLSVersion enum
+        return False
+
+    if isinstance(version, str):
+        version = ssl.TLSVersion.__members__[version]
+
+    # check compile time flags like ssl.HAS_TLSv1_2
+    if not getattr(ssl, f'HAS_{version.name}'):
+        return False
+
+    # check runtime and dynamic crypto policy settings. A TLS version may
+    # be compiled in but disabled by a policy or config option.
+    ctx = ssl.SSLContext()
+    if (
+            hasattr(ctx, 'minimum_version') and
+            ctx.minimum_version != ssl.TLSVersion.MINIMUM_SUPPORTED and
+            version < ctx.minimum_version
+    ):
+        return False
+    if (
+        hasattr(ctx, 'maximum_version') and
+        ctx.maximum_version != ssl.TLSVersion.MAXIMUM_SUPPORTED and
+        version > ctx.maximum_version
+    ):
+        return False
+
+    return True
+
+
+def requires_tls_version(version):
+    """Decorator to skip tests when a required TLS version is not available
+
+    :param version: TLS version name or ssl.TLSVersion member
+    :return:
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kw):
+            if not has_tls_version(version):
+                raise unittest.SkipTest(f"{version} is not available.")
+            else:
+                return func(*args, **kw)
+        return wrapper
+    return decorator
+
+
+requires_minimum_version = unittest.skipUnless(
+    hasattr(ssl.SSLContext, 'minimum_version'),
+    "required OpenSSL >= 1.1.0g"
+)
+
+
 def handle_error(prefix):
     exc_format = ' '.join(traceback.format_exception(*sys.exc_info()))
     if support.verbose:
@@ -1124,19 +1206,23 @@ def test_hostname_checks_common_name(self):
             with self.assertRaises(AttributeError):
                 ctx.hostname_checks_common_name = True
 
-    @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'),
-                         "required OpenSSL 1.1.0g")
+    @requires_minimum_version
+    @unittest.skipIf(IS_LIBRESSL, "see bpo-34001")
     def test_min_max_version(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         # OpenSSL default is MINIMUM_SUPPORTED, however some vendors like
         # Fedora override the setting to TLS 1.0.
+        minimum_range = {
+            # stock OpenSSL
+            ssl.TLSVersion.MINIMUM_SUPPORTED,
+            # Fedora 29 uses TLS 1.0 by default
+            ssl.TLSVersion.TLSv1,
+            # RHEL 8 uses TLS 1.2 by default
+            ssl.TLSVersion.TLSv1_2
+        }
+
         self.assertIn(
-            ctx.minimum_version,
-            {ssl.TLSVersion.MINIMUM_SUPPORTED,
-             # Fedora 29 uses TLS 1.0 by default
-             ssl.TLSVersion.TLSv1,
-             # RHEL 8 uses TLS 1.2 by default
-             ssl.TLSVersion.TLSv1_2}
+            ctx.minimum_version, minimum_range
         )
         self.assertEqual(
             ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED
@@ -1182,8 +1268,8 @@ def test_min_max_version(self):
 
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1)
 
-        self.assertEqual(
-            ctx.minimum_version, ssl.TLSVersion.MINIMUM_SUPPORTED
+        self.assertIn(
+            ctx.minimum_version, minimum_range
         )
         self.assertEqual(
             ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED
@@ -2723,6 +2809,8 @@ def test_echo(self):
         for protocol in PROTOCOLS:
             if protocol in {ssl.PROTOCOL_TLS_CLIENT, ssl.PROTOCOL_TLS_SERVER}:
                 continue
+            if not has_tls_protocol(protocol):
+                continue
             with self.subTest(protocol=ssl._PROTOCOL_NAMES[protocol]):
                 context = ssl.SSLContext(protocol)
                 context.load_cert_chain(CERTFILE)
@@ -3014,7 +3102,7 @@ def test_wrong_cert_tls12(self):
             else:
                 self.fail("Use of invalid cert should have failed!")
 
-    @unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3")
+    @requires_tls_version('TLSv1_3')
     def test_wrong_cert_tls13(self):
         client_context, server_context, hostname = testing_context()
         # load client cert that is not signed by trusted CA
@@ -3109,9 +3197,7 @@ def test_ssl_cert_verify_error(self):
                     self.assertIn(msg, repr(e))
                     self.assertIn('certificate verify failed', repr(e))
 
-    @skip_if_broken_ubuntu_ssl
-    @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'),
-                         "OpenSSL is compiled without SSLv2 support")
+    @requires_tls_version('SSLv2')
     def test_protocol_sslv2(self):
         """Connecting to an SSLv2 server with various client options"""
         if support.verbose:
@@ -3120,7 +3206,7 @@ def test_protocol_sslv2(self):
         try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL)
         try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED)
         try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLS, False)
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False)
         # SSLv23 client with specific SSL options
@@ -3138,7 +3224,7 @@ def test_PROTOCOL_TLS(self):
         """Connecting to an SSLv23 server with various client options"""
         if support.verbose:
             sys.stdout.write("\n")
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try:
                 try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv2, True)
             except OSError as x:
@@ -3147,35 +3233,36 @@ def test_PROTOCOL_TLS(self):
                     sys.stdout.write(
                         " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n"
                         % str(x))
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True)
-        try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1')
+        if has_tls_version('TLSv1'):
+            try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1')
 
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL)
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_OPTIONAL)
-        try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL)
+        if has_tls_version('TLSv1'):
+            try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL)
 
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED)
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_REQUIRED)
-        try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED)
+        if has_tls_version('TLSv1'):
+            try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED)
 
         # Server with specific SSL options
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False,
                            server_options=ssl.OP_NO_SSLv3)
         # Will choose TLSv1
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True,
                            server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
-        try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, False,
-                           server_options=ssl.OP_NO_TLSv1)
-
+        if has_tls_version('TLSv1'):
+            try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, False,
+                               server_options=ssl.OP_NO_TLSv1)
 
-    @skip_if_broken_ubuntu_ssl
-    @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv3'),
-                         "OpenSSL is compiled without SSLv3 support")
+    @requires_tls_version('SSLv3')
     def test_protocol_sslv3(self):
         """Connecting to an SSLv3 server with various client options"""
         if support.verbose:
@@ -3183,7 +3270,7 @@ def test_protocol_sslv3(self):
         try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3')
         try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL)
         try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED)
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
         try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLS, False,
                            client_options=ssl.OP_NO_SSLv3)
@@ -3193,7 +3280,7 @@ def test_protocol_sslv3(self):
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLS,
                                False, client_options=ssl.OP_NO_SSLv2)
 
-    @skip_if_broken_ubuntu_ssl
+    @requires_tls_version('TLSv1')
     def test_protocol_tlsv1(self):
         """Connecting to a TLSv1 server with various client options"""
         if support.verbose:
@@ -3201,36 +3288,32 @@ def test_protocol_tlsv1(self):
         try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1')
         try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL)
         try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED)
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLS, False,
                            client_options=ssl.OP_NO_TLSv1)
 
-    @skip_if_broken_ubuntu_ssl
-    @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_1"),
-                         "TLS version 1.1 not supported.")
+    @requires_tls_version('TLSv1_1')
     def test_protocol_tlsv1_1(self):
         """Connecting to a TLSv1.1 server with various client options.
            Testing against older TLS versions."""
         if support.verbose:
             sys.stdout.write("\n")
         try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1')
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False)
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLS, False,
                            client_options=ssl.OP_NO_TLSv1_1)
 
         try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1')
-        try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False)
-        try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False)
+        try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False)
+        try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False)
 
-    @skip_if_broken_ubuntu_ssl
-    @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"),
-                         "TLS version 1.2 not supported.")
+    @requires_tls_version('TLSv1_2')
     def test_protocol_tlsv1_2(self):
         """Connecting to a TLSv1.2 server with various client options.
            Testing against older TLS versions."""
@@ -3239,9 +3322,9 @@ def test_protocol_tlsv1_2(self):
         try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2',
                            server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,
                            client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,)
-        if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        if has_tls_version('SSLv2'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False)
-        if hasattr(ssl, 'PROTOCOL_SSLv3'):
+        if has_tls_version('SSLv3'):
             try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False)
         try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLS, False,
                            client_options=ssl.OP_NO_TLSv1_2)
@@ -3684,7 +3767,7 @@ def test_version_basic(self):
                 self.assertIs(s.version(), None)
                 self.assertIs(s._sslobj, None)
                 s.connect((HOST, server.port))
-                if IS_OPENSSL_1_1_1 and ssl.HAS_TLSv1_3:
+                if IS_OPENSSL_1_1_1 and has_tls_version('TLSv1_3'):
                     self.assertEqual(s.version(), 'TLSv1.3')
                 elif ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
                     self.assertEqual(s.version(), 'TLSv1.2')
@@ -3693,8 +3776,7 @@ def test_version_basic(self):
             self.assertIs(s._sslobj, None)
             self.assertIs(s.version(), None)
 
-    @unittest.skipUnless(ssl.HAS_TLSv1_3,
-                         "test requires TLSv1.3 enabled OpenSSL")
+    @requires_tls_version('TLSv1_3')
     def test_tls1_3(self):
         context = ssl.SSLContext(ssl.PROTOCOL_TLS)
         context.load_cert_chain(CERTFILE)
@@ -3711,9 +3793,9 @@ def test_tls1_3(self):
                 })
                 self.assertEqual(s.version(), 'TLSv1.3')
 
-    @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'),
-                         "required OpenSSL 1.1.0g")
-    def test_min_max_version(self):
+    @requires_minimum_version
+    @requires_tls_version('TLSv1_2')
+    def test_min_max_version_tlsv1_2(self):
         client_context, server_context, hostname = testing_context()
         # client TLSv1.0 to 1.2
         client_context.minimum_version = ssl.TLSVersion.TLSv1
@@ -3728,7 +3810,13 @@ def test_min_max_version(self):
                 s.connect((HOST, server.port))
                 self.assertEqual(s.version(), 'TLSv1.2')
 
+    @requires_minimum_version
+    @requires_tls_version('TLSv1_1')
+    def test_min_max_version_tlsv1_1(self):
+        client_context, server_context, hostname = testing_context()
         # client 1.0 to 1.2, server 1.0 to 1.1
+        client_context.minimum_version = ssl.TLSVersion.TLSv1
+        client_context.maximum_version = ssl.TLSVersion.TLSv1_2
         server_context.minimum_version = ssl.TLSVersion.TLSv1
         server_context.maximum_version = ssl.TLSVersion.TLSv1_1
 
@@ -3738,6 +3826,10 @@ def test_min_max_version(self):
                 s.connect((HOST, server.port))
                 self.assertEqual(s.version(), 'TLSv1.1')
 
+    @requires_minimum_version
+    @requires_tls_version('TLSv1_2')
+    def test_min_max_version_mismatch(self):
+        client_context, server_context, hostname = testing_context()
         # client 1.0, server 1.2 (mismatch)
         server_context.minimum_version = ssl.TLSVersion.TLSv1_2
         server_context.maximum_version = ssl.TLSVersion.TLSv1_2
@@ -3750,10 +3842,8 @@ def test_min_max_version(self):
                     s.connect((HOST, server.port))
                 self.assertIn("alert", str(e.exception))
 
-
-    @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'),
-                         "required OpenSSL 1.1.0g")
-    @unittest.skipUnless(ssl.HAS_SSLv3, "requires SSLv3 support")
+    @requires_minimum_version
+    @requires_tls_version('SSLv3')
     def test_min_max_version_sslv3(self):
         client_context, server_context, hostname = testing_context()
         server_context.minimum_version = ssl.TLSVersion.SSLv3
@@ -4272,7 +4362,7 @@ def test_session_handling(self):
                                  'Session refers to a different SSLContext.')
 
 
- at unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3")
+ at unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3")
 class TestPostHandshakeAuth(unittest.TestCase):
     def test_pha_setter(self):
         protocols = [
diff --git a/Misc/NEWS.d/next/Tests/2019-09-25-14-40-57.bpo-38275.-kdveI.rst b/Misc/NEWS.d/next/Tests/2019-09-25-14-40-57.bpo-38275.-kdveI.rst
new file mode 100644
index 000000000000..893c0f137aea
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-25-14-40-57.bpo-38275.-kdveI.rst
@@ -0,0 +1,4 @@
+test_ssl now handles disabled TLS/SSL versions better. OpenSSL's crypto
+policy and run-time settings are recognized and tests for disabled versions
+are skipped. Tests also accept more TLS minimum_versions for platforms that
+override OpenSSL's default with strict settings.


From webhook-mailer at python.org  Fri Sep 27 03:54:42 2019
From: webhook-mailer at python.org (Ivan Levkivskyi)
Date: Fri, 27 Sep 2019 07:54:42 -0000
Subject: [Python-checkins] cleanup ababstractproperty in typing.py (GH-16432)
Message-ID: <mailman.115.1569570884.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/6ce03ec627680ce0829a5b3067fab7faed21b533
commit: 6ce03ec627680ce0829a5b3067fab7faed21b533
branch: master
author: HongWeipeng <961365124 at qq.com>
committer: Ivan Levkivskyi <levkivskyi at gmail.com>
date: 2019-09-27T08:54:26+01:00
summary:

cleanup ababstractproperty in typing.py (GH-16432)

files:
M Lib/typing.py

diff --git a/Lib/typing.py b/Lib/typing.py
index b1ac33e00e70..2c75a7696487 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -17,7 +17,7 @@
 * Wrapper submodules for re and io related types.
 """
 
-from abc import abstractmethod, abstractproperty, ABCMeta
+from abc import abstractmethod, ABCMeta
 import collections
 import collections.abc
 import contextlib
@@ -1794,11 +1794,13 @@ class IO(Generic[AnyStr]):
 
     __slots__ = ()
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def mode(self) -> str:
         pass
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def name(self) -> str:
         pass
 
@@ -1894,23 +1896,28 @@ class TextIO(IO[str]):
 
     __slots__ = ()
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def buffer(self) -> BinaryIO:
         pass
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def encoding(self) -> str:
         pass
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def errors(self) -> Optional[str]:
         pass
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def line_buffering(self) -> bool:
         pass
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def newlines(self) -> Any:
         pass
 


From webhook-mailer at python.org  Fri Sep 27 04:13:48 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 27 Sep 2019 08:13:48 -0000
Subject: [Python-checkins] cleanup ababstractproperty in typing.py (GH-16432)
Message-ID: <mailman.116.1569572029.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/b2c2a0c02c6056dc73311a8a8a295249c454913a
commit: b2c2a0c02c6056dc73311a8a8a295249c454913a
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-27T01:13:38-07:00
summary:

cleanup ababstractproperty in typing.py (GH-16432)

(cherry picked from commit 6ce03ec627680ce0829a5b3067fab7faed21b533)

Co-authored-by: HongWeipeng <961365124 at qq.com>

files:
M Lib/typing.py

diff --git a/Lib/typing.py b/Lib/typing.py
index 048db3175e61..825be10812de 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -17,7 +17,7 @@
 * Wrapper submodules for re and io related types.
 """
 
-from abc import abstractmethod, abstractproperty, ABCMeta
+from abc import abstractmethod, ABCMeta
 import collections
 import collections.abc
 import contextlib
@@ -1823,11 +1823,13 @@ class IO(Generic[AnyStr]):
 
     __slots__ = ()
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def mode(self) -> str:
         pass
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def name(self) -> str:
         pass
 
@@ -1923,23 +1925,28 @@ class TextIO(IO[str]):
 
     __slots__ = ()
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def buffer(self) -> BinaryIO:
         pass
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def encoding(self) -> str:
         pass
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def errors(self) -> Optional[str]:
         pass
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def line_buffering(self) -> bool:
         pass
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def newlines(self) -> Any:
         pass
 


From webhook-mailer at python.org  Fri Sep 27 07:11:35 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 27 Sep 2019 11:11:35 -0000
Subject: [Python-checkins] bpo-38206: Clarify tp_dealloc requirements for
 heap allocated types. (GH-16248)
Message-ID: <mailman.117.1569582697.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/5faff977adbe089e1f91a5916ccb2160a22dd292
commit: 5faff977adbe089e1f91a5916ccb2160a22dd292
branch: master
author: Ammar Askar <ammar at ammaraskar.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-27T04:11:27-07:00
summary:

bpo-38206: Clarify tp_dealloc requirements for heap allocated types. (GH-16248)



As mentioned in the bpo ticket, this mistake came up on two reviews:
- https://github.com/python/cpython/pull/16127#pullrequestreview-288312751
- https://github.com/python/cpython/pull/16071#pullrequestreview-287819525

Would be nice to have it documented in a more permanent place than 3.8's whatsnew entry.


https://bugs.python.org/issue38206



Automerge-Triggered-By: @encukou

files:
M Doc/c-api/type.rst
M Doc/c-api/typeobj.rst

diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index 36309dd5254a..73311ec083a8 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -118,7 +118,8 @@ The following functions and structs are used to create
 
 .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
 
-   Creates and returns a heap type object from the *spec*.
+   Creates and returns a heap type object from the *spec*
+   (:const:`Py_TPFLAGS_HEAPTYPE`).
 
    If *bases* is a tuple, the created heap type contains all types contained
    in it as base types.
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index 7c7a79129ccc..bb7675007430 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -654,9 +654,9 @@ and :c:type:`PyType_Type` effectively act as defaults.)
    the instance is still in existence, but there are no references to it.  The
    destructor function should free all references which the instance owns, free all
    memory buffers owned by the instance (using the freeing function corresponding
-   to the allocation function used to allocate the buffer), and finally (as its
-   last action) call the type's :c:member:`~PyTypeObject.tp_free` function.  If the type is not
-   subtypable (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is
+   to the allocation function used to allocate the buffer), and call the type's
+   :c:member:`~PyTypeObject.tp_free` function.  If the type is not subtypable
+   (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is
    permissible to call the object deallocator directly instead of via
    :c:member:`~PyTypeObject.tp_free`.  The object deallocator should be the one used to allocate the
    instance; this is normally :c:func:`PyObject_Del` if the instance was allocated
@@ -664,6 +664,21 @@ and :c:type:`PyType_Type` effectively act as defaults.)
    :c:func:`PyObject_GC_Del` if the instance was allocated using
    :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`.
 
+   Finally, if the type is heap allocated (:const:`Py_TPFLAGS_HEAPTYPE`), the
+   deallocator should decrement the reference count for its type object after
+   calling the type deallocator. In order to avoid dangling pointers, the
+   recommended way to achieve this is:
+
+   .. code-block:: c
+
+     static void foo_dealloc(foo_object *self) {
+         PyTypeObject *tp = Py_TYPE(self);
+         // free references and buffers here
+         tp->tp_free(self);
+         Py_DECREF(tp);
+     }
+
+
    **Inheritance:**
 
    This field is inherited by subtypes.
@@ -1021,7 +1036,8 @@ and :c:type:`PyType_Type` effectively act as defaults.)
 
    .. data:: Py_TPFLAGS_HEAPTYPE
 
-      This bit is set when the type object itself is allocated on the heap.  In this
+      This bit is set when the type object itself is allocated on the heap, for
+      example, types created dynamically using :c:func:`PyType_FromSpec`.  In this
       case, the :attr:`ob_type` field of its instances is considered a reference to
       the type, and the type object is INCREF'ed when a new instance is created, and
       DECREF'ed when an instance is destroyed (this does not apply to instances of


From webhook-mailer at python.org  Fri Sep 27 07:18:27 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 27 Sep 2019 11:18:27 -0000
Subject: [Python-checkins] bpo-38206: Clarify tp_dealloc requirements for
 heap allocated types. (GH-16248)
Message-ID: <mailman.118.1569583109.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/14ddca726a8cd337d0461934374b5e6bf65bf812
commit: 14ddca726a8cd337d0461934374b5e6bf65bf812
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-27T04:18:24-07:00
summary:

bpo-38206: Clarify tp_dealloc requirements for heap allocated types. (GH-16248)


As mentioned in the bpo ticket, this mistake came up on two reviews:
- https://github.com/python/cpython/pull/16127GH-pullrequestreview-288312751
- https://github.com/python/cpython/pull/16071GH-pullrequestreview-287819525

Would be nice to have it documented in a more permanent place than 3.8's whatsnew entry.

https://bugs.python.org/issue38206

Automerge-Triggered-By: @encukou
(cherry picked from commit 5faff977adbe089e1f91a5916ccb2160a22dd292)

Co-authored-by: Ammar Askar <ammar at ammaraskar.com>

files:
M Doc/c-api/type.rst
M Doc/c-api/typeobj.rst

diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index 6416951ac5a9..239355a80241 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -118,7 +118,8 @@ The following functions and structs are used to create
 
 .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
 
-   Creates and returns a heap type object from the *spec*.
+   Creates and returns a heap type object from the *spec*
+   (:const:`Py_TPFLAGS_HEAPTYPE`).
 
    If *bases* is a tuple, the created heap type contains all types contained
    in it as base types.
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index 7c7a79129ccc..bb7675007430 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -654,9 +654,9 @@ and :c:type:`PyType_Type` effectively act as defaults.)
    the instance is still in existence, but there are no references to it.  The
    destructor function should free all references which the instance owns, free all
    memory buffers owned by the instance (using the freeing function corresponding
-   to the allocation function used to allocate the buffer), and finally (as its
-   last action) call the type's :c:member:`~PyTypeObject.tp_free` function.  If the type is not
-   subtypable (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is
+   to the allocation function used to allocate the buffer), and call the type's
+   :c:member:`~PyTypeObject.tp_free` function.  If the type is not subtypable
+   (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is
    permissible to call the object deallocator directly instead of via
    :c:member:`~PyTypeObject.tp_free`.  The object deallocator should be the one used to allocate the
    instance; this is normally :c:func:`PyObject_Del` if the instance was allocated
@@ -664,6 +664,21 @@ and :c:type:`PyType_Type` effectively act as defaults.)
    :c:func:`PyObject_GC_Del` if the instance was allocated using
    :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`.
 
+   Finally, if the type is heap allocated (:const:`Py_TPFLAGS_HEAPTYPE`), the
+   deallocator should decrement the reference count for its type object after
+   calling the type deallocator. In order to avoid dangling pointers, the
+   recommended way to achieve this is:
+
+   .. code-block:: c
+
+     static void foo_dealloc(foo_object *self) {
+         PyTypeObject *tp = Py_TYPE(self);
+         // free references and buffers here
+         tp->tp_free(self);
+         Py_DECREF(tp);
+     }
+
+
    **Inheritance:**
 
    This field is inherited by subtypes.
@@ -1021,7 +1036,8 @@ and :c:type:`PyType_Type` effectively act as defaults.)
 
    .. data:: Py_TPFLAGS_HEAPTYPE
 
-      This bit is set when the type object itself is allocated on the heap.  In this
+      This bit is set when the type object itself is allocated on the heap, for
+      example, types created dynamically using :c:func:`PyType_FromSpec`.  In this
       case, the :attr:`ob_type` field of its instances is considered a reference to
       the type, and the type object is INCREF'ed when a new instance is created, and
       DECREF'ed when an instance is destroyed (this does not apply to instances of


From webhook-mailer at python.org  Fri Sep 27 09:04:01 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 27 Sep 2019 13:04:01 -0000
Subject: [Python-checkins] bpo-38270: More fixes for strict crypto policy
 (GH-16418)
Message-ID: <mailman.119.1569589441.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/90558158093c0ad893102158fd3c2dd9f864e82e
commit: 90558158093c0ad893102158fd3c2dd9f864e82e
branch: master
author: Christian Heimes <christian at python.org>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-27T06:03:53-07:00
summary:

bpo-38270: More fixes for strict crypto policy (GH-16418)



test_hmac and test_hashlib test built-in hashing implementations and
OpenSSL-based hashing implementations. Add more checks to skip OpenSSL
implementations when a strict crypto policy is active.

Use EVP_DigestInit_ex() instead of EVP_DigestInit() to initialize the
EVP context. The EVP_DigestInit() function clears alls flags and breaks
usedforsecurity flag again.

Signed-off-by: Christian Heimes <christian at python.org>



https://bugs.python.org/issue38270

files:
M Lib/test/support/__init__.py
M Lib/test/test_hashlib.py
M Lib/test/test_hmac.py
M Modules/_hashopenssl.c

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index e401090c2146..d593fc18d96e 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -69,6 +69,11 @@
 except ImportError:
     resource = None
 
+try:
+    import _hashlib
+except ImportError:
+    _hashlib = None
+
 __all__ = [
     # globals
     "PIPE_MAX_SIZE", "verbose", "max_memuse", "use_resources", "failfast",
@@ -86,8 +91,8 @@
     "create_empty_file", "can_symlink", "fs_is_case_insensitive",
     # unittest
     "is_resource_enabled", "requires", "requires_freebsd_version",
-    "requires_linux_version", "requires_mac_ver", "check_syntax_error",
-    "check_syntax_warning",
+    "requires_linux_version", "requires_mac_ver", "requires_hashdigest",
+    "check_syntax_error", "check_syntax_warning",
     "TransientResource", "time_out", "socket_peer_reset", "ioerror_peer_reset",
     "transient_internet", "BasicTestRunner", "run_unittest", "run_doctest",
     "skip_unless_symlink", "requires_gzip", "requires_bz2", "requires_lzma",
@@ -649,12 +654,16 @@ def wrapper(*args, **kw):
     return decorator
 
 
-def requires_hashdigest(digestname):
+def requires_hashdigest(digestname, openssl=None, usedforsecurity=True):
     """Decorator raising SkipTest if a hashing algorithm is not available
 
     The hashing algorithm could be missing or blocked by a strict crypto
     policy.
 
+    If 'openssl' is True, then the decorator checks that OpenSSL provides
+    the algorithm. Otherwise the check falls back to built-in
+    implementations. The usedforsecurity flag is passed to the constructor.
+
     ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS
     ValueError: unsupported hash type md4
     """
@@ -662,7 +671,10 @@ def decorator(func):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
             try:
-                hashlib.new(digestname)
+                if openssl and _hashlib is not None:
+                    _hashlib.new(digestname, usedforsecurity=usedforsecurity)
+                else:
+                    hashlib.new(digestname, usedforsecurity=usedforsecurity)
             except ValueError:
                 raise unittest.SkipTest(
                     f"hash digest '{digestname}' is not available."
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index d55de02f9164..0e30b2fb11f2 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -8,6 +8,7 @@
 
 import array
 from binascii import unhexlify
+import functools
 import hashlib
 import importlib
 import itertools
@@ -18,6 +19,7 @@
 import warnings
 from test import support
 from test.support import _4G, bigmemtest, import_fresh_module
+from test.support import requires_hashdigest
 from http.client import HTTPException
 
 # Were we compiled --with-pydebug or with #define Py_DEBUG?
@@ -119,6 +121,7 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs):
             constructors.add(_test_algorithm_via_hashlib_new)
 
         _hashlib = self._conditional_import_module('_hashlib')
+        self._hashlib = _hashlib
         if _hashlib:
             # These two algorithms should always be present when this module
             # is compiled.  If not, something was compiled wrong.
@@ -127,7 +130,13 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs):
             for algorithm, constructors in self.constructors_to_test.items():
                 constructor = getattr(_hashlib, 'openssl_'+algorithm, None)
                 if constructor:
-                    constructors.add(constructor)
+                    try:
+                        constructor()
+                    except ValueError:
+                        # default constructor blocked by crypto policy
+                        pass
+                    else:
+                        constructors.add(constructor)
 
         def add_builtin_constructor(name):
             constructor = getattr(hashlib, "__get_builtin_constructor")(name)
@@ -193,6 +202,9 @@ def test_usedforsecurity(self):
             cons(b'', usedforsecurity=False)
         hashlib.new("sha256", usedforsecurity=True)
         hashlib.new("sha256", usedforsecurity=False)
+        if self._hashlib is not None:
+            self._hashlib.new("md5", usedforsecurity=False)
+            self._hashlib.openssl_md5(usedforsecurity=False)
 
     def test_unknown_hash(self):
         self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam')
diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py
index 2c09de8b26fa..1bbf201727d7 100644
--- a/Lib/test/test_hmac.py
+++ b/Lib/test/test_hmac.py
@@ -21,7 +21,7 @@ def wrapper(*args, **kwargs):
 
 class TestVectorsTestCase(unittest.TestCase):
 
-    @requires_hashdigest('md5')
+    @requires_hashdigest('md5', openssl=True)
     def test_md5_vectors(self):
         # Test the HMAC module against test vectors from the RFC.
 
@@ -79,7 +79,7 @@ def md5test(key, data, digest):
                  b"and Larger Than One Block-Size Data"),
                 "6f630fad67cda0ee1fb1f562db3aa53e")
 
-    @requires_hashdigest('sha1')
+    @requires_hashdigest('sha1', openssl=True)
     def test_sha_vectors(self):
         def shatest(key, data, digest):
             h = hmac.HMAC(key, data, digestmod=hashlib.sha1)
@@ -272,19 +272,19 @@ def hmactest(key, data, hexdigests):
                                    '134676fb6de0446065c97440fa8c6a58',
                  })
 
-    @requires_hashdigest('sha224')
+    @requires_hashdigest('sha224', openssl=True)
     def test_sha224_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64)
 
-    @requires_hashdigest('sha256')
+    @requires_hashdigest('sha256', openssl=True)
     def test_sha256_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64)
 
-    @requires_hashdigest('sha384')
+    @requires_hashdigest('sha384', openssl=True)
     def test_sha384_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128)
 
-    @requires_hashdigest('sha512')
+    @requires_hashdigest('sha512', openssl=True)
     def test_sha512_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128)
 
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 48511f72bc63..b147dbe8b3c5 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -552,7 +552,7 @@ EVPnew(const EVP_MD *digest,
     }
 
 
-    if (!EVP_DigestInit(self->ctx, digest)) {
+    if (!EVP_DigestInit_ex(self->ctx, digest, NULL)) {
         _setException(PyExc_ValueError);
         Py_DECREF(self);
         return NULL;


From webhook-mailer at python.org  Fri Sep 27 10:53:41 2019
From: webhook-mailer at python.org (Eric Snow)
Date: Fri, 27 Sep 2019 14:53:41 -0000
Subject: [Python-checkins] bpo-38187: Fix a refleak in Tools/c-analyzer.
 (gh-16304)
Message-ID: <mailman.120.1569596022.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/6693f730e0eb77d9453f73a3da33b78a97e996ee
commit: 6693f730e0eb77d9453f73a3da33b78a97e996ee
branch: master
author: Eric Snow <ericsnowcurrently at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-27T15:53:34+01:00
summary:

bpo-38187: Fix a refleak in Tools/c-analyzer. (gh-16304)

The "Slot" helper (descriptor) is leaking references due to its caching mechanism. The change includes a partial fix to Slot, but also adds Variable.storage to replace the problematic use of Slot.

https://bugs.python.org/issue38187

files:
M Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py
M Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py
M Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py
M Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py
M Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py
M Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py
M Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py
M Tools/c-analyzer/c_analyzer_common/_generate.py
M Tools/c-analyzer/c_analyzer_common/known.py
M Tools/c-analyzer/c_analyzer_common/util.py
M Tools/c-analyzer/c_parser/info.py
M Tools/c-analyzer/c_parser/naive.py
M Tools/c-analyzer/c_symbols/resolve.py

diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py
index e69de29bb2d1..bc502ef32d29 100644
--- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py
@@ -0,0 +1,6 @@
+import os.path
+from test.support import load_package_tests
+
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py
index 93100e0438cd..215023da577b 100644
--- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py
@@ -15,9 +15,6 @@ class FromFileTests(unittest.TestCase):
 
     _return_read_tsv = ()
 
-    def tearDown(self):
-        Variable._isglobal.instances.clear()
-
     @property
     def calls(self):
         try:
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py
index e69de29bb2d1..bc502ef32d29 100644
--- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py
@@ -0,0 +1,6 @@
+import os.path
+from test.support import load_package_tests
+
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py
index b29f966fd2ad..828899201b7c 100644
--- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py
@@ -64,7 +64,9 @@ def test_typical(self):
                                          **self.kwargs))
 
         self.assertEqual(found, [
+            info.Variable.from_parts('dir1/spam.c', None, 'var1', 'int'),
             info.Variable.from_parts('dir1/spam.c', None, 'var2', 'static int'),
+            info.Variable.from_parts('dir1/spam.c', None, 'var3', 'char *'),
             info.Variable.from_parts('dir1/eggs.c', None, 'var1', 'static int'),
             info.Variable.from_parts('dir1/eggs.c', 'func1', 'var2', 'static char *'),
             ])
@@ -299,7 +301,7 @@ def test_typical(self):
             info.Variable.from_parts('src1/spam.c', None, 'var1', 'static const char *'),
             info.Variable.from_parts('src1/spam.c', None, 'var1b', 'const char *'),
             info.Variable.from_parts('src1/spam.c', 'ham', 'initialized', 'static int'),
-            info.Variable.from_parts('src1/spam.c', 'ham', 'result', 'int'),
+            info.Variable.from_parts('src1/spam.c', 'ham', 'result', 'int'),  # skipped
             info.Variable.from_parts('src1/spam.c', None, 'var2', 'static PyObject *'),
             info.Variable.from_parts('src1/eggs.c', 'tofu', 'ready', 'static int'),
             info.Variable.from_parts('src1/spam.c', None, 'freelist', 'static (PyTupleObject *)[10]'),
@@ -318,6 +320,7 @@ def test_typical(self):
 
         self.assertEqual(found, [
             info.Variable.from_parts('src1/spam.c', None, 'var1', 'static const char *'),
+            info.Variable.from_parts('src1/spam.c', None, 'var1b', 'const char *'),
             info.Variable.from_parts('src1/spam.c', 'ham', 'initialized', 'static int'),
             info.Variable.from_parts('src1/spam.c', None, 'var2', 'static PyObject *'),
             info.Variable.from_parts('src1/eggs.c', 'tofu', 'ready', 'static int'),
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py
index e69de29bb2d1..bc502ef32d29 100644
--- a/Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py
@@ -0,0 +1,6 @@
+import os.path
+from test.support import load_package_tests
+
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py
index 1dfe5d066a33..d1a966c58904 100644
--- a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py
@@ -4,7 +4,7 @@
 from ..util import PseudoStr, StrProxy, Object
 from .. import tool_imports_for_tests
 with tool_imports_for_tests():
-    from c_analyzer_common.info import ID
+    from c_analyzer_common.info import ID, UNKNOWN
     from c_parser.info import (
         normalize_vartype, Variable,
         )
@@ -31,38 +31,47 @@ class VariableTests(unittest.TestCase):
 
     VALID_ARGS = (
             ('x/y/z/spam.c', 'func', 'eggs'),
+            'static',
             'int',
             )
     VALID_KWARGS = dict(zip(Variable._fields, VALID_ARGS))
     VALID_EXPECTED = VALID_ARGS
 
     def test_init_typical_global(self):
-        static = Variable(
-                id=ID(
-                    filename='x/y/z/spam.c',
-                    funcname=None,
-                    name='eggs',
-                    ),
-                vartype='int',
-                )
+        for storage in ('static', 'extern', 'implicit'):
+            with self.subTest(storage):
+                static = Variable(
+                        id=ID(
+                            filename='x/y/z/spam.c',
+                            funcname=None,
+                            name='eggs',
+                            ),
+                        storage=storage,
+                        vartype='int',
+                        )
 
-        self.assertEqual(static, (
-                ('x/y/z/spam.c', None, 'eggs'),
-                'int',
-                ))
+                self.assertEqual(static, (
+                        ('x/y/z/spam.c', None, 'eggs'),
+                        storage,
+                        'int',
+                        ))
 
     def test_init_typical_local(self):
-        static = Variable(
-                id=ID(
-                    filename='x/y/z/spam.c',
-                    funcname='func',
-                    name='eggs',
-                    ),
-                vartype='int',
-                )
+        for storage in ('static', 'local'):
+            with self.subTest(storage):
+                static = Variable(
+                        id=ID(
+                            filename='x/y/z/spam.c',
+                            funcname='func',
+                            name='eggs',
+                            ),
+                        storage=storage,
+                        vartype='int',
+                        )
 
         self.assertEqual(static, (
                 ('x/y/z/spam.c', 'func', 'eggs'),
+                storage,
                 'int',
                 ))
 
@@ -71,10 +80,12 @@ def test_init_all_missing(self):
             with self.subTest(repr(value)):
                 static = Variable(
                         id=value,
+                        storage=value,
                         vartype=value,
                         )
 
                 self.assertEqual(static, (
+                        None,
                         None,
                         None,
                         ))
@@ -89,34 +100,42 @@ def test_init_all_coerced(self):
                     PseudoStr('func'),
                     PseudoStr('spam'),
                     ),
+                 storage=PseudoStr('static'),
                  vartype=PseudoStr('int'),
                  ),
              (id,
+              'static',
               'int',
               )),
             ('non-str 1',
              dict(
                  id=id,
+                 storage=Object(),
                  vartype=Object(),
                  ),
              (id,
+              '<object>',
               '<object>',
               )),
             ('non-str 2',
              dict(
                  id=id,
+                 storage=StrProxy('static'),
                  vartype=StrProxy('variable'),
                  ),
              (id,
+              'static',
               'variable',
               )),
             ('non-str',
              dict(
                  id=id,
-                 vartype=('a', 'b', 'c'),
+                 storage=('a', 'b', 'c'),
+                 vartype=('x', 'y', 'z'),
                  ),
              (id,
               "('a', 'b', 'c')",
+              "('x', 'y', 'z')",
               )),
             ]
         for summary, kwargs, expected in tests:
@@ -134,36 +153,43 @@ def test_init_all_coerced(self):
     def test_iterable(self):
         static = Variable(**self.VALID_KWARGS)
 
-        id, vartype = static
+        id, storage, vartype = static
 
-        values = (id, vartype)
+        values = (id, storage, vartype)
         for value, expected in zip(values, self.VALID_EXPECTED):
             self.assertEqual(value, expected)
 
     def test_fields(self):
-        static = Variable(('a', 'b', 'z'), 'x')
+        static = Variable(('a', 'b', 'z'), 'x', 'y')
 
         self.assertEqual(static.id, ('a', 'b', 'z'))
-        self.assertEqual(static.vartype, 'x')
+        self.assertEqual(static.storage, 'x')
+        self.assertEqual(static.vartype, 'y')
 
     def test___getattr__(self):
-        static = Variable(('a', 'b', 'z'), 'x')
+        static = Variable(('a', 'b', 'z'), 'x', 'y')
 
         self.assertEqual(static.filename, 'a')
         self.assertEqual(static.funcname, 'b')
         self.assertEqual(static.name, 'z')
 
     def test_validate_typical(self):
-        static = Variable(
-                id=ID(
-                    filename='x/y/z/spam.c',
-                    funcname='func',
-                    name='eggs',
-                    ),
-                vartype='int',
-                )
+        validstorage = ('static', 'extern', 'implicit', 'local')
+        self.assertEqual(set(validstorage), set(Variable.STORAGE))
+
+        for storage in validstorage:
+            with self.subTest(storage):
+                static = Variable(
+                        id=ID(
+                            filename='x/y/z/spam.c',
+                            funcname='func',
+                            name='eggs',
+                            ),
+                        storage=storage,
+                        vartype='int',
+                        )
 
-        static.validate()  # This does not fail.
+                static.validate()  # This does not fail.
 
     def test_validate_missing_field(self):
         for field in Variable._fields:
@@ -171,6 +197,13 @@ def test_validate_missing_field(self):
                 static = Variable(**self.VALID_KWARGS)
                 static = static._replace(**{field: None})
 
+                with self.assertRaises(TypeError):
+                    static.validate()
+        for field in ('storage', 'vartype'):
+            with self.subTest(field):
+                static = Variable(**self.VALID_KWARGS)
+                static = static._replace(**{field: UNKNOWN})
+
                 with self.assertRaises(TypeError):
                     static.validate()
 
@@ -185,6 +218,7 @@ def test_validate_bad_field(self):
                 ) + badch
         tests = [
             ('id', ()),  # Any non-empty str is okay.
+            ('storage', ('external', 'global') + notnames),
             ('vartype', ()),  # Any non-empty str is okay.
             ]
         seen = set()
@@ -199,6 +233,8 @@ def test_validate_bad_field(self):
                         static.validate()
 
         for field, invalid in tests:
+            if field == 'id':
+                continue
             valid = seen - set(invalid)
             for value in valid:
                 with self.subTest(f'{field}={value!r}'):
diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py
index e69de29bb2d1..bc502ef32d29 100644
--- a/Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py
+++ b/Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py
@@ -0,0 +1,6 @@
+import os.path
+from test.support import load_package_tests
+
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)
diff --git a/Tools/c-analyzer/c_analyzer_common/_generate.py b/Tools/c-analyzer/c_analyzer_common/_generate.py
index 1629aa6b5210..9b2fc9edb5c8 100644
--- a/Tools/c-analyzer/c_analyzer_common/_generate.py
+++ b/Tools/c-analyzer/c_analyzer_common/_generate.py
@@ -262,7 +262,7 @@ def _known(symbol):
                 raise
     if symbol.name not in decl:
         decl = decl + symbol.name
-    return Variable(varid, decl)
+    return Variable(varid, 'static', decl)
 
 
 def known_row(varid, decl):
@@ -291,7 +291,7 @@ def known_rows(symbols, *,
             except KeyError:
                 found = _find_match(symbol, cache, filenames)
                 if found is None:
-                    found = Variable(symbol.id, UNKNOWN)
+                    found = Variable(symbol.id, UNKNOWN, UNKNOWN)
             yield _as_known(found.id, found.vartype)
     else:
         raise NotImplementedError  # XXX incorporate KNOWN
diff --git a/Tools/c-analyzer/c_analyzer_common/known.py b/Tools/c-analyzer/c_analyzer_common/known.py
index a0c6dfa5aa47..dec1e1d2e092 100644
--- a/Tools/c-analyzer/c_analyzer_common/known.py
+++ b/Tools/c-analyzer/c_analyzer_common/known.py
@@ -34,34 +34,41 @@ def from_file(infile, *,
         id = ID(filename, funcname, name)
         if kind == 'variable':
             values = known['variables']
-            value = Variable(id, declaration)
-            value._isglobal = _is_global(declaration) or id.funcname is None
+            if funcname:
+                storage = _get_storage(declaration) or 'local'
+            else:
+                storage = _get_storage(declaration) or 'implicit'
+            value = Variable(id, storage, declaration)
         else:
             raise ValueError(f'unsupported kind in row {row}')
-        if value.name == 'id' and declaration == UNKNOWN:
-            # None of these are variables.
-            declaration = 'int id';
-        else:
-            value.validate()
+        value.validate()
+#        if value.name == 'id' and declaration == UNKNOWN:
+#            # None of these are variables.
+#            declaration = 'int id';
+#        else:
+#            value.validate()
         values[id] = value
     return known
 
 
-def _is_global(vartype):
+def _get_storage(decl):
     # statics
-    if vartype.startswith('static '):
-        return True
-    if vartype.startswith(('Py_LOCAL(', 'Py_LOCAL_INLINE(')):
-        return True
-    if vartype.startswith(('_Py_IDENTIFIER(', '_Py_static_string(')):
-        return True
-    if vartype.startswith('PyDoc_VAR('):
-        return True
-    if vartype.startswith(('SLOT1BINFULL(', 'SLOT1BIN(')):
-        return True
-    if vartype.startswith('WRAP_METHOD('):
-        return True
+    if decl.startswith('static '):
+        return 'static'
+    if decl.startswith(('Py_LOCAL(', 'Py_LOCAL_INLINE(')):
+        return 'static'
+    if decl.startswith(('_Py_IDENTIFIER(', '_Py_static_string(')):
+        return 'static'
+    if decl.startswith('PyDoc_VAR('):
+        return 'static'
+    if decl.startswith(('SLOT1BINFULL(', 'SLOT1BIN(')):
+        return 'static'
+    if decl.startswith('WRAP_METHOD('):
+        return 'static'
     # public extern
-    if vartype.startswith('PyAPI_DATA('):
-        return True
-    return False
+    if decl.startswith('extern '):
+        return 'extern'
+    if decl.startswith('PyAPI_DATA('):
+        return 'extern'
+    # implicit or local
+    return None
diff --git a/Tools/c-analyzer/c_analyzer_common/util.py b/Tools/c-analyzer/c_analyzer_common/util.py
index 511c54f17834..43d0bb6e6656 100644
--- a/Tools/c-analyzer/c_analyzer_common/util.py
+++ b/Tools/c-analyzer/c_analyzer_common/util.py
@@ -82,6 +82,13 @@ def __init__(self, initial=_NOT_SET, *,
         self.default = default
         self.readonly = readonly
 
+        # The instance cache is not inherently tied to the normal
+        # lifetime of the instances.  So must do something in order to
+        # avoid keeping the instances alive by holding a reference here.
+        # Ideally we would use weakref.WeakValueDictionary to do this.
+        # However, most builtin types do not support weakrefs.  So
+        # instead we monkey-patch __del__ on the attached class to clear
+        # the instance.
         self.instances = {}
         self.name = None
 
@@ -89,6 +96,12 @@ def __set_name__(self, cls, name):
         if self.name is not None:
             raise TypeError('already used')
         self.name = name
+        try:
+            slotnames = cls.__slot_names__
+        except AttributeError:
+            slotnames = cls.__slot_names__ = []
+        slotnames.append(name)
+        self._ensure___del__(cls, slotnames)
 
     def __get__(self, obj, cls):
         if obj is None:  # called on the class
@@ -115,7 +128,23 @@ def __set__(self, obj, value):
     def __delete__(self, obj):
         if self.readonly:
             raise AttributeError(f'{self.name} is readonly')
-        self.instances[id(obj)] = self.default
+        self.instances[id(obj)] = self.default  # XXX refleak?
+
+    def _ensure___del__(self, cls, slotnames):  # See the comment in __init__().
+        try:
+            old___del__ = cls.__del__
+        except AttributeError:
+            old___del__ = (lambda s: None)
+        else:
+            if getattr(old___del__, '_slotted', False):
+                return
+
+        def __del__(_self):
+            for name in slotnames:
+                delattr(_self, name)
+            old___del__(_self)
+        __del__._slotted = True
+        cls.__del__ = __del__
 
     def set(self, obj, value):
         """Update the cached value for an object.
diff --git a/Tools/c-analyzer/c_parser/info.py b/Tools/c-analyzer/c_parser/info.py
index d7368b48cde3..a4e32d75eed7 100644
--- a/Tools/c-analyzer/c_parser/info.py
+++ b/Tools/c-analyzer/c_parser/info.py
@@ -1,4 +1,5 @@
 from collections import namedtuple
+import re
 
 from c_analyzer_common import info, util
 from c_analyzer_common.util import classonly, _NTBase
@@ -15,28 +16,53 @@ def normalize_vartype(vartype):
     return str(vartype)
 
 
+def extract_storage(decl, *, isfunc=False):
+    """Return (storage, vartype) based on the given declaration.
+
+    The default storage is "implicit" or "local".
+    """
+    if decl == info.UNKNOWN:
+        return decl, decl
+    if decl.startswith('static '):
+        return 'static', decl
+        #return 'static', decl.partition(' ')[2].strip()
+    elif decl.startswith('extern '):
+        return 'extern', decl
+        #return 'extern', decl.partition(' ')[2].strip()
+    elif re.match('.*\b(static|extern)\b', decl):
+        raise NotImplementedError
+    elif isfunc:
+        return 'local', decl
+    else:
+        return 'implicit', decl
+
+
 class Variable(_NTBase,
-               namedtuple('Variable', 'id vartype')):
+               namedtuple('Variable', 'id storage vartype')):
     """Information about a single variable declaration."""
 
     __slots__ = ()
-    _isglobal = util.Slot()
 
-    def __del__(self):
-        del self._isglobal
+    STORAGE = (
+            'static',
+            'extern',
+            'implicit',
+            'local',
+            )
 
     @classonly
-    def from_parts(cls, filename, funcname, name, vartype, isglobal=False):
+    def from_parts(cls, filename, funcname, name, decl, storage=None):
+        if storage is None:
+            storage, decl = extract_storage(decl, isfunc=funcname)
         id = info.ID(filename, funcname, name)
-        self = cls(id, vartype)
-        if isglobal:
-            self._isglobal = True
+        self = cls(id, storage, decl)
         return self
 
-    def __new__(cls, id, vartype):
+    def __new__(cls, id, storage, vartype):
         self = super().__new__(
                 cls,
                 id=info.ID.from_raw(id),
+                storage=str(storage) if storage else None,
                 vartype=normalize_vartype(vartype) if vartype else None,
                 )
         return self
@@ -63,18 +89,17 @@ def validate(self):
         """Fail if the object is invalid (i.e. init with bad data)."""
         self._validate_id()
 
+        if self.storage is None or self.storage == info.UNKNOWN:
+            raise TypeError('missing storage')
+        elif self.storage not in self.STORAGE:
+            raise ValueError(f'unsupported storage {self.storage:r}')
+
         if self.vartype is None or self.vartype == info.UNKNOWN:
             raise TypeError('missing vartype')
 
     @property
     def isglobal(self):
-        try:
-            return self._isglobal
-        except AttributeError:
-            # XXX Include extern variables.
-            # XXX Ignore functions.
-            self._isglobal = ('static' in self.vartype.split())
-            return self._isglobal
+        return self.storage != 'local'
 
     @property
     def isconst(self):
diff --git a/Tools/c-analyzer/c_parser/naive.py b/Tools/c-analyzer/c_parser/naive.py
index e0370cc3d1d4..160f96c279e2 100644
--- a/Tools/c-analyzer/c_parser/naive.py
+++ b/Tools/c-analyzer/c_parser/naive.py
@@ -163,7 +163,7 @@ def find_variables(varids, filenames=None, *,
             srcfiles = [varid.filename]
         else:
             if not filenames:
-                yield Variable(varid, UNKNOWN)
+                yield Variable(varid, UNKNOWN, UNKNOWN)
                 continue
             srcfiles = filenames
         for filename in srcfiles:
@@ -177,4 +177,4 @@ def find_variables(varids, filenames=None, *,
                 used.add(found)
                 break
         else:
-            yield Variable(varid, UNKNOWN)
+            yield Variable(varid, UNKNOWN, UNKNOWN)
diff --git a/Tools/c-analyzer/c_symbols/resolve.py b/Tools/c-analyzer/c_symbols/resolve.py
index dc876ae0b759..56210cefeb82 100644
--- a/Tools/c-analyzer/c_symbols/resolve.py
+++ b/Tools/c-analyzer/c_symbols/resolve.py
@@ -68,14 +68,11 @@ def find_in_source(symbol, dirnames, *,
     if symbol.funcname and symbol.funcname != UNKNOWN:
         raise NotImplementedError
 
-    (filename, funcname, vartype
+    (filename, funcname, decl
      ) = _find_symbol(symbol.name, filenames, _perfilecache)
     if filename == UNKNOWN:
         return None
-    return info.Variable(
-            id=(filename, funcname, symbol.name),
-            vartype=vartype,
-            )
+    return info.Variable.from_parts(filename, funcname, symbol.name, decl)
 
 
 def get_resolver(knownvars=None, dirnames=None, *,
@@ -144,6 +141,7 @@ def symbols_to_variables(symbols, *,
             #raise NotImplementedError(symbol)
             resolved = info.Variable(
                     id=symbol.id,
+                    storage=UNKNOWN,
                     vartype=UNKNOWN,
                     )
         yield resolved


From webhook-mailer at python.org  Fri Sep 27 13:03:09 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Fri, 27 Sep 2019 17:03:09 -0000
Subject: [Python-checkins] bpo-32820: Simplify __format__ implementation for
 ipaddress. (GH-16378)
Message-ID: <mailman.121.1569603789.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/5d6f5b629394066a5249af25cc01f1a1f0edc138
commit: 5d6f5b629394066a5249af25cc01f1a1f0edc138
branch: master
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-27T20:02:58+03:00
summary:

bpo-32820: Simplify __format__ implementation for ipaddress. (GH-16378)

Also cache the compiled RE for parsing the format specifier.

files:
M Lib/ipaddress.py

diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
index c389f0528823..7d80a52c158a 100644
--- a/Lib/ipaddress.py
+++ b/Lib/ipaddress.py
@@ -560,6 +560,8 @@ def __reduce__(self):
         return self.__class__, (str(self),)
 
 
+_address_fmt_re = None
+
 @functools.total_ordering
 class _BaseAddress(_IPAddressBase):
 
@@ -623,72 +625,49 @@ def __format__(self, fmt):
 
         Supported presentation types are:
         's': returns the IP address as a string (default)
-        'b' or 'n': converts to binary and returns a zero-padded string
+        'b': converts to binary and returns a zero-padded string
         'X' or 'x': converts to upper- or lower-case hex and returns a zero-padded string
+        'n': the same as 'b' for IPv4 and 'x' for IPv6
 
         For binary and hex presentation types, the alternate form specifier
         '#' and the grouping option '_' are supported.
         """
 
-
         # Support string formatting
         if not fmt or fmt[-1] == 's':
-            # let format() handle it
             return format(str(self), fmt)
 
         # From here on down, support for 'bnXx'
+        global _address_fmt_re
+        if _address_fmt_re is None:
+            import re
+            _address_fmt_re = re.compile('(#?)(_?)([xbnX])')
 
-        import re
-        fmt_re = '^(?P<alternate>#?)(?P<grouping>_?)(?P<fmt_base>[xbnX]){1}$'
-        m = re.match(fmt_re, fmt)
+        m = _address_fmt_re.fullmatch(fmt)
         if not m:
             return super().__format__(fmt)
 
-        groupdict = m.groupdict()
-        alternate = groupdict['alternate']
-        grouping = groupdict['grouping']
-        fmt_base = groupdict['fmt_base']
+        alternate, grouping, fmt_base = m.groups()
 
         # Set some defaults
         if fmt_base == 'n':
             if self._version == 4:
                 fmt_base = 'b'  # Binary is default for ipv4
-            if self._version == 6:
+            else:
                 fmt_base = 'x'  # Hex is default for ipv6
 
-        # Handle binary formatting
         if fmt_base == 'b':
-            if self._version == 4:
-                # resulting string is '0b' + 32 bits
-                #  plus 7 _ if needed
-                padlen = IPV4LENGTH+2 + (7*len(grouping))
-            elif self._version == 6:
-                # resulting string is '0b' + 128 bits
-                #  plus 31 _ if needed
-                padlen = IPV6LENGTH+2 + (31*len(grouping))
-
-        # Handle hex formatting
-        elif fmt_base in 'Xx':
-            if self._version == 4:
-                # resulting string is '0x' + 8 hex digits
-                # plus a single _ if needed
-                padlen = int(IPV4LENGTH/4)+2 + len(grouping)
-            elif self._version == 6:
-                # resulting string is '0x' + 32 hex digits
-                # plus 7 _ if needed
-                padlen = int(IPV6LENGTH/4)+2 + (7*len(grouping))
-
-        retstr = f'{int(self):#0{padlen}{grouping}{fmt_base}}'
+            padlen = self._max_prefixlen
+        else:
+            padlen = self._max_prefixlen // 4
 
-        if fmt_base == 'X':
-            retstr = retstr.upper()
+        if grouping:
+            padlen += padlen // 4 - 1
 
-        # If alternate is not set, strip the two leftmost
-        #  characters ('0b')
-        if not alternate:
-            retstr = retstr[2:]
+        if alternate:
+            padlen += 2  # 0b or 0x
 
-        return retstr
+        return format(int(self), f'{alternate}0{padlen}{grouping}{fmt_base}')
 
 
 @functools.total_ordering


From webhook-mailer at python.org  Fri Sep 27 14:01:39 2019
From: webhook-mailer at python.org (Serhiy Storchaka)
Date: Fri, 27 Sep 2019 18:01:39 -0000
Subject: [Python-checkins] Fix typo in the "Porting to Python 3.8" section.
 (GH-16435)
Message-ID: <mailman.122.1569607300.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/dd6117c6d7859fee57751593cd56f0862131de8b
commit: dd6117c6d7859fee57751593cd56f0862131de8b
branch: master
author: bariod <35639254+bariod at users.noreply.github.com>
committer: Serhiy Storchaka <storchaka at gmail.com>
date: 2019-09-27T21:01:33+03:00
summary:

Fix typo in the "Porting to Python 3.8" section. (GH-16435)

files:
M Doc/whatsnew/3.8.rst

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 0995cb3b9119..4cab2f7af118 100755
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -1531,7 +1531,7 @@ Changes in Python behavior
   :class:`int`, :class:`float`, :class:`complex` and few classes from
   the standard library.  They now inherit ``__str__()`` from :class:`object`.
   As result, defining the ``__repr__()`` method in the subclass of these
-  classes will affect they string representation.
+  classes will affect their string representation.
   (Contributed by Serhiy Storchaka in :issue:`36793`.)
 
 * On AIX, :attr:`sys.platform` doesn't contain the major version anymore.


From webhook-mailer at python.org  Fri Sep 27 15:59:41 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Fri, 27 Sep 2019 19:59:41 -0000
Subject: [Python-checkins] bpo-38243,
 xmlrpc.server: Escape the server_title (GH-16373)
Message-ID: <mailman.123.1569614382.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/e8650a4f8c7fb76f570d4ca9c1fbe44e91c8dfaa
commit: e8650a4f8c7fb76f570d4ca9c1fbe44e91c8dfaa
branch: master
author: Dong-hee Na <donghee.na92 at gmail.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-09-27T21:59:37+02:00
summary:

bpo-38243, xmlrpc.server: Escape the server_title (GH-16373)

Escape the server title of xmlrpc.server.DocXMLRPCServer
when rendering the document page as HTML.

files:
A Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst
M Lib/test/test_docxmlrpc.py
M Lib/xmlrpc/server.py

diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py
index 116e626740df..7d3e30cbee96 100644
--- a/Lib/test/test_docxmlrpc.py
+++ b/Lib/test/test_docxmlrpc.py
@@ -1,5 +1,6 @@
 from xmlrpc.server import DocXMLRPCServer
 import http.client
+import re
 import sys
 import threading
 import unittest
@@ -192,6 +193,21 @@ def test_annotations(self):
              b'method_annotation</strong></a>(x: bytes)</dt></dl>'),
             response.read())
 
+    def test_server_title_escape(self):
+        # bpo-38243: Ensure that the server title and documentation
+        # are escaped for HTML.
+        self.serv.set_server_title('test_title<script>')
+        self.serv.set_server_documentation('test_documentation<script>')
+        self.assertEqual('test_title<script>', self.serv.server_title)
+        self.assertEqual('test_documentation<script>',
+                self.serv.server_documentation)
+
+        generated = self.serv.generate_html_documentation()
+        title = re.search(r'<title>(.+?)</title>', generated).group()
+        documentation = re.search(r'<p><tt>(.+?)</tt></p>', generated).group()
+        self.assertEqual('<title>Python: test_title&lt;script&gt;</title>', title)
+        self.assertEqual('<p><tt>test_documentation&lt;script&gt;</tt></p>', documentation)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py
index f1c467eb1b2b..32aba4df4c7e 100644
--- a/Lib/xmlrpc/server.py
+++ b/Lib/xmlrpc/server.py
@@ -108,6 +108,7 @@ def export_add(self, x, y):
 from http.server import BaseHTTPRequestHandler
 from functools import partial
 from inspect import signature
+import html
 import http.server
 import socketserver
 import sys
@@ -894,7 +895,7 @@ def generate_html_documentation(self):
                                 methods
                             )
 
-        return documenter.page(self.server_title, documentation)
+        return documenter.page(html.escape(self.server_title), documentation)
 
 class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
     """XML-RPC and documentation request handler class.
diff --git a/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst b/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst
new file mode 100644
index 000000000000..98d7be129573
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst
@@ -0,0 +1,3 @@
+Escape the server title of :class:`xmlrpc.server.DocXMLRPCServer`
+when rendering the document page as HTML.
+(Contributed by Dong-hee Na in :issue:`38243`.)


From webhook-mailer at python.org  Fri Sep 27 16:18:20 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 27 Sep 2019 20:18:20 -0000
Subject: [Python-checkins] bpo-38243,
 xmlrpc.server: Escape the server_title (GH-16373)
Message-ID: <mailman.124.1569615501.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/39a0c7555530e31c6941a78da19b6a5b61170687
commit: 39a0c7555530e31c6941a78da19b6a5b61170687
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-27T13:18:14-07:00
summary:

bpo-38243, xmlrpc.server: Escape the server_title (GH-16373)


Escape the server title of xmlrpc.server.DocXMLRPCServer
when rendering the document page as HTML.
(cherry picked from commit e8650a4f8c7fb76f570d4ca9c1fbe44e91c8dfaa)

Co-authored-by: Dong-hee Na <donghee.na92 at gmail.com>

files:
A Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst
M Lib/test/test_docxmlrpc.py
M Lib/xmlrpc/server.py

diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py
index f077f05f5b4f..38215659b67d 100644
--- a/Lib/test/test_docxmlrpc.py
+++ b/Lib/test/test_docxmlrpc.py
@@ -1,5 +1,6 @@
 from xmlrpc.server import DocXMLRPCServer
 import http.client
+import re
 import sys
 import threading
 from test import support
@@ -193,6 +194,21 @@ def test_annotations(self):
              b'method_annotation</strong></a>(x: bytes)</dt></dl>'),
             response.read())
 
+    def test_server_title_escape(self):
+        # bpo-38243: Ensure that the server title and documentation
+        # are escaped for HTML.
+        self.serv.set_server_title('test_title<script>')
+        self.serv.set_server_documentation('test_documentation<script>')
+        self.assertEqual('test_title<script>', self.serv.server_title)
+        self.assertEqual('test_documentation<script>',
+                self.serv.server_documentation)
+
+        generated = self.serv.generate_html_documentation()
+        title = re.search(r'<title>(.+?)</title>', generated).group()
+        documentation = re.search(r'<p><tt>(.+?)</tt></p>', generated).group()
+        self.assertEqual('<title>Python: test_title&lt;script&gt;</title>', title)
+        self.assertEqual('<p><tt>test_documentation&lt;script&gt;</tt></p>', documentation)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py
index f1c467eb1b2b..32aba4df4c7e 100644
--- a/Lib/xmlrpc/server.py
+++ b/Lib/xmlrpc/server.py
@@ -108,6 +108,7 @@ def export_add(self, x, y):
 from http.server import BaseHTTPRequestHandler
 from functools import partial
 from inspect import signature
+import html
 import http.server
 import socketserver
 import sys
@@ -894,7 +895,7 @@ def generate_html_documentation(self):
                                 methods
                             )
 
-        return documenter.page(self.server_title, documentation)
+        return documenter.page(html.escape(self.server_title), documentation)
 
 class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
     """XML-RPC and documentation request handler class.
diff --git a/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst b/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst
new file mode 100644
index 000000000000..98d7be129573
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst
@@ -0,0 +1,3 @@
+Escape the server title of :class:`xmlrpc.server.DocXMLRPCServer`
+when rendering the document page as HTML.
+(Contributed by Dong-hee Na in :issue:`38243`.)


From webhook-mailer at python.org  Fri Sep 27 16:19:45 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Fri, 27 Sep 2019 20:19:45 -0000
Subject: [Python-checkins] bpo-38243,
 xmlrpc.server: Escape the server_title (GH-16373)
Message-ID: <mailman.125.1569615586.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/6447b9f9bd27e1f6b04cef674dd3a7ab27bf4f28
commit: 6447b9f9bd27e1f6b04cef674dd3a7ab27bf4f28
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-27T13:19:41-07:00
summary:

bpo-38243, xmlrpc.server: Escape the server_title (GH-16373)


Escape the server title of xmlrpc.server.DocXMLRPCServer
when rendering the document page as HTML.
(cherry picked from commit e8650a4f8c7fb76f570d4ca9c1fbe44e91c8dfaa)

Co-authored-by: Dong-hee Na <donghee.na92 at gmail.com>

files:
A Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst
M Lib/test/test_docxmlrpc.py
M Lib/xmlrpc/server.py

diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py
index 116e626740df..7d3e30cbee96 100644
--- a/Lib/test/test_docxmlrpc.py
+++ b/Lib/test/test_docxmlrpc.py
@@ -1,5 +1,6 @@
 from xmlrpc.server import DocXMLRPCServer
 import http.client
+import re
 import sys
 import threading
 import unittest
@@ -192,6 +193,21 @@ def test_annotations(self):
              b'method_annotation</strong></a>(x: bytes)</dt></dl>'),
             response.read())
 
+    def test_server_title_escape(self):
+        # bpo-38243: Ensure that the server title and documentation
+        # are escaped for HTML.
+        self.serv.set_server_title('test_title<script>')
+        self.serv.set_server_documentation('test_documentation<script>')
+        self.assertEqual('test_title<script>', self.serv.server_title)
+        self.assertEqual('test_documentation<script>',
+                self.serv.server_documentation)
+
+        generated = self.serv.generate_html_documentation()
+        title = re.search(r'<title>(.+?)</title>', generated).group()
+        documentation = re.search(r'<p><tt>(.+?)</tt></p>', generated).group()
+        self.assertEqual('<title>Python: test_title&lt;script&gt;</title>', title)
+        self.assertEqual('<p><tt>test_documentation&lt;script&gt;</tt></p>', documentation)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py
index f1c467eb1b2b..32aba4df4c7e 100644
--- a/Lib/xmlrpc/server.py
+++ b/Lib/xmlrpc/server.py
@@ -108,6 +108,7 @@ def export_add(self, x, y):
 from http.server import BaseHTTPRequestHandler
 from functools import partial
 from inspect import signature
+import html
 import http.server
 import socketserver
 import sys
@@ -894,7 +895,7 @@ def generate_html_documentation(self):
                                 methods
                             )
 
-        return documenter.page(self.server_title, documentation)
+        return documenter.page(html.escape(self.server_title), documentation)
 
 class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
     """XML-RPC and documentation request handler class.
diff --git a/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst b/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst
new file mode 100644
index 000000000000..98d7be129573
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst
@@ -0,0 +1,3 @@
+Escape the server title of :class:`xmlrpc.server.DocXMLRPCServer`
+when rendering the document page as HTML.
+(Contributed by Dong-hee Na in :issue:`38243`.)


From webhook-mailer at python.org  Fri Sep 27 18:44:38 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Fri, 27 Sep 2019 22:44:38 -0000
Subject: [Python-checkins] [3.8] bpo-38136: Updates await_count and
 call_count to be different things (GH-16192) (GH-16431)
Message-ID: <mailman.127.1569624280.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/52bdd414ed9da7c62c312c542803753986a0040a
commit: 52bdd414ed9da7c62c312c542803753986a0040a
branch: 3.8
author: Lisa Roach <lisaroach14 at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-27T15:44:34-07:00
summary:

[3.8] bpo-38136: Updates await_count and call_count to be different things (GH-16192) (GH-16431)

files:
A Misc/NEWS.d/next/Library/2019-09-16-09-54-42.bpo-38136.MdI-Zb.rst
M Doc/library/unittest.mock.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py
M Lib/unittest/test/testmock/testmock.py

diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index b446ddb3598f..300f28c8e2ce 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -514,21 +514,6 @@ the *new_callable* argument to :func:`patch`.
             >>> mock.call_count
             2
 
-        For :class:`AsyncMock` the :attr:`call_count` is only iterated if the function
-        has been awaited:
-
-            >>> mock = AsyncMock()
-            >>> mock()  # doctest: +SKIP
-            <coroutine object AsyncMockMixin._mock_call at ...>
-            >>> mock.call_count
-            0
-            >>> async def main():
-            ...     await mock()
-            ...
-            >>> asyncio.run(main())
-            >>> mock.call_count
-            1
-
     .. attribute:: return_value
 
         Set this to configure the value returned by calling the mock:
@@ -907,19 +892,22 @@ object::
 
   .. method:: assert_awaited()
 
-      Assert that the mock was awaited at least once.
+      Assert that the mock was awaited at least once. Note that this is separate
+      from the object having been called, the ``await`` keyword must be used:
 
           >>> mock = AsyncMock()
-          >>> async def main():
-          ...     await mock()
+          >>> async def main(coroutine_mock):
+          ...     await coroutine_mock
           ...
-          >>> asyncio.run(main())
+          >>> coroutine_mock = mock()
+          >>> mock.called
+          True
           >>> mock.assert_awaited()
-          >>> mock_2 = AsyncMock()
-          >>> mock_2.assert_awaited()
           Traceback (most recent call last):
           ...
           AssertionError: Expected mock to have been awaited.
+          >>> asyncio.run(main(coroutine_mock))
+          >>> mock.assert_awaited()
 
   .. method:: assert_awaited_once()
 
@@ -1004,14 +992,15 @@ object::
         ...     await mock(*args, **kwargs)
         ...
         >>> calls = [call("foo"), call("bar")]
-        >>> mock.assert_has_calls(calls)
+        >>> mock.assert_has_awaits(calls)
         Traceback (most recent call last):
         ...
-        AssertionError: Calls not found.
+        AssertionError: Awaits not found.
         Expected: [call('foo'), call('bar')]
+        Actual: []
         >>> asyncio.run(main('foo'))
         >>> asyncio.run(main('bar'))
-        >>> mock.assert_has_calls(calls)
+        >>> mock.assert_has_awaits(calls)
 
   .. method:: assert_not_awaited()
 
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 30f4663e5ef4..5ea562428a12 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -1070,14 +1070,20 @@ def __call__(self, /, *args, **kwargs):
         # can't use self in-case a function / method we are mocking uses self
         # in the signature
         self._mock_check_sig(*args, **kwargs)
+        self._increment_mock_call(*args, **kwargs)
         return self._mock_call(*args, **kwargs)
 
 
     def _mock_call(self, /, *args, **kwargs):
+        return self._execute_mock_call(*args, **kwargs)
+
+    def _increment_mock_call(self, /, *args, **kwargs):
         self.called = True
         self.call_count += 1
 
         # handle call_args
+        # needs to be set here so assertions on call arguments pass before
+        # execution in the case of awaited calls
         _call = _Call((args, kwargs), two=True)
         self.call_args = _call
         self.call_args_list.append(_call)
@@ -1117,6 +1123,10 @@ def _mock_call(self, /, *args, **kwargs):
             # follow the parental chain:
             _new_parent = _new_parent._mock_new_parent
 
+    def _execute_mock_call(self, /, *args, **kwargs):
+        # seperate from _increment_mock_call so that awaited functions are
+        # executed seperately from their call
+
         effect = self.side_effect
         if effect is not None:
             if _is_exception(effect):
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index af53210d97b3..86b0d0ee9a13 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -3,8 +3,8 @@
 import re
 import unittest
 
-from unittest.mock import (call, AsyncMock, patch, MagicMock, create_autospec,
-                           _AwaitEvent)
+from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock,
+                           create_autospec, _AwaitEvent, sentinel, _CallList)
 
 
 def tearDownModule():
@@ -591,11 +591,173 @@ class AsyncMockAssert(unittest.TestCase):
     def setUp(self):
         self.mock = AsyncMock()
 
-    async def _runnable_test(self, *args):
-        if not args:
-            await self.mock()
-        else:
-            await self.mock(*args)
+    async def _runnable_test(self, *args, **kwargs):
+        await self.mock(*args, **kwargs)
+
+    async def _await_coroutine(self, coroutine):
+        return await coroutine
+
+    def test_assert_called_but_not_awaited(self):
+        mock = AsyncMock(AsyncClass)
+        with self.assertWarns(RuntimeWarning):
+            # Will raise a warning because never awaited
+            mock.async_method()
+        self.assertTrue(asyncio.iscoroutinefunction(mock.async_method))
+        mock.async_method.assert_called()
+        mock.async_method.assert_called_once()
+        mock.async_method.assert_called_once_with()
+        with self.assertRaises(AssertionError):
+            mock.assert_awaited()
+        with self.assertRaises(AssertionError):
+            mock.async_method.assert_awaited()
+
+    def test_assert_called_then_awaited(self):
+        mock = AsyncMock(AsyncClass)
+        mock_coroutine = mock.async_method()
+        mock.async_method.assert_called()
+        mock.async_method.assert_called_once()
+        mock.async_method.assert_called_once_with()
+        with self.assertRaises(AssertionError):
+            mock.async_method.assert_awaited()
+
+        asyncio.run(self._await_coroutine(mock_coroutine))
+        # Assert we haven't re-called the function
+        mock.async_method.assert_called_once()
+        mock.async_method.assert_awaited()
+        mock.async_method.assert_awaited_once()
+        mock.async_method.assert_awaited_once_with()
+
+    def test_assert_called_and_awaited_at_same_time(self):
+        with self.assertRaises(AssertionError):
+            self.mock.assert_awaited()
+
+        with self.assertRaises(AssertionError):
+            self.mock.assert_called()
+
+        asyncio.run(self._runnable_test())
+        self.mock.assert_called_once()
+        self.mock.assert_awaited_once()
+
+    def test_assert_called_twice_and_awaited_once(self):
+        mock = AsyncMock(AsyncClass)
+        coroutine = mock.async_method()
+        with self.assertWarns(RuntimeWarning):
+            # The first call will be awaited so no warning there
+            # But this call will never get awaited, so it will warn here
+            mock.async_method()
+        with self.assertRaises(AssertionError):
+            mock.async_method.assert_awaited()
+        mock.async_method.assert_called()
+        asyncio.run(self._await_coroutine(coroutine))
+        mock.async_method.assert_awaited()
+        mock.async_method.assert_awaited_once()
+
+    def test_assert_called_once_and_awaited_twice(self):
+        mock = AsyncMock(AsyncClass)
+        coroutine = mock.async_method()
+        mock.async_method.assert_called_once()
+        asyncio.run(self._await_coroutine(coroutine))
+        with self.assertRaises(RuntimeError):
+            # Cannot reuse already awaited coroutine
+            asyncio.run(self._await_coroutine(coroutine))
+        mock.async_method.assert_awaited()
+
+    def test_assert_awaited_but_not_called(self):
+        with self.assertRaises(AssertionError):
+            self.mock.assert_awaited()
+        with self.assertRaises(AssertionError):
+            self.mock.assert_called()
+        with self.assertRaises(TypeError):
+            # You cannot await an AsyncMock, it must be a coroutine
+            asyncio.run(self._await_coroutine(self.mock))
+
+        with self.assertRaises(AssertionError):
+            self.mock.assert_awaited()
+        with self.assertRaises(AssertionError):
+            self.mock.assert_called()
+
+    def test_assert_has_calls_not_awaits(self):
+        kalls = [call('foo')]
+        with self.assertWarns(RuntimeWarning):
+            # Will raise a warning because never awaited
+            self.mock('foo')
+        self.mock.assert_has_calls(kalls)
+        with self.assertRaises(AssertionError):
+            self.mock.assert_has_awaits(kalls)
+
+    def test_assert_has_mock_calls_on_async_mock_no_spec(self):
+        with self.assertWarns(RuntimeWarning):
+            # Will raise a warning because never awaited
+            self.mock()
+        kalls_empty = [('', (), {})]
+        self.assertEqual(self.mock.mock_calls, kalls_empty)
+
+        with self.assertWarns(RuntimeWarning):
+            # Will raise a warning because never awaited
+            self.mock('foo')
+            self.mock('baz')
+        mock_kalls = ([call(), call('foo'), call('baz')])
+        self.assertEqual(self.mock.mock_calls, mock_kalls)
+
+    def test_assert_has_mock_calls_on_async_mock_with_spec(self):
+        a_class_mock = AsyncMock(AsyncClass)
+        with self.assertWarns(RuntimeWarning):
+            # Will raise a warning because never awaited
+            a_class_mock.async_method()
+        kalls_empty = [('', (), {})]
+        self.assertEqual(a_class_mock.async_method.mock_calls, kalls_empty)
+        self.assertEqual(a_class_mock.mock_calls, [call.async_method()])
+
+        with self.assertWarns(RuntimeWarning):
+            # Will raise a warning because never awaited
+            a_class_mock.async_method(1, 2, 3, a=4, b=5)
+        method_kalls = [call(), call(1, 2, 3, a=4, b=5)]
+        mock_kalls = [call.async_method(), call.async_method(1, 2, 3, a=4, b=5)]
+        self.assertEqual(a_class_mock.async_method.mock_calls, method_kalls)
+        self.assertEqual(a_class_mock.mock_calls, mock_kalls)
+
+    def test_async_method_calls_recorded(self):
+        with self.assertWarns(RuntimeWarning):
+            # Will raise warnings because never awaited
+            self.mock.something(3, fish=None)
+            self.mock.something_else.something(6, cake=sentinel.Cake)
+
+        self.assertEqual(self.mock.method_calls, [
+            ("something", (3,), {'fish': None}),
+            ("something_else.something", (6,), {'cake': sentinel.Cake})
+        ],
+            "method calls not recorded correctly")
+        self.assertEqual(self.mock.something_else.method_calls,
+                         [("something", (6,), {'cake': sentinel.Cake})],
+                         "method calls not recorded correctly")
+
+    def test_async_arg_lists(self):
+        def assert_attrs(mock):
+            names = ('call_args_list', 'method_calls', 'mock_calls')
+            for name in names:
+                attr = getattr(mock, name)
+                self.assertIsInstance(attr, _CallList)
+                self.assertIsInstance(attr, list)
+                self.assertEqual(attr, [])
+
+        assert_attrs(self.mock)
+        with self.assertWarns(RuntimeWarning):
+            # Will raise warnings because never awaited
+            self.mock()
+            self.mock(1, 2)
+            self.mock(a=3)
+
+        self.mock.reset_mock()
+        assert_attrs(self.mock)
+
+        a_mock = AsyncMock(AsyncClass)
+        with self.assertWarns(RuntimeWarning):
+            # Will raise warnings because never awaited
+            a_mock.async_method()
+            a_mock.async_method(1, a=3)
+
+        a_mock.reset_mock()
+        assert_attrs(a_mock)
 
     def test_assert_awaited(self):
         with self.assertRaises(AssertionError):
@@ -641,20 +803,20 @@ def test_assert_awaited_once_with(self):
 
     def test_assert_any_wait(self):
         with self.assertRaises(AssertionError):
-            self.mock.assert_any_await('NormalFoo')
+            self.mock.assert_any_await('foo')
 
-        asyncio.run(self._runnable_test('foo'))
+        asyncio.run(self._runnable_test('baz'))
         with self.assertRaises(AssertionError):
-            self.mock.assert_any_await('NormalFoo')
+            self.mock.assert_any_await('foo')
 
-        asyncio.run(self._runnable_test('NormalFoo'))
-        self.mock.assert_any_await('NormalFoo')
+        asyncio.run(self._runnable_test('foo'))
+        self.mock.assert_any_await('foo')
 
         asyncio.run(self._runnable_test('SomethingElse'))
-        self.mock.assert_any_await('NormalFoo')
+        self.mock.assert_any_await('foo')
 
     def test_assert_has_awaits_no_order(self):
-        calls = [call('NormalFoo'), call('baz')]
+        calls = [call('foo'), call('baz')]
 
         with self.assertRaises(AssertionError) as cm:
             self.mock.assert_has_awaits(calls)
@@ -664,7 +826,7 @@ def test_assert_has_awaits_no_order(self):
         with self.assertRaises(AssertionError):
             self.mock.assert_has_awaits(calls)
 
-        asyncio.run(self._runnable_test('NormalFoo'))
+        asyncio.run(self._runnable_test('foo'))
         with self.assertRaises(AssertionError):
             self.mock.assert_has_awaits(calls)
 
@@ -675,7 +837,7 @@ def test_assert_has_awaits_no_order(self):
         self.mock.assert_has_awaits(calls)
 
     def test_assert_has_awaits_ordered(self):
-        calls = [call('NormalFoo'), call('baz')]
+        calls = [call('foo'), call('baz')]
         with self.assertRaises(AssertionError):
             self.mock.assert_has_awaits(calls, any_order=True)
 
@@ -683,11 +845,11 @@ def test_assert_has_awaits_ordered(self):
         with self.assertRaises(AssertionError):
             self.mock.assert_has_awaits(calls, any_order=True)
 
-        asyncio.run(self._runnable_test('foo'))
+        asyncio.run(self._runnable_test('bamf'))
         with self.assertRaises(AssertionError):
             self.mock.assert_has_awaits(calls, any_order=True)
 
-        asyncio.run(self._runnable_test('NormalFoo'))
+        asyncio.run(self._runnable_test('foo'))
         self.mock.assert_has_awaits(calls, any_order=True)
 
         asyncio.run(self._runnable_test('qux'))
diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py
index 817c54822630..6dc2725427ec 100644
--- a/Lib/unittest/test/testmock/testmock.py
+++ b/Lib/unittest/test/testmock/testmock.py
@@ -842,6 +842,7 @@ def test():
     def test_setting_call(self):
         mock = Mock()
         def __call__(self, a):
+            self._increment_mock_call(a)
             return self._mock_call(a)
 
         type(mock).__call__ = __call__
@@ -2043,7 +2044,7 @@ def trace(frame, event, arg):  # pragma: no cover
             )
 
             mocks = [
-                Mock, MagicMock, NonCallableMock, NonCallableMagicMock
+                Mock, MagicMock, NonCallableMock, NonCallableMagicMock, AsyncMock
             ]
 
             for mock in mocks:
diff --git a/Misc/NEWS.d/next/Library/2019-09-16-09-54-42.bpo-38136.MdI-Zb.rst b/Misc/NEWS.d/next/Library/2019-09-16-09-54-42.bpo-38136.MdI-Zb.rst
new file mode 100644
index 000000000000..78cad245e67d
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-16-09-54-42.bpo-38136.MdI-Zb.rst
@@ -0,0 +1,3 @@
+Changes AsyncMock call count and await count to be two different counters.
+Now await count only counts when a coroutine has been awaited, not when it
+has been called, and vice-versa. Update the documentation around this.


From webhook-mailer at python.org  Fri Sep 27 22:28:38 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Sat, 28 Sep 2019 02:28:38 -0000
Subject: [Python-checkins] bpo-38304: Add PyConfig.struct_size (GH-16451)
Message-ID: <mailman.128.1569637720.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/441b10cf2855955c86565f8d59e72c2efc0f0a57
commit: 441b10cf2855955c86565f8d59e72c2efc0f0a57
branch: master
author: Victor Stinner <vstinner at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-28T04:28:35+02:00
summary:

bpo-38304: Add PyConfig.struct_size (GH-16451)

Add a new struct_size field to PyPreConfig and PyConfig structures to
allow to modify these structures in the future without breaking the
backward compatibility.

* Replace private _config_version field with public struct_size field
  in PyPreConfig and PyConfig.
* Public PyPreConfig_InitIsolatedConfig() and
  PyPreConfig_InitPythonConfig()
  return type becomes PyStatus, instead of void.
* Internal _PyConfig_InitCompatConfig(),
  _PyPreConfig_InitCompatConfig(), _PyPreConfig_InitFromConfig(),
  _PyPreConfig_InitFromPreConfig() return type becomes PyStatus,
  instead of void.
* Remove _Py_CONFIG_VERSION
* Update the Initialization Configuration documentation.

files:
A Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst
M Doc/c-api/init_config.rst
M Include/cpython/initconfig.h
M Include/internal/pycore_initconfig.h
M Modules/main.c
M Programs/_freeze_importlib.c
M Programs/_testembed.c
M Python/frozenmain.c
M Python/initconfig.c
M Python/pathconfig.c
M Python/preconfig.c
M Python/pylifecycle.c
M Python/pystate.c

diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index 0c3c725c841a..58e417441d13 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -194,18 +194,25 @@ PyPreConfig
    * Configure the LC_CTYPE locale
    * Set the UTF-8 mode
 
+   The :c:member:`struct_size` field must be explicitly initialized to
+   ``sizeof(PyPreConfig)``.
+
    Function to initialize a preconfiguration:
 
-   .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig)
+   .. c:function:: PyStatus PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig)
 
       Initialize the preconfiguration with :ref:`Python Configuration
       <init-python-config>`.
 
-   .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig)
+   .. c:function:: PyStatus PyPreConfig_InitPythonConfig(PyPreConfig *preconfig)
 
       Initialize the preconfiguration with :ref:`Isolated Configuration
       <init-isolated-conf>`.
 
+   The caller of these functions is responsible to handle exceptions (error or
+   exit) using :c:func:`PyStatus_Exception` and
+   :c:func:`Py_ExitStatusException`.
+
    Structure fields:
 
    .. c:member:: int allocator
@@ -267,6 +274,13 @@ PyPreConfig
       same way the regular Python parses command line arguments: see
       :ref:`Command Line Arguments <using-on-cmdline>`.
 
+   .. c:member:: size_t struct_size
+
+      Size of the structure in bytes: must be initialized to
+      ``sizeof(PyPreConfig)``.
+
+      Field used for API and ABI compatibility.
+
    .. c:member:: int use_environment
 
       See :c:member:`PyConfig.use_environment`.
@@ -316,12 +330,18 @@ the preinitialization.
 
 Example using the preinitialization to enable the UTF-8 Mode::
 
+    PyStatus status;
     PyPreConfig preconfig;
-    PyPreConfig_InitPythonConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = PyPreConfig_InitPythonConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
 
     preconfig.utf8_mode = 1;
 
-    PyStatus status = Py_PreInitialize(&preconfig);
+    status = Py_PreInitialize(&preconfig);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
     }
@@ -340,6 +360,9 @@ PyConfig
 
    Structure containing most parameters to configure Python.
 
+   The :c:member:`struct_size` field must be explicitly initialized to
+   ``sizeof(PyConfig)``.
+
    Structure methods:
 
    .. c:function:: PyStatus PyConfig_InitPythonConfig(PyConfig *config)
@@ -656,6 +679,13 @@ PyConfig
       Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and
       :data:`sys.stderr`.
 
+   .. c:member:: size_t struct_size
+
+      Size of the structure in bytes: must be initialized to
+      ``sizeof(PyConfig)``.
+
+      Field used for API and ABI compatibility.
+
    .. c:member:: int tracemalloc
 
       If non-zero, call :func:`tracemalloc.start` at startup.
@@ -718,6 +748,7 @@ Example setting the program name::
     {
         PyStatus status;
         PyConfig config;
+        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
@@ -750,6 +781,7 @@ configuration, and then override some parameters::
     {
         PyStatus status;
         PyConfig config;
+        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
@@ -835,8 +867,9 @@ Example of customized Python always running in isolated mode::
 
     int main(int argc, char **argv)
     {
-        PyConfig config;
         PyStatus status;
+        PyConfig config;
+        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
@@ -1028,6 +1061,7 @@ phases::
     {
         PyStatus status;
         PyConfig config;
+        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
index 047f511f3104..d5effb8397fc 100644
--- a/Include/cpython/initconfig.h
+++ b/Include/cpython/initconfig.h
@@ -45,8 +45,10 @@ PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list,
 /* --- PyPreConfig ----------------------------------------------- */
 
 typedef struct {
-    int _config_version;  /* Internal configuration version,
-                             used for ABI compatibility */
+    /* Size of the structure in bytes: must be initialized to
+       sizeof(PyPreConfig). Field used for API and ABI compatibility. */
+    size_t struct_size;
+
     int _config_init;     /* _PyConfigInitEnum value */
 
     /* Parse Py_PreInitializeFromBytesArgs() arguments?
@@ -122,15 +124,17 @@ typedef struct {
     int allocator;
 } PyPreConfig;
 
-PyAPI_FUNC(void) PyPreConfig_InitPythonConfig(PyPreConfig *config);
-PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config);
+PyAPI_FUNC(PyStatus) PyPreConfig_InitPythonConfig(PyPreConfig *config);
+PyAPI_FUNC(PyStatus) PyPreConfig_InitIsolatedConfig(PyPreConfig *config);
 
 
 /* --- PyConfig ---------------------------------------------- */
 
 typedef struct {
-    int _config_version;  /* Internal configuration version,
-                             used for ABI compatibility */
+    /* Size of the structure in bytes: must be initialized to
+       sizeof(PyConfig). Field used for API and ABI compatibility. */
+    size_t struct_size;
+
     int _config_init;     /* _PyConfigInitEnum value */
 
     int isolated;         /* Isolated mode? see PyPreConfig.isolated */
@@ -403,7 +407,6 @@ typedef struct {
 
     /* If equal to 0, stop Python initialization before the "main" phase */
     int _init_main;
-
 } PyConfig;
 
 PyAPI_FUNC(PyStatus) PyConfig_InitPythonConfig(PyConfig *config);
diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h
index a7c1994eca16..dcdb61695cab 100644
--- a/Include/internal/pycore_initconfig.h
+++ b/Include/internal/pycore_initconfig.h
@@ -40,6 +40,8 @@ extern "C" {
     (err._type == _PyStatus_TYPE_EXIT)
 #define _PyStatus_EXCEPTION(err) \
     (err._type != _PyStatus_TYPE_OK)
+#define _PyStatus_UPDATE_FUNC(err) \
+    do { err.func = _PyStatus_GET_FUNC(); } while (0)
 
 /* --- PyWideStringList ------------------------------------------------ */
 
@@ -118,11 +120,11 @@ extern PyStatus _PyPreCmdline_Read(_PyPreCmdline *cmdline,
 
 /* --- PyPreConfig ----------------------------------------------- */
 
-PyAPI_FUNC(void) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig);
-extern void _PyPreConfig_InitFromConfig(
+PyAPI_FUNC(PyStatus) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig);
+extern PyStatus _PyPreConfig_InitFromConfig(
     PyPreConfig *preconfig,
     const PyConfig *config);
-extern void _PyPreConfig_InitFromPreConfig(
+extern PyStatus _PyPreConfig_InitFromPreConfig(
     PyPreConfig *preconfig,
     const PyPreConfig *config2);
 extern PyObject* _PyPreConfig_AsDict(const PyPreConfig *preconfig);
@@ -135,8 +137,6 @@ extern PyStatus _PyPreConfig_Write(const PyPreConfig *preconfig);
 
 /* --- PyConfig ---------------------------------------------- */
 
-#define _Py_CONFIG_VERSION 1
-
 typedef enum {
     /* Py_Initialize() API: backward compatibility with Python 3.6 and 3.7 */
     _PyConfig_INIT_COMPAT = 1,
@@ -144,7 +144,7 @@ typedef enum {
     _PyConfig_INIT_ISOLATED = 3
 } _PyConfigInitEnum;
 
-PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config);
+PyAPI_FUNC(PyStatus) _PyConfig_InitCompatConfig(PyConfig *config);
 extern PyStatus _PyConfig_Copy(
     PyConfig *config,
     const PyConfig *config2);
diff --git a/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst
new file mode 100644
index 000000000000..b797933d05d6
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst	
@@ -0,0 +1,3 @@
+Add a new ``struct_size`` field to :c:type:`PyPreConfig` and :c:type:`PyConfig`
+structures to allow to modify these structures in the future without breaking
+the backward compatibility.
diff --git a/Modules/main.c b/Modules/main.c
index fdc7e0dabf17..ef6c66a8dbe5 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -53,13 +53,20 @@ pymain_init(const _PyArgv *args)
 #endif
 
     PyPreConfig preconfig;
-    PyPreConfig_InitPythonConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = PyPreConfig_InitPythonConfig(&preconfig);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
     status = _Py_PreInitializeFromPyArgv(&preconfig, args);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
     status = PyConfig_InitPythonConfig(&config);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c
index 74735f279c58..7c494c2786c0 100644
--- a/Programs/_freeze_importlib.c
+++ b/Programs/_freeze_importlib.c
@@ -78,6 +78,7 @@ main(int argc, char *argv[])
 
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitIsolatedConfig(&config);
     if (PyStatus_Exception(status)) {
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index 83c266b885af..a37594524f3d 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -385,7 +385,12 @@ static int check_init_compat_config(int preinit)
 
     if (preinit) {
         PyPreConfig preconfig;
-        _PyPreConfig_InitCompatConfig(&preconfig);
+        preconfig.struct_size = sizeof(PyPreConfig);
+
+        status = _PyPreConfig_InitCompatConfig(&preconfig);
+        if (PyStatus_Exception(status)) {
+            Py_ExitStatusException(status);
+        }
 
         status = Py_PreInitialize(&preconfig);
         if (PyStatus_Exception(status)) {
@@ -394,7 +399,13 @@ static int check_init_compat_config(int preinit)
     }
 
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+
+    status = _PyConfig_InitCompatConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
     config_set_program_name(&config);
     init_from_config_clear(&config);
 
@@ -470,7 +481,12 @@ static int test_init_from_config(void)
     PyStatus status;
 
     PyPreConfig preconfig;
-    _PyPreConfig_InitCompatConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitCompatConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
 
     putenv("PYTHONMALLOC=malloc_debug");
     preconfig.allocator = PYMEM_ALLOCATOR_MALLOC;
@@ -485,7 +501,12 @@ static int test_init_from_config(void)
     }
 
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+
+    status = _PyConfig_InitCompatConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
     config.install_signal_handlers = 0;
 
     /* FIXME: test use_environment */
@@ -617,6 +638,8 @@ static int check_init_parse_argv(int parse_argv)
     PyStatus status;
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -702,6 +725,8 @@ static int test_init_python_env(void)
     set_all_env_vars();
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -755,6 +780,8 @@ static int test_init_isolated_flag(void)
 
     /* Test PyConfig.isolated=1 */
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -779,7 +806,13 @@ static int test_preinit_isolated1(void)
     PyStatus status;
 
     PyPreConfig preconfig;
-    _PyPreConfig_InitCompatConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitCompatConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
     preconfig.isolated = 1;
 
     status = Py_PreInitialize(&preconfig);
@@ -788,7 +821,12 @@ static int test_preinit_isolated1(void)
     }
 
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+
+    status = _PyConfig_InitCompatConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
     config_set_program_name(&config);
     set_all_env_vars();
     init_from_config_clear(&config);
@@ -805,7 +843,13 @@ static int test_preinit_isolated2(void)
     PyStatus status;
 
     PyPreConfig preconfig;
-    _PyPreConfig_InitCompatConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitCompatConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
     preconfig.isolated = 0;
 
     status = Py_PreInitialize(&preconfig);
@@ -815,7 +859,11 @@ static int test_preinit_isolated2(void)
 
     /* Test PyConfig.isolated=1 */
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+    status = _PyConfig_InitCompatConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
 
     Py_IsolatedFlag = 0;
     config.isolated = 1;
@@ -835,7 +883,12 @@ static int test_preinit_dont_parse_argv(void)
     PyStatus status;
 
     PyPreConfig preconfig;
-    PyPreConfig_InitIsolatedConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = PyPreConfig_InitIsolatedConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
 
     preconfig.isolated = 0;
 
@@ -852,6 +905,7 @@ static int test_preinit_dont_parse_argv(void)
     }
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitIsolatedConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -877,6 +931,7 @@ static int test_preinit_parse_argv(void)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -934,7 +989,12 @@ static int check_preinit_isolated_config(int preinit)
 
     if (preinit) {
         PyPreConfig preconfig;
-        PyPreConfig_InitIsolatedConfig(&preconfig);
+        preconfig.struct_size = sizeof(PyPreConfig);
+
+        status = PyPreConfig_InitIsolatedConfig(&preconfig);
+        if (PyStatus_Exception(status)) {
+            Py_ExitStatusException(status);
+        }
 
         status = Py_PreInitialize(&preconfig);
         if (PyStatus_Exception(status)) {
@@ -947,6 +1007,8 @@ static int check_preinit_isolated_config(int preinit)
     }
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitIsolatedConfig(&config);
     if (PyStatus_Exception(status)) {
         PyConfig_Clear(&config);
@@ -996,7 +1058,12 @@ static int check_init_python_config(int preinit)
 
     if (preinit) {
         PyPreConfig preconfig;
-        PyPreConfig_InitPythonConfig(&preconfig);
+        preconfig.struct_size = sizeof(PyPreConfig);
+
+        status = PyPreConfig_InitPythonConfig(&preconfig);
+        if (PyStatus_Exception(status)) {
+            Py_ExitStatusException(status);
+        }
 
         status = Py_PreInitialize(&preconfig);
         if (PyStatus_Exception(status)) {
@@ -1005,6 +1072,8 @@ static int check_init_python_config(int preinit)
     }
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -1035,7 +1104,13 @@ static int test_init_dont_configure_locale(void)
     PyStatus status;
 
     PyPreConfig preconfig;
-    PyPreConfig_InitPythonConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = PyPreConfig_InitPythonConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
     preconfig.configure_locale = 0;
     preconfig.coerce_c_locale = 1;
     preconfig.coerce_c_locale_warn = 1;
@@ -1046,6 +1121,8 @@ static int test_init_dont_configure_locale(void)
     }
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -1063,6 +1140,8 @@ static int test_init_dev_mode(void)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -1286,6 +1365,8 @@ static int run_audit_run_test(int argc, wchar_t **argv, void *test)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -1334,6 +1415,8 @@ static int test_init_read_set(void)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -1382,6 +1465,8 @@ static int test_init_sys_add(void)
     PySys_AddWarnOption(L"ignore:::sysadd_warnoption");
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     PyStatus status;
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1450,7 +1535,12 @@ static int test_init_setpath_config(void)
 {
     PyStatus status;
     PyPreConfig preconfig;
-    PyPreConfig_InitPythonConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = PyPreConfig_InitPythonConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
 
     /* Explicitly preinitializes with Python preconfiguration to avoid
       Py_SetPath() implicit preinitialization with compat preconfiguration. */
@@ -1474,6 +1564,7 @@ static int test_init_setpath_config(void)
     putenv("TESTPATH=");
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1531,6 +1622,8 @@ static int test_init_run_main(void)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -1546,6 +1639,7 @@ static int test_init_main(void)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1577,6 +1671,7 @@ static int test_run_main(void)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
diff --git a/Python/frozenmain.c b/Python/frozenmain.c
index c56938ab4899..76309e9e5da2 100644
--- a/Python/frozenmain.c
+++ b/Python/frozenmain.c
@@ -40,6 +40,7 @@ Py_FrozenMain(int argc, char **argv)
     }
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         PyConfig_Clear(&config);
diff --git a/Python/initconfig.c b/Python/initconfig.c
index 9f04e3d183ab..62c868d5cbc9 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -528,6 +528,18 @@ Py_GetArgcArgv(int *argc, wchar_t ***argv)
      ? _PyStatus_ERR("cannot decode " NAME) \
      : _PyStatus_NO_MEMORY())
 
+
+static PyStatus
+config_check_struct_size(const PyConfig *config)
+{
+    if (config->struct_size != sizeof(PyConfig)) {
+        return _PyStatus_ERR("unsupported PyConfig structure size "
+                             "(Python version mismatch?)");
+    }
+    return _PyStatus_OK();
+}
+
+
 /* Free memory allocated in config, but don't clear all attributes */
 void
 PyConfig_Clear(PyConfig *config)
@@ -568,12 +580,19 @@ PyConfig_Clear(PyConfig *config)
 }
 
 
-void
+PyStatus
 _PyConfig_InitCompatConfig(PyConfig *config)
 {
+    size_t struct_size = config->struct_size;
     memset(config, 0, sizeof(*config));
+    config->struct_size = struct_size;
+
+    PyStatus status = config_check_struct_size(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
 
-    config->_config_version = _Py_CONFIG_VERSION;
     config->_config_init = (int)_PyConfig_INIT_COMPAT;
     config->isolated = -1;
     config->use_environment = -1;
@@ -603,13 +622,17 @@ _PyConfig_InitCompatConfig(PyConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_stdio = -1;
 #endif
+    return _PyStatus_OK();
 }
 
 
-static void
+static PyStatus
 config_init_defaults(PyConfig *config)
 {
-    _PyConfig_InitCompatConfig(config);
+    PyStatus status = _PyConfig_InitCompatConfig(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     config->isolated = 0;
     config->use_environment = 1;
@@ -628,13 +651,18 @@ config_init_defaults(PyConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_stdio = 0;
 #endif
+    return _PyStatus_OK();
 }
 
 
 PyStatus
 PyConfig_InitPythonConfig(PyConfig *config)
 {
-    config_init_defaults(config);
+    PyStatus status = config_init_defaults(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
 
     config->_config_init = (int)_PyConfig_INIT_PYTHON;
     config->configure_c_stdio = 1;
@@ -647,7 +675,11 @@ PyConfig_InitPythonConfig(PyConfig *config)
 PyStatus
 PyConfig_InitIsolatedConfig(PyConfig *config)
 {
-    config_init_defaults(config);
+    PyStatus status = config_init_defaults(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
 
     config->_config_init = (int)_PyConfig_INIT_ISOLATED;
     config->isolated = 1;
@@ -742,6 +774,19 @@ PyStatus
 _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
 {
     PyStatus status;
+
+    status = config_check_struct_size(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
+
+    status = config_check_struct_size(config2);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
+
     PyConfig_Clear(config);
 
 #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
@@ -2204,7 +2249,12 @@ core_read_precmdline(PyConfig *config, _PyPreCmdline *precmdline)
     }
 
     PyPreConfig preconfig;
-    _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     _PyPreConfig_GetConfig(&preconfig, config);
 
@@ -2385,6 +2435,12 @@ PyConfig_Read(PyConfig *config)
     PyStatus status;
     PyWideStringList orig_argv = PyWideStringList_INIT;
 
+    status = config_check_struct_size(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
+
     status = _Py_PreInitializeFromConfig(config, NULL);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index 812614532800..6886ab7c42d6 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -434,7 +434,12 @@ pathconfig_global_read(_PyPathConfig *pathconfig)
 {
     PyStatus status;
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+
+    status = _PyConfig_InitCompatConfig(&config);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto done;
+    }
 
     /* Call _PyConfig_InitPathConfig() */
     status = PyConfig_Read(&config);
diff --git a/Python/preconfig.c b/Python/preconfig.c
index 8be6533eace0..d18b01dc4586 100644
--- a/Python/preconfig.c
+++ b/Python/preconfig.c
@@ -269,12 +269,30 @@ _PyPreCmdline_Read(_PyPreCmdline *cmdline, const PyPreConfig *preconfig)
 /* --- PyPreConfig ----------------------------------------------- */
 
 
-void
+static PyStatus
+preconfig_check_struct_size(PyPreConfig *config)
+{
+    if (config->struct_size != sizeof(PyPreConfig)) {
+        return _PyStatus_ERR("unsupported PyPreConfig structure size "
+                             "(Python version mismatch?)");
+    }
+    return _PyStatus_OK();
+}
+
+
+PyStatus
 _PyPreConfig_InitCompatConfig(PyPreConfig *config)
 {
+    size_t struct_size = config->struct_size;
     memset(config, 0, sizeof(*config));
+    config->struct_size = struct_size;
+
+    PyStatus status = preconfig_check_struct_size(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
 
-    config->_config_version = _Py_CONFIG_VERSION;
     config->_config_init = (int)_PyConfig_INIT_COMPAT;
     config->parse_argv = 0;
     config->isolated = -1;
@@ -295,13 +313,18 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_fs_encoding = -1;
 #endif
+    return _PyStatus_OK();
 }
 
 
-void
+PyStatus
 PyPreConfig_InitPythonConfig(PyPreConfig *config)
 {
-    _PyPreConfig_InitCompatConfig(config);
+    PyStatus status = _PyPreConfig_InitCompatConfig(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
 
     config->_config_init = (int)_PyConfig_INIT_PYTHON;
     config->isolated = 0;
@@ -316,13 +339,18 @@ PyPreConfig_InitPythonConfig(PyPreConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_fs_encoding = 0;
 #endif
+    return _PyStatus_OK();
 }
 
 
-void
+PyStatus
 PyPreConfig_InitIsolatedConfig(PyPreConfig *config)
 {
-    _PyPreConfig_InitCompatConfig(config);
+    PyStatus status = _PyPreConfig_InitCompatConfig(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
 
     config->_config_init = (int)_PyConfig_INIT_ISOLATED;
     config->configure_locale = 0;
@@ -333,41 +361,55 @@ PyPreConfig_InitIsolatedConfig(PyPreConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_fs_encoding = 0;
 #endif
+    return _PyStatus_OK();
 }
 
 
-void
+PyStatus
 _PyPreConfig_InitFromPreConfig(PyPreConfig *config,
                                const PyPreConfig *config2)
 {
-    PyPreConfig_InitPythonConfig(config);
+    PyStatus status = PyPreConfig_InitPythonConfig(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
     preconfig_copy(config, config2);
+    return _PyStatus_OK();
 }
 
 
-void
+PyStatus
 _PyPreConfig_InitFromConfig(PyPreConfig *preconfig, const PyConfig *config)
 {
+    PyStatus status;
     _PyConfigInitEnum config_init = (_PyConfigInitEnum)config->_config_init;
     switch (config_init) {
     case _PyConfig_INIT_PYTHON:
-        PyPreConfig_InitPythonConfig(preconfig);
+        status = PyPreConfig_InitPythonConfig(preconfig);
         break;
     case _PyConfig_INIT_ISOLATED:
-        PyPreConfig_InitIsolatedConfig(preconfig);
+        status = PyPreConfig_InitIsolatedConfig(preconfig);
         break;
     case _PyConfig_INIT_COMPAT:
     default:
-        _PyPreConfig_InitCompatConfig(preconfig);
+        status = _PyPreConfig_InitCompatConfig(preconfig);
     }
+
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
     _PyPreConfig_GetConfig(preconfig, config);
+    return _PyStatus_OK();
 }
 
 
 static void
 preconfig_copy(PyPreConfig *config, const PyPreConfig *config2)
 {
-    assert(config2->_config_version == _Py_CONFIG_VERSION);
+    assert(config->struct_size == sizeof(PyPreConfig));
+
 #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
 
     COPY_ATTR(_config_init);
@@ -787,6 +829,12 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args)
         return status;
     }
 
+    status = preconfig_check_struct_size(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
+
     preconfig_get_global_vars(config);
 
     /* Copy LC_CTYPE locale, since it's modified later */
@@ -801,7 +849,12 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args)
 
     /* Save the config to be able to restore it if encodings change */
     PyPreConfig save_config;
-    _PyPreConfig_InitFromPreConfig(&save_config, config);
+    save_config.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitFromPreConfig(&save_config, config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     /* Set LC_CTYPE to the user preferred locale */
     if (config->configure_locale) {
@@ -923,7 +976,12 @@ PyStatus
 _PyPreConfig_Write(const PyPreConfig *src_config)
 {
     PyPreConfig config;
-    _PyPreConfig_InitFromPreConfig(&config, src_config);
+    config.struct_size = sizeof(PyPreConfig);
+
+    PyStatus status = _PyPreConfig_InitFromPreConfig(&config, src_config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     if (_PyRuntime.core_initialized) {
         /* bpo-34008: Calling this functions after Py_Initialize() ignores
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index eed583a4fdbc..ea0d7a5ee2b9 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -735,7 +735,12 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args)
     runtime->preinitializing = 1;
 
     PyPreConfig config;
-    _PyPreConfig_InitFromPreConfig(&config, src_config);
+    config.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitFromPreConfig(&config, src_config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     status = _PyPreConfig_Read(&config, args);
     if (_PyStatus_EXCEPTION(status)) {
@@ -794,7 +799,12 @@ _Py_PreInitializeFromConfig(const PyConfig *config,
     }
 
     PyPreConfig preconfig;
-    _PyPreConfig_InitFromConfig(&preconfig, config);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitFromConfig(&preconfig, config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     if (!config->parse_argv) {
         return Py_PreInitialize(&preconfig);
@@ -842,7 +852,12 @@ pyinit_core(_PyRuntimeState *runtime,
     }
 
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+
+    status = _PyConfig_InitCompatConfig(&config);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto done;
+    }
 
     status = _PyConfig_Copy(&config, src_config);
     if (_PyStatus_EXCEPTION(status)) {
@@ -1064,7 +1079,13 @@ Py_InitializeEx(int install_sigs)
     }
 
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+
+    status = _PyConfig_InitCompatConfig(&config);
+    if (_PyStatus_EXCEPTION(status)) {
+        Py_ExitStatusException(status);
+    }
+
     config.install_signal_handlers = install_sigs;
 
     status = Py_InitializeFromConfig(&config);
diff --git a/Python/pystate.c b/Python/pystate.c
index 7dd8b7f866b5..0f0cb2299557 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -60,7 +60,12 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime)
 
     _PyGC_Initialize(&runtime->gc);
     _PyEval_Initialize(&runtime->ceval);
-    PyPreConfig_InitPythonConfig(&runtime->preconfig);
+
+    runtime->preconfig.struct_size = sizeof(PyPreConfig);
+    PyStatus status = PyPreConfig_InitPythonConfig(&runtime->preconfig);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     runtime->gilstate.check_enabled = 1;
 
@@ -204,6 +209,7 @@ PyInterpreterState_New(void)
     memset(interp, 0, sizeof(*interp));
     interp->id_refcount = -1;
 
+    interp->config.struct_size = sizeof(PyConfig);
     PyStatus status = PyConfig_InitPythonConfig(&interp->config);
     if (_PyStatus_EXCEPTION(status)) {
         /* Don't report status to caller: PyConfig_InitPythonConfig()


From webhook-mailer at python.org  Fri Sep 27 22:50:47 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Sat, 28 Sep 2019 02:50:47 -0000
Subject: [Python-checkins] bpo-38304: Add PyConfig.struct_size (GH-16451)
 (GH-16453)
Message-ID: <mailman.129.1569639048.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/6e128382b3e7f83d91d5e73d83a051ff5fb6469c
commit: 6e128382b3e7f83d91d5e73d83a051ff5fb6469c
branch: 3.8
author: Victor Stinner <vstinner at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-28T04:50:43+02:00
summary:

bpo-38304: Add PyConfig.struct_size (GH-16451) (GH-16453)

Add a new struct_size field to PyPreConfig and PyConfig structures to
allow to modify these structures in the future without breaking the
backward compatibility.

* Replace private _config_version field with public struct_size field
  in PyPreConfig and PyConfig.
* Public PyPreConfig_InitIsolatedConfig() and
  PyPreConfig_InitPythonConfig()
  return type becomes PyStatus, instead of void.
* Internal _PyConfig_InitCompatConfig(),
  _PyPreConfig_InitCompatConfig(), _PyPreConfig_InitFromConfig(),
  _PyPreConfig_InitFromPreConfig() return type becomes PyStatus,
  instead of void.
* Remove _Py_CONFIG_VERSION
* Update the Initialization Configuration documentation.

(cherry picked from commit 441b10cf2855955c86565f8d59e72c2efc0f0a57)

files:
A Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst
M Doc/c-api/init_config.rst
M Include/cpython/initconfig.h
M Include/internal/pycore_initconfig.h
M Modules/main.c
M Programs/_freeze_importlib.c
M Programs/_testembed.c
M Python/frozenmain.c
M Python/initconfig.c
M Python/pathconfig.c
M Python/preconfig.c
M Python/pylifecycle.c
M Python/pystate.c

diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index 0c3c725c841a..58e417441d13 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -194,18 +194,25 @@ PyPreConfig
    * Configure the LC_CTYPE locale
    * Set the UTF-8 mode
 
+   The :c:member:`struct_size` field must be explicitly initialized to
+   ``sizeof(PyPreConfig)``.
+
    Function to initialize a preconfiguration:
 
-   .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig)
+   .. c:function:: PyStatus PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig)
 
       Initialize the preconfiguration with :ref:`Python Configuration
       <init-python-config>`.
 
-   .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig)
+   .. c:function:: PyStatus PyPreConfig_InitPythonConfig(PyPreConfig *preconfig)
 
       Initialize the preconfiguration with :ref:`Isolated Configuration
       <init-isolated-conf>`.
 
+   The caller of these functions is responsible to handle exceptions (error or
+   exit) using :c:func:`PyStatus_Exception` and
+   :c:func:`Py_ExitStatusException`.
+
    Structure fields:
 
    .. c:member:: int allocator
@@ -267,6 +274,13 @@ PyPreConfig
       same way the regular Python parses command line arguments: see
       :ref:`Command Line Arguments <using-on-cmdline>`.
 
+   .. c:member:: size_t struct_size
+
+      Size of the structure in bytes: must be initialized to
+      ``sizeof(PyPreConfig)``.
+
+      Field used for API and ABI compatibility.
+
    .. c:member:: int use_environment
 
       See :c:member:`PyConfig.use_environment`.
@@ -316,12 +330,18 @@ the preinitialization.
 
 Example using the preinitialization to enable the UTF-8 Mode::
 
+    PyStatus status;
     PyPreConfig preconfig;
-    PyPreConfig_InitPythonConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = PyPreConfig_InitPythonConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
 
     preconfig.utf8_mode = 1;
 
-    PyStatus status = Py_PreInitialize(&preconfig);
+    status = Py_PreInitialize(&preconfig);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
     }
@@ -340,6 +360,9 @@ PyConfig
 
    Structure containing most parameters to configure Python.
 
+   The :c:member:`struct_size` field must be explicitly initialized to
+   ``sizeof(PyConfig)``.
+
    Structure methods:
 
    .. c:function:: PyStatus PyConfig_InitPythonConfig(PyConfig *config)
@@ -656,6 +679,13 @@ PyConfig
       Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and
       :data:`sys.stderr`.
 
+   .. c:member:: size_t struct_size
+
+      Size of the structure in bytes: must be initialized to
+      ``sizeof(PyConfig)``.
+
+      Field used for API and ABI compatibility.
+
    .. c:member:: int tracemalloc
 
       If non-zero, call :func:`tracemalloc.start` at startup.
@@ -718,6 +748,7 @@ Example setting the program name::
     {
         PyStatus status;
         PyConfig config;
+        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
@@ -750,6 +781,7 @@ configuration, and then override some parameters::
     {
         PyStatus status;
         PyConfig config;
+        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
@@ -835,8 +867,9 @@ Example of customized Python always running in isolated mode::
 
     int main(int argc, char **argv)
     {
-        PyConfig config;
         PyStatus status;
+        PyConfig config;
+        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
@@ -1028,6 +1061,7 @@ phases::
     {
         PyStatus status;
         PyConfig config;
+        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
index 047f511f3104..d5effb8397fc 100644
--- a/Include/cpython/initconfig.h
+++ b/Include/cpython/initconfig.h
@@ -45,8 +45,10 @@ PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list,
 /* --- PyPreConfig ----------------------------------------------- */
 
 typedef struct {
-    int _config_version;  /* Internal configuration version,
-                             used for ABI compatibility */
+    /* Size of the structure in bytes: must be initialized to
+       sizeof(PyPreConfig). Field used for API and ABI compatibility. */
+    size_t struct_size;
+
     int _config_init;     /* _PyConfigInitEnum value */
 
     /* Parse Py_PreInitializeFromBytesArgs() arguments?
@@ -122,15 +124,17 @@ typedef struct {
     int allocator;
 } PyPreConfig;
 
-PyAPI_FUNC(void) PyPreConfig_InitPythonConfig(PyPreConfig *config);
-PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config);
+PyAPI_FUNC(PyStatus) PyPreConfig_InitPythonConfig(PyPreConfig *config);
+PyAPI_FUNC(PyStatus) PyPreConfig_InitIsolatedConfig(PyPreConfig *config);
 
 
 /* --- PyConfig ---------------------------------------------- */
 
 typedef struct {
-    int _config_version;  /* Internal configuration version,
-                             used for ABI compatibility */
+    /* Size of the structure in bytes: must be initialized to
+       sizeof(PyConfig). Field used for API and ABI compatibility. */
+    size_t struct_size;
+
     int _config_init;     /* _PyConfigInitEnum value */
 
     int isolated;         /* Isolated mode? see PyPreConfig.isolated */
@@ -403,7 +407,6 @@ typedef struct {
 
     /* If equal to 0, stop Python initialization before the "main" phase */
     int _init_main;
-
 } PyConfig;
 
 PyAPI_FUNC(PyStatus) PyConfig_InitPythonConfig(PyConfig *config);
diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h
index a7c1994eca16..dcdb61695cab 100644
--- a/Include/internal/pycore_initconfig.h
+++ b/Include/internal/pycore_initconfig.h
@@ -40,6 +40,8 @@ extern "C" {
     (err._type == _PyStatus_TYPE_EXIT)
 #define _PyStatus_EXCEPTION(err) \
     (err._type != _PyStatus_TYPE_OK)
+#define _PyStatus_UPDATE_FUNC(err) \
+    do { err.func = _PyStatus_GET_FUNC(); } while (0)
 
 /* --- PyWideStringList ------------------------------------------------ */
 
@@ -118,11 +120,11 @@ extern PyStatus _PyPreCmdline_Read(_PyPreCmdline *cmdline,
 
 /* --- PyPreConfig ----------------------------------------------- */
 
-PyAPI_FUNC(void) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig);
-extern void _PyPreConfig_InitFromConfig(
+PyAPI_FUNC(PyStatus) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig);
+extern PyStatus _PyPreConfig_InitFromConfig(
     PyPreConfig *preconfig,
     const PyConfig *config);
-extern void _PyPreConfig_InitFromPreConfig(
+extern PyStatus _PyPreConfig_InitFromPreConfig(
     PyPreConfig *preconfig,
     const PyPreConfig *config2);
 extern PyObject* _PyPreConfig_AsDict(const PyPreConfig *preconfig);
@@ -135,8 +137,6 @@ extern PyStatus _PyPreConfig_Write(const PyPreConfig *preconfig);
 
 /* --- PyConfig ---------------------------------------------- */
 
-#define _Py_CONFIG_VERSION 1
-
 typedef enum {
     /* Py_Initialize() API: backward compatibility with Python 3.6 and 3.7 */
     _PyConfig_INIT_COMPAT = 1,
@@ -144,7 +144,7 @@ typedef enum {
     _PyConfig_INIT_ISOLATED = 3
 } _PyConfigInitEnum;
 
-PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config);
+PyAPI_FUNC(PyStatus) _PyConfig_InitCompatConfig(PyConfig *config);
 extern PyStatus _PyConfig_Copy(
     PyConfig *config,
     const PyConfig *config2);
diff --git a/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst
new file mode 100644
index 000000000000..b797933d05d6
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst	
@@ -0,0 +1,3 @@
+Add a new ``struct_size`` field to :c:type:`PyPreConfig` and :c:type:`PyConfig`
+structures to allow to modify these structures in the future without breaking
+the backward compatibility.
diff --git a/Modules/main.c b/Modules/main.c
index b8a1c9b79ce3..13936add4f3e 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -53,13 +53,20 @@ pymain_init(const _PyArgv *args)
 #endif
 
     PyPreConfig preconfig;
-    PyPreConfig_InitPythonConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = PyPreConfig_InitPythonConfig(&preconfig);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
     status = _Py_PreInitializeFromPyArgv(&preconfig, args);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
     status = PyConfig_InitPythonConfig(&config);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c
index 74735f279c58..7c494c2786c0 100644
--- a/Programs/_freeze_importlib.c
+++ b/Programs/_freeze_importlib.c
@@ -78,6 +78,7 @@ main(int argc, char *argv[])
 
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitIsolatedConfig(&config);
     if (PyStatus_Exception(status)) {
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index 83c266b885af..a37594524f3d 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -385,7 +385,12 @@ static int check_init_compat_config(int preinit)
 
     if (preinit) {
         PyPreConfig preconfig;
-        _PyPreConfig_InitCompatConfig(&preconfig);
+        preconfig.struct_size = sizeof(PyPreConfig);
+
+        status = _PyPreConfig_InitCompatConfig(&preconfig);
+        if (PyStatus_Exception(status)) {
+            Py_ExitStatusException(status);
+        }
 
         status = Py_PreInitialize(&preconfig);
         if (PyStatus_Exception(status)) {
@@ -394,7 +399,13 @@ static int check_init_compat_config(int preinit)
     }
 
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+
+    status = _PyConfig_InitCompatConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
     config_set_program_name(&config);
     init_from_config_clear(&config);
 
@@ -470,7 +481,12 @@ static int test_init_from_config(void)
     PyStatus status;
 
     PyPreConfig preconfig;
-    _PyPreConfig_InitCompatConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitCompatConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
 
     putenv("PYTHONMALLOC=malloc_debug");
     preconfig.allocator = PYMEM_ALLOCATOR_MALLOC;
@@ -485,7 +501,12 @@ static int test_init_from_config(void)
     }
 
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+
+    status = _PyConfig_InitCompatConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
     config.install_signal_handlers = 0;
 
     /* FIXME: test use_environment */
@@ -617,6 +638,8 @@ static int check_init_parse_argv(int parse_argv)
     PyStatus status;
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -702,6 +725,8 @@ static int test_init_python_env(void)
     set_all_env_vars();
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -755,6 +780,8 @@ static int test_init_isolated_flag(void)
 
     /* Test PyConfig.isolated=1 */
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -779,7 +806,13 @@ static int test_preinit_isolated1(void)
     PyStatus status;
 
     PyPreConfig preconfig;
-    _PyPreConfig_InitCompatConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitCompatConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
     preconfig.isolated = 1;
 
     status = Py_PreInitialize(&preconfig);
@@ -788,7 +821,12 @@ static int test_preinit_isolated1(void)
     }
 
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+
+    status = _PyConfig_InitCompatConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
     config_set_program_name(&config);
     set_all_env_vars();
     init_from_config_clear(&config);
@@ -805,7 +843,13 @@ static int test_preinit_isolated2(void)
     PyStatus status;
 
     PyPreConfig preconfig;
-    _PyPreConfig_InitCompatConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitCompatConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
     preconfig.isolated = 0;
 
     status = Py_PreInitialize(&preconfig);
@@ -815,7 +859,11 @@ static int test_preinit_isolated2(void)
 
     /* Test PyConfig.isolated=1 */
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+    status = _PyConfig_InitCompatConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
 
     Py_IsolatedFlag = 0;
     config.isolated = 1;
@@ -835,7 +883,12 @@ static int test_preinit_dont_parse_argv(void)
     PyStatus status;
 
     PyPreConfig preconfig;
-    PyPreConfig_InitIsolatedConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = PyPreConfig_InitIsolatedConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
 
     preconfig.isolated = 0;
 
@@ -852,6 +905,7 @@ static int test_preinit_dont_parse_argv(void)
     }
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitIsolatedConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -877,6 +931,7 @@ static int test_preinit_parse_argv(void)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -934,7 +989,12 @@ static int check_preinit_isolated_config(int preinit)
 
     if (preinit) {
         PyPreConfig preconfig;
-        PyPreConfig_InitIsolatedConfig(&preconfig);
+        preconfig.struct_size = sizeof(PyPreConfig);
+
+        status = PyPreConfig_InitIsolatedConfig(&preconfig);
+        if (PyStatus_Exception(status)) {
+            Py_ExitStatusException(status);
+        }
 
         status = Py_PreInitialize(&preconfig);
         if (PyStatus_Exception(status)) {
@@ -947,6 +1007,8 @@ static int check_preinit_isolated_config(int preinit)
     }
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitIsolatedConfig(&config);
     if (PyStatus_Exception(status)) {
         PyConfig_Clear(&config);
@@ -996,7 +1058,12 @@ static int check_init_python_config(int preinit)
 
     if (preinit) {
         PyPreConfig preconfig;
-        PyPreConfig_InitPythonConfig(&preconfig);
+        preconfig.struct_size = sizeof(PyPreConfig);
+
+        status = PyPreConfig_InitPythonConfig(&preconfig);
+        if (PyStatus_Exception(status)) {
+            Py_ExitStatusException(status);
+        }
 
         status = Py_PreInitialize(&preconfig);
         if (PyStatus_Exception(status)) {
@@ -1005,6 +1072,8 @@ static int check_init_python_config(int preinit)
     }
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -1035,7 +1104,13 @@ static int test_init_dont_configure_locale(void)
     PyStatus status;
 
     PyPreConfig preconfig;
-    PyPreConfig_InitPythonConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = PyPreConfig_InitPythonConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
     preconfig.configure_locale = 0;
     preconfig.coerce_c_locale = 1;
     preconfig.coerce_c_locale_warn = 1;
@@ -1046,6 +1121,8 @@ static int test_init_dont_configure_locale(void)
     }
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -1063,6 +1140,8 @@ static int test_init_dev_mode(void)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -1286,6 +1365,8 @@ static int run_audit_run_test(int argc, wchar_t **argv, void *test)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -1334,6 +1415,8 @@ static int test_init_read_set(void)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -1382,6 +1465,8 @@ static int test_init_sys_add(void)
     PySys_AddWarnOption(L"ignore:::sysadd_warnoption");
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     PyStatus status;
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1450,7 +1535,12 @@ static int test_init_setpath_config(void)
 {
     PyStatus status;
     PyPreConfig preconfig;
-    PyPreConfig_InitPythonConfig(&preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = PyPreConfig_InitPythonConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
 
     /* Explicitly preinitializes with Python preconfiguration to avoid
       Py_SetPath() implicit preinitialization with compat preconfiguration. */
@@ -1474,6 +1564,7 @@ static int test_init_setpath_config(void)
     putenv("TESTPATH=");
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1531,6 +1622,8 @@ static int test_init_run_main(void)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -1546,6 +1639,7 @@ static int test_init_main(void)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1577,6 +1671,7 @@ static int test_run_main(void)
 {
     PyStatus status;
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
diff --git a/Python/frozenmain.c b/Python/frozenmain.c
index c56938ab4899..76309e9e5da2 100644
--- a/Python/frozenmain.c
+++ b/Python/frozenmain.c
@@ -40,6 +40,7 @@ Py_FrozenMain(int argc, char **argv)
     }
 
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         PyConfig_Clear(&config);
diff --git a/Python/initconfig.c b/Python/initconfig.c
index a527271452ea..dbc9de55a582 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -528,6 +528,18 @@ Py_GetArgcArgv(int *argc, wchar_t ***argv)
      ? _PyStatus_ERR("cannot decode " NAME) \
      : _PyStatus_NO_MEMORY())
 
+
+static PyStatus
+config_check_struct_size(const PyConfig *config)
+{
+    if (config->struct_size != sizeof(PyConfig)) {
+        return _PyStatus_ERR("unsupported PyConfig structure size "
+                             "(Python version mismatch?)");
+    }
+    return _PyStatus_OK();
+}
+
+
 /* Free memory allocated in config, but don't clear all attributes */
 void
 PyConfig_Clear(PyConfig *config)
@@ -568,12 +580,19 @@ PyConfig_Clear(PyConfig *config)
 }
 
 
-void
+PyStatus
 _PyConfig_InitCompatConfig(PyConfig *config)
 {
+    size_t struct_size = config->struct_size;
     memset(config, 0, sizeof(*config));
+    config->struct_size = struct_size;
+
+    PyStatus status = config_check_struct_size(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
 
-    config->_config_version = _Py_CONFIG_VERSION;
     config->_config_init = (int)_PyConfig_INIT_COMPAT;
     config->isolated = -1;
     config->use_environment = -1;
@@ -603,13 +622,17 @@ _PyConfig_InitCompatConfig(PyConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_stdio = -1;
 #endif
+    return _PyStatus_OK();
 }
 
 
-static void
+static PyStatus
 config_init_defaults(PyConfig *config)
 {
-    _PyConfig_InitCompatConfig(config);
+    PyStatus status = _PyConfig_InitCompatConfig(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     config->isolated = 0;
     config->use_environment = 1;
@@ -628,13 +651,18 @@ config_init_defaults(PyConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_stdio = 0;
 #endif
+    return _PyStatus_OK();
 }
 
 
 PyStatus
 PyConfig_InitPythonConfig(PyConfig *config)
 {
-    config_init_defaults(config);
+    PyStatus status = config_init_defaults(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
 
     config->_config_init = (int)_PyConfig_INIT_PYTHON;
     config->configure_c_stdio = 1;
@@ -647,7 +675,11 @@ PyConfig_InitPythonConfig(PyConfig *config)
 PyStatus
 PyConfig_InitIsolatedConfig(PyConfig *config)
 {
-    config_init_defaults(config);
+    PyStatus status = config_init_defaults(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
 
     config->_config_init = (int)_PyConfig_INIT_ISOLATED;
     config->isolated = 1;
@@ -742,6 +774,19 @@ PyStatus
 _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
 {
     PyStatus status;
+
+    status = config_check_struct_size(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
+
+    status = config_check_struct_size(config2);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
+
     PyConfig_Clear(config);
 
 #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
@@ -2199,7 +2244,12 @@ core_read_precmdline(PyConfig *config, _PyPreCmdline *precmdline)
     }
 
     PyPreConfig preconfig;
-    _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     _PyPreConfig_GetConfig(&preconfig, config);
 
@@ -2338,6 +2388,12 @@ PyConfig_Read(PyConfig *config)
     PyStatus status;
     PyWideStringList orig_argv = PyWideStringList_INIT;
 
+    status = config_check_struct_size(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
+
     status = _Py_PreInitializeFromConfig(config, NULL);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index d5b8b1acff8a..41ef7d28d066 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -434,7 +434,12 @@ pathconfig_global_read(_PyPathConfig *pathconfig)
 {
     PyStatus status;
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+
+    status = _PyConfig_InitCompatConfig(&config);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto done;
+    }
 
     /* Call _PyConfig_InitPathConfig() */
     status = PyConfig_Read(&config);
diff --git a/Python/preconfig.c b/Python/preconfig.c
index 8be6533eace0..d18b01dc4586 100644
--- a/Python/preconfig.c
+++ b/Python/preconfig.c
@@ -269,12 +269,30 @@ _PyPreCmdline_Read(_PyPreCmdline *cmdline, const PyPreConfig *preconfig)
 /* --- PyPreConfig ----------------------------------------------- */
 
 
-void
+static PyStatus
+preconfig_check_struct_size(PyPreConfig *config)
+{
+    if (config->struct_size != sizeof(PyPreConfig)) {
+        return _PyStatus_ERR("unsupported PyPreConfig structure size "
+                             "(Python version mismatch?)");
+    }
+    return _PyStatus_OK();
+}
+
+
+PyStatus
 _PyPreConfig_InitCompatConfig(PyPreConfig *config)
 {
+    size_t struct_size = config->struct_size;
     memset(config, 0, sizeof(*config));
+    config->struct_size = struct_size;
+
+    PyStatus status = preconfig_check_struct_size(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
 
-    config->_config_version = _Py_CONFIG_VERSION;
     config->_config_init = (int)_PyConfig_INIT_COMPAT;
     config->parse_argv = 0;
     config->isolated = -1;
@@ -295,13 +313,18 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_fs_encoding = -1;
 #endif
+    return _PyStatus_OK();
 }
 
 
-void
+PyStatus
 PyPreConfig_InitPythonConfig(PyPreConfig *config)
 {
-    _PyPreConfig_InitCompatConfig(config);
+    PyStatus status = _PyPreConfig_InitCompatConfig(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
 
     config->_config_init = (int)_PyConfig_INIT_PYTHON;
     config->isolated = 0;
@@ -316,13 +339,18 @@ PyPreConfig_InitPythonConfig(PyPreConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_fs_encoding = 0;
 #endif
+    return _PyStatus_OK();
 }
 
 
-void
+PyStatus
 PyPreConfig_InitIsolatedConfig(PyPreConfig *config)
 {
-    _PyPreConfig_InitCompatConfig(config);
+    PyStatus status = _PyPreConfig_InitCompatConfig(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
 
     config->_config_init = (int)_PyConfig_INIT_ISOLATED;
     config->configure_locale = 0;
@@ -333,41 +361,55 @@ PyPreConfig_InitIsolatedConfig(PyPreConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_fs_encoding = 0;
 #endif
+    return _PyStatus_OK();
 }
 
 
-void
+PyStatus
 _PyPreConfig_InitFromPreConfig(PyPreConfig *config,
                                const PyPreConfig *config2)
 {
-    PyPreConfig_InitPythonConfig(config);
+    PyStatus status = PyPreConfig_InitPythonConfig(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
     preconfig_copy(config, config2);
+    return _PyStatus_OK();
 }
 
 
-void
+PyStatus
 _PyPreConfig_InitFromConfig(PyPreConfig *preconfig, const PyConfig *config)
 {
+    PyStatus status;
     _PyConfigInitEnum config_init = (_PyConfigInitEnum)config->_config_init;
     switch (config_init) {
     case _PyConfig_INIT_PYTHON:
-        PyPreConfig_InitPythonConfig(preconfig);
+        status = PyPreConfig_InitPythonConfig(preconfig);
         break;
     case _PyConfig_INIT_ISOLATED:
-        PyPreConfig_InitIsolatedConfig(preconfig);
+        status = PyPreConfig_InitIsolatedConfig(preconfig);
         break;
     case _PyConfig_INIT_COMPAT:
     default:
-        _PyPreConfig_InitCompatConfig(preconfig);
+        status = _PyPreConfig_InitCompatConfig(preconfig);
     }
+
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
     _PyPreConfig_GetConfig(preconfig, config);
+    return _PyStatus_OK();
 }
 
 
 static void
 preconfig_copy(PyPreConfig *config, const PyPreConfig *config2)
 {
-    assert(config2->_config_version == _Py_CONFIG_VERSION);
+    assert(config->struct_size == sizeof(PyPreConfig));
+
 #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
 
     COPY_ATTR(_config_init);
@@ -787,6 +829,12 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args)
         return status;
     }
 
+    status = preconfig_check_struct_size(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyStatus_UPDATE_FUNC(status);
+        return status;
+    }
+
     preconfig_get_global_vars(config);
 
     /* Copy LC_CTYPE locale, since it's modified later */
@@ -801,7 +849,12 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args)
 
     /* Save the config to be able to restore it if encodings change */
     PyPreConfig save_config;
-    _PyPreConfig_InitFromPreConfig(&save_config, config);
+    save_config.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitFromPreConfig(&save_config, config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     /* Set LC_CTYPE to the user preferred locale */
     if (config->configure_locale) {
@@ -923,7 +976,12 @@ PyStatus
 _PyPreConfig_Write(const PyPreConfig *src_config)
 {
     PyPreConfig config;
-    _PyPreConfig_InitFromPreConfig(&config, src_config);
+    config.struct_size = sizeof(PyPreConfig);
+
+    PyStatus status = _PyPreConfig_InitFromPreConfig(&config, src_config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     if (_PyRuntime.core_initialized) {
         /* bpo-34008: Calling this functions after Py_Initialize() ignores
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 42a062e10865..c154d060ad8d 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -731,7 +731,12 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args)
     runtime->preinitializing = 1;
 
     PyPreConfig config;
-    _PyPreConfig_InitFromPreConfig(&config, src_config);
+    config.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitFromPreConfig(&config, src_config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     status = _PyPreConfig_Read(&config, args);
     if (_PyStatus_EXCEPTION(status)) {
@@ -790,7 +795,12 @@ _Py_PreInitializeFromConfig(const PyConfig *config,
     }
 
     PyPreConfig preconfig;
-    _PyPreConfig_InitFromConfig(&preconfig, config);
+    preconfig.struct_size = sizeof(PyPreConfig);
+
+    status = _PyPreConfig_InitFromConfig(&preconfig, config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     if (!config->parse_argv) {
         return Py_PreInitialize(&preconfig);
@@ -838,7 +848,12 @@ pyinit_core(_PyRuntimeState *runtime,
     }
 
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+
+    status = _PyConfig_InitCompatConfig(&config);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto done;
+    }
 
     status = _PyConfig_Copy(&config, src_config);
     if (_PyStatus_EXCEPTION(status)) {
@@ -1061,7 +1076,13 @@ Py_InitializeEx(int install_sigs)
     }
 
     PyConfig config;
-    _PyConfig_InitCompatConfig(&config);
+    config.struct_size = sizeof(PyConfig);
+
+    status = _PyConfig_InitCompatConfig(&config);
+    if (_PyStatus_EXCEPTION(status)) {
+        Py_ExitStatusException(status);
+    }
+
     config.install_signal_handlers = install_sigs;
 
     status = Py_InitializeFromConfig(&config);
diff --git a/Python/pystate.c b/Python/pystate.c
index f2924a8ff263..f0662fa70657 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -60,7 +60,12 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime)
 
     _PyGC_Initialize(&runtime->gc);
     _PyEval_Initialize(&runtime->ceval);
-    PyPreConfig_InitPythonConfig(&runtime->preconfig);
+
+    runtime->preconfig.struct_size = sizeof(PyPreConfig);
+    PyStatus status = PyPreConfig_InitPythonConfig(&runtime->preconfig);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     runtime->gilstate.check_enabled = 1;
 
@@ -205,6 +210,7 @@ PyInterpreterState_New(void)
     interp->id_refcount = -1;
     interp->check_interval = 100;
 
+    interp->config.struct_size = sizeof(PyConfig);
     PyStatus status = PyConfig_InitPythonConfig(&interp->config);
     if (_PyStatus_EXCEPTION(status)) {
         /* Don't report status to caller: PyConfig_InitPythonConfig()


From webhook-mailer at python.org  Sat Sep 28 03:33:06 2019
From: webhook-mailer at python.org (Ned Deily)
Date: Sat, 28 Sep 2019 07:33:06 -0000
Subject: [Python-checkins] bpo-38243,
 xmlrpc.server: Escape the server_title (GH-16373) (GH-16441)
Message-ID: <mailman.130.1569655988.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/1698cacfb924d1df452e78d11a4bf81ae7777389
commit: 1698cacfb924d1df452e78d11a4bf81ae7777389
branch: 3.6
author: Victor Stinner <vstinner at redhat.com>
committer: Ned Deily <nad at python.org>
date: 2019-09-28T03:33:00-04:00
summary:

bpo-38243, xmlrpc.server: Escape the server_title (GH-16373) (GH-16441)

Escape the server title of xmlrpc.server.DocXMLRPCServer
when rendering the document page as HTML.

(cherry picked from commit e8650a4f8c7fb76f570d4ca9c1fbe44e91c8dfaa)

files:
A Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst
M Lib/test/test_docxmlrpc.py
M Lib/xmlrpc/server.py

diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py
index 00903337c07c..d2adb21af0fb 100644
--- a/Lib/test/test_docxmlrpc.py
+++ b/Lib/test/test_docxmlrpc.py
@@ -1,5 +1,6 @@
 from xmlrpc.server import DocXMLRPCServer
 import http.client
+import re
 import sys
 from test import support
 threading = support.import_module('threading')
@@ -193,6 +194,21 @@ def test_annotations(self):
              b'method_annotation</strong></a>(x: bytes)</dt></dl>'),
             response.read())
 
+    def test_server_title_escape(self):
+        # bpo-38243: Ensure that the server title and documentation
+        # are escaped for HTML.
+        self.serv.set_server_title('test_title<script>')
+        self.serv.set_server_documentation('test_documentation<script>')
+        self.assertEqual('test_title<script>', self.serv.server_title)
+        self.assertEqual('test_documentation<script>',
+                self.serv.server_documentation)
+
+        generated = self.serv.generate_html_documentation()
+        title = re.search(r'<title>(.+?)</title>', generated).group()
+        documentation = re.search(r'<p><tt>(.+?)</tt></p>', generated).group()
+        self.assertEqual('<title>Python: test_title&lt;script&gt;</title>', title)
+        self.assertEqual('<p><tt>test_documentation&lt;script&gt;</tt></p>', documentation)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py
index 3e0dca027f06..efe593748968 100644
--- a/Lib/xmlrpc/server.py
+++ b/Lib/xmlrpc/server.py
@@ -106,6 +106,7 @@ def export_add(self, x, y):
 
 from xmlrpc.client import Fault, dumps, loads, gzip_encode, gzip_decode
 from http.server import BaseHTTPRequestHandler
+import html
 import http.server
 import socketserver
 import sys
@@ -904,7 +905,7 @@ def generate_html_documentation(self):
                                 methods
                             )
 
-        return documenter.page(self.server_title, documentation)
+        return documenter.page(html.escape(self.server_title), documentation)
 
 class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
     """XML-RPC and documentation request handler class.
diff --git a/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst b/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst
new file mode 100644
index 000000000000..98d7be129573
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst
@@ -0,0 +1,3 @@
+Escape the server title of :class:`xmlrpc.server.DocXMLRPCServer`
+when rendering the document page as HTML.
+(Contributed by Dong-hee Na in :issue:`38243`.)


From webhook-mailer at python.org  Sat Sep 28 08:32:04 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Sat, 28 Sep 2019 12:32:04 -0000
Subject: [Python-checkins] bpo-38216,
 bpo-36274: Allow subclasses to separately override validation and
 encoding behavior (GH-16448)
Message-ID: <mailman.131.1569673925.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/7774d7831e8809795c64ce27f7df52674581d298
commit: 7774d7831e8809795c64ce27f7df52674581d298
branch: master
author: Jason R. Coombs <jaraco at jaraco.com>
committer: GitHub <noreply at github.com>
date: 2019-09-28T08:32:01-04:00
summary:

bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448)

* bpo-38216: Allow bypassing input validation

* bpo-36274: Also allow the URL encoding to be overridden.

* bpo-38216, bpo-36274: Add tests demonstrating a hook for overriding validation, test demonstrating override encoding, and a test to capture expectation of the interface for the URL.

* Call with skip_host to avoid tripping on the host checking in the URL.

* Remove obsolete comment.

* Make _prepare_path_encoding its own attr.

This makes overriding just that simpler.

Also, don't use the := operator to make backporting easier.

* Add a news entry.

* _prepare_path_encoding -> _encode_prepared_path()

* Once again separate the path validation and request encoding, drastically simplifying the behavior. Drop the guarantee that all processing happens in _prepare_path.

files:
A Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst
M Lib/http/client.py
M Lib/test/test_httplib.py

diff --git a/Lib/http/client.py b/Lib/http/client.py
index f61267e108a5..33a434733f8a 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -1085,18 +1085,15 @@ def putrequest(self, method, url, skip_host=False,
         else:
             raise CannotSendRequest(self.__state)
 
-        # Save the method we use, we need it later in the response phase
+        # Save the method for use later in the response phase
         self._method = method
-        if not url:
-            url = '/'
-        # Prevent CVE-2019-9740.
-        if match := _contains_disallowed_url_pchar_re.search(url):
-            raise InvalidURL(f"URL can't contain control characters. {url!r} "
-                             f"(found at least {match.group()!r})")
+
+        url = url or '/'
+        self._validate_path(url)
+
         request = '%s %s %s' % (method, url, self._http_vsn_str)
 
-        # Non-ASCII characters should have been eliminated earlier
-        self._output(request.encode('ascii'))
+        self._output(self._encode_request(request))
 
         if self._http_vsn == 11:
             # Issue some standard headers for better HTTP/1.1 compliance
@@ -1174,6 +1171,18 @@ def putrequest(self, method, url, skip_host=False,
             # For HTTP/1.0, the server will assume "not chunked"
             pass
 
+    def _encode_request(self, request):
+        # ASCII also helps prevent CVE-2019-9740.
+        return request.encode('ascii')
+
+    def _validate_path(self, url):
+        """Validate a url for putrequest."""
+        # Prevent CVE-2019-9740.
+        match = _contains_disallowed_url_pchar_re.search(url)
+        if match:
+            raise InvalidURL(f"URL can't contain control characters. {url!r} "
+                             f"(found at least {match.group()!r})")
+
     def putheader(self, header, *values):
         """Send a request header line to the server.
 
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
index 1121224da443..e1aa41421feb 100644
--- a/Lib/test/test_httplib.py
+++ b/Lib/test/test_httplib.py
@@ -1155,6 +1155,34 @@ def run_server():
         thread.join()
         self.assertEqual(result, b"proxied data\n")
 
+    def test_putrequest_override_validation(self):
+        """
+        It should be possible to override the default validation
+        behavior in putrequest (bpo-38216).
+        """
+        class UnsafeHTTPConnection(client.HTTPConnection):
+            def _validate_path(self, url):
+                pass
+
+        conn = UnsafeHTTPConnection('example.com')
+        conn.sock = FakeSocket('')
+        conn.putrequest('GET', '/\x00')
+
+    def test_putrequest_override_encoding(self):
+        """
+        It should be possible to override the default encoding
+        to transmit bytes in another encoding even if invalid
+        (bpo-36274).
+        """
+        class UnsafeHTTPConnection(client.HTTPConnection):
+            def _encode_request(self, str_url):
+                return str_url.encode('utf-8')
+
+        conn = UnsafeHTTPConnection('example.com')
+        conn.sock = FakeSocket('')
+        conn.putrequest('GET', '/?')
+
+
 class ExtendedReadTest(TestCase):
     """
     Test peek(), read1(), readline()
@@ -1279,6 +1307,7 @@ def test_peek_0(self):
         p = self.resp.peek(0)
         self.assertLessEqual(0, len(p))
 
+
 class ExtendedReadTestChunked(ExtendedReadTest):
     """
     Test peek(), read1(), readline() in chunked mode
diff --git a/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst
new file mode 100644
index 000000000000..ac8e2b042d92
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst
@@ -0,0 +1,4 @@
+Allow the rare code that wants to send invalid http requests from the
+`http.client` library a way to do so.  The fixes for bpo-30458 led to
+breakage for some projects that were relying on this ability to test their
+own behavior in the face of bad requests.


From webhook-mailer at python.org  Sat Sep 28 09:15:09 2019
From: webhook-mailer at python.org (Jason R. Coombs)
Date: Sat, 28 Sep 2019 13:15:09 -0000
Subject: [Python-checkins] [3.7] bpo-38216,
 bpo-36274: Allow subclasses to separately override validation and
 encoding behavior (GH-16448) (GH-16461)
Message-ID: <mailman.132.1569676510.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/80dd66ac278ecbabbf843526e3a56f5031da9562
commit: 80dd66ac278ecbabbf843526e3a56f5031da9562
branch: 3.7
author: Jason R. Coombs <jaraco at jaraco.com>
committer: GitHub <noreply at github.com>
date: 2019-09-28T09:15:05-04:00
summary:

[3.7] bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448) (GH-16461)

* bpo-38216: Allow bypassing input validation

* bpo-36274: Also allow the URL encoding to be overridden.

* bpo-38216, bpo-36274: Add tests demonstrating a hook for overriding validation, test demonstrating override encoding, and a test to capture expectation of the interface for the URL.

* Call with skip_host to avoid tripping on the host checking in the URL.

* Remove obsolete comment.

* Make _prepare_path_encoding its own attr.

This makes overriding just that simpler.

Also, don't use the := operator to make backporting easier.

* Add a news entry.

* _prepare_path_encoding -> _encode_prepared_path()

* Once again separate the path validation and request encoding, drastically simplifying the behavior. Drop the guarantee that all processing happens in _prepare_path..
(cherry picked from commit 7774d7831e8809795c64ce27f7df52674581d298)

Co-authored-by: Jason R. Coombs <jaraco at jaraco.com>

files:
A Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst
M Lib/http/client.py
M Lib/test/test_httplib.py

diff --git a/Lib/http/client.py b/Lib/http/client.py
index dd23edcd597c..f7a183c77403 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -1107,19 +1107,15 @@ def putrequest(self, method, url, skip_host=False,
         else:
             raise CannotSendRequest(self.__state)
 
-        # Save the method we use, we need it later in the response phase
+        # Save the method for use later in the response phase
         self._method = method
-        if not url:
-            url = '/'
-        # Prevent CVE-2019-9740.
-        match = _contains_disallowed_url_pchar_re.search(url)
-        if match:
-            raise InvalidURL(f"URL can't contain control characters. {url!r} "
-                             f"(found at least {match.group()!r})")
+
+        url = url or '/'
+        self._validate_path(url)
+
         request = '%s %s %s' % (method, url, self._http_vsn_str)
 
-        # Non-ASCII characters should have been eliminated earlier
-        self._output(request.encode('ascii'))
+        self._output(self._encode_request(request))
 
         if self._http_vsn == 11:
             # Issue some standard headers for better HTTP/1.1 compliance
@@ -1197,6 +1193,18 @@ def putrequest(self, method, url, skip_host=False,
             # For HTTP/1.0, the server will assume "not chunked"
             pass
 
+    def _encode_request(self, request):
+        # ASCII also helps prevent CVE-2019-9740.
+        return request.encode('ascii')
+
+    def _validate_path(self, url):
+        """Validate a url for putrequest."""
+        # Prevent CVE-2019-9740.
+        match = _contains_disallowed_url_pchar_re.search(url)
+        if match:
+            raise InvalidURL(f"URL can't contain control characters. {url!r} "
+                             f"(found at least {match.group()!r})")
+
     def putheader(self, header, *values):
         """Send a request header line to the server.
 
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
index c4246671586f..9facb2e36017 100644
--- a/Lib/test/test_httplib.py
+++ b/Lib/test/test_httplib.py
@@ -1157,6 +1157,34 @@ def run_server():
         thread.join()
         self.assertEqual(result, b"proxied data\n")
 
+    def test_putrequest_override_validation(self):
+        """
+        It should be possible to override the default validation
+        behavior in putrequest (bpo-38216).
+        """
+        class UnsafeHTTPConnection(client.HTTPConnection):
+            def _validate_path(self, url):
+                pass
+
+        conn = UnsafeHTTPConnection('example.com')
+        conn.sock = FakeSocket('')
+        conn.putrequest('GET', '/\x00')
+
+    def test_putrequest_override_encoding(self):
+        """
+        It should be possible to override the default encoding
+        to transmit bytes in another encoding even if invalid
+        (bpo-36274).
+        """
+        class UnsafeHTTPConnection(client.HTTPConnection):
+            def _encode_request(self, str_url):
+                return str_url.encode('utf-8')
+
+        conn = UnsafeHTTPConnection('example.com')
+        conn.sock = FakeSocket('')
+        conn.putrequest('GET', '/?')
+
+
 class ExtendedReadTest(TestCase):
     """
     Test peek(), read1(), readline()
@@ -1281,6 +1309,7 @@ def test_peek_0(self):
         p = self.resp.peek(0)
         self.assertLessEqual(0, len(p))
 
+
 class ExtendedReadTestChunked(ExtendedReadTest):
     """
     Test peek(), read1(), readline() in chunked mode
diff --git a/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst
new file mode 100644
index 000000000000..ac8e2b042d92
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst
@@ -0,0 +1,4 @@
+Allow the rare code that wants to send invalid http requests from the
+`http.client` library a way to do so.  The fixes for bpo-30458 led to
+breakage for some projects that were relying on this ability to test their
+own behavior in the face of bad requests.


From webhook-mailer at python.org  Sat Sep 28 10:23:38 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sat, 28 Sep 2019 14:23:38 -0000
Subject: [Python-checkins] bpo-38216,
 bpo-36274: Allow subclasses to separately override validation and
 encoding behavior (GH-16448)
Message-ID: <mailman.133.1569680619.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/8f478b489ae11633d2609dff0ef21d0e1a857417
commit: 8f478b489ae11633d2609dff0ef21d0e1a857417
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-28T07:23:34-07:00
summary:

bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448)


* bpo-38216: Allow bypassing input validation

* bpo-36274: Also allow the URL encoding to be overridden.

* bpo-38216, bpo-36274: Add tests demonstrating a hook for overriding validation, test demonstrating override encoding, and a test to capture expectation of the interface for the URL.

* Call with skip_host to avoid tripping on the host checking in the URL.

* Remove obsolete comment.

* Make _prepare_path_encoding its own attr.

This makes overriding just that simpler.

Also, don't use the := operator to make backporting easier.

* Add a news entry.

* _prepare_path_encoding -> _encode_prepared_path()

* Once again separate the path validation and request encoding, drastically simplifying the behavior. Drop the guarantee that all processing happens in _prepare_path.
(cherry picked from commit 7774d7831e8809795c64ce27f7df52674581d298)

Co-authored-by: Jason R. Coombs <jaraco at jaraco.com>

files:
A Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst
M Lib/http/client.py
M Lib/test/test_httplib.py

diff --git a/Lib/http/client.py b/Lib/http/client.py
index f61267e108a5..33a434733f8a 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -1085,18 +1085,15 @@ def putrequest(self, method, url, skip_host=False,
         else:
             raise CannotSendRequest(self.__state)
 
-        # Save the method we use, we need it later in the response phase
+        # Save the method for use later in the response phase
         self._method = method
-        if not url:
-            url = '/'
-        # Prevent CVE-2019-9740.
-        if match := _contains_disallowed_url_pchar_re.search(url):
-            raise InvalidURL(f"URL can't contain control characters. {url!r} "
-                             f"(found at least {match.group()!r})")
+
+        url = url or '/'
+        self._validate_path(url)
+
         request = '%s %s %s' % (method, url, self._http_vsn_str)
 
-        # Non-ASCII characters should have been eliminated earlier
-        self._output(request.encode('ascii'))
+        self._output(self._encode_request(request))
 
         if self._http_vsn == 11:
             # Issue some standard headers for better HTTP/1.1 compliance
@@ -1174,6 +1171,18 @@ def putrequest(self, method, url, skip_host=False,
             # For HTTP/1.0, the server will assume "not chunked"
             pass
 
+    def _encode_request(self, request):
+        # ASCII also helps prevent CVE-2019-9740.
+        return request.encode('ascii')
+
+    def _validate_path(self, url):
+        """Validate a url for putrequest."""
+        # Prevent CVE-2019-9740.
+        match = _contains_disallowed_url_pchar_re.search(url)
+        if match:
+            raise InvalidURL(f"URL can't contain control characters. {url!r} "
+                             f"(found at least {match.group()!r})")
+
     def putheader(self, header, *values):
         """Send a request header line to the server.
 
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
index 1121224da443..e1aa41421feb 100644
--- a/Lib/test/test_httplib.py
+++ b/Lib/test/test_httplib.py
@@ -1155,6 +1155,34 @@ def run_server():
         thread.join()
         self.assertEqual(result, b"proxied data\n")
 
+    def test_putrequest_override_validation(self):
+        """
+        It should be possible to override the default validation
+        behavior in putrequest (bpo-38216).
+        """
+        class UnsafeHTTPConnection(client.HTTPConnection):
+            def _validate_path(self, url):
+                pass
+
+        conn = UnsafeHTTPConnection('example.com')
+        conn.sock = FakeSocket('')
+        conn.putrequest('GET', '/\x00')
+
+    def test_putrequest_override_encoding(self):
+        """
+        It should be possible to override the default encoding
+        to transmit bytes in another encoding even if invalid
+        (bpo-36274).
+        """
+        class UnsafeHTTPConnection(client.HTTPConnection):
+            def _encode_request(self, str_url):
+                return str_url.encode('utf-8')
+
+        conn = UnsafeHTTPConnection('example.com')
+        conn.sock = FakeSocket('')
+        conn.putrequest('GET', '/?')
+
+
 class ExtendedReadTest(TestCase):
     """
     Test peek(), read1(), readline()
@@ -1279,6 +1307,7 @@ def test_peek_0(self):
         p = self.resp.peek(0)
         self.assertLessEqual(0, len(p))
 
+
 class ExtendedReadTestChunked(ExtendedReadTest):
     """
     Test peek(), read1(), readline() in chunked mode
diff --git a/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst
new file mode 100644
index 000000000000..ac8e2b042d92
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst
@@ -0,0 +1,4 @@
+Allow the rare code that wants to send invalid http requests from the
+`http.client` library a way to do so.  The fixes for bpo-30458 led to
+breakage for some projects that were relying on this ability to test their
+own behavior in the face of bad requests.


From webhook-mailer at python.org  Sat Sep 28 10:49:19 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Sat, 28 Sep 2019 14:49:19 -0000
Subject: [Python-checkins] bpo-38115: Deal with invalid bytecode offsets in
 lnotab (GH-16079)
Message-ID: <mailman.134.1569682160.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/c8165036f374cd2ee64d4314eeb2514f7acb5026
commit: c8165036f374cd2ee64d4314eeb2514f7acb5026
branch: master
author: T. Wouters <thomas at python.org>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-28T07:49:15-07:00
summary:

bpo-38115: Deal with invalid bytecode offsets in lnotab (GH-16079)

Document that lnotab can contain invalid bytecode offsets (because of
terrible reasons that are difficult to fix). Make dis.findlinestarts()
ignore invalid offsets in lnotab. All other uses of lnotab in CPython
(various reimplementations of addr2line or line2addr in Python, C and gdb)
already ignore this, because they take an address to look for, instead.

Add tests for the result of dis.findlinestarts() on wacky constructs in
test_peepholer.py, because it's the easiest place to add them.

files:
A Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst
M Lib/dis.py
M Lib/test/test_peepholer.py
M Objects/lnotab_notes.txt

diff --git a/Lib/dis.py b/Lib/dis.py
index a25fb2b41764..10e5f7fb08ab 100644
--- a/Lib/dis.py
+++ b/Lib/dis.py
@@ -454,6 +454,7 @@ def findlinestarts(code):
     """
     byte_increments = code.co_lnotab[0::2]
     line_increments = code.co_lnotab[1::2]
+    bytecode_len = len(code.co_code)
 
     lastlineno = None
     lineno = code.co_firstlineno
@@ -464,6 +465,10 @@ def findlinestarts(code):
                 yield (addr, lineno)
                 lastlineno = lineno
             addr += byte_incr
+            if addr >= bytecode_len:
+                # The rest of the lnotab byte offsets are past the end of
+                # the bytecode, so the lines were optimized away.
+                return
         if line_incr >= 0x80:
             # line_increments is an array of 8-bit signed integers
             line_incr -= 0x100
diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py
index 47dee33076c5..23cc36c60537 100644
--- a/Lib/test/test_peepholer.py
+++ b/Lib/test/test_peepholer.py
@@ -40,6 +40,20 @@ def check_jump_targets(self, code):
                 self.fail(f'{instr.opname} at {instr.offset} '
                           f'jumps to {tgt.opname} at {tgt.offset}')
 
+    def check_lnotab(self, code):
+        "Check that the lnotab byte offsets are sensible."
+        code = dis._get_code_object(code)
+        lnotab = list(dis.findlinestarts(code))
+        # Don't bother checking if the line info is sensible, because
+        # most of the line info we can get at comes from lnotab.
+        min_bytecode = min(t[0] for t in lnotab)
+        max_bytecode = max(t[0] for t in lnotab)
+        self.assertGreaterEqual(min_bytecode, 0)
+        self.assertLess(max_bytecode, len(code.co_code))
+        # This could conceivably test more (and probably should, as there
+        # aren't very many tests of lnotab), if peepholer wasn't scheduled
+        # to be replaced anyway.
+
     def test_unot(self):
         # UNARY_NOT POP_JUMP_IF_FALSE  -->  POP_JUMP_IF_TRUE'
         def unot(x):
@@ -48,6 +62,7 @@ def unot(x):
         self.assertNotInBytecode(unot, 'UNARY_NOT')
         self.assertNotInBytecode(unot, 'POP_JUMP_IF_FALSE')
         self.assertInBytecode(unot, 'POP_JUMP_IF_TRUE')
+        self.check_lnotab(unot)
 
     def test_elim_inversion_of_is_or_in(self):
         for line, cmp_op in (
@@ -58,6 +73,7 @@ def test_elim_inversion_of_is_or_in(self):
             ):
             code = compile(line, '', 'single')
             self.assertInBytecode(code, 'COMPARE_OP', cmp_op)
+            self.check_lnotab(code)
 
     def test_global_as_constant(self):
         # LOAD_GLOBAL None/True/False  -->  LOAD_CONST None/True/False
@@ -75,6 +91,7 @@ def h():
         for func, elem in ((f, None), (g, True), (h, False)):
             self.assertNotInBytecode(func, 'LOAD_GLOBAL')
             self.assertInBytecode(func, 'LOAD_CONST', elem)
+            self.check_lnotab(func)
 
         def f():
             'Adding a docstring made this test fail in Py2.5.0'
@@ -82,6 +99,7 @@ def f():
 
         self.assertNotInBytecode(f, 'LOAD_GLOBAL')
         self.assertInBytecode(f, 'LOAD_CONST', None)
+        self.check_lnotab(f)
 
     def test_while_one(self):
         # Skip over:  LOAD_CONST trueconst  POP_JUMP_IF_FALSE xx
@@ -93,6 +111,7 @@ def f():
             self.assertNotInBytecode(f, elem)
         for elem in ('JUMP_ABSOLUTE',):
             self.assertInBytecode(f, elem)
+        self.check_lnotab(f)
 
     def test_pack_unpack(self):
         for line, elem in (
@@ -104,6 +123,7 @@ def test_pack_unpack(self):
             self.assertInBytecode(code, elem)
             self.assertNotInBytecode(code, 'BUILD_TUPLE')
             self.assertNotInBytecode(code, 'UNPACK_TUPLE')
+            self.check_lnotab(code)
 
     def test_folding_of_tuples_of_constants(self):
         for line, elem in (
@@ -116,6 +136,7 @@ def test_folding_of_tuples_of_constants(self):
             code = compile(line,'','single')
             self.assertInBytecode(code, 'LOAD_CONST', elem)
             self.assertNotInBytecode(code, 'BUILD_TUPLE')
+            self.check_lnotab(code)
 
         # Long tuples should be folded too.
         code = compile(repr(tuple(range(10000))),'','single')
@@ -124,6 +145,7 @@ def test_folding_of_tuples_of_constants(self):
         load_consts = [instr for instr in dis.get_instructions(code)
                               if instr.opname == 'LOAD_CONST']
         self.assertEqual(len(load_consts), 2)
+        self.check_lnotab(code)
 
         # Bug 1053819:  Tuple of constants misidentified when presented with:
         # . . . opcode_with_arg 100   unary_opcode   BUILD_TUPLE 1  . . .
@@ -141,6 +163,7 @@ def crater():
                 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
                 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
             ],)
+        self.check_lnotab(crater)
 
     def test_folding_of_lists_of_constants(self):
         for line, elem in (
@@ -153,6 +176,7 @@ def test_folding_of_lists_of_constants(self):
             code = compile(line, '', 'single')
             self.assertInBytecode(code, 'LOAD_CONST', elem)
             self.assertNotInBytecode(code, 'BUILD_LIST')
+            self.check_lnotab(code)
 
     def test_folding_of_sets_of_constants(self):
         for line, elem in (
@@ -166,6 +190,7 @@ def test_folding_of_sets_of_constants(self):
             code = compile(line, '', 'single')
             self.assertNotInBytecode(code, 'BUILD_SET')
             self.assertInBytecode(code, 'LOAD_CONST', elem)
+            self.check_lnotab(code)
 
         # Ensure that the resulting code actually works:
         def f(a):
@@ -176,9 +201,11 @@ def g(a):
 
         self.assertTrue(f(3))
         self.assertTrue(not f(4))
+        self.check_lnotab(f)
 
         self.assertTrue(not g(3))
         self.assertTrue(g(4))
+        self.check_lnotab(g)
 
 
     def test_folding_of_binops_on_constants(self):
@@ -203,41 +230,50 @@ def test_folding_of_binops_on_constants(self):
             self.assertInBytecode(code, 'LOAD_CONST', elem)
             for instr in dis.get_instructions(code):
                 self.assertFalse(instr.opname.startswith('BINARY_'))
+            self.check_lnotab(code)
 
         # Verify that unfoldables are skipped
         code = compile('a=2+"b"', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', 2)
         self.assertInBytecode(code, 'LOAD_CONST', 'b')
+        self.check_lnotab(code)
 
         # Verify that large sequences do not result from folding
         code = compile('a="x"*10000', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', 10000)
         self.assertNotIn("x"*10000, code.co_consts)
+        self.check_lnotab(code)
         code = compile('a=1<<1000', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', 1000)
         self.assertNotIn(1<<1000, code.co_consts)
+        self.check_lnotab(code)
         code = compile('a=2**1000', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', 1000)
         self.assertNotIn(2**1000, code.co_consts)
+        self.check_lnotab(code)
 
     def test_binary_subscr_on_unicode(self):
         # valid code get optimized
         code = compile('"foo"[0]', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', 'f')
         self.assertNotInBytecode(code, 'BINARY_SUBSCR')
+        self.check_lnotab(code)
         code = compile('"\u0061\uffff"[1]', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', '\uffff')
         self.assertNotInBytecode(code,'BINARY_SUBSCR')
+        self.check_lnotab(code)
 
         # With PEP 393, non-BMP char get optimized
         code = compile('"\U00012345"[0]', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', '\U00012345')
         self.assertNotInBytecode(code, 'BINARY_SUBSCR')
+        self.check_lnotab(code)
 
         # invalid code doesn't get optimized
         # out of range
         code = compile('"fuu"[10]', '', 'single')
         self.assertInBytecode(code, 'BINARY_SUBSCR')
+        self.check_lnotab(code)
 
     def test_folding_of_unaryops_on_constants(self):
         for line, elem in (
@@ -252,13 +288,15 @@ def test_folding_of_unaryops_on_constants(self):
             self.assertInBytecode(code, 'LOAD_CONST', elem)
             for instr in dis.get_instructions(code):
                 self.assertFalse(instr.opname.startswith('UNARY_'))
+            self.check_lnotab(code)
 
         # Check that -0.0 works after marshaling
         def negzero():
             return -(1.0-1.0)
 
-        for instr in dis.get_instructions(code):
+        for instr in dis.get_instructions(negzero):
             self.assertFalse(instr.opname.startswith('UNARY_'))
+        self.check_lnotab(negzero)
 
         # Verify that unfoldables are skipped
         for line, elem, opname in (
@@ -268,6 +306,7 @@ def negzero():
             code = compile(line, '', 'single')
             self.assertInBytecode(code, 'LOAD_CONST', elem)
             self.assertInBytecode(code, opname)
+            self.check_lnotab(code)
 
     def test_elim_extra_return(self):
         # RETURN LOAD_CONST None RETURN  -->  RETURN
@@ -277,6 +316,7 @@ def f(x):
         returns = [instr for instr in dis.get_instructions(f)
                           if instr.opname == 'RETURN_VALUE']
         self.assertEqual(len(returns), 1)
+        self.check_lnotab(f)
 
     def test_elim_jump_to_return(self):
         # JUMP_FORWARD to RETURN -->  RETURN
@@ -290,6 +330,7 @@ def f(cond, true_value, false_value):
         returns = [instr for instr in dis.get_instructions(f)
                           if instr.opname == 'RETURN_VALUE']
         self.assertEqual(len(returns), 2)
+        self.check_lnotab(f)
 
     def test_elim_jump_to_uncond_jump(self):
         # POP_JUMP_IF_FALSE to JUMP_FORWARD --> POP_JUMP_IF_FALSE to non-jump
@@ -302,6 +343,7 @@ def f():
             else:
                 baz()
         self.check_jump_targets(f)
+        self.check_lnotab(f)
 
     def test_elim_jump_to_uncond_jump2(self):
         # POP_JUMP_IF_FALSE to JUMP_ABSOLUTE --> POP_JUMP_IF_FALSE to non-jump
@@ -312,6 +354,7 @@ def f():
                     or d):
                     a = foo()
         self.check_jump_targets(f)
+        self.check_lnotab(f)
 
     def test_elim_jump_to_uncond_jump3(self):
         # Intentionally use two-line expressions to test issue37213.
@@ -320,18 +363,21 @@ def f(a, b, c):
             return ((a and b)
                     and c)
         self.check_jump_targets(f)
+        self.check_lnotab(f)
         self.assertEqual(count_instr_recursively(f, 'JUMP_IF_FALSE_OR_POP'), 2)
         # JUMP_IF_TRUE_OR_POP to JUMP_IF_TRUE_OR_POP --> JUMP_IF_TRUE_OR_POP to non-jump
         def f(a, b, c):
             return ((a or b)
                     or c)
         self.check_jump_targets(f)
+        self.check_lnotab(f)
         self.assertEqual(count_instr_recursively(f, 'JUMP_IF_TRUE_OR_POP'), 2)
         # JUMP_IF_FALSE_OR_POP to JUMP_IF_TRUE_OR_POP --> POP_JUMP_IF_FALSE to non-jump
         def f(a, b, c):
             return ((a and b)
                     or c)
         self.check_jump_targets(f)
+        self.check_lnotab(f)
         self.assertNotInBytecode(f, 'JUMP_IF_FALSE_OR_POP')
         self.assertInBytecode(f, 'JUMP_IF_TRUE_OR_POP')
         self.assertInBytecode(f, 'POP_JUMP_IF_FALSE')
@@ -340,6 +386,7 @@ def f(a, b, c):
             return ((a or b)
                     and c)
         self.check_jump_targets(f)
+        self.check_lnotab(f)
         self.assertNotInBytecode(f, 'JUMP_IF_TRUE_OR_POP')
         self.assertInBytecode(f, 'JUMP_IF_FALSE_OR_POP')
         self.assertInBytecode(f, 'POP_JUMP_IF_TRUE')
@@ -360,6 +407,7 @@ def f(cond1, cond2):
         returns = [instr for instr in dis.get_instructions(f)
                           if instr.opname == 'RETURN_VALUE']
         self.assertLessEqual(len(returns), 6)
+        self.check_lnotab(f)
 
     def test_elim_jump_after_return2(self):
         # Eliminate dead code: jumps immediately after returns can't be reached
@@ -374,6 +422,7 @@ def f(cond1, cond2):
         returns = [instr for instr in dis.get_instructions(f)
                           if instr.opname == 'RETURN_VALUE']
         self.assertLessEqual(len(returns), 2)
+        self.check_lnotab(f)
 
     def test_make_function_doesnt_bail(self):
         def f():
@@ -381,6 +430,7 @@ def g()->1+1:
                 pass
             return g
         self.assertNotInBytecode(f, 'BINARY_ADD')
+        self.check_lnotab(f)
 
     def test_constant_folding(self):
         # Issue #11244: aggressive constant folding.
@@ -401,17 +451,20 @@ def test_constant_folding(self):
                 self.assertFalse(instr.opname.startswith('UNARY_'))
                 self.assertFalse(instr.opname.startswith('BINARY_'))
                 self.assertFalse(instr.opname.startswith('BUILD_'))
+            self.check_lnotab(code)
 
     def test_in_literal_list(self):
         def containtest():
             return x in [a, b]
         self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0)
+        self.check_lnotab(containtest)
 
     def test_iterate_literal_list(self):
         def forloop():
             for x in [a, b]:
                 pass
         self.assertEqual(count_instr_recursively(forloop, 'BUILD_LIST'), 0)
+        self.check_lnotab(forloop)
 
     def test_condition_with_binop_with_bools(self):
         def f():
@@ -419,6 +472,7 @@ def f():
                 return 1
             return 0
         self.assertEqual(f(), 1)
+        self.check_lnotab(f)
 
     def test_if_with_if_expression(self):
         # Check bpo-37289
@@ -427,6 +481,19 @@ def f(x):
                 return True
             return False
         self.assertTrue(f(True))
+        self.check_lnotab(f)
+
+    def test_trailing_nops(self):
+        # Check the lnotab of a function that even after trivial
+        # optimization has trailing nops, which the lnotab adjustment has to
+        # handle properly (bpo-38115).
+        def f(x):
+            while 1:
+                return 3
+            while 1:
+                return 5
+            return 6
+        self.check_lnotab(f)
 
 
 class TestBuglets(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst b/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst
new file mode 100644
index 000000000000..5119c0546e36
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst
@@ -0,0 +1 @@
+Fix a bug in dis.findlinestarts() where it would return invalid bytecode offsets. Document that a code object's co_lnotab can contain invalid bytecode offsets.
\ No newline at end of file
diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt
index 3dab2b986616..71a297971828 100644
--- a/Objects/lnotab_notes.txt
+++ b/Objects/lnotab_notes.txt
@@ -3,7 +3,9 @@ All about co_lnotab, the line number table.
 Code objects store a field named co_lnotab.  This is an array of unsigned bytes
 disguised as a Python bytes object.  It is used to map bytecode offsets to
 source code line #s for tracebacks and to identify line number boundaries for
-line tracing.
+line tracing. Because of internals of the peephole optimizer, it's possible
+for lnotab to contain bytecode offsets that are no longer valid (for example
+if the optimizer removed the last line in a function).
 
 The array is conceptually a compressed list of
     (bytecode offset increment, line number increment)


From webhook-mailer at python.org  Sat Sep 28 11:04:48 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Sat, 28 Sep 2019 15:04:48 -0000
Subject: [Python-checkins] bpo-37408: Precise that Tarfile "format" argument
 only concerns writing. (GH-14389)
Message-ID: <mailman.135.1569683089.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/c5a7e0ce194c0eafe82eb3e431881012398e7d46
commit: c5a7e0ce194c0eafe82eb3e431881012398e7d46
branch: master
author: Pascal Chambon <pythoniks at gmail.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-28T08:04:44-07:00
summary:

bpo-37408: Precise that Tarfile "format" argument only concerns writing. (GH-14389)

files:
M Doc/library/tarfile.rst

diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst
index f25af8ca6a33..c34f2c4a5702 100644
--- a/Doc/library/tarfile.rst
+++ b/Doc/library/tarfile.rst
@@ -290,9 +290,10 @@ be finalized; only the internally used file object will be closed. See the
 
       *fileobj* is not closed, when :class:`TarFile` is closed.
 
-   *format* controls the archive format. It must be one of the constants
+   *format* controls the archive format for writing. It must be one of the constants
    :const:`USTAR_FORMAT`, :const:`GNU_FORMAT` or :const:`PAX_FORMAT` that are
-   defined at module level.
+   defined at module level. When reading, format will be automatically detected, even
+   if different formats are present in a single archive.
 
    The *tarinfo* argument can be used to replace the default :class:`TarInfo` class
    with a different one.


From webhook-mailer at python.org  Sat Sep 28 11:22:04 2019
From: webhook-mailer at python.org (Gregory P. Smith)
Date: Sat, 28 Sep 2019 15:22:04 -0000
Subject: [Python-checkins] bpo-38115: Deal with invalid bytecode offsets in
 lnotab (GH-16079) (GH-16464)
Message-ID: <mailman.136.1569684126.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/36c6fa968016a46a39c3cdbd0a17ea5490dfa343
commit: 36c6fa968016a46a39c3cdbd0a17ea5490dfa343
branch: 3.8
author: Gregory P. Smith <greg at krypto.org>
committer: GitHub <noreply at github.com>
date: 2019-09-28T08:22:00-07:00
summary:

bpo-38115: Deal with invalid bytecode offsets in lnotab (GH-16079) (GH-16464)

Document that lnotab can contain invalid bytecode offsets (because of
terrible reasons that are difficult to fix). Make dis.findlinestarts()
ignore invalid offsets in lnotab. All other uses of lnotab in CPython
(various reimplementations of addr2line or line2addr in Python, C and gdb)
already ignore this, because they take an address to look for, instead.

Add tests for the result of dis.findlinestarts() on wacky constructs in
test_peepholer.py, because it's the easiest place to add them.

(cherry picked from commit c8165036f374cd2ee64d4314eeb2514f7acb5026)

files:
A Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst
M Lib/dis.py
M Lib/test/test_peepholer.py
M Objects/lnotab_notes.txt

diff --git a/Lib/dis.py b/Lib/dis.py
index a25fb2b41764..10e5f7fb08ab 100644
--- a/Lib/dis.py
+++ b/Lib/dis.py
@@ -454,6 +454,7 @@ def findlinestarts(code):
     """
     byte_increments = code.co_lnotab[0::2]
     line_increments = code.co_lnotab[1::2]
+    bytecode_len = len(code.co_code)
 
     lastlineno = None
     lineno = code.co_firstlineno
@@ -464,6 +465,10 @@ def findlinestarts(code):
                 yield (addr, lineno)
                 lastlineno = lineno
             addr += byte_incr
+            if addr >= bytecode_len:
+                # The rest of the lnotab byte offsets are past the end of
+                # the bytecode, so the lines were optimized away.
+                return
         if line_incr >= 0x80:
             # line_increments is an array of 8-bit signed integers
             line_incr -= 0x100
diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py
index 9c206d1d09c0..01c00369156b 100644
--- a/Lib/test/test_peepholer.py
+++ b/Lib/test/test_peepholer.py
@@ -40,6 +40,20 @@ def check_jump_targets(self, code):
                 self.fail(f'{instr.opname} at {instr.offset} '
                           f'jumps to {tgt.opname} at {tgt.offset}')
 
+    def check_lnotab(self, code):
+        "Check that the lnotab byte offsets are sensible."
+        code = dis._get_code_object(code)
+        lnotab = list(dis.findlinestarts(code))
+        # Don't bother checking if the line info is sensible, because
+        # most of the line info we can get at comes from lnotab.
+        min_bytecode = min(t[0] for t in lnotab)
+        max_bytecode = max(t[0] for t in lnotab)
+        self.assertGreaterEqual(min_bytecode, 0)
+        self.assertLess(max_bytecode, len(code.co_code))
+        # This could conceivably test more (and probably should, as there
+        # aren't very many tests of lnotab), if peepholer wasn't scheduled
+        # to be replaced anyway.
+
     def test_unot(self):
         # UNARY_NOT POP_JUMP_IF_FALSE  -->  POP_JUMP_IF_TRUE'
         def unot(x):
@@ -48,6 +62,7 @@ def unot(x):
         self.assertNotInBytecode(unot, 'UNARY_NOT')
         self.assertNotInBytecode(unot, 'POP_JUMP_IF_FALSE')
         self.assertInBytecode(unot, 'POP_JUMP_IF_TRUE')
+        self.check_lnotab(unot)
 
     def test_elim_inversion_of_is_or_in(self):
         for line, cmp_op in (
@@ -58,6 +73,7 @@ def test_elim_inversion_of_is_or_in(self):
             ):
             code = compile(line, '', 'single')
             self.assertInBytecode(code, 'COMPARE_OP', cmp_op)
+            self.check_lnotab(code)
 
     def test_global_as_constant(self):
         # LOAD_GLOBAL None/True/False  -->  LOAD_CONST None/True/False
@@ -75,6 +91,7 @@ def h():
         for func, elem in ((f, None), (g, True), (h, False)):
             self.assertNotInBytecode(func, 'LOAD_GLOBAL')
             self.assertInBytecode(func, 'LOAD_CONST', elem)
+            self.check_lnotab(func)
 
         def f():
             'Adding a docstring made this test fail in Py2.5.0'
@@ -82,6 +99,7 @@ def f():
 
         self.assertNotInBytecode(f, 'LOAD_GLOBAL')
         self.assertInBytecode(f, 'LOAD_CONST', None)
+        self.check_lnotab(f)
 
     def test_while_one(self):
         # Skip over:  LOAD_CONST trueconst  POP_JUMP_IF_FALSE xx
@@ -93,6 +111,7 @@ def f():
             self.assertNotInBytecode(f, elem)
         for elem in ('JUMP_ABSOLUTE',):
             self.assertInBytecode(f, elem)
+        self.check_lnotab(f)
 
     def test_pack_unpack(self):
         for line, elem in (
@@ -104,6 +123,7 @@ def test_pack_unpack(self):
             self.assertInBytecode(code, elem)
             self.assertNotInBytecode(code, 'BUILD_TUPLE')
             self.assertNotInBytecode(code, 'UNPACK_TUPLE')
+            self.check_lnotab(code)
 
     def test_folding_of_tuples_of_constants(self):
         for line, elem in (
@@ -116,6 +136,7 @@ def test_folding_of_tuples_of_constants(self):
             code = compile(line,'','single')
             self.assertInBytecode(code, 'LOAD_CONST', elem)
             self.assertNotInBytecode(code, 'BUILD_TUPLE')
+            self.check_lnotab(code)
 
         # Long tuples should be folded too.
         code = compile(repr(tuple(range(10000))),'','single')
@@ -124,6 +145,7 @@ def test_folding_of_tuples_of_constants(self):
         load_consts = [instr for instr in dis.get_instructions(code)
                               if instr.opname == 'LOAD_CONST']
         self.assertEqual(len(load_consts), 2)
+        self.check_lnotab(code)
 
         # Bug 1053819:  Tuple of constants misidentified when presented with:
         # . . . opcode_with_arg 100   unary_opcode   BUILD_TUPLE 1  . . .
@@ -141,6 +163,7 @@ def crater():
                 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
                 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
             ],)
+        self.check_lnotab(crater)
 
     def test_folding_of_lists_of_constants(self):
         for line, elem in (
@@ -153,6 +176,7 @@ def test_folding_of_lists_of_constants(self):
             code = compile(line, '', 'single')
             self.assertInBytecode(code, 'LOAD_CONST', elem)
             self.assertNotInBytecode(code, 'BUILD_LIST')
+            self.check_lnotab(code)
 
     def test_folding_of_sets_of_constants(self):
         for line, elem in (
@@ -166,6 +190,7 @@ def test_folding_of_sets_of_constants(self):
             code = compile(line, '', 'single')
             self.assertNotInBytecode(code, 'BUILD_SET')
             self.assertInBytecode(code, 'LOAD_CONST', elem)
+            self.check_lnotab(code)
 
         # Ensure that the resulting code actually works:
         def f(a):
@@ -176,9 +201,11 @@ def g(a):
 
         self.assertTrue(f(3))
         self.assertTrue(not f(4))
+        self.check_lnotab(f)
 
         self.assertTrue(not g(3))
         self.assertTrue(g(4))
+        self.check_lnotab(g)
 
 
     def test_folding_of_binops_on_constants(self):
@@ -203,41 +230,50 @@ def test_folding_of_binops_on_constants(self):
             self.assertInBytecode(code, 'LOAD_CONST', elem)
             for instr in dis.get_instructions(code):
                 self.assertFalse(instr.opname.startswith('BINARY_'))
+            self.check_lnotab(code)
 
         # Verify that unfoldables are skipped
         code = compile('a=2+"b"', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', 2)
         self.assertInBytecode(code, 'LOAD_CONST', 'b')
+        self.check_lnotab(code)
 
         # Verify that large sequences do not result from folding
         code = compile('a="x"*10000', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', 10000)
         self.assertNotIn("x"*10000, code.co_consts)
+        self.check_lnotab(code)
         code = compile('a=1<<1000', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', 1000)
         self.assertNotIn(1<<1000, code.co_consts)
+        self.check_lnotab(code)
         code = compile('a=2**1000', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', 1000)
         self.assertNotIn(2**1000, code.co_consts)
+        self.check_lnotab(code)
 
     def test_binary_subscr_on_unicode(self):
         # valid code get optimized
         code = compile('"foo"[0]', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', 'f')
         self.assertNotInBytecode(code, 'BINARY_SUBSCR')
+        self.check_lnotab(code)
         code = compile('"\u0061\uffff"[1]', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', '\uffff')
         self.assertNotInBytecode(code,'BINARY_SUBSCR')
+        self.check_lnotab(code)
 
         # With PEP 393, non-BMP char get optimized
         code = compile('"\U00012345"[0]', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', '\U00012345')
         self.assertNotInBytecode(code, 'BINARY_SUBSCR')
+        self.check_lnotab(code)
 
         # invalid code doesn't get optimized
         # out of range
         code = compile('"fuu"[10]', '', 'single')
         self.assertInBytecode(code, 'BINARY_SUBSCR')
+        self.check_lnotab(code)
 
     def test_folding_of_unaryops_on_constants(self):
         for line, elem in (
@@ -252,13 +288,15 @@ def test_folding_of_unaryops_on_constants(self):
             self.assertInBytecode(code, 'LOAD_CONST', elem)
             for instr in dis.get_instructions(code):
                 self.assertFalse(instr.opname.startswith('UNARY_'))
+            self.check_lnotab(code)
 
         # Check that -0.0 works after marshaling
         def negzero():
             return -(1.0-1.0)
 
-        for instr in dis.get_instructions(code):
+        for instr in dis.get_instructions(negzero):
             self.assertFalse(instr.opname.startswith('UNARY_'))
+        self.check_lnotab(negzero)
 
         # Verify that unfoldables are skipped
         for line, elem, opname in (
@@ -268,6 +306,7 @@ def negzero():
             code = compile(line, '', 'single')
             self.assertInBytecode(code, 'LOAD_CONST', elem)
             self.assertInBytecode(code, opname)
+            self.check_lnotab(code)
 
     def test_elim_extra_return(self):
         # RETURN LOAD_CONST None RETURN  -->  RETURN
@@ -277,6 +316,7 @@ def f(x):
         returns = [instr for instr in dis.get_instructions(f)
                           if instr.opname == 'RETURN_VALUE']
         self.assertEqual(len(returns), 1)
+        self.check_lnotab(f)
 
     def test_elim_jump_to_return(self):
         # JUMP_FORWARD to RETURN -->  RETURN
@@ -290,6 +330,7 @@ def f(cond, true_value, false_value):
         returns = [instr for instr in dis.get_instructions(f)
                           if instr.opname == 'RETURN_VALUE']
         self.assertEqual(len(returns), 2)
+        self.check_lnotab(f)
 
     def test_elim_jump_to_uncond_jump(self):
         # POP_JUMP_IF_FALSE to JUMP_FORWARD --> POP_JUMP_IF_FALSE to non-jump
@@ -302,6 +343,7 @@ def f():
             else:
                 baz()
         self.check_jump_targets(f)
+        self.check_lnotab(f)
 
     def test_elim_jump_to_uncond_jump2(self):
         # POP_JUMP_IF_FALSE to JUMP_ABSOLUTE --> POP_JUMP_IF_FALSE to non-jump
@@ -312,6 +354,7 @@ def f():
                     or d):
                     a = foo()
         self.check_jump_targets(f)
+        self.check_lnotab(f)
 
     def test_elim_jump_to_uncond_jump3(self):
         # Intentionally use two-line expressions to test issue37213.
@@ -320,18 +363,21 @@ def f(a, b, c):
             return ((a and b)
                     and c)
         self.check_jump_targets(f)
+        self.check_lnotab(f)
         self.assertEqual(count_instr_recursively(f, 'JUMP_IF_FALSE_OR_POP'), 2)
         # JUMP_IF_TRUE_OR_POP to JUMP_IF_TRUE_OR_POP --> JUMP_IF_TRUE_OR_POP to non-jump
         def f(a, b, c):
             return ((a or b)
                     or c)
         self.check_jump_targets(f)
+        self.check_lnotab(f)
         self.assertEqual(count_instr_recursively(f, 'JUMP_IF_TRUE_OR_POP'), 2)
         # JUMP_IF_FALSE_OR_POP to JUMP_IF_TRUE_OR_POP --> POP_JUMP_IF_FALSE to non-jump
         def f(a, b, c):
             return ((a and b)
                     or c)
         self.check_jump_targets(f)
+        self.check_lnotab(f)
         self.assertNotInBytecode(f, 'JUMP_IF_FALSE_OR_POP')
         self.assertInBytecode(f, 'JUMP_IF_TRUE_OR_POP')
         self.assertInBytecode(f, 'POP_JUMP_IF_FALSE')
@@ -340,6 +386,7 @@ def f(a, b, c):
             return ((a or b)
                     and c)
         self.check_jump_targets(f)
+        self.check_lnotab(f)
         self.assertNotInBytecode(f, 'JUMP_IF_TRUE_OR_POP')
         self.assertInBytecode(f, 'JUMP_IF_FALSE_OR_POP')
         self.assertInBytecode(f, 'POP_JUMP_IF_TRUE')
@@ -360,6 +407,7 @@ def f(cond1, cond2):
         returns = [instr for instr in dis.get_instructions(f)
                           if instr.opname == 'RETURN_VALUE']
         self.assertLessEqual(len(returns), 6)
+        self.check_lnotab(f)
 
     def test_elim_jump_after_return2(self):
         # Eliminate dead code: jumps immediately after returns can't be reached
@@ -374,6 +422,7 @@ def f(cond1, cond2):
         returns = [instr for instr in dis.get_instructions(f)
                           if instr.opname == 'RETURN_VALUE']
         self.assertLessEqual(len(returns), 2)
+        self.check_lnotab(f)
 
     def test_make_function_doesnt_bail(self):
         def f():
@@ -381,6 +430,7 @@ def g()->1+1:
                 pass
             return g
         self.assertNotInBytecode(f, 'BINARY_ADD')
+        self.check_lnotab(f)
 
     def test_constant_folding(self):
         # Issue #11244: aggressive constant folding.
@@ -401,17 +451,20 @@ def test_constant_folding(self):
                 self.assertFalse(instr.opname.startswith('UNARY_'))
                 self.assertFalse(instr.opname.startswith('BINARY_'))
                 self.assertFalse(instr.opname.startswith('BUILD_'))
+            self.check_lnotab(code)
 
     def test_in_literal_list(self):
         def containtest():
             return x in [a, b]
         self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0)
+        self.check_lnotab(containtest)
 
     def test_iterate_literal_list(self):
         def forloop():
             for x in [a, b]:
                 pass
         self.assertEqual(count_instr_recursively(forloop, 'BUILD_LIST'), 0)
+        self.check_lnotab(forloop)
 
     def test_condition_with_binop_with_bools(self):
         def f():
@@ -419,6 +472,28 @@ def f():
                 return 1
             return 0
         self.assertEqual(f(), 1)
+        self.check_lnotab(f)
+
+    def test_if_with_if_expression(self):  # XXX does this belong in 3.8?
+        # Check bpo-37289
+        def f(x):
+            if (True if x else False):
+                return True
+            return False
+        self.assertTrue(f(True))
+        self.check_lnotab(f)
+
+    def test_trailing_nops(self):
+        # Check the lnotab of a function that even after trivial
+        # optimization has trailing nops, which the lnotab adjustment has to
+        # handle properly (bpo-38115).
+        def f(x):
+            while 1:
+                return 3
+            while 1:
+                return 5
+            return 6
+        self.check_lnotab(f)
 
 
 class TestBuglets(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst b/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst
new file mode 100644
index 000000000000..5119c0546e36
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-13-09-24-58.bpo-38115.BOO-Y1.rst
@@ -0,0 +1 @@
+Fix a bug in dis.findlinestarts() where it would return invalid bytecode offsets. Document that a code object's co_lnotab can contain invalid bytecode offsets.
\ No newline at end of file
diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt
index 3dab2b986616..71a297971828 100644
--- a/Objects/lnotab_notes.txt
+++ b/Objects/lnotab_notes.txt
@@ -3,7 +3,9 @@ All about co_lnotab, the line number table.
 Code objects store a field named co_lnotab.  This is an array of unsigned bytes
 disguised as a Python bytes object.  It is used to map bytecode offsets to
 source code line #s for tracebacks and to identify line number boundaries for
-line tracing.
+line tracing. Because of internals of the peephole optimizer, it's possible
+for lnotab to contain bytecode offsets that are no longer valid (for example
+if the optimizer removed the last line in a function).
 
 The array is conceptually a compressed list of
     (bytecode offset increment, line number increment)


From webhook-mailer at python.org  Sat Sep 28 12:44:20 2019
From: webhook-mailer at python.org (Ned Deily)
Date: Sat, 28 Sep 2019 16:44:20 -0000
Subject: [Python-checkins] [3.6] bpo-38216,
 bpo-36274: Allow subclasses to separately override validation and
 encoding behavior (GH-16448) (GH-16462)
Message-ID: <mailman.137.1569689061.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/5b18ce60b432d1dfa6f6988be07dd55646201a9b
commit: 5b18ce60b432d1dfa6f6988be07dd55646201a9b
branch: 3.6
author: Jason R. Coombs <jaraco at jaraco.com>
committer: Ned Deily <nad at python.org>
date: 2019-09-28T12:44:12-04:00
summary:

[3.6] bpo-38216, bpo-36274: Allow subclasses to separately override validation and encoding behavior (GH-16448) (GH-16462)

(cherry picked from commit 7774d7831e8809795c64ce27f7df52674581d298)

Co-authored-by: Jason R. Coombs <jaraco at jaraco.com>

files:
A Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst
M Lib/http/client.py
M Lib/test/test_httplib.py

diff --git a/Lib/http/client.py b/Lib/http/client.py
index 1a6bd8ac42eb..d4821f1a96e0 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -1117,19 +1117,15 @@ def putrequest(self, method, url, skip_host=False,
         else:
             raise CannotSendRequest(self.__state)
 
-        # Save the method we use, we need it later in the response phase
+        # Save the method for use later in the response phase
         self._method = method
-        if not url:
-            url = '/'
-        # Prevent CVE-2019-9740.
-        match = _contains_disallowed_url_pchar_re.search(url)
-        if match:
-            raise InvalidURL(f"URL can't contain control characters. {url!r} "
-                             f"(found at least {match.group()!r})")
+
+        url = url or '/'
+        self._validate_path(url)
+
         request = '%s %s %s' % (method, url, self._http_vsn_str)
 
-        # Non-ASCII characters should have been eliminated earlier
-        self._output(request.encode('ascii'))
+        self._output(self._encode_request(request))
 
         if self._http_vsn == 11:
             # Issue some standard headers for better HTTP/1.1 compliance
@@ -1207,6 +1203,18 @@ def putrequest(self, method, url, skip_host=False,
             # For HTTP/1.0, the server will assume "not chunked"
             pass
 
+    def _encode_request(self, request):
+        # ASCII also helps prevent CVE-2019-9740.
+        return request.encode('ascii')
+
+    def _validate_path(self, url):
+        """Validate a url for putrequest."""
+        # Prevent CVE-2019-9740.
+        match = _contains_disallowed_url_pchar_re.search(url)
+        if match:
+            raise InvalidURL(f"URL can't contain control characters. {url!r} "
+                             f"(found at least {match.group()!r})")
+
     def putheader(self, header, *values):
         """Send a request header line to the server.
 
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
index 714d521cf0e5..14d42d483773 100644
--- a/Lib/test/test_httplib.py
+++ b/Lib/test/test_httplib.py
@@ -1132,6 +1132,34 @@ def run_server():
         thread.join()
         self.assertEqual(result, b"proxied data\n")
 
+    def test_putrequest_override_validation(self):
+        """
+        It should be possible to override the default validation
+        behavior in putrequest (bpo-38216).
+        """
+        class UnsafeHTTPConnection(client.HTTPConnection):
+            def _validate_path(self, url):
+                pass
+
+        conn = UnsafeHTTPConnection('example.com')
+        conn.sock = FakeSocket('')
+        conn.putrequest('GET', '/\x00')
+
+    def test_putrequest_override_encoding(self):
+        """
+        It should be possible to override the default encoding
+        to transmit bytes in another encoding even if invalid
+        (bpo-36274).
+        """
+        class UnsafeHTTPConnection(client.HTTPConnection):
+            def _encode_request(self, str_url):
+                return str_url.encode('utf-8')
+
+        conn = UnsafeHTTPConnection('example.com')
+        conn.sock = FakeSocket('')
+        conn.putrequest('GET', '/?')
+
+
 class ExtendedReadTest(TestCase):
     """
     Test peek(), read1(), readline()
@@ -1256,6 +1284,7 @@ def test_peek_0(self):
         p = self.resp.peek(0)
         self.assertLessEqual(0, len(p))
 
+
 class ExtendedReadTestChunked(ExtendedReadTest):
     """
     Test peek(), read1(), readline() in chunked mode
diff --git a/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst
new file mode 100644
index 000000000000..ac8e2b042d92
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-27-15-24-45.bpo-38216.-7yvZR.rst
@@ -0,0 +1,4 @@
+Allow the rare code that wants to send invalid http requests from the
+`http.client` library a way to do so.  The fixes for bpo-30458 led to
+breakage for some projects that were relying on this ability to test their
+own behavior in the face of bad requests.


From webhook-mailer at python.org  Sat Sep 28 20:12:56 2019
From: webhook-mailer at python.org (Raymond Hettinger)
Date: Sun, 29 Sep 2019 00:12:56 -0000
Subject: [Python-checkins] bpo-38310: Predict BUILD_MAP_UNPACK_WITH_CALL ->
 CALL_FUNCTION_EX. (GH-16467)
Message-ID: <mailman.138.1569715977.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/f185a73249189bc3b75b4dc26e9ec2990a587b17
commit: f185a73249189bc3b75b4dc26e9ec2990a587b17
branch: master
author: Brandt Bucher <brandtbucher at gmail.com>
committer: Raymond Hettinger <rhettinger at users.noreply.github.com>
date: 2019-09-28T17:12:49-07:00
summary:

bpo-38310: Predict BUILD_MAP_UNPACK_WITH_CALL -> CALL_FUNCTION_EX. (GH-16467)

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-28-22-54-25.bpo-38310.YDTbEo.rst
M Python/ceval.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-28-22-54-25.bpo-38310.YDTbEo.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-28-22-54-25.bpo-38310.YDTbEo.rst
new file mode 100644
index 000000000000..5c3086b607d0
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-28-22-54-25.bpo-38310.YDTbEo.rst	
@@ -0,0 +1 @@
+Predict ``BUILD_MAP_UNPACK_WITH_CALL`` -> ``CALL_FUNCTION_EX`` opcode pairs in the main interpreter loop. Patch by Brandt Bucher.
\ No newline at end of file
diff --git a/Python/ceval.c b/Python/ceval.c
index 964ce1333694..a7d2ea80069a 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -2943,6 +2943,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
             while (oparg--)
                 Py_DECREF(POP());
             PUSH(sum);
+            PREDICT(CALL_FUNCTION_EX);
             DISPATCH();
         }
 
@@ -3522,6 +3523,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
         }
 
         case TARGET(CALL_FUNCTION_EX): {
+            PREDICTED(CALL_FUNCTION_EX);
             PyObject *func, *callargs, *kwargs = NULL, *result;
             if (oparg & 0x01) {
                 kwargs = POP();


From webhook-mailer at python.org  Sat Sep 28 21:42:50 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Sun, 29 Sep 2019 01:42:50 -0000
Subject: [Python-checkins] bpo-38108: Makes mock objects inherit from Base
 (GH-16060)
Message-ID: <mailman.139.1569721372.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/9a7d9519506ae807ca48ff02e2ea117ebac3450e
commit: 9a7d9519506ae807ca48ff02e2ea117ebac3450e
branch: master
author: Lisa Roach <lisaroach14 at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-28T18:42:44-07:00
summary:

bpo-38108: Makes mock objects inherit from Base (GH-16060)

files:
A Misc/NEWS.d/next/Library/2019-09-25-21-37-02.bpo-38108.Jr9HU6.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py
M Lib/unittest/test/testmock/testmagicmethods.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 0cd7af65b940..27aa3bc65c9a 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -409,7 +409,7 @@ def __new__(cls, /, *args, **kw):
             if spec_arg and _is_async_obj(spec_arg):
                 bases = (AsyncMockMixin, cls)
         new = type(cls.__name__, bases, {'__doc__': cls.__doc__})
-        instance = object.__new__(new)
+        instance = _safe_super(NonCallableMock, cls).__new__(new)
         return instance
 
 
@@ -990,17 +990,18 @@ def _get_child_mock(self, /, **kw):
 
         _type = type(self)
         if issubclass(_type, MagicMock) and _new_name in _async_method_magics:
+            # Any asynchronous magic becomes an AsyncMock
             klass = AsyncMock
-        elif _new_name  in _sync_async_magics:
-            # Special case these ones b/c users will assume they are async,
-            # but they are actually sync (ie. __aiter__)
-            klass = MagicMock
         elif issubclass(_type, AsyncMockMixin):
-            klass = AsyncMock
+            if _new_name in _all_sync_magics:
+                # Any synchronous magic becomes a MagicMock
+                klass = MagicMock
+            else:
+                klass = AsyncMock
         elif not issubclass(_type, CallableMixin):
             if issubclass(_type, NonCallableMagicMock):
                 klass = MagicMock
-            elif issubclass(_type, NonCallableMock) :
+            elif issubclass(_type, NonCallableMock):
                 klass = Mock
         else:
             klass = _type.__mro__[1]
@@ -1886,6 +1887,7 @@ def _patch_stopall():
     "round trunc floor ceil "
     "bool next "
     "fspath "
+    "aiter "
 )
 
 numerics = (
@@ -2024,7 +2026,7 @@ def _set_return_value(mock, method, name):
 
 
 
-class MagicMixin(object):
+class MagicMixin(Base):
     def __init__(self, /, *args, **kw):
         self._mock_set_magics()  # make magic work for kwargs in init
         _safe_super(MagicMixin, self).__init__(*args, **kw)
@@ -2032,13 +2034,14 @@ def __init__(self, /, *args, **kw):
 
 
     def _mock_set_magics(self):
-        these_magics = _magics
+        orig_magics = _magics | _async_method_magics
+        these_magics = orig_magics
 
         if getattr(self, "_mock_methods", None) is not None:
-            these_magics = _magics.intersection(self._mock_methods)
+            these_magics = orig_magics.intersection(self._mock_methods)
 
             remove_magics = set()
-            remove_magics = _magics - these_magics
+            remove_magics = orig_magics - these_magics
 
             for entry in remove_magics:
                 if entry in type(self).__dict__:
@@ -2066,33 +2069,13 @@ def mock_add_spec(self, spec, spec_set=False):
         self._mock_set_magics()
 
 
-class AsyncMagicMixin:
+class AsyncMagicMixin(MagicMixin):
     def __init__(self, /, *args, **kw):
-        self._mock_set_async_magics()  # make magic work for kwargs in init
+        self._mock_set_magics()  # make magic work for kwargs in init
         _safe_super(AsyncMagicMixin, self).__init__(*args, **kw)
-        self._mock_set_async_magics()  # fix magic broken by upper level init
-
-    def _mock_set_async_magics(self):
-        these_magics = _async_magics
-
-        if getattr(self, "_mock_methods", None) is not None:
-            these_magics = _async_magics.intersection(self._mock_methods)
-            remove_magics = _async_magics - these_magics
-
-            for entry in remove_magics:
-                if entry in type(self).__dict__:
-                    # remove unneeded magic methods
-                    delattr(self, entry)
-
-        # don't overwrite existing attributes if called a second time
-        these_magics = these_magics - set(type(self).__dict__)
-
-        _type = type(self)
-        for entry in these_magics:
-            setattr(_type, entry, MagicProxy(entry, self))
-
+        self._mock_set_magics()  # fix magic broken by upper level init
 
-class MagicMock(MagicMixin, AsyncMagicMixin, Mock):
+class MagicMock(MagicMixin, Mock):
     """
     MagicMock is a subclass of Mock with default implementations
     of most of the magic methods. You can use MagicMock without having to
@@ -2114,7 +2097,7 @@ def mock_add_spec(self, spec, spec_set=False):
 
 
 
-class MagicProxy(object):
+class MagicProxy(Base):
     def __init__(self, name, parent):
         self.name = name
         self.parent = parent
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index fde1e4a690b2..624fadca2649 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -379,6 +379,43 @@ def test_add_side_effect_iterable(self):
                 RuntimeError('coroutine raised StopIteration')
             )
 
+class AsyncMagicMethods(unittest.TestCase):
+    def test_async_magic_methods_return_async_mocks(self):
+        m_mock = MagicMock()
+        self.assertIsInstance(m_mock.__aenter__, AsyncMock)
+        self.assertIsInstance(m_mock.__aexit__, AsyncMock)
+        self.assertIsInstance(m_mock.__anext__, AsyncMock)
+        # __aiter__ is actually a synchronous object
+        # so should return a MagicMock
+        self.assertIsInstance(m_mock.__aiter__, MagicMock)
+
+    def test_sync_magic_methods_return_magic_mocks(self):
+        a_mock = AsyncMock()
+        self.assertIsInstance(a_mock.__enter__, MagicMock)
+        self.assertIsInstance(a_mock.__exit__, MagicMock)
+        self.assertIsInstance(a_mock.__next__, MagicMock)
+        self.assertIsInstance(a_mock.__len__, MagicMock)
+
+    def test_magicmock_has_async_magic_methods(self):
+        m_mock = MagicMock()
+        self.assertTrue(hasattr(m_mock, "__aenter__"))
+        self.assertTrue(hasattr(m_mock, "__aexit__"))
+        self.assertTrue(hasattr(m_mock, "__anext__"))
+
+    def test_asyncmock_has_sync_magic_methods(self):
+        a_mock = AsyncMock()
+        self.assertTrue(hasattr(a_mock, "__enter__"))
+        self.assertTrue(hasattr(a_mock, "__exit__"))
+        self.assertTrue(hasattr(a_mock, "__next__"))
+        self.assertTrue(hasattr(a_mock, "__len__"))
+
+    def test_magic_methods_are_async_functions(self):
+        m_mock = MagicMock()
+        self.assertIsInstance(m_mock.__aenter__, AsyncMock)
+        self.assertIsInstance(m_mock.__aexit__, AsyncMock)
+        # AsyncMocks are also coroutine functions
+        self.assertTrue(asyncio.iscoroutinefunction(m_mock.__aenter__))
+        self.assertTrue(asyncio.iscoroutinefunction(m_mock.__aexit__))
 
 class AsyncContextManagerTest(unittest.TestCase):
 
@@ -406,24 +443,6 @@ def __init__(self):
                 val = await response.json()
                 return val
 
-    def test_async_magic_methods_are_async_mocks_with_magicmock(self):
-        cm_mock = MagicMock(self.WithAsyncContextManager())
-        self.assertIsInstance(cm_mock.__aenter__, AsyncMock)
-        self.assertIsInstance(cm_mock.__aexit__, AsyncMock)
-
-    def test_magicmock_has_async_magic_methods(self):
-        cm = MagicMock(name='magic_cm')
-        self.assertTrue(hasattr(cm, "__aenter__"))
-        self.assertTrue(hasattr(cm, "__aexit__"))
-
-    def test_magic_methods_are_async_functions(self):
-        cm = MagicMock(name='magic_cm')
-        self.assertIsInstance(cm.__aenter__, AsyncMock)
-        self.assertIsInstance(cm.__aexit__, AsyncMock)
-        # AsyncMocks are also coroutine functions
-        self.assertTrue(asyncio.iscoroutinefunction(cm.__aenter__))
-        self.assertTrue(asyncio.iscoroutinefunction(cm.__aexit__))
-
     def test_set_return_value_of_aenter(self):
         def inner_test(mock_type):
             pc = self.ProductionCode()
diff --git a/Lib/unittest/test/testmock/testmagicmethods.py b/Lib/unittest/test/testmock/testmagicmethods.py
index 57f85e951e20..76b3a560de09 100644
--- a/Lib/unittest/test/testmock/testmagicmethods.py
+++ b/Lib/unittest/test/testmock/testmagicmethods.py
@@ -271,9 +271,6 @@ def test_magic_mock_equality(self):
         self.assertEqual(mock == mock, True)
         self.assertEqual(mock != mock, False)
 
-
-    # This should be fixed with issue38163
-    @unittest.expectedFailure
     def test_asyncmock_defaults(self):
         mock = AsyncMock()
         self.assertEqual(int(mock), 1)
diff --git a/Misc/NEWS.d/next/Library/2019-09-25-21-37-02.bpo-38108.Jr9HU6.rst b/Misc/NEWS.d/next/Library/2019-09-25-21-37-02.bpo-38108.Jr9HU6.rst
new file mode 100644
index 000000000000..d7eea367e482
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-25-21-37-02.bpo-38108.Jr9HU6.rst
@@ -0,0 +1,2 @@
+Any synchronous magic methods on an AsyncMock now return a MagicMock. Any
+asynchronous magic methods on a MagicMock now return an AsyncMock.


From webhook-mailer at python.org  Sun Sep 29 08:00:39 2019
From: webhook-mailer at python.org (Andrew Svetlov)
Date: Sun, 29 Sep 2019 12:00:39 -0000
Subject: [Python-checkins] bpo-38019: correctly handle pause/resume reading
 of closed asyncio unix pipe (GH-16472)
Message-ID: <mailman.140.1569758439.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/58498bc7178608b1ab031994ca09c43889ce3e76
commit: 58498bc7178608b1ab031994ca09c43889ce3e76
branch: master
author: Andrew Svetlov <andrew.svetlov at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-29T15:00:35+03:00
summary:

bpo-38019: correctly handle pause/resume reading of closed asyncio unix pipe (GH-16472)

files:
A Misc/NEWS.d/next/Library/2019-09-29-13-50-24.bpo-38019.6MoOE3.rst
M Lib/asyncio/unix_events.py
M Lib/test/test_asyncio/test_unix_events.py

diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py
index cbbb1065aeda..d8f653045aee 100644
--- a/Lib/asyncio/unix_events.py
+++ b/Lib/asyncio/unix_events.py
@@ -445,6 +445,7 @@ def __init__(self, loop, pipe, protocol, waiter=None, extra=None):
         self._fileno = pipe.fileno()
         self._protocol = protocol
         self._closing = False
+        self._paused = False
 
         mode = os.fstat(self._fileno).st_mode
         if not (stat.S_ISFIFO(mode) or
@@ -506,10 +507,20 @@ def _read_ready(self):
                 self._loop.call_soon(self._call_connection_lost, None)
 
     def pause_reading(self):
+        if self._closing or self._paused:
+            return
+        self._paused = True
         self._loop._remove_reader(self._fileno)
+        if self._loop.get_debug():
+            logger.debug("%r pauses reading", self)
 
     def resume_reading(self):
+        if self._closing or not self._paused:
+            return
+        self._paused = False
         self._loop._add_reader(self._fileno, self._read_ready)
+        if self._loop.get_debug():
+            logger.debug("%r resumes reading", self)
 
     def set_protocol(self, protocol):
         self._protocol = protocol
diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py
index 02ce7e89da09..5487b7afef83 100644
--- a/Lib/test/test_asyncio/test_unix_events.py
+++ b/Lib/test/test_asyncio/test_unix_events.py
@@ -736,6 +736,7 @@ def test_pause_reading(self, m_read):
     @mock.patch('os.read')
     def test_resume_reading(self, m_read):
         tr = self.read_pipe_transport()
+        tr.pause_reading()
         tr.resume_reading()
         self.loop.assert_reader(5, tr._read_ready)
 
@@ -790,6 +791,32 @@ def test__call_connection_lost_with_err(self):
         self.assertIsNone(tr._protocol)
         self.assertIsNone(tr._loop)
 
+    def test_pause_reading_on_closed_pipe(self):
+        tr = self.read_pipe_transport()
+        tr.close()
+        test_utils.run_briefly(self.loop)
+        self.assertIsNone(tr._loop)
+        tr.pause_reading()
+
+    def test_pause_reading_on_paused_pipe(self):
+        tr = self.read_pipe_transport()
+        tr.pause_reading()
+        # the second call should do nothing
+        tr.pause_reading()
+
+    def test_resume_reading_on_closed_pipe(self):
+        tr = self.read_pipe_transport()
+        tr.close()
+        test_utils.run_briefly(self.loop)
+        self.assertIsNone(tr._loop)
+        tr.resume_reading()
+
+    def test_resume_reading_on_paused_pipe(self):
+        tr = self.read_pipe_transport()
+        # the pipe is not paused
+        # resuming should do nothing
+        tr.resume_reading()
+
 
 class UnixWritePipeTransportTests(test_utils.TestCase):
 
diff --git a/Misc/NEWS.d/next/Library/2019-09-29-13-50-24.bpo-38019.6MoOE3.rst b/Misc/NEWS.d/next/Library/2019-09-29-13-50-24.bpo-38019.6MoOE3.rst
new file mode 100644
index 000000000000..aa5a23de2460
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-29-13-50-24.bpo-38019.6MoOE3.rst
@@ -0,0 +1 @@
+Correctly handle pause/resume reading of closed asyncio unix pipe.


From webhook-mailer at python.org  Sun Sep 29 08:19:15 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sun, 29 Sep 2019 12:19:15 -0000
Subject: [Python-checkins] bpo-38019: correctly handle pause/resume reading
 of closed asyncio unix pipe (GH-16472)
Message-ID: <mailman.141.1569759556.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/1c3e4691bb41fa070d6f4836cb03005123e53e4b
commit: 1c3e4691bb41fa070d6f4836cb03005123e53e4b
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-29T05:19:11-07:00
summary:

bpo-38019: correctly handle pause/resume reading of closed asyncio unix pipe (GH-16472)

(cherry picked from commit 58498bc7178608b1ab031994ca09c43889ce3e76)

Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-29-13-50-24.bpo-38019.6MoOE3.rst
M Lib/asyncio/unix_events.py
M Lib/test/test_asyncio/test_unix_events.py

diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py
index a0fc996d23a0..7034fc28d3e1 100644
--- a/Lib/asyncio/unix_events.py
+++ b/Lib/asyncio/unix_events.py
@@ -431,6 +431,7 @@ def __init__(self, loop, pipe, protocol, waiter=None, extra=None):
         self._fileno = pipe.fileno()
         self._protocol = protocol
         self._closing = False
+        self._paused = False
 
         mode = os.fstat(self._fileno).st_mode
         if not (stat.S_ISFIFO(mode) or
@@ -492,10 +493,20 @@ def _read_ready(self):
                 self._loop.call_soon(self._call_connection_lost, None)
 
     def pause_reading(self):
+        if self._closing or self._paused:
+            return
+        self._paused = True
         self._loop._remove_reader(self._fileno)
+        if self._loop.get_debug():
+            logger.debug("%r pauses reading", self)
 
     def resume_reading(self):
+        if self._closing or not self._paused:
+            return
+        self._paused = False
         self._loop._add_reader(self._fileno, self._read_ready)
+        if self._loop.get_debug():
+            logger.debug("%r resumes reading", self)
 
     def set_protocol(self, protocol):
         self._protocol = protocol
diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py
index ec171fa83da3..66a6bc168031 100644
--- a/Lib/test/test_asyncio/test_unix_events.py
+++ b/Lib/test/test_asyncio/test_unix_events.py
@@ -722,6 +722,7 @@ def test_pause_reading(self, m_read):
     @mock.patch('os.read')
     def test_resume_reading(self, m_read):
         tr = self.read_pipe_transport()
+        tr.pause_reading()
         tr.resume_reading()
         self.loop.assert_reader(5, tr._read_ready)
 
@@ -776,6 +777,32 @@ def test__call_connection_lost_with_err(self):
         self.assertIsNone(tr._protocol)
         self.assertIsNone(tr._loop)
 
+    def test_pause_reading_on_closed_pipe(self):
+        tr = self.read_pipe_transport()
+        tr.close()
+        test_utils.run_briefly(self.loop)
+        self.assertIsNone(tr._loop)
+        tr.pause_reading()
+
+    def test_pause_reading_on_paused_pipe(self):
+        tr = self.read_pipe_transport()
+        tr.pause_reading()
+        # the second call should do nothing
+        tr.pause_reading()
+
+    def test_resume_reading_on_closed_pipe(self):
+        tr = self.read_pipe_transport()
+        tr.close()
+        test_utils.run_briefly(self.loop)
+        self.assertIsNone(tr._loop)
+        tr.resume_reading()
+
+    def test_resume_reading_on_paused_pipe(self):
+        tr = self.read_pipe_transport()
+        # the pipe is not paused
+        # resuming should do nothing
+        tr.resume_reading()
+
 
 class UnixWritePipeTransportTests(test_utils.TestCase):
 
diff --git a/Misc/NEWS.d/next/Library/2019-09-29-13-50-24.bpo-38019.6MoOE3.rst b/Misc/NEWS.d/next/Library/2019-09-29-13-50-24.bpo-38019.6MoOE3.rst
new file mode 100644
index 000000000000..aa5a23de2460
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-29-13-50-24.bpo-38019.6MoOE3.rst
@@ -0,0 +1 @@
+Correctly handle pause/resume reading of closed asyncio unix pipe.


From webhook-mailer at python.org  Sun Sep 29 08:20:19 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sun, 29 Sep 2019 12:20:19 -0000
Subject: [Python-checkins] bpo-38019: correctly handle pause/resume reading
 of closed asyncio unix pipe (GH-16472)
Message-ID: <mailman.142.1569759620.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/19cd5951ec4480c7cfcbcc379b36902312cfb8b8
commit: 19cd5951ec4480c7cfcbcc379b36902312cfb8b8
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-29T05:20:15-07:00
summary:

bpo-38019: correctly handle pause/resume reading of closed asyncio unix pipe (GH-16472)

(cherry picked from commit 58498bc7178608b1ab031994ca09c43889ce3e76)

Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-29-13-50-24.bpo-38019.6MoOE3.rst
M Lib/asyncio/unix_events.py
M Lib/test/test_asyncio/test_unix_events.py

diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py
index cbbb1065aeda..d8f653045aee 100644
--- a/Lib/asyncio/unix_events.py
+++ b/Lib/asyncio/unix_events.py
@@ -445,6 +445,7 @@ def __init__(self, loop, pipe, protocol, waiter=None, extra=None):
         self._fileno = pipe.fileno()
         self._protocol = protocol
         self._closing = False
+        self._paused = False
 
         mode = os.fstat(self._fileno).st_mode
         if not (stat.S_ISFIFO(mode) or
@@ -506,10 +507,20 @@ def _read_ready(self):
                 self._loop.call_soon(self._call_connection_lost, None)
 
     def pause_reading(self):
+        if self._closing or self._paused:
+            return
+        self._paused = True
         self._loop._remove_reader(self._fileno)
+        if self._loop.get_debug():
+            logger.debug("%r pauses reading", self)
 
     def resume_reading(self):
+        if self._closing or not self._paused:
+            return
+        self._paused = False
         self._loop._add_reader(self._fileno, self._read_ready)
+        if self._loop.get_debug():
+            logger.debug("%r resumes reading", self)
 
     def set_protocol(self, protocol):
         self._protocol = protocol
diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py
index 02ce7e89da09..5487b7afef83 100644
--- a/Lib/test/test_asyncio/test_unix_events.py
+++ b/Lib/test/test_asyncio/test_unix_events.py
@@ -736,6 +736,7 @@ def test_pause_reading(self, m_read):
     @mock.patch('os.read')
     def test_resume_reading(self, m_read):
         tr = self.read_pipe_transport()
+        tr.pause_reading()
         tr.resume_reading()
         self.loop.assert_reader(5, tr._read_ready)
 
@@ -790,6 +791,32 @@ def test__call_connection_lost_with_err(self):
         self.assertIsNone(tr._protocol)
         self.assertIsNone(tr._loop)
 
+    def test_pause_reading_on_closed_pipe(self):
+        tr = self.read_pipe_transport()
+        tr.close()
+        test_utils.run_briefly(self.loop)
+        self.assertIsNone(tr._loop)
+        tr.pause_reading()
+
+    def test_pause_reading_on_paused_pipe(self):
+        tr = self.read_pipe_transport()
+        tr.pause_reading()
+        # the second call should do nothing
+        tr.pause_reading()
+
+    def test_resume_reading_on_closed_pipe(self):
+        tr = self.read_pipe_transport()
+        tr.close()
+        test_utils.run_briefly(self.loop)
+        self.assertIsNone(tr._loop)
+        tr.resume_reading()
+
+    def test_resume_reading_on_paused_pipe(self):
+        tr = self.read_pipe_transport()
+        # the pipe is not paused
+        # resuming should do nothing
+        tr.resume_reading()
+
 
 class UnixWritePipeTransportTests(test_utils.TestCase):
 
diff --git a/Misc/NEWS.d/next/Library/2019-09-29-13-50-24.bpo-38019.6MoOE3.rst b/Misc/NEWS.d/next/Library/2019-09-29-13-50-24.bpo-38019.6MoOE3.rst
new file mode 100644
index 000000000000..aa5a23de2460
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-29-13-50-24.bpo-38019.6MoOE3.rst
@@ -0,0 +1 @@
+Correctly handle pause/resume reading of closed asyncio unix pipe.


From webhook-mailer at python.org  Sun Sep 29 19:40:22 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Sun, 29 Sep 2019 23:40:22 -0000
Subject: [Python-checkins] bpo-38317: Fix PyConfig.warnoptions priority
 (GH-16478)
Message-ID: <mailman.143.1569800422.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/fb4ae152a9930f0e00cae8b2807f534058cf341a
commit: fb4ae152a9930f0e00cae8b2807f534058cf341a
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-30T01:40:17+02:00
summary:

bpo-38317: Fix PyConfig.warnoptions priority (GH-16478)

Fix warnings options priority: PyConfig.warnoptions has the highest
priority, as stated in the PEP 587.

* Document options order in PyConfig.warnoptions documentation.
* Make PyWideStringList_INIT macro private: replace "Py" prefix
  with "_Py".
* test_embed: add test_init_warnoptions().

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-30-00-56-21.bpo-38317.pmqlIQ.rst
M Doc/c-api/init_config.rst
M Include/cpython/initconfig.h
M Include/internal/pycore_initconfig.h
M Include/internal/pycore_pylifecycle.h
M Lib/test/test_embed.py
M Programs/_testembed.c
M Python/initconfig.c
M Python/preconfig.c
M Python/sysmodule.c

diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index 58e417441d13..4c77d5d1e0e8 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -704,7 +704,13 @@ PyConfig
 
    .. c:member:: PyWideStringList warnoptions
 
-      Options of the :mod:`warnings` module to build warnings filters.
+      :data:`sys.warnoptions`: options of the :mod:`warnings` module to build
+      warnings filters: lowest to highest priority.
+
+      The :mod:`warnings` module adds :data:`sys.warnoptions` in the reverse
+      order: the last :c:member:`PyConfig.warnoptions` item becomes the first
+      item of :data:`warnings.filters` which is checked first (highest
+      priority).
 
    .. c:member:: int write_bytecode
 
diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
index d5effb8397fc..8ce5622c5c40 100644
--- a/Include/cpython/initconfig.h
+++ b/Include/cpython/initconfig.h
@@ -220,7 +220,10 @@ typedef struct {
     wchar_t *program_name;
 
     PyWideStringList xoptions;     /* Command line -X options */
-    PyWideStringList warnoptions;  /* Warnings options */
+
+    /* Warnings options: lowest to highest priority. warnings.filters
+       is built in the reverse order (highest to lowest priority). */
+    PyWideStringList warnoptions;
 
     /* If equal to zero, disable the import of the module site and the
        site-dependent manipulations of sys.path that it entails. Also disable
diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h
index dcdb61695cab..eb6490f5a87f 100644
--- a/Include/internal/pycore_initconfig.h
+++ b/Include/internal/pycore_initconfig.h
@@ -45,7 +45,7 @@ extern "C" {
 
 /* --- PyWideStringList ------------------------------------------------ */
 
-#define PyWideStringList_INIT (PyWideStringList){.length = 0, .items = NULL}
+#define _PyWideStringList_INIT (PyWideStringList){.length = 0, .items = NULL}
 
 #ifndef NDEBUG
 PyAPI_FUNC(int) _PyWideStringList_CheckConsistency(const PyWideStringList *list);
diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index cdf5c09a4262..b0a98f5d2597 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -41,7 +41,7 @@ extern PyStatus _PySys_Create(
     PyThreadState *tstate,
     PyObject **sysmod_p);
 extern PyStatus _PySys_SetPreliminaryStderr(PyObject *sysdict);
-extern PyStatus _PySys_ReadPreinitWarnOptions(PyConfig *config);
+extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options);
 extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config);
 extern int _PySys_InitMain(
     _PyRuntimeState *runtime,
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index ab086e2332da..b87863a372a6 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -746,9 +746,9 @@ def test_init_from_config(self):
                 'cmdline_xoption',
             ],
             'warnoptions': [
-                'config_warnoption',
                 'cmdline_warnoption',
                 'default::BytesWarning',
+                'config_warnoption',
             ],
             'run_command': 'pass\n',
 
@@ -952,9 +952,9 @@ def test_init_sys_add(self):
                 'faulthandler',
             ],
             'warnoptions': [
-                'ignore:::config_warnoption',
                 'ignore:::cmdline_warnoption',
                 'ignore:::sysadd_warnoption',
+                'ignore:::config_warnoption',
             ],
         }
         self.check_all_configs("test_init_sys_add", config, api=API_PYTHON)
@@ -1268,6 +1268,30 @@ def get_func(name):
         self.assertEqual(Py_GetProgramFullPath(), config['executable'])
         self.assertEqual(Py_GetPythonHome(), config['home'])
 
+    def test_init_warnoptions(self):
+        # lowest to highest priority
+        warnoptions = [
+            'ignore:::PyConfig_Insert0',      # PyWideStringList_Insert(0)
+            'default',                        # PyConfig.dev_mode=1
+            'ignore:::env1',                  # PYTHONWARNINGS env var
+            'ignore:::env2',                  # PYTHONWARNINGS env var
+            'ignore:::cmdline1',              # -W opt command line option
+            'ignore:::cmdline2',              # -W opt command line option
+            'default::BytesWarning',          # PyConfig.bytes_warnings=1
+            'ignore:::PySys_AddWarnOption1',  # PySys_AddWarnOption()
+            'ignore:::PySys_AddWarnOption2',  # PySys_AddWarnOption()
+            'ignore:::PyConfig_BeforeRead',   # PyConfig.warnoptions
+            'ignore:::PyConfig_AfterRead']    # PyWideStringList_Append()
+        preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG)
+        config = {
+            'dev_mode': 1,
+            'faulthandler': 1,
+            'bytes_warning': 1,
+            'warnoptions': warnoptions,
+        }
+        self.check_all_configs("test_init_warnoptions", config, preconfig,
+                               api=API_PYTHON)
+
 
 class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
     def test_open_code_hook(self):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-30-00-56-21.bpo-38317.pmqlIQ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-30-00-56-21.bpo-38317.pmqlIQ.rst
new file mode 100644
index 000000000000..b6d07474cf52
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-30-00-56-21.bpo-38317.pmqlIQ.rst	
@@ -0,0 +1,2 @@
+Fix warnings options priority: ``PyConfig.warnoptions`` has the highest
+priority, as stated in the :pep:`587`.
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index a37594524f3d..c8600d58f0b4 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -1603,6 +1603,64 @@ static int test_init_setpythonhome(void)
 }
 
 
+static int test_init_warnoptions(void)
+{
+    PyStatus status;
+    putenv("PYTHONWARNINGS=ignore:::env1,ignore:::env2");
+
+    PySys_AddWarnOption(L"ignore:::PySys_AddWarnOption1");
+    PySys_AddWarnOption(L"ignore:::PySys_AddWarnOption2");
+
+    PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
+    status = PyConfig_InitPythonConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    config.dev_mode = 1;
+    config.bytes_warning = 1;
+
+    config_set_program_name(&config);
+
+    status = PyWideStringList_Append(&config.warnoptions,
+                                     L"ignore:::PyConfig_BeforeRead");
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    wchar_t* argv[] = {
+        L"python3",
+        L"-Wignore:::cmdline1",
+        L"-Wignore:::cmdline2"};
+    config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv);
+    config.parse_argv = 1;
+
+    status = PyConfig_Read(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    status = PyWideStringList_Append(&config.warnoptions,
+                                     L"ignore:::PyConfig_AfterRead");
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    status = PyWideStringList_Insert(&config.warnoptions,
+                                     0, L"ignore:::PyConfig_Insert0");
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    init_from_config_clear(&config);
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
 static void configure_init_main(PyConfig *config)
 {
     wchar_t* argv[] = {
@@ -1746,6 +1804,7 @@ static struct TestCase TestCases[] = {
     {"test_init_setpath", test_init_setpath},
     {"test_init_setpath_config", test_init_setpath_config},
     {"test_init_setpythonhome", test_init_setpythonhome},
+    {"test_init_warnoptions", test_init_warnoptions},
     {"test_run_main", test_run_main},
 
     {"test_open_code_hook", test_open_code_hook},
diff --git a/Python/initconfig.c b/Python/initconfig.c
index 62c868d5cbc9..dec4bf2e4d57 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -273,7 +273,7 @@ _PyWideStringList_Copy(PyWideStringList *list, const PyWideStringList *list2)
         return 0;
     }
 
-    PyWideStringList copy = PyWideStringList_INIT;
+    PyWideStringList copy = _PyWideStringList_INIT;
 
     size_t size = list2->length * sizeof(list2->items[0]);
     copy.items = PyMem_RawMalloc(size);
@@ -2095,63 +2095,83 @@ config_init_env_warnoptions(PyConfig *config, PyWideStringList *warnoptions)
 
 
 static PyStatus
-config_add_warnoption(PyConfig *config, const wchar_t *option)
+warnoptions_append(PyConfig *config, PyWideStringList *options,
+                   const wchar_t *option)
 {
+    /* config_init_warnoptions() add existing config warnoptions at the end:
+       ensure that the new option is not already present in this list to
+       prevent change the options order whne config_init_warnoptions() is
+       called twice. */
     if (_PyWideStringList_Find(&config->warnoptions, option)) {
         /* Already present: do nothing */
         return _PyStatus_OK();
     }
-    return PyWideStringList_Append(&config->warnoptions, option);
+    if (_PyWideStringList_Find(options, option)) {
+        /* Already present: do nothing */
+        return _PyStatus_OK();
+    }
+    return PyWideStringList_Append(options, option);
+}
+
+
+static PyStatus
+warnoptions_extend(PyConfig *config, PyWideStringList *options,
+                   const PyWideStringList *options2)
+{
+    const Py_ssize_t len = options2->length;
+    wchar_t *const *items = options2->items;
+
+    for (Py_ssize_t i = 0; i < len; i++) {
+        PyStatus status = warnoptions_append(config, options, items[i]);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
+    }
+    return _PyStatus_OK();
 }
 
 
 static PyStatus
 config_init_warnoptions(PyConfig *config,
                         const PyWideStringList *cmdline_warnoptions,
-                        const PyWideStringList *env_warnoptions)
+                        const PyWideStringList *env_warnoptions,
+                        const PyWideStringList *sys_warnoptions)
 {
     PyStatus status;
+    PyWideStringList options = _PyWideStringList_INIT;
 
-    /* The priority order for warnings configuration is (highest precedence
-     * first):
+    /* Priority of warnings options, lowest to highest:
      *
-     * - early PySys_AddWarnOption() calls
-     * - the BytesWarning filter, if needed ('-b', '-bb')
-     * - any '-W' command line options; then
-     * - the 'PYTHONWARNINGS' environment variable; then
-     * - the dev mode filter ('-X dev', 'PYTHONDEVMODE'); then
      * - any implicit filters added by _warnings.c/warnings.py
+     * - PyConfig.dev_mode: "default" filter
+     * - PYTHONWARNINGS environment variable
+     * - '-W' command line options
+     * - PyConfig.bytes_warning ('-b' and '-bb' command line options):
+     *   "default::BytesWarning" or "error::BytesWarning" filter
+     * - early PySys_AddWarnOption() calls
+     * - PyConfig.warnoptions
      *
-     * All settings except the last are passed to the warnings module via
-     * the `sys.warnoptions` list. Since the warnings module works on the basis
-     * of "the most recently added filter will be checked first", we add
-     * the lowest precedence entries first so that later entries override them.
+     * PyConfig.warnoptions is copied to sys.warnoptions. Since the warnings
+     * module works on the basis of "the most recently added filter will be
+     * checked first", we add the lowest precedence entries first so that later
+     * entries override them.
      */
 
     if (config->dev_mode) {
-        status = config_add_warnoption(config, L"default");
+        status = warnoptions_append(config, &options, L"default");
         if (_PyStatus_EXCEPTION(status)) {
-            return status;
+            goto error;
         }
     }
 
-    Py_ssize_t i;
-    const PyWideStringList *options;
-
-    options = env_warnoptions;
-    for (i = 0; i < options->length; i++) {
-        status = config_add_warnoption(config, options->items[i]);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
+    status = warnoptions_extend(config, &options, env_warnoptions);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto error;
     }
 
-    options = cmdline_warnoptions;
-    for (i = 0; i < options->length; i++) {
-        status = config_add_warnoption(config, options->items[i]);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
+    status = warnoptions_extend(config, &options, cmdline_warnoptions);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto error;
     }
 
     /* If the bytes_warning_flag isn't set, bytesobject.c and bytearrayobject.c
@@ -2166,19 +2186,30 @@ config_init_warnoptions(PyConfig *config,
         else {
             filter = L"default::BytesWarning";
         }
-        status = config_add_warnoption(config, filter);
+        status = warnoptions_append(config, &options, filter);
         if (_PyStatus_EXCEPTION(status)) {
-            return status;
+            goto error;
         }
     }
 
-    /* Handle early PySys_AddWarnOption() calls */
-    status = _PySys_ReadPreinitWarnOptions(config);
+    status = warnoptions_extend(config, &options, sys_warnoptions);
     if (_PyStatus_EXCEPTION(status)) {
-        return status;
+        goto error;
+    }
+
+    /* Always add all PyConfig.warnoptions options */
+    status = _PyWideStringList_Extend(&options, &config->warnoptions);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto error;
     }
 
+    _PyWideStringList_Clear(&config->warnoptions);
+    config->warnoptions = options;
     return _PyStatus_OK();
+
+error:
+    _PyWideStringList_Clear(&options);
+    return status;
 }
 
 
@@ -2186,7 +2217,7 @@ static PyStatus
 config_update_argv(PyConfig *config, Py_ssize_t opt_index)
 {
     const PyWideStringList *cmdline_argv = &config->argv;
-    PyWideStringList config_argv = PyWideStringList_INIT;
+    PyWideStringList config_argv = _PyWideStringList_INIT;
 
     /* Copy argv to be able to modify it (to force -c/-m) */
     if (cmdline_argv->length <= opt_index) {
@@ -2306,8 +2337,9 @@ static PyStatus
 config_read_cmdline(PyConfig *config)
 {
     PyStatus status;
-    PyWideStringList cmdline_warnoptions = PyWideStringList_INIT;
-    PyWideStringList env_warnoptions = PyWideStringList_INIT;
+    PyWideStringList cmdline_warnoptions = _PyWideStringList_INIT;
+    PyWideStringList env_warnoptions = _PyWideStringList_INIT;
+    PyWideStringList sys_warnoptions = _PyWideStringList_INIT;
 
     if (config->parse_argv < 0) {
         config->parse_argv = 1;
@@ -2351,9 +2383,16 @@ config_read_cmdline(PyConfig *config)
         }
     }
 
+    /* Handle early PySys_AddWarnOption() calls */
+    status = _PySys_ReadPreinitWarnOptions(&sys_warnoptions);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto done;
+    }
+
     status = config_init_warnoptions(config,
                                      &cmdline_warnoptions,
-                                     &env_warnoptions);
+                                     &env_warnoptions,
+                                     &sys_warnoptions);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
     }
@@ -2363,6 +2402,7 @@ config_read_cmdline(PyConfig *config)
 done:
     _PyWideStringList_Clear(&cmdline_warnoptions);
     _PyWideStringList_Clear(&env_warnoptions);
+    _PyWideStringList_Clear(&sys_warnoptions);
     return status;
 }
 
@@ -2433,7 +2473,7 @@ PyStatus
 PyConfig_Read(PyConfig *config)
 {
     PyStatus status;
-    PyWideStringList orig_argv = PyWideStringList_INIT;
+    PyWideStringList orig_argv = _PyWideStringList_INIT;
 
     status = config_check_struct_size(config);
     if (_PyStatus_EXCEPTION(status)) {
diff --git a/Python/preconfig.c b/Python/preconfig.c
index d18b01dc4586..01c72f5d6bad 100644
--- a/Python/preconfig.c
+++ b/Python/preconfig.c
@@ -75,7 +75,7 @@ _Py_SetFileSystemEncoding(const char *encoding, const char *errors)
 PyStatus
 _PyArgv_AsWstrList(const _PyArgv *args, PyWideStringList *list)
 {
-    PyWideStringList wargv = PyWideStringList_INIT;
+    PyWideStringList wargv = _PyWideStringList_INIT;
     if (args->use_bytes_argv) {
         size_t size = sizeof(wchar_t*) * args->argc;
         wargv.items = (wchar_t **)PyMem_RawMalloc(size);
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 9cc7b4b27c15..92935408a58e 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -2037,13 +2037,13 @@ _clear_preinit_entries(_Py_PreInitEntry *optionlist)
 
 
 PyStatus
-_PySys_ReadPreinitWarnOptions(PyConfig *config)
+_PySys_ReadPreinitWarnOptions(PyWideStringList *options)
 {
     PyStatus status;
     _Py_PreInitEntry entry;
 
     for (entry = _preinit_warnoptions; entry != NULL; entry = entry->next) {
-        status = PyWideStringList_Append(&config->warnoptions, entry->value);
+        status = PyWideStringList_Append(options, entry->value);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }


From webhook-mailer at python.org  Sun Sep 29 19:59:01 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Sun, 29 Sep 2019 23:59:01 -0000
Subject: [Python-checkins] bpo-38317: Fix PyConfig.warnoptions priority
 (GH-16478)
Message-ID: <mailman.144.1569801542.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/c9ed9e6fc76323ed537fb79d4232bcd27d82c57e
commit: c9ed9e6fc76323ed537fb79d4232bcd27d82c57e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2019-09-29T16:58:57-07:00
summary:

bpo-38317: Fix PyConfig.warnoptions priority (GH-16478)


Fix warnings options priority: PyConfig.warnoptions has the highest
priority, as stated in the PEP 587.

* Document options order in PyConfig.warnoptions documentation.
* Make PyWideStringList_INIT macro private: replace "Py" prefix
  with "_Py".
* test_embed: add test_init_warnoptions().
(cherry picked from commit fb4ae152a9930f0e00cae8b2807f534058cf341a)

Co-authored-by: Victor Stinner <vstinner at redhat.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-30-00-56-21.bpo-38317.pmqlIQ.rst
M Doc/c-api/init_config.rst
M Include/cpython/initconfig.h
M Include/internal/pycore_initconfig.h
M Include/internal/pycore_pylifecycle.h
M Lib/test/test_embed.py
M Programs/_testembed.c
M Python/initconfig.c
M Python/preconfig.c
M Python/sysmodule.c

diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index 58e417441d13..4c77d5d1e0e8 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -704,7 +704,13 @@ PyConfig
 
    .. c:member:: PyWideStringList warnoptions
 
-      Options of the :mod:`warnings` module to build warnings filters.
+      :data:`sys.warnoptions`: options of the :mod:`warnings` module to build
+      warnings filters: lowest to highest priority.
+
+      The :mod:`warnings` module adds :data:`sys.warnoptions` in the reverse
+      order: the last :c:member:`PyConfig.warnoptions` item becomes the first
+      item of :data:`warnings.filters` which is checked first (highest
+      priority).
 
    .. c:member:: int write_bytecode
 
diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
index d5effb8397fc..8ce5622c5c40 100644
--- a/Include/cpython/initconfig.h
+++ b/Include/cpython/initconfig.h
@@ -220,7 +220,10 @@ typedef struct {
     wchar_t *program_name;
 
     PyWideStringList xoptions;     /* Command line -X options */
-    PyWideStringList warnoptions;  /* Warnings options */
+
+    /* Warnings options: lowest to highest priority. warnings.filters
+       is built in the reverse order (highest to lowest priority). */
+    PyWideStringList warnoptions;
 
     /* If equal to zero, disable the import of the module site and the
        site-dependent manipulations of sys.path that it entails. Also disable
diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h
index dcdb61695cab..eb6490f5a87f 100644
--- a/Include/internal/pycore_initconfig.h
+++ b/Include/internal/pycore_initconfig.h
@@ -45,7 +45,7 @@ extern "C" {
 
 /* --- PyWideStringList ------------------------------------------------ */
 
-#define PyWideStringList_INIT (PyWideStringList){.length = 0, .items = NULL}
+#define _PyWideStringList_INIT (PyWideStringList){.length = 0, .items = NULL}
 
 #ifndef NDEBUG
 PyAPI_FUNC(int) _PyWideStringList_CheckConsistency(const PyWideStringList *list);
diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index cb568d28fb99..d4f0ae2da72e 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -41,7 +41,7 @@ extern PyStatus _PySys_Create(
     PyInterpreterState *interp,
     PyObject **sysmod_p);
 extern PyStatus _PySys_SetPreliminaryStderr(PyObject *sysdict);
-extern PyStatus _PySys_ReadPreinitWarnOptions(PyConfig *config);
+extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options);
 extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config);
 extern int _PySys_InitMain(
     _PyRuntimeState *runtime,
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index ed2b96fbfc1f..da79d7af0502 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -746,9 +746,9 @@ def test_init_from_config(self):
                 'cmdline_xoption',
             ],
             'warnoptions': [
-                'config_warnoption',
                 'cmdline_warnoption',
                 'default::BytesWarning',
+                'config_warnoption',
             ],
             'run_command': 'pass\n',
 
@@ -951,9 +951,9 @@ def test_init_sys_add(self):
                 'faulthandler',
             ],
             'warnoptions': [
-                'ignore:::config_warnoption',
                 'ignore:::cmdline_warnoption',
                 'ignore:::sysadd_warnoption',
+                'ignore:::config_warnoption',
             ],
         }
         self.check_all_configs("test_init_sys_add", config, api=API_PYTHON)
@@ -1267,6 +1267,30 @@ def get_func(name):
         self.assertEqual(Py_GetProgramFullPath(), config['executable'])
         self.assertEqual(Py_GetPythonHome(), config['home'])
 
+    def test_init_warnoptions(self):
+        # lowest to highest priority
+        warnoptions = [
+            'ignore:::PyConfig_Insert0',      # PyWideStringList_Insert(0)
+            'default',                        # PyConfig.dev_mode=1
+            'ignore:::env1',                  # PYTHONWARNINGS env var
+            'ignore:::env2',                  # PYTHONWARNINGS env var
+            'ignore:::cmdline1',              # -W opt command line option
+            'ignore:::cmdline2',              # -W opt command line option
+            'default::BytesWarning',          # PyConfig.bytes_warnings=1
+            'ignore:::PySys_AddWarnOption1',  # PySys_AddWarnOption()
+            'ignore:::PySys_AddWarnOption2',  # PySys_AddWarnOption()
+            'ignore:::PyConfig_BeforeRead',   # PyConfig.warnoptions
+            'ignore:::PyConfig_AfterRead']    # PyWideStringList_Append()
+        preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG)
+        config = {
+            'dev_mode': 1,
+            'faulthandler': 1,
+            'bytes_warning': 1,
+            'warnoptions': warnoptions,
+        }
+        self.check_all_configs("test_init_warnoptions", config, preconfig,
+                               api=API_PYTHON)
+
 
 class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
     def test_open_code_hook(self):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-30-00-56-21.bpo-38317.pmqlIQ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-30-00-56-21.bpo-38317.pmqlIQ.rst
new file mode 100644
index 000000000000..b6d07474cf52
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-30-00-56-21.bpo-38317.pmqlIQ.rst	
@@ -0,0 +1,2 @@
+Fix warnings options priority: ``PyConfig.warnoptions`` has the highest
+priority, as stated in the :pep:`587`.
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index a37594524f3d..c8600d58f0b4 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -1603,6 +1603,64 @@ static int test_init_setpythonhome(void)
 }
 
 
+static int test_init_warnoptions(void)
+{
+    PyStatus status;
+    putenv("PYTHONWARNINGS=ignore:::env1,ignore:::env2");
+
+    PySys_AddWarnOption(L"ignore:::PySys_AddWarnOption1");
+    PySys_AddWarnOption(L"ignore:::PySys_AddWarnOption2");
+
+    PyConfig config;
+    config.struct_size = sizeof(PyConfig);
+
+    status = PyConfig_InitPythonConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    config.dev_mode = 1;
+    config.bytes_warning = 1;
+
+    config_set_program_name(&config);
+
+    status = PyWideStringList_Append(&config.warnoptions,
+                                     L"ignore:::PyConfig_BeforeRead");
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    wchar_t* argv[] = {
+        L"python3",
+        L"-Wignore:::cmdline1",
+        L"-Wignore:::cmdline2"};
+    config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv);
+    config.parse_argv = 1;
+
+    status = PyConfig_Read(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    status = PyWideStringList_Append(&config.warnoptions,
+                                     L"ignore:::PyConfig_AfterRead");
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    status = PyWideStringList_Insert(&config.warnoptions,
+                                     0, L"ignore:::PyConfig_Insert0");
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    init_from_config_clear(&config);
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
 static void configure_init_main(PyConfig *config)
 {
     wchar_t* argv[] = {
@@ -1746,6 +1804,7 @@ static struct TestCase TestCases[] = {
     {"test_init_setpath", test_init_setpath},
     {"test_init_setpath_config", test_init_setpath_config},
     {"test_init_setpythonhome", test_init_setpythonhome},
+    {"test_init_warnoptions", test_init_warnoptions},
     {"test_run_main", test_run_main},
 
     {"test_open_code_hook", test_open_code_hook},
diff --git a/Python/initconfig.c b/Python/initconfig.c
index dbc9de55a582..fb04b4a2d22b 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -273,7 +273,7 @@ _PyWideStringList_Copy(PyWideStringList *list, const PyWideStringList *list2)
         return 0;
     }
 
-    PyWideStringList copy = PyWideStringList_INIT;
+    PyWideStringList copy = _PyWideStringList_INIT;
 
     size_t size = list2->length * sizeof(list2->items[0]);
     copy.items = PyMem_RawMalloc(size);
@@ -2095,63 +2095,83 @@ config_init_env_warnoptions(PyConfig *config, PyWideStringList *warnoptions)
 
 
 static PyStatus
-config_add_warnoption(PyConfig *config, const wchar_t *option)
+warnoptions_append(PyConfig *config, PyWideStringList *options,
+                   const wchar_t *option)
 {
+    /* config_init_warnoptions() add existing config warnoptions at the end:
+       ensure that the new option is not already present in this list to
+       prevent change the options order whne config_init_warnoptions() is
+       called twice. */
     if (_PyWideStringList_Find(&config->warnoptions, option)) {
         /* Already present: do nothing */
         return _PyStatus_OK();
     }
-    return PyWideStringList_Append(&config->warnoptions, option);
+    if (_PyWideStringList_Find(options, option)) {
+        /* Already present: do nothing */
+        return _PyStatus_OK();
+    }
+    return PyWideStringList_Append(options, option);
+}
+
+
+static PyStatus
+warnoptions_extend(PyConfig *config, PyWideStringList *options,
+                   const PyWideStringList *options2)
+{
+    const Py_ssize_t len = options2->length;
+    wchar_t *const *items = options2->items;
+
+    for (Py_ssize_t i = 0; i < len; i++) {
+        PyStatus status = warnoptions_append(config, options, items[i]);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
+    }
+    return _PyStatus_OK();
 }
 
 
 static PyStatus
 config_init_warnoptions(PyConfig *config,
                         const PyWideStringList *cmdline_warnoptions,
-                        const PyWideStringList *env_warnoptions)
+                        const PyWideStringList *env_warnoptions,
+                        const PyWideStringList *sys_warnoptions)
 {
     PyStatus status;
+    PyWideStringList options = _PyWideStringList_INIT;
 
-    /* The priority order for warnings configuration is (highest precedence
-     * first):
+    /* Priority of warnings options, lowest to highest:
      *
-     * - early PySys_AddWarnOption() calls
-     * - the BytesWarning filter, if needed ('-b', '-bb')
-     * - any '-W' command line options; then
-     * - the 'PYTHONWARNINGS' environment variable; then
-     * - the dev mode filter ('-X dev', 'PYTHONDEVMODE'); then
      * - any implicit filters added by _warnings.c/warnings.py
+     * - PyConfig.dev_mode: "default" filter
+     * - PYTHONWARNINGS environment variable
+     * - '-W' command line options
+     * - PyConfig.bytes_warning ('-b' and '-bb' command line options):
+     *   "default::BytesWarning" or "error::BytesWarning" filter
+     * - early PySys_AddWarnOption() calls
+     * - PyConfig.warnoptions
      *
-     * All settings except the last are passed to the warnings module via
-     * the `sys.warnoptions` list. Since the warnings module works on the basis
-     * of "the most recently added filter will be checked first", we add
-     * the lowest precedence entries first so that later entries override them.
+     * PyConfig.warnoptions is copied to sys.warnoptions. Since the warnings
+     * module works on the basis of "the most recently added filter will be
+     * checked first", we add the lowest precedence entries first so that later
+     * entries override them.
      */
 
     if (config->dev_mode) {
-        status = config_add_warnoption(config, L"default");
+        status = warnoptions_append(config, &options, L"default");
         if (_PyStatus_EXCEPTION(status)) {
-            return status;
+            goto error;
         }
     }
 
-    Py_ssize_t i;
-    const PyWideStringList *options;
-
-    options = env_warnoptions;
-    for (i = 0; i < options->length; i++) {
-        status = config_add_warnoption(config, options->items[i]);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
+    status = warnoptions_extend(config, &options, env_warnoptions);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto error;
     }
 
-    options = cmdline_warnoptions;
-    for (i = 0; i < options->length; i++) {
-        status = config_add_warnoption(config, options->items[i]);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
+    status = warnoptions_extend(config, &options, cmdline_warnoptions);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto error;
     }
 
     /* If the bytes_warning_flag isn't set, bytesobject.c and bytearrayobject.c
@@ -2166,19 +2186,30 @@ config_init_warnoptions(PyConfig *config,
         else {
             filter = L"default::BytesWarning";
         }
-        status = config_add_warnoption(config, filter);
+        status = warnoptions_append(config, &options, filter);
         if (_PyStatus_EXCEPTION(status)) {
-            return status;
+            goto error;
         }
     }
 
-    /* Handle early PySys_AddWarnOption() calls */
-    status = _PySys_ReadPreinitWarnOptions(config);
+    status = warnoptions_extend(config, &options, sys_warnoptions);
     if (_PyStatus_EXCEPTION(status)) {
-        return status;
+        goto error;
+    }
+
+    /* Always add all PyConfig.warnoptions options */
+    status = _PyWideStringList_Extend(&options, &config->warnoptions);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto error;
     }
 
+    _PyWideStringList_Clear(&config->warnoptions);
+    config->warnoptions = options;
     return _PyStatus_OK();
+
+error:
+    _PyWideStringList_Clear(&options);
+    return status;
 }
 
 
@@ -2186,7 +2217,7 @@ static PyStatus
 config_update_argv(PyConfig *config, Py_ssize_t opt_index)
 {
     const PyWideStringList *cmdline_argv = &config->argv;
-    PyWideStringList config_argv = PyWideStringList_INIT;
+    PyWideStringList config_argv = _PyWideStringList_INIT;
 
     /* Copy argv to be able to modify it (to force -c/-m) */
     if (cmdline_argv->length <= opt_index) {
@@ -2270,8 +2301,9 @@ static PyStatus
 config_read_cmdline(PyConfig *config)
 {
     PyStatus status;
-    PyWideStringList cmdline_warnoptions = PyWideStringList_INIT;
-    PyWideStringList env_warnoptions = PyWideStringList_INIT;
+    PyWideStringList cmdline_warnoptions = _PyWideStringList_INIT;
+    PyWideStringList env_warnoptions = _PyWideStringList_INIT;
+    PyWideStringList sys_warnoptions = _PyWideStringList_INIT;
 
     if (config->parse_argv < 0) {
         config->parse_argv = 1;
@@ -2304,9 +2336,16 @@ config_read_cmdline(PyConfig *config)
         }
     }
 
+    /* Handle early PySys_AddWarnOption() calls */
+    status = _PySys_ReadPreinitWarnOptions(&sys_warnoptions);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto done;
+    }
+
     status = config_init_warnoptions(config,
                                      &cmdline_warnoptions,
-                                     &env_warnoptions);
+                                     &env_warnoptions,
+                                     &sys_warnoptions);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
     }
@@ -2316,6 +2355,7 @@ config_read_cmdline(PyConfig *config)
 done:
     _PyWideStringList_Clear(&cmdline_warnoptions);
     _PyWideStringList_Clear(&env_warnoptions);
+    _PyWideStringList_Clear(&sys_warnoptions);
     return status;
 }
 
@@ -2386,7 +2426,7 @@ PyStatus
 PyConfig_Read(PyConfig *config)
 {
     PyStatus status;
-    PyWideStringList orig_argv = PyWideStringList_INIT;
+    PyWideStringList orig_argv = _PyWideStringList_INIT;
 
     status = config_check_struct_size(config);
     if (_PyStatus_EXCEPTION(status)) {
diff --git a/Python/preconfig.c b/Python/preconfig.c
index d18b01dc4586..01c72f5d6bad 100644
--- a/Python/preconfig.c
+++ b/Python/preconfig.c
@@ -75,7 +75,7 @@ _Py_SetFileSystemEncoding(const char *encoding, const char *errors)
 PyStatus
 _PyArgv_AsWstrList(const _PyArgv *args, PyWideStringList *list)
 {
-    PyWideStringList wargv = PyWideStringList_INIT;
+    PyWideStringList wargv = _PyWideStringList_INIT;
     if (args->use_bytes_argv) {
         size_t size = sizeof(wchar_t*) * args->argc;
         wargv.items = (wchar_t **)PyMem_RawMalloc(size);
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 5175158a3b00..5b0fb813b456 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -2071,13 +2071,13 @@ _clear_preinit_entries(_Py_PreInitEntry *optionlist)
 
 
 PyStatus
-_PySys_ReadPreinitWarnOptions(PyConfig *config)
+_PySys_ReadPreinitWarnOptions(PyWideStringList *options)
 {
     PyStatus status;
     _Py_PreInitEntry entry;
 
     for (entry = _preinit_warnoptions; entry != NULL; entry = entry->next) {
-        status = PyWideStringList_Append(&config->warnoptions, entry->value);
+        status = PyWideStringList_Append(options, entry->value);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }


From webhook-mailer at python.org  Mon Sep 30 00:01:34 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Mon, 30 Sep 2019 04:01:34 -0000
Subject: [Python-checkins] bpo-38161: Removes _AwaitEvent from AsyncMock.
 (GH-16443)
Message-ID: <mailman.145.1569816095.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/25e115ec00b5f75e3589c9f21013c47c21e1753f
commit: 25e115ec00b5f75e3589c9f21013c47c21e1753f
branch: master
author: Lisa Roach <lisaroach14 at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-29T21:01:28-07:00
summary:

bpo-38161: Removes _AwaitEvent from AsyncMock. (GH-16443)

files:
A Misc/NEWS.d/next/Library/2019-09-27-16-31-28.bpo-38161.zehai1.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 27aa3bc65c9a..e797d2f5b5e1 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -246,7 +246,6 @@ def _setup_async_mock(mock):
     mock.await_count = 0
     mock.await_args = None
     mock.await_args_list = _CallList()
-    mock.awaited = _AwaitEvent(mock)
 
     # Mock is not configured yet so the attributes are set
     # to a function and then the corresponding mock helper function
@@ -2116,7 +2115,6 @@ def __get__(self, obj, _type=None):
 
 
 class AsyncMockMixin(Base):
-    awaited = _delegating_property('awaited')
     await_count = _delegating_property('await_count')
     await_args = _delegating_property('await_args')
     await_args_list = _delegating_property('await_args_list')
@@ -2130,7 +2128,6 @@ def __init__(self, /, *args, **kwargs):
         # It is set through __dict__ because when spec_set is True, this
         # attribute is likely undefined.
         self.__dict__['_is_coroutine'] = asyncio.coroutines._is_coroutine
-        self.__dict__['_mock_awaited'] = _AwaitEvent(self)
         self.__dict__['_mock_await_count'] = 0
         self.__dict__['_mock_await_args'] = None
         self.__dict__['_mock_await_args_list'] = _CallList()
@@ -2159,7 +2156,6 @@ def __init__(self, /, *args, **kwargs):
                 self.await_count += 1
                 self.await_args = _call
                 self.await_args_list.append(_call)
-                await self.awaited._notify()
 
         return await proxy()
 
@@ -2890,35 +2886,3 @@ def __aiter__(self):
         except StopIteration:
             pass
         raise StopAsyncIteration
-
-
-class _AwaitEvent:
-    def __init__(self, mock):
-        self._mock = mock
-        self._condition = None
-
-    async def _notify(self):
-        condition = self._get_condition()
-        try:
-            await condition.acquire()
-            condition.notify_all()
-        finally:
-            condition.release()
-
-    def _get_condition(self):
-        """
-        Creation of condition is delayed, to minimize the chance of using the
-        wrong loop.
-        A user may create a mock with _AwaitEvent before selecting the
-        execution loop.  Requiring a user to delay creation is error-prone and
-        inflexible. Instead, condition is created when user actually starts to
-        use the mock.
-        """
-        # No synchronization is needed:
-        #   - asyncio is thread unsafe
-        #   - there are no awaits here, method will be executed without
-        #   switching asyncio context.
-        if self._condition is None:
-            self._condition = asyncio.Condition()
-
-        return self._condition
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index 624fadca2649..5a9e93e40bc4 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -4,7 +4,7 @@
 import unittest
 
 from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock,
-                           create_autospec, _AwaitEvent, sentinel, _CallList)
+                           create_autospec, sentinel, _CallList)
 
 
 def tearDownModule():
@@ -178,7 +178,6 @@ def test_create_autospec(self):
         self.assertEqual(spec.await_count, 0)
         self.assertIsNone(spec.await_args)
         self.assertEqual(spec.await_args_list, [])
-        self.assertIsInstance(spec.awaited, _AwaitEvent)
         spec.assert_not_awaited()
 
         asyncio.run(main())
@@ -212,7 +211,6 @@ def test_patch_with_autospec(self):
                 self.assertEqual(mock_method.await_count, 0)
                 self.assertEqual(mock_method.await_args_list, [])
                 self.assertIsNone(mock_method.await_args)
-                self.assertIsInstance(mock_method.awaited, _AwaitEvent)
                 mock_method.assert_not_awaited()
 
                 await awaitable
diff --git a/Misc/NEWS.d/next/Library/2019-09-27-16-31-28.bpo-38161.zehai1.rst b/Misc/NEWS.d/next/Library/2019-09-27-16-31-28.bpo-38161.zehai1.rst
new file mode 100644
index 000000000000..0077033c654f
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-27-16-31-28.bpo-38161.zehai1.rst
@@ -0,0 +1 @@
+Removes _AwaitEvent from AsyncMock.


From webhook-mailer at python.org  Mon Sep 30 00:02:50 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Mon, 30 Sep 2019 04:02:50 -0000
Subject: [Python-checkins] bpo-38108: Makes mock objects inherit from Base
 (GH-16060) (GH-16470)
Message-ID: <mailman.146.1569816171.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/b76ab352405df105c2d459fc66ef8dc98e47b37c
commit: b76ab352405df105c2d459fc66ef8dc98e47b37c
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Lisa Roach <lisaroach14 at gmail.com>
date: 2019-09-29T21:02:46-07:00
summary:

bpo-38108: Makes mock objects inherit from Base (GH-16060) (GH-16470)

files:
A Misc/NEWS.d/next/Library/2019-09-25-21-37-02.bpo-38108.Jr9HU6.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py
M Lib/unittest/test/testmock/testmagicmethods.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 5ea562428a12..8c542cfab787 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -415,7 +415,7 @@ def __new__(cls, /, *args, **kw):
                 if _is_async_obj(bound_args[spec_arg[0]]):
                     bases = (AsyncMockMixin, cls,)
         new = type(cls.__name__, bases, {'__doc__': cls.__doc__})
-        instance = object.__new__(new)
+        instance = _safe_super(NonCallableMock, cls).__new__(new)
         return instance
 
 
@@ -995,17 +995,18 @@ def _get_child_mock(self, /, **kw):
 
         _type = type(self)
         if issubclass(_type, MagicMock) and _new_name in _async_method_magics:
+            # Any asynchronous magic becomes an AsyncMock
             klass = AsyncMock
-        elif _new_name  in _sync_async_magics:
-            # Special case these ones b/c users will assume they are async,
-            # but they are actually sync (ie. __aiter__)
-            klass = MagicMock
         elif issubclass(_type, AsyncMockMixin):
-            klass = AsyncMock
+            if _new_name in _all_sync_magics:
+                # Any synchronous magic becomes a MagicMock
+                klass = MagicMock
+            else:
+                klass = AsyncMock
         elif not issubclass(_type, CallableMixin):
             if issubclass(_type, NonCallableMagicMock):
                 klass = MagicMock
-            elif issubclass(_type, NonCallableMock) :
+            elif issubclass(_type, NonCallableMock):
                 klass = Mock
         else:
             klass = _type.__mro__[1]
@@ -1872,6 +1873,7 @@ def _patch_stopall():
     "round trunc floor ceil "
     "bool next "
     "fspath "
+    "aiter "
 )
 
 numerics = (
@@ -2010,7 +2012,7 @@ def _set_return_value(mock, method, name):
 
 
 
-class MagicMixin(object):
+class MagicMixin(Base):
     def __init__(self, /, *args, **kw):
         self._mock_set_magics()  # make magic work for kwargs in init
         _safe_super(MagicMixin, self).__init__(*args, **kw)
@@ -2018,13 +2020,14 @@ def __init__(self, /, *args, **kw):
 
 
     def _mock_set_magics(self):
-        these_magics = _magics
+        orig_magics = _magics | _async_method_magics
+        these_magics = orig_magics
 
         if getattr(self, "_mock_methods", None) is not None:
-            these_magics = _magics.intersection(self._mock_methods)
+            these_magics = orig_magics.intersection(self._mock_methods)
 
             remove_magics = set()
-            remove_magics = _magics - these_magics
+            remove_magics = orig_magics - these_magics
 
             for entry in remove_magics:
                 if entry in type(self).__dict__:
@@ -2052,33 +2055,13 @@ def mock_add_spec(self, spec, spec_set=False):
         self._mock_set_magics()
 
 
-class AsyncMagicMixin:
+class AsyncMagicMixin(MagicMixin):
     def __init__(self, /, *args, **kw):
-        self._mock_set_async_magics()  # make magic work for kwargs in init
+        self._mock_set_magics()  # make magic work for kwargs in init
         _safe_super(AsyncMagicMixin, self).__init__(*args, **kw)
-        self._mock_set_async_magics()  # fix magic broken by upper level init
-
-    def _mock_set_async_magics(self):
-        these_magics = _async_magics
-
-        if getattr(self, "_mock_methods", None) is not None:
-            these_magics = _async_magics.intersection(self._mock_methods)
-            remove_magics = _async_magics - these_magics
-
-            for entry in remove_magics:
-                if entry in type(self).__dict__:
-                    # remove unneeded magic methods
-                    delattr(self, entry)
-
-        # don't overwrite existing attributes if called a second time
-        these_magics = these_magics - set(type(self).__dict__)
-
-        _type = type(self)
-        for entry in these_magics:
-            setattr(_type, entry, MagicProxy(entry, self))
-
+        self._mock_set_magics()  # fix magic broken by upper level init
 
-class MagicMock(MagicMixin, AsyncMagicMixin, Mock):
+class MagicMock(MagicMixin, Mock):
     """
     MagicMock is a subclass of Mock with default implementations
     of most of the magic methods. You can use MagicMock without having to
@@ -2100,7 +2083,7 @@ def mock_add_spec(self, spec, spec_set=False):
 
 
 
-class MagicProxy(object):
+class MagicProxy(Base):
     def __init__(self, name, parent):
         self.name = name
         self.parent = parent
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index 86b0d0ee9a13..df5ac5e3528c 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -375,6 +375,43 @@ def test_add_side_effect_iterable(self):
                 RuntimeError('coroutine raised StopIteration')
             )
 
+class AsyncMagicMethods(unittest.TestCase):
+    def test_async_magic_methods_return_async_mocks(self):
+        m_mock = MagicMock()
+        self.assertIsInstance(m_mock.__aenter__, AsyncMock)
+        self.assertIsInstance(m_mock.__aexit__, AsyncMock)
+        self.assertIsInstance(m_mock.__anext__, AsyncMock)
+        # __aiter__ is actually a synchronous object
+        # so should return a MagicMock
+        self.assertIsInstance(m_mock.__aiter__, MagicMock)
+
+    def test_sync_magic_methods_return_magic_mocks(self):
+        a_mock = AsyncMock()
+        self.assertIsInstance(a_mock.__enter__, MagicMock)
+        self.assertIsInstance(a_mock.__exit__, MagicMock)
+        self.assertIsInstance(a_mock.__next__, MagicMock)
+        self.assertIsInstance(a_mock.__len__, MagicMock)
+
+    def test_magicmock_has_async_magic_methods(self):
+        m_mock = MagicMock()
+        self.assertTrue(hasattr(m_mock, "__aenter__"))
+        self.assertTrue(hasattr(m_mock, "__aexit__"))
+        self.assertTrue(hasattr(m_mock, "__anext__"))
+
+    def test_asyncmock_has_sync_magic_methods(self):
+        a_mock = AsyncMock()
+        self.assertTrue(hasattr(a_mock, "__enter__"))
+        self.assertTrue(hasattr(a_mock, "__exit__"))
+        self.assertTrue(hasattr(a_mock, "__next__"))
+        self.assertTrue(hasattr(a_mock, "__len__"))
+
+    def test_magic_methods_are_async_functions(self):
+        m_mock = MagicMock()
+        self.assertIsInstance(m_mock.__aenter__, AsyncMock)
+        self.assertIsInstance(m_mock.__aexit__, AsyncMock)
+        # AsyncMocks are also coroutine functions
+        self.assertTrue(asyncio.iscoroutinefunction(m_mock.__aenter__))
+        self.assertTrue(asyncio.iscoroutinefunction(m_mock.__aexit__))
 
 class AsyncContextManagerTest(unittest.TestCase):
     class WithAsyncContextManager:
@@ -402,24 +439,6 @@ def __init__(self):
                 val = await response.json()
                 return val
 
-    def test_async_magic_methods_are_async_mocks_with_magicmock(self):
-        cm_mock = MagicMock(self.WithAsyncContextManager())
-        self.assertIsInstance(cm_mock.__aenter__, AsyncMock)
-        self.assertIsInstance(cm_mock.__aexit__, AsyncMock)
-
-    def test_magicmock_has_async_magic_methods(self):
-        cm = MagicMock(name='magic_cm')
-        self.assertTrue(hasattr(cm, "__aenter__"))
-        self.assertTrue(hasattr(cm, "__aexit__"))
-
-    def test_magic_methods_are_async_functions(self):
-        cm = MagicMock(name='magic_cm')
-        self.assertIsInstance(cm.__aenter__, AsyncMock)
-        self.assertIsInstance(cm.__aexit__, AsyncMock)
-        # AsyncMocks are also coroutine functions
-        self.assertTrue(asyncio.iscoroutinefunction(cm.__aenter__))
-        self.assertTrue(asyncio.iscoroutinefunction(cm.__aexit__))
-
     def test_set_return_value_of_aenter(self):
         def inner_test(mock_type):
             pc = self.ProductionCode()
diff --git a/Lib/unittest/test/testmock/testmagicmethods.py b/Lib/unittest/test/testmock/testmagicmethods.py
index 57f85e951e20..76b3a560de09 100644
--- a/Lib/unittest/test/testmock/testmagicmethods.py
+++ b/Lib/unittest/test/testmock/testmagicmethods.py
@@ -271,9 +271,6 @@ def test_magic_mock_equality(self):
         self.assertEqual(mock == mock, True)
         self.assertEqual(mock != mock, False)
 
-
-    # This should be fixed with issue38163
-    @unittest.expectedFailure
     def test_asyncmock_defaults(self):
         mock = AsyncMock()
         self.assertEqual(int(mock), 1)
diff --git a/Misc/NEWS.d/next/Library/2019-09-25-21-37-02.bpo-38108.Jr9HU6.rst b/Misc/NEWS.d/next/Library/2019-09-25-21-37-02.bpo-38108.Jr9HU6.rst
new file mode 100644
index 000000000000..d7eea367e482
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-25-21-37-02.bpo-38108.Jr9HU6.rst
@@ -0,0 +1,2 @@
+Any synchronous magic methods on an AsyncMock now return a MagicMock. Any
+asynchronous magic methods on a MagicMock now return an AsyncMock.


From webhook-mailer at python.org  Mon Sep 30 00:23:37 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Mon, 30 Sep 2019 04:23:37 -0000
Subject: [Python-checkins] bpo-38161: Removes _AwaitEvent from AsyncMock.
 (GH-16443) (GH-16481)
Message-ID: <mailman.147.1569817418.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/36e7e4aabb662e86e9dace1a6447492f45868654
commit: 36e7e4aabb662e86e9dace1a6447492f45868654
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Lisa Roach <lisaroach14 at gmail.com>
date: 2019-09-29T21:23:33-07:00
summary:

bpo-38161: Removes _AwaitEvent from AsyncMock. (GH-16443) (GH-16481)

files:
A Misc/NEWS.d/next/Library/2019-09-27-16-31-28.bpo-38161.zehai1.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 8c542cfab787..5d32f88c6b78 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -246,7 +246,6 @@ def _setup_async_mock(mock):
     mock.await_count = 0
     mock.await_args = None
     mock.await_args_list = _CallList()
-    mock.awaited = _AwaitEvent(mock)
 
     # Mock is not configured yet so the attributes are set
     # to a function and then the corresponding mock helper function
@@ -2102,7 +2101,6 @@ def __get__(self, obj, _type=None):
 
 
 class AsyncMockMixin(Base):
-    awaited = _delegating_property('awaited')
     await_count = _delegating_property('await_count')
     await_args = _delegating_property('await_args')
     await_args_list = _delegating_property('await_args_list')
@@ -2116,7 +2114,6 @@ def __init__(self, /, *args, **kwargs):
         # It is set through __dict__ because when spec_set is True, this
         # attribute is likely undefined.
         self.__dict__['_is_coroutine'] = asyncio.coroutines._is_coroutine
-        self.__dict__['_mock_awaited'] = _AwaitEvent(self)
         self.__dict__['_mock_await_count'] = 0
         self.__dict__['_mock_await_args'] = None
         self.__dict__['_mock_await_args_list'] = _CallList()
@@ -2145,7 +2142,6 @@ def __init__(self, /, *args, **kwargs):
                 self.await_count += 1
                 self.await_args = _call
                 self.await_args_list.append(_call)
-                await self.awaited._notify()
 
         return await proxy()
 
@@ -2878,35 +2874,3 @@ def __aiter__(self):
         except StopIteration:
             pass
         raise StopAsyncIteration
-
-
-class _AwaitEvent:
-    def __init__(self, mock):
-        self._mock = mock
-        self._condition = None
-
-    async def _notify(self):
-        condition = self._get_condition()
-        try:
-            await condition.acquire()
-            condition.notify_all()
-        finally:
-            condition.release()
-
-    def _get_condition(self):
-        """
-        Creation of condition is delayed, to minimize the chance of using the
-        wrong loop.
-        A user may create a mock with _AwaitEvent before selecting the
-        execution loop.  Requiring a user to delay creation is error-prone and
-        inflexible. Instead, condition is created when user actually starts to
-        use the mock.
-        """
-        # No synchronization is needed:
-        #   - asyncio is thread unsafe
-        #   - there are no awaits here, method will be executed without
-        #   switching asyncio context.
-        if self._condition is None:
-            self._condition = asyncio.Condition()
-
-        return self._condition
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index df5ac5e3528c..ed8aefc2a866 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -4,7 +4,7 @@
 import unittest
 
 from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock,
-                           create_autospec, _AwaitEvent, sentinel, _CallList)
+                           create_autospec, sentinel, _CallList)
 
 
 def tearDownModule():
@@ -178,7 +178,6 @@ def test_create_autospec(self):
         self.assertEqual(spec.await_count, 0)
         self.assertIsNone(spec.await_args)
         self.assertEqual(spec.await_args_list, [])
-        self.assertIsInstance(spec.awaited, _AwaitEvent)
         spec.assert_not_awaited()
 
         asyncio.run(main())
@@ -208,7 +207,6 @@ def test_patch_with_autospec(self):
                 self.assertEqual(mock_method.await_count, 0)
                 self.assertEqual(mock_method.await_args_list, [])
                 self.assertIsNone(mock_method.await_args)
-                self.assertIsInstance(mock_method.awaited, _AwaitEvent)
                 mock_method.assert_not_awaited()
 
                 await awaitable
diff --git a/Misc/NEWS.d/next/Library/2019-09-27-16-31-28.bpo-38161.zehai1.rst b/Misc/NEWS.d/next/Library/2019-09-27-16-31-28.bpo-38161.zehai1.rst
new file mode 100644
index 000000000000..0077033c654f
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-27-16-31-28.bpo-38161.zehai1.rst
@@ -0,0 +1 @@
+Removes _AwaitEvent from AsyncMock.


From webhook-mailer at python.org  Mon Sep 30 00:51:59 2019
From: webhook-mailer at python.org (Giampaolo Rodola)
Date: Mon, 30 Sep 2019 04:51:59 -0000
Subject: [Python-checkins] bpo-37096: Add large-file tests for modules using
 sendfile(2) (GH-13676)
Message-ID: <mailman.148.1569819120.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/5bcc6d89bcb622a6786fff632fabdcaf67dbb4e2
commit: 5bcc6d89bcb622a6786fff632fabdcaf67dbb4e2
branch: master
author: Giampaolo Rodola <g.rodola at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-30T12:51:55+08:00
summary:

bpo-37096: Add large-file tests for modules using sendfile(2) (GH-13676)

files:
M Lib/test/test_largefile.py

diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py
index 8870c721ab0e..6c8813e06260 100644
--- a/Lib/test/test_largefile.py
+++ b/Lib/test/test_largefile.py
@@ -5,17 +5,19 @@
 import stat
 import sys
 import unittest
-from test.support import TESTFN, requires, unlink, bigmemtest
+import socket
+import shutil
+import threading
+from test.support import TESTFN, requires, unlink, bigmemtest, find_unused_port
 import io  # C implementation of io
 import _pyio as pyio # Python implementation of io
 
 # size of file to create (>2 GiB; 2 GiB == 2,147,483,648 bytes)
 size = 2_500_000_000
+TESTFN2 = TESTFN + '2'
+
 
 class LargeFileTest:
-    """Test that each file function works as expected for large
-    (i.e. > 2 GiB) files.
-    """
 
     def setUp(self):
         if os.path.exists(TESTFN):
@@ -44,6 +46,13 @@ def tearDownClass(cls):
         if not os.stat(TESTFN)[stat.ST_SIZE] == 0:
             raise cls.failureException('File was not truncated by opening '
                                        'with mode "wb"')
+        unlink(TESTFN2)
+
+
+class TestFileMethods(LargeFileTest):
+    """Test that each file function works as expected for large
+    (i.e. > 2 GiB) files.
+    """
 
     # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes,
     # so memuse=2 is needed
@@ -140,6 +149,72 @@ def test_seekable(self):
                 f.seek(pos)
                 self.assertTrue(f.seekable())
 
+
+class TestCopyfile(LargeFileTest, unittest.TestCase):
+    open = staticmethod(io.open)
+
+    def test_it(self):
+        # Internally shutil.copyfile() can use "fast copy" methods like
+        # os.sendfile().
+        size = os.path.getsize(TESTFN)
+        shutil.copyfile(TESTFN, TESTFN2)
+        self.assertEqual(os.path.getsize(TESTFN2), size)
+        with open(TESTFN2, 'rb') as f:
+            self.assertEqual(f.read(5), b'z\x00\x00\x00\x00')
+            f.seek(size - 5)
+            self.assertEqual(f.read(), b'\x00\x00\x00\x00a')
+
+
+ at unittest.skipIf(not hasattr(os, 'sendfile'), 'sendfile not supported')
+class TestSocketSendfile(LargeFileTest, unittest.TestCase):
+    open = staticmethod(io.open)
+    timeout = 3
+
+    def setUp(self):
+        super().setUp()
+        self.thread = None
+
+    def tearDown(self):
+        super().tearDown()
+        if self.thread is not None:
+            self.thread.join(self.timeout)
+            self.thread = None
+
+    def tcp_server(self, sock):
+        def run(sock):
+            with sock:
+                conn, _ = sock.accept()
+                with conn, open(TESTFN2, 'wb') as f:
+                    event.wait(self.timeout)
+                    while True:
+                        chunk = conn.recv(65536)
+                        if not chunk:
+                            return
+                        f.write(chunk)
+
+        event = threading.Event()
+        sock.settimeout(self.timeout)
+        self.thread = threading.Thread(target=run, args=(sock, ))
+        self.thread.start()
+        event.set()
+
+    def test_it(self):
+        port = find_unused_port()
+        with socket.create_server(("", port)) as sock:
+            self.tcp_server(sock)
+            with socket.create_connection(("127.0.0.1", port)) as client:
+                with open(TESTFN, 'rb') as f:
+                    client.sendfile(f)
+        self.tearDown()
+
+        size = os.path.getsize(TESTFN)
+        self.assertEqual(os.path.getsize(TESTFN2), size)
+        with open(TESTFN2, 'rb') as f:
+            self.assertEqual(f.read(5), b'z\x00\x00\x00\x00')
+            f.seek(size - 5)
+            self.assertEqual(f.read(), b'\x00\x00\x00\x00a')
+
+
 def setUpModule():
     try:
         import signal
@@ -176,14 +251,18 @@ def setUpModule():
             unlink(TESTFN)
 
 
-class CLargeFileTest(LargeFileTest, unittest.TestCase):
+class CLargeFileTest(TestFileMethods, unittest.TestCase):
     open = staticmethod(io.open)
 
-class PyLargeFileTest(LargeFileTest, unittest.TestCase):
+
+class PyLargeFileTest(TestFileMethods, unittest.TestCase):
     open = staticmethod(pyio.open)
 
+
 def tearDownModule():
     unlink(TESTFN)
+    unlink(TESTFN2)
+
 
 if __name__ == '__main__':
     unittest.main()


From webhook-mailer at python.org  Mon Sep 30 00:56:51 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Mon, 30 Sep 2019 04:56:51 -0000
Subject: [Python-checkins] bpo-38163: Child mocks detect their type as sync
 or async (GH-16471)
Message-ID: <mailman.149.1569819411.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/3667e1ee6c90e6d3b6a745cd590ece87118f81ad
commit: 3667e1ee6c90e6d3b6a745cd590ece87118f81ad
branch: master
author: Lisa Roach <lisaroach14 at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-29T21:56:47-07:00
summary:

bpo-38163: Child mocks detect their type as sync or async (GH-16471)

files:
A Misc/NEWS.d/next/Library/2019-09-28-20-16-40.bpo-38163.x51-vK.rst
M Doc/library/unittest.mock.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py

diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index 300f28c8e2ce..3fa3cfdaf21a 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -865,7 +865,7 @@ object::
     True
 
   The result of ``mock()`` is an async function which will have the outcome
-  of ``side_effect`` or ``return_value``:
+  of ``side_effect`` or ``return_value`` after it has been awaited:
 
   - if ``side_effect`` is a function, the async function will return the
     result of that function,
@@ -890,6 +890,32 @@ object::
     >>> mock()  # doctest: +SKIP
     <coroutine object AsyncMockMixin._mock_call at ...>
 
+
+  Setting the *spec* of a :class:`Mock`, :class:`MagicMock`, or :class:`AsyncMock`
+  to a class with asynchronous and synchronous functions will automatically
+  detect the synchronous functions and set them as :class:`MagicMock` (if the
+  parent mock is :class:`AsyncMock` or :class:`MagicMock`) or :class:`Mock` (if
+  the parent mock is :class:`Mock`). All asynchronous functions will be
+  :class:`AsyncMock`.
+
+  >>> class ExampleClass:
+  ...     def sync_foo():
+  ...         pass
+  ...     async def async_foo():
+  ...         pass
+  ...
+  >>> a_mock = AsyncMock(ExampleClass)
+  >>> a_mock.sync_foo
+  <MagicMock name='mock.sync_foo' id='...'>
+  >>> a_mock.async_foo
+  <AsyncMock name='mock.async_foo' id='...'>
+  >>> mock = Mock(ExampleClass)
+  >>> mock.sync_foo
+  <Mock name='mock.sync_foo' id='...'>
+  >>> mock.async_foo
+  <AsyncMock name='mock.async_foo' id='...'>
+
+
   .. method:: assert_awaited()
 
       Assert that the mock was awaited at least once. Note that this is separate
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index e797d2f5b5e1..a48132c5b1cb 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -992,8 +992,9 @@ def _get_child_mock(self, /, **kw):
             # Any asynchronous magic becomes an AsyncMock
             klass = AsyncMock
         elif issubclass(_type, AsyncMockMixin):
-            if _new_name in _all_sync_magics:
-                # Any synchronous magic becomes a MagicMock
+            if (_new_name in _all_sync_magics or
+                    self._mock_methods and _new_name in self._mock_methods):
+                # Any synchronous method on AsyncMock becomes a MagicMock
                 klass = MagicMock
             else:
                 klass = AsyncMock
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index 5a9e93e40bc4..6b2d49d3e44d 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -3,7 +3,7 @@
 import re
 import unittest
 
-from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock,
+from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock, Mock,
                            create_autospec, sentinel, _CallList)
 
 
@@ -232,33 +232,50 @@ def test_patch_with_autospec(self):
 
 
 class AsyncSpecTest(unittest.TestCase):
-    def test_spec_as_async_positional_magicmock(self):
-        mock = MagicMock(async_func)
-        self.assertIsInstance(mock, MagicMock)
-        m = mock()
-        self.assertTrue(inspect.isawaitable(m))
-        asyncio.run(m)
+    def test_spec_normal_methods_on_class(self):
+        def inner_test(mock_type):
+            mock = mock_type(AsyncClass)
+            self.assertIsInstance(mock.async_method, AsyncMock)
+            self.assertIsInstance(mock.normal_method, MagicMock)
 
-    def test_spec_as_async_kw_magicmock(self):
-        mock = MagicMock(spec=async_func)
-        self.assertIsInstance(mock, MagicMock)
-        m = mock()
-        self.assertTrue(inspect.isawaitable(m))
-        asyncio.run(m)
+        for mock_type in [AsyncMock, MagicMock]:
+            with self.subTest(f"test method types with {mock_type}"):
+                inner_test(mock_type)
 
-    def test_spec_as_async_kw_AsyncMock(self):
-        mock = AsyncMock(spec=async_func)
-        self.assertIsInstance(mock, AsyncMock)
-        m = mock()
-        self.assertTrue(inspect.isawaitable(m))
-        asyncio.run(m)
+    def test_spec_normal_methods_on_class_with_mock(self):
+        mock = Mock(AsyncClass)
+        self.assertIsInstance(mock.async_method, AsyncMock)
+        self.assertIsInstance(mock.normal_method, Mock)
 
-    def test_spec_as_async_positional_AsyncMock(self):
-        mock = AsyncMock(async_func)
-        self.assertIsInstance(mock, AsyncMock)
-        m = mock()
-        self.assertTrue(inspect.isawaitable(m))
-        asyncio.run(m)
+    def test_spec_mock_type_kw(self):
+        def inner_test(mock_type):
+            async_mock = mock_type(spec=async_func)
+            self.assertIsInstance(async_mock, mock_type)
+            with self.assertWarns(RuntimeWarning):
+                # Will raise a warning because never awaited
+                self.assertTrue(inspect.isawaitable(async_mock()))
+
+            sync_mock = mock_type(spec=normal_func)
+            self.assertIsInstance(sync_mock, mock_type)
+
+        for mock_type in [AsyncMock, MagicMock, Mock]:
+            with self.subTest(f"test spec kwarg with {mock_type}"):
+                inner_test(mock_type)
+
+    def test_spec_mock_type_positional(self):
+        def inner_test(mock_type):
+            async_mock = mock_type(async_func)
+            self.assertIsInstance(async_mock, mock_type)
+            with self.assertWarns(RuntimeWarning):
+                # Will raise a warning because never awaited
+                self.assertTrue(inspect.isawaitable(async_mock()))
+
+            sync_mock = mock_type(normal_func)
+            self.assertIsInstance(sync_mock, mock_type)
+
+        for mock_type in [AsyncMock, MagicMock, Mock]:
+            with self.subTest(f"test spec positional with {mock_type}"):
+                inner_test(mock_type)
 
     def test_spec_as_normal_kw_AsyncMock(self):
         mock = AsyncMock(spec=normal_func)
diff --git a/Misc/NEWS.d/next/Library/2019-09-28-20-16-40.bpo-38163.x51-vK.rst b/Misc/NEWS.d/next/Library/2019-09-28-20-16-40.bpo-38163.x51-vK.rst
new file mode 100644
index 000000000000..5f7db26e05ce
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-28-20-16-40.bpo-38163.x51-vK.rst
@@ -0,0 +1,4 @@
+Child mocks will now detect their type as either synchronous or
+asynchronous, asynchronous child mocks will be AsyncMocks and synchronous
+child mocks will be either MagicMock or Mock (depending on their parent
+type).


From webhook-mailer at python.org  Mon Sep 30 00:59:59 2019
From: webhook-mailer at python.org (Yury Selivanov)
Date: Mon, 30 Sep 2019 04:59:59 -0000
Subject: [Python-checkins] bpo-38242: Revert "bpo-36889: Merge asyncio
 streams (GH-13251)" (#16482)
Message-ID: <mailman.150.1569819601.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/6758e6e12a71ef5530146161881f88df1fa43382
commit: 6758e6e12a71ef5530146161881f88df1fa43382
branch: master
author: Yury Selivanov <yury at edgedb.com>
committer: GitHub <noreply at github.com>
date: 2019-09-29T21:59:55-07:00
summary:

bpo-38242: Revert "bpo-36889: Merge asyncio streams (GH-13251)" (#16482)

See https://bugs.python.org/issue38242 for more details

files:
A Misc/NEWS.d/next/Library/2019-09-30-00-15-27.bpo-38242.uPIyAc.rst
M Doc/library/asyncio-api-index.rst
M Doc/library/asyncio-eventloop.rst
M Doc/library/asyncio-protocol.rst
M Doc/library/asyncio-stream.rst
M Lib/asyncio/__init__.py
M Lib/asyncio/streams.py
M Lib/asyncio/subprocess.py
M Lib/test/test_asyncio/test_buffered_proto.py
M Lib/test/test_asyncio/test_pep492.py
M Lib/test/test_asyncio/test_streams.py
M Lib/test/test_asyncio/test_subprocess.py
M Lib/test/test_asyncio/test_windows_events.py

diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst
index 716cf09dc99e..d5b5659abc65 100644
--- a/Doc/library/asyncio-api-index.rst
+++ b/Doc/library/asyncio-api-index.rst
@@ -132,47 +132,23 @@ High-level APIs to work with network IO.
     :widths: 50 50
     :class: full-width-table
 
-    * - ``await`` :func:`connect`
-      -  Establish a TCP connection to send and receive data.
-
     * - ``await`` :func:`open_connection`
-      -  Establish a TCP connection. (Deprecated in favor of :func:`connect`)
-
-    * - ``await`` :func:`connect_unix`
-      -  Establish a Unix socket connection to send and receive data.
+      -  Establish a TCP connection.
 
     * - ``await`` :func:`open_unix_connection`
-      -  Establish a Unix socket connection. (Deprecated in favor of :func:`connect_unix`)
-
-    * - :class:`StreamServer`
-      - Start a TCP server.
+      -  Establish a Unix socket connection.
 
     * - ``await`` :func:`start_server`
-      - Start a TCP server. (Deprecated in favor of :class:`StreamServer`)
-
-    * - :class:`UnixStreamServer`
-      - Start a Unix socket server.
+      - Start a TCP server.
 
     * - ``await`` :func:`start_unix_server`
-      - Start a Unix socket server. (Deprecated in favor of :class:`UnixStreamServer`)
-
-    * - :func:`connect_read_pipe`
-      - Establish a connection to :term:`file-like object <file object>` *pipe*
-        to receive data.
-
-    * - :func:`connect_write_pipe`
-      - Establish a connection to :term:`file-like object <file object>` *pipe*
-        to send data.
-
-    * - :class:`Stream`
-      - Stream is a single object combining APIs of :class:`StreamReader` and
-        :class:`StreamWriter`.
+      - Start a Unix socket server.
 
     * - :class:`StreamReader`
-      - High-level async/await object to receive network data. (Deprecated in favor of :class:`Stream`)
+      - High-level async/await object to receive network data.
 
     * - :class:`StreamWriter`
-      - High-level async/await object to send network data. (Deprecated in favor of :class:`Stream`)
+      - High-level async/await object to send network data.
 
 
 .. rubric:: Examples
diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index 2fd4cf30bd45..045787e0ce78 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -1637,7 +1637,8 @@ Wait until a file descriptor received some data using the
      :meth:`loop.create_connection` method.
 
    * Another similar :ref:`example <asyncio_example_create_connection-streams>`
-     using the high-level :func:`asyncio.connect` function and streams.
+     using the high-level :func:`asyncio.open_connection` function
+     and streams.
 
 
 .. _asyncio_example_unix_signals:
diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index cb0317ea20c1..67ca12108168 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -809,7 +809,7 @@ data, and waits until the connection is closed::
 .. seealso::
 
    The :ref:`TCP echo client using streams <asyncio-tcp-echo-client-streams>`
-   example uses the high-level :func:`asyncio.connect` function.
+   example uses the high-level :func:`asyncio.open_connection` function.
 
 
 .. _asyncio-udp-echo-server-protocol:
@@ -978,7 +978,7 @@ Wait until a socket receives data using the
 
    The :ref:`register an open socket to wait for data using streams
    <asyncio_example_create_connection-streams>` example uses high-level streams
-   created by the :func:`asyncio.connect` function in a coroutine.
+   created by the :func:`open_connection` function in a coroutine.
 
 .. _asyncio_example_subprocess_proto:
 
diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst
index feebd227eb8c..471e6e9099c2 100644
--- a/Doc/library/asyncio-stream.rst
+++ b/Doc/library/asyncio-stream.rst
@@ -18,12 +18,19 @@ streams::
     import asyncio
 
     async def tcp_echo_client(message):
-        async with asyncio.connect('127.0.0.1', 8888) as stream:
-            print(f'Send: {message!r}')
-            await stream.write(message.encode())
+        reader, writer = await asyncio.open_connection(
+            '127.0.0.1', 8888)
 
-            data = await stream.read(100)
-            print(f'Received: {data.decode()!r}')
+        print(f'Send: {message!r}')
+        writer.write(message.encode())
+        await writer.drain()
+
+        data = await reader.read(100)
+        print(f'Received: {data.decode()!r}')
+
+        print('Close the connection')
+        writer.close()
+        await writer.wait_closed()
 
     asyncio.run(tcp_echo_client('Hello World!'))
 
@@ -37,31 +44,6 @@ The following top-level asyncio functions can be used to create
 and work with streams:
 
 
-.. coroutinefunction:: connect(host=None, port=None, \*, \
-                               limit=2**16, ssl=None, family=0, \
-                               proto=0, flags=0, sock=None, local_addr=None, \
-                               server_hostname=None, ssl_handshake_timeout=None, \
-                               happy_eyeballs_delay=None, interleave=None)
-
-   Connect to TCP socket on *host* : *port* address and return a :class:`Stream`
-   object of mode :attr:`StreamMode.READWRITE`.
-
-   *limit* determines the buffer size limit used by the returned :class:`Stream`
-   instance. By default the *limit* is set to 64 KiB.
-
-   The rest of the arguments are passed directly to :meth:`loop.create_connection`.
-
-   The function can be used with ``await`` to get a connected stream::
-
-       stream = await asyncio.connect('127.0.0.1', 8888)
-
-   The function can also be used as an async context manager::
-
-       async with asyncio.connect('127.0.0.1', 8888) as stream:
-           ...
-
-   .. versionadded:: 3.8
-
 .. coroutinefunction:: open_connection(host=None, port=None, \*, \
                           loop=None, limit=None, ssl=None, family=0, \
                           proto=0, flags=0, sock=None, local_addr=None, \
@@ -87,12 +69,8 @@ and work with streams:
 
       The *ssl_handshake_timeout* parameter.
 
-   .. deprecated-removed:: 3.8 3.10
-
-      `open_connection()` is deprecated in favor of :func:`connect`.
-
 .. coroutinefunction:: start_server(client_connected_cb, host=None, \
-                          port=None, \*, loop=None, limit=2**16, \
+                          port=None, \*, loop=None, limit=None, \
                           family=socket.AF_UNSPEC, \
                           flags=socket.AI_PASSIVE, sock=None, \
                           backlog=100, ssl=None, reuse_address=None, \
@@ -124,60 +102,9 @@ and work with streams:
 
       The *ssl_handshake_timeout* and *start_serving* parameters.
 
-   .. deprecated-removed:: 3.8 3.10
-
-      `start_server()` is deprecated if favor of :class:`StreamServer`
-
-.. coroutinefunction:: connect_read_pipe(pipe, *, limit=2**16)
-
-   Takes a :term:`file-like object <file object>` *pipe* to return a
-   :class:`Stream` object of the mode :attr:`StreamMode.READ` that has
-   similar API of :class:`StreamReader`. It can also be used as an async context manager.
-
-   *limit* determines the buffer size limit used by the returned :class:`Stream`
-   instance. By default the limit is set to 64 KiB.
-
-   .. versionadded:: 3.8
-
-.. coroutinefunction:: connect_write_pipe(pipe, *, limit=2**16)
-
-   Takes a :term:`file-like object <file object>` *pipe* to return a
-   :class:`Stream` object of the mode :attr:`StreamMode.WRITE` that has
-   similar API of :class:`StreamWriter`. It can also be used as an async context manager.
-
-   *limit* determines the buffer size limit used by the returned :class:`Stream`
-   instance. By default the limit is set to 64 KiB.
-
-   .. versionadded:: 3.8
 
 .. rubric:: Unix Sockets
 
-.. function:: connect_unix(path=None, *, limit=2**16, ssl=None, \
-                           sock=None, server_hostname=None, \
-                           ssl_handshake_timeout=None)
-
-   Establish a Unix socket connection to socket with *path* address and
-   return an awaitable :class:`Stream` object of the mode :attr:`StreamMode.READWRITE`
-   that can be used as a reader and a writer.
-
-   *limit* determines the buffer size limit used by the returned :class:`Stream`
-   instance. By default the *limit* is set to 64 KiB.
-
-   The rest of the arguments are passed directly to :meth:`loop.create_unix_connection`.
-
-   The function can be used with ``await`` to get a connected stream::
-
-       stream = await asyncio.connect_unix('/tmp/example.sock')
-
-   The function can also be used as an async context manager::
-
-       async with asyncio.connect_unix('/tmp/example.sock') as stream:
-           ...
-
-   .. availability:: Unix.
-
-   .. versionadded:: 3.8
-
 .. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, \
                         limit=None, ssl=None, sock=None, \
                         server_hostname=None, ssl_handshake_timeout=None)
@@ -199,10 +126,6 @@ and work with streams:
 
       The *path* parameter can now be a :term:`path-like object`
 
-   .. deprecated-removed:: 3.8 3.10
-
-      ``open_unix_connection()`` is deprecated if favor of :func:`connect_unix`.
-
 
 .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \
                           \*, loop=None, limit=None, sock=None, \
@@ -225,349 +148,6 @@ and work with streams:
 
       The *path* parameter can now be a :term:`path-like object`.
 
-   .. deprecated-removed:: 3.8 3.10
-
-      ``start_unix_server()`` is deprecated in favor of :class:`UnixStreamServer`.
-
-
----------
-
-StreamServer
-============
-
-.. class:: StreamServer(client_connected_cb, /, host=None, port=None, *, \
-                        limit=2**16, family=socket.AF_UNSPEC, \
-                        flags=socket.AI_PASSIVE, sock=None, backlog=100, \
-                        ssl=None, reuse_address=None, reuse_port=None, \
-                        ssl_handshake_timeout=None, shutdown_timeout=60)
-
-   The *client_connected_cb* callback is called whenever a new client
-   connection is established.  It receives a :class:`Stream` object of the
-   mode :attr:`StreamMode.READWRITE`.
-
-   *client_connected_cb* can be a plain callable or a
-   :ref:`coroutine function <coroutine>`; if it is a coroutine function,
-   it will be automatically scheduled as a :class:`Task`.
-
-   *limit* determines the buffer size limit used by the
-   returned :class:`Stream` instance.  By default the *limit*
-   is set to 64 KiB.
-
-   The rest of the arguments are passed directly to
-   :meth:`loop.create_server`.
-
-   .. coroutinemethod:: start_serving()
-
-      Binds to the given host and port to start the server.
-
-   .. coroutinemethod:: serve_forever()
-
-      Start accepting connections until the coroutine is cancelled.
-      Cancellation of ``serve_forever`` task causes the server
-      to be closed.
-
-      This method can be called if the server is already accepting
-      connections.  Only one ``serve_forever`` task can exist per
-      one *Server* object.
-
-   .. method:: is_serving()
-
-      Returns ``True`` if the server is bound and currently serving.
-
-   .. method:: bind()
-
-      Bind the server to the given *host* and *port*. This method is
-      automatically called during ``__aenter__`` when :class:`StreamServer` is
-      used as an async context manager.
-
-   .. method:: is_bound()
-
-      Return ``True`` if the server is bound.
-
-   .. coroutinemethod:: abort()
-
-      Closes the connection and cancels all pending tasks.
-
-   .. coroutinemethod:: close()
-
-      Closes the connection. This method is automatically called during
-      ``__aexit__`` when :class:`StreamServer` is used as an async context
-      manager.
-
-   .. attribute:: sockets
-
-      Returns a tuple of socket objects the server is bound to.
-
-   .. versionadded:: 3.8
-
-
-UnixStreamServer
-================
-
-.. class:: UnixStreamServer(client_connected_cb, /, path=None, *, \
-                            limit=2**16, sock=None, backlog=100, \
-                            ssl=None, ssl_handshake_timeout=None, shutdown_timeout=60)
-
-   The *client_connected_cb* callback is called whenever a new client
-   connection is established.  It receives a :class:`Stream` object of the
-   mode :attr:`StreamMode.READWRITE`.
-
-   *client_connected_cb* can be a plain callable or a
-   :ref:`coroutine function <coroutine>`; if it is a coroutine function,
-   it will be automatically scheduled as a :class:`Task`.
-
-   *limit* determines the buffer size limit used by the
-   returned :class:`Stream` instance.  By default the *limit*
-   is set to 64 KiB.
-
-   The rest of the arguments are passed directly to
-   :meth:`loop.create_unix_server`.
-
-   .. coroutinemethod:: start_serving()
-
-      Binds to the given host and port to start the server.
-
-   .. method:: is_serving()
-
-      Returns ``True`` if the server is bound and currently serving.
-
-   .. method:: bind()
-
-      Bind the server to the given *host* and *port*. This method is
-      automatically called during ``__aenter__`` when :class:`UnixStreamServer` is
-      used as an async context manager.
-
-   .. method:: is_bound()
-
-      Return ``True`` if the server is bound.
-
-   .. coroutinemethod:: abort()
-
-      Closes the connection and cancels all pending tasks.
-
-   .. coroutinemethod:: close()
-
-      Closes the connection. This method is automatically called during
-      ``__aexit__`` when :class:`UnixStreamServer` is used as an async context
-      manager.
-
-   .. attribute:: sockets
-
-      Returns a tuple of socket objects the server is bound to.
-
-   .. availability:: Unix.
-
-   .. versionadded:: 3.8
-
-Stream
-======
-
-.. class:: Stream
-
-   Represents a Stream object that provides APIs to read and write data
-   to the IO stream . It includes the API provided by :class:`StreamReader`
-   and :class:`StreamWriter`. It can also be used as :term:`asynchronous iterator`
-   where :meth:`readline` is used. It raises :exc:`StopAsyncIteration` when
-   :meth:`readline` returns empty data.
-
-   Do not instantiate *Stream* objects directly; use API like :func:`connect`
-   and :class:`StreamServer` instead.
-
-   .. versionadded:: 3.8
-
-   .. attribute:: mode
-
-      Returns the mode of the stream which is a :class:`StreamMode` value. It could
-      be one of the below:
-
-      * :attr:`StreamMode.READ` - Connection can receive data.
-      * :attr:`StreamMode.WRITE` - Connection can send data.
-      * :attr:`StreamMode.READWRITE` - Connection can send and receive data.
-
-   .. coroutinemethod:: abort()
-
-      Aborts the connection immediately, without waiting for the send buffer to drain.
-
-   .. method:: at_eof()
-
-      Return ``True`` if the buffer is empty.
-
-   .. method:: can_write_eof()
-
-      Return *True* if the underlying transport supports
-      the :meth:`write_eof` method, *False* otherwise.
-
-   .. method:: close()
-
-      The method closes the stream and the underlying socket.
-
-      It is possible to directly await on the `close()` method::
-
-         await stream.close()
-
-      The ``await`` pauses the current coroutine until the stream and the underlying
-      socket are closed (and SSL shutdown is performed for a secure connection).
-
-   .. coroutinemethod:: drain()
-
-      Wait until it is appropriate to resume writing to the stream.
-      Example::
-
-          stream.write(data)
-          await stream.drain()
-
-      This is a flow control method that interacts with the underlying
-      IO write buffer.  When the size of the buffer reaches
-      the high watermark, *drain()* blocks until the size of the
-      buffer is drained down to the low watermark and writing can
-      be resumed.  When there is nothing to wait for, the :meth:`drain`
-      returns immediately.
-
-      .. deprecated:: 3.8
-
-      It is recommended to directly await on the `write()` method instead::
-
-         await stream.write(data)
-
-   .. method:: get_extra_info(name, default=None)
-
-      Access optional transport information; see
-      :meth:`BaseTransport.get_extra_info` for details.
-
-   .. method:: is_closing()
-
-      Return ``True`` if the stream is closed or in the process of
-      being closed.
-
-   .. coroutinemethod:: read(n=-1)
-
-      Read up to *n* bytes.  If *n* is not provided, or set to ``-1``,
-      read until EOF and return all read bytes.
-
-      If EOF was received and the internal buffer is empty,
-      return an empty ``bytes`` object.
-
-   .. coroutinemethod:: readexactly(n)
-
-      Read exactly *n* bytes.
-
-      Raise an :exc:`IncompleteReadError` if EOF is reached before *n*
-      can be read.  Use the :attr:`IncompleteReadError.partial`
-      attribute to get the partially read data.
-
-   .. coroutinemethod:: readline()
-
-      Read one line, where "line" is a sequence of bytes
-      ending with ``\n``.
-
-      If EOF is received and ``\n`` was not found, the method
-      returns partially read data.
-
-      If EOF is received and the internal buffer is empty,
-      return an empty ``bytes`` object.
-
-   .. coroutinemethod:: readuntil(separator=b'\\n')
-
-      Read data from the stream until *separator* is found.
-
-      On success, the data and separator will be removed from the
-      internal buffer (consumed). Returned data will include the
-      separator at the end.
-
-      If the amount of data read exceeds the configured stream limit, a
-      :exc:`LimitOverrunError` exception is raised, and the data
-      is left in the internal buffer and can be read again.
-
-      If EOF is reached before the complete separator is found,
-      an :exc:`IncompleteReadError` exception is raised, and the internal
-      buffer is reset.  The :attr:`IncompleteReadError.partial` attribute
-      may contain a portion of the separator.
-
-   .. coroutinemethod:: sendfile(file, offset=0, count=None, *, fallback=True)
-
-      Sends a *file* over the stream using an optimized syscall if available.
-
-      For other parameters meaning please see :meth:`AbstractEventloop.sendfile`.
-
-   .. coroutinemethod:: start_tls(sslcontext, *, server_hostname=None, \
-                                  ssl_handshake_timeout=None)
-
-      Upgrades the existing transport-based connection to TLS.
-
-      For other parameters meaning please see :meth:`AbstractEventloop.start_tls`.
-
-   .. coroutinemethod:: wait_closed()
-
-      Wait until the stream is closed.
-
-      Should be called after :meth:`close` to wait until the underlying
-      connection is closed.
-
-   .. coroutinemethod:: write(data)
-
-      Write *data* to the underlying socket; wait until the data is sent, e.g.::
-
-         await stream.write(data)
-
-   .. method:: write(data)
-
-      The method attempts to write the *data* to the underlying socket immediately.
-      If that fails, the data is queued in an internal write buffer until it can be
-      sent. :meth:`drain` can be used to flush the underlying buffer once writing is
-      available::
-
-         stream.write(data)
-         await stream.drain()
-
-      .. deprecated:: 3.8
-
-      It is recommended to directly await on the `write()` method instead::
-
-          await stream.write(data)
-
-   .. method:: writelines(data)
-
-      The method writes a list (or any iterable) of bytes to the underlying socket
-      immediately.
-      If that fails, the data is queued in an internal write buffer until it can be
-      sent.
-
-      It is possible to directly await on the `writelines()` method::
-
-         await stream.writelines(lines)
-
-      The ``await`` pauses the current coroutine until the data is written to the
-      socket.
-
-   .. method:: write_eof()
-
-      Close the write end of the stream after the buffered write
-      data is flushed.
-
-
-StreamMode
-==========
-
-.. class:: StreamMode
-
-   A subclass of :class:`enum.Flag` that defines a set of values that can be
-   used to determine the ``mode`` of :class:`Stream` objects.
-
-   .. data:: READ
-
-   The stream object is readable and provides the API of :class:`StreamReader`.
-
-   .. data:: WRITE
-
-   The stream object is writeable and provides the API of :class:`StreamWriter`.
-
-   .. data:: READWRITE
-
-   The stream object is readable and writeable and provides the API of both
-   :class:`StreamReader` and :class:`StreamWriter`.
-
-  .. versionadded:: 3.8
-
 
 StreamReader
 ============
@@ -629,7 +209,8 @@ StreamReader
 
    .. method:: at_eof()
 
-      Return ``True`` if the buffer is empty.
+      Return ``True`` if the buffer is empty and :meth:`feed_eof`
+      was called.
 
 
 StreamWriter
@@ -650,22 +231,11 @@ StreamWriter
       If that fails, the data is queued in an internal write buffer until it can be
       sent.
 
-      Starting with Python 3.8, it is possible to directly await on the `write()`
-      method::
-
-         await stream.write(data)
-
-      The ``await`` pauses the current coroutine until the data is written to the
-      socket.
-
-      Below is an equivalent code that works with Python <= 3.7::
+      The method should be used along with the ``drain()`` method::
 
          stream.write(data)
          await stream.drain()
 
-      .. versionchanged:: 3.8
-         Support ``await stream.write(...)`` syntax.
-
    .. method:: writelines(data)
 
       The method writes a list (or any iterable) of bytes to the underlying socket
@@ -673,42 +243,20 @@ StreamWriter
       If that fails, the data is queued in an internal write buffer until it can be
       sent.
 
-      Starting with Python 3.8, it is possible to directly await on the `writelines()`
-      method::
-
-         await stream.writelines(lines)
-
-      The ``await`` pauses the current coroutine until the data is written to the
-      socket.
-
-      Below is an equivalent code that works with Python <= 3.7::
+      The method should be used along with the ``drain()`` method::
 
          stream.writelines(lines)
          await stream.drain()
 
-      .. versionchanged:: 3.8
-         Support ``await stream.writelines()`` syntax.
-
    .. method:: close()
 
       The method closes the stream and the underlying socket.
 
-      Starting with Python 3.8, it is possible to directly await on the `close()`
-      method::
-
-         await stream.close()
-
-      The ``await`` pauses the current coroutine until the stream and the underlying
-      socket are closed (and SSL shutdown is performed for a secure connection).
-
-      Below is an equivalent code that works with Python <= 3.7::
+      The method should be used along with the ``wait_closed()`` method::
 
          stream.close()
          await stream.wait_closed()
 
-      .. versionchanged:: 3.8
-         Support ``await stream.close()`` syntax.
-
    .. method:: can_write_eof()
 
       Return *True* if the underlying transport supports
@@ -768,17 +316,22 @@ Examples
 TCP echo client using streams
 -----------------------------
 
-TCP echo client using the :func:`asyncio.connect` function::
+TCP echo client using the :func:`asyncio.open_connection` function::
 
     import asyncio
 
     async def tcp_echo_client(message):
-        async with asyncio.connect('127.0.0.1', 8888) as stream:
-            print(f'Send: {message!r}')
-            await stream.write(message.encode())
+        reader, writer = await asyncio.open_connection(
+            '127.0.0.1', 8888)
+
+        print(f'Send: {message!r}')
+        writer.write(message.encode())
+
+        data = await reader.read(100)
+        print(f'Received: {data.decode()!r}')
 
-            data = await stream.read(100)
-            print(f'Received: {data.decode()!r}')
+        print('Close the connection')
+        writer.close()
 
     asyncio.run(tcp_echo_client('Hello World!'))
 
@@ -794,28 +347,32 @@ TCP echo client using the :func:`asyncio.connect` function::
 TCP echo server using streams
 -----------------------------
 
-TCP echo server using the :class:`asyncio.StreamServer` class::
+TCP echo server using the :func:`asyncio.start_server` function::
 
     import asyncio
 
-    async def handle_echo(stream):
-        data = await stream.read(100)
+    async def handle_echo(reader, writer):
+        data = await reader.read(100)
         message = data.decode()
-        addr = stream.get_extra_info('peername')
+        addr = writer.get_extra_info('peername')
 
         print(f"Received {message!r} from {addr!r}")
 
         print(f"Send: {message!r}")
-        await stream.write(data)
+        writer.write(data)
+        await writer.drain()
 
         print("Close the connection")
-        await stream.close()
+        writer.close()
 
     async def main():
-        async with asyncio.StreamServer(
-                handle_echo, '127.0.0.1', 8888) as server:
-            addr = server.sockets[0].getsockname()
-            print(f'Serving on {addr}')
+        server = await asyncio.start_server(
+            handle_echo, '127.0.0.1', 8888)
+
+        addr = server.sockets[0].getsockname()
+        print(f'Serving on {addr}')
+
+        async with server:
             await server.serve_forever()
 
     asyncio.run(main())
@@ -839,9 +396,11 @@ Simple example querying HTTP headers of the URL passed on the command line::
     async def print_http_headers(url):
         url = urllib.parse.urlsplit(url)
         if url.scheme == 'https':
-            stream = await asyncio.connect(url.hostname, 443, ssl=True)
+            reader, writer = await asyncio.open_connection(
+                url.hostname, 443, ssl=True)
         else:
-            stream = await asyncio.connect(url.hostname, 80)
+            reader, writer = await asyncio.open_connection(
+                url.hostname, 80)
 
         query = (
             f"HEAD {url.path or '/'} HTTP/1.0\r\n"
@@ -849,14 +408,18 @@ Simple example querying HTTP headers of the URL passed on the command line::
             f"\r\n"
         )
 
-        stream.write(query.encode('latin-1'))
-        while (line := await stream.readline()):
+        writer.write(query.encode('latin-1'))
+        while True:
+            line = await reader.readline()
+            if not line:
+                break
+
             line = line.decode('latin1').rstrip()
             if line:
                 print(f'HTTP header> {line}')
 
         # Ignore the body, close the socket
-        await stream.close()
+        writer.close()
 
     url = sys.argv[1]
     asyncio.run(print_http_headers(url))
@@ -877,7 +440,7 @@ Register an open socket to wait for data using streams
 ------------------------------------------------------
 
 Coroutine waiting until a socket receives data using the
-:func:`asyncio.connect` function::
+:func:`open_connection` function::
 
     import asyncio
     import socket
@@ -891,15 +454,17 @@ Coroutine waiting until a socket receives data using the
         rsock, wsock = socket.socketpair()
 
         # Register the open socket to wait for data.
-        async with asyncio.connect(sock=rsock) as stream:
-            # Simulate the reception of data from the network
-            loop.call_soon(wsock.send, 'abc'.encode())
+        reader, writer = await asyncio.open_connection(sock=rsock)
+
+        # Simulate the reception of data from the network
+        loop.call_soon(wsock.send, 'abc'.encode())
 
-            # Wait for data
-            data = await stream.read(100)
+        # Wait for data
+        data = await reader.read(100)
 
-            # Got data, we are done: close the socket
-            print("Received:", data.decode())
+        # Got data, we are done: close the socket
+        print("Received:", data.decode())
+        writer.close()
 
         # Close the second socket
         wsock.close()
diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py
index a6a29dbfecd5..28c2e2c429f3 100644
--- a/Lib/asyncio/__init__.py
+++ b/Lib/asyncio/__init__.py
@@ -3,7 +3,6 @@
 # flake8: noqa
 
 import sys
-import warnings
 
 # This relies on each of the submodules having an __all__ variable.
 from .base_events import *
@@ -44,40 +43,3 @@
 else:
     from .unix_events import *  # pragma: no cover
     __all__ += unix_events.__all__
-
-
-__all__ += ('StreamReader', 'StreamWriter', 'StreamReaderProtocol')  # deprecated
-
-
-def __getattr__(name):
-    global StreamReader, StreamWriter, StreamReaderProtocol
-    if name == 'StreamReader':
-        warnings.warn("StreamReader is deprecated since Python 3.8 "
-                      "in favor of Stream, and scheduled for removal "
-                      "in Python 3.10",
-                      DeprecationWarning,
-                      stacklevel=2)
-        from .streams import StreamReader as sr
-        StreamReader = sr
-        return StreamReader
-    if name == 'StreamWriter':
-        warnings.warn("StreamWriter is deprecated since Python 3.8 "
-                      "in favor of Stream, and scheduled for removal "
-                      "in Python 3.10",
-                      DeprecationWarning,
-                      stacklevel=2)
-        from .streams import StreamWriter as sw
-        StreamWriter = sw
-        return StreamWriter
-    if name == 'StreamReaderProtocol':
-        warnings.warn("Using asyncio internal class StreamReaderProtocol "
-                      "is deprecated since Python 3.8 "
-                      " and scheduled for removal "
-                      "in Python 3.10",
-                      DeprecationWarning,
-                      stacklevel=2)
-        from .streams import StreamReaderProtocol as srp
-        StreamReaderProtocol = srp
-        return StreamReaderProtocol
-
-    raise AttributeError(f"module {__name__} has no attribute {name}")
diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py
index b709dc11477e..795530e6f69e 100644
--- a/Lib/asyncio/streams.py
+++ b/Lib/asyncio/streams.py
@@ -1,19 +1,14 @@
 __all__ = (
-    'Stream', 'StreamMode',
-    'open_connection', 'start_server',
-    'connect', 'connect_read_pipe', 'connect_write_pipe',
-    'StreamServer')
+    'StreamReader', 'StreamWriter', 'StreamReaderProtocol',
+    'open_connection', 'start_server')
 
-import enum
 import socket
 import sys
 import warnings
 import weakref
 
 if hasattr(socket, 'AF_UNIX'):
-    __all__ += ('open_unix_connection', 'start_unix_server',
-                'connect_unix',
-                'UnixStreamServer')
+    __all__ += ('open_unix_connection', 'start_unix_server')
 
 from . import coroutines
 from . import events
@@ -21,155 +16,12 @@
 from . import format_helpers
 from . import protocols
 from .log import logger
-from . import tasks
+from .tasks import sleep
 
 
 _DEFAULT_LIMIT = 2 ** 16  # 64 KiB
 
 
-class StreamMode(enum.Flag):
-    READ = enum.auto()
-    WRITE = enum.auto()
-    READWRITE = READ | WRITE
-
-
-def _ensure_can_read(mode):
-    if not mode & StreamMode.READ:
-        raise RuntimeError("The stream is write-only")
-
-
-def _ensure_can_write(mode):
-    if not mode & StreamMode.WRITE:
-        raise RuntimeError("The stream is read-only")
-
-
-class _ContextManagerHelper:
-    __slots__ = ('_awaitable', '_result')
-
-    def __init__(self, awaitable):
-        self._awaitable = awaitable
-        self._result = None
-
-    def __await__(self):
-        return self._awaitable.__await__()
-
-    async def __aenter__(self):
-        ret = await self._awaitable
-        result = await ret.__aenter__()
-        self._result = result
-        return result
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        return await self._result.__aexit__(exc_type, exc_val, exc_tb)
-
-
-def connect(host=None, port=None, *,
-            limit=_DEFAULT_LIMIT,
-            ssl=None, family=0, proto=0,
-            flags=0, sock=None, local_addr=None,
-            server_hostname=None,
-            ssl_handshake_timeout=None,
-            happy_eyeballs_delay=None, interleave=None):
-    """Connect to TCP socket on *host* : *port* address to send and receive data.
-
-    *limit* determines the buffer size limit used by the returned `Stream`
-    instance. By default the *limit* is set to 64 KiB.
-
-    The rest of the arguments are passed directly to `loop.create_connection()`.
-    """
-    # Design note:
-    # Don't use decorator approach but explicit non-async
-    # function to fail fast and explicitly
-    # if passed arguments don't match the function signature
-    return _ContextManagerHelper(_connect(host, port, limit,
-                                          ssl, family, proto,
-                                          flags, sock, local_addr,
-                                          server_hostname,
-                                          ssl_handshake_timeout,
-                                          happy_eyeballs_delay,
-                                          interleave))
-
-
-async def _connect(host, port,
-                  limit,
-                  ssl, family, proto,
-                  flags, sock, local_addr,
-                  server_hostname,
-                  ssl_handshake_timeout,
-                  happy_eyeballs_delay, interleave):
-    loop = events.get_running_loop()
-    stream = Stream(mode=StreamMode.READWRITE,
-                    limit=limit,
-                    loop=loop,
-                    _asyncio_internal=True)
-    await loop.create_connection(
-        lambda: _StreamProtocol(stream, loop=loop,
-                                _asyncio_internal=True),
-        host, port,
-        ssl=ssl, family=family, proto=proto,
-        flags=flags, sock=sock, local_addr=local_addr,
-        server_hostname=server_hostname,
-        ssl_handshake_timeout=ssl_handshake_timeout,
-        happy_eyeballs_delay=happy_eyeballs_delay, interleave=interleave)
-    return stream
-
-
-def connect_read_pipe(pipe, *, limit=_DEFAULT_LIMIT):
-    """Establish a connection to a file-like object *pipe* to receive data.
-
-    Takes a file-like object *pipe* to return a Stream object of the mode
-    StreamMode.READ that has similar API of StreamReader. It can also be used
-    as an async context manager.
-    """
-
-    # Design note:
-    # Don't use decorator approach but explicit non-async
-    # function to fail fast and explicitly
-    # if passed arguments don't match the function signature
-    return _ContextManagerHelper(_connect_read_pipe(pipe, limit))
-
-
-async def _connect_read_pipe(pipe, limit):
-    loop = events.get_running_loop()
-    stream = Stream(mode=StreamMode.READ,
-                    limit=limit,
-                    loop=loop,
-                    _asyncio_internal=True)
-    await loop.connect_read_pipe(
-        lambda: _StreamProtocol(stream, loop=loop,
-                                _asyncio_internal=True),
-        pipe)
-    return stream
-
-
-def connect_write_pipe(pipe, *, limit=_DEFAULT_LIMIT):
-    """Establish a connection to a file-like object *pipe* to send data.
-
-    Takes a file-like object *pipe* to return a Stream object of the mode
-    StreamMode.WRITE that has similar API of StreamWriter. It can also be used
-    as an async context manager.
-    """
-
-    # Design note:
-    # Don't use decorator approach but explicit non-async
-    # function to fail fast and explicitly
-    # if passed arguments don't match the function signature
-    return _ContextManagerHelper(_connect_write_pipe(pipe, limit))
-
-
-async def _connect_write_pipe(pipe, limit):
-    loop = events.get_running_loop()
-    stream = Stream(mode=StreamMode.WRITE,
-                    limit=limit,
-                    loop=loop,
-                    _asyncio_internal=True)
-    await loop.connect_write_pipe(
-        lambda: _StreamProtocol(stream, loop=loop,
-                                _asyncio_internal=True),
-        pipe)
-    return stream
-
-
 async def open_connection(host=None, port=None, *,
                           loop=None, limit=_DEFAULT_LIMIT, **kwds):
     """A wrapper for create_connection() returning a (reader, writer) pair.
@@ -189,11 +41,6 @@ def connect_write_pipe(pipe, *, limit=_DEFAULT_LIMIT):
     StreamReaderProtocol classes, just copy the code -- there's
     really nothing special here except some convenience.)
     """
-    warnings.warn("open_connection() is deprecated since Python 3.8 "
-                  "in favor of connect(), and scheduled for removal "
-                  "in Python 3.10",
-                  DeprecationWarning,
-                  stacklevel=2)
     if loop is None:
         loop = events.get_event_loop()
     else:
@@ -201,7 +48,7 @@ def connect_write_pipe(pipe, *, limit=_DEFAULT_LIMIT):
                       "and scheduled for removal in Python 3.10.",
                       DeprecationWarning, stacklevel=2)
     reader = StreamReader(limit=limit, loop=loop)
-    protocol = StreamReaderProtocol(reader, loop=loop, _asyncio_internal=True)
+    protocol = StreamReaderProtocol(reader, loop=loop)
     transport, _ = await loop.create_connection(
         lambda: protocol, host, port, **kwds)
     writer = StreamWriter(transport, protocol, reader, loop)
@@ -231,11 +78,6 @@ def connect_write_pipe(pipe, *, limit=_DEFAULT_LIMIT):
     The return value is the same as loop.create_server(), i.e. a
     Server object which can be used to stop the service.
     """
-    warnings.warn("start_server() is deprecated since Python 3.8 "
-                  "in favor of StreamServer(), and scheduled for removal "
-                  "in Python 3.10",
-                  DeprecationWarning,
-                  stacklevel=2)
     if loop is None:
         loop = events.get_event_loop()
     else:
@@ -246,201 +88,18 @@ def connect_write_pipe(pipe, *, limit=_DEFAULT_LIMIT):
     def factory():
         reader = StreamReader(limit=limit, loop=loop)
         protocol = StreamReaderProtocol(reader, client_connected_cb,
-                                        loop=loop,
-                                        _asyncio_internal=True)
+                                        loop=loop)
         return protocol
 
     return await loop.create_server(factory, host, port, **kwds)
 
 
-class _BaseStreamServer:
-    # Design notes.
-    # StreamServer and UnixStreamServer are exposed as FINAL classes,
-    # not function factories.
-    # async with serve(host, port) as server:
-    #      server.start_serving()
-    # looks ugly.
-    # The class doesn't provide API for enumerating connected streams
-    # It can be a subject for improvements in Python 3.9
-
-    _server_impl = None
-
-    def __init__(self, client_connected_cb,
-                 /,
-                 limit=_DEFAULT_LIMIT,
-                 shutdown_timeout=60,
-                 _asyncio_internal=False):
-        if not _asyncio_internal:
-            raise RuntimeError("_ServerStream is a private asyncio class")
-        self._client_connected_cb = client_connected_cb
-        self._limit = limit
-        self._loop = events.get_running_loop()
-        self._streams = {}
-        self._shutdown_timeout = shutdown_timeout
-
-    def __init_subclass__(cls):
-        if not cls.__module__.startswith('asyncio.'):
-            raise TypeError(f"asyncio.{cls.__name__} "
-                            "class cannot be inherited from")
-
-    async def bind(self):
-        if self._server_impl is not None:
-            return
-        self._server_impl = await self._bind()
-
-    def is_bound(self):
-        return self._server_impl is not None
-
-    @property
-    def sockets(self):
-        # multiple value for socket bound to both IPv4 and IPv6 families
-        if self._server_impl is None:
-            return ()
-        return self._server_impl.sockets
-
-    def is_serving(self):
-        if self._server_impl is None:
-            return False
-        return self._server_impl.is_serving()
-
-    async def start_serving(self):
-        await self.bind()
-        await self._server_impl.start_serving()
-
-    async def serve_forever(self):
-        await self.start_serving()
-        await self._server_impl.serve_forever()
-
-    async def close(self):
-        if self._server_impl is None:
-            return
-        self._server_impl.close()
-        streams = list(self._streams.keys())
-        active_tasks = list(self._streams.values())
-        if streams:
-            await tasks.wait([stream.close() for stream in streams])
-        await self._server_impl.wait_closed()
-        self._server_impl = None
-        await self._shutdown_active_tasks(active_tasks)
-
-    async def abort(self):
-        if self._server_impl is None:
-            return
-        self._server_impl.close()
-        streams = list(self._streams.keys())
-        active_tasks = list(self._streams.values())
-        if streams:
-            await tasks.wait([stream.abort() for stream in streams])
-        await self._server_impl.wait_closed()
-        self._server_impl = None
-        await self._shutdown_active_tasks(active_tasks)
-
-    async def __aenter__(self):
-        await self.bind()
-        return self
-
-    async def __aexit__(self, exc_type, exc_value, exc_tb):
-        await self.close()
-
-    def _attach(self, stream, task):
-        self._streams[stream] = task
-
-    def _detach(self, stream, task):
-        del self._streams[stream]
-
-    async def _shutdown_active_tasks(self, active_tasks):
-        if not active_tasks:
-            return
-        # NOTE: tasks finished with exception are reported
-        # by the Task.__del__() method.
-        done, pending = await tasks.wait(active_tasks,
-                                         timeout=self._shutdown_timeout)
-        if not pending:
-            return
-        for task in pending:
-            task.cancel()
-        done, pending = await tasks.wait(pending,
-                                         timeout=self._shutdown_timeout)
-        for task in pending:
-            self._loop.call_exception_handler({
-                "message": (f'{task!r} ignored cancellation request '
-                            f'from a closing {self!r}'),
-                "stream_server": self
-            })
-
-    def __repr__(self):
-        ret = [f'{self.__class__.__name__}']
-        if self.is_serving():
-            ret.append('serving')
-        if self.sockets:
-            ret.append(f'sockets={self.sockets!r}')
-        return '<' + ' '.join(ret) + '>'
-
-    def __del__(self, _warn=warnings.warn):
-        if self._server_impl is not None:
-            _warn(f"unclosed stream server {self!r}",
-                  ResourceWarning, source=self)
-            self._server_impl.close()
-
-
-class StreamServer(_BaseStreamServer):
-
-    def __init__(self, client_connected_cb, /, host=None, port=None, *,
-                 limit=_DEFAULT_LIMIT,
-                 family=socket.AF_UNSPEC,
-                 flags=socket.AI_PASSIVE, sock=None, backlog=100,
-                 ssl=None, reuse_address=None, reuse_port=None,
-                 ssl_handshake_timeout=None,
-                 shutdown_timeout=60):
-        super().__init__(client_connected_cb,
-                         limit=limit,
-                         shutdown_timeout=shutdown_timeout,
-                         _asyncio_internal=True)
-        self._host = host
-        self._port = port
-        self._family = family
-        self._flags = flags
-        self._sock = sock
-        self._backlog = backlog
-        self._ssl = ssl
-        self._reuse_address = reuse_address
-        self._reuse_port = reuse_port
-        self._ssl_handshake_timeout = ssl_handshake_timeout
-
-    async def _bind(self):
-        def factory():
-            protocol = _ServerStreamProtocol(self,
-                                             self._limit,
-                                             self._client_connected_cb,
-                                             loop=self._loop,
-                                             _asyncio_internal=True)
-            return protocol
-        return await self._loop.create_server(
-            factory,
-            self._host,
-            self._port,
-            start_serving=False,
-            family=self._family,
-            flags=self._flags,
-            sock=self._sock,
-            backlog=self._backlog,
-            ssl=self._ssl,
-            reuse_address=self._reuse_address,
-            reuse_port=self._reuse_port,
-            ssl_handshake_timeout=self._ssl_handshake_timeout)
-
-
 if hasattr(socket, 'AF_UNIX'):
     # UNIX Domain Sockets are supported on this platform
 
     async def open_unix_connection(path=None, *,
                                    loop=None, limit=_DEFAULT_LIMIT, **kwds):
         """Similar to `open_connection` but works with UNIX Domain Sockets."""
-        warnings.warn("open_unix_connection() is deprecated since Python 3.8 "
-                      "in favor of connect_unix(), and scheduled for removal "
-                      "in Python 3.10",
-                      DeprecationWarning,
-                      stacklevel=2)
         if loop is None:
             loop = events.get_event_loop()
         else:
@@ -448,62 +107,15 @@ def factory():
                           "and scheduled for removal in Python 3.10.",
                           DeprecationWarning, stacklevel=2)
         reader = StreamReader(limit=limit, loop=loop)
-        protocol = StreamReaderProtocol(reader, loop=loop,
-                                        _asyncio_internal=True)
+        protocol = StreamReaderProtocol(reader, loop=loop)
         transport, _ = await loop.create_unix_connection(
             lambda: protocol, path, **kwds)
         writer = StreamWriter(transport, protocol, reader, loop)
         return reader, writer
 
-
-    def connect_unix(path=None, *,
-                     limit=_DEFAULT_LIMIT,
-                     ssl=None, sock=None,
-                     server_hostname=None,
-                     ssl_handshake_timeout=None):
-        """Similar to `connect()` but works with UNIX Domain Sockets."""
-        # Design note:
-        # Don't use decorator approach but explicit non-async
-        # function to fail fast and explicitly
-        # if passed arguments don't match the function signature
-        return _ContextManagerHelper(_connect_unix(path,
-                                                   limit,
-                                                   ssl, sock,
-                                                   server_hostname,
-                                                   ssl_handshake_timeout))
-
-
-    async def _connect_unix(path,
-                           limit,
-                           ssl, sock,
-                           server_hostname,
-                           ssl_handshake_timeout):
-        """Similar to `connect()` but works with UNIX Domain Sockets."""
-        loop = events.get_running_loop()
-        stream = Stream(mode=StreamMode.READWRITE,
-                        limit=limit,
-                        loop=loop,
-                        _asyncio_internal=True)
-        await loop.create_unix_connection(
-            lambda: _StreamProtocol(stream,
-                                    loop=loop,
-                                    _asyncio_internal=True),
-            path,
-            ssl=ssl,
-            sock=sock,
-            server_hostname=server_hostname,
-            ssl_handshake_timeout=ssl_handshake_timeout)
-        return stream
-
-
     async def start_unix_server(client_connected_cb, path=None, *,
                                 loop=None, limit=_DEFAULT_LIMIT, **kwds):
         """Similar to `start_server` but works with UNIX Domain Sockets."""
-        warnings.warn("start_unix_server() is deprecated since Python 3.8 "
-                      "in favor of UnixStreamServer(), and scheduled "
-                      "for removal in Python 3.10",
-                      DeprecationWarning,
-                      stacklevel=2)
         if loop is None:
             loop = events.get_event_loop()
         else:
@@ -514,48 +126,11 @@ def connect_unix(path=None, *,
         def factory():
             reader = StreamReader(limit=limit, loop=loop)
             protocol = StreamReaderProtocol(reader, client_connected_cb,
-                                            loop=loop,
-                                            _asyncio_internal=True)
+                                            loop=loop)
             return protocol
 
         return await loop.create_unix_server(factory, path, **kwds)
 
-    class UnixStreamServer(_BaseStreamServer):
-
-        def __init__(self, client_connected_cb, /, path=None, *,
-                     limit=_DEFAULT_LIMIT,
-                     sock=None,
-                     backlog=100,
-                     ssl=None,
-                     ssl_handshake_timeout=None,
-                     shutdown_timeout=60):
-            super().__init__(client_connected_cb,
-                             limit=limit,
-                             shutdown_timeout=shutdown_timeout,
-                             _asyncio_internal=True)
-            self._path = path
-            self._sock = sock
-            self._backlog = backlog
-            self._ssl = ssl
-            self._ssl_handshake_timeout = ssl_handshake_timeout
-
-        async def _bind(self):
-            def factory():
-                protocol = _ServerStreamProtocol(self,
-                                                 self._limit,
-                                                 self._client_connected_cb,
-                                                 loop=self._loop,
-                                                 _asyncio_internal=True)
-                return protocol
-            return await self._loop.create_unix_server(
-                factory,
-                self._path,
-                start_serving=False,
-                sock=self._sock,
-                backlog=self._backlog,
-                ssl=self._ssl,
-                ssl_handshake_timeout=self._ssl_handshake_timeout)
-
 
 class FlowControlMixin(protocols.Protocol):
     """Reusable flow control logic for StreamWriter.drain().
@@ -567,20 +142,11 @@ class FlowControlMixin(protocols.Protocol):
     StreamWriter.drain() must wait for _drain_helper() coroutine.
     """
 
-    def __init__(self, loop=None, *, _asyncio_internal=False):
+    def __init__(self, loop=None):
         if loop is None:
             self._loop = events.get_event_loop()
         else:
             self._loop = loop
-        if not _asyncio_internal:
-            # NOTE:
-            # Avoid inheritance from FlowControlMixin
-            # Copy-paste the code to your project
-            # if you need flow control helpers
-            warnings.warn(f"{self.__class__} should be instantiated "
-                          "by asyncio internals only, "
-                          "please avoid its creation from user code",
-                          DeprecationWarning)
         self._paused = False
         self._drain_waiter = None
         self._connection_lost = False
@@ -634,8 +200,6 @@ def _get_close_waiter(self, stream):
         raise NotImplementedError
 
 
-# begin legacy stream APIs
-
 class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
     """Helper class to adapt between Protocol and StreamReader.
 
@@ -645,47 +209,103 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
     call inappropriate methods of the protocol.)
     """
 
-    def __init__(self, stream_reader, client_connected_cb=None, loop=None,
-                 *, _asyncio_internal=False):
-        super().__init__(loop=loop, _asyncio_internal=_asyncio_internal)
-        self._stream_reader = stream_reader
+    _source_traceback = None
+
+    def __init__(self, stream_reader, client_connected_cb=None, loop=None):
+        super().__init__(loop=loop)
+        if stream_reader is not None:
+            self._stream_reader_wr = weakref.ref(stream_reader,
+                                                 self._on_reader_gc)
+            self._source_traceback = stream_reader._source_traceback
+        else:
+            self._stream_reader_wr = None
+        if client_connected_cb is not None:
+            # This is a stream created by the `create_server()` function.
+            # Keep a strong reference to the reader until a connection
+            # is established.
+            self._strong_reader = stream_reader
+        self._reject_connection = False
         self._stream_writer = None
+        self._transport = None
         self._client_connected_cb = client_connected_cb
         self._over_ssl = False
         self._closed = self._loop.create_future()
 
+    def _on_reader_gc(self, wr):
+        transport = self._transport
+        if transport is not None:
+            # connection_made was called
+            context = {
+                'message': ('An open stream object is being garbage '
+                            'collected; call "stream.close()" explicitly.')
+            }
+            if self._source_traceback:
+                context['source_traceback'] = self._source_traceback
+            self._loop.call_exception_handler(context)
+            transport.abort()
+        else:
+            self._reject_connection = True
+        self._stream_reader_wr = None
+
+    @property
+    def _stream_reader(self):
+        if self._stream_reader_wr is None:
+            return None
+        return self._stream_reader_wr()
+
     def connection_made(self, transport):
-        self._stream_reader.set_transport(transport)
+        if self._reject_connection:
+            context = {
+                'message': ('An open stream was garbage collected prior to '
+                            'establishing network connection; '
+                            'call "stream.close()" explicitly.')
+            }
+            if self._source_traceback:
+                context['source_traceback'] = self._source_traceback
+            self._loop.call_exception_handler(context)
+            transport.abort()
+            return
+        self._transport = transport
+        reader = self._stream_reader
+        if reader is not None:
+            reader.set_transport(transport)
         self._over_ssl = transport.get_extra_info('sslcontext') is not None
         if self._client_connected_cb is not None:
             self._stream_writer = StreamWriter(transport, self,
-                                               self._stream_reader,
+                                               reader,
                                                self._loop)
-            res = self._client_connected_cb(self._stream_reader,
+            res = self._client_connected_cb(reader,
                                             self._stream_writer)
             if coroutines.iscoroutine(res):
                 self._loop.create_task(res)
+            self._strong_reader = None
 
     def connection_lost(self, exc):
-        if self._stream_reader is not None:
+        reader = self._stream_reader
+        if reader is not None:
             if exc is None:
-                self._stream_reader.feed_eof()
+                reader.feed_eof()
             else:
-                self._stream_reader.set_exception(exc)
+                reader.set_exception(exc)
         if not self._closed.done():
             if exc is None:
                 self._closed.set_result(None)
             else:
                 self._closed.set_exception(exc)
         super().connection_lost(exc)
-        self._stream_reader = None
+        self._stream_reader_wr = None
         self._stream_writer = None
+        self._transport = None
 
     def data_received(self, data):
-        self._stream_reader.feed_data(data)
+        reader = self._stream_reader
+        if reader is not None:
+            reader.feed_data(data)
 
     def eof_received(self):
-        self._stream_reader.feed_eof()
+        reader = self._stream_reader
+        if reader is not None:
+            reader.feed_eof()
         if self._over_ssl:
             # Prevent a warning in SSLProtocol.eof_received:
             # "returning true from eof_received()
@@ -693,6 +313,9 @@ def eof_received(self):
             return False
         return True
 
+    def _get_close_waiter(self, stream):
+        return self._closed
+
     def __del__(self):
         # Prevent reports about unhandled exceptions.
         # Better than self._closed._log_traceback = False hack
@@ -718,6 +341,8 @@ def __init__(self, transport, protocol, reader, loop):
         assert reader is None or isinstance(reader, StreamReader)
         self._reader = reader
         self._loop = loop
+        self._complete_fut = self._loop.create_future()
+        self._complete_fut.set_result(None)
 
     def __repr__(self):
         info = [self.__class__.__name__, f'transport={self._transport!r}']
@@ -748,7 +373,7 @@ def is_closing(self):
         return self._transport.is_closing()
 
     async def wait_closed(self):
-        await self._protocol._closed
+        await self._protocol._get_close_waiter(self)
 
     def get_extra_info(self, name, default=None):
         return self._transport.get_extra_info(name, default)
@@ -766,18 +391,23 @@ def get_extra_info(self, name, default=None):
             if exc is not None:
                 raise exc
         if self._transport.is_closing():
+            # Wait for protocol.connection_lost() call
+            # Raise connection closing error if any,
+            # ConnectionResetError otherwise
             # Yield to the event loop so connection_lost() may be
             # called.  Without this, _drain_helper() would return
             # immediately, and code that calls
             #     write(...); await drain()
             # in a loop would never call connection_lost(), so it
             # would not see an error when the socket is closed.
-            await tasks.sleep(0, loop=self._loop)
+            await sleep(0)
         await self._protocol._drain_helper()
 
 
 class StreamReader:
 
+    _source_traceback = None
+
     def __init__(self, limit=_DEFAULT_LIMIT, loop=None):
         # The line length limit is  a security feature;
         # it also doubles as half the buffer limit.
@@ -796,6 +426,9 @@ def __init__(self, limit=_DEFAULT_LIMIT, loop=None):
         self._exception = None
         self._transport = None
         self._paused = False
+        if self._loop.get_debug():
+            self._source_traceback = format_helpers.extract_stack(
+                sys._getframe(1))
 
     def __repr__(self):
         info = ['StreamReader']
@@ -1123,706 +756,3 @@ def __aiter__(self):
         if val == b'':
             raise StopAsyncIteration
         return val
-
-
-# end legacy stream APIs
-
-
-class _BaseStreamProtocol(FlowControlMixin, protocols.Protocol):
-    """Helper class to adapt between Protocol and StreamReader.
-
-    (This is a helper class instead of making StreamReader itself a
-    Protocol subclass, because the StreamReader has other potential
-    uses, and to prevent the user of the StreamReader to accidentally
-    call inappropriate methods of the protocol.)
-    """
-
-    _stream = None  # initialized in derived classes
-
-    def __init__(self, loop=None,
-                 *, _asyncio_internal=False):
-        super().__init__(loop=loop, _asyncio_internal=_asyncio_internal)
-        self._transport = None
-        self._over_ssl = False
-        self._closed = self._loop.create_future()
-
-    def connection_made(self, transport):
-        self._transport = transport
-        self._over_ssl = transport.get_extra_info('sslcontext') is not None
-
-    def connection_lost(self, exc):
-        stream = self._stream
-        if stream is not None:
-            if exc is None:
-                stream._feed_eof()
-            else:
-                stream._set_exception(exc)
-        if not self._closed.done():
-            if exc is None:
-                self._closed.set_result(None)
-            else:
-                self._closed.set_exception(exc)
-        super().connection_lost(exc)
-        self._transport = None
-
-    def data_received(self, data):
-        stream = self._stream
-        if stream is not None:
-            stream._feed_data(data)
-
-    def eof_received(self):
-        stream = self._stream
-        if stream is not None:
-            stream._feed_eof()
-        if self._over_ssl:
-            # Prevent a warning in SSLProtocol.eof_received:
-            # "returning true from eof_received()
-            # has no effect when using ssl"
-            return False
-        return True
-
-    def _get_close_waiter(self, stream):
-        return self._closed
-
-    def __del__(self):
-        # Prevent reports about unhandled exceptions.
-        # Better than self._closed._log_traceback = False hack
-        closed = self._get_close_waiter(self._stream)
-        if closed.done() and not closed.cancelled():
-            closed.exception()
-
-
-class _StreamProtocol(_BaseStreamProtocol):
-    _source_traceback = None
-
-    def __init__(self, stream, loop=None,
-                 *, _asyncio_internal=False):
-        super().__init__(loop=loop, _asyncio_internal=_asyncio_internal)
-        self._source_traceback = stream._source_traceback
-        self._stream_wr = weakref.ref(stream, self._on_gc)
-        self._reject_connection = False
-
-    def _on_gc(self, wr):
-        transport = self._transport
-        if transport is not None:
-            # connection_made was called
-            context = {
-                'message': ('An open stream object is being garbage '
-                            'collected; call "stream.close()" explicitly.')
-            }
-            if self._source_traceback:
-                context['source_traceback'] = self._source_traceback
-            self._loop.call_exception_handler(context)
-            transport.abort()
-        else:
-            self._reject_connection = True
-        self._stream_wr = None
-
-    @property
-    def _stream(self):
-        if self._stream_wr is None:
-            return None
-        return self._stream_wr()
-
-    def connection_made(self, transport):
-        if self._reject_connection:
-            context = {
-                'message': ('An open stream was garbage collected prior to '
-                            'establishing network connection; '
-                            'call "stream.close()" explicitly.')
-            }
-            if self._source_traceback:
-                context['source_traceback'] = self._source_traceback
-            self._loop.call_exception_handler(context)
-            transport.abort()
-            return
-        super().connection_made(transport)
-        stream = self._stream
-        if stream is None:
-            return
-        stream._set_transport(transport)
-        stream._protocol = self
-
-    def connection_lost(self, exc):
-        super().connection_lost(exc)
-        self._stream_wr = None
-
-
-class _ServerStreamProtocol(_BaseStreamProtocol):
-    def __init__(self, server, limit, client_connected_cb, loop=None,
-                 *, _asyncio_internal=False):
-        super().__init__(loop=loop, _asyncio_internal=_asyncio_internal)
-        assert self._closed
-        self._client_connected_cb = client_connected_cb
-        self._limit = limit
-        self._server = server
-        self._task = None
-
-    def connection_made(self, transport):
-        super().connection_made(transport)
-        stream = Stream(mode=StreamMode.READWRITE,
-                        transport=transport,
-                        protocol=self,
-                        limit=self._limit,
-                        loop=self._loop,
-                        is_server_side=True,
-                        _asyncio_internal=True)
-        self._stream = stream
-        # If self._client_connected_cb(self._stream) fails
-        # the exception is logged by transport
-        self._task = self._loop.create_task(
-            self._client_connected_cb(self._stream))
-        self._server._attach(stream, self._task)
-
-    def connection_lost(self, exc):
-        super().connection_lost(exc)
-        self._server._detach(self._stream, self._task)
-        self._stream = None
-
-
-class _OptionalAwait:
-    # The class doesn't create a coroutine
-    # if not awaited
-    # It prevents "coroutine is never awaited" message
-
-    __slots___ = ('_method',)
-
-    def __init__(self, method):
-        self._method = method
-
-    def __await__(self):
-        return self._method().__await__()
-
-
-class Stream:
-    """Wraps a Transport.
-
-    This exposes write(), writelines(), [can_]write_eof(),
-    get_extra_info() and close().  It adds drain() which returns an
-    optional Future on which you can wait for flow control.  It also
-    adds a transport property which references the Transport
-    directly.
-    """
-
-    _source_traceback = None
-
-    def __init__(self, mode, *,
-                 transport=None,
-                 protocol=None,
-                 loop=None,
-                 limit=_DEFAULT_LIMIT,
-                 is_server_side=False,
-                 _asyncio_internal=False):
-        if not _asyncio_internal:
-            raise RuntimeError(f"{self.__class__} should be instantiated "
-                               "by asyncio internals only")
-        self._mode = mode
-        self._transport = transport
-        self._protocol = protocol
-        self._is_server_side = is_server_side
-
-        # The line length limit is  a security feature;
-        # it also doubles as half the buffer limit.
-
-        if limit <= 0:
-            raise ValueError('Limit cannot be <= 0')
-
-        self._limit = limit
-        if loop is None:
-            self._loop = events.get_event_loop()
-        else:
-            self._loop = loop
-        self._buffer = bytearray()
-        self._eof = False    # Whether we're done.
-        self._waiter = None  # A future used by _wait_for_data()
-        self._exception = None
-        self._paused = False
-        self._complete_fut = self._loop.create_future()
-        self._complete_fut.set_result(None)
-
-        if self._loop.get_debug():
-            self._source_traceback = format_helpers.extract_stack(
-                sys._getframe(1))
-
-    def __repr__(self):
-        info = [self.__class__.__name__]
-        info.append(f'mode={self._mode}')
-        if self._buffer:
-            info.append(f'{len(self._buffer)} bytes')
-        if self._eof:
-            info.append('eof')
-        if self._limit != _DEFAULT_LIMIT:
-            info.append(f'limit={self._limit}')
-        if self._waiter:
-            info.append(f'waiter={self._waiter!r}')
-        if self._exception:
-            info.append(f'exception={self._exception!r}')
-        if self._transport:
-            info.append(f'transport={self._transport!r}')
-        if self._paused:
-            info.append('paused')
-        return '<{}>'.format(' '.join(info))
-
-    @property
-    def mode(self):
-        return self._mode
-
-    def is_server_side(self):
-        return self._is_server_side
-
-    @property
-    def transport(self):
-        warnings.warn("Stream.transport attribute is deprecated "
-                      "since Python 3.8 and is scheduled for removal in 3.10; "
-                      "it is an internal API",
-                      DeprecationWarning,
-                      stacklevel=2)
-        return self._transport
-
-    def write(self, data):
-        _ensure_can_write(self._mode)
-        self._transport.write(data)
-        return self._fast_drain()
-
-    def writelines(self, data):
-        _ensure_can_write(self._mode)
-        self._transport.writelines(data)
-        return self._fast_drain()
-
-    def _fast_drain(self):
-        # The helper tries to use fast-path to return already existing
-        # complete future object if underlying transport is not paused
-        # and actual waiting for writing resume is not needed
-        exc = self.exception()
-        if exc is not None:
-            fut = self._loop.create_future()
-            fut.set_exception(exc)
-            return fut
-        if not self._transport.is_closing():
-            if self._protocol._connection_lost:
-                fut = self._loop.create_future()
-                fut.set_exception(ConnectionResetError('Connection lost'))
-                return fut
-            if not self._protocol._paused:
-                # fast path, the stream is not paused
-                # no need to wait for resume signal
-                return self._complete_fut
-        return _OptionalAwait(self.drain)
-
-    def write_eof(self):
-        _ensure_can_write(self._mode)
-        return self._transport.write_eof()
-
-    def can_write_eof(self):
-        if not self._mode.is_write():
-            return False
-        return self._transport.can_write_eof()
-
-    def close(self):
-        self._transport.close()
-        return _OptionalAwait(self.wait_closed)
-
-    def is_closing(self):
-        return self._transport.is_closing()
-
-    async def abort(self):
-        self._transport.abort()
-        await self.wait_closed()
-
-    async def wait_closed(self):
-        await self._protocol._get_close_waiter(self)
-
-    def get_extra_info(self, name, default=None):
-        return self._transport.get_extra_info(name, default)
-
-    async def drain(self):
-        """Flush the write buffer.
-
-        The intended use is to write
-
-          w.write(data)
-          await w.drain()
-        """
-        _ensure_can_write(self._mode)
-        exc = self.exception()
-        if exc is not None:
-            raise exc
-        if self._transport.is_closing():
-            # Wait for protocol.connection_lost() call
-            # Raise connection closing error if any,
-            # ConnectionResetError otherwise
-            await tasks.sleep(0)
-        await self._protocol._drain_helper()
-
-    async def sendfile(self, file, offset=0, count=None, *, fallback=True):
-        await self.drain()  # check for stream mode and exceptions
-        return await self._loop.sendfile(self._transport, file,
-                                         offset, count, fallback=fallback)
-
-    async def start_tls(self, sslcontext, *,
-                        server_hostname=None,
-                        ssl_handshake_timeout=None):
-        await self.drain()  # check for stream mode and exceptions
-        transport = await self._loop.start_tls(
-            self._transport, self._protocol, sslcontext,
-            server_side=self._is_server_side,
-            server_hostname=server_hostname,
-            ssl_handshake_timeout=ssl_handshake_timeout)
-        self._transport = transport
-        self._protocol._transport = transport
-        self._protocol._over_ssl = True
-
-    def exception(self):
-        return self._exception
-
-    def set_exception(self, exc):
-        warnings.warn("Stream.set_exception() is deprecated "
-                      "since Python 3.8 and is scheduled for removal in 3.10; "
-                      "it is an internal API",
-                      DeprecationWarning,
-                      stacklevel=2)
-        self._set_exception(exc)
-
-    def _set_exception(self, exc):
-        self._exception = exc
-
-        waiter = self._waiter
-        if waiter is not None:
-            self._waiter = None
-            if not waiter.cancelled():
-                waiter.set_exception(exc)
-
-    def _wakeup_waiter(self):
-        """Wakeup read*() functions waiting for data or EOF."""
-        waiter = self._waiter
-        if waiter is not None:
-            self._waiter = None
-            if not waiter.cancelled():
-                waiter.set_result(None)
-
-    def set_transport(self, transport):
-        warnings.warn("Stream.set_transport() is deprecated "
-                      "since Python 3.8 and is scheduled for removal in 3.10; "
-                      "it is an internal API",
-                      DeprecationWarning,
-                      stacklevel=2)
-        self._set_transport(transport)
-
-    def _set_transport(self, transport):
-        if transport is self._transport:
-            return
-        assert self._transport is None, 'Transport already set'
-        self._transport = transport
-
-    def _maybe_resume_transport(self):
-        if self._paused and len(self._buffer) <= self._limit:
-            self._paused = False
-            self._transport.resume_reading()
-
-    def feed_eof(self):
-        warnings.warn("Stream.feed_eof() is deprecated "
-                      "since Python 3.8 and is scheduled for removal in 3.10; "
-                      "it is an internal API",
-                      DeprecationWarning,
-                      stacklevel=2)
-        self._feed_eof()
-
-    def _feed_eof(self):
-        self._eof = True
-        self._wakeup_waiter()
-
-    def at_eof(self):
-        """Return True if the buffer is empty and 'feed_eof' was called."""
-        return self._eof and not self._buffer
-
-    def feed_data(self, data):
-        warnings.warn("Stream.feed_data() is deprecated "
-                      "since Python 3.8 and is scheduled for removal in 3.10; "
-                      "it is an internal API",
-                      DeprecationWarning,
-                      stacklevel=2)
-        self._feed_data(data)
-
-    def _feed_data(self, data):
-        _ensure_can_read(self._mode)
-        assert not self._eof, 'feed_data after feed_eof'
-
-        if not data:
-            return
-
-        self._buffer.extend(data)
-        self._wakeup_waiter()
-
-        if (self._transport is not None and
-                not self._paused and
-                len(self._buffer) > 2 * self._limit):
-            try:
-                self._transport.pause_reading()
-            except NotImplementedError:
-                # The transport can't be paused.
-                # We'll just have to buffer all data.
-                # Forget the transport so we don't keep trying.
-                self._transport = None
-            else:
-                self._paused = True
-
-    async def _wait_for_data(self, func_name):
-        """Wait until feed_data() or feed_eof() is called.
-
-        If stream was paused, automatically resume it.
-        """
-        # StreamReader uses a future to link the protocol feed_data() method
-        # to a read coroutine. Running two read coroutines at the same time
-        # would have an unexpected behaviour. It would not possible to know
-        # which coroutine would get the next data.
-        if self._waiter is not None:
-            raise RuntimeError(
-                f'{func_name}() called while another coroutine is '
-                f'already waiting for incoming data')
-
-        assert not self._eof, '_wait_for_data after EOF'
-
-        # Waiting for data while paused will make deadlock, so prevent it.
-        # This is essential for readexactly(n) for case when n > self._limit.
-        if self._paused:
-            self._paused = False
-            self._transport.resume_reading()
-
-        self._waiter = self._loop.create_future()
-        try:
-            await self._waiter
-        finally:
-            self._waiter = None
-
-    async def readline(self):
-        """Read chunk of data from the stream until newline (b'\n') is found.
-
-        On success, return chunk that ends with newline. If only partial
-        line can be read due to EOF, return incomplete line without
-        terminating newline. When EOF was reached while no bytes read, empty
-        bytes object is returned.
-
-        If limit is reached, ValueError will be raised. In that case, if
-        newline was found, complete line including newline will be removed
-        from internal buffer. Else, internal buffer will be cleared. Limit is
-        compared against part of the line without newline.
-
-        If stream was paused, this function will automatically resume it if
-        needed.
-        """
-        _ensure_can_read(self._mode)
-        sep = b'\n'
-        seplen = len(sep)
-        try:
-            line = await self.readuntil(sep)
-        except exceptions.IncompleteReadError as e:
-            return e.partial
-        except exceptions.LimitOverrunError as e:
-            if self._buffer.startswith(sep, e.consumed):
-                del self._buffer[:e.consumed + seplen]
-            else:
-                self._buffer.clear()
-            self._maybe_resume_transport()
-            raise ValueError(e.args[0])
-        return line
-
-    async def readuntil(self, separator=b'\n'):
-        """Read data from the stream until ``separator`` is found.
-
-        On success, the data and separator will be removed from the
-        internal buffer (consumed). Returned data will include the
-        separator at the end.
-
-        Configured stream limit is used to check result. Limit sets the
-        maximal length of data that can be returned, not counting the
-        separator.
-
-        If an EOF occurs and the complete separator is still not found,
-        an IncompleteReadError exception will be raised, and the internal
-        buffer will be reset.  The IncompleteReadError.partial attribute
-        may contain the separator partially.
-
-        If the data cannot be read because of over limit, a
-        LimitOverrunError exception  will be raised, and the data
-        will be left in the internal buffer, so it can be read again.
-        """
-        _ensure_can_read(self._mode)
-        seplen = len(separator)
-        if seplen == 0:
-            raise ValueError('Separator should be at least one-byte string')
-
-        if self._exception is not None:
-            raise self._exception
-
-        # Consume whole buffer except last bytes, which length is
-        # one less than seplen. Let's check corner cases with
-        # separator='SEPARATOR':
-        # * we have received almost complete separator (without last
-        #   byte). i.e buffer='some textSEPARATO'. In this case we
-        #   can safely consume len(separator) - 1 bytes.
-        # * last byte of buffer is first byte of separator, i.e.
-        #   buffer='abcdefghijklmnopqrS'. We may safely consume
-        #   everything except that last byte, but this require to
-        #   analyze bytes of buffer that match partial separator.
-        #   This is slow and/or require FSM. For this case our
-        #   implementation is not optimal, since require rescanning
-        #   of data that is known to not belong to separator. In
-        #   real world, separator will not be so long to notice
-        #   performance problems. Even when reading MIME-encoded
-        #   messages :)
-
-        # `offset` is the number of bytes from the beginning of the buffer
-        # where there is no occurrence of `separator`.
-        offset = 0
-
-        # Loop until we find `separator` in the buffer, exceed the buffer size,
-        # or an EOF has happened.
-        while True:
-            buflen = len(self._buffer)
-
-            # Check if we now have enough data in the buffer for `separator` to
-            # fit.
-            if buflen - offset >= seplen:
-                isep = self._buffer.find(separator, offset)
-
-                if isep != -1:
-                    # `separator` is in the buffer. `isep` will be used later
-                    # to retrieve the data.
-                    break
-
-                # see upper comment for explanation.
-                offset = buflen + 1 - seplen
-                if offset > self._limit:
-                    raise exceptions.LimitOverrunError(
-                        'Separator is not found, and chunk exceed the limit',
-                        offset)
-
-            # Complete message (with full separator) may be present in buffer
-            # even when EOF flag is set. This may happen when the last chunk
-            # adds data which makes separator be found. That's why we check for
-            # EOF *ater* inspecting the buffer.
-            if self._eof:
-                chunk = bytes(self._buffer)
-                self._buffer.clear()
-                raise exceptions.IncompleteReadError(chunk, None)
-
-            # _wait_for_data() will resume reading if stream was paused.
-            await self._wait_for_data('readuntil')
-
-        if isep > self._limit:
-            raise exceptions.LimitOverrunError(
-                'Separator is found, but chunk is longer than limit', isep)
-
-        chunk = self._buffer[:isep + seplen]
-        del self._buffer[:isep + seplen]
-        self._maybe_resume_transport()
-        return bytes(chunk)
-
-    async def read(self, n=-1):
-        """Read up to `n` bytes from the stream.
-
-        If n is not provided, or set to -1, read until EOF and return all read
-        bytes. If the EOF was received and the internal buffer is empty, return
-        an empty bytes object.
-
-        If n is zero, return empty bytes object immediately.
-
-        If n is positive, this function try to read `n` bytes, and may return
-        less or equal bytes than requested, but at least one byte. If EOF was
-        received before any byte is read, this function returns empty byte
-        object.
-
-        Returned value is not limited with limit, configured at stream
-        creation.
-
-        If stream was paused, this function will automatically resume it if
-        needed.
-        """
-        _ensure_can_read(self._mode)
-
-        if self._exception is not None:
-            raise self._exception
-
-        if n == 0:
-            return b''
-
-        if n < 0:
-            # This used to just loop creating a new waiter hoping to
-            # collect everything in self._buffer, but that would
-            # deadlock if the subprocess sends more than self.limit
-            # bytes.  So just call self.read(self._limit) until EOF.
-            blocks = []
-            while True:
-                block = await self.read(self._limit)
-                if not block:
-                    break
-                blocks.append(block)
-            return b''.join(blocks)
-
-        if not self._buffer and not self._eof:
-            await self._wait_for_data('read')
-
-        # This will work right even if buffer is less than n bytes
-        data = bytes(self._buffer[:n])
-        del self._buffer[:n]
-
-        self._maybe_resume_transport()
-        return data
-
-    async def readexactly(self, n):
-        """Read exactly `n` bytes.
-
-        Raise an IncompleteReadError if EOF is reached before `n` bytes can be
-        read. The IncompleteReadError.partial attribute of the exception will
-        contain the partial read bytes.
-
-        if n is zero, return empty bytes object.
-
-        Returned value is not limited with limit, configured at stream
-        creation.
-
-        If stream was paused, this function will automatically resume it if
-        needed.
-        """
-        _ensure_can_read(self._mode)
-        if n < 0:
-            raise ValueError('readexactly size can not be less than zero')
-
-        if self._exception is not None:
-            raise self._exception
-
-        if n == 0:
-            return b''
-
-        while len(self._buffer) < n:
-            if self._eof:
-                incomplete = bytes(self._buffer)
-                self._buffer.clear()
-                raise exceptions.IncompleteReadError(incomplete, n)
-
-            await self._wait_for_data('readexactly')
-
-        if len(self._buffer) == n:
-            data = bytes(self._buffer)
-            self._buffer.clear()
-        else:
-            data = bytes(self._buffer[:n])
-            del self._buffer[:n]
-        self._maybe_resume_transport()
-        return data
-
-    def __aiter__(self):
-        _ensure_can_read(self._mode)
-        return self
-
-    async def __anext__(self):
-        val = await self.readline()
-        if val == b'':
-            raise StopAsyncIteration
-        return val
-
-    async def __aenter__(self):
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        await self.close()
diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py
index bddfb0146402..c9506b158302 100644
--- a/Lib/asyncio/subprocess.py
+++ b/Lib/asyncio/subprocess.py
@@ -19,16 +19,14 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
                                protocols.SubprocessProtocol):
     """Like StreamReaderProtocol, but for a subprocess."""
 
-    def __init__(self, limit, loop, *, _asyncio_internal=False):
-        super().__init__(loop=loop, _asyncio_internal=_asyncio_internal)
+    def __init__(self, limit, loop):
+        super().__init__(loop=loop)
         self._limit = limit
         self.stdin = self.stdout = self.stderr = None
         self._transport = None
         self._process_exited = False
         self._pipe_fds = []
         self._stdin_closed = self._loop.create_future()
-        self._stdout_closed = self._loop.create_future()
-        self._stderr_closed = self._loop.create_future()
 
     def __repr__(self):
         info = [self.__class__.__name__]
@@ -42,35 +40,27 @@ def __repr__(self):
 
     def connection_made(self, transport):
         self._transport = transport
+
         stdout_transport = transport.get_pipe_transport(1)
         if stdout_transport is not None:
-            self.stdout = streams.Stream(mode=streams.StreamMode.READ,
-                                         transport=stdout_transport,
-                                         protocol=self,
-                                         limit=self._limit,
-                                         loop=self._loop,
-                                         _asyncio_internal=True)
-            self.stdout._set_transport(stdout_transport)
+            self.stdout = streams.StreamReader(limit=self._limit,
+                                               loop=self._loop)
+            self.stdout.set_transport(stdout_transport)
             self._pipe_fds.append(1)
 
         stderr_transport = transport.get_pipe_transport(2)
         if stderr_transport is not None:
-            self.stderr = streams.Stream(mode=streams.StreamMode.READ,
-                                         transport=stderr_transport,
-                                         protocol=self,
-                                         limit=self._limit,
-                                         loop=self._loop,
-                                         _asyncio_internal=True)
-            self.stderr._set_transport(stderr_transport)
+            self.stderr = streams.StreamReader(limit=self._limit,
+                                               loop=self._loop)
+            self.stderr.set_transport(stderr_transport)
             self._pipe_fds.append(2)
 
         stdin_transport = transport.get_pipe_transport(0)
         if stdin_transport is not None:
-            self.stdin = streams.Stream(mode=streams.StreamMode.WRITE,
-                                        transport=stdin_transport,
-                                        protocol=self,
-                                        loop=self._loop,
-                                        _asyncio_internal=True)
+            self.stdin = streams.StreamWriter(stdin_transport,
+                                              protocol=self,
+                                              reader=None,
+                                              loop=self._loop)
 
     def pipe_data_received(self, fd, data):
         if fd == 1:
@@ -80,7 +70,7 @@ def pipe_data_received(self, fd, data):
         else:
             reader = None
         if reader is not None:
-            reader._feed_data(data)
+            reader.feed_data(data)
 
     def pipe_connection_lost(self, fd, exc):
         if fd == 0:
@@ -101,9 +91,9 @@ def pipe_connection_lost(self, fd, exc):
             reader = None
         if reader is not None:
             if exc is None:
-                reader._feed_eof()
+                reader.feed_eof()
             else:
-                reader._set_exception(exc)
+                reader.set_exception(exc)
 
         if fd in self._pipe_fds:
             self._pipe_fds.remove(fd)
@@ -121,20 +111,10 @@ def _maybe_close_transport(self):
     def _get_close_waiter(self, stream):
         if stream is self.stdin:
             return self._stdin_closed
-        elif stream is self.stdout:
-            return self._stdout_closed
-        elif stream is self.stderr:
-            return self._stderr_closed
 
 
 class Process:
-    def __init__(self, transport, protocol, loop, *, _asyncio_internal=False):
-        if not _asyncio_internal:
-            warnings.warn(f"{self.__class__} should be instantiated "
-                          "by asyncio internals only, "
-                          "please avoid its creation from user code",
-                          DeprecationWarning)
-
+    def __init__(self, transport, protocol, loop):
         self._transport = transport
         self._protocol = protocol
         self._loop = loop
@@ -232,13 +212,12 @@ def kill(self):
         )
 
     protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
-                                                        loop=loop,
-                                                        _asyncio_internal=True)
+                                                        loop=loop)
     transport, protocol = await loop.subprocess_shell(
         protocol_factory,
         cmd, stdin=stdin, stdout=stdout,
         stderr=stderr, **kwds)
-    return Process(transport, protocol, loop, _asyncio_internal=True)
+    return Process(transport, protocol, loop)
 
 
 async def create_subprocess_exec(program, *args, stdin=None, stdout=None,
@@ -253,11 +232,10 @@ def kill(self):
                       stacklevel=2
         )
     protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
-                                                        loop=loop,
-                                                        _asyncio_internal=True)
+                                                        loop=loop)
     transport, protocol = await loop.subprocess_exec(
         protocol_factory,
         program, *args,
         stdin=stdin, stdout=stdout,
         stderr=stderr, **kwds)
-    return Process(transport, protocol, loop, _asyncio_internal=True)
+    return Process(transport, protocol, loop)
diff --git a/Lib/test/test_asyncio/test_buffered_proto.py b/Lib/test/test_asyncio/test_buffered_proto.py
index b1531fb9343f..f24e363ebfcf 100644
--- a/Lib/test/test_asyncio/test_buffered_proto.py
+++ b/Lib/test/test_asyncio/test_buffered_proto.py
@@ -58,10 +58,9 @@ def on_buf(buf):
             writer.close()
             await writer.wait_closed()
 
-        with self.assertWarns(DeprecationWarning):
-            srv = self.loop.run_until_complete(
-                asyncio.start_server(
-                    on_server_client, '127.0.0.1', 0))
+        srv = self.loop.run_until_complete(
+            asyncio.start_server(
+                on_server_client, '127.0.0.1', 0))
 
         addr = srv.sockets[0].getsockname()
         self.loop.run_until_complete(
diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py
index 58a609444285..a1f27dd5721c 100644
--- a/Lib/test/test_asyncio/test_pep492.py
+++ b/Lib/test/test_asyncio/test_pep492.py
@@ -95,11 +95,9 @@ class StreamReaderTests(BaseTest):
     def test_readline(self):
         DATA = b'line1\nline2\nline3'
 
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(DATA)
-        stream._feed_eof()
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(DATA)
+        stream.feed_eof()
 
         async def reader():
             data = []
diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py
index 6325ee3983cc..b9413ab35fc5 100644
--- a/Lib/test/test_asyncio/test_streams.py
+++ b/Lib/test/test_asyncio/test_streams.py
@@ -1,8 +1,6 @@
 """Tests for streams.py."""
 
-import contextlib
 import gc
-import io
 import os
 import queue
 import pickle
@@ -18,7 +16,6 @@
     ssl = None
 
 import asyncio
-from asyncio.streams import _StreamProtocol, _ensure_can_read, _ensure_can_write
 from test.test_asyncio import utils as test_utils
 
 
@@ -26,24 +23,6 @@ def tearDownModule():
     asyncio.set_event_loop_policy(None)
 
 
-class StreamModeTests(unittest.TestCase):
-    def test__ensure_can_read_ok(self):
-        self.assertIsNone(_ensure_can_read(asyncio.StreamMode.READ))
-        self.assertIsNone(_ensure_can_read(asyncio.StreamMode.READWRITE))
-
-    def test__ensure_can_read_fail(self):
-        with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-            _ensure_can_read(asyncio.StreamMode.WRITE)
-
-    def test__ensure_can_write_ok(self):
-        self.assertIsNone(_ensure_can_write(asyncio.StreamMode.WRITE))
-        self.assertIsNone(_ensure_can_write(asyncio.StreamMode.READWRITE))
-
-    def test__ensure_can_write_fail(self):
-        with self.assertRaisesRegex(RuntimeError, "The stream is read-only"):
-            _ensure_can_write(asyncio.StreamMode.READ)
-
-
 class StreamTests(test_utils.TestCase):
 
     DATA = b'line1\nline2\nline3\n'
@@ -63,8 +42,7 @@ def tearDown(self):
 
     @mock.patch('asyncio.streams.events')
     def test_ctor_global_loop(self, m_events):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader()
         self.assertIs(stream._loop, m_events.get_event_loop.return_value)
 
     def _basetest_open_connection(self, open_connection_fut):
@@ -100,8 +78,7 @@ def _basetest_open_connection_no_loop_ssl(self, open_connection_fut):
         self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
         try:
             with self.assertWarns(DeprecationWarning):
-                reader, writer = self.loop.run_until_complete(
-                    open_connection_fut)
+                reader, writer = self.loop.run_until_complete(open_connection_fut)
         finally:
             asyncio.set_event_loop(None)
         writer.write(b'GET / HTTP/1.0\r\n\r\n')
@@ -161,27 +138,21 @@ def test_open_unix_connection_error(self):
             self._basetest_open_connection_error(conn_fut)
 
     def test_feed_empty_data(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
-        stream._feed_data(b'')
+        stream.feed_data(b'')
         self.assertEqual(b'', stream._buffer)
 
     def test_feed_nonempty_data(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
-        stream._feed_data(self.DATA)
+        stream.feed_data(self.DATA)
         self.assertEqual(self.DATA, stream._buffer)
 
     def test_read_zero(self):
         # Read zero bytes.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(self.DATA)
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(self.DATA)
 
         data = self.loop.run_until_complete(stream.read(0))
         self.assertEqual(b'', data)
@@ -189,13 +160,11 @@ def test_read_zero(self):
 
     def test_read(self):
         # Read bytes.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         read_task = self.loop.create_task(stream.read(30))
 
         def cb():
-            stream._feed_data(self.DATA)
+            stream.feed_data(self.DATA)
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -204,11 +173,9 @@ def cb():
 
     def test_read_line_breaks(self):
         # Read bytes without line breaks.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'line1')
-        stream._feed_data(b'line2')
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'line1')
+        stream.feed_data(b'line2')
 
         data = self.loop.run_until_complete(stream.read(5))
 
@@ -217,13 +184,11 @@ def test_read_line_breaks(self):
 
     def test_read_eof(self):
         # Read bytes, stop at eof.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         read_task = self.loop.create_task(stream.read(1024))
 
         def cb():
-            stream._feed_eof()
+            stream.feed_eof()
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -232,15 +197,13 @@ def cb():
 
     def test_read_until_eof(self):
         # Read all bytes until eof.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         read_task = self.loop.create_task(stream.read(-1))
 
         def cb():
-            stream._feed_data(b'chunk1\n')
-            stream._feed_data(b'chunk2')
-            stream._feed_eof()
+            stream.feed_data(b'chunk1\n')
+            stream.feed_data(b'chunk2')
+            stream.feed_eof()
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -249,34 +212,26 @@ def cb():
         self.assertEqual(b'', stream._buffer)
 
     def test_read_exception(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'line\n')
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'line\n')
 
         data = self.loop.run_until_complete(stream.read(2))
         self.assertEqual(b'li', data)
 
-        stream._set_exception(ValueError())
+        stream.set_exception(ValueError())
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.read(2))
 
     def test_invalid_limit(self):
         with self.assertRaisesRegex(ValueError, 'imit'):
-            asyncio.Stream(mode=asyncio.StreamMode.READ,
-                           limit=0, loop=self.loop,
-                           _asyncio_internal=True)
+            asyncio.StreamReader(limit=0, loop=self.loop)
 
         with self.assertRaisesRegex(ValueError, 'imit'):
-            asyncio.Stream(mode=asyncio.StreamMode.READ,
-                           limit=-1, loop=self.loop,
-                           _asyncio_internal=True)
+            asyncio.StreamReader(limit=-1, loop=self.loop)
 
     def test_read_limit(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=3, loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'chunk')
+        stream = asyncio.StreamReader(limit=3, loop=self.loop)
+        stream.feed_data(b'chunk')
         data = self.loop.run_until_complete(stream.read(5))
         self.assertEqual(b'chunk', data)
         self.assertEqual(b'', stream._buffer)
@@ -284,16 +239,14 @@ def test_read_limit(self):
     def test_readline(self):
         # Read one line. 'readline' will need to wait for the data
         # to come from 'cb'
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'chunk1 ')
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'chunk1 ')
         read_task = self.loop.create_task(stream.readline())
 
         def cb():
-            stream._feed_data(b'chunk2 ')
-            stream._feed_data(b'chunk3 ')
-            stream._feed_data(b'\n chunk4')
+            stream.feed_data(b'chunk2 ')
+            stream.feed_data(b'chunk3 ')
+            stream.feed_data(b'\n chunk4')
         self.loop.call_soon(cb)
 
         line = self.loop.run_until_complete(read_task)
@@ -301,26 +254,22 @@ def cb():
         self.assertEqual(b' chunk4', stream._buffer)
 
     def test_readline_limit_with_existing_data(self):
-        # Read one line. The data is in Stream's buffer
+        # Read one line. The data is in StreamReader's buffer
         # before the event loop is run.
 
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=3, loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'li')
-        stream._feed_data(b'ne1\nline2\n')
+        stream = asyncio.StreamReader(limit=3, loop=self.loop)
+        stream.feed_data(b'li')
+        stream.feed_data(b'ne1\nline2\n')
 
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readline())
         # The buffer should contain the remaining data after exception
         self.assertEqual(b'line2\n', stream._buffer)
 
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=3, loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'li')
-        stream._feed_data(b'ne1')
-        stream._feed_data(b'li')
+        stream = asyncio.StreamReader(limit=3, loop=self.loop)
+        stream.feed_data(b'li')
+        stream.feed_data(b'ne1')
+        stream.feed_data(b'li')
 
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readline())
@@ -332,34 +281,30 @@ def test_readline_limit_with_existing_data(self):
         self.assertEqual(b'', stream._buffer)
 
     def test_at_eof(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         self.assertFalse(stream.at_eof())
 
-        stream._feed_data(b'some data\n')
+        stream.feed_data(b'some data\n')
         self.assertFalse(stream.at_eof())
 
         self.loop.run_until_complete(stream.readline())
         self.assertFalse(stream.at_eof())
 
-        stream._feed_data(b'some data\n')
-        stream._feed_eof()
+        stream.feed_data(b'some data\n')
+        stream.feed_eof()
         self.loop.run_until_complete(stream.readline())
         self.assertTrue(stream.at_eof())
 
     def test_readline_limit(self):
-        # Read one line. Streams are fed with data after
+        # Read one line. StreamReaders are fed with data after
         # their 'readline' methods are called.
 
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=7, loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(limit=7, loop=self.loop)
         def cb():
-            stream._feed_data(b'chunk1')
-            stream._feed_data(b'chunk2')
-            stream._feed_data(b'chunk3\n')
-            stream._feed_eof()
+            stream.feed_data(b'chunk1')
+            stream.feed_data(b'chunk2')
+            stream.feed_data(b'chunk3\n')
+            stream.feed_eof()
         self.loop.call_soon(cb)
 
         self.assertRaises(
@@ -368,14 +313,12 @@ def cb():
         # a ValueError it should be empty.
         self.assertEqual(b'', stream._buffer)
 
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=7, loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(limit=7, loop=self.loop)
         def cb():
-            stream._feed_data(b'chunk1')
-            stream._feed_data(b'chunk2\n')
-            stream._feed_data(b'chunk3\n')
-            stream._feed_eof()
+            stream.feed_data(b'chunk1')
+            stream.feed_data(b'chunk2\n')
+            stream.feed_data(b'chunk3\n')
+            stream.feed_eof()
         self.loop.call_soon(cb)
 
         self.assertRaises(
@@ -383,20 +326,18 @@ def cb():
         self.assertEqual(b'chunk3\n', stream._buffer)
 
         # check strictness of the limit
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=7, loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'1234567\n')
+        stream = asyncio.StreamReader(limit=7, loop=self.loop)
+        stream.feed_data(b'1234567\n')
         line = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'1234567\n', line)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'12345678\n')
+        stream.feed_data(b'12345678\n')
         with self.assertRaises(ValueError) as cm:
             self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'12345678')
+        stream.feed_data(b'12345678')
         with self.assertRaises(ValueError) as cm:
             self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'', stream._buffer)
@@ -404,11 +345,9 @@ def cb():
     def test_readline_nolimit_nowait(self):
         # All needed data for the first 'readline' call will be
         # in the buffer.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(self.DATA[:6])
-        stream._feed_data(self.DATA[6:])
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(self.DATA[:6])
+        stream.feed_data(self.DATA[6:])
 
         line = self.loop.run_until_complete(stream.readline())
 
@@ -416,29 +355,23 @@ def test_readline_nolimit_nowait(self):
         self.assertEqual(b'line2\nline3\n', stream._buffer)
 
     def test_readline_eof(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'some data')
-        stream._feed_eof()
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'some data')
+        stream.feed_eof()
 
         line = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'some data', line)
 
     def test_readline_empty_eof(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_eof()
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_eof()
 
         line = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'', line)
 
     def test_readline_read_byte_count(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(self.DATA)
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(self.DATA)
 
         self.loop.run_until_complete(stream.readline())
 
@@ -448,89 +381,79 @@ def test_readline_read_byte_count(self):
         self.assertEqual(b'ine3\n', stream._buffer)
 
     def test_readline_exception(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'line\n')
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'line\n')
 
         data = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'line\n', data)
 
-        stream._set_exception(ValueError())
+        stream.set_exception(ValueError())
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readline())
         self.assertEqual(b'', stream._buffer)
 
     def test_readuntil_separator(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         with self.assertRaisesRegex(ValueError, 'Separator should be'):
             self.loop.run_until_complete(stream.readuntil(separator=b''))
 
     def test_readuntil_multi_chunks(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
-        stream._feed_data(b'lineAAA')
+        stream.feed_data(b'lineAAA')
         data = self.loop.run_until_complete(stream.readuntil(separator=b'AAA'))
         self.assertEqual(b'lineAAA', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'lineAAA')
+        stream.feed_data(b'lineAAA')
         data = self.loop.run_until_complete(stream.readuntil(b'AAA'))
         self.assertEqual(b'lineAAA', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'lineAAAxxx')
+        stream.feed_data(b'lineAAAxxx')
         data = self.loop.run_until_complete(stream.readuntil(b'AAA'))
         self.assertEqual(b'lineAAA', data)
         self.assertEqual(b'xxx', stream._buffer)
 
     def test_readuntil_multi_chunks_1(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
-        stream._feed_data(b'QWEaa')
-        stream._feed_data(b'XYaa')
-        stream._feed_data(b'a')
+        stream.feed_data(b'QWEaa')
+        stream.feed_data(b'XYaa')
+        stream.feed_data(b'a')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'QWEaaXYaaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'QWEaa')
-        stream._feed_data(b'XYa')
-        stream._feed_data(b'aa')
+        stream.feed_data(b'QWEaa')
+        stream.feed_data(b'XYa')
+        stream.feed_data(b'aa')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'QWEaaXYaaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'aaa')
+        stream.feed_data(b'aaa')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'aaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'Xaaa')
+        stream.feed_data(b'Xaaa')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'Xaaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'XXX')
-        stream._feed_data(b'a')
-        stream._feed_data(b'a')
-        stream._feed_data(b'a')
+        stream.feed_data(b'XXX')
+        stream.feed_data(b'a')
+        stream.feed_data(b'a')
+        stream.feed_data(b'a')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'XXXaaa', data)
         self.assertEqual(b'', stream._buffer)
 
     def test_readuntil_eof(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'some dataAA')
-        stream._feed_eof()
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'some dataAA')
+        stream.feed_eof()
 
         with self.assertRaises(asyncio.IncompleteReadError) as cm:
             self.loop.run_until_complete(stream.readuntil(b'AAA'))
@@ -539,18 +462,15 @@ def test_readuntil_eof(self):
         self.assertEqual(b'', stream._buffer)
 
     def test_readuntil_limit_found_sep(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop, limit=3,
-                                _asyncio_internal=True)
-        stream._feed_data(b'some dataAA')
-
+        stream = asyncio.StreamReader(loop=self.loop, limit=3)
+        stream.feed_data(b'some dataAA')
         with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                     'not found') as cm:
             self.loop.run_until_complete(stream.readuntil(b'AAA'))
 
         self.assertEqual(b'some dataAA', stream._buffer)
 
-        stream._feed_data(b'A')
+        stream.feed_data(b'A')
         with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                     'is found') as cm:
             self.loop.run_until_complete(stream.readuntil(b'AAA'))
@@ -559,10 +479,8 @@ def test_readuntil_limit_found_sep(self):
 
     def test_readexactly_zero_or_less(self):
         # Read exact number of bytes (zero or less).
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(self.DATA)
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(self.DATA)
 
         data = self.loop.run_until_complete(stream.readexactly(0))
         self.assertEqual(b'', data)
@@ -574,17 +492,15 @@ def test_readexactly_zero_or_less(self):
 
     def test_readexactly(self):
         # Read exact number of bytes.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
         n = 2 * len(self.DATA)
         read_task = self.loop.create_task(stream.readexactly(n))
 
         def cb():
-            stream._feed_data(self.DATA)
-            stream._feed_data(self.DATA)
-            stream._feed_data(self.DATA)
+            stream.feed_data(self.DATA)
+            stream.feed_data(self.DATA)
+            stream.feed_data(self.DATA)
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -592,25 +508,21 @@ def cb():
         self.assertEqual(self.DATA, stream._buffer)
 
     def test_readexactly_limit(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=3, loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'chunk')
+        stream = asyncio.StreamReader(limit=3, loop=self.loop)
+        stream.feed_data(b'chunk')
         data = self.loop.run_until_complete(stream.readexactly(5))
         self.assertEqual(b'chunk', data)
         self.assertEqual(b'', stream._buffer)
 
     def test_readexactly_eof(self):
         # Read exact number of bytes (eof).
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         n = 2 * len(self.DATA)
         read_task = self.loop.create_task(stream.readexactly(n))
 
         def cb():
-            stream._feed_data(self.DATA)
-            stream._feed_eof()
+            stream.feed_data(self.DATA)
+            stream.feed_eof()
         self.loop.call_soon(cb)
 
         with self.assertRaises(asyncio.IncompleteReadError) as cm:
@@ -622,35 +534,29 @@ def cb():
         self.assertEqual(b'', stream._buffer)
 
     def test_readexactly_exception(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'line\n')
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'line\n')
 
         data = self.loop.run_until_complete(stream.readexactly(2))
         self.assertEqual(b'li', data)
 
-        stream._set_exception(ValueError())
+        stream.set_exception(ValueError())
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readexactly(2))
 
     def test_exception(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         self.assertIsNone(stream.exception())
 
         exc = ValueError()
-        stream._set_exception(exc)
+        stream.set_exception(exc)
         self.assertIs(stream.exception(), exc)
 
     def test_exception_waiter(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
         async def set_err():
-            stream._set_exception(ValueError())
+            stream.set_exception(ValueError())
 
         t1 = self.loop.create_task(stream.readline())
         t2 = self.loop.create_task(set_err())
@@ -660,16 +566,14 @@ def test_exception_waiter(self):
         self.assertRaises(ValueError, t1.result)
 
     def test_exception_cancel(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
         t = self.loop.create_task(stream.readline())
         test_utils.run_briefly(self.loop)
         t.cancel()
         test_utils.run_briefly(self.loop)
         # The following line fails if set_exception() isn't careful.
-        stream._set_exception(RuntimeError('message'))
+        stream.set_exception(RuntimeError('message'))
         test_utils.run_briefly(self.loop)
         self.assertIs(stream._waiter, None)
 
@@ -829,7 +733,7 @@ def stop(self):
     def test_read_all_from_pipe_reader(self):
         # See asyncio issue 168.  This test is derived from the example
         # subprocess_attach_read_pipe.py, but we configure the
-        # Stream's limit so that twice it is less than the size
+        # StreamReader's limit so that twice it is less than the size
         # of the data writter.  Also we must explicitly attach a child
         # watcher to the event loop.
 
@@ -843,11 +747,8 @@ def test_read_all_from_pipe_reader(self):
         args = [sys.executable, '-c', code, str(wfd)]
 
         pipe = open(rfd, 'rb', 0)
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop, limit=1,
-                                _asyncio_internal=True)
-        protocol = _StreamProtocol(stream, loop=self.loop,
-                                   _asyncio_internal=True)
+        reader = asyncio.StreamReader(loop=self.loop, limit=1)
+        protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop)
         transport, _ = self.loop.run_until_complete(
             self.loop.connect_read_pipe(lambda: protocol, pipe))
 
@@ -865,30 +766,29 @@ def test_read_all_from_pipe_reader(self):
             asyncio.set_child_watcher(None)
 
         os.close(wfd)
-        data = self.loop.run_until_complete(stream.read(-1))
+        data = self.loop.run_until_complete(reader.read(-1))
         self.assertEqual(data, b'data')
 
     def test_streamreader_constructor(self):
         self.addCleanup(asyncio.set_event_loop, None)
         asyncio.set_event_loop(self.loop)
 
-        # asyncio issue #184: Ensure that _StreamProtocol constructor
+        # asyncio issue #184: Ensure that StreamReaderProtocol constructor
         # retrieves the current loop if the loop parameter is not set
-        reader = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                _asyncio_internal=True)
+        reader = asyncio.StreamReader()
         self.assertIs(reader._loop, self.loop)
 
     def test_streamreaderprotocol_constructor(self):
         self.addCleanup(asyncio.set_event_loop, None)
         asyncio.set_event_loop(self.loop)
 
-        # asyncio issue #184: Ensure that _StreamProtocol constructor
+        # asyncio issue #184: Ensure that StreamReaderProtocol constructor
         # retrieves the current loop if the loop parameter is not set
-        stream = mock.Mock()
-        protocol = _StreamProtocol(stream, _asyncio_internal=True)
+        reader = mock.Mock()
+        protocol = asyncio.StreamReaderProtocol(reader)
         self.assertIs(protocol._loop, self.loop)
 
-    def test_drain_raises_deprecated(self):
+    def test_drain_raises(self):
         # See http://bugs.python.org/issue25441
 
         # This test should not use asyncio for the mock server; the
@@ -902,7 +802,7 @@ def test_drain_raises_deprecated(self):
 
         def server():
             # Runs in a separate thread.
-            with socket.create_server(('127.0.0.1', 0)) as sock:
+            with socket.create_server(('localhost', 0)) as sock:
                 addr = sock.getsockname()
                 q.put(addr)
                 clt, _ = sock.accept()
@@ -933,106 +833,48 @@ def server():
         thread.join()
         self.assertEqual([], messages)
 
-    def test_drain_raises(self):
-        # See http://bugs.python.org/issue25441
-
-        # This test should not use asyncio for the mock server; the
-        # whole point of the test is to test for a bug in drain()
-        # where it never gives up the event loop but the socket is
-        # closed on the  server side.
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        q = queue.Queue()
-
-        def server():
-            # Runs in a separate thread.
-            with socket.create_server(('localhost', 0)) as sock:
-                addr = sock.getsockname()
-                q.put(addr)
-                clt, _ = sock.accept()
-                clt.close()
-
-        async def client(host, port):
-            stream = await asyncio.connect(host, port)
-
-            while True:
-                stream.write(b"foo\n")
-                await stream.drain()
-
-        # Start the server thread and wait for it to be listening.
-        thread = threading.Thread(target=server)
-        thread.setDaemon(True)
-        thread.start()
-        addr = q.get()
-
-        # Should not be stuck in an infinite loop.
-        with self.assertRaises((ConnectionResetError, ConnectionAbortedError,
-                                BrokenPipeError)):
-            self.loop.run_until_complete(client(*addr))
-
-        # Clean up the thread.  (Only on success; on failure, it may
-        # be stuck in accept().)
-        thread.join()
-        self.assertEqual([], messages)
-
     def test___repr__(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        self.assertEqual("<Stream mode=StreamMode.READ>", repr(stream))
+        stream = asyncio.StreamReader(loop=self.loop)
+        self.assertEqual("<StreamReader>", repr(stream))
 
     def test___repr__nondefault_limit(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop, limit=123,
-                                _asyncio_internal=True)
-        self.assertEqual("<Stream mode=StreamMode.READ limit=123>", repr(stream))
+        stream = asyncio.StreamReader(loop=self.loop, limit=123)
+        self.assertEqual("<StreamReader limit=123>", repr(stream))
 
     def test___repr__eof(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_eof()
-        self.assertEqual("<Stream mode=StreamMode.READ eof>", repr(stream))
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_eof()
+        self.assertEqual("<StreamReader eof>", repr(stream))
 
     def test___repr__data(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'data')
-        self.assertEqual("<Stream mode=StreamMode.READ 4 bytes>", repr(stream))
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'data')
+        self.assertEqual("<StreamReader 4 bytes>", repr(stream))
 
     def test___repr__exception(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         exc = RuntimeError()
-        stream._set_exception(exc)
-        self.assertEqual("<Stream mode=StreamMode.READ exception=RuntimeError()>",
+        stream.set_exception(exc)
+        self.assertEqual("<StreamReader exception=RuntimeError()>",
                          repr(stream))
 
     def test___repr__waiter(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._waiter = self.loop.create_future()
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream._waiter = asyncio.Future(loop=self.loop)
         self.assertRegex(
             repr(stream),
-            r"<Stream .+ waiter=<Future pending[\S ]*>>")
+            r"<StreamReader waiter=<Future pending[\S ]*>>")
         stream._waiter.set_result(None)
         self.loop.run_until_complete(stream._waiter)
         stream._waiter = None
-        self.assertEqual("<Stream mode=StreamMode.READ>", repr(stream))
+        self.assertEqual("<StreamReader>", repr(stream))
 
     def test___repr__transport(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         stream._transport = mock.Mock()
         stream._transport.__repr__ = mock.Mock()
         stream._transport.__repr__.return_value = "<Transport>"
-        self.assertEqual("<Stream mode=StreamMode.READ transport=<Transport>>",
-                         repr(stream))
+        self.assertEqual("<StreamReader transport=<Transport>>", repr(stream))
 
     def test_IncompleteReadError_pickleable(self):
         e = asyncio.IncompleteReadError(b'abc', 10)
@@ -1051,7 +893,7 @@ def test_LimitOverrunError_pickleable(self):
                 self.assertEqual(str(e), str(e2))
                 self.assertEqual(e.consumed, e2.consumed)
 
-    def test_wait_closed_on_close_deprecated(self):
+    def test_wait_closed_on_close(self):
         with test_utils.run_test_server() as httpd:
             with self.assertWarns(DeprecationWarning):
                 rd, wr = self.loop.run_until_complete(
@@ -1069,24 +911,7 @@ def test_wait_closed_on_close_deprecated(self):
             self.assertTrue(wr.is_closing())
             self.loop.run_until_complete(wr.wait_closed())
 
-    def test_wait_closed_on_close(self):
-        with test_utils.run_test_server() as httpd:
-            stream = self.loop.run_until_complete(
-                asyncio.connect(*httpd.address))
-
-            stream.write(b'GET / HTTP/1.0\r\n\r\n')
-            f = stream.readline()
-            data = self.loop.run_until_complete(f)
-            self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-            f = stream.read()
-            data = self.loop.run_until_complete(f)
-            self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-            self.assertFalse(stream.is_closing())
-            stream.close()
-            self.assertTrue(stream.is_closing())
-            self.loop.run_until_complete(stream.wait_closed())
-
-    def test_wait_closed_on_close_with_unread_data_deprecated(self):
+    def test_wait_closed_on_close_with_unread_data(self):
         with test_utils.run_test_server() as httpd:
             with self.assertWarns(DeprecationWarning):
                 rd, wr = self.loop.run_until_complete(
@@ -1099,44 +924,33 @@ def test_wait_closed_on_close_with_unread_data_deprecated(self):
             wr.close()
             self.loop.run_until_complete(wr.wait_closed())
 
-    def test_wait_closed_on_close_with_unread_data(self):
-        with test_utils.run_test_server() as httpd:
-            stream = self.loop.run_until_complete(
-                asyncio.connect(*httpd.address))
-
-            stream.write(b'GET / HTTP/1.0\r\n\r\n')
-            f = stream.readline()
-            data = self.loop.run_until_complete(f)
-            self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-            stream.close()
-            self.loop.run_until_complete(stream.wait_closed())
-
     def test_del_stream_before_sock_closing(self):
         messages = []
         self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
 
-        async def test():
-
-            with test_utils.run_test_server() as httpd:
-                stream = await asyncio.connect(*httpd.address)
-                sock = stream.get_extra_info('socket')
-                self.assertNotEqual(sock.fileno(), -1)
+        with test_utils.run_test_server() as httpd:
+            with self.assertWarns(DeprecationWarning):
+                rd, wr = self.loop.run_until_complete(
+                    asyncio.open_connection(*httpd.address, loop=self.loop))
+            sock = wr.get_extra_info('socket')
+            self.assertNotEqual(sock.fileno(), -1)
 
-                await stream.write(b'GET / HTTP/1.0\r\n\r\n')
-                data = await stream.readline()
-                self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
+            wr.write(b'GET / HTTP/1.0\r\n\r\n')
+            f = rd.readline()
+            data = self.loop.run_until_complete(f)
+            self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
 
-                # drop refs to reader/writer
-                del stream
-                gc.collect()
-                # make a chance to close the socket
-                await asyncio.sleep(0)
+            # drop refs to reader/writer
+            del rd
+            del wr
+            gc.collect()
+            # make a chance to close the socket
+            test_utils.run_briefly(self.loop)
 
-                self.assertEqual(1, len(messages), messages)
-                self.assertEqual(sock.fileno(), -1)
+            self.assertEqual(1, len(messages))
+            self.assertEqual(sock.fileno(), -1)
 
-        self.loop.run_until_complete(test())
-        self.assertEqual(1, len(messages), messages)
+        self.assertEqual(1, len(messages))
         self.assertEqual('An open stream object is being garbage '
                          'collected; call "stream.close()" explicitly.',
                          messages[0]['message'])
@@ -1146,12 +960,9 @@ def test_del_stream_before_connection_made(self):
         self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
 
         with test_utils.run_test_server() as httpd:
-            stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                    loop=self.loop,
-                                    _asyncio_internal=True)
-            pr = _StreamProtocol(stream, loop=self.loop,
-                                 _asyncio_internal=True)
-            del stream
+            rd = asyncio.StreamReader(loop=self.loop)
+            pr = asyncio.StreamReaderProtocol(rd, loop=self.loop)
+            del rd
             gc.collect()
             tr, _ = self.loop.run_until_complete(
                 self.loop.create_connection(
@@ -1168,14 +979,15 @@ def test_del_stream_before_connection_made(self):
 
     def test_async_writer_api(self):
         async def inner(httpd):
-            stream = await asyncio.connect(*httpd.address)
+            rd, wr = await asyncio.open_connection(*httpd.address)
 
-            await stream.write(b'GET / HTTP/1.0\r\n\r\n')
-            data = await stream.readline()
+            wr.write(b'GET / HTTP/1.0\r\n\r\n')
+            data = await rd.readline()
             self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-            data = await stream.read()
+            data = await rd.read()
             self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-            await stream.close()
+            wr.close()
+            await wr.wait_closed()
 
         messages = []
         self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
@@ -1187,16 +999,17 @@ def test_async_writer_api(self):
 
     def test_async_writer_api_exception_after_close(self):
         async def inner(httpd):
-            stream = await asyncio.connect(*httpd.address)
+            rd, wr = await asyncio.open_connection(*httpd.address)
 
-            await stream.write(b'GET / HTTP/1.0\r\n\r\n')
-            data = await stream.readline()
+            wr.write(b'GET / HTTP/1.0\r\n\r\n')
+            data = await rd.readline()
             self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-            data = await stream.read()
+            data = await rd.read()
             self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-            stream.close()
+            wr.close()
             with self.assertRaises(ConnectionResetError):
-                await stream.write(b'data')
+                wr.write(b'data')
+                await wr.drain()
 
         messages = []
         self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
@@ -1227,587 +1040,6 @@ def test_eof_feed_when_closing_writer(self):
 
         self.assertEqual(messages, [])
 
-    def test_stream_reader_create_warning(self):
-        with contextlib.suppress(AttributeError):
-            del asyncio.StreamReader
-        with self.assertWarns(DeprecationWarning):
-            asyncio.StreamReader
-
-    def test_stream_writer_create_warning(self):
-        with contextlib.suppress(AttributeError):
-            del asyncio.StreamWriter
-        with self.assertWarns(DeprecationWarning):
-            asyncio.StreamWriter
-
-    def test_stream_reader_forbidden_ops(self):
-        async def inner():
-            stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                    _asyncio_internal=True)
-            with self.assertRaisesRegex(RuntimeError, "The stream is read-only"):
-                await stream.write(b'data')
-            with self.assertRaisesRegex(RuntimeError, "The stream is read-only"):
-                await stream.writelines([b'data', b'other'])
-            with self.assertRaisesRegex(RuntimeError, "The stream is read-only"):
-                stream.write_eof()
-            with self.assertRaisesRegex(RuntimeError, "The stream is read-only"):
-                await stream.drain()
-
-        self.loop.run_until_complete(inner())
-
-    def test_stream_writer_forbidden_ops(self):
-        async def inner():
-            stream = asyncio.Stream(mode=asyncio.StreamMode.WRITE,
-                                    _asyncio_internal=True)
-            with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                stream._feed_data(b'data')
-            with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                await stream.readline()
-            with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                await stream.readuntil()
-            with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                await stream.read()
-            with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                await stream.readexactly(10)
-            with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                async for chunk in stream:
-                    pass
-
-        self.loop.run_until_complete(inner())
-
-    def _basetest_connect(self, stream):
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-
-        stream.write(b'GET / HTTP/1.0\r\n\r\n')
-        f = stream.readline()
-        data = self.loop.run_until_complete(f)
-        self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-        f = stream.read()
-        data = self.loop.run_until_complete(f)
-        self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-        stream.close()
-        self.loop.run_until_complete(stream.wait_closed())
-
-        self.assertEqual([], messages)
-
-    def test_connect(self):
-        with test_utils.run_test_server() as httpd:
-            stream = self.loop.run_until_complete(
-                asyncio.connect(*httpd.address))
-            self.assertFalse(stream.is_server_side())
-            self._basetest_connect(stream)
-
-    @support.skip_unless_bind_unix_socket
-    def test_connect_unix(self):
-        with test_utils.run_test_unix_server() as httpd:
-            stream = self.loop.run_until_complete(
-                asyncio.connect_unix(httpd.address))
-            self._basetest_connect(stream)
-
-    def test_stream_async_context_manager(self):
-        async def test(httpd):
-            stream = await asyncio.connect(*httpd.address)
-            async with stream:
-                await stream.write(b'GET / HTTP/1.0\r\n\r\n')
-                data = await stream.readline()
-                self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-                data = await stream.read()
-                self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-            self.assertTrue(stream.is_closing())
-
-        with test_utils.run_test_server() as httpd:
-            self.loop.run_until_complete(test(httpd))
-
-    def test_connect_async_context_manager(self):
-        async def test(httpd):
-            async with asyncio.connect(*httpd.address) as stream:
-                await stream.write(b'GET / HTTP/1.0\r\n\r\n')
-                data = await stream.readline()
-                self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-                data = await stream.read()
-                self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-            self.assertTrue(stream.is_closing())
-
-        with test_utils.run_test_server() as httpd:
-            self.loop.run_until_complete(test(httpd))
-
-    @support.skip_unless_bind_unix_socket
-    def test_connect_unix_async_context_manager(self):
-        async def test(httpd):
-            async with asyncio.connect_unix(httpd.address) as stream:
-                await stream.write(b'GET / HTTP/1.0\r\n\r\n')
-                data = await stream.readline()
-                self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-                data = await stream.read()
-                self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-            self.assertTrue(stream.is_closing())
-
-        with test_utils.run_test_unix_server() as httpd:
-            self.loop.run_until_complete(test(httpd))
-
-    def test_stream_server(self):
-
-        async def handle_client(stream):
-            self.assertTrue(stream.is_server_side())
-            data = await stream.readline()
-            await stream.write(data)
-            await stream.close()
-
-        async def client(srv):
-            addr = srv.sockets[0].getsockname()
-            stream = await asyncio.connect(*addr)
-            # send a line
-            await stream.write(b"hello world!\n")
-            # read it back
-            msgback = await stream.readline()
-            await stream.close()
-            self.assertEqual(msgback, b"hello world!\n")
-            await srv.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client, '127.0.0.1', 0) as server:
-                await server.start_serving()
-                task = asyncio.create_task(client(server))
-                with contextlib.suppress(asyncio.CancelledError):
-                    await server.serve_forever()
-                await task
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(test())
-        self.assertEqual(messages, [])
-
-    @support.skip_unless_bind_unix_socket
-    def test_unix_stream_server(self):
-
-        async def handle_client(stream):
-            data = await stream.readline()
-            await stream.write(data)
-            await stream.close()
-
-        async def client(srv):
-            addr = srv.sockets[0].getsockname()
-            stream = await asyncio.connect_unix(addr)
-            # send a line
-            await stream.write(b"hello world!\n")
-            # read it back
-            msgback = await stream.readline()
-            await stream.close()
-            self.assertEqual(msgback, b"hello world!\n")
-            await srv.close()
-
-        async def test():
-            with test_utils.unix_socket_path() as path:
-                async with asyncio.UnixStreamServer(handle_client, path) as server:
-                    await server.start_serving()
-                    task = asyncio.create_task(client(server))
-                    with contextlib.suppress(asyncio.CancelledError):
-                        await server.serve_forever()
-                    await task
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(test())
-        self.assertEqual(messages, [])
-
-    def test_stream_server_inheritance_forbidden(self):
-        with self.assertRaises(TypeError):
-            class MyServer(asyncio.StreamServer):
-                pass
-
-    @support.skip_unless_bind_unix_socket
-    def test_unix_stream_server_inheritance_forbidden(self):
-        with self.assertRaises(TypeError):
-            class MyServer(asyncio.UnixStreamServer):
-                pass
-
-    def test_stream_server_bind(self):
-        async def handle_client(stream):
-            await stream.close()
-
-        async def test():
-            srv = asyncio.StreamServer(handle_client, '127.0.0.1', 0)
-            self.assertFalse(srv.is_bound())
-            self.assertEqual(0, len(srv.sockets))
-            await srv.bind()
-            self.assertTrue(srv.is_bound())
-            self.assertEqual(1, len(srv.sockets))
-            await srv.close()
-            self.assertFalse(srv.is_bound())
-            self.assertEqual(0, len(srv.sockets))
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(test())
-        self.assertEqual(messages, [])
-
-    def test_stream_server_bind_async_with(self):
-        async def handle_client(stream):
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client, '127.0.0.1', 0) as srv:
-                self.assertTrue(srv.is_bound())
-                self.assertEqual(1, len(srv.sockets))
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(test())
-        self.assertEqual(messages, [])
-
-    def test_stream_server_start_serving(self):
-        async def handle_client(stream):
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client, '127.0.0.1', 0) as srv:
-                self.assertFalse(srv.is_serving())
-                await srv.start_serving()
-                self.assertTrue(srv.is_serving())
-                await srv.close()
-                self.assertFalse(srv.is_serving())
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(test())
-        self.assertEqual(messages, [])
-
-    def test_stream_server_close(self):
-        server_stream_aborted = False
-        fut1 = self.loop.create_future()
-        fut2 = self.loop.create_future()
-
-        async def handle_client(stream):
-            data = await stream.readexactly(4)
-            self.assertEqual(b'data', data)
-            fut1.set_result(None)
-            await fut2
-            self.assertEqual(b'', await stream.readline())
-            nonlocal server_stream_aborted
-            server_stream_aborted = True
-
-        async def client(srv):
-            addr = srv.sockets[0].getsockname()
-            stream = await asyncio.connect(*addr)
-            await stream.write(b'data')
-            await fut2
-            self.assertEqual(b'', await stream.readline())
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client, '127.0.0.1', 0) as server:
-                await server.start_serving()
-                task = asyncio.create_task(client(server))
-                await fut1
-                fut2.set_result(None)
-                await server.close()
-                await task
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(asyncio.wait_for(test(), 60.0))
-        self.assertEqual(messages, [])
-        self.assertTrue(fut1.done())
-        self.assertTrue(fut2.done())
-        self.assertTrue(server_stream_aborted)
-
-    def test_stream_server_abort(self):
-        server_stream_aborted = False
-        fut1 = self.loop.create_future()
-        fut2 = self.loop.create_future()
-
-        async def handle_client(stream):
-            data = await stream.readexactly(4)
-            self.assertEqual(b'data', data)
-            fut1.set_result(None)
-            await fut2
-            self.assertEqual(b'', await stream.readline())
-            nonlocal server_stream_aborted
-            server_stream_aborted = True
-
-        async def client(srv):
-            addr = srv.sockets[0].getsockname()
-            stream = await asyncio.connect(*addr)
-            await stream.write(b'data')
-            await fut2
-            self.assertEqual(b'', await stream.readline())
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client, '127.0.0.1', 0) as server:
-                await server.start_serving()
-                task = asyncio.create_task(client(server))
-                await fut1
-                fut2.set_result(None)
-                await server.abort()
-                await task
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(asyncio.wait_for(test(), 60.0))
-        self.assertEqual(messages, [])
-        self.assertTrue(fut1.done())
-        self.assertTrue(fut2.done())
-        self.assertTrue(server_stream_aborted)
-
-    def test_stream_shutdown_hung_task(self):
-        fut1 = self.loop.create_future()
-        fut2 = self.loop.create_future()
-        cancelled = self.loop.create_future()
-
-        async def handle_client(stream):
-            data = await stream.readexactly(4)
-            self.assertEqual(b'data', data)
-            fut1.set_result(None)
-            await fut2
-            try:
-                while True:
-                    await asyncio.sleep(0.01)
-            except asyncio.CancelledError:
-                cancelled.set_result(None)
-                raise
-
-        async def client(srv):
-            addr = srv.sockets[0].getsockname()
-            stream = await asyncio.connect(*addr)
-            await stream.write(b'data')
-            await fut2
-            self.assertEqual(b'', await stream.readline())
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client,
-                                            '127.0.0.1',
-                                            0,
-                                            shutdown_timeout=0.3) as server:
-                await server.start_serving()
-                task = asyncio.create_task(client(server))
-                await fut1
-                fut2.set_result(None)
-                await server.close()
-                await task
-                await cancelled
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(asyncio.wait_for(test(), 60.0))
-        self.assertEqual(messages, [])
-        self.assertTrue(fut1.done())
-        self.assertTrue(fut2.done())
-        self.assertTrue(cancelled.done())
-
-    def test_stream_shutdown_hung_task_prevents_cancellation(self):
-        fut1 = self.loop.create_future()
-        fut2 = self.loop.create_future()
-        cancelled = self.loop.create_future()
-        do_handle_client = True
-
-        async def handle_client(stream):
-            data = await stream.readexactly(4)
-            self.assertEqual(b'data', data)
-            fut1.set_result(None)
-            await fut2
-            while do_handle_client:
-                with contextlib.suppress(asyncio.CancelledError):
-                    await asyncio.sleep(0.01)
-            cancelled.set_result(None)
-
-        async def client(srv):
-            addr = srv.sockets[0].getsockname()
-            stream = await asyncio.connect(*addr)
-            await stream.write(b'data')
-            await fut2
-            self.assertEqual(b'', await stream.readline())
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client,
-                                            '127.0.0.1',
-                                            0,
-                                            shutdown_timeout=0.3) as server:
-                await server.start_serving()
-                task = asyncio.create_task(client(server))
-                await fut1
-                fut2.set_result(None)
-                await server.close()
-                nonlocal do_handle_client
-                do_handle_client = False
-                await task
-                await cancelled
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(asyncio.wait_for(test(), 60.0))
-        self.assertEqual(1, len(messages))
-        self.assertRegex(messages[0]['message'],
-                         "<Task pending .+ ignored cancellation request")
-        self.assertTrue(fut1.done())
-        self.assertTrue(fut2.done())
-        self.assertTrue(cancelled.done())
-
-    def test_sendfile(self):
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-
-        with open(support.TESTFN, 'wb') as fp:
-            fp.write(b'data\n')
-        self.addCleanup(support.unlink, support.TESTFN)
-
-        async def serve_callback(stream):
-            data = await stream.readline()
-            await stream.write(b'ack-' + data)
-            data = await stream.readline()
-            await stream.write(b'ack-' + data)
-            data = await stream.readline()
-            await stream.write(b'ack-' + data)
-            await stream.close()
-
-        async def do_connect(host, port):
-            stream = await asyncio.connect(host, port)
-            await stream.write(b'begin\n')
-            data = await stream.readline()
-            self.assertEqual(b'ack-begin\n', data)
-            with open(support.TESTFN, 'rb') as fp:
-                await stream.sendfile(fp)
-            data = await stream.readline()
-            self.assertEqual(b'ack-data\n', data)
-            await stream.write(b'end\n')
-            data = await stream.readline()
-            self.assertEqual(data, b'ack-end\n')
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(serve_callback, '127.0.0.1', 0) as srv:
-                await srv.start_serving()
-                await do_connect(*srv.sockets[0].getsockname())
-
-        self.loop.run_until_complete(test())
-
-        self.assertEqual([], messages)
-
-
-    @unittest.skipIf(ssl is None, 'No ssl module')
-    def test_connect_start_tls(self):
-        with test_utils.run_test_server(use_ssl=True) as httpd:
-            # connect without SSL but upgrade to TLS just after
-            # connection is established
-            stream = self.loop.run_until_complete(
-                asyncio.connect(*httpd.address))
-
-            self.loop.run_until_complete(
-                stream.start_tls(
-                    sslcontext=test_utils.dummy_ssl_context()))
-            self._basetest_connect(stream)
-
-    def test_repr_unbound(self):
-        async def serve(stream):
-            pass
-
-        async def test():
-            srv = asyncio.StreamServer(serve)
-            self.assertEqual('<StreamServer>', repr(srv))
-            await srv.close()
-
-        self.loop.run_until_complete(test())
-
-    def test_repr_bound(self):
-        async def serve(stream):
-            pass
-
-        async def test():
-            srv = asyncio.StreamServer(serve, '127.0.0.1', 0)
-            await srv.bind()
-            self.assertRegex(repr(srv), r'<StreamServer sockets=\(.+\)>')
-            await srv.close()
-
-        self.loop.run_until_complete(test())
-
-    def test_repr_serving(self):
-        async def serve(stream):
-            pass
-
-        async def test():
-            srv = asyncio.StreamServer(serve, '127.0.0.1', 0)
-            await srv.start_serving()
-            self.assertRegex(repr(srv), r'<StreamServer serving sockets=\(.+\)>')
-            await srv.close()
-
-        self.loop.run_until_complete(test())
-
-
-    @unittest.skipUnless(sys.platform != 'win32',
-                         "Don't support pipes for Windows")
-    def test_read_pipe(self):
-        async def test():
-            rpipe, wpipe = os.pipe()
-            pipeobj = io.open(rpipe, 'rb', 1024)
-
-            async with asyncio.connect_read_pipe(pipeobj) as stream:
-                self.assertEqual(stream.mode, asyncio.StreamMode.READ)
-
-                os.write(wpipe, b'1')
-                data = await stream.readexactly(1)
-                self.assertEqual(data, b'1')
-
-                os.write(wpipe, b'2345')
-                data = await stream.readexactly(4)
-                self.assertEqual(data, b'2345')
-                os.close(wpipe)
-
-        self.loop.run_until_complete(test())
-
-    @unittest.skipUnless(sys.platform != 'win32',
-                         "Don't support pipes for Windows")
-    def test_write_pipe(self):
-        async def test():
-            rpipe, wpipe = os.pipe()
-            pipeobj = io.open(wpipe, 'wb', 1024)
-
-            async with asyncio.connect_write_pipe(pipeobj) as stream:
-                self.assertEqual(stream.mode, asyncio.StreamMode.WRITE)
-
-                await stream.write(b'1')
-                data = os.read(rpipe, 1024)
-                self.assertEqual(data, b'1')
-
-                await stream.write(b'2345')
-                data = os.read(rpipe, 1024)
-                self.assertEqual(data, b'2345')
-
-                os.close(rpipe)
-
-        self.loop.run_until_complete(test())
-
-    def test_stream_ctor_forbidden(self):
-        with self.assertRaisesRegex(RuntimeError,
-                                    "should be instantiated "
-                                    "by asyncio internals only"):
-            asyncio.Stream(asyncio.StreamMode.READWRITE)
-
-    def test_deprecated_methods(self):
-        async def f():
-            return asyncio.Stream(mode=asyncio.StreamMode.READWRITE,
-                                  _asyncio_internal=True)
-
-        stream = self.loop.run_until_complete(f())
-
-        tr = mock.Mock()
-
-        with self.assertWarns(DeprecationWarning):
-            stream.set_transport(tr)
-
-        with self.assertWarns(DeprecationWarning):
-            stream.transport is tr
-
-        with self.assertWarns(DeprecationWarning):
-            stream.feed_data(b'data')
-
-        with self.assertWarns(DeprecationWarning):
-            stream.feed_eof()
-
-        with self.assertWarns(DeprecationWarning):
-            stream.set_exception(ConnectionResetError("test"))
-
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py
index 3ad18e5c5166..fe8cfa61b1b2 100644
--- a/Lib/test/test_asyncio/test_subprocess.py
+++ b/Lib/test/test_asyncio/test_subprocess.py
@@ -582,18 +582,6 @@ def test_read_stdout_after_process_exit(self):
 
         self.loop.run_until_complete(execute())
 
-    def test_subprocess_protocol_create_warning(self):
-        with self.assertWarns(DeprecationWarning):
-            subprocess.SubprocessStreamProtocol(limit=10, loop=self.loop)
-
-    def test_process_create_warning(self):
-        proto = subprocess.SubprocessStreamProtocol(limit=10, loop=self.loop,
-                                                    _asyncio_internal=True)
-        transp = mock.Mock()
-
-        with self.assertWarns(DeprecationWarning):
-            subprocess.Process(transp, proto, loop=self.loop)
-
     def test_create_subprocess_exec_text_mode_fails(self):
         async def execute():
             with self.assertRaises(ValueError):
diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py
index d0ba19391fa0..9ed10fc20f81 100644
--- a/Lib/test/test_asyncio/test_windows_events.py
+++ b/Lib/test/test_asyncio/test_windows_events.py
@@ -15,7 +15,6 @@
 
 import asyncio
 from asyncio import windows_events
-from asyncio.streams import _StreamProtocol
 from test.test_asyncio import utils as test_utils
 
 
@@ -118,16 +117,14 @@ def test_pipe(self):
 
         clients = []
         for i in range(5):
-            stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                    loop=self.loop, _asyncio_internal=True)
-            protocol = _StreamProtocol(stream,
-                                       loop=self.loop,
-                                       _asyncio_internal=True)
+            stream_reader = asyncio.StreamReader(loop=self.loop)
+            protocol = asyncio.StreamReaderProtocol(stream_reader,
+                                                    loop=self.loop)
             trans, proto = await self.loop.create_pipe_connection(
                 lambda: protocol, ADDRESS)
             self.assertIsInstance(trans, asyncio.Transport)
             self.assertEqual(protocol, proto)
-            clients.append((stream, trans))
+            clients.append((stream_reader, trans))
 
         for i, (r, w) in enumerate(clients):
             w.write('lower-{}\n'.format(i).encode())
@@ -136,7 +133,6 @@ def test_pipe(self):
             response = await r.readline()
             self.assertEqual(response, 'LOWER-{}\n'.format(i).encode())
             w.close()
-            await r.close()
 
         server.close()
 
diff --git a/Misc/NEWS.d/next/Library/2019-09-30-00-15-27.bpo-38242.uPIyAc.rst b/Misc/NEWS.d/next/Library/2019-09-30-00-15-27.bpo-38242.uPIyAc.rst
new file mode 100644
index 000000000000..be9da891b866
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-30-00-15-27.bpo-38242.uPIyAc.rst
@@ -0,0 +1 @@
+Revert the new asyncio Streams API


From webhook-mailer at python.org  Mon Sep 30 01:22:48 2019
From: webhook-mailer at python.org (Lisa Roach)
Date: Mon, 30 Sep 2019 05:22:48 -0000
Subject: [Python-checkins] [3.8] bpo-38163: Child mocks detect their type as
 sync or async (GH-16471) (GH-16484)
Message-ID: <mailman.151.1569820970.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/21f24ead90c22d0e2c2ebf14a64b37d99de54b33
commit: 21f24ead90c22d0e2c2ebf14a64b37d99de54b33
branch: 3.8
author: Lisa Roach <lisaroach14 at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-09-29T22:22:44-07:00
summary:

[3.8] bpo-38163: Child mocks detect their type as sync or async (GH-16471) (GH-16484)

files:
A Misc/NEWS.d/next/Library/2019-09-28-20-16-40.bpo-38163.x51-vK.rst
M Doc/library/unittest.mock.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py

diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index 300f28c8e2ce..3fa3cfdaf21a 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -865,7 +865,7 @@ object::
     True
 
   The result of ``mock()`` is an async function which will have the outcome
-  of ``side_effect`` or ``return_value``:
+  of ``side_effect`` or ``return_value`` after it has been awaited:
 
   - if ``side_effect`` is a function, the async function will return the
     result of that function,
@@ -890,6 +890,32 @@ object::
     >>> mock()  # doctest: +SKIP
     <coroutine object AsyncMockMixin._mock_call at ...>
 
+
+  Setting the *spec* of a :class:`Mock`, :class:`MagicMock`, or :class:`AsyncMock`
+  to a class with asynchronous and synchronous functions will automatically
+  detect the synchronous functions and set them as :class:`MagicMock` (if the
+  parent mock is :class:`AsyncMock` or :class:`MagicMock`) or :class:`Mock` (if
+  the parent mock is :class:`Mock`). All asynchronous functions will be
+  :class:`AsyncMock`.
+
+  >>> class ExampleClass:
+  ...     def sync_foo():
+  ...         pass
+  ...     async def async_foo():
+  ...         pass
+  ...
+  >>> a_mock = AsyncMock(ExampleClass)
+  >>> a_mock.sync_foo
+  <MagicMock name='mock.sync_foo' id='...'>
+  >>> a_mock.async_foo
+  <AsyncMock name='mock.async_foo' id='...'>
+  >>> mock = Mock(ExampleClass)
+  >>> mock.sync_foo
+  <Mock name='mock.sync_foo' id='...'>
+  >>> mock.async_foo
+  <AsyncMock name='mock.async_foo' id='...'>
+
+
   .. method:: assert_awaited()
 
       Assert that the mock was awaited at least once. Note that this is separate
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 5d32f88c6b78..488ab1c23de4 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -997,8 +997,9 @@ def _get_child_mock(self, /, **kw):
             # Any asynchronous magic becomes an AsyncMock
             klass = AsyncMock
         elif issubclass(_type, AsyncMockMixin):
-            if _new_name in _all_sync_magics:
-                # Any synchronous magic becomes a MagicMock
+            if (_new_name in _all_sync_magics or
+                    self._mock_methods and _new_name in self._mock_methods):
+                # Any synchronous method on AsyncMock becomes a MagicMock
                 klass = MagicMock
             else:
                 klass = AsyncMock
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py
index ed8aefc2a866..7671d55a8cfb 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -3,7 +3,7 @@
 import re
 import unittest
 
-from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock,
+from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock, Mock,
                            create_autospec, sentinel, _CallList)
 
 
@@ -228,33 +228,50 @@ def test_patch_with_autospec(self):
 
 
 class AsyncSpecTest(unittest.TestCase):
-    def test_spec_as_async_positional_magicmock(self):
-        mock = MagicMock(async_func)
-        self.assertIsInstance(mock, MagicMock)
-        m = mock()
-        self.assertTrue(inspect.isawaitable(m))
-        asyncio.run(m)
+    def test_spec_normal_methods_on_class(self):
+        def inner_test(mock_type):
+            mock = mock_type(AsyncClass)
+            self.assertIsInstance(mock.async_method, AsyncMock)
+            self.assertIsInstance(mock.normal_method, MagicMock)
 
-    def test_spec_as_async_kw_magicmock(self):
-        mock = MagicMock(spec=async_func)
-        self.assertIsInstance(mock, MagicMock)
-        m = mock()
-        self.assertTrue(inspect.isawaitable(m))
-        asyncio.run(m)
+        for mock_type in [AsyncMock, MagicMock]:
+            with self.subTest(f"test method types with {mock_type}"):
+                inner_test(mock_type)
 
-    def test_spec_as_async_kw_AsyncMock(self):
-        mock = AsyncMock(spec=async_func)
-        self.assertIsInstance(mock, AsyncMock)
-        m = mock()
-        self.assertTrue(inspect.isawaitable(m))
-        asyncio.run(m)
+    def test_spec_normal_methods_on_class_with_mock(self):
+        mock = Mock(AsyncClass)
+        self.assertIsInstance(mock.async_method, AsyncMock)
+        self.assertIsInstance(mock.normal_method, Mock)
 
-    def test_spec_as_async_positional_AsyncMock(self):
-        mock = AsyncMock(async_func)
-        self.assertIsInstance(mock, AsyncMock)
-        m = mock()
-        self.assertTrue(inspect.isawaitable(m))
-        asyncio.run(m)
+    def test_spec_mock_type_kw(self):
+        def inner_test(mock_type):
+            async_mock = mock_type(spec=async_func)
+            self.assertIsInstance(async_mock, mock_type)
+            with self.assertWarns(RuntimeWarning):
+                # Will raise a warning because never awaited
+                self.assertTrue(inspect.isawaitable(async_mock()))
+
+            sync_mock = mock_type(spec=normal_func)
+            self.assertIsInstance(sync_mock, mock_type)
+
+        for mock_type in [AsyncMock, MagicMock, Mock]:
+            with self.subTest(f"test spec kwarg with {mock_type}"):
+                inner_test(mock_type)
+
+    def test_spec_mock_type_positional(self):
+        def inner_test(mock_type):
+            async_mock = mock_type(async_func)
+            self.assertIsInstance(async_mock, mock_type)
+            with self.assertWarns(RuntimeWarning):
+                # Will raise a warning because never awaited
+                self.assertTrue(inspect.isawaitable(async_mock()))
+
+            sync_mock = mock_type(normal_func)
+            self.assertIsInstance(sync_mock, mock_type)
+
+        for mock_type in [AsyncMock, MagicMock, Mock]:
+            with self.subTest(f"test spec positional with {mock_type}"):
+                inner_test(mock_type)
 
     def test_spec_as_normal_kw_AsyncMock(self):
         mock = AsyncMock(spec=normal_func)
diff --git a/Misc/NEWS.d/next/Library/2019-09-28-20-16-40.bpo-38163.x51-vK.rst b/Misc/NEWS.d/next/Library/2019-09-28-20-16-40.bpo-38163.x51-vK.rst
new file mode 100644
index 000000000000..5f7db26e05ce
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-28-20-16-40.bpo-38163.x51-vK.rst
@@ -0,0 +1,4 @@
+Child mocks will now detect their type as either synchronous or
+asynchronous, asynchronous child mocks will be AsyncMocks and synchronous
+child mocks will be either MagicMock or Mock (depending on their parent
+type).


From webhook-mailer at python.org  Mon Sep 30 01:30:21 2019
From: webhook-mailer at python.org (Yury Selivanov)
Date: Mon, 30 Sep 2019 05:30:21 -0000
Subject: [Python-checkins] bpo-38242: Revert "bpo-36889: Merge asyncio
 streams (GH-13251)" (#16482) (#16485)
Message-ID: <mailman.152.1569821422.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/1c19d656a79a00f58361ceb61c0a6d1faf90c686
commit: 1c19d656a79a00f58361ceb61c0a6d1faf90c686
branch: 3.8
author: Yury Selivanov <yury at edgedb.com>
committer: GitHub <noreply at github.com>
date: 2019-09-29T22:30:17-07:00
summary:

bpo-38242: Revert "bpo-36889: Merge asyncio streams (GH-13251)" (#16482) (#16485)

See https://bugs.python.org/issue38242 for more details

files:
A Misc/NEWS.d/next/Library/2019-09-30-00-15-27.bpo-38242.uPIyAc.rst
M Doc/library/asyncio-api-index.rst
M Doc/library/asyncio-eventloop.rst
M Doc/library/asyncio-protocol.rst
M Doc/library/asyncio-stream.rst
M Lib/asyncio/__init__.py
M Lib/asyncio/streams.py
M Lib/asyncio/subprocess.py
M Lib/test/test_asyncio/test_buffered_proto.py
M Lib/test/test_asyncio/test_pep492.py
M Lib/test/test_asyncio/test_streams.py
M Lib/test/test_asyncio/test_subprocess.py
M Lib/test/test_asyncio/test_windows_events.py

diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst
index 716cf09dc99e..d5b5659abc65 100644
--- a/Doc/library/asyncio-api-index.rst
+++ b/Doc/library/asyncio-api-index.rst
@@ -132,47 +132,23 @@ High-level APIs to work with network IO.
     :widths: 50 50
     :class: full-width-table
 
-    * - ``await`` :func:`connect`
-      -  Establish a TCP connection to send and receive data.
-
     * - ``await`` :func:`open_connection`
-      -  Establish a TCP connection. (Deprecated in favor of :func:`connect`)
-
-    * - ``await`` :func:`connect_unix`
-      -  Establish a Unix socket connection to send and receive data.
+      -  Establish a TCP connection.
 
     * - ``await`` :func:`open_unix_connection`
-      -  Establish a Unix socket connection. (Deprecated in favor of :func:`connect_unix`)
-
-    * - :class:`StreamServer`
-      - Start a TCP server.
+      -  Establish a Unix socket connection.
 
     * - ``await`` :func:`start_server`
-      - Start a TCP server. (Deprecated in favor of :class:`StreamServer`)
-
-    * - :class:`UnixStreamServer`
-      - Start a Unix socket server.
+      - Start a TCP server.
 
     * - ``await`` :func:`start_unix_server`
-      - Start a Unix socket server. (Deprecated in favor of :class:`UnixStreamServer`)
-
-    * - :func:`connect_read_pipe`
-      - Establish a connection to :term:`file-like object <file object>` *pipe*
-        to receive data.
-
-    * - :func:`connect_write_pipe`
-      - Establish a connection to :term:`file-like object <file object>` *pipe*
-        to send data.
-
-    * - :class:`Stream`
-      - Stream is a single object combining APIs of :class:`StreamReader` and
-        :class:`StreamWriter`.
+      - Start a Unix socket server.
 
     * - :class:`StreamReader`
-      - High-level async/await object to receive network data. (Deprecated in favor of :class:`Stream`)
+      - High-level async/await object to receive network data.
 
     * - :class:`StreamWriter`
-      - High-level async/await object to send network data. (Deprecated in favor of :class:`Stream`)
+      - High-level async/await object to send network data.
 
 
 .. rubric:: Examples
diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index 8f7974be66ea..f763fd5f036d 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -1625,7 +1625,8 @@ Wait until a file descriptor received some data using the
      :meth:`loop.create_connection` method.
 
    * Another similar :ref:`example <asyncio_example_create_connection-streams>`
-     using the high-level :func:`asyncio.connect` function and streams.
+     using the high-level :func:`asyncio.open_connection` function
+     and streams.
 
 
 .. _asyncio_example_unix_signals:
diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index cb0317ea20c1..67ca12108168 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -809,7 +809,7 @@ data, and waits until the connection is closed::
 .. seealso::
 
    The :ref:`TCP echo client using streams <asyncio-tcp-echo-client-streams>`
-   example uses the high-level :func:`asyncio.connect` function.
+   example uses the high-level :func:`asyncio.open_connection` function.
 
 
 .. _asyncio-udp-echo-server-protocol:
@@ -978,7 +978,7 @@ Wait until a socket receives data using the
 
    The :ref:`register an open socket to wait for data using streams
    <asyncio_example_create_connection-streams>` example uses high-level streams
-   created by the :func:`asyncio.connect` function in a coroutine.
+   created by the :func:`open_connection` function in a coroutine.
 
 .. _asyncio_example_subprocess_proto:
 
diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst
index feebd227eb8c..471e6e9099c2 100644
--- a/Doc/library/asyncio-stream.rst
+++ b/Doc/library/asyncio-stream.rst
@@ -18,12 +18,19 @@ streams::
     import asyncio
 
     async def tcp_echo_client(message):
-        async with asyncio.connect('127.0.0.1', 8888) as stream:
-            print(f'Send: {message!r}')
-            await stream.write(message.encode())
+        reader, writer = await asyncio.open_connection(
+            '127.0.0.1', 8888)
 
-            data = await stream.read(100)
-            print(f'Received: {data.decode()!r}')
+        print(f'Send: {message!r}')
+        writer.write(message.encode())
+        await writer.drain()
+
+        data = await reader.read(100)
+        print(f'Received: {data.decode()!r}')
+
+        print('Close the connection')
+        writer.close()
+        await writer.wait_closed()
 
     asyncio.run(tcp_echo_client('Hello World!'))
 
@@ -37,31 +44,6 @@ The following top-level asyncio functions can be used to create
 and work with streams:
 
 
-.. coroutinefunction:: connect(host=None, port=None, \*, \
-                               limit=2**16, ssl=None, family=0, \
-                               proto=0, flags=0, sock=None, local_addr=None, \
-                               server_hostname=None, ssl_handshake_timeout=None, \
-                               happy_eyeballs_delay=None, interleave=None)
-
-   Connect to TCP socket on *host* : *port* address and return a :class:`Stream`
-   object of mode :attr:`StreamMode.READWRITE`.
-
-   *limit* determines the buffer size limit used by the returned :class:`Stream`
-   instance. By default the *limit* is set to 64 KiB.
-
-   The rest of the arguments are passed directly to :meth:`loop.create_connection`.
-
-   The function can be used with ``await`` to get a connected stream::
-
-       stream = await asyncio.connect('127.0.0.1', 8888)
-
-   The function can also be used as an async context manager::
-
-       async with asyncio.connect('127.0.0.1', 8888) as stream:
-           ...
-
-   .. versionadded:: 3.8
-
 .. coroutinefunction:: open_connection(host=None, port=None, \*, \
                           loop=None, limit=None, ssl=None, family=0, \
                           proto=0, flags=0, sock=None, local_addr=None, \
@@ -87,12 +69,8 @@ and work with streams:
 
       The *ssl_handshake_timeout* parameter.
 
-   .. deprecated-removed:: 3.8 3.10
-
-      `open_connection()` is deprecated in favor of :func:`connect`.
-
 .. coroutinefunction:: start_server(client_connected_cb, host=None, \
-                          port=None, \*, loop=None, limit=2**16, \
+                          port=None, \*, loop=None, limit=None, \
                           family=socket.AF_UNSPEC, \
                           flags=socket.AI_PASSIVE, sock=None, \
                           backlog=100, ssl=None, reuse_address=None, \
@@ -124,60 +102,9 @@ and work with streams:
 
       The *ssl_handshake_timeout* and *start_serving* parameters.
 
-   .. deprecated-removed:: 3.8 3.10
-
-      `start_server()` is deprecated if favor of :class:`StreamServer`
-
-.. coroutinefunction:: connect_read_pipe(pipe, *, limit=2**16)
-
-   Takes a :term:`file-like object <file object>` *pipe* to return a
-   :class:`Stream` object of the mode :attr:`StreamMode.READ` that has
-   similar API of :class:`StreamReader`. It can also be used as an async context manager.
-
-   *limit* determines the buffer size limit used by the returned :class:`Stream`
-   instance. By default the limit is set to 64 KiB.
-
-   .. versionadded:: 3.8
-
-.. coroutinefunction:: connect_write_pipe(pipe, *, limit=2**16)
-
-   Takes a :term:`file-like object <file object>` *pipe* to return a
-   :class:`Stream` object of the mode :attr:`StreamMode.WRITE` that has
-   similar API of :class:`StreamWriter`. It can also be used as an async context manager.
-
-   *limit* determines the buffer size limit used by the returned :class:`Stream`
-   instance. By default the limit is set to 64 KiB.
-
-   .. versionadded:: 3.8
 
 .. rubric:: Unix Sockets
 
-.. function:: connect_unix(path=None, *, limit=2**16, ssl=None, \
-                           sock=None, server_hostname=None, \
-                           ssl_handshake_timeout=None)
-
-   Establish a Unix socket connection to socket with *path* address and
-   return an awaitable :class:`Stream` object of the mode :attr:`StreamMode.READWRITE`
-   that can be used as a reader and a writer.
-
-   *limit* determines the buffer size limit used by the returned :class:`Stream`
-   instance. By default the *limit* is set to 64 KiB.
-
-   The rest of the arguments are passed directly to :meth:`loop.create_unix_connection`.
-
-   The function can be used with ``await`` to get a connected stream::
-
-       stream = await asyncio.connect_unix('/tmp/example.sock')
-
-   The function can also be used as an async context manager::
-
-       async with asyncio.connect_unix('/tmp/example.sock') as stream:
-           ...
-
-   .. availability:: Unix.
-
-   .. versionadded:: 3.8
-
 .. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, \
                         limit=None, ssl=None, sock=None, \
                         server_hostname=None, ssl_handshake_timeout=None)
@@ -199,10 +126,6 @@ and work with streams:
 
       The *path* parameter can now be a :term:`path-like object`
 
-   .. deprecated-removed:: 3.8 3.10
-
-      ``open_unix_connection()`` is deprecated if favor of :func:`connect_unix`.
-
 
 .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \
                           \*, loop=None, limit=None, sock=None, \
@@ -225,349 +148,6 @@ and work with streams:
 
       The *path* parameter can now be a :term:`path-like object`.
 
-   .. deprecated-removed:: 3.8 3.10
-
-      ``start_unix_server()`` is deprecated in favor of :class:`UnixStreamServer`.
-
-
----------
-
-StreamServer
-============
-
-.. class:: StreamServer(client_connected_cb, /, host=None, port=None, *, \
-                        limit=2**16, family=socket.AF_UNSPEC, \
-                        flags=socket.AI_PASSIVE, sock=None, backlog=100, \
-                        ssl=None, reuse_address=None, reuse_port=None, \
-                        ssl_handshake_timeout=None, shutdown_timeout=60)
-
-   The *client_connected_cb* callback is called whenever a new client
-   connection is established.  It receives a :class:`Stream` object of the
-   mode :attr:`StreamMode.READWRITE`.
-
-   *client_connected_cb* can be a plain callable or a
-   :ref:`coroutine function <coroutine>`; if it is a coroutine function,
-   it will be automatically scheduled as a :class:`Task`.
-
-   *limit* determines the buffer size limit used by the
-   returned :class:`Stream` instance.  By default the *limit*
-   is set to 64 KiB.
-
-   The rest of the arguments are passed directly to
-   :meth:`loop.create_server`.
-
-   .. coroutinemethod:: start_serving()
-
-      Binds to the given host and port to start the server.
-
-   .. coroutinemethod:: serve_forever()
-
-      Start accepting connections until the coroutine is cancelled.
-      Cancellation of ``serve_forever`` task causes the server
-      to be closed.
-
-      This method can be called if the server is already accepting
-      connections.  Only one ``serve_forever`` task can exist per
-      one *Server* object.
-
-   .. method:: is_serving()
-
-      Returns ``True`` if the server is bound and currently serving.
-
-   .. method:: bind()
-
-      Bind the server to the given *host* and *port*. This method is
-      automatically called during ``__aenter__`` when :class:`StreamServer` is
-      used as an async context manager.
-
-   .. method:: is_bound()
-
-      Return ``True`` if the server is bound.
-
-   .. coroutinemethod:: abort()
-
-      Closes the connection and cancels all pending tasks.
-
-   .. coroutinemethod:: close()
-
-      Closes the connection. This method is automatically called during
-      ``__aexit__`` when :class:`StreamServer` is used as an async context
-      manager.
-
-   .. attribute:: sockets
-
-      Returns a tuple of socket objects the server is bound to.
-
-   .. versionadded:: 3.8
-
-
-UnixStreamServer
-================
-
-.. class:: UnixStreamServer(client_connected_cb, /, path=None, *, \
-                            limit=2**16, sock=None, backlog=100, \
-                            ssl=None, ssl_handshake_timeout=None, shutdown_timeout=60)
-
-   The *client_connected_cb* callback is called whenever a new client
-   connection is established.  It receives a :class:`Stream` object of the
-   mode :attr:`StreamMode.READWRITE`.
-
-   *client_connected_cb* can be a plain callable or a
-   :ref:`coroutine function <coroutine>`; if it is a coroutine function,
-   it will be automatically scheduled as a :class:`Task`.
-
-   *limit* determines the buffer size limit used by the
-   returned :class:`Stream` instance.  By default the *limit*
-   is set to 64 KiB.
-
-   The rest of the arguments are passed directly to
-   :meth:`loop.create_unix_server`.
-
-   .. coroutinemethod:: start_serving()
-
-      Binds to the given host and port to start the server.
-
-   .. method:: is_serving()
-
-      Returns ``True`` if the server is bound and currently serving.
-
-   .. method:: bind()
-
-      Bind the server to the given *host* and *port*. This method is
-      automatically called during ``__aenter__`` when :class:`UnixStreamServer` is
-      used as an async context manager.
-
-   .. method:: is_bound()
-
-      Return ``True`` if the server is bound.
-
-   .. coroutinemethod:: abort()
-
-      Closes the connection and cancels all pending tasks.
-
-   .. coroutinemethod:: close()
-
-      Closes the connection. This method is automatically called during
-      ``__aexit__`` when :class:`UnixStreamServer` is used as an async context
-      manager.
-
-   .. attribute:: sockets
-
-      Returns a tuple of socket objects the server is bound to.
-
-   .. availability:: Unix.
-
-   .. versionadded:: 3.8
-
-Stream
-======
-
-.. class:: Stream
-
-   Represents a Stream object that provides APIs to read and write data
-   to the IO stream . It includes the API provided by :class:`StreamReader`
-   and :class:`StreamWriter`. It can also be used as :term:`asynchronous iterator`
-   where :meth:`readline` is used. It raises :exc:`StopAsyncIteration` when
-   :meth:`readline` returns empty data.
-
-   Do not instantiate *Stream* objects directly; use API like :func:`connect`
-   and :class:`StreamServer` instead.
-
-   .. versionadded:: 3.8
-
-   .. attribute:: mode
-
-      Returns the mode of the stream which is a :class:`StreamMode` value. It could
-      be one of the below:
-
-      * :attr:`StreamMode.READ` - Connection can receive data.
-      * :attr:`StreamMode.WRITE` - Connection can send data.
-      * :attr:`StreamMode.READWRITE` - Connection can send and receive data.
-
-   .. coroutinemethod:: abort()
-
-      Aborts the connection immediately, without waiting for the send buffer to drain.
-
-   .. method:: at_eof()
-
-      Return ``True`` if the buffer is empty.
-
-   .. method:: can_write_eof()
-
-      Return *True* if the underlying transport supports
-      the :meth:`write_eof` method, *False* otherwise.
-
-   .. method:: close()
-
-      The method closes the stream and the underlying socket.
-
-      It is possible to directly await on the `close()` method::
-
-         await stream.close()
-
-      The ``await`` pauses the current coroutine until the stream and the underlying
-      socket are closed (and SSL shutdown is performed for a secure connection).
-
-   .. coroutinemethod:: drain()
-
-      Wait until it is appropriate to resume writing to the stream.
-      Example::
-
-          stream.write(data)
-          await stream.drain()
-
-      This is a flow control method that interacts with the underlying
-      IO write buffer.  When the size of the buffer reaches
-      the high watermark, *drain()* blocks until the size of the
-      buffer is drained down to the low watermark and writing can
-      be resumed.  When there is nothing to wait for, the :meth:`drain`
-      returns immediately.
-
-      .. deprecated:: 3.8
-
-      It is recommended to directly await on the `write()` method instead::
-
-         await stream.write(data)
-
-   .. method:: get_extra_info(name, default=None)
-
-      Access optional transport information; see
-      :meth:`BaseTransport.get_extra_info` for details.
-
-   .. method:: is_closing()
-
-      Return ``True`` if the stream is closed or in the process of
-      being closed.
-
-   .. coroutinemethod:: read(n=-1)
-
-      Read up to *n* bytes.  If *n* is not provided, or set to ``-1``,
-      read until EOF and return all read bytes.
-
-      If EOF was received and the internal buffer is empty,
-      return an empty ``bytes`` object.
-
-   .. coroutinemethod:: readexactly(n)
-
-      Read exactly *n* bytes.
-
-      Raise an :exc:`IncompleteReadError` if EOF is reached before *n*
-      can be read.  Use the :attr:`IncompleteReadError.partial`
-      attribute to get the partially read data.
-
-   .. coroutinemethod:: readline()
-
-      Read one line, where "line" is a sequence of bytes
-      ending with ``\n``.
-
-      If EOF is received and ``\n`` was not found, the method
-      returns partially read data.
-
-      If EOF is received and the internal buffer is empty,
-      return an empty ``bytes`` object.
-
-   .. coroutinemethod:: readuntil(separator=b'\\n')
-
-      Read data from the stream until *separator* is found.
-
-      On success, the data and separator will be removed from the
-      internal buffer (consumed). Returned data will include the
-      separator at the end.
-
-      If the amount of data read exceeds the configured stream limit, a
-      :exc:`LimitOverrunError` exception is raised, and the data
-      is left in the internal buffer and can be read again.
-
-      If EOF is reached before the complete separator is found,
-      an :exc:`IncompleteReadError` exception is raised, and the internal
-      buffer is reset.  The :attr:`IncompleteReadError.partial` attribute
-      may contain a portion of the separator.
-
-   .. coroutinemethod:: sendfile(file, offset=0, count=None, *, fallback=True)
-
-      Sends a *file* over the stream using an optimized syscall if available.
-
-      For other parameters meaning please see :meth:`AbstractEventloop.sendfile`.
-
-   .. coroutinemethod:: start_tls(sslcontext, *, server_hostname=None, \
-                                  ssl_handshake_timeout=None)
-
-      Upgrades the existing transport-based connection to TLS.
-
-      For other parameters meaning please see :meth:`AbstractEventloop.start_tls`.
-
-   .. coroutinemethod:: wait_closed()
-
-      Wait until the stream is closed.
-
-      Should be called after :meth:`close` to wait until the underlying
-      connection is closed.
-
-   .. coroutinemethod:: write(data)
-
-      Write *data* to the underlying socket; wait until the data is sent, e.g.::
-
-         await stream.write(data)
-
-   .. method:: write(data)
-
-      The method attempts to write the *data* to the underlying socket immediately.
-      If that fails, the data is queued in an internal write buffer until it can be
-      sent. :meth:`drain` can be used to flush the underlying buffer once writing is
-      available::
-
-         stream.write(data)
-         await stream.drain()
-
-      .. deprecated:: 3.8
-
-      It is recommended to directly await on the `write()` method instead::
-
-          await stream.write(data)
-
-   .. method:: writelines(data)
-
-      The method writes a list (or any iterable) of bytes to the underlying socket
-      immediately.
-      If that fails, the data is queued in an internal write buffer until it can be
-      sent.
-
-      It is possible to directly await on the `writelines()` method::
-
-         await stream.writelines(lines)
-
-      The ``await`` pauses the current coroutine until the data is written to the
-      socket.
-
-   .. method:: write_eof()
-
-      Close the write end of the stream after the buffered write
-      data is flushed.
-
-
-StreamMode
-==========
-
-.. class:: StreamMode
-
-   A subclass of :class:`enum.Flag` that defines a set of values that can be
-   used to determine the ``mode`` of :class:`Stream` objects.
-
-   .. data:: READ
-
-   The stream object is readable and provides the API of :class:`StreamReader`.
-
-   .. data:: WRITE
-
-   The stream object is writeable and provides the API of :class:`StreamWriter`.
-
-   .. data:: READWRITE
-
-   The stream object is readable and writeable and provides the API of both
-   :class:`StreamReader` and :class:`StreamWriter`.
-
-  .. versionadded:: 3.8
-
 
 StreamReader
 ============
@@ -629,7 +209,8 @@ StreamReader
 
    .. method:: at_eof()
 
-      Return ``True`` if the buffer is empty.
+      Return ``True`` if the buffer is empty and :meth:`feed_eof`
+      was called.
 
 
 StreamWriter
@@ -650,22 +231,11 @@ StreamWriter
       If that fails, the data is queued in an internal write buffer until it can be
       sent.
 
-      Starting with Python 3.8, it is possible to directly await on the `write()`
-      method::
-
-         await stream.write(data)
-
-      The ``await`` pauses the current coroutine until the data is written to the
-      socket.
-
-      Below is an equivalent code that works with Python <= 3.7::
+      The method should be used along with the ``drain()`` method::
 
          stream.write(data)
          await stream.drain()
 
-      .. versionchanged:: 3.8
-         Support ``await stream.write(...)`` syntax.
-
    .. method:: writelines(data)
 
       The method writes a list (or any iterable) of bytes to the underlying socket
@@ -673,42 +243,20 @@ StreamWriter
       If that fails, the data is queued in an internal write buffer until it can be
       sent.
 
-      Starting with Python 3.8, it is possible to directly await on the `writelines()`
-      method::
-
-         await stream.writelines(lines)
-
-      The ``await`` pauses the current coroutine until the data is written to the
-      socket.
-
-      Below is an equivalent code that works with Python <= 3.7::
+      The method should be used along with the ``drain()`` method::
 
          stream.writelines(lines)
          await stream.drain()
 
-      .. versionchanged:: 3.8
-         Support ``await stream.writelines()`` syntax.
-
    .. method:: close()
 
       The method closes the stream and the underlying socket.
 
-      Starting with Python 3.8, it is possible to directly await on the `close()`
-      method::
-
-         await stream.close()
-
-      The ``await`` pauses the current coroutine until the stream and the underlying
-      socket are closed (and SSL shutdown is performed for a secure connection).
-
-      Below is an equivalent code that works with Python <= 3.7::
+      The method should be used along with the ``wait_closed()`` method::
 
          stream.close()
          await stream.wait_closed()
 
-      .. versionchanged:: 3.8
-         Support ``await stream.close()`` syntax.
-
    .. method:: can_write_eof()
 
       Return *True* if the underlying transport supports
@@ -768,17 +316,22 @@ Examples
 TCP echo client using streams
 -----------------------------
 
-TCP echo client using the :func:`asyncio.connect` function::
+TCP echo client using the :func:`asyncio.open_connection` function::
 
     import asyncio
 
     async def tcp_echo_client(message):
-        async with asyncio.connect('127.0.0.1', 8888) as stream:
-            print(f'Send: {message!r}')
-            await stream.write(message.encode())
+        reader, writer = await asyncio.open_connection(
+            '127.0.0.1', 8888)
+
+        print(f'Send: {message!r}')
+        writer.write(message.encode())
+
+        data = await reader.read(100)
+        print(f'Received: {data.decode()!r}')
 
-            data = await stream.read(100)
-            print(f'Received: {data.decode()!r}')
+        print('Close the connection')
+        writer.close()
 
     asyncio.run(tcp_echo_client('Hello World!'))
 
@@ -794,28 +347,32 @@ TCP echo client using the :func:`asyncio.connect` function::
 TCP echo server using streams
 -----------------------------
 
-TCP echo server using the :class:`asyncio.StreamServer` class::
+TCP echo server using the :func:`asyncio.start_server` function::
 
     import asyncio
 
-    async def handle_echo(stream):
-        data = await stream.read(100)
+    async def handle_echo(reader, writer):
+        data = await reader.read(100)
         message = data.decode()
-        addr = stream.get_extra_info('peername')
+        addr = writer.get_extra_info('peername')
 
         print(f"Received {message!r} from {addr!r}")
 
         print(f"Send: {message!r}")
-        await stream.write(data)
+        writer.write(data)
+        await writer.drain()
 
         print("Close the connection")
-        await stream.close()
+        writer.close()
 
     async def main():
-        async with asyncio.StreamServer(
-                handle_echo, '127.0.0.1', 8888) as server:
-            addr = server.sockets[0].getsockname()
-            print(f'Serving on {addr}')
+        server = await asyncio.start_server(
+            handle_echo, '127.0.0.1', 8888)
+
+        addr = server.sockets[0].getsockname()
+        print(f'Serving on {addr}')
+
+        async with server:
             await server.serve_forever()
 
     asyncio.run(main())
@@ -839,9 +396,11 @@ Simple example querying HTTP headers of the URL passed on the command line::
     async def print_http_headers(url):
         url = urllib.parse.urlsplit(url)
         if url.scheme == 'https':
-            stream = await asyncio.connect(url.hostname, 443, ssl=True)
+            reader, writer = await asyncio.open_connection(
+                url.hostname, 443, ssl=True)
         else:
-            stream = await asyncio.connect(url.hostname, 80)
+            reader, writer = await asyncio.open_connection(
+                url.hostname, 80)
 
         query = (
             f"HEAD {url.path or '/'} HTTP/1.0\r\n"
@@ -849,14 +408,18 @@ Simple example querying HTTP headers of the URL passed on the command line::
             f"\r\n"
         )
 
-        stream.write(query.encode('latin-1'))
-        while (line := await stream.readline()):
+        writer.write(query.encode('latin-1'))
+        while True:
+            line = await reader.readline()
+            if not line:
+                break
+
             line = line.decode('latin1').rstrip()
             if line:
                 print(f'HTTP header> {line}')
 
         # Ignore the body, close the socket
-        await stream.close()
+        writer.close()
 
     url = sys.argv[1]
     asyncio.run(print_http_headers(url))
@@ -877,7 +440,7 @@ Register an open socket to wait for data using streams
 ------------------------------------------------------
 
 Coroutine waiting until a socket receives data using the
-:func:`asyncio.connect` function::
+:func:`open_connection` function::
 
     import asyncio
     import socket
@@ -891,15 +454,17 @@ Coroutine waiting until a socket receives data using the
         rsock, wsock = socket.socketpair()
 
         # Register the open socket to wait for data.
-        async with asyncio.connect(sock=rsock) as stream:
-            # Simulate the reception of data from the network
-            loop.call_soon(wsock.send, 'abc'.encode())
+        reader, writer = await asyncio.open_connection(sock=rsock)
+
+        # Simulate the reception of data from the network
+        loop.call_soon(wsock.send, 'abc'.encode())
 
-            # Wait for data
-            data = await stream.read(100)
+        # Wait for data
+        data = await reader.read(100)
 
-            # Got data, we are done: close the socket
-            print("Received:", data.decode())
+        # Got data, we are done: close the socket
+        print("Received:", data.decode())
+        writer.close()
 
         # Close the second socket
         wsock.close()
diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py
index a6a29dbfecd5..28c2e2c429f3 100644
--- a/Lib/asyncio/__init__.py
+++ b/Lib/asyncio/__init__.py
@@ -3,7 +3,6 @@
 # flake8: noqa
 
 import sys
-import warnings
 
 # This relies on each of the submodules having an __all__ variable.
 from .base_events import *
@@ -44,40 +43,3 @@
 else:
     from .unix_events import *  # pragma: no cover
     __all__ += unix_events.__all__
-
-
-__all__ += ('StreamReader', 'StreamWriter', 'StreamReaderProtocol')  # deprecated
-
-
-def __getattr__(name):
-    global StreamReader, StreamWriter, StreamReaderProtocol
-    if name == 'StreamReader':
-        warnings.warn("StreamReader is deprecated since Python 3.8 "
-                      "in favor of Stream, and scheduled for removal "
-                      "in Python 3.10",
-                      DeprecationWarning,
-                      stacklevel=2)
-        from .streams import StreamReader as sr
-        StreamReader = sr
-        return StreamReader
-    if name == 'StreamWriter':
-        warnings.warn("StreamWriter is deprecated since Python 3.8 "
-                      "in favor of Stream, and scheduled for removal "
-                      "in Python 3.10",
-                      DeprecationWarning,
-                      stacklevel=2)
-        from .streams import StreamWriter as sw
-        StreamWriter = sw
-        return StreamWriter
-    if name == 'StreamReaderProtocol':
-        warnings.warn("Using asyncio internal class StreamReaderProtocol "
-                      "is deprecated since Python 3.8 "
-                      " and scheduled for removal "
-                      "in Python 3.10",
-                      DeprecationWarning,
-                      stacklevel=2)
-        from .streams import StreamReaderProtocol as srp
-        StreamReaderProtocol = srp
-        return StreamReaderProtocol
-
-    raise AttributeError(f"module {__name__} has no attribute {name}")
diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py
index 20c642a001d2..795530e6f69e 100644
--- a/Lib/asyncio/streams.py
+++ b/Lib/asyncio/streams.py
@@ -1,19 +1,14 @@
 __all__ = (
-    'Stream', 'StreamMode',
-    'open_connection', 'start_server',
-    'connect', 'connect_read_pipe', 'connect_write_pipe',
-    'StreamServer')
+    'StreamReader', 'StreamWriter', 'StreamReaderProtocol',
+    'open_connection', 'start_server')
 
-import enum
 import socket
 import sys
 import warnings
 import weakref
 
 if hasattr(socket, 'AF_UNIX'):
-    __all__ += ('open_unix_connection', 'start_unix_server',
-                'connect_unix',
-                'UnixStreamServer')
+    __all__ += ('open_unix_connection', 'start_unix_server')
 
 from . import coroutines
 from . import events
@@ -21,155 +16,12 @@
 from . import format_helpers
 from . import protocols
 from .log import logger
-from . import tasks
+from .tasks import sleep
 
 
 _DEFAULT_LIMIT = 2 ** 16  # 64 KiB
 
 
-class StreamMode(enum.Flag):
-    READ = enum.auto()
-    WRITE = enum.auto()
-    READWRITE = READ | WRITE
-
-
-def _ensure_can_read(mode):
-    if not mode & StreamMode.READ:
-        raise RuntimeError("The stream is write-only")
-
-
-def _ensure_can_write(mode):
-    if not mode & StreamMode.WRITE:
-        raise RuntimeError("The stream is read-only")
-
-
-class _ContextManagerHelper:
-    __slots__ = ('_awaitable', '_result')
-
-    def __init__(self, awaitable):
-        self._awaitable = awaitable
-        self._result = None
-
-    def __await__(self):
-        return self._awaitable.__await__()
-
-    async def __aenter__(self):
-        ret = await self._awaitable
-        result = await ret.__aenter__()
-        self._result = result
-        return result
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        return await self._result.__aexit__(exc_type, exc_val, exc_tb)
-
-
-def connect(host=None, port=None, *,
-            limit=_DEFAULT_LIMIT,
-            ssl=None, family=0, proto=0,
-            flags=0, sock=None, local_addr=None,
-            server_hostname=None,
-            ssl_handshake_timeout=None,
-            happy_eyeballs_delay=None, interleave=None):
-    """Connect to TCP socket on *host* : *port* address to send and receive data.
-
-    *limit* determines the buffer size limit used by the returned `Stream`
-    instance. By default the *limit* is set to 64 KiB.
-
-    The rest of the arguments are passed directly to `loop.create_connection()`.
-    """
-    # Design note:
-    # Don't use decorator approach but explicit non-async
-    # function to fail fast and explicitly
-    # if passed arguments don't match the function signature
-    return _ContextManagerHelper(_connect(host, port, limit,
-                                          ssl, family, proto,
-                                          flags, sock, local_addr,
-                                          server_hostname,
-                                          ssl_handshake_timeout,
-                                          happy_eyeballs_delay,
-                                          interleave))
-
-
-async def _connect(host, port,
-                  limit,
-                  ssl, family, proto,
-                  flags, sock, local_addr,
-                  server_hostname,
-                  ssl_handshake_timeout,
-                  happy_eyeballs_delay, interleave):
-    loop = events.get_running_loop()
-    stream = Stream(mode=StreamMode.READWRITE,
-                    limit=limit,
-                    loop=loop,
-                    _asyncio_internal=True)
-    await loop.create_connection(
-        lambda: _StreamProtocol(stream, loop=loop,
-                                _asyncio_internal=True),
-        host, port,
-        ssl=ssl, family=family, proto=proto,
-        flags=flags, sock=sock, local_addr=local_addr,
-        server_hostname=server_hostname,
-        ssl_handshake_timeout=ssl_handshake_timeout,
-        happy_eyeballs_delay=happy_eyeballs_delay, interleave=interleave)
-    return stream
-
-
-def connect_read_pipe(pipe, *, limit=_DEFAULT_LIMIT):
-    """Establish a connection to a file-like object *pipe* to receive data.
-
-    Takes a file-like object *pipe* to return a Stream object of the mode
-    StreamMode.READ that has similar API of StreamReader. It can also be used
-    as an async context manager.
-    """
-
-    # Design note:
-    # Don't use decorator approach but explicit non-async
-    # function to fail fast and explicitly
-    # if passed arguments don't match the function signature
-    return _ContextManagerHelper(_connect_read_pipe(pipe, limit))
-
-
-async def _connect_read_pipe(pipe, limit):
-    loop = events.get_running_loop()
-    stream = Stream(mode=StreamMode.READ,
-                    limit=limit,
-                    loop=loop,
-                    _asyncio_internal=True)
-    await loop.connect_read_pipe(
-        lambda: _StreamProtocol(stream, loop=loop,
-                                _asyncio_internal=True),
-        pipe)
-    return stream
-
-
-def connect_write_pipe(pipe, *, limit=_DEFAULT_LIMIT):
-    """Establish a connection to a file-like object *pipe* to send data.
-
-    Takes a file-like object *pipe* to return a Stream object of the mode
-    StreamMode.WRITE that has similar API of StreamWriter. It can also be used
-    as an async context manager.
-    """
-
-    # Design note:
-    # Don't use decorator approach but explicit non-async
-    # function to fail fast and explicitly
-    # if passed arguments don't match the function signature
-    return _ContextManagerHelper(_connect_write_pipe(pipe, limit))
-
-
-async def _connect_write_pipe(pipe, limit):
-    loop = events.get_running_loop()
-    stream = Stream(mode=StreamMode.WRITE,
-                    limit=limit,
-                    loop=loop,
-                    _asyncio_internal=True)
-    await loop.connect_write_pipe(
-        lambda: _StreamProtocol(stream, loop=loop,
-                                _asyncio_internal=True),
-        pipe)
-    return stream
-
-
 async def open_connection(host=None, port=None, *,
                           loop=None, limit=_DEFAULT_LIMIT, **kwds):
     """A wrapper for create_connection() returning a (reader, writer) pair.
@@ -189,11 +41,6 @@ def connect_write_pipe(pipe, *, limit=_DEFAULT_LIMIT):
     StreamReaderProtocol classes, just copy the code -- there's
     really nothing special here except some convenience.)
     """
-    warnings.warn("open_connection() is deprecated since Python 3.8 "
-                  "in favor of connect(), and scheduled for removal "
-                  "in Python 3.10",
-                  DeprecationWarning,
-                  stacklevel=2)
     if loop is None:
         loop = events.get_event_loop()
     else:
@@ -201,7 +48,7 @@ def connect_write_pipe(pipe, *, limit=_DEFAULT_LIMIT):
                       "and scheduled for removal in Python 3.10.",
                       DeprecationWarning, stacklevel=2)
     reader = StreamReader(limit=limit, loop=loop)
-    protocol = StreamReaderProtocol(reader, loop=loop, _asyncio_internal=True)
+    protocol = StreamReaderProtocol(reader, loop=loop)
     transport, _ = await loop.create_connection(
         lambda: protocol, host, port, **kwds)
     writer = StreamWriter(transport, protocol, reader, loop)
@@ -231,11 +78,6 @@ def connect_write_pipe(pipe, *, limit=_DEFAULT_LIMIT):
     The return value is the same as loop.create_server(), i.e. a
     Server object which can be used to stop the service.
     """
-    warnings.warn("start_server() is deprecated since Python 3.8 "
-                  "in favor of StreamServer(), and scheduled for removal "
-                  "in Python 3.10",
-                  DeprecationWarning,
-                  stacklevel=2)
     if loop is None:
         loop = events.get_event_loop()
     else:
@@ -246,201 +88,18 @@ def connect_write_pipe(pipe, *, limit=_DEFAULT_LIMIT):
     def factory():
         reader = StreamReader(limit=limit, loop=loop)
         protocol = StreamReaderProtocol(reader, client_connected_cb,
-                                        loop=loop,
-                                        _asyncio_internal=True)
+                                        loop=loop)
         return protocol
 
     return await loop.create_server(factory, host, port, **kwds)
 
 
-class _BaseStreamServer:
-    # Design notes.
-    # StreamServer and UnixStreamServer are exposed as FINAL classes,
-    # not function factories.
-    # async with serve(host, port) as server:
-    #      server.start_serving()
-    # looks ugly.
-    # The class doesn't provide API for enumerating connected streams
-    # It can be a subject for improvements in Python 3.9
-
-    _server_impl = None
-
-    def __init__(self, client_connected_cb,
-                 /,
-                 limit=_DEFAULT_LIMIT,
-                 shutdown_timeout=60,
-                 _asyncio_internal=False):
-        if not _asyncio_internal:
-            raise RuntimeError("_ServerStream is a private asyncio class")
-        self._client_connected_cb = client_connected_cb
-        self._limit = limit
-        self._loop = events.get_running_loop()
-        self._streams = {}
-        self._shutdown_timeout = shutdown_timeout
-
-    def __init_subclass__(cls):
-        if not cls.__module__.startswith('asyncio.'):
-            raise TypeError(f"asyncio.{cls.__name__} "
-                            "class cannot be inherited from")
-
-    async def bind(self):
-        if self._server_impl is not None:
-            return
-        self._server_impl = await self._bind()
-
-    def is_bound(self):
-        return self._server_impl is not None
-
-    @property
-    def sockets(self):
-        # multiple value for socket bound to both IPv4 and IPv6 families
-        if self._server_impl is None:
-            return ()
-        return self._server_impl.sockets
-
-    def is_serving(self):
-        if self._server_impl is None:
-            return False
-        return self._server_impl.is_serving()
-
-    async def start_serving(self):
-        await self.bind()
-        await self._server_impl.start_serving()
-
-    async def serve_forever(self):
-        await self.start_serving()
-        await self._server_impl.serve_forever()
-
-    async def close(self):
-        if self._server_impl is None:
-            return
-        self._server_impl.close()
-        streams = list(self._streams.keys())
-        active_tasks = list(self._streams.values())
-        if streams:
-            await tasks.wait([stream.close() for stream in streams])
-        await self._server_impl.wait_closed()
-        self._server_impl = None
-        await self._shutdown_active_tasks(active_tasks)
-
-    async def abort(self):
-        if self._server_impl is None:
-            return
-        self._server_impl.close()
-        streams = list(self._streams.keys())
-        active_tasks = list(self._streams.values())
-        if streams:
-            await tasks.wait([stream.abort() for stream in streams])
-        await self._server_impl.wait_closed()
-        self._server_impl = None
-        await self._shutdown_active_tasks(active_tasks)
-
-    async def __aenter__(self):
-        await self.bind()
-        return self
-
-    async def __aexit__(self, exc_type, exc_value, exc_tb):
-        await self.close()
-
-    def _attach(self, stream, task):
-        self._streams[stream] = task
-
-    def _detach(self, stream, task):
-        del self._streams[stream]
-
-    async def _shutdown_active_tasks(self, active_tasks):
-        if not active_tasks:
-            return
-        # NOTE: tasks finished with exception are reported
-        # by the Task.__del__() method.
-        done, pending = await tasks.wait(active_tasks,
-                                         timeout=self._shutdown_timeout)
-        if not pending:
-            return
-        for task in pending:
-            task.cancel()
-        done, pending = await tasks.wait(pending,
-                                         timeout=self._shutdown_timeout)
-        for task in pending:
-            self._loop.call_exception_handler({
-                "message": (f'{task!r} ignored cancellation request '
-                            f'from a closing {self!r}'),
-                "stream_server": self
-            })
-
-    def __repr__(self):
-        ret = [f'{self.__class__.__name__}']
-        if self.is_serving():
-            ret.append('serving')
-        if self.sockets:
-            ret.append(f'sockets={self.sockets!r}')
-        return '<' + ' '.join(ret) + '>'
-
-    def __del__(self, _warn=warnings.warn):
-        if self._server_impl is not None:
-            _warn(f"unclosed stream server {self!r}",
-                  ResourceWarning, source=self)
-            self._server_impl.close()
-
-
-class StreamServer(_BaseStreamServer):
-
-    def __init__(self, client_connected_cb, /, host=None, port=None, *,
-                 limit=_DEFAULT_LIMIT,
-                 family=socket.AF_UNSPEC,
-                 flags=socket.AI_PASSIVE, sock=None, backlog=100,
-                 ssl=None, reuse_address=None, reuse_port=None,
-                 ssl_handshake_timeout=None,
-                 shutdown_timeout=60):
-        super().__init__(client_connected_cb,
-                         limit=limit,
-                         shutdown_timeout=shutdown_timeout,
-                         _asyncio_internal=True)
-        self._host = host
-        self._port = port
-        self._family = family
-        self._flags = flags
-        self._sock = sock
-        self._backlog = backlog
-        self._ssl = ssl
-        self._reuse_address = reuse_address
-        self._reuse_port = reuse_port
-        self._ssl_handshake_timeout = ssl_handshake_timeout
-
-    async def _bind(self):
-        def factory():
-            protocol = _ServerStreamProtocol(self,
-                                             self._limit,
-                                             self._client_connected_cb,
-                                             loop=self._loop,
-                                             _asyncio_internal=True)
-            return protocol
-        return await self._loop.create_server(
-            factory,
-            self._host,
-            self._port,
-            start_serving=False,
-            family=self._family,
-            flags=self._flags,
-            sock=self._sock,
-            backlog=self._backlog,
-            ssl=self._ssl,
-            reuse_address=self._reuse_address,
-            reuse_port=self._reuse_port,
-            ssl_handshake_timeout=self._ssl_handshake_timeout)
-
-
 if hasattr(socket, 'AF_UNIX'):
     # UNIX Domain Sockets are supported on this platform
 
     async def open_unix_connection(path=None, *,
                                    loop=None, limit=_DEFAULT_LIMIT, **kwds):
         """Similar to `open_connection` but works with UNIX Domain Sockets."""
-        warnings.warn("open_unix_connection() is deprecated since Python 3.8 "
-                      "in favor of connect_unix(), and scheduled for removal "
-                      "in Python 3.10",
-                      DeprecationWarning,
-                      stacklevel=2)
         if loop is None:
             loop = events.get_event_loop()
         else:
@@ -448,62 +107,15 @@ def factory():
                           "and scheduled for removal in Python 3.10.",
                           DeprecationWarning, stacklevel=2)
         reader = StreamReader(limit=limit, loop=loop)
-        protocol = StreamReaderProtocol(reader, loop=loop,
-                                        _asyncio_internal=True)
+        protocol = StreamReaderProtocol(reader, loop=loop)
         transport, _ = await loop.create_unix_connection(
             lambda: protocol, path, **kwds)
         writer = StreamWriter(transport, protocol, reader, loop)
         return reader, writer
 
-
-    def connect_unix(path=None, *,
-                     limit=_DEFAULT_LIMIT,
-                     ssl=None, sock=None,
-                     server_hostname=None,
-                     ssl_handshake_timeout=None):
-        """Similar to `connect()` but works with UNIX Domain Sockets."""
-        # Design note:
-        # Don't use decorator approach but explicit non-async
-        # function to fail fast and explicitly
-        # if passed arguments don't match the function signature
-        return _ContextManagerHelper(_connect_unix(path,
-                                                   limit,
-                                                   ssl, sock,
-                                                   server_hostname,
-                                                   ssl_handshake_timeout))
-
-
-    async def _connect_unix(path,
-                           limit,
-                           ssl, sock,
-                           server_hostname,
-                           ssl_handshake_timeout):
-        """Similar to `connect()` but works with UNIX Domain Sockets."""
-        loop = events.get_running_loop()
-        stream = Stream(mode=StreamMode.READWRITE,
-                        limit=limit,
-                        loop=loop,
-                        _asyncio_internal=True)
-        await loop.create_unix_connection(
-            lambda: _StreamProtocol(stream,
-                                    loop=loop,
-                                    _asyncio_internal=True),
-            path,
-            ssl=ssl,
-            sock=sock,
-            server_hostname=server_hostname,
-            ssl_handshake_timeout=ssl_handshake_timeout)
-        return stream
-
-
     async def start_unix_server(client_connected_cb, path=None, *,
                                 loop=None, limit=_DEFAULT_LIMIT, **kwds):
         """Similar to `start_server` but works with UNIX Domain Sockets."""
-        warnings.warn("start_unix_server() is deprecated since Python 3.8 "
-                      "in favor of UnixStreamServer(), and scheduled "
-                      "for removal in Python 3.10",
-                      DeprecationWarning,
-                      stacklevel=2)
         if loop is None:
             loop = events.get_event_loop()
         else:
@@ -514,48 +126,11 @@ def connect_unix(path=None, *,
         def factory():
             reader = StreamReader(limit=limit, loop=loop)
             protocol = StreamReaderProtocol(reader, client_connected_cb,
-                                            loop=loop,
-                                            _asyncio_internal=True)
+                                            loop=loop)
             return protocol
 
         return await loop.create_unix_server(factory, path, **kwds)
 
-    class UnixStreamServer(_BaseStreamServer):
-
-        def __init__(self, client_connected_cb, /, path=None, *,
-                     limit=_DEFAULT_LIMIT,
-                     sock=None,
-                     backlog=100,
-                     ssl=None,
-                     ssl_handshake_timeout=None,
-                     shutdown_timeout=60):
-            super().__init__(client_connected_cb,
-                             limit=limit,
-                             shutdown_timeout=shutdown_timeout,
-                             _asyncio_internal=True)
-            self._path = path
-            self._sock = sock
-            self._backlog = backlog
-            self._ssl = ssl
-            self._ssl_handshake_timeout = ssl_handshake_timeout
-
-        async def _bind(self):
-            def factory():
-                protocol = _ServerStreamProtocol(self,
-                                                 self._limit,
-                                                 self._client_connected_cb,
-                                                 loop=self._loop,
-                                                 _asyncio_internal=True)
-                return protocol
-            return await self._loop.create_unix_server(
-                factory,
-                self._path,
-                start_serving=False,
-                sock=self._sock,
-                backlog=self._backlog,
-                ssl=self._ssl,
-                ssl_handshake_timeout=self._ssl_handshake_timeout)
-
 
 class FlowControlMixin(protocols.Protocol):
     """Reusable flow control logic for StreamWriter.drain().
@@ -567,20 +142,11 @@ class FlowControlMixin(protocols.Protocol):
     StreamWriter.drain() must wait for _drain_helper() coroutine.
     """
 
-    def __init__(self, loop=None, *, _asyncio_internal=False):
+    def __init__(self, loop=None):
         if loop is None:
             self._loop = events.get_event_loop()
         else:
             self._loop = loop
-        if not _asyncio_internal:
-            # NOTE:
-            # Avoid inheritance from FlowControlMixin
-            # Copy-paste the code to your project
-            # if you need flow control helpers
-            warnings.warn(f"{self.__class__} should be instaniated "
-                          "by asyncio internals only, "
-                          "please avoid its creation from user code",
-                          DeprecationWarning)
         self._paused = False
         self._drain_waiter = None
         self._connection_lost = False
@@ -634,8 +200,6 @@ def _get_close_waiter(self, stream):
         raise NotImplementedError
 
 
-# begin legacy stream APIs
-
 class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
     """Helper class to adapt between Protocol and StreamReader.
 
@@ -645,47 +209,103 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
     call inappropriate methods of the protocol.)
     """
 
-    def __init__(self, stream_reader, client_connected_cb=None, loop=None,
-                 *, _asyncio_internal=False):
-        super().__init__(loop=loop, _asyncio_internal=_asyncio_internal)
-        self._stream_reader = stream_reader
+    _source_traceback = None
+
+    def __init__(self, stream_reader, client_connected_cb=None, loop=None):
+        super().__init__(loop=loop)
+        if stream_reader is not None:
+            self._stream_reader_wr = weakref.ref(stream_reader,
+                                                 self._on_reader_gc)
+            self._source_traceback = stream_reader._source_traceback
+        else:
+            self._stream_reader_wr = None
+        if client_connected_cb is not None:
+            # This is a stream created by the `create_server()` function.
+            # Keep a strong reference to the reader until a connection
+            # is established.
+            self._strong_reader = stream_reader
+        self._reject_connection = False
         self._stream_writer = None
+        self._transport = None
         self._client_connected_cb = client_connected_cb
         self._over_ssl = False
         self._closed = self._loop.create_future()
 
+    def _on_reader_gc(self, wr):
+        transport = self._transport
+        if transport is not None:
+            # connection_made was called
+            context = {
+                'message': ('An open stream object is being garbage '
+                            'collected; call "stream.close()" explicitly.')
+            }
+            if self._source_traceback:
+                context['source_traceback'] = self._source_traceback
+            self._loop.call_exception_handler(context)
+            transport.abort()
+        else:
+            self._reject_connection = True
+        self._stream_reader_wr = None
+
+    @property
+    def _stream_reader(self):
+        if self._stream_reader_wr is None:
+            return None
+        return self._stream_reader_wr()
+
     def connection_made(self, transport):
-        self._stream_reader.set_transport(transport)
+        if self._reject_connection:
+            context = {
+                'message': ('An open stream was garbage collected prior to '
+                            'establishing network connection; '
+                            'call "stream.close()" explicitly.')
+            }
+            if self._source_traceback:
+                context['source_traceback'] = self._source_traceback
+            self._loop.call_exception_handler(context)
+            transport.abort()
+            return
+        self._transport = transport
+        reader = self._stream_reader
+        if reader is not None:
+            reader.set_transport(transport)
         self._over_ssl = transport.get_extra_info('sslcontext') is not None
         if self._client_connected_cb is not None:
             self._stream_writer = StreamWriter(transport, self,
-                                               self._stream_reader,
+                                               reader,
                                                self._loop)
-            res = self._client_connected_cb(self._stream_reader,
+            res = self._client_connected_cb(reader,
                                             self._stream_writer)
             if coroutines.iscoroutine(res):
                 self._loop.create_task(res)
+            self._strong_reader = None
 
     def connection_lost(self, exc):
-        if self._stream_reader is not None:
+        reader = self._stream_reader
+        if reader is not None:
             if exc is None:
-                self._stream_reader.feed_eof()
+                reader.feed_eof()
             else:
-                self._stream_reader.set_exception(exc)
+                reader.set_exception(exc)
         if not self._closed.done():
             if exc is None:
                 self._closed.set_result(None)
             else:
                 self._closed.set_exception(exc)
         super().connection_lost(exc)
-        self._stream_reader = None
+        self._stream_reader_wr = None
         self._stream_writer = None
+        self._transport = None
 
     def data_received(self, data):
-        self._stream_reader.feed_data(data)
+        reader = self._stream_reader
+        if reader is not None:
+            reader.feed_data(data)
 
     def eof_received(self):
-        self._stream_reader.feed_eof()
+        reader = self._stream_reader
+        if reader is not None:
+            reader.feed_eof()
         if self._over_ssl:
             # Prevent a warning in SSLProtocol.eof_received:
             # "returning true from eof_received()
@@ -693,6 +313,9 @@ def eof_received(self):
             return False
         return True
 
+    def _get_close_waiter(self, stream):
+        return self._closed
+
     def __del__(self):
         # Prevent reports about unhandled exceptions.
         # Better than self._closed._log_traceback = False hack
@@ -718,6 +341,8 @@ def __init__(self, transport, protocol, reader, loop):
         assert reader is None or isinstance(reader, StreamReader)
         self._reader = reader
         self._loop = loop
+        self._complete_fut = self._loop.create_future()
+        self._complete_fut.set_result(None)
 
     def __repr__(self):
         info = [self.__class__.__name__, f'transport={self._transport!r}']
@@ -748,7 +373,7 @@ def is_closing(self):
         return self._transport.is_closing()
 
     async def wait_closed(self):
-        await self._protocol._closed
+        await self._protocol._get_close_waiter(self)
 
     def get_extra_info(self, name, default=None):
         return self._transport.get_extra_info(name, default)
@@ -766,18 +391,23 @@ def get_extra_info(self, name, default=None):
             if exc is not None:
                 raise exc
         if self._transport.is_closing():
+            # Wait for protocol.connection_lost() call
+            # Raise connection closing error if any,
+            # ConnectionResetError otherwise
             # Yield to the event loop so connection_lost() may be
             # called.  Without this, _drain_helper() would return
             # immediately, and code that calls
             #     write(...); await drain()
             # in a loop would never call connection_lost(), so it
             # would not see an error when the socket is closed.
-            await tasks.sleep(0, loop=self._loop)
+            await sleep(0)
         await self._protocol._drain_helper()
 
 
 class StreamReader:
 
+    _source_traceback = None
+
     def __init__(self, limit=_DEFAULT_LIMIT, loop=None):
         # The line length limit is  a security feature;
         # it also doubles as half the buffer limit.
@@ -796,6 +426,9 @@ def __init__(self, limit=_DEFAULT_LIMIT, loop=None):
         self._exception = None
         self._transport = None
         self._paused = False
+        if self._loop.get_debug():
+            self._source_traceback = format_helpers.extract_stack(
+                sys._getframe(1))
 
     def __repr__(self):
         info = ['StreamReader']
@@ -1123,706 +756,3 @@ def __aiter__(self):
         if val == b'':
             raise StopAsyncIteration
         return val
-
-
-# end legacy stream APIs
-
-
-class _BaseStreamProtocol(FlowControlMixin, protocols.Protocol):
-    """Helper class to adapt between Protocol and StreamReader.
-
-    (This is a helper class instead of making StreamReader itself a
-    Protocol subclass, because the StreamReader has other potential
-    uses, and to prevent the user of the StreamReader to accidentally
-    call inappropriate methods of the protocol.)
-    """
-
-    _stream = None  # initialized in derived classes
-
-    def __init__(self, loop=None,
-                 *, _asyncio_internal=False):
-        super().__init__(loop=loop, _asyncio_internal=_asyncio_internal)
-        self._transport = None
-        self._over_ssl = False
-        self._closed = self._loop.create_future()
-
-    def connection_made(self, transport):
-        self._transport = transport
-        self._over_ssl = transport.get_extra_info('sslcontext') is not None
-
-    def connection_lost(self, exc):
-        stream = self._stream
-        if stream is not None:
-            if exc is None:
-                stream._feed_eof()
-            else:
-                stream._set_exception(exc)
-        if not self._closed.done():
-            if exc is None:
-                self._closed.set_result(None)
-            else:
-                self._closed.set_exception(exc)
-        super().connection_lost(exc)
-        self._transport = None
-
-    def data_received(self, data):
-        stream = self._stream
-        if stream is not None:
-            stream._feed_data(data)
-
-    def eof_received(self):
-        stream = self._stream
-        if stream is not None:
-            stream._feed_eof()
-        if self._over_ssl:
-            # Prevent a warning in SSLProtocol.eof_received:
-            # "returning true from eof_received()
-            # has no effect when using ssl"
-            return False
-        return True
-
-    def _get_close_waiter(self, stream):
-        return self._closed
-
-    def __del__(self):
-        # Prevent reports about unhandled exceptions.
-        # Better than self._closed._log_traceback = False hack
-        closed = self._get_close_waiter(self._stream)
-        if closed.done() and not closed.cancelled():
-            closed.exception()
-
-
-class _StreamProtocol(_BaseStreamProtocol):
-    _source_traceback = None
-
-    def __init__(self, stream, loop=None,
-                 *, _asyncio_internal=False):
-        super().__init__(loop=loop, _asyncio_internal=_asyncio_internal)
-        self._source_traceback = stream._source_traceback
-        self._stream_wr = weakref.ref(stream, self._on_gc)
-        self._reject_connection = False
-
-    def _on_gc(self, wr):
-        transport = self._transport
-        if transport is not None:
-            # connection_made was called
-            context = {
-                'message': ('An open stream object is being garbage '
-                            'collected; call "stream.close()" explicitly.')
-            }
-            if self._source_traceback:
-                context['source_traceback'] = self._source_traceback
-            self._loop.call_exception_handler(context)
-            transport.abort()
-        else:
-            self._reject_connection = True
-        self._stream_wr = None
-
-    @property
-    def _stream(self):
-        if self._stream_wr is None:
-            return None
-        return self._stream_wr()
-
-    def connection_made(self, transport):
-        if self._reject_connection:
-            context = {
-                'message': ('An open stream was garbage collected prior to '
-                            'establishing network connection; '
-                            'call "stream.close()" explicitly.')
-            }
-            if self._source_traceback:
-                context['source_traceback'] = self._source_traceback
-            self._loop.call_exception_handler(context)
-            transport.abort()
-            return
-        super().connection_made(transport)
-        stream = self._stream
-        if stream is None:
-            return
-        stream._set_transport(transport)
-        stream._protocol = self
-
-    def connection_lost(self, exc):
-        super().connection_lost(exc)
-        self._stream_wr = None
-
-
-class _ServerStreamProtocol(_BaseStreamProtocol):
-    def __init__(self, server, limit, client_connected_cb, loop=None,
-                 *, _asyncio_internal=False):
-        super().__init__(loop=loop, _asyncio_internal=_asyncio_internal)
-        assert self._closed
-        self._client_connected_cb = client_connected_cb
-        self._limit = limit
-        self._server = server
-        self._task = None
-
-    def connection_made(self, transport):
-        super().connection_made(transport)
-        stream = Stream(mode=StreamMode.READWRITE,
-                        transport=transport,
-                        protocol=self,
-                        limit=self._limit,
-                        loop=self._loop,
-                        is_server_side=True,
-                        _asyncio_internal=True)
-        self._stream = stream
-        # If self._client_connected_cb(self._stream) fails
-        # the exception is logged by transport
-        self._task = self._loop.create_task(
-            self._client_connected_cb(self._stream))
-        self._server._attach(stream, self._task)
-
-    def connection_lost(self, exc):
-        super().connection_lost(exc)
-        self._server._detach(self._stream, self._task)
-        self._stream = None
-
-
-class _OptionalAwait:
-    # The class doesn't create a coroutine
-    # if not awaited
-    # It prevents "coroutine is never awaited" message
-
-    __slots___ = ('_method',)
-
-    def __init__(self, method):
-        self._method = method
-
-    def __await__(self):
-        return self._method().__await__()
-
-
-class Stream:
-    """Wraps a Transport.
-
-    This exposes write(), writelines(), [can_]write_eof(),
-    get_extra_info() and close().  It adds drain() which returns an
-    optional Future on which you can wait for flow control.  It also
-    adds a transport property which references the Transport
-    directly.
-    """
-
-    _source_traceback = None
-
-    def __init__(self, mode, *,
-                 transport=None,
-                 protocol=None,
-                 loop=None,
-                 limit=_DEFAULT_LIMIT,
-                 is_server_side=False,
-                 _asyncio_internal=False):
-        if not _asyncio_internal:
-            raise RuntimeError(f"{self.__class__} should be instantiated "
-                               "by asyncio internals only")
-        self._mode = mode
-        self._transport = transport
-        self._protocol = protocol
-        self._is_server_side = is_server_side
-
-        # The line length limit is  a security feature;
-        # it also doubles as half the buffer limit.
-
-        if limit <= 0:
-            raise ValueError('Limit cannot be <= 0')
-
-        self._limit = limit
-        if loop is None:
-            self._loop = events.get_event_loop()
-        else:
-            self._loop = loop
-        self._buffer = bytearray()
-        self._eof = False    # Whether we're done.
-        self._waiter = None  # A future used by _wait_for_data()
-        self._exception = None
-        self._paused = False
-        self._complete_fut = self._loop.create_future()
-        self._complete_fut.set_result(None)
-
-        if self._loop.get_debug():
-            self._source_traceback = format_helpers.extract_stack(
-                sys._getframe(1))
-
-    def __repr__(self):
-        info = [self.__class__.__name__]
-        info.append(f'mode={self._mode}')
-        if self._buffer:
-            info.append(f'{len(self._buffer)} bytes')
-        if self._eof:
-            info.append('eof')
-        if self._limit != _DEFAULT_LIMIT:
-            info.append(f'limit={self._limit}')
-        if self._waiter:
-            info.append(f'waiter={self._waiter!r}')
-        if self._exception:
-            info.append(f'exception={self._exception!r}')
-        if self._transport:
-            info.append(f'transport={self._transport!r}')
-        if self._paused:
-            info.append('paused')
-        return '<{}>'.format(' '.join(info))
-
-    @property
-    def mode(self):
-        return self._mode
-
-    def is_server_side(self):
-        return self._is_server_side
-
-    @property
-    def transport(self):
-        warnings.warn("Stream.transport attribute is deprecated "
-                      "since Python 3.8 and is scheduled for removal in 3.10; "
-                      "it is an internal API",
-                      DeprecationWarning,
-                      stacklevel=2)
-        return self._transport
-
-    def write(self, data):
-        _ensure_can_write(self._mode)
-        self._transport.write(data)
-        return self._fast_drain()
-
-    def writelines(self, data):
-        _ensure_can_write(self._mode)
-        self._transport.writelines(data)
-        return self._fast_drain()
-
-    def _fast_drain(self):
-        # The helper tries to use fast-path to return already existing
-        # complete future object if underlying transport is not paused
-        # and actual waiting for writing resume is not needed
-        exc = self.exception()
-        if exc is not None:
-            fut = self._loop.create_future()
-            fut.set_exception(exc)
-            return fut
-        if not self._transport.is_closing():
-            if self._protocol._connection_lost:
-                fut = self._loop.create_future()
-                fut.set_exception(ConnectionResetError('Connection lost'))
-                return fut
-            if not self._protocol._paused:
-                # fast path, the stream is not paused
-                # no need to wait for resume signal
-                return self._complete_fut
-        return _OptionalAwait(self.drain)
-
-    def write_eof(self):
-        _ensure_can_write(self._mode)
-        return self._transport.write_eof()
-
-    def can_write_eof(self):
-        if not self._mode.is_write():
-            return False
-        return self._transport.can_write_eof()
-
-    def close(self):
-        self._transport.close()
-        return _OptionalAwait(self.wait_closed)
-
-    def is_closing(self):
-        return self._transport.is_closing()
-
-    async def abort(self):
-        self._transport.abort()
-        await self.wait_closed()
-
-    async def wait_closed(self):
-        await self._protocol._get_close_waiter(self)
-
-    def get_extra_info(self, name, default=None):
-        return self._transport.get_extra_info(name, default)
-
-    async def drain(self):
-        """Flush the write buffer.
-
-        The intended use is to write
-
-          w.write(data)
-          await w.drain()
-        """
-        _ensure_can_write(self._mode)
-        exc = self.exception()
-        if exc is not None:
-            raise exc
-        if self._transport.is_closing():
-            # Wait for protocol.connection_lost() call
-            # Raise connection closing error if any,
-            # ConnectionResetError otherwise
-            await tasks.sleep(0)
-        await self._protocol._drain_helper()
-
-    async def sendfile(self, file, offset=0, count=None, *, fallback=True):
-        await self.drain()  # check for stream mode and exceptions
-        return await self._loop.sendfile(self._transport, file,
-                                         offset, count, fallback=fallback)
-
-    async def start_tls(self, sslcontext, *,
-                        server_hostname=None,
-                        ssl_handshake_timeout=None):
-        await self.drain()  # check for stream mode and exceptions
-        transport = await self._loop.start_tls(
-            self._transport, self._protocol, sslcontext,
-            server_side=self._is_server_side,
-            server_hostname=server_hostname,
-            ssl_handshake_timeout=ssl_handshake_timeout)
-        self._transport = transport
-        self._protocol._transport = transport
-        self._protocol._over_ssl = True
-
-    def exception(self):
-        return self._exception
-
-    def set_exception(self, exc):
-        warnings.warn("Stream.set_exception() is deprecated "
-                      "since Python 3.8 and is scheduled for removal in 3.10; "
-                      "it is an internal API",
-                      DeprecationWarning,
-                      stacklevel=2)
-        self._set_exception(exc)
-
-    def _set_exception(self, exc):
-        self._exception = exc
-
-        waiter = self._waiter
-        if waiter is not None:
-            self._waiter = None
-            if not waiter.cancelled():
-                waiter.set_exception(exc)
-
-    def _wakeup_waiter(self):
-        """Wakeup read*() functions waiting for data or EOF."""
-        waiter = self._waiter
-        if waiter is not None:
-            self._waiter = None
-            if not waiter.cancelled():
-                waiter.set_result(None)
-
-    def set_transport(self, transport):
-        warnings.warn("Stream.set_transport() is deprecated "
-                      "since Python 3.8 and is scheduled for removal in 3.10; "
-                      "it is an internal API",
-                      DeprecationWarning,
-                      stacklevel=2)
-        self._set_transport(transport)
-
-    def _set_transport(self, transport):
-        if transport is self._transport:
-            return
-        assert self._transport is None, 'Transport already set'
-        self._transport = transport
-
-    def _maybe_resume_transport(self):
-        if self._paused and len(self._buffer) <= self._limit:
-            self._paused = False
-            self._transport.resume_reading()
-
-    def feed_eof(self):
-        warnings.warn("Stream.feed_eof() is deprecated "
-                      "since Python 3.8 and is scheduled for removal in 3.10; "
-                      "it is an internal API",
-                      DeprecationWarning,
-                      stacklevel=2)
-        self._feed_eof()
-
-    def _feed_eof(self):
-        self._eof = True
-        self._wakeup_waiter()
-
-    def at_eof(self):
-        """Return True if the buffer is empty and 'feed_eof' was called."""
-        return self._eof and not self._buffer
-
-    def feed_data(self, data):
-        warnings.warn("Stream.feed_data() is deprecated "
-                      "since Python 3.8 and is scheduled for removal in 3.10; "
-                      "it is an internal API",
-                      DeprecationWarning,
-                      stacklevel=2)
-        self._feed_data(data)
-
-    def _feed_data(self, data):
-        _ensure_can_read(self._mode)
-        assert not self._eof, 'feed_data after feed_eof'
-
-        if not data:
-            return
-
-        self._buffer.extend(data)
-        self._wakeup_waiter()
-
-        if (self._transport is not None and
-                not self._paused and
-                len(self._buffer) > 2 * self._limit):
-            try:
-                self._transport.pause_reading()
-            except NotImplementedError:
-                # The transport can't be paused.
-                # We'll just have to buffer all data.
-                # Forget the transport so we don't keep trying.
-                self._transport = None
-            else:
-                self._paused = True
-
-    async def _wait_for_data(self, func_name):
-        """Wait until feed_data() or feed_eof() is called.
-
-        If stream was paused, automatically resume it.
-        """
-        # StreamReader uses a future to link the protocol feed_data() method
-        # to a read coroutine. Running two read coroutines at the same time
-        # would have an unexpected behaviour. It would not possible to know
-        # which coroutine would get the next data.
-        if self._waiter is not None:
-            raise RuntimeError(
-                f'{func_name}() called while another coroutine is '
-                f'already waiting for incoming data')
-
-        assert not self._eof, '_wait_for_data after EOF'
-
-        # Waiting for data while paused will make deadlock, so prevent it.
-        # This is essential for readexactly(n) for case when n > self._limit.
-        if self._paused:
-            self._paused = False
-            self._transport.resume_reading()
-
-        self._waiter = self._loop.create_future()
-        try:
-            await self._waiter
-        finally:
-            self._waiter = None
-
-    async def readline(self):
-        """Read chunk of data from the stream until newline (b'\n') is found.
-
-        On success, return chunk that ends with newline. If only partial
-        line can be read due to EOF, return incomplete line without
-        terminating newline. When EOF was reached while no bytes read, empty
-        bytes object is returned.
-
-        If limit is reached, ValueError will be raised. In that case, if
-        newline was found, complete line including newline will be removed
-        from internal buffer. Else, internal buffer will be cleared. Limit is
-        compared against part of the line without newline.
-
-        If stream was paused, this function will automatically resume it if
-        needed.
-        """
-        _ensure_can_read(self._mode)
-        sep = b'\n'
-        seplen = len(sep)
-        try:
-            line = await self.readuntil(sep)
-        except exceptions.IncompleteReadError as e:
-            return e.partial
-        except exceptions.LimitOverrunError as e:
-            if self._buffer.startswith(sep, e.consumed):
-                del self._buffer[:e.consumed + seplen]
-            else:
-                self._buffer.clear()
-            self._maybe_resume_transport()
-            raise ValueError(e.args[0])
-        return line
-
-    async def readuntil(self, separator=b'\n'):
-        """Read data from the stream until ``separator`` is found.
-
-        On success, the data and separator will be removed from the
-        internal buffer (consumed). Returned data will include the
-        separator at the end.
-
-        Configured stream limit is used to check result. Limit sets the
-        maximal length of data that can be returned, not counting the
-        separator.
-
-        If an EOF occurs and the complete separator is still not found,
-        an IncompleteReadError exception will be raised, and the internal
-        buffer will be reset.  The IncompleteReadError.partial attribute
-        may contain the separator partially.
-
-        If the data cannot be read because of over limit, a
-        LimitOverrunError exception  will be raised, and the data
-        will be left in the internal buffer, so it can be read again.
-        """
-        _ensure_can_read(self._mode)
-        seplen = len(separator)
-        if seplen == 0:
-            raise ValueError('Separator should be at least one-byte string')
-
-        if self._exception is not None:
-            raise self._exception
-
-        # Consume whole buffer except last bytes, which length is
-        # one less than seplen. Let's check corner cases with
-        # separator='SEPARATOR':
-        # * we have received almost complete separator (without last
-        #   byte). i.e buffer='some textSEPARATO'. In this case we
-        #   can safely consume len(separator) - 1 bytes.
-        # * last byte of buffer is first byte of separator, i.e.
-        #   buffer='abcdefghijklmnopqrS'. We may safely consume
-        #   everything except that last byte, but this require to
-        #   analyze bytes of buffer that match partial separator.
-        #   This is slow and/or require FSM. For this case our
-        #   implementation is not optimal, since require rescanning
-        #   of data that is known to not belong to separator. In
-        #   real world, separator will not be so long to notice
-        #   performance problems. Even when reading MIME-encoded
-        #   messages :)
-
-        # `offset` is the number of bytes from the beginning of the buffer
-        # where there is no occurrence of `separator`.
-        offset = 0
-
-        # Loop until we find `separator` in the buffer, exceed the buffer size,
-        # or an EOF has happened.
-        while True:
-            buflen = len(self._buffer)
-
-            # Check if we now have enough data in the buffer for `separator` to
-            # fit.
-            if buflen - offset >= seplen:
-                isep = self._buffer.find(separator, offset)
-
-                if isep != -1:
-                    # `separator` is in the buffer. `isep` will be used later
-                    # to retrieve the data.
-                    break
-
-                # see upper comment for explanation.
-                offset = buflen + 1 - seplen
-                if offset > self._limit:
-                    raise exceptions.LimitOverrunError(
-                        'Separator is not found, and chunk exceed the limit',
-                        offset)
-
-            # Complete message (with full separator) may be present in buffer
-            # even when EOF flag is set. This may happen when the last chunk
-            # adds data which makes separator be found. That's why we check for
-            # EOF *ater* inspecting the buffer.
-            if self._eof:
-                chunk = bytes(self._buffer)
-                self._buffer.clear()
-                raise exceptions.IncompleteReadError(chunk, None)
-
-            # _wait_for_data() will resume reading if stream was paused.
-            await self._wait_for_data('readuntil')
-
-        if isep > self._limit:
-            raise exceptions.LimitOverrunError(
-                'Separator is found, but chunk is longer than limit', isep)
-
-        chunk = self._buffer[:isep + seplen]
-        del self._buffer[:isep + seplen]
-        self._maybe_resume_transport()
-        return bytes(chunk)
-
-    async def read(self, n=-1):
-        """Read up to `n` bytes from the stream.
-
-        If n is not provided, or set to -1, read until EOF and return all read
-        bytes. If the EOF was received and the internal buffer is empty, return
-        an empty bytes object.
-
-        If n is zero, return empty bytes object immediately.
-
-        If n is positive, this function try to read `n` bytes, and may return
-        less or equal bytes than requested, but at least one byte. If EOF was
-        received before any byte is read, this function returns empty byte
-        object.
-
-        Returned value is not limited with limit, configured at stream
-        creation.
-
-        If stream was paused, this function will automatically resume it if
-        needed.
-        """
-        _ensure_can_read(self._mode)
-
-        if self._exception is not None:
-            raise self._exception
-
-        if n == 0:
-            return b''
-
-        if n < 0:
-            # This used to just loop creating a new waiter hoping to
-            # collect everything in self._buffer, but that would
-            # deadlock if the subprocess sends more than self.limit
-            # bytes.  So just call self.read(self._limit) until EOF.
-            blocks = []
-            while True:
-                block = await self.read(self._limit)
-                if not block:
-                    break
-                blocks.append(block)
-            return b''.join(blocks)
-
-        if not self._buffer and not self._eof:
-            await self._wait_for_data('read')
-
-        # This will work right even if buffer is less than n bytes
-        data = bytes(self._buffer[:n])
-        del self._buffer[:n]
-
-        self._maybe_resume_transport()
-        return data
-
-    async def readexactly(self, n):
-        """Read exactly `n` bytes.
-
-        Raise an IncompleteReadError if EOF is reached before `n` bytes can be
-        read. The IncompleteReadError.partial attribute of the exception will
-        contain the partial read bytes.
-
-        if n is zero, return empty bytes object.
-
-        Returned value is not limited with limit, configured at stream
-        creation.
-
-        If stream was paused, this function will automatically resume it if
-        needed.
-        """
-        _ensure_can_read(self._mode)
-        if n < 0:
-            raise ValueError('readexactly size can not be less than zero')
-
-        if self._exception is not None:
-            raise self._exception
-
-        if n == 0:
-            return b''
-
-        while len(self._buffer) < n:
-            if self._eof:
-                incomplete = bytes(self._buffer)
-                self._buffer.clear()
-                raise exceptions.IncompleteReadError(incomplete, n)
-
-            await self._wait_for_data('readexactly')
-
-        if len(self._buffer) == n:
-            data = bytes(self._buffer)
-            self._buffer.clear()
-        else:
-            data = bytes(self._buffer[:n])
-            del self._buffer[:n]
-        self._maybe_resume_transport()
-        return data
-
-    def __aiter__(self):
-        _ensure_can_read(self._mode)
-        return self
-
-    async def __anext__(self):
-        val = await self.readline()
-        if val == b'':
-            raise StopAsyncIteration
-        return val
-
-    async def __aenter__(self):
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        await self.close()
diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py
index ce504b8b0cee..c9506b158302 100644
--- a/Lib/asyncio/subprocess.py
+++ b/Lib/asyncio/subprocess.py
@@ -19,16 +19,14 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
                                protocols.SubprocessProtocol):
     """Like StreamReaderProtocol, but for a subprocess."""
 
-    def __init__(self, limit, loop, *, _asyncio_internal=False):
-        super().__init__(loop=loop, _asyncio_internal=_asyncio_internal)
+    def __init__(self, limit, loop):
+        super().__init__(loop=loop)
         self._limit = limit
         self.stdin = self.stdout = self.stderr = None
         self._transport = None
         self._process_exited = False
         self._pipe_fds = []
         self._stdin_closed = self._loop.create_future()
-        self._stdout_closed = self._loop.create_future()
-        self._stderr_closed = self._loop.create_future()
 
     def __repr__(self):
         info = [self.__class__.__name__]
@@ -42,35 +40,27 @@ def __repr__(self):
 
     def connection_made(self, transport):
         self._transport = transport
+
         stdout_transport = transport.get_pipe_transport(1)
         if stdout_transport is not None:
-            self.stdout = streams.Stream(mode=streams.StreamMode.READ,
-                                         transport=stdout_transport,
-                                         protocol=self,
-                                         limit=self._limit,
-                                         loop=self._loop,
-                                         _asyncio_internal=True)
-            self.stdout._set_transport(stdout_transport)
+            self.stdout = streams.StreamReader(limit=self._limit,
+                                               loop=self._loop)
+            self.stdout.set_transport(stdout_transport)
             self._pipe_fds.append(1)
 
         stderr_transport = transport.get_pipe_transport(2)
         if stderr_transport is not None:
-            self.stderr = streams.Stream(mode=streams.StreamMode.READ,
-                                         transport=stderr_transport,
-                                         protocol=self,
-                                         limit=self._limit,
-                                         loop=self._loop,
-                                         _asyncio_internal=True)
-            self.stderr._set_transport(stderr_transport)
+            self.stderr = streams.StreamReader(limit=self._limit,
+                                               loop=self._loop)
+            self.stderr.set_transport(stderr_transport)
             self._pipe_fds.append(2)
 
         stdin_transport = transport.get_pipe_transport(0)
         if stdin_transport is not None:
-            self.stdin = streams.Stream(mode=streams.StreamMode.WRITE,
-                                        transport=stdin_transport,
-                                        protocol=self,
-                                        loop=self._loop,
-                                        _asyncio_internal=True)
+            self.stdin = streams.StreamWriter(stdin_transport,
+                                              protocol=self,
+                                              reader=None,
+                                              loop=self._loop)
 
     def pipe_data_received(self, fd, data):
         if fd == 1:
@@ -80,7 +70,7 @@ def pipe_data_received(self, fd, data):
         else:
             reader = None
         if reader is not None:
-            reader._feed_data(data)
+            reader.feed_data(data)
 
     def pipe_connection_lost(self, fd, exc):
         if fd == 0:
@@ -101,9 +91,9 @@ def pipe_connection_lost(self, fd, exc):
             reader = None
         if reader is not None:
             if exc is None:
-                reader._feed_eof()
+                reader.feed_eof()
             else:
-                reader._set_exception(exc)
+                reader.set_exception(exc)
 
         if fd in self._pipe_fds:
             self._pipe_fds.remove(fd)
@@ -121,20 +111,10 @@ def _maybe_close_transport(self):
     def _get_close_waiter(self, stream):
         if stream is self.stdin:
             return self._stdin_closed
-        elif stream is self.stdout:
-            return self._stdout_closed
-        elif stream is self.stderr:
-            return self._stderr_closed
 
 
 class Process:
-    def __init__(self, transport, protocol, loop, *, _asyncio_internal=False):
-        if not _asyncio_internal:
-            warnings.warn(f"{self.__class__} should be instaniated "
-                          "by asyncio internals only, "
-                          "please avoid its creation from user code",
-                          DeprecationWarning)
-
+    def __init__(self, transport, protocol, loop):
         self._transport = transport
         self._protocol = protocol
         self._loop = loop
@@ -232,13 +212,12 @@ def kill(self):
         )
 
     protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
-                                                        loop=loop,
-                                                        _asyncio_internal=True)
+                                                        loop=loop)
     transport, protocol = await loop.subprocess_shell(
         protocol_factory,
         cmd, stdin=stdin, stdout=stdout,
         stderr=stderr, **kwds)
-    return Process(transport, protocol, loop, _asyncio_internal=True)
+    return Process(transport, protocol, loop)
 
 
 async def create_subprocess_exec(program, *args, stdin=None, stdout=None,
@@ -253,11 +232,10 @@ def kill(self):
                       stacklevel=2
         )
     protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
-                                                        loop=loop,
-                                                        _asyncio_internal=True)
+                                                        loop=loop)
     transport, protocol = await loop.subprocess_exec(
         protocol_factory,
         program, *args,
         stdin=stdin, stdout=stdout,
         stderr=stderr, **kwds)
-    return Process(transport, protocol, loop, _asyncio_internal=True)
+    return Process(transport, protocol, loop)
diff --git a/Lib/test/test_asyncio/test_buffered_proto.py b/Lib/test/test_asyncio/test_buffered_proto.py
index b1531fb9343f..f24e363ebfcf 100644
--- a/Lib/test/test_asyncio/test_buffered_proto.py
+++ b/Lib/test/test_asyncio/test_buffered_proto.py
@@ -58,10 +58,9 @@ def on_buf(buf):
             writer.close()
             await writer.wait_closed()
 
-        with self.assertWarns(DeprecationWarning):
-            srv = self.loop.run_until_complete(
-                asyncio.start_server(
-                    on_server_client, '127.0.0.1', 0))
+        srv = self.loop.run_until_complete(
+            asyncio.start_server(
+                on_server_client, '127.0.0.1', 0))
 
         addr = srv.sockets[0].getsockname()
         self.loop.run_until_complete(
diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py
index 58a609444285..a1f27dd5721c 100644
--- a/Lib/test/test_asyncio/test_pep492.py
+++ b/Lib/test/test_asyncio/test_pep492.py
@@ -95,11 +95,9 @@ class StreamReaderTests(BaseTest):
     def test_readline(self):
         DATA = b'line1\nline2\nline3'
 
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(DATA)
-        stream._feed_eof()
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(DATA)
+        stream.feed_eof()
 
         async def reader():
             data = []
diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py
index 6325ee3983cc..b9413ab35fc5 100644
--- a/Lib/test/test_asyncio/test_streams.py
+++ b/Lib/test/test_asyncio/test_streams.py
@@ -1,8 +1,6 @@
 """Tests for streams.py."""
 
-import contextlib
 import gc
-import io
 import os
 import queue
 import pickle
@@ -18,7 +16,6 @@
     ssl = None
 
 import asyncio
-from asyncio.streams import _StreamProtocol, _ensure_can_read, _ensure_can_write
 from test.test_asyncio import utils as test_utils
 
 
@@ -26,24 +23,6 @@ def tearDownModule():
     asyncio.set_event_loop_policy(None)
 
 
-class StreamModeTests(unittest.TestCase):
-    def test__ensure_can_read_ok(self):
-        self.assertIsNone(_ensure_can_read(asyncio.StreamMode.READ))
-        self.assertIsNone(_ensure_can_read(asyncio.StreamMode.READWRITE))
-
-    def test__ensure_can_read_fail(self):
-        with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-            _ensure_can_read(asyncio.StreamMode.WRITE)
-
-    def test__ensure_can_write_ok(self):
-        self.assertIsNone(_ensure_can_write(asyncio.StreamMode.WRITE))
-        self.assertIsNone(_ensure_can_write(asyncio.StreamMode.READWRITE))
-
-    def test__ensure_can_write_fail(self):
-        with self.assertRaisesRegex(RuntimeError, "The stream is read-only"):
-            _ensure_can_write(asyncio.StreamMode.READ)
-
-
 class StreamTests(test_utils.TestCase):
 
     DATA = b'line1\nline2\nline3\n'
@@ -63,8 +42,7 @@ def tearDown(self):
 
     @mock.patch('asyncio.streams.events')
     def test_ctor_global_loop(self, m_events):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader()
         self.assertIs(stream._loop, m_events.get_event_loop.return_value)
 
     def _basetest_open_connection(self, open_connection_fut):
@@ -100,8 +78,7 @@ def _basetest_open_connection_no_loop_ssl(self, open_connection_fut):
         self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
         try:
             with self.assertWarns(DeprecationWarning):
-                reader, writer = self.loop.run_until_complete(
-                    open_connection_fut)
+                reader, writer = self.loop.run_until_complete(open_connection_fut)
         finally:
             asyncio.set_event_loop(None)
         writer.write(b'GET / HTTP/1.0\r\n\r\n')
@@ -161,27 +138,21 @@ def test_open_unix_connection_error(self):
             self._basetest_open_connection_error(conn_fut)
 
     def test_feed_empty_data(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
-        stream._feed_data(b'')
+        stream.feed_data(b'')
         self.assertEqual(b'', stream._buffer)
 
     def test_feed_nonempty_data(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
-        stream._feed_data(self.DATA)
+        stream.feed_data(self.DATA)
         self.assertEqual(self.DATA, stream._buffer)
 
     def test_read_zero(self):
         # Read zero bytes.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(self.DATA)
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(self.DATA)
 
         data = self.loop.run_until_complete(stream.read(0))
         self.assertEqual(b'', data)
@@ -189,13 +160,11 @@ def test_read_zero(self):
 
     def test_read(self):
         # Read bytes.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         read_task = self.loop.create_task(stream.read(30))
 
         def cb():
-            stream._feed_data(self.DATA)
+            stream.feed_data(self.DATA)
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -204,11 +173,9 @@ def cb():
 
     def test_read_line_breaks(self):
         # Read bytes without line breaks.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'line1')
-        stream._feed_data(b'line2')
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'line1')
+        stream.feed_data(b'line2')
 
         data = self.loop.run_until_complete(stream.read(5))
 
@@ -217,13 +184,11 @@ def test_read_line_breaks(self):
 
     def test_read_eof(self):
         # Read bytes, stop at eof.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         read_task = self.loop.create_task(stream.read(1024))
 
         def cb():
-            stream._feed_eof()
+            stream.feed_eof()
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -232,15 +197,13 @@ def cb():
 
     def test_read_until_eof(self):
         # Read all bytes until eof.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         read_task = self.loop.create_task(stream.read(-1))
 
         def cb():
-            stream._feed_data(b'chunk1\n')
-            stream._feed_data(b'chunk2')
-            stream._feed_eof()
+            stream.feed_data(b'chunk1\n')
+            stream.feed_data(b'chunk2')
+            stream.feed_eof()
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -249,34 +212,26 @@ def cb():
         self.assertEqual(b'', stream._buffer)
 
     def test_read_exception(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'line\n')
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'line\n')
 
         data = self.loop.run_until_complete(stream.read(2))
         self.assertEqual(b'li', data)
 
-        stream._set_exception(ValueError())
+        stream.set_exception(ValueError())
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.read(2))
 
     def test_invalid_limit(self):
         with self.assertRaisesRegex(ValueError, 'imit'):
-            asyncio.Stream(mode=asyncio.StreamMode.READ,
-                           limit=0, loop=self.loop,
-                           _asyncio_internal=True)
+            asyncio.StreamReader(limit=0, loop=self.loop)
 
         with self.assertRaisesRegex(ValueError, 'imit'):
-            asyncio.Stream(mode=asyncio.StreamMode.READ,
-                           limit=-1, loop=self.loop,
-                           _asyncio_internal=True)
+            asyncio.StreamReader(limit=-1, loop=self.loop)
 
     def test_read_limit(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=3, loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'chunk')
+        stream = asyncio.StreamReader(limit=3, loop=self.loop)
+        stream.feed_data(b'chunk')
         data = self.loop.run_until_complete(stream.read(5))
         self.assertEqual(b'chunk', data)
         self.assertEqual(b'', stream._buffer)
@@ -284,16 +239,14 @@ def test_read_limit(self):
     def test_readline(self):
         # Read one line. 'readline' will need to wait for the data
         # to come from 'cb'
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'chunk1 ')
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'chunk1 ')
         read_task = self.loop.create_task(stream.readline())
 
         def cb():
-            stream._feed_data(b'chunk2 ')
-            stream._feed_data(b'chunk3 ')
-            stream._feed_data(b'\n chunk4')
+            stream.feed_data(b'chunk2 ')
+            stream.feed_data(b'chunk3 ')
+            stream.feed_data(b'\n chunk4')
         self.loop.call_soon(cb)
 
         line = self.loop.run_until_complete(read_task)
@@ -301,26 +254,22 @@ def cb():
         self.assertEqual(b' chunk4', stream._buffer)
 
     def test_readline_limit_with_existing_data(self):
-        # Read one line. The data is in Stream's buffer
+        # Read one line. The data is in StreamReader's buffer
         # before the event loop is run.
 
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=3, loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'li')
-        stream._feed_data(b'ne1\nline2\n')
+        stream = asyncio.StreamReader(limit=3, loop=self.loop)
+        stream.feed_data(b'li')
+        stream.feed_data(b'ne1\nline2\n')
 
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readline())
         # The buffer should contain the remaining data after exception
         self.assertEqual(b'line2\n', stream._buffer)
 
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=3, loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'li')
-        stream._feed_data(b'ne1')
-        stream._feed_data(b'li')
+        stream = asyncio.StreamReader(limit=3, loop=self.loop)
+        stream.feed_data(b'li')
+        stream.feed_data(b'ne1')
+        stream.feed_data(b'li')
 
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readline())
@@ -332,34 +281,30 @@ def test_readline_limit_with_existing_data(self):
         self.assertEqual(b'', stream._buffer)
 
     def test_at_eof(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         self.assertFalse(stream.at_eof())
 
-        stream._feed_data(b'some data\n')
+        stream.feed_data(b'some data\n')
         self.assertFalse(stream.at_eof())
 
         self.loop.run_until_complete(stream.readline())
         self.assertFalse(stream.at_eof())
 
-        stream._feed_data(b'some data\n')
-        stream._feed_eof()
+        stream.feed_data(b'some data\n')
+        stream.feed_eof()
         self.loop.run_until_complete(stream.readline())
         self.assertTrue(stream.at_eof())
 
     def test_readline_limit(self):
-        # Read one line. Streams are fed with data after
+        # Read one line. StreamReaders are fed with data after
         # their 'readline' methods are called.
 
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=7, loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(limit=7, loop=self.loop)
         def cb():
-            stream._feed_data(b'chunk1')
-            stream._feed_data(b'chunk2')
-            stream._feed_data(b'chunk3\n')
-            stream._feed_eof()
+            stream.feed_data(b'chunk1')
+            stream.feed_data(b'chunk2')
+            stream.feed_data(b'chunk3\n')
+            stream.feed_eof()
         self.loop.call_soon(cb)
 
         self.assertRaises(
@@ -368,14 +313,12 @@ def cb():
         # a ValueError it should be empty.
         self.assertEqual(b'', stream._buffer)
 
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=7, loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(limit=7, loop=self.loop)
         def cb():
-            stream._feed_data(b'chunk1')
-            stream._feed_data(b'chunk2\n')
-            stream._feed_data(b'chunk3\n')
-            stream._feed_eof()
+            stream.feed_data(b'chunk1')
+            stream.feed_data(b'chunk2\n')
+            stream.feed_data(b'chunk3\n')
+            stream.feed_eof()
         self.loop.call_soon(cb)
 
         self.assertRaises(
@@ -383,20 +326,18 @@ def cb():
         self.assertEqual(b'chunk3\n', stream._buffer)
 
         # check strictness of the limit
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=7, loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'1234567\n')
+        stream = asyncio.StreamReader(limit=7, loop=self.loop)
+        stream.feed_data(b'1234567\n')
         line = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'1234567\n', line)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'12345678\n')
+        stream.feed_data(b'12345678\n')
         with self.assertRaises(ValueError) as cm:
             self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'12345678')
+        stream.feed_data(b'12345678')
         with self.assertRaises(ValueError) as cm:
             self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'', stream._buffer)
@@ -404,11 +345,9 @@ def cb():
     def test_readline_nolimit_nowait(self):
         # All needed data for the first 'readline' call will be
         # in the buffer.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(self.DATA[:6])
-        stream._feed_data(self.DATA[6:])
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(self.DATA[:6])
+        stream.feed_data(self.DATA[6:])
 
         line = self.loop.run_until_complete(stream.readline())
 
@@ -416,29 +355,23 @@ def test_readline_nolimit_nowait(self):
         self.assertEqual(b'line2\nline3\n', stream._buffer)
 
     def test_readline_eof(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'some data')
-        stream._feed_eof()
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'some data')
+        stream.feed_eof()
 
         line = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'some data', line)
 
     def test_readline_empty_eof(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_eof()
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_eof()
 
         line = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'', line)
 
     def test_readline_read_byte_count(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(self.DATA)
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(self.DATA)
 
         self.loop.run_until_complete(stream.readline())
 
@@ -448,89 +381,79 @@ def test_readline_read_byte_count(self):
         self.assertEqual(b'ine3\n', stream._buffer)
 
     def test_readline_exception(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'line\n')
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'line\n')
 
         data = self.loop.run_until_complete(stream.readline())
         self.assertEqual(b'line\n', data)
 
-        stream._set_exception(ValueError())
+        stream.set_exception(ValueError())
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readline())
         self.assertEqual(b'', stream._buffer)
 
     def test_readuntil_separator(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         with self.assertRaisesRegex(ValueError, 'Separator should be'):
             self.loop.run_until_complete(stream.readuntil(separator=b''))
 
     def test_readuntil_multi_chunks(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
-        stream._feed_data(b'lineAAA')
+        stream.feed_data(b'lineAAA')
         data = self.loop.run_until_complete(stream.readuntil(separator=b'AAA'))
         self.assertEqual(b'lineAAA', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'lineAAA')
+        stream.feed_data(b'lineAAA')
         data = self.loop.run_until_complete(stream.readuntil(b'AAA'))
         self.assertEqual(b'lineAAA', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'lineAAAxxx')
+        stream.feed_data(b'lineAAAxxx')
         data = self.loop.run_until_complete(stream.readuntil(b'AAA'))
         self.assertEqual(b'lineAAA', data)
         self.assertEqual(b'xxx', stream._buffer)
 
     def test_readuntil_multi_chunks_1(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
-        stream._feed_data(b'QWEaa')
-        stream._feed_data(b'XYaa')
-        stream._feed_data(b'a')
+        stream.feed_data(b'QWEaa')
+        stream.feed_data(b'XYaa')
+        stream.feed_data(b'a')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'QWEaaXYaaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'QWEaa')
-        stream._feed_data(b'XYa')
-        stream._feed_data(b'aa')
+        stream.feed_data(b'QWEaa')
+        stream.feed_data(b'XYa')
+        stream.feed_data(b'aa')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'QWEaaXYaaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'aaa')
+        stream.feed_data(b'aaa')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'aaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'Xaaa')
+        stream.feed_data(b'Xaaa')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'Xaaa', data)
         self.assertEqual(b'', stream._buffer)
 
-        stream._feed_data(b'XXX')
-        stream._feed_data(b'a')
-        stream._feed_data(b'a')
-        stream._feed_data(b'a')
+        stream.feed_data(b'XXX')
+        stream.feed_data(b'a')
+        stream.feed_data(b'a')
+        stream.feed_data(b'a')
         data = self.loop.run_until_complete(stream.readuntil(b'aaa'))
         self.assertEqual(b'XXXaaa', data)
         self.assertEqual(b'', stream._buffer)
 
     def test_readuntil_eof(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'some dataAA')
-        stream._feed_eof()
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'some dataAA')
+        stream.feed_eof()
 
         with self.assertRaises(asyncio.IncompleteReadError) as cm:
             self.loop.run_until_complete(stream.readuntil(b'AAA'))
@@ -539,18 +462,15 @@ def test_readuntil_eof(self):
         self.assertEqual(b'', stream._buffer)
 
     def test_readuntil_limit_found_sep(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop, limit=3,
-                                _asyncio_internal=True)
-        stream._feed_data(b'some dataAA')
-
+        stream = asyncio.StreamReader(loop=self.loop, limit=3)
+        stream.feed_data(b'some dataAA')
         with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                     'not found') as cm:
             self.loop.run_until_complete(stream.readuntil(b'AAA'))
 
         self.assertEqual(b'some dataAA', stream._buffer)
 
-        stream._feed_data(b'A')
+        stream.feed_data(b'A')
         with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                     'is found') as cm:
             self.loop.run_until_complete(stream.readuntil(b'AAA'))
@@ -559,10 +479,8 @@ def test_readuntil_limit_found_sep(self):
 
     def test_readexactly_zero_or_less(self):
         # Read exact number of bytes (zero or less).
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(self.DATA)
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(self.DATA)
 
         data = self.loop.run_until_complete(stream.readexactly(0))
         self.assertEqual(b'', data)
@@ -574,17 +492,15 @@ def test_readexactly_zero_or_less(self):
 
     def test_readexactly(self):
         # Read exact number of bytes.
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
         n = 2 * len(self.DATA)
         read_task = self.loop.create_task(stream.readexactly(n))
 
         def cb():
-            stream._feed_data(self.DATA)
-            stream._feed_data(self.DATA)
-            stream._feed_data(self.DATA)
+            stream.feed_data(self.DATA)
+            stream.feed_data(self.DATA)
+            stream.feed_data(self.DATA)
         self.loop.call_soon(cb)
 
         data = self.loop.run_until_complete(read_task)
@@ -592,25 +508,21 @@ def cb():
         self.assertEqual(self.DATA, stream._buffer)
 
     def test_readexactly_limit(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                limit=3, loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'chunk')
+        stream = asyncio.StreamReader(limit=3, loop=self.loop)
+        stream.feed_data(b'chunk')
         data = self.loop.run_until_complete(stream.readexactly(5))
         self.assertEqual(b'chunk', data)
         self.assertEqual(b'', stream._buffer)
 
     def test_readexactly_eof(self):
         # Read exact number of bytes (eof).
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         n = 2 * len(self.DATA)
         read_task = self.loop.create_task(stream.readexactly(n))
 
         def cb():
-            stream._feed_data(self.DATA)
-            stream._feed_eof()
+            stream.feed_data(self.DATA)
+            stream.feed_eof()
         self.loop.call_soon(cb)
 
         with self.assertRaises(asyncio.IncompleteReadError) as cm:
@@ -622,35 +534,29 @@ def cb():
         self.assertEqual(b'', stream._buffer)
 
     def test_readexactly_exception(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'line\n')
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'line\n')
 
         data = self.loop.run_until_complete(stream.readexactly(2))
         self.assertEqual(b'li', data)
 
-        stream._set_exception(ValueError())
+        stream.set_exception(ValueError())
         self.assertRaises(
             ValueError, self.loop.run_until_complete, stream.readexactly(2))
 
     def test_exception(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         self.assertIsNone(stream.exception())
 
         exc = ValueError()
-        stream._set_exception(exc)
+        stream.set_exception(exc)
         self.assertIs(stream.exception(), exc)
 
     def test_exception_waiter(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
         async def set_err():
-            stream._set_exception(ValueError())
+            stream.set_exception(ValueError())
 
         t1 = self.loop.create_task(stream.readline())
         t2 = self.loop.create_task(set_err())
@@ -660,16 +566,14 @@ def test_exception_waiter(self):
         self.assertRaises(ValueError, t1.result)
 
     def test_exception_cancel(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
 
         t = self.loop.create_task(stream.readline())
         test_utils.run_briefly(self.loop)
         t.cancel()
         test_utils.run_briefly(self.loop)
         # The following line fails if set_exception() isn't careful.
-        stream._set_exception(RuntimeError('message'))
+        stream.set_exception(RuntimeError('message'))
         test_utils.run_briefly(self.loop)
         self.assertIs(stream._waiter, None)
 
@@ -829,7 +733,7 @@ def stop(self):
     def test_read_all_from_pipe_reader(self):
         # See asyncio issue 168.  This test is derived from the example
         # subprocess_attach_read_pipe.py, but we configure the
-        # Stream's limit so that twice it is less than the size
+        # StreamReader's limit so that twice it is less than the size
         # of the data writter.  Also we must explicitly attach a child
         # watcher to the event loop.
 
@@ -843,11 +747,8 @@ def test_read_all_from_pipe_reader(self):
         args = [sys.executable, '-c', code, str(wfd)]
 
         pipe = open(rfd, 'rb', 0)
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop, limit=1,
-                                _asyncio_internal=True)
-        protocol = _StreamProtocol(stream, loop=self.loop,
-                                   _asyncio_internal=True)
+        reader = asyncio.StreamReader(loop=self.loop, limit=1)
+        protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop)
         transport, _ = self.loop.run_until_complete(
             self.loop.connect_read_pipe(lambda: protocol, pipe))
 
@@ -865,30 +766,29 @@ def test_read_all_from_pipe_reader(self):
             asyncio.set_child_watcher(None)
 
         os.close(wfd)
-        data = self.loop.run_until_complete(stream.read(-1))
+        data = self.loop.run_until_complete(reader.read(-1))
         self.assertEqual(data, b'data')
 
     def test_streamreader_constructor(self):
         self.addCleanup(asyncio.set_event_loop, None)
         asyncio.set_event_loop(self.loop)
 
-        # asyncio issue #184: Ensure that _StreamProtocol constructor
+        # asyncio issue #184: Ensure that StreamReaderProtocol constructor
         # retrieves the current loop if the loop parameter is not set
-        reader = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                _asyncio_internal=True)
+        reader = asyncio.StreamReader()
         self.assertIs(reader._loop, self.loop)
 
     def test_streamreaderprotocol_constructor(self):
         self.addCleanup(asyncio.set_event_loop, None)
         asyncio.set_event_loop(self.loop)
 
-        # asyncio issue #184: Ensure that _StreamProtocol constructor
+        # asyncio issue #184: Ensure that StreamReaderProtocol constructor
         # retrieves the current loop if the loop parameter is not set
-        stream = mock.Mock()
-        protocol = _StreamProtocol(stream, _asyncio_internal=True)
+        reader = mock.Mock()
+        protocol = asyncio.StreamReaderProtocol(reader)
         self.assertIs(protocol._loop, self.loop)
 
-    def test_drain_raises_deprecated(self):
+    def test_drain_raises(self):
         # See http://bugs.python.org/issue25441
 
         # This test should not use asyncio for the mock server; the
@@ -902,7 +802,7 @@ def test_drain_raises_deprecated(self):
 
         def server():
             # Runs in a separate thread.
-            with socket.create_server(('127.0.0.1', 0)) as sock:
+            with socket.create_server(('localhost', 0)) as sock:
                 addr = sock.getsockname()
                 q.put(addr)
                 clt, _ = sock.accept()
@@ -933,106 +833,48 @@ def server():
         thread.join()
         self.assertEqual([], messages)
 
-    def test_drain_raises(self):
-        # See http://bugs.python.org/issue25441
-
-        # This test should not use asyncio for the mock server; the
-        # whole point of the test is to test for a bug in drain()
-        # where it never gives up the event loop but the socket is
-        # closed on the  server side.
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        q = queue.Queue()
-
-        def server():
-            # Runs in a separate thread.
-            with socket.create_server(('localhost', 0)) as sock:
-                addr = sock.getsockname()
-                q.put(addr)
-                clt, _ = sock.accept()
-                clt.close()
-
-        async def client(host, port):
-            stream = await asyncio.connect(host, port)
-
-            while True:
-                stream.write(b"foo\n")
-                await stream.drain()
-
-        # Start the server thread and wait for it to be listening.
-        thread = threading.Thread(target=server)
-        thread.setDaemon(True)
-        thread.start()
-        addr = q.get()
-
-        # Should not be stuck in an infinite loop.
-        with self.assertRaises((ConnectionResetError, ConnectionAbortedError,
-                                BrokenPipeError)):
-            self.loop.run_until_complete(client(*addr))
-
-        # Clean up the thread.  (Only on success; on failure, it may
-        # be stuck in accept().)
-        thread.join()
-        self.assertEqual([], messages)
-
     def test___repr__(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        self.assertEqual("<Stream mode=StreamMode.READ>", repr(stream))
+        stream = asyncio.StreamReader(loop=self.loop)
+        self.assertEqual("<StreamReader>", repr(stream))
 
     def test___repr__nondefault_limit(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop, limit=123,
-                                _asyncio_internal=True)
-        self.assertEqual("<Stream mode=StreamMode.READ limit=123>", repr(stream))
+        stream = asyncio.StreamReader(loop=self.loop, limit=123)
+        self.assertEqual("<StreamReader limit=123>", repr(stream))
 
     def test___repr__eof(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_eof()
-        self.assertEqual("<Stream mode=StreamMode.READ eof>", repr(stream))
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_eof()
+        self.assertEqual("<StreamReader eof>", repr(stream))
 
     def test___repr__data(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._feed_data(b'data')
-        self.assertEqual("<Stream mode=StreamMode.READ 4 bytes>", repr(stream))
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream.feed_data(b'data')
+        self.assertEqual("<StreamReader 4 bytes>", repr(stream))
 
     def test___repr__exception(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         exc = RuntimeError()
-        stream._set_exception(exc)
-        self.assertEqual("<Stream mode=StreamMode.READ exception=RuntimeError()>",
+        stream.set_exception(exc)
+        self.assertEqual("<StreamReader exception=RuntimeError()>",
                          repr(stream))
 
     def test___repr__waiter(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
-        stream._waiter = self.loop.create_future()
+        stream = asyncio.StreamReader(loop=self.loop)
+        stream._waiter = asyncio.Future(loop=self.loop)
         self.assertRegex(
             repr(stream),
-            r"<Stream .+ waiter=<Future pending[\S ]*>>")
+            r"<StreamReader waiter=<Future pending[\S ]*>>")
         stream._waiter.set_result(None)
         self.loop.run_until_complete(stream._waiter)
         stream._waiter = None
-        self.assertEqual("<Stream mode=StreamMode.READ>", repr(stream))
+        self.assertEqual("<StreamReader>", repr(stream))
 
     def test___repr__transport(self):
-        stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                loop=self.loop,
-                                _asyncio_internal=True)
+        stream = asyncio.StreamReader(loop=self.loop)
         stream._transport = mock.Mock()
         stream._transport.__repr__ = mock.Mock()
         stream._transport.__repr__.return_value = "<Transport>"
-        self.assertEqual("<Stream mode=StreamMode.READ transport=<Transport>>",
-                         repr(stream))
+        self.assertEqual("<StreamReader transport=<Transport>>", repr(stream))
 
     def test_IncompleteReadError_pickleable(self):
         e = asyncio.IncompleteReadError(b'abc', 10)
@@ -1051,7 +893,7 @@ def test_LimitOverrunError_pickleable(self):
                 self.assertEqual(str(e), str(e2))
                 self.assertEqual(e.consumed, e2.consumed)
 
-    def test_wait_closed_on_close_deprecated(self):
+    def test_wait_closed_on_close(self):
         with test_utils.run_test_server() as httpd:
             with self.assertWarns(DeprecationWarning):
                 rd, wr = self.loop.run_until_complete(
@@ -1069,24 +911,7 @@ def test_wait_closed_on_close_deprecated(self):
             self.assertTrue(wr.is_closing())
             self.loop.run_until_complete(wr.wait_closed())
 
-    def test_wait_closed_on_close(self):
-        with test_utils.run_test_server() as httpd:
-            stream = self.loop.run_until_complete(
-                asyncio.connect(*httpd.address))
-
-            stream.write(b'GET / HTTP/1.0\r\n\r\n')
-            f = stream.readline()
-            data = self.loop.run_until_complete(f)
-            self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-            f = stream.read()
-            data = self.loop.run_until_complete(f)
-            self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-            self.assertFalse(stream.is_closing())
-            stream.close()
-            self.assertTrue(stream.is_closing())
-            self.loop.run_until_complete(stream.wait_closed())
-
-    def test_wait_closed_on_close_with_unread_data_deprecated(self):
+    def test_wait_closed_on_close_with_unread_data(self):
         with test_utils.run_test_server() as httpd:
             with self.assertWarns(DeprecationWarning):
                 rd, wr = self.loop.run_until_complete(
@@ -1099,44 +924,33 @@ def test_wait_closed_on_close_with_unread_data_deprecated(self):
             wr.close()
             self.loop.run_until_complete(wr.wait_closed())
 
-    def test_wait_closed_on_close_with_unread_data(self):
-        with test_utils.run_test_server() as httpd:
-            stream = self.loop.run_until_complete(
-                asyncio.connect(*httpd.address))
-
-            stream.write(b'GET / HTTP/1.0\r\n\r\n')
-            f = stream.readline()
-            data = self.loop.run_until_complete(f)
-            self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-            stream.close()
-            self.loop.run_until_complete(stream.wait_closed())
-
     def test_del_stream_before_sock_closing(self):
         messages = []
         self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
 
-        async def test():
-
-            with test_utils.run_test_server() as httpd:
-                stream = await asyncio.connect(*httpd.address)
-                sock = stream.get_extra_info('socket')
-                self.assertNotEqual(sock.fileno(), -1)
+        with test_utils.run_test_server() as httpd:
+            with self.assertWarns(DeprecationWarning):
+                rd, wr = self.loop.run_until_complete(
+                    asyncio.open_connection(*httpd.address, loop=self.loop))
+            sock = wr.get_extra_info('socket')
+            self.assertNotEqual(sock.fileno(), -1)
 
-                await stream.write(b'GET / HTTP/1.0\r\n\r\n')
-                data = await stream.readline()
-                self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
+            wr.write(b'GET / HTTP/1.0\r\n\r\n')
+            f = rd.readline()
+            data = self.loop.run_until_complete(f)
+            self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
 
-                # drop refs to reader/writer
-                del stream
-                gc.collect()
-                # make a chance to close the socket
-                await asyncio.sleep(0)
+            # drop refs to reader/writer
+            del rd
+            del wr
+            gc.collect()
+            # make a chance to close the socket
+            test_utils.run_briefly(self.loop)
 
-                self.assertEqual(1, len(messages), messages)
-                self.assertEqual(sock.fileno(), -1)
+            self.assertEqual(1, len(messages))
+            self.assertEqual(sock.fileno(), -1)
 
-        self.loop.run_until_complete(test())
-        self.assertEqual(1, len(messages), messages)
+        self.assertEqual(1, len(messages))
         self.assertEqual('An open stream object is being garbage '
                          'collected; call "stream.close()" explicitly.',
                          messages[0]['message'])
@@ -1146,12 +960,9 @@ def test_del_stream_before_connection_made(self):
         self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
 
         with test_utils.run_test_server() as httpd:
-            stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                    loop=self.loop,
-                                    _asyncio_internal=True)
-            pr = _StreamProtocol(stream, loop=self.loop,
-                                 _asyncio_internal=True)
-            del stream
+            rd = asyncio.StreamReader(loop=self.loop)
+            pr = asyncio.StreamReaderProtocol(rd, loop=self.loop)
+            del rd
             gc.collect()
             tr, _ = self.loop.run_until_complete(
                 self.loop.create_connection(
@@ -1168,14 +979,15 @@ def test_del_stream_before_connection_made(self):
 
     def test_async_writer_api(self):
         async def inner(httpd):
-            stream = await asyncio.connect(*httpd.address)
+            rd, wr = await asyncio.open_connection(*httpd.address)
 
-            await stream.write(b'GET / HTTP/1.0\r\n\r\n')
-            data = await stream.readline()
+            wr.write(b'GET / HTTP/1.0\r\n\r\n')
+            data = await rd.readline()
             self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-            data = await stream.read()
+            data = await rd.read()
             self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-            await stream.close()
+            wr.close()
+            await wr.wait_closed()
 
         messages = []
         self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
@@ -1187,16 +999,17 @@ def test_async_writer_api(self):
 
     def test_async_writer_api_exception_after_close(self):
         async def inner(httpd):
-            stream = await asyncio.connect(*httpd.address)
+            rd, wr = await asyncio.open_connection(*httpd.address)
 
-            await stream.write(b'GET / HTTP/1.0\r\n\r\n')
-            data = await stream.readline()
+            wr.write(b'GET / HTTP/1.0\r\n\r\n')
+            data = await rd.readline()
             self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-            data = await stream.read()
+            data = await rd.read()
             self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-            stream.close()
+            wr.close()
             with self.assertRaises(ConnectionResetError):
-                await stream.write(b'data')
+                wr.write(b'data')
+                await wr.drain()
 
         messages = []
         self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
@@ -1227,587 +1040,6 @@ def test_eof_feed_when_closing_writer(self):
 
         self.assertEqual(messages, [])
 
-    def test_stream_reader_create_warning(self):
-        with contextlib.suppress(AttributeError):
-            del asyncio.StreamReader
-        with self.assertWarns(DeprecationWarning):
-            asyncio.StreamReader
-
-    def test_stream_writer_create_warning(self):
-        with contextlib.suppress(AttributeError):
-            del asyncio.StreamWriter
-        with self.assertWarns(DeprecationWarning):
-            asyncio.StreamWriter
-
-    def test_stream_reader_forbidden_ops(self):
-        async def inner():
-            stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                    _asyncio_internal=True)
-            with self.assertRaisesRegex(RuntimeError, "The stream is read-only"):
-                await stream.write(b'data')
-            with self.assertRaisesRegex(RuntimeError, "The stream is read-only"):
-                await stream.writelines([b'data', b'other'])
-            with self.assertRaisesRegex(RuntimeError, "The stream is read-only"):
-                stream.write_eof()
-            with self.assertRaisesRegex(RuntimeError, "The stream is read-only"):
-                await stream.drain()
-
-        self.loop.run_until_complete(inner())
-
-    def test_stream_writer_forbidden_ops(self):
-        async def inner():
-            stream = asyncio.Stream(mode=asyncio.StreamMode.WRITE,
-                                    _asyncio_internal=True)
-            with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                stream._feed_data(b'data')
-            with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                await stream.readline()
-            with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                await stream.readuntil()
-            with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                await stream.read()
-            with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                await stream.readexactly(10)
-            with self.assertRaisesRegex(RuntimeError, "The stream is write-only"):
-                async for chunk in stream:
-                    pass
-
-        self.loop.run_until_complete(inner())
-
-    def _basetest_connect(self, stream):
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-
-        stream.write(b'GET / HTTP/1.0\r\n\r\n')
-        f = stream.readline()
-        data = self.loop.run_until_complete(f)
-        self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-        f = stream.read()
-        data = self.loop.run_until_complete(f)
-        self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-        stream.close()
-        self.loop.run_until_complete(stream.wait_closed())
-
-        self.assertEqual([], messages)
-
-    def test_connect(self):
-        with test_utils.run_test_server() as httpd:
-            stream = self.loop.run_until_complete(
-                asyncio.connect(*httpd.address))
-            self.assertFalse(stream.is_server_side())
-            self._basetest_connect(stream)
-
-    @support.skip_unless_bind_unix_socket
-    def test_connect_unix(self):
-        with test_utils.run_test_unix_server() as httpd:
-            stream = self.loop.run_until_complete(
-                asyncio.connect_unix(httpd.address))
-            self._basetest_connect(stream)
-
-    def test_stream_async_context_manager(self):
-        async def test(httpd):
-            stream = await asyncio.connect(*httpd.address)
-            async with stream:
-                await stream.write(b'GET / HTTP/1.0\r\n\r\n')
-                data = await stream.readline()
-                self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-                data = await stream.read()
-                self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-            self.assertTrue(stream.is_closing())
-
-        with test_utils.run_test_server() as httpd:
-            self.loop.run_until_complete(test(httpd))
-
-    def test_connect_async_context_manager(self):
-        async def test(httpd):
-            async with asyncio.connect(*httpd.address) as stream:
-                await stream.write(b'GET / HTTP/1.0\r\n\r\n')
-                data = await stream.readline()
-                self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-                data = await stream.read()
-                self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-            self.assertTrue(stream.is_closing())
-
-        with test_utils.run_test_server() as httpd:
-            self.loop.run_until_complete(test(httpd))
-
-    @support.skip_unless_bind_unix_socket
-    def test_connect_unix_async_context_manager(self):
-        async def test(httpd):
-            async with asyncio.connect_unix(httpd.address) as stream:
-                await stream.write(b'GET / HTTP/1.0\r\n\r\n')
-                data = await stream.readline()
-                self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
-                data = await stream.read()
-                self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
-            self.assertTrue(stream.is_closing())
-
-        with test_utils.run_test_unix_server() as httpd:
-            self.loop.run_until_complete(test(httpd))
-
-    def test_stream_server(self):
-
-        async def handle_client(stream):
-            self.assertTrue(stream.is_server_side())
-            data = await stream.readline()
-            await stream.write(data)
-            await stream.close()
-
-        async def client(srv):
-            addr = srv.sockets[0].getsockname()
-            stream = await asyncio.connect(*addr)
-            # send a line
-            await stream.write(b"hello world!\n")
-            # read it back
-            msgback = await stream.readline()
-            await stream.close()
-            self.assertEqual(msgback, b"hello world!\n")
-            await srv.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client, '127.0.0.1', 0) as server:
-                await server.start_serving()
-                task = asyncio.create_task(client(server))
-                with contextlib.suppress(asyncio.CancelledError):
-                    await server.serve_forever()
-                await task
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(test())
-        self.assertEqual(messages, [])
-
-    @support.skip_unless_bind_unix_socket
-    def test_unix_stream_server(self):
-
-        async def handle_client(stream):
-            data = await stream.readline()
-            await stream.write(data)
-            await stream.close()
-
-        async def client(srv):
-            addr = srv.sockets[0].getsockname()
-            stream = await asyncio.connect_unix(addr)
-            # send a line
-            await stream.write(b"hello world!\n")
-            # read it back
-            msgback = await stream.readline()
-            await stream.close()
-            self.assertEqual(msgback, b"hello world!\n")
-            await srv.close()
-
-        async def test():
-            with test_utils.unix_socket_path() as path:
-                async with asyncio.UnixStreamServer(handle_client, path) as server:
-                    await server.start_serving()
-                    task = asyncio.create_task(client(server))
-                    with contextlib.suppress(asyncio.CancelledError):
-                        await server.serve_forever()
-                    await task
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(test())
-        self.assertEqual(messages, [])
-
-    def test_stream_server_inheritance_forbidden(self):
-        with self.assertRaises(TypeError):
-            class MyServer(asyncio.StreamServer):
-                pass
-
-    @support.skip_unless_bind_unix_socket
-    def test_unix_stream_server_inheritance_forbidden(self):
-        with self.assertRaises(TypeError):
-            class MyServer(asyncio.UnixStreamServer):
-                pass
-
-    def test_stream_server_bind(self):
-        async def handle_client(stream):
-            await stream.close()
-
-        async def test():
-            srv = asyncio.StreamServer(handle_client, '127.0.0.1', 0)
-            self.assertFalse(srv.is_bound())
-            self.assertEqual(0, len(srv.sockets))
-            await srv.bind()
-            self.assertTrue(srv.is_bound())
-            self.assertEqual(1, len(srv.sockets))
-            await srv.close()
-            self.assertFalse(srv.is_bound())
-            self.assertEqual(0, len(srv.sockets))
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(test())
-        self.assertEqual(messages, [])
-
-    def test_stream_server_bind_async_with(self):
-        async def handle_client(stream):
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client, '127.0.0.1', 0) as srv:
-                self.assertTrue(srv.is_bound())
-                self.assertEqual(1, len(srv.sockets))
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(test())
-        self.assertEqual(messages, [])
-
-    def test_stream_server_start_serving(self):
-        async def handle_client(stream):
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client, '127.0.0.1', 0) as srv:
-                self.assertFalse(srv.is_serving())
-                await srv.start_serving()
-                self.assertTrue(srv.is_serving())
-                await srv.close()
-                self.assertFalse(srv.is_serving())
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(test())
-        self.assertEqual(messages, [])
-
-    def test_stream_server_close(self):
-        server_stream_aborted = False
-        fut1 = self.loop.create_future()
-        fut2 = self.loop.create_future()
-
-        async def handle_client(stream):
-            data = await stream.readexactly(4)
-            self.assertEqual(b'data', data)
-            fut1.set_result(None)
-            await fut2
-            self.assertEqual(b'', await stream.readline())
-            nonlocal server_stream_aborted
-            server_stream_aborted = True
-
-        async def client(srv):
-            addr = srv.sockets[0].getsockname()
-            stream = await asyncio.connect(*addr)
-            await stream.write(b'data')
-            await fut2
-            self.assertEqual(b'', await stream.readline())
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client, '127.0.0.1', 0) as server:
-                await server.start_serving()
-                task = asyncio.create_task(client(server))
-                await fut1
-                fut2.set_result(None)
-                await server.close()
-                await task
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(asyncio.wait_for(test(), 60.0))
-        self.assertEqual(messages, [])
-        self.assertTrue(fut1.done())
-        self.assertTrue(fut2.done())
-        self.assertTrue(server_stream_aborted)
-
-    def test_stream_server_abort(self):
-        server_stream_aborted = False
-        fut1 = self.loop.create_future()
-        fut2 = self.loop.create_future()
-
-        async def handle_client(stream):
-            data = await stream.readexactly(4)
-            self.assertEqual(b'data', data)
-            fut1.set_result(None)
-            await fut2
-            self.assertEqual(b'', await stream.readline())
-            nonlocal server_stream_aborted
-            server_stream_aborted = True
-
-        async def client(srv):
-            addr = srv.sockets[0].getsockname()
-            stream = await asyncio.connect(*addr)
-            await stream.write(b'data')
-            await fut2
-            self.assertEqual(b'', await stream.readline())
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client, '127.0.0.1', 0) as server:
-                await server.start_serving()
-                task = asyncio.create_task(client(server))
-                await fut1
-                fut2.set_result(None)
-                await server.abort()
-                await task
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(asyncio.wait_for(test(), 60.0))
-        self.assertEqual(messages, [])
-        self.assertTrue(fut1.done())
-        self.assertTrue(fut2.done())
-        self.assertTrue(server_stream_aborted)
-
-    def test_stream_shutdown_hung_task(self):
-        fut1 = self.loop.create_future()
-        fut2 = self.loop.create_future()
-        cancelled = self.loop.create_future()
-
-        async def handle_client(stream):
-            data = await stream.readexactly(4)
-            self.assertEqual(b'data', data)
-            fut1.set_result(None)
-            await fut2
-            try:
-                while True:
-                    await asyncio.sleep(0.01)
-            except asyncio.CancelledError:
-                cancelled.set_result(None)
-                raise
-
-        async def client(srv):
-            addr = srv.sockets[0].getsockname()
-            stream = await asyncio.connect(*addr)
-            await stream.write(b'data')
-            await fut2
-            self.assertEqual(b'', await stream.readline())
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client,
-                                            '127.0.0.1',
-                                            0,
-                                            shutdown_timeout=0.3) as server:
-                await server.start_serving()
-                task = asyncio.create_task(client(server))
-                await fut1
-                fut2.set_result(None)
-                await server.close()
-                await task
-                await cancelled
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(asyncio.wait_for(test(), 60.0))
-        self.assertEqual(messages, [])
-        self.assertTrue(fut1.done())
-        self.assertTrue(fut2.done())
-        self.assertTrue(cancelled.done())
-
-    def test_stream_shutdown_hung_task_prevents_cancellation(self):
-        fut1 = self.loop.create_future()
-        fut2 = self.loop.create_future()
-        cancelled = self.loop.create_future()
-        do_handle_client = True
-
-        async def handle_client(stream):
-            data = await stream.readexactly(4)
-            self.assertEqual(b'data', data)
-            fut1.set_result(None)
-            await fut2
-            while do_handle_client:
-                with contextlib.suppress(asyncio.CancelledError):
-                    await asyncio.sleep(0.01)
-            cancelled.set_result(None)
-
-        async def client(srv):
-            addr = srv.sockets[0].getsockname()
-            stream = await asyncio.connect(*addr)
-            await stream.write(b'data')
-            await fut2
-            self.assertEqual(b'', await stream.readline())
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(handle_client,
-                                            '127.0.0.1',
-                                            0,
-                                            shutdown_timeout=0.3) as server:
-                await server.start_serving()
-                task = asyncio.create_task(client(server))
-                await fut1
-                fut2.set_result(None)
-                await server.close()
-                nonlocal do_handle_client
-                do_handle_client = False
-                await task
-                await cancelled
-
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-        self.loop.run_until_complete(asyncio.wait_for(test(), 60.0))
-        self.assertEqual(1, len(messages))
-        self.assertRegex(messages[0]['message'],
-                         "<Task pending .+ ignored cancellation request")
-        self.assertTrue(fut1.done())
-        self.assertTrue(fut2.done())
-        self.assertTrue(cancelled.done())
-
-    def test_sendfile(self):
-        messages = []
-        self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))
-
-        with open(support.TESTFN, 'wb') as fp:
-            fp.write(b'data\n')
-        self.addCleanup(support.unlink, support.TESTFN)
-
-        async def serve_callback(stream):
-            data = await stream.readline()
-            await stream.write(b'ack-' + data)
-            data = await stream.readline()
-            await stream.write(b'ack-' + data)
-            data = await stream.readline()
-            await stream.write(b'ack-' + data)
-            await stream.close()
-
-        async def do_connect(host, port):
-            stream = await asyncio.connect(host, port)
-            await stream.write(b'begin\n')
-            data = await stream.readline()
-            self.assertEqual(b'ack-begin\n', data)
-            with open(support.TESTFN, 'rb') as fp:
-                await stream.sendfile(fp)
-            data = await stream.readline()
-            self.assertEqual(b'ack-data\n', data)
-            await stream.write(b'end\n')
-            data = await stream.readline()
-            self.assertEqual(data, b'ack-end\n')
-            await stream.close()
-
-        async def test():
-            async with asyncio.StreamServer(serve_callback, '127.0.0.1', 0) as srv:
-                await srv.start_serving()
-                await do_connect(*srv.sockets[0].getsockname())
-
-        self.loop.run_until_complete(test())
-
-        self.assertEqual([], messages)
-
-
-    @unittest.skipIf(ssl is None, 'No ssl module')
-    def test_connect_start_tls(self):
-        with test_utils.run_test_server(use_ssl=True) as httpd:
-            # connect without SSL but upgrade to TLS just after
-            # connection is established
-            stream = self.loop.run_until_complete(
-                asyncio.connect(*httpd.address))
-
-            self.loop.run_until_complete(
-                stream.start_tls(
-                    sslcontext=test_utils.dummy_ssl_context()))
-            self._basetest_connect(stream)
-
-    def test_repr_unbound(self):
-        async def serve(stream):
-            pass
-
-        async def test():
-            srv = asyncio.StreamServer(serve)
-            self.assertEqual('<StreamServer>', repr(srv))
-            await srv.close()
-
-        self.loop.run_until_complete(test())
-
-    def test_repr_bound(self):
-        async def serve(stream):
-            pass
-
-        async def test():
-            srv = asyncio.StreamServer(serve, '127.0.0.1', 0)
-            await srv.bind()
-            self.assertRegex(repr(srv), r'<StreamServer sockets=\(.+\)>')
-            await srv.close()
-
-        self.loop.run_until_complete(test())
-
-    def test_repr_serving(self):
-        async def serve(stream):
-            pass
-
-        async def test():
-            srv = asyncio.StreamServer(serve, '127.0.0.1', 0)
-            await srv.start_serving()
-            self.assertRegex(repr(srv), r'<StreamServer serving sockets=\(.+\)>')
-            await srv.close()
-
-        self.loop.run_until_complete(test())
-
-
-    @unittest.skipUnless(sys.platform != 'win32',
-                         "Don't support pipes for Windows")
-    def test_read_pipe(self):
-        async def test():
-            rpipe, wpipe = os.pipe()
-            pipeobj = io.open(rpipe, 'rb', 1024)
-
-            async with asyncio.connect_read_pipe(pipeobj) as stream:
-                self.assertEqual(stream.mode, asyncio.StreamMode.READ)
-
-                os.write(wpipe, b'1')
-                data = await stream.readexactly(1)
-                self.assertEqual(data, b'1')
-
-                os.write(wpipe, b'2345')
-                data = await stream.readexactly(4)
-                self.assertEqual(data, b'2345')
-                os.close(wpipe)
-
-        self.loop.run_until_complete(test())
-
-    @unittest.skipUnless(sys.platform != 'win32',
-                         "Don't support pipes for Windows")
-    def test_write_pipe(self):
-        async def test():
-            rpipe, wpipe = os.pipe()
-            pipeobj = io.open(wpipe, 'wb', 1024)
-
-            async with asyncio.connect_write_pipe(pipeobj) as stream:
-                self.assertEqual(stream.mode, asyncio.StreamMode.WRITE)
-
-                await stream.write(b'1')
-                data = os.read(rpipe, 1024)
-                self.assertEqual(data, b'1')
-
-                await stream.write(b'2345')
-                data = os.read(rpipe, 1024)
-                self.assertEqual(data, b'2345')
-
-                os.close(rpipe)
-
-        self.loop.run_until_complete(test())
-
-    def test_stream_ctor_forbidden(self):
-        with self.assertRaisesRegex(RuntimeError,
-                                    "should be instantiated "
-                                    "by asyncio internals only"):
-            asyncio.Stream(asyncio.StreamMode.READWRITE)
-
-    def test_deprecated_methods(self):
-        async def f():
-            return asyncio.Stream(mode=asyncio.StreamMode.READWRITE,
-                                  _asyncio_internal=True)
-
-        stream = self.loop.run_until_complete(f())
-
-        tr = mock.Mock()
-
-        with self.assertWarns(DeprecationWarning):
-            stream.set_transport(tr)
-
-        with self.assertWarns(DeprecationWarning):
-            stream.transport is tr
-
-        with self.assertWarns(DeprecationWarning):
-            stream.feed_data(b'data')
-
-        with self.assertWarns(DeprecationWarning):
-            stream.feed_eof()
-
-        with self.assertWarns(DeprecationWarning):
-            stream.set_exception(ConnectionResetError("test"))
-
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py
index 3ad18e5c5166..fe8cfa61b1b2 100644
--- a/Lib/test/test_asyncio/test_subprocess.py
+++ b/Lib/test/test_asyncio/test_subprocess.py
@@ -582,18 +582,6 @@ def test_read_stdout_after_process_exit(self):
 
         self.loop.run_until_complete(execute())
 
-    def test_subprocess_protocol_create_warning(self):
-        with self.assertWarns(DeprecationWarning):
-            subprocess.SubprocessStreamProtocol(limit=10, loop=self.loop)
-
-    def test_process_create_warning(self):
-        proto = subprocess.SubprocessStreamProtocol(limit=10, loop=self.loop,
-                                                    _asyncio_internal=True)
-        transp = mock.Mock()
-
-        with self.assertWarns(DeprecationWarning):
-            subprocess.Process(transp, proto, loop=self.loop)
-
     def test_create_subprocess_exec_text_mode_fails(self):
         async def execute():
             with self.assertRaises(ValueError):
diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py
index d0ba19391fa0..9ed10fc20f81 100644
--- a/Lib/test/test_asyncio/test_windows_events.py
+++ b/Lib/test/test_asyncio/test_windows_events.py
@@ -15,7 +15,6 @@
 
 import asyncio
 from asyncio import windows_events
-from asyncio.streams import _StreamProtocol
 from test.test_asyncio import utils as test_utils
 
 
@@ -118,16 +117,14 @@ def test_pipe(self):
 
         clients = []
         for i in range(5):
-            stream = asyncio.Stream(mode=asyncio.StreamMode.READ,
-                                    loop=self.loop, _asyncio_internal=True)
-            protocol = _StreamProtocol(stream,
-                                       loop=self.loop,
-                                       _asyncio_internal=True)
+            stream_reader = asyncio.StreamReader(loop=self.loop)
+            protocol = asyncio.StreamReaderProtocol(stream_reader,
+                                                    loop=self.loop)
             trans, proto = await self.loop.create_pipe_connection(
                 lambda: protocol, ADDRESS)
             self.assertIsInstance(trans, asyncio.Transport)
             self.assertEqual(protocol, proto)
-            clients.append((stream, trans))
+            clients.append((stream_reader, trans))
 
         for i, (r, w) in enumerate(clients):
             w.write('lower-{}\n'.format(i).encode())
@@ -136,7 +133,6 @@ def test_pipe(self):
             response = await r.readline()
             self.assertEqual(response, 'LOWER-{}\n'.format(i).encode())
             w.close()
-            await r.close()
 
         server.close()
 
diff --git a/Misc/NEWS.d/next/Library/2019-09-30-00-15-27.bpo-38242.uPIyAc.rst b/Misc/NEWS.d/next/Library/2019-09-30-00-15-27.bpo-38242.uPIyAc.rst
new file mode 100644
index 000000000000..be9da891b866
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-30-00-15-27.bpo-38242.uPIyAc.rst
@@ -0,0 +1 @@
+Revert the new asyncio Streams API


From webhook-mailer at python.org  Mon Sep 30 01:59:15 2019
From: webhook-mailer at python.org (Yury Selivanov)
Date: Mon, 30 Sep 2019 05:59:15 -0000
Subject: [Python-checkins] bpo-30773: Fix ag_running;
 prohibit running athrow/asend/aclose in parallel (#7468)
Message-ID: <mailman.153.1569823156.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/fc4a044a3c54ce21e9ed150f7d769fb479d34c49
commit: fc4a044a3c54ce21e9ed150f7d769fb479d34c49
branch: master
author: Yury Selivanov <yury at magic.io>
committer: GitHub <noreply at github.com>
date: 2019-09-29T22:59:11-07:00
summary:

bpo-30773: Fix ag_running; prohibit running athrow/asend/aclose in parallel (#7468)

files:
A Misc/NEWS.d/next/Core and Builtins/2018-06-07-01-01-20.bpo-30773.C31rVE.rst
M Include/genobject.h
M Lib/test/test_asyncgen.py
M Objects/genobject.c

diff --git a/Include/genobject.h b/Include/genobject.h
index 3b3b68b56b90..96f8dcc74d8c 100644
--- a/Include/genobject.h
+++ b/Include/genobject.h
@@ -79,6 +79,8 @@ typedef struct {
     /* Flag is set to 1 when aclose() is called for the first time, or
        when a StopAsyncIteration exception is raised. */
     int ag_closed;
+
+    int ag_running_async;
 } PyAsyncGenObject;
 
 PyAPI_DATA(PyTypeObject) PyAsyncGen_Type;
diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py
index 3a8d5fd4007f..23eb6a411cd1 100644
--- a/Lib/test/test_asyncgen.py
+++ b/Lib/test/test_asyncgen.py
@@ -133,24 +133,6 @@ def async_iterate(g):
                     break
             return res
 
-        def async_iterate(g):
-            res = []
-            while True:
-                try:
-                    g.__anext__().__next__()
-                except StopAsyncIteration:
-                    res.append('STOP')
-                    break
-                except StopIteration as ex:
-                    if ex.args:
-                        res.append(ex.args[0])
-                    else:
-                        res.append('EMPTY StopIteration')
-                        break
-                except Exception as ex:
-                    res.append(str(type(ex)))
-            return res
-
         sync_gen_result = sync_iterate(sync_gen)
         async_gen_result = async_iterate(async_gen)
         self.assertEqual(sync_gen_result, async_gen_result)
@@ -176,19 +158,22 @@ def test_async_gen_iteration_02(self):
 
         g = gen()
         ai = g.__aiter__()
-        self.assertEqual(ai.__anext__().__next__(), ('result',))
+
+        an = ai.__anext__()
+        self.assertEqual(an.__next__(), ('result',))
 
         try:
-            ai.__anext__().__next__()
+            an.__next__()
         except StopIteration as ex:
             self.assertEqual(ex.args[0], 123)
         else:
             self.fail('StopIteration was not raised')
 
-        self.assertEqual(ai.__anext__().__next__(), ('result',))
+        an = ai.__anext__()
+        self.assertEqual(an.__next__(), ('result',))
 
         try:
-            ai.__anext__().__next__()
+            an.__next__()
         except StopAsyncIteration as ex:
             self.assertFalse(ex.args)
         else:
@@ -212,10 +197,11 @@ def test_async_gen_exception_04(self):
 
         g = gen()
         ai = g.__aiter__()
-        self.assertEqual(ai.__anext__().__next__(), ('result',))
+        an = ai.__anext__()
+        self.assertEqual(an.__next__(), ('result',))
 
         try:
-            ai.__anext__().__next__()
+            an.__next__()
         except StopIteration as ex:
             self.assertEqual(ex.args[0], 123)
         else:
@@ -646,17 +632,13 @@ def test_async_gen_asyncio_aclose_08(self):
             gen = foo()
             it = gen.__aiter__()
             self.assertEqual(await it.__anext__(), 1)
-            t = self.loop.create_task(it.__anext__())
-            await asyncio.sleep(0.01)
             await gen.aclose()
-            return t
 
-        t = self.loop.run_until_complete(run())
+        self.loop.run_until_complete(run())
         self.assertEqual(DONE, 1)
 
         # Silence ResourceWarnings
         fut.cancel()
-        t.cancel()
         self.loop.run_until_complete(asyncio.sleep(0.01))
 
     def test_async_gen_asyncio_gc_aclose_09(self):
@@ -1053,46 +1035,18 @@ def test_async_gen_asyncio_shutdown_01(self):
 
         self.loop.run_until_complete(asyncio.sleep(0.1))
 
-        self.loop.run_until_complete(self.loop.shutdown_asyncgens())
-        self.assertEqual(finalized, 2)
-
         # Silence warnings
         t1.cancel()
         t2.cancel()
-        self.loop.run_until_complete(asyncio.sleep(0.1))
 
-    def test_async_gen_asyncio_shutdown_02(self):
-        logged = 0
-
-        def logger(loop, context):
-            nonlocal logged
-            self.assertIn('asyncgen', context)
-            expected = 'an error occurred during closing of asynchronous'
-            if expected in context['message']:
-                logged += 1
-
-        async def waiter(timeout):
-            try:
-                await asyncio.sleep(timeout)
-                yield 1
-            finally:
-                1 / 0
-
-        async def wait():
-            async for _ in waiter(1):
-                pass
-
-        t = self.loop.create_task(wait())
-        self.loop.run_until_complete(asyncio.sleep(0.1))
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(t1)
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(t2)
 
-        self.loop.set_exception_handler(logger)
         self.loop.run_until_complete(self.loop.shutdown_asyncgens())
 
-        self.assertEqual(logged, 1)
-
-        # Silence warnings
-        t.cancel()
-        self.loop.run_until_complete(asyncio.sleep(0.1))
+        self.assertEqual(finalized, 2)
 
     def test_async_gen_expression_01(self):
         async def arange(n):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-06-07-01-01-20.bpo-30773.C31rVE.rst b/Misc/NEWS.d/next/Core and Builtins/2018-06-07-01-01-20.bpo-30773.C31rVE.rst
new file mode 100644
index 000000000000..501ee527e8a9
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2018-06-07-01-01-20.bpo-30773.C31rVE.rst	
@@ -0,0 +1,2 @@
+Prohibit parallel running of aclose() / asend() / athrow(). Fix ag_running
+to reflect the actual running status of the AG.
diff --git a/Objects/genobject.c b/Objects/genobject.c
index 82e6e55868f6..b72248caf9cc 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -1326,7 +1326,8 @@ static PyGetSetDef async_gen_getsetlist[] = {
 
 static PyMemberDef async_gen_memberlist[] = {
     {"ag_frame",   T_OBJECT, offsetof(PyAsyncGenObject, ag_frame),   READONLY},
-    {"ag_running", T_BOOL,   offsetof(PyAsyncGenObject, ag_running), READONLY},
+    {"ag_running", T_BOOL,   offsetof(PyAsyncGenObject, ag_running_async),
+        READONLY},
     {"ag_code",    T_OBJECT, offsetof(PyAsyncGenObject, ag_code),    READONLY},
     {NULL}      /* Sentinel */
 };
@@ -1420,6 +1421,7 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
     o->ag_finalizer = NULL;
     o->ag_closed = 0;
     o->ag_hooks_inited = 0;
+    o->ag_running_async = 0;
     return (PyObject*)o;
 }
 
@@ -1467,6 +1469,7 @@ async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result)
             gen->ag_closed = 1;
         }
 
+        gen->ag_running_async = 0;
         return NULL;
     }
 
@@ -1474,6 +1477,7 @@ async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result)
         /* async yield */
         _PyGen_SetStopIterationValue(((_PyAsyncGenWrappedValue*)result)->agw_val);
         Py_DECREF(result);
+        gen->ag_running_async = 0;
         return NULL;
     }
 
@@ -1518,12 +1522,20 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg)
     }
 
     if (o->ags_state == AWAITABLE_STATE_INIT) {
+        if (o->ags_gen->ag_running_async) {
+            PyErr_SetString(
+                PyExc_RuntimeError,
+                "anext(): asynchronous generator is already running");
+            return NULL;
+        }
+
         if (arg == NULL || arg == Py_None) {
             arg = o->ags_sendval;
         }
         o->ags_state = AWAITABLE_STATE_ITER;
     }
 
+    o->ags_gen->ag_running_async = 1;
     result = gen_send_ex((PyGenObject*)o->ags_gen, arg, 0, 0);
     result = async_gen_unwrap_value(o->ags_gen, result);
 
@@ -1787,8 +1799,23 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
     }
 
     if (o->agt_state == AWAITABLE_STATE_INIT) {
+        if (o->agt_gen->ag_running_async) {
+            if (o->agt_args == NULL) {
+                PyErr_SetString(
+                    PyExc_RuntimeError,
+                    "aclose(): asynchronous generator is already running");
+            }
+            else {
+                PyErr_SetString(
+                    PyExc_RuntimeError,
+                    "athrow(): asynchronous generator is already running");
+            }
+            return NULL;
+        }
+
         if (o->agt_gen->ag_closed) {
-            PyErr_SetNone(PyExc_StopIteration);
+            o->agt_state = AWAITABLE_STATE_CLOSED;
+            PyErr_SetNone(PyExc_StopAsyncIteration);
             return NULL;
         }
 
@@ -1798,6 +1825,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
         }
 
         o->agt_state = AWAITABLE_STATE_ITER;
+        o->agt_gen->ag_running_async = 1;
 
         if (o->agt_args == NULL) {
             /* aclose() mode */
@@ -1843,6 +1871,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
         /* aclose() mode */
         if (retval) {
             if (_PyAsyncGenWrappedValue_CheckExact(retval)) {
+                o->agt_gen->ag_running_async = 0;
                 Py_DECREF(retval);
                 goto yield_close;
             }
@@ -1856,11 +1885,13 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
     }
 
 yield_close:
+    o->agt_gen->ag_running_async = 0;
     PyErr_SetString(
         PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG);
     return NULL;
 
 check_error:
+    o->agt_gen->ag_running_async = 0;
     if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) ||
             PyErr_ExceptionMatches(PyExc_GeneratorExit))
     {
@@ -1895,6 +1926,7 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *args)
     } else {
         /* aclose() mode */
         if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) {
+            o->agt_gen->ag_running_async = 0;
             Py_DECREF(retval);
             PyErr_SetString(PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG);
             return NULL;


From webhook-mailer at python.org  Mon Sep 30 02:19:10 2019
From: webhook-mailer at python.org (Yury Selivanov)
Date: Mon, 30 Sep 2019 06:19:10 -0000
Subject: [Python-checkins] bpo-30773: Fix ag_running;
 prohibit running athrow/asend/aclose in parallel (GH-7468) (#16486)
Message-ID: <mailman.154.1569824351.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/2f87a7dc5a1ad7f37787f0adee242c931643f878
commit: 2f87a7dc5a1ad7f37787f0adee242c931643f878
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Yury Selivanov <yury at magic.io>
date: 2019-09-29T23:19:02-07:00
summary:

bpo-30773: Fix ag_running; prohibit running athrow/asend/aclose in parallel (GH-7468) (#16486)

(cherry picked from commit fc4a044a3c54ce21e9ed150f7d769fb479d34c49)

Co-authored-by: Yury Selivanov <yury at magic.io>

files:
A Misc/NEWS.d/next/Core and Builtins/2018-06-07-01-01-20.bpo-30773.C31rVE.rst
M Include/genobject.h
M Lib/test/test_asyncgen.py
M Objects/genobject.c

diff --git a/Include/genobject.h b/Include/genobject.h
index 6755963f332f..59ede281f44f 100644
--- a/Include/genobject.h
+++ b/Include/genobject.h
@@ -80,6 +80,8 @@ typedef struct {
     /* Flag is set to 1 when aclose() is called for the first time, or
        when a StopAsyncIteration exception is raised. */
     int ag_closed;
+
+    int ag_running_async;
 } PyAsyncGenObject;
 
 PyAPI_DATA(PyTypeObject) PyAsyncGen_Type;
diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py
index 3a8d5fd4007f..23eb6a411cd1 100644
--- a/Lib/test/test_asyncgen.py
+++ b/Lib/test/test_asyncgen.py
@@ -133,24 +133,6 @@ def async_iterate(g):
                     break
             return res
 
-        def async_iterate(g):
-            res = []
-            while True:
-                try:
-                    g.__anext__().__next__()
-                except StopAsyncIteration:
-                    res.append('STOP')
-                    break
-                except StopIteration as ex:
-                    if ex.args:
-                        res.append(ex.args[0])
-                    else:
-                        res.append('EMPTY StopIteration')
-                        break
-                except Exception as ex:
-                    res.append(str(type(ex)))
-            return res
-
         sync_gen_result = sync_iterate(sync_gen)
         async_gen_result = async_iterate(async_gen)
         self.assertEqual(sync_gen_result, async_gen_result)
@@ -176,19 +158,22 @@ def test_async_gen_iteration_02(self):
 
         g = gen()
         ai = g.__aiter__()
-        self.assertEqual(ai.__anext__().__next__(), ('result',))
+
+        an = ai.__anext__()
+        self.assertEqual(an.__next__(), ('result',))
 
         try:
-            ai.__anext__().__next__()
+            an.__next__()
         except StopIteration as ex:
             self.assertEqual(ex.args[0], 123)
         else:
             self.fail('StopIteration was not raised')
 
-        self.assertEqual(ai.__anext__().__next__(), ('result',))
+        an = ai.__anext__()
+        self.assertEqual(an.__next__(), ('result',))
 
         try:
-            ai.__anext__().__next__()
+            an.__next__()
         except StopAsyncIteration as ex:
             self.assertFalse(ex.args)
         else:
@@ -212,10 +197,11 @@ def test_async_gen_exception_04(self):
 
         g = gen()
         ai = g.__aiter__()
-        self.assertEqual(ai.__anext__().__next__(), ('result',))
+        an = ai.__anext__()
+        self.assertEqual(an.__next__(), ('result',))
 
         try:
-            ai.__anext__().__next__()
+            an.__next__()
         except StopIteration as ex:
             self.assertEqual(ex.args[0], 123)
         else:
@@ -646,17 +632,13 @@ def test_async_gen_asyncio_aclose_08(self):
             gen = foo()
             it = gen.__aiter__()
             self.assertEqual(await it.__anext__(), 1)
-            t = self.loop.create_task(it.__anext__())
-            await asyncio.sleep(0.01)
             await gen.aclose()
-            return t
 
-        t = self.loop.run_until_complete(run())
+        self.loop.run_until_complete(run())
         self.assertEqual(DONE, 1)
 
         # Silence ResourceWarnings
         fut.cancel()
-        t.cancel()
         self.loop.run_until_complete(asyncio.sleep(0.01))
 
     def test_async_gen_asyncio_gc_aclose_09(self):
@@ -1053,46 +1035,18 @@ def test_async_gen_asyncio_shutdown_01(self):
 
         self.loop.run_until_complete(asyncio.sleep(0.1))
 
-        self.loop.run_until_complete(self.loop.shutdown_asyncgens())
-        self.assertEqual(finalized, 2)
-
         # Silence warnings
         t1.cancel()
         t2.cancel()
-        self.loop.run_until_complete(asyncio.sleep(0.1))
 
-    def test_async_gen_asyncio_shutdown_02(self):
-        logged = 0
-
-        def logger(loop, context):
-            nonlocal logged
-            self.assertIn('asyncgen', context)
-            expected = 'an error occurred during closing of asynchronous'
-            if expected in context['message']:
-                logged += 1
-
-        async def waiter(timeout):
-            try:
-                await asyncio.sleep(timeout)
-                yield 1
-            finally:
-                1 / 0
-
-        async def wait():
-            async for _ in waiter(1):
-                pass
-
-        t = self.loop.create_task(wait())
-        self.loop.run_until_complete(asyncio.sleep(0.1))
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(t1)
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(t2)
 
-        self.loop.set_exception_handler(logger)
         self.loop.run_until_complete(self.loop.shutdown_asyncgens())
 
-        self.assertEqual(logged, 1)
-
-        # Silence warnings
-        t.cancel()
-        self.loop.run_until_complete(asyncio.sleep(0.1))
+        self.assertEqual(finalized, 2)
 
     def test_async_gen_expression_01(self):
         async def arange(n):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-06-07-01-01-20.bpo-30773.C31rVE.rst b/Misc/NEWS.d/next/Core and Builtins/2018-06-07-01-01-20.bpo-30773.C31rVE.rst
new file mode 100644
index 000000000000..501ee527e8a9
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2018-06-07-01-01-20.bpo-30773.C31rVE.rst	
@@ -0,0 +1,2 @@
+Prohibit parallel running of aclose() / asend() / athrow(). Fix ag_running
+to reflect the actual running status of the AG.
diff --git a/Objects/genobject.c b/Objects/genobject.c
index f1e9fec78a46..6285219bb2d7 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -1342,7 +1342,8 @@ static PyGetSetDef async_gen_getsetlist[] = {
 
 static PyMemberDef async_gen_memberlist[] = {
     {"ag_frame",   T_OBJECT, offsetof(PyAsyncGenObject, ag_frame),   READONLY},
-    {"ag_running", T_BOOL,   offsetof(PyAsyncGenObject, ag_running), READONLY},
+    {"ag_running", T_BOOL,   offsetof(PyAsyncGenObject, ag_running_async),
+        READONLY},
     {"ag_code",    T_OBJECT, offsetof(PyAsyncGenObject, ag_code),    READONLY},
     {NULL}      /* Sentinel */
 };
@@ -1436,6 +1437,7 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
     o->ag_finalizer = NULL;
     o->ag_closed = 0;
     o->ag_hooks_inited = 0;
+    o->ag_running_async = 0;
     return (PyObject*)o;
 }
 
@@ -1483,6 +1485,7 @@ async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result)
             gen->ag_closed = 1;
         }
 
+        gen->ag_running_async = 0;
         return NULL;
     }
 
@@ -1490,6 +1493,7 @@ async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result)
         /* async yield */
         _PyGen_SetStopIterationValue(((_PyAsyncGenWrappedValue*)result)->agw_val);
         Py_DECREF(result);
+        gen->ag_running_async = 0;
         return NULL;
     }
 
@@ -1534,12 +1538,20 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg)
     }
 
     if (o->ags_state == AWAITABLE_STATE_INIT) {
+        if (o->ags_gen->ag_running_async) {
+            PyErr_SetString(
+                PyExc_RuntimeError,
+                "anext(): asynchronous generator is already running");
+            return NULL;
+        }
+
         if (arg == NULL || arg == Py_None) {
             arg = o->ags_sendval;
         }
         o->ags_state = AWAITABLE_STATE_ITER;
     }
 
+    o->ags_gen->ag_running_async = 1;
     result = gen_send_ex((PyGenObject*)o->ags_gen, arg, 0, 0);
     result = async_gen_unwrap_value(o->ags_gen, result);
 
@@ -1803,8 +1815,23 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
     }
 
     if (o->agt_state == AWAITABLE_STATE_INIT) {
+        if (o->agt_gen->ag_running_async) {
+            if (o->agt_args == NULL) {
+                PyErr_SetString(
+                    PyExc_RuntimeError,
+                    "aclose(): asynchronous generator is already running");
+            }
+            else {
+                PyErr_SetString(
+                    PyExc_RuntimeError,
+                    "athrow(): asynchronous generator is already running");
+            }
+            return NULL;
+        }
+
         if (o->agt_gen->ag_closed) {
-            PyErr_SetNone(PyExc_StopIteration);
+            o->agt_state = AWAITABLE_STATE_CLOSED;
+            PyErr_SetNone(PyExc_StopAsyncIteration);
             return NULL;
         }
 
@@ -1814,6 +1841,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
         }
 
         o->agt_state = AWAITABLE_STATE_ITER;
+        o->agt_gen->ag_running_async = 1;
 
         if (o->agt_args == NULL) {
             /* aclose() mode */
@@ -1859,6 +1887,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
         /* aclose() mode */
         if (retval) {
             if (_PyAsyncGenWrappedValue_CheckExact(retval)) {
+                o->agt_gen->ag_running_async = 0;
                 Py_DECREF(retval);
                 goto yield_close;
             }
@@ -1872,11 +1901,13 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
     }
 
 yield_close:
+    o->agt_gen->ag_running_async = 0;
     PyErr_SetString(
         PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG);
     return NULL;
 
 check_error:
+    o->agt_gen->ag_running_async = 0;
     if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) ||
             PyErr_ExceptionMatches(PyExc_GeneratorExit))
     {
@@ -1911,6 +1942,7 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *args)
     } else {
         /* aclose() mode */
         if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) {
+            o->agt_gen->ag_running_async = 0;
             Py_DECREF(retval);
             PyErr_SetString(PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG);
             return NULL;


From webhook-mailer at python.org  Mon Sep 30 06:52:38 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 30 Sep 2019 10:52:38 -0000
Subject: [Python-checkins] bpo-38304: Fix PyConfig usage in python_uwp.cpp
 (GH-16487)
Message-ID: <mailman.155.1569840759.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/89f8177dcfdbcf17c85bb6998c946c9f42bf6e27
commit: 89f8177dcfdbcf17c85bb6998c946c9f42bf6e27
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-30T12:52:29+02:00
summary:

bpo-38304: Fix PyConfig usage in python_uwp.cpp (GH-16487)

* Set PyPreConfig.struct_size and PyConfig.struct_size as required by
  the API.
* PyPreConfig_InitPythonConfig() can now fail: check PyStatus result.

files:
M PC/python_uwp.cpp

diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp
index 06c1dd353655..2656d188c250 100644
--- a/PC/python_uwp.cpp
+++ b/PC/python_uwp.cpp
@@ -167,7 +167,10 @@ wmain(int argc, wchar_t **argv)
     PyStatus status;
 
     PyPreConfig preconfig;
+    preconfig.struct_size = sizeof(PyPreConfig);
+
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     const wchar_t *moduleName = NULL;
     const wchar_t *p = wcsrchr(argv[0], L'\\');
@@ -186,7 +189,10 @@ wmain(int argc, wchar_t **argv)
         }
     }
 
-    PyPreConfig_InitPythonConfig(&preconfig);
+    status = PyPreConfig_InitPythonConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        goto fail_without_config;
+    }
     if (!moduleName) {
         status = Py_PreInitializeFromArgs(&preconfig, argc, argv);
         if (PyStatus_Exception(status)) {


From webhook-mailer at python.org  Mon Sep 30 06:53:20 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 30 Sep 2019 10:53:20 -0000
Subject: [Python-checkins] bpo-38304: Fix PyConfig usage in python_uwp.cpp
 (GH-16488)
Message-ID: <mailman.156.1569840802.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/81f6b031c46721478372d77fe2e55aa1f8300ae1
commit: 81f6b031c46721478372d77fe2e55aa1f8300ae1
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-30T12:53:17+02:00
summary:

bpo-38304: Fix PyConfig usage in python_uwp.cpp (GH-16488)

* Set PyPreConfig.struct_size and PyConfig.struct_size as required by
  the API.
* PyPreConfig_InitPythonConfig() can now fail: check PyStatus result.

files:
M PC/python_uwp.cpp

diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp
index 06c1dd353655..2656d188c250 100644
--- a/PC/python_uwp.cpp
+++ b/PC/python_uwp.cpp
@@ -167,7 +167,10 @@ wmain(int argc, wchar_t **argv)
     PyStatus status;
 
     PyPreConfig preconfig;
+    preconfig.struct_size = sizeof(PyPreConfig);
+
     PyConfig config;
+    config.struct_size = sizeof(PyConfig);
 
     const wchar_t *moduleName = NULL;
     const wchar_t *p = wcsrchr(argv[0], L'\\');
@@ -186,7 +189,10 @@ wmain(int argc, wchar_t **argv)
         }
     }
 
-    PyPreConfig_InitPythonConfig(&preconfig);
+    status = PyPreConfig_InitPythonConfig(&preconfig);
+    if (PyStatus_Exception(status)) {
+        goto fail_without_config;
+    }
     if (!moduleName) {
         status = Py_PreInitializeFromArgs(&preconfig, argc, argv);
         if (PyStatus_Exception(status)) {


From webhook-mailer at python.org  Mon Sep 30 08:49:41 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 30 Sep 2019 12:49:41 -0000
Subject: [Python-checkins] bpo-38322: Fix gotlandmark() of PC/getpathp.c
 (GH-16489)
Message-ID: <mailman.157.1569847783.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/dec39716ca93ee2e8d9b94915ece33014eb58e9e
commit: dec39716ca93ee2e8d9b94915ece33014eb58e9e
branch: master
author: Victor Stinner <vstinner at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-30T14:49:34+02:00
summary:

bpo-38322: Fix gotlandmark() of PC/getpathp.c (GH-16489)

Write the filename into a temporary buffer instead of reusing prefix.
The problem is that join() modifies prefix inplace. If prefix is not
normalized, join() can make prefix shorter and so gotlandmark()
does modify prefix instead of returning it unmodified.

files:
M PC/getpathp.c

diff --git a/PC/getpathp.c b/PC/getpathp.c
index 8bac592aefdb..04f24d986f66 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -315,15 +315,13 @@ canonicalize(wchar_t *buffer, const wchar_t *path)
    'prefix' is null terminated in bounds.  join() ensures
    'landmark' can not overflow prefix if too long. */
 static int
-gotlandmark(wchar_t *prefix, const wchar_t *landmark)
+gotlandmark(const wchar_t *prefix, const wchar_t *landmark)
 {
-    int ok;
-    Py_ssize_t n = wcsnlen_s(prefix, MAXPATHLEN);
-
-    join(prefix, landmark);
-    ok = ismodule(prefix, FALSE);
-    prefix[n] = '\0';
-    return ok;
+    wchar_t filename[MAXPATHLEN+1];
+    memset(filename, 0, sizeof(filename));
+    wcscpy_s(filename, Py_ARRAY_LENGTH(filename), prefix);
+    join(filename, landmark);
+    return ismodule(filename, FALSE);
 }
 
 


From webhook-mailer at python.org  Mon Sep 30 08:49:46 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 30 Sep 2019 12:49:46 -0000
Subject: [Python-checkins] bpo-38322: Fix gotlandmark() of PC/getpathp.c
 (GH-16490)
Message-ID: <mailman.158.1569847787.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/18c4ba9f33868761e374a725d497902863d59ea9
commit: 18c4ba9f33868761e374a725d497902863d59ea9
branch: 3.8
author: Victor Stinner <vstinner at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-30T14:49:42+02:00
summary:

bpo-38322: Fix gotlandmark() of PC/getpathp.c (GH-16490)

Write the filename into a temporary buffer instead of reusing prefix.
The problem is that join() modifies prefix inplace. If prefix is not
normalized, join() can make prefix shorter and so gotlandmark()
does modify prefix instead of returning it unmodified.

files:
M PC/getpathp.c

diff --git a/PC/getpathp.c b/PC/getpathp.c
index 8bac592aefdb..04f24d986f66 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -315,15 +315,13 @@ canonicalize(wchar_t *buffer, const wchar_t *path)
    'prefix' is null terminated in bounds.  join() ensures
    'landmark' can not overflow prefix if too long. */
 static int
-gotlandmark(wchar_t *prefix, const wchar_t *landmark)
+gotlandmark(const wchar_t *prefix, const wchar_t *landmark)
 {
-    int ok;
-    Py_ssize_t n = wcsnlen_s(prefix, MAXPATHLEN);
-
-    join(prefix, landmark);
-    ok = ismodule(prefix, FALSE);
-    prefix[n] = '\0';
-    return ok;
+    wchar_t filename[MAXPATHLEN+1];
+    memset(filename, 0, sizeof(filename));
+    wcscpy_s(filename, Py_ARRAY_LENGTH(filename), prefix);
+    join(filename, landmark);
+    return ismodule(filename, FALSE);
 }
 
 


From webhook-mailer at python.org  Mon Sep 30 10:41:40 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 30 Sep 2019 14:41:40 -0000
Subject: [Python-checkins] bpo-38321: Fix _asynciomodule.c compiler warning
 (GH-16493)
Message-ID: <mailman.159.1569854500.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/efe74b6369a8d08f27c69703fcc1686966e51068
commit: efe74b6369a8d08f27c69703fcc1686966e51068
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-30T16:41:34+02:00
summary:

bpo-38321: Fix _asynciomodule.c compiler warning (GH-16493)

bpo-38248, bpo-38321: Fix warning:

    modules\_asynciomodule.c(2667):
    warning C4102: 'set_exception': unreferenced label

The related goto has been removed by
commit edad4d89e357c92f70c0324b937845d652b20afd.

files:
M Modules/_asynciomodule.c

diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c
index 8ee0d7a7e14c..b67afd41768f 100644
--- a/Modules/_asynciomodule.c
+++ b/Modules/_asynciomodule.c
@@ -2657,7 +2657,6 @@ task_step_impl(TaskObj *task, PyObject *exc)
         /* Some other exception; pop it and call Task.set_exception() */
         PyErr_Fetch(&et, &ev, &tb);
 
-set_exception:
         assert(et);
         if (!ev || !PyObject_TypeCheck(ev, (PyTypeObject *) et)) {
             PyErr_NormalizeException(&et, &ev, &tb);


From webhook-mailer at python.org  Mon Sep 30 11:50:06 2019
From: webhook-mailer at python.org (Vinay Sajip)
Date: Mon, 30 Sep 2019 15:50:06 -0000
Subject: [Python-checkins] bpo-38321: Fix
 PyCStructUnionType_update_stgdict() warning (GH-16492)
Message-ID: <mailman.160.1569858607.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/c9a413ede47171a224c72dd34122005170caaad4
commit: c9a413ede47171a224c72dd34122005170caaad4
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: Vinay Sajip <vinay_sajip at yahoo.co.uk>
date: 2019-09-30T16:49:47+01:00
summary:

bpo-38321: Fix PyCStructUnionType_update_stgdict() warning (GH-16492)

bpo-22273, bpo-38321: Fix following warning:

    modules\_ctypes\stgdict.c(704):
    warning C4244: 'initializing': conversion from 'Py_ssize_t' to 'int', possible loss of data

files:
M Modules/_ctypes/stgdict.c

diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c
index 947e9c25592f..b794f10a2cb7 100644
--- a/Modules/_ctypes/stgdict.c
+++ b/Modules/_ctypes/stgdict.c
@@ -701,7 +701,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
                 assert(actual_type_index <= MAX_ELEMENTS);
             }
             else {
-                int length = dict->length;
+                Py_ssize_t length = dict->length;
                 StgDictObject *edict;
 
                 edict = PyType_stgdict(dict->proto);


From webhook-mailer at python.org  Mon Sep 30 15:14:49 2019
From: webhook-mailer at python.org (Victor Stinner)
Date: Mon, 30 Sep 2019 19:14:49 -0000
Subject: [Python-checkins] bpo-38321: Fix _testcapimodule.c warning
 (GH-16494)
Message-ID: <mailman.161.1569870890.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/aca8c406ada3bb547765b262bed3ac0cc6be8dd3
commit: aca8c406ada3bb547765b262bed3ac0cc6be8dd3
branch: master
author: Victor Stinner <vstinner at python.org>
committer: GitHub <noreply at github.com>
date: 2019-09-30T21:14:26+02:00
summary:

bpo-38321: Fix _testcapimodule.c warning (GH-16494)

Fix the following warning:

    modules\_testcapimodule.c(6409):
    warning C4146: unary minus operator applied to unsigned type,
    result still unsigned

files:
M Modules/_testcapimodule.c

diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 30a786cd3455..3f3ed664880a 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -1952,7 +1952,7 @@ unicode_asutf8andsize(PyObject *self, PyObject *args)
         return NULL;
     }
 
-    buffer = PyUnicode_AsUTF8AndSize(unicode, &utf8_len); 
+    buffer = PyUnicode_AsUTF8AndSize(unicode, &utf8_len);
     if (buffer == NULL) {
         return NULL;
     }
@@ -6406,7 +6406,7 @@ static PyType_Spec HeapCTypeWithDict_spec = {
 
 static struct PyMemberDef heapctypewithnegativedict_members[] = {
     {"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)},
-    {"__dictoffset__", T_PYSSIZET, -sizeof(void*), READONLY},
+    {"__dictoffset__", T_PYSSIZET, -(Py_ssize_t)sizeof(void*), READONLY},
     {NULL} /* Sentinel */
 };
 


From webhook-mailer at python.org  Mon Sep 30 18:46:50 2019
From: webhook-mailer at python.org (Nick Coghlan)
Date: Mon, 30 Sep 2019 22:46:50 -0000
Subject: [Python-checkins] bpo-38304: Remove PyConfig.struct_size (GH-16500)
Message-ID: <mailman.162.1569883611.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/bdace21b769998396d0ccc8da99a8ca9b507bfdf
commit: bdace21b769998396d0ccc8da99a8ca9b507bfdf
branch: 3.8
author: Victor Stinner <vstinner at redhat.com>
committer: Nick Coghlan <ncoghlan at gmail.com>
date: 2019-10-01T08:46:42+10:00
summary:

bpo-38304: Remove PyConfig.struct_size (GH-16500)

For now, we'll rely on the fact that the config structures aren't covered by the stable ABI.

We may revisit this in the future if we further explore the idea of offering a stable embedding API.

files:
D Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst
M Doc/c-api/init_config.rst
M Include/cpython/initconfig.h
M Include/internal/pycore_initconfig.h
M Modules/main.c
M PC/python_uwp.cpp
M Programs/_freeze_importlib.c
M Programs/_testembed.c
M Python/frozenmain.c
M Python/initconfig.c
M Python/pathconfig.c
M Python/preconfig.c
M Python/pylifecycle.c
M Python/pystate.c

diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index 4c77d5d1e0e8..15b8b94d5a74 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -194,25 +194,18 @@ PyPreConfig
    * Configure the LC_CTYPE locale
    * Set the UTF-8 mode
 
-   The :c:member:`struct_size` field must be explicitly initialized to
-   ``sizeof(PyPreConfig)``.
-
    Function to initialize a preconfiguration:
 
-   .. c:function:: PyStatus PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig)
+   .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig)
 
       Initialize the preconfiguration with :ref:`Python Configuration
       <init-python-config>`.
 
-   .. c:function:: PyStatus PyPreConfig_InitPythonConfig(PyPreConfig *preconfig)
+   .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig)
 
       Initialize the preconfiguration with :ref:`Isolated Configuration
       <init-isolated-conf>`.
 
-   The caller of these functions is responsible to handle exceptions (error or
-   exit) using :c:func:`PyStatus_Exception` and
-   :c:func:`Py_ExitStatusException`.
-
    Structure fields:
 
    .. c:member:: int allocator
@@ -274,13 +267,6 @@ PyPreConfig
       same way the regular Python parses command line arguments: see
       :ref:`Command Line Arguments <using-on-cmdline>`.
 
-   .. c:member:: size_t struct_size
-
-      Size of the structure in bytes: must be initialized to
-      ``sizeof(PyPreConfig)``.
-
-      Field used for API and ABI compatibility.
-
    .. c:member:: int use_environment
 
       See :c:member:`PyConfig.use_environment`.
@@ -332,12 +318,7 @@ Example using the preinitialization to enable the UTF-8 Mode::
 
     PyStatus status;
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
-
-    status = PyPreConfig_InitPythonConfig(&preconfig);
-    if (PyStatus_Exception(status)) {
-        Py_ExitStatusException(status);
-    }
+    PyPreConfig_InitPythonConfig(&preconfig);
 
     preconfig.utf8_mode = 1;
 
@@ -360,9 +341,6 @@ PyConfig
 
    Structure containing most parameters to configure Python.
 
-   The :c:member:`struct_size` field must be explicitly initialized to
-   ``sizeof(PyConfig)``.
-
    Structure methods:
 
    .. c:function:: PyStatus PyConfig_InitPythonConfig(PyConfig *config)
@@ -679,13 +657,6 @@ PyConfig
       Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and
       :data:`sys.stderr`.
 
-   .. c:member:: size_t struct_size
-
-      Size of the structure in bytes: must be initialized to
-      ``sizeof(PyConfig)``.
-
-      Field used for API and ABI compatibility.
-
    .. c:member:: int tracemalloc
 
       If non-zero, call :func:`tracemalloc.start` at startup.
@@ -754,7 +725,6 @@ Example setting the program name::
     {
         PyStatus status;
         PyConfig config;
-        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
@@ -787,7 +757,6 @@ configuration, and then override some parameters::
     {
         PyStatus status;
         PyConfig config;
-        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
@@ -875,7 +844,6 @@ Example of customized Python always running in isolated mode::
     {
         PyStatus status;
         PyConfig config;
-        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
@@ -1067,7 +1035,6 @@ phases::
     {
         PyStatus status;
         PyConfig config;
-        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
index 8ce5622c5c40..25e02cea10dc 100644
--- a/Include/cpython/initconfig.h
+++ b/Include/cpython/initconfig.h
@@ -45,10 +45,6 @@ PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list,
 /* --- PyPreConfig ----------------------------------------------- */
 
 typedef struct {
-    /* Size of the structure in bytes: must be initialized to
-       sizeof(PyPreConfig). Field used for API and ABI compatibility. */
-    size_t struct_size;
-
     int _config_init;     /* _PyConfigInitEnum value */
 
     /* Parse Py_PreInitializeFromBytesArgs() arguments?
@@ -124,17 +120,13 @@ typedef struct {
     int allocator;
 } PyPreConfig;
 
-PyAPI_FUNC(PyStatus) PyPreConfig_InitPythonConfig(PyPreConfig *config);
-PyAPI_FUNC(PyStatus) PyPreConfig_InitIsolatedConfig(PyPreConfig *config);
+PyAPI_FUNC(void) PyPreConfig_InitPythonConfig(PyPreConfig *config);
+PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config);
 
 
 /* --- PyConfig ---------------------------------------------- */
 
 typedef struct {
-    /* Size of the structure in bytes: must be initialized to
-       sizeof(PyConfig). Field used for API and ABI compatibility. */
-    size_t struct_size;
-
     int _config_init;     /* _PyConfigInitEnum value */
 
     int isolated;         /* Isolated mode? see PyPreConfig.isolated */
diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h
index eb6490f5a87f..e9810bcf90a8 100644
--- a/Include/internal/pycore_initconfig.h
+++ b/Include/internal/pycore_initconfig.h
@@ -120,8 +120,8 @@ extern PyStatus _PyPreCmdline_Read(_PyPreCmdline *cmdline,
 
 /* --- PyPreConfig ----------------------------------------------- */
 
-PyAPI_FUNC(PyStatus) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig);
-extern PyStatus _PyPreConfig_InitFromConfig(
+PyAPI_FUNC(void) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig);
+extern void _PyPreConfig_InitFromConfig(
     PyPreConfig *preconfig,
     const PyConfig *config);
 extern PyStatus _PyPreConfig_InitFromPreConfig(
diff --git a/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst b/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst
deleted file mode 100644
index b797933d05d6..000000000000
--- a/Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst	
+++ /dev/null
@@ -1,3 +0,0 @@
-Add a new ``struct_size`` field to :c:type:`PyPreConfig` and :c:type:`PyConfig`
-structures to allow to modify these structures in the future without breaking
-the backward compatibility.
diff --git a/Modules/main.c b/Modules/main.c
index 13936add4f3e..66af89f56122 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -53,12 +53,7 @@ pymain_init(const _PyArgv *args)
 #endif
 
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
-
-    status = PyPreConfig_InitPythonConfig(&preconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
+    PyPreConfig_InitPythonConfig(&preconfig);
 
     status = _Py_PreInitializeFromPyArgv(&preconfig, args);
     if (_PyStatus_EXCEPTION(status)) {
@@ -66,7 +61,7 @@ pymain_init(const _PyArgv *args)
     }
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp
index 2656d188c250..758c8f73725e 100644
--- a/PC/python_uwp.cpp
+++ b/PC/python_uwp.cpp
@@ -165,12 +165,8 @@ int
 wmain(int argc, wchar_t **argv)
 {
     PyStatus status;
-
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
-
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     const wchar_t *moduleName = NULL;
     const wchar_t *p = wcsrchr(argv[0], L'\\');
@@ -189,10 +185,7 @@ wmain(int argc, wchar_t **argv)
         }
     }
 
-    status = PyPreConfig_InitPythonConfig(&preconfig);
-    if (PyStatus_Exception(status)) {
-        goto fail_without_config;
-    }
+    PyPreConfig_InitPythonConfig(&preconfig);
     if (!moduleName) {
         status = Py_PreInitializeFromArgs(&preconfig, argc, argv);
         if (PyStatus_Exception(status)) {
diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c
index 7c494c2786c0..74735f279c58 100644
--- a/Programs/_freeze_importlib.c
+++ b/Programs/_freeze_importlib.c
@@ -78,7 +78,6 @@ main(int argc, char *argv[])
 
     PyStatus status;
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitIsolatedConfig(&config);
     if (PyStatus_Exception(status)) {
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index c8600d58f0b4..d2d4c7f8c651 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -385,12 +385,7 @@ static int check_init_compat_config(int preinit)
 
     if (preinit) {
         PyPreConfig preconfig;
-        preconfig.struct_size = sizeof(PyPreConfig);
-
-        status = _PyPreConfig_InitCompatConfig(&preconfig);
-        if (PyStatus_Exception(status)) {
-            Py_ExitStatusException(status);
-        }
+        _PyPreConfig_InitCompatConfig(&preconfig);
 
         status = Py_PreInitialize(&preconfig);
         if (PyStatus_Exception(status)) {
@@ -399,7 +394,6 @@ static int check_init_compat_config(int preinit)
     }
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = _PyConfig_InitCompatConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -478,15 +472,8 @@ static int test_init_global_config(void)
 
 static int test_init_from_config(void)
 {
-    PyStatus status;
-
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
-
-    status = _PyPreConfig_InitCompatConfig(&preconfig);
-    if (PyStatus_Exception(status)) {
-        Py_ExitStatusException(status);
-    }
+    _PyPreConfig_InitCompatConfig(&preconfig);
 
     putenv("PYTHONMALLOC=malloc_debug");
     preconfig.allocator = PYMEM_ALLOCATOR_MALLOC;
@@ -495,13 +482,12 @@ static int test_init_from_config(void)
     Py_UTF8Mode = 0;
     preconfig.utf8_mode = 1;
 
-    status = Py_PreInitialize(&preconfig);
+    PyStatus status = Py_PreInitialize(&preconfig);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
     }
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = _PyConfig_InitCompatConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -638,7 +624,6 @@ static int check_init_parse_argv(int parse_argv)
     PyStatus status;
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -725,7 +710,6 @@ static int test_init_python_env(void)
     set_all_env_vars();
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -780,7 +764,6 @@ static int test_init_isolated_flag(void)
 
     /* Test PyConfig.isolated=1 */
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -803,25 +786,17 @@ static int test_init_isolated_flag(void)
 /* PyPreConfig.isolated=1, PyConfig.isolated=0 */
 static int test_preinit_isolated1(void)
 {
-    PyStatus status;
-
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
-
-    status = _PyPreConfig_InitCompatConfig(&preconfig);
-    if (PyStatus_Exception(status)) {
-        Py_ExitStatusException(status);
-    }
+    _PyPreConfig_InitCompatConfig(&preconfig);
 
     preconfig.isolated = 1;
 
-    status = Py_PreInitialize(&preconfig);
+    PyStatus status = Py_PreInitialize(&preconfig);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
     }
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = _PyConfig_InitCompatConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -840,26 +815,19 @@ static int test_preinit_isolated1(void)
 /* PyPreConfig.isolated=0, PyConfig.isolated=1 */
 static int test_preinit_isolated2(void)
 {
-    PyStatus status;
-
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
-
-    status = _PyPreConfig_InitCompatConfig(&preconfig);
-    if (PyStatus_Exception(status)) {
-        Py_ExitStatusException(status);
-    }
+    _PyPreConfig_InitCompatConfig(&preconfig);
 
     preconfig.isolated = 0;
 
-    status = Py_PreInitialize(&preconfig);
+    PyStatus status = Py_PreInitialize(&preconfig);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
     }
 
     /* Test PyConfig.isolated=1 */
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
+
     status = _PyConfig_InitCompatConfig(&config);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
@@ -880,15 +848,8 @@ static int test_preinit_isolated2(void)
 
 static int test_preinit_dont_parse_argv(void)
 {
-    PyStatus status;
-
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
-
-    status = PyPreConfig_InitIsolatedConfig(&preconfig);
-    if (PyStatus_Exception(status)) {
-        Py_ExitStatusException(status);
-    }
+    PyPreConfig_InitIsolatedConfig(&preconfig);
 
     preconfig.isolated = 0;
 
@@ -899,13 +860,13 @@ static int test_preinit_dont_parse_argv(void)
                        L"-X", L"dev",
                        L"-X", L"utf8",
                        L"script.py"};
-    status = Py_PreInitializeFromArgs(&preconfig, Py_ARRAY_LENGTH(argv), argv);
+    PyStatus status = Py_PreInitializeFromArgs(&preconfig,
+                                               Py_ARRAY_LENGTH(argv), argv);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
     }
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitIsolatedConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -931,7 +892,6 @@ static int test_preinit_parse_argv(void)
 {
     PyStatus status;
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -989,12 +949,7 @@ static int check_preinit_isolated_config(int preinit)
 
     if (preinit) {
         PyPreConfig preconfig;
-        preconfig.struct_size = sizeof(PyPreConfig);
-
-        status = PyPreConfig_InitIsolatedConfig(&preconfig);
-        if (PyStatus_Exception(status)) {
-            Py_ExitStatusException(status);
-        }
+        PyPreConfig_InitIsolatedConfig(&preconfig);
 
         status = Py_PreInitialize(&preconfig);
         if (PyStatus_Exception(status)) {
@@ -1007,7 +962,6 @@ static int check_preinit_isolated_config(int preinit)
     }
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitIsolatedConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1058,12 +1012,7 @@ static int check_init_python_config(int preinit)
 
     if (preinit) {
         PyPreConfig preconfig;
-        preconfig.struct_size = sizeof(PyPreConfig);
-
-        status = PyPreConfig_InitPythonConfig(&preconfig);
-        if (PyStatus_Exception(status)) {
-            Py_ExitStatusException(status);
-        }
+        PyPreConfig_InitPythonConfig(&preconfig);
 
         status = Py_PreInitialize(&preconfig);
         if (PyStatus_Exception(status)) {
@@ -1072,7 +1021,6 @@ static int check_init_python_config(int preinit)
     }
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1101,27 +1049,19 @@ static int test_init_python_config(void)
 
 static int test_init_dont_configure_locale(void)
 {
-    PyStatus status;
-
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
-
-    status = PyPreConfig_InitPythonConfig(&preconfig);
-    if (PyStatus_Exception(status)) {
-        Py_ExitStatusException(status);
-    }
+    PyPreConfig_InitPythonConfig(&preconfig);
 
     preconfig.configure_locale = 0;
     preconfig.coerce_c_locale = 1;
     preconfig.coerce_c_locale_warn = 1;
 
-    status = Py_PreInitialize(&preconfig);
+    PyStatus status = Py_PreInitialize(&preconfig);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
     }
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1140,7 +1080,6 @@ static int test_init_dev_mode(void)
 {
     PyStatus status;
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1365,7 +1304,6 @@ static int run_audit_run_test(int argc, wchar_t **argv, void *test)
 {
     PyStatus status;
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1415,7 +1353,6 @@ static int test_init_read_set(void)
 {
     PyStatus status;
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1465,7 +1402,6 @@ static int test_init_sys_add(void)
     PySys_AddWarnOption(L"ignore:::sysadd_warnoption");
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     PyStatus status;
     status = PyConfig_InitPythonConfig(&config);
@@ -1533,18 +1469,12 @@ static int test_init_setpath(void)
 
 static int test_init_setpath_config(void)
 {
-    PyStatus status;
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
-
-    status = PyPreConfig_InitPythonConfig(&preconfig);
-    if (PyStatus_Exception(status)) {
-        Py_ExitStatusException(status);
-    }
+    PyPreConfig_InitPythonConfig(&preconfig);
 
     /* Explicitly preinitializes with Python preconfiguration to avoid
       Py_SetPath() implicit preinitialization with compat preconfiguration. */
-    status = Py_PreInitialize(&preconfig);
+    PyStatus status = Py_PreInitialize(&preconfig);
     if (PyStatus_Exception(status)) {
         Py_ExitStatusException(status);
     }
@@ -1564,7 +1494,6 @@ static int test_init_setpath_config(void)
     putenv("TESTPATH=");
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1612,7 +1541,6 @@ static int test_init_warnoptions(void)
     PySys_AddWarnOption(L"ignore:::PySys_AddWarnOption2");
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1680,7 +1608,6 @@ static int test_init_run_main(void)
 {
     PyStatus status;
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1697,7 +1624,6 @@ static int test_init_main(void)
 {
     PyStatus status;
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
@@ -1729,7 +1655,6 @@ static int test_run_main(void)
 {
     PyStatus status;
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
diff --git a/Python/frozenmain.c b/Python/frozenmain.c
index 76309e9e5da2..c56938ab4899 100644
--- a/Python/frozenmain.c
+++ b/Python/frozenmain.c
@@ -40,7 +40,6 @@ Py_FrozenMain(int argc, char **argv)
     }
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
     status = PyConfig_InitPythonConfig(&config);
     if (PyStatus_Exception(status)) {
         PyConfig_Clear(&config);
diff --git a/Python/initconfig.c b/Python/initconfig.c
index fb04b4a2d22b..9164c7cfc8e8 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -529,17 +529,6 @@ Py_GetArgcArgv(int *argc, wchar_t ***argv)
      : _PyStatus_NO_MEMORY())
 
 
-static PyStatus
-config_check_struct_size(const PyConfig *config)
-{
-    if (config->struct_size != sizeof(PyConfig)) {
-        return _PyStatus_ERR("unsupported PyConfig structure size "
-                             "(Python version mismatch?)");
-    }
-    return _PyStatus_OK();
-}
-
-
 /* Free memory allocated in config, but don't clear all attributes */
 void
 PyConfig_Clear(PyConfig *config)
@@ -583,15 +572,7 @@ PyConfig_Clear(PyConfig *config)
 PyStatus
 _PyConfig_InitCompatConfig(PyConfig *config)
 {
-    size_t struct_size = config->struct_size;
     memset(config, 0, sizeof(*config));
-    config->struct_size = struct_size;
-
-    PyStatus status = config_check_struct_size(config);
-    if (_PyStatus_EXCEPTION(status)) {
-        _PyStatus_UPDATE_FUNC(status);
-        return status;
-    }
 
     config->_config_init = (int)_PyConfig_INIT_COMPAT;
     config->isolated = -1;
@@ -775,18 +756,6 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
 {
     PyStatus status;
 
-    status = config_check_struct_size(config);
-    if (_PyStatus_EXCEPTION(status)) {
-        _PyStatus_UPDATE_FUNC(status);
-        return status;
-    }
-
-    status = config_check_struct_size(config2);
-    if (_PyStatus_EXCEPTION(status)) {
-        _PyStatus_UPDATE_FUNC(status);
-        return status;
-    }
-
     PyConfig_Clear(config);
 
 #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
@@ -2275,7 +2244,6 @@ core_read_precmdline(PyConfig *config, _PyPreCmdline *precmdline)
     }
 
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
 
     status = _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig);
     if (_PyStatus_EXCEPTION(status)) {
@@ -2428,12 +2396,6 @@ PyConfig_Read(PyConfig *config)
     PyStatus status;
     PyWideStringList orig_argv = _PyWideStringList_INIT;
 
-    status = config_check_struct_size(config);
-    if (_PyStatus_EXCEPTION(status)) {
-        _PyStatus_UPDATE_FUNC(status);
-        return status;
-    }
-
     status = _Py_PreInitializeFromConfig(config, NULL);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index 41ef7d28d066..ff4bbf2a97eb 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -434,7 +434,6 @@ pathconfig_global_read(_PyPathConfig *pathconfig)
 {
     PyStatus status;
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = _PyConfig_InitCompatConfig(&config);
     if (_PyStatus_EXCEPTION(status)) {
diff --git a/Python/preconfig.c b/Python/preconfig.c
index 01c72f5d6bad..89a6227fa672 100644
--- a/Python/preconfig.c
+++ b/Python/preconfig.c
@@ -269,29 +269,10 @@ _PyPreCmdline_Read(_PyPreCmdline *cmdline, const PyPreConfig *preconfig)
 /* --- PyPreConfig ----------------------------------------------- */
 
 
-static PyStatus
-preconfig_check_struct_size(PyPreConfig *config)
-{
-    if (config->struct_size != sizeof(PyPreConfig)) {
-        return _PyStatus_ERR("unsupported PyPreConfig structure size "
-                             "(Python version mismatch?)");
-    }
-    return _PyStatus_OK();
-}
-
-
-PyStatus
+void
 _PyPreConfig_InitCompatConfig(PyPreConfig *config)
 {
-    size_t struct_size = config->struct_size;
     memset(config, 0, sizeof(*config));
-    config->struct_size = struct_size;
-
-    PyStatus status = preconfig_check_struct_size(config);
-    if (_PyStatus_EXCEPTION(status)) {
-        _PyStatus_UPDATE_FUNC(status);
-        return status;
-    }
 
     config->_config_init = (int)_PyConfig_INIT_COMPAT;
     config->parse_argv = 0;
@@ -313,18 +294,13 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_fs_encoding = -1;
 #endif
-    return _PyStatus_OK();
 }
 
 
-PyStatus
+void
 PyPreConfig_InitPythonConfig(PyPreConfig *config)
 {
-    PyStatus status = _PyPreConfig_InitCompatConfig(config);
-    if (_PyStatus_EXCEPTION(status)) {
-        _PyStatus_UPDATE_FUNC(status);
-        return status;
-    }
+    _PyPreConfig_InitCompatConfig(config);
 
     config->_config_init = (int)_PyConfig_INIT_PYTHON;
     config->isolated = 0;
@@ -339,18 +315,13 @@ PyPreConfig_InitPythonConfig(PyPreConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_fs_encoding = 0;
 #endif
-    return _PyStatus_OK();
 }
 
 
-PyStatus
+void
 PyPreConfig_InitIsolatedConfig(PyPreConfig *config)
 {
-    PyStatus status = _PyPreConfig_InitCompatConfig(config);
-    if (_PyStatus_EXCEPTION(status)) {
-        _PyStatus_UPDATE_FUNC(status);
-        return status;
-    }
+    _PyPreConfig_InitCompatConfig(config);
 
     config->_config_init = (int)_PyConfig_INIT_ISOLATED;
     config->configure_locale = 0;
@@ -361,7 +332,6 @@ PyPreConfig_InitIsolatedConfig(PyPreConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_fs_encoding = 0;
 #endif
-    return _PyStatus_OK();
 }
 
 
@@ -369,47 +339,35 @@ PyStatus
 _PyPreConfig_InitFromPreConfig(PyPreConfig *config,
                                const PyPreConfig *config2)
 {
-    PyStatus status = PyPreConfig_InitPythonConfig(config);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-
+    PyPreConfig_InitPythonConfig(config);
     preconfig_copy(config, config2);
     return _PyStatus_OK();
 }
 
 
-PyStatus
+void
 _PyPreConfig_InitFromConfig(PyPreConfig *preconfig, const PyConfig *config)
 {
-    PyStatus status;
     _PyConfigInitEnum config_init = (_PyConfigInitEnum)config->_config_init;
     switch (config_init) {
     case _PyConfig_INIT_PYTHON:
-        status = PyPreConfig_InitPythonConfig(preconfig);
+        PyPreConfig_InitPythonConfig(preconfig);
         break;
     case _PyConfig_INIT_ISOLATED:
-        status = PyPreConfig_InitIsolatedConfig(preconfig);
+        PyPreConfig_InitIsolatedConfig(preconfig);
         break;
     case _PyConfig_INIT_COMPAT:
     default:
-        status = _PyPreConfig_InitCompatConfig(preconfig);
-    }
-
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+        _PyPreConfig_InitCompatConfig(preconfig);
     }
 
     _PyPreConfig_GetConfig(preconfig, config);
-    return _PyStatus_OK();
 }
 
 
 static void
 preconfig_copy(PyPreConfig *config, const PyPreConfig *config2)
 {
-    assert(config->struct_size == sizeof(PyPreConfig));
-
 #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
 
     COPY_ATTR(_config_init);
@@ -829,12 +787,6 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args)
         return status;
     }
 
-    status = preconfig_check_struct_size(config);
-    if (_PyStatus_EXCEPTION(status)) {
-        _PyStatus_UPDATE_FUNC(status);
-        return status;
-    }
-
     preconfig_get_global_vars(config);
 
     /* Copy LC_CTYPE locale, since it's modified later */
@@ -849,7 +801,6 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args)
 
     /* Save the config to be able to restore it if encodings change */
     PyPreConfig save_config;
-    save_config.struct_size = sizeof(PyPreConfig);
 
     status = _PyPreConfig_InitFromPreConfig(&save_config, config);
     if (_PyStatus_EXCEPTION(status)) {
@@ -976,7 +927,6 @@ PyStatus
 _PyPreConfig_Write(const PyPreConfig *src_config)
 {
     PyPreConfig config;
-    config.struct_size = sizeof(PyPreConfig);
 
     PyStatus status = _PyPreConfig_InitFromPreConfig(&config, src_config);
     if (_PyStatus_EXCEPTION(status)) {
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index c154d060ad8d..5174f749c68d 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -731,7 +731,6 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args)
     runtime->preinitializing = 1;
 
     PyPreConfig config;
-    config.struct_size = sizeof(PyPreConfig);
 
     status = _PyPreConfig_InitFromPreConfig(&config, src_config);
     if (_PyStatus_EXCEPTION(status)) {
@@ -795,12 +794,8 @@ _Py_PreInitializeFromConfig(const PyConfig *config,
     }
 
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
 
-    status = _PyPreConfig_InitFromConfig(&preconfig, config);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
+    _PyPreConfig_InitFromConfig(&preconfig, config);
 
     if (!config->parse_argv) {
         return Py_PreInitialize(&preconfig);
@@ -848,7 +843,6 @@ pyinit_core(_PyRuntimeState *runtime,
     }
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = _PyConfig_InitCompatConfig(&config);
     if (_PyStatus_EXCEPTION(status)) {
@@ -1076,7 +1070,6 @@ Py_InitializeEx(int install_sigs)
     }
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
 
     status = _PyConfig_InitCompatConfig(&config);
     if (_PyStatus_EXCEPTION(status)) {
diff --git a/Python/pystate.c b/Python/pystate.c
index f0662fa70657..098280d76dfc 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -61,11 +61,7 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime)
     _PyGC_Initialize(&runtime->gc);
     _PyEval_Initialize(&runtime->ceval);
 
-    runtime->preconfig.struct_size = sizeof(PyPreConfig);
-    PyStatus status = PyPreConfig_InitPythonConfig(&runtime->preconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
+    PyPreConfig_InitPythonConfig(&runtime->preconfig);
 
     runtime->gilstate.check_enabled = 1;
 
@@ -210,7 +206,6 @@ PyInterpreterState_New(void)
     interp->id_refcount = -1;
     interp->check_interval = 100;
 
-    interp->config.struct_size = sizeof(PyConfig);
     PyStatus status = PyConfig_InitPythonConfig(&interp->config);
     if (_PyStatus_EXCEPTION(status)) {
         /* Don't report status to caller: PyConfig_InitPythonConfig()


From webhook-mailer at python.org  Mon Sep 30 20:12:28 2019
From: webhook-mailer at python.org (Yury Selivanov)
Date: Tue, 01 Oct 2019 00:12:28 -0000
Subject: [Python-checkins] Fix and improve `asyncio.run()` docs (GH-16403)
Message-ID: <mailman.163.1569888749.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/e407013089259e4c0b271703e1975bbcd578a2d5
commit: e407013089259e4c0b271703e1975bbcd578a2d5
branch: master
author: Kyle Stanley <aeros167 at gmail.com>
committer: Yury Selivanov <yury at magic.io>
date: 2019-09-30T17:12:21-07:00
summary:

Fix and improve `asyncio.run()` docs (GH-16403)

files:
M Doc/glossary.rst
M Doc/library/asyncio-task.rst
M Lib/asyncio/runners.py

diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index e601e8b36984..9ce0357f1cb4 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -247,7 +247,7 @@ Glossary
       Fortran contiguous arrays, the first index varies the fastest.
 
    coroutine
-      Coroutines is a more generalized form of subroutines. Subroutines are
+      Coroutines are a more generalized form of subroutines. Subroutines are
       entered at one point and exited at another point.  Coroutines can be
       entered, exited, and resumed at many different points.  They can be
       implemented with the :keyword:`async def` statement.  See also
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index 9f55a3534a5e..155887a3ab3a 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -212,6 +212,8 @@ Running an asyncio Program
 
 .. function:: run(coro, \*, debug=False)
 
+    Execute the :term:`coroutine` *coro* and return the result.
+
     This function runs the passed coroutine, taking care of
     managing the asyncio event loop, *finalizing asynchronous
     generators*, and closing the threadpool.
@@ -225,10 +227,6 @@ Running an asyncio Program
     the end.  It should be used as a main entry point for asyncio
     programs, and should ideally only be called once.
 
-    Return a result of *coro* execution, or raise a :exc:`RuntimeError`
-    if ``asyncio.run()`` is called from a running event loop, or a
-    :exc:`ValueError` if *coro* is not a courutine.
-
     Example::
 
         async def main():
diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py
index 904102bf82be..03ce33300eba 100644
--- a/Lib/asyncio/runners.py
+++ b/Lib/asyncio/runners.py
@@ -6,7 +6,7 @@
 
 
 def run(main, *, debug=False):
-    """Run a coroutine.
+    """Execute the coroutine and return the result.
 
     This function runs the passed coroutine, taking care of
     managing the asyncio event loop and finalizing asynchronous
@@ -21,10 +21,6 @@ def run(main, *, debug=False):
     It should be used as a main entry point for asyncio programs, and should
     ideally only be called once.
 
-    Return a result of *coro* execution, or raise a RuntimeError
-    if `asyncio.run()`is called from a running event loop, or a ValueError
-    if `main` is not a courutine.
-
     Example:
 
         async def main():


From webhook-mailer at python.org  Mon Sep 30 21:46:48 2019
From: webhook-mailer at python.org (Yury Selivanov)
Date: Tue, 01 Oct 2019 01:46:48 -0000
Subject: [Python-checkins] Fix and improve `asyncio.run()` docs (GH-16403)
 (GH-16504)
Message-ID: <mailman.164.1569894409.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/2f644c0dc951868181e380f3d1cb4854e3a3152e
commit: 2f644c0dc951868181e380f3d1cb4854e3a3152e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Yury Selivanov <yury at magic.io>
date: 2019-09-30T18:46:43-07:00
summary:

Fix and improve `asyncio.run()` docs (GH-16403) (GH-16504)

(cherry picked from commit e407013089259e4c0b271703e1975bbcd578a2d5)

Co-authored-by: Kyle Stanley <aeros167 at gmail.com>

files:
M Doc/glossary.rst
M Doc/library/asyncio-task.rst
M Lib/asyncio/runners.py

diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index e601e8b36984..9ce0357f1cb4 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -247,7 +247,7 @@ Glossary
       Fortran contiguous arrays, the first index varies the fastest.
 
    coroutine
-      Coroutines is a more generalized form of subroutines. Subroutines are
+      Coroutines are a more generalized form of subroutines. Subroutines are
       entered at one point and exited at another point.  Coroutines can be
       entered, exited, and resumed at many different points.  They can be
       implemented with the :keyword:`async def` statement.  See also
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index 321b4f457659..682923379391 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -212,6 +212,8 @@ Running an asyncio Program
 
 .. function:: run(coro, \*, debug=False)
 
+    Execute the :term:`coroutine` *coro* and return the result.
+
     This function runs the passed coroutine, taking care of
     managing the asyncio event loop and *finalizing asynchronous
     generators*.
@@ -225,10 +227,6 @@ Running an asyncio Program
     the end.  It should be used as a main entry point for asyncio
     programs, and should ideally only be called once.
 
-    Return a result of *coro* execution, or raise a :exc:`RuntimeError`
-    if ``asyncio.run()`` is called from a running event loop, or a
-    :exc:`ValueError` if *coro* is not a courutine.
-
     Example::
 
         async def main():
diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py
index df686386b8fd..2e37e18b45fc 100644
--- a/Lib/asyncio/runners.py
+++ b/Lib/asyncio/runners.py
@@ -6,7 +6,7 @@
 
 
 def run(main, *, debug=False):
-    """Run a coroutine.
+    """Execute the coroutine and return the result.
 
     This function runs the passed coroutine, taking care of
     managing the asyncio event loop and finalizing asynchronous
@@ -21,10 +21,6 @@ def run(main, *, debug=False):
     It should be used as a main entry point for asyncio programs, and should
     ideally only be called once.
 
-    Return a result of *coro* execution, or raise a RuntimeError
-    if `asyncio.run()`is called from a running event loop, or a ValueError
-    if `main` is not a courutine.
-
     Example:
 
         async def main():


From webhook-mailer at python.org  Mon Sep 30 21:47:17 2019
From: webhook-mailer at python.org (Yury Selivanov)
Date: Tue, 01 Oct 2019 01:47:17 -0000
Subject: [Python-checkins] Fix and improve `asyncio.run()` docs (GH-16403)
 (GH-16505)
Message-ID: <mailman.165.1569894438.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/2eb1dac46e256e7f9be71021e4630c0a3e9b4b03
commit: 2eb1dac46e256e7f9be71021e4630c0a3e9b4b03
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Yury Selivanov <yury at magic.io>
date: 2019-09-30T18:47:13-07:00
summary:

Fix and improve `asyncio.run()` docs (GH-16403) (GH-16505)

(cherry picked from commit e407013089259e4c0b271703e1975bbcd578a2d5)

Co-authored-by: Kyle Stanley <aeros167 at gmail.com>

files:
M Doc/glossary.rst
M Doc/library/asyncio-task.rst
M Lib/asyncio/runners.py

diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 8af4373b11d7..ed588612aeab 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -243,7 +243,7 @@ Glossary
       Fortran contiguous arrays, the first index varies the fastest.
 
    coroutine
-      Coroutines is a more generalized form of subroutines. Subroutines are
+      Coroutines are a more generalized form of subroutines. Subroutines are
       entered at one point and exited at another point.  Coroutines can be
       entered, exited, and resumed at many different points.  They can be
       implemented with the :keyword:`async def` statement.  See also
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index a9f6f9cfc4b6..9a4cda188788 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -212,6 +212,8 @@ Running an asyncio Program
 
 .. function:: run(coro, \*, debug=False)
 
+    Execute the :term:`coroutine` *coro* and return the result.
+
     This function runs the passed coroutine, taking care of
     managing the asyncio event loop and *finalizing asynchronous
     generators*.
@@ -225,10 +227,6 @@ Running an asyncio Program
     the end.  It should be used as a main entry point for asyncio
     programs, and should ideally only be called once.
 
-    Return a result of *coro* execution, or raise a :exc:`RuntimeError`
-    if ``asyncio.run()`` is called from a running event loop, or a
-    :exc:`ValueError` if *coro* is not a courutine.
-
     Example::
 
         async def main():
diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py
index df686386b8fd..2e37e18b45fc 100644
--- a/Lib/asyncio/runners.py
+++ b/Lib/asyncio/runners.py
@@ -6,7 +6,7 @@
 
 
 def run(main, *, debug=False):
-    """Run a coroutine.
+    """Execute the coroutine and return the result.
 
     This function runs the passed coroutine, taking care of
     managing the asyncio event loop and finalizing asynchronous
@@ -21,10 +21,6 @@ def run(main, *, debug=False):
     It should be used as a main entry point for asyncio programs, and should
     ideally only be called once.
 
-    Return a result of *coro* execution, or raise a RuntimeError
-    if `asyncio.run()`is called from a running event loop, or a ValueError
-    if `main` is not a courutine.
-
     Example:
 
         async def main():


From webhook-mailer at python.org  Mon Sep 30 22:41:29 2019
From: webhook-mailer at python.org (Miss Islington (bot))
Date: Tue, 01 Oct 2019 02:41:29 -0000
Subject: [Python-checkins] bpo-32689: Updates shutil.move to allow for Path
 objects to be used as source arg (GH-15326)
Message-ID: <mailman.166.1569897690.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/cf57cabef82c4689ce9796bb1fcdb125fa05efcb
commit: cf57cabef82c4689ce9796bb1fcdb125fa05efcb
branch: master
author: Maxwell A McKinnon <maxwell.mckinnon at maximintegrated.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-09-30T19:41:16-07:00
summary:

bpo-32689: Updates shutil.move to allow for Path objects to be used as source arg (GH-15326)



Important work originally done by @emilyemorehouse two years ago and nearly ready to go in.

This bug has affected many people and in some cases has been a dealbreaker to the adoption of the otherwise wonderful pathlib and PEP519. https://stackoverflow.com/questions/33625931/copy-file-with-pathlib-in-python.

This adds the outstanding test request from that PR @vstinner (https://github.com/python/cpython/pull/5393).

Test fails without the change, passes with it, along with every other test in test_shutil.

Some variants were experimented with to make the one line change and the most performant one was picked.


# Added Test for PathLike directory destination, the current fail case

```
Lib/test/test_shutil.py::TestMove::test_move_file_pathlike FAILED                                                               [100%]

============================================================== FAILURES ===============================================================
__________________________________________________ TestMove.test_move_file_pathlike ___________________________________________________

self = <test.test_shutil.TestMove testMethod=test_move_file_pathlike>

    def test_move_file_pathlike(self):
        # Move a file to another location on the same filesystem.
        src = pathlib.Path(self.src_file)
>       self._check_move_file(src, self.dst_dir, self.dst_file)

Lib/test/test_shutil.py:1563:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
Lib/test/test_shutil.py:1545: in _check_move_file
    shutil.move(src, dst)
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/shutil.py:562: in move
    real_dst = os.path.join(dst, _basename(src))
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

path = PosixPath('/var/folders/r2/psq74t5x3nbfzlph8bh2pvdw0000gn/T/tmp9ie0wh9_/foo')

    def _basename(path):
        # A basename() variant which first strips the trailing slash, if present.
        # Thus we always get the last component of the path, even for directories.
        sep = os.path.sep + (os.path.altsep or '')
>       return os.path.basename(path.rstrip(sep))
E       AttributeError: 'PosixPath' object has no attribute 'rstrip'

/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/shutil.py:526: AttributeError
============================================== 1 failed, 102 deselected in 0.30 seconds ===============================================
```

After change:

```
========================================================= test session starts =========================================================
platform darwin -- Python 3.7.4, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- /Users/maxwellmckinnon/.venvs/TA3.7/bin/python3.7
cachedir: .pytest_cache
rootdir: /Users/maxwellmckinnon/dev/cpython
plugins: cov-2.7.1, mock-1.10.4
collected 103 items / 102 deselected / 1 selected

Lib/test/test_shutil.py::TestMove::test_move_file_pathlike PASSED                                                               [100%]

============================================== 1 passed, 102 deselected in 0.06 seconds ===============================================
```

Running all the tests in test_shutil.py
```
?? pytest Lib/test/test_shutil.py -v
========================================================= test session starts =========================================================
platform darwin -- Python 3.7.4, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- /Users/maxwellmckinnon/.venvs/TA3.7/bin/python3.7
cachedir: .pytest_cache
rootdir: /Users/maxwellmckinnon/dev/cpython
plugins: cov-2.7.1, mock-1.10.4
collected 103 items

Lib/test/test_shutil.py::TestShutil::test_chown PASSED                                                                          [  0%]
Lib/test/test_shutil.py::TestShutil::test_copy PASSED                                                                           [  1%]
...
Lib/test/test_shutil.py::TermsizeTests::test_stty_match SKIPPED                                                                 [ 99%]
Lib/test/test_shutil.py::PublicAPITests::test_module_all_attribute PASSED                                                       [100%]

================================================ 96 passed, 7 skipped in 1.25 seconds =================================================
```

# Performance Considerations
Is it considered poor form to get rid of _basename altogether and make use of pathlib in the move function? I'm not sure if the idea is for all these modules to strictly avoid circular dependencies. They are already using os.path which is just as much a citizen in 3.8 as pathlib right?

e.g.

`real_dst = os.path.join(dst, _basename(src))`
becomes
`real_dst = Path(dst) / Path(src).name`

I've looked around and familiarized myself, and I now think importing pathlib here is fine. My only remaining concern is that of performance.

Here's the performance difference for this step. 

```
In [46]: %timeit real_dst = os.path.join("a/b/c", _basename('b/'))
2.71 ?s ? 62.6 ns per loop (mean ? std. dev. of 7 runs, 100000 loops each)

In [47]: %timeit real_dst = Path("a/b/c") / Path('b/').name
12.4 ?s ? 65.3 ns per loop (mean ? std. dev. of 7 runs, 100000 loops each)
```

Is 10us significant or insignificant compared to the least expensive operation this function will do? I don't know. Let's find out.

```
In [55]: %timeit os.rename('/tmp/a/a.txt', '/tmp/a/b.txt'); os.rename('/tmp/a/b.txt', '/tmp/a/a.txt')
124 ?s ? 2.18 ?s per loop (mean ? std. dev. of 7 runs, 10000 loops each)
```
62us to rename. 10us seems significant enough that we wouldn't want to favor the Path sugar suggestion. 16% speed decrease from adding the 10us.

What do people think? I was hoping to get to use pathlib.Path here, but I suspect for this low level move, it should be as fast as possible, and 16% is not worth one line of sugary code to me.



https://bugs.python.org/issue32689



Automerge-Triggered-By: @gvanrossum

files:
A Misc/NEWS.d/next/Library/2018-02-13-17-58-30.bpo-32689.a-3SnH.rst
M Doc/library/shutil.rst
M Lib/shutil.py
M Lib/test/test_shutil.py

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index 174b7e875a38..59390d0e907e 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -355,6 +355,9 @@ Directory and files operations
       copy the file more efficiently. See
       :ref:`shutil-platform-dependent-efficient-copy-operations` section.
 
+   .. versionchanged:: 3.9
+      Accepts a :term:`path-like object` for both *src* and *dst*.
+
 .. function:: disk_usage(path)
 
    Return disk usage statistics about the given path as a :term:`named tuple`
diff --git a/Lib/shutil.py b/Lib/shutil.py
index 5c1255a6713d..f0d03366368e 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -730,8 +730,20 @@ def onerror(*args):
 rmtree.avoids_symlink_attacks = _use_fd_functions
 
 def _basename(path):
-    # A basename() variant which first strips the trailing slash, if present.
-    # Thus we always get the last component of the path, even for directories.
+    """A basename() variant which first strips the trailing slash, if present.
+    Thus we always get the last component of the path, even for directories.
+
+    path: Union[PathLike, str]
+
+    e.g.
+    >>> os.path.basename('/bar/foo')
+    'foo'
+    >>> os.path.basename('/bar/foo/')
+    ''
+    >>> _basename('/bar/foo/')
+    'foo'
+    """
+    path = os.fspath(path)
     sep = os.path.sep + (os.path.altsep or '')
     return os.path.basename(path.rstrip(sep))
 
@@ -769,7 +781,10 @@ def move(src, dst, copy_function=copy2):
             os.rename(src, dst)
             return
 
+        # Using _basename instead of os.path.basename is important, as we must
+        # ignore any trailing slash to avoid the basename returning ''
         real_dst = os.path.join(dst, _basename(src))
+
         if os.path.exists(real_dst):
             raise Error("Destination path '%s' already exists" % real_dst)
     try:
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index ab0f96d25135..428d4f341715 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1835,6 +1835,16 @@ def test_move_file_to_dir(self):
         # Move a file inside an existing dir on the same filesystem.
         self._check_move_file(self.src_file, self.dst_dir, self.dst_file)
 
+    def test_move_file_to_dir_pathlike_src(self):
+        # Move a pathlike file to another location on the same filesystem.
+        src = pathlib.Path(self.src_file)
+        self._check_move_file(src, self.dst_dir, self.dst_file)
+
+    def test_move_file_to_dir_pathlike_dst(self):
+        # Move a file to another pathlike location on the same filesystem.
+        dst = pathlib.Path(self.dst_dir)
+        self._check_move_file(self.src_file, dst, self.dst_file)
+
     @mock_rename
     def test_move_file_other_fs(self):
         # Move a file to an existing dir on another filesystem.
diff --git a/Misc/NEWS.d/next/Library/2018-02-13-17-58-30.bpo-32689.a-3SnH.rst b/Misc/NEWS.d/next/Library/2018-02-13-17-58-30.bpo-32689.a-3SnH.rst
new file mode 100644
index 000000000000..d435351a3a75
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-02-13-17-58-30.bpo-32689.a-3SnH.rst
@@ -0,0 +1,2 @@
+Update :func:`shutil.move` function to allow for Path objects to be used as
+source argument. Patch by Emily Morehouse and Maxwell "5.13b" McKinnon.


From webhook-mailer at python.org  Mon Sep 30 23:40:59 2019
From: webhook-mailer at python.org (Giampaolo Rodola)
Date: Tue, 01 Oct 2019 03:40:59 -0000
Subject: [Python-checkins] bpo-38319: Fix shutil._fastcopy_sendfile(): set
 sendfile() max block size (GH-16491)
Message-ID: <mailman.167.1569901260.32504.python-checkins@python.org>

https://github.com/python/cpython/commit/94e165096fd65e8237e60de570fb609604ab94c9
commit: 94e165096fd65e8237e60de570fb609604ab94c9
branch: master
author: Giampaolo Rodola <g.rodola at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-10-01T11:40:54+08:00
summary:

bpo-38319: Fix shutil._fastcopy_sendfile(): set sendfile() max block size (GH-16491)

files:
A Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst
M Lib/shutil.py
M Lib/socket.py

diff --git a/Lib/shutil.py b/Lib/shutil.py
index f0d03366368e..f97de788d9d4 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -135,9 +135,13 @@ def _fastcopy_sendfile(fsrc, fdst):
     # should not make any difference, also in case the file content
     # changes while being copied.
     try:
-        blocksize = max(os.fstat(infd).st_size, 2 ** 23)  # min 8MB
-    except Exception:
-        blocksize = 2 ** 27  # 128MB
+        blocksize = max(os.fstat(infd).st_size, 2 ** 23)  # min 8MiB
+    except OSError:
+        blocksize = 2 ** 27  # 128MiB
+    # On 32-bit architectures truncate to 1GiB to avoid OverflowError,
+    # see bpo-38319.
+    if sys.maxsize < 2 ** 32:
+        blocksize = min(blocksize, 2 ** 30)
 
     offset = 0
     while True:
diff --git a/Lib/socket.py b/Lib/socket.py
index e5989d9dfd40..84a5dcb0daf2 100755
--- a/Lib/socket.py
+++ b/Lib/socket.py
@@ -356,8 +356,8 @@ def _sendfile_use_sendfile(self, file, offset=0, count=None):
                 raise _GiveupOnSendfile(err)  # not a regular file
             if not fsize:
                 return 0  # empty file
-            blocksize = fsize if not count else count
-
+            # Truncate to 1GiB to avoid OverflowError, see bpo-38319.
+            blocksize = min(count or fsize, 2 ** 30)
             timeout = self.gettimeout()
             if timeout == 0:
                 raise ValueError("non-blocking sockets are not supported")
diff --git a/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst b/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst
new file mode 100644
index 000000000000..376a9e459de2
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst
@@ -0,0 +1,2 @@
+sendfile() used in socket and shutil modules was raising OverflowError for
+files >= 2GiB on 32-bit architectures.  (patch by Giampaolo Rodola)


From webhook-mailer at python.org  Mon Sep  9 10:52:38 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Mon, 09 Sep 2019 14:52:38 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.490.1599366859.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 bpo-36797: Fix a dead link in  Doc/distutils/apiref (GH-15700) (GH-15704)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/cabcbbe7a518d53cd7dbb915ceccf4970c2d=
0ea5
commit: cabcbbe7a518d53cd7dbb915ceccf4970c2d0ea5
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-09T16:52:34+02:00
summary:

bpo-36797: Fix a dead link in  Doc/distutils/apiref (GH-15700) (GH-15704)

https://bugs.python.org/issue36797
(cherry picked from commit 2c2b561967c1916855399f809e30ae0ba7e09ae2)

Co-authored-by: Miro Hron=C4=8Dok <miro at hroncok.cz>

files:
A Misc/NEWS.d/next/Documentation/2019-09-05-14-47-51.bpo-36797.KN9Ga5.rst
M Doc/distutils/apiref.rst

diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst
index 937f19f57be9..a42d2d3559f2 100644
--- a/Doc/distutils/apiref.rst
+++ b/Doc/distutils/apiref.rst
@@ -6,12 +6,12 @@ API Reference
=20
 .. seealso::
=20
-   `New and changed setup.py arguments in setuptools <setuptools-setup-py>`_
+   `New and changed setup.py arguments in setuptools`_
       The ``setuptools`` project adds new capabilities to the ``setup`` func=
tion
       and other APIs, makes the API consistent across different Python versi=
ons,
       and is hence recommended over using ``distutils`` directly.
=20
-.. _setuptools-setup-py: https://setuptools.readthedocs.io/en/latest/setupto=
ols.html#new-and-changed-setup-keywords
+.. _New and changed setup.py arguments in setuptools: https://setuptools.rea=
dthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords
=20
 .. include:: ./_setuptools_disclaimer.rst
=20
diff --git a/Misc/NEWS.d/next/Documentation/2019-09-05-14-47-51.bpo-36797.KN9=
Ga5.rst b/Misc/NEWS.d/next/Documentation/2019-09-05-14-47-51.bpo-36797.KN9Ga5=
.rst
new file mode 100644
index 000000000000..62a3c176b1c7
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-09-05-14-47-51.bpo-36797.KN9Ga5.rst
@@ -0,0 +1 @@
+Fix a dead link in the distutils API Reference.


From webhook-mailer at python.org  Mon Sep  9 11:00:47 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Mon, 09 Sep 2019 15:00:47 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.491.1599366862.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Minor changes in Doc/faq/library. (#15449)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/88b24f96aedbe546d7d3248089d94e874edc=
0e11
commit: 88b24f96aedbe546d7d3248089d94e874edc0e11
branch: master
author: Antoine <43954001+awecx at users.noreply.github.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-09T17:00:43+02:00
summary:

Minor changes in Doc/faq/library. (#15449)

* Minor changes.

* Update Doc/faq/library.rst

Co-Authored-By: Kyle Stanley <aeros167 at gmail.com>

* Apply suggestions from aeros167.

* Update Doc/faq/library.rst

Co-Authored-By: Kyle Stanley <aeros167 at gmail.com>

* Apply suggestions from aeros167 + re-add a "a" that was accidentally delete=
d.

files:
M Doc/faq/library.rst

diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst
index ab92a879a885..97058b5806a3 100644
--- a/Doc/faq/library.rst
+++ b/Doc/faq/library.rst
@@ -125,7 +125,7 @@ argument list.  It is called as ::
=20
    handler(signum, frame)
=20
-so it should be declared with two arguments::
+so it should be declared with two parameters::
=20
    def handler(signum, frame):
        ...
@@ -159,9 +159,9 @@ The "global main logic" of your program may be as simple =
as ::
=20
 at the bottom of the main module of your program.
=20
-Once your program is organized as a tractable collection of functions and cl=
ass
-behaviours you should write test functions that exercise the behaviours.  A =
test
-suite that automates a sequence of tests can be associated with each module.
+Once your program is organized as a tractable collection of function and cla=
ss
+behaviours, you should write test functions that exercise the behaviours.  A
+test suite that automates a sequence of tests can be associated with each mo=
dule.
 This sounds like a lot of work, but since Python is so terse and flexible it=
's
 surprisingly easy.  You can make coding much more pleasant and fun by writing
 your test functions in parallel with the "production code", since this makes=
 it
@@ -295,7 +295,7 @@ queue as there are threads.
 How do I parcel out work among a bunch of worker threads?
 ---------------------------------------------------------
=20
-The easiest way is to use the new :mod:`concurrent.futures` module,
+The easiest way is to use the :mod:`concurrent.futures` module,
 especially the :mod:`~concurrent.futures.ThreadPoolExecutor` class.
=20
 Or, if you want fine control over the dispatching algorithm, you can write
@@ -679,7 +679,7 @@ How can I mimic CGI form submission (METHOD=3DPOST)?
 I would like to retrieve web pages that are the result of POSTing a form. Is
 there existing code that would let me do this easily?
=20
-Yes. Here's a simple example that uses urllib.request::
+Yes. Here's a simple example that uses :mod:`urllib.request`::
=20
    #!/usr/local/bin/python
=20
@@ -765,20 +765,21 @@ The :mod:`select` module is commonly used to help with =
asynchronous I/O on
 sockets.
=20
 To prevent the TCP connect from blocking, you can set the socket to non-bloc=
king
-mode.  Then when you do the ``connect()``, you will either connect immediate=
ly
+mode.  Then when you do the :meth:`socket.connect`, you will either connect =
immediately
 (unlikely) or get an exception that contains the error number as ``.errno``.
 ``errno.EINPROGRESS`` indicates that the connection is in progress, but hasn=
't
 finished yet.  Different OSes will return different values, so you're going =
to
 have to check what's returned on your system.
=20
-You can use the ``connect_ex()`` method to avoid creating an exception.  It =
will
-just return the errno value.  To poll, you can call ``connect_ex()`` again l=
ater
+You can use the :meth:`socket.connect_ex` method to avoid creating an except=
ion.  It will
+just return the errno value.  To poll, you can call :meth:`socket.connect_ex=
` again later
 -- ``0`` or ``errno.EISCONN`` indicate that you're connected -- or you can p=
ass this
-socket to select to check if it's writable.
+socket to :meth:`select.select` to check if it's writable.
=20
 .. note::
-   The :mod:`asyncore` module presents a framework-like approach to the prob=
lem
-   of writing non-blocking networking code.
+   The :mod:`asyncio` module provides a general purpose single-threaded and
+   concurrent asynchronous library, which can be used for writing non-blocki=
ng
+   network code.
    The third-party `Twisted <https://twistedmatrix.com/trac/>`_ library is
    a popular and feature-rich alternative.
=20
@@ -832,8 +833,8 @@ There are also many other specialized generators in this =
module, such as:
=20
 Some higher-level functions operate on sequences directly, such as:
=20
-* ``choice(S)`` chooses random element from a given sequence
-* ``shuffle(L)`` shuffles a list in-place, i.e. permutes it randomly
+* ``choice(S)`` chooses a random element from a given sequence.
+* ``shuffle(L)`` shuffles a list in-place, i.e. permutes it randomly.
=20
 There's also a ``Random`` class you can instantiate to create independent
 multiple random number generators.


From webhook-mailer at python.org  Wed Sep 11 07:26:35 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 11:26:35 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.492.1599366864.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Minor ReST formatting fixes in subprocess docs (#14876)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/1a13efb7e05b545def26f29c954751fdb6b2=
2fa3
commit: 1a13efb7e05b545def26f29c954751fdb6b22fa3
branch: master
author: Tim Hoffmann <2836374+timhoffm at users.noreply.github.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T13:26:31+02:00
summary:

Minor ReST formatting fixes in subprocess docs (#14876)

files:
M Doc/library/subprocess.rst

diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
index 167ed9a6ead4..954e0fec1182 100644
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -1048,7 +1048,7 @@ calls these functions.
    Run the command described by *args*.  Wait for command to complete, then
    return the :attr:`~Popen.returncode` attribute.
=20
-   Code needing to capture stdout or stderr should use :func:`run` instead:
+   Code needing to capture stdout or stderr should use :func:`run` instead::
=20
        run(...).returncode
=20
@@ -1076,7 +1076,7 @@ calls these functions.
    :exc:`CalledProcessError` object will have the return code in the
    :attr:`~CalledProcessError.returncode` attribute.
=20
-   Code needing to capture stdout or stderr should use :func:`run` instead:
+   Code needing to capture stdout or stderr should use :func:`run` instead::
=20
        run(..., check=3DTrue)
=20
@@ -1198,8 +1198,8 @@ becomes::
    p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
    output =3D p2.communicate()[0]
=20
-The p1.stdout.close() call after starting the p2 is important in order for p1
-to receive a SIGPIPE if p2 exits before p1.
+The ``p1.stdout.close()`` call after starting the p2 is important in order f=
or
+p1 to receive a SIGPIPE if p2 exits before p1.
=20
 Alternatively, for trusted input, the shell's own pipeline support may still
 be used directly:


From webhook-mailer at python.org  Wed Sep 11 08:43:33 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 12:43:33 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.493.1599366865.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Fix typo in fastsearch comments (GH-14608)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/60bba83b5d9947fb3106325293e3a4e9c9cd=
ea7e
commit: 60bba83b5d9947fb3106325293e3a4e9c9cdea7e
branch: master
author: Valentin Haenel <esc at users.noreply.github.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T14:43:29+02:00
summary:

Doc: Fix typo in fastsearch comments (GH-14608)

files:
M Objects/stringlib/fastsearch.h

diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h
index 46fcf356d0d0..56a4467d3538 100644
--- a/Objects/stringlib/fastsearch.h
+++ b/Objects/stringlib/fastsearch.h
@@ -52,7 +52,7 @@ STRINGLIB(find_char)(const STRINGLIB_CHAR* s, Py_ssize_t n,=
 STRINGLIB_CHAR ch)
             return (p - s);
         return -1;
 #else
-        /* use memchr if we can choose a needle without two many likely
+        /* use memchr if we can choose a needle without too many likely
            false positives */
         const STRINGLIB_CHAR *s1, *e1;
         unsigned char needle =3D ch & 0xff;
@@ -111,7 +111,7 @@ STRINGLIB(rfind_char)(const STRINGLIB_CHAR* s, Py_ssize_t=
 n, STRINGLIB_CHAR ch)
             return (p - s);
         return -1;
 #else
-        /* use memrchr if we can choose a needle without two many likely
+        /* use memrchr if we can choose a needle without too many likely
            false positives */
         const STRINGLIB_CHAR *s1;
         Py_ssize_t n1;


From webhook-mailer at python.org  Wed Sep 11 09:01:22 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 13:01:22 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.494.1599366867.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Use walrus operator in example. (GH-15934)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/e1d455f3a3b82c2e08d5e133bcbab5a181b6=
6cfb
commit: e1d455f3a3b82c2e08d5e133bcbab5a181b66cfb
branch: master
author: Julien Palard <julien at palard.fr>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T15:01:18+02:00
summary:

Doc: Use walrus operator in example. (GH-15934)

files:
M Doc/library/http.client.rst

diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst
index 48bc35ca76cc..db26d56af30f 100644
--- a/Doc/library/http.client.rst
+++ b/Doc/library/http.client.rst
@@ -516,10 +516,7 @@ Here is an example session that uses the ``GET`` method::
    >>> # The following example demonstrates reading data in chunks.
    >>> conn.request("GET", "/")
    >>> r1 =3D conn.getresponse()
-   >>> while True:
-   ...     chunk =3D r1.read(200)  # 200 bytes
-   ...     if not chunk:
-   ...          break
+   >>> while chunk :=3D r1.read(200):
    ...     print(repr(chunk))
    b'<!doctype html>\n<!--[if"...
    ...


From webhook-mailer at python.org  Wed Sep 11 09:05:53 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 13:05:53 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.495.1599366868.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Fix typo in fastsearch comments (GH-14608) (GH-15933)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/6a17751589841346b09f5eab747ae52bbf93=
4348
commit: 6a17751589841346b09f5eab747ae52bbf934348
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T15:05:49+02:00
summary:

Doc: Fix typo in fastsearch comments (GH-14608) (GH-15933)

(cherry picked from commit 60bba83b5d9947fb3106325293e3a4e9c9cdea7e)

Co-authored-by: Valentin Haenel <esc at users.noreply.github.com>

files:
M Objects/stringlib/fastsearch.h

diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h
index 46fcf356d0d0..56a4467d3538 100644
--- a/Objects/stringlib/fastsearch.h
+++ b/Objects/stringlib/fastsearch.h
@@ -52,7 +52,7 @@ STRINGLIB(find_char)(const STRINGLIB_CHAR* s, Py_ssize_t n,=
 STRINGLIB_CHAR ch)
             return (p - s);
         return -1;
 #else
-        /* use memchr if we can choose a needle without two many likely
+        /* use memchr if we can choose a needle without too many likely
            false positives */
         const STRINGLIB_CHAR *s1, *e1;
         unsigned char needle =3D ch & 0xff;
@@ -111,7 +111,7 @@ STRINGLIB(rfind_char)(const STRINGLIB_CHAR* s, Py_ssize_t=
 n, STRINGLIB_CHAR ch)
             return (p - s);
         return -1;
 #else
-        /* use memrchr if we can choose a needle without two many likely
+        /* use memrchr if we can choose a needle without too many likely
            false positives */
         const STRINGLIB_CHAR *s1;
         Py_ssize_t n1;


From webhook-mailer at python.org  Wed Sep 11 09:17:52 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 13:17:52 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.496.1599366870.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: [3.8] Doc: Use walrus operator in example. (GH-15934) (GH-15936)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/c97ee1d3d8fdcbe6ca560767b53219802745=
7d80
commit: c97ee1d3d8fdcbe6ca560767b532198027457d80
branch: 3.8
author: St=C3=A9phane Wirtel <stephane at wirtel.be>
committer: GitHub <noreply at github.com>
date: 2019-09-11T15:17:48+02:00
summary:

[3.8] Doc: Use walrus operator in example. (GH-15934) (GH-15936)

(cherry picked from commit e1d455f3a3b82c2e08d5e133bcbab5a181b66cfb)

Co-authored-by: Julien Palard <julien at palard.fr>

files:
M Doc/library/http.client.rst

diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst
index 48bc35ca76cc..db26d56af30f 100644
--- a/Doc/library/http.client.rst
+++ b/Doc/library/http.client.rst
@@ -516,10 +516,7 @@ Here is an example session that uses the ``GET`` method::
    >>> # The following example demonstrates reading data in chunks.
    >>> conn.request("GET", "/")
    >>> r1 =3D conn.getresponse()
-   >>> while True:
-   ...     chunk =3D r1.read(200)  # 200 bytes
-   ...     if not chunk:
-   ...          break
+   >>> while chunk :=3D r1.read(200):
    ...     print(repr(chunk))
    b'<!doctype html>\n<!--[if"...
    ...


From webhook-mailer at python.org  Wed Sep 11 09:36:54 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 13:36:54 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.497.1599366872.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-33944: note about the intended use of code in .pth files
 (GH-10131) (GH-15942)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/d1d968d45df1a900b0600c4d296b180aa978=
336d
commit: d1d968d45df1a900b0600c4d296b180aa978336d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T15:36:50+02:00
summary:

bpo-33944: note about the intended use of code in .pth files (GH-10131) (GH-1=
5942)

https://bugs.python.org/issue33944
(cherry picked from commit f9b5840fb4497a9e2ba2c1f01ad0dafba04c8496)

Co-authored-by: native-api <ivan_pozdeev at mail.ru>

files:
A Misc/NEWS.d/next/Documentation/2018-10-26-18-10-29.bpo-33944.V1YeOA.rst
M Doc/library/site.rst

diff --git a/Doc/library/site.rst b/Doc/library/site.rst
index 9e4d402c1f5e..e1ca160c107b 100644
--- a/Doc/library/site.rst
+++ b/Doc/library/site.rst
@@ -61,6 +61,19 @@ directory rather than a file.  No item is added to ``sys.p=
ath`` more than
 once.  Blank lines and lines beginning with ``#`` are skipped.  Lines starti=
ng
 with ``import`` (followed by space or tab) are executed.
=20
+.. note::
+
+   An executable line in a :file:`.pth` file is run at every Python startup,
+   regardless of whether a particular module is actually going to be used.
+   Its impact should thus be kept to a minimum.
+   The primary intended purpose of executable lines is to make the
+   corresponding module(s) importable
+   (load 3rd-party import hooks, adjust :envvar:`PATH` etc).
+   Any other initialization is supposed to be done upon a module's
+   actual import, if and when it happens.
+   Limiting a code chunk to a single line is a deliberate measure
+   to discourage putting anything more complex here.
+
 .. index::
    single: package
    triple: path; configuration; file
diff --git a/Misc/NEWS.d/next/Documentation/2018-10-26-18-10-29.bpo-33944.V1Y=
eOA.rst b/Misc/NEWS.d/next/Documentation/2018-10-26-18-10-29.bpo-33944.V1YeOA=
.rst
new file mode 100644
index 000000000000..29bec88c9649
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2018-10-26-18-10-29.bpo-33944.V1YeOA.rst
@@ -0,0 +1 @@
+Added a note about the intended use of code in .pth files.


From webhook-mailer at python.org  Wed Sep 11 09:37:53 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 13:37:53 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.498.1599366874.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 bpo-33459: Fix "tuple displays" term in Expressions.rst (GH-6760) (GH-15939)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/4c2fa5c474af81487c7a8d4188f9afd80d4d=
769b
commit: 4c2fa5c474af81487c7a8d4188f9afd80d4d769b
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T15:37:49+02:00
summary:

bpo-33459: Fix "tuple displays" term in Expressions.rst (GH-6760) (GH-15939)

https://bugs.python.org/issue33459

Automerge-Triggered-By: @matrixise
(cherry picked from commit dc269971091710563a0d730a0d4b084901826c15)

Co-authored-by: Andre Delfino <adelfino at gmail.com>

files:
M Doc/reference/expressions.rst

diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst
index 990600ec1888..0a61ded2e3ab 100644
--- a/Doc/reference/expressions.rst
+++ b/Doc/reference/expressions.rst
@@ -148,9 +148,8 @@ immutable, the same rules as for literals apply (i.e., tw=
o occurrences of the em
 tuple may or may not yield the same object).
=20
 .. index::
-   single: comma; tuple display
-   pair: tuple; display
-   single: , (comma); tuple display
+   single: comma
+   single: , (comma)
=20
 Note that tuples are not formed by the parentheses, but rather by use of the
 comma operator.  The exception is the empty tuple, for which parentheses *ar=
e*
@@ -1833,7 +1832,8 @@ precedence and have a left-to-right chaining feature as=
 described in the
 | ``x[index]``, ``x[index:index]``,             | Subscription, slicing,    =
          |
 | ``x(arguments...)``, ``x.attribute``          | call, attribute reference =
          |
 +-----------------------------------------------+---------------------------=
----------+
-| ``(expressions...)``,                         | Binding or tuple display, =
          |
+| ``(expressions...)``,                         | Binding or parenthesized  =
          |
+|                                               | expression,               =
          |
 | ``[expressions...]``,                         | list display,             =
          |
 | ``{key: value...}``,                          | dictionary display,       =
          |
 | ``{expressions...}``                          | set display               =
          |


From webhook-mailer at python.org  Wed Sep 11 09:38:21 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 13:38:21 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.499.1599366875.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 bpo-33459: Fix "tuple displays" term in Expressions.rst (GH-6760) (GH-15940)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/4d2babd99022bf43dc659d9fe5700c1fc138=
08c4
commit: 4d2babd99022bf43dc659d9fe5700c1fc13808c4
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T15:38:17+02:00
summary:

bpo-33459: Fix "tuple displays" term in Expressions.rst (GH-6760) (GH-15940)

https://bugs.python.org/issue33459

Automerge-Triggered-By: @matrixise
(cherry picked from commit dc269971091710563a0d730a0d4b084901826c15)

Co-authored-by: Andre Delfino <adelfino at gmail.com>

files:
M Doc/reference/expressions.rst

diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst
index ee13c5f4c6fc..73a2f271ca54 100644
--- a/Doc/reference/expressions.rst
+++ b/Doc/reference/expressions.rst
@@ -148,9 +148,8 @@ immutable, the same rules as for literals apply (i.e., tw=
o occurrences of the em
 tuple may or may not yield the same object).
=20
 .. index::
-   single: comma; tuple display
-   pair: tuple; display
-   single: , (comma); tuple display
+   single: comma
+   single: , (comma)
=20
 Note that tuples are not formed by the parentheses, but rather by use of the
 comma operator.  The exception is the empty tuple, for which parentheses *ar=
e*
@@ -1822,7 +1821,8 @@ precedence and have a left-to-right chaining feature as=
 described in the
 | ``x[index]``, ``x[index:index]``,             | Subscription, slicing,    =
          |
 | ``x(arguments...)``, ``x.attribute``          | call, attribute reference =
          |
 +-----------------------------------------------+---------------------------=
----------+
-| ``(expressions...)``,                         | Binding or tuple display, =
          |
+| ``(expressions...)``,                         | Binding or parenthesized  =
          |
+|                                               | expression,               =
          |
 | ``[expressions...]``,                         | list display,             =
          |
 | ``{key: value...}``,                          | dictionary display,       =
          |
 | ``{expressions...}``                          | set display               =
          |


From webhook-mailer at python.org  Wed Sep 11 10:18:04 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 14:18:04 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.500.1599366877.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Update ftplib.all_errors documentation to match code (#15026)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/51adfc6ed2c6c466dda84edc11d9b2b8ae29=
03e8
commit: 51adfc6ed2c6c466dda84edc11d9b2b8ae2903e8
branch: master
author: Andrew Scheller <github at loowis.durge.org>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T16:18:00+02:00
summary:

Update ftplib.all_errors documentation to match code (#15026)

The documentation doesn't mention the `EOFError` that https://github.com/pyth=
on/cpython/blob/master/Lib/ftplib.py#L66 includes

files:
M Doc/library/ftplib.rst

diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst
index e006d019b837..7423413d209a 100644
--- a/Doc/library/ftplib.rst
+++ b/Doc/library/ftplib.rst
@@ -144,7 +144,7 @@ The module defines the following items:
    The set of all exceptions (as a tuple) that methods of :class:`FTP`
    instances may raise as a result of problems with the FTP connection (as
    opposed to programming errors made by the caller).  This set includes the
-   four exceptions listed above as well as :exc:`OSError`.
+   four exceptions listed above as well as :exc:`OSError` and :exc:`EOFError=
`.
=20
=20
 .. seealso::


From webhook-mailer at python.org  Wed Sep 11 10:25:29 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 14:25:29 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.501.1599366879.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Indicate locations of parse_qs, parse_qsl, escape (GH-14828)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/1abf54336fd2cb545c453d22dd5501392b33=
50b2
commit: 1abf54336fd2cb545c453d22dd5501392b3350b2
branch: master
author: Simon Willison <swillison at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T16:25:25+02:00
summary:

Doc: Indicate locations of parse_qs, parse_qsl, escape (GH-14828)

Since they have been removed from cgi it's useful to remind people where they
can be found instead.

files:
M Doc/whatsnew/3.8.rst

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 07202ea4ff08..aefe9b53746f 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -1467,7 +1467,8 @@ The following features and APIs have been removed from =
Python 3.8:
   script is tied to. (Contributed by Brett Cannon in :issue:`25427`.)
=20
 * ``parse_qs``, ``parse_qsl``, and ``escape`` are removed from :mod:`cgi`
-  module.  They are deprecated from Python 3.2 or older.
+  module.  They are deprecated from Python 3.2 or older. They should be impo=
rted
+  from the ``urllib.parse`` and ``html`` modules instead.
=20
 * ``filemode`` function is removed from :mod:`tarfile` module.
   It is not documented and deprecated since Python 3.3.


From webhook-mailer at python.org  Wed Sep 11 10:37:42 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 14:37:42 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.502.1599366880.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: closes bpo-37252: Fix devpoll tests. (GH-14017) (GH-15948)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/b4808c1265722b8a03ab436e37c7c337074a=
cb15
commit: b4808c1265722b8a03ab436e37c7c337074acb15
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T16:37:38+02:00
summary:

closes bpo-37252: Fix devpoll tests. (GH-14017) (GH-15948)

(cherry picked from commit 95da826db9ed4acbc81f32296f14429c06bd2124)

Co-authored-by: Jakub Kul=C3=ADk <Kulikjak at gmail.com>

files:
A Misc/NEWS.d/next/Tests/2019-06-12-14-30-29.bpo-37252.4o-uLs.rst
M Lib/test/test_devpoll.py

diff --git a/Lib/test/test_devpoll.py b/Lib/test/test_devpoll.py
index c133c811b197..110c00686273 100644
--- a/Lib/test/test_devpoll.py
+++ b/Lib/test/test_devpoll.py
@@ -109,7 +109,7 @@ def test_close(self):
         # operations must fail with ValueError("I/O operation on closed ...")
         self.assertRaises(ValueError, devpoll.modify, fd, select.POLLIN)
         self.assertRaises(ValueError, devpoll.poll)
-        self.assertRaises(ValueError, devpoll.register, fd, fd, select.POLLI=
N)
+        self.assertRaises(ValueError, devpoll.register, fd, select.POLLIN)
         self.assertRaises(ValueError, devpoll.unregister, fd)
=20
     def test_fd_non_inheritable(self):
@@ -122,9 +122,9 @@ def test_events_mask_overflow(self):
         w, r =3D os.pipe()
         pollster.register(w)
         # Issue #17919
-        self.assertRaises(OverflowError, pollster.register, 0, -1)
+        self.assertRaises(ValueError, pollster.register, 0, -1)
         self.assertRaises(OverflowError, pollster.register, 0, 1 << 64)
-        self.assertRaises(OverflowError, pollster.modify, 1, -1)
+        self.assertRaises(ValueError, pollster.modify, 1, -1)
         self.assertRaises(OverflowError, pollster.modify, 1, 1 << 64)
=20
     @cpython_only
diff --git a/Misc/NEWS.d/next/Tests/2019-06-12-14-30-29.bpo-37252.4o-uLs.rst =
b/Misc/NEWS.d/next/Tests/2019-06-12-14-30-29.bpo-37252.4o-uLs.rst
new file mode 100644
index 000000000000..1bd7d2872f13
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-06-12-14-30-29.bpo-37252.4o-uLs.rst
@@ -0,0 +1,2 @@
+Fix assertions in ``test_close`` and ``test_events_mask_overflow`` devpoll
+tests.


From webhook-mailer at python.org  Wed Sep 11 11:12:14 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 15:12:14 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.503.1599366882.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-35224: Additional documentation for Assignment Expressions
 (GH-15935) (GH-15967)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/be2aa58fdc29cf13aabff6d6712e7853e94e=
88f8
commit: be2aa58fdc29cf13aabff6d6712e7853e94e88f8
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T17:12:09+02:00
summary:

bpo-35224: Additional documentation for Assignment Expressions (GH-15935) (GH=
-15967)

Add or update assignment expression documentation for:
- FAQ - Design
- Reference - Expressions
- Reference - Lexical Analysis

https://bugs.python.org/issue35224

Automerge-Triggered-By: @matrixise
(cherry picked from commit 6357c95716d89ac1f80587fbc4133df8d2e8396c)

Co-authored-by: Emily Morehouse <emily at cuttlesoft.com>

files:
M Doc/faq/design.rst
M Doc/reference/expressions.rst
M Doc/reference/lexical_analysis.rst
M Doc/whatsnew/3.8.rst

diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst
index e2d63a0323da..81c0f474ac16 100644
--- a/Doc/faq/design.rst
+++ b/Doc/faq/design.rst
@@ -149,66 +149,15 @@ to tell Python which namespace to use.
 Why can't I use an assignment in an expression?
 -----------------------------------------------
=20
-Many people used to C or Perl complain that they want to use this C idiom:
+Starting in Python 3.8, you can!
=20
-.. code-block:: c
+Assignment expressions using the walrus operator `:=3D` assign a variable in=
 an
+expression::
=20
-   while (line =3D readline(f)) {
-       // do something with line
-   }
-
-where in Python you're forced to write this::
-
-   while True:
-       line =3D f.readline()
-       if not line:
-           break
-       ...  # do something with line
-
-The reason for not allowing assignment in Python expressions is a common,
-hard-to-find bug in those other languages, caused by this construct:
-
-.. code-block:: c
-
-    if (x =3D 0) {
-        // error handling
-    }
-    else {
-        // code that only works for nonzero x
-    }
-
-The error is a simple typo: ``x =3D 0``, which assigns 0 to the variable ``x=
``,
-was written while the comparison ``x =3D=3D 0`` is certainly what was intend=
ed.
-
-Many alternatives have been proposed.  Most are hacks that save some typing =
but
-use arbitrary or cryptic syntax or keywords, and fail the simple criterion f=
or
-language change proposals: it should intuitively suggest the proper meaning =
to a
-human reader who has not yet been introduced to the construct.
-
-An interesting phenomenon is that most experienced Python programmers recogn=
ize
-the ``while True`` idiom and don't seem to be missing the assignment in
-expression construct much; it's only newcomers who express a strong desire to
-add this to the language.
-
-There's an alternative way of spelling this that seems attractive but is
-generally less robust than the "while True" solution::
-
-   line =3D f.readline()
-   while line:
-       ...  # do something with line...
-       line =3D f.readline()
-
-The problem with this is that if you change your mind about exactly how you =
get
-the next line (e.g. you want to change it into ``sys.stdin.readline()``) you
-have to remember to change two places in your program -- the second occurren=
ce
-is hidden at the bottom of the loop.
-
-The best approach is to use iterators, making it possible to loop through
-objects using the ``for`` statement.  For example, :term:`file objects
-<file object>` support the iterator protocol, so you can write simply::
+   while chunk :=3D fp.read(200):
+      print(chunk)
=20
-   for line in f:
-       ...  # do something with line...
+See :pep:`572` for more information.
=20
=20
=20
diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst
index 73a2f271ca54..d9db33a78568 100644
--- a/Doc/reference/expressions.rst
+++ b/Doc/reference/expressions.rst
@@ -1784,6 +1784,8 @@ precedence and have a left-to-right chaining feature as=
 described in the
 +-----------------------------------------------+---------------------------=
----------+
 | Operator                                      | Description               =
          |
 +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D+=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D+
+| ``:=3D``                                        | Assignment expression   =
            |
++-----------------------------------------------+---------------------------=
----------+
 | :keyword:`lambda`                             | Lambda expression         =
          |
 +-----------------------------------------------+---------------------------=
----------+
 | :keyword:`if <if_expr>` -- :keyword:`!else`   | Conditional expression    =
          |
diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analy=
sis.rst
index 7e1e17edb2d8..c0e13b53698e 100644
--- a/Doc/reference/lexical_analysis.rst
+++ b/Doc/reference/lexical_analysis.rst
@@ -887,7 +887,7 @@ The following tokens are operators:
=20
=20
    +       -       *       **      /       //      %      @
-   <<      >>      &       |       ^       ~
+   <<      >>      &       |       ^       ~       :=3D
    <       >       <=3D      >=3D      =3D=3D      !=3D
=20
=20
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index aefe9b53746f..c2455f487b14 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -122,8 +122,6 @@ See :pep:`572` for a full description.
=20
 (Contributed by Emily Morehouse in :issue:`35224`.)
=20
-.. TODO: Emily will sprint on docs at PyCon US 2019.
-
=20
 Positional-only parameters
 --------------------------


From webhook-mailer at python.org  Wed Sep 11 11:29:48 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 15:29:48 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.504.1599366884.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 bpo-34331: Fix incorrectly pluralized abstract class error message. (GH-8670)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/4a12a178f4a6b9a59d97fecc727f2b6b28df=
c85f
commit: 4a12a178f4a6b9a59d97fecc727f2b6b28dfc85f
branch: master
author: Daniel Andrade <dangro at users.noreply.github.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T17:29:44+02:00
summary:

bpo-34331: Fix incorrectly pluralized abstract class error message. (GH-8670)

files:
A Misc/NEWS.d/next/C API/2018-08-04-00-59-44.bpo-34331.iaUkmU.rst
M Lib/test/test_abc.py
M Objects/typeobject.c

diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py
index 9f5afb241aea..000e5838e3c7 100644
--- a/Lib/test/test_abc.py
+++ b/Lib/test/test_abc.py
@@ -149,6 +149,25 @@ def foo(): return 4
             self.assertEqual(D.foo(), 4)
             self.assertEqual(D().foo(), 4)
=20
+        def test_object_new_with_one_abstractmethod(self):
+            class C(metaclass=3Dabc_ABCMeta):
+                @abc.abstractmethod
+                def method_one(self):
+                    pass
+            msg =3D r"class C with abstract method method_one"
+            self.assertRaisesRegex(TypeError, msg, C)
+
+        def test_object_new_with_many_abstractmethods(self):
+            class C(metaclass=3Dabc_ABCMeta):
+                @abc.abstractmethod
+                def method_one(self):
+                    pass
+                @abc.abstractmethod
+                def method_two(self):
+                    pass
+            msg =3D r"class C with abstract methods method_one, method_two"
+            self.assertRaisesRegex(TypeError, msg, C)
+
         def test_abstractmethod_integration(self):
             for abstractthing in [abc.abstractmethod, abc.abstractproperty,
                                   abc.abstractclassmethod,
diff --git a/Misc/NEWS.d/next/C API/2018-08-04-00-59-44.bpo-34331.iaUkmU.rst =
b/Misc/NEWS.d/next/C API/2018-08-04-00-59-44.bpo-34331.iaUkmU.rst
new file mode 100644
index 000000000000..e45e0915dbf5
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2018-08-04-00-59-44.bpo-34331.iaUkmU.rst=09
@@ -0,0 +1,2 @@
+Use singular/plural noun in error message when instantiating an abstract
+class with non-overriden abstract method(s).
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 7575e5580bf9..dfdac9e2e4f5 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -3753,6 +3753,7 @@ object_new(PyTypeObject *type, PyObject *args, PyObject=
 *kwds)
         PyObject *joined;
         PyObject *comma;
         _Py_static_string(comma_id, ", ");
+        Py_ssize_t method_count;
=20
         /* Compute ", ".join(sorted(type.__abstractmethods__))
            into joined. */
@@ -3773,14 +3774,18 @@ object_new(PyTypeObject *type, PyObject *args, PyObje=
ct *kwds)
             return NULL;
         }
         joined =3D PyUnicode_Join(comma, sorted_methods);
+        method_count =3D PyObject_Length(sorted_methods);
         Py_DECREF(sorted_methods);
         if (joined =3D=3D NULL)
             return NULL;
+        if (method_count =3D=3D -1)
+            return NULL;
=20
         PyErr_Format(PyExc_TypeError,
                      "Can't instantiate abstract class %s "
-                     "with abstract methods %U",
+                     "with abstract method%s %U",
                      type->tp_name,
+                     method_count > 1 ? "s" : "",
                      joined);
         Py_DECREF(joined);
         return NULL;


From webhook-mailer at python.org  Wed Sep 11 11:37:18 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 15:37:18 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.505.1599366885.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Improve clarity of try-return-finally-return (GH-15677)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/0cc27417f2cd399c432d7dda9aeca1d81af7=
6936
commit: 0cc27417f2cd399c432d7dda9aeca1d81af76936
branch: master
author: toonarmycaptain <toonarmycaptain at hotmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T17:37:13+02:00
summary:

Improve clarity of try-return-finally-return (GH-15677)

Clarify execution in try-return-finally-return case.

files:
M Doc/tutorial/errors.rst
M Misc/ACKS

diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst
index e9a63e425279..75a49259d440 100644
--- a/Doc/tutorial/errors.rst
+++ b/Doc/tutorial/errors.rst
@@ -388,15 +388,28 @@ example::
      File "<stdin>", line 2, in <module>
    KeyboardInterrupt
=20
-A *finally clause* is always executed before leaving the :keyword:`try`
-statement, whether an exception has occurred or not. When an exception has
-occurred in the :keyword:`!try` clause and has not been handled by an
-:keyword:`except` clause (or it has occurred in an :keyword:`!except` or
-:keyword:`!else` clause), it is re-raised after the :keyword:`finally` claus=
e has
-been executed.  The :keyword:`!finally` clause is also executed "on the way =
out"
-when any other clause of the :keyword:`!try` statement is left via a
-:keyword:`break`, :keyword:`continue` or :keyword:`return` statement.  A more
-complicated example::
+If a :keyword:`finally` clause is present, the :keyword:`finally` clause wil=
l execute as the last task before the :keyword:`try` statement completes. The=
 :keyword:`finally` clause runs whether or not the :keyword:`try` statement p=
roduces an exception. The following points discuss more complex cases when an=
 exception occurs:
+
+* If an exception occurs during execution of the :keyword:`!try` clause, the=
 exception may be handled by an :keyword:`except` clause. In all cases, the e=
xception is re-raised after the :keyword:`!finally` clause has been executed.
+
+* An exception could occur during execution of an :keyword:`!except` or :key=
word:`!else` clause. Again, the exception is re-raised after the :keyword:`!f=
inally` clause has been executed.
+
+* If the :keyword:`!try` statement reaches a :keyword:`break`, :keyword:`con=
tinue` or :keyword:`return` statement, the :keyword:`finally` clause will exe=
cute just prior to the :keyword:`break`, :keyword:`continue` or :keyword:`ret=
urn` statement's execution.
+
+* If a :keyword:`finally` clause includes a :keyword:`return` statement, the=
 :keyword:`finally` clause's :keyword:`return` statement will execute before,=
 and instead of, the :keyword:`return` statement in a :keyword:`try` clause.
+
+For example::
+
+   >>> def bool_return(): -> bool:
+   ...     try:
+   ...         return True
+   ...     finally:
+   ...         return False
+   ...
+   >>> bool_return()
+   False
+
+A more complicated example::
=20
    >>> def divide(x, y):
    ...     try:
diff --git a/Misc/ACKS b/Misc/ACKS
index d37d2bdcc43c..f321131b2ab4 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -58,6 +58,7 @@ Jon Anglin
 Michele Angrisano
 Ankur Ankan
 Heidi Annexstad
+David Antonini
 Ramchandra Apte
 =C3=89ric Araujo
 Alexandru Ardelean


From webhook-mailer at python.org  Wed Sep 11 12:45:56 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 16:45:56 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.506.1599366887.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-38117: Test with OpenSSL 1.1.1d (GH-15983)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/58ab13479d854491ac9207bacfae25e8b18b=
044a
commit: 58ab13479d854491ac9207bacfae25e8b18b044a
branch: master
author: Christian Heimes <christian at python.org>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T18:45:52+02:00
summary:

bpo-38117: Test with OpenSSL 1.1.1d (GH-15983)

Signed-off-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Tests/2019-09-11-17-22-32.bpo-38117.X7LgGY.rst
M .azure-pipelines/ci.yml
M .azure-pipelines/pr.yml
M .travis.yml
M Tools/ssl/multissltests.py

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index 12ff7d4c55c9..fe10e45b6792 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -59,7 +59,7 @@ jobs:
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1d
=20
   steps:
   - template: ./posix-steps.yml
@@ -116,7 +116,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1d
=20
   steps:
   - template: ./posix-steps.yml
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
index e3e24ae2a4b0..09209fc0c954 100644
--- a/.azure-pipelines/pr.yml
+++ b/.azure-pipelines/pr.yml
@@ -59,7 +59,7 @@ jobs:
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1d
=20
   steps:
   - template: ./posix-steps.yml
@@ -116,7 +116,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1c
+    openssl_version: 1.1.1d
=20
   steps:
   - template: ./posix-steps.yml
diff --git a/.travis.yml b/.travis.yml
index 98b33393f52c..d4275fde6424 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,7 +11,7 @@ cache:
=20
 env:
   global:
-    - OPENSSL=3D1.1.1c
+    - OPENSSL=3D1.1.1d
     - OPENSSL_DIR=3D"$HOME/multissl/openssl/${OPENSSL}"
     - PATH=3D"${OPENSSL_DIR}/bin:$PATH"
     - CFLAGS=3D"-I${OPENSSL_DIR}/include"
diff --git a/Misc/NEWS.d/next/Tests/2019-09-11-17-22-32.bpo-38117.X7LgGY.rst =
b/Misc/NEWS.d/next/Tests/2019-09-11-17-22-32.bpo-38117.X7LgGY.rst
new file mode 100644
index 000000000000..d3280375516a
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-09-11-17-22-32.bpo-38117.X7LgGY.rst
@@ -0,0 +1 @@
+Test with OpenSSL 1.1.1d
diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py
index 7fda4df55a67..d816810d5707 100755
--- a/Tools/ssl/multissltests.py
+++ b/Tools/ssl/multissltests.py
@@ -45,17 +45,16 @@
 ]
=20
 OPENSSL_RECENT_VERSIONS =3D [
-    "1.0.2s",
-    "1.1.0k",
-    "1.1.1c",
+    "1.0.2t",
+    "1.1.0l",
+    "1.1.1d",
 ]
=20
 LIBRESSL_OLD_VERSIONS =3D [
-    "2.9.2",
 ]
=20
 LIBRESSL_RECENT_VERSIONS =3D [
-    "2.8.3",
+    "2.9.2",
 ]
=20
 # store files in ../multissl


From webhook-mailer at python.org  Wed Sep 11 13:10:00 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 17:10:00 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.507.1599366889.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-37750: Add doc of PyBuffer_FromContiguous (GH-15988) (GH-15990)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/4cab7eb9e1f4afc44bac4889de034e031d46=
2e7a
commit: 4cab7eb9e1f4afc44bac4889de034e031d462e7a
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T19:09:57+02:00
summary:

bpo-37750: Add doc of PyBuffer_FromContiguous (GH-15988) (GH-15990)

https://bugs.python.org/issue37750

Automerge-Triggered-By: @matrixise
(cherry picked from commit 5a56ce4a0e820fefcd598b94715a7ff7e199858d)

Co-authored-by: Hai Shi <shihai1992 at gmail.com>

files:
M Doc/c-api/buffer.rst

diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index fdb8bd19d218..06941e100409 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -481,6 +481,13 @@ Buffer-related functions
    *indices* must point to an array of ``view->ndim`` indices.
=20
=20
+.. c:function:: int PyBuffer_FromContiguous(Py_buffer *view, void *buf, Py_s=
size_t len, char fort)
+
+   Copy contiguous *len* bytes from *buf* to *view*.
+   *fort* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering).
+   ``0`` is returned on success, ``-1`` on error.
+
+
 .. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssiz=
e_t len, char order)
=20
    Copy *len* bytes from *src* to its contiguous representation in *buf*.


From webhook-mailer at python.org  Wed Sep 11 13:10:27 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 17:10:27 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.508.1599366891.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-37750: Add doc of PyBuffer_FromContiguous (GH-15988) (GH-15991)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/c112faff158a05c1e71f9e1957364ed756ef=
bcde
commit: c112faff158a05c1e71f9e1957364ed756efbcde
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T19:10:23+02:00
summary:

bpo-37750: Add doc of PyBuffer_FromContiguous (GH-15988) (GH-15991)

https://bugs.python.org/issue37750

Automerge-Triggered-By: @matrixise
(cherry picked from commit 5a56ce4a0e820fefcd598b94715a7ff7e199858d)

Co-authored-by: Hai Shi <shihai1992 at gmail.com>

files:
M Doc/c-api/buffer.rst

diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index 71e5b32f1fe5..19ec7aaadfa7 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -481,6 +481,13 @@ Buffer-related functions
    *indices* must point to an array of ``view->ndim`` indices.
=20
=20
+.. c:function:: int PyBuffer_FromContiguous(Py_buffer *view, void *buf, Py_s=
size_t len, char fort)
+
+   Copy contiguous *len* bytes from *buf* to *view*.
+   *fort* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering).
+   ``0`` is returned on success, ``-1`` on error.
+
+
 .. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssiz=
e_t len, char order)
=20
    Copy *len* bytes from *src* to its contiguous representation in *buf*.


From webhook-mailer at python.org  Wed Sep 11 13:34:22 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 17:34:22 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.509.1599366892.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-37698: Update doc of PyBuffer_ToContiguous (GH-14992) (GH-15998)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/c62da14776d12276ed8f6ecbc74c0db8243b=
4260
commit: c62da14776d12276ed8f6ecbc74c0db8243b4260
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T19:34:19+02:00
summary:

bpo-37698: Update doc of PyBuffer_ToContiguous (GH-14992) (GH-15998)

https://bugs.python.org/issue37698
(cherry picked from commit 15f5a7527b87e11fcf23069c147fd4cb7d42cfb0)

Co-authored-by: Hai Shi <shihai1992 at gmail.com>

files:
M Doc/c-api/buffer.rst

diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index 19ec7aaadfa7..b639ba6ff787 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -491,8 +491,8 @@ Buffer-related functions
 .. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssiz=
e_t len, char order)
=20
    Copy *len* bytes from *src* to its contiguous representation in *buf*.
-   *order* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering).
-   ``0`` is returned on success, ``-1`` on error.
+   *order* can be ``'C'`` or ``'F'`` or ``'A'`` (for C-style or Fortran-style
+   ordering or either one). ``0`` is returned on success, ``-1`` on error.
=20
    This function fails if *len* !=3D *src->len*.
=20


From webhook-mailer at python.org  Wed Sep 11 13:36:04 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 17:36:04 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.510.1599366894.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 [3.8] bpo-37698: Update doc of PyBuffer_ToContiguous (GH-14992) (GH-15999)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/965e53a9deb1c7f02c68dee119eae94c3be8=
af15
commit: 965e53a9deb1c7f02c68dee119eae94c3be8af15
branch: 3.8
author: St=C3=A9phane Wirtel <stephane at wirtel.be>
committer: GitHub <noreply at github.com>
date: 2019-09-11T19:36:00+02:00
summary:

[3.8] bpo-37698: Update doc of PyBuffer_ToContiguous (GH-14992) (GH-15999)

https://bugs.python.org/issue37698
(cherry picked from commit 15f5a7527b87e11fcf23069c147fd4cb7d42cfb0)

Co-authored-by: Hai Shi <shihai1992 at gmail.com>

files:
M Doc/c-api/buffer.rst

diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index 06941e100409..67140511ef74 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -491,8 +491,8 @@ Buffer-related functions
 .. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssiz=
e_t len, char order)
=20
    Copy *len* bytes from *src* to its contiguous representation in *buf*.
-   *order* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering).
-   ``0`` is returned on success, ``-1`` on error.
+   *order* can be ``'C'`` or ``'F'`` or ``'A'`` (for C-style or Fortran-style
+   ordering or either one). ``0`` is returned on success, ``-1`` on error.
=20
    This function fails if *len* !=3D *src->len*.
=20


From webhook-mailer at python.org  Wed Sep 11 13:42:12 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 17:42:12 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.511.1599366896.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Improve clarity of try-return-finally-return (GH-15677) (GH-15982)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/e4be7c8ba1ab11ccf13237996f80c0dd0b9b=
4f14
commit: e4be7c8ba1ab11ccf13237996f80c0dd0b9b4f14
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T19:42:09+02:00
summary:

Improve clarity of try-return-finally-return (GH-15677) (GH-15982)

Clarify execution in try-return-finally-return case.
(cherry picked from commit 0cc27417f2cd399c432d7dda9aeca1d81af76936)

Co-authored-by: toonarmycaptain <toonarmycaptain at hotmail.com>

files:
M Doc/tutorial/errors.rst
M Misc/ACKS

diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst
index 4e287bbd8d29..0ddf0cc0042d 100644
--- a/Doc/tutorial/errors.rst
+++ b/Doc/tutorial/errors.rst
@@ -341,15 +341,28 @@ example::
      File "<stdin>", line 2, in <module>
    KeyboardInterrupt
=20
-A *finally clause* is always executed before leaving the :keyword:`try`
-statement, whether an exception has occurred or not. When an exception has
-occurred in the :keyword:`!try` clause and has not been handled by an
-:keyword:`except` clause (or it has occurred in an :keyword:`!except` or
-:keyword:`!else` clause), it is re-raised after the :keyword:`finally` claus=
e has
-been executed.  The :keyword:`!finally` clause is also executed "on the way =
out"
-when any other clause of the :keyword:`!try` statement is left via a
-:keyword:`break`, :keyword:`continue` or :keyword:`return` statement.  A more
-complicated example::
+If a :keyword:`finally` clause is present, the :keyword:`finally` clause wil=
l execute as the last task before the :keyword:`try` statement completes. The=
 :keyword:`finally` clause runs whether or not the :keyword:`try` statement p=
roduces an exception. The following points discuss more complex cases when an=
 exception occurs:
+
+* If an exception occurs during execution of the :keyword:`!try` clause, the=
 exception may be handled by an :keyword:`except` clause. In all cases, the e=
xception is re-raised after the :keyword:`!finally` clause has been executed.
+
+* An exception could occur during execution of an :keyword:`!except` or :key=
word:`!else` clause. Again, the exception is re-raised after the :keyword:`!f=
inally` clause has been executed.
+
+* If the :keyword:`!try` statement reaches a :keyword:`break`, :keyword:`con=
tinue` or :keyword:`return` statement, the :keyword:`finally` clause will exe=
cute just prior to the :keyword:`break`, :keyword:`continue` or :keyword:`ret=
urn` statement's execution.
+
+* If a :keyword:`finally` clause includes a :keyword:`return` statement, the=
 :keyword:`finally` clause's :keyword:`return` statement will execute before,=
 and instead of, the :keyword:`return` statement in a :keyword:`try` clause.
+
+For example::
+
+   >>> def bool_return(): -> bool:
+   ...     try:
+   ...         return True
+   ...     finally:
+   ...         return False
+   ...
+   >>> bool_return()
+   False
+
+A more complicated example::
=20
    >>> def divide(x, y):
    ...     try:
diff --git a/Misc/ACKS b/Misc/ACKS
index 35fcec2b2088..7afc92e7bb76 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -58,6 +58,7 @@ Jon Anglin
 Michele Angrisano
 Ankur Ankan
 Heidi Annexstad
+David Antonini
 Ramchandra Apte
 =C3=89ric Araujo
 Alexandru Ardelean


From webhook-mailer at python.org  Thu Sep 12 04:19:23 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 08:19:23 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.512.1599366902.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Update pickle.rst (GH-14128) (GH-16014)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/eb19c45298361d75f744dcda295fa635a849=
0c3c
commit: eb19c45298361d75f744dcda295fa635a8490c3c
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-12T09:19:12+01:00
summary:

Doc: Update pickle.rst (GH-14128) (GH-16014)

* Edits for readability and grammar
(cherry picked from commit 362f5350eb5e2c7bfb0b0a8c306a2e128c3aee93)

Co-authored-by: G=C3=A9ry Ogam <gery.ogam at gmail.com>

files:
M Doc/library/pickle.rst

diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst
index 9442efa2b667..eb58178e0e92 100644
--- a/Doc/library/pickle.rst
+++ b/Doc/library/pickle.rst
@@ -211,8 +211,9 @@ process more convenient:
=20
 .. function:: dump(obj, file, protocol=3DNone, \*, fix_imports=3DTrue, buffe=
r_callback=3DNone)
=20
-   Write a pickled representation of *obj* to the open :term:`file object` *=
file*.
-   This is equivalent to ``Pickler(file, protocol).dump(obj)``.
+   Write the pickled representation of the object *obj* to the open
+   :term:`file object` *file*.  This is equivalent to
+   ``Pickler(file, protocol).dump(obj)``.
=20
    Arguments *file*, *protocol*, *fix_imports* and *buffer_callback* have
    the same meaning as in the :class:`Pickler` constructor.
@@ -222,7 +223,7 @@ process more convenient:
=20
 .. function:: dumps(obj, protocol=3DNone, \*, fix_imports=3DTrue, buffer_cal=
lback=3DNone)
=20
-   Return the pickled representation of the object as a :class:`bytes` objec=
t,
+   Return the pickled representation of the object *obj* as a :class:`bytes`=
 object,
    instead of writing it to a file.
=20
    Arguments *protocol*, *fix_imports* and *buffer_callback* have the same
@@ -233,13 +234,13 @@ process more convenient:
=20
 .. function:: load(file, \*, fix_imports=3DTrue, encoding=3D"ASCII", errors=
=3D"strict", buffers=3DNone)
=20
-   Read a pickled object representation from the open :term:`file object`
+   Read the pickled representation of an object from the open :term:`file ob=
ject`
    *file* and return the reconstituted object hierarchy specified therein.
    This is equivalent to ``Unpickler(file).load()``.
=20
    The protocol version of the pickle is detected automatically, so no
-   protocol argument is needed.  Bytes past the pickled object's
-   representation are ignored.
+   protocol argument is needed.  Bytes past the pickled representation
+   of the object are ignored.
=20
    Arguments *file*, *fix_imports*, *encoding*, *errors*, *strict* and *buff=
ers*
    have the same meaning as in the :class:`Unpickler` constructor.
@@ -249,12 +250,12 @@ process more convenient:
=20
 .. function:: loads(bytes_object, \*, fix_imports=3DTrue, encoding=3D"ASCII"=
, errors=3D"strict", buffers=3DNone)
=20
-   Read a pickled object hierarchy from a :class:`bytes` object and return t=
he
-   reconstituted object hierarchy specified therein.
+   Return the reconstituted object hierarchy of the pickled representation
+   *bytes_object* of an object.
=20
    The protocol version of the pickle is detected automatically, so no
-   protocol argument is needed.  Bytes past the pickled object's
-   representation are ignored.
+   protocol argument is needed.  Bytes past the pickled representation
+   of the object are ignored.
=20
    Arguments *file*, *fix_imports*, *encoding*, *errors*, *strict* and *buff=
ers*
    have the same meaning as in the :class:`Unpickler` constructor.
@@ -325,7 +326,7 @@ The :mod:`pickle` module exports three classes, :class:`P=
ickler`,
=20
    .. method:: dump(obj)
=20
-      Write a pickled representation of *obj* to the open file object given =
in
+      Write the pickled representation of *obj* to the open file object give=
n in
       the constructor.
=20
    .. method:: persistent_id(obj)
@@ -426,9 +427,10 @@ The :mod:`pickle` module exports three classes, :class:`=
Pickler`,
=20
    .. method:: load()
=20
-      Read a pickled object representation from the open file object given in
-      the constructor, and return the reconstituted object hierarchy specifi=
ed
-      therein.  Bytes past the pickled object's representation are ignored.
+      Read the pickled representation of an object from the open file object
+      given in the constructor, and return the reconstituted object hierarchy
+      specified therein.  Bytes past the pickled representation of the object
+      are ignored.
=20
    .. method:: persistent_load(pid)
=20
@@ -731,13 +733,13 @@ alphanumeric characters (for protocol 0) [#]_ or just a=
n arbitrary object (for
 any newer protocol).
=20
 The resolution of such persistent IDs is not defined by the :mod:`pickle`
-module; it will delegate this resolution to the user defined methods on the
+module; it will delegate this resolution to the user-defined methods on the
 pickler and unpickler, :meth:`~Pickler.persistent_id` and
 :meth:`~Unpickler.persistent_load` respectively.
=20
-To pickle objects that have an external persistent id, the pickler must have=
 a
+To pickle objects that have an external persistent ID, the pickler must have=
 a
 custom :meth:`~Pickler.persistent_id` method that takes an object as an
-argument and returns either ``None`` or the persistent id for that object.
+argument and returns either ``None`` or the persistent ID for that object.
 When ``None`` is returned, the pickler simply pickles the object as normal.
 When a persistent ID string is returned, the pickler will pickle that object,
 along with a marker so that the unpickler will recognize it as a persistent =
ID.


From webhook-mailer at python.org  Thu Sep 12 04:19:37 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 08:19:37 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.513.1599366904.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Update pickle.rst (GH-14128) (GH-16015)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/ff2a5c0204bd08cde2f2c10605e1f9b442b5=
a7d6
commit: ff2a5c0204bd08cde2f2c10605e1f9b442b5a7d6
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-12T09:19:33+01:00
summary:

Doc: Update pickle.rst (GH-14128) (GH-16015)

* Edits for readability and grammar
(cherry picked from commit 362f5350eb5e2c7bfb0b0a8c306a2e128c3aee93)

Co-authored-by: G=C3=A9ry Ogam <gery.ogam at gmail.com>

files:
M Doc/library/pickle.rst

diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst
index 2d01d8b7ab25..f741976c9306 100644
--- a/Doc/library/pickle.rst
+++ b/Doc/library/pickle.rst
@@ -189,8 +189,9 @@ process more convenient:
=20
 .. function:: dump(obj, file, protocol=3DNone, \*, fix_imports=3DTrue)
=20
-   Write a pickled representation of *obj* to the open :term:`file object` *=
file*.
-   This is equivalent to ``Pickler(file, protocol).dump(obj)``.
+   Write the pickled representation of the object *obj* to the open
+   :term:`file object` *file*.  This is equivalent to
+   ``Pickler(file, protocol).dump(obj)``.
=20
    The optional *protocol* argument, an integer, tells the pickler to use
    the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`.
@@ -208,7 +209,7 @@ process more convenient:
=20
 .. function:: dumps(obj, protocol=3DNone, \*, fix_imports=3DTrue)
=20
-   Return the pickled representation of the object as a :class:`bytes` objec=
t,
+   Return the pickled representation of the object *obj* as a :class:`bytes`=
 object,
    instead of writing it to a file.
=20
    Arguments *protocol* and *fix_imports* have the same meaning as in
@@ -216,13 +217,13 @@ process more convenient:
=20
 .. function:: load(file, \*, fix_imports=3DTrue, encoding=3D"ASCII", errors=
=3D"strict")
=20
-   Read a pickled object representation from the open :term:`file object`
+   Read the pickled representation of an object from the open :term:`file ob=
ject`
    *file* and return the reconstituted object hierarchy specified therein.
    This is equivalent to ``Unpickler(file).load()``.
=20
    The protocol version of the pickle is detected automatically, so no
-   protocol argument is needed.  Bytes past the pickled object's
-   representation are ignored.
+   protocol argument is needed.  Bytes past the pickled representation
+   of the object are ignored.
=20
    The argument *file* must have two methods, a read() method that takes an
    integer argument, and a readline() method that requires no arguments.  Bo=
th
@@ -243,12 +244,12 @@ process more convenient:
=20
 .. function:: loads(bytes_object, \*, fix_imports=3DTrue, encoding=3D"ASCII"=
, errors=3D"strict")
=20
-   Read a pickled object hierarchy from a :class:`bytes` object and return t=
he
-   reconstituted object hierarchy specified therein.
+   Return the reconstituted object hierarchy of the pickled representation
+   *bytes_object* of an object.
=20
    The protocol version of the pickle is detected automatically, so no
-   protocol argument is needed.  Bytes past the pickled object's
-   representation are ignored.
+   protocol argument is needed.  Bytes past the pickled representation
+   of the object are ignored.
=20
    Optional keyword arguments are *fix_imports*, *encoding* and *errors*,
    which are used to control compatibility support for pickle stream generat=
ed
@@ -310,7 +311,7 @@ The :mod:`pickle` module exports two classes, :class:`Pic=
kler` and
=20
    .. method:: dump(obj)
=20
-      Write a pickled representation of *obj* to the open file object given =
in
+      Write the pickled representation of *obj* to the open file object give=
n in
       the constructor.
=20
    .. method:: persistent_id(obj)
@@ -382,9 +383,10 @@ The :mod:`pickle` module exports two classes, :class:`Pi=
ckler` and
=20
    .. method:: load()
=20
-      Read a pickled object representation from the open file object given in
-      the constructor, and return the reconstituted object hierarchy specifi=
ed
-      therein.  Bytes past the pickled object's representation are ignored.
+      Read the pickled representation of an object from the open file object
+      given in the constructor, and return the reconstituted object hierarchy
+      specified therein.  Bytes past the pickled representation of the object
+      are ignored.
=20
    .. method:: persistent_load(pid)
=20
@@ -649,13 +651,13 @@ alphanumeric characters (for protocol 0) [#]_ or just a=
n arbitrary object (for
 any newer protocol).
=20
 The resolution of such persistent IDs is not defined by the :mod:`pickle`
-module; it will delegate this resolution to the user defined methods on the
+module; it will delegate this resolution to the user-defined methods on the
 pickler and unpickler, :meth:`~Pickler.persistent_id` and
 :meth:`~Unpickler.persistent_load` respectively.
=20
-To pickle objects that have an external persistent id, the pickler must have=
 a
+To pickle objects that have an external persistent ID, the pickler must have=
 a
 custom :meth:`~Pickler.persistent_id` method that takes an object as an
-argument and returns either ``None`` or the persistent id for that object.
+argument and returns either ``None`` or the persistent ID for that object.
 When ``None`` is returned, the pickler simply pickles the object as normal.
 When a persistent ID string is returned, the pickler will pickle that object,
 along with a marker so that the unpickler will recognize it as a persistent =
ID.


From webhook-mailer at python.org  Thu Sep 12 05:03:03 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 09:03:03 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.514.1599366906.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-18578: Rename and document test.bytecode_helper as
 test.support.bytecode_helper (GH-15168)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/92777d5e5aed1753bafe07265dbe98b2d271=
815b
commit: 92777d5e5aed1753bafe07265dbe98b2d271815b
branch: master
author: Joannah Nanjekye <33177550+nanjekyejoannah at users.noreply.github.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-12T10:02:59+01:00
summary:

bpo-18578: Rename and document test.bytecode_helper as test.support.bytecode_=
helper (GH-15168)

Rename and document test.bytecode_helper as test.support.bytecode_helper

files:
A Lib/test/support/bytecode_helper.py
A Misc/NEWS.d/next/Library/2019-08-07-19-34-07.bpo-18578.xfvdb_.rst
D Lib/test/bytecode_helper.py
M Doc/library/test.rst
M Lib/test/test_dis.py
M Lib/test/test_peepholer.py

diff --git a/Doc/library/test.rst b/Doc/library/test.rst
index 6eef5c65499a..5dde55cdf951 100644
--- a/Doc/library/test.rst
+++ b/Doc/library/test.rst
@@ -1503,3 +1503,33 @@ script execution tests.
    containing the *source*.  If *compiled* is ``True``, both source files wi=
ll
    be compiled and added to the zip package.  Return a tuple of the full zip
    path and the archive name for the zip file.
+
+
+:mod:`test.support.bytecode_helper` --- Support tools for testing correct by=
tecode generation
+=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
+
+.. module:: test.support.bytecode_helper
+   :synopsis: Support tools for testing correct bytecode generation.
+
+The :mod:`test.support.bytecode_helper` module provides support for testing
+and inspecting bytecode generation.
+
+The module defines the follwing class:
+
+.. class:: BytecodeTestCase(unittest.TestCase)
+
+   This class has custom assertion methods for inspecting bytecode.
+
+.. method:: BytecodeTestCase.get_disassembly_as_string(co)
+
+   Return the disassembly of *co* as string.
+
+
+.. method:: BytecodeTestCase.assertInBytecode(x, opname, argval=3D_UNSPECIFI=
ED)
+
+   Return instr if *opname* is found, otherwise throws :exc:`AssertionError`.
+
+
+.. method:: BytecodeTestCase.assertNotInBytecode(x, opname, argval=3D_UNSPEC=
IFIED)
+
+   Throws :exc:`AssertionError` if *opname* is found.
diff --git a/Lib/test/bytecode_helper.py b/Lib/test/support/bytecode_helper.py
similarity index 91%
rename from Lib/test/bytecode_helper.py
rename to Lib/test/support/bytecode_helper.py
index 347d60337d76..348e277c1658 100644
--- a/Lib/test/bytecode_helper.py
+++ b/Lib/test/support/bytecode_helper.py
@@ -15,7 +15,7 @@ def get_disassembly_as_string(self, co):
         return s.getvalue()
=20
     def assertInBytecode(self, x, opname, argval=3D_UNSPECIFIED):
-        """Returns instr if op is found, otherwise throws AssertionError"""
+        """Returns instr if opname is found, otherwise throws AssertionError=
"""
         for instr in dis.get_instructions(x):
             if instr.opname =3D=3D opname:
                 if argval is _UNSPECIFIED or instr.argval =3D=3D argval:
@@ -29,7 +29,7 @@ def assertInBytecode(self, x, opname, argval=3D_UNSPECIFIED=
):
         self.fail(msg)
=20
     def assertNotInBytecode(self, x, opname, argval=3D_UNSPECIFIED):
-        """Throws AssertionError if op is found"""
+        """Throws AssertionError if opname is found"""
         for instr in dis.get_instructions(x):
             if instr.opname =3D=3D opname:
                 disassembly =3D self.get_disassembly_as_string(x)
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
index 11f97e6c0838..fecf203cc9b5 100644
--- a/Lib/test/test_dis.py
+++ b/Lib/test/test_dis.py
@@ -1,7 +1,7 @@
 # Minimal tests for dis module
=20
 from test.support import captured_stdout
-from test.bytecode_helper import BytecodeTestCase
+from test.support.bytecode_helper import BytecodeTestCase
 import unittest
 import sys
 import dis
diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py
index c90a53210a93..47dee33076c5 100644
--- a/Lib/test/test_peepholer.py
+++ b/Lib/test/test_peepholer.py
@@ -1,7 +1,7 @@
 import dis
 import unittest
=20
-from test.bytecode_helper import BytecodeTestCase
+from test.support.bytecode_helper import BytecodeTestCase
=20
=20
 def count_instr_recursively(f, opname):
diff --git a/Misc/NEWS.d/next/Library/2019-08-07-19-34-07.bpo-18578.xfvdb_.rs=
t b/Misc/NEWS.d/next/Library/2019-08-07-19-34-07.bpo-18578.xfvdb_.rst
new file mode 100644
index 000000000000..cc0e02fdb394
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-07-19-34-07.bpo-18578.xfvdb_.rst
@@ -0,0 +1,2 @@
+Renamed and documented `test.bytecode_helper` as `test.support.bytecode_help=
er`.
+Patch by Joannah Nanjekye.
\ No newline at end of file


From webhook-mailer at python.org  Thu Sep 12 05:41:40 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 09:41:40 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.515.1599366908.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Add example of dict() function with positional and keyword
 arguments (GH-15220)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/7544497ad322322050f3cb64ba2f4cb7f38e=
ddee
commit: 7544497ad322322050f3cb64ba2f4cb7f38eddee
branch: master
author: Georgy Frolov <gosha at fro.lv>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-12T10:41:36+01:00
summary:

Doc: Add example of dict() function with positional and keyword arguments (GH=
-15220)

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index c78f4ba8f068..426d768b170e 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -4213,7 +4213,8 @@ pairs within braces, for example: ``{'jack': 4098, 'sjo=
erd': 4127}`` or ``{4098:
       >>> c =3D dict(zip(['one', 'two', 'three'], [1, 2, 3]))
       >>> d =3D dict([('two', 2), ('one', 1), ('three', 3)])
       >>> e =3D dict({'three': 3, 'one': 1, 'two': 2})
-      >>> a =3D=3D b =3D=3D c =3D=3D d =3D=3D e
+      >>> f =3D dict({'one': 1, 'three': 3}, two=3D2)
+      >>> a =3D=3D b =3D=3D c =3D=3D d =3D=3D e =3D=3D f
       True
=20
    Providing keyword arguments as in the first example only works for keys t=
hat


From webhook-mailer at python.org  Thu Sep 12 07:10:08 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 11:10:08 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.516.1599366910.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-32008: Prefer client or TLSv1_2 in examples (GH-5797)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/894d0f7d5542ee04556ec1bee8c58506f7c9=
16d4
commit: 894d0f7d5542ee04556ec1bee8c58506f7c916d4
branch: master
author: Christian Heimes <christian at python.org>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-12T12:10:05+01:00
summary:

bpo-32008: Prefer client or TLSv1_2 in examples (GH-5797)

Prefer client or TLSv1_2 in examples

Signed-off-by: Christian Heimes <christian at python.org>

files:
M Doc/library/ssl.rst

diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index 2db48ae841da..b7af496e1cfe 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -1882,13 +1882,15 @@ to speed up repeated connections from the same client=
s.
    :meth:`~SSLContext.wrap_socket` in order to match the hostname.  Enabling
    hostname checking automatically sets :attr:`~SSLContext.verify_mode` from
    :data:`CERT_NONE` to :data:`CERT_REQUIRED`.  It cannot be set back to
-   :data:`CERT_NONE` as long as hostname checking is enabled.
+   :data:`CERT_NONE` as long as hostname checking is enabled. The
+   :data:`PROTOCOL_TLS_CLIENT` protocol enables hostname checking by default.
+   With other protocols, hostname checking must be enabled explicitly.
=20
    Example::
=20
       import socket, ssl
=20
-      context =3D ssl.SSLContext()
+      context =3D ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
       context.verify_mode =3D ssl.CERT_REQUIRED
       context.check_hostname =3D True
       context.load_default_certs()
@@ -2217,19 +2219,23 @@ If you prefer to tune security settings yourself, you=
 might create
 a context from scratch (but beware that you might not get the settings
 right)::
=20
-   >>> context =3D ssl.SSLContext()
-   >>> context.verify_mode =3D ssl.CERT_REQUIRED
-   >>> context.check_hostname =3D True
+   >>> context =3D ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
    >>> context.load_verify_locations("/etc/ssl/certs/ca-bundle.crt")
=20
 (this snippet assumes your operating system places a bundle of all CA
 certificates in ``/etc/ssl/certs/ca-bundle.crt``; if not, you'll get an
 error and have to adjust the location)
=20
+The :data:`PROTOCOL_TLS_CLIENT` protocol configures the context for cert
+validation and hostname verification. :attr:`~SSLContext.verify_mode` is
+set to :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` is set
+to ``True``. All other protocols create SSL contexts with insecure defaults.
+
 When you use the context to connect to a server, :const:`CERT_REQUIRED`
-validates the server certificate: it ensures that the server certificate
-was signed with one of the CA certificates, and checks the signature for
-correctness::
+and :attr:`~SSLContext.check_hostname` validate the server certificate: it
+ensures that the server certificate was signed with one of the CA
+certificates, checks the signature for correctness, and verifies other
+properties like validity and identity of the hostname::
=20
    >>> conn =3D context.wrap_socket(socket.socket(socket.AF_INET),
    ...                            server_hostname=3D"www.python.org")


From webhook-mailer at python.org  Thu Sep 12 07:20:45 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 11:20:45 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.517.1599366912.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-32008: Prefer client or TLSv1_2 in examples (GH-5797) (GH-16026)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/07b4148f3932ed22f9cd8476f49bd0079d09=
3590
commit: 07b4148f3932ed22f9cd8476f49bd0079d093590
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-12T12:20:41+01:00
summary:

bpo-32008: Prefer client or TLSv1_2 in examples (GH-5797) (GH-16026)

Prefer client or TLSv1_2 in examples

Signed-off-by: Christian Heimes <christian at python.org>
(cherry picked from commit 894d0f7d5542ee04556ec1bee8c58506f7c916d4)

Co-authored-by: Christian Heimes <christian at python.org>

files:
M Doc/library/ssl.rst

diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index 1133ebf25f5b..b22136810d28 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -1873,13 +1873,15 @@ to speed up repeated connections from the same client=
s.
    :meth:`~SSLContext.wrap_socket` in order to match the hostname.  Enabling
    hostname checking automatically sets :attr:`~SSLContext.verify_mode` from
    :data:`CERT_NONE` to :data:`CERT_REQUIRED`.  It cannot be set back to
-   :data:`CERT_NONE` as long as hostname checking is enabled.
+   :data:`CERT_NONE` as long as hostname checking is enabled. The
+   :data:`PROTOCOL_TLS_CLIENT` protocol enables hostname checking by default.
+   With other protocols, hostname checking must be enabled explicitly.
=20
    Example::
=20
       import socket, ssl
=20
-      context =3D ssl.SSLContext()
+      context =3D ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
       context.verify_mode =3D ssl.CERT_REQUIRED
       context.check_hostname =3D True
       context.load_default_certs()
@@ -2181,19 +2183,23 @@ If you prefer to tune security settings yourself, you=
 might create
 a context from scratch (but beware that you might not get the settings
 right)::
=20
-   >>> context =3D ssl.SSLContext()
-   >>> context.verify_mode =3D ssl.CERT_REQUIRED
-   >>> context.check_hostname =3D True
+   >>> context =3D ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
    >>> context.load_verify_locations("/etc/ssl/certs/ca-bundle.crt")
=20
 (this snippet assumes your operating system places a bundle of all CA
 certificates in ``/etc/ssl/certs/ca-bundle.crt``; if not, you'll get an
 error and have to adjust the location)
=20
+The :data:`PROTOCOL_TLS_CLIENT` protocol configures the context for cert
+validation and hostname verification. :attr:`~SSLContext.verify_mode` is
+set to :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` is set
+to ``True``. All other protocols create SSL contexts with insecure defaults.
+
 When you use the context to connect to a server, :const:`CERT_REQUIRED`
-validates the server certificate: it ensures that the server certificate
-was signed with one of the CA certificates, and checks the signature for
-correctness::
+and :attr:`~SSLContext.check_hostname` validate the server certificate: it
+ensures that the server certificate was signed with one of the CA
+certificates, checks the signature for correctness, and verifies other
+properties like validity and identity of the hostname::
=20
    >>> conn =3D context.wrap_socket(socket.socket(socket.AF_INET),
    ...                            server_hostname=3D"www.python.org")


From webhook-mailer at python.org  Thu Sep 12 07:25:05 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 11:25:05 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.518.1599366913.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 [3.8] bpo-32008: Prefer client or TLSv1_2 in examples (GH-5797) (GH-16027)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/1fc84b64f9f740f2dc089da1d061dfdd5b43=
8d3c
commit: 1fc84b64f9f740f2dc089da1d061dfdd5b438d3c
branch: 3.8
author: St=C3=A9phane Wirtel <stephane at wirtel.be>
committer: GitHub <noreply at github.com>
date: 2019-09-12T12:25:02+01:00
summary:

[3.8] bpo-32008: Prefer client or TLSv1_2 in examples (GH-5797) (GH-16027)

Prefer client or TLSv1_2 in examples

Signed-off-by: Christian Heimes <christian at python.org>
(cherry picked from commit 894d0f7d5542ee04556ec1bee8c58506f7c916d4)

Co-authored-by: Christian Heimes <christian at python.org>

files:
M Doc/library/ssl.rst

diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index 279af5728913..9b0ba8ed1b8b 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -1882,13 +1882,15 @@ to speed up repeated connections from the same client=
s.
    :meth:`~SSLContext.wrap_socket` in order to match the hostname.  Enabling
    hostname checking automatically sets :attr:`~SSLContext.verify_mode` from
    :data:`CERT_NONE` to :data:`CERT_REQUIRED`.  It cannot be set back to
-   :data:`CERT_NONE` as long as hostname checking is enabled.
+   :data:`CERT_NONE` as long as hostname checking is enabled. The
+   :data:`PROTOCOL_TLS_CLIENT` protocol enables hostname checking by default.
+   With other protocols, hostname checking must be enabled explicitly.
=20
    Example::
=20
       import socket, ssl
=20
-      context =3D ssl.SSLContext()
+      context =3D ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
       context.verify_mode =3D ssl.CERT_REQUIRED
       context.check_hostname =3D True
       context.load_default_certs()
@@ -2217,19 +2219,23 @@ If you prefer to tune security settings yourself, you=
 might create
 a context from scratch (but beware that you might not get the settings
 right)::
=20
-   >>> context =3D ssl.SSLContext()
-   >>> context.verify_mode =3D ssl.CERT_REQUIRED
-   >>> context.check_hostname =3D True
+   >>> context =3D ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
    >>> context.load_verify_locations("/etc/ssl/certs/ca-bundle.crt")
=20
 (this snippet assumes your operating system places a bundle of all CA
 certificates in ``/etc/ssl/certs/ca-bundle.crt``; if not, you'll get an
 error and have to adjust the location)
=20
+The :data:`PROTOCOL_TLS_CLIENT` protocol configures the context for cert
+validation and hostname verification. :attr:`~SSLContext.verify_mode` is
+set to :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` is set
+to ``True``. All other protocols create SSL contexts with insecure defaults.
+
 When you use the context to connect to a server, :const:`CERT_REQUIRED`
-validates the server certificate: it ensures that the server certificate
-was signed with one of the CA certificates, and checks the signature for
-correctness::
+and :attr:`~SSLContext.check_hostname` validate the server certificate: it
+ensures that the server certificate was signed with one of the CA
+certificates, checks the signature for correctness, and verifies other
+properties like validity and identity of the hostname::
=20
    >>> conn =3D context.wrap_socket(socket.socket(socket.AF_INET),
    ...                            server_hostname=3D"www.python.org")


From webhook-mailer at python.org  Thu Sep 12 08:11:24 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 12:11:24 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.519.1599366915.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-26868: Fix example usage of PyModule_AddObject. (#15725)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/224b8aaa7e8f67f748e8b7b6a4a77a25f655=
4651
commit: 224b8aaa7e8f67f748e8b7b6a4a77a25f6554651
branch: master
author: Brandt Bucher <brandtbucher at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-12T13:11:20+01:00
summary:

bpo-26868: Fix example usage of PyModule_AddObject. (#15725)

* Add a note to the PyModule_AddObject docs.

* Correct example usages of PyModule_AddObject.

* Whitespace.

* Clean up wording.

* =F0=9F=93=9C=F0=9F=A4=96 Added by blurb_it.

* First code review.

* Add < 0 in the tests with PyModule_AddObject

files:
A Misc/NEWS.d/next/Documentation/2019-09-07-15-55-46.bpo-26868.Raw0Gd.rst
M Doc/c-api/module.rst
M Doc/extending/extending.rst
M Doc/extending/newtypes_tutorial.rst
M Doc/includes/custom.c
M Doc/includes/custom2.c
M Doc/includes/custom3.c
M Doc/includes/custom4.c
M Doc/includes/sublist.c

diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index 68cbda2938f3..feca1ec2a057 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -417,7 +417,22 @@ state:
=20
    Add an object to *module* as *name*.  This is a convenience function whic=
h can
    be used from the module's initialization function.  This steals a referen=
ce to
-   *value*.  Return ``-1`` on error, ``0`` on success.
+   *value* on success.  Return ``-1`` on error, ``0`` on success.
+
+   .. note::
+
+      Unlike other functions that steal references, ``PyModule_AddObject()``=
 only
+      decrements the reference count of *value* **on success**.
+
+      This means that its return value must be checked, and calling code must
+      :c:func:`Py_DECREF` *value* manually on error. Example usage::
+
+         Py_INCREF(spam);
+         if (PyModule_AddObject(module, "spam", spam) < 0) {
+             Py_DECREF(module);
+             Py_DECREF(spam);
+             return NULL;
+         }
=20
 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *na=
me, long value)
=20
diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst
index e459514b2b58..5b4ea8220e23 100644
--- a/Doc/extending/extending.rst
+++ b/Doc/extending/extending.rst
@@ -209,7 +209,7 @@ usually declare a static object variable at the beginning=
 of your file::
    static PyObject *SpamError;
=20
 and initialize it in your module's initialization function (:c:func:`PyInit_=
spam`)
-with an exception object (leaving out the error checking for now)::
+with an exception object::
=20
    PyMODINIT_FUNC
    PyInit_spam(void)
@@ -221,8 +221,14 @@ with an exception object (leaving out the error checking=
 for now)::
            return NULL;
=20
        SpamError =3D PyErr_NewException("spam.error", NULL, NULL);
-       Py_INCREF(SpamError);
-       PyModule_AddObject(m, "error", SpamError);
+       Py_XINCREF(SpamError);
+       if (PyModule_AddObject(m, "error", SpamError) < 0) {
+           Py_XDECREF(SpamError);
+           Py_CLEAR(SpamError);
+           Py_DECREF(m);
+           return NULL;
+       }
+
        return m;
    }
=20
@@ -1261,8 +1267,12 @@ function must take care of initializing the C API poin=
ter array::
        /* Create a Capsule containing the API pointer array's address */
        c_api_object =3D PyCapsule_New((void *)PySpam_API, "spam._C_API", NUL=
L);
=20
-       if (c_api_object !=3D NULL)
-           PyModule_AddObject(m, "_C_API", c_api_object);
+       if (PyModule_AddObject(m, "_C_API", c_api_object) < 0) {
+           Py_XDECREF(c_api_object);
+           Py_DECREF(m);
+           return NULL;
+       }
+
        return m;
    }
=20
diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tut=
orial.rst
index 59c8cc01222b..94ca747c7aeb 100644
--- a/Doc/extending/newtypes_tutorial.rst
+++ b/Doc/extending/newtypes_tutorial.rst
@@ -179,7 +179,12 @@ This initializes the :class:`Custom` type, filling in a =
number of members
 to the appropriate default values, including :attr:`ob_type` that we initial=
ly
 set to *NULL*. ::
=20
-   PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+   Py_INCREF(&CustomType);
+   if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+       Py_DECREF(&CustomType);
+       PY_DECREF(m);
+       return NULL;
+   }
=20
 This adds the type to the module dictionary.  This allows us to create
 :class:`Custom` instances by calling the :class:`Custom` class:
@@ -864,7 +869,12 @@ function::
            return NULL;
=20
        Py_INCREF(&SubListType);
-       PyModule_AddObject(m, "SubList", (PyObject *) &SubListType);
+       if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) {
+           Py_DECREF(&SubListType);
+           Py_DECREF(m);
+           return NULL;
+       }
+
        return m;
    }
=20
diff --git a/Doc/includes/custom.c b/Doc/includes/custom.c
index 13d16f5424ae..bda32e2ad81d 100644
--- a/Doc/includes/custom.c
+++ b/Doc/includes/custom.c
@@ -35,6 +35,11 @@ PyInit_custom(void)
         return NULL;
=20
     Py_INCREF(&CustomType);
-    PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+    if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+        Py_DECREF(&CustomType);
+        PY_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Doc/includes/custom2.c b/Doc/includes/custom2.c
index 6477a19dafed..5bacab7a2a97 100644
--- a/Doc/includes/custom2.c
+++ b/Doc/includes/custom2.c
@@ -128,6 +128,11 @@ PyInit_custom2(void)
         return NULL;
=20
     Py_INCREF(&CustomType);
-    PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+    if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+        Py_DECREF(&CustomType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Doc/includes/custom3.c b/Doc/includes/custom3.c
index 213d0864ce1c..2b7a99ecf96c 100644
--- a/Doc/includes/custom3.c
+++ b/Doc/includes/custom3.c
@@ -179,6 +179,11 @@ PyInit_custom3(void)
         return NULL;
=20
     Py_INCREF(&CustomType);
-    PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+    if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+        Py_DECREF(&CustomType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Doc/includes/custom4.c b/Doc/includes/custom4.c
index b0b2906dbdc8..584992fc5f1a 100644
--- a/Doc/includes/custom4.c
+++ b/Doc/includes/custom4.c
@@ -193,6 +193,11 @@ PyInit_custom4(void)
         return NULL;
=20
     Py_INCREF(&CustomType);
-    PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+    if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
+        Py_DECREF(&CustomType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Doc/includes/sublist.c b/Doc/includes/sublist.c
index 76ff93948cfd..b2c26e73ebaf 100644
--- a/Doc/includes/sublist.c
+++ b/Doc/includes/sublist.c
@@ -59,6 +59,11 @@ PyInit_sublist(void)
         return NULL;
=20
     Py_INCREF(&SubListType);
-    PyModule_AddObject(m, "SubList", (PyObject *) &SubListType);
+    if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) {
+        Py_DECREF(&SubListType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
 }
diff --git a/Misc/NEWS.d/next/Documentation/2019-09-07-15-55-46.bpo-26868.Raw=
0Gd.rst b/Misc/NEWS.d/next/Documentation/2019-09-07-15-55-46.bpo-26868.Raw0Gd=
.rst
new file mode 100644
index 000000000000..c668092b41a6
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-09-07-15-55-46.bpo-26868.Raw0Gd.rst
@@ -0,0 +1 @@
+Fix example usage of :c:func:`PyModule_AddObject` to properly handle errors.
\ No newline at end of file


From webhook-mailer at python.org  Thu Sep 12 08:42:11 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 12:42:11 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.520.1599366916.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-38132: Simplify _hashopenssl code (GH-16023)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/5a4f82f457049b5b07b6fba4ca42bc1ecf59=
7976
commit: 5a4f82f457049b5b07b6fba4ca42bc1ecf597976
branch: master
author: Christian Heimes <christian at python.org>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-12T13:42:07+01:00
summary:

bpo-38132: Simplify _hashopenssl code (GH-16023)

Signed-off-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Library/2019-09-12-12-47-35.bpo-38132.KSFx1F.rst
M Lib/test/test_hashlib.py
M Modules/_hashopenssl.c
M Modules/clinic/_hashopenssl.c.h

diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index f83f73a0cc45..b7b04a37f0d0 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -194,6 +194,9 @@ def test_unknown_hash(self):
         self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam=
')
         self.assertRaises(TypeError, hashlib.new, 1)
=20
+    def test_new_upper_to_lower(self):
+        self.assertEqual(hashlib.new("SHA256").name, "sha256")
+
     def test_get_builtin_constructor(self):
         get_builtin_constructor =3D getattr(hashlib,
                                           '__get_builtin_constructor')
diff --git a/Misc/NEWS.d/next/Library/2019-09-12-12-47-35.bpo-38132.KSFx1F.rs=
t b/Misc/NEWS.d/next/Library/2019-09-12-12-47-35.bpo-38132.KSFx1F.rst
new file mode 100644
index 000000000000..c2e3b6290d9e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-12-12-47-35.bpo-38132.KSFx1F.rst
@@ -0,0 +1,3 @@
+The OpenSSL hashlib wrapper uses a simpler implementation. Several Macros
+and pointless caches are gone. The hash name now comes from OpenSSL's EVP.
+The algorithm name stays the same, except it is now always lower case.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 5e76853b3411..798317fb58b9 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -30,7 +30,6 @@
=20
 typedef struct {
     PyObject_HEAD
-    PyObject            *name;  /* name of this hash algorithm */
     EVP_MD_CTX          *ctx;   /* OpenSSL message digest context */
     PyThread_type_lock   lock;  /* OpenSSL context lock */
 } EVPobject;
@@ -38,18 +37,6 @@ typedef struct {
=20
 static PyTypeObject EVPtype;
=20
-
-#define DEFINE_CONSTS_FOR_NEW(Name)  \
-    static PyObject *CONST_ ## Name ## _name_obj =3D NULL; \
-    static EVP_MD_CTX *CONST_new_ ## Name ## _ctx_p =3D NULL;
-
-DEFINE_CONSTS_FOR_NEW(md5)
-DEFINE_CONSTS_FOR_NEW(sha1)
-DEFINE_CONSTS_FOR_NEW(sha224)
-DEFINE_CONSTS_FOR_NEW(sha256)
-DEFINE_CONSTS_FOR_NEW(sha384)
-DEFINE_CONSTS_FOR_NEW(sha512)
-
 #include "clinic/_hashopenssl.c.h"
 /*[clinic input]
 module _hashlib
@@ -90,16 +77,13 @@ _setException(PyObject *exc)
 /* LCOV_EXCL_STOP */
=20
 static EVPobject *
-newEVPobject(PyObject *name)
+newEVPobject(void)
 {
     EVPobject *retval =3D (EVPobject *)PyObject_New(EVPobject, &EVPtype);
     if (retval =3D=3D NULL) {
         return NULL;
     }
=20
-    /* save the name for .name to return */
-    Py_INCREF(name);
-    retval->name =3D name;
     retval->lock =3D NULL;
=20
     retval->ctx =3D EVP_MD_CTX_new();
@@ -139,7 +123,6 @@ EVP_dealloc(EVPobject *self)
     if (self->lock !=3D NULL)
         PyThread_free_lock(self->lock);
     EVP_MD_CTX_free(self->ctx);
-    Py_XDECREF(self->name);
     PyObject_Del(self);
 }
=20
@@ -167,7 +150,7 @@ EVP_copy_impl(EVPobject *self)
 {
     EVPobject *newobj;
=20
-    if ( (newobj =3D newEVPobject(self->name))=3D=3DNULL)
+    if ( (newobj =3D newEVPobject())=3D=3DNULL)
         return NULL;
=20
     if (!locked_EVP_MD_CTX_copy(newobj->ctx, self)) {
@@ -307,10 +290,20 @@ EVP_get_digest_size(EVPobject *self, void *closure)
     return PyLong_FromLong(size);
 }
=20
-static PyMemberDef EVP_members[] =3D {
-    {"name", T_OBJECT, offsetof(EVPobject, name), READONLY, PyDoc_STR("algor=
ithm name.")},
-    {NULL}  /* Sentinel */
-};
+static PyObject *
+EVP_get_name(EVPobject *self, void *closure)
+{
+    const char *name =3D EVP_MD_name(EVP_MD_CTX_md(self->ctx));
+    PyObject *name_obj, *name_lower;
+
+    name_obj =3D PyUnicode_FromString(name);
+    if (!name_obj) {
+        return NULL;
+    }
+    name_lower =3D PyObject_CallMethod(name_obj, "lower", NULL);
+    Py_DECREF(name_obj);
+    return name_lower;
+}
=20
 static PyGetSetDef EVP_getseters[] =3D {
     {"digest_size",
@@ -321,6 +314,10 @@ static PyGetSetDef EVP_getseters[] =3D {
      (getter)EVP_get_block_size, NULL,
      NULL,
      NULL},
+    {"name",
+     (getter)EVP_get_name, NULL,
+     NULL,
+     PyDoc_STR("algorithm name.")},
     {NULL}  /* Sentinel */
 };
=20
@@ -328,7 +325,14 @@ static PyGetSetDef EVP_getseters[] =3D {
 static PyObject *
 EVP_repr(EVPobject *self)
 {
-    return PyUnicode_FromFormat("<%U HASH object @ %p>", self->name, self);
+    PyObject *name_obj, *repr;
+    name_obj =3D EVP_get_name(self, NULL);
+    if (!name_obj) {
+        return NULL;
+    }
+    repr =3D PyUnicode_FromFormat("<%U HASH object @ %p>", name_obj, self);
+    Py_DECREF(name_obj);
+    return repr;
 }
=20
 PyDoc_STRVAR(hashtype_doc,
@@ -379,7 +383,7 @@ static PyTypeObject EVPtype =3D {
     0,                  /*tp_iter*/
     0,                  /*tp_iternext*/
     EVP_methods,        /* tp_methods */
-    EVP_members,        /* tp_members */
+    NULL,        /* tp_members */
     EVP_getseters,      /* tp_getset */
     0,                  /* tp_base */
     0,                  /* tp_dict */
@@ -389,28 +393,23 @@ static PyTypeObject EVPtype =3D {
 };
=20
 static PyObject *
-EVPnew(PyObject *name_obj,
-       const EVP_MD *digest, const EVP_MD_CTX *initial_ctx,
+EVPnew(const EVP_MD *digest,
        const unsigned char *cp, Py_ssize_t len)
 {
     EVPobject *self;
=20
-    if (!digest && !initial_ctx) {
+    if (!digest) {
         PyErr_SetString(PyExc_ValueError, "unsupported hash type");
         return NULL;
     }
=20
-    if ((self =3D newEVPobject(name_obj)) =3D=3D NULL)
+    if ((self =3D newEVPobject()) =3D=3D NULL)
         return NULL;
=20
-    if (initial_ctx) {
-        EVP_MD_CTX_copy(self->ctx, initial_ctx);
-    } else {
-        if (!EVP_DigestInit(self->ctx, digest)) {
-            _setException(PyExc_ValueError);
-            Py_DECREF(self);
-            return NULL;
-        }
+    if (!EVP_DigestInit(self->ctx, digest)) {
+        _setException(PyExc_ValueError);
+        Py_DECREF(self);
+        return NULL;
     }
=20
     if (cp && len) {
@@ -462,13 +461,132 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyO=
bject *data_obj)
=20
     digest =3D EVP_get_digestbyname(name);
=20
-    ret_obj =3D EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf, vie=
w.len);
+    ret_obj =3D EVPnew(digest, (unsigned char*)view.buf, view.len);
=20
     if (data_obj)
         PyBuffer_Release(&view);
     return ret_obj;
 }
=20
+static PyObject*
+EVP_fast_new(PyObject *module, PyObject *data_obj, const EVP_MD *digest)
+{
+    Py_buffer view =3D { 0 };
+    PyObject *ret_obj;
+
+    if (data_obj)
+        GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view);
+
+    ret_obj =3D EVPnew(digest, (unsigned char*)view.buf, view.len);
+
+    if (data_obj)
+        PyBuffer_Release(&view);
+
+    return ret_obj;
+}
+
+/*[clinic input]
+_hashlib.openssl_md5
+
+    string as data_obj: object(py_default=3D"b''") =3D NULL
+
+Returns a md5 hash object; optionally initialized with a string
+
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj)
+/*[clinic end generated code: output=3D6caae75b73e22c3f input=3D52010d3869e1=
b1a7]*/
+{
+    return EVP_fast_new(module, data_obj, EVP_md5());
+}
+
+
+/*[clinic input]
+_hashlib.openssl_sha1
+
+    string as data_obj: object(py_default=3D"b''") =3D NULL
+
+Returns a sha1 hash object; optionally initialized with a string
+
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj)
+/*[clinic end generated code: output=3D07606d8f75153e61 input=3D16807d30e4aa=
8ae9]*/
+{
+    return EVP_fast_new(module, data_obj, EVP_sha1());
+}
+
+
+/*[clinic input]
+_hashlib.openssl_sha224
+
+    string as data_obj: object(py_default=3D"b''") =3D NULL
+
+Returns a sha224 hash object; optionally initialized with a string
+
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj)
+/*[clinic end generated code: output=3D55e848761bcef0c9 input=3D5dbc2f1d84eb=
459b]*/
+{
+    return EVP_fast_new(module, data_obj, EVP_sha224());
+}
+
+
+/*[clinic input]
+_hashlib.openssl_sha256
+
+    string as data_obj: object(py_default=3D"b''") =3D NULL
+
+Returns a sha256 hash object; optionally initialized with a string
+
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj)
+/*[clinic end generated code: output=3D05851d7cce34ac65 input=3Da68a5d21cda5=
a80f]*/
+{
+    return EVP_fast_new(module, data_obj, EVP_sha256());
+}
+
+
+/*[clinic input]
+_hashlib.openssl_sha384
+
+    string as data_obj: object(py_default=3D"b''") =3D NULL
+
+Returns a sha384 hash object; optionally initialized with a string
+
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj)
+/*[clinic end generated code: output=3D5101a4704a932c2f input=3D6bdfa006622b=
64ea]*/
+{
+    return EVP_fast_new(module, data_obj, EVP_sha384());
+}
+
+
+/*[clinic input]
+_hashlib.openssl_sha512
+
+    string as data_obj: object(py_default=3D"b''") =3D NULL
+
+Returns a sha512 hash object; optionally initialized with a string
+
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj)
+/*[clinic end generated code: output=3D20c8e63ee560a5cb input=3Dece50182ad4b=
76a6]*/
+{
+    return EVP_fast_new(module, data_obj, EVP_sha512());
+}
+
+
 /*[clinic input]
 _hashlib.pbkdf2_hmac as pbkdf2_hmac
=20
@@ -800,76 +918,6 @@ generate_hash_name_list(void)
     return state.set;
 }
=20
-
-/*
- *  This macro generates constructor function definitions for specific
- *  hash algorithms.  These constructors are much faster than calling
- *  the generic one passing it a python string and are noticeably
- *  faster than calling a python new() wrapper.  That is important for
- *  code that wants to make hashes of a bunch of small strings.
- *  The first call will lazy-initialize, which reports an exception
- *  if initialization fails.
- */
-#define GEN_CONSTRUCTOR(NAME)  \
-    static PyObject * \
-    EVP_new_ ## NAME (PyObject *self, PyObject *const *args, Py_ssize_t narg=
s) \
-    { \
-        PyObject *data_obj =3D NULL; \
-        Py_buffer view =3D { 0 }; \
-        PyObject *ret_obj; \
-     \
-        if (!_PyArg_ParseStack(args, nargs, "|O:" #NAME , &data_obj)) { \
-            return NULL; \
-        } \
-     \
-        if (CONST_new_ ## NAME ## _ctx_p =3D=3D NULL) { \
-            EVP_MD_CTX *ctx_p =3D EVP_MD_CTX_new(); \
-            if (!EVP_get_digestbyname(#NAME) || \
-                !EVP_DigestInit(ctx_p, EVP_get_digestbyname(#NAME))) { \
-                _setException(PyExc_ValueError); \
-                EVP_MD_CTX_free(ctx_p); \
-                return NULL; \
-            } \
-            CONST_new_ ## NAME ## _ctx_p =3D ctx_p; \
-        } \
-     \
-        if (data_obj) \
-            GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); \
-     \
-        ret_obj =3D EVPnew( \
-                    CONST_ ## NAME ## _name_obj, \
-                    NULL, \
-                    CONST_new_ ## NAME ## _ctx_p, \
-                    (unsigned char*)view.buf, \
-                    view.len); \
-     \
-        if (data_obj) \
-            PyBuffer_Release(&view); \
-        return ret_obj; \
-    }
-
-/* a PyMethodDef structure for the constructor */
-#define CONSTRUCTOR_METH_DEF(NAME)  \
-    {"openssl_" #NAME, (PyCFunction)(void(*)(void))EVP_new_ ## NAME, METH_FA=
STCALL, \
-        PyDoc_STR("Returns a " #NAME \
-                  " hash object; optionally initialized with a string") \
-    }
-
-/* used in the init function to setup a constructor: initialize OpenSSL
-   constructor constants if they haven't been initialized already.  */
-#define INIT_CONSTRUCTOR_CONSTANTS(NAME)  do { \
-    if (CONST_ ## NAME ## _name_obj =3D=3D NULL) { \
-        CONST_ ## NAME ## _name_obj =3D PyUnicode_FromString(#NAME); \
-    } \
-} while (0);
-
-GEN_CONSTRUCTOR(md5)
-GEN_CONSTRUCTOR(sha1)
-GEN_CONSTRUCTOR(sha224)
-GEN_CONSTRUCTOR(sha256)
-GEN_CONSTRUCTOR(sha384)
-GEN_CONSTRUCTOR(sha512)
-
 /* List of functions exported by this module */
=20
 static struct PyMethodDef EVP_functions[] =3D {
@@ -877,12 +925,12 @@ static struct PyMethodDef EVP_functions[] =3D {
     PBKDF2_HMAC_METHODDEF
     _HASHLIB_SCRYPT_METHODDEF
     _HASHLIB_HMAC_DIGEST_METHODDEF
-    CONSTRUCTOR_METH_DEF(md5),
-    CONSTRUCTOR_METH_DEF(sha1),
-    CONSTRUCTOR_METH_DEF(sha224),
-    CONSTRUCTOR_METH_DEF(sha256),
-    CONSTRUCTOR_METH_DEF(sha384),
-    CONSTRUCTOR_METH_DEF(sha512),
+    _HASHLIB_OPENSSL_MD5_METHODDEF
+    _HASHLIB_OPENSSL_SHA1_METHODDEF
+    _HASHLIB_OPENSSL_SHA224_METHODDEF
+    _HASHLIB_OPENSSL_SHA256_METHODDEF
+    _HASHLIB_OPENSSL_SHA384_METHODDEF
+    _HASHLIB_OPENSSL_SHA512_METHODDEF
     {NULL,      NULL}            /* Sentinel */
 };
=20
@@ -939,12 +987,5 @@ PyInit__hashlib(void)
     Py_INCREF((PyObject *)&EVPtype);
     PyModule_AddObject(m, "HASH", (PyObject *)&EVPtype);
=20
-    /* these constants are used by the convenience constructors */
-    INIT_CONSTRUCTOR_CONSTANTS(md5);
-    INIT_CONSTRUCTOR_CONSTANTS(sha1);
-    INIT_CONSTRUCTOR_CONSTANTS(sha224);
-    INIT_CONSTRUCTOR_CONSTANTS(sha256);
-    INIT_CONSTRUCTOR_CONSTANTS(sha384);
-    INIT_CONSTRUCTOR_CONSTANTS(sha512);
     return m;
 }
diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h
index 854d92537736..9aaea47e8328 100644
--- a/Modules/clinic/_hashopenssl.c.h
+++ b/Modules/clinic/_hashopenssl.c.h
@@ -109,6 +109,228 @@ EVP_new(PyObject *module, PyObject *const *args, Py_ssi=
ze_t nargs, PyObject *kwn
     return return_value;
 }
=20
+PyDoc_STRVAR(_hashlib_openssl_md5__doc__,
+"openssl_md5($module, /, string=3Db\'\')\n"
+"--\n"
+"\n"
+"Returns a md5 hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_MD5_METHODDEF    \
+    {"openssl_md5", (PyCFunction)(void(*)(void))_hashlib_openssl_md5, METH_F=
ASTCALL|METH_KEYWORDS, _hashlib_openssl_md5__doc__},
+
+static PyObject *
+_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj);
+
+static PyObject *
+_hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nar=
gs, PyObject *kwnames)
+{
+    PyObject *return_value =3D NULL;
+    static const char * const _keywords[] =3D {"string", NULL};
+    static _PyArg_Parser _parser =3D {NULL, _keywords, "openssl_md5", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs =3D nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0=
) - 0;
+    PyObject *data_obj =3D NULL;
+
+    args =3D _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, =
1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_pos;
+    }
+    data_obj =3D args[0];
+skip_optional_pos:
+    return_value =3D _hashlib_openssl_md5_impl(module, data_obj);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_hashlib_openssl_sha1__doc__,
+"openssl_sha1($module, /, string=3Db\'\')\n"
+"--\n"
+"\n"
+"Returns a sha1 hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_SHA1_METHODDEF    \
+    {"openssl_sha1", (PyCFunction)(void(*)(void))_hashlib_openssl_sha1, METH=
_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha1__doc__},
+
+static PyObject *
+_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj);
+
+static PyObject *
+_hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t na=
rgs, PyObject *kwnames)
+{
+    PyObject *return_value =3D NULL;
+    static const char * const _keywords[] =3D {"string", NULL};
+    static _PyArg_Parser _parser =3D {NULL, _keywords, "openssl_sha1", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs =3D nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0=
) - 0;
+    PyObject *data_obj =3D NULL;
+
+    args =3D _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, =
1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_pos;
+    }
+    data_obj =3D args[0];
+skip_optional_pos:
+    return_value =3D _hashlib_openssl_sha1_impl(module, data_obj);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_hashlib_openssl_sha224__doc__,
+"openssl_sha224($module, /, string=3Db\'\')\n"
+"--\n"
+"\n"
+"Returns a sha224 hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_SHA224_METHODDEF    \
+    {"openssl_sha224", (PyCFunction)(void(*)(void))_hashlib_openssl_sha224, =
METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha224__doc__},
+
+static PyObject *
+_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj);
+
+static PyObject *
+_hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t =
nargs, PyObject *kwnames)
+{
+    PyObject *return_value =3D NULL;
+    static const char * const _keywords[] =3D {"string", NULL};
+    static _PyArg_Parser _parser =3D {NULL, _keywords, "openssl_sha224", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs =3D nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0=
) - 0;
+    PyObject *data_obj =3D NULL;
+
+    args =3D _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, =
1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_pos;
+    }
+    data_obj =3D args[0];
+skip_optional_pos:
+    return_value =3D _hashlib_openssl_sha224_impl(module, data_obj);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_hashlib_openssl_sha256__doc__,
+"openssl_sha256($module, /, string=3Db\'\')\n"
+"--\n"
+"\n"
+"Returns a sha256 hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_SHA256_METHODDEF    \
+    {"openssl_sha256", (PyCFunction)(void(*)(void))_hashlib_openssl_sha256, =
METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha256__doc__},
+
+static PyObject *
+_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj);
+
+static PyObject *
+_hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t =
nargs, PyObject *kwnames)
+{
+    PyObject *return_value =3D NULL;
+    static const char * const _keywords[] =3D {"string", NULL};
+    static _PyArg_Parser _parser =3D {NULL, _keywords, "openssl_sha256", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs =3D nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0=
) - 0;
+    PyObject *data_obj =3D NULL;
+
+    args =3D _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, =
1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_pos;
+    }
+    data_obj =3D args[0];
+skip_optional_pos:
+    return_value =3D _hashlib_openssl_sha256_impl(module, data_obj);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_hashlib_openssl_sha384__doc__,
+"openssl_sha384($module, /, string=3Db\'\')\n"
+"--\n"
+"\n"
+"Returns a sha384 hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_SHA384_METHODDEF    \
+    {"openssl_sha384", (PyCFunction)(void(*)(void))_hashlib_openssl_sha384, =
METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha384__doc__},
+
+static PyObject *
+_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj);
+
+static PyObject *
+_hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t =
nargs, PyObject *kwnames)
+{
+    PyObject *return_value =3D NULL;
+    static const char * const _keywords[] =3D {"string", NULL};
+    static _PyArg_Parser _parser =3D {NULL, _keywords, "openssl_sha384", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs =3D nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0=
) - 0;
+    PyObject *data_obj =3D NULL;
+
+    args =3D _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, =
1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_pos;
+    }
+    data_obj =3D args[0];
+skip_optional_pos:
+    return_value =3D _hashlib_openssl_sha384_impl(module, data_obj);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_hashlib_openssl_sha512__doc__,
+"openssl_sha512($module, /, string=3Db\'\')\n"
+"--\n"
+"\n"
+"Returns a sha512 hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_SHA512_METHODDEF    \
+    {"openssl_sha512", (PyCFunction)(void(*)(void))_hashlib_openssl_sha512, =
METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha512__doc__},
+
+static PyObject *
+_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj);
+
+static PyObject *
+_hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t =
nargs, PyObject *kwnames)
+{
+    PyObject *return_value =3D NULL;
+    static const char * const _keywords[] =3D {"string", NULL};
+    static _PyArg_Parser _parser =3D {NULL, _keywords, "openssl_sha512", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs =3D nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0=
) - 0;
+    PyObject *data_obj =3D NULL;
+
+    args =3D _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, =
1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_pos;
+    }
+    data_obj =3D args[0];
+skip_optional_pos:
+    return_value =3D _hashlib_openssl_sha512_impl(module, data_obj);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(pbkdf2_hmac__doc__,
 "pbkdf2_hmac($module, /, hash_name, password, salt, iterations,\n"
 "            dklen=3DNone)\n"
@@ -401,4 +623,4 @@ _hashlib_hmac_digest(PyObject *module, PyObject *const *a=
rgs, Py_ssize_t nargs,
 #ifndef _HASHLIB_SCRYPT_METHODDEF
     #define _HASHLIB_SCRYPT_METHODDEF
 #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */
-/*[clinic end generated code: output=3Dcfe686cb2fa042e1 input=3Da9049054013a=
1b77]*/
+/*[clinic end generated code: output=3D38c2637f67e9bb79 input=3Da9049054013a=
1b77]*/


From webhook-mailer at python.org  Thu Sep 12 09:03:54 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 13:03:54 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.521.1599366918.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-38132: Simplify _hashopenssl code (GH-16023) (#16040)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/67b90a079c39066aa9d51690fe5678c36ccb=
c258
commit: 67b90a079c39066aa9d51690fe5678c36ccbc258
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-12T14:03:50+01:00
summary:

bpo-38132: Simplify _hashopenssl code (GH-16023) (#16040)

Signed-off-by: Christian Heimes <christian at python.org>
(cherry picked from commit 5a4f82f457049b5b07b6fba4ca42bc1ecf597976)

Co-authored-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Library/2019-09-12-12-47-35.bpo-38132.KSFx1F.rst
M Lib/test/test_hashlib.py
M Modules/_hashopenssl.c
M Modules/clinic/_hashopenssl.c.h

diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index f83f73a0cc45..b7b04a37f0d0 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -194,6 +194,9 @@ def test_unknown_hash(self):
         self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam=
')
         self.assertRaises(TypeError, hashlib.new, 1)
=20
+    def test_new_upper_to_lower(self):
+        self.assertEqual(hashlib.new("SHA256").name, "sha256")
+
     def test_get_builtin_constructor(self):
         get_builtin_constructor =3D getattr(hashlib,
                                           '__get_builtin_constructor')
diff --git a/Misc/NEWS.d/next/Library/2019-09-12-12-47-35.bpo-38132.KSFx1F.rs=
t b/Misc/NEWS.d/next/Library/2019-09-12-12-47-35.bpo-38132.KSFx1F.rst
new file mode 100644
index 000000000000..c2e3b6290d9e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-12-12-47-35.bpo-38132.KSFx1F.rst
@@ -0,0 +1,3 @@
+The OpenSSL hashlib wrapper uses a simpler implementation. Several Macros
+and pointless caches are gone. The hash name now comes from OpenSSL's EVP.
+The algorithm name stays the same, except it is now always lower case.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 5e76853b3411..798317fb58b9 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -30,7 +30,6 @@
=20
 typedef struct {
     PyObject_HEAD
-    PyObject            *name;  /* name of this hash algorithm */
     EVP_MD_CTX          *ctx;   /* OpenSSL message digest context */
     PyThread_type_lock   lock;  /* OpenSSL context lock */
 } EVPobject;
@@ -38,18 +37,6 @@ typedef struct {
=20
 static PyTypeObject EVPtype;
=20
-
-#define DEFINE_CONSTS_FOR_NEW(Name)  \
-    static PyObject *CONST_ ## Name ## _name_obj =3D NULL; \
-    static EVP_MD_CTX *CONST_new_ ## Name ## _ctx_p =3D NULL;
-
-DEFINE_CONSTS_FOR_NEW(md5)
-DEFINE_CONSTS_FOR_NEW(sha1)
-DEFINE_CONSTS_FOR_NEW(sha224)
-DEFINE_CONSTS_FOR_NEW(sha256)
-DEFINE_CONSTS_FOR_NEW(sha384)
-DEFINE_CONSTS_FOR_NEW(sha512)
-
 #include "clinic/_hashopenssl.c.h"
 /*[clinic input]
 module _hashlib
@@ -90,16 +77,13 @@ _setException(PyObject *exc)
 /* LCOV_EXCL_STOP */
=20
 static EVPobject *
-newEVPobject(PyObject *name)
+newEVPobject(void)
 {
     EVPobject *retval =3D (EVPobject *)PyObject_New(EVPobject, &EVPtype);
     if (retval =3D=3D NULL) {
         return NULL;
     }
=20
-    /* save the name for .name to return */
-    Py_INCREF(name);
-    retval->name =3D name;
     retval->lock =3D NULL;
=20
     retval->ctx =3D EVP_MD_CTX_new();
@@ -139,7 +123,6 @@ EVP_dealloc(EVPobject *self)
     if (self->lock !=3D NULL)
         PyThread_free_lock(self->lock);
     EVP_MD_CTX_free(self->ctx);
-    Py_XDECREF(self->name);
     PyObject_Del(self);
 }
=20
@@ -167,7 +150,7 @@ EVP_copy_impl(EVPobject *self)
 {
     EVPobject *newobj;
=20
-    if ( (newobj =3D newEVPobject(self->name))=3D=3DNULL)
+    if ( (newobj =3D newEVPobject())=3D=3DNULL)
         return NULL;
=20
     if (!locked_EVP_MD_CTX_copy(newobj->ctx, self)) {
@@ -307,10 +290,20 @@ EVP_get_digest_size(EVPobject *self, void *closure)
     return PyLong_FromLong(size);
 }
=20
-static PyMemberDef EVP_members[] =3D {
-    {"name", T_OBJECT, offsetof(EVPobject, name), READONLY, PyDoc_STR("algor=
ithm name.")},
-    {NULL}  /* Sentinel */
-};
+static PyObject *
+EVP_get_name(EVPobject *self, void *closure)
+{
+    const char *name =3D EVP_MD_name(EVP_MD_CTX_md(self->ctx));
+    PyObject *name_obj, *name_lower;
+
+    name_obj =3D PyUnicode_FromString(name);
+    if (!name_obj) {
+        return NULL;
+    }
+    name_lower =3D PyObject_CallMethod(name_obj, "lower", NULL);
+    Py_DECREF(name_obj);
+    return name_lower;
+}
=20
 static PyGetSetDef EVP_getseters[] =3D {
     {"digest_size",
@@ -321,6 +314,10 @@ static PyGetSetDef EVP_getseters[] =3D {
      (getter)EVP_get_block_size, NULL,
      NULL,
      NULL},
+    {"name",
+     (getter)EVP_get_name, NULL,
+     NULL,
+     PyDoc_STR("algorithm name.")},
     {NULL}  /* Sentinel */
 };
=20
@@ -328,7 +325,14 @@ static PyGetSetDef EVP_getseters[] =3D {
 static PyObject *
 EVP_repr(EVPobject *self)
 {
-    return PyUnicode_FromFormat("<%U HASH object @ %p>", self->name, self);
+    PyObject *name_obj, *repr;
+    name_obj =3D EVP_get_name(self, NULL);
+    if (!name_obj) {
+        return NULL;
+    }
+    repr =3D PyUnicode_FromFormat("<%U HASH object @ %p>", name_obj, self);
+    Py_DECREF(name_obj);
+    return repr;
 }
=20
 PyDoc_STRVAR(hashtype_doc,
@@ -379,7 +383,7 @@ static PyTypeObject EVPtype =3D {
     0,                  /*tp_iter*/
     0,                  /*tp_iternext*/
     EVP_methods,        /* tp_methods */
-    EVP_members,        /* tp_members */
+    NULL,        /* tp_members */
     EVP_getseters,      /* tp_getset */
     0,                  /* tp_base */
     0,                  /* tp_dict */
@@ -389,28 +393,23 @@ static PyTypeObject EVPtype =3D {
 };
=20
 static PyObject *
-EVPnew(PyObject *name_obj,
-       const EVP_MD *digest, const EVP_MD_CTX *initial_ctx,
+EVPnew(const EVP_MD *digest,
        const unsigned char *cp, Py_ssize_t len)
 {
     EVPobject *self;
=20
-    if (!digest && !initial_ctx) {
+    if (!digest) {
         PyErr_SetString(PyExc_ValueError, "unsupported hash type");
         return NULL;
     }
=20
-    if ((self =3D newEVPobject(name_obj)) =3D=3D NULL)
+    if ((self =3D newEVPobject()) =3D=3D NULL)
         return NULL;
=20
-    if (initial_ctx) {
-        EVP_MD_CTX_copy(self->ctx, initial_ctx);
-    } else {
-        if (!EVP_DigestInit(self->ctx, digest)) {
-            _setException(PyExc_ValueError);
-            Py_DECREF(self);
-            return NULL;
-        }
+    if (!EVP_DigestInit(self->ctx, digest)) {
+        _setException(PyExc_ValueError);
+        Py_DECREF(self);
+        return NULL;
     }
=20
     if (cp && len) {
@@ -462,13 +461,132 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyO=
bject *data_obj)
=20
     digest =3D EVP_get_digestbyname(name);
=20
-    ret_obj =3D EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf, vie=
w.len);
+    ret_obj =3D EVPnew(digest, (unsigned char*)view.buf, view.len);
=20
     if (data_obj)
         PyBuffer_Release(&view);
     return ret_obj;
 }
=20
+static PyObject*
+EVP_fast_new(PyObject *module, PyObject *data_obj, const EVP_MD *digest)
+{
+    Py_buffer view =3D { 0 };
+    PyObject *ret_obj;
+
+    if (data_obj)
+        GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view);
+
+    ret_obj =3D EVPnew(digest, (unsigned char*)view.buf, view.len);
+
+    if (data_obj)
+        PyBuffer_Release(&view);
+
+    return ret_obj;
+}
+
+/*[clinic input]
+_hashlib.openssl_md5
+
+    string as data_obj: object(py_default=3D"b''") =3D NULL
+
+Returns a md5 hash object; optionally initialized with a string
+
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj)
+/*[clinic end generated code: output=3D6caae75b73e22c3f input=3D52010d3869e1=
b1a7]*/
+{
+    return EVP_fast_new(module, data_obj, EVP_md5());
+}
+
+
+/*[clinic input]
+_hashlib.openssl_sha1
+
+    string as data_obj: object(py_default=3D"b''") =3D NULL
+
+Returns a sha1 hash object; optionally initialized with a string
+
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj)
+/*[clinic end generated code: output=3D07606d8f75153e61 input=3D16807d30e4aa=
8ae9]*/
+{
+    return EVP_fast_new(module, data_obj, EVP_sha1());
+}
+
+
+/*[clinic input]
+_hashlib.openssl_sha224
+
+    string as data_obj: object(py_default=3D"b''") =3D NULL
+
+Returns a sha224 hash object; optionally initialized with a string
+
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj)
+/*[clinic end generated code: output=3D55e848761bcef0c9 input=3D5dbc2f1d84eb=
459b]*/
+{
+    return EVP_fast_new(module, data_obj, EVP_sha224());
+}
+
+
+/*[clinic input]
+_hashlib.openssl_sha256
+
+    string as data_obj: object(py_default=3D"b''") =3D NULL
+
+Returns a sha256 hash object; optionally initialized with a string
+
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj)
+/*[clinic end generated code: output=3D05851d7cce34ac65 input=3Da68a5d21cda5=
a80f]*/
+{
+    return EVP_fast_new(module, data_obj, EVP_sha256());
+}
+
+
+/*[clinic input]
+_hashlib.openssl_sha384
+
+    string as data_obj: object(py_default=3D"b''") =3D NULL
+
+Returns a sha384 hash object; optionally initialized with a string
+
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj)
+/*[clinic end generated code: output=3D5101a4704a932c2f input=3D6bdfa006622b=
64ea]*/
+{
+    return EVP_fast_new(module, data_obj, EVP_sha384());
+}
+
+
+/*[clinic input]
+_hashlib.openssl_sha512
+
+    string as data_obj: object(py_default=3D"b''") =3D NULL
+
+Returns a sha512 hash object; optionally initialized with a string
+
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj)
+/*[clinic end generated code: output=3D20c8e63ee560a5cb input=3Dece50182ad4b=
76a6]*/
+{
+    return EVP_fast_new(module, data_obj, EVP_sha512());
+}
+
+
 /*[clinic input]
 _hashlib.pbkdf2_hmac as pbkdf2_hmac
=20
@@ -800,76 +918,6 @@ generate_hash_name_list(void)
     return state.set;
 }
=20
-
-/*
- *  This macro generates constructor function definitions for specific
- *  hash algorithms.  These constructors are much faster than calling
- *  the generic one passing it a python string and are noticeably
- *  faster than calling a python new() wrapper.  That is important for
- *  code that wants to make hashes of a bunch of small strings.
- *  The first call will lazy-initialize, which reports an exception
- *  if initialization fails.
- */
-#define GEN_CONSTRUCTOR(NAME)  \
-    static PyObject * \
-    EVP_new_ ## NAME (PyObject *self, PyObject *const *args, Py_ssize_t narg=
s) \
-    { \
-        PyObject *data_obj =3D NULL; \
-        Py_buffer view =3D { 0 }; \
-        PyObject *ret_obj; \
-     \
-        if (!_PyArg_ParseStack(args, nargs, "|O:" #NAME , &data_obj)) { \
-            return NULL; \
-        } \
-     \
-        if (CONST_new_ ## NAME ## _ctx_p =3D=3D NULL) { \
-            EVP_MD_CTX *ctx_p =3D EVP_MD_CTX_new(); \
-            if (!EVP_get_digestbyname(#NAME) || \
-                !EVP_DigestInit(ctx_p, EVP_get_digestbyname(#NAME))) { \
-                _setException(PyExc_ValueError); \
-                EVP_MD_CTX_free(ctx_p); \
-                return NULL; \
-            } \
-            CONST_new_ ## NAME ## _ctx_p =3D ctx_p; \
-        } \
-     \
-        if (data_obj) \
-            GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); \
-     \
-        ret_obj =3D EVPnew( \
-                    CONST_ ## NAME ## _name_obj, \
-                    NULL, \
-                    CONST_new_ ## NAME ## _ctx_p, \
-                    (unsigned char*)view.buf, \
-                    view.len); \
-     \
-        if (data_obj) \
-            PyBuffer_Release(&view); \
-        return ret_obj; \
-    }
-
-/* a PyMethodDef structure for the constructor */
-#define CONSTRUCTOR_METH_DEF(NAME)  \
-    {"openssl_" #NAME, (PyCFunction)(void(*)(void))EVP_new_ ## NAME, METH_FA=
STCALL, \
-        PyDoc_STR("Returns a " #NAME \
-                  " hash object; optionally initialized with a string") \
-    }
-
-/* used in the init function to setup a constructor: initialize OpenSSL
-   constructor constants if they haven't been initialized already.  */
-#define INIT_CONSTRUCTOR_CONSTANTS(NAME)  do { \
-    if (CONST_ ## NAME ## _name_obj =3D=3D NULL) { \
-        CONST_ ## NAME ## _name_obj =3D PyUnicode_FromString(#NAME); \
-    } \
-} while (0);
-
-GEN_CONSTRUCTOR(md5)
-GEN_CONSTRUCTOR(sha1)
-GEN_CONSTRUCTOR(sha224)
-GEN_CONSTRUCTOR(sha256)
-GEN_CONSTRUCTOR(sha384)
-GEN_CONSTRUCTOR(sha512)
-
 /* List of functions exported by this module */
=20
 static struct PyMethodDef EVP_functions[] =3D {
@@ -877,12 +925,12 @@ static struct PyMethodDef EVP_functions[] =3D {
     PBKDF2_HMAC_METHODDEF
     _HASHLIB_SCRYPT_METHODDEF
     _HASHLIB_HMAC_DIGEST_METHODDEF
-    CONSTRUCTOR_METH_DEF(md5),
-    CONSTRUCTOR_METH_DEF(sha1),
-    CONSTRUCTOR_METH_DEF(sha224),
-    CONSTRUCTOR_METH_DEF(sha256),
-    CONSTRUCTOR_METH_DEF(sha384),
-    CONSTRUCTOR_METH_DEF(sha512),
+    _HASHLIB_OPENSSL_MD5_METHODDEF
+    _HASHLIB_OPENSSL_SHA1_METHODDEF
+    _HASHLIB_OPENSSL_SHA224_METHODDEF
+    _HASHLIB_OPENSSL_SHA256_METHODDEF
+    _HASHLIB_OPENSSL_SHA384_METHODDEF
+    _HASHLIB_OPENSSL_SHA512_METHODDEF
     {NULL,      NULL}            /* Sentinel */
 };
=20
@@ -939,12 +987,5 @@ PyInit__hashlib(void)
     Py_INCREF((PyObject *)&EVPtype);
     PyModule_AddObject(m, "HASH", (PyObject *)&EVPtype);
=20
-    /* these constants are used by the convenience constructors */
-    INIT_CONSTRUCTOR_CONSTANTS(md5);
-    INIT_CONSTRUCTOR_CONSTANTS(sha1);
-    INIT_CONSTRUCTOR_CONSTANTS(sha224);
-    INIT_CONSTRUCTOR_CONSTANTS(sha256);
-    INIT_CONSTRUCTOR_CONSTANTS(sha384);
-    INIT_CONSTRUCTOR_CONSTANTS(sha512);
     return m;
 }
diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h
index 854d92537736..9aaea47e8328 100644
--- a/Modules/clinic/_hashopenssl.c.h
+++ b/Modules/clinic/_hashopenssl.c.h
@@ -109,6 +109,228 @@ EVP_new(PyObject *module, PyObject *const *args, Py_ssi=
ze_t nargs, PyObject *kwn
     return return_value;
 }
=20
+PyDoc_STRVAR(_hashlib_openssl_md5__doc__,
+"openssl_md5($module, /, string=3Db\'\')\n"
+"--\n"
+"\n"
+"Returns a md5 hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_MD5_METHODDEF    \
+    {"openssl_md5", (PyCFunction)(void(*)(void))_hashlib_openssl_md5, METH_F=
ASTCALL|METH_KEYWORDS, _hashlib_openssl_md5__doc__},
+
+static PyObject *
+_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj);
+
+static PyObject *
+_hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nar=
gs, PyObject *kwnames)
+{
+    PyObject *return_value =3D NULL;
+    static const char * const _keywords[] =3D {"string", NULL};
+    static _PyArg_Parser _parser =3D {NULL, _keywords, "openssl_md5", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs =3D nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0=
) - 0;
+    PyObject *data_obj =3D NULL;
+
+    args =3D _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, =
1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_pos;
+    }
+    data_obj =3D args[0];
+skip_optional_pos:
+    return_value =3D _hashlib_openssl_md5_impl(module, data_obj);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_hashlib_openssl_sha1__doc__,
+"openssl_sha1($module, /, string=3Db\'\')\n"
+"--\n"
+"\n"
+"Returns a sha1 hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_SHA1_METHODDEF    \
+    {"openssl_sha1", (PyCFunction)(void(*)(void))_hashlib_openssl_sha1, METH=
_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha1__doc__},
+
+static PyObject *
+_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj);
+
+static PyObject *
+_hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t na=
rgs, PyObject *kwnames)
+{
+    PyObject *return_value =3D NULL;
+    static const char * const _keywords[] =3D {"string", NULL};
+    static _PyArg_Parser _parser =3D {NULL, _keywords, "openssl_sha1", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs =3D nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0=
) - 0;
+    PyObject *data_obj =3D NULL;
+
+    args =3D _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, =
1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_pos;
+    }
+    data_obj =3D args[0];
+skip_optional_pos:
+    return_value =3D _hashlib_openssl_sha1_impl(module, data_obj);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_hashlib_openssl_sha224__doc__,
+"openssl_sha224($module, /, string=3Db\'\')\n"
+"--\n"
+"\n"
+"Returns a sha224 hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_SHA224_METHODDEF    \
+    {"openssl_sha224", (PyCFunction)(void(*)(void))_hashlib_openssl_sha224, =
METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha224__doc__},
+
+static PyObject *
+_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj);
+
+static PyObject *
+_hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t =
nargs, PyObject *kwnames)
+{
+    PyObject *return_value =3D NULL;
+    static const char * const _keywords[] =3D {"string", NULL};
+    static _PyArg_Parser _parser =3D {NULL, _keywords, "openssl_sha224", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs =3D nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0=
) - 0;
+    PyObject *data_obj =3D NULL;
+
+    args =3D _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, =
1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_pos;
+    }
+    data_obj =3D args[0];
+skip_optional_pos:
+    return_value =3D _hashlib_openssl_sha224_impl(module, data_obj);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_hashlib_openssl_sha256__doc__,
+"openssl_sha256($module, /, string=3Db\'\')\n"
+"--\n"
+"\n"
+"Returns a sha256 hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_SHA256_METHODDEF    \
+    {"openssl_sha256", (PyCFunction)(void(*)(void))_hashlib_openssl_sha256, =
METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha256__doc__},
+
+static PyObject *
+_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj);
+
+static PyObject *
+_hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t =
nargs, PyObject *kwnames)
+{
+    PyObject *return_value =3D NULL;
+    static const char * const _keywords[] =3D {"string", NULL};
+    static _PyArg_Parser _parser =3D {NULL, _keywords, "openssl_sha256", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs =3D nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0=
) - 0;
+    PyObject *data_obj =3D NULL;
+
+    args =3D _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, =
1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_pos;
+    }
+    data_obj =3D args[0];
+skip_optional_pos:
+    return_value =3D _hashlib_openssl_sha256_impl(module, data_obj);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_hashlib_openssl_sha384__doc__,
+"openssl_sha384($module, /, string=3Db\'\')\n"
+"--\n"
+"\n"
+"Returns a sha384 hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_SHA384_METHODDEF    \
+    {"openssl_sha384", (PyCFunction)(void(*)(void))_hashlib_openssl_sha384, =
METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha384__doc__},
+
+static PyObject *
+_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj);
+
+static PyObject *
+_hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t =
nargs, PyObject *kwnames)
+{
+    PyObject *return_value =3D NULL;
+    static const char * const _keywords[] =3D {"string", NULL};
+    static _PyArg_Parser _parser =3D {NULL, _keywords, "openssl_sha384", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs =3D nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0=
) - 0;
+    PyObject *data_obj =3D NULL;
+
+    args =3D _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, =
1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_pos;
+    }
+    data_obj =3D args[0];
+skip_optional_pos:
+    return_value =3D _hashlib_openssl_sha384_impl(module, data_obj);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_hashlib_openssl_sha512__doc__,
+"openssl_sha512($module, /, string=3Db\'\')\n"
+"--\n"
+"\n"
+"Returns a sha512 hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_SHA512_METHODDEF    \
+    {"openssl_sha512", (PyCFunction)(void(*)(void))_hashlib_openssl_sha512, =
METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha512__doc__},
+
+static PyObject *
+_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj);
+
+static PyObject *
+_hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t =
nargs, PyObject *kwnames)
+{
+    PyObject *return_value =3D NULL;
+    static const char * const _keywords[] =3D {"string", NULL};
+    static _PyArg_Parser _parser =3D {NULL, _keywords, "openssl_sha512", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs =3D nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0=
) - 0;
+    PyObject *data_obj =3D NULL;
+
+    args =3D _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, =
1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_pos;
+    }
+    data_obj =3D args[0];
+skip_optional_pos:
+    return_value =3D _hashlib_openssl_sha512_impl(module, data_obj);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(pbkdf2_hmac__doc__,
 "pbkdf2_hmac($module, /, hash_name, password, salt, iterations,\n"
 "            dklen=3DNone)\n"
@@ -401,4 +623,4 @@ _hashlib_hmac_digest(PyObject *module, PyObject *const *a=
rgs, Py_ssize_t nargs,
 #ifndef _HASHLIB_SCRYPT_METHODDEF
     #define _HASHLIB_SCRYPT_METHODDEF
 #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */
-/*[clinic end generated code: output=3Dcfe686cb2fa042e1 input=3Da9049054013a=
1b77]*/
+/*[clinic end generated code: output=3D38c2637f67e9bb79 input=3Da9049054013a=
1b77]*/


From webhook-mailer at python.org  Thu Sep 12 09:19:10 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 13:19:10 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.522.1599366920.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Add -m reference in context of code execution (GH-16045)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/a45b0efdeaf27c3f8a630fb8c3670df6f0d4=
2d20
commit: a45b0efdeaf27c3f8a630fb8c3670df6f0d42d20
branch: master
author: Julien Palard <julien at palard.fr>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-12T14:19:06+01:00
summary:

Doc: Add -m reference in context of code execution (GH-16045)

files:
M Doc/reference/executionmodel.rst

diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.=
rst
index 49cb86b56084..55ac01b6a844 100644
--- a/Doc/reference/executionmodel.rst
+++ b/Doc/reference/executionmodel.rst
@@ -22,7 +22,9 @@ The following are blocks: a module, a function body, and a =
class definition.
 Each command typed interactively is a block.  A script file (a file given as
 standard input to the interpreter or specified as a command line argument to=
 the
 interpreter) is a code block.  A script command (a command specified on the
-interpreter command line with the :option:`-c` option) is a code block.  The=
 string
+interpreter command line with the :option:`-c` option) is a code block.
+A module run as a top level script (as module ``__main__``) from the command
+line using a :option:`-m` argument is also a code block. The string
 argument passed to the built-in functions :func:`eval` and :func:`exec` is a
 code block.
=20


From webhook-mailer at python.org  Thu Sep 12 09:30:51 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 13:30:51 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.523.1599366922.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-38132: Check EVP_DigestUpdate for error (GH-16041)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/8c74574e0aaf1a00719fbc9acbdc27a39235=
20aa
commit: 8c74574e0aaf1a00719fbc9acbdc27a3923520aa
branch: master
author: Christian Heimes <christian at python.org>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-12T14:30:47+01:00
summary:

bpo-38132: Check EVP_DigestUpdate for error (GH-16041)

files:
M Modules/_hashopenssl.c

diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 798317fb58b9..bea7e5ed3123 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -96,7 +96,7 @@ newEVPobject(void)
     return retval;
 }
=20
-static void
+static int
 EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len)
 {
     unsigned int process;
@@ -108,11 +108,12 @@ EVP_hash(EVPobject *self, const void *vp, Py_ssize_t le=
n)
             process =3D Py_SAFE_DOWNCAST(len, Py_ssize_t, unsigned int);
         if (!EVP_DigestUpdate(self->ctx, (const void*)cp, process)) {
             _setException(PyExc_ValueError);
-            break;
+            return -1;
         }
         len -=3D process;
         cp +=3D process;
     }
+    return 0;
 }
=20
 /* Internal methods for a hash object */
@@ -243,6 +244,7 @@ static PyObject *
 EVP_update(EVPobject *self, PyObject *obj)
 /*[clinic end generated code: output=3Dec1d55ed2432e966 input=3D9b30ec848f01=
5501]*/
 {
+    int result;
     Py_buffer view;
=20
     GET_BUFFER_VIEW_OR_ERROUT(obj, &view);
@@ -255,14 +257,17 @@ EVP_update(EVPobject *self, PyObject *obj)
     if (self->lock !=3D NULL) {
         Py_BEGIN_ALLOW_THREADS
         PyThread_acquire_lock(self->lock, 1);
-        EVP_hash(self, view.buf, view.len);
+        result =3D EVP_hash(self, view.buf, view.len);
         PyThread_release_lock(self->lock);
         Py_END_ALLOW_THREADS
     } else {
-        EVP_hash(self, view.buf, view.len);
+        result =3D EVP_hash(self, view.buf, view.len);
     }
=20
     PyBuffer_Release(&view);
+
+    if (result =3D=3D -1)
+        return NULL;
     Py_RETURN_NONE;
 }
=20
@@ -396,6 +401,7 @@ static PyObject *
 EVPnew(const EVP_MD *digest,
        const unsigned char *cp, Py_ssize_t len)
 {
+    int result =3D 0;
     EVPobject *self;
=20
     if (!digest) {
@@ -415,10 +421,14 @@ EVPnew(const EVP_MD *digest,
     if (cp && len) {
         if (len >=3D HASHLIB_GIL_MINSIZE) {
             Py_BEGIN_ALLOW_THREADS
-            EVP_hash(self, cp, len);
+            result =3D EVP_hash(self, cp, len);
             Py_END_ALLOW_THREADS
         } else {
-            EVP_hash(self, cp, len);
+            result =3D EVP_hash(self, cp, len);
+        }
+        if (result =3D=3D -1) {
+            Py_DECREF(self);
+            return NULL;
         }
     }
=20


From webhook-mailer at python.org  Thu Sep 12 11:34:27 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 12 Sep 2019 15:34:27 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.524.1599366924.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-37908: Add an example of ArgumentParser.exit() (GH-15455)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/b1a2abdb06408ffc4f13d6ff50351ad49c99=
afc0
commit: b1a2abdb06408ffc4f13d6ff50351ad49c99afc0
branch: master
author: Hai Shi <shihai1992 at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-12T16:34:24+01:00
summary:

bpo-37908: Add an example of ArgumentParser.exit() (GH-15455)

Co-Authored-By: Brandt Bucher <brandtbucher at gmail.com>

files:
M Doc/library/argparse.rst

diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index c2cf7d320b3f..6dffd2e9325e 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -2029,7 +2029,14 @@ Exiting methods
 .. method:: ArgumentParser.exit(status=3D0, message=3DNone)
=20
    This method terminates the program, exiting with the specified *status*
-   and, if given, it prints a *message* before that.
+   and, if given, it prints a *message* before that. The user can override
+   this method to handle these steps differently::
+
+    class ErrorCatchingArgumentParser(argparse.ArgumentParser):
+        def exit(self, status=3D0, message=3DNone):
+            if status:
+                raise Exception(f'Exiting because of an error: {message}')
+            exit(status)
=20
 .. method:: ArgumentParser.error(message)
=20


From webhook-mailer at python.org  Fri Sep 13 05:20:09 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Fri, 13 Sep 2019 09:20:09 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.525.1599366926.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Add example snippet for str.isupper() (GH-14681)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/7183064e9e2a1b19f2ff469a65a34e709448=
3881
commit: 7183064e9e2a1b19f2ff469a65a34e7094483881
branch: master
author: Kishore Vancheeshwaran <24776049+kishvanchee at users.noreply.github.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-13T10:20:01+01:00
summary:

Doc: Add example snippet for str.isupper() (GH-14681)

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 426d768b170e..a21c5538cbcc 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -1782,6 +1782,16 @@ expression support in the :mod:`re` module).
    Return true if all cased characters [4]_ in the string are uppercase and
    there is at least one cased character, false otherwise.
=20
+      >>> 'BANANA'.isupper()
+      True
+      >>> 'banana'.isupper()
+      False
+      >>> 'baNana'.isupper()
+      False
+      >>> ' '.isupper()
+      False
+
+
=20
 .. method:: str.join(iterable)
=20


From webhook-mailer at python.org  Fri Sep 13 06:17:47 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Fri, 13 Sep 2019 10:17:47 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.526.1599366928.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-8538: Add support for boolean actions to argparse (GH-11478)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/6a517c674907c195660fa9178a7b561de49c=
c721
commit: 6a517c674907c195660fa9178a7b561de49cc721
branch: master
author: R=C3=A9mi Lapeyre <remi.lapeyre at henki.fr>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-13T11:17:43+01:00
summary:

bpo-8538: Add support for boolean actions to argparse (GH-11478)

Co-Authored-By: remilapeyre <remi.lapeyre at henki.fr>

files:
A Misc/NEWS.d/next/Library/2019-01-09-16-18-52.bpo-8538.PfVZia.rst
M Doc/library/argparse.rst
M Lib/argparse.py
M Lib/test/test_argparse.py

diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index 6dffd2e9325e..a8aeca41a70a 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -839,9 +839,19 @@ how the command-line arguments should be handled. The su=
pplied actions are:
     Namespace(foo=3D['f1', 'f2', 'f3', 'f4'])
=20
 You may also specify an arbitrary action by passing an Action subclass or
-other object that implements the same interface.  The recommended way to do
-this is to extend :class:`Action`, overriding the ``__call__`` method
-and optionally the ``__init__`` method.
+other object that implements the same interface. The ``BooleanOptionalAction=
``
+is available in ``argparse`` and adds support for boolean actions such as
+``--foo`` and ``--no-foo``::
+
+    >>> import argparse
+    >>> parser =3D argparse.ArgumentParser()
+    >>> parser.add_argument('--foo', action=3Dargparse.BooleanOptionalAction)
+    >>> parser.parse_args(['--no-foo'])
+    Namespace(foo=3DFalse)
+
+The recommended way to create a custom action is to extend :class:`Action`,
+overriding the ``__call__`` method and optionally the ``__init__`` and
+``format_usage`` methods.
=20
 An example of a custom action::
=20
@@ -1361,6 +1371,9 @@ Action instances should be callable, so subclasses must=
 override the
 The ``__call__`` method may perform arbitrary actions, but will typically set
 attributes on the ``namespace`` based on ``dest`` and ``values``.
=20
+Action subclasses can define a ``format_usage`` method that takes no argument
+and return a string which will be used when printing the usage of the progra=
m.
+If such method is not provided, a sensible default will be used.
=20
 The parse_args() method
 -----------------------
diff --git a/Lib/argparse.py b/Lib/argparse.py
index 370692bd1bf4..13af7ac23921 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -67,6 +67,7 @@
     'ArgumentParser',
     'ArgumentError',
     'ArgumentTypeError',
+    'BooleanOptionalAction',
     'FileType',
     'HelpFormatter',
     'ArgumentDefaultsHelpFormatter',
@@ -454,7 +455,7 @@ def _format_actions_usage(self, actions, groups):
                 # if the Optional doesn't take a value, format is:
                 #    -s or --long
                 if action.nargs =3D=3D 0:
-                    part =3D '%s' % option_string
+                    part =3D action.format_usage()
=20
                 # if the Optional takes a value, format is:
                 #    -s ARGS or --long ARGS
@@ -842,9 +843,53 @@ def _get_kwargs(self):
         ]
         return [(name, getattr(self, name)) for name in names]
=20
+    def format_usage(self):
+        return self.option_strings[0]
+
     def __call__(self, parser, namespace, values, option_string=3DNone):
         raise NotImplementedError(_('.__call__() not defined'))
=20
+class BooleanOptionalAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const=3DNone,
+                 default=3DNone,
+                 type=3DNone,
+                 choices=3DNone,
+                 required=3DFalse,
+                 help=3DNone,
+                 metavar=3DNone):
+
+        _option_strings =3D []
+        for option_string in option_strings:
+            _option_strings.append(option_string)
+
+            if option_string.startswith('--'):
+                option_string =3D '--no-' + option_string[2:]
+                _option_strings.append(option_string)
+
+        if help is not None and default is not None:
+            help +=3D f" (default: {default})"
+
+        super().__init__(
+            option_strings=3D_option_strings,
+            dest=3Ddest,
+            nargs=3D0,
+            default=3Ddefault,
+            type=3Dtype,
+            choices=3Dchoices,
+            required=3Drequired,
+            help=3Dhelp,
+            metavar=3Dmetavar)
+
+    def __call__(self, parser, namespace, values, option_string=3DNone):
+        if option_string in self.option_strings:
+            setattr(namespace, self.dest, not option_string.startswith('--no=
-'))
+
+    def format_usage(self):
+        return ' | '.join(self.option_strings)
+
=20
 class _StoreAction(Action):
=20
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index 464db2906998..a97c921852c7 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -686,6 +686,30 @@ class TestOptionalsActionStoreTrue(ParserTestCase):
         ('--apple', NS(apple=3DTrue)),
     ]
=20
+class TestBooleanOptionalAction(ParserTestCase):
+    """Tests BooleanOptionalAction"""
+
+    argument_signatures =3D [Sig('--foo', action=3Dargparse.BooleanOptionalA=
ction)]
+    failures =3D ['--foo bar', '--foo=3Dbar']
+    successes =3D [
+        ('', NS(foo=3DNone)),
+        ('--foo', NS(foo=3DTrue)),
+        ('--no-foo', NS(foo=3DFalse)),
+        ('--foo --no-foo', NS(foo=3DFalse)),  # useful for aliases
+        ('--no-foo --foo', NS(foo=3DTrue)),
+    ]
+
+class TestBooleanOptionalActionRequired(ParserTestCase):
+    """Tests BooleanOptionalAction required"""
+
+    argument_signatures =3D [
+        Sig('--foo', required=3DTrue, action=3Dargparse.BooleanOptionalActio=
n)
+    ]
+    failures =3D ['']
+    successes =3D [
+        ('--foo', NS(foo=3DTrue)),
+        ('--no-foo', NS(foo=3DFalse)),
+    ]
=20
 class TestOptionalsActionAppend(ParserTestCase):
     """Tests the append action for an Optional"""
@@ -3456,6 +3480,10 @@ class TestHelpUsage(HelpTestCase):
         Sig('a', help=3D'a'),
         Sig('b', help=3D'b', nargs=3D2),
         Sig('c', help=3D'c', nargs=3D'?'),
+        Sig('--foo', help=3D'Whether to foo', action=3Dargparse.BooleanOptio=
nalAction),
+        Sig('--bar', help=3D'Whether to bar', default=3DTrue,
+                     action=3Dargparse.BooleanOptionalAction),
+        Sig('-f', '--foobar', '--barfoo', action=3Dargparse.BooleanOptionalA=
ction),
     ]
     argument_group_signatures =3D [
         (Sig('group'), [
@@ -3466,26 +3494,32 @@ class TestHelpUsage(HelpTestCase):
         ])
     ]
     usage =3D '''\
-        usage: PROG [-h] [-w W [W ...]] [-x [X [X ...]]] [-y [Y]] [-z Z Z Z]
+        usage: PROG [-h] [-w W [W ...]] [-x [X [X ...]]] [--foo | --no-foo]
+                    [--bar | --no-bar]
+                    [-f | --foobar | --no-foobar | --barfoo | --no-barfoo] [=
-y [Y]]
+                    [-z Z Z Z]
                     a b b [c] [d [d ...]] e [e ...]
         '''
     help =3D usage + '''\
=20
         positional arguments:
-          a               a
-          b               b
-          c               c
+          a                     a
+          b                     b
+          c                     c
=20
         optional arguments:
-          -h, --help      show this help message and exit
-          -w W [W ...]    w
-          -x [X [X ...]]  x
+          -h, --help            show this help message and exit
+          -w W [W ...]          w
+          -x [X [X ...]]        x
+          --foo, --no-foo       Whether to foo
+          --bar, --no-bar       Whether to bar (default: True)
+          -f, --foobar, --no-foobar, --barfoo, --no-barfoo
=20
         group:
-          -y [Y]          y
-          -z Z Z Z        z
-          d               d
-          e               e
+          -y [Y]                y
+          -z Z Z Z              z
+          d                     d
+          e                     e
         '''
     version =3D ''
=20
diff --git a/Misc/NEWS.d/next/Library/2019-01-09-16-18-52.bpo-8538.PfVZia.rst=
 b/Misc/NEWS.d/next/Library/2019-01-09-16-18-52.bpo-8538.PfVZia.rst
new file mode 100644
index 000000000000..94249ab1e434
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-01-09-16-18-52.bpo-8538.PfVZia.rst
@@ -0,0 +1,2 @@
+Add support for boolean actions like ``--foo`` and ``--no-foo`` to argparse.
+Patch contributed by R=C3=A9mi Lapeyre.


From webhook-mailer at python.org  Fri Sep 13 07:40:11 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Fri, 13 Sep 2019 11:40:11 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.527.1599366929.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-12707: deprecate info(), geturl(), getcode() methods in favor of
 headers, url, and status properties for HTTPResponse and addinfourl
 (GH-11447)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/ff2e18286560e981f4e09afb0d2448ea9944=
14d8
commit: ff2e18286560e981f4e09afb0d2448ea994414d8
branch: master
author: Ashwin Ramaswami <aramaswamis at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-13T12:40:07+01:00
summary:

bpo-12707: deprecate info(), geturl(), getcode() methods in favor of headers,=
 url, and status properties for HTTPResponse and addinfourl (GH-11447)

Co-Authored-By: epicfaace <aramaswamis at gmail.com>

files:
A Misc/NEWS.d/next/Documentation/2019-08-27-01-14-59.bpo-12707.Yj3_7_.rst
M Doc/library/http.client.rst
M Doc/library/urllib.request.rst
M Lib/test/test_urllib.py
M Lib/test/test_urllib_response.py
M Lib/urllib/request.py
M Lib/urllib/response.py

diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst
index db26d56af30f..a458ab77efd8 100644
--- a/Doc/library/http.client.rst
+++ b/Doc/library/http.client.rst
@@ -484,6 +484,14 @@ statement.
=20
    HTTP protocol version used by server.  10 for HTTP/1.0, 11 for HTTP/1.1.
=20
+.. attribute:: HTTPResponse.url
+
+   URL of the resource retrieved, commonly used to determine if a redirect w=
as followed.
+
+.. attribute:: HTTPResponse.headers
+
+   Headers of the response in the form of an :class:`email.message.EmailMess=
age` instance.
+
 .. attribute:: HTTPResponse.status
=20
    Status code returned by server.
@@ -501,6 +509,21 @@ statement.
=20
    Is ``True`` if the stream is closed.
=20
+.. method:: HTTPResponse.geturl()
+
+   .. deprecated:: 3.9
+      Deprecated in favor of :attr:`~HTTPResponse.url`.
+
+.. method:: HTTPResponse.info()
+
+   .. deprecated:: 3.9
+      Deprecated in favor of :attr:`~HTTPResponse.headers`.
+
+.. method:: HTTPResponse.getstatus()
+
+   .. deprecated:: 3.9
+      Deprecated in favor of :attr:`~HTTPResponse.status`.
+
 Examples
 --------
=20
diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst
index 448bc6785470..a903d605cf4e 100644
--- a/Doc/library/urllib.request.rst
+++ b/Doc/library/urllib.request.rst
@@ -55,16 +55,8 @@ The :mod:`urllib.request` module defines the following fun=
ctions:
    The *cadefault* parameter is ignored.
=20
    This function always returns an object which can work as a
-   :term:`context manager` and has methods such as
-
-   * :meth:`~urllib.response.addinfourl.geturl` --- return the URL of the re=
source retrieved,
-     commonly used to determine if a redirect was followed
-
-   * :meth:`~urllib.response.addinfourl.info` --- return the meta-informatio=
n of the page, such as headers,
-     in the form of an :func:`email.message_from_string` instance (see
-     `Quick Reference to HTTP Headers <http://jkorpela.fi/http.html>`_)
-
-   * :meth:`~urllib.response.addinfourl.getcode` -- return the HTTP status c=
ode of the response.
+   :term:`context manager` and has the properties *url*, *headers*, and *sta=
tus*.
+   See :class:`urllib.response.addinfourl` for more detail on these properti=
es.
=20
    For HTTP and HTTPS URLs, this function returns a
    :class:`http.client.HTTPResponse` object slightly modified. In addition
@@ -1585,9 +1577,42 @@ some point in the future.
    :synopsis: Response classes used by urllib.
=20
 The :mod:`urllib.response` module defines functions and classes which define=
 a
-minimal file like interface, including ``read()`` and ``readline()``. The
-typical response object is an addinfourl instance, which defines an ``info()=
``
-method and that returns headers and a ``geturl()`` method that returns the u=
rl.
-Functions defined by this module are used internally by the
-:mod:`urllib.request` module.
+minimal file-like interface, including ``read()`` and ``readline()``.
+Functions defined by this module are used internally by the :mod:`urllib.req=
uest` module.
+The typical response object is a :class:`urllib.response.addinfourl` instanc=
e:
+
+.. class:: addinfourl
+
+   .. attribute:: url
+
+      URL of the resource retrieved, commonly used to determine if a redirec=
t was followed.
+
+   .. attribute:: headers
+
+      Returns the headers of the response in the form of an :class:`~email.m=
essage.EmailMessage` instance.
+
+   .. attribute:: status
+
+      .. versionadded:: 3.9
+
+      Status code returned by server.
+
+   .. method:: geturl()
+
+      .. deprecated:: 3.9
+         Deprecated in favor of :attr:`~addinfourl.url`.
+
+   .. method:: info()
+
+      .. deprecated:: 3.9
+         Deprecated in favor of :attr:`~addinfourl.headers`.
+
+   .. attribute:: code
+
+      .. deprecated:: 3.9
+         Deprecated in favor of :attr:`~addinfourl.status`.
+
+   .. method:: getstatus()
=20
+      .. deprecated:: 3.9
+         Deprecated in favor of :attr:`~addinfourl.status`.
diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
index 8895421f97fc..9a6b5f66b7a1 100644
--- a/Lib/test/test_urllib.py
+++ b/Lib/test/test_urllib.py
@@ -194,6 +194,15 @@ def test_close(self):
         # by the tearDown() method for the test
         self.returned_obj.close()
=20
+    def test_headers(self):
+        self.assertIsInstance(self.returned_obj.headers, email.message.Messa=
ge)
+
+    def test_url(self):
+        self.assertEqual(self.returned_obj.url, self.pathname)
+
+    def test_status(self):
+        self.assertIsNone(self.returned_obj.status)
+
     def test_info(self):
         self.assertIsInstance(self.returned_obj.info(), email.message.Messag=
e)
=20
diff --git a/Lib/test/test_urllib_response.py b/Lib/test/test_urllib_response=
.py
index 0eb59426ccb6..73d2ef0424f4 100644
--- a/Lib/test/test_urllib_response.py
+++ b/Lib/test/test_urllib_response.py
@@ -42,6 +42,7 @@ def closehook():
     def test_addinfo(self):
         info =3D urllib.response.addinfo(self.fp, self.test_headers)
         self.assertEqual(info.info(), self.test_headers)
+        self.assertEqual(info.headers, self.test_headers)
=20
     def test_addinfourl(self):
         url =3D "http://www.python.org"
@@ -51,6 +52,9 @@ def test_addinfourl(self):
         self.assertEqual(infourl.info(), self.test_headers)
         self.assertEqual(infourl.geturl(), url)
         self.assertEqual(infourl.getcode(), code)
+        self.assertEqual(infourl.headers, self.test_headers)
+        self.assertEqual(infourl.url, url)
+        self.assertEqual(infourl.status, code)
=20
     def tearDown(self):
         self.sock.close()
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
index f6ce9cb6d586..267a90dd80db 100644
--- a/Lib/urllib/request.py
+++ b/Lib/urllib/request.py
@@ -163,18 +163,10 @@ def urlopen(url, data=3DNone, timeout=3Dsocket._GLOBAL_=
DEFAULT_TIMEOUT,
=20
     The *cadefault* parameter is ignored.
=20
-    This function always returns an object which can work as a context
-    manager and has methods such as
=20
-    * geturl() - return the URL of the resource retrieved, commonly used to
-      determine if a redirect was followed
-
-    * info() - return the meta-information of the page, such as headers, in =
the
-      form of an email.message_from_string() instance (see Quick Reference to
-      HTTP Headers)
-
-    * getcode() - return the HTTP status code of the response.  Raises URLEr=
ror
-      on errors.
+    This function always returns an object which can work as a
+    context manager and has the properties url, headers, and status.
+    See urllib.response.addinfourl for more detail on these properties.
=20
     For HTTP and HTTPS URLs, this function returns a http.client.HTTPResponse
     object slightly modified. In addition to the three new methods above, the
diff --git a/Lib/urllib/response.py b/Lib/urllib/response.py
index 4778118dbb17..5a2c3cc78c39 100644
--- a/Lib/urllib/response.py
+++ b/Lib/urllib/response.py
@@ -73,6 +73,10 @@ def __init__(self, fp, headers, url, code=3DNone):
         self.url =3D url
         self.code =3D code
=20
+    @property
+    def status(self):
+        return self.code
+
     def getcode(self):
         return self.code
=20
diff --git a/Misc/NEWS.d/next/Documentation/2019-08-27-01-14-59.bpo-12707.Yj3=
_7_.rst b/Misc/NEWS.d/next/Documentation/2019-08-27-01-14-59.bpo-12707.Yj3_7_=
.rst
new file mode 100644
index 000000000000..423e8310d737
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2019-08-27-01-14-59.bpo-12707.Yj3_7_.rst
@@ -0,0 +1 @@
+Deprecate info(), geturl(), getcode() methods in favor of the headers, url, =
and status properties, respectively, for HTTPResponse and addinfourl. Also de=
precate the code attribute of addinfourl in favor of the status attribute. Pa=
tch by Ashwin Ramaswami
\ No newline at end of file


From webhook-mailer at python.org  Fri Sep 13 09:01:06 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Fri, 13 Sep 2019 13:01:06 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.528.1599366931.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Improve consistency of os.path.normcase with other os.path
 functions (GH-14004)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/53f78ec9e181f544b1a4575f32d42e296e05=
e4b4
commit: 53f78ec9e181f544b1a4575f32d42e296e05e4b4
branch: master
author: Kexuan Sun <me at kianasun.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-13T14:01:02+01:00
summary:

Doc: Improve consistency of os.path.normcase with other os.path functions (GH=
-14004)

files:
M Doc/library/os.path.rst

diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst
index 640ceecbc78f..a5abacf02144 100644
--- a/Doc/library/os.path.rst
+++ b/Doc/library/os.path.rst
@@ -328,8 +328,6 @@ the :mod:`glob` module.)
    Normalize the case of a pathname.  On Windows, convert all characters in =
the
    pathname to lowercase, and also convert forward slashes to backward slash=
es.
    On other operating systems, return the path unchanged.
-   Raise a :exc:`TypeError` if the type of *path* is not ``str`` or ``bytes`=
` (directly
-   or indirectly through the :class:`os.PathLike` interface).
=20
    .. versionchanged:: 3.6
       Accepts a :term:`path-like object`.


From webhook-mailer at python.org  Fri Sep 13 09:36:13 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Fri, 13 Sep 2019 13:36:13 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.529.1599366932.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-26468: Doc: improve the documentation of shutil.copy2 when it can
 fail. (GH-13765)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/9585f46b97931d2640c3343dfe03aed15beb=
9fea
commit: 9585f46b97931d2640c3343dfe03aed15beb9fea
branch: master
author: Windson yang <wiwindson at outlook.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-13T14:36:09+01:00
summary:

bpo-26468: Doc: improve the documentation of shutil.copy2 when it can fail. (=
GH-13765)

files:
M Doc/library/shutil.rst

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index 88c5f62dd4c9..174b7e875a38 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -187,7 +187,8 @@ Directory and files operations
    However, this functionality is not available on all platforms.
    On platforms where some or all of this functionality is
    unavailable, :func:`copy2` will preserve all the metadata
-   it can; :func:`copy2` never returns failure.
+   it can; :func:`copy2` never raises an exception because it
+   cannot preserve file metadata.
=20
    :func:`copy2` uses :func:`copystat` to copy the file metadata.
    Please see :func:`copystat` for more information


From webhook-mailer at python.org  Fri Sep 13 10:07:10 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Fri, 13 Sep 2019 14:07:10 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.530.1599366934.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-25068: urllib.request.ProxyHandler now lowercases the dict keys
 (GH-13489)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/b761e3aed1fbada4572a776f6a0d3c4be491=
d595
commit: b761e3aed1fbada4572a776f6a0d3c4be491d595
branch: master
author: Zackery Spytz <zspytz at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-13T15:07:07+01:00
summary:

bpo-25068: urllib.request.ProxyHandler now lowercases the dict keys (GH-13489)

files:
A Misc/NEWS.d/next/Library/2019-05-22-04-52-35.bpo-25068.vR_rC-.rst
M Lib/test/test_urllib2.py
M Lib/urllib/request.py

diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
index c228fa7d1dc1..186a96765971 100644
--- a/Lib/test/test_urllib2.py
+++ b/Lib/test/test_urllib2.py
@@ -1342,21 +1342,22 @@ def http_open(self, req):
                 self.assertTrue(request.startswith(expected), repr(request))
=20
     def test_proxy(self):
-        o =3D OpenerDirector()
-        ph =3D urllib.request.ProxyHandler(dict(http=3D"proxy.example.com:31=
28"))
-        o.add_handler(ph)
-        meth_spec =3D [
-            [("http_open", "return response")]
-            ]
-        handlers =3D add_ordered_mock_handlers(o, meth_spec)
-
-        req =3D Request("http://acme.example.com/")
-        self.assertEqual(req.host, "acme.example.com")
-        o.open(req)
-        self.assertEqual(req.host, "proxy.example.com:3128")
-
-        self.assertEqual([(handlers[0], "http_open")],
-                         [tup[0:2] for tup in o.calls])
+        u =3D "proxy.example.com:3128"
+        for d in dict(http=3Du), dict(HTTP=3Du):
+            o =3D OpenerDirector()
+            ph =3D urllib.request.ProxyHandler(d)
+            o.add_handler(ph)
+            meth_spec =3D [
+                [("http_open", "return response")]
+                ]
+            handlers =3D add_ordered_mock_handlers(o, meth_spec)
+
+            req =3D Request("http://acme.example.com/")
+            self.assertEqual(req.host, "acme.example.com")
+            o.open(req)
+            self.assertEqual(req.host, u)
+            self.assertEqual([(handlers[0], "http_open")],
+                             [tup[0:2] for tup in o.calls])
=20
     def test_proxy_no_proxy(self):
         os.environ['no_proxy'] =3D 'python.org'
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
index 267a90dd80db..721c152179e9 100644
--- a/Lib/urllib/request.py
+++ b/Lib/urllib/request.py
@@ -792,6 +792,7 @@ def __init__(self, proxies=3DNone):
         assert hasattr(proxies, 'keys'), "proxies must be a mapping"
         self.proxies =3D proxies
         for type, url in proxies.items():
+            type =3D type.lower()
             setattr(self, '%s_open' % type,
                     lambda r, proxy=3Durl, type=3Dtype, meth=3Dself.proxy_op=
en:
                         meth(r, proxy, type))
diff --git a/Misc/NEWS.d/next/Library/2019-05-22-04-52-35.bpo-25068.vR_rC-.rs=
t b/Misc/NEWS.d/next/Library/2019-05-22-04-52-35.bpo-25068.vR_rC-.rst
new file mode 100644
index 000000000000..89de83bce029
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-22-04-52-35.bpo-25068.vR_rC-.rst
@@ -0,0 +1,2 @@
+:class:`urllib.request.ProxyHandler` now lowercases the keys of the passed
+dictionary.


From webhook-mailer at python.org  Fri Sep 13 12:40:59 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Fri, 13 Sep 2019 16:40:59 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.531.1599366936.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-38122: minor fixes to AsyncMock spec handling (GH-16099)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/14fd925a18fe3db0922a7d798e373102fe7a=
8a9c
commit: 14fd925a18fe3db0922a7d798e373102fe7a8a9c
branch: master
author: Michael Foord <voidspace at users.noreply.github.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-13T17:40:56+01:00
summary:

bpo-38122: minor fixes to AsyncMock spec handling (GH-16099)

files:
M Lib/unittest/mock.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 513cd5c924d4..b33d58a2cb0d 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -402,18 +402,12 @@ def __new__(cls, /, *args, **kw):
         # so we can create magic methods on the
         # class without stomping on other mocks
         bases =3D (cls,)
-        if not issubclass(cls, AsyncMock):
+        if not issubclass(cls, AsyncMockMixin):
             # Check if spec is an async object or function
-            sig =3D inspect.signature(NonCallableMock.__init__)
-            bound_args =3D sig.bind_partial(cls, *args, **kw).arguments
-            spec_arg =3D [
-                arg for arg in bound_args.keys()
-                if arg.startswith('spec')
-            ]
-            if spec_arg:
-                # what if spec_set is different than spec?
-                if _is_async_obj(bound_args[spec_arg[0]]):
-                    bases =3D (AsyncMockMixin, cls,)
+            bound_args =3D _MOCK_SIG.bind_partial(cls, *args, **kw).arguments
+            spec_arg =3D bound_args.get('spec_set', bound_args.get('spec'))
+            if spec_arg and _is_async_obj(spec_arg):
+                bases =3D (AsyncMockMixin, cls)
         new =3D type(cls.__name__, bases, {'__doc__': cls.__doc__})
         instance =3D object.__new__(new)
         return instance
@@ -1020,6 +1014,9 @@ def _calls_repr(self, prefix=3D"Calls"):
         return f"\n{prefix}: {safe_repr(self.mock_calls)}."
=20
=20
+_MOCK_SIG =3D inspect.signature(NonCallableMock.__init__)
+
+
 class _AnyComparer(list):
     """A list which checks if it contains a call which may have an
     argument of ANY, flipping the components of item and self from


From webhook-mailer at python.org  Fri Sep 13 13:10:57 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Fri, 13 Sep 2019 17:10:57 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.532.1599366937.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: [3.8] bpo-38150: Fix refleak in the finalizer of a _testcapimodule
 type (GH-16115) (GH-16118)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/53ff2ca89feb90c1fbf2605321bcef35b457=
f16f
commit: 53ff2ca89feb90c1fbf2605321bcef35b457f16f
branch: 3.8
author: St=C3=A9phane Wirtel <stephane at wirtel.be>
committer: GitHub <noreply at github.com>
date: 2019-09-13T18:10:53+01:00
summary:

[3.8] bpo-38150: Fix refleak in the finalizer of a _testcapimodule type (GH-1=
6115) (GH-16118)

The PyLong created in the finalizer was not being cleaned up

https://bugs.python.org/issue38150

Automerge-Triggered-By: @matrixise
(cherry picked from commit a67ac2f2d9550e5a36d28f9b6eeacf6575dda2d5)

Co-authored-by: Eddie Elizondo <eelizondo at fb.com>

files:
M Modules/_testcapimodule.c

diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index cb9206ca4eba..cccf9249040d 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -6090,7 +6090,7 @@ static void
 heapctypesubclasswithfinalizer_finalize(PyObject *self)
 {
     PyObject *error_type, *error_value, *error_traceback, *m;
-    PyObject *oldtype =3D NULL, *newtype =3D NULL;
+    PyObject *oldtype =3D NULL, *newtype =3D NULL, *refcnt =3D NULL;
=20
     /* Save the current exception, if any. */
     PyErr_Fetch(&error_type, &error_value, &error_traceback);
@@ -6108,18 +6108,26 @@ heapctypesubclasswithfinalizer_finalize(PyObject *sel=
f)
     if (PyObject_SetAttrString(self, "__class__", newtype) < 0) {
         goto cleanup_finalize;
     }
-    if (PyObject_SetAttrString(
-        oldtype, "refcnt_in_del", PyLong_FromSsize_t(Py_REFCNT(oldtype))) < =
0) {
+    refcnt =3D PyLong_FromSsize_t(Py_REFCNT(oldtype));
+    if (refcnt =3D=3D NULL) {
         goto cleanup_finalize;
     }
-    if (PyObject_SetAttrString(
-        newtype, "refcnt_in_del", PyLong_FromSsize_t(Py_REFCNT(newtype))) < =
0) {
+    if (PyObject_SetAttrString(oldtype, "refcnt_in_del", refcnt) < 0) {
+        goto cleanup_finalize;
+    }
+    Py_DECREF(refcnt);
+    refcnt =3D PyLong_FromSsize_t(Py_REFCNT(newtype));
+    if (refcnt =3D=3D NULL) {
+        goto cleanup_finalize;
+    }
+    if (PyObject_SetAttrString(newtype, "refcnt_in_del", refcnt) < 0) {
         goto cleanup_finalize;
     }
=20
 cleanup_finalize:
     Py_XDECREF(oldtype);
     Py_XDECREF(newtype);
+    Py_XDECREF(refcnt);
=20
     /* Restore the saved exception. */
     PyErr_Restore(error_type, error_value, error_traceback);


From webhook-mailer at python.org  Fri Sep 13 13:20:25 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Fri, 13 Sep 2019 17:20:25 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.533.1599366939.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 bpo-32790: Add info about alt format using # for 'g' in chart (GH-6624)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/d44542f9a231bf725ecd82eb640a672c759a=
8227
commit: d44542f9a231bf725ecd82eb640a672c759a8227
branch: master
author: bchhabra2490 <bchhabra2490 at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-13T18:20:21+01:00
summary:

bpo-32790: Add info about alt format using # for 'g' in chart (GH-6624)

files:
M Doc/library/string.rst

diff --git a/Doc/library/string.rst b/Doc/library/string.rst
index af52af0482c4..e2983db1ac83 100644
--- a/Doc/library/string.rst
+++ b/Doc/library/string.rst
@@ -508,7 +508,8 @@ The available presentation types for floating point and d=
ecimal values are:
    |         | with presentation type ``'e'`` and precision ``p-1``.    |
    |         | In both cases insignificant trailing zeros are removed   |
    |         | from the significand, and the decimal point is also      |
-   |         | removed if there are no remaining digits following it.   |
+   |         | removed if there are no remaining digits following it,   |
+   |         | unless the ``'#'`` option is used.                       |
    |         |                                                          |
    |         | Positive and negative infinity, positive and negative    |
    |         | zero, and nans, are formatted as ``inf``, ``-inf``,      |


From webhook-mailer at python.org  Fri Sep 13 13:32:31 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Fri, 13 Sep 2019 17:32:31 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.534.1599366940.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: fcntl.lockf() is more powerful than written (GH-6750)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/77cd0ceab2f6c1696fb1d31115c2f880b2e2=
1934
commit: 77cd0ceab2f6c1696fb1d31115c2f880b2e21934
branch: master
author: Eric O. LEBIGOT (EOL) <lebigot at users.noreply.github.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-13T18:32:28+01:00
summary:

Doc: fcntl.lockf() is more powerful than written (GH-6750)

files:
M Doc/library/fcntl.rst

diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst
index 6a66f5b66481..da39c3187a1e 100644
--- a/Doc/library/fcntl.rst
+++ b/Doc/library/fcntl.rst
@@ -124,7 +124,8 @@ The module defines the following functions:
 .. function:: lockf(fd, cmd, len=3D0, start=3D0, whence=3D0)
=20
    This is essentially a wrapper around the :func:`~fcntl.fcntl` locking cal=
ls.
-   *fd* is the file descriptor of the file to lock or unlock, and *cmd*
+   *fd* is the file descriptor (file objects providing a :meth:`~io.IOBase.f=
ileno`
+   method are accepted as well) of the file to lock or unlock, and *cmd*
    is one of the following values:
=20
    * :const:`LOCK_UN` -- unlock
@@ -175,4 +176,3 @@ using the :func:`flock` call may be better.
       present in the :mod:`os` module (on BSD only), the :func:`os.open`
       function provides an alternative to the :func:`lockf` and :func:`flock`
       functions.
-


From webhook-mailer at python.org  Fri Sep 13 13:42:39 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Fri, 13 Sep 2019 17:42:39 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.535.1599366942.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-34706: Preserve subclassing in inspect.Signature.from_callable
 (GH-16108) (GH-16113)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/cf25765cf79a5bd19ba90a535a51367d6177=
f2bc
commit: cf25765cf79a5bd19ba90a535a51367d6177f2bc
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-13T18:42:35+01:00
summary:

bpo-34706: Preserve subclassing in inspect.Signature.from_callable (GH-16108)=
 (GH-16113)

https://bugs.python.org/issue34706

Specifically in the case of a class that does not override its
constructor signature inherited from object.

These are Buck Evan @bukzor's changes cherrypicked from GH-9344.
(cherry picked from commit 5b9ff7a0dcb16d6f5c3cd4f1f52e0ca6a4bde586)

Co-authored-by: Gregory P. Smith <greg at krypto.org>

files:
A Misc/NEWS.d/next/Library/2019-09-13-14-54-33.bpo-34706.HWVpOY.rst
M Lib/inspect.py
M Lib/test/test_inspect.py

diff --git a/Lib/inspect.py b/Lib/inspect.py
index 0a57749ccdd4..c2a1ed4148e7 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -2367,7 +2367,7 @@ def _signature_from_callable(obj, *,
                 if (obj.__init__ is object.__init__ and
                     obj.__new__ is object.__new__):
                     # Return a signature of 'object' builtin.
-                    return signature(object)
+                    return sigcls.from_callable(object)
                 else:
                     raise ValueError(
                         'no signature found for builtin type {!r}'.format(ob=
j))
diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py
index 1cd4ea28939d..db9ac36f7198 100644
--- a/Lib/test/test_inspect.py
+++ b/Lib/test/test_inspect.py
@@ -3157,14 +3157,21 @@ def test_signature_from_callable_python_obj(self):
         class MySignature(inspect.Signature): pass
         def foo(a, *, b:1): pass
         foo_sig =3D MySignature.from_callable(foo)
-        self.assertTrue(isinstance(foo_sig, MySignature))
+        self.assertIsInstance(foo_sig, MySignature)
+
+    def test_signature_from_callable_class(self):
+        # A regression test for a class inheriting its signature from `objec=
t`.
+        class MySignature(inspect.Signature): pass
+        class foo: pass
+        foo_sig =3D MySignature.from_callable(foo)
+        self.assertIsInstance(foo_sig, MySignature)
=20
     @unittest.skipIf(MISSING_C_DOCSTRINGS,
                      "Signature information for builtins requires docstrings=
")
     def test_signature_from_callable_builtin_obj(self):
         class MySignature(inspect.Signature): pass
         sig =3D MySignature.from_callable(_pickle.Pickler)
-        self.assertTrue(isinstance(sig, MySignature))
+        self.assertIsInstance(sig, MySignature)
=20
     def test_signature_definition_order_preserved_on_kwonly(self):
         for fn in signatures_with_lexicographic_keyword_only_parameters():
diff --git a/Misc/NEWS.d/next/Library/2019-09-13-14-54-33.bpo-34706.HWVpOY.rs=
t b/Misc/NEWS.d/next/Library/2019-09-13-14-54-33.bpo-34706.HWVpOY.rst
new file mode 100644
index 000000000000..1df3742abd27
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-13-14-54-33.bpo-34706.HWVpOY.rst
@@ -0,0 +1 @@
+Preserve subclassing in inspect.Signature.from_callable.


From webhook-mailer at python.org  Fri Sep 13 13:42:58 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Fri, 13 Sep 2019 17:42:58 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.536.1599366944.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-34706: Preserve subclassing in inspect.Signature.from_callable
 (GH-16108) (GH-16114)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/3906920cfecba83d529f8755f5ec2e53e49f=
9e90
commit: 3906920cfecba83d529f8755f5ec2e53e49f9e90
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-13T18:42:53+01:00
summary:

bpo-34706: Preserve subclassing in inspect.Signature.from_callable (GH-16108)=
 (GH-16114)

https://bugs.python.org/issue34706

Specifically in the case of a class that does not override its
constructor signature inherited from object.

These are Buck Evan @bukzor's changes cherrypicked from GH-9344.
(cherry picked from commit 5b9ff7a0dcb16d6f5c3cd4f1f52e0ca6a4bde586)

Co-authored-by: Gregory P. Smith <greg at krypto.org>

files:
A Misc/NEWS.d/next/Library/2019-09-13-14-54-33.bpo-34706.HWVpOY.rst
M Lib/inspect.py
M Lib/test/test_inspect.py

diff --git a/Lib/inspect.py b/Lib/inspect.py
index 7f0517207d15..bbecd98f94ab 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -2358,7 +2358,7 @@ def _signature_from_callable(obj, *,
                 if (obj.__init__ is object.__init__ and
                     obj.__new__ is object.__new__):
                     # Return a signature of 'object' builtin.
-                    return signature(object)
+                    return sigcls.from_callable(object)
                 else:
                     raise ValueError(
                         'no signature found for builtin type {!r}'.format(ob=
j))
diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py
index f3eb2f62c37e..75ac40ba4f81 100644
--- a/Lib/test/test_inspect.py
+++ b/Lib/test/test_inspect.py
@@ -3050,14 +3050,21 @@ def test_signature_from_callable_python_obj(self):
         class MySignature(inspect.Signature): pass
         def foo(a, *, b:1): pass
         foo_sig =3D MySignature.from_callable(foo)
-        self.assertTrue(isinstance(foo_sig, MySignature))
+        self.assertIsInstance(foo_sig, MySignature)
+
+    def test_signature_from_callable_class(self):
+        # A regression test for a class inheriting its signature from `objec=
t`.
+        class MySignature(inspect.Signature): pass
+        class foo: pass
+        foo_sig =3D MySignature.from_callable(foo)
+        self.assertIsInstance(foo_sig, MySignature)
=20
     @unittest.skipIf(MISSING_C_DOCSTRINGS,
                      "Signature information for builtins requires docstrings=
")
     def test_signature_from_callable_builtin_obj(self):
         class MySignature(inspect.Signature): pass
         sig =3D MySignature.from_callable(_pickle.Pickler)
-        self.assertTrue(isinstance(sig, MySignature))
+        self.assertIsInstance(sig, MySignature)
=20
     def test_signature_definition_order_preserved_on_kwonly(self):
         for fn in signatures_with_lexicographic_keyword_only_parameters():
diff --git a/Misc/NEWS.d/next/Library/2019-09-13-14-54-33.bpo-34706.HWVpOY.rs=
t b/Misc/NEWS.d/next/Library/2019-09-13-14-54-33.bpo-34706.HWVpOY.rst
new file mode 100644
index 000000000000..1df3742abd27
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-13-14-54-33.bpo-34706.HWVpOY.rst
@@ -0,0 +1 @@
+Preserve subclassing in inspect.Signature.from_callable.


From webhook-mailer at python.org  Sat Sep 14 02:17:14 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Sat, 14 Sep 2019 06:17:14 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.537.1599366945.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Fix link to window.getch in curses documentation (GH-16132)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/a26ace19bddea2d7a999a6de8286b3f27b13=
2f35
commit: a26ace19bddea2d7a999a6de8286b3f27b132f35
branch: master
author: Anthony Sottile <asottile at umich.edu>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-14T07:17:09+01:00
summary:

Doc: Fix link to window.getch in curses documentation (GH-16132)

files:
M Doc/library/curses.rst

diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst
index c88d3520e988..f1803c685a9e 100644
--- a/Doc/library/curses.rst
+++ b/Doc/library/curses.rst
@@ -1283,7 +1283,7 @@ The :mod:`curses` module defines the following data mem=
bers:
=20
 .. data:: ERR
=20
-   Some curses routines  that  return  an integer, such as  :func:`getch`, r=
eturn
+   Some curses routines  that  return  an integer, such as :meth:`~window.ge=
tch`, return
    :const:`ERR` upon failure.
=20
=20


From webhook-mailer at python.org  Sat Sep 14 02:25:13 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Sat, 14 Sep 2019 06:25:13 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.538.1599366947.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: [3.8] Doc: Fix link to window.getch in curses documentation
 (GH-16132) (GH-16135)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/f05d39d42cf4a7d6bcca8b3f7ef2601ee9eb=
f160
commit: f05d39d42cf4a7d6bcca8b3f7ef2601ee9ebf160
branch: 3.8
author: St=C3=A9phane Wirtel <stephane at wirtel.be>
committer: GitHub <noreply at github.com>
date: 2019-09-14T07:25:10+01:00
summary:

[3.8] Doc: Fix link to window.getch in curses documentation (GH-16132) (GH-16=
135)

(cherry picked from commit a26ace19bddea2d7a999a6de8286b3f27b132f35)

Co-authored-by: Anthony Sottile <asottile at umich.edu>

files:
M Doc/library/curses.rst

diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst
index 7d1e7538a292..cbd68834b6cb 100644
--- a/Doc/library/curses.rst
+++ b/Doc/library/curses.rst
@@ -1283,7 +1283,7 @@ The :mod:`curses` module defines the following data mem=
bers:
=20
 .. data:: ERR
=20
-   Some curses routines  that  return  an integer, such as  :func:`getch`, r=
eturn
+   Some curses routines  that  return  an integer, such as :meth:`~window.ge=
tch`, return
    :const:`ERR` upon failure.
=20
=20


From webhook-mailer at python.org  Mon Sep 16 12:18:23 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Mon, 16 Sep 2019 16:18:23 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.539.1599366949.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 Doc: Add link of GNU Readline library to interpreter tutorial (GH-16152)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/f18242be16714da2cfe013dbadfaf2e31d97=
1562
commit: f18242be16714da2cfe013dbadfaf2e31d971562
branch: master
author: Adorilson Bezerra <adorilson at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-16T18:18:04+02:00
summary:

Doc: Add link of GNU Readline library to interpreter tutorial (GH-16152)

files:
M Doc/tutorial/interpreter.rst

diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst
index aeb6aa0d8e86..ffab8fa4e0d8 100644
--- a/Doc/tutorial/interpreter.rst
+++ b/Doc/tutorial/interpreter.rst
@@ -34,13 +34,14 @@ status.  If that doesn't work, you can exit the interpret=
er by typing the
 following command: ``quit()``.
=20
 The interpreter's line-editing features include interactive editing, history
-substitution and code completion on systems that support readline.  Perhaps =
the
-quickest check to see whether command line editing is supported is typing
-:kbd:`Control-P` to the first Python prompt you get.  If it beeps, you have =
command
-line editing; see Appendix :ref:`tut-interacting` for an introduction to the
-keys.  If nothing appears to happen, or if ``^P`` is echoed, command line
-editing isn't available; you'll only be able to use backspace to remove
-characters from the current line.
+substitution and code completion on systems that support the `GNU Readline
+<https://tiswww.case.edu/php/chet/readline/rltop.html>`_ library.
+Perhaps the quickest check to see whether command line editing is supported =
is
+typing :kbd:`Control-P` to the first Python prompt you get.  If it beeps, you
+have command line editing; see Appendix :ref:`tut-interacting` for an
+introduction to the keys.  If nothing appears to happen, or if ``^P`` is
+echoed, command line editing isn't available; you'll only be able to use
+backspace to remove characters from the current line.
=20
 The interpreter operates somewhat like the Unix shell: when called with stan=
dard
 input connected to a tty device, it reads and executes commands interactivel=
y;


From webhook-mailer at python.org  Mon Sep 16 12:29:55 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Mon, 16 Sep 2019 16:29:55 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.540.1599366950.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Add link of GNU Readline library to interpreter tutorial
 (GH-16152) (GH-16188)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/ad4ce9edf3936728b3d74acb1e8b7e84a801=
20c1
commit: ad4ce9edf3936728b3d74acb1e8b7e84a80120c1
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-16T18:29:50+02:00
summary:

Doc: Add link of GNU Readline library to interpreter tutorial (GH-16152) (GH-=
16188)

(cherry picked from commit f18242be16714da2cfe013dbadfaf2e31d971562)

Co-authored-by: Adorilson Bezerra <adorilson at gmail.com>

files:
M Doc/tutorial/interpreter.rst

diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst
index 647dd72b374f..daa417364244 100644
--- a/Doc/tutorial/interpreter.rst
+++ b/Doc/tutorial/interpreter.rst
@@ -34,13 +34,14 @@ status.  If that doesn't work, you can exit the interpret=
er by typing the
 following command: ``quit()``.
=20
 The interpreter's line-editing features include interactive editing, history
-substitution and code completion on systems that support readline.  Perhaps =
the
-quickest check to see whether command line editing is supported is typing
-:kbd:`Control-P` to the first Python prompt you get.  If it beeps, you have =
command
-line editing; see Appendix :ref:`tut-interacting` for an introduction to the
-keys.  If nothing appears to happen, or if ``^P`` is echoed, command line
-editing isn't available; you'll only be able to use backspace to remove
-characters from the current line.
+substitution and code completion on systems that support the `GNU Readline
+<https://tiswww.case.edu/php/chet/readline/rltop.html>`_ library.
+Perhaps the quickest check to see whether command line editing is supported =
is
+typing :kbd:`Control-P` to the first Python prompt you get.  If it beeps, you
+have command line editing; see Appendix :ref:`tut-interacting` for an
+introduction to the keys.  If nothing appears to happen, or if ``^P`` is
+echoed, command line editing isn't available; you'll only be able to use
+backspace to remove characters from the current line.
=20
 The interpreter operates somewhat like the Unix shell: when called with stan=
dard
 input connected to a tty device, it reads and executes commands interactivel=
y;


From webhook-mailer at python.org  Mon Sep 16 12:30:08 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Mon, 16 Sep 2019 16:30:08 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.541.1599366952.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Add link of GNU Readline library to interpreter tutorial
 (GH-16152) (GH-16189)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/9dc381c0aa464d9a41e433ea87dd153aafb8=
026e
commit: 9dc381c0aa464d9a41e433ea87dd153aafb8026e
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-16T18:30:04+02:00
summary:

Doc: Add link of GNU Readline library to interpreter tutorial (GH-16152) (GH-=
16189)

(cherry picked from commit f18242be16714da2cfe013dbadfaf2e31d971562)

Co-authored-by: Adorilson Bezerra <adorilson at gmail.com>

files:
M Doc/tutorial/interpreter.rst

diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst
index 60f13fcb4894..320befdbb9d0 100644
--- a/Doc/tutorial/interpreter.rst
+++ b/Doc/tutorial/interpreter.rst
@@ -34,13 +34,14 @@ status.  If that doesn't work, you can exit the interpret=
er by typing the
 following command: ``quit()``.
=20
 The interpreter's line-editing features include interactive editing, history
-substitution and code completion on systems that support readline.  Perhaps =
the
-quickest check to see whether command line editing is supported is typing
-:kbd:`Control-P` to the first Python prompt you get.  If it beeps, you have =
command
-line editing; see Appendix :ref:`tut-interacting` for an introduction to the
-keys.  If nothing appears to happen, or if ``^P`` is echoed, command line
-editing isn't available; you'll only be able to use backspace to remove
-characters from the current line.
+substitution and code completion on systems that support the `GNU Readline
+<https://tiswww.case.edu/php/chet/readline/rltop.html>`_ library.
+Perhaps the quickest check to see whether command line editing is supported =
is
+typing :kbd:`Control-P` to the first Python prompt you get.  If it beeps, you
+have command line editing; see Appendix :ref:`tut-interacting` for an
+introduction to the keys.  If nothing appears to happen, or if ``^P`` is
+echoed, command line editing isn't available; you'll only be able to use
+backspace to remove characters from the current line.
=20
 The interpreter operates somewhat like the Unix shell: when called with stan=
dard
 input connected to a tty device, it reads and executes commands interactivel=
y;


From webhook-mailer at python.org  Mon Sep 16 12:34:49 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Mon, 16 Sep 2019 16:34:49 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.542.1599366954.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-38100: Fix spelling error in unittest.mock code (GH-16168)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/a9187c31185fe7ea47271839898416400cc3=
d976
commit: a9187c31185fe7ea47271839898416400cc3d976
branch: master
author: marcoramirezmx <55331462+marcoramirezmx at users.noreply.github.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-16T18:34:46+02:00
summary:

bpo-38100: Fix spelling error in unittest.mock code (GH-16168)

files:
M Lib/unittest/mock.py

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index b33d58a2cb0d..74d32af9bf99 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -1988,9 +1988,9 @@ def _set_return_value(mock, method, name):
         method.return_value =3D fixed
         return
=20
-    return_calulator =3D _calculate_return_value.get(name)
-    if return_calulator is not None:
-        return_value =3D return_calulator(mock)
+    return_calculator =3D _calculate_return_value.get(name)
+    if return_calculator is not None:
+        return_value =3D return_calculator(mock)
         method.return_value =3D return_value
         return
=20


From webhook-mailer at python.org  Mon Sep 16 14:36:18 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Mon, 16 Sep 2019 18:36:18 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.543.1599366955.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Fix Wikipedia link for functools.lru_cache (GH-16183)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/336b3064d8981bc7f76c5cc6f6a0527df697=
71d6
commit: 336b3064d8981bc7f76c5cc6f6a0527df69771d6
branch: master
author: amist <amistern0 at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-16T20:36:14+02:00
summary:

Doc: Fix Wikipedia link for functools.lru_cache (GH-16183)

files:
M Lib/functools.py

diff --git a/Lib/functools.py b/Lib/functools.py
index f87d9c571873..3192bd02d93e 100644
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -482,7 +482,7 @@ def lru_cache(maxsize=3D128, typed=3DFalse):
     with f.cache_info().  Clear the cache and statistics with f.cache_clear(=
).
     Access the underlying function with f.__wrapped__.
=20
-    See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
+    See:  http://en.wikipedia.org/wiki/Cache_replacement_policies#Least_rece=
ntly_used_(LRU)
=20
     """
=20


From webhook-mailer at python.org  Tue Sep 17 00:57:23 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Tue, 17 Sep 2019 04:57:23 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.544.1599366957.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Add list(dict) in stdtypes library (GH-16209)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/63dedef48bba9d54f13b958237696505fa66=
5796
commit: 63dedef48bba9d54f13b958237696505fa665796
branch: master
author: Adorilson Bezerra <adorilson at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-17T06:57:14+02:00
summary:

Doc: Add list(dict) in stdtypes library (GH-16209)

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index a21c5538cbcc..3895a972a72a 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -4234,6 +4234,10 @@ pairs within braces, for example: ``{'jack': 4098, 'sj=
oerd': 4127}`` or ``{4098:
    These are the operations that dictionaries support (and therefore, custom
    mapping types should support too):
=20
+   .. describe:: list(d)
+
+      Return a list of all the keys used in the dictionary *d*.
+
    .. describe:: len(d)
=20
       Return the number of items in the dictionary *d*.


From webhook-mailer at python.org  Tue Sep 17 01:04:23 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Tue, 17 Sep 2019 05:04:23 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.545.1599366959.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Add list(dict) in stdtypes library (GH-16209) (GH-16211)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/89f132a3286640b764a69b8945cbb3a499ad=
6bab
commit: 89f132a3286640b764a69b8945cbb3a499ad6bab
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-17T07:04:19+02:00
summary:

Doc: Add list(dict) in stdtypes library (GH-16209) (GH-16211)

(cherry picked from commit 63dedef48bba9d54f13b958237696505fa665796)

Co-authored-by: Adorilson Bezerra <adorilson at gmail.com>

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 1903864a55e2..7a8faba00fd9 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -4141,6 +4141,10 @@ pairs within braces, for example: ``{'jack': 4098, 'sj=
oerd': 4127}`` or ``{4098:
    These are the operations that dictionaries support (and therefore, custom
    mapping types should support too):
=20
+   .. describe:: list(d)
+
+      Return a list of all the keys used in the dictionary *d*.
+
    .. describe:: len(d)
=20
       Return the number of items in the dictionary *d*.


From webhook-mailer at python.org  Tue Sep 17 01:05:36 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Tue, 17 Sep 2019 05:05:36 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.546.1599366961.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Add list(dict) in stdtypes library (GH-16209) (GH-16210)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/d90bb231c27708f60e9812e0066e3620bc8c=
fe58
commit: d90bb231c27708f60e9812e0066e3620bc8cfe58
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-17T07:05:33+02:00
summary:

Doc: Add list(dict) in stdtypes library (GH-16209) (GH-16210)

(cherry picked from commit 63dedef48bba9d54f13b958237696505fa665796)

Co-authored-by: Adorilson Bezerra <adorilson at gmail.com>

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 32bd7d2b894d..06b0c00ed10a 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -4209,6 +4209,10 @@ pairs within braces, for example: ``{'jack': 4098, 'sj=
oerd': 4127}`` or ``{4098:
    These are the operations that dictionaries support (and therefore, custom
    mapping types should support too):
=20
+   .. describe:: list(d)
+
+      Return a list of all the keys used in the dictionary *d*.
+
    .. describe:: len(d)
=20
       Return the number of items in the dictionary *d*.


From webhook-mailer at python.org  Wed Sep 11 13:42:25 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 17:42:25 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.547.1599366963.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Improve clarity of try-return-finally-return (GH-15677) (GH-15981)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/c0acc0e53e6724c8162f797b94934d53c643=
23d6
commit: c0acc0e53e6724c8162f797b94934d53c64323d6
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T19:42:21+02:00
summary:

Improve clarity of try-return-finally-return (GH-15677) (GH-15981)

Clarify execution in try-return-finally-return case.
(cherry picked from commit 0cc27417f2cd399c432d7dda9aeca1d81af76936)

Co-authored-by: toonarmycaptain <toonarmycaptain at hotmail.com>

files:
M Doc/tutorial/errors.rst
M Misc/ACKS

diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst
index 4e287bbd8d29..0ddf0cc0042d 100644
--- a/Doc/tutorial/errors.rst
+++ b/Doc/tutorial/errors.rst
@@ -341,15 +341,28 @@ example::
      File "<stdin>", line 2, in <module>
    KeyboardInterrupt
=20
-A *finally clause* is always executed before leaving the :keyword:`try`
-statement, whether an exception has occurred or not. When an exception has
-occurred in the :keyword:`!try` clause and has not been handled by an
-:keyword:`except` clause (or it has occurred in an :keyword:`!except` or
-:keyword:`!else` clause), it is re-raised after the :keyword:`finally` claus=
e has
-been executed.  The :keyword:`!finally` clause is also executed "on the way =
out"
-when any other clause of the :keyword:`!try` statement is left via a
-:keyword:`break`, :keyword:`continue` or :keyword:`return` statement.  A more
-complicated example::
+If a :keyword:`finally` clause is present, the :keyword:`finally` clause wil=
l execute as the last task before the :keyword:`try` statement completes. The=
 :keyword:`finally` clause runs whether or not the :keyword:`try` statement p=
roduces an exception. The following points discuss more complex cases when an=
 exception occurs:
+
+* If an exception occurs during execution of the :keyword:`!try` clause, the=
 exception may be handled by an :keyword:`except` clause. In all cases, the e=
xception is re-raised after the :keyword:`!finally` clause has been executed.
+
+* An exception could occur during execution of an :keyword:`!except` or :key=
word:`!else` clause. Again, the exception is re-raised after the :keyword:`!f=
inally` clause has been executed.
+
+* If the :keyword:`!try` statement reaches a :keyword:`break`, :keyword:`con=
tinue` or :keyword:`return` statement, the :keyword:`finally` clause will exe=
cute just prior to the :keyword:`break`, :keyword:`continue` or :keyword:`ret=
urn` statement's execution.
+
+* If a :keyword:`finally` clause includes a :keyword:`return` statement, the=
 :keyword:`finally` clause's :keyword:`return` statement will execute before,=
 and instead of, the :keyword:`return` statement in a :keyword:`try` clause.
+
+For example::
+
+   >>> def bool_return(): -> bool:
+   ...     try:
+   ...         return True
+   ...     finally:
+   ...         return False
+   ...
+   >>> bool_return()
+   False
+
+A more complicated example::
=20
    >>> def divide(x, y):
    ...     try:
diff --git a/Misc/ACKS b/Misc/ACKS
index adc15c94cd31..406009c4e201 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -58,6 +58,7 @@ Jon Anglin
 Michele Angrisano
 Ankur Ankan
 Heidi Annexstad
+David Antonini
 Ramchandra Apte
 =C3=89ric Araujo
 Alexandru Ardelean


From webhook-mailer at python.org  Wed Sep 11 13:54:00 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 17:54:00 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.548.1599366965.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Fix missing negation. (GH-14640) (GH-15985)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/bf0c76cdf64625e88681f1c7fe9bdae64df0=
7e3a
commit: bf0c76cdf64625e88681f1c7fe9bdae64df07e3a
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T19:53:56+02:00
summary:

Doc: Fix missing negation. (GH-14640) (GH-15985)

Reported by Hug Capella on docs at .

Automerge-Triggered-By: @matrixise
(cherry picked from commit 1fae844451b120b93880d9360f288c70e125520c)

Co-authored-by: Julien Palard <julien at palard.fr>

files:
M Doc/library/re.rst

diff --git a/Doc/library/re.rst b/Doc/library/re.rst
index 23deb1f01b86f..a23e45f2eb52f 100644
--- a/Doc/library/re.rst
+++ b/Doc/library/re.rst
@@ -537,8 +537,8 @@ character ``'$'``.
    Matches any character which is not a word character. This is
    the opposite of ``\w``. If the :const:`ASCII` flag is used this
    becomes the equivalent of ``[^a-zA-Z0-9_]``.  If the :const:`LOCALE` flag=
 is
-   used, matches characters considered alphanumeric in the current locale
-   and the underscore.
+   used, matches characters which are neither alphanumeric in the current lo=
cale
+   nor the underscore.
=20
 .. index:: single: \Z; in regular expressions
=20


From webhook-mailer at python.org  Wed Sep 11 13:59:16 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Wed, 11 Sep 2019 17:59:16 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.549.1599366966.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-34001: Fix test_ssl with LibreSSL (GH-13783) (#15997)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/d6ac67f48f5079efc3fa4be3316a9578edb5=
6e0d
commit: d6ac67f48f5079efc3fa4be3316a9578edb56e0d
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-11T19:59:13+02:00
summary:

bpo-34001: Fix test_ssl with LibreSSL (GH-13783) (#15997)

(cherry picked from commit c9bc49c5f6e26a7c958307c2ac338951a7534d9a)

Co-authored-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Tests/2019-06-03-20-47-10.bpo-34001.KvYx9z.rst
M Lib/test/test_ssl.py

diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 48c4fecabd14..a32b0bcc7e22 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -1106,6 +1106,7 @@ def test_hostname_checks_common_name(self):
=20
     @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'),
                          "required OpenSSL 1.1.0g")
+    @unittest.skipIf(IS_LIBRESSL, "see bpo-34001")
     def test_min_max_version(self):
         ctx =3D ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         # OpenSSL default is MINIMUM_SUPPORTED, however some vendors like
@@ -3726,8 +3727,8 @@ def test_min_max_version(self):
                 self.assertEqual(s.version(), 'TLSv1.1')
=20
         # client 1.0, server 1.2 (mismatch)
-        server_context.minimum_version =3D ssl.TLSVersion.TLSv1_2
         server_context.maximum_version =3D ssl.TLSVersion.TLSv1_2
+        server_context.minimum_version =3D ssl.TLSVersion.TLSv1_2
         client_context.maximum_version =3D ssl.TLSVersion.TLSv1
         client_context.maximum_version =3D ssl.TLSVersion.TLSv1
         with ThreadedEchoServer(context=3Dserver_context) as server:
diff --git a/Misc/NEWS.d/next/Tests/2019-06-03-20-47-10.bpo-34001.KvYx9z.rst =
b/Misc/NEWS.d/next/Tests/2019-06-03-20-47-10.bpo-34001.KvYx9z.rst
new file mode 100644
index 000000000000..35c65a8fbf0f
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-06-03-20-47-10.bpo-34001.KvYx9z.rst
@@ -0,0 +1,2 @@
+Make test_ssl pass with LibreSSL. LibreSSL handles minimum and maximum TLS
+version differently than OpenSSL.


From webhook-mailer at python.org  Tue Sep 17 06:42:33 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Tue, 17 Sep 2019 10:42:33 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.550.1599366968.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 Doc: Fix grammar/spelling in ssl.VERIFY_CRL_CHECK_LEAF docs (GH-16221)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/219fb9d65ef7e5363eccc9dde0988bb085db=
1c86
commit: 219fb9d65ef7e5363eccc9dde0988bb085db1c86
branch: master
author: J=C3=B6rn Heissler <joernheissler at users.noreply.github.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-17T12:42:30+02:00
summary:

Doc: Fix grammar/spelling in ssl.VERIFY_CRL_CHECK_LEAF docs (GH-16221)

files:
M Doc/library/ssl.rst

diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index b7af496e1cfe..923bd87f2d7e 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -613,9 +613,9 @@ Constants
 .. data:: VERIFY_CRL_CHECK_LEAF
=20
    Possible value for :attr:`SSLContext.verify_flags`. In this mode, only the
-   peer cert is check but non of the intermediate CA certificates. The mode
+   peer cert is checked but none of the intermediate CA certificates. The mo=
de
    requires a valid CRL that is signed by the peer cert's issuer (its direct
-   ancestor CA). If no proper has been loaded
+   ancestor CA). If no proper CRL has has been loaded with
    :attr:`SSLContext.load_verify_locations`, validation will fail.
=20
    .. versionadded:: 3.4


From webhook-mailer at python.org  Thu Sep 19 01:24:33 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 19 Sep 2019 05:24:33 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.551.1599366970.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Corrected syntax for return annotation (GH-16265)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/9ab6038fe843e1193d795eb58fd5931b44be=
5a96
commit: 9ab6038fe843e1193d795eb58fd5931b44be5a96
branch: master
author: Jason Plurad <pluradj at us.ibm.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-19T07:24:29+02:00
summary:

Doc: Corrected syntax for return annotation (GH-16265)

Signed-off-by: Jason Plurad <pluradj at us.ibm.com>

files:
M Doc/tutorial/errors.rst

diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst
index 75a49259d440..c820e42ac471 100644
--- a/Doc/tutorial/errors.rst
+++ b/Doc/tutorial/errors.rst
@@ -400,7 +400,7 @@ If a :keyword:`finally` clause is present, the :keyword:`=
finally` clause will ex
=20
 For example::
=20
-   >>> def bool_return(): -> bool:
+   >>> def bool_return():
    ...     try:
    ...         return True
    ...     finally:


From webhook-mailer at python.org  Thu Sep 19 01:37:42 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 19 Sep 2019 05:37:42 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.552.1599366972.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: [3.7] bpo-38218: Doc: Corrected syntax for return annotation
 (GH-16265) (GH-16275)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/20d3bce3effd7cca71a9bf3caa247eef2791=
517b
commit: 20d3bce3effd7cca71a9bf3caa247eef2791517b
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-19T07:37:38+02:00
summary:

[3.7] bpo-38218: Doc: Corrected syntax for return annotation (GH-16265) (GH-1=
6275)

Signed-off-by: Jason Plurad <pluradj at us.ibm.com>
(cherry picked from commit 9ab6038fe843e1193d795eb58fd5931b44be5a96)

Co-authored-by: Jason Plurad <pluradj at us.ibm.com>

files:
M Doc/tutorial/errors.rst

diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst
index 0ddf0cc0042d..9585f0673aec 100644
--- a/Doc/tutorial/errors.rst
+++ b/Doc/tutorial/errors.rst
@@ -353,7 +353,7 @@ If a :keyword:`finally` clause is present, the :keyword:`=
finally` clause will ex
=20
 For example::
=20
-   >>> def bool_return(): -> bool:
+   >>> def bool_return():
    ...     try:
    ...         return True
    ...     finally:


From webhook-mailer at python.org  Thu Sep 19 01:43:01 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 19 Sep 2019 05:43:01 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.553.1599366974.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: [3.8] bpo-38218: Doc: Corrected syntax for return annotation
 (GH-16265) (GH-16274)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/6612a4fd36c7f2d71b00da12a3ad4ce61967=
0cd2
commit: 6612a4fd36c7f2d71b00da12a3ad4ce619670cd2
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-19T07:42:57+02:00
summary:

[3.8] bpo-38218: Doc: Corrected syntax for return annotation (GH-16265) (GH-1=
6274)

Signed-off-by: Jason Plurad <pluradj at us.ibm.com>
(cherry picked from commit 9ab6038fe843e1193d795eb58fd5931b44be5a96)

Co-authored-by: Jason Plurad <pluradj at us.ibm.com>

files:
M Doc/tutorial/errors.rst

diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst
index 0ddf0cc0042d..9585f0673aec 100644
--- a/Doc/tutorial/errors.rst
+++ b/Doc/tutorial/errors.rst
@@ -353,7 +353,7 @@ If a :keyword:`finally` clause is present, the :keyword:`=
finally` clause will ex
=20
 For example::
=20
-   >>> def bool_return(): -> bool:
+   >>> def bool_return():
    ...     try:
    ...         return True
    ...     finally:


From webhook-mailer at python.org  Thu Sep 19 01:51:27 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 19 Sep 2019 05:51:27 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.554.1599366976.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Clarify dict equality irrespective of ordering. (GH-16266)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/3171d67a6aaf7fe88685b3a80644f0284686=
ef63
commit: 3171d67a6aaf7fe88685b3a80644f0284686ef63
branch: master
author: toonarmycaptain <toonarmycaptain at hotmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-19T07:51:23+02:00
summary:

Doc: Clarify dict equality irrespective of ordering. (GH-16266)

files:
M Doc/library/stdtypes.rst

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 3895a972a72a..e8b3aba4b909 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -4381,7 +4381,7 @@ pairs within braces, for example: ``{'jack': 4098, 'sjo=
erd': 4127}`` or ``{4098:
          False
=20
    Dictionaries compare equal if and only if they have the same ``(key,
-   value)`` pairs. Order comparisons ('<', '<=3D', '>=3D', '>') raise
+   value)`` pairs (regardless of ordering). Order comparisons ('<', '<=3D', =
'>=3D', '>') raise
    :exc:`TypeError`.
=20
    Dictionaries preserve insertion order.  Note that updating a key does not


From webhook-mailer at python.org  Thu Sep 19 02:41:42 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 19 Sep 2019 06:41:42 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.555.1599366978.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Fix _PyTraceMalloc_Fini() definition (GH-16259) (GH-16278)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/8d9efc4a2750e0512c0d2af27c70c3dea018=
a4e6
commit: 8d9efc4a2750e0512c0d2af27c70c3dea018a4e6
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-19T08:41:35+02:00
summary:

Fix _PyTraceMalloc_Fini() definition (GH-16259) (GH-16278)

The function return type is void, not int.
(cherry picked from commit d299b8b47d9dbea8fec2c2abcb6c32ad32f5f4d1)

Co-authored-by: Victor Stinner <vstinner at redhat.com>

files:
M Include/internal/pycore_pylifecycle.h

diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_=
pylifecycle.h
index a21eb0142233..852280e84027 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -83,7 +83,7 @@ extern void _PyUnicode_Fini(void);
 extern void PyLong_Fini(void);
 extern void _PyFaulthandler_Fini(void);
 extern void _PyHash_Fini(void);
-extern int _PyTraceMalloc_Fini(void);
+extern void _PyTraceMalloc_Fini(void);
 extern void _PyWarnings_Fini(PyInterpreterState *interp);
=20
 extern void _PyGILState_Init(


From webhook-mailer at python.org  Sat Sep 21 01:22:58 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Sat, 21 Sep 2019 05:22:58 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.556.1599366980.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Fix spelling errors of 'initial' in enum.py (GH-16314)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/bb16fb2cb8dca1fa1d67621c3175793eda7d=
bdc2
commit: bb16fb2cb8dca1fa1d67621c3175793eda7dbdc2
branch: master
author: HongWeipeng <hongweichen8888 at sina.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-21T07:22:54+02:00
summary:

Doc: Fix spelling errors of 'initial' in enum.py (GH-16314)

files:
M Lib/enum.py

diff --git a/Lib/enum.py b/Lib/enum.py
index 8f82a6da9958..8a6e5d2e46d6 100644
--- a/Lib/enum.py
+++ b/Lib/enum.py
@@ -672,7 +672,7 @@ def _generate_next_value_(name, start, count, last_values=
):
         Generate the next value when not given.
=20
         name: the name of the member
-        start: the initital start value or None
+        start: the initial start value or None
         count: the number of existing members
         last_value: the last value assigned or None
         """


From webhook-mailer at python.org  Sat Sep 21 01:29:13 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Sat, 21 Sep 2019 05:29:13 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.557.1599366981.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Remove provisional note for asyncio.run() (GH-16310) (GH-16311)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/960f52adc966748123a2d2b5dceb0482e090=
c15b
commit: 960f52adc966748123a2d2b5dceb0482e090c15b
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-21T07:29:09+02:00
summary:

Doc: Remove provisional note for asyncio.run() (GH-16310) (GH-16311)

Based on a comment from @asvetlov https://github.com/python/cpython/pull/1573=
5GH-discussion_r323619076, this removes the provisional note for ``asyncio.ru=
n()`` in the documentation.

Automerge-Triggered-By: @1st1
(cherry picked from commit 77af2290e594479002aa7d8f914d1f38b14a4854)

Co-authored-by: Kyle Stanley <aeros167 at gmail.com>

files:
M Doc/library/asyncio-task.rst

diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index 57e0e07ad3bb..7efec5b6c1df 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -226,8 +226,6 @@ Running an asyncio Program
     programs, and should ideally only be called once.
=20
     .. versionadded:: 3.7
-       **Important:** this function has been added to asyncio in
-       Python 3.7 on a :term:`provisional basis <provisional api>`.
=20
=20
 Creating Tasks


From webhook-mailer at python.org  Sat Sep 21 02:00:08 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Sat, 21 Sep 2019 06:00:08 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.558.1599366983.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: [3.8] bpo-38093: Correctly returns AsyncMock for async subclasses.
 (GH-15947) (GH-16299)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/865bb685a67798eb98dcf5f3a852e08c7779=
2998
commit: 865bb685a67798eb98dcf5f3a852e08c77792998
branch: 3.8
author: Lisa Roach <lisaroach14 at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-21T08:00:04+02:00
summary:

[3.8] bpo-38093: Correctly returns AsyncMock for async subclasses. (GH-15947)=
 (GH-16299)

(cherry picked from commit 8b03f943c37e07fb2394acdcfacd066647f9b1fd)

Co-authored-by: Lisa Roach <lisaroach14 at gmail.com>

files:
A Misc/NEWS.d/next/Library/2019-09-11-14-45-30.bpo-38093.yQ6k7y.rst
M Doc/library/unittest.mock-examples.rst
M Lib/unittest/mock.py
M Lib/unittest/test/testmock/testasync.py
M Lib/unittest/test/testmock/testmagicmethods.py

diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mo=
ck-examples.rst
index cf6b671b5bee..e650bb1e23e0 100644
--- a/Doc/library/unittest.mock-examples.rst
+++ b/Doc/library/unittest.mock-examples.rst
@@ -14,7 +14,7 @@
=20
     import asyncio
     import unittest
-    from unittest.mock import Mock, MagicMock, patch, call, sentinel
+    from unittest.mock import Mock, MagicMock, AsyncMock, patch, call, senti=
nel
=20
     class SomeClass:
         attribute =3D 'this is a doctest'
@@ -280,14 +280,16 @@ function returns is what the call returns:
 Mocking asynchronous iterators
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
=20
-Since Python 3.8, ``MagicMock`` has support to mock :ref:`async-iterators`
-through ``__aiter__``. The :attr:`~Mock.return_value` attribute of ``__aiter=
__``
-can be used to set the return values to be used for iteration.
+Since Python 3.8, ``AsyncMock`` and ``MagicMock`` have support to mock
+:ref:`async-iterators` through ``__aiter__``. The :attr:`~Mock.return_value`
+attribute of ``__aiter__`` can be used to set the return values to be used f=
or
+iteration.
=20
-    >>> mock =3D MagicMock()
+    >>> mock =3D MagicMock()  # AsyncMock also works here
     >>> mock.__aiter__.return_value =3D [1, 2, 3]
     >>> async def main():
     ...     return [i async for i in mock]
+    ...
     >>> asyncio.run(main())
     [1, 2, 3]
=20
@@ -295,24 +297,25 @@ can be used to set the return values to be used for ite=
ration.
 Mocking asynchronous context manager
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
=20
-Since Python 3.8, ``MagicMock`` has support to mock
-:ref:`async-context-managers` through ``__aenter__`` and ``__aexit__``. The
-return value of ``__aenter__`` is an :class:`AsyncMock`.
+Since Python 3.8, ``AsyncMock`` and ``MagicMock`` have support to mock
+:ref:`async-context-managers` through ``__aenter__`` and ``__aexit__``.
+By default, ``__aenter__`` and ``__aexit__`` are ``AsyncMock`` instances that
+return an async function.
=20
     >>> class AsyncContextManager:
-    ...
     ...     async def __aenter__(self):
     ...         return self
-    ...
-    ...     async def __aexit__(self):
+    ...     async def __aexit__(self, exc_type, exc, tb):
     ...         pass
-    >>> mock_instance =3D MagicMock(AsyncContextManager())
+    ...
+    >>> mock_instance =3D MagicMock(AsyncContextManager())  # AsyncMock also=
 works here
     >>> async def main():
     ...     async with mock_instance as result:
     ...         pass
+    ...
     >>> asyncio.run(main())
-    >>> mock_instance.__aenter__.assert_called_once()
-    >>> mock_instance.__aexit__.assert_called_once()
+    >>> mock_instance.__aenter__.assert_awaited_once()
+    >>> mock_instance.__aexit__.assert_awaited_once()
=20
=20
 Creating a Mock from an Existing Object
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 9fd5c3ce9d0a..0c7545bd2b7f 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -988,9 +988,13 @@ def _get_child_mock(self, /, **kw):
         _type =3D type(self)
         if issubclass(_type, MagicMock) and _new_name in _async_method_magic=
s:
             klass =3D AsyncMock
-        if issubclass(_type, AsyncMockMixin):
+        elif _new_name  in _sync_async_magics:
+            # Special case these ones b/c users will assume they are async,
+            # but they are actually sync (ie. __aiter__)
             klass =3D MagicMock
-        if not issubclass(_type, CallableMixin):
+        elif issubclass(_type, AsyncMockMixin):
+            klass =3D AsyncMock
+        elif not issubclass(_type, CallableMixin):
             if issubclass(_type, NonCallableMagicMock):
                 klass =3D MagicMock
             elif issubclass(_type, NonCallableMock) :
@@ -1867,7 +1871,7 @@ def _patch_stopall():
     '__reduce__', '__reduce_ex__', '__getinitargs__', '__getnewargs__',
     '__getstate__', '__setstate__', '__getformat__', '__setformat__',
     '__repr__', '__dir__', '__subclasses__', '__format__',
-    '__getnewargs_ex__', '__aenter__', '__aexit__', '__anext__', '__aiter__',
+    '__getnewargs_ex__',
 }
=20
=20
@@ -1886,10 +1890,12 @@ def method(self, /, *args, **kw):
=20
 # Magic methods used for async `with` statements
 _async_method_magics =3D {"__aenter__", "__aexit__", "__anext__"}
-# `__aiter__` is a plain function but used with async calls
-_async_magics =3D _async_method_magics | {"__aiter__"}
+# Magic methods that are only used with async calls but are synchronous func=
tions themselves
+_sync_async_magics =3D {"__aiter__"}
+_async_magics =3D _async_method_magics | _sync_async_magics
=20
-_all_magics =3D _magics | _non_defaults
+_all_sync_magics =3D _magics | _non_defaults
+_all_magics =3D _all_sync_magics | _async_magics
=20
 _unsupported_magics =3D {
     '__getattr__', '__setattr__',
diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/test=
mock/testasync.py
index 4660bea3e123..9b50b6d86bbd 100644
--- a/Lib/unittest/test/testmock/testasync.py
+++ b/Lib/unittest/test/testmock/testasync.py
@@ -377,10 +377,6 @@ def test_add_side_effect_iterable(self):
=20
 class AsyncContextManagerTest(unittest.TestCase):
     class WithAsyncContextManager:
-        def __init__(self):
-            self.entered =3D False
-            self.exited =3D False
-
         async def __aenter__(self, *args, **kwargs):
             self.entered =3D True
             return self
@@ -388,32 +384,81 @@ def __init__(self):
         async def __aexit__(self, *args, **kwargs):
             self.exited =3D True
=20
-    def test_magic_methods_are_async_mocks(self):
-        mock =3D MagicMock(self.WithAsyncContextManager())
-        self.assertIsInstance(mock.__aenter__, AsyncMock)
-        self.assertIsInstance(mock.__aexit__, AsyncMock)
+    class WithSyncContextManager:
+        def __enter__(self, *args, **kwargs):
+            return self
+
+        def __exit__(self, *args, **kwargs):
+            pass
+
+    class ProductionCode:
+        # Example real-world(ish) code
+        def __init__(self):
+            self.session =3D None
+
+        async def main(self):
+            async with self.session.post('https://python.org') as response:
+                val =3D await response.json()
+                return val
+
+    def test_async_magic_methods_are_async_mocks_with_magicmock(self):
+        cm_mock =3D MagicMock(self.WithAsyncContextManager())
+        self.assertIsInstance(cm_mock.__aenter__, AsyncMock)
+        self.assertIsInstance(cm_mock.__aexit__, AsyncMock)
+
+    def test_magicmock_has_async_magic_methods(self):
+        cm =3D MagicMock(name=3D'magic_cm')
+        self.assertTrue(hasattr(cm, "__aenter__"))
+        self.assertTrue(hasattr(cm, "__aexit__"))
+
+    def test_magic_methods_are_async_functions(self):
+        cm =3D MagicMock(name=3D'magic_cm')
+        self.assertIsInstance(cm.__aenter__, AsyncMock)
+        self.assertIsInstance(cm.__aexit__, AsyncMock)
+        # AsyncMocks are also coroutine functions
+        self.assertTrue(asyncio.iscoroutinefunction(cm.__aenter__))
+        self.assertTrue(asyncio.iscoroutinefunction(cm.__aexit__))
+
+    def test_set_return_value_of_aenter(self):
+        def inner_test(mock_type):
+            pc =3D self.ProductionCode()
+            pc.session =3D MagicMock(name=3D'sessionmock')
+            cm =3D mock_type(name=3D'magic_cm')
+            response =3D AsyncMock(name=3D'response')
+            response.json =3D AsyncMock(return_value=3D{'json': 123})
+            cm.__aenter__.return_value =3D response
+            pc.session.post.return_value =3D cm
+            result =3D asyncio.run(pc.main())
+            self.assertEqual(result, {'json': 123})
+
+        for mock_type in [AsyncMock, MagicMock]:
+            with self.subTest(f"test set return value of aenter with {mock_t=
ype}"):
+                inner_test(mock_type)
=20
     def test_mock_supports_async_context_manager(self):
-        called =3D False
-        instance =3D self.WithAsyncContextManager()
-        mock_instance =3D MagicMock(instance)
+        def inner_test(mock_type):
+            called =3D False
+            cm =3D self.WithAsyncContextManager()
+            cm_mock =3D mock_type(cm)
+
+            async def use_context_manager():
+                nonlocal called
+                async with cm_mock as result:
+                    called =3D True
+                return result
=20
-        async def use_context_manager():
-            nonlocal called
-            async with mock_instance as result:
-                called =3D True
-            return result
-
-        result =3D asyncio.run(use_context_manager())
-        self.assertFalse(instance.entered)
-        self.assertFalse(instance.exited)
-        self.assertTrue(called)
-        self.assertTrue(mock_instance.entered)
-        self.assertTrue(mock_instance.exited)
-        self.assertTrue(mock_instance.__aenter__.called)
-        self.assertTrue(mock_instance.__aexit__.called)
-        self.assertIsNot(mock_instance, result)
-        self.assertIsInstance(result, AsyncMock)
+            cm_result =3D asyncio.run(use_context_manager())
+            self.assertTrue(called)
+            self.assertTrue(cm_mock.__aenter__.called)
+            self.assertTrue(cm_mock.__aexit__.called)
+            cm_mock.__aenter__.assert_awaited()
+            cm_mock.__aexit__.assert_awaited()
+            # We mock __aenter__ so it does not return self
+            self.assertIsNot(cm_mock, cm_result)
+
+        for mock_type in [AsyncMock, MagicMock]:
+            with self.subTest(f"test context manager magics with {mock_type}=
"):
+                inner_test(mock_type)
=20
     def test_mock_customize_async_context_manager(self):
         instance =3D self.WithAsyncContextManager()
@@ -481,27 +526,30 @@ def __aiter__(self):
=20
             raise StopAsyncIteration
=20
-    def test_mock_aiter_and_anext(self):
-        instance =3D self.WithAsyncIterator()
-        mock_instance =3D MagicMock(instance)
-
-        self.assertEqual(asyncio.iscoroutine(instance.__aiter__),
-                         asyncio.iscoroutine(mock_instance.__aiter__))
-        self.assertEqual(asyncio.iscoroutine(instance.__anext__),
-                         asyncio.iscoroutine(mock_instance.__anext__))
-
-        iterator =3D instance.__aiter__()
-        if asyncio.iscoroutine(iterator):
-            iterator =3D asyncio.run(iterator)
-
-        mock_iterator =3D mock_instance.__aiter__()
-        if asyncio.iscoroutine(mock_iterator):
-            mock_iterator =3D asyncio.run(mock_iterator)
+    def test_aiter_set_return_value(self):
+        mock_iter =3D AsyncMock(name=3D"tester")
+        mock_iter.__aiter__.return_value =3D [1, 2, 3]
+        async def main():
+            return [i async for i in mock_iter]
+        result =3D asyncio.run(main())
+        self.assertEqual(result, [1, 2, 3])
+
+    def test_mock_aiter_and_anext_asyncmock(self):
+        def inner_test(mock_type):
+            instance =3D self.WithAsyncIterator()
+            mock_instance =3D mock_type(instance)
+            # Check that the mock and the real thing bahave the same
+            # __aiter__ is not actually async, so not a coroutinefunction
+            self.assertFalse(asyncio.iscoroutinefunction(instance.__aiter__))
+            self.assertFalse(asyncio.iscoroutinefunction(mock_instance.__ait=
er__))
+            # __anext__ is async
+            self.assertTrue(asyncio.iscoroutinefunction(instance.__anext__))
+            self.assertTrue(asyncio.iscoroutinefunction(mock_instance.__anex=
t__))
+
+        for mock_type in [AsyncMock, MagicMock]:
+            with self.subTest(f"test aiter and anext corourtine with {mock_t=
ype}"):
+                inner_test(mock_type)
=20
-        self.assertEqual(asyncio.iscoroutine(iterator.__aiter__),
-                         asyncio.iscoroutine(mock_iterator.__aiter__))
-        self.assertEqual(asyncio.iscoroutine(iterator.__anext__),
-                         asyncio.iscoroutine(mock_iterator.__anext__))
=20
     def test_mock_async_for(self):
         async def iterate(iterator):
@@ -512,19 +560,30 @@ def test_mock_async_for(self):
             return accumulator
=20
         expected =3D ["FOO", "BAR", "BAZ"]
-        with self.subTest("iterate through default value"):
-            mock_instance =3D MagicMock(self.WithAsyncIterator())
-            self.assertEqual([], asyncio.run(iterate(mock_instance)))
+        def test_default(mock_type):
+            mock_instance =3D mock_type(self.WithAsyncIterator())
+            self.assertEqual(asyncio.run(iterate(mock_instance)), [])
+
=20
-        with self.subTest("iterate through set return_value"):
-            mock_instance =3D MagicMock(self.WithAsyncIterator())
+        def test_set_return_value(mock_type):
+            mock_instance =3D mock_type(self.WithAsyncIterator())
             mock_instance.__aiter__.return_value =3D expected[:]
-            self.assertEqual(expected, asyncio.run(iterate(mock_instance)))
+            self.assertEqual(asyncio.run(iterate(mock_instance)), expected)
=20
-        with self.subTest("iterate through set return_value iterator"):
-            mock_instance =3D MagicMock(self.WithAsyncIterator())
+        def test_set_return_value_iter(mock_type):
+            mock_instance =3D mock_type(self.WithAsyncIterator())
             mock_instance.__aiter__.return_value =3D iter(expected[:])
-            self.assertEqual(expected, asyncio.run(iterate(mock_instance)))
+            self.assertEqual(asyncio.run(iterate(mock_instance)), expected)
+
+        for mock_type in [AsyncMock, MagicMock]:
+            with self.subTest(f"default value with {mock_type}"):
+                test_default(mock_type)
+
+            with self.subTest(f"set return_value with {mock_type}"):
+                test_set_return_value(mock_type)
+
+            with self.subTest(f"set return_value iterator with {mock_type}"):
+                test_set_return_value_iter(mock_type)
=20
=20
 class AsyncMockAssert(unittest.TestCase):
diff --git a/Lib/unittest/test/testmock/testmagicmethods.py b/Lib/unittest/te=
st/testmock/testmagicmethods.py
index 130a3397ba0d..57f85e951e20 100644
--- a/Lib/unittest/test/testmock/testmagicmethods.py
+++ b/Lib/unittest/test/testmock/testmagicmethods.py
@@ -1,8 +1,9 @@
+import asyncio
 import math
 import unittest
 import os
 import sys
-from unittest.mock import Mock, MagicMock, _magics
+from unittest.mock import AsyncMock, Mock, MagicMock, _magics
=20
=20
=20
@@ -271,6 +272,34 @@ def test_magic_mock_equality(self):
         self.assertEqual(mock !=3D mock, False)
=20
=20
+    # This should be fixed with issue38163
+    @unittest.expectedFailure
+    def test_asyncmock_defaults(self):
+        mock =3D AsyncMock()
+        self.assertEqual(int(mock), 1)
+        self.assertEqual(complex(mock), 1j)
+        self.assertEqual(float(mock), 1.0)
+        self.assertNotIn(object(), mock)
+        self.assertEqual(len(mock), 0)
+        self.assertEqual(list(mock), [])
+        self.assertEqual(hash(mock), object.__hash__(mock))
+        self.assertEqual(str(mock), object.__str__(mock))
+        self.assertTrue(bool(mock))
+        self.assertEqual(round(mock), mock.__round__())
+        self.assertEqual(math.trunc(mock), mock.__trunc__())
+        self.assertEqual(math.floor(mock), mock.__floor__())
+        self.assertEqual(math.ceil(mock), mock.__ceil__())
+        self.assertTrue(asyncio.iscoroutinefunction(mock.__aexit__))
+        self.assertTrue(asyncio.iscoroutinefunction(mock.__aenter__))
+        self.assertIsInstance(mock.__aenter__, AsyncMock)
+        self.assertIsInstance(mock.__aexit__, AsyncMock)
+
+        # in Python 3 oct and hex use __index__
+        # so these tests are for __index__ in py3k
+        self.assertEqual(oct(mock), '0o1')
+        self.assertEqual(hex(mock), '0x1')
+        # how to test __sizeof__ ?
+
     def test_magicmock_defaults(self):
         mock =3D MagicMock()
         self.assertEqual(int(mock), 1)
@@ -286,6 +315,10 @@ def test_magicmock_defaults(self):
         self.assertEqual(math.trunc(mock), mock.__trunc__())
         self.assertEqual(math.floor(mock), mock.__floor__())
         self.assertEqual(math.ceil(mock), mock.__ceil__())
+        self.assertTrue(asyncio.iscoroutinefunction(mock.__aexit__))
+        self.assertTrue(asyncio.iscoroutinefunction(mock.__aenter__))
+        self.assertIsInstance(mock.__aenter__, AsyncMock)
+        self.assertIsInstance(mock.__aexit__, AsyncMock)
=20
         # in Python 3 oct and hex use __index__
         # so these tests are for __index__ in py3k
diff --git a/Misc/NEWS.d/next/Library/2019-09-11-14-45-30.bpo-38093.yQ6k7y.rs=
t b/Misc/NEWS.d/next/Library/2019-09-11-14-45-30.bpo-38093.yQ6k7y.rst
new file mode 100644
index 000000000000..24a53013cec7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-11-14-45-30.bpo-38093.yQ6k7y.rst
@@ -0,0 +1,2 @@
+Fixes AsyncMock so it doesn't crash when used with AsyncContextManagers
+or AsyncIterators.


From webhook-mailer at python.org  Sat Sep 21 02:18:44 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Sat, 21 Sep 2019 06:18:44 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.559.1599366985.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 Doc: Add whatsnew entry for loop.shutdown_default_executor() (GH-16308)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/030fe8e34d3879ffbd57b702b111ee6d4e01=
1441
commit: 030fe8e34d3879ffbd57b702b111ee6d4e011441
branch: master
author: Kyle Stanley <aeros167 at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-21T08:18:40+02:00
summary:

Doc: Add whatsnew entry for loop.shutdown_default_executor() (GH-16308)

files:
M Doc/whatsnew/3.9.rst

diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
index deaefc753916..200d9627ca52 100644
--- a/Doc/whatsnew/3.9.rst
+++ b/Doc/whatsnew/3.9.rst
@@ -116,6 +116,13 @@ Added the *indent* option to :func:`~ast.dump` which all=
ows it to produce a
 multiline indented output.
 (Contributed by Serhiy Storchaka in :issue:`37995`.)
=20
+asyncio
+-------
+
+Added a new couroutine :meth:`loop.shutdown_default_executor` that schedules
+a shutdown for the default executor that waits on the threadpool to finish
+closing. Also, :func:`asyncio.run` has been updated to use the new coroutine.
+(Contributed by Kyle Stanley in :issue:`34037`.)
=20
 threading
 ---------


From webhook-mailer at python.org  Thu Sep 26 03:01:22 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 26 Sep 2019 07:01:22 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.560.1599366986.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Use the `with` statement in the first example of the ftplib doc.
 (GH-16271)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/5d326abf2cb4891b78d9319a81bffb3974b5=
b745
commit: 5d326abf2cb4891b78d9319a81bffb3974b5b745
branch: master
author: St=C3=A9phane Wirtel <stephane at wirtel.be>
committer: GitHub <noreply at github.com>
date: 2019-09-26T09:01:18+02:00
summary:

Doc: Use the `with` statement in the first example of the ftplib doc. (GH-162=
71)

files:
M Doc/library/ftplib.rst

diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst
index 7423413d209a..db0212367340 100644
--- a/Doc/library/ftplib.rst
+++ b/Doc/library/ftplib.rst
@@ -33,7 +33,8 @@ Here's a sample session using the :mod:`ftplib` module::
    drwxr-sr-x    4 1176     1176         4096 Nov 17  2008 project
    drwxr-xr-x    3 1176     1176         4096 Oct 10  2012 tools
    '226 Directory send OK.'
-   >>> ftp.retrbinary('RETR README', open('README', 'wb').write)
+   >>> with open('README', 'wb') as fp:
+   >>>     ftp.retrbinary('RETR README', fp.write)
    '226 Transfer complete.'
    >>> ftp.quit()
=20


From webhook-mailer at python.org  Thu Sep 26 03:06:49 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 26 Sep 2019 07:06:49 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.561.1599366988.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Several fixes and improvements for 3.9 whatsnew (GH-16375)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/6a10d59541f8ec9fa548e012d4595b39d141=
e1cc
commit: 6a10d59541f8ec9fa548e012d4595b39d141e1cc
branch: master
author: Kyle Stanley <aeros167 at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-26T09:06:46+02:00
summary:

Doc: Several fixes and improvements for 3.9 whatsnew (GH-16375)

files:
M Doc/whatsnew/3.9.rst

diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
index 200d9627ca52..690b536fd881 100644
--- a/Doc/whatsnew/3.9.rst
+++ b/Doc/whatsnew/3.9.rst
@@ -75,27 +75,27 @@ New Features
 Other Language Changes
 =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
=20
-* :func:`builtins.__import__` now raises :exc:`ImportError` instead of
-  :exc:`ValueError` as used to occur when a relative import went past
+* :func:`__import__` now raises :exc:`ImportError` instead of
+  :exc:`ValueError`, which used to occur when a relative import went past
   its top-level package.
   (Contributed by Ngalim Siregar in :issue:`37444`.)
=20
=20
 * Python now gets the absolute path of the script filename specified on
   the command line (ex: ``python3 script.py``): the ``__file__`` attribute of
-  the ``__main__`` module, ``sys.argv[0]`` and ``sys.path[0]`` become an
+  the :mod:`__main__` module, ``sys.argv[0]`` and ``sys.path[0]`` become an
   absolute path, rather than a relative path. These paths now remain valid
   after the current directory is changed by :func:`os.chdir`. As a side effe=
ct,
-  a traceback also displays the absolute path for ``__main__`` module frames=
 in
-  this case.
+  a traceback also displays the absolute path for :mod:`__main__` module fra=
mes
+  in this case.
   (Contributed by Victor Stinner in :issue:`20443`.)
=20
 * In development mode and in debug build, *encoding* and *errors* arguments =
are
   now checked on string encoding and decoding operations. Examples:
   :func:`open`, :meth:`str.encode` and :meth:`bytes.decode`.
=20
-  By default, for best performances, the *errors* argument is only checked at
-  the first encoding/decoding error, and the *encoding* argument is sometimes
+  By default, for best performance, the *errors* argument is only checked at
+  the first encoding/decoding error and the *encoding* argument is sometimes
   ignored for empty strings.
   (Contributed by Victor Stinner in :issue:`37388`.)
=20
@@ -119,15 +119,16 @@ multiline indented output.
 asyncio
 -------
=20
-Added a new couroutine :meth:`loop.shutdown_default_executor` that schedules
-a shutdown for the default executor that waits on the threadpool to finish
-closing. Also, :func:`asyncio.run` has been updated to use the new coroutine.
+Added a new :term:`coroutine` :meth:`~asyncio.loop.shutdown_default_executor`
+that schedules a shutdown for the default executor that waits on the
+:class:`~concurrent.futures.ThreadPoolExecutor` to finish closing. Also,
+:func:`asyncio.run` has been updated to use the new :term:`coroutine`.
 (Contributed by Kyle Stanley in :issue:`34037`.)
=20
 threading
 ---------
=20
-In a subinterpreter, spawning a daemon thread now raises an exception. Daemon
+In a subinterpreter, spawning a daemon thread now raises a :exc:`RuntimeErro=
r`. Daemon
 threads were never supported in subinterpreters. Previously, the subinterpre=
ter
 finalization crashed with a Python fatal error if a daemon thread was still
 running.
@@ -164,8 +165,8 @@ Optimizations
 Build and C API Changes
 =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
=20
-* Add a new public :c:func:`PyObject_CallNoArgs` function to the C API:
-  call a callable Python object without any arguments. It is the most effici=
ent
+* Add a new public :c:func:`PyObject_CallNoArgs` function to the C API, which
+  calls a callable Python object without any arguments. It is the most effic=
ient
   way to call a callable Python object without any argument.
   (Contributed by Victor Stinner in :issue:`37194`.)
=20
@@ -176,18 +177,18 @@ Deprecated
=20
 * Currently :func:`math.factorial` accepts :class:`float` instances with
   non-negative integer values (like ``5.0``).  It raises a :exc:`ValueError`
-  for non-integral and negative floats.  It is deprecated now.  In future
+  for non-integral and negative floats.  It is now deprecated.  In future
   Python versions it will raise a :exc:`TypeError` for all floats.
   (Contributed by Serhiy Storchaka in :issue:`37315`.)
=20
 * The :mod:`parser` module is deprecated and will be removed in future versi=
ons
-  of Python. For the majority of use cases users can leverage the Abstract S=
yntax
+  of Python. For the majority of use cases, users can leverage the Abstract =
Syntax
   Tree (AST) generation and compilation stage, using the :mod:`ast` module.
=20
 * The :mod:`random` module currently accepts any hashable type as a
   possible seed value.  Unfortunately, some of those types are not
   guaranteed to have a deterministic hash value.  After Python 3.9,
-  the module will restrict its seeds to *None*, :class:`int`,
+  the module will restrict its seeds to :const:`None`, :class:`int`,
   :class:`float`, :class:`str`, :class:`bytes`, and :class:`bytearray`.
=20
=20
@@ -195,7 +196,7 @@ Removed
 =3D=3D=3D=3D=3D=3D=3D
=20
 * The undocumented ``sys.callstats()`` function has been removed. Since Pyth=
on
-  3.7, it was deprecated and always returned ``None``. It required a special
+  3.7, it was deprecated and always returned :const:`None`. It required a sp=
ecial
   build option ``CALL_PROFILE`` which was already removed in Python 3.7.
   (Contributed by Victor Stinner in :issue:`37414`.)
=20
@@ -213,7 +214,7 @@ Removed
   (Contributed by Victor Stinner in :issue:`37312`.)
=20
 * ``aifc.openfp()`` alias to ``aifc.open()``, ``sunau.openfp()`` alias to
-  ``sunau.open()``, and ``wave.openfp()`` alias to ``wave.open()`` have been
+  ``sunau.open()``, and ``wave.openfp()`` alias to :func:`wave.open()` have =
been
   removed. They were deprecated since Python 3.7.
   (Contributed by Victor Stinner in :issue:`37320`.)
=20
@@ -229,16 +230,16 @@ Removed
   (Contributed by Serhiy Storchaka in :issue:`36543`.)
=20
 * The old :mod:`plistlib` API has been removed, it was deprecated since Pyth=
on
-  3.4. Use the :func:`load`, :func:`loads`, :func:`dump`, and :func:`dumps`
-  functions. Additionally, the ``use_builtin_types`` parameter was removed,
-  standard :class:`bytes` objects are always used.
+  3.4. Use the :func:`~plistlib.load`, :func:`~plistlib.loads`, :func:`~plis=
tlib.dump`, and
+  :func:`~plistlib.dumps` functions. Additionally, the *use_builtin_types* p=
arameter was
+  removed, standard :class:`bytes` objects are always used instead.
   (Contributed by Jon Janzen in :issue:`36409`.)
=20
-* ``PyThreadState_DeleteCurrent()`` has been removed. It was not documented.
+* The C function ``PyThreadState_DeleteCurrent()`` has been removed. It was =
not documented.
   (Contributed by Joannah Nanjekye in :issue:`37878`.)
=20
 * The C function ``PyGen_NeedsFinalizing`` has been removed. It was not
-  documented, tested or used anywhere within CPython after the implementation
+  documented, tested, or used anywhere within CPython after the implementati=
on
   of :pep:`442`. Patch by Joannah Nanjekye.
   (Contributed by Joannah Nanjekye in :issue:`15088`)
=20
@@ -253,11 +254,10 @@ that may require changes to your code.
 Changes in the Python API
 -------------------------
=20
-* :func:`builtins.__import__` and :func:`importlib.util.resolve_name` now ra=
ise
+* :func:`__import__` and :func:`importlib.util.resolve_name` now raise
   :exc:`ImportError` where it previously raised :exc:`ValueError`. Callers
   catching the specific exception type and supporting both Python 3.9 and
-  earlier versions will need to catch both:
-  ``except (ImportError, ValueError):``
+  earlier versions will need to catch both using ``except (ImportError, Valu=
eError):``.
=20
 * The :mod:`venv` activation scripts no longer special-case when
   ``__VENV_PROMPT__`` is set to ``""``.


From webhook-mailer at python.org  Thu Sep 26 03:12:30 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 26 Sep 2019 07:12:30 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.562.1599366990.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Use the `with` statement in the first example of the ftplib doc.
 (GH-16271) (GH-16413)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/80bde157368fd45d5a5bfd6e8564a14bdec5=
494c
commit: 80bde157368fd45d5a5bfd6e8564a14bdec5494c
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-26T09:12:26+02:00
summary:

Doc: Use the `with` statement in the first example of the ftplib doc. (GH-162=
71) (GH-16413)

(cherry picked from commit 5d326abf2cb4891b78d9319a81bffb3974b5b745)

Co-authored-by: St=C3=A9phane Wirtel <stephane at wirtel.be>

files:
M Doc/library/ftplib.rst

diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst
index 6c39f9a59fc1..d9701f4871d9 100644
--- a/Doc/library/ftplib.rst
+++ b/Doc/library/ftplib.rst
@@ -33,7 +33,8 @@ Here's a sample session using the :mod:`ftplib` module::
    drwxr-sr-x    4 1176     1176         4096 Nov 17  2008 project
    drwxr-xr-x    3 1176     1176         4096 Oct 10  2012 tools
    '226 Directory send OK.'
-   >>> ftp.retrbinary('RETR README', open('README', 'wb').write)
+   >>> with open('README', 'wb') as fp:
+   >>>     ftp.retrbinary('RETR README', fp.write)
    '226 Transfer complete.'
    >>> ftp.quit()
=20


From webhook-mailer at python.org  Thu Sep 26 03:12:43 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 26 Sep 2019 07:12:43 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.563.1599366992.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Doc: Use the `with` statement in the first example of the ftplib doc.
 (GH-16271) (GH-16412)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/b3c35fea13838426525f87366261efa50e5c=
0864
commit: b3c35fea13838426525f87366261efa50e5c0864
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-26T09:12:39+02:00
summary:

Doc: Use the `with` statement in the first example of the ftplib doc. (GH-162=
71) (GH-16412)

(cherry picked from commit 5d326abf2cb4891b78d9319a81bffb3974b5b745)

Co-authored-by: St=C3=A9phane Wirtel <stephane at wirtel.be>

files:
M Doc/library/ftplib.rst

diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst
index 7423413d209a..db0212367340 100644
--- a/Doc/library/ftplib.rst
+++ b/Doc/library/ftplib.rst
@@ -33,7 +33,8 @@ Here's a sample session using the :mod:`ftplib` module::
    drwxr-sr-x    4 1176     1176         4096 Nov 17  2008 project
    drwxr-xr-x    3 1176     1176         4096 Oct 10  2012 tools
    '226 Directory send OK.'
-   >>> ftp.retrbinary('RETR README', open('README', 'wb').write)
+   >>> with open('README', 'wb') as fp:
+   >>>     ftp.retrbinary('RETR README', fp.write)
    '226 Transfer complete.'
    >>> ftp.quit()
=20


From webhook-mailer at python.org  Thu Sep 26 05:09:17 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 26 Sep 2019 09:09:17 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.564.1599366993.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 bpo-38130: Fix error in explaining when an exception is re-raised (GH-16016)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/1ad7be2f16cc9955f271f57a5089602bb41e=
ee85
commit: 1ad7be2f16cc9955f271f57a5089602bb41eee85
branch: master
author: Mohammad Dehghan <md.unicorn at gmail.com>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-26T11:09:05+02:00
summary:

bpo-38130: Fix error in explaining when an exception is re-raised (GH-16016)

Co-Authored-By: Ashwin Ramaswami <aramaswamis at gmail.com>

files:
M Doc/tutorial/errors.rst

diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst
index c820e42ac471..8f86eca02483 100644
--- a/Doc/tutorial/errors.rst
+++ b/Doc/tutorial/errors.rst
@@ -390,7 +390,7 @@ example::
=20
 If a :keyword:`finally` clause is present, the :keyword:`finally` clause wil=
l execute as the last task before the :keyword:`try` statement completes. The=
 :keyword:`finally` clause runs whether or not the :keyword:`try` statement p=
roduces an exception. The following points discuss more complex cases when an=
 exception occurs:
=20
-* If an exception occurs during execution of the :keyword:`!try` clause, the=
 exception may be handled by an :keyword:`except` clause. In all cases, the e=
xception is re-raised after the :keyword:`!finally` clause has been executed.
+* If an exception occurs during execution of the :keyword:`!try` clause, the=
 exception may be handled by an :keyword:`except` clause. If the exception is=
 not handled by an :keyword:`except` clause, the exception is re-raised after=
 the :keyword:`!finally` clause has been executed.
=20
 * An exception could occur during execution of an :keyword:`!except` or :key=
word:`!else` clause. Again, the exception is re-raised after the :keyword:`!f=
inally` clause has been executed.
=20


From webhook-mailer at python.org  Thu Sep 26 05:15:27 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 26 Sep 2019 09:15:27 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.565.1599366995.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-38130: Fix error in explaining when an exception is re-raised
 (GH-16016) (GH-16416)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/5017a645b0be53c4f399ae39ac897539e841=
6ca6
commit: 5017a645b0be53c4f399ae39ac897539e8416ca6
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-26T11:15:21+02:00
summary:

bpo-38130: Fix error in explaining when an exception is re-raised (GH-16016) =
(GH-16416)

Co-Authored-By: Ashwin Ramaswami <aramaswamis at gmail.com>
(cherry picked from commit 1ad7be2f16cc9955f271f57a5089602bb41eee85)

Co-authored-by: Mohammad Dehghan <md.unicorn at gmail.com>

files:
M Doc/tutorial/errors.rst

diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst
index 9585f0673aec..4bc7184d1078 100644
--- a/Doc/tutorial/errors.rst
+++ b/Doc/tutorial/errors.rst
@@ -343,7 +343,7 @@ example::
=20
 If a :keyword:`finally` clause is present, the :keyword:`finally` clause wil=
l execute as the last task before the :keyword:`try` statement completes. The=
 :keyword:`finally` clause runs whether or not the :keyword:`try` statement p=
roduces an exception. The following points discuss more complex cases when an=
 exception occurs:
=20
-* If an exception occurs during execution of the :keyword:`!try` clause, the=
 exception may be handled by an :keyword:`except` clause. In all cases, the e=
xception is re-raised after the :keyword:`!finally` clause has been executed.
+* If an exception occurs during execution of the :keyword:`!try` clause, the=
 exception may be handled by an :keyword:`except` clause. If the exception is=
 not handled by an :keyword:`except` clause, the exception is re-raised after=
 the :keyword:`!finally` clause has been executed.
=20
 * An exception could occur during execution of an :keyword:`!except` or :key=
word:`!else` clause. Again, the exception is re-raised after the :keyword:`!f=
inally` clause has been executed.
=20


From webhook-mailer at python.org  Thu Sep 26 05:16:44 2019
From: webhook-mailer at python.org (=?utf-8?q?St=C3=A9phane?= Wirtel)
Date: Thu, 26 Sep 2019 09:16:44 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.566.1599366997.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-38130: Fix error in explaining when an exception is re-raised
 (GH-16016) (GH-16415)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/68040edb79895c577e2526ad5f30b1b161b2=
c32b
commit: 68040edb79895c577e2526ad5f30b1b161b2c32b
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: St=C3=A9phane Wirtel <stephane at wirtel.be>
date: 2019-09-26T11:16:38+02:00
summary:

bpo-38130: Fix error in explaining when an exception is re-raised (GH-16016) =
(GH-16415)

Co-Authored-By: Ashwin Ramaswami <aramaswamis at gmail.com>
(cherry picked from commit 1ad7be2f16cc9955f271f57a5089602bb41eee85)

Co-authored-by: Mohammad Dehghan <md.unicorn at gmail.com>

files:
M Doc/tutorial/errors.rst

diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst
index 9585f0673aec..4bc7184d1078 100644
--- a/Doc/tutorial/errors.rst
+++ b/Doc/tutorial/errors.rst
@@ -343,7 +343,7 @@ example::
=20
 If a :keyword:`finally` clause is present, the :keyword:`finally` clause wil=
l execute as the last task before the :keyword:`try` statement completes. The=
 :keyword:`finally` clause runs whether or not the :keyword:`try` statement p=
roduces an exception. The following points discuss more complex cases when an=
 exception occurs:
=20
-* If an exception occurs during execution of the :keyword:`!try` clause, the=
 exception may be handled by an :keyword:`except` clause. In all cases, the e=
xception is re-raised after the :keyword:`!finally` clause has been executed.
+* If an exception occurs during execution of the :keyword:`!try` clause, the=
 exception may be handled by an :keyword:`except` clause. If the exception is=
 not handled by an :keyword:`except` clause, the exception is re-raised after=
 the :keyword:`!finally` clause has been executed.
=20
 * An exception could occur during execution of an :keyword:`!except` or :key=
word:`!else` clause. Again, the exception is re-raised after the :keyword:`!f=
inally` clause has been executed.
=20


From webhook-mailer at python.org  Fri Sep 27 21:44:36 2019
From: webhook-mailer at python.org (=?utf-8?q?Jes=C3=BAs?= Cea)
Date: Sat, 28 Sep 2019 01:44:36 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.567.1599366998.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 bpo-38301: In Solaris family, we must be sure to use '-D_REENTRANT' (#16446)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/52d1b86bde2b772a76919c76991c32638495=
4bf1
commit: 52d1b86bde2b772a76919c76991c326384954bf1
branch: master
author: Jes=C3=BAs Cea <jcea at jcea.es>
committer: GitHub <noreply at github.com>
date: 2019-09-28T03:44:32+02:00
summary:

bpo-38301: In Solaris family, we must be sure to use '-D_REENTRANT' (#16446)

files:
A Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst
M configure
M configure.ac

diff --git a/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst =
b/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst
new file mode 100644
index 000000000000..59c9a76385e2
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst
@@ -0,0 +1,2 @@
+In Solaris family, we must be sure to use ``-D_REENTRANT``.
+Patch by Jes=C3=BAs Cea Avi=C3=B3n.
diff --git a/configure b/configure
index bb8fbc26c74e..f1979c1b8124 100755
--- a/configure
+++ b/configure
@@ -10500,6 +10500,9 @@ then
     $as_echo "#define _REENTRANT 1" >>confdefs.h
=20
     posix_threads=3Dyes
+    if test "$ac_sys_system" =3D "SunOS"; then
+        CFLAGS=3D"$CFLAGS -D_REENTRANT"
+    fi
 elif test "$ac_cv_kpthread" =3D "yes"
 then
     CC=3D"$CC -Kpthread"
diff --git a/configure.ac b/configure.ac
index 69e2a5ea556c..917a9087185e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3063,6 +3063,9 @@ then
     # Defining _REENTRANT on system with POSIX threads should not hurt.
     AC_DEFINE(_REENTRANT)
     posix_threads=3Dyes
+    if test "$ac_sys_system" =3D "SunOS"; then
+        CFLAGS=3D"$CFLAGS -D_REENTRANT"
+    fi
 elif test "$ac_cv_kpthread" =3D "yes"
 then
     CC=3D"$CC -Kpthread"


From webhook-mailer at python.org  Fri Sep 27 22:20:35 2019
From: webhook-mailer at python.org (=?utf-8?q?Jes=C3=BAs?= Cea)
Date: Sat, 28 Sep 2019 02:20:35 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.568.1599367000.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-38301: In Solaris family, we must be sure to use '-D_REENTRANT'
 (GH-16446) (#16449)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/8750bce9884335d43ac06989f64473ed877c=
5b63
commit: 8750bce9884335d43ac06989f64473ed877c5b63
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: Jes=C3=BAs Cea <jcea at jcea.es>
date: 2019-09-28T04:20:31+02:00
summary:

bpo-38301: In Solaris family, we must be sure to use '-D_REENTRANT' (GH-16446=
) (#16449)

(cherry picked from commit 52d1b86bde2b772a76919c76991c326384954bf1)

Co-authored-by: Jes=C3=BAs Cea <jcea at jcea.es>

files:
A Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst
M configure
M configure.ac

diff --git a/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst =
b/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst
new file mode 100644
index 000000000000..59c9a76385e2
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst
@@ -0,0 +1,2 @@
+In Solaris family, we must be sure to use ``-D_REENTRANT``.
+Patch by Jes=C3=BAs Cea Avi=C3=B3n.
diff --git a/configure b/configure
index 379ac6408eab..2a933cdbeb61 100755
--- a/configure
+++ b/configure
@@ -10500,6 +10500,9 @@ then
     $as_echo "#define _REENTRANT 1" >>confdefs.h
=20
     posix_threads=3Dyes
+    if test "$ac_sys_system" =3D "SunOS"; then
+        CFLAGS=3D"$CFLAGS -D_REENTRANT"
+    fi
 elif test "$ac_cv_kpthread" =3D "yes"
 then
     CC=3D"$CC -Kpthread"
diff --git a/configure.ac b/configure.ac
index 7b3b0b8be48b..a189d42c2c36 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3063,6 +3063,9 @@ then
     # Defining _REENTRANT on system with POSIX threads should not hurt.
     AC_DEFINE(_REENTRANT)
     posix_threads=3Dyes
+    if test "$ac_sys_system" =3D "SunOS"; then
+        CFLAGS=3D"$CFLAGS -D_REENTRANT"
+    fi
 elif test "$ac_cv_kpthread" =3D "yes"
 then
     CC=3D"$CC -Kpthread"


From webhook-mailer at python.org  Fri Sep 27 22:21:28 2019
From: webhook-mailer at python.org (=?utf-8?q?Jes=C3=BAs?= Cea)
Date: Sat, 28 Sep 2019 02:21:28 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.569.1599367002.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-38301: In Solaris family, we must be sure to use '-D_REENTRANT'
 (GH-16446) (#16450)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/6112b91bb827b07ab83e3e720aa055772a2c=
8b9d
commit: 6112b91bb827b07ab83e3e720aa055772a2c8b9d
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: Jes=C3=BAs Cea <jcea at jcea.es>
date: 2019-09-28T04:21:24+02:00
summary:

bpo-38301: In Solaris family, we must be sure to use '-D_REENTRANT' (GH-16446=
) (#16450)

(cherry picked from commit 52d1b86bde2b772a76919c76991c326384954bf1)

Co-authored-by: Jes=C3=BAs Cea <jcea at jcea.es>

files:
A Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst
M configure
M configure.ac

diff --git a/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst =
b/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst
new file mode 100644
index 000000000000..59c9a76385e2
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst
@@ -0,0 +1,2 @@
+In Solaris family, we must be sure to use ``-D_REENTRANT``.
+Patch by Jes=C3=BAs Cea Avi=C3=B3n.
diff --git a/configure b/configure
index cfdbb6cfeda6..0f466db4aa0a 100755
--- a/configure
+++ b/configure
@@ -10489,6 +10489,9 @@ then
     $as_echo "#define _REENTRANT 1" >>confdefs.h
=20
     posix_threads=3Dyes
+    if test "$ac_sys_system" =3D "SunOS"; then
+        CFLAGS=3D"$CFLAGS -D_REENTRANT"
+    fi
 elif test "$ac_cv_kpthread" =3D "yes"
 then
     CC=3D"$CC -Kpthread"
diff --git a/configure.ac b/configure.ac
index 93f3b0d5a6be..1ef0df70e686 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3079,6 +3079,9 @@ then
     # Defining _REENTRANT on system with POSIX threads should not hurt.
     AC_DEFINE(_REENTRANT)
     posix_threads=3Dyes
+    if test "$ac_sys_system" =3D "SunOS"; then
+        CFLAGS=3D"$CFLAGS -D_REENTRANT"
+    fi
 elif test "$ac_cv_kpthread" =3D "yes"
 then
     CC=3D"$CC -Kpthread"


From webhook-mailer at python.org  Fri Sep 27 23:09:28 2019
From: webhook-mailer at python.org (=?utf-8?q?Jes=C3=BAs?= Cea)
Date: Sat, 28 Sep 2019 03:09:28 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.570.1599367004.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: [2.7] bpo-38301: In Solaris family, we must be sure to use
 '-D_REENTRANT' (GH-16446). (#16454)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/598f676880662fb453ff98fda42b7b7068e5=
be32
commit: 598f676880662fb453ff98fda42b7b7068e5be32
branch: 2.7
author: Jes=C3=BAs Cea <jcea at jcea.es>
committer: GitHub <noreply at github.com>
date: 2019-09-28T05:09:24+02:00
summary:

[2.7] bpo-38301: In Solaris family, we must be sure to use '-D_REENTRANT' (GH=
-16446). (#16454)

(cherry picked from commit 52d1b86bde2b772a76919c76991c326384954bf1)

Co-authored-by: Jes=C3=BAs Cea <jcea at jcea.es>

files:
A Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst
M configure
M configure.ac

diff --git a/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst =
b/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst
new file mode 100644
index 000000000000..59c9a76385e2
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-09-28-02-37-11.bpo-38301.123456.rst
@@ -0,0 +1,2 @@
+In Solaris family, we must be sure to use ``-D_REENTRANT``.
+Patch by Jes=C3=BAs Cea Avi=C3=B3n.
diff --git a/configure b/configure
index 67300fe2b6e1..63d675312da9 100755
--- a/configure
+++ b/configure
@@ -9602,6 +9602,9 @@ then
=20
     posix_threads=3Dyes
     THREADOBJ=3D"Python/thread.o"
+    if test "$ac_sys_system" =3D "SunOS"; then
+        CFLAGS=3D"$CFLAGS -D_REENTRANT"
+    fi
 elif test "$ac_cv_kpthread" =3D "yes"
 then
     CC=3D"$CC -Kpthread"
diff --git a/configure.ac b/configure.ac
index 36df3d02a2d8..efe6922b5de2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2625,6 +2625,9 @@ then
     AC_DEFINE(_REENTRANT)
     posix_threads=3Dyes
     THREADOBJ=3D"Python/thread.o"   =20
+    if test "$ac_sys_system" =3D "SunOS"; then
+        CFLAGS=3D"$CFLAGS -D_REENTRANT"
+    fi
 elif test "$ac_cv_kpthread" =3D "yes"
 then
     CC=3D"$CC -Kpthread"


From webhook-mailer at python.org  Mon Sep 30 03:06:34 2019
From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa)
Date: Mon, 30 Sep 2019 07:06:34 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.571.1599367005.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: bpo-37408: Precise that Tarfile "format" argument only concerns
 writing. (GH-14389) (#16465)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/1aeb720d64021e544043ca4619a44a7da0cd=
00dd
commit: 1aeb720d64021e544043ca4619a44a7da0cd00dd
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: =C5=81ukasz Langa <lukasz at langa.pl>
date: 2019-09-30T09:06:28+02:00
summary:

bpo-37408: Precise that Tarfile "format" argument only concerns writing. (GH-=
14389) (#16465)

(cherry picked from commit c5a7e0ce194c0eafe82eb3e431881012398e7d46)

Co-authored-by: Pascal Chambon <pythoniks at gmail.com>

files:
M Doc/library/tarfile.rst

diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst
index f25af8ca6a33..c34f2c4a5702 100644
--- a/Doc/library/tarfile.rst
+++ b/Doc/library/tarfile.rst
@@ -290,9 +290,10 @@ be finalized; only the internally used file object will =
be closed. See the
=20
       *fileobj* is not closed, when :class:`TarFile` is closed.
=20
-   *format* controls the archive format. It must be one of the constants
+   *format* controls the archive format for writing. It must be one of the c=
onstants
    :const:`USTAR_FORMAT`, :const:`GNU_FORMAT` or :const:`PAX_FORMAT` that are
-   defined at module level.
+   defined at module level. When reading, format will be automatically detec=
ted, even
+   if different formats are present in a single archive.
=20
    The *tarinfo* argument can be used to replace the default :class:`TarInfo=
` class
    with a different one.


From webhook-mailer at python.org  Mon Sep 30 03:10:43 2019
From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa)
Date: Mon, 30 Sep 2019 07:10:43 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.572.1599367007.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject:
 [3.8] bpo-38270: More fixes for strict crypto policy (GH-16418) (#16437)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/bfca56b3ddb4bb0a98931ebcd1257af2afa9=
b2ae
commit: bfca56b3ddb4bb0a98931ebcd1257af2afa9b2ae
branch: 3.8
author: Christian Heimes <christian at python.org>
committer: =C5=81ukasz Langa <lukasz at langa.pl>
date: 2019-09-30T09:10:38+02:00
summary:

[3.8] bpo-38270: More fixes for strict crypto policy (GH-16418) (#16437)

test_hmac and test_hashlib test built-in hashing implementations and
OpenSSL-based hashing implementations. Add more checks to skip OpenSSL
implementations when a strict crypto policy is active.

Use EVP_DigestInit_ex() instead of EVP_DigestInit() to initialize the
EVP context. The EVP_DigestInit() function clears alls flags and breaks
usedforsecurity flag again.

Signed-off-by: Christian Heimes <christian at python.org>

https://bugs.python.org/issue38270.
(cherry picked from commit 90558158093c0ad893102158fd3c2dd9f864e82e)

Co-authored-by: Christian Heimes <christian at python.org>

files:
M Lib/test/support/__init__.py
M Lib/test/test_hashlib.py
M Lib/test/test_hmac.py
M Modules/_hashopenssl.c

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index ad69ec94fb82..2ff7640535e9 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -69,6 +69,11 @@
 except ImportError:
     resource =3D None
=20
+try:
+    import _hashlib
+except ImportError:
+    _hashlib =3D None
+
 __all__ =3D [
     # globals
     "PIPE_MAX_SIZE", "verbose", "max_memuse", "use_resources", "failfast",
@@ -86,8 +91,8 @@
     "create_empty_file", "can_symlink", "fs_is_case_insensitive",
     # unittest
     "is_resource_enabled", "requires", "requires_freebsd_version",
-    "requires_linux_version", "requires_mac_ver", "check_syntax_error",
-    "check_syntax_warning",
+    "requires_linux_version", "requires_mac_ver", "requires_hashdigest",
+    "check_syntax_error", "check_syntax_warning",
     "TransientResource", "time_out", "socket_peer_reset", "ioerror_peer_rese=
t",
     "transient_internet", "BasicTestRunner", "run_unittest", "run_doctest",
     "skip_unless_symlink", "requires_gzip", "requires_bz2", "requires_lzma",
@@ -647,12 +652,16 @@ def wrapper(*args, **kw):
     return decorator
=20
=20
-def requires_hashdigest(digestname):
+def requires_hashdigest(digestname, openssl=3DNone):
     """Decorator raising SkipTest if a hashing algorithm is not available
=20
     The hashing algorithm could be missing or blocked by a strict crypto
     policy.
=20
+    If 'openssl' is True, then the decorator checks that OpenSSL provides
+    the algorithm. Otherwise the check falls back to built-in
+    implementations.
+
     ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for =
FIPS
     ValueError: unsupported hash type md4
     """
@@ -660,7 +669,10 @@ def decorator(func):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
             try:
-                hashlib.new(digestname)
+                if openssl and _hashlib is not None:
+                    _hashlib.new(digestname)
+                else:
+                    hashlib.new(digestname)
             except ValueError:
                 raise unittest.SkipTest(
                     f"hash digest '{digestname}' is not available."
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index e7cc6ccdc602..8b53d23ef525 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -8,6 +8,7 @@
=20
 import array
 from binascii import unhexlify
+import functools
 import hashlib
 import importlib
 import itertools
@@ -18,6 +19,7 @@
 import warnings
 from test import support
 from test.support import _4G, bigmemtest, import_fresh_module
+from test.support import requires_hashdigest
 from http.client import HTTPException
=20
 # Were we compiled --with-pydebug or with #define Py_DEBUG?
@@ -119,6 +121,7 @@ def _test_algorithm_via_hashlib_new(data=3DNone, _alg=3Da=
lgorithm, **kwargs):
             constructors.add(_test_algorithm_via_hashlib_new)
=20
         _hashlib =3D self._conditional_import_module('_hashlib')
+        self._hashlib =3D _hashlib
         if _hashlib:
             # These two algorithms should always be present when this module
             # is compiled.  If not, something was compiled wrong.
@@ -127,7 +130,13 @@ def _test_algorithm_via_hashlib_new(data=3DNone, _alg=3D=
algorithm, **kwargs):
             for algorithm, constructors in self.constructors_to_test.items():
                 constructor =3D getattr(_hashlib, 'openssl_'+algorithm, None)
                 if constructor:
-                    constructors.add(constructor)
+                    try:
+                        constructor()
+                    except ValueError:
+                        # default constructor blocked by crypto policy
+                        pass
+                    else:
+                        constructors.add(constructor)
=20
         def add_builtin_constructor(name):
             constructor =3D getattr(hashlib, "__get_builtin_constructor")(na=
me)
diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py
index 2c09de8b26fa..1bbf201727d7 100644
--- a/Lib/test/test_hmac.py
+++ b/Lib/test/test_hmac.py
@@ -21,7 +21,7 @@ def wrapper(*args, **kwargs):
=20
 class TestVectorsTestCase(unittest.TestCase):
=20
-    @requires_hashdigest('md5')
+    @requires_hashdigest('md5', openssl=3DTrue)
     def test_md5_vectors(self):
         # Test the HMAC module against test vectors from the RFC.
=20
@@ -79,7 +79,7 @@ def md5test(key, data, digest):
                  b"and Larger Than One Block-Size Data"),
                 "6f630fad67cda0ee1fb1f562db3aa53e")
=20
-    @requires_hashdigest('sha1')
+    @requires_hashdigest('sha1', openssl=3DTrue)
     def test_sha_vectors(self):
         def shatest(key, data, digest):
             h =3D hmac.HMAC(key, data, digestmod=3Dhashlib.sha1)
@@ -272,19 +272,19 @@ def hmactest(key, data, hexdigests):
                                    '134676fb6de0446065c97440fa8c6a58',
                  })
=20
-    @requires_hashdigest('sha224')
+    @requires_hashdigest('sha224', openssl=3DTrue)
     def test_sha224_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64)
=20
-    @requires_hashdigest('sha256')
+    @requires_hashdigest('sha256', openssl=3DTrue)
     def test_sha256_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64)
=20
-    @requires_hashdigest('sha384')
+    @requires_hashdigest('sha384', openssl=3DTrue)
     def test_sha384_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128)
=20
-    @requires_hashdigest('sha512')
+    @requires_hashdigest('sha512', openssl=3DTrue)
     def test_sha512_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128)
=20
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 84906a4da4f5..1513e4e35edc 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -556,7 +556,7 @@ EVPnew(const EVP_MD *digest,
     if ((self =3D newEVPobject()) =3D=3D NULL)
         return NULL;
=20
-    if (!EVP_DigestInit(self->ctx, digest)) {
+    if (!EVP_DigestInit_ex(self->ctx, digest, NULL)) {
         _setException(PyExc_ValueError);
         Py_DECREF(self);
         return NULL;


From webhook-mailer at python.org  Mon Sep 30 13:06:49 2019
From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa)
Date: Mon, 30 Sep 2019 17:06:49 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.573.1599367009.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Clear weakrefs in garbage found by the GC (#16495)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/bcda460baf25062ab68622b3f043f52b9db4=
d21d
commit: bcda460baf25062ab68622b3f043f52b9db4d21d
branch: master
author: Neil Schemenauer <nas-github at arctrix.com>
committer: =C5=81ukasz Langa <lukasz at langa.pl>
date: 2019-09-30T19:06:45+02:00
summary:

Clear weakrefs in garbage found by the GC (#16495)

Fix a bug due to the interaction of weakrefs and the cyclic garbage
collector. We must clear any weakrefs in garbage in order to prevent
their callbacks from executing and causing a crash.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-30-09-33-21.bpo-38006.UYlJum.rst
M Modules/gcmodule.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-30-09-33-21.bpo-38006=
.UYlJum.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-30-09-33-21.bpo-3800=
6.UYlJum.rst
new file mode 100644
index 000000000000..c2dafae0afab
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-30-09-33-21.bpo-38006.UYlJum=
.rst=09
@@ -0,0 +1,3 @@
+Fix a bug due to the interaction of weakrefs and the cyclic garbage
+collector. We must clear any weakrefs in garbage in order to prevent their
+callbacks from executing and causing a crash.
diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
index f6e841d52ff9..b95f676663a3 100644
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -678,6 +678,21 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old)
         op =3D FROM_GC(gc);
         next =3D GC_NEXT(gc);
=20
+        if (PyWeakref_Check(op)) {
+            /* A weakref inside the unreachable set must be cleared.  If we
+             * allow its callback to execute inside delete_garbage(), it
+             * could expose objects that have tp_clear already called on
+             * them.  Or, it could resurrect unreachable objects.  One way
+             * this can happen is if some container objects do not implement
+             * tp_traverse.  Then, wr_object can be outside the unreachable
+             * set but can be deallocated as a result of breaking the
+             * reference cycle.  If we don't clear the weakref, the callback
+             * will run and potentially cause a crash.  See bpo-38006 for
+             * one example.
+             */
+            _PyWeakref_ClearRef((PyWeakReference *)op);
+        }
+
         if (! PyType_SUPPORTS_WEAKREFS(Py_TYPE(op)))
             continue;
=20
@@ -733,6 +748,8 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old)
              * is moved to wrcb_to_call in this case.
              */
             if (gc_is_collecting(AS_GC(wr))) {
+                /* it should already have been cleared above */
+                assert(wr->wr_object =3D=3D Py_None);
                 continue;
             }
=20


From webhook-mailer at python.org  Mon Sep 30 13:27:51 2019
From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa)
Date: Mon, 30 Sep 2019 17:27:51 -0000
Subject: [Python-checkins] (no subject)
Message-ID: <mailman.574.1599367010.9553.python-checkins@python.org>




To: python-checkins at python.org
Subject: Clear weakrefs in garbage found by the GC (GH-16495) (#16499)
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0

https://github.com/python/cpython/commit/92ca515ee1efbdc51678e12d105ad642c9b9=
cc13
commit: 92ca515ee1efbdc51678e12d105ad642c9b9cc13
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.co=
m>
committer: =C5=81ukasz Langa <lukasz at langa.pl>
date: 2019-09-30T19:27:46+02:00
summary:

Clear weakrefs in garbage found by the GC (GH-16495) (#16499)

Fix a bug due to the interaction of weakrefs and the cyclic garbage
collector. We must clear any weakrefs in garbage in order to prevent
their callbacks from executing and causing a crash.
(cherry picked from commit bcda460baf25062ab68622b3f043f52b9db4d21d)

Co-authored-by: Neil Schemenauer <nas-github at arctrix.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2019-09-30-09-33-21.bpo-38006.UYlJum.rst
M Modules/gcmodule.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-30-09-33-21.bpo-38006=
.UYlJum.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-30-09-33-21.bpo-3800=
6.UYlJum.rst
new file mode 100644
index 000000000000..c2dafae0afab
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-09-30-09-33-21.bpo-38006.UYlJum=
.rst=09
@@ -0,0 +1,3 @@
+Fix a bug due to the interaction of weakrefs and the cyclic garbage
+collector. We must clear any weakrefs in garbage in order to prevent their
+callbacks from executing and causing a crash.
diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
index b2ee56623cd4..8bdbafef167e 100644
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -678,6 +678,21 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old)
         op =3D FROM_GC(gc);
         next =3D GC_NEXT(gc);
=20
+        if (PyWeakref_Check(op)) {
+            /* A weakref inside the unreachable set must be cleared.  If we
+             * allow its callback to execute inside delete_garbage(), it
+             * could expose objects that have tp_clear already called on
+             * them.  Or, it could resurrect unreachable objects.  One way
+             * this can happen is if some container objects do not implement
+             * tp_traverse.  Then, wr_object can be outside the unreachable
+             * set but can be deallocated as a result of breaking the
+             * reference cycle.  If we don't clear the weakref, the callback
+             * will run and potentially cause a crash.  See bpo-38006 for
+             * one example.
+             */
+            _PyWeakref_ClearRef((PyWeakReference *)op);
+        }
+
         if (! PyType_SUPPORTS_WEAKREFS(Py_TYPE(op)))
             continue;
=20
@@ -733,6 +748,8 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old)
              * is moved to wrcb_to_call in this case.
              */
             if (gc_is_collecting(AS_GC(wr))) {
+                /* it should already have been cleared above */
+                assert(wr->wr_object =3D=3D Py_None);
                 continue;
             }
=20